vm/gce: accept un-tar-ed image

vm/gce differs from other VM types in that it accepts image
in a weird, GCE-specific format (namely, image named disk.raw
is put into .tar.gz file). This makes it impossible to write
generic code that creates images for any VM types.

Make vm/gce accept just image like e.g. vm/qemu
and handle own specifics internally.

7 files changed, 117 insertions, 163 deletions

diff --git a/pkg/gcs/gcs.go b/pkg/gcs/gcs.go
index 0121d2aeb..97883256a 100644
--- a/pkg/gcs/gcs.go
+++ b/pkg/gcs/gcs.go
@@ -80,25 +80,35 @@ func (client *Client) Read(gcsFile string) (*File, error) {
 }
 
 func (client *Client) UploadFile(localFile, gcsFile string) error {
-	bucket, filename, err := split(gcsFile)
+	local, err := os.Open(localFile)
 	if err != nil {
 		return err
 	}
-	local, err := os.Open(localFile)
+	defer local.Close()
+
+	w, err := client.FileWriter(gcsFile)
 	if err != nil {
 		return err
 	}
-	defer local.Close()
-	bkt := client.client.Bucket(bucket)
-	f := bkt.Object(filename)
-	w := f.NewWriter(client.ctx)
 	defer w.Close()
+
 	if _, err := io.Copy(w, local); err != nil {
 		return err
 	}
 	return nil
 }
 
+func (client *Client) FileWriter(gcsFile string) (io.WriteCloser, error) {
+	bucket, filename, err := split(gcsFile)
+	if err != nil {
+		return nil, err
+	}
+	bkt := client.client.Bucket(bucket)
+	f := bkt.Object(filename)
+	w := f.NewWriter(client.ctx)
+	return w, nil
+}
+
 func split(file string) (bucket, filename string, err error) {
 	pos := strings.IndexByte(file, '/')
 	if pos == -1 {
diff --git a/pkg/kernel/generated.go b/pkg/kernel/generated.go
index 0e6ba5b89..e9b00b345 100644
--- a/pkg/kernel/generated.go
+++ b/pkg/kernel/generated.go
@@ -8,17 +8,12 @@ const createImageScript = `#!/bin/bash
 set -eux
 
 if [ ! -e $1/sbin/init ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
+	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage"
 	exit 1
 fi
 
 if [ "$(basename $2)" != "bzImage" ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
-	exit 1
-fi
-
-if [ "$(basename $3)" != "vmlinux" ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
+	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage"
 	exit 1
 fi
 
@@ -75,10 +70,4 @@ sudo grub-install --boot-directory=disk.mnt/boot --no-floppy /dev/nbd0
 sudo umount disk.mnt
 rm -rf disk.mnt
 sudo qemu-nbd -d /dev/nbd0
-tar -Szcf disk.tar.gz disk.raw
-mkdir -p obj
-cp $3 obj/
-echo -n "$4" > tag
-tar -czvf image.tar.gz disk.tar.gz key tag obj/vmlinux
-rm -rf tag obj
 `
diff --git a/pkg/kernel/kernel.go b/pkg/kernel/kernel.go
index 9e9b4b32d..5992900f7 100644
--- a/pkg/kernel/kernel.go
+++ b/pkg/kernel/kernel.go
@@ -60,8 +60,8 @@ func Build(dir, compiler, config string, fullConfig bool) error {
 
 // CreateImage creates a disk image that is suitable for syzkaller.
 // Kernel is taken from kernelDir, userspace system is taken from userspaceDir.
-// The resulting image is marked with tag and copied to the specified image file.
-func CreateImage(kernelDir, userspaceDir, tag, image string) error {
+// Produces image and root ssh key in the specified files.
+func CreateImage(kernelDir, userspaceDir, image, sshkey string) error {
 	tempDir, err := ioutil.TempDir("", "syz-build")
 	if err != nil {
 		return err
@@ -71,12 +71,14 @@ func CreateImage(kernelDir, userspaceDir, tag, image string) error {
 	if err := ioutil.WriteFile(scriptFile, []byte(createImageScript), 0700); err != nil {
 		return fmt.Errorf("failed to write script file: %v", err)
 	}
-	vmlinux := filepath.Join(kernelDir, "vmlinux")
-	bzImage := filepath.Join(kernelDir, "arch/x86/boot/bzImage")
-	if _, err := osutil.RunCmd(time.Hour, tempDir, scriptFile, userspaceDir, bzImage, vmlinux, tag); err != nil {
+	bzImage := filepath.Join(kernelDir, filepath.FromSlash("arch/x86/boot/bzImage"))
+	if _, err := osutil.RunCmd(time.Hour, tempDir, scriptFile, userspaceDir, bzImage); err != nil {
 		return fmt.Errorf("image build failed: %v", err)
 	}
-	if err := os.Rename(filepath.Join(tempDir, "image.tar.gz"), image); err != nil {
+	if err := os.Rename(filepath.Join(tempDir, "disk.raw"), image); err != nil {
+		return err
+	}
+	if err := os.Rename(filepath.Join(tempDir, "key"), sshkey); err != nil {
 		return err
 	}
 	return nil
diff --git a/syz-gce/generated.go b/syz-gce/generated.go
index 5cc845b5f..af090863e 100644
--- a/syz-gce/generated.go
+++ b/syz-gce/generated.go
@@ -297,84 +297,3 @@ CONFIG_LLC2=y
 CONFIG_TIPC=y
 CONFIG_TIPC_MEDIA_UDP=y
 `
-
-const createImageScript = `#!/bin/bash
-
-
-set -eux
-
-if [ ! -e $1/sbin/init ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
-	exit 1
-fi
-
-if [ "$(basename $2)" != "bzImage" ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
-	exit 1
-fi
-
-if [ "$(basename $3)" != "vmlinux" ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
-	exit 1
-fi
-
-sudo umount disk.mnt || true
-sudo qemu-nbd -d /dev/nbd0 || true
-rm -rf disk.mnt disk.raw tag obj || true
-
-sudo modprobe nbd
-fallocate -l 2G disk.raw
-sudo qemu-nbd -c /dev/nbd0 --format=raw disk.raw
-mkdir -p disk.mnt
-echo -en "o\nn\np\n1\n2048\n\na\n1\nw\n" | sudo fdisk /dev/nbd0
-until [ -e /dev/nbd0p1 ]; do sleep 1; done
-sudo mkfs.ext4 /dev/nbd0p1
-sudo mount /dev/nbd0p1 disk.mnt
-sudo cp -a $1/. disk.mnt/.
-sudo cp $2 disk.mnt/vmlinuz
-sudo sed -i "/^root/ { s/:x:/::/ }" disk.mnt/etc/passwd
-echo "T0:23:respawn:/sbin/getty -L ttyS0 115200 vt100" | sudo tee -a disk.mnt/etc/inittab
-echo -en "auto lo\niface lo inet loopback\nauto eth0\niface eth0 inet dhcp\n" | sudo tee disk.mnt/etc/network/interfaces
-echo "debugfs /sys/kernel/debug debugfs defaults 0 0" | sudo tee -a disk.mnt/etc/fstab
-echo "kernel.printk = 7 4 1 3" | sudo tee -a disk.mnt/etc/sysctl.conf
-echo "debug.exception-trace = 0" | sudo tee -a disk.mnt/etc/sysctl.conf
-echo "net.core.bpf_jit_enable = 1" | sudo tee -a disk.mnt/etc/sysctl.conf
-echo "net.core.bpf_jit_harden = 2" | sudo tee -a disk.mnt/etc/sysctl.conf
-echo "net.ipv4.ping_group_range = 0 65535" | sudo tee -a disk.mnt/etc/sysctl.conf
-echo -en "127.0.0.1\tlocalhost\n" | sudo tee disk.mnt/etc/hosts
-echo "nameserver 8.8.8.8" | sudo tee -a disk.mnt/etc/resolve.conf
-echo "ClientAliveInterval 420" | sudo tee -a disk.mnt/etc/ssh/sshd_config
-echo "syzkaller" | sudo tee disk.mnt/etc/hostname
-rm -f key key.pub
-ssh-keygen -f key -t rsa -N ""
-sudo mkdir -p disk.mnt/root/.ssh
-sudo cp key.pub disk.mnt/root/.ssh/authorized_keys
-sudo chown root disk.mnt/root/.ssh/authorized_keys
-sudo mkdir -p disk.mnt/boot/grub
-cat << EOF | sudo tee disk.mnt/boot/grub/grub.cfg
-terminal_input console
-terminal_output console
-set timeout=0
-menuentry 'linux' --class gnu-linux --class gnu --class os {
-	insmod vbe
-	insmod vga
-	insmod video_bochs
-	insmod video_cirrus
-	insmod gzio
-	insmod part_msdos
-	insmod ext2
-	set root='(hd0,1)'
-	linux /vmlinuz root=/dev/sda1 console=ttyS0 earlyprintk=serial vsyscall=native rodata=n ftrace_dump_on_oops=orig_cpu oops=panic panic_on_warn=1 nmi_watchdog=panic panic=86400 kvm-intel.nested=1 kvm-intel.unrestricted_guest=1 kvm-intel.vmm_exclusive=1 kvm-intel.fasteoi=1 kvm-intel.ept=1 kvm-intel.flexpriority=1 kvm-intel.vpid=1 kvm-intel.emulate_invalid_guest_state=1 kvm-intel.eptad=1 kvm-intel.enable_shadow_vmcs=1 kvm-intel.pml=1 kvm-intel.enable_apicv=1
-}
-EOF
-sudo grub-install --boot-directory=disk.mnt/boot --no-floppy /dev/nbd0
-sudo umount disk.mnt
-rm -rf disk.mnt
-sudo qemu-nbd -d /dev/nbd0
-tar -Szcf disk.tar.gz disk.raw
-mkdir -p obj
-cp $3 obj/
-echo -n "$4" > tag
-tar -czvf image.tar.gz disk.tar.gz key tag obj/vmlinux
-rm -rf tag obj
-`
diff --git a/syz-gce/syz-gce.go b/syz-gce/syz-gce.go
index df0395af3..2a472f32b 100644
--- a/syz-gce/syz-gce.go
+++ b/syz-gce/syz-gce.go
@@ -6,9 +6,6 @@
 //go:generate bash -c "echo -en 'const syzconfig = `\n' >> generated.go"
 //go:generate bash -c "cat kernel.config | grep -v '#' >> generated.go"
 //go:generate bash -c "echo -en '`\n\n' >> generated.go"
-//go:generate bash -c "echo -en 'const createImageScript = `#!/bin/bash\n' >> generated.go"
-//go:generate bash -c "cat ../tools/create-gce-image.sh | grep -v '#' >> generated.go"
-//go:generate bash -c "echo -en '`\n\n' >> generated.go"
 
 // syz-gce runs syz-manager on GCE in a continous loop handling image/syzkaller updates.
 // It downloads test image from GCS, downloads and builds syzkaller, then starts syz-manager
@@ -352,7 +349,7 @@ func (a *LocalBuildAction) Build() error {
 		return err
 	}
 	for _, p := range patches {
-		if err := a.apply(p); err != nil {
+		if err := applyPatch(dir, p); err != nil {
 			return err
 		}
 	}
@@ -368,43 +365,31 @@ func (a *LocalBuildAction) Build() error {
 	if err := kernel.Build(dir, a.Compiler, config, full); err != nil {
 		return fmt.Errorf("build failed: %v", err)
 	}
-	scriptFile := filepath.Join(a.Dir, "create-gce-image.sh")
-	if err := ioutil.WriteFile(scriptFile, []byte(createImageScript), 0700); err != nil {
-		return fmt.Errorf("failed to write script file: %v", err)
-	}
 	Logf(0, "building image...")
-	vmlinux := filepath.Join(dir, "vmlinux")
-	bzImage := filepath.Join(dir, "arch/x86/boot/bzImage")
-	if _, err := runCmd(a.Dir, scriptFile, a.UserspaceDir, bzImage, vmlinux, hash); err != nil {
+	if err := kernel.CreateImage(dir, a.UserspaceDir, "image/disk.raw", "image/key"); err != nil {
 		return fmt.Errorf("image build failed: %v", err)
 	}
-	os.Remove(filepath.Join(a.Dir, "disk.raw"))
-	os.Remove(filepath.Join(a.Dir, "image.tar.gz"))
-	os.MkdirAll("image/obj", 0700)
 	if err := ioutil.WriteFile("image/tag", []byte(hash), 0600); err != nil {
 		return fmt.Errorf("failed to write tag file: %v", err)
 	}
-	if err := os.Rename(filepath.Join(a.Dir, "key"), "image/key"); err != nil {
-		return fmt.Errorf("failed to rename key file: %v", err)
-	}
+	os.MkdirAll("image/obj", 0700)
+	vmlinux := filepath.Join(dir, "vmlinux")
 	if err := os.Rename(vmlinux, "image/obj/vmlinux"); err != nil {
 		return fmt.Errorf("failed to rename vmlinux file: %v", err)
 	}
-	if err := os.Rename(filepath.Join(a.Dir, "disk.tar.gz"), "image/disk.tar.gz"); err != nil {
-		return fmt.Errorf("failed to rename vmlinux file: %v", err)
-	}
 	return nil
 }
 
-func (a *LocalBuildAction) apply(p dashboard.Patch) error {
+func applyPatch(kernelDir string, p dashboard.Patch) error {
 	// Do --dry-run first to not mess with partially consistent state.
 	cmd := exec.Command("patch", "-p1", "--force", "--ignore-whitespace", "--dry-run")
-	cmd.Dir = filepath.Join(a.Dir, "linux")
+	cmd.Dir = kernelDir
 	cmd.Stdin = bytes.NewReader(p.Diff)
 	if output, err := cmd.CombinedOutput(); err != nil {
-		// If it reverses clean, then it's already applied (seems to be the easiest way to detect it).
+		// If it reverses clean, then it's already applied
+		// (seems to be the easiest way to detect it).
 		cmd = exec.Command("patch", "-p1", "--force", "--ignore-whitespace", "--reverse", "--dry-run")
-		cmd.Dir = filepath.Join(a.Dir, "linux")
+		cmd.Dir = kernelDir
 		cmd.Stdin = bytes.NewReader(p.Diff)
 		if _, err := cmd.CombinedOutput(); err == nil {
 			Logf(0, "patch already present: %v", p.Title)
@@ -415,7 +400,7 @@ func (a *LocalBuildAction) apply(p dashboard.Patch) error {
 	}
 	// Now apply for real.
 	cmd = exec.Command("patch", "-p1", "--force", "--ignore-whitespace")
-	cmd.Dir = filepath.Join(a.Dir, "linux")
+	cmd.Dir = kernelDir
 	cmd.Stdin = bytes.NewReader(p.Diff)
 	if output, err := cmd.CombinedOutput(); err != nil {
 		return fmt.Errorf("patch '%v' failed after dry run:\n%s", p.Title, output)
@@ -479,7 +464,7 @@ func writeManagerConfig(cfg *Config, httpPort int, file string) error {
 		Tag:              string(tag),
 		Syzkaller:        "gopath/src/github.com/google/syzkaller",
 		Type:             "gce",
-		Image:            "image/disk.tar.gz",
+		Image:            "image/disk.raw",
 		Sshkey:           sshKey,
 		Sandbox:          cfg.Sandbox,
 		Procs:            cfg.Procs,
diff --git a/tools/create-gce-image.sh b/tools/create-gce-image.sh
index 29ed2cb0d..3329fcfe0 100755
--- a/tools/create-gce-image.sh
+++ b/tools/create-gce-image.sh
@@ -15,34 +15,35 @@
 #   note: kernel modules are not supported
 #
 # Usage:
-#   ./create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'
+#   ./create-gce-image.sh /dir/with/user/space/system /path/to/bzImage
 #
-# The image can then be uploaded to GCS with:
-#   gsutil cp disk.tar.gz gs://my-images
-# and then my-images/disk.tar.gz can be used to create new GCE bootable image.
-# image.tar.gz can be used with syz-gce.
+# Outputs are (in the current dir):
+# - disk.raw: the image
+# - key: root ssh key
+# The script can also create/delete temp files in the current dir.
+#
+# The image then needs to be compressed with:
+#   tar -Sczf disk.tar.gz disk.raw
+# and uploaded to GCS with:
+#   gsutil cp disk.tar.gz gs://my-images/image.tar.gz
+# finally, my-images/image.tar.gz can be used to create a new GCE image.
 #
 # The image can be tested locally with e.g.:
-#   qemu-system-x86_64 -hda disk.raw -net user,host=10.0.2.10,hostfwd=tcp::10022-:22 -net nic -enable-kvm -m 2G -display none -serial stdio
+#   qemu-system-x86_64 -hda disk.raw -net user,host=10.0.2.10,hostfwd=tcp::10022-:22 \
+#       -net nic -enable-kvm -m 2G -display none -serial stdio
 # once the kernel boots, you can ssh into it with:
-#   ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes -p 10022 -i key root@localhost
-#
-# Note: the script creates and deletes some failes in cwd.
+#   ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o IdentitiesOnly=yes \
+#       -p 10022 -i key root@localhost
 
 set -eux
 
 if [ ! -e $1/sbin/init ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
+	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage"
 	exit 1
 fi
 
 if [ "$(basename $2)" != "bzImage" ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
-	exit 1
-fi
-
-if [ "$(basename $3)" != "vmlinux" ]; then
-	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage /path/to/vmlinux 'image tag'"
+	echo "usage: create-gce-image.sh /dir/with/user/space/system /path/to/bzImage"
 	exit 1
 fi
 
@@ -104,9 +105,3 @@ sudo grub-install --boot-directory=disk.mnt/boot --no-floppy /dev/nbd0
 sudo umount disk.mnt
 rm -rf disk.mnt
 sudo qemu-nbd -d /dev/nbd0
-tar -Sczf disk.tar.gz disk.raw
-mkdir -p obj
-cp $3 obj/
-echo -n "$4" > tag
-tar -czf image.tar.gz disk.tar.gz key tag obj/vmlinux
-rm -rf tag obj
diff --git a/vm/gce/gce.go b/vm/gce/gce.go
index eeae077a5..0ba38298e 100644
--- a/vm/gce/gce.go
+++ b/vm/gce/gce.go
@@ -12,8 +12,12 @@
 package gce
 
 import (
+	"archive/tar"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
@@ -88,18 +92,14 @@ func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
 	}
 	Logf(0, "GCE initialized: running on %v, internal IP %v, project %v, zone %v",
 		GCE.Instance, GCE.InternalIP, GCE.ProjectID, GCE.ZoneID)
-	GCS, err := gcs.NewClient()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create GCS client: %v", err)
-	}
-	defer GCS.Close()
+
 	gcsImage := filepath.Join(cfg.GCS_Path, env.Name+"-image.tar.gz")
-	gceImage := env.Name
 	Logf(0, "uploading image to %v...", gcsImage)
-	if err := GCS.UploadFile(env.Image, gcsImage); err != nil {
-		return nil, fmt.Errorf("failed to upload image: %v", err)
+	if err := uploadImageToGCS(env.Image, gcsImage); err != nil {
+		return nil, err
 	}
-	Logf(0, "creating GCE image...")
+	gceImage := env.Name
+	Logf(0, "creating GCE image %v...", gceImage)
 	if err := GCE.DeleteImage(gceImage); err != nil {
 		return nil, fmt.Errorf("failed to delete GCE image: %v", err)
 	}
@@ -315,6 +315,60 @@ func waitInstanceBoot(ip, sshKey, sshUser string) error {
 	return fmt.Errorf("can't ssh into the instance")
 }
 
+func uploadImageToGCS(localImage, gcsImage string) error {
+	GCS, err := gcs.NewClient()
+	if err != nil {
+		return fmt.Errorf("failed to create GCS client: %v", err)
+	}
+	defer GCS.Close()
+
+	localReader, err := os.Open(localImage)
+	if err != nil {
+		return fmt.Errorf("failed to open image file: %v")
+	}
+	defer localReader.Close()
+	localStat, err := localReader.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat image file: %v")
+	}
+
+	gcsWriter, err := GCS.FileWriter(gcsImage)
+	if err != nil {
+		return fmt.Errorf("failed to upload image: %v", err)
+	}
+	defer gcsWriter.Close()
+
+	gzipWriter := gzip.NewWriter(gcsWriter)
+	tarWriter := tar.NewWriter(gzipWriter)
+	tarHeader := &tar.Header{
+		Name:     "disk.raw",
+		Typeflag: tar.TypeReg,
+		Mode:     0640,
+		Size:     localStat.Size(),
+		ModTime:  time.Now(),
+		Uid:      0,
+		Uname:    "root",
+		Gid:      0,
+		Gname:    "",
+	}
+	if err := tarWriter.WriteHeader(tarHeader); err != nil {
+		return fmt.Errorf("failed to write image tar header: %v", err)
+	}
+	if _, err := io.Copy(tarWriter, localReader); err != nil {
+		return fmt.Errorf("failed to write image file: %v", err)
+	}
+	if err := tarWriter.Close(); err != nil {
+		return fmt.Errorf("failed to write image file: %v", err)
+	}
+	if err := gzipWriter.Close(); err != nil {
+		return fmt.Errorf("failed to write image file: %v", err)
+	}
+	if err := gcsWriter.Close(); err != nil {
+		return fmt.Errorf("failed to write image file: %v", err)
+	}
+	return nil
+}
+
 func sshArgs(sshKey, portArg string, port int) []string {
 	return []string{
 		portArg, fmt.Sprint(port),