vm/gce: add support GCE VMs

5 files changed, 480 insertions, 27 deletions

diff --git a/config/config.go b/config/config.go
index 3983ba002..eea8374a9 100644
--- a/config/config.go
+++ b/config/config.go
@@ -46,6 +46,8 @@ type Config struct {
 	// "namespace": create a new namespace for fuzzer using CLONE_NEWNS/CLONE_NEWNET/CLONE_NEWPID/etc,
 	//	requires building kernel with CONFIG_NAMESPACES, CONFIG_UTS_NS, CONFIG_USER_NS, CONFIG_PID_NS and CONFIG_NET_NS.
 
+	Machine_Type string // GCE machine type (e.g. "n1-highcpu-2")
+
 	Cover bool // use kcov coverage (default: true)
 	Leak  bool // do memory leak checking
 
@@ -116,6 +118,11 @@ func parse(data []byte) (*Config, map[int]bool, []*regexp.Regexp, error) {
 			return nil, nil, nil, fmt.Errorf("specify at least 1 adb device")
 		}
 		cfg.Count = len(cfg.Devices)
+	case "gce":
+		if cfg.Machine_Type == "" {
+			return nil, nil, nil, fmt.Errorf("machine_type parameter is empty (required for gce)")
+		}
+		fallthrough
 	default:
 		if cfg.Count <= 0 || cfg.Count > 1000 {
 			return nil, nil, nil, fmt.Errorf("invalid config param count: %v, want (1, 1000]", cfg.Count)
@@ -243,19 +250,20 @@ func CreateVMConfig(cfg *Config, index int) (*vm.Config, error) {
 		return nil, fmt.Errorf("failed to create instance temp dir: %v", err)
 	}
 	vmCfg := &vm.Config{
-		Name:     fmt.Sprintf("%v-%v", cfg.Type, index),
-		Index:    index,
-		Workdir:  workdir,
-		Bin:      cfg.Bin,
-		Kernel:   cfg.Kernel,
-		Cmdline:  cfg.Cmdline,
-		Image:    cfg.Image,
-		Initrd:   cfg.Initrd,
-		Sshkey:   cfg.Sshkey,
-		Executor: filepath.Join(cfg.Syzkaller, "bin", "syz-executor"),
-		Cpu:      cfg.Cpu,
-		Mem:      cfg.Mem,
-		Debug:    cfg.Debug,
+		Name:        fmt.Sprintf("%v-%v", cfg.Type, index),
+		Index:       index,
+		Workdir:     workdir,
+		Bin:         cfg.Bin,
+		Kernel:      cfg.Kernel,
+		Cmdline:     cfg.Cmdline,
+		Image:       cfg.Image,
+		Initrd:      cfg.Initrd,
+		Sshkey:      cfg.Sshkey,
+		Executor:    filepath.Join(cfg.Syzkaller, "bin", "syz-executor"),
+		Cpu:         cfg.Cpu,
+		Mem:         cfg.Mem,
+		Debug:       cfg.Debug,
+		MachineType: cfg.Machine_Type,
 	}
 	if len(cfg.Devices) != 0 {
 		vmCfg.Device = cfg.Devices[index]
@@ -293,6 +301,7 @@ func checkUnknownFields(data []byte) (string, error) {
 		"Disable_Syscalls",
 		"Suppressions",
 		"Initrd",
+		"Machine_Type",
 	}
 	f := make(map[string]interface{})
 	if err := json.Unmarshal(data, &f); err != nil {
diff --git a/syz-manager/manager.go b/syz-manager/manager.go
index ed3fc2766..9836c6080 100644
--- a/syz-manager/manager.go
+++ b/syz-manager/manager.go
@@ -30,6 +30,7 @@ import (
 	"github.com/google/syzkaller/sys"
 	"github.com/google/syzkaller/vm"
 	_ "github.com/google/syzkaller/vm/adb"
+	_ "github.com/google/syzkaller/vm/gce"
 	_ "github.com/google/syzkaller/vm/kvm"
 	_ "github.com/google/syzkaller/vm/local"
 	_ "github.com/google/syzkaller/vm/qemu"
diff --git a/tools/syz-repro/repro.go b/tools/syz-repro/repro.go
index f1fd621b0..de685acce 100644
--- a/tools/syz-repro/repro.go
+++ b/tools/syz-repro/repro.go
@@ -22,6 +22,7 @@ import (
 	"github.com/google/syzkaller/report"
 	"github.com/google/syzkaller/vm"
 	_ "github.com/google/syzkaller/vm/adb"
+	_ "github.com/google/syzkaller/vm/gce"
 	_ "github.com/google/syzkaller/vm/kvm"
 	_ "github.com/google/syzkaller/vm/qemu"
 )
diff --git a/vm/gce/gce.go b/vm/gce/gce.go
new file mode 100644
index 000000000..2c40305a5
--- /dev/null
+++ b/vm/gce/gce.go
@@ -0,0 +1,441 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// Package gce allows to use Google Compute Engine (GCE) virtual machines as VMs.
+// It is assumed that syz-manager also runs on GCE as VMs are created in the current project/zone.
+// See https://cloud.google.com/compute/docs for details.
+// In particular, how to build GCE-compatible images:
+// https://cloud.google.com/compute/docs/tutorials/building-images
+// Working with serial console:
+// https://cloud.google.com/compute/docs/instances/interacting-with-serial-console
+// API reference:
+// https://cloud.google.com/compute/docs/reference/latest/
+// and Go API wrappers:
+// https://godoc.org/google.golang.org/api/compute/v0.beta
+package gce
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/google/syzkaller/vm"
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/compute/v0.beta"
+	"google.golang.org/api/googleapi"
+)
+
+func init() {
+	vm.Register("gce", ctor)
+}
+
+type instance struct {
+	cfg     *vm.Config
+	name    string
+	ip      string
+	offset  int64
+	sshkey  string // per-instance private ssh key
+	workdir string
+	closed  chan bool
+}
+
+var (
+	initOnce       sync.Once
+	computeService *compute.Service
+	projectID      string
+	zoneID         string
+	internalIP     string
+
+	// apiCallTicker ticks regularly, preventing us from accidentally making
+	// GCE API calls too quickly. Our quota is 20 QPS, but we temporarily
+	// limit ourselves to less than that.
+	apiRateGate = time.NewTicker(time.Second / 10).C
+)
+
+func initGCE() {
+	ctx := context.Background()
+	tokenSource, err := google.DefaultTokenSource(ctx, compute.CloudPlatformScope)
+	if err != nil {
+		log.Fatalf("failed to get a token source: %v", err)
+	}
+	httpClient := oauth2.NewClient(ctx, tokenSource)
+	computeService, _ = compute.New(httpClient)
+	// Obtain project name, zone and current instance IP address.
+	projectID, err = getMeta("project/project-id")
+	if err != nil {
+		log.Fatalf("failed to query gce project-id: %v", err)
+	}
+	zoneID, err = getMeta("instance/zone")
+	if err != nil {
+		log.Fatalf("failed to query gce zone: %v", err)
+	}
+	if i := strings.LastIndexByte(zoneID, '/'); i != -1 {
+		zoneID = zoneID[i+1:] // the query returns some nonsense prefix
+	}
+	instID, err := getMeta("instance/id")
+	if err != nil {
+		log.Fatalf("failed to query gce instance id: %v", err)
+	}
+	instances, err := computeService.Instances.List(projectID, zoneID).Do()
+	if err != nil {
+		log.Fatalf("error getting instance list: %v", err)
+	}
+	// Finds this instance internal IP.
+	instName := ""
+	for _, inst := range instances.Items {
+		if fmt.Sprint(inst.Id) != instID {
+			continue
+		}
+		instName = inst.Name
+		for _, iface := range inst.NetworkInterfaces {
+			if strings.HasPrefix(iface.NetworkIP, "10.") {
+				internalIP = iface.NetworkIP
+				break
+			}
+		}
+		break
+	}
+	if instName == "" || internalIP == "" {
+		log.Fatalf("failed to get current instance name and internal IP")
+	}
+	log.Printf("gce initialized: running on %v, internal IP, %v project %v, zone %v", instName, internalIP, projectID, zoneID)
+}
+
+func ctor(cfg *vm.Config) (vm.Instance, error) {
+	initOnce.Do(initGCE)
+	name := fmt.Sprintf("syzkaller-%v", cfg.Index)
+	ok := false
+	defer func() {
+		if !ok {
+			os.RemoveAll(cfg.Workdir)
+		}
+	}()
+
+	// Create SSH key for the instance.
+	sshkey := filepath.Join(cfg.Workdir, "key")
+	keygen := exec.Command("ssh-keygen", "-t", "rsa", "-b", "2048", "-N", "", "-C", "syzkaller", "-f", sshkey)
+	if out, err := keygen.CombinedOutput(); err != nil {
+		return nil, fmt.Errorf("failed to execute ssh-keygen: %v\n%s", err, out)
+	}
+	sshkeyPub, err := ioutil.ReadFile(sshkey + ".pub")
+	if err != nil {
+		return nil, fmt.Errorf("failed to read file: %v", err)
+	}
+
+	log.Printf("deleting instance: %v", name)
+	if err := deleteInstance(name); err != nil {
+		return nil, err
+	}
+	log.Printf("creating instance: %v", name)
+	ip, err := createInstance(name, cfg.MachineType, cfg.Image, string(sshkeyPub))
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if !ok {
+			deleteInstance(name)
+		}
+	}()
+	log.Printf("wait instance to boot: %v (%v)", name, ip)
+	if err := waitInstanceBoot(ip, cfg.Sshkey); err != nil {
+		return nil, err
+	}
+	ok = true
+	inst := &instance{
+		cfg:    cfg,
+		name:   name,
+		ip:     ip,
+		sshkey: sshkey,
+		closed: make(chan bool),
+	}
+	return inst, nil
+}
+
+func (inst *instance) Close() {
+	close(inst.closed)
+	deleteInstance(inst.name)
+	os.RemoveAll(inst.cfg.Workdir)
+}
+
+func (inst *instance) Forward(port int) (string, error) {
+	return fmt.Sprintf("%v:%v", internalIP, port), nil
+}
+
+func (inst *instance) Copy(hostSrc string) (string, error) {
+	vmDst := filepath.Join("/", filepath.Base(hostSrc))
+	args := append(sshArgs(inst.cfg.Sshkey, "-P", 22), hostSrc, "root@"+inst.name+":"+vmDst)
+	cmd := exec.Command("scp", args...)
+	if err := cmd.Start(); err != nil {
+		return "", err
+	}
+	done := make(chan bool)
+	go func() {
+		select {
+		case <-time.After(time.Minute):
+			cmd.Process.Kill()
+		case <-done:
+		}
+	}()
+	err := cmd.Wait()
+	close(done)
+	if err != nil {
+		return "", err
+	}
+	return vmDst, nil
+}
+
+func (inst *instance) Run(timeout time.Duration, command string) (<-chan []byte, <-chan error, error) {
+	conRpipe, conWpipe, err := vm.LongPipe()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	conAddr := fmt.Sprintf("%v.%v.%v.syzkaller.port=1@ssh-serialport.googleapis.com", projectID, zoneID, inst.name)
+	conArgs := append(sshArgs(inst.sshkey, "-p", 9600), conAddr)
+	con := exec.Command("ssh", conArgs...)
+	con.Env = []string{}
+	con.Stdout = conWpipe
+	con.Stderr = conWpipe
+	if _, err := con.StdinPipe(); err != nil { // SSH would close connection on stdin EOF
+		conRpipe.Close()
+		conWpipe.Close()
+		return nil, nil, err
+	}
+	if err := con.Start(); err != nil {
+		conRpipe.Close()
+		conWpipe.Close()
+		return nil, nil, fmt.Errorf("failed to connect to console server: %v", err)
+
+	}
+	conWpipe.Close()
+	conDone := make(chan error, 1)
+	go func() {
+		err := con.Wait()
+		conDone <- fmt.Errorf("console connection closed: %v", err)
+	}()
+
+	sshRpipe, sshWpipe, err := vm.LongPipe()
+	if err != nil {
+		con.Process.Kill()
+		sshRpipe.Close()
+		return nil, nil, err
+	}
+	args := append(sshArgs(inst.cfg.Sshkey, "-p", 22), "root@"+inst.name, command)
+	ssh := exec.Command("ssh", args...)
+	ssh.Stdout = sshWpipe
+	ssh.Stderr = sshWpipe
+	if err := ssh.Start(); err != nil {
+		con.Process.Kill()
+		conRpipe.Close()
+		sshRpipe.Close()
+		sshWpipe.Close()
+		return nil, nil, fmt.Errorf("failed to connect to instance: %v", err)
+	}
+	sshWpipe.Close()
+	sshDone := make(chan error, 1)
+	go func() {
+		err := ssh.Wait()
+		sshDone <- fmt.Errorf("ssh exited: %v", err)
+	}()
+
+	merger := vm.NewOutputMerger(nil)
+	merger.Add(conRpipe)
+	merger.Add(sshRpipe)
+
+	errc := make(chan error, 1)
+	signal := func(err error) {
+		select {
+		case errc <- err:
+		default:
+		}
+	}
+
+	go func() {
+		select {
+		case <-time.After(timeout):
+			signal(vm.TimeoutErr)
+			con.Process.Kill()
+			ssh.Process.Kill()
+		case <-inst.closed:
+			signal(fmt.Errorf("instance closed"))
+			con.Process.Kill()
+			ssh.Process.Kill()
+		case err := <-conDone:
+			signal(err)
+			ssh.Process.Kill()
+		case err := <-sshDone:
+			signal(err)
+			con.Process.Kill()
+		}
+		merger.Wait()
+	}()
+	return merger.Output, errc, nil
+}
+
+func getMeta(path string) (string, error) {
+	req, err := http.NewRequest("GET", "http://metadata.google.internal/computeMetadata/v1/"+path, nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Add("Metadata-Flavor", "Google")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+func waitInstanceBoot(ip, sshkey string) error {
+	for i := 0; i < 100; i++ {
+		if !vm.SleepInterruptible(5 * time.Second) {
+			return fmt.Errorf("shutdown in progress")
+		}
+		cmd := exec.Command("ssh", append(sshArgs(sshkey, "-p", 22), "root@"+ip, "pwd")...)
+		if _, err := cmd.CombinedOutput(); err == nil {
+			return nil
+		}
+	}
+	return fmt.Errorf("can't ssh into the instance")
+}
+
+func createInstance(name, machineType, image, sshkey string) (string, error) {
+	prefix := "https://www.googleapis.com/compute/v1/projects/" + projectID
+	instance := &compute.Instance{
+		Name:        name,
+		Description: "syzkaller worker",
+		MachineType: prefix + "/zones/" + zoneID + "/machineTypes/" + machineType,
+		Disks: []*compute.AttachedDisk{
+			{
+				AutoDelete: true,
+				Boot:       true,
+				Type:       "PERSISTENT",
+				InitializeParams: &compute.AttachedDiskInitializeParams{
+					DiskName:    name,
+					SourceImage: prefix + "/global/images/" + image,
+				},
+			},
+		},
+		Metadata: &compute.Metadata{
+			Items: []*compute.MetadataItems{
+				{
+					Key:   "ssh-keys",
+					Value: "syzkaller:" + sshkey,
+				},
+				{
+					Key:   "serial-port-enable",
+					Value: "1",
+				},
+			},
+		},
+		NetworkInterfaces: []*compute.NetworkInterface{
+			&compute.NetworkInterface{
+				Network: "global/networks/default",
+			},
+		},
+		Scheduling: &compute.Scheduling{
+			AutomaticRestart:  false,
+			Preemptible:       false,
+			OnHostMaintenance: "MIGRATE",
+		},
+	}
+
+	<-apiRateGate
+	op, err := computeService.Instances.Insert(projectID, zoneID, instance).Do()
+	if err != nil {
+		return "", fmt.Errorf("failed to create instance: %v", err)
+	}
+	if err := waitForCompletion("create", op.Name, false); err != nil {
+		return "", err
+	}
+
+	<-apiRateGate
+	inst, err := computeService.Instances.Get(projectID, zoneID, name).Do()
+	if err != nil {
+		return "", fmt.Errorf("error getting instance %s details after creation: %v", name, err)
+	}
+
+	// Finds its internal IP.
+	ip := ""
+	for _, iface := range inst.NetworkInterfaces {
+		if strings.HasPrefix(iface.NetworkIP, "10.") {
+			ip = iface.NetworkIP
+			break
+		}
+	}
+	if ip == "" {
+		return "", fmt.Errorf("didn't find instance internal IP address")
+	}
+	return ip, nil
+}
+
+func deleteInstance(name string) error {
+	<-apiRateGate
+	op, err := computeService.Instances.Delete(projectID, zoneID, name).Do()
+	apiErr, ok := err.(*googleapi.Error)
+	if ok && apiErr.Code == 404 {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to delete instance: %v", err)
+	}
+	if err := waitForCompletion("delete", op.Name, true); err != nil {
+		return err
+	}
+	return nil
+}
+
+func waitForCompletion(desc, opName string, ignoreNotFound bool) error {
+	for {
+		time.Sleep(2 * time.Second)
+		<-apiRateGate
+		op, err := computeService.ZoneOperations.Get(projectID, zoneID, opName).Do()
+		if err != nil {
+			return fmt.Errorf("failed to get %v operation %v: %v", desc, opName, err)
+		}
+		switch op.Status {
+		case "PENDING", "RUNNING":
+			continue
+		case "DONE":
+			if op.Error != nil {
+				reason := ""
+				for _, operr := range op.Error.Errors {
+					if ignoreNotFound && operr.Code == "RESOURCE_NOT_FOUND" {
+						return nil
+					}
+					reason += fmt.Sprintf("%+v.", operr)
+				}
+				return fmt.Errorf("%v operation failed: %v", desc, reason)
+			}
+			return nil
+		default:
+			return fmt.Errorf("unknown %v operation status %q: %+v", desc, op.Status, op)
+		}
+	}
+}
+
+func sshArgs(sshKey, portArg string, port int) []string {
+	return []string{
+		portArg, fmt.Sprint(port),
+		"-i", sshKey,
+		"-F", "/dev/null",
+		"-o", "UserKnownHostsFile=/dev/null",
+		"-o", "BatchMode=yes",
+		"-o", "IdentitiesOnly=yes",
+		"-o", "StrictHostKeyChecking=no",
+		"-o", "ConnectTimeout=5",
+	}
+}
diff --git a/vm/vm.go b/vm/vm.go
index 0d2d73bd8..b2d75bf10 100644
--- a/vm/vm.go
+++ b/vm/vm.go
@@ -34,20 +34,21 @@ type Instance interface {
 }
 
 type Config struct {
-	Name     string
-	Index    int
-	Workdir  string
-	Bin      string
-	Initrd   string
-	Kernel   string
-	Cmdline  string
-	Image    string
-	Sshkey   string
-	Executor string
-	Device   string
-	Cpu      int
-	Mem      int
-	Debug    bool
+	Name        string
+	Index       int
+	Workdir     string
+	Bin         string
+	Initrd      string
+	Kernel      string
+	Cmdline     string
+	Image       string
+	Sshkey      string
+	Executor    string
+	Device      string
+	MachineType string
+	Cpu         int
+	Mem         int
+	Debug       bool
 }
 
 type ctorFunc func(cfg *Config) (Instance, error)