Add a bhyve VM backend (#1150)

* vm: add bhyve support

bhyve is FreeBSD's native hypervisor.  Because it is missing snapshot
support and user networking, some additional configuration on the host
is required.  However, unlike QEMU on FreeBSD, bhyve can make use of
hardware virtualization features and is thus faster.

* docs/freebsd: document bhyve support

1 files changed, 342 insertions, 0 deletions

diff --git a/vm/bhyve/bhyve.go b/vm/bhyve/bhyve.go
new file mode 100644
index 000000000..4cbc8d480
--- /dev/null
+++ b/vm/bhyve/bhyve.go
@@ -0,0 +1,342 @@
+// Copyright 2019 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package bhyve
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/google/syzkaller/pkg/config"
+	"github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/vm/vmimpl"
+)
+
+func init() {
+	vmimpl.Register("bhyve", ctor, true)
+}
+
+type Config struct {
+	Bridge  string `json:"bridge"`  // name of network bridge device
+	Count   int    `json:"count"`   // number of VMs to use
+	HostIP  string `json:"hostip"`  // VM host IP address
+	Mem     string `json:"mem"`     // amount of VM memory
+	Dataset string `json:"dataset"` // ZFS dataset containing VM image
+}
+
+type Pool struct {
+	env *vmimpl.Env
+	cfg *Config
+}
+
+type instance struct {
+	cfg      *Config
+	snapshot string
+	tapdev   string
+	image    string
+	debug    bool
+	os       string
+	sshkey   string
+	sshuser  string
+	sshhost  string
+	merger   *vmimpl.OutputMerger
+	vmName   string
+	bhyve    *exec.Cmd
+}
+
+var ipRegex = regexp.MustCompile(`bound to (([0-9]+\.){3}[0-9]+) `)
+var tapRegex = regexp.MustCompile(`^tap[0-9]+`)
+
+func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
+	cfg := &Config{
+		Count: 1,
+		Mem:   "512M",
+	}
+	if err := config.LoadData(env.Config, cfg); err != nil {
+		return nil, fmt.Errorf("failed to parse bhyve vm config: %v", err)
+	}
+	if cfg.Count < 1 || cfg.Count > 128 {
+		return nil, fmt.Errorf("invalid config param count: %v, want [1-128]", cfg.Count)
+	}
+	if env.Debug && cfg.Count > 1 {
+		log.Logf(0, "limiting number of VMs from %v to 1 in debug mode", cfg.Count)
+		cfg.Count = 1
+	}
+	pool := &Pool{
+		cfg: cfg,
+		env: env,
+	}
+	return pool, nil
+}
+
+func (pool *Pool) Count() int {
+	return pool.cfg.Count
+}
+
+func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
+	inst := &instance{
+		cfg:     pool.cfg,
+		debug:   pool.env.Debug,
+		os:      pool.env.OS,
+		sshkey:  pool.env.SSHKey,
+		sshuser: pool.env.SSHUser,
+		vmName:  fmt.Sprintf("syzkaller-%v-%v", pool.env.Name, index),
+	}
+
+	dataset := inst.cfg.Dataset
+	mountpoint, err := osutil.RunCmd(time.Minute, "", "zfs", "get", "-H", "-o", "value", "mountpoint", dataset)
+	if err != nil {
+		return nil, err
+	}
+
+	snapshot := fmt.Sprintf("%v@bhyve-%v", dataset, inst.vmName)
+	clone := fmt.Sprintf("%v/bhyve-%v", dataset, inst.vmName)
+
+	prefix := strings.TrimSuffix(string(mountpoint), "\n") + "/"
+	image := strings.TrimPrefix(pool.env.Image, prefix)
+	if image == pool.env.Image {
+		return nil, fmt.Errorf("image file %v not contained in dataset %v", image, prefix)
+	}
+	inst.image = prefix + fmt.Sprintf("bhyve-%v", inst.vmName) + "/" + image
+
+	// Stop the instance from a previous run in case it's still running.
+	osutil.RunCmd(time.Minute, "", "bhyvectl", "--destroy", fmt.Sprintf("--vm=%v", inst.vmName))
+	// Destroy a lingering snapshot and clone.
+	osutil.RunCmd(time.Minute, "", "zfs", "destroy", "-R", snapshot)
+
+	// Create a snapshot of the data set containing the VM image.  bhyve will
+	// use a clone of the snapshot, which gets recreated every time the VM
+	// is restarted.  This is all to work around bhyve's current lack of an
+	// image snapshot facility.
+	if _, err := osutil.RunCmd(time.Minute, "", "zfs", "snapshot", snapshot); err != nil {
+		inst.Close()
+		return nil, err
+	}
+	inst.snapshot = snapshot
+	if _, err := osutil.RunCmd(time.Minute, "", "zfs", "clone", snapshot, clone); err != nil {
+		inst.Close()
+		return nil, err
+	}
+
+	tapdev, err := osutil.RunCmd(time.Minute, "", "ifconfig", "tap", "create")
+	if err != nil {
+		inst.Close()
+		return nil, err
+	}
+	inst.tapdev = tapRegex.FindString(string(tapdev))
+	if _, err := osutil.RunCmd(time.Minute, "", "ifconfig", inst.cfg.Bridge, "addm", inst.tapdev); err != nil {
+		inst.Close()
+		return nil, err
+	}
+
+	if err := inst.Boot(); err != nil {
+		inst.Close()
+		return nil, err
+	}
+
+	return inst, nil
+}
+
+func (inst *instance) Boot() error {
+	loaderArgs := []string{
+		"-c", "stdio",
+		"-m", inst.cfg.Mem,
+		"-d", inst.image,
+		"-e", "autoboot_delay=0",
+		inst.vmName,
+	}
+
+	// Stop the instance from the previous run in case it's still running.
+	osutil.RunCmd(time.Minute, "", "bhyvectl", "--destroy", fmt.Sprintf("--vm=%v", inst.vmName))
+
+	_, err := osutil.RunCmd(time.Minute, "", "bhyveload", loaderArgs...)
+	if err != nil {
+		return err
+	}
+
+	bhyveArgs := []string{
+		"-H", "-A", "-P",
+		"-c", "1",
+		"-m", inst.cfg.Mem,
+		"-s", "0:0,hostbridge",
+		"-s", "1:0,lpc",
+		"-s", fmt.Sprintf("2:0,virtio-net,%v", inst.tapdev),
+		"-s", fmt.Sprintf("3:0,virtio-blk,%v", inst.image),
+		"-l", "com1,stdio",
+		inst.vmName,
+	}
+
+	outr, outw, err := osutil.LongPipe()
+	if err != nil {
+		return err
+	}
+
+	bhyve := osutil.Command("bhyve", bhyveArgs...)
+	bhyve.Stdout = outw
+	bhyve.Stderr = outw
+	if err := bhyve.Start(); err != nil {
+		outr.Close()
+		outw.Close()
+		return err
+	}
+	outw.Close()
+	outw = nil
+	inst.bhyve = bhyve
+
+	var tee io.Writer
+	if inst.debug {
+		tee = os.Stdout
+	}
+	inst.merger = vmimpl.NewOutputMerger(tee)
+	inst.merger.Add("console", outr)
+	outr = nil
+
+	var bootOutput []byte
+	bootOutputStop := make(chan bool)
+	ipch := make(chan string, 1)
+	go func() {
+		gotip := false
+		for {
+			select {
+			case out := <-inst.merger.Output:
+				bootOutput = append(bootOutput, out...)
+			case <-bootOutputStop:
+				close(bootOutputStop)
+				return
+			}
+			if gotip {
+				continue
+			}
+			if ip := parseIP(bootOutput); ip != "" {
+				ipch <- ip
+				gotip = true
+			}
+		}
+	}()
+
+	select {
+	case ip := <-ipch:
+		inst.sshhost = ip
+	case <-inst.merger.Err:
+		bootOutputStop <- true
+		<-bootOutputStop
+		return vmimpl.BootError{Title: "bhyve exited", Output: bootOutput}
+	case <-time.After(10 * time.Minute):
+		bootOutputStop <- true
+		<-bootOutputStop
+		return vmimpl.BootError{Title: "no IP found", Output: bootOutput}
+	}
+
+	if err := vmimpl.WaitForSSH(inst.debug, 10*time.Minute, inst.sshhost,
+		inst.sshkey, inst.sshuser, inst.os, 22, nil); err != nil {
+		bootOutputStop <- true
+		<-bootOutputStop
+		return vmimpl.MakeBootError(err, bootOutput)
+	}
+	bootOutputStop <- true
+	return nil
+}
+
+func (inst *instance) Close() {
+	if inst.bhyve != nil {
+		inst.bhyve.Process.Kill()
+		inst.bhyve.Wait()
+		inst.bhyve = nil
+	}
+	if inst.snapshot != "" {
+		osutil.RunCmd(time.Minute, "", "zfs", "destroy", "-R", inst.snapshot)
+		inst.snapshot = ""
+	}
+	if inst.tapdev != "" {
+		osutil.RunCmd(time.Minute, "", "ifconfig", inst.tapdev, "destroy")
+		inst.tapdev = ""
+	}
+}
+
+func (inst *instance) Forward(port int) (string, error) {
+	return fmt.Sprintf("%v:%v", inst.cfg.HostIP, port), nil
+}
+
+func (inst *instance) Copy(hostSrc string) (string, error) {
+	vmDst := filepath.Join("/root", filepath.Base(hostSrc))
+	args := append(vmimpl.SCPArgs(inst.debug, inst.sshkey, 22),
+		hostSrc, inst.sshuser+"@"+inst.sshhost+":"+vmDst)
+	if inst.debug {
+		log.Logf(0, "running command: scp %#v", args)
+	}
+	_, err := osutil.RunCmd(10*time.Minute, "", "scp", args...)
+	if err != nil {
+		return "", err
+	}
+	return vmDst, nil
+}
+
+func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
+	<-chan []byte, <-chan error, error) {
+	rpipe, wpipe, err := osutil.LongPipe()
+	if err != nil {
+		return nil, nil, err
+	}
+	inst.merger.Add("ssh", rpipe)
+
+	args := append(vmimpl.SSHArgs(inst.debug, inst.sshkey, 22),
+		inst.sshuser+"@"+inst.sshhost, command)
+	if inst.debug {
+		log.Logf(0, "running command: ssh %#v", args)
+	}
+	cmd := osutil.Command("ssh", args...)
+	cmd.Stdout = wpipe
+	cmd.Stderr = wpipe
+	if err := cmd.Start(); err != nil {
+		wpipe.Close()
+		return nil, nil, err
+	}
+	wpipe.Close()
+	errc := make(chan error, 1)
+	signal := func(err error) {
+		select {
+		case errc <- err:
+		default:
+		}
+	}
+
+	go func() {
+		select {
+		case <-time.After(timeout):
+			signal(vmimpl.ErrTimeout)
+		case <-stop:
+			signal(vmimpl.ErrTimeout)
+		case err := <-inst.merger.Err:
+			cmd.Process.Kill()
+			if cmdErr := cmd.Wait(); cmdErr == nil {
+				// If the command exited successfully, we got EOF error from merger.
+				// But in this case no error has happened and the EOF is expected.
+				err = nil
+			}
+			signal(err)
+			return
+		}
+		cmd.Process.Kill()
+		cmd.Wait()
+	}()
+	return inst.merger.Output, errc, nil
+}
+
+func (inst *instance) Diagnose() ([]byte, bool) {
+	return nil, false
+}
+
+func parseIP(output []byte) string {
+	matches := ipRegex.FindSubmatch(output)
+	if len(matches) < 2 {
+		return ""
+	}
+	return string(matches[1])
+}