lkvm support

3 files changed, 420 insertions, 0 deletions

diff --git a/ipc/ipc.go b/ipc/ipc.go
index aa7a0e3a5..bf5f38207 100644
--- a/ipc/ipc.go
+++ b/ipc/ipc.go
@@ -267,9 +267,11 @@ func (env *Env) execBin() (output, strace []byte, failed, hanged bool, err0 erro
 func createMapping(size int) (f *os.File, mem []byte, err error) {
 	f, err = ioutil.TempFile("./", "syzkaller-shm")
 	if err != nil {
+		err = fmt.Errorf("failed to create temp file: %v", err)
 		return
 	}
 	if err = f.Truncate(int64(size)); err != nil {
+		err = fmt.Errorf("failed to truncate shm file: %v", err)
 		f.Close()
 		os.Remove(f.Name())
 		return
@@ -277,11 +279,13 @@ func createMapping(size int) (f *os.File, mem []byte, err error) {
 	f.Close()
 	f, err = os.OpenFile(f.Name(), os.O_RDWR, 0)
 	if err != nil {
+		err = fmt.Errorf("failed to open shm file: %v", err)
 		os.Remove(f.Name())
 		return
 	}
 	mem, err = syscall.Mmap(int(f.Fd()), 0, size, syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
 	if err != nil {
+		err = fmt.Errorf("failed to mmap shm file: %v", err)
 		f.Close()
 		os.Remove(f.Name())
 		return
diff --git a/manager/main.go b/manager/main.go
index f83338918..079055b0b 100644
--- a/manager/main.go
+++ b/manager/main.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/google/syzkaller/sys"
 	"github.com/google/syzkaller/vm"
+	_ "github.com/google/syzkaller/vm/kvm"
 	_ "github.com/google/syzkaller/vm/local"
 	_ "github.com/google/syzkaller/vm/qemu"
 )
diff --git a/vm/kvm/kvm.go b/vm/kvm/kvm.go
new file mode 100644
index 000000000..c95d63a7e
--- /dev/null
+++ b/vm/kvm/kvm.go
@@ -0,0 +1,415 @@
+// Copyright 2015 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package kvm
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"sync"
+	"time"
+
+	"github.com/google/syzkaller/vm"
+)
+
+const hostAddr = "192.168.33.1"
+const logOutput = true
+
+func init() {
+	vm.Register("kvm", ctor)
+}
+
+type kvm struct {
+	params
+	workdir   string
+	crashdir  string
+	callsFlag string
+	id        int
+	mgrPort   int
+}
+
+type params struct {
+	Lkvm     string
+	Kernel   string
+	Cmdline  string
+	Fuzzer   string
+	Executor string
+	Cpu      int
+	Mem      int
+}
+
+func ctor(workdir string, syscalls map[int]bool, port, index int, paramsData []byte) (vm.Instance, error) {
+	p := new(params)
+	if err := json.Unmarshal(paramsData, p); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal kvm params: %v", err)
+	}
+	if _, err := os.Stat(p.Kernel); err != nil {
+		return nil, fmt.Errorf("kernel '%v' does not exist: %v", p.Kernel, err)
+	}
+	if _, err := os.Stat(p.Fuzzer); err != nil {
+		return nil, fmt.Errorf("fuzzer binary '%v' does not exist: %v", p.Fuzzer, err)
+	}
+	if _, err := os.Stat(p.Executor); err != nil {
+		return nil, fmt.Errorf("executor binary '%v' does not exist: %v", p.Executor, err)
+	}
+	if p.Lkvm == "" {
+		p.Lkvm = "lkvm"
+	}
+	if p.Cpu <= 0 || p.Cpu > 1024 {
+		return nil, fmt.Errorf("bad kvm cpu: %v, want [1-1024]", p.Cpu)
+	}
+	if p.Mem < 128 || p.Mem > 1048576 {
+		return nil, fmt.Errorf("bad kvm mem: %v, want [128-1048576]", p.Mem)
+	}
+
+	crashdir := filepath.Join(workdir, "crashes")
+	os.MkdirAll(crashdir, 0770)
+
+	workdir = filepath.Join(workdir, "kvm")
+	os.MkdirAll(workdir, 0770)
+
+	vm := &kvm{
+		params:   *p,
+		workdir:  workdir,
+		crashdir: crashdir,
+		id:       index,
+		mgrPort:  port,
+	}
+
+	if len(syscalls) != 0 {
+		buf := new(bytes.Buffer)
+		for c := range syscalls {
+			fmt.Fprintf(buf, ",%v", c)
+		}
+		vm.callsFlag = "-calls=" + buf.String()[1:]
+	}
+
+	return vm, nil
+}
+
+func (vm *kvm) Run() {
+	log.Printf("kvm/%v: started\n", vm.id)
+	sandbox := fmt.Sprintf("syz-%v", vm.id)
+	sandboxPath := filepath.Join(os.Getenv("HOME"), ".lkvm", sandbox)
+	for run := 0; ; run++ {
+		logname := filepath.Join(vm.workdir, fmt.Sprintf("log%v-%v-%v", vm.id, run, time.Now().Unix()))
+		var logf *os.File
+		if logOutput {
+			var err error
+			logf, err = os.Create(logname)
+			if err != nil {
+				log.Printf("failed to create log file: %v", err)
+				time.Sleep(10 * time.Second)
+				continue
+			}
+		}
+		rpipe, wpipe, err := os.Pipe()
+		if err != nil {
+			log.Printf("failed to create pipe: %v", err)
+			if logf != nil {
+				logf.Close()
+			}
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		os.RemoveAll(sandboxPath)
+		out, err := exec.Command(vm.Lkvm, "setup", sandbox).CombinedOutput()
+		if err != nil {
+			log.Printf("failed to lkvm setup: %v\n%s", err, out)
+			if logf != nil {
+				logf.Close()
+			}
+			rpipe.Close()
+			wpipe.Close()
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		if err := copyFile(vm.Fuzzer, filepath.Join(sandboxPath, "/syzkaller_fuzzer")); err != nil {
+			log.Printf("failed to copy file into sandbox: %v", err)
+			os.RemoveAll(sandboxPath)
+			if logf != nil {
+				logf.Close()
+			}
+			rpipe.Close()
+			wpipe.Close()
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		if err := copyFile(vm.Executor, filepath.Join(sandboxPath, "/syzkaller_executor")); err != nil {
+			log.Printf("failed to copy file into sandbox: %v", err)
+			os.RemoveAll(sandboxPath)
+			if logf != nil {
+				logf.Close()
+			}
+			rpipe.Close()
+			wpipe.Close()
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		inst := &Instance{
+			id:          vm.id,
+			crashdir:    vm.crashdir,
+			params:      vm.params,
+			name:        fmt.Sprintf("kvm/%v-%v", vm.id, run),
+			sandbox:     sandbox,
+			sandboxPath: sandboxPath,
+			callsFlag:   vm.callsFlag,
+			log:         logf,
+			rpipe:       rpipe,
+			wpipe:       wpipe,
+			mgrPort:     vm.mgrPort,
+			cmds:        make(map[*Command]bool),
+		}
+		inst.Run()
+		inst.Shutdown()
+		time.Sleep(10 * time.Second)
+	}
+}
+
+type Instance struct {
+	params
+	sync.Mutex
+	id          int
+	crashdir    string
+	name        string
+	sandbox     string
+	sandboxPath string
+	callsFlag   string
+	log         *os.File
+	rpipe       *os.File
+	wpipe       *os.File
+	mgrPort     int
+	cmds        map[*Command]bool
+	kvm         *Command
+}
+
+type Command struct {
+	sync.Mutex
+	cmd    *exec.Cmd
+	done   chan struct{}
+	failed bool
+	out    []byte
+	outpos int
+}
+
+func (inst *Instance) Run() {
+	var outputMu sync.Mutex
+	var output []byte
+	go func() {
+		var buf [64 << 10]byte
+		for {
+			n, err := inst.rpipe.Read(buf[:])
+			if n != 0 {
+				outputMu.Lock()
+				output = append(output, buf[:n]...)
+				outputMu.Unlock()
+				if inst.log != nil {
+					inst.log.Write(buf[:n])
+				}
+			}
+			if err != nil {
+				break
+			}
+		}
+	}()
+
+	// Start the instance.
+	inst.kvm = inst.CreateCommand(
+		inst.Lkvm, "sandbox",
+		"--disk", inst.sandbox,
+		fmt.Sprintf("--mem=%v", inst.Mem),
+		fmt.Sprintf("--cpus=%v", inst.Cpu),
+		"--kernel", inst.Kernel,
+		"--network", "mode=user",
+		"--", "/syzkaller_fuzzer",
+		"-name", inst.name,
+		"-executor", "/syzkaller_executor",
+		"-manager", fmt.Sprintf("%v:%v", hostAddr, inst.mgrPort),
+		inst.callsFlag,
+	)
+
+	start := time.Now()
+	deadline := start.Add(time.Hour)
+	lastOutput := time.Now()
+	lastOutputLen := 0
+	matchPos := 0
+	crashRe := regexp.MustCompile("\\[ cut here \\]|Kernel panic| BUG: | WARNING: | INFO: |unable to handle kernel NULL pointer dereference|general protection fault")
+	const contextSize = 64 << 10
+	for range time.NewTicker(5 * time.Second).C {
+		outputMu.Lock()
+		if lastOutputLen != len(output) {
+			lastOutput = time.Now()
+		}
+		if loc := crashRe.FindAllIndex(output[matchPos:], -1); len(loc) != 0 {
+			// Give it some time to finish writing the error message.
+			outputMu.Unlock()
+			time.Sleep(5 * time.Second)
+			outputMu.Lock()
+			loc = crashRe.FindAllIndex(output[matchPos:], -1)
+			start := loc[0][0] - contextSize
+			if start < 0 {
+				start = 0
+			}
+			end := loc[len(loc)-1][1] + contextSize
+			if end > len(output) {
+				end = len(output)
+			}
+			inst.SaveCrasher(output[start:end])
+		}
+		if len(output) > 2*contextSize {
+			copy(output, output[len(output)-contextSize:])
+			output = output[:contextSize]
+		}
+		matchPos = len(output) - 128
+		if matchPos < 0 {
+			matchPos = 0
+		}
+		lastOutputLen = len(output)
+		outputMu.Unlock()
+
+		if time.Since(lastOutput) > 3*time.Minute {
+			time.Sleep(time.Second)
+			outputMu.Lock()
+			output = append(output, "\nno output from fuzzer, restarting\n"...)
+			inst.SaveCrasher(output)
+			outputMu.Unlock()
+			inst.Logf("no output from fuzzer, restarting")
+			inst.kvm.cmd.Process.Kill()
+			inst.kvm.cmd.Process.Kill()
+			return
+		}
+		if inst.kvm.Exited() {
+			time.Sleep(time.Second)
+			outputMu.Lock()
+			output = append(output, "\nfuzzer binary stopped or lost connection\n"...)
+			inst.SaveCrasher(output)
+			outputMu.Unlock()
+			inst.Logf("fuzzer binary stopped or lost connection")
+			return
+		}
+		if time.Now().After(deadline) {
+			inst.Logf("running for long enough, restarting")
+			inst.kvm.cmd.Process.Kill()
+			inst.kvm.cmd.Process.Kill()
+			return
+		}
+	}
+}
+
+func (inst *Instance) SaveCrasher(output []byte) {
+	ioutil.WriteFile(filepath.Join(inst.crashdir, fmt.Sprintf("crash%v-%v", inst.id, time.Now().UnixNano())), output, 0660)
+}
+
+func (inst *Instance) Shutdown() {
+	defer func() {
+		os.RemoveAll(inst.sandboxPath)
+		inst.rpipe.Close()
+		inst.wpipe.Close()
+		if inst.log != nil {
+			inst.log.Close()
+		}
+	}()
+	if inst.kvm.cmd == nil {
+		// CreateCommand should have been failed very early.
+		return
+	}
+	for try := 0; try < 10; try++ {
+		inst.kvm.cmd.Process.Kill()
+		time.Sleep(time.Second)
+		inst.Lock()
+		n := len(inst.cmds)
+		inst.Unlock()
+		if n == 0 {
+			return
+		}
+	}
+	inst.Logf("hanged processes after kill")
+	inst.Lock()
+	for cmd := range inst.cmds {
+		cmd.cmd.Process.Kill()
+		cmd.cmd.Process.Kill()
+	}
+	inst.Unlock()
+	time.Sleep(3 * time.Second)
+}
+
+func (inst *Instance) CreateCommand(args ...string) *Command {
+	if inst.log != nil {
+		fmt.Fprintf(inst.log, "executing command: %v\n", args)
+	}
+	cmd := &Command{}
+	cmd.done = make(chan struct{})
+	cmd.cmd = exec.Command(args[0], args[1:]...)
+	cmd.cmd.Stdout = inst.wpipe
+	cmd.cmd.Stderr = inst.wpipe
+	if err := cmd.cmd.Start(); err != nil {
+		inst.Logf("failed to start command '%v': %v\n", args, err)
+		cmd.failed = true
+		close(cmd.done)
+		return cmd
+	}
+	inst.Lock()
+	inst.cmds[cmd] = true
+	inst.Unlock()
+	go func() {
+		err := cmd.cmd.Wait()
+		inst.Lock()
+		delete(inst.cmds, cmd)
+		inst.Unlock()
+		if inst.log != nil {
+			fmt.Fprintf(inst.log, "command '%v' exited: %v\n", args, err)
+		}
+		cmd.failed = err != nil
+		close(cmd.done)
+	}()
+	return cmd
+}
+
+func (inst *Instance) Logf(str string, args ...interface{}) {
+	fmt.Fprintf(inst.wpipe, str+"\n", args...)
+	log.Printf("%v: "+str, append([]interface{}{inst.name}, args...)...)
+}
+
+func (cmd *Command) Wait(max time.Duration) bool {
+	select {
+	case <-cmd.done:
+		return !cmd.failed
+	case <-time.After(max):
+		return false
+	}
+}
+
+func (cmd *Command) Exited() bool {
+	select {
+	case <-cmd.done:
+		return true
+	default:
+		return false
+	}
+}
+
+func copyFile(oldfn, newfn string) error {
+	oldf, err := os.Open(oldfn)
+	if err != nil {
+		return err
+	}
+	defer oldf.Close()
+	newf, err := os.Create(newfn)
+	if err != nil {
+		return err
+	}
+	defer newf.Close()
+	_, err = io.Copy(newf, oldf)
+	if err != nil {
+		return err
+	}
+	return nil
+}