pkg/runtest: add package for syzkaller program unit-testing

Package runtest is a driver for end-to-end testing of syzkaller programs.
It tests program execution via both executor and csource,
with different sandboxes and execution modes (threaded, repeated, etc).
It can run test OS programs locally via run_test.go
and all other real OS programs via tools/syz-runtest
which uses manager config to wind up VMs.
Test programs are located in sys/*/test/* files.

Update #603

2 files changed, 483 insertions, 0 deletions

diff --git a/pkg/runtest/run.go b/pkg/runtest/run.go
new file mode 100644
index 000000000..0c79a531f
--- /dev/null
+++ b/pkg/runtest/run.go
@@ -0,0 +1,411 @@
+// Copyright 2018 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// Package runtest is a driver for end-to-end testing of syzkaller programs.
+// It tests program execution via both executor and csource,
+// with different sandboxes and execution modes (threaded, repeated, etc).
+// It can run test OS programs locally via run_test.go
+// and all other real OS programs via tools/syz-runtest
+// which uses manager config to wind up VMs.
+// Test programs are located in sys/*/test/* files.
+package runtest
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/google/syzkaller/pkg/csource"
+	"github.com/google/syzkaller/pkg/host"
+	"github.com/google/syzkaller/pkg/ipc"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/sys/targets"
+)
+
+type RunRequest struct {
+	Bin    string
+	P      *prog.Prog
+	Cfg    *ipc.Config
+	Opts   *ipc.ExecOpts
+	Repeat int
+
+	Done   chan struct{}
+	Output []byte
+	Info   [][]ipc.CallInfo
+	Err    error
+
+	results []int
+	name    string
+	broken  string
+	skip    string
+}
+
+type Context struct {
+	Dir          string
+	Target       *prog.Target
+	Features     *host.Features
+	EnabledCalls map[string]map[*prog.Syscall]bool
+	Requests     chan *RunRequest
+	LogFunc      func(text string)
+}
+
+func (ctx *Context) log(msg string, args ...interface{}) {
+	ctx.LogFunc(fmt.Sprintf(msg, args...))
+}
+
+func (ctx *Context) Run() error {
+	progs := make(chan *RunRequest, 2*cap(ctx.Requests))
+	errc := make(chan error, 1)
+	go func() {
+		defer close(progs)
+		defer close(ctx.Requests)
+		errc <- ctx.generatePrograms(progs)
+	}()
+	var ok, fail, broken, skip int
+	for req := range progs {
+		<-req.Done
+		if req.Bin != "" {
+			os.Remove(req.Bin)
+		}
+		result := ""
+		if req.broken != "" {
+			broken++
+			result = fmt.Sprintf("BROKEN (%v)", req.broken)
+		} else if req.skip != "" {
+			skip++
+			result = fmt.Sprintf("SKIP (%v)", req.skip)
+		} else {
+			if req.Err == nil {
+				req.Err = checkResult(req)
+			}
+			if req.Err != nil {
+				fail++
+				result = fmt.Sprintf("FAIL: %v",
+					strings.Replace(req.Err.Error(), "\n", "\n\t", -1))
+				if len(req.Output) != 0 {
+					result += fmt.Sprintf("\n\t%s",
+						strings.Replace(string(req.Output), "\n", "\n\t", -1))
+				}
+			} else {
+				ok++
+				result = "OK"
+			}
+		}
+		ctx.log("%-36v: %v", req.name, result)
+	}
+	if err := <-errc; err != nil {
+		return err
+	}
+	ctx.log("ok: %v, broken: %v, skip: %v, fail: %v", ok, broken, skip, fail)
+	if fail != 0 {
+		return fmt.Errorf("tests failed")
+	}
+	return nil
+}
+
+func (ctx *Context) generatePrograms(progs chan *RunRequest) error {
+	files, err := ioutil.ReadDir(ctx.Dir)
+	if err != nil {
+		return fmt.Errorf("failed to read %v: %v", ctx.Dir, err)
+	}
+	cover := []bool{false}
+	if ctx.Features[host.FeatureCoverage].Enabled {
+		cover = append(cover, true)
+	}
+	var sandboxes []string
+	for sandbox := range ctx.EnabledCalls {
+		sandboxes = append(sandboxes, sandbox)
+	}
+	sort.Strings(sandboxes)
+	sysTarget := targets.Get(ctx.Target.OS, ctx.Target.Arch)
+	closedDone := make(chan struct{})
+	close(closedDone)
+	for _, file := range files {
+		if strings.HasSuffix(file.Name(), "~") {
+			continue
+		}
+		p, results, err := ctx.parseProg(file.Name())
+		if err != nil {
+			return err
+		}
+	nextSandbox:
+		for _, sandbox := range sandboxes {
+			name := fmt.Sprintf("%v %v", file.Name(), sandbox)
+			for _, call := range p.Calls {
+				if !ctx.EnabledCalls[sandbox][call.Meta] {
+					progs <- &RunRequest{
+						Done: closedDone,
+						name: name,
+						skip: fmt.Sprintf("unsupported call %v", call.Meta.Name),
+					}
+					continue nextSandbox
+				}
+			}
+			for _, threaded := range []bool{false, true} {
+				name := name
+				if threaded {
+					name += "/thr"
+				}
+				for _, cov := range cover {
+					if sandbox == "" {
+						break // executor does not support empty sandbox
+					}
+					name := name
+					if cov {
+						name += "/cover"
+					}
+					req, err := ctx.createSyzTest(p, sandbox, threaded, cov)
+					if err != nil {
+						return err
+					}
+					ctx.produceTest(progs, req, name, results)
+				}
+				for _, times := range []int{1, 3} {
+					name := name
+					if times > 1 {
+						name += "/repeat"
+					}
+					name += " C"
+					if !sysTarget.ExecutorUsesForkServer && times > 1 {
+						// Non-fork loop implementation does not support repetition.
+						progs <- &RunRequest{
+							Done:   closedDone,
+							name:   name,
+							broken: "non-forking loop",
+						}
+						continue
+					}
+					req, err := ctx.createCTest(p, sandbox, threaded, times)
+					if err != nil {
+						return err
+					}
+					ctx.produceTest(progs, req, name, results)
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (ctx *Context) parseProg(filename string) (*prog.Prog, []int, error) {
+	data, err := ioutil.ReadFile(filepath.Join(ctx.Dir, filename))
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to read %v: %v", filename, err)
+	}
+	p, err := ctx.Target.Deserialize(data)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to deserialize %v: %v", filename, err)
+	}
+	results := make([]int, len(p.Calls))
+	for i, call := range p.Calls {
+		switch call.Comment {
+		case "":
+		case "EINVAL":
+			results[i] = int(syscall.EINVAL)
+		case "EBADF":
+			results[i] = int(syscall.EBADF)
+		case "ENOMEM":
+			results[i] = int(syscall.ENOMEM)
+		default:
+			return nil, nil, fmt.Errorf("%v: unknown comment %q", filename, call.Comment)
+		}
+	}
+	return p, results, nil
+}
+
+func (ctx *Context) produceTest(progs chan *RunRequest, req *RunRequest, name string, results []int) {
+	req.name = name
+	req.results = results
+	req.Done = make(chan struct{})
+	ctx.Requests <- req
+	progs <- req
+}
+
+func (ctx *Context) createSyzTest(p *prog.Prog, sandbox string, threaded, cov bool) (*RunRequest, error) {
+	sysTarget := targets.Get(p.Target.OS, p.Target.Arch)
+	cfg := new(ipc.Config)
+	opts := new(ipc.ExecOpts)
+	if sysTarget.ExecutorUsesShmem {
+		cfg.Flags |= ipc.FlagUseShmem
+	}
+	if sysTarget.ExecutorUsesForkServer {
+		cfg.Flags |= ipc.FlagUseForkServer
+	}
+	switch sandbox {
+	case "namespace":
+		cfg.Flags |= ipc.FlagSandboxNamespace
+	case "setuid":
+		cfg.Flags |= ipc.FlagSandboxSetuid
+	}
+	if threaded {
+		opts.Flags |= ipc.FlagThreaded | ipc.FlagCollide
+	}
+	if cov {
+		cfg.Flags |= ipc.FlagSignal
+		opts.Flags |= ipc.FlagCollectCover | ipc.FlagDedupCover
+	}
+	if ctx.Features[host.FeatureNetworkInjection].Enabled {
+		cfg.Flags |= ipc.FlagEnableTun
+	}
+	if ctx.Features[host.FeatureNetworkDevices].Enabled {
+		cfg.Flags |= ipc.FlagEnableNetDev
+	}
+	req := &RunRequest{
+		P:      p,
+		Cfg:    cfg,
+		Opts:   opts,
+		Repeat: 3,
+	}
+	return req, nil
+}
+
+func (ctx *Context) createCTest(p *prog.Prog, sandbox string, threaded bool, times int) (*RunRequest, error) {
+	opts := csource.Options{
+		Threaded:    threaded,
+		Collide:     false,
+		Repeat:      times > 1,
+		RepeatTimes: times,
+		Procs:       1,
+		Sandbox:     sandbox,
+		UseTmpDir:   true,
+		HandleSegv:  true,
+		Trace:       true,
+	}
+	if sandbox != "" {
+		if ctx.Features[host.FeatureNetworkInjection].Enabled {
+			opts.EnableTun = true
+		}
+		if ctx.Features[host.FeatureNetworkDevices].Enabled {
+			opts.EnableNetdev = true
+		}
+	}
+	src, err := csource.Write(p, opts)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create C source: %v", err)
+	}
+	bin, err := csource.Build(p.Target, src)
+	if err != nil {
+		return nil, fmt.Errorf("failed to build C program: %v", err)
+	}
+	req := &RunRequest{
+		P:      p,
+		Bin:    bin,
+		Repeat: times,
+	}
+	return req, nil
+}
+
+func checkResult(req *RunRequest) error {
+	if req.Bin != "" {
+		var err error
+		if req.Info, err = parseBinOutput(req); err != nil {
+			return err
+		}
+	}
+	if req.Repeat != len(req.Info) {
+		return fmt.Errorf("should repeat %v times, but repeated %v",
+			req.Repeat, len(req.Info))
+	}
+	for run, info := range req.Info {
+		for i, inf := range info {
+			if inf.Flags&ipc.CallExecuted == 0 {
+				return fmt.Errorf("run %v: call %v is not executed", run, i)
+			}
+			if inf.Errno != req.results[i] {
+				return fmt.Errorf("run %v: wrong call %v result %v, want %v",
+					run, i, inf.Errno, req.results[i])
+			}
+		}
+	}
+	return nil
+}
+
+func parseBinOutput(req *RunRequest) ([][]ipc.CallInfo, error) {
+	var infos [][]ipc.CallInfo
+	s := bufio.NewScanner(bytes.NewReader(req.Output))
+	re := regexp.MustCompile("^### call=([0-9]+) errno=([0-9]+)$")
+	for s.Scan() {
+		if s.Text() == "### start" {
+			infos = append(infos, make([]ipc.CallInfo, len(req.P.Calls)))
+		}
+		match := re.FindSubmatch(s.Bytes())
+		if match == nil {
+			continue
+		}
+		if len(infos) == 0 {
+			return nil, fmt.Errorf("call completed without start")
+		}
+		call, err := strconv.ParseUint(string(match[1]), 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse call %q in %q",
+				string(match[1]), s.Text())
+		}
+		errno, err := strconv.ParseUint(string(match[2]), 10, 32)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse errno %q in %q",
+				string(match[2]), s.Text())
+		}
+		info := infos[len(infos)-1]
+		if call >= uint64(len(info)) {
+			return nil, fmt.Errorf("bad call index %v", call)
+		}
+		if info[call].Flags != 0 {
+			return nil, fmt.Errorf("double result for call %v", call)
+		}
+		info[call].Flags |= ipc.CallExecuted
+		info[call].Errno = int(errno)
+	}
+	return infos, nil
+}
+
+func RunTest(req *RunRequest, executor string) {
+	if req.Bin != "" {
+		tmpDir, err := ioutil.TempDir("", "syz-runtest")
+		if err != nil {
+			req.Err = fmt.Errorf("failed to create temp dir: %v", err)
+			return
+		}
+		defer os.RemoveAll(tmpDir)
+		req.Output, req.Err = osutil.RunCmd(15*time.Second, tmpDir, req.Bin)
+		return
+	}
+	req.Cfg.Executor = executor
+	env, err := ipc.MakeEnv(req.Cfg, 0)
+	if err != nil {
+		req.Err = fmt.Errorf("failed to create ipc env: %v", err)
+		return
+	}
+	defer env.Close()
+	for run := 0; run < req.Repeat; run++ {
+		output, info, failed, hanged, err := env.Exec(req.Opts, req.P)
+		req.Output = append(req.Output, output...)
+		if err != nil {
+			req.Err = fmt.Errorf("run %v: failed to run: %v", run, err)
+			return
+		}
+		if failed {
+			req.Err = fmt.Errorf("run %v: failed", run)
+			return
+		}
+		if hanged {
+			req.Err = fmt.Errorf("run %v: hanged", run)
+			return
+		}
+		for i := range info {
+			// Detach them because they point into the output shmem region.
+			info[i].Signal = append([]uint32{}, info[i].Signal...)
+			info[i].Cover = append([]uint32{}, info[i].Cover...)
+		}
+		req.Info = append(req.Info, info)
+	}
+}
diff --git a/pkg/runtest/run_test.go b/pkg/runtest/run_test.go
new file mode 100644
index 000000000..c856f58be
--- /dev/null
+++ b/pkg/runtest/run_test.go
@@ -0,0 +1,72 @@
+// Copyright 2018 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package runtest
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/google/syzkaller/pkg/csource"
+	"github.com/google/syzkaller/pkg/host"
+	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/sys/targets"
+	_ "github.com/google/syzkaller/sys/test/gen" // pull in the test target
+)
+
+func Test(t *testing.T) {
+	for _, sysTarget := range targets.List["test"] {
+		sysTarget1 := targets.Get(sysTarget.OS, sysTarget.Arch)
+		t.Run(sysTarget1.Arch, func(t *testing.T) {
+			t.Parallel()
+			test(t, sysTarget1)
+		})
+	}
+}
+
+func test(t *testing.T, sysTarget *targets.Target) {
+	target, err := prog.GetTarget(sysTarget.OS, sysTarget.Arch)
+	if err != nil {
+		t.Fatal(err)
+	}
+	executor, err := csource.BuildFile(target, filepath.FromSlash("../../executor/executor.cc"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.Remove(executor)
+	features, err := host.Check(target)
+	if err != nil {
+		t.Fatalf("failed to detect host features: %v", err)
+	}
+	calls, _, err := host.DetectSupportedSyscalls(target, "none")
+	if err != nil {
+		t.Fatalf("failed to detect supported syscalls: %v", err)
+	}
+	enabledCalls := map[string]map[*prog.Syscall]bool{
+		"":     calls,
+		"none": calls,
+	}
+	requests := make(chan *RunRequest, 2*runtime.GOMAXPROCS(0))
+	go func() {
+		for req := range requests {
+			RunTest(req, executor)
+			close(req.Done)
+		}
+	}()
+	ctx := &Context{
+		Dir:          filepath.Join("..", "..", "sys", target.OS, "test"),
+		Target:       target,
+		Features:     features,
+		EnabledCalls: enabledCalls,
+		Requests:     requests,
+		LogFunc: func(text string) {
+			t.Helper()
+			t.Logf(text)
+		},
+	}
+	if err := ctx.Run(); err != nil {
+		t.Fatal(err)
+	}
+}