tools/syz-testbed: support multiple targets

Refactor the code of syz-testbed to enable support of different targets.

This required the following changes:
- Instead of doing mass start and mass restart of instances, let them be
  more independent.
- Support different types of instances, which may have different
  expectations of the target behavior. E.g. syz-manager normally never
  exits, while syz-repro is expected to exit after a certain time.
- Factor out stats management, as stat entries may be different for
  different targets.
- Introduce locking to TestbedCtx and Checkout, as slices there can be
  modified at arbitrary times.

1 files changed, 87 insertions, 47 deletions

diff --git a/tools/syz-testbed/instance.go b/tools/syz-testbed/instance.go
index 888ad9ff7..f34df8a98 100644
--- a/tools/syz-testbed/instance.go
+++ b/tools/syz-testbed/instance.go
@@ -14,28 +14,35 @@ import (
 	"github.com/google/syzkaller/pkg/osutil"
 )
 
+type Instance interface {
+	Run() error
+	Stop()
+	FetchResult() (RunResult, error)
+}
+
 // The essential information about an active instance.
-type Instance struct {
+type InstanceCommon struct {
 	Name            string
-	Workdir         string
-	BenchFile       string
 	LogFile         string
 	ExecCommand     string
 	ExecCommandArgs []string
 	stopChannel     chan bool
 }
 
-func (inst *Instance) Run() error {
+func (inst *InstanceCommon) Run() error {
 	const stopDelay = time.Minute
 
-	logfile, err := os.Create(inst.LogFile)
-	if err != nil {
-		return fmt.Errorf("[%s] failed to create logfile: %s", inst.Name, err)
-	}
 	log.Printf("[%s] starting", inst.Name)
 	cmd := osutil.GraciousCommand(inst.ExecCommand, inst.ExecCommandArgs...)
-	cmd.Stdout = logfile
-	cmd.Stderr = logfile
+
+	if inst.LogFile != "" {
+		logfile, err := os.Create(inst.LogFile)
+		if err != nil {
+			return fmt.Errorf("[%s] failed to create logfile: %s", inst.Name, err)
+		}
+		cmd.Stdout = logfile
+		cmd.Stderr = logfile
+	}
 
 	complete := make(chan error)
 	cmd.Start()
@@ -44,8 +51,8 @@ func (inst *Instance) Run() error {
 	}()
 
 	select {
-	case err := <-complete:
-		return fmt.Errorf("[%s] stopped: %s", inst.Name, err)
+	case <-complete:
+		return nil
 	case <-inst.stopChannel:
 		// TODO: handle other OSes?
 		cmd.Process.Signal(os.Interrupt)
@@ -62,14 +69,20 @@ func (inst *Instance) Run() error {
 	}
 }
 
-func (inst *Instance) Stop() {
+func (inst *InstanceCommon) Stop() {
 	select {
 	case inst.stopChannel <- true:
 	default:
 	}
 }
 
-func (inst *Instance) FetchResult() (*RunResult, error) {
+type SyzManagerInstance struct {
+	InstanceCommon
+	SyzkallerInfo
+	RunTime time.Duration
+}
+
+func (inst *SyzManagerInstance) FetchResult() (RunResult, error) {
 	bugs, err := collectBugs(inst.Workdir)
 	if err != nil {
 		return nil, err
@@ -78,39 +91,45 @@ func (inst *Instance) FetchResult() (*RunResult, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &RunResult{
-		Workdir:     inst.Workdir,
+	return &SyzManagerResult{
 		Bugs:        bugs,
 		StatRecords: records,
 	}, nil
 }
 
-func (ctx *TestbedContext) NewInstance(checkout *Checkout, mgrName string) (*Instance, error) {
-	defer func() {
-		ctx.NextInstanceID++
+func (inst *SyzManagerInstance) Run() error {
+	ret := make(chan error, 1)
+	go func() {
+		ret <- inst.InstanceCommon.Run()
 	}()
-	name := fmt.Sprintf("%s-%d", checkout.Name, ctx.NextInstanceID)
-	managerCfgPath := filepath.Join(checkout.Path, fmt.Sprintf("syz-%d.cnf", ctx.NextInstanceID))
-	workdir := filepath.Join(checkout.Path, fmt.Sprintf("workdir_%d", ctx.NextInstanceID))
-	bench := filepath.Join(checkout.Path, fmt.Sprintf("bench-%d.txt", ctx.NextInstanceID))
-	logFile := filepath.Join(checkout.Path, fmt.Sprintf("log-%d.txt", ctx.NextInstanceID))
 
-	log.Printf("[%s] Generating workdir", name)
+	select {
+	case err := <-ret:
+		// Syz-managers are not supposed to stop themselves under normal circumstances.
+		// If one of them did stop, there must have been a very good reason to do so.
+		return fmt.Errorf("[%s] stopped: %v", inst.Name, err)
+	case <-time.After(inst.RunTime):
+		inst.Stop()
+		<-ret
+		return nil
+	}
+}
+
+type SyzkallerInfo struct {
+	Workdir   string
+	CfgFile   string
+	BenchFile string
+}
+
+func SetupSyzkallerInstance(mgrName, folder string, checkout *Checkout) (*SyzkallerInfo, error) {
+	workdir := filepath.Join(folder, "workdir")
+	log.Printf("[%s] Generating workdir", mgrName)
 	err := osutil.MkdirAll(workdir)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create workdir %s", workdir)
 	}
-
-	if ctx.Config.Corpus != "" {
-		log.Printf("[%s] Copying corpus", name)
-		corpusPath := filepath.Join(workdir, "corpus.db")
-		err = osutil.CopyFile(ctx.Config.Corpus, corpusPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to copy corpus from %s: %s", ctx.Config.Corpus, err)
-		}
-	}
-
-	log.Printf("[%s] Generating syz-manager config", name)
+	log.Printf("[%s] Generating syz-manager config", mgrName)
+	cfgFile := filepath.Join(folder, "manager.cfg")
 	managerCfg, err := config.PatchJSON(checkout.ManagerConfig, map[string]interface{}{
 		"name":      mgrName,
 		"workdir":   workdir,
@@ -119,19 +138,40 @@ func (ctx *TestbedContext) NewInstance(checkout *Checkout, mgrName string) (*Ins
 	if err != nil {
 		return nil, fmt.Errorf("failed to patch mgr config")
 	}
-
-	err = osutil.WriteFile(managerCfgPath, managerCfg)
+	err = osutil.WriteFile(cfgFile, managerCfg)
 	if err != nil {
-		return nil, fmt.Errorf("failed to save manager config to %s: %s", managerCfgPath, err)
+		return nil, fmt.Errorf("failed to save manager config to %s: %s", cfgFile, err)
 	}
+	return &SyzkallerInfo{
+		Workdir:   workdir,
+		CfgFile:   cfgFile,
+		BenchFile: filepath.Join(folder, "bench.txt"),
+	}, nil
+}
 
-	return &Instance{
-		Name:            name,
-		Workdir:         workdir,
-		BenchFile:       bench,
-		LogFile:         logFile,
-		ExecCommand:     filepath.Join(checkout.Path, "bin", "syz-manager"),
-		ExecCommandArgs: []string{"-config", managerCfgPath, "-bench", bench},
-		stopChannel:     make(chan bool, 1),
+func (t *SyzManagerTarget) newSyzManagerInstance(slotName, uniqName string, checkout *Checkout) (Instance, error) {
+	folder := filepath.Join(checkout.Path, fmt.Sprintf("run-%s", uniqName))
+	common, err := SetupSyzkallerInstance(slotName, folder, checkout)
+	if err != nil {
+		return nil, err
+	}
+	if t.config.Corpus != "" {
+		log.Printf("[%s] Copying corpus", uniqName)
+		corpusPath := filepath.Join(common.Workdir, "corpus.db")
+		err = osutil.CopyFile(t.config.Corpus, corpusPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to copy corpus from %s: %s", t.config.Corpus, err)
+		}
+	}
+	return &SyzManagerInstance{
+		InstanceCommon: InstanceCommon{
+			Name:            uniqName,
+			LogFile:         filepath.Join(folder, "log.txt"),
+			ExecCommand:     filepath.Join(checkout.Path, "bin", "syz-manager"),
+			ExecCommandArgs: []string{"-config", common.CfgFile, "-bench", common.BenchFile},
+			stopChannel:     make(chan bool, 1),
+		},
+		SyzkallerInfo: *common,
+		RunTime:       t.config.RunTime.Duration,
 	}, nil
 }