tools/syz-testbed: produce results indefinitely

Creation of instances only at the start of the tool limits the scale of
the experiments.

Instead of setting up instances at the beginning, create, start and
stop them dynamically. This allows to conduct much larger experiments
while keeping the maximal resource consuption under control. When an
instance has been executing for the specified time, it is stopped and
replaced by another one.

Introduce two new parameters:
- max_instances - the maximal number of simultaneously running instances.
- run_hours - the lifetime (in hours) of a single instance.

The longer the tool runs, the more test results will be accumulated for
each checkout specified in the configuration file.

1 files changed, 67 insertions, 75 deletions

diff --git a/tools/syz-testbed/stats.go b/tools/syz-testbed/stats.go
index 9b5ed7133..f244bcbda 100644
--- a/tools/syz-testbed/stats.go
+++ b/tools/syz-testbed/stats.go
@@ -19,6 +19,25 @@ type BugInfo struct {
 	Title string
 }
 
+// The information collected from a syz-manager instance.
+type RunResult struct {
+	Workdir     string
+	Bugs        []BugInfo
+	StatRecords []map[string]uint64
+}
+
+// The grouping of single instance results. Taken by all stat generating routines.
+type RunResultGroup struct {
+	Name    string
+	Results []*RunResult
+}
+
+// Different "views" of the statistics, e.g. only completed instances or completed + running.
+type StatView struct {
+	Name   string
+	Groups []RunResultGroup
+}
+
 // TODO: we're implementing this functionaity at least the 3rd time (see syz-manager/html
 // and tools/reporter). Create a more generic implementation and put it into a globally
 // visible package.
@@ -57,7 +76,7 @@ func readBenches(benchFile string) ([]map[string]uint64, error) {
 	return ret, nil
 }
 
-func avgStats(infos []map[string]uint64) map[string]uint64 {
+func avgBenches(infos []map[string]uint64) map[string]uint64 {
 	ret := make(map[string]uint64)
 	if len(infos) == 0 {
 		return ret
@@ -75,29 +94,25 @@ func avgStats(infos []map[string]uint64) map[string]uint64 {
 
 type BugSummary struct {
 	title string
-	found map[*CheckoutInfo]bool
+	found map[string]bool
 }
 
 // If there are several instances belonging to a single checkout, we're interested in the
 // set of bugs found by at least one of those instances.
-func summarizeBugs(checkouts []*CheckoutInfo) ([]*BugSummary, error) {
+func summarizeBugs(groups []RunResultGroup) ([]*BugSummary, error) {
 	bugsMap := make(map[string]*BugSummary)
-	for _, checkout := range checkouts {
-		for _, instance := range checkout.Instances {
-			bugs, err := collectBugs(instance.Workdir)
-			if err != nil {
-				return nil, err
-			}
-			for _, bug := range bugs {
+	for _, group := range groups {
+		for _, result := range group.Results {
+			for _, bug := range result.Bugs {
 				summary := bugsMap[bug.Title]
 				if summary == nil {
 					summary = &BugSummary{
 						title: bug.Title,
-						found: make(map[*CheckoutInfo]bool),
+						found: make(map[string]bool),
 					}
 					bugsMap[bug.Title] = summary
 				}
-				summary.found[checkout] = true
+				summary.found[group.Name] = true
 			}
 		}
 	}
@@ -110,13 +125,13 @@ func summarizeBugs(checkouts []*CheckoutInfo) ([]*BugSummary, error) {
 
 // For each checkout, take the union of sets of bugs found by each instance.
 // Then output these unions as a single table.
-func generateBugTable(checkouts []*CheckoutInfo) ([][]string, error) {
+func (view StatView) GenerateBugTable() ([][]string, error) {
 	table := [][]string{}
 	titles := []string{""}
-	for _, checkout := range checkouts {
-		titles = append(titles, checkout.Name)
+	for _, group := range view.Groups {
+		titles = append(titles, group.Name)
 	}
-	summaries, err := summarizeBugs(checkouts)
+	summaries, err := summarizeBugs(view.Groups)
 	if err != nil {
 		return nil, err
 	}
@@ -124,9 +139,9 @@ func generateBugTable(checkouts []*CheckoutInfo) ([][]string, error) {
 	table = append(table, titles)
 	for _, bug := range summaries {
 		row := []string{bug.title}
-		for _, checkout := range checkouts {
+		for _, group := range view.Groups {
 			val := ""
-			if bug.found[checkout] {
+			if bug.found[group.Name] {
 				val = "YES"
 			}
 			row = append(row, val)
@@ -136,26 +151,32 @@ func generateBugTable(checkouts []*CheckoutInfo) ([][]string, error) {
 	return table, nil
 }
 
-type StatGroup struct {
-	Name      string
-	Instances []InstanceInfo
+func (group RunResultGroup) AvgStatRecords() []map[string]uint64 {
+	ret := []map[string]uint64{}
+	for i := 0; ; i++ {
+		toAvg := []map[string]uint64{}
+		for _, result := range group.Results {
+			if i < len(result.StatRecords) {
+				toAvg = append(toAvg, result.StatRecords[i])
+			}
+		}
+		if len(toAvg) != len(group.Results) || len(toAvg) == 0 {
+			break
+		}
+		ret = append(ret, avgBenches(toAvg))
+	}
+	return ret
 }
 
-func genericStatsTable(groups []StatGroup) ([][]string, error) {
+func (view StatView) StatsTable() ([][]string, error) {
 	// Map: stats key x group name -> value.
 	cells := make(map[string]map[string]string)
-	for _, group := range groups {
-		infos := []map[string]uint64{}
-		for _, instance := range group.Instances {
-			records, err := readBenches(instance.BenchFile)
-			if err != nil {
-				return nil, err
-			}
-			if len(records) > 0 {
-				infos = append(infos, records[len(records)-1])
-			}
+	for _, group := range view.Groups {
+		avgBench := group.AvgStatRecords()
+		if len(avgBench) == 0 {
+			continue
 		}
-		for key, value := range avgStats(infos) {
+		for key, value := range avgBench[len(avgBench)-1] {
 			if _, ok := cells[key]; !ok {
 				cells[key] = make(map[string]string)
 			}
@@ -163,13 +184,13 @@ func genericStatsTable(groups []StatGroup) ([][]string, error) {
 		}
 	}
 	title := []string{""}
-	for _, group := range groups {
+	for _, group := range view.Groups {
 		title = append(title, group.Name)
 	}
 	table := [][]string{title}
 	for key, valuesMap := range cells {
 		row := []string{key}
-		for _, group := range groups {
+		for _, group := range view.Groups {
 			row = append(row, valuesMap[group.Name])
 		}
 		table = append(table, row)
@@ -177,56 +198,27 @@ func genericStatsTable(groups []StatGroup) ([][]string, error) {
 	return table, nil
 }
 
-func checkoutStatsTable(checkouts []*CheckoutInfo) ([][]string, error) {
-	groups := []StatGroup{}
-	for _, checkout := range checkouts {
-		groups = append(groups, StatGroup{
-			Name:      checkout.Name,
-			Instances: checkout.Instances,
-		})
-	}
-	return genericStatsTable(groups)
-}
-
-func instanceStatsTable(checkouts []*CheckoutInfo) ([][]string, error) {
-	groups := []StatGroup{}
-	for _, checkout := range checkouts {
-		for _, instance := range checkout.Instances {
-			groups = append(groups, StatGroup{
-				Name:      instance.Name,
-				Instances: []InstanceInfo{instance},
+func (view StatView) InstanceStatsTable() ([][]string, error) {
+	newView := StatView{}
+	for _, group := range view.Groups {
+		for i, result := range group.Results {
+			newView.Groups = append(newView.Groups, RunResultGroup{
+				Name:    fmt.Sprintf("%s-%d", group.Name, i),
+				Results: []*RunResult{result},
 			})
 		}
 	}
-	return genericStatsTable(groups)
+	return newView.StatsTable()
 }
 
 // Average bench files of several instances into a single bench file.
-func saveAvgBenchFile(checkout *CheckoutInfo, fileName string) error {
-	allRecords := [][]map[string]uint64{}
-	for _, instance := range checkout.Instances {
-		records, err := readBenches(instance.BenchFile)
-		if err != nil {
-			return err
-		}
-		allRecords = append(allRecords, records)
-	}
+func (group *RunResultGroup) SaveAvgBenchFile(fileName string) error {
 	f, err := os.Create(fileName)
 	if err != nil {
 		return err
 	}
 	defer f.Close()
-	for i := 0; ; i++ {
-		toAvg := []map[string]uint64{}
-		for _, records := range allRecords {
-			if i < len(records) {
-				toAvg = append(toAvg, records[i])
-			}
-		}
-		if len(toAvg) != len(allRecords) {
-			break
-		}
-		averaged := avgStats(toAvg)
+	for _, averaged := range group.AvgStatRecords() {
 		data, err := json.MarshalIndent(averaged, "", "  ")
 		if err != nil {
 			return err
@@ -238,7 +230,7 @@ func saveAvgBenchFile(checkout *CheckoutInfo, fileName string) error {
 	return nil
 }
 
-func saveTableAsCsv(table [][]string, fileName string) error {
+func SaveTableAsCsv(table [][]string, fileName string) error {
 	f, err := os.Create(fileName)
 	if err != nil {
 		return err