syz-manager: display job lists for triage and smash jobs

This will let us understand better what exactly the fuzzer was doing.

3 files changed, 84 insertions, 12 deletions

diff --git a/pkg/fuzzer/fuzzer.go b/pkg/fuzzer/fuzzer.go
index afd131fc4..38a967251 100644
--- a/pkg/fuzzer/fuzzer.go
+++ b/pkg/fuzzer/fuzzer.go
@@ -29,6 +29,7 @@ type Fuzzer struct {
 	rnd          *rand.Rand
 	target       *prog.Target
 	hintsLimiter prog.HintsLimiter
+	runningJobs  map[jobIntrospector]struct{}
 
 	ct           *prog.ChoiceTable
 	ctProgs      int
@@ -50,9 +51,10 @@ func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand,
 		Config: cfg,
 		Cover:  newCover(),
 
-		ctx:    ctx,
-		rnd:    rnd,
-		target: target,
+		ctx:         ctx,
+		rnd:         rnd,
+		target:      target,
+		runningJobs: map[jobIntrospector]struct{}{},
 
 		// We're okay to lose some of the messages -- if we are already
 		// regenerating the table, we don't want to repeat it right away.
@@ -251,10 +253,24 @@ func (fuzzer *Fuzzer) startJob(stat *stat.Val, newJob job) {
 	fuzzer.Logf(2, "started %T", newJob)
 	go func() {
 		stat.Add(1)
+		defer stat.Add(-1)
+
 		fuzzer.statJobs.Add(1)
+		defer fuzzer.statJobs.Add(-1)
+
+		if obj, ok := newJob.(jobIntrospector); ok {
+			fuzzer.mu.Lock()
+			fuzzer.runningJobs[obj] = struct{}{}
+			fuzzer.mu.Unlock()
+
+			defer func() {
+				fuzzer.mu.Lock()
+				delete(fuzzer.runningJobs, obj)
+				fuzzer.mu.Unlock()
+			}()
+		}
+
 		newJob.run(fuzzer)
-		fuzzer.statJobs.Add(-1)
-		stat.Add(-1)
 	}()
 }
 
@@ -354,6 +370,17 @@ func (fuzzer *Fuzzer) ChoiceTable() *prog.ChoiceTable {
 	return fuzzer.ct
 }
 
+func (fuzzer *Fuzzer) RunningJobs() []JobInfo {
+	fuzzer.mu.Lock()
+	defer fuzzer.mu.Unlock()
+
+	var ret []JobInfo
+	for item := range fuzzer.runningJobs {
+		ret = append(ret, item.info())
+	}
+	return ret
+}
+
 func (fuzzer *Fuzzer) logCurrentStats() {
 	for {
 		select {
diff --git a/pkg/fuzzer/job.go b/pkg/fuzzer/job.go
index 7a89006f1..92ef5deb5 100644
--- a/pkg/fuzzer/job.go
+++ b/pkg/fuzzer/job.go
@@ -4,8 +4,11 @@
 package fuzzer
 
 import (
+	"fmt"
 	"math/rand"
+	"sort"
 	"sync"
+	"sync/atomic"
 
 	"github.com/google/syzkaller/pkg/corpus"
 	"github.com/google/syzkaller/pkg/cover"
@@ -19,6 +22,18 @@ type job interface {
 	run(fuzzer *Fuzzer)
 }
 
+type jobIntrospector interface {
+	info() JobInfo
+}
+
+type JobInfo struct {
+	ID    string
+	Calls []string
+	Type  string
+	Prog  *prog.Prog
+	Execs int
+}
+
 func genProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *queue.Request {
 	p := fuzzer.target.Generate(rnd,
 		prog.RecommendedCalls,
@@ -61,6 +76,7 @@ type triageJob struct {
 	queue    queue.Executor
 	// Set of calls that gave potential new coverage.
 	calls map[int]*triageCall
+	execs atomic.Int32
 }
 
 type triageCall struct {
@@ -107,6 +123,7 @@ const (
 )
 
 func (job *triageJob) execute(req *queue.Request, flags ProgFlags) *queue.Result {
+	defer job.execs.Add(1)
 	req.Important = true // All triage executions are important.
 	return job.fuzzer.executeWithFlags(job.queue, req, flags)
 }
@@ -365,10 +382,25 @@ func getSignalAndCover(p *prog.Prog, info *flatrpc.ProgInfo, call int) signal.Si
 	return signal.FromRaw(inf.Signal, signalPrio(p, inf, call))
 }
 
+func (job *triageJob) info() JobInfo {
+	ret := JobInfo{
+		ID:    fmt.Sprintf("%p", job),
+		Type:  "triage",
+		Execs: int(job.execs.Load()),
+		Prog:  job.p,
+	}
+	for id := range job.calls {
+		ret.Calls = append(ret.Calls, job.p.CallName(id))
+	}
+	sort.Strings(ret.Calls)
+	return ret
+}
+
 type smashJob struct {
-	exec queue.Executor
-	p    *prog.Prog
-	call int
+	exec  queue.Executor
+	p     *prog.Prog
+	call  int
+	execs atomic.Int32
 }
 
 func (job *smashJob) run(fuzzer *Fuzzer) {
@@ -390,6 +422,17 @@ func (job *smashJob) run(fuzzer *Fuzzer) {
 		if result.Stop() {
 			return
 		}
+		job.execs.Add(1)
+	}
+}
+
+func (job *smashJob) info() JobInfo {
+	return JobInfo{
+		ID:    fmt.Sprintf("%p", job),
+		Type:  "smash",
+		Execs: int(job.execs.Load()),
+		Prog:  job.p,
+		Calls: []string{job.p.CallName(job.call)},
 	}
 }
 
diff --git a/pkg/fuzzer/stats.go b/pkg/fuzzer/stats.go
index 073afab29..f0039d500 100644
--- a/pkg/fuzzer/stats.go
+++ b/pkg/fuzzer/stats.go
@@ -33,11 +33,13 @@ func newStats() Stats {
 			stat.Console, stat.Graph("corpus")),
 		statNewInputs: stat.New("new inputs", "Potential untriaged corpus candidates",
 			stat.Graph("corpus")),
-		statJobs:       stat.New("fuzzer jobs", "Total running fuzzer jobs", stat.NoGraph),
-		statJobsTriage: stat.New("triage jobs", "Running triage jobs", stat.StackedGraph("jobs")),
+		statJobs: stat.New("fuzzer jobs", "Total running fuzzer jobs", stat.NoGraph),
+		statJobsTriage: stat.New("triage jobs", "Running triage jobs", stat.StackedGraph("jobs"),
+			stat.Link("/jobs?type=triage")),
 		statJobsTriageCandidate: stat.New("candidate triage jobs", "Running candidate triage jobs",
-			stat.StackedGraph("jobs")),
-		statJobsSmash:          stat.New("smash jobs", "Running smash jobs", stat.StackedGraph("jobs")),
+			stat.StackedGraph("jobs"), stat.Link("/jobs?type=triage")),
+		statJobsSmash: stat.New("smash jobs", "Running smash jobs", stat.StackedGraph("jobs"),
+			stat.Link("/jobs?type=smash")),
 		statJobsFaultInjection: stat.New("fault jobs", "Running fault injection jobs", stat.StackedGraph("jobs")),
 		statJobsHints:          stat.New("hints jobs", "Running hints jobs", stat.StackedGraph("jobs")),
 		statExecTime:           stat.New("prog exec time", "Test program execution time (ms)", stat.Distribution{}),