syz-manager: display job lists for triage and smash jobs

This will let us understand better what exactly the fuzzer was doing.

4 files changed, 177 insertions, 12 deletions

diff --git a/pkg/fuzzer/fuzzer.go b/pkg/fuzzer/fuzzer.go
index afd131fc4..38a967251 100644
--- a/pkg/fuzzer/fuzzer.go
+++ b/pkg/fuzzer/fuzzer.go
@@ -29,6 +29,7 @@ type Fuzzer struct {
 	rnd          *rand.Rand
 	target       *prog.Target
 	hintsLimiter prog.HintsLimiter
+	runningJobs  map[jobIntrospector]struct{}
 
 	ct           *prog.ChoiceTable
 	ctProgs      int
@@ -50,9 +51,10 @@ func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand,
 		Config: cfg,
 		Cover:  newCover(),
 
-		ctx:    ctx,
-		rnd:    rnd,
-		target: target,
+		ctx:         ctx,
+		rnd:         rnd,
+		target:      target,
+		runningJobs: map[jobIntrospector]struct{}{},
 
 		// We're okay to lose some of the messages -- if we are already
 		// regenerating the table, we don't want to repeat it right away.
@@ -251,10 +253,24 @@ func (fuzzer *Fuzzer) startJob(stat *stat.Val, newJob job) {
 	fuzzer.Logf(2, "started %T", newJob)
 	go func() {
 		stat.Add(1)
+		defer stat.Add(-1)
+
 		fuzzer.statJobs.Add(1)
+		defer fuzzer.statJobs.Add(-1)
+
+		if obj, ok := newJob.(jobIntrospector); ok {
+			fuzzer.mu.Lock()
+			fuzzer.runningJobs[obj] = struct{}{}
+			fuzzer.mu.Unlock()
+
+			defer func() {
+				fuzzer.mu.Lock()
+				delete(fuzzer.runningJobs, obj)
+				fuzzer.mu.Unlock()
+			}()
+		}
+
 		newJob.run(fuzzer)
-		fuzzer.statJobs.Add(-1)
-		stat.Add(-1)
 	}()
 }
 
@@ -354,6 +370,17 @@ func (fuzzer *Fuzzer) ChoiceTable() *prog.ChoiceTable {
 	return fuzzer.ct
 }
 
+func (fuzzer *Fuzzer) RunningJobs() []JobInfo {
+	fuzzer.mu.Lock()
+	defer fuzzer.mu.Unlock()
+
+	var ret []JobInfo
+	for item := range fuzzer.runningJobs {
+		ret = append(ret, item.info())
+	}
+	return ret
+}
+
 func (fuzzer *Fuzzer) logCurrentStats() {
 	for {
 		select {
diff --git a/pkg/fuzzer/job.go b/pkg/fuzzer/job.go
index 7a89006f1..92ef5deb5 100644
--- a/pkg/fuzzer/job.go
+++ b/pkg/fuzzer/job.go
@@ -4,8 +4,11 @@
 package fuzzer
 
 import (
+	"fmt"
 	"math/rand"
+	"sort"
 	"sync"
+	"sync/atomic"
 
 	"github.com/google/syzkaller/pkg/corpus"
 	"github.com/google/syzkaller/pkg/cover"
@@ -19,6 +22,18 @@ type job interface {
 	run(fuzzer *Fuzzer)
 }
 
+type jobIntrospector interface {
+	info() JobInfo
+}
+
+type JobInfo struct {
+	ID    string
+	Calls []string
+	Type  string
+	Prog  *prog.Prog
+	Execs int
+}
+
 func genProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *queue.Request {
 	p := fuzzer.target.Generate(rnd,
 		prog.RecommendedCalls,
@@ -61,6 +76,7 @@ type triageJob struct {
 	queue    queue.Executor
 	// Set of calls that gave potential new coverage.
 	calls map[int]*triageCall
+	execs atomic.Int32
 }
 
 type triageCall struct {
@@ -107,6 +123,7 @@ const (
 )
 
 func (job *triageJob) execute(req *queue.Request, flags ProgFlags) *queue.Result {
+	defer job.execs.Add(1)
 	req.Important = true // All triage executions are important.
 	return job.fuzzer.executeWithFlags(job.queue, req, flags)
 }
@@ -365,10 +382,25 @@ func getSignalAndCover(p *prog.Prog, info *flatrpc.ProgInfo, call int) signal.Si
 	return signal.FromRaw(inf.Signal, signalPrio(p, inf, call))
 }
 
+func (job *triageJob) info() JobInfo {
+	ret := JobInfo{
+		ID:    fmt.Sprintf("%p", job),
+		Type:  "triage",
+		Execs: int(job.execs.Load()),
+		Prog:  job.p,
+	}
+	for id := range job.calls {
+		ret.Calls = append(ret.Calls, job.p.CallName(id))
+	}
+	sort.Strings(ret.Calls)
+	return ret
+}
+
 type smashJob struct {
-	exec queue.Executor
-	p    *prog.Prog
-	call int
+	exec  queue.Executor
+	p     *prog.Prog
+	call  int
+	execs atomic.Int32
 }
 
 func (job *smashJob) run(fuzzer *Fuzzer) {
@@ -390,6 +422,17 @@ func (job *smashJob) run(fuzzer *Fuzzer) {
 		if result.Stop() {
 			return
 		}
+		job.execs.Add(1)
+	}
+}
+
+func (job *smashJob) info() JobInfo {
+	return JobInfo{
+		ID:    fmt.Sprintf("%p", job),
+		Type:  "smash",
+		Execs: int(job.execs.Load()),
+		Prog:  job.p,
+		Calls: []string{job.p.CallName(job.call)},
 	}
 }
 
diff --git a/pkg/fuzzer/stats.go b/pkg/fuzzer/stats.go
index 073afab29..f0039d500 100644
--- a/pkg/fuzzer/stats.go
+++ b/pkg/fuzzer/stats.go
@@ -33,11 +33,13 @@ func newStats() Stats {
 			stat.Console, stat.Graph("corpus")),
 		statNewInputs: stat.New("new inputs", "Potential untriaged corpus candidates",
 			stat.Graph("corpus")),
-		statJobs:       stat.New("fuzzer jobs", "Total running fuzzer jobs", stat.NoGraph),
-		statJobsTriage: stat.New("triage jobs", "Running triage jobs", stat.StackedGraph("jobs")),
+		statJobs: stat.New("fuzzer jobs", "Total running fuzzer jobs", stat.NoGraph),
+		statJobsTriage: stat.New("triage jobs", "Running triage jobs", stat.StackedGraph("jobs"),
+			stat.Link("/jobs?type=triage")),
 		statJobsTriageCandidate: stat.New("candidate triage jobs", "Running candidate triage jobs",
-			stat.StackedGraph("jobs")),
-		statJobsSmash:          stat.New("smash jobs", "Running smash jobs", stat.StackedGraph("jobs")),
+			stat.StackedGraph("jobs"), stat.Link("/jobs?type=triage")),
+		statJobsSmash: stat.New("smash jobs", "Running smash jobs", stat.StackedGraph("jobs"),
+			stat.Link("/jobs?type=smash")),
 		statJobsFaultInjection: stat.New("fault jobs", "Running fault injection jobs", stat.StackedGraph("jobs")),
 		statJobsHints:          stat.New("hints jobs", "Running hints jobs", stat.StackedGraph("jobs")),
 		statExecTime:           stat.New("prog exec time", "Test program execution time (ms)", stat.Distribution{}),
diff --git a/syz-manager/http.go b/syz-manager/http.go
index b1880556b..4dea4f7e5 100644
--- a/syz-manager/http.go
+++ b/syz-manager/http.go
@@ -21,6 +21,7 @@ import (
 	"time"
 
 	"github.com/google/syzkaller/pkg/cover"
+	"github.com/google/syzkaller/pkg/fuzzer"
 	"github.com/google/syzkaller/pkg/html/pages"
 	"github.com/google/syzkaller/pkg/log"
 	"github.com/google/syzkaller/pkg/osutil"
@@ -62,6 +63,7 @@ func (mgr *Manager) initHTTP() {
 	handle("/input", mgr.httpInput)
 	handle("/debuginput", mgr.httpDebugInput)
 	handle("/modules", mgr.modulesInfo)
+	handle("/jobs", mgr.httpJobs)
 	// Browsers like to request this, without special handler this goes to / handler.
 	handle("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {})
 
@@ -650,6 +652,50 @@ func (mgr *Manager) collectCrashes(workdir string) ([]*UICrashType, error) {
 	return crashTypes, nil
 }
 
+func (mgr *Manager) httpJobs(w http.ResponseWriter, r *http.Request) {
+	var list []fuzzer.JobInfo
+	if fuzzer := mgr.fuzzer.Load(); fuzzer != nil {
+		list = fuzzer.RunningJobs()
+	}
+	if key := r.FormValue("id"); key != "" {
+		for _, item := range list {
+			if item.ID == key {
+				w.Write(item.Prog.Serialize())
+				return
+			}
+		}
+		http.Error(w, "invalid job id (the job has likely already finished)", http.StatusBadRequest)
+		return
+	}
+	jobType := r.FormValue("type")
+	data := UIJobList{}
+	switch jobType {
+	case "triage":
+		data.Title = "triage jobs"
+	case "smash":
+		data.Title = "smash jobs"
+	default:
+		http.Error(w, "unknown job type", http.StatusBadRequest)
+		return
+	}
+	for _, item := range list {
+		if item.Type != jobType {
+			continue
+		}
+		data.Jobs = append(data.Jobs, UIJobInfo{
+			ID:    item.ID,
+			Short: item.Prog.String(),
+			Execs: item.Execs,
+			Calls: strings.Join(item.Calls, ", "),
+		})
+	}
+	sort.Slice(data.Jobs, func(i, j int) bool {
+		a, b := data.Jobs[i], data.Jobs[j]
+		return a.Short < b.Short
+	})
+	executeTemplate(w, jobListTemplate, data)
+}
+
 func readCrash(workdir, dir string, repros map[string]bool, start int64, full bool) *UICrashType {
 	if len(dir) != 40 {
 		return nil
@@ -1141,3 +1187,50 @@ var rawCoverTemplate = pages.Create(`
 </table>
 </body></html>
 `)
+
+type UIJobList struct {
+	Title string
+	Jobs  []UIJobInfo
+}
+
+type UIJobInfo struct {
+	ID    string
+	Short string
+	Calls string
+	Execs int
+}
+
+var jobListTemplate = pages.Create(`
+<!doctype html>
+<html>
+<head>
+	<title>{{.Title}}</title>
+	{{HEAD}}
+<style>
+table td {
+	max-width: 600pt;
+	word-break: break-all;
+	overflow-wrap: break-word;
+	white-space: normal;
+}
+</style>
+</head>
+<body>
+
+<table class="list_table">
+	<caption>{{.Title}} ({{len .Jobs}}):</caption>
+	<tr>
+		<th>Program</th>
+		<th>Calls</th>
+		<th>Execs</th>
+	</tr>
+	{{range $job := $.Jobs}}
+	<tr>
+		<td><a href='/jobs?id={{$job.ID}}'>{{$job.Short}}</a></td>
+		<td>{{$job.Calls}}</td>
+		<td>{{$job.Execs}}</td>
+	</tr>
+	{{end}}
+</table>
+</body></html>
+`)