all: refactor stats

Add ability for each package to create and export own stats.
Each stat is self-contained, describes how it should be presented,
and there is not need to copy them from one package to another.
Stats also keep historical data and allow building graphs over time.

6 files changed, 105 insertions, 105 deletions

diff --git a/pkg/fuzzer/cover.go b/pkg/fuzzer/cover.go
index 31d1fee1b..03580128d 100644
--- a/pkg/fuzzer/cover.go
+++ b/pkg/fuzzer/cover.go
@@ -7,6 +7,7 @@ import (
 	"sync"
 
 	"github.com/google/syzkaller/pkg/signal"
+	"github.com/google/syzkaller/pkg/stats"
 )
 
 // Cover keeps track of the signal known to the fuzzer.
@@ -17,6 +18,13 @@ type Cover struct {
 	dropSignal signal.Signal // the newly dropped max signal
 }
 
+func newCover() *Cover {
+	cover := new(Cover)
+	stats.Create("max signal", "Maximum fuzzing signal (including flakes)",
+		stats.Graph("signal"), stats.LenOf(&cover.maxSignal, &cover.mu))
+	return cover
+}
+
 // Signal that should no longer be chased after.
 // It is not returned in GrabSignalDelta().
 func (cover *Cover) AddMaxSignal(sign signal.Signal) {
@@ -61,18 +69,6 @@ func (cover *Cover) GrabSignalDelta() (plus, minus signal.Signal) {
 	return
 }
 
-type CoverStats struct {
-	MaxSignal int
-}
-
-func (cover *Cover) Stats() CoverStats {
-	cover.mu.RLock()
-	defer cover.mu.RUnlock()
-	return CoverStats{
-		MaxSignal: len(cover.maxSignal),
-	}
-}
-
 func (cover *Cover) subtract(delta signal.Signal) {
 	cover.mu.Lock()
 	defer cover.mu.Unlock()
diff --git a/pkg/fuzzer/fuzzer.go b/pkg/fuzzer/fuzzer.go
index 6a1df6ba6..6ed79aa9a 100644
--- a/pkg/fuzzer/fuzzer.go
+++ b/pkg/fuzzer/fuzzer.go
@@ -16,10 +16,12 @@ import (
 	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/pkg/rpctype"
 	"github.com/google/syzkaller/pkg/signal"
+	"github.com/google/syzkaller/pkg/stats"
 	"github.com/google/syzkaller/prog"
 )
 
 type Fuzzer struct {
+	Stats
 	Config *Config
 	Cover  *Cover
 
@@ -36,9 +38,6 @@ type Fuzzer struct {
 
 	nextExec  *priorityQueue[*Request]
 	nextJobID atomic.Int64
-
-	runningJobs      atomic.Int64
-	queuedCandidates atomic.Int64
 }
 
 func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand,
@@ -49,8 +48,9 @@ func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand,
 		}
 	}
 	f := &Fuzzer{
+		Stats:  newStats(),
 		Config: cfg,
-		Cover:  &Cover{},
+		Cover:  newCover(),
 
 		ctx:    ctx,
 		stats:  map[string]uint64{},
@@ -94,7 +94,7 @@ type Request struct {
 	SignalFilter signal.Signal // If specified, the resulting signal MAY be a subset of it.
 	// Fields that are only relevant within pkg/fuzzer.
 	flags   ProgTypes
-	stat    string
+	stat    *stats.Val
 	resultC chan *Result
 }
 
@@ -117,10 +117,10 @@ func (fuzzer *Fuzzer) Done(req *Request, res *Result) {
 	if req.resultC != nil {
 		req.resultC <- res
 	}
-	// Update stats.
-	fuzzer.mu.Lock()
-	fuzzer.stats[req.stat]++
-	fuzzer.mu.Unlock()
+	if res.Info != nil {
+		fuzzer.statExecTime.Add(int(res.Info.Elapsed.Milliseconds()))
+	}
+	req.stat.Add(1)
 }
 
 func (fuzzer *Fuzzer) triageProgCall(p *prog.Prog, info *ipc.CallInfo, call int,
@@ -140,7 +140,7 @@ func (fuzzer *Fuzzer) triageProgCall(p *prog.Prog, info *ipc.CallInfo, call int,
 		return
 	}
 	fuzzer.Logf(2, "found new signal in call %d in %s", call, p)
-	fuzzer.startJob(&triageJob{
+	fuzzer.startJob(fuzzer.statJobsTriage, &triageJob{
 		p:           p.Clone(),
 		call:        call,
 		info:        *info,
@@ -171,10 +171,8 @@ type Candidate struct {
 
 func (fuzzer *Fuzzer) NextInput() *Request {
 	req := fuzzer.nextInput()
-	if req.stat == statCandidate {
-		if fuzzer.queuedCandidates.Add(-1) < 0 {
-			panic("queuedCandidates is out of sync")
-		}
+	if req.stat == fuzzer.statExecCandidate {
+		fuzzer.StatCandidates.Add(-1)
 	}
 	return req
 }
@@ -209,7 +207,7 @@ func (fuzzer *Fuzzer) nextInput() *Request {
 	return genProgRequest(fuzzer, rnd)
 }
 
-func (fuzzer *Fuzzer) startJob(newJob job) {
+func (fuzzer *Fuzzer) startJob(stat *stats.Val, newJob job) {
 	fuzzer.Logf(2, "started %T", newJob)
 	if impl, ok := newJob.(jobSaveID); ok {
 		// E.g. for big and slow hint jobs, we would prefer not to serialize them,
@@ -217,9 +215,11 @@ func (fuzzer *Fuzzer) startJob(newJob job) {
 		impl.saveID(-fuzzer.nextJobID.Add(1))
 	}
 	go func() {
-		fuzzer.runningJobs.Add(1)
+		stat.Add(1)
+		fuzzer.statJobs.Add(1)
 		newJob.run(fuzzer)
-		fuzzer.runningJobs.Add(-1)
+		fuzzer.statJobs.Add(-1)
+		stat.Add(-1)
 	}()
 }
 
@@ -231,9 +231,9 @@ func (fuzzer *Fuzzer) Logf(level int, msg string, args ...interface{}) {
 }
 
 func (fuzzer *Fuzzer) AddCandidates(candidates []Candidate) {
-	fuzzer.queuedCandidates.Add(int64(len(candidates)))
+	fuzzer.StatCandidates.Add(len(candidates))
 	for _, candidate := range candidates {
-		fuzzer.pushExec(candidateRequest(candidate), priority{candidatePrio})
+		fuzzer.pushExec(candidateRequest(fuzzer, candidate), priority{candidatePrio})
 	}
 }
 
@@ -248,9 +248,6 @@ func (fuzzer *Fuzzer) nextRand() int64 {
 }
 
 func (fuzzer *Fuzzer) pushExec(req *Request, prio priority) {
-	if req.stat == "" {
-		panic("Request.Stat field must be set")
-	}
 	if req.NeedHints && (req.NeedCover || req.NeedSignal != rpctype.NoSignal) {
 		panic("Request.NeedHints is mutually exclusive with other fields")
 	}
@@ -327,7 +324,7 @@ func (fuzzer *Fuzzer) logCurrentStats() {
 		runtime.ReadMemStats(&m)
 
 		str := fmt.Sprintf("exec queue size: %d, running jobs: %d, heap (MB): %d",
-			fuzzer.nextExec.Len(), fuzzer.runningJobs.Load(), m.Alloc/1000/1000)
+			fuzzer.nextExec.Len(), fuzzer.statJobs.Val(), m.Alloc/1000/1000)
 		fuzzer.Logf(0, "%s", str)
 	}
 }
diff --git a/pkg/fuzzer/fuzzer_test.go b/pkg/fuzzer/fuzzer_test.go
index 0f7a0ae58..87068064f 100644
--- a/pkg/fuzzer/fuzzer_test.go
+++ b/pkg/fuzzer/fuzzer_test.go
@@ -159,15 +159,13 @@ func TestRotate(t *testing.T) {
 	})
 	fuzzer.Cover.AddMaxSignal(fakeSignal(1000))
 
-	stats := fuzzer.Stats()
-	assert.Equal(t, 1000, stats.MaxSignal)
-	assert.Equal(t, 100, stats.Signal)
+	assert.Equal(t, 1000, len(fuzzer.Cover.maxSignal))
+	assert.Equal(t, 100, corpusObj.StatSignal.Val())
 
 	// Rotate some of the signal.
 	fuzzer.RotateMaxSignal(200)
-	stats = fuzzer.Stats()
-	assert.Equal(t, 800, stats.MaxSignal)
-	assert.Equal(t, 100, stats.Signal)
+	assert.Equal(t, 800, len(fuzzer.Cover.maxSignal))
+	assert.Equal(t, 100, corpusObj.StatSignal.Val())
 
 	plus, minus := fuzzer.Cover.GrabSignalDelta()
 	assert.Equal(t, 0, plus.Len())
@@ -175,9 +173,8 @@ func TestRotate(t *testing.T) {
 
 	// Rotate the rest.
 	fuzzer.RotateMaxSignal(1000)
-	stats = fuzzer.Stats()
-	assert.Equal(t, 100, stats.MaxSignal)
-	assert.Equal(t, 100, stats.Signal)
+	assert.Equal(t, 100, len(fuzzer.Cover.maxSignal))
+	assert.Equal(t, 100, corpusObj.StatSignal.Val())
 	plus, minus = fuzzer.Cover.GrabSignalDelta()
 	assert.Equal(t, 0, plus.Len())
 	assert.Equal(t, 700, minus.Len())
@@ -230,9 +227,9 @@ func (f *testFuzzer) oneMore() bool {
 	defer f.mu.Unlock()
 	f.iter++
 	if f.iter%100 == 0 {
-		stat := f.fuzzer.Stats()
 		f.t.Logf("<iter %d>: corpus %d, signal %d, max signal %d, crash types %d, running jobs %d",
-			f.iter, stat.Progs, stat.Signal, stat.MaxSignal, len(f.crashes), stat.RunningJobs)
+			f.iter, f.fuzzer.Config.Corpus.StatProgs.Val(), f.fuzzer.Config.Corpus.StatSignal.Val(),
+			len(f.fuzzer.Cover.maxSignal), len(f.crashes), f.fuzzer.statJobs.Val())
 	}
 	return f.iter < f.iterLimit &&
 		(f.expectedCrashes == nil || len(f.crashes) != len(f.expectedCrashes))
diff --git a/pkg/fuzzer/job.go b/pkg/fuzzer/job.go
index e287b9599..2086016bf 100644
--- a/pkg/fuzzer/job.go
+++ b/pkg/fuzzer/job.go
@@ -12,6 +12,7 @@ import (
 	"github.com/google/syzkaller/pkg/ipc"
 	"github.com/google/syzkaller/pkg/rpctype"
 	"github.com/google/syzkaller/pkg/signal"
+	"github.com/google/syzkaller/pkg/stats"
 	"github.com/google/syzkaller/prog"
 )
 
@@ -71,7 +72,7 @@ func genProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *Request {
 	return &Request{
 		Prog:       p,
 		NeedSignal: rpctype.NewSignal,
-		stat:       statGenerate,
+		stat:       fuzzer.statExecGenerate,
 	}
 }
 
@@ -90,11 +91,11 @@ func mutateProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *Request {
 	return &Request{
 		Prog:       newP,
 		NeedSignal: rpctype.NewSignal,
-		stat:       statFuzz,
+		stat:       fuzzer.statExecFuzz,
 	}
 }
 
-func candidateRequest(input Candidate) *Request {
+func candidateRequest(fuzzer *Fuzzer, input Candidate) *Request {
 	flags := progCandidate
 	if input.Minimized {
 		flags |= progMinimized
@@ -105,7 +106,7 @@ func candidateRequest(input Candidate) *Request {
 	return &Request{
 		Prog:       input.Prog,
 		NeedSignal: rpctype.NewSignal,
-		stat:       statCandidate,
+		stat:       fuzzer.statExecCandidate,
 		flags:      flags,
 	}
 }
@@ -131,10 +132,11 @@ func triageJobPrio(flags ProgTypes) jobPriority {
 }
 
 func (job *triageJob) run(fuzzer *Fuzzer) {
+	fuzzer.statNewInputs.Add(1)
 	callName := fmt.Sprintf("call #%v %v", job.call, job.p.CallName(job.call))
 	fuzzer.Logf(3, "triaging input for %v (new signal=%v)", callName, job.newSignal.Len())
 	// Compute input coverage and non-flaky signal for minimization.
-	info, stop := job.deflake(fuzzer.exec, fuzzer.Config.FetchRawCover)
+	info, stop := job.deflake(fuzzer.exec, fuzzer.statExecTriage, fuzzer.Config.FetchRawCover)
 	if stop || info.newStableSignal.Empty() {
 		return
 	}
@@ -149,7 +151,7 @@ func (job *triageJob) run(fuzzer *Fuzzer) {
 	}
 	fuzzer.Logf(2, "added new input for %v to the corpus: %s", callName, job.p)
 	if job.flags&progSmashed == 0 {
-		fuzzer.startJob(&smashJob{
+		fuzzer.startJob(fuzzer.statJobsSmash, &smashJob{
 			p:    job.p.Clone(),
 			call: job.call,
 		})
@@ -171,7 +173,8 @@ type deflakedCover struct {
 	rawCover        []uint32
 }
 
-func (job *triageJob) deflake(exec func(job, *Request) *Result, rawCover bool) (info deflakedCover, stop bool) {
+func (job *triageJob) deflake(exec func(job, *Request) *Result, stat *stats.Val, rawCover bool) (
+	info deflakedCover, stop bool) {
 	// As demonstrated in #4639, programs reproduce with a very high, but not 100% probability.
 	// The triage algorithm must tolerate this, so let's pick the signal that is common
 	// to 3 out of 5 runs.
@@ -196,7 +199,7 @@ func (job *triageJob) deflake(exec func(job, *Request) *Result, rawCover bool) (
 			NeedSignal:   rpctype.AllSignal,
 			NeedCover:    true,
 			NeedRawCover: rawCover,
-			stat:         statTriage,
+			stat:         stat,
 			flags:        progInTriage,
 		})
 		if result.Stop {
@@ -236,7 +239,7 @@ func (job *triageJob) minimize(fuzzer *Fuzzer, newSignal signal.Signal) (stop bo
 					Prog:         p1,
 					NeedSignal:   rpctype.AllSignal,
 					SignalFilter: newSignal,
-					stat:         statMinimize,
+					stat:         fuzzer.statExecMinimize,
 				})
 				if result.Stop {
 					stop = true
@@ -292,7 +295,10 @@ func (job *smashJob) priority() priority {
 func (job *smashJob) run(fuzzer *Fuzzer) {
 	fuzzer.Logf(2, "smashing the program %s (call=%d):", job.p, job.call)
 	if fuzzer.Config.Comparisons && job.call >= 0 {
-		fuzzer.startJob(newHintsJob(job.p.Clone(), job.call))
+		fuzzer.startJob(fuzzer.statJobsHints, &hintsJob{
+			p:    job.p.Clone(),
+			call: job.call,
+		})
 	}
 
 	const iters = 75
@@ -306,7 +312,7 @@ func (job *smashJob) run(fuzzer *Fuzzer) {
 		result := fuzzer.exec(job, &Request{
 			Prog:       p,
 			NeedSignal: rpctype.NewSignal,
-			stat:       statSmash,
+			stat:       fuzzer.statExecSmash,
 		})
 		if result.Stop {
 			return
@@ -314,7 +320,7 @@ func (job *smashJob) run(fuzzer *Fuzzer) {
 		if fuzzer.Config.Collide {
 			result := fuzzer.exec(job, &Request{
 				Prog: randomCollide(p, rnd),
-				stat: statCollide,
+				stat: fuzzer.statExecCollide,
 			})
 			if result.Stop {
 				return
@@ -356,7 +362,7 @@ func (job *smashJob) faultInjection(fuzzer *Fuzzer) {
 		newProg.Calls[job.call].Props.FailNth = nth
 		result := fuzzer.exec(job, &Request{
 			Prog: job.p,
-			stat: statSmash,
+			stat: fuzzer.statExecSmash,
 		})
 		if result.Stop {
 			return
@@ -374,13 +380,6 @@ type hintsJob struct {
 	call int
 }
 
-func newHintsJob(p *prog.Prog, call int) *hintsJob {
-	return &hintsJob{
-		p:    p,
-		call: call,
-	}
-}
-
 func (job *hintsJob) priority() priority {
 	return priority{smashPrio}
 }
@@ -391,7 +390,7 @@ func (job *hintsJob) run(fuzzer *Fuzzer) {
 	result := fuzzer.exec(job, &Request{
 		Prog:      p,
 		NeedHints: true,
-		stat:      statSeed,
+		stat:      fuzzer.statExecSeed,
 	})
 	if result.Stop || result.Info == nil {
 		return
@@ -404,7 +403,7 @@ func (job *hintsJob) run(fuzzer *Fuzzer) {
 			result := fuzzer.exec(job, &Request{
 				Prog:       p,
 				NeedSignal: rpctype.NewSignal,
-				stat:       statHint,
+				stat:       fuzzer.statExecHint,
 			})
 			return !result.Stop
 		})
diff --git a/pkg/fuzzer/job_test.go b/pkg/fuzzer/job_test.go
index d9f7873dc..70efc0bb6 100644
--- a/pkg/fuzzer/job_test.go
+++ b/pkg/fuzzer/job_test.go
@@ -32,7 +32,7 @@ func TestDeflakeFail(t *testing.T) {
 		run++
 		// For first, we return 0 and 1. For second, 1 and 2. And so on.
 		return fakeResult(0, []uint32{uint32(run), uint32(run + 1)}, []uint32{10, 20})
-	}, false)
+	}, nil, false)
 	assert.False(t, stop)
 	assert.Equal(t, 5, run)
 	assert.Empty(t, ret.stableSignal.ToRaw())
@@ -69,7 +69,7 @@ func TestDeflakeSuccess(t *testing.T) {
 		// We expect it to have finished earlier.
 		t.Fatal("only 4 runs were expected")
 		return nil
-	}, false)
+	}, nil, false)
 	assert.False(t, stop)
 	// Cover is a union of all coverages.
 	assert.ElementsMatch(t, []uint32{10, 20, 30, 40}, ret.cover.Serialize())
diff --git a/pkg/fuzzer/stats.go b/pkg/fuzzer/stats.go
index 044febc64..38bef0405 100644
--- a/pkg/fuzzer/stats.go
+++ b/pkg/fuzzer/stats.go
@@ -3,44 +3,55 @@
 
 package fuzzer
 
-import "github.com/google/syzkaller/pkg/corpus"
-
-const (
-	statGenerate       = "exec gen"
-	statFuzz           = "exec fuzz"
-	statCandidate      = "exec candidate"
-	statTriage         = "exec triage"
-	statMinimize       = "exec minimize"
-	statSmash          = "exec smash"
-	statHint           = "exec hints"
-	statSeed           = "exec seeds"
-	statCollide        = "exec collide"
-	statExecTotal      = "exec total"
-	statBufferTooSmall = "buffer too small"
-)
+import "github.com/google/syzkaller/pkg/stats"
 
 type Stats struct {
-	CoverStats
-	corpus.Stats
-	Candidates  int
-	RunningJobs int
-	// Let's keep stats in Named as long as the rest of the code does not depend
-	// on their specific values.
-	Named map[string]uint64
+	StatCandidates    *stats.Val
+	statNewInputs     *stats.Val
+	statJobs          *stats.Val
+	statJobsTriage    *stats.Val
+	statJobsSmash     *stats.Val
+	statJobsHints     *stats.Val
+	statExecTime      *stats.Val
+	statExecGenerate  *stats.Val
+	statExecFuzz      *stats.Val
+	statExecCandidate *stats.Val
+	statExecTriage    *stats.Val
+	statExecMinimize  *stats.Val
+	statExecSmash     *stats.Val
+	statExecHint      *stats.Val
+	statExecSeed      *stats.Val
+	statExecCollide   *stats.Val
 }
 
-func (fuzzer *Fuzzer) Stats() Stats {
-	ret := Stats{
-		CoverStats:  fuzzer.Cover.Stats(),
-		Stats:       fuzzer.Config.Corpus.Stats(),
-		Candidates:  int(fuzzer.queuedCandidates.Load()),
-		RunningJobs: int(fuzzer.runningJobs.Load()),
-		Named:       make(map[string]uint64),
-	}
-	fuzzer.mu.Lock()
-	defer fuzzer.mu.Unlock()
-	for k, v := range fuzzer.stats {
-		ret.Named[k] = v
+func newStats() Stats {
+	return Stats{
+		StatCandidates: stats.Create("candidates", "Number of candidate programs in triage queue",
+			stats.Graph("corpus")),
+		statNewInputs: stats.Create("new inputs", "Potential untriaged corpus candidates",
+			stats.Graph("corpus")),
+		statJobs:       stats.Create("fuzzer jobs", "Total running fuzzer jobs", stats.NoGraph),
+		statJobsTriage: stats.Create("triage jobs", "Running triage jobs", stats.StackedGraph("jobs")),
+		statJobsSmash:  stats.Create("smash jobs", "Running smash jobs", stats.StackedGraph("jobs")),
+		statJobsHints:  stats.Create("hints jobs", "Running hints jobs", stats.StackedGraph("jobs")),
+		statExecTime:   stats.Create("prog exec time", "Test program execution time (ms)", stats.Distribution{}),
+		statExecGenerate: stats.Create("exec gen", "Executions of generated programs", stats.Rate{},
+			stats.StackedGraph("exec")),
+		statExecFuzz: stats.Create("exec fuzz", "Executions of mutated programs",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecCandidate: stats.Create("exec candidate", "Executions of candidate programs",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecTriage: stats.Create("exec triage", "Executions of corpus triage programs",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecMinimize: stats.Create("exec minimize", "Executions of programs during minimization",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecSmash: stats.Create("exec smash", "Executions of smashed programs",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecHint: stats.Create("exec hints", "Executions of programs generated using hints",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecSeed: stats.Create("exec seeds", "Executions of programs for hints extraction",
+			stats.Rate{}, stats.StackedGraph("exec")),
+		statExecCollide: stats.Create("exec collide", "Executions of programs in collide mode",
+			stats.Rate{}, stats.StackedGraph("exec")),
 	}
-	return ret
 }