From 1be1a06281dccada078a2a51e8b483811af8f596 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 29 Mar 2024 15:02:10 +0100 Subject: all: refactor stats Add ability for each package to create and export own stats. Each stat is self-contained, describes how it should be presented, and there is not need to copy them from one package to another. Stats also keep historical data and allow building graphs over time. --- pkg/fuzzer/cover.go | 20 +++++------- pkg/fuzzer/fuzzer.go | 43 ++++++++++++------------- pkg/fuzzer/fuzzer_test.go | 19 +++++------ pkg/fuzzer/job.go | 43 ++++++++++++------------- pkg/fuzzer/job_test.go | 4 +-- pkg/fuzzer/stats.go | 81 +++++++++++++++++++++++++++-------------------- 6 files changed, 105 insertions(+), 105 deletions(-) (limited to 'pkg/fuzzer') diff --git a/pkg/fuzzer/cover.go b/pkg/fuzzer/cover.go index 31d1fee1b..03580128d 100644 --- a/pkg/fuzzer/cover.go +++ b/pkg/fuzzer/cover.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/google/syzkaller/pkg/signal" + "github.com/google/syzkaller/pkg/stats" ) // Cover keeps track of the signal known to the fuzzer. @@ -17,6 +18,13 @@ type Cover struct { dropSignal signal.Signal // the newly dropped max signal } +func newCover() *Cover { + cover := new(Cover) + stats.Create("max signal", "Maximum fuzzing signal (including flakes)", + stats.Graph("signal"), stats.LenOf(&cover.maxSignal, &cover.mu)) + return cover +} + // Signal that should no longer be chased after. // It is not returned in GrabSignalDelta(). func (cover *Cover) AddMaxSignal(sign signal.Signal) { @@ -61,18 +69,6 @@ func (cover *Cover) GrabSignalDelta() (plus, minus signal.Signal) { return } -type CoverStats struct { - MaxSignal int -} - -func (cover *Cover) Stats() CoverStats { - cover.mu.RLock() - defer cover.mu.RUnlock() - return CoverStats{ - MaxSignal: len(cover.maxSignal), - } -} - func (cover *Cover) subtract(delta signal.Signal) { cover.mu.Lock() defer cover.mu.Unlock() diff --git a/pkg/fuzzer/fuzzer.go b/pkg/fuzzer/fuzzer.go index 6a1df6ba6..6ed79aa9a 100644 --- a/pkg/fuzzer/fuzzer.go +++ b/pkg/fuzzer/fuzzer.go @@ -16,10 +16,12 @@ import ( "github.com/google/syzkaller/pkg/ipc" "github.com/google/syzkaller/pkg/rpctype" "github.com/google/syzkaller/pkg/signal" + "github.com/google/syzkaller/pkg/stats" "github.com/google/syzkaller/prog" ) type Fuzzer struct { + Stats Config *Config Cover *Cover @@ -36,9 +38,6 @@ type Fuzzer struct { nextExec *priorityQueue[*Request] nextJobID atomic.Int64 - - runningJobs atomic.Int64 - queuedCandidates atomic.Int64 } func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand, @@ -49,8 +48,9 @@ func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand, } } f := &Fuzzer{ + Stats: newStats(), Config: cfg, - Cover: &Cover{}, + Cover: newCover(), ctx: ctx, stats: map[string]uint64{}, @@ -94,7 +94,7 @@ type Request struct { SignalFilter signal.Signal // If specified, the resulting signal MAY be a subset of it. // Fields that are only relevant within pkg/fuzzer. flags ProgTypes - stat string + stat *stats.Val resultC chan *Result } @@ -117,10 +117,10 @@ func (fuzzer *Fuzzer) Done(req *Request, res *Result) { if req.resultC != nil { req.resultC <- res } - // Update stats. - fuzzer.mu.Lock() - fuzzer.stats[req.stat]++ - fuzzer.mu.Unlock() + if res.Info != nil { + fuzzer.statExecTime.Add(int(res.Info.Elapsed.Milliseconds())) + } + req.stat.Add(1) } func (fuzzer *Fuzzer) triageProgCall(p *prog.Prog, info *ipc.CallInfo, call int, @@ -140,7 +140,7 @@ func (fuzzer *Fuzzer) triageProgCall(p *prog.Prog, info *ipc.CallInfo, call int, return } fuzzer.Logf(2, "found new signal in call %d in %s", call, p) - fuzzer.startJob(&triageJob{ + fuzzer.startJob(fuzzer.statJobsTriage, &triageJob{ p: p.Clone(), call: call, info: *info, @@ -171,10 +171,8 @@ type Candidate struct { func (fuzzer *Fuzzer) NextInput() *Request { req := fuzzer.nextInput() - if req.stat == statCandidate { - if fuzzer.queuedCandidates.Add(-1) < 0 { - panic("queuedCandidates is out of sync") - } + if req.stat == fuzzer.statExecCandidate { + fuzzer.StatCandidates.Add(-1) } return req } @@ -209,7 +207,7 @@ func (fuzzer *Fuzzer) nextInput() *Request { return genProgRequest(fuzzer, rnd) } -func (fuzzer *Fuzzer) startJob(newJob job) { +func (fuzzer *Fuzzer) startJob(stat *stats.Val, newJob job) { fuzzer.Logf(2, "started %T", newJob) if impl, ok := newJob.(jobSaveID); ok { // E.g. for big and slow hint jobs, we would prefer not to serialize them, @@ -217,9 +215,11 @@ func (fuzzer *Fuzzer) startJob(newJob job) { impl.saveID(-fuzzer.nextJobID.Add(1)) } go func() { - fuzzer.runningJobs.Add(1) + stat.Add(1) + fuzzer.statJobs.Add(1) newJob.run(fuzzer) - fuzzer.runningJobs.Add(-1) + fuzzer.statJobs.Add(-1) + stat.Add(-1) }() } @@ -231,9 +231,9 @@ func (fuzzer *Fuzzer) Logf(level int, msg string, args ...interface{}) { } func (fuzzer *Fuzzer) AddCandidates(candidates []Candidate) { - fuzzer.queuedCandidates.Add(int64(len(candidates))) + fuzzer.StatCandidates.Add(len(candidates)) for _, candidate := range candidates { - fuzzer.pushExec(candidateRequest(candidate), priority{candidatePrio}) + fuzzer.pushExec(candidateRequest(fuzzer, candidate), priority{candidatePrio}) } } @@ -248,9 +248,6 @@ func (fuzzer *Fuzzer) nextRand() int64 { } func (fuzzer *Fuzzer) pushExec(req *Request, prio priority) { - if req.stat == "" { - panic("Request.Stat field must be set") - } if req.NeedHints && (req.NeedCover || req.NeedSignal != rpctype.NoSignal) { panic("Request.NeedHints is mutually exclusive with other fields") } @@ -327,7 +324,7 @@ func (fuzzer *Fuzzer) logCurrentStats() { runtime.ReadMemStats(&m) str := fmt.Sprintf("exec queue size: %d, running jobs: %d, heap (MB): %d", - fuzzer.nextExec.Len(), fuzzer.runningJobs.Load(), m.Alloc/1000/1000) + fuzzer.nextExec.Len(), fuzzer.statJobs.Val(), m.Alloc/1000/1000) fuzzer.Logf(0, "%s", str) } } diff --git a/pkg/fuzzer/fuzzer_test.go b/pkg/fuzzer/fuzzer_test.go index 0f7a0ae58..87068064f 100644 --- a/pkg/fuzzer/fuzzer_test.go +++ b/pkg/fuzzer/fuzzer_test.go @@ -159,15 +159,13 @@ func TestRotate(t *testing.T) { }) fuzzer.Cover.AddMaxSignal(fakeSignal(1000)) - stats := fuzzer.Stats() - assert.Equal(t, 1000, stats.MaxSignal) - assert.Equal(t, 100, stats.Signal) + assert.Equal(t, 1000, len(fuzzer.Cover.maxSignal)) + assert.Equal(t, 100, corpusObj.StatSignal.Val()) // Rotate some of the signal. fuzzer.RotateMaxSignal(200) - stats = fuzzer.Stats() - assert.Equal(t, 800, stats.MaxSignal) - assert.Equal(t, 100, stats.Signal) + assert.Equal(t, 800, len(fuzzer.Cover.maxSignal)) + assert.Equal(t, 100, corpusObj.StatSignal.Val()) plus, minus := fuzzer.Cover.GrabSignalDelta() assert.Equal(t, 0, plus.Len()) @@ -175,9 +173,8 @@ func TestRotate(t *testing.T) { // Rotate the rest. fuzzer.RotateMaxSignal(1000) - stats = fuzzer.Stats() - assert.Equal(t, 100, stats.MaxSignal) - assert.Equal(t, 100, stats.Signal) + assert.Equal(t, 100, len(fuzzer.Cover.maxSignal)) + assert.Equal(t, 100, corpusObj.StatSignal.Val()) plus, minus = fuzzer.Cover.GrabSignalDelta() assert.Equal(t, 0, plus.Len()) assert.Equal(t, 700, minus.Len()) @@ -230,9 +227,9 @@ func (f *testFuzzer) oneMore() bool { defer f.mu.Unlock() f.iter++ if f.iter%100 == 0 { - stat := f.fuzzer.Stats() f.t.Logf(": corpus %d, signal %d, max signal %d, crash types %d, running jobs %d", - f.iter, stat.Progs, stat.Signal, stat.MaxSignal, len(f.crashes), stat.RunningJobs) + f.iter, f.fuzzer.Config.Corpus.StatProgs.Val(), f.fuzzer.Config.Corpus.StatSignal.Val(), + len(f.fuzzer.Cover.maxSignal), len(f.crashes), f.fuzzer.statJobs.Val()) } return f.iter < f.iterLimit && (f.expectedCrashes == nil || len(f.crashes) != len(f.expectedCrashes)) diff --git a/pkg/fuzzer/job.go b/pkg/fuzzer/job.go index e287b9599..2086016bf 100644 --- a/pkg/fuzzer/job.go +++ b/pkg/fuzzer/job.go @@ -12,6 +12,7 @@ import ( "github.com/google/syzkaller/pkg/ipc" "github.com/google/syzkaller/pkg/rpctype" "github.com/google/syzkaller/pkg/signal" + "github.com/google/syzkaller/pkg/stats" "github.com/google/syzkaller/prog" ) @@ -71,7 +72,7 @@ func genProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *Request { return &Request{ Prog: p, NeedSignal: rpctype.NewSignal, - stat: statGenerate, + stat: fuzzer.statExecGenerate, } } @@ -90,11 +91,11 @@ func mutateProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *Request { return &Request{ Prog: newP, NeedSignal: rpctype.NewSignal, - stat: statFuzz, + stat: fuzzer.statExecFuzz, } } -func candidateRequest(input Candidate) *Request { +func candidateRequest(fuzzer *Fuzzer, input Candidate) *Request { flags := progCandidate if input.Minimized { flags |= progMinimized @@ -105,7 +106,7 @@ func candidateRequest(input Candidate) *Request { return &Request{ Prog: input.Prog, NeedSignal: rpctype.NewSignal, - stat: statCandidate, + stat: fuzzer.statExecCandidate, flags: flags, } } @@ -131,10 +132,11 @@ func triageJobPrio(flags ProgTypes) jobPriority { } func (job *triageJob) run(fuzzer *Fuzzer) { + fuzzer.statNewInputs.Add(1) callName := fmt.Sprintf("call #%v %v", job.call, job.p.CallName(job.call)) fuzzer.Logf(3, "triaging input for %v (new signal=%v)", callName, job.newSignal.Len()) // Compute input coverage and non-flaky signal for minimization. - info, stop := job.deflake(fuzzer.exec, fuzzer.Config.FetchRawCover) + info, stop := job.deflake(fuzzer.exec, fuzzer.statExecTriage, fuzzer.Config.FetchRawCover) if stop || info.newStableSignal.Empty() { return } @@ -149,7 +151,7 @@ func (job *triageJob) run(fuzzer *Fuzzer) { } fuzzer.Logf(2, "added new input for %v to the corpus: %s", callName, job.p) if job.flags&progSmashed == 0 { - fuzzer.startJob(&smashJob{ + fuzzer.startJob(fuzzer.statJobsSmash, &smashJob{ p: job.p.Clone(), call: job.call, }) @@ -171,7 +173,8 @@ type deflakedCover struct { rawCover []uint32 } -func (job *triageJob) deflake(exec func(job, *Request) *Result, rawCover bool) (info deflakedCover, stop bool) { +func (job *triageJob) deflake(exec func(job, *Request) *Result, stat *stats.Val, rawCover bool) ( + info deflakedCover, stop bool) { // As demonstrated in #4639, programs reproduce with a very high, but not 100% probability. // The triage algorithm must tolerate this, so let's pick the signal that is common // to 3 out of 5 runs. @@ -196,7 +199,7 @@ func (job *triageJob) deflake(exec func(job, *Request) *Result, rawCover bool) ( NeedSignal: rpctype.AllSignal, NeedCover: true, NeedRawCover: rawCover, - stat: statTriage, + stat: stat, flags: progInTriage, }) if result.Stop { @@ -236,7 +239,7 @@ func (job *triageJob) minimize(fuzzer *Fuzzer, newSignal signal.Signal) (stop bo Prog: p1, NeedSignal: rpctype.AllSignal, SignalFilter: newSignal, - stat: statMinimize, + stat: fuzzer.statExecMinimize, }) if result.Stop { stop = true @@ -292,7 +295,10 @@ func (job *smashJob) priority() priority { func (job *smashJob) run(fuzzer *Fuzzer) { fuzzer.Logf(2, "smashing the program %s (call=%d):", job.p, job.call) if fuzzer.Config.Comparisons && job.call >= 0 { - fuzzer.startJob(newHintsJob(job.p.Clone(), job.call)) + fuzzer.startJob(fuzzer.statJobsHints, &hintsJob{ + p: job.p.Clone(), + call: job.call, + }) } const iters = 75 @@ -306,7 +312,7 @@ func (job *smashJob) run(fuzzer *Fuzzer) { result := fuzzer.exec(job, &Request{ Prog: p, NeedSignal: rpctype.NewSignal, - stat: statSmash, + stat: fuzzer.statExecSmash, }) if result.Stop { return @@ -314,7 +320,7 @@ func (job *smashJob) run(fuzzer *Fuzzer) { if fuzzer.Config.Collide { result := fuzzer.exec(job, &Request{ Prog: randomCollide(p, rnd), - stat: statCollide, + stat: fuzzer.statExecCollide, }) if result.Stop { return @@ -356,7 +362,7 @@ func (job *smashJob) faultInjection(fuzzer *Fuzzer) { newProg.Calls[job.call].Props.FailNth = nth result := fuzzer.exec(job, &Request{ Prog: job.p, - stat: statSmash, + stat: fuzzer.statExecSmash, }) if result.Stop { return @@ -374,13 +380,6 @@ type hintsJob struct { call int } -func newHintsJob(p *prog.Prog, call int) *hintsJob { - return &hintsJob{ - p: p, - call: call, - } -} - func (job *hintsJob) priority() priority { return priority{smashPrio} } @@ -391,7 +390,7 @@ func (job *hintsJob) run(fuzzer *Fuzzer) { result := fuzzer.exec(job, &Request{ Prog: p, NeedHints: true, - stat: statSeed, + stat: fuzzer.statExecSeed, }) if result.Stop || result.Info == nil { return @@ -404,7 +403,7 @@ func (job *hintsJob) run(fuzzer *Fuzzer) { result := fuzzer.exec(job, &Request{ Prog: p, NeedSignal: rpctype.NewSignal, - stat: statHint, + stat: fuzzer.statExecHint, }) return !result.Stop }) diff --git a/pkg/fuzzer/job_test.go b/pkg/fuzzer/job_test.go index d9f7873dc..70efc0bb6 100644 --- a/pkg/fuzzer/job_test.go +++ b/pkg/fuzzer/job_test.go @@ -32,7 +32,7 @@ func TestDeflakeFail(t *testing.T) { run++ // For first, we return 0 and 1. For second, 1 and 2. And so on. return fakeResult(0, []uint32{uint32(run), uint32(run + 1)}, []uint32{10, 20}) - }, false) + }, nil, false) assert.False(t, stop) assert.Equal(t, 5, run) assert.Empty(t, ret.stableSignal.ToRaw()) @@ -69,7 +69,7 @@ func TestDeflakeSuccess(t *testing.T) { // We expect it to have finished earlier. t.Fatal("only 4 runs were expected") return nil - }, false) + }, nil, false) assert.False(t, stop) // Cover is a union of all coverages. assert.ElementsMatch(t, []uint32{10, 20, 30, 40}, ret.cover.Serialize()) diff --git a/pkg/fuzzer/stats.go b/pkg/fuzzer/stats.go index 044febc64..38bef0405 100644 --- a/pkg/fuzzer/stats.go +++ b/pkg/fuzzer/stats.go @@ -3,44 +3,55 @@ package fuzzer -import "github.com/google/syzkaller/pkg/corpus" - -const ( - statGenerate = "exec gen" - statFuzz = "exec fuzz" - statCandidate = "exec candidate" - statTriage = "exec triage" - statMinimize = "exec minimize" - statSmash = "exec smash" - statHint = "exec hints" - statSeed = "exec seeds" - statCollide = "exec collide" - statExecTotal = "exec total" - statBufferTooSmall = "buffer too small" -) +import "github.com/google/syzkaller/pkg/stats" type Stats struct { - CoverStats - corpus.Stats - Candidates int - RunningJobs int - // Let's keep stats in Named as long as the rest of the code does not depend - // on their specific values. - Named map[string]uint64 + StatCandidates *stats.Val + statNewInputs *stats.Val + statJobs *stats.Val + statJobsTriage *stats.Val + statJobsSmash *stats.Val + statJobsHints *stats.Val + statExecTime *stats.Val + statExecGenerate *stats.Val + statExecFuzz *stats.Val + statExecCandidate *stats.Val + statExecTriage *stats.Val + statExecMinimize *stats.Val + statExecSmash *stats.Val + statExecHint *stats.Val + statExecSeed *stats.Val + statExecCollide *stats.Val } -func (fuzzer *Fuzzer) Stats() Stats { - ret := Stats{ - CoverStats: fuzzer.Cover.Stats(), - Stats: fuzzer.Config.Corpus.Stats(), - Candidates: int(fuzzer.queuedCandidates.Load()), - RunningJobs: int(fuzzer.runningJobs.Load()), - Named: make(map[string]uint64), - } - fuzzer.mu.Lock() - defer fuzzer.mu.Unlock() - for k, v := range fuzzer.stats { - ret.Named[k] = v +func newStats() Stats { + return Stats{ + StatCandidates: stats.Create("candidates", "Number of candidate programs in triage queue", + stats.Graph("corpus")), + statNewInputs: stats.Create("new inputs", "Potential untriaged corpus candidates", + stats.Graph("corpus")), + statJobs: stats.Create("fuzzer jobs", "Total running fuzzer jobs", stats.NoGraph), + statJobsTriage: stats.Create("triage jobs", "Running triage jobs", stats.StackedGraph("jobs")), + statJobsSmash: stats.Create("smash jobs", "Running smash jobs", stats.StackedGraph("jobs")), + statJobsHints: stats.Create("hints jobs", "Running hints jobs", stats.StackedGraph("jobs")), + statExecTime: stats.Create("prog exec time", "Test program execution time (ms)", stats.Distribution{}), + statExecGenerate: stats.Create("exec gen", "Executions of generated programs", stats.Rate{}, + stats.StackedGraph("exec")), + statExecFuzz: stats.Create("exec fuzz", "Executions of mutated programs", + stats.Rate{}, stats.StackedGraph("exec")), + statExecCandidate: stats.Create("exec candidate", "Executions of candidate programs", + stats.Rate{}, stats.StackedGraph("exec")), + statExecTriage: stats.Create("exec triage", "Executions of corpus triage programs", + stats.Rate{}, stats.StackedGraph("exec")), + statExecMinimize: stats.Create("exec minimize", "Executions of programs during minimization", + stats.Rate{}, stats.StackedGraph("exec")), + statExecSmash: stats.Create("exec smash", "Executions of smashed programs", + stats.Rate{}, stats.StackedGraph("exec")), + statExecHint: stats.Create("exec hints", "Executions of programs generated using hints", + stats.Rate{}, stats.StackedGraph("exec")), + statExecSeed: stats.Create("exec seeds", "Executions of programs for hints extraction", + stats.Rate{}, stats.StackedGraph("exec")), + statExecCollide: stats.Create("exec collide", "Executions of programs in collide mode", + stats.Rate{}, stats.StackedGraph("exec")), } - return ret } -- cgit mrf-deployment