From 42a1ab121510d4ace36fddeb7396bdb7f28bd489 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Thu, 24 Oct 2024 18:38:29 +0200 Subject: pkg/corpus: move focus area configuration to the constructor Set Corpus in HTTPServer dynamically. Refactor syz-manager and syz-diff accordingly. --- pkg/corpus/corpus.go | 41 ++++++++++++++------------------ pkg/corpus/prio_test.go | 3 +-- pkg/manager/http.go | 63 ++++++++++++++++++++++++++++++++++++++----------- syz-manager/manager.go | 33 ++++++++++++++++---------- tools/syz-diff/diff.go | 10 ++++---- 5 files changed, 92 insertions(+), 58 deletions(-) diff --git a/pkg/corpus/corpus.go b/pkg/corpus/corpus.go index 487457376..99abd8188 100644 --- a/pkg/corpus/corpus.go +++ b/pkg/corpus/corpus.go @@ -50,6 +50,10 @@ func NewCorpus(ctx context.Context) *Corpus { } func NewMonitoredCorpus(ctx context.Context, updates chan<- NewItemEvent) *Corpus { + return NewFocusedCorpus(ctx, updates, nil) +} + +func NewFocusedCorpus(ctx context.Context, updates chan<- NewItemEvent, areas []FocusArea) *Corpus { corpus := &Corpus{ ctx: ctx, progsMap: make(map[string]*Item), @@ -62,6 +66,20 @@ func NewMonitoredCorpus(ctx context.Context, updates chan<- NewItemEvent) *Corpu stat.LenOf(&corpus.signal, &corpus.mu)) corpus.StatCover = stat.New("coverage", "Source coverage in the corpus", stat.Console, stat.Link("/cover"), stat.Prometheus("syz_corpus_cover"), stat.LenOf(&corpus.cover, &corpus.mu)) + for _, area := range areas { + obj := &ProgramsList{} + if len(areas) > 1 && area.Name != "" { + // Only show extra statistics if there's more than one area. + stat.New("corpus ["+area.Name+"]", + fmt.Sprintf("Corpus programs of the focus area %q", area.Name), + stat.Console, stat.Graph("corpus"), + stat.LenOf(&obj.progs, &corpus.mu)) + } + corpus.focusAreas = append(corpus.focusAreas, &focusAreaState{ + FocusArea: area, + ProgramsList: obj, + }) + } return corpus } @@ -106,29 +124,6 @@ type NewItemEvent struct { NewCover []uint64 } -// SetFocusAreas can only be called once on an empty corpus. -func (corpus *Corpus) SetFocusAreas(areas []FocusArea) { - corpus.mu.Lock() - defer corpus.mu.Unlock() - if len(corpus.progsMap) > 0 { - panic("SetFocusAreas() is called on a non-empty corpus") - } - for _, area := range areas { - obj := &ProgramsList{} - if len(areas) > 1 && area.Name != "" { - // Only show extra statistics if there's more than one area. - stat.New("corpus ["+area.Name+"]", - fmt.Sprintf("Corpus programs of the focus area %q", area.Name), - stat.Console, stat.Graph("corpus"), - stat.LenOf(&obj.progs, &corpus.mu)) - } - corpus.focusAreas = append(corpus.focusAreas, &focusAreaState{ - FocusArea: area, - ProgramsList: obj, - }) - } -} - func (corpus *Corpus) Save(inp NewInput) { progData := inp.Prog.Serialize() sig := hash.String(progData) diff --git a/pkg/corpus/prio_test.go b/pkg/corpus/prio_test.go index 6328ba3e5..71507c2b1 100644 --- a/pkg/corpus/prio_test.go +++ b/pkg/corpus/prio_test.go @@ -51,8 +51,7 @@ func TestChooseProgram(t *testing.T) { func TestFocusAreas(t *testing.T) { target := getTarget(t, targets.TestOS, targets.TestArch64) - corpus := NewCorpus(context.Background()) - corpus.SetFocusAreas([]FocusArea{ + corpus := NewFocusedCorpus(context.Background(), nil, []FocusArea{ { CoverPCs: map[uint64]struct{}{ 0: {}, diff --git a/pkg/manager/http.go b/pkg/manager/http.go index c94ce727f..aa54c66eb 100644 --- a/pkg/manager/http.go +++ b/pkg/manager/http.go @@ -49,11 +49,11 @@ type HTTPServer struct { // To be set once. Cfg *mgrconfig.Config StartTime time.Time - Corpus *corpus.Corpus CrashStore *CrashStore DiffStore *DiffFuzzerStore // Set dynamically. + Corpus atomic.Pointer[corpus.Corpus] Fuzzer atomic.Pointer[fuzzer.Fuzzer] Cover atomic.Pointer[CoverageInfo] ReproLoop atomic.Pointer[ReproLoop] @@ -172,10 +172,12 @@ func (serv *HTTPServer) httpExpertMode(w http.ResponseWriter, r *http.Request) { func (serv *HTTPServer) httpSyscalls(w http.ResponseWriter, r *http.Request) { var calls map[string]*corpus.CallCov - if obj := serv.EnabledSyscalls.Load(); obj != nil { - calls = serv.Corpus.CallCover() + syscallsObj := serv.EnabledSyscalls.Load() + corpusObj := serv.Corpus.Load() + if corpusObj != nil && syscallsObj != nil { + calls = corpusObj.CallCover() // Add enabled, but not yet covered calls. - for call := range obj.(map[*prog.Syscall]bool) { + for call := range syscallsObj.(map[*prog.Syscall]bool) { if calls[call.Name] == nil { calls[call.Name] = new(corpus.CallCov) } @@ -321,11 +323,16 @@ func (serv *HTTPServer) httpCrash(w http.ResponseWriter, r *http.Request) { } func (serv *HTTPServer) httpCorpus(w http.ResponseWriter, r *http.Request) { + corpus := serv.Corpus.Load() + if corpus == nil { + http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError) + return + } data := UICorpus{ Call: r.FormValue("call"), RawCover: serv.Cfg.RawCover, } - for _, inp := range serv.Corpus.Items() { + for _, inp := range corpus.Items() { if data.Call != "" && data.Call != inp.StringCall() { continue } @@ -416,6 +423,12 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f return } + corpus := serv.Corpus.Load() + if corpus == nil { + http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError) + return + } + rg, err := coverInfo.ReportGenerator.Get() if err != nil { http.Error(w, fmt.Sprintf("failed to generate coverage profile: %v", err), http.StatusInternalServerError) @@ -431,7 +444,7 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f var progs []cover.Prog if sig := r.FormValue("input"); sig != "" { - inp := serv.Corpus.Item(sig) + inp := corpus.Item(sig) if inp == nil { http.Error(w, "unknown input hash", http.StatusInternalServerError) return @@ -456,7 +469,7 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f } } else { call := r.FormValue("call") - for _, inp := range serv.Corpus.Items() { + for _, inp := range corpus.Items() { if call != "" && call != inp.StringCall() { continue } @@ -511,8 +524,13 @@ func (serv *HTTPServer) httpCoverCover(w http.ResponseWriter, r *http.Request, f } func (serv *HTTPServer) httpCoverFallback(w http.ResponseWriter, r *http.Request) { + corpus := serv.Corpus.Load() + if corpus == nil { + http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError) + return + } calls := make(map[int][]int) - for s := range serv.Corpus.Signal() { + for s := range corpus.Signal() { id, errno := prog.DecodeFallbackSignal(uint64(s)) calls[id] = append(calls[id], errno) } @@ -548,6 +566,12 @@ func (serv *HTTPServer) httpFileCover(w http.ResponseWriter, r *http.Request) { } func (serv *HTTPServer) httpPrio(w http.ResponseWriter, r *http.Request) { + corpus := serv.Corpus.Load() + if corpus == nil { + http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError) + return + } + callName := r.FormValue("call") call := serv.Cfg.Target.SyscallMap[callName] if call == nil { @@ -555,11 +579,12 @@ func (serv *HTTPServer) httpPrio(w http.ResponseWriter, r *http.Request) { return } - var corpus []*prog.Prog - for _, inp := range serv.Corpus.Items() { - corpus = append(corpus, inp.Prog) + var progs []*prog.Prog + for _, inp := range corpus.Items() { + progs = append(progs, inp.Prog) } - prios := serv.Cfg.Target.CalculatePriorities(corpus) + + prios := serv.Cfg.Target.CalculatePriorities(progs) data := &UIPrioData{Call: callName} for i, p := range prios[call.ID] { @@ -589,7 +614,12 @@ func (serv *HTTPServer) httpFile(w http.ResponseWriter, r *http.Request) { } func (serv *HTTPServer) httpInput(w http.ResponseWriter, r *http.Request) { - inp := serv.Corpus.Item(r.FormValue("sig")) + corpus := serv.Corpus.Load() + if corpus == nil { + http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError) + return + } + inp := corpus.Item(r.FormValue("sig")) if inp == nil { http.Error(w, "can't find the input", http.StatusInternalServerError) return @@ -599,7 +629,12 @@ func (serv *HTTPServer) httpInput(w http.ResponseWriter, r *http.Request) { } func (serv *HTTPServer) httpDebugInput(w http.ResponseWriter, r *http.Request) { - inp := serv.Corpus.Item(r.FormValue("sig")) + corpus := serv.Corpus.Load() + if corpus == nil { + http.Error(w, "the corpus information is not yet available", http.StatusInternalServerError) + return + } + inp := corpus.Item(r.FormValue("sig")) if inp == nil { http.Error(w, "can't find the input", http.StatusInternalServerError) return diff --git a/syz-manager/manager.go b/syz-manager/manager.go index 96164af63..922639407 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -208,12 +208,10 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { log.Fatalf("%v", err) } - corpusUpdates := make(chan corpus.NewItemEvent, 128) mgr := &Manager{ cfg: cfg, mode: mode, vmPool: vmPool, - corpus: corpus.NewMonitoredCorpus(context.Background(), corpusUpdates), corpusPreload: make(chan []fuzzer.Candidate), target: cfg.Target, sysTarget: cfg.SysTarget, @@ -235,7 +233,6 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { mgr.http = &manager.HTTPServer{ Cfg: cfg, StartTime: time.Now(), - Corpus: mgr.corpus, CrashStore: mgr.crashStore, } @@ -246,7 +243,6 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { close(mgr.corpusPreload) } go mgr.http.Serve() - go mgr.corpusInputHandler(corpusUpdates) go mgr.trackUsedFiles() // Create RPC server for fuzzers. @@ -1031,11 +1027,16 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map statSyscalls := stat.New("syscalls", "Number of enabled syscalls", stat.Simple, stat.NoGraph, stat.Link("/syscalls")) statSyscalls.Add(len(enabledSyscalls)) - corpus := mgr.loadCorpus(enabledSyscalls) + candidates := mgr.loadCorpus(enabledSyscalls) mgr.setPhaseLocked(phaseLoadedCorpus) opts := fuzzer.DefaultExecOpts(mgr.cfg, features, *flagDebug) if mgr.mode == ModeFuzzing { + corpusUpdates := make(chan corpus.NewItemEvent, 128) + mgr.corpus = corpus.NewFocusedCorpus(context.Background(), + corpusUpdates, mgr.coverFilters.Areas) + mgr.http.Corpus.Store(mgr.corpus) + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) fuzzerObj := fuzzer.NewFuzzer(context.Background(), &fuzzer.Config{ Corpus: mgr.corpus, @@ -1059,10 +1060,11 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map return !mgr.saturatedCalls[call] }, }, rnd, mgr.target) - fuzzerObj.AddCandidates(corpus) + fuzzerObj.AddCandidates(candidates) mgr.fuzzer.Store(fuzzerObj) mgr.http.Fuzzer.Store(fuzzerObj) + go mgr.corpusInputHandler(corpusUpdates) go mgr.corpusMinimization() go mgr.fuzzerLoop(fuzzerObj) if mgr.dash != nil { @@ -1085,7 +1087,7 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map return source } else if mgr.mode == ModeCorpusRun { ctx := &corpusRunner{ - candidates: corpus, + candidates: candidates, rnd: rand.New(rand.NewSource(time.Now().UnixNano())), } return queue.DefaultOpts(ctx, opts) @@ -1267,14 +1269,20 @@ func (mgr *Manager) dashboardReporter() { var lastFuzzingTime time.Duration var lastCrashes, lastSuppressedCrashes, lastExecs uint64 for range time.NewTicker(time.Minute).C { + mgr.mu.Lock() + corpus := mgr.corpus + mgr.mu.Unlock() + if corpus == nil { + continue + } mgr.mu.Lock() req := &dashapi.ManagerStatsReq{ Name: mgr.cfg.Name, Addr: webAddr, UpTime: time.Duration(mgr.statUptime.Val()) * time.Second, - Corpus: uint64(mgr.corpus.StatProgs.Val()), - PCs: uint64(mgr.corpus.StatCover.Val()), - Cover: uint64(mgr.corpus.StatSignal.Val()), + Corpus: uint64(corpus.StatProgs.Val()), + PCs: uint64(corpus.StatCover.Val()), + Cover: uint64(corpus.StatSignal.Val()), CrashTypes: uint64(mgr.statCrashTypes.Val()), FuzzingTime: time.Duration(mgr.statFuzzingTime.Val()) - lastFuzzingTime, Crashes: uint64(mgr.statCrashes.Val()) - lastCrashes, @@ -1283,8 +1291,8 @@ func (mgr *Manager) dashboardReporter() { } if mgr.phase >= phaseTriagedCorpus && !triageInfoSent { triageInfoSent = true - req.TriagedCoverage = uint64(mgr.corpus.StatSignal.Val()) - req.TriagedPCs = uint64(mgr.corpus.StatCover.Val()) + req.TriagedCoverage = uint64(corpus.StatSignal.Val()) + req.TriagedPCs = uint64(corpus.StatCover.Val()) } mgr.mu.Unlock() @@ -1337,7 +1345,6 @@ func (mgr *Manager) CoverageFilter(modules []*vminfo.KernelModule) []uint64 { ReportGenerator: mgr.reportGenerator, CoverFilter: filters.ExecutorFilter, }) - mgr.corpus.SetFocusAreas(filters.Areas) var pcs []uint64 for pc := range filters.ExecutorFilter { pcs = append(pcs, pc) diff --git a/tools/syz-diff/diff.go b/tools/syz-diff/diff.go index e593205bc..6a6a0991d 100644 --- a/tools/syz-diff/diff.go +++ b/tools/syz-diff/diff.go @@ -90,7 +90,6 @@ func main() { http: &manager.HTTPServer{ Cfg: newCfg, StartTime: time.Now(), - Corpus: new.corpus, DiffStore: store, }, } @@ -232,7 +231,6 @@ type kernelContext struct { ctx context.Context cfg *mgrconfig.Config reporter *report.Reporter - corpus *corpus.Corpus fuzzer atomic.Pointer[fuzzer.Fuzzer] serv rpcserver.Server servStats rpcserver.Stats @@ -256,7 +254,6 @@ func setup(ctx context.Context, name string, cfg *mgrconfig.Config) *kernelConte name: name, ctx: ctx, cfg: cfg, - corpus: corpus.NewCorpus(ctx), crashes: make(chan *report.Report, 128), candidates: make(chan []fuzzer.Candidate), servStats: rpcserver.NewNamedStats(name), @@ -320,8 +317,9 @@ func (kc *kernelContext) MachineChecked(features flatrpc.Feature, syscalls map[* func (kc *kernelContext) setupFuzzer(features flatrpc.Feature, syscalls map[*prog.Syscall]bool) queue.Source { rnd := rand.New(rand.NewSource(time.Now().UnixNano())) - fuzzerObj := fuzzer.NewFuzzer(context.Background(), &fuzzer.Config{ - Corpus: kc.corpus, + corpusObj := corpus.NewFocusedCorpus(kc.ctx, nil, kc.coverFilters.Areas) + fuzzerObj := fuzzer.NewFuzzer(kc.ctx, &fuzzer.Config{ + Corpus: corpusObj, Coverage: kc.cfg.Cover, // TODO: it may be unstable between different revisions though. // For now it's only kept true because it seems to increase repro chances in local runs (???). @@ -343,6 +341,7 @@ func (kc *kernelContext) setupFuzzer(features flatrpc.Feature, syscalls map[*pro if kc.http != nil { kc.http.Fuzzer.Store(fuzzerObj) kc.http.EnabledSyscalls.Store(syscalls) + kc.http.Corpus.Store(corpusObj) } filtered := manager.FilterCandidates(<-kc.candidates, syscalls, false).Candidates @@ -376,7 +375,6 @@ func (kc *kernelContext) CoverageFilter(modules []*vminfo.KernelModule) []uint64 log.Fatalf("failed to init coverage filter: %v", err) } kc.coverFilters = filters - kc.corpus.SetFocusAreas(filters.Areas) log.Logf(0, "cover filter size: %d", len(filters.ExecutorFilter)) if kc.http != nil { kc.http.Cover.Store(&manager.CoverageInfo{ -- cgit mrf-deployment