From a72ab0ee0af4b1d0d401c09436e1e9ff04bc46e6 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Wed, 4 Sep 2024 17:24:25 +0200 Subject: pkg/manager: move coverage filter code out of syz-manager This will enable the reuse of the functionality elsewhere. --- pkg/manager/covfilter.go | 140 ++++++++++++++++++++++++++++++++++++ pkg/manager/report_generator.go | 70 ++++++++++++++++++ pkg/mgrconfig/config.go | 4 +- pkg/mgrconfig/load.go | 4 +- syz-manager/cover.go | 48 ------------- syz-manager/covfilter.go | 152 ---------------------------------------- syz-manager/http.go | 15 ++-- syz-manager/manager.go | 13 ++++ 8 files changed, 237 insertions(+), 209 deletions(-) create mode 100644 pkg/manager/covfilter.go create mode 100644 pkg/manager/report_generator.go delete mode 100644 syz-manager/cover.go delete mode 100644 syz-manager/covfilter.go diff --git a/pkg/manager/covfilter.go b/pkg/manager/covfilter.go new file mode 100644 index 000000000..420eb40b6 --- /dev/null +++ b/pkg/manager/covfilter.go @@ -0,0 +1,140 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package manager + +import ( + "bufio" + "fmt" + "os" + "regexp" + "sort" + "strconv" + + "github.com/google/syzkaller/pkg/cover/backend" + "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/mgrconfig" +) + +func CreateCoverageFilter(source *ReportGeneratorWrapper, covCfg mgrconfig.CovFilterCfg) ([]uint64, + map[uint64]struct{}, error) { + if covCfg.Empty() { + return nil, nil, nil + } + rg, err := source.Get() + if err != nil { + return nil, nil, err + } + pcs := make(map[uint64]struct{}) + foreachSymbol := func(apply func(*backend.ObjectUnit)) { + for _, sym := range rg.Symbols { + apply(&sym.ObjectUnit) + } + } + if err := covFilterAddFilter(pcs, covCfg.Functions, foreachSymbol); err != nil { + return nil, nil, err + } + foreachUnit := func(apply func(*backend.ObjectUnit)) { + for _, unit := range rg.Units { + apply(&unit.ObjectUnit) + } + } + if err := covFilterAddFilter(pcs, covCfg.Files, foreachUnit); err != nil { + return nil, nil, err + } + if err := covFilterAddRawPCs(pcs, covCfg.RawPCs); err != nil { + return nil, nil, err + } + // Copy pcs into execPCs. This is used to filter coverage in the executor. + execPCs := make([]uint64, 0, len(pcs)) + for pc := range pcs { + execPCs = append(execPCs, pc) + } + // PCs from CMPs are deleted to calculate `filtered coverage` statistics. + for _, sym := range rg.Symbols { + for _, pc := range sym.CMPs { + delete(pcs, pc) + } + } + return execPCs, pcs, nil +} + +func covFilterAddFilter(pcs map[uint64]struct{}, filters []string, foreach func(func(*backend.ObjectUnit))) error { + res, err := compileRegexps(filters) + if err != nil { + return err + } + used := make(map[*regexp.Regexp][]string) + foreach(func(unit *backend.ObjectUnit) { + for _, re := range res { + if re.MatchString(unit.Name) { + // We add both coverage points and comparison interception points + // because executor filters comparisons as well. + for _, pc := range unit.PCs { + pcs[pc] = struct{}{} + } + for _, pc := range unit.CMPs { + pcs[pc] = struct{}{} + } + used[re] = append(used[re], unit.Name) + break + } + } + }) + for _, re := range res { + sort.Strings(used[re]) + log.Logf(0, "coverage filter: %v: %v", re, used[re]) + } + if len(res) != len(used) { + return fmt.Errorf("some filters don't match anything") + } + return nil +} + +func covFilterAddRawPCs(pcs map[uint64]struct{}, rawPCsFiles []string) error { + re := regexp.MustCompile(`(0x[0-9a-f]+)(?:: (0x[0-9a-f]+))?`) + for _, f := range rawPCsFiles { + rawFile, err := os.Open(f) + if err != nil { + return fmt.Errorf("failed to open raw PCs file: %w", err) + } + defer rawFile.Close() + s := bufio.NewScanner(rawFile) + for s.Scan() { + match := re.FindStringSubmatch(s.Text()) + if match == nil { + return fmt.Errorf("bad line: %q", s.Text()) + } + pc, err := strconv.ParseUint(match[1], 0, 64) + if err != nil { + return err + } + weight, err := strconv.ParseUint(match[2], 0, 32) + if match[2] != "" && err != nil { + return err + } + // If no weight is detected, set the weight to 0x1 by default. + if match[2] == "" || weight < 1 { + weight = 1 + } + _ = weight // currently unused + pcs[pc] = struct{}{} + } + if err := s.Err(); err != nil { + return err + } + } + return nil +} + +func compileRegexps(regexpStrings []string) ([]*regexp.Regexp, error) { + var regexps []*regexp.Regexp + for _, rs := range regexpStrings { + r, err := regexp.Compile(rs) + if err != nil { + return nil, fmt.Errorf("failed to compile regexp: %w", err) + } + regexps = append(regexps, r) + } + return regexps, nil +} diff --git a/pkg/manager/report_generator.go b/pkg/manager/report_generator.go new file mode 100644 index 000000000..b3d293ef1 --- /dev/null +++ b/pkg/manager/report_generator.go @@ -0,0 +1,70 @@ +// Copyright 2015 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package manager + +import ( + "fmt" + "sync" + + "github.com/google/syzkaller/pkg/cover" + "github.com/google/syzkaller/pkg/cover/backend" + "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/mgrconfig" + "github.com/google/syzkaller/pkg/vminfo" +) + +type ReportGeneratorWrapper struct { + cfg *mgrconfig.Config + modules []*vminfo.KernelModule + + mu sync.Mutex + initialized bool + cached *cover.ReportGenerator +} + +func ReportGeneratorCache(cfg *mgrconfig.Config) *ReportGeneratorWrapper { + return &ReportGeneratorWrapper{cfg: cfg} +} + +func (w *ReportGeneratorWrapper) Get() (*cover.ReportGenerator, error) { + w.mu.Lock() + defer w.mu.Unlock() + if !w.initialized { + return nil, fmt.Errorf("report generator creation before Init() is called") + } + if w.cached == nil { + log.Logf(0, "initializing coverage information...") + rg, err := cover.MakeReportGenerator(w.cfg, w.cfg.KernelSubsystem, w.modules, w.cfg.RawCover) + if err != nil { + return nil, err + } + w.cached = rg + } + return w.cached, nil +} + +func (w *ReportGeneratorWrapper) Init(modules []*vminfo.KernelModule) { + w.mu.Lock() + defer w.mu.Unlock() + if w.initialized { + panic("Init() called twice") + } + w.initialized = true + w.modules = modules +} + +func (w *ReportGeneratorWrapper) Reset() { + w.mu.Lock() + defer w.mu.Unlock() + w.cached = nil +} + +func CoverToPCs(cfg *mgrconfig.Config, cov []uint64) []uint64 { + pcs := make([]uint64, 0, len(cov)) + for _, pc := range cov { + prev := backend.PreviousInstructionPC(cfg.SysTarget, cfg.Type, pc) + pcs = append(pcs, prev) + } + return pcs +} diff --git a/pkg/mgrconfig/config.go b/pkg/mgrconfig/config.go index 2ea0d8ad9..f4857e613 100644 --- a/pkg/mgrconfig/config.go +++ b/pkg/mgrconfig/config.go @@ -152,7 +152,7 @@ type Config struct { // "pcs": specify raw PC table files name. // Each line of the file should be: "64-bit-pc:32-bit-weight\n". // eg. "0xffffffff81000000:0x10\n" - CovFilter covFilterCfg `json:"cover_filter,omitempty"` + CovFilter CovFilterCfg `json:"cover_filter,omitempty"` // For each prog in the corpus, remember the raw array of PCs obtained from the kernel. // It can be useful for debugging syzkaller descriptions and syzkaller itself. @@ -242,7 +242,7 @@ type Subsystem struct { Paths []string `json:"path"` } -type covFilterCfg struct { +type CovFilterCfg struct { Files []string `json:"files,omitempty"` Functions []string `json:"functions,omitempty"` RawPCs []string `json:"pcs,omitempty"` diff --git a/pkg/mgrconfig/load.go b/pkg/mgrconfig/load.go index aa22b3ea8..76a0bcf14 100644 --- a/pkg/mgrconfig/load.go +++ b/pkg/mgrconfig/load.go @@ -260,8 +260,8 @@ func checkNonEmpty(fields ...string) error { return nil } -func (cfg *Config) HasCovFilter() bool { - return len(cfg.CovFilter.Functions)+len(cfg.CovFilter.Files)+len(cfg.CovFilter.RawPCs) != 0 +func (cov *CovFilterCfg) Empty() bool { + return len(cov.Functions)+len(cov.Files)+len(cov.RawPCs) == 0 } func (cfg *Config) CompleteKernelDirs() { diff --git a/syz-manager/cover.go b/syz-manager/cover.go deleted file mode 100644 index 19ae5f668..000000000 --- a/syz-manager/cover.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2015 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "sync" - - "github.com/google/syzkaller/pkg/cover" - "github.com/google/syzkaller/pkg/cover/backend" - "github.com/google/syzkaller/pkg/log" - "github.com/google/syzkaller/pkg/mgrconfig" - "github.com/google/syzkaller/pkg/vminfo" -) - -var ( - cachedRepGenMu sync.Mutex - cachedRepGen *cover.ReportGenerator -) - -func getReportGenerator(cfg *mgrconfig.Config, modules []*vminfo.KernelModule) (*cover.ReportGenerator, error) { - cachedRepGenMu.Lock() - defer cachedRepGenMu.Unlock() - if cachedRepGen == nil { - log.Logf(0, "initializing coverage information...") - rg, err := cover.MakeReportGenerator(cfg, cfg.KernelSubsystem, modules, cfg.RawCover) - if err != nil { - return nil, err - } - cachedRepGen = rg - } - return cachedRepGen, nil -} - -func resetReportGenerator() { - cachedRepGenMu.Lock() - defer cachedRepGenMu.Unlock() - cachedRepGen = nil -} - -func coverToPCs(cfg *mgrconfig.Config, cov []uint64) []uint64 { - pcs := make([]uint64, 0, len(cov)) - for _, pc := range cov { - prev := backend.PreviousInstructionPC(cfg.SysTarget, cfg.Type, pc) - pcs = append(pcs, prev) - } - return pcs -} diff --git a/syz-manager/covfilter.go b/syz-manager/covfilter.go deleted file mode 100644 index ca6e4175a..000000000 --- a/syz-manager/covfilter.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2020 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "bufio" - "fmt" - "os" - "regexp" - "sort" - "strconv" - - "github.com/google/syzkaller/pkg/cover/backend" - "github.com/google/syzkaller/pkg/log" - "github.com/google/syzkaller/pkg/mgrconfig" - "github.com/google/syzkaller/pkg/vminfo" -) - -func (mgr *Manager) CoverageFilter(modules []*vminfo.KernelModule) []uint64 { - execFilter, filter, err := createCoverageFilter(mgr.cfg, modules) - if err != nil { - log.Fatalf("failed to init coverage filter: %v", err) - } - mgr.modules = modules - mgr.coverFilter = filter - return execFilter -} - -func createCoverageFilter(cfg *mgrconfig.Config, modules []*vminfo.KernelModule) ([]uint64, - map[uint64]struct{}, error) { - if !cfg.HasCovFilter() { - return nil, nil, nil - } - // Always initialize ReportGenerator because RPCServer.NewInput will need it to filter coverage. - rg, err := getReportGenerator(cfg, modules) - if err != nil { - return nil, nil, err - } - pcs := make(map[uint64]struct{}) - foreachSymbol := func(apply func(*backend.ObjectUnit)) { - for _, sym := range rg.Symbols { - apply(&sym.ObjectUnit) - } - } - if err := covFilterAddFilter(pcs, cfg.CovFilter.Functions, foreachSymbol); err != nil { - return nil, nil, err - } - foreachUnit := func(apply func(*backend.ObjectUnit)) { - for _, unit := range rg.Units { - apply(&unit.ObjectUnit) - } - } - if err := covFilterAddFilter(pcs, cfg.CovFilter.Files, foreachUnit); err != nil { - return nil, nil, err - } - if err := covFilterAddRawPCs(pcs, cfg.CovFilter.RawPCs); err != nil { - return nil, nil, err - } - // Copy pcs into execPCs. This is used to filter coverage in the executor. - execPCs := make([]uint64, 0, len(pcs)) - for pc := range pcs { - execPCs = append(execPCs, pc) - } - // PCs from CMPs are deleted to calculate `filtered coverage` statistics. - for _, sym := range rg.Symbols { - for _, pc := range sym.CMPs { - delete(pcs, pc) - } - } - return execPCs, pcs, nil -} - -func covFilterAddFilter(pcs map[uint64]struct{}, filters []string, foreach func(func(*backend.ObjectUnit))) error { - res, err := compileRegexps(filters) - if err != nil { - return err - } - used := make(map[*regexp.Regexp][]string) - foreach(func(unit *backend.ObjectUnit) { - for _, re := range res { - if re.MatchString(unit.Name) { - // We add both coverage points and comparison interception points - // because executor filters comparisons as well. - for _, pc := range unit.PCs { - pcs[pc] = struct{}{} - } - for _, pc := range unit.CMPs { - pcs[pc] = struct{}{} - } - used[re] = append(used[re], unit.Name) - break - } - } - }) - for _, re := range res { - sort.Strings(used[re]) - log.Logf(0, "coverage filter: %v: %v", re, used[re]) - } - if len(res) != len(used) { - return fmt.Errorf("some filters don't match anything") - } - return nil -} - -func covFilterAddRawPCs(pcs map[uint64]struct{}, rawPCsFiles []string) error { - re := regexp.MustCompile(`(0x[0-9a-f]+)(?:: (0x[0-9a-f]+))?`) - for _, f := range rawPCsFiles { - rawFile, err := os.Open(f) - if err != nil { - return fmt.Errorf("failed to open raw PCs file: %w", err) - } - defer rawFile.Close() - s := bufio.NewScanner(rawFile) - for s.Scan() { - match := re.FindStringSubmatch(s.Text()) - if match == nil { - return fmt.Errorf("bad line: %q", s.Text()) - } - pc, err := strconv.ParseUint(match[1], 0, 64) - if err != nil { - return err - } - weight, err := strconv.ParseUint(match[2], 0, 32) - if match[2] != "" && err != nil { - return err - } - // If no weight is detected, set the weight to 0x1 by default. - if match[2] == "" || weight < 1 { - weight = 1 - } - _ = weight // currently unused - pcs[pc] = struct{}{} - } - if err := s.Err(); err != nil { - return err - } - } - return nil -} - -func compileRegexps(regexpStrings []string) ([]*regexp.Regexp, error) { - var regexps []*regexp.Regexp - for _, rs := range regexpStrings { - r, err := regexp.Compile(rs) - if err != nil { - return nil, fmt.Errorf("failed to compile regexp: %w", err) - } - regexps = append(regexps, r) - } - return regexps, nil -} diff --git a/syz-manager/http.go b/syz-manager/http.go index 288d043fc..2e5f703a9 100644 --- a/syz-manager/http.go +++ b/syz-manager/http.go @@ -24,6 +24,7 @@ import ( "github.com/google/syzkaller/pkg/fuzzer" "github.com/google/syzkaller/pkg/html/pages" "github.com/google/syzkaller/pkg/log" + "github.com/google/syzkaller/pkg/manager" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/stat" "github.com/google/syzkaller/pkg/vcs" @@ -332,7 +333,7 @@ func (mgr *Manager) httpCoverCover(w http.ResponseWriter, r *http.Request, funcF return } - rg, err := getReportGenerator(mgr.cfg, mgr.modules) + rg, err := mgr.reportGenerator.Get() if err != nil { http.Error(w, fmt.Sprintf("failed to generate coverage profile: %v", err), http.StatusInternalServerError) return @@ -340,7 +341,7 @@ func (mgr *Manager) httpCoverCover(w http.ResponseWriter, r *http.Request, funcF if r.FormValue("flush") != "" { defer func() { - resetReportGenerator() + mgr.reportGenerator.Reset() debug.FreeOSMemory() }() } @@ -362,13 +363,13 @@ func (mgr *Manager) httpCoverCover(w http.ResponseWriter, r *http.Request, funcF progs = append(progs, cover.Prog{ Sig: sig, Data: string(inp.Prog.Serialize()), - PCs: coverToPCs(mgr.cfg, inp.Updates[updateID].RawCover), + PCs: manager.CoverToPCs(mgr.cfg, inp.Updates[updateID].RawCover), }) } else { progs = append(progs, cover.Prog{ Sig: sig, Data: string(inp.Prog.Serialize()), - PCs: coverToPCs(mgr.cfg, inp.Cover), + PCs: manager.CoverToPCs(mgr.cfg, inp.Cover), }) } } else { @@ -380,7 +381,7 @@ func (mgr *Manager) httpCoverCover(w http.ResponseWriter, r *http.Request, funcF progs = append(progs, cover.Prog{ Sig: inp.Sig, Data: string(inp.Prog.Serialize()), - PCs: coverToPCs(mgr.cfg, inp.Cover), + PCs: manager.CoverToPCs(mgr.cfg, inp.Cover), }) } } @@ -562,6 +563,10 @@ func (mgr *Manager) httpDebugInput(w http.ResponseWriter, r *http.Request) { } func (mgr *Manager) modulesInfo(w http.ResponseWriter, r *http.Request) { + if !mgr.checkDone.Load() { + http.Error(w, "info is not ready, please try again later after fuzzer started", http.StatusInternalServerError) + return + } modules, err := json.MarshalIndent(mgr.modules, "", "\t") if err != nil { fmt.Fprintf(w, "unable to create JSON modules info: %v", err) diff --git a/syz-manager/manager.go b/syz-manager/manager.go index d88ca816e..db181d3a3 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -89,6 +89,7 @@ type Manager struct { crashTypes map[string]bool enabledFeatures flatrpc.Feature checkDone atomic.Bool + reportGenerator *manager.ReportGeneratorWrapper fresh bool expertMode bool modules []*vminfo.KernelModule @@ -229,6 +230,7 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { externalReproQueue: make(chan *manager.Crash, 10), crashes: make(chan *manager.Crash, 10), saturatedCalls: make(map[string]bool), + reportGenerator: manager.ReportGeneratorCache(cfg), } if *flagDebug { @@ -1424,6 +1426,17 @@ func (mgr *Manager) dashboardReproTasks() { } } +func (mgr *Manager) CoverageFilter(modules []*vminfo.KernelModule) []uint64 { + mgr.reportGenerator.Init(modules) + execFilter, filter, err := manager.CreateCoverageFilter(mgr.reportGenerator, mgr.cfg.CovFilter) + if err != nil { + log.Fatalf("failed to init coverage filter: %v", err) + } + mgr.modules = modules + mgr.coverFilter = filter + return execFilter +} + func publicWebAddr(addr string) string { _, port, err := net.SplitHostPort(addr) if err == nil && port != "" { -- cgit mrf-deployment