From 8b02337e33dbcc9aae5b80ebe850733ca7c8fac7 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Thu, 3 Oct 2024 17:48:33 +0200 Subject: pkg/manager: factor out the HTTP server code Decouple it from syz-manager. Remove a lot of no longer necessary mutex calls. --- syz-manager/http.go | 1242 ------------------------------------------------ syz-manager/hub.go | 4 +- syz-manager/manager.go | 183 ++----- 3 files changed, 50 insertions(+), 1379 deletions(-) delete mode 100644 syz-manager/http.go (limited to 'syz-manager') diff --git a/syz-manager/http.go b/syz-manager/http.go deleted file mode 100644 index 2e5f703a9..000000000 --- a/syz-manager/http.go +++ /dev/null @@ -1,1242 +0,0 @@ -// Copyright 2015 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "html/template" - "io" - "net/http" - _ "net/http/pprof" - "os" - "path/filepath" - "regexp" - "runtime/debug" - "sort" - "strconv" - "strings" - "time" - - "github.com/google/syzkaller/pkg/cover" - "github.com/google/syzkaller/pkg/fuzzer" - "github.com/google/syzkaller/pkg/html/pages" - "github.com/google/syzkaller/pkg/log" - "github.com/google/syzkaller/pkg/manager" - "github.com/google/syzkaller/pkg/osutil" - "github.com/google/syzkaller/pkg/stat" - "github.com/google/syzkaller/pkg/vcs" - "github.com/google/syzkaller/prog" - "github.com/google/syzkaller/vm/dispatcher" - "github.com/gorilla/handlers" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promhttp" -) - -func (mgr *Manager) initHTTP() { - handle := func(pattern string, handler func(http.ResponseWriter, *http.Request)) { - http.Handle(pattern, handlers.CompressHandler(http.HandlerFunc(handler))) - } - handle("/", mgr.httpSummary) - handle("/config", mgr.httpConfig) - handle("/expert_mode", mgr.httpExpertMode) - handle("/stats", mgr.httpStats) - handle("/vms", mgr.httpVMs) - handle("/vm", mgr.httpVM) - handle("/metrics", promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}).ServeHTTP) - handle("/syscalls", mgr.httpSyscalls) - handle("/corpus", mgr.httpCorpus) - handle("/corpus.db", mgr.httpDownloadCorpus) - handle("/crash", mgr.httpCrash) - handle("/cover", mgr.httpCover) - handle("/subsystemcover", mgr.httpSubsystemCover) - handle("/modulecover", mgr.httpModuleCover) - handle("/prio", mgr.httpPrio) - handle("/file", mgr.httpFile) - handle("/report", mgr.httpReport) - handle("/rawcover", mgr.httpRawCover) - handle("/rawcoverfiles", mgr.httpRawCoverFiles) - handle("/filterpcs", mgr.httpFilterPCs) - handle("/funccover", mgr.httpFuncCover) - handle("/filecover", mgr.httpFileCover) - handle("/input", mgr.httpInput) - handle("/debuginput", mgr.httpDebugInput) - handle("/modules", mgr.modulesInfo) - handle("/jobs", mgr.httpJobs) - // Browsers like to request this, without special handler this goes to / handler. - handle("/favicon.ico", func(w http.ResponseWriter, r *http.Request) {}) - - log.Logf(0, "serving http on http://%v", mgr.cfg.HTTP) - go func() { - err := http.ListenAndServe(mgr.cfg.HTTP, nil) - if err != nil { - log.Fatalf("failed to listen on %v: %v", mgr.cfg.HTTP, err) - } - }() -} - -func (mgr *Manager) httpSummary(w http.ResponseWriter, r *http.Request) { - revision, link := revisionAndLink() - data := &UISummaryData{ - Name: mgr.cfg.Name, - Revision: revision, - RevisionLink: link, - Expert: mgr.expertMode, - Log: log.CachedLogOutput(), - } - - level := stat.Simple - if mgr.expertMode { - level = stat.All - } - for _, stat := range stat.Collect(level) { - data.Stats = append(data.Stats, UIStat{ - Name: stat.Name, - Value: stat.Value, - Hint: stat.Desc, - Link: stat.Link, - }) - } - - var err error - if data.Crashes, err = mgr.collectCrashes(mgr.cfg.Workdir); err != nil { - http.Error(w, fmt.Sprintf("failed to collect crashes: %v", err), http.StatusInternalServerError) - return - } - executeTemplate(w, summaryTemplate, data) -} - -func revisionAndLink() (string, string) { - var revision string - var link string - if len(prog.GitRevisionBase) > 8 { - revision = prog.GitRevisionBase[:8] - link = vcs.LogLink(vcs.SyzkallerRepo, prog.GitRevisionBase) - } else { - revision = prog.GitRevisionBase - link = "" - } - - return revision, link -} - -func (mgr *Manager) httpConfig(w http.ResponseWriter, r *http.Request) { - data, err := json.MarshalIndent(mgr.cfg, "", "\t") - if err != nil { - http.Error(w, fmt.Sprintf("failed to encode json: %v", err), - http.StatusInternalServerError) - return - } - w.Write(data) -} - -func (mgr *Manager) httpExpertMode(w http.ResponseWriter, r *http.Request) { - mgr.expertMode = !mgr.expertMode - http.Redirect(w, r, "/", http.StatusFound) -} - -func (mgr *Manager) httpSyscalls(w http.ResponseWriter, r *http.Request) { - data := &UISyscallsData{ - Name: mgr.cfg.Name, - } - for c, cc := range mgr.collectSyscallInfo() { - var syscallID *int - if syscall, ok := mgr.target.SyscallMap[c]; ok { - syscallID = &syscall.ID - } - data.Calls = append(data.Calls, UICallType{ - Name: c, - ID: syscallID, - Inputs: cc.Count, - Cover: len(cc.Cover), - }) - } - sort.Slice(data.Calls, func(i, j int) bool { - return data.Calls[i].Name < data.Calls[j].Name - }) - executeTemplate(w, syscallsTemplate, data) -} - -func (mgr *Manager) httpStats(w http.ResponseWriter, r *http.Request) { - executeTemplate(w, pages.StatsTemplate, stat.RenderGraphs()) -} - -func (mgr *Manager) httpVMs(w http.ResponseWriter, r *http.Request) { - data := &UIVMData{ - Name: mgr.cfg.Name, - } - // TODO: we could also query vmLoop for VMs that are idle (waiting to start reproducing), - // and query the exact bug that is being reproduced by a VM. - for id, state := range mgr.pool.State() { - name := fmt.Sprintf("#%d", id) - info := UIVMInfo{ - Name: name, - State: "unknown", - Since: time.Since(state.LastUpdate), - } - switch state.State { - case dispatcher.StateOffline: - info.State = "offline" - case dispatcher.StateBooting: - info.State = "booting" - case dispatcher.StateWaiting: - info.State = "waiting" - case dispatcher.StateRunning: - info.State = "running: " + state.Status - } - if state.Reserved { - info.State = "[reserved] " + info.State - } - if state.MachineInfo != nil { - info.MachineInfo = fmt.Sprintf("/vm?type=machine-info&id=%d", id) - } - if state.DetailedStatus != nil { - info.DetailedStatus = fmt.Sprintf("/vm?type=detailed-status&id=%v", id) - } - data.VMs = append(data.VMs, info) - } - executeTemplate(w, vmsTemplate, data) -} - -func (mgr *Manager) httpVM(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", ctTextPlain) - id, err := strconv.Atoi(r.FormValue("id")) - infos := mgr.pool.State() - if err != nil || id < 0 || id >= len(infos) { - http.Error(w, "invalid instance id", http.StatusBadRequest) - return - } - info := infos[id] - switch r.FormValue("type") { - case "machine-info": - if info.MachineInfo != nil { - w.Write(info.MachineInfo()) - } - case "detailed-status": - if info.DetailedStatus != nil { - w.Write(info.DetailedStatus()) - } - default: - w.Write([]byte("unknown info type")) - } -} - -func (mgr *Manager) httpCrash(w http.ResponseWriter, r *http.Request) { - crashID := r.FormValue("id") - crash := readCrash(mgr.cfg.Workdir, crashID, nil, mgr.firstConnect.Load(), true) - if crash == nil { - http.Error(w, "failed to read crash info", http.StatusInternalServerError) - return - } - executeTemplate(w, crashTemplate, crash) -} - -func (mgr *Manager) httpCorpus(w http.ResponseWriter, r *http.Request) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - data := UICorpus{ - Call: r.FormValue("call"), - RawCover: mgr.cfg.RawCover, - } - for _, inp := range mgr.corpus.Items() { - if data.Call != "" && data.Call != inp.StringCall() { - continue - } - data.Inputs = append(data.Inputs, &UIInput{ - Sig: inp.Sig, - Short: inp.Prog.String(), - Cover: len(inp.Cover), - }) - } - sort.Slice(data.Inputs, func(i, j int) bool { - a, b := data.Inputs[i], data.Inputs[j] - if a.Cover != b.Cover { - return a.Cover > b.Cover - } - return a.Short < b.Short - }) - executeTemplate(w, corpusTemplate, data) -} - -func (mgr *Manager) httpDownloadCorpus(w http.ResponseWriter, r *http.Request) { - corpus := filepath.Join(mgr.cfg.Workdir, "corpus.db") - file, err := os.Open(corpus) - if err != nil { - http.Error(w, fmt.Sprintf("failed to open corpus : %v", err), http.StatusInternalServerError) - return - } - defer file.Close() - buf, err := io.ReadAll(file) - if err != nil { - http.Error(w, fmt.Sprintf("failed to read corpus : %v", err), http.StatusInternalServerError) - return - } - w.Write(buf) -} - -const ( - DoHTML int = iota - DoSubsystemCover - DoModuleCover - DoFuncCover - DoFileCover - DoRawCoverFiles - DoRawCover - DoFilterPCs - DoCoverJSONL -) - -func (mgr *Manager) httpCover(w http.ResponseWriter, r *http.Request) { - if !mgr.cfg.Cover { - mgr.httpCoverFallback(w, r) - return - } - if r.FormValue("jsonl") == "1" { - mgr.httpCoverCover(w, r, DoCoverJSONL) - return - } - mgr.httpCoverCover(w, r, DoHTML) -} - -func (mgr *Manager) httpSubsystemCover(w http.ResponseWriter, r *http.Request) { - if !mgr.cfg.Cover { - mgr.httpCoverFallback(w, r) - return - } - mgr.httpCoverCover(w, r, DoSubsystemCover) -} - -func (mgr *Manager) httpModuleCover(w http.ResponseWriter, r *http.Request) { - if !mgr.cfg.Cover { - mgr.httpCoverFallback(w, r) - return - } - mgr.httpCoverCover(w, r, DoModuleCover) -} - -const ctTextPlain = "text/plain; charset=utf-8" -const ctApplicationJSON = "application/json" - -func (mgr *Manager) httpCoverCover(w http.ResponseWriter, r *http.Request, funcFlag int) { - if !mgr.cfg.Cover { - http.Error(w, "coverage is not enabled", http.StatusInternalServerError) - return - } - - // Don't hold the mutex while creating report generator and generating the report, - // these operations take lots of time. - if !mgr.checkDone.Load() { - http.Error(w, "coverage is not ready, please try again later after fuzzer started", http.StatusInternalServerError) - return - } - - rg, err := mgr.reportGenerator.Get() - if err != nil { - http.Error(w, fmt.Sprintf("failed to generate coverage profile: %v", err), http.StatusInternalServerError) - return - } - - if r.FormValue("flush") != "" { - defer func() { - mgr.reportGenerator.Reset() - debug.FreeOSMemory() - }() - } - - mgr.mu.Lock() - var progs []cover.Prog - if sig := r.FormValue("input"); sig != "" { - inp := mgr.corpus.Item(sig) - if inp == nil { - http.Error(w, "unknown input hash", http.StatusInternalServerError) - return - } - if r.FormValue("update_id") != "" { - updateID, err := strconv.Atoi(r.FormValue("update_id")) - if err != nil || updateID < 0 || updateID >= len(inp.Updates) { - http.Error(w, "bad call_id", http.StatusBadRequest) - return - } - progs = append(progs, cover.Prog{ - Sig: sig, - Data: string(inp.Prog.Serialize()), - PCs: manager.CoverToPCs(mgr.cfg, inp.Updates[updateID].RawCover), - }) - } else { - progs = append(progs, cover.Prog{ - Sig: sig, - Data: string(inp.Prog.Serialize()), - PCs: manager.CoverToPCs(mgr.cfg, inp.Cover), - }) - } - } else { - call := r.FormValue("call") - for _, inp := range mgr.corpus.Items() { - if call != "" && call != inp.StringCall() { - continue - } - progs = append(progs, cover.Prog{ - Sig: inp.Sig, - Data: string(inp.Prog.Serialize()), - PCs: manager.CoverToPCs(mgr.cfg, inp.Cover), - }) - } - } - mgr.mu.Unlock() - - var coverFilter map[uint64]struct{} - if r.FormValue("filter") != "" || funcFlag == DoFilterPCs { - if mgr.coverFilter == nil { - http.Error(w, "cover is not filtered in config", http.StatusInternalServerError) - return - } - coverFilter = mgr.coverFilter - } - - params := cover.HandlerParams{ - Progs: progs, - Filter: coverFilter, - Debug: r.FormValue("debug") != "", - Force: r.FormValue("force") != "", - } - - type handlerFuncType func(w io.Writer, params cover.HandlerParams) error - flagToFunc := map[int]struct { - Do handlerFuncType - contentType string - }{ - DoHTML: {rg.DoHTML, ""}, - DoSubsystemCover: {rg.DoSubsystemCover, ""}, - DoModuleCover: {rg.DoModuleCover, ""}, - DoFuncCover: {rg.DoFuncCover, ctTextPlain}, - DoFileCover: {rg.DoFileCover, ctTextPlain}, - DoRawCoverFiles: {rg.DoRawCoverFiles, ctTextPlain}, - DoRawCover: {rg.DoRawCover, ctTextPlain}, - DoFilterPCs: {rg.DoFilterPCs, ctTextPlain}, - DoCoverJSONL: {rg.DoCoverJSONL, ctApplicationJSON}, - } - - if ct := flagToFunc[funcFlag].contentType; ct != "" { - w.Header().Set("Content-Type", ct) - } - - if err := flagToFunc[funcFlag].Do(w, params); err != nil { - http.Error(w, fmt.Sprintf("failed to generate coverage profile: %v", err), http.StatusInternalServerError) - return - } -} - -func (mgr *Manager) httpCoverFallback(w http.ResponseWriter, r *http.Request) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - calls := make(map[int][]int) - for s := range mgr.corpus.Signal() { - id, errno := prog.DecodeFallbackSignal(uint64(s)) - calls[id] = append(calls[id], errno) - } - data := &UIFallbackCoverData{} - for call := range mgr.targetEnabledSyscalls { - errnos := calls[call.ID] - sort.Ints(errnos) - successful := 0 - for len(errnos) != 0 && errnos[0] == 0 { - successful++ - errnos = errnos[1:] - } - data.Calls = append(data.Calls, UIFallbackCall{ - Name: call.Name, - Successful: successful, - Errnos: errnos, - }) - } - sort.Slice(data.Calls, func(i, j int) bool { - return data.Calls[i].Name < data.Calls[j].Name - }) - executeTemplate(w, fallbackCoverTemplate, data) -} - -func (mgr *Manager) httpFuncCover(w http.ResponseWriter, r *http.Request) { - mgr.httpCoverCover(w, r, DoFuncCover) -} - -func (mgr *Manager) httpFileCover(w http.ResponseWriter, r *http.Request) { - mgr.httpCoverCover(w, r, DoFileCover) -} - -func (mgr *Manager) httpPrio(w http.ResponseWriter, r *http.Request) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - callName := r.FormValue("call") - call := mgr.target.SyscallMap[callName] - if call == nil { - http.Error(w, fmt.Sprintf("unknown call: %v", callName), http.StatusInternalServerError) - return - } - - var corpus []*prog.Prog - for _, inp := range mgr.corpus.Items() { - corpus = append(corpus, inp.Prog) - } - prios := mgr.target.CalculatePriorities(corpus) - - data := &UIPrioData{Call: callName} - for i, p := range prios[call.ID] { - data.Prios = append(data.Prios, UIPrio{mgr.target.Syscalls[i].Name, p}) - } - sort.Slice(data.Prios, func(i, j int) bool { - return data.Prios[i].Prio > data.Prios[j].Prio - }) - executeTemplate(w, prioTemplate, data) -} - -func (mgr *Manager) httpFile(w http.ResponseWriter, r *http.Request) { - file := filepath.Clean(r.FormValue("name")) - if !strings.HasPrefix(file, "crashes/") && !strings.HasPrefix(file, "corpus/") { - http.Error(w, "oh, oh, oh!", http.StatusInternalServerError) - return - } - file = filepath.Join(mgr.cfg.Workdir, file) - f, err := os.Open(file) - if err != nil { - http.Error(w, "failed to open the file", http.StatusInternalServerError) - return - } - defer f.Close() - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - io.Copy(w, f) -} - -func (mgr *Manager) httpInput(w http.ResponseWriter, r *http.Request) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - inp := mgr.corpus.Item(r.FormValue("sig")) - if inp == nil { - http.Error(w, "can't find the input", http.StatusInternalServerError) - return - } - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Write(inp.Prog.Serialize()) -} - -func (mgr *Manager) httpDebugInput(w http.ResponseWriter, r *http.Request) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - inp := mgr.corpus.Item(r.FormValue("sig")) - if inp == nil { - http.Error(w, "can't find the input", http.StatusInternalServerError) - return - } - getIDs := func(callID int) []int { - ret := []int{} - for id, update := range inp.Updates { - if update.Call == callID { - ret = append(ret, id) - } - } - return ret - } - data := []UIRawCallCover{} - for pos, line := range strings.Split(string(inp.Prog.Serialize()), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - data = append(data, UIRawCallCover{ - Sig: r.FormValue("sig"), - Call: line, - UpdateIDs: getIDs(pos), - }) - } - extraIDs := getIDs(-1) - if len(extraIDs) > 0 { - data = append(data, UIRawCallCover{ - Sig: r.FormValue("sig"), - Call: ".extra", - UpdateIDs: extraIDs, - }) - } - executeTemplate(w, rawCoverTemplate, data) -} - -func (mgr *Manager) modulesInfo(w http.ResponseWriter, r *http.Request) { - if !mgr.checkDone.Load() { - http.Error(w, "info is not ready, please try again later after fuzzer started", http.StatusInternalServerError) - return - } - modules, err := json.MarshalIndent(mgr.modules, "", "\t") - if err != nil { - fmt.Fprintf(w, "unable to create JSON modules info: %v", err) - return - } - w.Header().Set("Content-Type", "application/json") - w.Write(modules) -} - -var alphaNumRegExp = regexp.MustCompile(`^[a-zA-Z0-9]*$`) - -func isAlphanumeric(s string) bool { - return alphaNumRegExp.MatchString(s) -} - -func (mgr *Manager) httpReport(w http.ResponseWriter, r *http.Request) { - mgr.mu.Lock() - defer mgr.mu.Unlock() - - crashID := r.FormValue("id") - if !isAlphanumeric(crashID) { - http.Error(w, "wrong id", http.StatusBadRequest) - return - } - - desc, err := os.ReadFile(filepath.Join(mgr.crashdir, crashID, "description")) - if err != nil { - http.Error(w, "failed to read description file", http.StatusInternalServerError) - return - } - tag, _ := os.ReadFile(filepath.Join(mgr.crashdir, crashID, "repro.tag")) - prog, _ := os.ReadFile(filepath.Join(mgr.crashdir, crashID, "repro.prog")) - cprog, _ := os.ReadFile(filepath.Join(mgr.crashdir, crashID, "repro.cprog")) - rep, _ := os.ReadFile(filepath.Join(mgr.crashdir, crashID, "repro.report")) - - commitDesc := "" - if len(tag) != 0 { - commitDesc = fmt.Sprintf(" on commit %s.", trimNewLines(tag)) - } - fmt.Fprintf(w, "Syzkaller hit '%s' bug%s.\n\n", trimNewLines(desc), commitDesc) - if len(rep) != 0 { - fmt.Fprintf(w, "%s\n\n", rep) - } - if len(prog) == 0 && len(cprog) == 0 { - fmt.Fprintf(w, "The bug is not reproducible.\n") - } else { - fmt.Fprintf(w, "Syzkaller reproducer:\n%s\n\n", prog) - if len(cprog) != 0 { - fmt.Fprintf(w, "C reproducer:\n%s\n\n", cprog) - } - } -} - -func (mgr *Manager) httpRawCover(w http.ResponseWriter, r *http.Request) { - mgr.httpCoverCover(w, r, DoRawCover) -} - -func (mgr *Manager) httpRawCoverFiles(w http.ResponseWriter, r *http.Request) { - mgr.httpCoverCover(w, r, DoRawCoverFiles) -} - -func (mgr *Manager) httpFilterPCs(w http.ResponseWriter, r *http.Request) { - mgr.httpCoverCover(w, r, DoFilterPCs) -} - -func (mgr *Manager) collectCrashes(workdir string) ([]*UICrashType, error) { - // Note: mu is not locked here. - var repros map[string]bool - if !mgr.cfg.VMLess && mgr.reproLoop != nil { - repros = mgr.reproLoop.Reproducing() - } - - crashdir := filepath.Join(workdir, "crashes") - dirs, err := osutil.ListDir(crashdir) - if err != nil { - return nil, err - } - var crashTypes []*UICrashType - for _, dir := range dirs { - crash := readCrash(workdir, dir, repros, mgr.firstConnect.Load(), false) - if crash != nil { - crashTypes = append(crashTypes, crash) - } - } - sort.Slice(crashTypes, func(i, j int) bool { - return strings.ToLower(crashTypes[i].Description) < strings.ToLower(crashTypes[j].Description) - }) - return crashTypes, nil -} - -func (mgr *Manager) httpJobs(w http.ResponseWriter, r *http.Request) { - var list []*fuzzer.JobInfo - if fuzzer := mgr.fuzzer.Load(); fuzzer != nil { - list = fuzzer.RunningJobs() - } - if key := r.FormValue("id"); key != "" { - for _, item := range list { - if item.ID() == key { - w.Write(item.Bytes()) - return - } - } - http.Error(w, "invalid job id (the job has likely already finished)", http.StatusBadRequest) - return - } - jobType := r.FormValue("type") - data := UIJobList{ - Title: fmt.Sprintf("%s jobs", jobType), - } - switch jobType { - case "triage": - case "smash": - case "hints": - default: - http.Error(w, "unknown job type", http.StatusBadRequest) - return - } - for _, item := range list { - if item.Type != jobType { - continue - } - data.Jobs = append(data.Jobs, UIJobInfo{ - ID: item.ID(), - Short: item.Name, - Execs: item.Execs.Load(), - Calls: strings.Join(item.Calls, ", "), - }) - } - sort.Slice(data.Jobs, func(i, j int) bool { - a, b := data.Jobs[i], data.Jobs[j] - return a.Short < b.Short - }) - executeTemplate(w, jobListTemplate, data) -} - -func readCrash(workdir, dir string, repros map[string]bool, start int64, full bool) *UICrashType { - if len(dir) != 40 { - return nil - } - crashdir := filepath.Join(workdir, "crashes") - descFile, err := os.Open(filepath.Join(crashdir, dir, "description")) - if err != nil { - return nil - } - defer descFile.Close() - descBytes, err := io.ReadAll(descFile) - if err != nil || len(descBytes) == 0 { - return nil - } - desc := string(trimNewLines(descBytes)) - stat, err := descFile.Stat() - if err != nil { - return nil - } - modTime := stat.ModTime() - descFile.Close() - - files, err := osutil.ListDir(filepath.Join(crashdir, dir)) - if err != nil { - return nil - } - var crashes []*UICrash - reproAttempts := 0 - hasRepro, hasCRepro := false, false - strace := "" - reports := make(map[string]bool) - for _, f := range files { - if strings.HasPrefix(f, "log") { - index, err := strconv.ParseUint(f[3:], 10, 64) - if err == nil { - crashes = append(crashes, &UICrash{ - Index: int(index), - }) - } - } else if strings.HasPrefix(f, "report") { - reports[f] = true - } else if f == "repro.prog" { - hasRepro = true - } else if f == "repro.cprog" { - hasCRepro = true - } else if f == "repro.report" { - } else if f == "repro0" || f == "repro1" || f == "repro2" { - reproAttempts++ - } else if f == "strace.log" { - strace = filepath.Join("crashes", dir, f) - } - } - - if full { - for _, crash := range crashes { - index := strconv.Itoa(crash.Index) - crash.Log = filepath.Join("crashes", dir, "log"+index) - if stat, err := os.Stat(filepath.Join(workdir, crash.Log)); err == nil { - crash.Time = stat.ModTime() - crash.Active = start != 0 && crash.Time.Unix() >= start - } - tag, _ := os.ReadFile(filepath.Join(crashdir, dir, "tag"+index)) - crash.Tag = string(tag) - reportFile := filepath.Join("crashes", dir, "report"+index) - if osutil.IsExist(filepath.Join(workdir, reportFile)) { - crash.Report = reportFile - } - } - sort.Slice(crashes, func(i, j int) bool { - return crashes[i].Time.After(crashes[j].Time) - }) - } - - triaged := reproStatus(hasRepro, hasCRepro, repros[desc], reproAttempts >= maxReproAttempts) - return &UICrashType{ - Description: desc, - LastTime: modTime, - Active: start != 0 && modTime.Unix() >= start, - ID: dir, - Count: len(crashes), - Triaged: triaged, - Strace: strace, - Crashes: crashes, - } -} - -func reproStatus(hasRepro, hasCRepro, reproducing, nonReproducible bool) string { - status := "" - if hasRepro { - status = "has repro" - if hasCRepro { - status = "has C repro" - } - } else if reproducing { - status = "reproducing" - } else if nonReproducible { - status = "non-reproducible" - } - return status -} - -func executeTemplate(w http.ResponseWriter, templ *template.Template, data interface{}) { - buf := new(bytes.Buffer) - if err := templ.Execute(buf, data); err != nil { - log.Logf(0, "failed to execute template: %v", err) - http.Error(w, fmt.Sprintf("failed to execute template: %v", err), http.StatusInternalServerError) - return - } - w.Write(buf.Bytes()) -} - -func trimNewLines(data []byte) []byte { - for len(data) > 0 && data[len(data)-1] == '\n' { - data = data[:len(data)-1] - } - return data -} - -type UISummaryData struct { - Name string - Revision string - RevisionLink string - Expert bool - Stats []UIStat - Crashes []*UICrashType - Log string -} - -type UIVMData struct { - Name string - VMs []UIVMInfo -} - -type UIVMInfo struct { - Name string - State string - Since time.Duration - MachineInfo string - DetailedStatus string -} - -type UISyscallsData struct { - Name string - Calls []UICallType -} - -type UICrashType struct { - Description string - LastTime time.Time - Active bool - ID string - Count int - Triaged string - Strace string - Crashes []*UICrash -} - -type UICrash struct { - Index int - Time time.Time - Active bool - Log string - Report string - Tag string -} - -type UIStat struct { - Name string - Value string - Hint string - Link string -} - -type UICallType struct { - Name string - ID *int - Inputs int - Cover int -} - -type UICorpus struct { - Call string - RawCover bool - Inputs []*UIInput -} - -type UIInput struct { - Sig string - Short string - Cover int -} - -var summaryTemplate = pages.Create(` - - - - {{.Name}} syzkaller - {{HEAD}} - - -{{.Name }} syzkaller -[config] -{{.Revision}} -{{if .Expert}}disable{{else}}enable{{end}} expert mode -
- - - - {{range $s := $.Stats}} - - - - - {{end}} -
Stats 📈
{{$s.Name}} - {{if $s.Link}} - {{$s.Value}} - {{else}} - {{$s.Value}} - {{end}} -
- - - - - - - - - - {{range $c := $.Crashes}} - - - - - - - {{end}} -
Crashes:
DescriptionCountLast TimeReport
{{$c.Description}}{{$c.Count}}{{formatTime $c.LastTime}} - {{if $c.Triaged}} - {{$c.Triaged}} - {{end}} - {{if $c.Strace}} - Strace - {{end}} -
- -Log: -
- - - -`) - -var vmsTemplate = pages.Create(` - - - - {{.Name }} syzkaller - {{HEAD}} - - - - - - - - - - - - - {{range $vm := $.VMs}} - - - - - - - - {{end}} -
VM Info:
NameStateSinceMachine InfoStatus
{{$vm.Name}}{{$vm.State}}{{formatDuration $vm.Since}}{{optlink $vm.MachineInfo "info"}}{{optlink $vm.DetailedStatus "status"}}
- -`) - -var syscallsTemplate = pages.Create(` - - - - {{.Name }} syzkaller - {{HEAD}} - - - - - - - - - - - - {{range $c := $.Calls}} - - - - - - - {{end}} -
Per-syscall coverage:
SyscallInputsCoveragePrio
{{$c.Name}}{{if $c.ID }} [{{$c.ID}}]{{end}}{{$c.Inputs}}{{$c.Cover}}prio
- -`) - -var crashTemplate = pages.Create(` - - - - {{.Description}} - {{HEAD}} - - -{{.Description}} - -{{if .Triaged}} -Report: {{.Triaged}} -{{end}} - - - - - - - - - - {{range $c := $.Crashes}} - - - - - {{end}} - - - - - {{end}} -
#LogReportTimeTag
{{$c.Index}}log - {{if $c.Report}} - report{{formatTime $c.Time}}{{formatTagHash $c.Tag}}
- -`) - -var corpusTemplate = pages.Create(` - - - - syzkaller corpus - {{HEAD}} - - - - - - - - - - {{range $inp := $.Inputs}} - - - - - {{end}} -
Corpus{{if $.Call}} for {{$.Call}}{{end}}:
CoverageProgram
- {{$inp.Cover}} - {{if $.RawCover}} - / [raw] - {{end}} - {{$inp.Short}}
- -`) - -type UIPrioData struct { - Call string - Prios []UIPrio -} - -type UIPrio struct { - Call string - Prio int32 -} - -var prioTemplate = pages.Create(` - - - - syzkaller priorities - {{HEAD}} - - - - - - - - - {{range $p := $.Prios}} - - - - - {{end}} -
Priorities for {{$.Call}}:
PrioCall
{{printf "%5v" $p.Prio}}{{$p.Call}}
- -`) - -type UIFallbackCoverData struct { - Calls []UIFallbackCall -} - -type UIFallbackCall struct { - Name string - Successful int - Errnos []int -} - -var fallbackCoverTemplate = pages.Create(` - - - - syzkaller coverage - {{HEAD}} - - - - - - - - - {{range $c := $.Calls}} - - - - - - {{end}} -
CallSuccessfulErrnos
{{$c.Name}}{{if $c.Successful}}{{$c.Successful}}{{end}}{{range $e := $c.Errnos}}{{$e}} {{end}}
- -`) - -type UIRawCallCover struct { - Sig string - Call string - UpdateIDs []int -} - -var rawCoverTemplate = pages.Create(` - - - - syzkaller raw cover - {{HEAD}} - - - - - - - - - - {{range $line := .}} - - - - - {{end}} -
Raw cover
LineLinks
{{$line.Call}} - {{range $id := $line.UpdateIDs}} - [{{$id}}] - {{end}} -
- -`) - -type UIJobList struct { - Title string - Jobs []UIJobInfo -} - -type UIJobInfo struct { - ID string - Short string - Calls string - Execs int32 -} - -var jobListTemplate = pages.Create(` - - - - {{.Title}} - {{HEAD}} - - - - - - - - - - - - {{range $job := $.Jobs}} - - - - - - {{end}} -
{{.Title}} ({{len .Jobs}}):
ProgramCallsExecs
{{$job.Short}}{{$job.Calls}}{{$job.Execs}}
- -`) diff --git a/syz-manager/hub.go b/syz-manager/hub.go index 014822bfe..4f0a02de0 100644 --- a/syz-manager/hub.go +++ b/syz-manager/hub.go @@ -39,13 +39,13 @@ func pickGetter(key string) keyGetter { } } -func (mgr *Manager) hubSyncLoop(keyGet keyGetter) { +func (mgr *Manager) hubSyncLoop(keyGet keyGetter, enabledSyscalls map[*prog.Syscall]bool) { hc := &HubConnector{ mgr: mgr, cfg: mgr.cfg, target: mgr.target, domain: mgr.cfg.TargetOS + "/" + mgr.cfg.HubDomain, - enabledCalls: mgr.targetEnabledSyscalls, + enabledCalls: enabledSyscalls, leak: mgr.enabledFeatures&flatrpc.FeatureLeak != 0, fresh: mgr.fresh, hubReproQueue: mgr.externalReproQueue, diff --git a/syz-manager/manager.go b/syz-manager/manager.go index 21df05001..ac451d720 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -31,7 +31,6 @@ import ( "github.com/google/syzkaller/pkg/fuzzer" "github.com/google/syzkaller/pkg/fuzzer/queue" "github.com/google/syzkaller/pkg/gce" - "github.com/google/syzkaller/pkg/hash" "github.com/google/syzkaller/pkg/log" "github.com/google/syzkaller/pkg/manager" "github.com/google/syzkaller/pkg/mgrconfig" @@ -79,8 +78,9 @@ type Manager struct { target *prog.Target sysTarget *targets.Target reporter *report.Reporter - crashdir string + crashStore *manager.CrashStore serv rpcserver.Server + http *manager.HTTPServer corpus *corpus.Corpus corpusDB *db.DB corpusDBMu sync.Mutex // for concurrent operations on corpusDB @@ -91,8 +91,6 @@ type Manager struct { checkDone atomic.Bool reportGenerator *manager.ReportGeneratorWrapper fresh bool - expertMode bool - modules []*vminfo.KernelModule coverFilter map[uint64]struct{} // includes only coverage PCs dash *dashapi.Dashboard @@ -100,11 +98,10 @@ type Manager struct { // cfg.DashboardOnlyRepro is set, so that we don't accidentially use dash for anything. dashRepro *dashapi.Dashboard - mu sync.Mutex - fuzzer atomic.Pointer[fuzzer.Fuzzer] - snapshotSource *queue.Distributor - phase int - targetEnabledSyscalls map[*prog.Syscall]bool + mu sync.Mutex + fuzzer atomic.Pointer[fuzzer.Fuzzer] + snapshotSource *queue.Distributor + phase int disabledHashes map[string]struct{} newRepros [][]byte @@ -204,8 +201,7 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { defer vmPool.Close() } - crashdir := filepath.Join(cfg.Workdir, "crashes") - osutil.MkdirAll(crashdir) + osutil.MkdirAll(cfg.Workdir) reporter, err := report.NewReporter(cfg) if err != nil { @@ -222,7 +218,7 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { target: cfg.Target, sysTarget: cfg.SysTarget, reporter: reporter, - crashdir: crashdir, + crashStore: manager.NewCrashStore(cfg), crashTypes: make(map[string]bool), disabledHashes: make(map[string]struct{}), memoryLeakFrames: make(map[string]bool), @@ -233,10 +229,15 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { saturatedCalls: make(map[string]bool), reportGenerator: manager.ReportGeneratorCache(cfg), } - if *flagDebug { mgr.cfg.Procs = 1 } + mgr.http = &manager.HTTPServer{ + Cfg: cfg, + StartTime: time.Now(), + Corpus: mgr.corpus, + CrashStore: mgr.crashStore, + } mgr.initStats() if mode == ModeFuzzing || mode == ModeCorpusTriage || mode == ModeCorpusRun { @@ -244,7 +245,7 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { } else { close(mgr.corpusPreload) } - mgr.initHTTP() // Creates HTTP server. + go mgr.http.Serve() go mgr.corpusInputHandler(corpusUpdates) go mgr.trackUsedFiles() @@ -296,7 +297,10 @@ func RunManager(mode Mode, cfg *mgrconfig.Config) { return } mgr.pool = vm.NewDispatcher(mgr.vmPool, mgr.fuzzerInstance) + mgr.http.Pool.Store(mgr.pool) mgr.reproLoop = manager.NewReproLoop(mgr, mgr.vmPool.Count()-mgr.cfg.FuzzingVMs, mgr.cfg.DashboardOnlyRepro) + mgr.http.ReproLoop.Store(mgr.reproLoop) + ctx := vm.ShutdownCtx() go mgr.processFuzzingResults(ctx) mgr.pool.Loop(ctx) @@ -480,8 +484,8 @@ func (mgr *Manager) preloadCorpus() { mgr.corpusPreload <- info.Candidates } -func (mgr *Manager) loadCorpus() []fuzzer.Candidate { - ret := manager.FilterCandidates(<-mgr.corpusPreload, mgr.targetEnabledSyscalls, true) +func (mgr *Manager) loadCorpus(enabledSyscalls map[*prog.Syscall]bool) []fuzzer.Candidate { + ret := manager.FilterCandidates(<-mgr.corpusPreload, enabledSyscalls, true) if mgr.cfg.PreserveCorpus { for _, hash := range ret.ModifiedHashes { // This program contains a disabled syscall. @@ -674,66 +678,25 @@ func (mgr *Manager) saveCrash(crash *manager.Crash) bool { return mgr.cfg.Reproduce && resp.NeedRepro } } - - sig := hash.Hash([]byte(crash.Title)) - id := sig.String() - dir := filepath.Join(mgr.crashdir, id) - osutil.MkdirAll(dir) - if err := osutil.WriteFile(filepath.Join(dir, "description"), []byte(crash.Title+"\n")); err != nil { - log.Logf(0, "failed to write crash: %v", err) + first, err := mgr.crashStore.SaveCrash(crash) + if err != nil { + log.Logf(0, "failed to save the cash: %v", err) + return false } - - // Save up to mgr.cfg.MaxCrashLogs reports, overwrite the oldest once we've reached that number. - // Newer reports are generally more useful. Overwriting is also needed - // to be able to understand if a particular bug still happens or already fixed. - oldestI := 0 - var oldestTime time.Time - for i := 0; i < mgr.cfg.MaxCrashLogs; i++ { - info, err := os.Stat(filepath.Join(dir, fmt.Sprintf("log%v", i))) - if err != nil { - oldestI = i - if i == 0 { - go mgr.emailCrash(crash) - } - break - } - if oldestTime.IsZero() || info.ModTime().Before(oldestTime) { - oldestI = i - oldestTime = info.ModTime() - } + if first { + go mgr.emailCrash(crash) } - writeOrRemove := func(name string, data []byte) { - filename := filepath.Join(dir, name+fmt.Sprint(oldestI)) - if len(data) == 0 { - os.Remove(filename) - return - } - osutil.WriteFile(filename, data) - } - writeOrRemove("log", crash.Output) - writeOrRemove("tag", []byte(mgr.cfg.Tag)) - writeOrRemove("report", crash.Report.Report) - writeOrRemove("machineInfo", crash.MachineInfo) return mgr.NeedRepro(crash) } -const maxReproAttempts = 3 - func (mgr *Manager) needLocalRepro(crash *manager.Crash) bool { if !mgr.cfg.Reproduce || crash.Corrupted || crash.Suppressed { return false } - sig := hash.Hash([]byte(crash.Title)) - dir := filepath.Join(mgr.crashdir, sig.String()) - if osutil.IsExist(filepath.Join(dir, "repro.prog")) { + if mgr.crashStore.HasRepro(crash.Title) { return false } - for i := 0; i < maxReproAttempts; i++ { - if !osutil.IsExist(filepath.Join(dir, fmt.Sprintf("repro%v", i))) { - return true - } - } - return false + return mgr.crashStore.MoreReproAttempts(crash.Title) } func (mgr *Manager) NeedRepro(crash *manager.Crash) bool { @@ -743,7 +706,10 @@ func (mgr *Manager) NeedRepro(crash *manager.Crash) bool { if crash.FromHub || crash.FromDashboard { return true } - if !mgr.checkDone.Load() || (mgr.enabledFeatures&flatrpc.FeatureLeak != 0 && + mgr.mu.Lock() + phase, features := mgr.phase, mgr.enabledFeatures + mgr.mu.Unlock() + if phase < phaseLoadedCorpus || (features&flatrpc.FeatureLeak != 0 && crash.Type != crash_pkg.MemoryLeak) { // Leak checking is very slow, don't bother reproducing other crashes on leak instance. return false @@ -797,14 +763,9 @@ func (mgr *Manager) saveFailedRepro(rep *report.Report, stats *repro.Stats) { return } } - dir := filepath.Join(mgr.crashdir, hash.String([]byte(rep.Title))) - osutil.MkdirAll(dir) - for i := 0; i < maxReproAttempts; i++ { - name := filepath.Join(dir, fmt.Sprintf("repro%v", i)) - if !osutil.IsExist(name) && len(reproLog) > 0 { - osutil.WriteFile(name, reproLog) - break - } + err := mgr.crashStore.SaveFailedRepro(rep.Title, reproLog) + if err != nil { + log.Logf(0, "failed to save repro log for %q: %v", rep.Title, err) } } @@ -880,45 +841,9 @@ func (mgr *Manager) saveRepro(res *manager.ReproResult) { return } } - - rep := repro.Report - dir := filepath.Join(mgr.crashdir, hash.String([]byte(rep.Title))) - osutil.MkdirAll(dir) - - if err := osutil.WriteFile(filepath.Join(dir, "description"), []byte(rep.Title+"\n")); err != nil { - log.Logf(0, "failed to write crash: %v", err) - } - osutil.WriteFile(filepath.Join(dir, "repro.prog"), append([]byte(opts), progText...)) - if mgr.cfg.Tag != "" { - osutil.WriteFile(filepath.Join(dir, "repro.tag"), []byte(mgr.cfg.Tag)) - } - if len(rep.Output) > 0 { - osutil.WriteFile(filepath.Join(dir, "repro.log"), rep.Output) - } - if len(rep.Report) > 0 { - osutil.WriteFile(filepath.Join(dir, "repro.report"), rep.Report) - } - if len(cprogText) > 0 { - osutil.WriteFile(filepath.Join(dir, "repro.cprog"), cprogText) - } - repro.Prog.ForEachAsset(func(name string, typ prog.AssetType, r io.Reader) { - fileName := filepath.Join(dir, name+".gz") - if err := osutil.WriteGzipStream(fileName, r); err != nil { - log.Logf(0, "failed to write crash asset: type %d, write error %v", typ, err) - } - }) - if res.Strace != nil { - // Unlike dashboard reporting, we save strace output separately from the original log. - if res.Strace.Error != nil { - osutil.WriteFile(filepath.Join(dir, "strace.error"), - []byte(fmt.Sprintf("%v", res.Strace.Error))) - } - if len(res.Strace.Output) > 0 { - osutil.WriteFile(filepath.Join(dir, "strace.log"), res.Strace.Output) - } - } - if reproLog := res.Stats.FullLog(); len(reproLog) > 0 { - osutil.WriteFile(filepath.Join(dir, "repro.stats"), reproLog) + err := mgr.crashStore.SaveRepro(res, append([]byte(opts), progText...), cprogText) + if err != nil { + log.Logf(0, "%s", err) } } @@ -1074,24 +999,6 @@ func setGuiltyFiles(crash *dashapi.Crash, report *report.Report) { } } -func (mgr *Manager) collectSyscallInfo() map[string]*corpus.CallCov { - mgr.mu.Lock() - enabledSyscalls := mgr.targetEnabledSyscalls - mgr.mu.Unlock() - - if enabledSyscalls == nil { - return nil - } - calls := mgr.corpus.CallCover() - // Add enabled, but not yet covered calls. - for call := range enabledSyscalls { - if calls[call.Name] == nil { - calls[call.Name] = new(corpus.CallCov) - } - } - return calls -} - func (mgr *Manager) BugFrames() (leaks, races []string) { mgr.mu.Lock() defer mgr.mu.Unlock() @@ -1121,12 +1028,12 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map panic("MachineChecked called twice") } mgr.enabledFeatures = features - mgr.targetEnabledSyscalls = enabledSyscalls + mgr.http.EnabledSyscalls.Store(enabledSyscalls) mgr.firstConnect.Store(time.Now().Unix()) statSyscalls := stat.New("syscalls", "Number of enabled syscalls", stat.Simple, stat.NoGraph, stat.Link("/syscalls")) statSyscalls.Add(len(enabledSyscalls)) - corpus := mgr.loadCorpus() + corpus := mgr.loadCorpus(enabledSyscalls) mgr.setPhaseLocked(phaseLoadedCorpus) opts := mgr.defaultExecOpts() @@ -1156,6 +1063,7 @@ func (mgr *Manager) MachineChecked(features flatrpc.Feature, enabledSyscalls map }, rnd, mgr.target) fuzzerObj.AddCandidates(corpus) mgr.fuzzer.Store(fuzzerObj) + mgr.http.Fuzzer.Store(fuzzerObj) go mgr.corpusMinimization() go mgr.fuzzerLoop(fuzzerObj) @@ -1301,7 +1209,8 @@ func (mgr *Manager) fuzzerLoop(fuzzer *fuzzer.Fuzzer) { } if mgr.cfg.HubClient != "" { mgr.setPhaseLocked(phaseTriagedCorpus) - go mgr.hubSyncLoop(pickGetter(mgr.cfg.HubKey)) + go mgr.hubSyncLoop(pickGetter(mgr.cfg.HubKey), + fuzzer.Config.EnabledCalls) } else { mgr.setPhaseLocked(phaseTriagedHub) } @@ -1452,8 +1361,12 @@ func (mgr *Manager) CoverageFilter(modules []*vminfo.KernelModule) []uint64 { if err != nil { log.Fatalf("failed to init coverage filter: %v", err) } - mgr.modules = modules mgr.coverFilter = filter + mgr.http.Cover.Store(&manager.CoverageInfo{ + Modules: modules, + ReportGenerator: mgr.reportGenerator, + CoverFilter: filter, + }) return execFilter } -- cgit mrf-deployment