From 75bb1b32609dc8e20e442a992f648e465c66cdf3 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 11 Nov 2024 14:36:02 +0100 Subject: tools/syz-reprolist: rename tool to syz-db-export --- dashboard/app/batch_main.go | 2 +- dashboard/app/batch_reproexport.go | 8 +-- dashboard/app/cron.yaml | 2 +- tools/syz-db-export/README.md | 25 +++++++ tools/syz-db-export/reprolist.go | 138 +++++++++++++++++++++++++++++++++++++ tools/syz-reprolist/README.md | 25 ------- tools/syz-reprolist/reprolist.go | 138 ------------------------------------- 7 files changed, 169 insertions(+), 169 deletions(-) create mode 100644 tools/syz-db-export/README.md create mode 100644 tools/syz-db-export/reprolist.go delete mode 100644 tools/syz-reprolist/README.md delete mode 100644 tools/syz-reprolist/reprolist.go diff --git a/dashboard/app/batch_main.go b/dashboard/app/batch_main.go index acf37ee8f..e1b514a26 100644 --- a/dashboard/app/batch_main.go +++ b/dashboard/app/batch_main.go @@ -17,7 +17,7 @@ import ( func initBatchProcessors() { http.HandleFunc("/cron/batch_coverage", handleBatchCoverage) - http.HandleFunc("/cron/batch_reproexport", handleBatchReproExport) + http.HandleFunc("/cron/batch_db_export", handleBatchDBExport) } // from https://cloud.google.com/batch/docs/samples/batch-create-script-job diff --git a/dashboard/app/batch_reproexport.go b/dashboard/app/batch_reproexport.go index 05e366544..b0361379f 100644 --- a/dashboard/app/batch_reproexport.go +++ b/dashboard/app/batch_reproexport.go @@ -13,7 +13,7 @@ import ( const exportTimeoutSeconds = 60 * 60 * 6 -func handleBatchReproExport(w http.ResponseWriter, r *http.Request) { +func handleBatchDBExport(w http.ResponseWriter, r *http.Request) { ctx := appengine.NewContext(r) for ns, nsConfig := range getConfig(ctx).Namespaces { if nsConfig.ReproExportPath == "" { @@ -23,19 +23,19 @@ func handleBatchReproExport(w http.ResponseWriter, r *http.Request) { Scopes: []string{"https://www.googleapis.com/auth/userinfo.email"}, } if err := createScriptJob(ctx, "syzkaller", "export-repro", - exportReproScript(ns, nsConfig.ReproExportPath), exportTimeoutSeconds, serviceAccount); err != nil { + exportDBScript(ns, nsConfig.ReproExportPath), exportTimeoutSeconds, serviceAccount); err != nil { log.Errorf(ctx, "createScriptJob: %s", err.Error()) } } } -func exportReproScript(srcNamespace, archivePath string) string { +func exportDBScript(srcNamespace, archivePath string) string { return "\n" + "git clone -q --depth 1 --branch master --single-branch https://github.com/google/syzkaller\n" + "cd syzkaller\n" + "token=$(gcloud auth print-access-token)\n" + "CI=1 ./tools/syz-env \"" + // CI=1 to suppress "The input device is not a TTY". - "go run ./tools/syz-reprolist/... -namespace " + srcNamespace + " -output export -token $token -j 10 && " + + "go run ./tools/syz-db-export/... -namespace " + srcNamespace + " -output export -token $token -j 10 && " + "tar -czf export.tar.gz ./export/ && " + "gsutil -q -m cp export.tar.gz " + archivePath + "\"" diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml index 1bb7717bc..a0acdf7d2 100644 --- a/dashboard/app/cron.yaml +++ b/dashboard/app/cron.yaml @@ -26,5 +26,5 @@ cron: - url: /cron/batch_coverage?days=true&months=true&steps=10 schedule: every 24 hours # Export reproducers every week. -- url: /cron/batch_reproexport +- url: /cron/batch_db_export schedule: every saturday 00:00 diff --git a/tools/syz-db-export/README.md b/tools/syz-db-export/README.md new file mode 100644 index 000000000..47b3248c9 --- /dev/null +++ b/tools/syz-db-export/README.md @@ -0,0 +1,25 @@ +# Syzbot DB export +Every week syzbot runs syz-db-export to export its databases: +1. [Upstream Linux](https://syzkaller.appspot.com/upstream) +db is [here](https://storage.googleapis.com/artifacts.syzkaller.appspot.com/shared-files/repro-export/upstream.tar.gz). +2. Contact us if you want see others. + +## Export structure +DB currently includes: +1. Bugs descriptions. +2. First C-Reproducer for every bug. + +It doesn't include: +1. Second+ C-Reproducers for every bug. +2. Syz-Reproducers. +3. Any reproducer related metadata (like triggering requirements). + +## How to export more data + +The best way to see more data exported is to modify the tool and send us PR with your changes. + +To reproduce locally what syzbot is doing for upstream Linux: +```golang +go run ./tools/syz-db-export/... -namespace upstream +``` +Extending tools/syz-db-export you can teach syzbot to export more. diff --git a/tools/syz-db-export/reprolist.go b/tools/syz-db-export/reprolist.go new file mode 100644 index 000000000..24a548423 --- /dev/null +++ b/tools/syz-db-export/reprolist.go @@ -0,0 +1,138 @@ +// Copyright 2019 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path" + "strings" + + "github.com/google/syzkaller/dashboard/api" + "golang.org/x/sync/errgroup" +) + +var ( + flagDashboard = flag.String("dashboard", "https://syzkaller.appspot.com", "dashboard address") + flagOutputDir = flag.String("output", "export", "output dir") + flagNamespace = flag.String("namespace", "upstream", "target namespace") + flagToken = flag.String("token", "", "gcp bearer token to disable throttling (contact syzbot first)\n"+ + "usage example: ./tools/syz-db-export -namespace upstream -token $(gcloud auth print-access-token)") + flagParallel = flag.Int("j", 2, "number of parallel threads") + flagVerbose = flag.Bool("v", false, "verbose output") +) + +func main() { + flag.Parse() + if err := os.MkdirAll(*flagOutputDir, 0755); err != nil { + log.Fatalf("alert: failed to create output dir: %v", err) + } + if *flagNamespace == "" { + log.Fatal("alert: namespace can't be empty") + } + if err := exportNamespace(); err != nil { + log.Fatalf("alert: error: %s", err.Error()) + } +} + +func exportNamespace() error { + cli := api.NewClient(*flagDashboard, *flagToken) + bugs, err := cli.BugGroups(*flagNamespace, api.BugGroupOpen|api.BugGroupFixed) + if err != nil { + return err + } + fmt.Printf("total %d bugs available\n", len(bugs)) + + iBugChan := make(chan int) + g, _ := errgroup.WithContext(context.Background()) + for i := 0; i < *flagParallel; i++ { + i := i + g.Go(func() error { + for iBug := range iBugChan { + bug, err := cli.Bug(bugs[iBug].Link) + if err != nil { + return err + } + if *flagVerbose { + fmt.Printf("[%v](%v/%v)saving bug %v\n", + i, iBug, len(bugs), bug.ID) + } + if err := saveBug(bug); err != nil { + return fmt.Errorf("saveBug(bugID=%s): %w", bug.ID, err) + } + cReproURL := bug.Crashes[0].CReproducerLink // export max 1 CRepro per bug + if cReproURL == "" { + continue + } + reproID := reproIDFromURL(cReproURL) + if *flagVerbose { + fmt.Printf("[%v](%v/%v)saving c-repro %v for bug %v\n", + i, iBug, len(bugs), reproID, bug.ID) + } + cReproBody, err := cli.Text(cReproURL) + if err != nil { + return err + } + if err := saveCRepro(bug.ID, reproID, cReproBody); err != nil { + return fmt.Errorf("saveRepro(bugID=%s, reproID=%s): %w", bug.ID, reproID, err) + } + } + return nil + }) + } + errChan := make(chan error) + go func() { + errChan <- g.Wait() + }() + for iBug := range bugs { + select { + case iBugChan <- iBug: + case err := <-errChan: + return err + } + } + close(iBugChan) + return g.Wait() +} + +// saceCRepro assumes the bug dir already exists. +func saveCRepro(bugID, reproID string, reproData []byte) error { + reproPath := path.Join(*flagOutputDir, "bugs", bugID, reproID+".c") + if err := os.WriteFile(reproPath, reproData, 0666); err != nil { + return fmt.Errorf("os.WriteFile: %w", err) + } + return nil +} + +func reproIDFromURL(url string) string { + parts := strings.Split(url, "&") + if len(parts) != 2 { + log.Panicf("can't split %s in two parts by ?", url) + } + parts = strings.Split(parts[1], "=") + if len(parts) != 2 { + log.Panicf("can't split %s in two parts by =", url) + } + return parts[1] +} + +func saveBug(bug *api.Bug) error { + jsonBytes, err := json.Marshal(bug) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + bugDir := path.Join(*flagOutputDir, "bugs", bug.ID) + if err := os.MkdirAll(bugDir, 0755); err != nil { + return fmt.Errorf("os.MkdirAll(%s): %w", bugDir, err) + } + bugDetailsPath := path.Join(bugDir, "details.json") + if err := os.WriteFile(bugDetailsPath, jsonBytes, 0666); err != nil { + return fmt.Errorf("os.WriteFile: %w", err) + } + return nil +} diff --git a/tools/syz-reprolist/README.md b/tools/syz-reprolist/README.md deleted file mode 100644 index ecaa09d2c..000000000 --- a/tools/syz-reprolist/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Syzbot DB export -Every week syzbot runs syz-reprolist to export its databases: -1. [Upstream Linux](https://syzkaller.appspot.com/upstream) -db is [here](https://storage.googleapis.com/artifacts.syzkaller.appspot.com/shared-files/repro-export/upstream.tar.gz). -2. Contact us if you want see others. - -## Export structure -DB currently includes: -1. First C-Reproducer for every bug. - -It doesn't include: -1. Second+ C-Reproducers for every bug. -2. Syz-Reproducers. -3. Bugs description itself. -4. Any reproducer related metadata (like triggering requirements). - -## How to export more data - -The best way to see more data exported is to modify the tool and send us PR with your changes. - -To reproduce locally what syzbot is doing for upstream Linux: -```golang -go run ./tools/syz-reprolist/... -namespace upstream -``` -Extending tools/syz-reprolist you can teach syzbot to export more. diff --git a/tools/syz-reprolist/reprolist.go b/tools/syz-reprolist/reprolist.go deleted file mode 100644 index ca90536ba..000000000 --- a/tools/syz-reprolist/reprolist.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "log" - "os" - "path" - "strings" - - "github.com/google/syzkaller/dashboard/api" - "golang.org/x/sync/errgroup" -) - -var ( - flagDashboard = flag.String("dashboard", "https://syzkaller.appspot.com", "dashboard address") - flagOutputDir = flag.String("output", "export", "output dir") - flagNamespace = flag.String("namespace", "upstream", "target namespace") - flagToken = flag.String("token", "", "gcp bearer token to disable throttling (contact syzbot first)\n"+ - "usage example: ./tools/syz-bot-export -namespace upstream -token $(gcloud auth print-access-token)") - flagParallel = flag.Int("j", 2, "number of parallel threads") - flagVerbose = flag.Bool("v", false, "verbose output") -) - -func main() { - flag.Parse() - if err := os.MkdirAll(*flagOutputDir, 0755); err != nil { - log.Fatalf("alert: failed to create output dir: %v", err) - } - if *flagNamespace == "" { - log.Fatal("alert: namespace can't be empty") - } - if err := exportNamespace(); err != nil { - log.Fatalf("alert: error: %s", err.Error()) - } -} - -func exportNamespace() error { - cli := api.NewClient(*flagDashboard, *flagToken) - bugs, err := cli.BugGroups(*flagNamespace, api.BugGroupOpen|api.BugGroupFixed) - if err != nil { - return err - } - fmt.Printf("total %d bugs available\n", len(bugs)) - - iBugChan := make(chan int) - g, _ := errgroup.WithContext(context.Background()) - for i := 0; i < *flagParallel; i++ { - i := i - g.Go(func() error { - for iBug := range iBugChan { - bug, err := cli.Bug(bugs[iBug].Link) - if err != nil { - return err - } - if *flagVerbose { - fmt.Printf("[%v](%v/%v)saving bug %v\n", - i, iBug, len(bugs), bug.ID) - } - if err := saveBug(bug); err != nil { - return fmt.Errorf("saveBug(bugID=%s): %w", bug.ID, err) - } - cReproURL := bug.Crashes[0].CReproducerLink // export max 1 CRepro per bug - if cReproURL == "" { - continue - } - reproID := reproIDFromURL(cReproURL) - if *flagVerbose { - fmt.Printf("[%v](%v/%v)saving c-repro %v for bug %v\n", - i, iBug, len(bugs), reproID, bug.ID) - } - cReproBody, err := cli.Text(cReproURL) - if err != nil { - return err - } - if err := saveCRepro(bug.ID, reproID, cReproBody); err != nil { - return fmt.Errorf("saveRepro(bugID=%s, reproID=%s): %w", bug.ID, reproID, err) - } - } - return nil - }) - } - errChan := make(chan error) - go func() { - errChan <- g.Wait() - }() - for iBug := range bugs { - select { - case iBugChan <- iBug: - case err := <-errChan: - return err - } - } - close(iBugChan) - return g.Wait() -} - -// saceCRepro assumes the bug dir already exists. -func saveCRepro(bugID, reproID string, reproData []byte) error { - reproPath := path.Join(*flagOutputDir, "bugs", bugID, reproID+".c") - if err := os.WriteFile(reproPath, reproData, 0666); err != nil { - return fmt.Errorf("os.WriteFile: %w", err) - } - return nil -} - -func reproIDFromURL(url string) string { - parts := strings.Split(url, "&") - if len(parts) != 2 { - log.Panicf("can't split %s in two parts by ?", url) - } - parts = strings.Split(parts[1], "=") - if len(parts) != 2 { - log.Panicf("can't split %s in two parts by =", url) - } - return parts[1] -} - -func saveBug(bug *api.Bug) error { - jsonBytes, err := json.Marshal(bug) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - bugDir := path.Join(*flagOutputDir, "bugs", bug.ID) - if err := os.MkdirAll(bugDir, 0755); err != nil { - return fmt.Errorf("os.MkdirAll(%s): %w", bugDir, err) - } - bugDetailsPath := path.Join(bugDir, "details.json") - if err := os.WriteFile(bugDetailsPath, jsonBytes, 0666); err != nil { - return fmt.Errorf("os.WriteFile: %w", err) - } - return nil -} -- cgit mrf-deployment