From 75bb1b32609dc8e20e442a992f648e465c66cdf3 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 11 Nov 2024 14:36:02 +0100 Subject: tools/syz-reprolist: rename tool to syz-db-export --- tools/syz-db-export/README.md | 25 +++++++ tools/syz-db-export/reprolist.go | 138 +++++++++++++++++++++++++++++++++++++++ tools/syz-reprolist/README.md | 25 ------- tools/syz-reprolist/reprolist.go | 138 --------------------------------------- 4 files changed, 163 insertions(+), 163 deletions(-) create mode 100644 tools/syz-db-export/README.md create mode 100644 tools/syz-db-export/reprolist.go delete mode 100644 tools/syz-reprolist/README.md delete mode 100644 tools/syz-reprolist/reprolist.go (limited to 'tools') diff --git a/tools/syz-db-export/README.md b/tools/syz-db-export/README.md new file mode 100644 index 000000000..47b3248c9 --- /dev/null +++ b/tools/syz-db-export/README.md @@ -0,0 +1,25 @@ +# Syzbot DB export +Every week syzbot runs syz-db-export to export its databases: +1. [Upstream Linux](https://syzkaller.appspot.com/upstream) +db is [here](https://storage.googleapis.com/artifacts.syzkaller.appspot.com/shared-files/repro-export/upstream.tar.gz). +2. Contact us if you want see others. + +## Export structure +DB currently includes: +1. Bugs descriptions. +2. First C-Reproducer for every bug. + +It doesn't include: +1. Second+ C-Reproducers for every bug. +2. Syz-Reproducers. +3. Any reproducer related metadata (like triggering requirements). + +## How to export more data + +The best way to see more data exported is to modify the tool and send us PR with your changes. + +To reproduce locally what syzbot is doing for upstream Linux: +```golang +go run ./tools/syz-db-export/... -namespace upstream +``` +Extending tools/syz-db-export you can teach syzbot to export more. diff --git a/tools/syz-db-export/reprolist.go b/tools/syz-db-export/reprolist.go new file mode 100644 index 000000000..24a548423 --- /dev/null +++ b/tools/syz-db-export/reprolist.go @@ -0,0 +1,138 @@ +// Copyright 2019 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path" + "strings" + + "github.com/google/syzkaller/dashboard/api" + "golang.org/x/sync/errgroup" +) + +var ( + flagDashboard = flag.String("dashboard", "https://syzkaller.appspot.com", "dashboard address") + flagOutputDir = flag.String("output", "export", "output dir") + flagNamespace = flag.String("namespace", "upstream", "target namespace") + flagToken = flag.String("token", "", "gcp bearer token to disable throttling (contact syzbot first)\n"+ + "usage example: ./tools/syz-db-export -namespace upstream -token $(gcloud auth print-access-token)") + flagParallel = flag.Int("j", 2, "number of parallel threads") + flagVerbose = flag.Bool("v", false, "verbose output") +) + +func main() { + flag.Parse() + if err := os.MkdirAll(*flagOutputDir, 0755); err != nil { + log.Fatalf("alert: failed to create output dir: %v", err) + } + if *flagNamespace == "" { + log.Fatal("alert: namespace can't be empty") + } + if err := exportNamespace(); err != nil { + log.Fatalf("alert: error: %s", err.Error()) + } +} + +func exportNamespace() error { + cli := api.NewClient(*flagDashboard, *flagToken) + bugs, err := cli.BugGroups(*flagNamespace, api.BugGroupOpen|api.BugGroupFixed) + if err != nil { + return err + } + fmt.Printf("total %d bugs available\n", len(bugs)) + + iBugChan := make(chan int) + g, _ := errgroup.WithContext(context.Background()) + for i := 0; i < *flagParallel; i++ { + i := i + g.Go(func() error { + for iBug := range iBugChan { + bug, err := cli.Bug(bugs[iBug].Link) + if err != nil { + return err + } + if *flagVerbose { + fmt.Printf("[%v](%v/%v)saving bug %v\n", + i, iBug, len(bugs), bug.ID) + } + if err := saveBug(bug); err != nil { + return fmt.Errorf("saveBug(bugID=%s): %w", bug.ID, err) + } + cReproURL := bug.Crashes[0].CReproducerLink // export max 1 CRepro per bug + if cReproURL == "" { + continue + } + reproID := reproIDFromURL(cReproURL) + if *flagVerbose { + fmt.Printf("[%v](%v/%v)saving c-repro %v for bug %v\n", + i, iBug, len(bugs), reproID, bug.ID) + } + cReproBody, err := cli.Text(cReproURL) + if err != nil { + return err + } + if err := saveCRepro(bug.ID, reproID, cReproBody); err != nil { + return fmt.Errorf("saveRepro(bugID=%s, reproID=%s): %w", bug.ID, reproID, err) + } + } + return nil + }) + } + errChan := make(chan error) + go func() { + errChan <- g.Wait() + }() + for iBug := range bugs { + select { + case iBugChan <- iBug: + case err := <-errChan: + return err + } + } + close(iBugChan) + return g.Wait() +} + +// saceCRepro assumes the bug dir already exists. +func saveCRepro(bugID, reproID string, reproData []byte) error { + reproPath := path.Join(*flagOutputDir, "bugs", bugID, reproID+".c") + if err := os.WriteFile(reproPath, reproData, 0666); err != nil { + return fmt.Errorf("os.WriteFile: %w", err) + } + return nil +} + +func reproIDFromURL(url string) string { + parts := strings.Split(url, "&") + if len(parts) != 2 { + log.Panicf("can't split %s in two parts by ?", url) + } + parts = strings.Split(parts[1], "=") + if len(parts) != 2 { + log.Panicf("can't split %s in two parts by =", url) + } + return parts[1] +} + +func saveBug(bug *api.Bug) error { + jsonBytes, err := json.Marshal(bug) + if err != nil { + return fmt.Errorf("json.Marshal: %w", err) + } + bugDir := path.Join(*flagOutputDir, "bugs", bug.ID) + if err := os.MkdirAll(bugDir, 0755); err != nil { + return fmt.Errorf("os.MkdirAll(%s): %w", bugDir, err) + } + bugDetailsPath := path.Join(bugDir, "details.json") + if err := os.WriteFile(bugDetailsPath, jsonBytes, 0666); err != nil { + return fmt.Errorf("os.WriteFile: %w", err) + } + return nil +} diff --git a/tools/syz-reprolist/README.md b/tools/syz-reprolist/README.md deleted file mode 100644 index ecaa09d2c..000000000 --- a/tools/syz-reprolist/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Syzbot DB export -Every week syzbot runs syz-reprolist to export its databases: -1. [Upstream Linux](https://syzkaller.appspot.com/upstream) -db is [here](https://storage.googleapis.com/artifacts.syzkaller.appspot.com/shared-files/repro-export/upstream.tar.gz). -2. Contact us if you want see others. - -## Export structure -DB currently includes: -1. First C-Reproducer for every bug. - -It doesn't include: -1. Second+ C-Reproducers for every bug. -2. Syz-Reproducers. -3. Bugs description itself. -4. Any reproducer related metadata (like triggering requirements). - -## How to export more data - -The best way to see more data exported is to modify the tool and send us PR with your changes. - -To reproduce locally what syzbot is doing for upstream Linux: -```golang -go run ./tools/syz-reprolist/... -namespace upstream -``` -Extending tools/syz-reprolist you can teach syzbot to export more. diff --git a/tools/syz-reprolist/reprolist.go b/tools/syz-reprolist/reprolist.go deleted file mode 100644 index ca90536ba..000000000 --- a/tools/syz-reprolist/reprolist.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2019 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package main - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "log" - "os" - "path" - "strings" - - "github.com/google/syzkaller/dashboard/api" - "golang.org/x/sync/errgroup" -) - -var ( - flagDashboard = flag.String("dashboard", "https://syzkaller.appspot.com", "dashboard address") - flagOutputDir = flag.String("output", "export", "output dir") - flagNamespace = flag.String("namespace", "upstream", "target namespace") - flagToken = flag.String("token", "", "gcp bearer token to disable throttling (contact syzbot first)\n"+ - "usage example: ./tools/syz-bot-export -namespace upstream -token $(gcloud auth print-access-token)") - flagParallel = flag.Int("j", 2, "number of parallel threads") - flagVerbose = flag.Bool("v", false, "verbose output") -) - -func main() { - flag.Parse() - if err := os.MkdirAll(*flagOutputDir, 0755); err != nil { - log.Fatalf("alert: failed to create output dir: %v", err) - } - if *flagNamespace == "" { - log.Fatal("alert: namespace can't be empty") - } - if err := exportNamespace(); err != nil { - log.Fatalf("alert: error: %s", err.Error()) - } -} - -func exportNamespace() error { - cli := api.NewClient(*flagDashboard, *flagToken) - bugs, err := cli.BugGroups(*flagNamespace, api.BugGroupOpen|api.BugGroupFixed) - if err != nil { - return err - } - fmt.Printf("total %d bugs available\n", len(bugs)) - - iBugChan := make(chan int) - g, _ := errgroup.WithContext(context.Background()) - for i := 0; i < *flagParallel; i++ { - i := i - g.Go(func() error { - for iBug := range iBugChan { - bug, err := cli.Bug(bugs[iBug].Link) - if err != nil { - return err - } - if *flagVerbose { - fmt.Printf("[%v](%v/%v)saving bug %v\n", - i, iBug, len(bugs), bug.ID) - } - if err := saveBug(bug); err != nil { - return fmt.Errorf("saveBug(bugID=%s): %w", bug.ID, err) - } - cReproURL := bug.Crashes[0].CReproducerLink // export max 1 CRepro per bug - if cReproURL == "" { - continue - } - reproID := reproIDFromURL(cReproURL) - if *flagVerbose { - fmt.Printf("[%v](%v/%v)saving c-repro %v for bug %v\n", - i, iBug, len(bugs), reproID, bug.ID) - } - cReproBody, err := cli.Text(cReproURL) - if err != nil { - return err - } - if err := saveCRepro(bug.ID, reproID, cReproBody); err != nil { - return fmt.Errorf("saveRepro(bugID=%s, reproID=%s): %w", bug.ID, reproID, err) - } - } - return nil - }) - } - errChan := make(chan error) - go func() { - errChan <- g.Wait() - }() - for iBug := range bugs { - select { - case iBugChan <- iBug: - case err := <-errChan: - return err - } - } - close(iBugChan) - return g.Wait() -} - -// saceCRepro assumes the bug dir already exists. -func saveCRepro(bugID, reproID string, reproData []byte) error { - reproPath := path.Join(*flagOutputDir, "bugs", bugID, reproID+".c") - if err := os.WriteFile(reproPath, reproData, 0666); err != nil { - return fmt.Errorf("os.WriteFile: %w", err) - } - return nil -} - -func reproIDFromURL(url string) string { - parts := strings.Split(url, "&") - if len(parts) != 2 { - log.Panicf("can't split %s in two parts by ?", url) - } - parts = strings.Split(parts[1], "=") - if len(parts) != 2 { - log.Panicf("can't split %s in two parts by =", url) - } - return parts[1] -} - -func saveBug(bug *api.Bug) error { - jsonBytes, err := json.Marshal(bug) - if err != nil { - return fmt.Errorf("json.Marshal: %w", err) - } - bugDir := path.Join(*flagOutputDir, "bugs", bug.ID) - if err := os.MkdirAll(bugDir, 0755); err != nil { - return fmt.Errorf("os.MkdirAll(%s): %w", bugDir, err) - } - bugDetailsPath := path.Join(bugDir, "details.json") - if err := os.WriteFile(bugDetailsPath, jsonBytes, 0666); err != nil { - return fmt.Errorf("os.WriteFile: %w", err) - } - return nil -} -- cgit mrf-deployment