From 5e7b4bcaa61e8bb9b1d1fbca21684fe490f69133 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 7 Oct 2024 14:09:32 +0200 Subject: dashboard/app: introduce batch_reproexport This PR exports the latest reproducer for every bug. Reproducers are exported to the "bug_id/repro_id.c" files. This approach allows to add some metadata files or export more reproducers/bug later. All the files are then archived and uploaded to the preconfigured location. --- tools/syz-reprolist/external_api.go | 60 ++++++++++++++++ tools/syz-reprolist/reprolist.go | 135 +++++++++++++++++++++++++++++++++++- 2 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 tools/syz-reprolist/external_api.go (limited to 'tools') diff --git a/tools/syz-reprolist/external_api.go b/tools/syz-reprolist/external_api.go new file mode 100644 index 000000000..36fe4ef68 --- /dev/null +++ b/tools/syz-reprolist/external_api.go @@ -0,0 +1,60 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "fmt" + "log" + "strings" +) + +type bugList struct { + Version int + Bugs []struct { + Link string + } +} + +func reproIDFromURL(url string) string { + parts := strings.Split(url, "&") + if len(parts) != 2 { + log.Panicf("can't split %s in two parts by ?", url) + } + parts = strings.Split(parts[1], "=") + if len(parts) != 2 { + log.Panicf("can't split %s in two parts by =", url) + } + return parts[1] +} + +func getBugList(jsonBugs []byte) ([]string, error) { + var bl bugList + if err := json.Unmarshal(jsonBugs, &bl); err != nil { + return nil, fmt.Errorf("json.Unmarshal: %w", err) + } + if bl.Version != 1 { + return nil, fmt.Errorf("unsupported export version %d", bl.Version) + } + res := []string{} + for _, b := range bl.Bugs { + res = append(res, b.Link) + } + return res, nil +} + +type BugDetails struct { + ID string + Crashes []struct { + CReproURL string `json:"c-reproducer"` + } +} + +func makeBugDetails(jsonDetails []byte) (*BugDetails, error) { + var bd BugDetails + if err := json.Unmarshal(jsonDetails, &bd); err != nil { + return nil, fmt.Errorf("json.Unmarshal: %w", err) + } + return &bd, nil +} diff --git a/tools/syz-reprolist/reprolist.go b/tools/syz-reprolist/reprolist.go index 3b0aa94d0..6970b2da3 100644 --- a/tools/syz-reprolist/reprolist.go +++ b/tools/syz-reprolist/reprolist.go @@ -4,10 +4,15 @@ package main import ( + "context" "flag" "fmt" + "html" + "io" "log" + "net/http" "os" + "path" "path/filepath" "runtime" "strings" @@ -19,6 +24,7 @@ import ( "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/vcs" "github.com/google/syzkaller/sys/targets" + "golang.org/x/sync/errgroup" ) var ( @@ -28,17 +34,27 @@ var ( flagOutputDir = flag.String("output", "repros", "output dir") flagSyzkallerDir = flag.String("syzkaller", ".", "syzkaller dir") flagOS = flag.String("os", runtime.GOOS, "target OS") + flagNamespace = flag.String("namespace", "", "target namespace") + flagToken = flag.String("token", "", "gcp bearer token to disable throttling (contact syzbot first)\n"+ + "usage example: ./tools/syz-reprolist -namespace upstream -token $(gcloud auth pring-access-token)") + flagParallel = flag.Int("j", 2, "number of parallel threads") ) func main() { flag.Parse() + if err := os.MkdirAll(*flagOutputDir, 0755); err != nil { + log.Fatalf("alert: failed to create output dir: %v", err) + } + if *flagNamespace != "" { + if err := exportNamespace(); err != nil { + log.Fatalf("alert: error: %s", err.Error()) + } + return + } clients := strings.Split(*flagAPIClients, ",") if len(clients) == 0 { log.Fatalf("api client is required") } - if err := os.MkdirAll(*flagOutputDir, 0755); err != nil { - log.Fatalf("failed to create output dir: %v", err) - } for _, client := range clients { log.Printf("processing client %v", client) dash, err := dashapi.New(client, *flagDashboard, *flagAPIKey) @@ -272,3 +288,116 @@ func containsCommit(hash string) bool { _, err := osutil.RunCmd(time.Hour, *flagSyzkallerDir, "git", "merge-base", "--is-ancestor", hash, "HEAD") return err == nil } + +func exportNamespace() error { + bugURLs, err := getFullBugList() + if err != nil { + return err + } + fmt.Printf("total %d bugs available\n", len(bugURLs)) + + iBugChan := make(chan int) + g, _ := errgroup.WithContext(context.Background()) + for i := 0; i < *flagParallel; i++ { + g.Go(func() error { + for iBug := range iBugChan { + bugURL := *flagDashboard + bugURLs[iBug] + bugBody, err := getJSONBody(bugURL) + if err != nil { + return fmt.Errorf("getJSONBody(%s): %w", bugURL, err) + } + bugDetails, err := makeBugDetails(bugBody) + if err != nil { + return fmt.Errorf("makeBugDetails: %w", err) + } + if cReproURL := bugDetails.Crashes[0].CReproURL; cReproURL != "" { // export max 1 CRepro per bug + reproID := reproIDFromURL(cReproURL) + fmt.Printf("[%d](%d/%d)saving c-repro %s for bug %s\n", + i, iBug, len(bugURLs), reproID, bugDetails.ID) + fullReproURL := *flagDashboard + html.UnescapeString(cReproURL) + cReproBody, err := getJSONBody(fullReproURL) + if err != nil { + return fmt.Errorf("getJSONBody(%s): %w", fullReproURL, err) + } + if err := saveCRepro(reproID, cReproBody); err != nil { + return fmt.Errorf("saveRepro(bugID=%s, reproID=%s): %w", bugDetails.ID, reproID, err) + } + } + } + return nil + }) + } + errChan := make(chan error) + go func() { + errChan <- g.Wait() + }() + for iBug := range bugURLs { + select { + case iBugChan <- iBug: + case err := <-errChan: + return err + } + } + close(iBugChan) + return g.Wait() +} + +func getFullBugList() ([]string, error) { + bugLists := []string{ + *flagDashboard + "/" + *flagNamespace, + *flagDashboard + "/" + *flagNamespace + "/fixed", + } + fullBugList := []string{} + for _, url := range bugLists { + fmt.Printf("loading bug list from %s\n", url) + body, err := getJSONBody(url) + if err != nil { + return nil, fmt.Errorf("getBody(%s): %w", url, err) + } + bugs, err := getBugList(body) + if err != nil { + return nil, fmt.Errorf("bugList: %w", err) + } + fullBugList = append(fullBugList, bugs...) + } + return fullBugList, nil +} + +func getJSONBody(url string) ([]byte, error) { + if strings.Contains(url, "?") { + url = url + "&json=1" + } else { + url = url + "?json=1" + } + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("http.NewRequest: %w", err) + } + if *flagToken != "" { + req.Header.Add("Authorization", "Bearer "+*flagToken) + } else { + time.Sleep(time.Second) // tolerate throttling + } + client := &http.Client{} + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("http.Get(%s): %w", url, err) + } + defer res.Body.Close() + body, err := io.ReadAll(res.Body) + if res.StatusCode > 299 { + return nil, fmt.Errorf("io.ReadAll failed with status code: %d and\nbody: %s", res.StatusCode, body) + } + if err != nil { + return nil, fmt.Errorf("io.ReadAll(body): %w", err) + } + return body, nil +} + +func saveCRepro(reproID string, reproData []byte) error { + reproPath := path.Join(*flagOutputDir, reproID+".c") + if err := os.WriteFile(reproPath, reproData, 0666); err != nil { + return fmt.Errorf("os.WriteFile: %w", err) + } + return nil +} -- cgit mrf-deployment