From b54aa474a6b13cc9b8c0a68f07d71873f30dfa02 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 9 Dec 2021 11:11:40 +0100 Subject: syz-ci: support uploading corpus.db Just as we upload coverage reports, it's useful to snapshot corpus.db for sharing/regression testing. --- syz-ci/manager.go | 73 +++++++++++++++++++++++++++++++++++-------------------- syz-ci/syz-ci.go | 13 ++++++---- 2 files changed, 55 insertions(+), 31 deletions(-) diff --git a/syz-ci/manager.go b/syz-ci/manager.go index 1b86db9a5..14e443c59 100644 --- a/syz-ci/manager.go +++ b/syz-ci/manager.go @@ -141,7 +141,7 @@ var kernelBuildSem = make(chan struct{}, 1) func (mgr *Manager) loop() { lastCommit := "" nextBuildTime := time.Now() - var managerRestartTime, coverUploadTime time.Time + var managerRestartTime, artifactUploadTime time.Time latestInfo := mgr.checkLatest() if latestInfo != nil && time.Since(latestInfo.Time) < kernelRebuildPeriod/2 && mgr.managercfg.TargetOS != targets.Fuchsia { @@ -167,10 +167,17 @@ loop: lastCommit, latestInfo, rebuildAfter = mgr.pollAndBuild(lastCommit, latestInfo) nextBuildTime = time.Now().Add(rebuildAfter) } - if !coverUploadTime.IsZero() && time.Now().After(coverUploadTime) { - coverUploadTime = time.Time{} - if err := mgr.uploadCoverReport(); err != nil { - mgr.Errorf("failed to upload cover report: %v", err) + if !artifactUploadTime.IsZero() && time.Now().After(artifactUploadTime) { + artifactUploadTime = time.Time{} + if mgr.managercfg.Cover && mgr.cfg.CoverUploadPath != "" { + if err := mgr.uploadCoverReport(); err != nil { + mgr.Errorf("failed to upload cover report: %v", err) + } + } + if mgr.cfg.CorpusUploadPath != "" { + if err := mgr.uploadCorpus(); err != nil { + mgr.Errorf("failed to upload corpus: %v", err) + } } } @@ -183,8 +190,8 @@ loop: if latestInfo != nil && (latestInfo.Time != managerRestartTime || mgr.cmd == nil) { managerRestartTime = latestInfo.Time mgr.restartManager() - if mgr.cmd != nil && mgr.managercfg.Cover && mgr.cfg.CoverUploadPath != "" { - coverUploadTime = time.Now().Add(6 * time.Hour) + if mgr.cmd != nil { + artifactUploadTime = time.Now().Add(6 * time.Hour) } } @@ -640,45 +647,59 @@ func (mgr *Manager) uploadCoverReport() error { } defer resp.Body.Close() // Upload coverage report. - coverUploadURL, err := url.Parse(mgr.cfg.CoverUploadPath) + return uploadFile(mgr.cfg.CoverUploadPath, mgr.name+".html", resp.Body) +} + +func (mgr *Manager) uploadCorpus() error { + f, err := os.Open(filepath.Join(mgr.workDir, "corpus.db")) + if err != nil { + return err + } + defer f.Close() + return uploadFile(mgr.cfg.CorpusUploadPath, mgr.name+"-corpus.db", f) +} + +func uploadFile(dstPath, name string, file io.Reader) error { + URL, err := url.Parse(dstPath) if err != nil { - return fmt.Errorf("failed to parse cover upload path: %v", err) - } - coverUploadURL.Path = path.Join(coverUploadURL.Path, mgr.name+".html") - coverUploadURLStr := coverUploadURL.String() - log.Logf(0, "%v: uploading cover report to %v", mgr.name, coverUploadURLStr) - if strings.HasPrefix(coverUploadURLStr, "gs://") { - return uploadCoverReportGCS(strings.TrimPrefix(coverUploadURLStr, "gs://"), resp.Body) - } else if strings.HasPrefix(coverUploadURLStr, "http://") || - strings.HasPrefix(coverUploadURLStr, "https://") { - return uploadCoverReportHTTPPut(coverUploadURLStr, resp.Body) - } else { // Use GCS as default to maintain backwards compatibility. - return uploadCoverReportGCS(coverUploadURLStr, resp.Body) + return fmt.Errorf("failed to parse upload path: %v", err) + } + URL.Path = path.Join(URL.Path, name) + URLStr := URL.String() + log.Logf(0, "uploading %v to %v", name, URLStr) + if strings.HasPrefix(URLStr, "gs://") { + return uploadFileGCS(strings.TrimPrefix(URLStr, "gs://"), file) + } + if strings.HasPrefix(URLStr, "http://") || + strings.HasPrefix(URLStr, "https://") { + return uploadFileHTTPPut(URLStr, file) } + // Use GCS as default to maintain backwards compatibility. + return uploadFileGCS(URLStr, file) } -func uploadCoverReportGCS(coverUploadURL string, coverReport io.Reader) error { +func uploadFileGCS(URL string, file io.Reader) error { GCS, err := gcs.NewClient() if err != nil { return fmt.Errorf("failed to create GCS client: %v", err) } defer GCS.Close() - gcsWriter, err := GCS.FileWriter(coverUploadURL) + gcsWriter, err := GCS.FileWriter(URL) if err != nil { return fmt.Errorf("failed to create GCS writer: %v", err) } - if _, err := io.Copy(gcsWriter, coverReport); err != nil { + if _, err := io.Copy(gcsWriter, file); err != nil { gcsWriter.Close() return fmt.Errorf("failed to copy report: %v", err) } if err := gcsWriter.Close(); err != nil { return fmt.Errorf("failed to close gcs writer: %v", err) } - return GCS.Publish(coverUploadURL) + return GCS.Publish(URL) } -func uploadCoverReportHTTPPut(coverUploadURL string, coverReport io.Reader) error { - req, err := http.NewRequest(http.MethodPut, coverUploadURL, coverReport) +func uploadFileHTTPPut(URL string, file io.Reader) error { + req, err := http.NewRequest(http.MethodPut, URL, file) if err != nil { return fmt.Errorf("failed to create HTTP PUT request: %v", err) } diff --git a/syz-ci/syz-ci.go b/syz-ci/syz-ci.go index fedc20550..0781812ab 100644 --- a/syz-ci/syz-ci.go +++ b/syz-ci/syz-ci.go @@ -91,12 +91,15 @@ type Config struct { SyzkallerBranch string `json:"syzkaller_branch"` // Defaults to "master". // Dir with additional syscall descriptions (.txt and .const files). SyzkallerDescriptions string `json:"syzkaller_descriptions"` - // Protocol-specific path to upload coverage reports from managers (optional). + // Path to upload coverage reports from managers (optional). // Supported protocols: GCS (gs://) and HTTP PUT (http:// or https://). - CoverUploadPath string `json:"cover_upload_path"` - BisectBinDir string `json:"bisect_bin_dir"` - Ccache string `json:"ccache"` - Managers []*ManagerConfig `json:"managers"` + CoverUploadPath string `json:"cover_upload_path"` + // Path to upload corpus.db from managers (optional). + // Supported protocols: GCS (gs://) and HTTP PUT (http:// or https://). + CorpusUploadPath string `json:"corpus_upload_path"` + BisectBinDir string `json:"bisect_bin_dir"` + Ccache string `json:"ccache"` + Managers []*ManagerConfig `json:"managers"` // Poll period for jobs in seconds (optional, defaults to 10 seconds) JobPollPeriod int `json:"job_poll_period"` // Poll period for commits in seconds (optional, defaults to 3600 seconds) -- cgit mrf-deployment