From a9e7e9e3a26f5e19a9d96c1a27717aa83e0bdbcd Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Fri, 9 Aug 2024 12:40:12 +0200 Subject: covermerger: add web-git source provider It allows to speed up the single file calculations because we don't need git clone. --- pkg/covermerger/covermerger.go | 2 +- pkg/covermerger/monorepo.go | 104 ----------------------------------- pkg/covermerger/provider_monorepo.go | 104 +++++++++++++++++++++++++++++++++++ pkg/covermerger/provider_web.go | 65 ++++++++++++++++++++++ 4 files changed, 170 insertions(+), 105 deletions(-) delete mode 100644 pkg/covermerger/monorepo.go create mode 100644 pkg/covermerger/provider_monorepo.go create mode 100644 pkg/covermerger/provider_web.go (limited to 'pkg') diff --git a/pkg/covermerger/covermerger.go b/pkg/covermerger/covermerger.go index 602a0a263..f9faa9387 100644 --- a/pkg/covermerger/covermerger.go +++ b/pkg/covermerger/covermerger.go @@ -122,7 +122,7 @@ type Config struct { Workdir string skipRepoClone bool Base RepoBranchCommit - FileVersProvider fileVersProvider + FileVersProvider FileVersProvider } func isSchema(fields, schema []string) bool { diff --git a/pkg/covermerger/monorepo.go b/pkg/covermerger/monorepo.go deleted file mode 100644 index 5ed67e311..000000000 --- a/pkg/covermerger/monorepo.go +++ /dev/null @@ -1,104 +0,0 @@ -// Copyright 2024 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package covermerger - -import ( - "fmt" - "log" - "sync" - - "github.com/google/syzkaller/pkg/vcs" - "github.com/google/syzkaller/sys/targets" -) - -type fileVersProvider interface { - GetFileVersions(c *Config, targetFilePath string, rbcs []RepoBranchCommit, - ) (fileVersions, error) -} - -type monoRepo struct { - branches map[RepoBranchCommit]struct{} - mu sync.RWMutex - repo vcs.Repo -} - -type fileVersions map[RepoBranchCommit]string - -func (mr *monoRepo) GetFileVersions(c *Config, targetFilePath string, rbcs []RepoBranchCommit, -) (fileVersions, error) { - mr.mu.RLock() - if !mr.allRepoBranchesPresent(rbcs) { - mr.mu.RUnlock() - if err := mr.cloneBranches(rbcs); err != nil { - return nil, fmt.Errorf("failed to clone repos: %w", err) - } - mr.mu.RLock() - } - defer mr.mu.RUnlock() - res := make(fileVersions) - for _, rbc := range rbcs { - fileBytes, err := mr.repo.Object(targetFilePath, rbc.Commit) - // It is ok if some file doesn't exist. It means we have repo FS diff. - if err != nil { - continue - } - res[rbc] = string(fileBytes) - } - return res, nil -} - -func (mr *monoRepo) allRepoBranchesPresent(rbcs []RepoBranchCommit) bool { - for _, rbc := range rbcs { - if !mr.repoBranchPresent(rbc) { - return false - } - } - return true -} - -func (mr *monoRepo) repoBranchPresent(rbc RepoBranchCommit) bool { - rbc.Commit = "" - _, ok := mr.branches[rbc] - return ok -} - -func (mr *monoRepo) addRepoBranch(rbc RepoBranchCommit) error { - rbc.Commit = "" - mr.branches[rbc] = struct{}{} - log.Printf("cloning repo: %s, branch: %s", rbc.Repo, rbc.Branch) - if rbc.Repo == "" || rbc.Branch == "" { - panic("repo and branch are needed") - } - if _, err := mr.repo.CheckoutBranch(rbc.Repo, rbc.Branch); err != nil { - return fmt.Errorf("failed to CheckoutBranch(repo %s, branch %s): %w", - rbc.Repo, rbc.Branch, err) - } - return nil -} - -func MakeMonoRepo(workdir string) fileVersProvider { - rbcPath := workdir + "/repos/linux_kernels" - mr := &monoRepo{ - branches: map[RepoBranchCommit]struct{}{}, - } - var err error - if mr.repo, err = vcs.NewRepo(targets.Linux, "none", rbcPath); err != nil { - panic(fmt.Sprintf("failed to create/open repo at %s: %s", rbcPath, err.Error())) - } - return mr -} - -func (mr *monoRepo) cloneBranches(rbcs []RepoBranchCommit) error { - mr.mu.Lock() - defer mr.mu.Unlock() - for _, rbc := range rbcs { - if mr.repoBranchPresent(rbc) { - continue - } - if err := mr.addRepoBranch(rbc); err != nil { - return err - } - } - return nil -} diff --git a/pkg/covermerger/provider_monorepo.go b/pkg/covermerger/provider_monorepo.go new file mode 100644 index 000000000..21d6792e2 --- /dev/null +++ b/pkg/covermerger/provider_monorepo.go @@ -0,0 +1,104 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package covermerger + +import ( + "fmt" + "log" + "sync" + + "github.com/google/syzkaller/pkg/vcs" + "github.com/google/syzkaller/sys/targets" +) + +type FileVersProvider interface { + GetFileVersions(c *Config, targetFilePath string, rbcs []RepoBranchCommit, + ) (fileVersions, error) +} + +type monoRepo struct { + branches map[RepoBranchCommit]struct{} + mu sync.RWMutex + repo vcs.Repo +} + +type fileVersions map[RepoBranchCommit]string + +func (mr *monoRepo) GetFileVersions(c *Config, targetFilePath string, rbcs []RepoBranchCommit, +) (fileVersions, error) { + mr.mu.RLock() + if !mr.allRepoBranchesPresent(rbcs) { + mr.mu.RUnlock() + if err := mr.cloneBranches(rbcs); err != nil { + return nil, fmt.Errorf("failed to clone repos: %w", err) + } + mr.mu.RLock() + } + defer mr.mu.RUnlock() + res := make(fileVersions) + for _, rbc := range rbcs { + fileBytes, err := mr.repo.Object(targetFilePath, rbc.Commit) + // It is ok if some file doesn't exist. It means we have repo FS diff. + if err != nil { + continue + } + res[rbc] = string(fileBytes) + } + return res, nil +} + +func (mr *monoRepo) allRepoBranchesPresent(rbcs []RepoBranchCommit) bool { + for _, rbc := range rbcs { + if !mr.repoBranchPresent(rbc) { + return false + } + } + return true +} + +func (mr *monoRepo) repoBranchPresent(rbc RepoBranchCommit) bool { + rbc.Commit = "" + _, ok := mr.branches[rbc] + return ok +} + +func (mr *monoRepo) addRepoBranch(rbc RepoBranchCommit) error { + rbc.Commit = "" + mr.branches[rbc] = struct{}{} + log.Printf("cloning repo: %s, branch: %s", rbc.Repo, rbc.Branch) + if rbc.Repo == "" || rbc.Branch == "" { + panic("repo and branch are needed") + } + if _, err := mr.repo.CheckoutBranch(rbc.Repo, rbc.Branch); err != nil { + return fmt.Errorf("failed to CheckoutBranch(repo %s, branch %s): %w", + rbc.Repo, rbc.Branch, err) + } + return nil +} + +func MakeMonoRepo(workdir string) FileVersProvider { + rbcPath := workdir + "/repos/linux_kernels" + mr := &monoRepo{ + branches: map[RepoBranchCommit]struct{}{}, + } + var err error + if mr.repo, err = vcs.NewRepo(targets.Linux, "none", rbcPath); err != nil { + panic(fmt.Sprintf("failed to create/open repo at %s: %s", rbcPath, err.Error())) + } + return mr +} + +func (mr *monoRepo) cloneBranches(rbcs []RepoBranchCommit) error { + mr.mu.Lock() + defer mr.mu.Unlock() + for _, rbc := range rbcs { + if mr.repoBranchPresent(rbc) { + continue + } + if err := mr.addRepoBranch(rbc); err != nil { + return err + } + } + return nil +} diff --git a/pkg/covermerger/provider_web.go b/pkg/covermerger/provider_web.go new file mode 100644 index 000000000..72933f280 --- /dev/null +++ b/pkg/covermerger/provider_web.go @@ -0,0 +1,65 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package covermerger + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" +) + +type webGit struct { +} + +func (mr *webGit) GetFileVersions(c *Config, targetFilePath string, rbcs []RepoBranchCommit, +) (fileVersions, error) { + res := make(fileVersions) + for _, rbc := range rbcs { + fileBytes, err := loadFile(targetFilePath, rbc.Repo, rbc.Commit) + // It is ok if some file doesn't exist. It means we have repo FS diff. + if err == errFileNotFound { + continue + } + if err != nil { + return nil, fmt.Errorf("failed to loadFile: %w", err) + } + res[rbc] = string(fileBytes) + } + return res, nil +} + +var errFileNotFound = errors.New("file not found") + +func loadFile(filePath, repo, commit string) ([]byte, error) { + uri := fmt.Sprintf("%s/plain/%s?id=%s", repo, filePath, commit) + u, err := url.Parse(uri) + if err != nil { + return nil, fmt.Errorf("failed to parse %v: %w", uri, err) + } + u.Scheme = "https" + uri = u.String() + res, err := http.Get(uri) + if err != nil { + return nil, fmt.Errorf("failed to http.Get: %w", err) + } + defer res.Body.Close() + + if res.StatusCode == 404 { + return nil, errFileNotFound + } + if res.StatusCode != 200 { + return nil, fmt.Errorf("error: status %d getting %s", res.StatusCode, uri) + } + body, err := io.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("failed to io.ReadAll from body: %w", err) + } + return body, nil +} + +func MakeWebGit() FileVersProvider { + return &webGit{} +} -- cgit mrf-deployment