From 58e8587f648cb149e15553e4a43749df340adc8b Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Wed, 27 Jun 2018 13:47:15 +0200 Subject: pkg/vcs: pave way for multi-vcs support Wrap current git interface in abstract interface. Provide constructor that create repo interface for the given os/vm. --- pkg/bisect/bisect.go | 24 +++-- pkg/instance/instance.go | 2 +- pkg/vcs/git.go | 246 +++++++++++++---------------------------------- pkg/vcs/git_test.go | 85 ++-------------- pkg/vcs/vcs.go | 180 ++++++++++++++++++++++++++++++++++ pkg/vcs/vcs_test.go | 81 ++++++++++++++++ 6 files changed, 348 insertions(+), 270 deletions(-) create mode 100644 pkg/vcs/vcs.go create mode 100644 pkg/vcs/vcs_test.go (limited to 'pkg') diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go index 3dab05290..3e136276e 100644 --- a/pkg/bisect/bisect.go +++ b/pkg/bisect/bisect.go @@ -51,6 +51,7 @@ type ReproConfig struct { type env struct { cfg *Config + repo vcs.Repo head *vcs.Commit inst *instance.Env numTests int @@ -63,8 +64,13 @@ type buildEnv struct { } func Run(cfg *Config) (*vcs.Commit, error) { + repo, err := vcs.NewRepo(cfg.Manager.TargetOS, cfg.Manager.Type, cfg.Manager.KernelSrc) + if err != nil { + return nil, err + } env := &env{ - cfg: cfg, + cfg: cfg, + repo: repo, } if cfg.Fix { env.log("searching for fixing commit since %v", cfg.Kernel.Commit) @@ -98,7 +104,7 @@ func (env *env) bisect() (*vcs.Commit, error) { if env.inst, err = instance.NewEnv(&cfg.Manager); err != nil { return nil, err } - if env.head, err = vcs.Poll(cfg.Manager.KernelSrc, cfg.Kernel.Repo, cfg.Kernel.Branch); err != nil { + if env.head, err = env.repo.Poll(cfg.Kernel.Repo, cfg.Kernel.Branch); err != nil { return nil, err } if err := build.Clean(cfg.Manager.TargetOS, cfg.Manager.TargetArch, @@ -109,7 +115,7 @@ func (env *env) bisect() (*vcs.Commit, error) { if err := env.inst.BuildSyzkaller(cfg.Syzkaller.Repo, cfg.Syzkaller.Commit); err != nil { return nil, err } - if _, err := vcs.SwitchCommit(cfg.Manager.KernelSrc, cfg.Kernel.Commit); err != nil { + if _, err := env.repo.SwitchCommit(cfg.Kernel.Commit); err != nil { return nil, err } if res, err := env.test(); err != nil { @@ -127,7 +133,7 @@ func (env *env) bisect() (*vcs.Commit, error) { if good == "" { return nil, nil // still not fixed } - return vcs.Bisect(cfg.Manager.KernelSrc, bad, good, cfg.Trace, func() (vcs.BisectResult, error) { + return env.repo.Bisect(bad, good, cfg.Trace, func() (vcs.BisectResult, error) { res, err := env.test() if cfg.Fix { if res == vcs.BisectBad { @@ -149,7 +155,7 @@ func (env *env) commitRange() (*vcs.Commit, string, string, error) { func (env *env) commitRangeForFix() (*vcs.Commit, string, string, error) { env.log("testing current HEAD %v", env.head.Hash) - if _, err := vcs.SwitchCommit(env.cfg.Manager.KernelSrc, env.head.Hash); err != nil { + if _, err := env.repo.SwitchCommit(env.head.Hash); err != nil { return nil, "", "", err } res, err := env.test() @@ -164,7 +170,7 @@ func (env *env) commitRangeForFix() (*vcs.Commit, string, string, error) { func (env *env) commitRangeForBug() (*vcs.Commit, string, string, error) { cfg := env.cfg - tags, err := vcs.PreviousReleaseTags(cfg.Manager.KernelSrc, cfg.Kernel.Commit) + tags, err := env.repo.PreviousReleaseTags(cfg.Kernel.Commit) if err != nil { return nil, "", "", err } @@ -183,7 +189,7 @@ func (env *env) commitRangeForBug() (*vcs.Commit, string, string, error) { lastBad := cfg.Kernel.Commit for i, tag := range tags { env.log("testing release %v", tag) - commit, err := vcs.SwitchCommit(cfg.Manager.KernelSrc, tag) + commit, err := env.repo.SwitchCommit(tag) if err != nil { return nil, "", "", err } @@ -207,7 +213,7 @@ func (env *env) commitRangeForBug() (*vcs.Commit, string, string, error) { func (env *env) test() (vcs.BisectResult, error) { cfg := env.cfg env.numTests++ - current, err := vcs.HeadCommit(cfg.Manager.KernelSrc) + current, err := env.repo.HeadCommit() if err != nil { return 0, err } @@ -303,7 +309,7 @@ func (env *env) processResults(current *vcs.Commit, results []error) (bad, good // Note: linux-specific. func (env *env) buildEnvForCommit(commit string) (*buildEnv, error) { cfg := env.cfg - tags, err := vcs.PreviousReleaseTags(cfg.Manager.KernelSrc, commit) + tags, err := env.repo.PreviousReleaseTags(commit) if err != nil { return nil, err } diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index a41a4cbc2..79616f3fa 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -59,7 +59,7 @@ func (env *Env) BuildSyzkaller(repo, commit string) error { if srcIndex == -1 { return fmt.Errorf("syzkaller path %q is not in GOPATH", cfg.Syzkaller) } - if _, err := vcs.CheckoutCommit(cfg.Syzkaller, repo, commit); err != nil { + if _, err := vcs.NewSyzkallerRepo(cfg.Syzkaller).CheckoutCommit(repo, commit); err != nil { return fmt.Errorf("failed to checkout syzkaller repo: %v", err) } cmd := osutil.Command("make", "target") diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go index e942b3c39..d1595aaa8 100644 --- a/pkg/vcs/git.go +++ b/pkg/vcs/git.go @@ -12,7 +12,6 @@ import ( "net/mail" "os" "os/exec" - "regexp" "sort" "strconv" "strings" @@ -21,21 +20,28 @@ import ( "github.com/google/syzkaller/pkg/osutil" ) -const ( - DateFormat = "Mon Jan 2 15:04:05 2006 -0700" - timeout = time.Hour // timeout for all git invocations -) +type git struct { + os string + vm string + dir string +} + +func newGit(os, vm, dir string) *git { + return &git{ + os: os, + vm: vm, + dir: dir, + } +} -// Poll checkouts the specified repository/branch in dir. -// This involves fetching/resetting/cloning as necessary to recover from all possible problems. -// Returns hash of the HEAD commit in the specified branch. -func Poll(dir, repo, branch string) (*Commit, error) { +func (git *git) Poll(repo, branch string) (*Commit, error) { + dir := git.dir runSandboxed(dir, "git", "bisect", "reset") runSandboxed(dir, "git", "reset", "--hard") origin, err := runSandboxed(dir, "git", "remote", "get-url", "origin") if err != nil || strings.TrimSpace(string(origin)) != repo { // The repo is here, but it has wrong origin (e.g. repo in config has changed), re-clone. - if err := clone(dir, repo, branch); err != nil { + if err := git.clone(repo, branch); err != nil { return nil, err } } @@ -44,27 +50,27 @@ func Poll(dir, repo, branch string) (*Commit, error) { // stuck with the local version forever (git checkout won't fail). if _, err := runSandboxed(dir, "git", "checkout", "origin/"+branch); err != nil { // No such branch (e.g. branch in config has changed), re-clone. - if err := clone(dir, repo, branch); err != nil { + if err := git.clone(repo, branch); err != nil { return nil, err } } if _, err := runSandboxed(dir, "git", "fetch", "--no-tags"); err != nil { // Something else is wrong, re-clone. - if err := clone(dir, repo, branch); err != nil { + if err := git.clone(repo, branch); err != nil { return nil, err } } if _, err := runSandboxed(dir, "git", "checkout", "origin/"+branch); err != nil { return nil, err } - return HeadCommit(dir) + return git.HeadCommit() } -// CheckoutBranch checkouts the specified repository/branch in dir. -func CheckoutBranch(dir, repo, branch string) (*Commit, error) { +func (git *git) CheckoutBranch(repo, branch string) (*Commit, error) { + dir := git.dir runSandboxed(dir, "git", "bisect", "reset") if _, err := runSandboxed(dir, "git", "reset", "--hard"); err != nil { - if err := initRepo(dir); err != nil { + if err := git.initRepo(); err != nil { return nil, err } } @@ -75,14 +81,14 @@ func CheckoutBranch(dir, repo, branch string) (*Commit, error) { if _, err := runSandboxed(dir, "git", "checkout", "FETCH_HEAD"); err != nil { return nil, err } - return HeadCommit(dir) + return git.HeadCommit() } -// CheckoutCommit checkouts the specified repository on the specified commit in dir. -func CheckoutCommit(dir, repo, commit string) (*Commit, error) { +func (git *git) CheckoutCommit(repo, commit string) (*Commit, error) { + dir := git.dir runSandboxed(dir, "git", "bisect", "reset") if _, err := runSandboxed(dir, "git", "reset", "--hard"); err != nil { - if err := initRepo(dir); err != nil { + if err := git.initRepo(); err != nil { return nil, err } } @@ -90,73 +96,65 @@ func CheckoutCommit(dir, repo, commit string) (*Commit, error) { if err != nil { return nil, err } - return SwitchCommit(dir, commit) + return git.SwitchCommit(commit) } -// SwitchCommit checkouts the specified commit without fetching. -func SwitchCommit(dir, commit string) (*Commit, error) { +func (git *git) SwitchCommit(commit string) (*Commit, error) { + dir := git.dir if _, err := runSandboxed(dir, "git", "checkout", commit); err != nil { return nil, err } - return HeadCommit(dir) + return git.HeadCommit() } -func clone(dir, repo, branch string) error { - if err := initRepo(dir); err != nil { +func (git *git) clone(repo, branch string) error { + if err := git.initRepo(); err != nil { return err } - if _, err := runSandboxed(dir, "git", "remote", "add", "origin", repo); err != nil { + if _, err := runSandboxed(git.dir, "git", "remote", "add", "origin", repo); err != nil { return err } - if _, err := runSandboxed(dir, "git", "fetch", "origin", branch); err != nil { + if _, err := runSandboxed(git.dir, "git", "fetch", "origin", branch); err != nil { return err } return nil } -func initRepo(dir string) error { - if err := os.RemoveAll(dir); err != nil { +func (git *git) initRepo() error { + if err := os.RemoveAll(git.dir); err != nil { return fmt.Errorf("failed to remove repo dir: %v", err) } - if err := osutil.MkdirAll(dir); err != nil { + if err := osutil.MkdirAll(git.dir); err != nil { return fmt.Errorf("failed to create repo dir: %v", err) } - if err := osutil.SandboxChown(dir); err != nil { + if err := osutil.SandboxChown(git.dir); err != nil { return err } - if _, err := runSandboxed(dir, "git", "init"); err != nil { + if _, err := runSandboxed(git.dir, "git", "init"); err != nil { return err } return nil } -type Commit struct { - Hash string - Title string - Author string - CC []string - Date time.Time +func (git *git) HeadCommit() (*Commit, error) { + return git.GetCommit("HEAD") } -// HeadCommit returns info about the HEAD commit of the current branch of git repository in dir. -func HeadCommit(dir string) (*Commit, error) { - return GetCommit(dir, "HEAD") -} - -func GetCommit(dir, commit string) (*Commit, error) { - output, err := runSandboxed(dir, "git", "log", "--format=%H%n%s%n%ae%n%ad%n%b", "-n", "1", commit) +func (git *git) GetCommit(commit string) (*Commit, error) { + output, err := runSandboxed(git.dir, "git", "log", "--format=%H%n%s%n%ae%n%ad%n%b", "-n", "1", commit) if err != nil { return nil, err } - return parseCommit(output) + return gitParseCommit(output) } -func parseCommit(output []byte) (*Commit, error) { +func gitParseCommit(output []byte) (*Commit, error) { lines := bytes.Split(output, []byte{'\n'}) if len(lines) < 4 || len(lines[0]) != 40 { return nil, fmt.Errorf("unexpected git log output: %q", output) } - date, err := time.Parse(DateFormat, string(lines[3])) + const dateFormat = "Mon Jan 2 15:04:05 2006 -0700" + date, err := time.Parse(dateFormat, string(lines[3])) if err != nil { return nil, fmt.Errorf("failed to parse date in git log output: %v\n%q", err, output) } @@ -191,12 +189,11 @@ func parseCommit(output []byte) (*Commit, error) { return com, nil } -// ListRecentCommits returns list of recent commit titles starting from baseCommit. -func ListRecentCommits(dir, baseCommit string) ([]string, error) { +func (git *git) ListRecentCommits(baseCommit string) ([]string, error) { // On upstream kernel this produces ~11MB of output. // Somewhat inefficient to collect whole output in a slice // and then convert to string, but should be bearable. - output, err := runSandboxed(dir, "git", "log", + output, err := runSandboxed(git.dir, "git", "log", "--pretty=format:%s", "--no-merges", "-n", "200000", baseCommit) if err != nil { return nil, err @@ -204,18 +201,10 @@ func ListRecentCommits(dir, baseCommit string) ([]string, error) { return strings.Split(string(output), "\n"), nil } -type FixCommit struct { - Tag string - Title string -} - -// ExtractFixTagsFromCommits extracts fixing tags for bugs from git log. -// Given email = "user@domain.com", it searches for tags of the form "user+tag@domain.com" -// and return pairs {tag, commit title}. -func ExtractFixTagsFromCommits(dir, baseCommit, email string) ([]FixCommit, error) { +func (git *git) ExtractFixTagsFromCommits(baseCommit, email string) ([]FixCommit, error) { since := time.Now().Add(-time.Hour * 24 * 365).Format("01-02-2006") cmd := exec.Command("git", "log", "--no-merges", "--since", since, baseCommit) - cmd.Dir = dir + cmd.Dir = git.dir stdout, err := cmd.StdoutPipe() if err != nil { return nil, err @@ -225,10 +214,10 @@ func ExtractFixTagsFromCommits(dir, baseCommit, email string) ([]FixCommit, erro } defer cmd.Wait() defer cmd.Process.Kill() - return extractFixTags(stdout, email) + return gitExtractFixTags(stdout, email) } -func extractFixTags(r io.Reader, email string) ([]FixCommit, error) { +func gitExtractFixTags(r io.Reader, email string) ([]FixCommit, error) { user, domain, err := splitEmail(email) if err != nil { return nil, fmt.Errorf("failed to parse email %q: %v", email, err) @@ -292,81 +281,11 @@ func splitEmail(email string) (user, domain string, err error) { return } -// CanonicalizeCommit returns commit title that can be used when checking -// if a particular commit is present in a git tree. -// Some trees add prefixes to commit titles during backporting, -// so we want e.g. commit "foo bar" match "BACKPORT: foo bar". -func CanonicalizeCommit(title string) string { - for _, prefix := range commitPrefixes { - if strings.HasPrefix(title, prefix) { - title = title[len(prefix):] - break - } - } - return strings.TrimSpace(title) -} - -var commitPrefixes = []string{ - "UPSTREAM:", - "CHROMIUM:", - "FROMLIST:", - "BACKPORT:", - "FROMGIT:", - "net-backports:", -} - -func Patch(dir string, patch []byte) error { - // Do --dry-run first to not mess with partially consistent state. - cmd := osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--dry-run") - if err := osutil.Sandbox(cmd, true, true); err != nil { - return err - } - cmd.Stdin = bytes.NewReader(patch) - cmd.Dir = dir - if output, err := cmd.CombinedOutput(); err != nil { - // If it reverses clean, then it's already applied - // (seems to be the easiest way to detect it). - cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--reverse", "--dry-run") - if err := osutil.Sandbox(cmd, true, true); err != nil { - return err - } - cmd.Stdin = bytes.NewReader(patch) - cmd.Dir = dir - if _, err := cmd.CombinedOutput(); err == nil { - return fmt.Errorf("patch is already applied") - } - return fmt.Errorf("failed to apply patch:\n%s", output) - } - // Now apply for real. - cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace") - if err := osutil.Sandbox(cmd, true, true); err != nil { - return err - } - cmd.Stdin = bytes.NewReader(patch) - cmd.Dir = dir - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to apply patch after dry run:\n%s", output) - } - return nil -} - -type BisectResult int - -const ( - BisectBad BisectResult = iota - BisectGood - BisectSkip -) - -// Bisect bisects good..bad commit range against the provided predicate (wrapper around git bisect). -// The predicate should return an error only if there is no way to proceed -// (it will abort the process), if possible it should prefer to return BisectSkip. -// Progress of the process is streamed to the provided trace. -// Returns the first commit on which the predicate returns BisectBad. -func Bisect(dir, bad, good string, trace io.Writer, pred func() (BisectResult, error)) (*Commit, error) { +func (git *git) Bisect(bad, good string, trace io.Writer, pred func() (BisectResult, error)) (*Commit, error) { + dir := git.dir runSandboxed(dir, "git", "bisect", "reset") runSandboxed(dir, "git", "reset", "--hard") - firstBad, err := GetCommit(dir, bad) + firstBad, err := git.GetCommit(bad) if err != nil { return nil, err } @@ -376,7 +295,7 @@ func Bisect(dir, bad, good string, trace io.Writer, pred func() (BisectResult, e } defer runSandboxed(dir, "git", "bisect", "reset") fmt.Fprintf(trace, "# git bisect start %v %v\n%s", bad, good, output) - current, err := HeadCommit(dir) + current, err := git.HeadCommit() if err != nil { return nil, err } @@ -398,7 +317,7 @@ func Bisect(dir, bad, good string, trace io.Writer, pred func() (BisectResult, e return nil, err } fmt.Fprintf(trace, "# git bisect %v %v\n%s", bisectTerms[res], current.Hash, output) - next, err := HeadCommit(dir) + next, err := git.HeadCommit() if err != nil { return nil, err } @@ -409,30 +328,29 @@ func Bisect(dir, bad, good string, trace io.Writer, pred func() (BisectResult, e } } -// PreviousReleaseTags returns list of preceding release tags that are reachable from the given commit. // Note: linux-specific. -func PreviousReleaseTags(dir, commit string) ([]string, error) { - output, err := runSandboxed(dir, "git", "tag", "--no-contains", commit, "--merged", commit, "v*.*") +func (git *git) PreviousReleaseTags(commit string) ([]string, error) { + output, err := runSandboxed(git.dir, "git", "tag", "--no-contains", commit, "--merged", commit, "v*.*") if err != nil { return nil, err } - return parseReleaseTags(output) + return gitParseReleaseTags(output) } -func parseReleaseTags(output []byte) ([]string, error) { +func gitParseReleaseTags(output []byte) ([]string, error) { var tags []string for _, tag := range bytes.Split(output, []byte{'\n'}) { - if releaseTagRe.Match(tag) && releaseTagToInt(string(tag)) != 0 { + if releaseTagRe.Match(tag) && gitReleaseTagToInt(string(tag)) != 0 { tags = append(tags, string(tag)) } } sort.Slice(tags, func(i, j int) bool { - return releaseTagToInt(tags[i]) > releaseTagToInt(tags[j]) + return gitReleaseTagToInt(tags[i]) > gitReleaseTagToInt(tags[j]) }) return tags, nil } -func releaseTagToInt(tag string) uint64 { +func gitReleaseTagToInt(tag string) uint64 { matches := releaseTagRe.FindStringSubmatchIndex(tag) v1, err := strconv.ParseUint(tag[matches[2]:matches[3]], 10, 64) if err != nil { @@ -460,37 +378,3 @@ func runSandboxed(dir, command string, args ...string) ([]byte, error) { } return osutil.Run(timeout, cmd) } - -// CheckRepoAddress does a best-effort approximate check of a git repo address. -func CheckRepoAddress(repo string) bool { - return gitRepoRe.MatchString(repo) -} - -// CheckBranch does a best-effort approximate check of a git branch name. -func CheckBranch(branch string) bool { - return gitBranchRe.MatchString(branch) -} - -func CheckCommitHash(hash string) bool { - if !gitHashRe.MatchString(hash) { - return false - } - ln := len(hash) - return ln == 8 || ln == 10 || ln == 12 || ln == 16 || ln == 20 || ln == 40 -} - -var ( - // nolint: lll - gitRepoRe = regexp.MustCompile(`^(git|ssh|http|https|ftp|ftps)://[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)+(:[0-9]+)?/[a-zA-Z0-9-_./]+\.git(/)?$`) - gitBranchRe = regexp.MustCompile("^[a-zA-Z0-9-_/.]{2,200}$") - gitHashRe = regexp.MustCompile("^[a-f0-9]+$") - releaseTagRe = regexp.MustCompile(`^v([0-9]+).([0-9]+)(?:\.([0-9]+))?$`) - ccRes = []*regexp.Regexp{ - regexp.MustCompile(`^Reviewed\-.*: (.*)$`), - regexp.MustCompile(`^[A-Za-z-]+\-and\-[Rr]eviewed\-.*: (.*)$`), - regexp.MustCompile(`^Acked\-.*: (.*)$`), - regexp.MustCompile(`^[A-Za-z-]+\-and\-[Aa]cked\-.*: (.*)$`), - regexp.MustCompile(`^Tested\-.*: (.*)$`), - regexp.MustCompile(`^[A-Za-z-]+\-and\-[Tt]ested\-.*: (.*)$`), - } -) diff --git a/pkg/vcs/git_test.go b/pkg/vcs/git_test.go index 6b7cf183e..095aca29a 100644 --- a/pkg/vcs/git_test.go +++ b/pkg/vcs/git_test.go @@ -10,7 +10,7 @@ import ( "time" ) -func TestParseCommit(t *testing.T) { +func TestGitParseCommit(t *testing.T) { tests := map[string]*Commit{ `2075b16e32c26e4031b9fd3cbe26c54676a8fcb5 rbtree: include rcu.h @@ -46,7 +46,7 @@ Signed-off-by: Linux Master }, } for input, com := range tests { - res, err := parseCommit([]byte(input)) + res, err := gitParseCommit([]byte(input)) if err != nil && com != nil { t.Fatalf("want %+v, got error: %v", com, err) } @@ -74,80 +74,7 @@ Signed-off-by: Linux Master } } -func TestCanonicalizeCommit(t *testing.T) { - tests := map[string]string{ - "foo bar": "foo bar", - " foo ": "foo", - "UPSTREAM: foo bar": "foo bar", - "BACKPORT: UPSTREAM: foo bar": "UPSTREAM: foo bar", - } - for in, want := range tests { - got := CanonicalizeCommit(in) - if got != want { - t.Errorf("input %q: got %q, want %q", in, got, want) - } - } -} - -func TestCheckRepoAddress(t *testing.T) { - testPredicate(t, CheckRepoAddress, map[string]bool{ - "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git": true, - "https://github.com/torvalds/linux.git": true, - "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git": true, - "git://git.cmpxchg.org/linux-mmots.git": true, - "https://anonscm.debian.org/git/kernel/linux.git": true, - "git://kernel.ubuntu.com/ubuntu/ubuntu-zesty.git": true, - "http://host.xz:123/path/to/repo.git/": true, - "": false, - "foobar": false, - "linux-next": false, - "foo://kernel.ubuntu.com/ubuntu/ubuntu-zesty.git": false, - "git://kernel/ubuntu.git": false, - "git://kernel.com/ubuntu": false, - "gitgit://kernel.ubuntu.com/ubuntu/ubuntu-zesty.git": false, - }) -} - -func TestCheckBranch(t *testing.T) { - testPredicate(t, CheckBranch, map[string]bool{ - "master": true, - "core/core": true, - "irq-irqdomain-for-linus": true, - "timers/2038": true, - "ubuntu-zesty/v4.9.4": true, - "WIP.locking/atomics": true, - "linux-4.9.y": true, - "abi_spec": true, - "@": false, - "": false, - }) -} - -func TestCheckCommitHash(t *testing.T) { - testPredicate(t, CheckCommitHash, map[string]bool{ - "ff12bea91c22bba93d3ffc3034d813d686bc7eeb": true, // 40 - "eae05cb0aaeae05cb0aa": true, // 20 - "449dd6984d0eaabb": true, // 16 - "449dd6984d0e": true, // 12 - "eae05cb0aa": true, // 10 - "eae05cb0": true, // 8 - "": false, - "aa": false, - "eae05cb0aab": false, - "xxxxxxxx": false, - }) -} - -func testPredicate(t *testing.T, fn func(string) bool, tests map[string]bool) { - for input, want := range tests { - res := fn(input) - if res != want { - t.Errorf("%v: got %v, want %v", input, res, want) - } - } -} - -func TestParseReleaseTags(t *testing.T) { +func TestGitParseReleaseTags(t *testing.T) { input := ` v3.1 v2.6.12 @@ -183,7 +110,7 @@ v1. "v2.6.13", "v2.6.12", } - got, err := parseReleaseTags([]byte(input)) + got, err := gitParseReleaseTags([]byte(input)) if err != nil { t.Fatal(err) } @@ -192,8 +119,8 @@ v1. } } -func TestExtractFixTags(t *testing.T) { - commits, err := extractFixTags(strings.NewReader(extractFixTagsInput), extractFixTagsEmail) +func TestGitExtractFixTags(t *testing.T) { + commits, err := gitExtractFixTags(strings.NewReader(extractFixTagsInput), extractFixTagsEmail) if err != nil { t.Fatal(err) } diff --git a/pkg/vcs/vcs.go b/pkg/vcs/vcs.go new file mode 100644 index 000000000..bef279bdc --- /dev/null +++ b/pkg/vcs/vcs.go @@ -0,0 +1,180 @@ +// Copyright 2018 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +// Package vcs provides helper functions for working with various repositories (e.g. git). +package vcs + +import ( + "bytes" + "fmt" + "io" + "regexp" + "strings" + "time" + + "github.com/google/syzkaller/pkg/osutil" +) + +type Repo interface { + // Poll checkouts the specified repository/branch. + // This involves fetching/resetting/cloning as necessary to recover from all possible problems. + // Returns hash of the HEAD commit in the specified branch. + Poll(repo, branch string) (*Commit, error) + + // CheckoutBranch checkouts the specified repository/branch. + CheckoutBranch(repo, branch string) (*Commit, error) + + // CheckoutCommit checkouts the specified repository on the specified commit. + CheckoutCommit(repo, commit string) (*Commit, error) + + // SwitchCommit checkouts the specified commit without fetching. + SwitchCommit(commit string) (*Commit, error) + + // HeadCommit returns info about the HEAD commit of the current branch of git repository. + HeadCommit() (*Commit, error) + + // ListRecentCommits returns list of recent commit titles starting from baseCommit. + ListRecentCommits(baseCommit string) ([]string, error) + + // ExtractFixTagsFromCommits extracts fixing tags for bugs from git log. + // Given email = "user@domain.com", it searches for tags of the form "user+tag@domain.com" + // and return pairs {tag, commit title}. + ExtractFixTagsFromCommits(baseCommit, email string) ([]FixCommit, error) + + // PreviousReleaseTags returns list of preceding release tags that are reachable from the given commit. + PreviousReleaseTags(commit string) ([]string, error) + + // Bisect bisects good..bad commit range against the provided predicate (wrapper around git bisect). + // The predicate should return an error only if there is no way to proceed + // (it will abort the process), if possible it should prefer to return BisectSkip. + // Progress of the process is streamed to the provided trace. + // Returns the first commit on which the predicate returns BisectBad. + Bisect(bad, good string, trace io.Writer, pred func() (BisectResult, error)) (*Commit, error) +} + +type Commit struct { + Hash string + Title string + Author string + CC []string + Date time.Time +} + +type FixCommit struct { + Tag string + Title string +} + +type BisectResult int + +const ( + BisectBad BisectResult = iota + BisectGood + BisectSkip +) + +func NewRepo(os, vm, dir string) (Repo, error) { + switch os { + case "linux": + return newGit(os, vm, dir), nil + } + return nil, fmt.Errorf("vcs is unsupported for %v", os) +} + +func NewSyzkallerRepo(dir string) Repo { + return newGit("syzkaller", "", dir) +} + +func Patch(dir string, patch []byte) error { + // Do --dry-run first to not mess with partially consistent state. + cmd := osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--dry-run") + if err := osutil.Sandbox(cmd, true, true); err != nil { + return err + } + cmd.Stdin = bytes.NewReader(patch) + cmd.Dir = dir + if output, err := cmd.CombinedOutput(); err != nil { + // If it reverses clean, then it's already applied + // (seems to be the easiest way to detect it). + cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--reverse", "--dry-run") + if err := osutil.Sandbox(cmd, true, true); err != nil { + return err + } + cmd.Stdin = bytes.NewReader(patch) + cmd.Dir = dir + if _, err := cmd.CombinedOutput(); err == nil { + return fmt.Errorf("patch is already applied") + } + return fmt.Errorf("failed to apply patch:\n%s", output) + } + // Now apply for real. + cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace") + if err := osutil.Sandbox(cmd, true, true); err != nil { + return err + } + cmd.Stdin = bytes.NewReader(patch) + cmd.Dir = dir + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to apply patch after dry run:\n%s", output) + } + return nil +} + +// CheckRepoAddress does a best-effort approximate check of a git repo address. +func CheckRepoAddress(repo string) bool { + return gitRepoRe.MatchString(repo) +} + +// CheckBranch does a best-effort approximate check of a git branch name. +func CheckBranch(branch string) bool { + return gitBranchRe.MatchString(branch) +} + +func CheckCommitHash(hash string) bool { + if !gitHashRe.MatchString(hash) { + return false + } + ln := len(hash) + return ln == 8 || ln == 10 || ln == 12 || ln == 16 || ln == 20 || ln == 40 +} + +const timeout = time.Hour // timeout for all git invocations + +var ( + // nolint: lll + gitRepoRe = regexp.MustCompile(`^(git|ssh|http|https|ftp|ftps)://[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)+(:[0-9]+)?/[a-zA-Z0-9-_./]+\.git(/)?$`) + gitBranchRe = regexp.MustCompile("^[a-zA-Z0-9-_/.]{2,200}$") + gitHashRe = regexp.MustCompile("^[a-f0-9]+$") + releaseTagRe = regexp.MustCompile(`^v([0-9]+).([0-9]+)(?:\.([0-9]+))?$`) + ccRes = []*regexp.Regexp{ + regexp.MustCompile(`^Reviewed\-.*: (.*)$`), + regexp.MustCompile(`^[A-Za-z-]+\-and\-[Rr]eviewed\-.*: (.*)$`), + regexp.MustCompile(`^Acked\-.*: (.*)$`), + regexp.MustCompile(`^[A-Za-z-]+\-and\-[Aa]cked\-.*: (.*)$`), + regexp.MustCompile(`^Tested\-.*: (.*)$`), + regexp.MustCompile(`^[A-Za-z-]+\-and\-[Tt]ested\-.*: (.*)$`), + } +) + +// CanonicalizeCommit returns commit title that can be used when checking +// if a particular commit is present in a git tree. +// Some trees add prefixes to commit titles during backporting, +// so we want e.g. commit "foo bar" match "BACKPORT: foo bar". +func CanonicalizeCommit(title string) string { + for _, prefix := range commitPrefixes { + if strings.HasPrefix(title, prefix) { + title = title[len(prefix):] + break + } + } + return strings.TrimSpace(title) +} + +var commitPrefixes = []string{ + "UPSTREAM:", + "CHROMIUM:", + "FROMLIST:", + "BACKPORT:", + "FROMGIT:", + "net-backports:", +} diff --git a/pkg/vcs/vcs_test.go b/pkg/vcs/vcs_test.go new file mode 100644 index 000000000..2c0660849 --- /dev/null +++ b/pkg/vcs/vcs_test.go @@ -0,0 +1,81 @@ +// Copyright 2018 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package vcs + +import ( + "testing" +) + +func TestCanonicalizeCommit(t *testing.T) { + tests := map[string]string{ + "foo bar": "foo bar", + " foo ": "foo", + "UPSTREAM: foo bar": "foo bar", + "BACKPORT: UPSTREAM: foo bar": "UPSTREAM: foo bar", + } + for in, want := range tests { + got := CanonicalizeCommit(in) + if got != want { + t.Errorf("input %q: got %q, want %q", in, got, want) + } + } +} + +func TestCheckRepoAddress(t *testing.T) { + testPredicate(t, CheckRepoAddress, map[string]bool{ + "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git": true, + "https://github.com/torvalds/linux.git": true, + "git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git": true, + "git://git.cmpxchg.org/linux-mmots.git": true, + "https://anonscm.debian.org/git/kernel/linux.git": true, + "git://kernel.ubuntu.com/ubuntu/ubuntu-zesty.git": true, + "http://host.xz:123/path/to/repo.git/": true, + "": false, + "foobar": false, + "linux-next": false, + "foo://kernel.ubuntu.com/ubuntu/ubuntu-zesty.git": false, + "git://kernel/ubuntu.git": false, + "git://kernel.com/ubuntu": false, + "gitgit://kernel.ubuntu.com/ubuntu/ubuntu-zesty.git": false, + }) +} + +func TestCheckBranch(t *testing.T) { + testPredicate(t, CheckBranch, map[string]bool{ + "master": true, + "core/core": true, + "irq-irqdomain-for-linus": true, + "timers/2038": true, + "ubuntu-zesty/v4.9.4": true, + "WIP.locking/atomics": true, + "linux-4.9.y": true, + "abi_spec": true, + "@": false, + "": false, + }) +} + +func TestCheckCommitHash(t *testing.T) { + testPredicate(t, CheckCommitHash, map[string]bool{ + "ff12bea91c22bba93d3ffc3034d813d686bc7eeb": true, // 40 + "eae05cb0aaeae05cb0aa": true, // 20 + "449dd6984d0eaabb": true, // 16 + "449dd6984d0e": true, // 12 + "eae05cb0aa": true, // 10 + "eae05cb0": true, // 8 + "": false, + "aa": false, + "eae05cb0aab": false, + "xxxxxxxx": false, + }) +} + +func testPredicate(t *testing.T, fn func(string) bool, tests map[string]bool) { + for input, want := range tests { + res := fn(input) + if res != want { + t.Errorf("%v: got %v, want %v", input, res, want) + } + } +} -- cgit mrf-deployment