dashboard/app: poll commits info

This implements 2 features:
 - syz-ci polls a set of additional repos to discover fixing commits sooner
   (e.g. it can now discover a fixing commit in netfilter tree before
   it reaches any of the tested trees).
 - syz-ci uploads info about commits to dashboard.
   For example, a user marks a bug as fixed by commit "foo: bar".
   syz-ci will find this commit in the main namespace repo
   and upload commmit hash/date/author to dashboard. This in turn
   allows to show links to fixing commits.

Fixes #691
Fixes #610

1 files changed, 102 insertions, 43 deletions

diff --git a/pkg/vcs/git.go b/pkg/vcs/git.go
index 45a7e340c..4f23bb16c 100644
--- a/pkg/vcs/git.go
+++ b/pkg/vcs/git.go
@@ -148,10 +148,10 @@ func (git *git) getCommit(commit string) (*Commit, error) {
 	if err != nil {
 		return nil, err
 	}
-	return gitParseCommit(output)
+	return gitParseCommit(output, nil, nil)
 }
 
-func gitParseCommit(output []byte) (*Commit, error) {
+func gitParseCommit(output, user, domain []byte) (*Commit, error) {
 	lines := bytes.Split(output, []byte{'\n'})
 	if len(lines) < 4 || len(lines[0]) != 40 {
 		return nil, fmt.Errorf("unexpected git log output: %q", output)
@@ -163,7 +163,29 @@ func gitParseCommit(output []byte) (*Commit, error) {
 	}
 	cc := make(map[string]bool)
 	cc[strings.ToLower(string(lines[2]))] = true
+	var tags []string
 	for _, line := range lines[4:] {
+		if user != nil {
+			userPos := bytes.Index(line, user)
+			if userPos != -1 {
+				domainPos := bytes.Index(line[userPos+len(user)+1:], domain)
+				if domainPos != -1 {
+					startPos := userPos + len(user)
+					endPos := userPos + len(user) + domainPos + 1
+					tag := string(line[startPos:endPos])
+					present := false
+					for _, tag1 := range tags {
+						if tag1 == tag {
+							present = true
+							break
+						}
+					}
+					if !present {
+						tags = append(tags, tag)
+					}
+				}
+			}
+		}
 		for _, re := range ccRes {
 			matches := re.FindSubmatchIndex(line)
 			if matches == nil {
@@ -187,26 +209,82 @@ func gitParseCommit(output []byte) (*Commit, error) {
 		Title:  string(lines[1]),
 		Author: string(lines[2]),
 		CC:     sortedCC,
+		Tags:   tags,
 		Date:   date,
 	}
 	return com, nil
 }
 
+func (git *git) GetCommitByTitle(title string) (*Commit, error) {
+	commits, _, err := git.GetCommitsByTitles([]string{title})
+	if err != nil || len(commits) == 0 {
+		return nil, err
+	}
+	return commits[0], nil
+}
+
+func (git *git) GetCommitsByTitles(titles []string) ([]*Commit, []string, error) {
+	var greps []string
+	m := make(map[string]string)
+	for _, title := range titles {
+		canonical := CanonicalizeCommit(title)
+		greps = append(greps, canonical)
+		m[canonical] = title
+	}
+	since := time.Now().Add(-time.Hour * 24 * 365 * 2).Format("01-02-2006")
+	commits, err := git.fetchCommits(since, "HEAD", "", "", greps, true)
+	if err != nil {
+		return nil, nil, err
+	}
+	var results []*Commit
+	for _, com := range commits {
+		canonical := CanonicalizeCommit(com.Title)
+		if orig := m[canonical]; orig != "" {
+			delete(m, canonical)
+			results = append(results, com)
+			com.Title = orig
+		}
+	}
+	var missing []string
+	for _, orig := range m {
+		missing = append(missing, orig)
+	}
+	return results, missing, nil
+}
+
 func (git *git) ListRecentCommits(baseCommit string) ([]string, error) {
 	// On upstream kernel this produces ~11MB of output.
 	// Somewhat inefficient to collect whole output in a slice
 	// and then convert to string, but should be bearable.
 	output, err := runSandboxed(git.dir, "git", "log",
-		"--pretty=format:%s", "--no-merges", "-n", "200000", baseCommit)
+		"--pretty=format:%s", "-n", "200000", baseCommit)
 	if err != nil {
 		return nil, err
 	}
 	return strings.Split(string(output), "\n"), nil
 }
 
-func (git *git) ExtractFixTagsFromCommits(baseCommit, email string) ([]FixCommit, error) {
+func (git *git) ExtractFixTagsFromCommits(baseCommit, email string) ([]*Commit, error) {
+	user, domain, err := splitEmail(email)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse email %q: %v", email, err)
+	}
+	grep := user + "+.*" + domain
 	since := time.Now().Add(-time.Hour * 24 * 365).Format("01-02-2006")
-	cmd := exec.Command("git", "log", "--no-merges", "--since", since, baseCommit)
+	return git.fetchCommits(since, baseCommit, user, domain, []string{grep}, false)
+}
+
+func (git *git) fetchCommits(since, base, user, domain string, greps []string, fixedStrings bool) ([]*Commit, error) {
+	const commitSeparator = "---===syzkaller-commit-separator===---"
+	args := []string{"log", "--since", since, "--format=%H%n%s%n%ae%n%ad%n%b%n" + commitSeparator}
+	if fixedStrings {
+		args = append(args, "--fixed-strings")
+	}
+	for _, grep := range greps {
+		args = append(args, "--grep", grep)
+	}
+	args = append(args, base)
+	cmd := exec.Command("git", args...)
 	cmd.Dir = git.dir
 	stdout, err := cmd.StdoutPipe()
 	if err != nil {
@@ -217,52 +295,33 @@ func (git *git) ExtractFixTagsFromCommits(baseCommit, email string) ([]FixCommit
 	}
 	defer cmd.Wait()
 	defer cmd.Process.Kill()
-	return gitExtractFixTags(stdout, email)
-}
-
-func gitExtractFixTags(r io.Reader, email string) ([]FixCommit, error) {
-	user, domain, err := splitEmail(email)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse email %q: %v", email, err)
-	}
 	var (
-		s           = bufio.NewScanner(r)
-		commits     []FixCommit
-		commitTitle = ""
-		commitStart = []byte("commit ")
-		bodyPrefix  = []byte("    ")
-		userBytes   = []byte(user + "+")
-		domainBytes = []byte(domain)
+		s           = bufio.NewScanner(stdout)
+		buf         = new(bytes.Buffer)
+		separator   = []byte(commitSeparator)
+		commits     []*Commit
+		userBytes   []byte
+		domainBytes []byte
 	)
+	if user != "" {
+		userBytes = []byte(user + "+")
+		domainBytes = []byte(domain)
+	}
 	for s.Scan() {
 		ln := s.Bytes()
-		if bytes.HasPrefix(ln, commitStart) {
-			commitTitle = ""
+		if !bytes.Equal(ln, separator) {
+			buf.Write(ln)
+			buf.WriteByte('\n')
 			continue
 		}
-		if !bytes.HasPrefix(ln, bodyPrefix) {
-			continue
-		}
-		ln = ln[len(bodyPrefix):]
-		if len(ln) == 0 {
-			continue
-		}
-		if commitTitle == "" {
-			commitTitle = string(ln)
-			continue
-		}
-		userPos := bytes.Index(ln, userBytes)
-		if userPos == -1 {
-			continue
+		com, err := gitParseCommit(buf.Bytes(), userBytes, domainBytes)
+		if err != nil {
+			return nil, err
 		}
-		domainPos := bytes.Index(ln[userPos+len(userBytes)+1:], domainBytes)
-		if domainPos == -1 {
-			continue
+		if user == "" || len(com.Tags) != 0 {
+			commits = append(commits, com)
 		}
-		startPos := userPos + len(userBytes)
-		endPos := userPos + len(userBytes) + domainPos + 1
-		tag := string(ln[startPos:endPos])
-		commits = append(commits, FixCommit{tag, commitTitle})
+		buf.Reset()
 	}
 	return commits, s.Err()
 }