dashboard/app: collect more info for better reports

Collect kernel build commit title/date.
Add support for kernel repo aliases (to be able
to say linux-next instead of full git repo address).
Collect on what managers a bug happened.
Reuse Crash.ReportLen as generic crash reporting priority.
Make it possible to prioritize reporting of particular
kernel repos and arches.

Fixes #473

8 files changed, 204 insertions, 132 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 5a0d3823d..c12fd62b0 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -269,20 +269,22 @@ func uploadBuild(c context.Context, ns string, req *dashapi.Build, typ BuildType
 		return false, err
 	}
 	build := &Build{
-		Namespace:       ns,
-		Manager:         req.Manager,
-		ID:              req.ID,
-		Type:            typ,
-		Time:            timeNow(c),
-		OS:              req.OS,
-		Arch:            req.Arch,
-		VMArch:          req.VMArch,
-		SyzkallerCommit: req.SyzkallerCommit,
-		CompilerID:      req.CompilerID,
-		KernelRepo:      req.KernelRepo,
-		KernelBranch:    req.KernelBranch,
-		KernelCommit:    req.KernelCommit,
-		KernelConfig:    configID,
+		Namespace:         ns,
+		Manager:           req.Manager,
+		ID:                req.ID,
+		Type:              typ,
+		Time:              timeNow(c),
+		OS:                req.OS,
+		Arch:              req.Arch,
+		VMArch:            req.VMArch,
+		SyzkallerCommit:   req.SyzkallerCommit,
+		CompilerID:        req.CompilerID,
+		KernelRepo:        req.KernelRepo,
+		KernelBranch:      req.KernelBranch,
+		KernelCommit:      req.KernelCommit,
+		KernelCommitTitle: req.KernelCommitTitle,
+		KernelCommitDate:  req.KernelCommitDate,
+		KernelConfig:      configID,
 	}
 	if _, err := datastore.Put(c, buildKey(c, ns, req.ID), build); err != nil {
 		return false, err
@@ -470,6 +472,10 @@ func reportCrash(c context.Context, ns string, req *dashapi.Crash) (*Bug, error)
 			return nil, err
 		}
 	}
+	build, err := loadBuild(c, ns, req.BuildID)
+	if err != nil {
+		return nil, err
+	}
 
 	now := timeNow(c)
 	reproLevel := ReproLevelNone
@@ -482,24 +488,21 @@ func reportCrash(c context.Context, ns string, req *dashapi.Crash) (*Bug, error)
 		now.Sub(bug.LastTime) > time.Hour ||
 		reproLevel != ReproLevelNone
 	if saveCrash {
-		build, err := loadBuild(c, ns, req.BuildID)
-		if err != nil {
-			return nil, err
+		// Reporting priority of this crash.
+		// Currently it is computed only from repository ReportingPriority and Arch,
+		// but can be extended to account for other factors as well.
+		prio := kernelRepoInfo(build).ReportingPriority * 1e6
+		if build.Arch == "amd64" {
+			prio += 1e3
 		}
-
 		crash := &Crash{
 			Manager:     build.Manager,
 			BuildID:     req.BuildID,
 			Time:        now,
 			Maintainers: req.Maintainers,
 			ReproOpts:   req.ReproOpts,
-			// We used to report crash with the longest report len to work around
-			// corrupted reports. Now that we explicitly detect corrupted reports,
-			// disable this sorting. When all old bugs are closed, we need to remove
-			// sorting by ReportLen from queryCrashesForBug.
-			ReportLen: 1e9,
+			ReportLen:   prio,
 		}
-
 		if crash.Log, err = putText(c, ns, "CrashLog", req.Log, false); err != nil {
 			return nil, err
 		}
@@ -537,6 +540,9 @@ func reportCrash(c context.Context, ns string, req *dashapi.Crash) (*Bug, error)
 		if len(req.Report) != 0 {
 			bug.HasReport = true
 		}
+		if !stringInList(bug.HappenedOn, build.Manager) {
+			bug.HappenedOn = append(bug.HappenedOn, build.Manager)
+		}
 		if _, err = datastore.Put(c, bugKey, bug); err != nil {
 			return fmt.Errorf("failed to put bug: %v", err)
 		}
diff --git a/dashboard/app/app_test.go b/dashboard/app/app_test.go
index 9507a5209..a8f3885e6 100644
--- a/dashboard/app/app_test.go
+++ b/dashboard/app/app_test.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/google/syzkaller/dashboard/dashapi"
 )
@@ -104,17 +105,21 @@ func (cfg *TestConfig) Validate() error {
 
 func testBuild(id int) *dashapi.Build {
 	return &dashapi.Build{
-		Manager:         fmt.Sprintf("manager%v", id),
-		ID:              fmt.Sprintf("build%v", id),
-		SyzkallerCommit: fmt.Sprintf("syzkaller_commit%v", id),
-		CompilerID:      fmt.Sprintf("compiler%v", id),
-		KernelRepo:      fmt.Sprintf("repo%v", id),
-		KernelBranch:    fmt.Sprintf("branch%v", id),
-		KernelCommit:    fmt.Sprintf("kernel_commit%v", id),
-		KernelConfig:    []byte(fmt.Sprintf("config%v", id)),
+		Manager:           fmt.Sprintf("manager%v", id),
+		ID:                fmt.Sprintf("build%v", id),
+		SyzkallerCommit:   fmt.Sprintf("syzkaller_commit%v", id),
+		CompilerID:        fmt.Sprintf("compiler%v", id),
+		KernelRepo:        fmt.Sprintf("repo%v", id),
+		KernelBranch:      fmt.Sprintf("branch%v", id),
+		KernelCommit:      fmt.Sprintf("kernel_commit%v", id),
+		KernelCommitTitle: fmt.Sprintf("kernel_commit_title%v", id),
+		KernelCommitDate:  buildCommitDate,
+		KernelConfig:      []byte(fmt.Sprintf("config%v", id)),
 	}
 }
 
+var buildCommitDate = time.Date(1, 2, 3, 4, 5, 6, 0, time.UTC)
+
 func testCrash(build *dashapi.Build, id int) *dashapi.Crash {
 	return &dashapi.Crash{
 		BuildID: build.ID,
diff --git a/dashboard/app/config.go b/dashboard/app/config.go
index cb1759d4c..6bf1a6953 100644
--- a/dashboard/app/config.go
+++ b/dashboard/app/config.go
@@ -28,6 +28,8 @@ type GlobalConfig struct {
 	// Each namespace has own reporting config, own API clients
 	// and bugs are not merged across namespaces.
 	Namespaces map[string]*Config
+	// Maps full repository address/branch to description of this repo.
+	KernelRepos map[string]KernelRepo
 }
 
 // Per-namespace config.
@@ -70,6 +72,14 @@ type ReportingType interface {
 	Validate() error
 }
 
+type KernelRepo struct {
+	// Alias is a short, readable name of a kernel repository.
+	Alias string
+	// ReportingPriority says if we need to prefer to report crashes in this
+	// repo over crashes in repos with lower value. Must be in [0-9] range.
+	ReportingPriority int
+}
+
 var (
 	clientNameRe = regexp.MustCompile("^[a-zA-Z0-9-_]{4,100}$")
 	clientKeyRe  = regexp.MustCompile("^[a-zA-Z0-9]{16,128}$")
@@ -151,6 +161,14 @@ func init() {
 			}
 		}
 	}
+	for repo, info := range config.KernelRepos {
+		if info.Alias == "" {
+			panic(fmt.Sprintf("empty kernel repo alias for %q", repo))
+		}
+		if prio := info.ReportingPriority; prio < 0 || prio > 9 {
+			panic(fmt.Sprintf("bad kernel repo reporting priority %v for %q", prio, repo))
+		}
+	}
 }
 
 func checkClients(clientNames map[string]bool, clients map[string]string) {
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index e64ca0e4d..c79e3ddc9 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -46,20 +46,22 @@ type ManagerStats struct {
 }
 
 type Build struct {
-	Namespace       string
-	Manager         string
-	ID              string // unique ID generated by syz-ci
-	Type            BuildType
-	Time            time.Time
-	OS              string
-	Arch            string
-	VMArch          string
-	SyzkallerCommit string
-	CompilerID      string
-	KernelRepo      string
-	KernelBranch    string
-	KernelCommit    string
-	KernelConfig    int64 // reference to KernelConfig text entity
+	Namespace         string
+	Manager           string
+	ID                string // unique ID generated by syz-ci
+	Type              BuildType
+	Time              time.Time
+	OS                string
+	Arch              string
+	VMArch            string
+	SyzkallerCommit   string
+	CompilerID        string
+	KernelRepo        string
+	KernelBranch      string
+	KernelCommit      string
+	KernelCommitTitle string    `datastore:",noindex"`
+	KernelCommitDate  time.Time `datastore:",noindex"`
+	KernelConfig      int64     // reference to KernelConfig text entity
 }
 
 type Bug struct {
@@ -77,7 +79,8 @@ type Bug struct {
 	Closed     time.Time
 	Reporting  []BugReporting
 	Commits    []string
-	PatchedOn  []string
+	HappenedOn []string `datastore:",noindex"` // list of managers
+	PatchedOn  []string `datastore:",noindex"` // list of managers
 }
 
 type BugReporting struct {
@@ -103,7 +106,10 @@ type Crash struct {
 	ReproOpts   []byte    `datastore:",noindex"`
 	ReproSyz    int64     // reference to ReproSyz text entity
 	ReproC      int64     // reference to ReproC text entity
-	ReportLen   int
+	// Custom crash priority for reporting (greater values are higher priority).
+	// For example, a crash in mainline kernel has higher priority than a crash in a side branch.
+	// For historical reasons this is called ReportLen.
+	ReportLen int
 }
 
 // ReportingState holds dynamic info associated with reporting.
@@ -315,6 +321,15 @@ func bugReportingHash(bugHash, reporting string) string {
 	return hash.String([]byte(fmt.Sprintf("%v-%v", bugHash, reporting)))[:hashLen]
 }
 
+func kernelRepoInfo(build *Build) KernelRepo {
+	repoID := build.KernelRepo + "/" + build.KernelBranch
+	info := config.KernelRepos[repoID]
+	if info.Alias == "" {
+		info.Alias = repoID
+	}
+	return info
+}
+
 func textLink(tag string, id int64) string {
 	if id == 0 {
 		return ""
diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml
index 1af746189..d0f14cfc8 100644
--- a/dashboard/app/index.yaml
+++ b/dashboard/app/index.yaml
@@ -43,6 +43,20 @@ indexes:
     direction: desc
   - name: ReproSyz
     direction: desc
+  - name: ReportLen
+    direction: desc
+  - name: Reported
+    direction: desc
+  - name: Time
+    direction: desc
+
+- kind: Crash
+  ancestor: yes
+  properties:
+  - name: ReproC
+    direction: desc
+  - name: ReproSyz
+    direction: desc
   - name: Reported
     direction: desc
   - name: ReportLen
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 892e8a906..a25c82610 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -225,26 +225,29 @@ func createBugReport(c context.Context, bug *Bug, crash *Crash, crashKey *datast
 	}
 
 	rep := &dashapi.BugReport{
-		Namespace:    bug.Namespace,
-		Config:       reportingConfig,
-		ID:           bugReporting.ID,
-		ExtID:        bugReporting.ExtID,
-		First:        bugReporting.Reported.IsZero(),
-		Title:        bug.displayTitle(),
-		Log:          crashLog,
-		Report:       report,
-		Maintainers:  crash.Maintainers,
-		OS:           build.OS,
-		Arch:         build.Arch,
-		VMArch:       build.VMArch,
-		CompilerID:   build.CompilerID,
-		KernelRepo:   build.KernelRepo,
-		KernelBranch: build.KernelBranch,
-		KernelCommit: build.KernelCommit,
-		KernelConfig: kernelConfig,
-		ReproC:       reproC,
-		ReproSyz:     reproSyz,
-		CrashID:      crashKey.IntID(),
+		Namespace:         bug.Namespace,
+		Config:            reportingConfig,
+		ID:                bugReporting.ID,
+		ExtID:             bugReporting.ExtID,
+		First:             bugReporting.Reported.IsZero(),
+		Title:             bug.displayTitle(),
+		Log:               crashLog,
+		Report:            report,
+		Maintainers:       crash.Maintainers,
+		OS:                build.OS,
+		Arch:              build.Arch,
+		VMArch:            build.VMArch,
+		CompilerID:        build.CompilerID,
+		KernelRepo:        build.KernelRepo,
+		KernelRepoAlias:   kernelRepoInfo(build).Alias,
+		KernelBranch:      build.KernelBranch,
+		KernelCommit:      build.KernelCommit,
+		KernelCommitTitle: build.KernelCommitTitle,
+		KernelCommitDate:  build.KernelCommitDate,
+		KernelConfig:      kernelConfig,
+		ReproC:            reproC,
+		ReproSyz:          reproSyz,
+		CrashID:           crashKey.IntID(),
 	}
 	if bugReporting.CC != "" {
 		rep.CC = strings.Split(bugReporting.CC, "|")
@@ -581,8 +584,8 @@ func queryCrashesForBug(c context.Context, bugKey *datastore.Key, limit int) (
 		Ancestor(bugKey).
 		Order("-ReproC").
 		Order("-ReproSyz").
-		Order("-Reported").
 		Order("-ReportLen").
+		Order("-Reported").
 		Order("-Time").
 		Limit(limit).
 		GetAll(c, &crashes)
diff --git a/dashboard/app/reporting_test.go b/dashboard/app/reporting_test.go
index bd743f33a..dbba2e626 100644
--- a/dashboard/app/reporting_test.go
+++ b/dashboard/app/reporting_test.go
@@ -46,20 +46,23 @@ func TestReportBug(t *testing.T) {
 		t.Fatalf("empty report ID")
 	}
 	want := &dashapi.BugReport{
-		Namespace:    "test1",
-		Config:       []byte(`{"Index":1}`),
-		ID:           rep.ID,
-		First:        true,
-		Title:        "title1",
-		Maintainers:  []string{"bar@foo.com", "foo@bar.com"},
-		CompilerID:   "compiler1",
-		KernelRepo:   "repo1",
-		KernelBranch: "branch1",
-		KernelCommit: "kernel_commit1",
-		KernelConfig: []byte("config1"),
-		Log:          []byte("log1"),
-		Report:       []byte("report1"),
-		CrashID:      rep.CrashID,
+		Namespace:         "test1",
+		Config:            []byte(`{"Index":1}`),
+		ID:                rep.ID,
+		First:             true,
+		Title:             "title1",
+		Maintainers:       []string{"bar@foo.com", "foo@bar.com"},
+		CompilerID:        "compiler1",
+		KernelRepo:        "repo1",
+		KernelRepoAlias:   "repo1/branch1",
+		KernelBranch:      "branch1",
+		KernelCommit:      "kernel_commit1",
+		KernelCommitTitle: build.KernelCommitTitle,
+		KernelCommitDate:  buildCommitDate,
+		KernelConfig:      []byte("config1"),
+		Log:               []byte("log1"),
+		Report:            []byte("report1"),
+		CrashID:           rep.CrashID,
 	}
 	c.expectEQ(rep, want)
 
@@ -214,20 +217,23 @@ func TestInvalidBug(t *testing.T) {
 		t.Fatalf("empty report ID")
 	}
 	want := &dashapi.BugReport{
-		Namespace:    "test1",
-		Config:       []byte(`{"Index":1}`),
-		ID:           rep.ID,
-		First:        true,
-		Title:        "title1 (2)",
-		CompilerID:   "compiler1",
-		KernelRepo:   "repo1",
-		KernelBranch: "branch1",
-		KernelCommit: "kernel_commit1",
-		KernelConfig: []byte("config1"),
-		Log:          []byte("log2"),
-		Report:       []byte("report2"),
-		ReproC:       []byte("int main() { return 1; }"),
-		CrashID:      rep.CrashID,
+		Namespace:         "test1",
+		Config:            []byte(`{"Index":1}`),
+		ID:                rep.ID,
+		First:             true,
+		Title:             "title1 (2)",
+		CompilerID:        "compiler1",
+		KernelRepo:        "repo1",
+		KernelRepoAlias:   "repo1/branch1",
+		KernelBranch:      "branch1",
+		KernelCommit:      "kernel_commit1",
+		KernelCommitTitle: build.KernelCommitTitle,
+		KernelCommitDate:  buildCommitDate,
+		KernelConfig:      []byte("config1"),
+		Log:               []byte("log2"),
+		Report:            []byte("report2"),
+		ReproC:            []byte("int main() { return 1; }"),
+		CrashID:           rep.CrashID,
 	}
 	c.expectEQ(rep, want)
 
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go
index e833fbfd2..c9b9ded15 100644
--- a/dashboard/dashapi/dashapi.go
+++ b/dashboard/dashapi/dashapi.go
@@ -35,19 +35,21 @@ func New(client, addr, key string) *Dashboard {
 
 // Build describes all aspects of a kernel build.
 type Build struct {
-	Manager         string
-	ID              string
-	OS              string
-	Arch            string
-	VMArch          string
-	SyzkallerCommit string
-	CompilerID      string
-	KernelRepo      string
-	KernelBranch    string
-	KernelCommit    string
-	KernelConfig    []byte
-	Commits         []string // see BuilderPoll
-	FixCommits      []FixCommit
+	Manager           string
+	ID                string
+	OS                string
+	Arch              string
+	VMArch            string
+	SyzkallerCommit   string
+	CompilerID        string
+	KernelRepo        string
+	KernelBranch      string
+	KernelCommit      string
+	KernelCommitTitle string
+	KernelCommitDate  time.Time
+	KernelConfig      []byte
+	Commits           []string // see BuilderPoll
+	FixCommits        []FixCommit
 }
 
 type FixCommit struct {
@@ -204,28 +206,31 @@ func (dash *Dashboard) LogError(name, msg string, args ...interface{}) {
 // BugReport describes a single bug.
 // Used by dashboard external reporting.
 type BugReport struct {
-	Namespace    string
-	Config       []byte
-	ID           string
-	JobID        string
-	ExtID        string // arbitrary reporting ID forwarded from BugUpdate.ExtID
-	First        bool   // Set for first report for this bug.
-	Title        string
-	Maintainers  []string
-	CC           []string // additional CC emails
-	OS           string
-	Arch         string
-	VMArch       string
-	CompilerID   string
-	KernelRepo   string
-	KernelBranch string
-	KernelCommit string
-	KernelConfig []byte
-	Log          []byte
-	Report       []byte
-	ReproC       []byte
-	ReproSyz     []byte
-	CrashID      int64 // returned back in BugUpdate
+	Namespace         string
+	Config            []byte
+	ID                string
+	JobID             string
+	ExtID             string // arbitrary reporting ID forwarded from BugUpdate.ExtID
+	First             bool   // Set for first report for this bug.
+	Title             string
+	Maintainers       []string
+	CC                []string // additional CC emails
+	OS                string
+	Arch              string
+	VMArch            string
+	CompilerID        string
+	KernelRepo        string
+	KernelRepoAlias   string
+	KernelBranch      string
+	KernelCommit      string
+	KernelCommitTitle string
+	KernelCommitDate  time.Time
+	KernelConfig      []byte
+	Log               []byte
+	Report            []byte
+	ReproC            []byte
+	ReproSyz          []byte
+	CrashID           int64 // returned back in BugUpdate
 
 	CrashTitle string // job execution crash title
 	Error      []byte // job execution error