dashboard/app: add support for Kcidb

Add support for publishing bugs to Kcidb.

Update #2144

6 files changed, 126 insertions, 1 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 4da3f2ac0..73a802d61 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -1014,6 +1014,10 @@ func apiLoadBug(c context.Context, ns string, r *http.Request, payload []byte) (
 	if bug.sanitizeAccess(AccessPublic) > AccessPublic {
 		return nil, nil
 	}
+	return loadBugReport(c, bug)
+}
+
+func loadBugReport(c context.Context, bug *Bug) (*dashapi.BugReport, error) {
 	crash, crashKey, err := findCrashForBug(c, bug)
 	if err != nil {
 		return nil, err
diff --git a/dashboard/app/app.yaml b/dashboard/app/app.yaml
index 353553d27..0451503dd 100644
--- a/dashboard/app/app.yaml
+++ b/dashboard/app/app.yaml
@@ -19,7 +19,7 @@ handlers:
 - url: /static
   static_dir: static
   secure: always
-- url: /(admin|email_poll)
+- url: /(admin|email_poll|kcidb_poll)
   script: auto
   login: admin
   secure: always
diff --git a/dashboard/app/config.go b/dashboard/app/config.go
index 312ae7997..fa80cca12 100644
--- a/dashboard/app/config.go
+++ b/dashboard/app/config.go
@@ -4,6 +4,7 @@
 package main
 
 import (
+	"bytes"
 	"encoding/json"
 	"fmt"
 	"net/mail"
@@ -83,6 +84,8 @@ type Config struct {
 	// Other repos are secondary repos, they may be tested or not.
 	// If not tested they are used to poll for fixing commits.
 	Repos []KernelRepo
+	// If not nil, bugs in this namespace will be exported to the specified Kcidb.
+	Kcidb *KcidbConfig
 }
 
 // ObsoletingConfig describes how bugs without reproducer should be obsoleted.
@@ -167,6 +170,17 @@ type KernelRepo struct {
 	BuildMaintainers []string
 }
 
+type KcidbConfig struct {
+	// Origin is how this system identified in Kcidb, e.g. "syzbot_foobar".
+	Origin string
+	// Project is Kcidb GCE project name, e.g. "kernelci-production".
+	Project string
+	// Topic is pubsub topic to publish messages to, e.g. "playground_kernelci_new".
+	Topic string
+	// Credentials is Google application credentials file contents to use for authorization.
+	Credentials []byte
+}
+
 var (
 	namespaceNameRe = regexp.MustCompile("^[a-zA-Z0-9-_.]{4,32}$")
 	clientNameRe    = regexp.MustCompile("^[a-zA-Z0-9-_.]{4,100}$")
@@ -216,6 +230,7 @@ func installConfig(cfg *GlobalConfig) {
 	initEmailReporting()
 	initHTTPHandlers()
 	initAPIHandlers()
+	initKcidb()
 }
 
 func checkConfig(cfg *GlobalConfig) {
@@ -299,6 +314,9 @@ func checkNamespace(ns string, cfg *Config, namespaces, clientNames map[string]b
 			return true
 		}
 	}
+	if cfg.Kcidb != nil {
+		checkKcidb(ns, cfg.Kcidb)
+	}
 	checkKernelRepos(ns, cfg)
 	checkNamespaceReporting(ns, cfg)
 }
@@ -396,6 +414,21 @@ func checkManager(ns, name string, mgr ConfigManager) {
 	}
 }
 
+func checkKcidb(ns string, kcidb *KcidbConfig) {
+	if !regexp.MustCompile("^[a-z0-9_]+$").MatchString(kcidb.Origin) {
+		panic(fmt.Sprintf("%v: bad Kcidb origin %q", ns, kcidb.Origin))
+	}
+	if kcidb.Project == "" {
+		panic(fmt.Sprintf("%v: empty Kcidb project", ns))
+	}
+	if kcidb.Topic == "" {
+		panic(fmt.Sprintf("%v: empty Kcidb topic", ns))
+	}
+	if !bytes.Contains(kcidb.Credentials, []byte("private_key")) {
+		panic(fmt.Sprintf("%v: empty Kcidb credentials", ns))
+	}
+}
+
 func checkConfigAccessLevel(current *AccessLevel, parent AccessLevel, what string) {
 	verifyAccessLevel(parent)
 	if *current == 0 {
diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml
index e98bcda69..1912abeec 100644
--- a/dashboard/app/cron.yaml
+++ b/dashboard/app/cron.yaml
@@ -4,6 +4,8 @@
 cron:
 - url: /email_poll
   schedule: every 1 minutes
+- url: /kcidb_poll
+  schedule: every 5 minutes
 - url: /_ah/datastore_admin/backup.create?name=backup&filesystem=gs&gs_bucket_name=syzkaller-backups&kind=Bug&kind=Build&kind=Crash&kind=CrashLog&kind=CrashReport&kind=Error&kind=Job&kind=KernelConfig&kind=Manager&kind=ManagerStats&kind=Patch&kind=ReportingState&kind=ReproC&kind=ReproSyz
   schedule: every monday 00:00
   target: ah-builtin-python-bundle
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index d00d90f11..252e3254f 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -95,6 +95,9 @@ type Bug struct {
 	HappenedOn     []string // list of managers
 	PatchedOn      []string `datastore:",noindex"` // list of managers
 	UNCC           []string // don't CC these emails on this bug
+	// 0 means we did not report this bug to Kcidb, 1 - reported.
+	// The idea is to extend this to a bitmask later: if reported a reproducer, if we reported bisection, etc.
+	KcidbStatus int64
 }
 
 type Commit struct {
diff --git a/dashboard/app/kcidb.go b/dashboard/app/kcidb.go
new file mode 100644
index 000000000..f19a78b9a
--- /dev/null
+++ b/dashboard/app/kcidb.go
@@ -0,0 +1,83 @@
+// Copyright 2020 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/google/syzkaller/pkg/kcidb"
+	"golang.org/x/net/context"
+	"google.golang.org/appengine"
+	db "google.golang.org/appengine/datastore"
+	"google.golang.org/appengine/log"
+)
+
+func initKcidb() {
+	http.HandleFunc("/kcidb_poll", handleKcidbPoll)
+}
+
+func handleKcidbPoll(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	for ns, cfg := range config.Namespaces {
+		if cfg.Kcidb == nil {
+			continue
+		}
+		if err := handleKcidbNamespce(c, ns, cfg.Kcidb); err != nil {
+			log.Errorf(c, "kcidb: %v failed: %v", ns, err)
+		}
+	}
+}
+
+func handleKcidbNamespce(c context.Context, ns string, cfg *KcidbConfig) error {
+	client, err := kcidb.NewClient(c, cfg.Origin, cfg.Project, cfg.Topic, cfg.Credentials)
+	if err != nil {
+		return err
+	}
+	defer client.Close()
+
+	filter := func(query *db.Query) *db.Query {
+		return query.Filter("Namespace=", ns).
+			Filter("Status=", BugStatusOpen)
+	}
+	reported := 0
+	return foreachBug(c, filter, func(bug *Bug, bugKey *db.Key) error {
+		if reported >= 30 ||
+			bug.KcidbStatus != 0 ||
+			bug.sanitizeAccess(AccessPublic) > AccessPublic ||
+			bug.Reporting[len(bug.Reporting)-1].Reported.IsZero() ||
+			timeSince(c, bug.LastTime) > 7*24*time.Hour {
+			return nil
+		}
+		reported++
+		return publishKcidbBug(c, client, bug, bugKey)
+	})
+}
+
+func publishKcidbBug(c context.Context, client *kcidb.Client, bug *Bug, bugKey *db.Key) error {
+	rep, err := loadBugReport(c, bug)
+	if err != nil {
+		return err
+	}
+	if err := client.Publish(rep); err != nil {
+		return err
+	}
+	tx := func(c context.Context) error {
+		bug := new(Bug)
+		if err := db.Get(c, bugKey, bug); err != nil {
+			return err
+		}
+		bug.KcidbStatus = 1
+		if _, err := db.Put(c, bugKey, bug); err != nil {
+			return fmt.Errorf("failed to put bug: %v", err)
+		}
+		return nil
+	}
+	if err := db.RunInTransaction(c, tx, nil); err != nil {
+		return err
+	}
+	log.Infof(c, "published bug to kcidb: %v:%v '%v'", bug.Namespace, bugKey.StringID(), bug.displayTitle())
+	return nil
+}