From a9767fb2a6393444e871a02e79a14ccfa2aef52b Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 29 Sep 2020 15:35:44 +0200 Subject: dashboard/app: add support for Kcidb Add support for publishing bugs to Kcidb. Update #2144 --- dashboard/app/api.go | 4 +++ dashboard/app/app.yaml | 2 +- dashboard/app/config.go | 33 +++++++++++++++++++ dashboard/app/cron.yaml | 2 ++ dashboard/app/entities.go | 3 ++ dashboard/app/kcidb.go | 83 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 dashboard/app/kcidb.go diff --git a/dashboard/app/api.go b/dashboard/app/api.go index 4da3f2ac0..73a802d61 100644 --- a/dashboard/app/api.go +++ b/dashboard/app/api.go @@ -1014,6 +1014,10 @@ func apiLoadBug(c context.Context, ns string, r *http.Request, payload []byte) ( if bug.sanitizeAccess(AccessPublic) > AccessPublic { return nil, nil } + return loadBugReport(c, bug) +} + +func loadBugReport(c context.Context, bug *Bug) (*dashapi.BugReport, error) { crash, crashKey, err := findCrashForBug(c, bug) if err != nil { return nil, err diff --git a/dashboard/app/app.yaml b/dashboard/app/app.yaml index 353553d27..0451503dd 100644 --- a/dashboard/app/app.yaml +++ b/dashboard/app/app.yaml @@ -19,7 +19,7 @@ handlers: - url: /static static_dir: static secure: always -- url: /(admin|email_poll) +- url: /(admin|email_poll|kcidb_poll) script: auto login: admin secure: always diff --git a/dashboard/app/config.go b/dashboard/app/config.go index 312ae7997..fa80cca12 100644 --- a/dashboard/app/config.go +++ b/dashboard/app/config.go @@ -4,6 +4,7 @@ package main import ( + "bytes" "encoding/json" "fmt" "net/mail" @@ -83,6 +84,8 @@ type Config struct { // Other repos are secondary repos, they may be tested or not. // If not tested they are used to poll for fixing commits. Repos []KernelRepo + // If not nil, bugs in this namespace will be exported to the specified Kcidb. + Kcidb *KcidbConfig } // ObsoletingConfig describes how bugs without reproducer should be obsoleted. @@ -167,6 +170,17 @@ type KernelRepo struct { BuildMaintainers []string } +type KcidbConfig struct { + // Origin is how this system identified in Kcidb, e.g. "syzbot_foobar". + Origin string + // Project is Kcidb GCE project name, e.g. "kernelci-production". + Project string + // Topic is pubsub topic to publish messages to, e.g. "playground_kernelci_new". + Topic string + // Credentials is Google application credentials file contents to use for authorization. + Credentials []byte +} + var ( namespaceNameRe = regexp.MustCompile("^[a-zA-Z0-9-_.]{4,32}$") clientNameRe = regexp.MustCompile("^[a-zA-Z0-9-_.]{4,100}$") @@ -216,6 +230,7 @@ func installConfig(cfg *GlobalConfig) { initEmailReporting() initHTTPHandlers() initAPIHandlers() + initKcidb() } func checkConfig(cfg *GlobalConfig) { @@ -299,6 +314,9 @@ func checkNamespace(ns string, cfg *Config, namespaces, clientNames map[string]b return true } } + if cfg.Kcidb != nil { + checkKcidb(ns, cfg.Kcidb) + } checkKernelRepos(ns, cfg) checkNamespaceReporting(ns, cfg) } @@ -396,6 +414,21 @@ func checkManager(ns, name string, mgr ConfigManager) { } } +func checkKcidb(ns string, kcidb *KcidbConfig) { + if !regexp.MustCompile("^[a-z0-9_]+$").MatchString(kcidb.Origin) { + panic(fmt.Sprintf("%v: bad Kcidb origin %q", ns, kcidb.Origin)) + } + if kcidb.Project == "" { + panic(fmt.Sprintf("%v: empty Kcidb project", ns)) + } + if kcidb.Topic == "" { + panic(fmt.Sprintf("%v: empty Kcidb topic", ns)) + } + if !bytes.Contains(kcidb.Credentials, []byte("private_key")) { + panic(fmt.Sprintf("%v: empty Kcidb credentials", ns)) + } +} + func checkConfigAccessLevel(current *AccessLevel, parent AccessLevel, what string) { verifyAccessLevel(parent) if *current == 0 { diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml index e98bcda69..1912abeec 100644 --- a/dashboard/app/cron.yaml +++ b/dashboard/app/cron.yaml @@ -4,6 +4,8 @@ cron: - url: /email_poll schedule: every 1 minutes +- url: /kcidb_poll + schedule: every 5 minutes - url: /_ah/datastore_admin/backup.create?name=backup&filesystem=gs&gs_bucket_name=syzkaller-backups&kind=Bug&kind=Build&kind=Crash&kind=CrashLog&kind=CrashReport&kind=Error&kind=Job&kind=KernelConfig&kind=Manager&kind=ManagerStats&kind=Patch&kind=ReportingState&kind=ReproC&kind=ReproSyz schedule: every monday 00:00 target: ah-builtin-python-bundle diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go index d00d90f11..252e3254f 100644 --- a/dashboard/app/entities.go +++ b/dashboard/app/entities.go @@ -95,6 +95,9 @@ type Bug struct { HappenedOn []string // list of managers PatchedOn []string `datastore:",noindex"` // list of managers UNCC []string // don't CC these emails on this bug + // 0 means we did not report this bug to Kcidb, 1 - reported. + // The idea is to extend this to a bitmask later: if reported a reproducer, if we reported bisection, etc. + KcidbStatus int64 } type Commit struct { diff --git a/dashboard/app/kcidb.go b/dashboard/app/kcidb.go new file mode 100644 index 000000000..f19a78b9a --- /dev/null +++ b/dashboard/app/kcidb.go @@ -0,0 +1,83 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "net/http" + "time" + + "github.com/google/syzkaller/pkg/kcidb" + "golang.org/x/net/context" + "google.golang.org/appengine" + db "google.golang.org/appengine/datastore" + "google.golang.org/appengine/log" +) + +func initKcidb() { + http.HandleFunc("/kcidb_poll", handleKcidbPoll) +} + +func handleKcidbPoll(w http.ResponseWriter, r *http.Request) { + c := appengine.NewContext(r) + for ns, cfg := range config.Namespaces { + if cfg.Kcidb == nil { + continue + } + if err := handleKcidbNamespce(c, ns, cfg.Kcidb); err != nil { + log.Errorf(c, "kcidb: %v failed: %v", ns, err) + } + } +} + +func handleKcidbNamespce(c context.Context, ns string, cfg *KcidbConfig) error { + client, err := kcidb.NewClient(c, cfg.Origin, cfg.Project, cfg.Topic, cfg.Credentials) + if err != nil { + return err + } + defer client.Close() + + filter := func(query *db.Query) *db.Query { + return query.Filter("Namespace=", ns). + Filter("Status=", BugStatusOpen) + } + reported := 0 + return foreachBug(c, filter, func(bug *Bug, bugKey *db.Key) error { + if reported >= 30 || + bug.KcidbStatus != 0 || + bug.sanitizeAccess(AccessPublic) > AccessPublic || + bug.Reporting[len(bug.Reporting)-1].Reported.IsZero() || + timeSince(c, bug.LastTime) > 7*24*time.Hour { + return nil + } + reported++ + return publishKcidbBug(c, client, bug, bugKey) + }) +} + +func publishKcidbBug(c context.Context, client *kcidb.Client, bug *Bug, bugKey *db.Key) error { + rep, err := loadBugReport(c, bug) + if err != nil { + return err + } + if err := client.Publish(rep); err != nil { + return err + } + tx := func(c context.Context) error { + bug := new(Bug) + if err := db.Get(c, bugKey, bug); err != nil { + return err + } + bug.KcidbStatus = 1 + if _, err := db.Put(c, bugKey, bug); err != nil { + return fmt.Errorf("failed to put bug: %v", err) + } + return nil + } + if err := db.RunInTransaction(c, tx, nil); err != nil { + return err + } + log.Infof(c, "published bug to kcidb: %v:%v '%v'", bug.Namespace, bugKey.StringID(), bug.displayTitle()) + return nil +} -- cgit mrf-deployment