From fd3bba535d0200374dad3bd872650a4ceb075cf2 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 30 Jun 2020 20:01:01 +0200 Subject: dashboard/app: cache per-namespace bug stats We used to show number of fixed bugs at the top of the main page. However, now with the button nagivation, "fixed" is shown on every page. Fetching and processing all bugs on every page would be unwise. Cache these stats in memcache. It will be useful to show more stats in future. --- dashboard/app/cache.go | 70 ++++++++++++++++++++++++++++++++++++++++++++ dashboard/app/handler.go | 6 ++++ dashboard/app/main.go | 4 --- dashboard/app/reporting.go | 20 +++++++++---- dashboard/app/templates.html | 22 +++++++------- 5 files changed, 102 insertions(+), 20 deletions(-) create mode 100644 dashboard/app/cache.go diff --git a/dashboard/app/cache.go b/dashboard/app/cache.go new file mode 100644 index 000000000..574324518 --- /dev/null +++ b/dashboard/app/cache.go @@ -0,0 +1,70 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "net/http" + "time" + + "golang.org/x/net/context" + "google.golang.org/appengine/memcache" +) + +type Cached struct { + Open int + Fixed int + Invalid int +} + +func CacheGet(c context.Context, r *http.Request, ns string) (*Cached, error) { + accessLevel := accessLevel(c, r) + key := fmt.Sprintf("%v-%v", ns, accessLevel) + v := new(Cached) + _, err := memcache.Gob.Get(c, key, v) + if err != nil && err != memcache.ErrCacheMiss { + return nil, err + } + if err == nil { + return v, nil + } + if v, err = buildCached(c, ns, accessLevel); err != nil { + return nil, err + } + item := &memcache.Item{ + Key: key, + Object: v, + Expiration: time.Hour, + } + if err := memcache.Gob.Set(c, item); err != nil { + return nil, err + } + return v, nil +} + +func buildCached(c context.Context, ns string, accessLevel AccessLevel) (*Cached, error) { + v := &Cached{} + bugs, _, err := loadNamespaceBugs(c, ns) + if err != nil { + return nil, err + } + for _, bug := range bugs { + switch bug.Status { + case BugStatusOpen: + if accessLevel < bug.sanitizeAccess(accessLevel) { + continue + } + if len(bug.Commits) == 0 { + v.Open++ + } else { + v.Fixed++ + } + case BugStatusFixed: + v.Fixed++ + case BugStatusInvalid: + v.Invalid++ + } + } + return v, nil +} diff --git a/dashboard/app/handler.go b/dashboard/app/handler.go index ed55831d8..2daa203d1 100644 --- a/dashboard/app/handler.go +++ b/dashboard/app/handler.go @@ -97,6 +97,7 @@ type uiHeader struct { AnalyticsTrackingID string Subpage string Namespace string + Cached *Cached Namespaces []uiNamespace } @@ -171,6 +172,11 @@ func commonHeader(c context.Context, r *http.Request, w http.ResponseWriter, ns h.Namespace = ns cookie.Namespace = ns encodeCookie(w, cookie) + cached, err := CacheGet(c, r, ns) + if err != nil { + return nil, err + } + h.Cached = cached } return h, nil } diff --git a/dashboard/app/main.go b/dashboard/app/main.go index 270a1750b..9c6144082 100644 --- a/dashboard/app/main.go +++ b/dashboard/app/main.go @@ -240,7 +240,6 @@ func handleFixed(c context.Context, w http.ResponseWriter, r *http.Request) erro return handleTerminalBugList(c, w, r, &TerminalBug{ Status: BugStatusFixed, Subpage: "/fixed", - Caption: "fixed", ShowPatch: true, }) } @@ -249,7 +248,6 @@ func handleInvalid(c context.Context, w http.ResponseWriter, r *http.Request) er return handleTerminalBugList(c, w, r, &TerminalBug{ Status: BugStatusInvalid, Subpage: "/invalid", - Caption: "invalid", ShowPatch: false, }) } @@ -257,7 +255,6 @@ func handleInvalid(c context.Context, w http.ResponseWriter, r *http.Request) er type TerminalBug struct { Status int Subpage string - Caption string ShowPatch bool } @@ -668,7 +665,6 @@ func fetchTerminalBugs(c context.Context, accessLevel AccessLevel, } res := &uiBugGroup{ Now: timeNow(c), - Caption: typ.Caption, ShowPatch: typ.ShowPatch, Namespace: ns, } diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go index da3de2232..23703c78b 100644 --- a/dashboard/app/reporting.go +++ b/dashboard/app/reporting.go @@ -44,9 +44,7 @@ func reportingPollBugs(c context.Context, typ string) []*dashapi.BugReport { log.Errorf(c, "%v", err) return nil } - bugs, _, err := loadAllBugs(c, func(query *db.Query) *db.Query { - return query.Filter("Status<", BugStatusFixed) - }) + bugs, _, err := loadOpenBugs(c) if err != nil { log.Errorf(c, "%v", err) return nil @@ -150,9 +148,7 @@ func needReport(c context.Context, typ string, state *ReportingState, bug *Bug) } func reportingPollNotifications(c context.Context, typ string) []*dashapi.BugNotification { - bugs, _, err := loadAllBugs(c, func(query *db.Query) *db.Query { - return query.Filter("Status<", BugStatusFixed) - }) + bugs, _, err := loadOpenBugs(c) if err != nil { log.Errorf(c, "%v", err) return nil @@ -549,6 +545,18 @@ func loadAllBugs(c context.Context, filter func(*db.Query) *db.Query) ([]*Bug, [ return bugs, keys, nil } +func loadNamespaceBugs(c context.Context, ns string) ([]*Bug, []*db.Key, error) { + return loadAllBugs(c, func(query *db.Query) *db.Query { + return query.Filter("Namespace=", ns) + }) +} + +func loadOpenBugs(c context.Context) ([]*Bug, []*db.Key, error) { + return loadAllBugs(c, func(query *db.Query) *db.Query { + return query.Filter("Status<", BugStatusFixed) + }) +} + func foreachBug(c context.Context, filter func(*db.Query) *db.Query, fn func(bug *Bug, key *db.Key) error) error { const batchSize = 1000 var cursor *db.Cursor diff --git a/dashboard/app/templates.html b/dashboard/app/templates.html index 15f3f3d53..5743368a1 100644 --- a/dashboard/app/templates.html +++ b/dashboard/app/templates.html @@ -58,11 +58,11 @@ Use of this source code is governed by Apache 2 LICENSE that can be found in the - 🐞 Open + 🐞 Open [{{$.Cached.Open}}] - 🐞 Fixed + 🐞 Fixed [{{$.Cached.Fixed}}] - 🐞 Invalid + 🐞 Invalid [{{$.Cached.Invalid}}] @@ -76,14 +76,16 @@ Use of this source code is governed by Apache 2 LICENSE that can be found in the {{if .}} {{if .Bugs}} - {{if $.Fragment}} - {{end}} - {{$.Caption}} ({{len $.Bugs}}): - {{if $.Fragment}}{{end}} - {{if $.ShowNamespace}} -- cgit mrf-deployment
- {{else}} - + {{if $.Caption}} + {{if $.Fragment}} + + {{else}} + + {{end}} + {{$.Caption}} ({{len $.Bugs}}): + {{if $.Fragment}}{{end}} +