From 82874357f78ea9aacf31d1da403372f573f45e2d Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Mon, 17 Jun 2024 11:18:56 +0200 Subject: dashboard/app: add coverage histogram --- dashboard/app/entities_spanner.go | 74 +++++++++++++++++++++++++++++++++++++++ dashboard/app/graphs.go | 40 +++++++++++++++++++++ dashboard/app/main.go | 1 + dashboard/app/templates.html | 4 ++- go.mod | 2 +- 5 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 dashboard/app/entities_spanner.go diff --git a/dashboard/app/entities_spanner.go b/dashboard/app/entities_spanner.go new file mode 100644 index 000000000..7f2c47b97 --- /dev/null +++ b/dashboard/app/entities_spanner.go @@ -0,0 +1,74 @@ +// Copyright 2024 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "context" + "fmt" + "os" + "time" + + "cloud.google.com/go/civil" + "cloud.google.com/go/spanner" + "google.golang.org/api/iterator" +) + +// This file contains definitions of entities stored in spanner. + +type CoverageHistory struct { + instrumented map[string]int64 + covered map[string]int64 +} + +// MergedCoverage uses dates, not time. +func MergedCoverage(ctx context.Context, ns string, fromDate, toDate civil.Date) (*CoverageHistory, error) { + projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") + client, err := spanner.NewClient(ctx, "projects/"+projectID+"/instances/syzbot/databases/coverage") + if err != nil { + panic(fmt.Sprintf("spanner.NewClient() failed: %s", err.Error())) + } + defer client.Close() + + stmt := spanner.Statement{ + SQL: `select + dateto targetdate, + cast(sum(instrumented) as INTEGER) as instrumented, + cast(sum(covered) as INTEGER) as covered + from "files" + where namespace=$1 and dateto>=$2 and dateto<=$3 + group by targetdate`, + Params: map[string]interface{}{ + "p1": ns, + "p2": fromDate, + "p3": toDate, + }, + } + + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() + res := &CoverageHistory{ + instrumented: map[string]int64{}, + covered: map[string]int64{}, + } + for { + row, err := iter.Next() + if err == iterator.Done { + break + } + if err != nil { + return nil, fmt.Errorf("failed to iter.Next() spanner DB: %w", err) + } + var r struct { + Targetdate time.Time + Instrumented int64 + Covered int64 + } + if err = row.ToStruct(&r); err != nil { + return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err) + } + res.instrumented[r.Targetdate.Format(time.DateOnly)] = r.Instrumented + res.covered[r.Targetdate.Format(time.DateOnly)] = r.Covered + } + return res, nil +} diff --git a/dashboard/app/graphs.go b/dashboard/app/graphs.go index f2c991db7..c363417d0 100644 --- a/dashboard/app/graphs.go +++ b/dashboard/app/graphs.go @@ -13,6 +13,7 @@ import ( "strconv" "time" + "cloud.google.com/go/civil" db "google.golang.org/appengine/v2/datastore" ) @@ -187,6 +188,45 @@ func handleFoundBugsGraph(c context.Context, w http.ResponseWriter, r *http.Requ return serveTemplate(w, "graph_histogram.html", data) } +func handleCoverageGraph(c context.Context, w http.ResponseWriter, r *http.Request) error { + hdr, err := commonHeader(c, r, w, "") + if err != nil { + return err + } + yesterday := civil.DateOf(time.Now().Add(-1 * 24 * time.Hour)) + monthAgo := yesterday.AddDays(-31) + hist, err := MergedCoverage(c, hdr.Namespace, monthAgo, yesterday) + if err != nil { + return err + } + dates := []string{} + for i := 31; i >= 0; i-- { + dates = append(dates, yesterday.AddDays(-i).String()) + } + cols := []uiGraphColumn{} + for _, date := range dates { + if _, ok := hist.covered[date]; !ok || hist.instrumented[date] == 0 { + cols = append(cols, uiGraphColumn{Hint: date, Vals: []uiGraphValue{{IsNull: true}}}) + } else { + cols = append(cols, uiGraphColumn{ + Vals: []uiGraphValue{{Val: float32(hist.covered[date]) / float32(hist.instrumented[date])}}, + Hint: date, + }) + } + } + data := &uiHistogramPage{ + Title: hdr.Namespace + " coverage", + Header: hdr, + Graph: &uiGraph{ + Headers: []uiGraphHeader{ + {Name: "Total", Color: "Red"}, + }, + Columns: cols, + }, + } + return serveTemplate(w, "graph_histogram.html", data) +} + func loadGraphBugs(c context.Context, ns string) ([]*Bug, error) { filter := func(query *db.Query) *db.Query { return query.Filter("Namespace=", ns) diff --git a/dashboard/app/main.go b/dashboard/app/main.go index 4e26fb86b..42a0a9da3 100644 --- a/dashboard/app/main.go +++ b/dashboard/app/main.go @@ -64,6 +64,7 @@ func initHTTPHandlers() { http.Handle("/"+ns+"/graph/fuzzing", handlerWrapper(handleGraphFuzzing)) http.Handle("/"+ns+"/graph/crashes", handlerWrapper(handleGraphCrashes)) http.Handle("/"+ns+"/graph/found-bugs", handlerWrapper(handleFoundBugsGraph)) + http.Handle("/"+ns+"/graph/coverage", handlerWrapper(handleCoverageGraph)) http.Handle("/"+ns+"/repos", handlerWrapper(handleRepos)) http.Handle("/"+ns+"/bug-summaries", handlerWrapper(handleBugSummaries)) http.Handle("/"+ns+"/subsystems", handlerWrapper(handleSubsystemsList)) diff --git a/dashboard/app/templates.html b/dashboard/app/templates.html index 4cfd5fb37..ffd7566ec 100644 --- a/dashboard/app/templates.html +++ b/dashboard/app/templates.html @@ -83,7 +83,9 @@ Use of this source code is governed by Apache 2 LICENSE that can be found in the 📈 Fuzzing - 📈 Crashes + 📈 Crashes + + 📈 Coverage {{if .ContactEmail}} diff --git a/go.mod b/go.mod index 8e2059e61..3077de4a9 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/google/syzkaller go 1.21 require ( + cloud.google.com/go v0.114.0 cloud.google.com/go/logging v1.10.0 cloud.google.com/go/profiler v0.4.0 cloud.google.com/go/pubsub v1.38.0 @@ -37,7 +38,6 @@ require ( require ( 4d63.com/gocheckcompilerdirectives v1.2.1 // indirect 4d63.com/gochecknoglobals v0.2.1 // indirect - cloud.google.com/go v0.114.0 // indirect cloud.google.com/go/ai v0.5.0 // indirect cloud.google.com/go/auth v0.4.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect -- cgit mrf-deployment