From b727b13b371c02598242821ea230ed2e9f53e305 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Wed, 6 Nov 2024 17:18:44 +0100 Subject: dashboard/app: read lines coverage from spanner We currently merge bigquery data for every line coverage request. Let's read cached lines coverage data from spanner instead. It allows to get only 1 file version from git and skip the data merge step. --- dashboard/app/graphs.go | 16 ++++++----- pkg/cover/file.go | 30 +++++++++++--------- pkg/coveragedb/spanner.go | 59 ++++++++++++++++++++++++++++++++++++++++ tools/syz-cover/syz-cover.go | 65 ++++++++++++++++++++++++-------------------- 4 files changed, 121 insertions(+), 49 deletions(-) diff --git a/dashboard/app/graphs.go b/dashboard/app/graphs.go index c8b2843cb..b05636431 100644 --- a/dashboard/app/graphs.go +++ b/dashboard/app/graphs.go @@ -275,17 +275,19 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques if err != nil { return fmt.Errorf("coveragedb.MakeTimePeriod: %w", err) } - dateFrom, dateTo := tp.DatesFromTo() mainNsRepo, _ := nsConfig.mainRepoBranch() + hitCounts, err := coveragedb.ReadLinesHitCount(c, hdr.Namespace, targetCommit, kernelFilePath, tp) + if err != nil { + return fmt.Errorf("coveragedb.ReadLinesHitCount: %w", err) + } + content, err := cover.RendFileCoverage( - c, hdr.Namespace, mainNsRepo, - targetCommit, "", // merge all commits to targetCommit + mainNsRepo, + targetCommit, kernelFilePath, makeProxyURIProvider(nsConfig.Coverage.WebGitURI), - dateFrom, - dateTo, - cover.DefaultHTMLRenderConfig(), - ) + &covermerger.MergeResult{HitCounts: hitCounts}, + cover.DefaultHTMLRenderConfig()) if err != nil { return fmt.Errorf("cover.RendFileCoverage: %w", err) } diff --git a/pkg/cover/file.go b/pkg/cover/file.go index f7c679293..368607ffe 100644 --- a/pkg/cover/file.go +++ b/pkg/cover/file.go @@ -9,7 +9,7 @@ import ( "html" "strings" - "cloud.google.com/go/civil" + "github.com/google/syzkaller/pkg/coveragedb" "github.com/google/syzkaller/pkg/covermerger" ) @@ -40,14 +40,18 @@ func DefaultHTMLRenderConfig() *CoverageRenderConfig { } } -func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, filePath string, - proxy covermerger.FuncProxyURI, - fromDate, toDate civil.Date, renderConfig *CoverageRenderConfig) (string, error) { - fileContent, err := covermerger.GetFileVersion(filePath, repo, forCommit) +func RendFileCoverage(repo, forCommit, filePath string, proxy covermerger.FuncProxyURI, + mr *covermerger.MergeResult, renderConfig *CoverageRenderConfig) (string, error) { + repoCommit := covermerger.RepoCommit{Repo: repo, Commit: forCommit} + files, err := covermerger.MakeWebGit(proxy).GetFileVersions(filePath, repoCommit) if err != nil { - return "", fmt.Errorf("failed to GetFileVersion for file %s, commit %s from repo %s: %w", - filePath, forCommit, repo, err) + return "", fmt.Errorf("failed to GetFileVersions: %w", err) } + return rendResult(files[repoCommit], mr, renderConfig), nil +} + +func GetMergeResult(c context.Context, ns, repo, forCommit, sourceCommit, filePath string, + proxy covermerger.FuncProxyURI, tp coveragedb.TimePeriod) (*covermerger.MergeResult, error) { config := &covermerger.Config{ Jobs: 1, Base: covermerger.RepoCommit{ @@ -58,6 +62,7 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file StoreDetails: true, } + fromDate, toDate := tp.DatesFromTo() dbReader := covermerger.MakeBQCSVReader() if err := dbReader.InitNsRecords(c, ns, @@ -66,23 +71,22 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file fromDate, toDate, ); err != nil { - return "", fmt.Errorf("failed to dbReader.InitNsRecords: %w", err) + return nil, fmt.Errorf("failed to dbReader.InitNsRecords: %w", err) } defer dbReader.Close() csvReader, err := dbReader.Reader() if err != nil { - return "", fmt.Errorf("failed to dbReader.Reader: %w", err) + return nil, fmt.Errorf("failed to dbReader.Reader: %w", err) } mergeResult, err := covermerger.MergeCSVData(config, csvReader) if err != nil { - return "", fmt.Errorf("error merging coverage: %w", err) + return nil, fmt.Errorf("error merging coverage: %w", err) } if _, exist := mergeResult[filePath]; !exist { - return "", fmt.Errorf("no merge result for file %s(fileSize %d)", filePath, len(fileContent)) + return nil, fmt.Errorf("no merge result for file %s", filePath) } - - return rendResult(fileContent, mergeResult[filePath], renderConfig), nil + return mergeResult[filePath], nil } func rendResult(content string, coverage *covermerger.MergeResult, renderConfig *CoverageRenderConfig) string { diff --git a/pkg/coveragedb/spanner.go b/pkg/coveragedb/spanner.go index 0ff3a3197..54fa2e192 100644 --- a/pkg/coveragedb/spanner.go +++ b/pkg/coveragedb/spanner.go @@ -6,6 +6,7 @@ package coveragedb import ( "context" "fmt" + "os" "time" "cloud.google.com/go/civil" @@ -89,6 +90,64 @@ func SaveMergeResult(ctx context.Context, projectID string, covMap map[string]*C return nil } +type linesCoverage struct { + LinesInstrumented []int64 + HitCounts []int64 +} + +func linesCoverageStmt(ns, filepath, commit string, timePeriod TimePeriod) spanner.Statement { + return spanner.Statement{ + SQL: ` +select + linesinstrumented, + hitcounts +from merge_history + join files + on merge_history.session = files.session +where + namespace=$1 and dateto=$2 and duration=$3 and filepath=$4 and commit=$5`, + Params: map[string]interface{}{ + "p1": ns, + "p2": timePeriod.DateTo, + "p3": timePeriod.Days, + "p4": filepath, + "p5": commit, + }, + } +} + +func ReadLinesHitCount(ctx context.Context, ns, commit, file string, tp TimePeriod, +) (map[int]int, error) { + projectID := os.Getenv("GOOGLE_CLOUD_PROJECT") + client, err := NewClient(ctx, projectID) + if err != nil { + return nil, fmt.Errorf("spanner.NewClient: %w", err) + } + defer client.Close() + + stmt := linesCoverageStmt(ns, file, commit, tp) + iter := client.Single().Query(ctx, stmt) + defer iter.Stop() + + row, err := iter.Next() + if err == iterator.Done { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("iter.Next: %w", err) + } + var r linesCoverage + if err = row.ToStruct(&r); err != nil { + return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err) + } + + res := map[int]int{} + for i, instrLine := range r.LinesInstrumented { + res[int(instrLine)] = int(r.HitCounts[i]) + } + return res, nil +} + func historyMutation(session string, template *HistoryRecord, totalRows int64) *spanner.Mutation { historyInsert, err := spanner.InsertOrUpdateStruct("merge_history", &HistoryRecord{ Session: session, diff --git a/tools/syz-cover/syz-cover.go b/tools/syz-cover/syz-cover.go index b640ff63d..4b4270992 100644 --- a/tools/syz-cover/syz-cover.go +++ b/tools/syz-cover/syz-cover.go @@ -30,6 +30,7 @@ import ( "io" "os" "os/exec" + "slices" "strconv" "strings" "time" @@ -51,9 +52,8 @@ var ( "modules JSON info obtained from /modules (optional)") flagNsHeatmap = flag.String("heatmap", "", "generate namespace heatmap") flagNsHeatmapGroupBy = flag.String("group-by", "dir", "dir or subsystem") - flagDateFrom = flag.String("from", - civil.DateOf(time.Now().Add(-14*24*time.Hour)).String(), "heatmap date from(optional)") - flagDateTo = flag.String("to", + flagPeriod = flag.String("period", "day", "time period(day[default], month, quarter)") + flagDateTo = flag.String("to", civil.DateOf(time.Now()).String(), "heatmap date to(optional)") flagProjectID = flag.String("project", "syzkaller", "spanner db project name") flagForFile = flag.String("for-file", "", "[optional]show file coverage") @@ -70,33 +70,26 @@ var ( "there are missing coverage callbacks") ) -func parseDates() (civil.Date, civil.Date) { - dateFrom, errDateFrom := civil.ParseDate(*flagDateFrom) - if errDateFrom != nil { - tool.Failf("failed to parse date from: %v", errDateFrom.Error()) - } - dateTo, errDateTo := civil.ParseDate(*flagDateTo) - if errDateTo != nil { - tool.Failf("failed to parse date to: %v", errDateTo.Error()) - } - return dateFrom, dateTo -} - -func periodsFromDays(from, to civil.Date) []coveragedb.TimePeriod { - if to.Before(from) { - panic("toDay can't be less than fromDay") - } - res := []coveragedb.TimePeriod{{DateTo: from, Days: 1, Type: coveragedb.DayPeriod}} - for ; from.Before(to); from = from.AddDays(1) { - res = append(res, coveragedb.TimePeriod{DateTo: from, Days: 1, Type: coveragedb.DayPeriod}) +func dayPeriods(tp coveragedb.TimePeriod) []coveragedb.TimePeriod { + var res []coveragedb.TimePeriod + for i := 0; i < tp.Days; i++ { + res = append(res, coveragedb.TimePeriod{DateTo: tp.DateTo.AddDays(-i), Days: 1, Type: coveragedb.DayPeriod}) } + slices.Reverse(res) return res } func toolBuildNsHeatmap() { buf := new(bytes.Buffer) - periods := periodsFromDays(parseDates()) - var err error + dateTo, err := civil.ParseDate(*flagDateTo) + if err != nil { + tool.Fail(err) + } + tp, err := coveragedb.MakeTimePeriod(dateTo, *flagPeriod) + if err != nil { + tool.Fail(err) + } + periods := dayPeriods(tp) switch *flagNsHeatmapGroupBy { case "dir": if err = cover.DoDirHeatMap(buf, *flagProjectID, *flagNsHeatmap, periods); err != nil { @@ -115,19 +108,33 @@ func toolBuildNsHeatmap() { } func toolFileCover() { - dateFrom, dateTo := parseDates() + dateTo, err := civil.ParseDate(*flagDateTo) + if err != nil { + tool.Failf("failed to parse date from: %v", err) + } + tp, err := coveragedb.MakeTimePeriod(dateTo, *flagPeriod) + if err != nil { + tool.Fail(err) + } config := cover.DefaultTextRenderConfig() config.ShowLineSourceExplanation = *flagDebug - details, err := cover.RendFileCoverage( - context.Background(), + mr, err := cover.GetMergeResult(context.Background(), *flagNamespace, *flagRepo, *flagCommit, *flagSourceCommit, *flagForFile, + nil, tp) + if err != nil { + tool.Fail(err) + } + + details, err := cover.RendFileCoverage( + *flagRepo, + *flagCommit, + *flagForFile, nil, // no proxy - get files directly from WebGits - dateFrom, - dateTo, + mr, config, ) if err != nil { -- cgit mrf-deployment