dashboard/app: read lines coverage from spanner

We currently merge bigquery data for every line coverage request.
Let's read cached lines coverage data from spanner instead.
It allows to get only 1 file version from git and skip the data merge step.

4 files changed, 121 insertions, 49 deletions

diff --git a/dashboard/app/graphs.go b/dashboard/app/graphs.go
index c8b2843cb..b05636431 100644
--- a/dashboard/app/graphs.go
+++ b/dashboard/app/graphs.go
@@ -275,17 +275,19 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques
 	if err != nil {
 		return fmt.Errorf("coveragedb.MakeTimePeriod: %w", err)
 	}
-	dateFrom, dateTo := tp.DatesFromTo()
 	mainNsRepo, _ := nsConfig.mainRepoBranch()
+	hitCounts, err := coveragedb.ReadLinesHitCount(c, hdr.Namespace, targetCommit, kernelFilePath, tp)
+	if err != nil {
+		return fmt.Errorf("coveragedb.ReadLinesHitCount: %w", err)
+	}
+
 	content, err := cover.RendFileCoverage(
-		c, hdr.Namespace, mainNsRepo,
-		targetCommit, "", // merge all commits to targetCommit
+		mainNsRepo,
+		targetCommit,
 		kernelFilePath,
 		makeProxyURIProvider(nsConfig.Coverage.WebGitURI),
-		dateFrom,
-		dateTo,
-		cover.DefaultHTMLRenderConfig(),
-	)
+		&covermerger.MergeResult{HitCounts: hitCounts},
+		cover.DefaultHTMLRenderConfig())
 	if err != nil {
 		return fmt.Errorf("cover.RendFileCoverage: %w", err)
 	}
diff --git a/pkg/cover/file.go b/pkg/cover/file.go
index f7c679293..368607ffe 100644
--- a/pkg/cover/file.go
+++ b/pkg/cover/file.go
@@ -9,7 +9,7 @@ import (
 	"html"
 	"strings"
 
-	"cloud.google.com/go/civil"
+	"github.com/google/syzkaller/pkg/coveragedb"
 	"github.com/google/syzkaller/pkg/covermerger"
 )
 
@@ -40,14 +40,18 @@ func DefaultHTMLRenderConfig() *CoverageRenderConfig {
 	}
 }
 
-func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, filePath string,
-	proxy covermerger.FuncProxyURI,
-	fromDate, toDate civil.Date, renderConfig *CoverageRenderConfig) (string, error) {
-	fileContent, err := covermerger.GetFileVersion(filePath, repo, forCommit)
+func RendFileCoverage(repo, forCommit, filePath string, proxy covermerger.FuncProxyURI,
+	mr *covermerger.MergeResult, renderConfig *CoverageRenderConfig) (string, error) {
+	repoCommit := covermerger.RepoCommit{Repo: repo, Commit: forCommit}
+	files, err := covermerger.MakeWebGit(proxy).GetFileVersions(filePath, repoCommit)
 	if err != nil {
-		return "", fmt.Errorf("failed to GetFileVersion for file %s, commit %s from repo %s: %w",
-			filePath, forCommit, repo, err)
+		return "", fmt.Errorf("failed to GetFileVersions: %w", err)
 	}
+	return rendResult(files[repoCommit], mr, renderConfig), nil
+}
+
+func GetMergeResult(c context.Context, ns, repo, forCommit, sourceCommit, filePath string,
+	proxy covermerger.FuncProxyURI, tp coveragedb.TimePeriod) (*covermerger.MergeResult, error) {
 	config := &covermerger.Config{
 		Jobs: 1,
 		Base: covermerger.RepoCommit{
@@ -58,6 +62,7 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file
 		StoreDetails:     true,
 	}
 
+	fromDate, toDate := tp.DatesFromTo()
 	dbReader := covermerger.MakeBQCSVReader()
 	if err := dbReader.InitNsRecords(c,
 		ns,
@@ -66,23 +71,22 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file
 		fromDate,
 		toDate,
 	); err != nil {
-		return "", fmt.Errorf("failed to dbReader.InitNsRecords: %w", err)
+		return nil, fmt.Errorf("failed to dbReader.InitNsRecords: %w", err)
 	}
 	defer dbReader.Close()
 	csvReader, err := dbReader.Reader()
 	if err != nil {
-		return "", fmt.Errorf("failed to dbReader.Reader: %w", err)
+		return nil, fmt.Errorf("failed to dbReader.Reader: %w", err)
 	}
 
 	mergeResult, err := covermerger.MergeCSVData(config, csvReader)
 	if err != nil {
-		return "", fmt.Errorf("error merging coverage: %w", err)
+		return nil, fmt.Errorf("error merging coverage: %w", err)
 	}
 	if _, exist := mergeResult[filePath]; !exist {
-		return "", fmt.Errorf("no merge result for file %s(fileSize %d)", filePath, len(fileContent))
+		return nil, fmt.Errorf("no merge result for file %s", filePath)
 	}
-
-	return rendResult(fileContent, mergeResult[filePath], renderConfig), nil
+	return mergeResult[filePath], nil
 }
 
 func rendResult(content string, coverage *covermerger.MergeResult, renderConfig *CoverageRenderConfig) string {
diff --git a/pkg/coveragedb/spanner.go b/pkg/coveragedb/spanner.go
index 0ff3a3197..54fa2e192 100644
--- a/pkg/coveragedb/spanner.go
+++ b/pkg/coveragedb/spanner.go
@@ -6,6 +6,7 @@ package coveragedb
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"cloud.google.com/go/civil"
@@ -89,6 +90,64 @@ func SaveMergeResult(ctx context.Context, projectID string, covMap map[string]*C
 	return nil
 }
 
+type linesCoverage struct {
+	LinesInstrumented []int64
+	HitCounts         []int64
+}
+
+func linesCoverageStmt(ns, filepath, commit string, timePeriod TimePeriod) spanner.Statement {
+	return spanner.Statement{
+		SQL: `
+select
+	linesinstrumented,
+	hitcounts
+from merge_history
+	join files
+		on merge_history.session = files.session
+where
+	namespace=$1 and dateto=$2 and duration=$3 and filepath=$4 and commit=$5`,
+		Params: map[string]interface{}{
+			"p1": ns,
+			"p2": timePeriod.DateTo,
+			"p3": timePeriod.Days,
+			"p4": filepath,
+			"p5": commit,
+		},
+	}
+}
+
+func ReadLinesHitCount(ctx context.Context, ns, commit, file string, tp TimePeriod,
+) (map[int]int, error) {
+	projectID := os.Getenv("GOOGLE_CLOUD_PROJECT")
+	client, err := NewClient(ctx, projectID)
+	if err != nil {
+		return nil, fmt.Errorf("spanner.NewClient: %w", err)
+	}
+	defer client.Close()
+
+	stmt := linesCoverageStmt(ns, file, commit, tp)
+	iter := client.Single().Query(ctx, stmt)
+	defer iter.Stop()
+
+	row, err := iter.Next()
+	if err == iterator.Done {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("iter.Next: %w", err)
+	}
+	var r linesCoverage
+	if err = row.ToStruct(&r); err != nil {
+		return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err)
+	}
+
+	res := map[int]int{}
+	for i, instrLine := range r.LinesInstrumented {
+		res[int(instrLine)] = int(r.HitCounts[i])
+	}
+	return res, nil
+}
+
 func historyMutation(session string, template *HistoryRecord, totalRows int64) *spanner.Mutation {
 	historyInsert, err := spanner.InsertOrUpdateStruct("merge_history", &HistoryRecord{
 		Session:   session,
diff --git a/tools/syz-cover/syz-cover.go b/tools/syz-cover/syz-cover.go
index b640ff63d..4b4270992 100644
--- a/tools/syz-cover/syz-cover.go
+++ b/tools/syz-cover/syz-cover.go
@@ -30,6 +30,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -51,9 +52,8 @@ var (
 		"modules JSON info obtained from /modules (optional)")
 	flagNsHeatmap        = flag.String("heatmap", "", "generate namespace heatmap")
 	flagNsHeatmapGroupBy = flag.String("group-by", "dir", "dir or subsystem")
-	flagDateFrom         = flag.String("from",
-		civil.DateOf(time.Now().Add(-14*24*time.Hour)).String(), "heatmap date from(optional)")
-	flagDateTo = flag.String("to",
+	flagPeriod           = flag.String("period", "day", "time period(day[default], month, quarter)")
+	flagDateTo           = flag.String("to",
 		civil.DateOf(time.Now()).String(), "heatmap date to(optional)")
 	flagProjectID = flag.String("project", "syzkaller", "spanner db project name")
 	flagForFile   = flag.String("for-file", "", "[optional]show file coverage")
@@ -70,33 +70,26 @@ var (
 		"there are missing coverage callbacks")
 )
 
-func parseDates() (civil.Date, civil.Date) {
-	dateFrom, errDateFrom := civil.ParseDate(*flagDateFrom)
-	if errDateFrom != nil {
-		tool.Failf("failed to parse date from: %v", errDateFrom.Error())
-	}
-	dateTo, errDateTo := civil.ParseDate(*flagDateTo)
-	if errDateTo != nil {
-		tool.Failf("failed to parse date to: %v", errDateTo.Error())
-	}
-	return dateFrom, dateTo
-}
-
-func periodsFromDays(from, to civil.Date) []coveragedb.TimePeriod {
-	if to.Before(from) {
-		panic("toDay can't be less than fromDay")
-	}
-	res := []coveragedb.TimePeriod{{DateTo: from, Days: 1, Type: coveragedb.DayPeriod}}
-	for ; from.Before(to); from = from.AddDays(1) {
-		res = append(res, coveragedb.TimePeriod{DateTo: from, Days: 1, Type: coveragedb.DayPeriod})
+func dayPeriods(tp coveragedb.TimePeriod) []coveragedb.TimePeriod {
+	var res []coveragedb.TimePeriod
+	for i := 0; i < tp.Days; i++ {
+		res = append(res, coveragedb.TimePeriod{DateTo: tp.DateTo.AddDays(-i), Days: 1, Type: coveragedb.DayPeriod})
 	}
+	slices.Reverse(res)
 	return res
 }
 
 func toolBuildNsHeatmap() {
 	buf := new(bytes.Buffer)
-	periods := periodsFromDays(parseDates())
-	var err error
+	dateTo, err := civil.ParseDate(*flagDateTo)
+	if err != nil {
+		tool.Fail(err)
+	}
+	tp, err := coveragedb.MakeTimePeriod(dateTo, *flagPeriod)
+	if err != nil {
+		tool.Fail(err)
+	}
+	periods := dayPeriods(tp)
 	switch *flagNsHeatmapGroupBy {
 	case "dir":
 		if err = cover.DoDirHeatMap(buf, *flagProjectID, *flagNsHeatmap, periods); err != nil {
@@ -115,19 +108,33 @@ func toolBuildNsHeatmap() {
 }
 
 func toolFileCover() {
-	dateFrom, dateTo := parseDates()
+	dateTo, err := civil.ParseDate(*flagDateTo)
+	if err != nil {
+		tool.Failf("failed to parse date from: %v", err)
+	}
+	tp, err := coveragedb.MakeTimePeriod(dateTo, *flagPeriod)
+	if err != nil {
+		tool.Fail(err)
+	}
 	config := cover.DefaultTextRenderConfig()
 	config.ShowLineSourceExplanation = *flagDebug
-	details, err := cover.RendFileCoverage(
-		context.Background(),
+	mr, err := cover.GetMergeResult(context.Background(),
 		*flagNamespace,
 		*flagRepo,
 		*flagCommit,
 		*flagSourceCommit,
 		*flagForFile,
+		nil, tp)
+	if err != nil {
+		tool.Fail(err)
+	}
+
+	details, err := cover.RendFileCoverage(
+		*flagRepo,
+		*flagCommit,
+		*flagForFile,
 		nil, // no proxy - get files directly from WebGits
-		dateFrom,
-		dateTo,
+		mr,
 		config,
 	)
 	if err != nil {