dashboard/app: read lines coverage from spanner

We currently merge bigquery data for every line coverage request.
Let's read cached lines coverage data from spanner instead.
It allows to get only 1 file version from git and skip the data merge step.

2 files changed, 76 insertions, 13 deletions

diff --git a/pkg/cover/file.go b/pkg/cover/file.go
index f7c679293..368607ffe 100644
--- a/pkg/cover/file.go
+++ b/pkg/cover/file.go
@@ -9,7 +9,7 @@ import (
 	"html"
 	"strings"
 
-	"cloud.google.com/go/civil"
+	"github.com/google/syzkaller/pkg/coveragedb"
 	"github.com/google/syzkaller/pkg/covermerger"
 )
 
@@ -40,14 +40,18 @@ func DefaultHTMLRenderConfig() *CoverageRenderConfig {
 	}
 }
 
-func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, filePath string,
-	proxy covermerger.FuncProxyURI,
-	fromDate, toDate civil.Date, renderConfig *CoverageRenderConfig) (string, error) {
-	fileContent, err := covermerger.GetFileVersion(filePath, repo, forCommit)
+func RendFileCoverage(repo, forCommit, filePath string, proxy covermerger.FuncProxyURI,
+	mr *covermerger.MergeResult, renderConfig *CoverageRenderConfig) (string, error) {
+	repoCommit := covermerger.RepoCommit{Repo: repo, Commit: forCommit}
+	files, err := covermerger.MakeWebGit(proxy).GetFileVersions(filePath, repoCommit)
 	if err != nil {
-		return "", fmt.Errorf("failed to GetFileVersion for file %s, commit %s from repo %s: %w",
-			filePath, forCommit, repo, err)
+		return "", fmt.Errorf("failed to GetFileVersions: %w", err)
 	}
+	return rendResult(files[repoCommit], mr, renderConfig), nil
+}
+
+func GetMergeResult(c context.Context, ns, repo, forCommit, sourceCommit, filePath string,
+	proxy covermerger.FuncProxyURI, tp coveragedb.TimePeriod) (*covermerger.MergeResult, error) {
 	config := &covermerger.Config{
 		Jobs: 1,
 		Base: covermerger.RepoCommit{
@@ -58,6 +62,7 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file
 		StoreDetails:     true,
 	}
 
+	fromDate, toDate := tp.DatesFromTo()
 	dbReader := covermerger.MakeBQCSVReader()
 	if err := dbReader.InitNsRecords(c,
 		ns,
@@ -66,23 +71,22 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file
 		fromDate,
 		toDate,
 	); err != nil {
-		return "", fmt.Errorf("failed to dbReader.InitNsRecords: %w", err)
+		return nil, fmt.Errorf("failed to dbReader.InitNsRecords: %w", err)
 	}
 	defer dbReader.Close()
 	csvReader, err := dbReader.Reader()
 	if err != nil {
-		return "", fmt.Errorf("failed to dbReader.Reader: %w", err)
+		return nil, fmt.Errorf("failed to dbReader.Reader: %w", err)
 	}
 
 	mergeResult, err := covermerger.MergeCSVData(config, csvReader)
 	if err != nil {
-		return "", fmt.Errorf("error merging coverage: %w", err)
+		return nil, fmt.Errorf("error merging coverage: %w", err)
 	}
 	if _, exist := mergeResult[filePath]; !exist {
-		return "", fmt.Errorf("no merge result for file %s(fileSize %d)", filePath, len(fileContent))
+		return nil, fmt.Errorf("no merge result for file %s", filePath)
 	}
-
-	return rendResult(fileContent, mergeResult[filePath], renderConfig), nil
+	return mergeResult[filePath], nil
 }
 
 func rendResult(content string, coverage *covermerger.MergeResult, renderConfig *CoverageRenderConfig) string {
diff --git a/pkg/coveragedb/spanner.go b/pkg/coveragedb/spanner.go
index 0ff3a3197..54fa2e192 100644
--- a/pkg/coveragedb/spanner.go
+++ b/pkg/coveragedb/spanner.go
@@ -6,6 +6,7 @@ package coveragedb
 import (
 	"context"
 	"fmt"
+	"os"
 	"time"
 
 	"cloud.google.com/go/civil"
@@ -89,6 +90,64 @@ func SaveMergeResult(ctx context.Context, projectID string, covMap map[string]*C
 	return nil
 }
 
+type linesCoverage struct {
+	LinesInstrumented []int64
+	HitCounts         []int64
+}
+
+func linesCoverageStmt(ns, filepath, commit string, timePeriod TimePeriod) spanner.Statement {
+	return spanner.Statement{
+		SQL: `
+select
+	linesinstrumented,
+	hitcounts
+from merge_history
+	join files
+		on merge_history.session = files.session
+where
+	namespace=$1 and dateto=$2 and duration=$3 and filepath=$4 and commit=$5`,
+		Params: map[string]interface{}{
+			"p1": ns,
+			"p2": timePeriod.DateTo,
+			"p3": timePeriod.Days,
+			"p4": filepath,
+			"p5": commit,
+		},
+	}
+}
+
+func ReadLinesHitCount(ctx context.Context, ns, commit, file string, tp TimePeriod,
+) (map[int]int, error) {
+	projectID := os.Getenv("GOOGLE_CLOUD_PROJECT")
+	client, err := NewClient(ctx, projectID)
+	if err != nil {
+		return nil, fmt.Errorf("spanner.NewClient: %w", err)
+	}
+	defer client.Close()
+
+	stmt := linesCoverageStmt(ns, file, commit, tp)
+	iter := client.Single().Query(ctx, stmt)
+	defer iter.Stop()
+
+	row, err := iter.Next()
+	if err == iterator.Done {
+		return nil, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("iter.Next: %w", err)
+	}
+	var r linesCoverage
+	if err = row.ToStruct(&r); err != nil {
+		return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err)
+	}
+
+	res := map[int]int{}
+	for i, instrLine := range r.LinesInstrumented {
+		res[int(instrLine)] = int(r.HitCounts[i])
+	}
+	return res, nil
+}
+
 func historyMutation(session string, template *HistoryRecord, totalRows int64) *spanner.Mutation {
 	historyInsert, err := spanner.InsertOrUpdateStruct("merge_history", &HistoryRecord{
 		Session:   session,