dashboard/app: add file coverage page

It directly uses the coverage signals from BigQuery.
There is no need to wait for the coverage_batch cron jobs.
Looks good for debugging.

Limitations:
1. It is slow. I know how to speed up but want to stabilize the UI first.
2. It is expensive because of the direct BQ requests. Limited to admin only because of it.
3. It merges only the commits reachable on github because of the gitweb throttling.

After the UI stabilization I'll save all the required artifacts to spanner and make this page publicly available.
To merge all the commits, not the github reachable only, http git caching instance is needed.

9 files changed, 145 insertions, 29 deletions

diff --git a/dashboard/app/graphs.go b/dashboard/app/graphs.go
index 87c7e1403..448095512 100644
--- a/dashboard/app/graphs.go
+++ b/dashboard/app/graphs.go
@@ -17,6 +17,8 @@ import (
 	"cloud.google.com/go/civil"
 	"github.com/google/syzkaller/pkg/cover"
 	"github.com/google/syzkaller/pkg/coveragedb"
+	"github.com/google/syzkaller/pkg/covermerger"
+	"github.com/google/syzkaller/pkg/validator"
 	db "google.golang.org/appengine/v2/datastore"
 )
 
@@ -239,6 +241,57 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 	})
 }
 
+func githubTorvaldsLinuxURI(filePath string, rc covermerger.RepoCommit) string {
+	return fmt.Sprintf("https://raw.githubusercontent.com/torvalds/linux/%s/%s", rc.Commit, filePath)
+}
+
+func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Request) error {
+	accessLevel := accessLevel(c, r)
+	if accessLevel != AccessAdmin {
+		return ErrAccess
+	}
+	hdr, err := commonHeader(c, r, w, "")
+	if err != nil {
+		return err
+	}
+	dateToStr := r.FormValue("dateto")
+	periodType := r.FormValue("period")
+	targetCommit := r.FormValue("commit")
+	kernelFilePath := r.FormValue("filepath")
+	if err := validator.AnyError("input validation failed",
+		validator.TimePeriodType(periodType, "period"),
+		validator.CommitHash(targetCommit, "commit"),
+		validator.KernelFilePath(kernelFilePath, "filepath"),
+	); err != nil {
+		return err
+	}
+	targetDate, err := civil.ParseDate(dateToStr)
+	if err != nil {
+		return fmt.Errorf("civil.ParseDate(%s): %w", dateToStr, err)
+	}
+	tp, err := coveragedb.MakeTimePeriod(targetDate, periodType)
+	if err != nil {
+		return fmt.Errorf("coveragedb.MakeTimePeriod: %w", err)
+	}
+	dateFrom, dateTo := tp.DatesFromTo()
+	mainNsRepo, _ := getNsConfig(c, hdr.Namespace).mainRepoBranch()
+	content, err := cover.RendFileCoverage(
+		c, hdr.Namespace, mainNsRepo,
+		targetCommit, "", // merge all commits to targetCommit
+		kernelFilePath,
+		githubTorvaldsLinuxURI,
+		dateFrom,
+		dateTo,
+		cover.DefaultHTMLRenderConfig(),
+	)
+	if err != nil {
+		return fmt.Errorf("cover.RendFileCoverage: %w", err)
+	}
+	w.Header().Set("Content-Type", "text/html")
+	w.Write([]byte(content))
+	return nil
+}
+
 func handleCoverageGraph(c context.Context, w http.ResponseWriter, r *http.Request) error {
 	hdr, err := commonHeader(c, r, w, "")
 	if err != nil {
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 3dae674de..24046de68 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -65,6 +65,7 @@ func initHTTPHandlers() {
 		http.Handle("/"+ns+"/graph/crashes", handlerWrapper(handleGraphCrashes))
 		http.Handle("/"+ns+"/graph/found-bugs", handlerWrapper(handleFoundBugsGraph))
 		if nsConfig.Coverage != nil {
+			http.Handle("/"+ns+"/graph/coverage/file", handlerWrapper(handleFileCoverage))
 			http.Handle("/"+ns+"/graph/coverage", handlerWrapper(handleCoverageGraph))
 			http.Handle("/"+ns+"/graph/coverage_heatmap", handlerWrapper(handleCoverageHeatmap))
 			if nsConfig.Subsystems.Service != nil {
diff --git a/pkg/cover/file.go b/pkg/cover/file.go
index bbb9a291e..f7c679293 100644
--- a/pkg/cover/file.go
+++ b/pkg/cover/file.go
@@ -6,6 +6,7 @@ package cover
 import (
 	"context"
 	"fmt"
+	"html"
 	"strings"
 
 	"cloud.google.com/go/civil"
@@ -15,7 +16,7 @@ import (
 type lineRender func(string, int, *covermerger.MergeResult, *CoverageRenderConfig) string
 
 type CoverageRenderConfig struct {
-	Render                    lineRender
+	RendLine                  lineRender
 	ShowLineCoverage          bool
 	ShowLineNumbers           bool
 	ShowLineSourceExplanation bool
@@ -23,7 +24,16 @@ type CoverageRenderConfig struct {
 
 func DefaultTextRenderConfig() *CoverageRenderConfig {
 	return &CoverageRenderConfig{
-		Render:                    RendTextLine,
+		RendLine:                  RendTextLine,
+		ShowLineCoverage:          true,
+		ShowLineNumbers:           true,
+		ShowLineSourceExplanation: false,
+	}
+}
+
+func DefaultHTMLRenderConfig() *CoverageRenderConfig {
+	return &CoverageRenderConfig{
+		RendLine:                  RendHTMLLine,
 		ShowLineCoverage:          true,
 		ShowLineNumbers:           true,
 		ShowLineSourceExplanation: false,
@@ -31,6 +41,7 @@ func DefaultTextRenderConfig() *CoverageRenderConfig {
 }
 
 func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, filePath string,
+	proxy covermerger.FuncProxyURI,
 	fromDate, toDate civil.Date, renderConfig *CoverageRenderConfig) (string, error) {
 	fileContent, err := covermerger.GetFileVersion(filePath, repo, forCommit)
 	if err != nil {
@@ -43,7 +54,7 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file
 			Repo:   repo,
 			Commit: forCommit,
 		},
-		FileVersProvider: covermerger.MakeWebGit(),
+		FileVersProvider: covermerger.MakeWebGit(proxy),
 		StoreDetails:     true,
 	}
 
@@ -67,6 +78,9 @@ func RendFileCoverage(c context.Context, ns, repo, forCommit, sourceCommit, file
 	if err != nil {
 		return "", fmt.Errorf("error merging coverage: %w", err)
 	}
+	if _, exist := mergeResult[filePath]; !exist {
+		return "", fmt.Errorf("no merge result for file %s(fileSize %d)", filePath, len(fileContent))
+	}
 
 	return rendResult(fileContent, mergeResult[filePath], renderConfig), nil
 }
@@ -79,11 +93,11 @@ func rendResult(content string, coverage *covermerger.MergeResult, renderConfig
 		}
 	}
 	srcLines := strings.Split(content, "\n")
-	var htmlLines []string
+	var resLines []string
 	for i, srcLine := range srcLines {
-		htmlLines = append(htmlLines, renderConfig.Render(srcLine, i+1, coverage, renderConfig))
+		resLines = append(resLines, renderConfig.RendLine(srcLine, i+1, coverage, renderConfig))
 	}
-	return strings.Join(htmlLines, "\n")
+	return strings.Join(resLines, "\n")
 }
 
 func RendTextLine(code string, line int, coverage *covermerger.MergeResult, config *CoverageRenderConfig) string {
@@ -111,6 +125,11 @@ func RendTextLine(code string, line int, coverage *covermerger.MergeResult, conf
 	return res
 }
 
+func RendHTMLLine(code string, line int, coverage *covermerger.MergeResult, config *CoverageRenderConfig) string {
+	textLine := RendTextLine(code, line, coverage, config)
+	return `<pre style="margin: 0">` + html.EscapeString(textLine) + "</pre>"
+}
+
 func mainSignalSource(sources []*covermerger.FileRecord) string {
 	res := ""
 	prevMax := -1
diff --git a/pkg/coveragedb/time_period.go b/pkg/coveragedb/time_period.go
index bfa1b2789..4e17e4273 100644
--- a/pkg/coveragedb/time_period.go
+++ b/pkg/coveragedb/time_period.go
@@ -5,6 +5,7 @@ package coveragedb
 
 import (
 	"errors"
+	"fmt"
 	"slices"
 	"sort"
 
@@ -16,6 +17,23 @@ type TimePeriod struct {
 	Days   int
 }
 
+// DatesFromTo returns the closed range [fromDate, toDate].
+func (tp *TimePeriod) DatesFromTo() (civil.Date, civil.Date) {
+	return tp.DateTo.AddDays(-tp.Days + 1), tp.DateTo
+}
+
+func MakeTimePeriod(targetDate civil.Date, periodType string) (TimePeriod, error) {
+	pOps, err := PeriodOps(periodType)
+	if err != nil {
+		return TimePeriod{}, err
+	}
+	tp := TimePeriod{DateTo: targetDate, Days: pOps.PointedPeriodDays(targetDate)}
+	if !pOps.IsValidPeriod(tp) {
+		return TimePeriod{}, fmt.Errorf("date %s doesn't point the period(%s) end", targetDate.String(), periodType)
+	}
+	return tp, nil
+}
+
 const (
 	DayPeriod     = "day"
 	MonthPeriod   = "month"
@@ -53,15 +71,15 @@ func PeriodOps(periodType string) (periodOps, error) {
 type periodOps interface {
 	IsValidPeriod(p TimePeriod) bool
 	lastPeriodDate(d civil.Date) civil.Date
-	pointedPeriodDays(d civil.Date) int
+	PointedPeriodDays(d civil.Date) int
 }
 
 func GenNPeriodsTill(n int, d civil.Date, po periodOps) []TimePeriod {
 	var res []TimePeriod
 	for i := 0; i < n; i++ {
 		d = po.lastPeriodDate(d)
-		res = append(res, TimePeriod{DateTo: d, Days: po.pointedPeriodDays(d)})
-		d = d.AddDays(-po.pointedPeriodDays(d))
+		res = append(res, TimePeriod{DateTo: d, Days: po.PointedPeriodDays(d)})
+		d = d.AddDays(-po.PointedPeriodDays(d))
 	}
 	slices.Reverse(res)
 	return res
@@ -77,7 +95,7 @@ func (dpo *DayPeriodOps) IsValidPeriod(p TimePeriod) bool {
 	return p.Days == 1
 }
 
-func (dpo *DayPeriodOps) pointedPeriodDays(d civil.Date) int {
+func (dpo *DayPeriodOps) PointedPeriodDays(d civil.Date) int {
 	return 1
 }
 
@@ -95,7 +113,7 @@ func (m *MonthPeriodOps) IsValidPeriod(p TimePeriod) bool {
 	return lmd == p.DateTo && p.Days == lmd.Day
 }
 
-func (m *MonthPeriodOps) pointedPeriodDays(d civil.Date) int {
+func (m *MonthPeriodOps) PointedPeriodDays(d civil.Date) int {
 	return m.lastPeriodDate(d).Day
 }
 
@@ -103,7 +121,7 @@ type QuarterPeriodOps struct{}
 
 func (q *QuarterPeriodOps) IsValidPeriod(p TimePeriod) bool {
 	lmd := q.lastPeriodDate(p.DateTo)
-	return lmd == p.DateTo && p.Days == q.pointedPeriodDays(lmd)
+	return lmd == p.DateTo && p.Days == q.PointedPeriodDays(lmd)
 }
 
 func (q *QuarterPeriodOps) lastPeriodDate(d civil.Date) civil.Date {
@@ -112,12 +130,12 @@ func (q *QuarterPeriodOps) lastPeriodDate(d civil.Date) civil.Date {
 	return (&MonthPeriodOps{}).lastPeriodDate(d)
 }
 
-func (q *QuarterPeriodOps) pointedPeriodDays(d civil.Date) int {
+func (q *QuarterPeriodOps) PointedPeriodDays(d civil.Date) int {
 	d = q.lastPeriodDate(d)
 	d.Day = 1
 	res := 0
 	for i := 0; i < 3; i++ {
-		res += (&MonthPeriodOps{}).pointedPeriodDays(d)
+		res += (&MonthPeriodOps{}).PointedPeriodDays(d)
 		d.Month--
 	}
 	return res
@@ -141,7 +159,7 @@ func PeriodsToMerge(srcDates, mergedPeriods []TimePeriod, srcRows, mergedRows []
 	periods := []TimePeriod{}
 	for periodEndDate := range periodRows {
 		periods = append(periods,
-			TimePeriod{DateTo: periodEndDate, Days: ops.pointedPeriodDays(periodEndDate)})
+			TimePeriod{DateTo: periodEndDate, Days: ops.PointedPeriodDays(periodEndDate)})
 	}
 	sort.Slice(periods, func(i, j int) bool {
 		return periods[i].DateTo.After(periods[j].DateTo)
diff --git a/pkg/coveragedb/time_period_test.go b/pkg/coveragedb/time_period_test.go
index 4ccd18f12..e985b162a 100644
--- a/pkg/coveragedb/time_period_test.go
+++ b/pkg/coveragedb/time_period_test.go
@@ -22,7 +22,7 @@ func TestDayPeriodOps(t *testing.T) {
 	assert.True(t, ops.IsValidPeriod(goodPeriod))
 	assert.False(t, ops.IsValidPeriod(badPeriod))
 
-	assert.Equal(t, 1, ops.pointedPeriodDays(d))
+	assert.Equal(t, 1, ops.PointedPeriodDays(d))
 
 	assert.Equal(t,
 		[]TimePeriod{
@@ -47,7 +47,7 @@ func TestMonthPeriodOps(t *testing.T) {
 	assert.False(t, ops.IsValidPeriod(badPeriod1))
 	assert.False(t, ops.IsValidPeriod(badPeriod2))
 
-	assert.Equal(t, 29, ops.pointedPeriodDays(midMonthDate))
+	assert.Equal(t, 29, ops.PointedPeriodDays(midMonthDate))
 
 	assert.Equal(t,
 		[]TimePeriod{
@@ -73,7 +73,7 @@ func TestQuarterPeriodOps(t *testing.T) {
 	assert.False(t, ops.IsValidPeriod(badPeriod1))
 	assert.False(t, ops.IsValidPeriod(badPeriod2))
 
-	assert.Equal(t, 31+29+31, ops.pointedPeriodDays(midQuarterDate))
+	assert.Equal(t, 31+29+31, ops.PointedPeriodDays(midQuarterDate))
 
 	assert.Equal(t,
 		[]TimePeriod{
@@ -279,3 +279,13 @@ func TestAtMostNLatestPeriods(t *testing.T) {
 	assert.Equal(t, []TimePeriod{makeTimePeriod("2024-06-06", 1)}, AtMostNLatestPeriods(sampleDays, 1))
 	assert.Equal(t, sampleDays, AtMostNLatestPeriods(sampleDays, 100))
 }
+
+func TestMakeTimePeriod(t *testing.T) {
+	tp, err := MakeTimePeriod(civil.Date{Year: 2024, Month: time.March, Day: 31}, QuarterPeriod)
+	assert.NoError(t, err)
+	assert.NotEqual(t, TimePeriod{}, tp)
+
+	tp, err = MakeTimePeriod(civil.Date{Year: 2024, Month: time.March, Day: 30}, QuarterPeriod)
+	assert.Error(t, err)
+	assert.Equal(t, TimePeriod{}, tp)
+}
diff --git a/pkg/covermerger/provider_web.go b/pkg/covermerger/provider_web.go
index 6acb120ed..2a1bee85a 100644
--- a/pkg/covermerger/provider_web.go
+++ b/pkg/covermerger/provider_web.go
@@ -11,14 +11,17 @@ import (
 	"net/url"
 )
 
+type FuncProxyURI func(filePath string, rc RepoCommit) string
+
 type webGit struct {
+	funcProxy FuncProxyURI
 }
 
 func (mr *webGit) GetFileVersions(targetFilePath string, repoCommits ...RepoCommit,
 ) (fileVersions, error) {
 	res := make(fileVersions)
 	for _, repoCommit := range repoCommits {
-		fileBytes, err := loadFile(targetFilePath, repoCommit.Repo, repoCommit.Commit)
+		fileBytes, err := mr.loadFile(targetFilePath, repoCommit.Repo, repoCommit.Commit)
 		// It is ok if some file doesn't exist. It means we have repo FS diff.
 		if err == errFileNotFound {
 			continue
@@ -33,10 +36,15 @@ func (mr *webGit) GetFileVersions(targetFilePath string, repoCommits ...RepoComm
 
 var errFileNotFound = errors.New("file not found")
 
-func loadFile(filePath, repo, commit string) ([]byte, error) {
-	uri := fmt.Sprintf("%s/plain/%s", repo, filePath)
-	if commit != "latest" {
-		uri += "?id=" + commit
+func (mr *webGit) loadFile(filePath, repo, commit string) ([]byte, error) {
+	var uri string
+	if mr.funcProxy != nil {
+		uri = mr.funcProxy(filePath, RepoCommit{Repo: repo, Commit: commit})
+	} else {
+		uri = fmt.Sprintf("%s/plain/%s", repo, filePath)
+		if commit != "latest" {
+			uri += "?id=" + commit
+		}
 	}
 	u, err := url.Parse(uri)
 	if err != nil {
@@ -54,7 +62,7 @@ func loadFile(filePath, repo, commit string) ([]byte, error) {
 		return nil, errFileNotFound
 	}
 	if res.StatusCode != 200 {
-		return nil, fmt.Errorf("error: status %d getting %s", res.StatusCode, uri)
+		return nil, fmt.Errorf("error: status %d getting '%s'", res.StatusCode, uri)
 	}
 	body, err := io.ReadAll(res.Body)
 	if err != nil {
@@ -63,13 +71,15 @@ func loadFile(filePath, repo, commit string) ([]byte, error) {
 	return body, nil
 }
 
-func MakeWebGit() FileVersProvider {
-	return &webGit{}
+func MakeWebGit(funcProxy FuncProxyURI) FileVersProvider {
+	return &webGit{
+		funcProxy: funcProxy,
+	}
 }
 
 func GetFileVersion(filePath, repo, commit string) (string, error) {
 	repoCommit := RepoCommit{repo, commit}
-	files, err := MakeWebGit().GetFileVersions(filePath, repoCommit)
+	files, err := MakeWebGit(nil).GetFileVersions(filePath, repoCommit)
 	if err != nil {
 		return "", fmt.Errorf("failed to GetFileVersions: %w", err)
 	}
diff --git a/pkg/validator/validator.go b/pkg/validator/validator.go
index 420830835..723cf2384 100644
--- a/pkg/validator/validator.go
+++ b/pkg/validator/validator.go
@@ -9,6 +9,7 @@ import (
 	"regexp"
 
 	"github.com/google/syzkaller/pkg/auth"
+	"github.com/google/syzkaller/pkg/coveragedb"
 )
 
 type Result struct {
@@ -50,11 +51,14 @@ var (
 	EmptyStr       = makeStrLenFunc("not empty", 0)
 	AlphaNumeric   = makeStrReFunc("not an alphanum", "^[a-zA-Z0-9]*$")
 	CommitHash     = makeCombinedStrFunc("not a hash", AlphaNumeric, makeStrLenFunc("len is not 40", 40))
-	KernelFilePath = makeStrReFunc("not a kernel file path", "^[./_a-zA-Z0-9]*$")
+	KernelFilePath = makeStrReFunc("not a kernel file path", "^[./-_a-zA-Z0-9]*$")
 	NamespaceName  = makeStrReFunc("not a namespace name", "^[a-zA-Z0-9-_.]{4,32}$")
 	DashClientName = makeStrReFunc("not a dashboard client name", "^[a-zA-Z0-9-_.]{4,100}$")
 	DashClientKey  = makeStrReFunc("not a dashboard client key",
 		"^([a-zA-Z0-9]{16,128})|("+regexp.QuoteMeta(auth.OauthMagic)+".*)$")
+	TimePeriodType = makeStrReFunc(fmt.Sprintf("bad time period, use (%s|%s|%s)",
+		coveragedb.DayPeriod, coveragedb.MonthPeriod, coveragedb.QuarterPeriod),
+		fmt.Sprintf("^(%s|%s|%s)$", coveragedb.DayPeriod, coveragedb.MonthPeriod, coveragedb.QuarterPeriod))
 )
 
 type strValidationFunc func(string, ...string) Result
diff --git a/tools/syz-cover/syz-cover.go b/tools/syz-cover/syz-cover.go
index bfab129da..47fa0df66 100644
--- a/tools/syz-cover/syz-cover.go
+++ b/tools/syz-cover/syz-cover.go
@@ -125,6 +125,7 @@ func toolFileCover() {
 		*flagCommit,
 		*flagSourceCommit,
 		*flagForFile,
+		nil, // no proxy - get files directly from WebGits
 		dateFrom,
 		dateTo,
 		config,
diff --git a/tools/syz-covermerger/syz_covermerger.go b/tools/syz-covermerger/syz_covermerger.go
index 1dcb1a197..3d9c8f711 100644
--- a/tools/syz-covermerger/syz_covermerger.go
+++ b/tools/syz-covermerger/syz_covermerger.go
@@ -39,7 +39,7 @@ func makeProvider() covermerger.FileVersProvider {
 	case "git-clone":
 		return covermerger.MakeMonoRepo(*flagWorkdir)
 	case "web-git":
-		return covermerger.MakeWebGit()
+		return covermerger.MakeWebGit(nil)
 	default:
 		panic(fmt.Sprintf("unknown provider %v", *flagSrcProvider))
 	}