dashboard/app: send coverage report to ns-defined email

We periodically send coverage reports for the regressions detection.

7 files changed, 296 insertions, 21 deletions

diff --git a/dashboard/app/app_test.go b/dashboard/app/app_test.go
index 876104f42..445fcf785 100644
--- a/dashboard/app/app_test.go
+++ b/dashboard/app/app_test.go
@@ -579,6 +579,31 @@ var testConfig = &GlobalConfig{
 			FindBugOriginTrees:     true,
 			RetestMissingBackports: true,
 		},
+		"coverage-tests": {
+			Coverage: &CoverageConfig{
+				EmailRegressionsTo:  "test@test.test",
+				RegressionThreshold: 1,
+			},
+			AccessLevel: AccessPublic,
+			Key:         "coveragetestskeycoveragetestskeycoveragetestskey",
+			Repos: []KernelRepo{
+				{
+					URL:    "git://syzkaller.org/test.git",
+					Branch: "main",
+					Alias:  "main",
+				},
+			},
+			Reporting: []Reporting{
+				{
+					Name:       "non-public",
+					DailyLimit: 1000,
+					Filter: func(bug *Bug) FilterResult {
+						return FilterReport
+					},
+					Config: &TestConfig{Index: 1},
+				},
+			},
+		},
 	},
 }
 
diff --git a/dashboard/app/config.go b/dashboard/app/config.go
index 836344954..96f631e0a 100644
--- a/dashboard/app/config.go
+++ b/dashboard/app/config.go
@@ -155,6 +155,7 @@ type ACLItem struct {
 }
 
 const defaultDashboardClientName = "coverage-merger"
+const defaultRegressionThreshold = 50
 
 type CoverageConfig struct {
 	BatchProject        string
@@ -167,6 +168,15 @@ type CoverageConfig struct {
 	// WebGitURI specifies where can we get the kernel file source code directly from AppEngine.
 	// It may be the Git or Gerrit compatible repo.
 	WebGitURI string
+
+	// EmailRegressionsTo species the regressions recipient.
+	// If empty, regression analysis is disabled.
+	EmailRegressionsTo string
+
+	// RegressionThreshold is a minimal basic block coverage drop in a file.
+	// The amount of files in the dir and other factors do not matter.
+	// Defaults to defaultRegressionThreshold.
+	RegressionThreshold int
 }
 
 // DiscussionEmailConfig defines the correspondence between an email and a DiscussionSource.
@@ -585,6 +595,16 @@ func checkNamespace(ns string, cfg *Config, namespaces, clientNames map[string]b
 	checkKernelRepos(ns, cfg, cfg.Repos)
 	checkNamespaceReporting(ns, cfg)
 	checkSubsystems(ns, cfg)
+	checkCoverageConfig(ns, cfg)
+}
+
+func checkCoverageConfig(ns string, cfg *Config) {
+	if cfg.Coverage == nil || cfg.Coverage.EmailRegressionsTo == "" {
+		return
+	}
+	if _, err := mail.ParseAddress(cfg.Coverage.EmailRegressionsTo); err != nil {
+		panic(fmt.Sprintf("bad cfg.Coverage.EmailRegressionsTo in '%s': %s", ns, err.Error()))
+	}
 }
 
 func checkSubsystems(ns string, cfg *Config) {
diff --git a/dashboard/app/coverage.go b/dashboard/app/coverage.go
index 305a9fad1..c600aab9e 100644
--- a/dashboard/app/coverage.go
+++ b/dashboard/app/coverage.go
@@ -17,6 +17,7 @@ import (
 	"github.com/google/syzkaller/pkg/coveragedb"
 	"github.com/google/syzkaller/pkg/coveragedb/spannerclient"
 	"github.com/google/syzkaller/pkg/covermerger"
+	"github.com/google/syzkaller/pkg/html/urlutil"
 	"github.com/google/syzkaller/pkg/validator"
 	"google.golang.org/appengine/v2"
 )
@@ -71,6 +72,59 @@ func handleSubsystemsCoverageHeatmap(c context.Context, w http.ResponseWriter, r
 	return handleHeatmap(c, w, r, cover.DoSubsystemsHeatMapStyleBodyJS)
 }
 
+type covPageParam int
+
+const (
+	// keep-sorted start
+	CommitHash covPageParam = iota
+	DateTo
+	FilePath
+	ManagerName
+	MinCoverLinesDrop
+	OrderByCoverDrop
+	PeriodCount
+	PeriodType
+	SubsystemName
+	UniqueOnly
+	// keep-sorted end
+)
+
+var covPageParams = map[covPageParam]string{
+	// keep-sorted start
+	CommitHash:        "commit",
+	DateTo:            "dateto",
+	FilePath:          "filepath",
+	ManagerName:       "manager",
+	MinCoverLinesDrop: "min-cover-lines-drop",
+	OrderByCoverDrop:  "order-by-cover-lines-drop",
+	PeriodCount:       "period_count",
+	PeriodType:        "period",
+	SubsystemName:     "subsystem",
+	UniqueOnly:        "unique-only",
+	// keep-sorted end
+}
+
+func coveragePageLink(ns, periodType, dateTo string, minDrop, periodCount int, orderByCoverDrop bool) string {
+	if periodType == "" {
+		periodType = coveragedb.MonthPeriod
+	}
+	url := "/" + ns + "/coverage"
+	url = urlutil.SetParam(url, covPageParams[PeriodType], periodType)
+	if periodCount != 0 {
+		url = urlutil.SetParam(url, covPageParams[PeriodCount], strconv.Itoa(periodCount))
+	}
+	if dateTo != "" {
+		url = urlutil.SetParam(url, covPageParams[DateTo], dateTo)
+	}
+	if minDrop > 0 {
+		url = urlutil.SetParam(url, covPageParams[MinCoverLinesDrop], strconv.Itoa(minDrop))
+	}
+	if orderByCoverDrop {
+		url = urlutil.SetParam(url, covPageParams[OrderByCoverDrop], "1")
+	}
+	return url
+}
+
 func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f funcStyleBodyJS) error {
 	hdr, err := commonHeader(c, r, w, "")
 	if err != nil {
@@ -80,10 +134,10 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 	if nsConfig.Coverage == nil {
 		return ErrClientNotFound
 	}
-	ss := r.FormValue("subsystem")
-	manager := r.FormValue("manager")
+	ss := r.FormValue(covPageParams[SubsystemName])
+	manager := r.FormValue(covPageParams[ManagerName])
 
-	periodType := r.FormValue("period")
+	periodType := r.FormValue(covPageParams[PeriodType])
 	if periodType == "" {
 		periodType = coveragedb.DayPeriod
 	}
@@ -94,7 +148,7 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 			periodType, ErrClientBadRequest)
 	}
 
-	periodCount := r.FormValue("period_count")
+	periodCount := r.FormValue(covPageParams[PeriodCount])
 	if periodCount == "" {
 		periodCount = "4"
 	}
@@ -104,7 +158,7 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 	}
 
 	dateTo := civil.DateOf(timeNow(c))
-	if customDate := r.FormValue("dateto"); customDate != "" {
+	if customDate := r.FormValue(covPageParams[DateTo]); customDate != "" {
 		if dateTo, err = civil.ParseDate(customDate); err != nil {
 			return fmt.Errorf("civil.ParseDate(%s): %w", customDate, err)
 		}
@@ -126,12 +180,12 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f
 	slices.Sort(managers)
 	slices.Sort(subsystems)
 
-	onlyUnique := r.FormValue("unique-only") == "1"
-	orderByCoverLinesDrop := r.FormValue("order-by-cover-lines-drop") == "1"
+	onlyUnique := r.FormValue(covPageParams[UniqueOnly]) == "1"
+	orderByCoverLinesDrop := r.FormValue(covPageParams[OrderByCoverDrop]) == "1"
 	// Prefixing "0" we don't fail on empty string.
-	minCoverLinesDrop, err := strconv.Atoi("0" + r.FormValue("min-cover-lines-drop"))
+	minCoverLinesDrop, err := strconv.Atoi("0" + r.FormValue(covPageParams[MinCoverLinesDrop]))
 	if err != nil {
-		return fmt.Errorf("min-cover-lines-drop should be integer")
+		return fmt.Errorf(covPageParams[MinCoverLinesDrop] + " should be integer")
 	}
 
 	var style template.CSS
@@ -182,18 +236,18 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques
 	if nsConfig.Coverage == nil || nsConfig.Coverage.WebGitURI == "" {
 		return ErrClientNotFound
 	}
-	dateToStr := r.FormValue("dateto")
-	periodType := r.FormValue("period")
-	targetCommit := r.FormValue("commit")
-	kernelFilePath := r.FormValue("filepath")
-	manager := r.FormValue("manager")
+	dateToStr := r.FormValue(covPageParams[DateTo])
+	periodType := r.FormValue(covPageParams[PeriodType])
+	targetCommit := r.FormValue(covPageParams[CommitHash])
+	kernelFilePath := r.FormValue(covPageParams[FilePath])
+	manager := r.FormValue(covPageParams[ManagerName])
 	if err := validator.AnyError("input validation failed",
-		validator.TimePeriodType(periodType, "period"),
-		validator.CommitHash(targetCommit, "commit"),
-		validator.KernelFilePath(kernelFilePath, "filepath"),
+		validator.TimePeriodType(periodType, covPageParams[PeriodType]),
+		validator.CommitHash(targetCommit, covPageParams[CommitHash]),
+		validator.KernelFilePath(kernelFilePath, covPageParams[FilePath]),
 		validator.AnyOk(
-			validator.Allowlisted(manager, []string{"", "*"}, "manager"),
-			validator.ManagerName(manager, "manager")),
+			validator.Allowlisted(manager, []string{"", "*"}, covPageParams[ManagerName]),
+			validator.ManagerName(manager, covPageParams[ManagerName])),
 	); err != nil {
 		return fmt.Errorf("%w: %w", err, ErrClientBadRequest)
 	}
@@ -205,7 +259,7 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques
 	if err != nil {
 		return fmt.Errorf("coveragedb.MakeTimePeriod: %w", err)
 	}
-	onlyUnique := r.FormValue("unique-only") == "1"
+	onlyUnique := r.FormValue(covPageParams[UniqueOnly]) == "1"
 	mainNsRepo, _ := nsConfig.mainRepoBranch()
 	client := GetCoverageDBClient(c)
 	if client == nil {
@@ -268,7 +322,7 @@ func handleCoverageGraph(c context.Context, w http.ResponseWriter, r *http.Reque
 	if nsConfig.Coverage == nil {
 		return ErrClientNotFound
 	}
-	periodType := r.FormValue("period")
+	periodType := r.FormValue(covPageParams[PeriodType])
 	if periodType == "" {
 		periodType = coveragedb.QuarterPeriod
 	}
diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml
index 2734c36df..182714daa 100644
--- a/dashboard/app/cron.yaml
+++ b/dashboard/app/cron.yaml
@@ -33,3 +33,9 @@ cron:
 # Export reproducers every week.
 - url: /cron/batch_db_export
   schedule: every saturday 00:00
+# Monthly coverage reports are regenerated every week.
+# Coverage data propagation may take up to ~48 hours.
+# 7 days + 2 days = 9 days is the maximum expected delay to guarantee monthly coverage data will not change.
+# We use 15 for convenience here.
+- url: /cron/email_coverage_reports
+  schedule: 15 of month 00:00
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 635af8cd2..dc7fa9eb9 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -20,7 +20,10 @@ import (
 	"text/tabwriter"
 	"time"
 
+	"cloud.google.com/go/civil"
 	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/cover"
+	"github.com/google/syzkaller/pkg/coveragedb"
 	"github.com/google/syzkaller/pkg/email"
 	"github.com/google/syzkaller/pkg/email/lore"
 	"github.com/google/syzkaller/pkg/html"
@@ -34,6 +37,7 @@ import (
 // Email reporting interface.
 
 func initEmailReporting() {
+	http.HandleFunc("/cron/email_coverage_reports", handleCoverageReports)
 	http.HandleFunc("/cron/email_poll", handleEmailPoll)
 	http.HandleFunc("/_ah/mail/", handleIncomingMail)
 	http.HandleFunc("/_ah/bounce", handleEmailBounce)
@@ -102,6 +106,115 @@ func (cfg *EmailConfig) Validate() error {
 	return nil
 }
 
+// handleCoverageReports sends a coverage report for the two full months preceding the current one.
+// Assuming it is called June 15, the monthly report will cover April-May diff.
+func handleCoverageReports(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	curHostPort := r.URL.Host
+	targetDate := civil.DateOf(timeNow(ctx)).AddMonths(-1)
+	periods, err := coveragedb.GenNPeriodsTill(2, targetDate, "month")
+	if err != nil {
+		msg := fmt.Sprintf("error generating coverage report: %s", err.Error())
+		log.Errorf(ctx, "%s", msg)
+		http.Error(w, "%s: %w", http.StatusBadRequest)
+		return
+	}
+	wg := sync.WaitGroup{}
+	for nsName, nsConfig := range getConfig(ctx).Namespaces {
+		if nsConfig.Coverage == nil || nsConfig.Coverage.EmailRegressionsTo == "" {
+			continue
+		}
+		emailTo := nsConfig.Coverage.EmailRegressionsTo
+		minDrop := defaultRegressionThreshold
+		if nsConfig.Coverage.RegressionThreshold > 0 {
+			minDrop = nsConfig.Coverage.RegressionThreshold
+		}
+
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err := sendNsCoverageReport(ctx, nsName, emailTo, curHostPort, periods, minDrop); err != nil {
+				msg := fmt.Sprintf("error generating coverage report for ns '%s': %s", nsName, err.Error())
+				log.Errorf(ctx, "%s", msg)
+				return
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func sendNsCoverageReport(ctx context.Context, ns, email, domain string,
+	period []coveragedb.TimePeriod, minDrop int) error {
+	var days int
+	for _, p := range period {
+		days += p.Days
+	}
+	periodFrom := fmt.Sprintf("%s %d", period[0].DateTo.Month.String(), period[0].DateTo.Year)
+	periodTo := fmt.Sprintf("%s %d", period[1].DateTo.Month.String(), period[1].DateTo.Year)
+	table, err := coverageTable(ctx, ns, period, minDrop)
+	if err != nil {
+		return fmt.Errorf("coverageTable: %w", err)
+	}
+	args := struct {
+		Namespace      string
+		PeriodFrom     string
+		PeriodFromDays int
+		PeriodTo       string
+		PeriodToDays   int
+		Link           string
+		Table          string
+	}{
+		Namespace:      ns,
+		PeriodFrom:     periodFrom,
+		PeriodFromDays: period[0].Days,
+		PeriodTo:       periodTo,
+		PeriodToDays:   period[1].Days,
+		Link: fmt.Sprintf("https://%s%s", domain,
+			coveragePageLink(ns, period[1].Type, period[1].DateTo.String(), minDrop, 2, true)),
+		Table: table,
+	}
+	title := fmt.Sprintf("%s coverage regression (%s)->(%s)", ns, periodFrom, periodTo)
+	err = sendMailTemplate(ctx, &mailSendParams{
+		templateName: "mail_ns_coverage.txt",
+		templateArg:  args,
+		title:        title,
+		cfg: &EmailConfig{
+			Email: email,
+		},
+		reportID: "coverage-report",
+	})
+	if err != nil {
+		err2 := fmt.Errorf("error generating coverage report: %w", err)
+		log.Errorf(ctx, "%s", err2.Error())
+		return err2
+	}
+	return nil
+}
+
+func coverageTable(ctx context.Context, ns string, fromTo []coveragedb.TimePeriod, minDrop int) (string, error) {
+	covAndDates, err := coveragedb.FilesCoverageWithDetails(
+		ctx,
+		GetCoverageDBClient(ctx),
+		&coveragedb.SelectScope{
+			Ns:      ns,
+			Periods: fromTo,
+		},
+		false)
+	if err != nil {
+		return "", fmt.Errorf("coveragedb.FilesCoverageWithDetails: %w", err)
+	}
+	templData := cover.FilesCoverageToTemplateData(covAndDates)
+	cover.FormatResult(templData, cover.Format{
+		OrderByCoveredLinesDrop:   true,
+		FilterMinCoveredLinesDrop: minDrop,
+	})
+	res := "Blocks diff,\tPath\n"
+	templData.Root.Visit(func(path string, summary int64) {
+		res += fmt.Sprintf("% 11d\t%s\n", summary, path)
+	})
+	return res, nil
+}
+
 // handleEmailPoll is called by cron and sends emails for new bugs, if any.
 func handleEmailPoll(w http.ResponseWriter, r *http.Request) {
 	c := appengine.NewContext(r)
diff --git a/dashboard/app/reporting_test.go b/dashboard/app/reporting_test.go
index 36ddcc4d1..c5c95101d 100644
--- a/dashboard/app/reporting_test.go
+++ b/dashboard/app/reporting_test.go
@@ -14,9 +14,12 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/coveragedb"
+	"github.com/google/syzkaller/pkg/coveragedb/mocks"
 	"github.com/google/syzkaller/pkg/email"
 	"github.com/google/syzkaller/sys/targets"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 func TestReportBug(t *testing.T) {
@@ -1365,3 +1368,53 @@ func TestReportRevokedBisectCrash(t *testing.T) {
 	// We already reported that the bug has a reproducer.
 	client.pollBugs(0)
 }
+
+func TestCoverageRegression(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	mTran1 := mocks.NewReadOnlyTransaction(t)
+	mTran1.On("Query", mock.Anything, mock.Anything).
+		Return(newRowIteratorMock(t, []*coveragedb.FileCoverageWithDetails{
+			{
+				Filepath:     "file_name.c",
+				Instrumented: 100,
+				Covered:      100,
+			},
+		})).Once()
+
+	mTran2 := mocks.NewReadOnlyTransaction(t)
+	mTran2.On("Query", mock.Anything, mock.Anything).
+		Return(newRowIteratorMock(t, []*coveragedb.FileCoverageWithDetails{
+			{
+				Filepath:     "file_name.c",
+				Instrumented: 0,
+			},
+		})).Once()
+
+	m := mocks.NewSpannerClient(t)
+	m.On("Single").
+		Return(mTran1).Once()
+	m.On("Single").
+		Return(mTran2).Once()
+
+	c.transformContext = func(ctx context.Context) context.Context {
+		return SetCoverageDBClient(ctx, m)
+	}
+	_, err := c.AuthGET(AccessAdmin, "/cron/email_coverage_reports")
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(c.emailSink))
+	msg := <-c.emailSink
+	assert.Equal(t, []string{"test@test.test"}, msg.To)
+	assert.Equal(t, "coverage-tests coverage regression (November 1999)->(December 1999)", msg.Subject)
+	wantLink := "https:///coverage-tests/coverage?" +
+		"dateto=1999-12-31&min-cover-lines-drop=1&order-by-cover-lines-drop=1&period=month&period_count=2"
+	assert.Equal(t, `Regressions happened in 'coverage-tests' from November 1999 (30 days) to December 1999 (31 days).
+Web version: `+wantLink+`
+
+Blocks diff,	Path
+       -100	
+       -100	/file_name.c
+
+`, msg.Body)
+}
diff --git a/dashboard/app/templates/mail_ns_coverage.txt b/dashboard/app/templates/mail_ns_coverage.txt
new file mode 100644
index 000000000..05eb7f934
--- /dev/null
+++ b/dashboard/app/templates/mail_ns_coverage.txt
@@ -0,0 +1,4 @@
+Regressions happened in '{{.Namespace}}' from {{.PeriodFrom}} ({{.PeriodFromDays}} days) to {{.PeriodTo}} ({{.PeriodToDays}} days).
+Web version: {{.Link}}
+
+{{.Table}}