From ce7952f4e369f2440b2bc369868df305c42bf7d6 Mon Sep 17 00:00:00 2001 From: Taras Madan Date: Thu, 10 Apr 2025 14:16:42 +0200 Subject: dashboard/app: send coverage report to ns-defined email We periodically send coverage reports for the regressions detection. --- dashboard/app/app_test.go | 25 ++++++ dashboard/app/config.go | 20 +++++ dashboard/app/coverage.go | 96 ++++++++++++++++++----- dashboard/app/cron.yaml | 6 ++ dashboard/app/reporting_email.go | 113 +++++++++++++++++++++++++++ dashboard/app/reporting_test.go | 53 +++++++++++++ dashboard/app/templates/mail_ns_coverage.txt | 4 + pkg/cover/heatmap.go | 14 +++- pkg/cover/heatmap_test.go | 2 +- 9 files changed, 308 insertions(+), 25 deletions(-) create mode 100644 dashboard/app/templates/mail_ns_coverage.txt diff --git a/dashboard/app/app_test.go b/dashboard/app/app_test.go index 876104f42..445fcf785 100644 --- a/dashboard/app/app_test.go +++ b/dashboard/app/app_test.go @@ -579,6 +579,31 @@ var testConfig = &GlobalConfig{ FindBugOriginTrees: true, RetestMissingBackports: true, }, + "coverage-tests": { + Coverage: &CoverageConfig{ + EmailRegressionsTo: "test@test.test", + RegressionThreshold: 1, + }, + AccessLevel: AccessPublic, + Key: "coveragetestskeycoveragetestskeycoveragetestskey", + Repos: []KernelRepo{ + { + URL: "git://syzkaller.org/test.git", + Branch: "main", + Alias: "main", + }, + }, + Reporting: []Reporting{ + { + Name: "non-public", + DailyLimit: 1000, + Filter: func(bug *Bug) FilterResult { + return FilterReport + }, + Config: &TestConfig{Index: 1}, + }, + }, + }, }, } diff --git a/dashboard/app/config.go b/dashboard/app/config.go index 836344954..96f631e0a 100644 --- a/dashboard/app/config.go +++ b/dashboard/app/config.go @@ -155,6 +155,7 @@ type ACLItem struct { } const defaultDashboardClientName = "coverage-merger" +const defaultRegressionThreshold = 50 type CoverageConfig struct { BatchProject string @@ -167,6 +168,15 @@ type CoverageConfig struct { // WebGitURI specifies where can we get the kernel file source code directly from AppEngine. // It may be the Git or Gerrit compatible repo. WebGitURI string + + // EmailRegressionsTo species the regressions recipient. + // If empty, regression analysis is disabled. + EmailRegressionsTo string + + // RegressionThreshold is a minimal basic block coverage drop in a file. + // The amount of files in the dir and other factors do not matter. + // Defaults to defaultRegressionThreshold. + RegressionThreshold int } // DiscussionEmailConfig defines the correspondence between an email and a DiscussionSource. @@ -585,6 +595,16 @@ func checkNamespace(ns string, cfg *Config, namespaces, clientNames map[string]b checkKernelRepos(ns, cfg, cfg.Repos) checkNamespaceReporting(ns, cfg) checkSubsystems(ns, cfg) + checkCoverageConfig(ns, cfg) +} + +func checkCoverageConfig(ns string, cfg *Config) { + if cfg.Coverage == nil || cfg.Coverage.EmailRegressionsTo == "" { + return + } + if _, err := mail.ParseAddress(cfg.Coverage.EmailRegressionsTo); err != nil { + panic(fmt.Sprintf("bad cfg.Coverage.EmailRegressionsTo in '%s': %s", ns, err.Error())) + } } func checkSubsystems(ns string, cfg *Config) { diff --git a/dashboard/app/coverage.go b/dashboard/app/coverage.go index 305a9fad1..c600aab9e 100644 --- a/dashboard/app/coverage.go +++ b/dashboard/app/coverage.go @@ -17,6 +17,7 @@ import ( "github.com/google/syzkaller/pkg/coveragedb" "github.com/google/syzkaller/pkg/coveragedb/spannerclient" "github.com/google/syzkaller/pkg/covermerger" + "github.com/google/syzkaller/pkg/html/urlutil" "github.com/google/syzkaller/pkg/validator" "google.golang.org/appengine/v2" ) @@ -71,6 +72,59 @@ func handleSubsystemsCoverageHeatmap(c context.Context, w http.ResponseWriter, r return handleHeatmap(c, w, r, cover.DoSubsystemsHeatMapStyleBodyJS) } +type covPageParam int + +const ( + // keep-sorted start + CommitHash covPageParam = iota + DateTo + FilePath + ManagerName + MinCoverLinesDrop + OrderByCoverDrop + PeriodCount + PeriodType + SubsystemName + UniqueOnly + // keep-sorted end +) + +var covPageParams = map[covPageParam]string{ + // keep-sorted start + CommitHash: "commit", + DateTo: "dateto", + FilePath: "filepath", + ManagerName: "manager", + MinCoverLinesDrop: "min-cover-lines-drop", + OrderByCoverDrop: "order-by-cover-lines-drop", + PeriodCount: "period_count", + PeriodType: "period", + SubsystemName: "subsystem", + UniqueOnly: "unique-only", + // keep-sorted end +} + +func coveragePageLink(ns, periodType, dateTo string, minDrop, periodCount int, orderByCoverDrop bool) string { + if periodType == "" { + periodType = coveragedb.MonthPeriod + } + url := "/" + ns + "/coverage" + url = urlutil.SetParam(url, covPageParams[PeriodType], periodType) + if periodCount != 0 { + url = urlutil.SetParam(url, covPageParams[PeriodCount], strconv.Itoa(periodCount)) + } + if dateTo != "" { + url = urlutil.SetParam(url, covPageParams[DateTo], dateTo) + } + if minDrop > 0 { + url = urlutil.SetParam(url, covPageParams[MinCoverLinesDrop], strconv.Itoa(minDrop)) + } + if orderByCoverDrop { + url = urlutil.SetParam(url, covPageParams[OrderByCoverDrop], "1") + } + return url +} + func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f funcStyleBodyJS) error { hdr, err := commonHeader(c, r, w, "") if err != nil { @@ -80,10 +134,10 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f if nsConfig.Coverage == nil { return ErrClientNotFound } - ss := r.FormValue("subsystem") - manager := r.FormValue("manager") + ss := r.FormValue(covPageParams[SubsystemName]) + manager := r.FormValue(covPageParams[ManagerName]) - periodType := r.FormValue("period") + periodType := r.FormValue(covPageParams[PeriodType]) if periodType == "" { periodType = coveragedb.DayPeriod } @@ -94,7 +148,7 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f periodType, ErrClientBadRequest) } - periodCount := r.FormValue("period_count") + periodCount := r.FormValue(covPageParams[PeriodCount]) if periodCount == "" { periodCount = "4" } @@ -104,7 +158,7 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f } dateTo := civil.DateOf(timeNow(c)) - if customDate := r.FormValue("dateto"); customDate != "" { + if customDate := r.FormValue(covPageParams[DateTo]); customDate != "" { if dateTo, err = civil.ParseDate(customDate); err != nil { return fmt.Errorf("civil.ParseDate(%s): %w", customDate, err) } @@ -126,12 +180,12 @@ func handleHeatmap(c context.Context, w http.ResponseWriter, r *http.Request, f slices.Sort(managers) slices.Sort(subsystems) - onlyUnique := r.FormValue("unique-only") == "1" - orderByCoverLinesDrop := r.FormValue("order-by-cover-lines-drop") == "1" + onlyUnique := r.FormValue(covPageParams[UniqueOnly]) == "1" + orderByCoverLinesDrop := r.FormValue(covPageParams[OrderByCoverDrop]) == "1" // Prefixing "0" we don't fail on empty string. - minCoverLinesDrop, err := strconv.Atoi("0" + r.FormValue("min-cover-lines-drop")) + minCoverLinesDrop, err := strconv.Atoi("0" + r.FormValue(covPageParams[MinCoverLinesDrop])) if err != nil { - return fmt.Errorf("min-cover-lines-drop should be integer") + return fmt.Errorf(covPageParams[MinCoverLinesDrop] + " should be integer") } var style template.CSS @@ -182,18 +236,18 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques if nsConfig.Coverage == nil || nsConfig.Coverage.WebGitURI == "" { return ErrClientNotFound } - dateToStr := r.FormValue("dateto") - periodType := r.FormValue("period") - targetCommit := r.FormValue("commit") - kernelFilePath := r.FormValue("filepath") - manager := r.FormValue("manager") + dateToStr := r.FormValue(covPageParams[DateTo]) + periodType := r.FormValue(covPageParams[PeriodType]) + targetCommit := r.FormValue(covPageParams[CommitHash]) + kernelFilePath := r.FormValue(covPageParams[FilePath]) + manager := r.FormValue(covPageParams[ManagerName]) if err := validator.AnyError("input validation failed", - validator.TimePeriodType(periodType, "period"), - validator.CommitHash(targetCommit, "commit"), - validator.KernelFilePath(kernelFilePath, "filepath"), + validator.TimePeriodType(periodType, covPageParams[PeriodType]), + validator.CommitHash(targetCommit, covPageParams[CommitHash]), + validator.KernelFilePath(kernelFilePath, covPageParams[FilePath]), validator.AnyOk( - validator.Allowlisted(manager, []string{"", "*"}, "manager"), - validator.ManagerName(manager, "manager")), + validator.Allowlisted(manager, []string{"", "*"}, covPageParams[ManagerName]), + validator.ManagerName(manager, covPageParams[ManagerName])), ); err != nil { return fmt.Errorf("%w: %w", err, ErrClientBadRequest) } @@ -205,7 +259,7 @@ func handleFileCoverage(c context.Context, w http.ResponseWriter, r *http.Reques if err != nil { return fmt.Errorf("coveragedb.MakeTimePeriod: %w", err) } - onlyUnique := r.FormValue("unique-only") == "1" + onlyUnique := r.FormValue(covPageParams[UniqueOnly]) == "1" mainNsRepo, _ := nsConfig.mainRepoBranch() client := GetCoverageDBClient(c) if client == nil { @@ -268,7 +322,7 @@ func handleCoverageGraph(c context.Context, w http.ResponseWriter, r *http.Reque if nsConfig.Coverage == nil { return ErrClientNotFound } - periodType := r.FormValue("period") + periodType := r.FormValue(covPageParams[PeriodType]) if periodType == "" { periodType = coveragedb.QuarterPeriod } diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml index 2734c36df..182714daa 100644 --- a/dashboard/app/cron.yaml +++ b/dashboard/app/cron.yaml @@ -33,3 +33,9 @@ cron: # Export reproducers every week. - url: /cron/batch_db_export schedule: every saturday 00:00 +# Monthly coverage reports are regenerated every week. +# Coverage data propagation may take up to ~48 hours. +# 7 days + 2 days = 9 days is the maximum expected delay to guarantee monthly coverage data will not change. +# We use 15 for convenience here. +- url: /cron/email_coverage_reports + schedule: 15 of month 00:00 diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go index 635af8cd2..dc7fa9eb9 100644 --- a/dashboard/app/reporting_email.go +++ b/dashboard/app/reporting_email.go @@ -20,7 +20,10 @@ import ( "text/tabwriter" "time" + "cloud.google.com/go/civil" "github.com/google/syzkaller/dashboard/dashapi" + "github.com/google/syzkaller/pkg/cover" + "github.com/google/syzkaller/pkg/coveragedb" "github.com/google/syzkaller/pkg/email" "github.com/google/syzkaller/pkg/email/lore" "github.com/google/syzkaller/pkg/html" @@ -34,6 +37,7 @@ import ( // Email reporting interface. func initEmailReporting() { + http.HandleFunc("/cron/email_coverage_reports", handleCoverageReports) http.HandleFunc("/cron/email_poll", handleEmailPoll) http.HandleFunc("/_ah/mail/", handleIncomingMail) http.HandleFunc("/_ah/bounce", handleEmailBounce) @@ -102,6 +106,115 @@ func (cfg *EmailConfig) Validate() error { return nil } +// handleCoverageReports sends a coverage report for the two full months preceding the current one. +// Assuming it is called June 15, the monthly report will cover April-May diff. +func handleCoverageReports(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + curHostPort := r.URL.Host + targetDate := civil.DateOf(timeNow(ctx)).AddMonths(-1) + periods, err := coveragedb.GenNPeriodsTill(2, targetDate, "month") + if err != nil { + msg := fmt.Sprintf("error generating coverage report: %s", err.Error()) + log.Errorf(ctx, "%s", msg) + http.Error(w, "%s: %w", http.StatusBadRequest) + return + } + wg := sync.WaitGroup{} + for nsName, nsConfig := range getConfig(ctx).Namespaces { + if nsConfig.Coverage == nil || nsConfig.Coverage.EmailRegressionsTo == "" { + continue + } + emailTo := nsConfig.Coverage.EmailRegressionsTo + minDrop := defaultRegressionThreshold + if nsConfig.Coverage.RegressionThreshold > 0 { + minDrop = nsConfig.Coverage.RegressionThreshold + } + + wg.Add(1) + go func() { + defer wg.Done() + if err := sendNsCoverageReport(ctx, nsName, emailTo, curHostPort, periods, minDrop); err != nil { + msg := fmt.Sprintf("error generating coverage report for ns '%s': %s", nsName, err.Error()) + log.Errorf(ctx, "%s", msg) + return + } + }() + } + wg.Wait() +} + +func sendNsCoverageReport(ctx context.Context, ns, email, domain string, + period []coveragedb.TimePeriod, minDrop int) error { + var days int + for _, p := range period { + days += p.Days + } + periodFrom := fmt.Sprintf("%s %d", period[0].DateTo.Month.String(), period[0].DateTo.Year) + periodTo := fmt.Sprintf("%s %d", period[1].DateTo.Month.String(), period[1].DateTo.Year) + table, err := coverageTable(ctx, ns, period, minDrop) + if err != nil { + return fmt.Errorf("coverageTable: %w", err) + } + args := struct { + Namespace string + PeriodFrom string + PeriodFromDays int + PeriodTo string + PeriodToDays int + Link string + Table string + }{ + Namespace: ns, + PeriodFrom: periodFrom, + PeriodFromDays: period[0].Days, + PeriodTo: periodTo, + PeriodToDays: period[1].Days, + Link: fmt.Sprintf("https://%s%s", domain, + coveragePageLink(ns, period[1].Type, period[1].DateTo.String(), minDrop, 2, true)), + Table: table, + } + title := fmt.Sprintf("%s coverage regression (%s)->(%s)", ns, periodFrom, periodTo) + err = sendMailTemplate(ctx, &mailSendParams{ + templateName: "mail_ns_coverage.txt", + templateArg: args, + title: title, + cfg: &EmailConfig{ + Email: email, + }, + reportID: "coverage-report", + }) + if err != nil { + err2 := fmt.Errorf("error generating coverage report: %w", err) + log.Errorf(ctx, "%s", err2.Error()) + return err2 + } + return nil +} + +func coverageTable(ctx context.Context, ns string, fromTo []coveragedb.TimePeriod, minDrop int) (string, error) { + covAndDates, err := coveragedb.FilesCoverageWithDetails( + ctx, + GetCoverageDBClient(ctx), + &coveragedb.SelectScope{ + Ns: ns, + Periods: fromTo, + }, + false) + if err != nil { + return "", fmt.Errorf("coveragedb.FilesCoverageWithDetails: %w", err) + } + templData := cover.FilesCoverageToTemplateData(covAndDates) + cover.FormatResult(templData, cover.Format{ + OrderByCoveredLinesDrop: true, + FilterMinCoveredLinesDrop: minDrop, + }) + res := "Blocks diff,\tPath\n" + templData.Root.Visit(func(path string, summary int64) { + res += fmt.Sprintf("% 11d\t%s\n", summary, path) + }) + return res, nil +} + // handleEmailPoll is called by cron and sends emails for new bugs, if any. func handleEmailPoll(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) diff --git a/dashboard/app/reporting_test.go b/dashboard/app/reporting_test.go index 36ddcc4d1..c5c95101d 100644 --- a/dashboard/app/reporting_test.go +++ b/dashboard/app/reporting_test.go @@ -14,9 +14,12 @@ import ( "github.com/google/go-cmp/cmp" "github.com/google/syzkaller/dashboard/dashapi" + "github.com/google/syzkaller/pkg/coveragedb" + "github.com/google/syzkaller/pkg/coveragedb/mocks" "github.com/google/syzkaller/pkg/email" "github.com/google/syzkaller/sys/targets" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) func TestReportBug(t *testing.T) { @@ -1365,3 +1368,53 @@ func TestReportRevokedBisectCrash(t *testing.T) { // We already reported that the bug has a reproducer. client.pollBugs(0) } + +func TestCoverageRegression(t *testing.T) { + c := NewCtx(t) + defer c.Close() + + mTran1 := mocks.NewReadOnlyTransaction(t) + mTran1.On("Query", mock.Anything, mock.Anything). + Return(newRowIteratorMock(t, []*coveragedb.FileCoverageWithDetails{ + { + Filepath: "file_name.c", + Instrumented: 100, + Covered: 100, + }, + })).Once() + + mTran2 := mocks.NewReadOnlyTransaction(t) + mTran2.On("Query", mock.Anything, mock.Anything). + Return(newRowIteratorMock(t, []*coveragedb.FileCoverageWithDetails{ + { + Filepath: "file_name.c", + Instrumented: 0, + }, + })).Once() + + m := mocks.NewSpannerClient(t) + m.On("Single"). + Return(mTran1).Once() + m.On("Single"). + Return(mTran2).Once() + + c.transformContext = func(ctx context.Context) context.Context { + return SetCoverageDBClient(ctx, m) + } + _, err := c.AuthGET(AccessAdmin, "/cron/email_coverage_reports") + assert.NoError(t, err) + assert.Equal(t, 1, len(c.emailSink)) + msg := <-c.emailSink + assert.Equal(t, []string{"test@test.test"}, msg.To) + assert.Equal(t, "coverage-tests coverage regression (November 1999)->(December 1999)", msg.Subject) + wantLink := "https:///coverage-tests/coverage?" + + "dateto=1999-12-31&min-cover-lines-drop=1&order-by-cover-lines-drop=1&period=month&period_count=2" + assert.Equal(t, `Regressions happened in 'coverage-tests' from November 1999 (30 days) to December 1999 (31 days). +Web version: `+wantLink+` + +Blocks diff, Path + -100 + -100 /file_name.c + +`, msg.Body) +} diff --git a/dashboard/app/templates/mail_ns_coverage.txt b/dashboard/app/templates/mail_ns_coverage.txt new file mode 100644 index 000000000..05eb7f934 --- /dev/null +++ b/dashboard/app/templates/mail_ns_coverage.txt @@ -0,0 +1,4 @@ +Regressions happened in '{{.Namespace}}' from {{.PeriodFrom}} ({{.PeriodFromDays}} days) to {{.PeriodTo}} ({{.PeriodToDays}} days). +Web version: {{.Link}} + +{{.Table}} diff --git a/pkg/cover/heatmap.go b/pkg/cover/heatmap.go index e6458c36b..672312fee 100644 --- a/pkg/cover/heatmap.go +++ b/pkg/cover/heatmap.go @@ -145,12 +145,20 @@ func (thm *templateHeatmapRow) prepareDataFor(pageColumns []pageColumnTarget) { } } +func (thm *templateHeatmapRow) Visit(v func(string, int64), path ...string) { + curPath := append(path, thm.Name) + v(strings.Join(curPath, "/"), thm.Summary) + for _, item := range thm.Items { + item.Visit(v, curPath...) + } +} + type pageColumnTarget struct { TimePeriod coveragedb.TimePeriod Commit string } -func filesCoverageToTemplateData(fCov []*coveragedb.FileCoverageWithDetails) *templateHeatmap { +func FilesCoverageToTemplateData(fCov []*coveragedb.FileCoverageWithDetails) *templateHeatmap { res := templateHeatmap{ Root: &templateHeatmapRow{ IsDir: true, @@ -223,7 +231,7 @@ func DoHeatMapStyleBodyJS( if err != nil { return "", "", "", fmt.Errorf("failed to FilesCoverageWithDetails: %w", err) } - templData := filesCoverageToTemplateData(covAndDates) + templData := FilesCoverageToTemplateData(covAndDates) templData.Subsystems = sss templData.Managers = managers FormatResult(templData, dataFilters) @@ -252,7 +260,7 @@ func DoSubsystemsHeatMapStyleBodyJS( ssCovAndDates = append(ssCovAndDates, &newRecord) } } - templData := filesCoverageToTemplateData(ssCovAndDates) + templData := FilesCoverageToTemplateData(ssCovAndDates) templData.Managers = managers FormatResult(templData, format) return stylesBodyJSTemplate(templData) diff --git a/pkg/cover/heatmap_test.go b/pkg/cover/heatmap_test.go index e4b490f23..42bcea4ec 100644 --- a/pkg/cover/heatmap_test.go +++ b/pkg/cover/heatmap_test.go @@ -150,7 +150,7 @@ func TestFilesCoverageToTemplateData(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - got := filesCoverageToTemplateData(test.input) + got := FilesCoverageToTemplateData(test.input) assert.EqualExportedValues(t, test.want, got) }) } -- cgit mrf-deployment