dashboard/app: aggregate months and quarters to db

1 files changed, 71 insertions, 13 deletions

diff --git a/dashboard/app/coverage_batch.go b/dashboard/app/coverage_batch.go
index ac2cbcc66..853e9c25c 100644
--- a/dashboard/app/coverage_batch.go
+++ b/dashboard/app/coverage_batch.go
@@ -13,6 +13,7 @@ import (
 	"cloud.google.com/go/batch/apiv1/batchpb"
 	"cloud.google.com/go/bigquery"
 	"cloud.google.com/go/civil"
+	"github.com/google/syzkaller/pkg/spanner/coveragedb"
 	"github.com/google/uuid"
 	"google.golang.org/api/iterator"
 	"google.golang.org/appengine/v2"
@@ -25,7 +26,7 @@ func initCoverageBatches() {
 }
 
 const (
-	daysPerBatch        = 5
+	runsPerBatch        = 5
 	daysToMerge         = 7
 	batchTimeoutSeconds = 60 * 60 * 6
 )
@@ -41,21 +42,35 @@ func handleBatchCoverage(w http.ResponseWriter, r *http.Request) {
 			log.Errorf(ctx, "can't find default repo or branch for ns %s", ns)
 			continue
 		}
-		dates, err := nsDatesToMerge(ctx, ns, daysToMerge, daysPerBatch)
+		periods, err := nsDatesToMerge(ctx, ns, daysToMerge, runsPerBatch)
 		if err != nil {
 			log.Errorf(ctx, "failed nsDatesToMerge(): %s", err)
-			continue
 		}
-		if len(dates) == 0 {
+		daysAvailable, rowsAvailable, err := nsDataAvailable(ctx, ns)
+		if err != nil {
+			log.Errorf(ctx, "failed nsDataAvailable(%s): %s", ns, err)
+		}
+		periodsMerged, rowsMerged, err := coveragedb.NsDataMerged(ctx, "syzkaller", ns)
+		if err != nil {
+			log.Errorf(ctx, "failed coveragedb.NsDataMerged(%s): %s", ns, err)
+		}
+		periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged,
+			&coveragedb.DayPeriodOps{})...)
+		periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged,
+			&coveragedb.MonthPeriodOps{})...)
+		periods = append(periods, coveragedb.PeriodsToMerge(daysAvailable, periodsMerged, rowsAvailable, rowsMerged,
+			&coveragedb.QuarterPeriodOps{})...)
+		if len(periods) == 0 {
 			log.Infof(ctx, "there is no new coverage for merging available in %s", ns)
 			continue
 		}
+		periods = coveragedb.AtMostNLatestPeriods(periods, runsPerBatch)
 		nsCovConfig := nsConfig.Coverage
 		if err := createScriptJob(
 			ctx,
 			nsCovConfig.BatchProject,
 			nsCovConfig.BatchServiceAccount,
-			batchScript(ns, repo, branch, 7, dates,
+			batchScript(ns, repo, branch, periods,
 				nsCovConfig.JobInitScript,
 				nsCovConfig.SyzEnvInitScript,
 				nsCovConfig.DashboardClientName),
@@ -65,7 +80,7 @@ func handleBatchCoverage(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func batchScript(ns, repo, branch string, days int, datesTo []civil.Date,
+func batchScript(ns, repo, branch string, periods []coveragedb.TimePeriod,
 	jobInitScript, syzEnvInitScript, clientName string) string {
 	if clientName == "" {
 		clientName = defaultDashboardClientName
@@ -78,14 +93,14 @@ func batchScript(ns, repo, branch string, days int, datesTo []civil.Date,
 	if syzEnvInitScript != "" {
 		script += syzEnvInitScript + "; "
 	}
-	for _, dateTo := range datesTo {
+	for _, period := range periods {
 		script += "./tools/syz-bq.sh" +
 			" -w ../workdir-cover-aggregation/" +
 			" -n " + ns +
 			" -r " + repo +
 			" -b " + branch +
-			" -d " + strconv.Itoa(days) +
-			" -t " + dateTo.String() +
+			" -d " + strconv.Itoa(period.Days) +
+			" -t " + period.DateTo.String() +
 			" -c " + clientName +
 			" 2>&1; " // we don't want stderr output to be logged as errors
 	}
@@ -173,7 +188,9 @@ func createScriptJob(ctx context.Context, projectID, serviceAccount, script stri
 	return nil
 }
 
-func nsDatesToMerge(ctx context.Context, ns string, days, maxRecords int64) ([]civil.Date, error) {
+// TODO: remove once coverage is switched to months and quarters
+// Reason: SQL is hard to craft, hard to reuse and hard to test.
+func nsDatesToMerge(ctx context.Context, ns string, days, maxRecords int) ([]coveragedb.TimePeriod, error) {
 	client, err := bigquery.NewClient(ctx, "syzkaller")
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize bigquery client: %w", err)
@@ -232,7 +249,7 @@ func nsDatesToMerge(ctx context.Context, ns string, days, maxRecords int64) ([]c
 		return nil, fmt.Errorf("failed to Read() from bigquery: %w", err)
 	}
 
-	var dates []civil.Date
+	var periods []coveragedb.TimePeriod
 	for {
 		var values struct {
 			Dateto civil.Date
@@ -244,8 +261,49 @@ func nsDatesToMerge(ctx context.Context, ns string, days, maxRecords int64) ([]c
 		if err != nil {
 			return nil, fmt.Errorf("failed to it.Next() bigquery records: %w", err)
 		}
-		dates = append(dates, values.Dateto)
+		periods = append(periods, coveragedb.TimePeriod{DateTo: values.Dateto, Days: days})
 	}
 
-	return dates, nil
+	return periods, nil
+}
+
+func nsDataAvailable(ctx context.Context, ns string) ([]coveragedb.TimePeriod, []int64, error) {
+	client, err := bigquery.NewClient(ctx, "syzkaller")
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to initialize bigquery client: %w", err)
+	}
+	if err := client.EnableStorageReadClient(ctx); err != nil {
+		return nil, nil, fmt.Errorf("failed to client.EnableStorageReadClient: %w", err)
+	}
+	q := client.Query(fmt.Sprintf(`
+	SELECT
+		PARSE_DATE('%%Y%%m%%d', partition_id) as partitiondate,
+		total_rows as records
+	FROM
+		syzkaller.syzbot_coverage.INFORMATION_SCHEMA.PARTITIONS
+	WHERE table_name LIKE '%s'
+	`, ns))
+	it, err := q.Read(ctx)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to Read() from bigquery: %w", err)
+	}
+
+	var periods []coveragedb.TimePeriod
+	var recordsCount []int64
+	for {
+		var values struct {
+			PartitionDate civil.Date
+			Records       int64
+		}
+		err = it.Next(&values)
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to it.Next() bigquery records: %w", err)
+		}
+		periods = append(periods, coveragedb.TimePeriod{DateTo: values.PartitionDate, Days: 1})
+		recordsCount = append(recordsCount, values.Records)
+	}
+	return periods, recordsCount, nil
 }