From b03dfde25a58c76d9fc303af81f1e6102c4d73cb Mon Sep 17 00:00:00 2001
From: Taras Madan <tarasmadan@google.com>
Date: Sun, 7 Jul 2024 14:37:08 +0200
Subject: tools: normalize covermerger db

---
 tools/syz-bq.sh                          | 20 ++++++++++----------
 tools/syz-covermerger/db.go              | 29 ++++++++++++++---------------
 tools/syz-covermerger/syz_covermerger.go |  2 +-
 3 files changed, 25 insertions(+), 26 deletions(-)

(limited to 'tools')

diff --git a/tools/syz-bq.sh b/tools/syz-bq.sh
index 7bf33bbca..4bdf0ccd8 100755
--- a/tools/syz-bq.sh
+++ b/tools/syz-bq.sh
@@ -49,22 +49,20 @@ then
   exit
 fi
 
+db="coverage"
+
 # it also allows to early check gcloud credentials
 echo "making sure spanner table 'files' exists"
 create_table=$( echo -n '
 CREATE TABLE IF NOT EXISTS
   files (
-    "namespace" text,
-    "repo" text,
-    "commit" text,
+    "session" text,
     "filepath" text,
-    "duration" bigint,
-    "dateto" date,
     "instrumented" bigint,
     "covered" bigint,
   PRIMARY KEY
-    (duration, dateto, commit, filepath) );')
-gcloud spanner databases ddl update coverage --instance=syzbot --project=syzkaller \
+    (session, filepath) );')
+gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \
  --ddl="$create_table"
 
 echo "making sure spanner table 'merge_history' exists"
@@ -73,13 +71,15 @@ CREATE TABLE IF NOT EXISTS
   merge_history (
     "namespace" text,
     "repo" text,
-    "commit" text,
     "duration" bigint,
     "dateto" date,
+    "session" text,
+    "time" timestamptz,
+    "commit" text,
     "totalrows" bigint,
   PRIMARY KEY
-    (duration, dateto, commit) );')
-gcloud spanner databases ddl update coverage --instance=syzbot --project=syzkaller \
+    (namespace, repo, duration, dateto) );')
+gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \
  --ddl="$create_table"
 
 echo "Workdir: $workdir"
diff --git a/tools/syz-covermerger/db.go b/tools/syz-covermerger/db.go
index 7ba72702d..30e51365d 100644
--- a/tools/syz-covermerger/db.go
+++ b/tools/syz-covermerger/db.go
@@ -6,25 +6,25 @@ package main
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"cloud.google.com/go/civil"
 	"cloud.google.com/go/spanner"
+	"github.com/google/uuid"
 )
 
 // TODO: move to dashAPI once tested? I'm not sure we'll benefit.
 
-type DBRecord struct {
-	Namespace    string
-	Repo         string
-	Commit       string
-	Duration     int64
-	DateTo       civil.Date
+type DBFilesRecord struct {
+	Session      string
 	FilePath     string
 	Instrumented int64
 	Covered      int64
 }
 
 type DBHistoryRecord struct {
+	Session   string
+	Time      time.Time
 	Namespace string
 	Repo      string
 	Commit    string
@@ -34,22 +34,19 @@ type DBHistoryRecord struct {
 }
 
 func saveToSpanner(ctx context.Context, projectID string, coverage map[string]*Coverage,
-	template *DBRecord, totalRows int64) {
+	template *DBHistoryRecord, totalRows int64) {
 	client, err := spanner.NewClient(ctx, "projects/"+projectID+"/instances/syzbot/databases/coverage")
 	if err != nil {
 		panic(fmt.Sprintf("spanner.NewClient() failed: %s", err.Error()))
 	}
 	defer client.Close()
 
+	session := uuid.New().String()
 	mutations := []*spanner.Mutation{}
 	for filePath, record := range coverage {
 		var insert *spanner.Mutation
-		if insert, err = spanner.InsertOrUpdateStruct("files", &DBRecord{
-			Namespace:    template.Namespace,
-			Repo:         template.Repo,
-			Commit:       template.Commit,
-			Duration:     template.Duration,
-			DateTo:       template.DateTo,
+		if insert, err = spanner.InsertOrUpdateStruct("files", &DBFilesRecord{
+			Session:      session,
 			FilePath:     filePath,
 			Instrumented: record.Instrumented,
 			Covered:      record.Covered,
@@ -57,10 +54,10 @@ func saveToSpanner(ctx context.Context, projectID string, coverage map[string]*C
 			panic(fmt.Sprintf("failed to spanner.InsertStruct(): %s", err.Error()))
 		}
 		mutations = append(mutations, insert)
-		// 80k mutations is a DB limit. 7 fields * 1k records is apx 7k mutations
+		// 80k mutations is a DB limit. 4 fields * 2k records is apx 8k mutations
 		// let keep this value 10x lower to have a room for indexes
 		// indexes update are also counted
-		if len(mutations) > 1000 {
+		if len(mutations) > 2000 {
 			if _, err = client.Apply(ctx, mutations); err != nil {
 				panic(fmt.Sprintf("failed to spanner.Apply(inserts): %s", err.Error()))
 			}
@@ -70,6 +67,8 @@ func saveToSpanner(ctx context.Context, projectID string, coverage map[string]*C
 
 	var historyInsert *spanner.Mutation
 	if historyInsert, err = spanner.InsertOrUpdateStruct("merge_history", &DBHistoryRecord{
+		Session:   session,
+		Time:      time.Now(),
 		Namespace: template.Namespace,
 		Repo:      template.Repo,
 		Commit:    template.Commit,
diff --git a/tools/syz-covermerger/syz_covermerger.go b/tools/syz-covermerger/syz_covermerger.go
index 15aab4abb..69e408da2 100644
--- a/tools/syz-covermerger/syz_covermerger.go
+++ b/tools/syz-covermerger/syz_covermerger.go
@@ -59,7 +59,7 @@ func main() {
 		}
 		coverage, _, _ := mergeResultsToCoverage(mergeResult)
 		saveToSpanner(context.Background(), *flagProjectID, coverage,
-			&DBRecord{
+			&DBHistoryRecord{
 				Namespace: *flagNamespace,
 				Repo:      *flagRepo,
 				Commit:    *flagCommit,
-- 
cgit mrf-deployment