syz-covermerger: store subsystems details in spanner

5 files changed, 101 insertions, 49 deletions

diff --git a/pkg/cover/heatmap.go b/pkg/cover/heatmap.go
index 1cdf00428..9a38a0ef8 100644
--- a/pkg/cover/heatmap.go
+++ b/pkg/cover/heatmap.go
@@ -17,7 +17,6 @@ import (
 	"cloud.google.com/go/civil"
 	"cloud.google.com/go/spanner"
 	"github.com/google/syzkaller/pkg/spanner/coveragedb"
-	"github.com/google/syzkaller/pkg/subsystem"
 	_ "github.com/google/syzkaller/pkg/subsystem/lists"
 	"golang.org/x/exp/maps"
 	"google.golang.org/api/iterator"
@@ -86,14 +85,15 @@ func (thm *templateHeatmapRow) prepareDataFor(dates []civil.Date) {
 	}
 }
 
-type fileCoverageAndDate struct {
+type fileCoverageWithDetails struct {
 	Filepath     string
 	Instrumented int64
 	Covered      int64
 	Dateto       civil.Date
+	Subsystems   []string
 }
 
-func filesCoverageToTemplateData(fCov []*fileCoverageAndDate) *templateHeatmap {
+func filesCoverageToTemplateData(fCov []*fileCoverageWithDetails) *templateHeatmap {
 	res := templateHeatmap{
 		Root: &templateHeatmapRow{
 			builder:      map[string]*templateHeatmapRow{},
@@ -123,8 +123,8 @@ func filesCoverageToTemplateData(fCov []*fileCoverageAndDate) *templateHeatmap {
 	return &res
 }
 
-func filesCoverageAndDates(ctx context.Context, projectID, ns string, fromDate, toDate civil.Date,
-) ([]*fileCoverageAndDate, error) {
+func filesCoverageWithDetails(ctx context.Context, projectID, ns string, fromDate, toDate civil.Date,
+) ([]*fileCoverageWithDetails, error) {
 	client, err := coveragedb.NewClient(ctx, projectID)
 	if err != nil {
 		return nil, fmt.Errorf("spanner.NewClient() failed: %s", err.Error())
@@ -137,10 +137,15 @@ select
   dateto,
   instrumented,
   covered,
-  filepath
-from merge_history join files
-  on merge_history.session = files.session
-where namespace=$1 and dateto>=$2 and dateto<=$3
+  files.filepath,
+  subsystems
+from merge_history
+  join files
+    on merge_history.session = files.session
+  join file_subsystems
+    on merge_history.namespace = file_subsystems.namespace and files.filepath = file_subsystems.filepath
+where
+  merge_history.namespace=$1 and dateto>=$2 and dateto<=$3
 `,
 		Params: map[string]interface{}{
 			"p1": ns,
@@ -151,7 +156,7 @@ where namespace=$1 and dateto>=$2 and dateto<=$3
 
 	iter := client.Single().Query(ctx, stmt)
 	defer iter.Stop()
-	res := []*fileCoverageAndDate{}
+	res := []*fileCoverageWithDetails{}
 	for {
 		row, err := iter.Next()
 		if err == iterator.Done {
@@ -160,7 +165,7 @@ where namespace=$1 and dateto>=$2 and dateto<=$3
 		if err != nil {
 			return nil, fmt.Errorf("failed to iter.Next() spanner DB: %w", err)
 		}
-		var r fileCoverageAndDate
+		var r fileCoverageWithDetails
 		if err = row.ToStruct(&r); err != nil {
 			return nil, fmt.Errorf("failed to row.ToStruct() spanner DB: %w", err)
 		}
@@ -188,9 +193,9 @@ func DoHeatMap(w io.Writer, projectID, ns string, dateFrom, dateTo civil.Date) e
 
 func DoHeatMapStyleBodyJS(projectID, ns string, dateFrom, dateTo civil.Date,
 ) (template.CSS, template.HTML, template.HTML, error) {
-	covAndDates, err := filesCoverageAndDates(context.Background(), projectID, ns, dateFrom, dateTo)
+	covAndDates, err := filesCoverageWithDetails(context.Background(), projectID, ns, dateFrom, dateTo)
 	if err != nil {
-		return "", "", "", fmt.Errorf("failed to filesCoverageAndDates: %w", err)
+		return "", "", "", fmt.Errorf("failed to filesCoverageWithDetails: %w", err)
 	}
 	templateData := filesCoverageToTemplateData(covAndDates)
 	var styles, body, js bytes.Buffer
@@ -209,25 +214,18 @@ func DoHeatMapStyleBodyJS(projectID, ns string, dateFrom, dateTo civil.Date,
 }
 
 func DoSubsystemsHeatMap(w io.Writer, projectID, ns string, dateFrom, dateTo civil.Date) error {
-	covAndDates, err := filesCoverageAndDates(context.Background(), projectID, ns, dateFrom, dateTo)
+	covWithDetails, err := filesCoverageWithDetails(context.Background(), projectID, ns, dateFrom, dateTo)
 	if err != nil {
 		panic(err)
 	}
-	ssMatcher := subsystem.MakePathMatcher(subsystem.GetList("linux"))
-	ssCache := make(map[string][]*subsystem.Subsystem)
-	var ssCovAndDates []*fileCoverageAndDate
-	for _, cad := range covAndDates {
-		sss := ssCache[cad.Filepath]
-		if sss == nil {
-			sss = ssMatcher.Match(cad.Filepath)
-			ssCache[cad.Filepath] = sss
-		}
-		for _, ss := range sss {
-			newRecord := fileCoverageAndDate{
-				Filepath:     ss.Name + "/" + cad.Filepath,
-				Instrumented: cad.Instrumented,
-				Covered:      cad.Covered,
-				Dateto:       cad.Dateto,
+	var ssCovAndDates []*fileCoverageWithDetails
+	for _, cwd := range covWithDetails {
+		for _, ssName := range cwd.Subsystems {
+			newRecord := fileCoverageWithDetails{
+				Filepath:     ssName + "/" + cwd.Filepath,
+				Instrumented: cwd.Instrumented,
+				Covered:      cwd.Covered,
+				Dateto:       cwd.Dateto,
 			}
 			ssCovAndDates = append(ssCovAndDates, &newRecord)
 		}
diff --git a/pkg/cover/heatmap_test.go b/pkg/cover/heatmap_test.go
index d450a5243..3d54f5b40 100644
--- a/pkg/cover/heatmap_test.go
+++ b/pkg/cover/heatmap_test.go
@@ -14,12 +14,12 @@ import (
 func TestFilesCoverageToTemplateData(t *testing.T) {
 	tests := []struct {
 		name  string
-		input []*fileCoverageAndDate
+		input []*fileCoverageWithDetails
 		want  *templateHeatmap
 	}{
 		{
 			name:  "empty input",
-			input: []*fileCoverageAndDate{},
+			input: []*fileCoverageWithDetails{},
 			want: &templateHeatmap{
 				Root: &templateHeatmapRow{
 					Items: []*templateHeatmapRow{},
@@ -28,7 +28,7 @@ func TestFilesCoverageToTemplateData(t *testing.T) {
 		},
 		{
 			name: "single file",
-			input: []*fileCoverageAndDate{
+			input: []*fileCoverageWithDetails{
 				{
 					Filepath:     "file1",
 					Instrumented: 1,
@@ -59,7 +59,7 @@ func TestFilesCoverageToTemplateData(t *testing.T) {
 		},
 		{
 			name: "tree data",
-			input: []*fileCoverageAndDate{
+			input: []*fileCoverageWithDetails{
 				{
 					Filepath:     "dir/file2",
 					Instrumented: 1,
diff --git a/pkg/spanner/coveragedb/coverage.go b/pkg/spanner/coveragedb/coverage.go
index b794806d1..b0ccedf07 100644
--- a/pkg/spanner/coveragedb/coverage.go
+++ b/pkg/spanner/coveragedb/coverage.go
@@ -18,6 +18,12 @@ type FilesRecord struct {
 	Covered      int64
 }
 
+type FileSubsystems struct {
+	Namespace  string
+	FilePath   string
+	Subsystems []string
+}
+
 type HistoryRecord struct {
 	Session   string
 	Time      time.Time
diff --git a/tools/syz-covermerger/db.go b/tools/syz-covermerger/db.go
index e0df6735f..2f347e6a7 100644
--- a/tools/syz-covermerger/db.go
+++ b/tools/syz-covermerger/db.go
@@ -10,6 +10,8 @@ import (
 
 	"cloud.google.com/go/spanner"
 	"github.com/google/syzkaller/pkg/spanner/coveragedb"
+	"github.com/google/syzkaller/pkg/subsystem"
+	_ "github.com/google/syzkaller/pkg/subsystem/lists"
 	"github.com/google/uuid"
 )
 
@@ -23,19 +25,15 @@ func saveToSpanner(ctx context.Context, projectID string, covMap map[string]*Cov
 	}
 	defer client.Close()
 
+	ssMatcher := subsystem.MakePathMatcher(subsystem.GetList("linux"))
+	ssCache := make(map[string][]string)
+
 	session := uuid.New().String()
 	mutations := []*spanner.Mutation{}
 	for filePath, record := range covMap {
-		var insert *spanner.Mutation
-		if insert, err = spanner.InsertOrUpdateStruct("files", &coveragedb.FilesRecord{
-			Session:      session,
-			FilePath:     filePath,
-			Instrumented: record.Instrumented,
-			Covered:      record.Covered,
-		}); err != nil {
-			panic(fmt.Sprintf("failed to spanner.InsertStruct(): %s", err.Error()))
-		}
-		mutations = append(mutations, insert)
+		mutations = append(mutations, fileRecordMutation(session, filePath, record))
+		subsystems := fileSubsystems(filePath, ssMatcher, ssCache)
+		mutations = append(mutations, fileSubsystemsMutation(template.Namespace, filePath, subsystems))
 		// 80k mutations is a DB limit. 4 fields * 2k records is apx 8k mutations
 		// let keep this value 10x lower to have a room for indexes
 		// indexes update are also counted
@@ -46,9 +44,14 @@ func saveToSpanner(ctx context.Context, projectID string, covMap map[string]*Cov
 			mutations = nil
 		}
 	}
+	mutations = append(mutations, historyMutation(session, template, totalRows))
+	if _, err = client.Apply(ctx, mutations); err != nil {
+		panic(fmt.Sprintf("failed to spanner.Apply(inserts): %s", err.Error()))
+	}
+}
 
-	var historyInsert *spanner.Mutation
-	if historyInsert, err = spanner.InsertOrUpdateStruct("merge_history", &coveragedb.HistoryRecord{
+func historyMutation(session string, template *coveragedb.HistoryRecord, totalRows int64) *spanner.Mutation {
+	historyInsert, err := spanner.InsertOrUpdateStruct("merge_history", &coveragedb.HistoryRecord{
 		Session:   session,
 		Time:      time.Now(),
 		Namespace: template.Namespace,
@@ -57,12 +60,45 @@ func saveToSpanner(ctx context.Context, projectID string, covMap map[string]*Cov
 		Duration:  template.Duration,
 		DateTo:    template.DateTo,
 		TotalRows: totalRows,
-	}); err != nil {
+	})
+	if err != nil {
 		panic(fmt.Sprintf("failed to spanner.InsertStruct(): %s", err.Error()))
 	}
-	mutations = append(mutations, historyInsert)
+	return historyInsert
+}
 
-	if _, err = client.Apply(ctx, mutations); err != nil {
-		panic(fmt.Sprintf("failed to spanner.Apply(inserts): %s", err.Error()))
+func fileRecordMutation(session, filePath string, record *Coverage) *spanner.Mutation {
+	insert, err := spanner.InsertOrUpdateStruct("files", &coveragedb.FilesRecord{
+		Session:      session,
+		FilePath:     filePath,
+		Instrumented: record.Instrumented,
+		Covered:      record.Covered,
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to fileRecordMutation(): %s", err.Error()))
+	}
+	return insert
+}
+
+func fileSubsystemsMutation(ns, filePath string, subsystems []string) *spanner.Mutation {
+	insert, err := spanner.InsertOrUpdateStruct("file_subsystems", &coveragedb.FileSubsystems{
+		Namespace:  ns,
+		FilePath:   filePath,
+		Subsystems: subsystems,
+	})
+	if err != nil {
+		panic(fmt.Sprintf("failed to fileSubsystemsMutation(): %s", err.Error()))
+	}
+	return insert
+}
+
+func fileSubsystems(filePath string, ssMatcher *subsystem.PathMatcher, ssCache map[string][]string) []string {
+	sss, cached := ssCache[filePath]
+	if !cached {
+		for _, match := range ssMatcher.Match(filePath) {
+			sss = append(sss, match.Name)
+		}
+		ssCache[filePath] = sss
 	}
+	return sss
 }
diff --git a/tools/syz-covermerger/init_db.sh b/tools/syz-covermerger/init_db.sh
index f7e185221..929eaca32 100644..100755
--- a/tools/syz-covermerger/init_db.sh
+++ b/tools/syz-covermerger/init_db.sh
@@ -35,3 +35,15 @@ CREATE TABLE IF NOT EXISTS
     (namespace, repo, duration, dateto) );')
 gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \
  --ddl="$create_table"
+
+echo "making sure spanner table 'file_subsystems' exists"
+create_table=$( echo -n '
+CREATE TABLE IF NOT EXISTS
+  file_subsystems (
+    "namespace" text,
+    "filepath" text,
+    "subsystems" text[],
+  PRIMARY KEY
+    (namespace, filepath) );')
+gcloud spanner databases ddl update $db --instance=syzbot --project=syzkaller \
+ --ddl="$create_table"