syz-cluster: store session test logs

Record the logs from the build and fuzzing steps.

11 files changed, 92 insertions, 28 deletions

diff --git a/pkg/build/linux.go b/pkg/build/linux.go
index 73baed5d8..47c84c551 100644
--- a/pkg/build/linux.go
+++ b/pkg/build/linux.go
@@ -177,7 +177,8 @@ func runMake(params Params, extraArgs ...string) error {
 		"KERNELVERSION=syzkaller",
 		"LOCALVERSION=-syzkaller",
 	)
-	_, err := osutil.Run(time.Hour, cmd)
+	output, err := osutil.Run(time.Hour, cmd)
+	params.Tracer.Log("Build log:\n%s", output)
 	return err
 }
 
diff --git a/syz-cluster/controller/api.go b/syz-cluster/controller/api.go
index e5cb18a27..3e25b2ba4 100644
--- a/syz-cluster/controller/api.go
+++ b/syz-cluster/controller/api.go
@@ -102,7 +102,7 @@ func (c ControllerAPI) uploadTest(w http.ResponseWriter, r *http.Request) {
 	if req == nil {
 		return
 	}
-	// TODO: add parameters validation.
+	// TODO: add parameters validation (and also of the Log size).
 	err := c.testService.Save(r.Context(), req)
 	if err != nil {
 		http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
diff --git a/syz-cluster/controller/api_test.go b/syz-cluster/controller/api_test.go
index bc205392b..99532bc77 100644
--- a/syz-cluster/controller/api_test.go
+++ b/syz-cluster/controller/api_test.go
@@ -67,6 +67,7 @@ func TestAPISaveFinding(t *testing.T) {
 		BaseBuildID: buildResp.ID,
 		TestName:    "test",
 		Result:      api.TestRunning,
+		Log:         []byte("some log"),
 	})
 	assert.NoError(t, err)
 
diff --git a/syz-cluster/controller/services.go b/syz-cluster/controller/services.go
index 1ec178956..fc1ce183b 100644
--- a/syz-cluster/controller/services.go
+++ b/syz-cluster/controller/services.go
@@ -143,28 +143,48 @@ func (s *BuildService) LastBuild(ctx context.Context, req *api.LastBuildReq) (*a
 }
 
 type SessionTestService struct {
-	testRepo *db.SessionTestRepository
+	testRepo    *db.SessionTestRepository
+	blobStorage blob.Storage
 }
 
 func NewSessionTestService(env *app.AppEnvironment) *SessionTestService {
 	return &SessionTestService{
-		testRepo: db.NewSessionTestRepository(env.Spanner),
+		testRepo:    db.NewSessionTestRepository(env.Spanner),
+		blobStorage: env.BlobStorage,
 	}
 }
 
 func (s *SessionTestService) Save(ctx context.Context, req *api.TestResult) error {
-	entity := &db.SessionTest{
-		SessionID: req.SessionID,
-		TestName:  req.TestName,
-		Result:    req.Result,
-		UpdatedAt: time.Now(),
+	entity, err := s.testRepo.Get(ctx, req.SessionID, req.TestName)
+	if err != nil {
+		return fmt.Errorf("failed to query the test: %w", err)
+	} else if entity == nil {
+		entity = &db.SessionTest{
+			SessionID: req.SessionID,
+			TestName:  req.TestName,
+		}
 	}
+	entity.Result = req.Result
+	entity.UpdatedAt = time.Now()
 	if req.BaseBuildID != "" {
 		entity.BaseBuildID = spanner.NullString{StringVal: req.BaseBuildID, Valid: true}
 	}
 	if req.PatchedBuildID != "" {
 		entity.PatchedBuildID = spanner.NullString{StringVal: req.PatchedBuildID, Valid: true}
 	}
+	if entity.LogURI != "" {
+		err := s.blobStorage.Update(entity.LogURI, bytes.NewReader(req.Log))
+		if err != nil {
+			return fmt.Errorf("failed to update the log: %w", err)
+		}
+	} else if len(req.Log) > 0 {
+		// TODO: it will leak if we fail to save the entity.
+		uri, err := s.blobStorage.Store(bytes.NewReader(req.Log))
+		if err != nil {
+			return fmt.Errorf("failed to save the log: %w", err)
+		}
+		entity.LogURI = uri
+	}
 	return s.testRepo.InsertOrUpdate(context.Background(), entity)
 }
 
diff --git a/syz-cluster/dashboard/handler.go b/syz-cluster/dashboard/handler.go
index f744a5ca9..33c838b85 100644
--- a/syz-cluster/dashboard/handler.go
+++ b/syz-cluster/dashboard/handler.go
@@ -187,6 +187,18 @@ func (h *dashboardHandler) findingInfo(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (h *dashboardHandler) sessionTestLog(w http.ResponseWriter, r *http.Request) {
+	test, err := h.sessionTestRepo.Get(r.Context(), r.PathValue("id"), r.FormValue("name"))
+	if err != nil {
+		http.Error(w, fmt.Sprint(err), http.StatusInternalServerError)
+		return
+	} else if test == nil {
+		http.Error(w, "there's no such test", http.StatusNotFound)
+		return
+	}
+	h.streamBlob(w, test.LogURI)
+}
+
 func (h *dashboardHandler) streamBlob(w http.ResponseWriter, uri string) {
 	if uri == "" {
 		return
diff --git a/syz-cluster/dashboard/main.go b/syz-cluster/dashboard/main.go
index 0497cc86b..064dd31da 100644
--- a/syz-cluster/dashboard/main.go
+++ b/syz-cluster/dashboard/main.go
@@ -27,6 +27,7 @@ func main() {
 		app.Fatalf("failed to set up handler: %v", err)
 	}
 	http.HandleFunc("/sessions/{id}/log", handler.sessionLog)
+	http.HandleFunc("/sessions/{id}/test_logs", handler.sessionTestLog)
 	http.HandleFunc("/series/{id}", handler.seriesInfo)
 	http.HandleFunc("/patches/{id}", handler.patchContent)
 	http.HandleFunc("/findings/{id}/{key}", handler.findingInfo)
diff --git a/syz-cluster/dashboard/templates/series.html b/syz-cluster/dashboard/templates/series.html
index adc10b8d2..042f9efba 100644
--- a/syz-cluster/dashboard/templates/series.html
+++ b/syz-cluster/dashboard/templates/series.html
@@ -117,7 +117,12 @@
                     <td>{{.TestName}}</td>
                     <td>{{if .BaseBuild}}{{.BaseBuild.CommitHash}}{{end}}</td>
                     <td>{{if .PatchedBuild}}{{.PatchedBuild.CommitHash}}[patched]{{end}}</td>
-                    <th>{{.Result}}</th>
+                    <th>
+                        {{.Result}}
+                        {{if .LogURI}}
+                            <a href="/sessions/{{.SessionID}}/test_logs?name={{.TestName}}" class="modal-link-raw">[Log]</a>
+                        {{end}}
+                    </th>
                 </tr>
                 {{if .Findings}}
                 <tr>
diff --git a/syz-cluster/pkg/db/entities.go b/syz-cluster/pkg/db/entities.go
index f0f4e97db..784a96a2e 100644
--- a/syz-cluster/pkg/db/entities.go
+++ b/syz-cluster/pkg/db/entities.go
@@ -82,6 +82,7 @@ type SessionTest struct {
 	UpdatedAt      time.Time          `spanner:"UpdatedAt"`
 	TestName       string             `spanner:"TestName"`
 	Result         string             `spanner:"Result"`
+	LogURI         string             `spanner:"LogURI"`
 }
 
 type Finding struct {
diff --git a/syz-cluster/pkg/db/migrations/1_initialize.up.sql b/syz-cluster/pkg/db/migrations/1_initialize.up.sql
index 22252666f..4612956e7 100644
--- a/syz-cluster/pkg/db/migrations/1_initialize.up.sql
+++ b/syz-cluster/pkg/db/migrations/1_initialize.up.sql
@@ -69,6 +69,7 @@ CREATE TABLE SessionTests (
     Result STRING(36) NOT NULL,
     BaseBuildID STRING(36),
     PatchedBuildID STRING(36),
+    LogURI STRING(256) NOT NULL,
     CONSTRAINT FK_SessionResults FOREIGN KEY (SessionID) REFERENCES Sessions (ID),
     CONSTRAINT ResultEnum CHECK (Result IN ('passed', 'failed', 'error', 'running')),
     CONSTRAINT FK_BaseBuild FOREIGN KEY (BaseBuildID) REFERENCES Builds (ID),
diff --git a/syz-cluster/pkg/db/session_test_repo.go b/syz-cluster/pkg/db/session_test_repo.go
index 4ece8a904..43ada137c 100644
--- a/syz-cluster/pkg/db/session_test_repo.go
+++ b/syz-cluster/pkg/db/session_test_repo.go
@@ -57,6 +57,19 @@ func (repo *SessionTestRepository) InsertOrUpdate(ctx context.Context, test *Ses
 	return err
 }
 
+func (repo *SessionTestRepository) Get(ctx context.Context, sessionID, testName string) (*SessionTest, error) {
+	stmt := spanner.Statement{
+		SQL: "SELECT * FROM `SessionTests` WHERE `SessionID` = @session AND `TestName` = @name",
+		Params: map[string]interface{}{
+			"session": sessionID,
+			"name":    testName,
+		},
+	}
+	iter := repo.client.Single().Query(ctx, stmt)
+	defer iter.Stop()
+	return readOne[SessionTest](iter)
+}
+
 type FullSessionTest struct {
 	*SessionTest
 	BaseBuild    *Build
diff --git a/syz-cluster/workflow/build-step/main.go b/syz-cluster/workflow/build-step/main.go
index bd226401e..148e9a0b3 100644
--- a/syz-cluster/workflow/build-step/main.go
+++ b/syz-cluster/workflow/build-step/main.go
@@ -4,6 +4,7 @@
 package main
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -59,7 +60,15 @@ func main() {
 			SeriesID:   req.SeriesID,
 		},
 	}
-	commit, err := checkoutKernel(req, series)
+
+	output := new(bytes.Buffer)
+	tracer := &debugtracer.GenericTracer{
+		WithTime:    false,
+		TraceWriter: output,
+		OutDir:      "",
+	}
+
+	commit, err := checkoutKernel(tracer, req, series)
 	if commit != nil {
 		uploadReq.CommitDate = commit.CommitDate
 	}
@@ -68,10 +77,11 @@ func main() {
 		log.Printf("failed to checkout: %v", err)
 		uploadReq.Log = []byte(err.Error())
 	} else {
-		err := buildKernel(req)
+		err := buildKernel(tracer, req)
 		if err == nil {
 			uploadReq.BuildSuccess = true
 		} else {
+			log.Printf("%s", output.Bytes())
 			log.Printf("failed to build: %v", err)
 			uploadReq.Log = []byte(err.Error())
 			finding = &api.Finding{
@@ -82,11 +92,12 @@ func main() {
 			}
 		}
 	}
-	reportResults(ctx, client, req.SeriesID != "", uploadReq, finding)
+	reportResults(ctx, client, req.SeriesID != "",
+		uploadReq, finding, output.Bytes())
 }
 
 func reportResults(ctx context.Context, client *api.Client, patched bool,
-	uploadReq *api.UploadBuildReq, finding *api.Finding) {
+	uploadReq *api.UploadBuildReq, finding *api.Finding, output []byte) {
 	buildInfo, err := client.UploadBuild(ctx, uploadReq)
 	if err != nil {
 		app.Fatalf("failed to upload build: %v", err)
@@ -100,6 +111,7 @@ func reportResults(ctx context.Context, client *api.Client, patched bool,
 		SessionID: *flagSession,
 		TestName:  *flagTestName,
 		Result:    api.TestFailed,
+		Log:       output,
 	}
 	if uploadReq.BuildSuccess {
 		testResult.Result = api.TestPassed
@@ -136,8 +148,8 @@ func readRequest() *api.BuildRequest {
 	return &req
 }
 
-func checkoutKernel(req *api.BuildRequest, series *api.Series) (*vcs.Commit, error) {
-	log.Printf("checking out %q", req.CommitHash)
+func checkoutKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest, series *api.Series) (*vcs.Commit, error) {
+	tracer.Log("checking out %q", req.CommitHash)
 	ops, err := triage.NewGitTreeOps(*flagRepository, true)
 	if err != nil {
 		return nil, err
@@ -151,13 +163,13 @@ func checkoutKernel(req *api.BuildRequest, series *api.Series) (*vcs.Commit, err
 		patches = series.Patches
 	}
 	if len(patches) > 0 {
-		log.Printf("applying %d patches", len(patches))
+		tracer.Log("applying %d patches", len(patches))
 	}
 	err = ops.ApplySeries(req.CommitHash, patches)
 	return commit, err
 }
 
-func buildKernel(req *api.BuildRequest) error {
+func buildKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest) error {
 	kernelConfig, err := os.ReadFile(filepath.Join("/kernel-configs", req.ConfigName))
 	if err != nil {
 		return fmt.Errorf("failed to read the kernel config: %w", err)
@@ -176,28 +188,25 @@ func buildKernel(req *api.BuildRequest) error {
 		Linker:       "ld.lld",
 		UserspaceDir: "/disk-images/buildroot_amd64_2024.09", // See the Dockerfile.
 		Config:       kernelConfig,
-		Tracer: &debugtracer.GenericTracer{
-			TraceWriter: os.Stdout,
-			OutDir:      "",
-		},
+		Tracer:       tracer,
 	}
-	log.Printf("started build: %q", req)
+	tracer.Log("started build: %q", req)
 	info, err := build.Image(params)
-	log.Printf("compiler: %q", info.CompilerID)
+	tracer.Log("compiler: %q", info.CompilerID)
 	if err != nil {
 		var kernelError *build.KernelError
 		var verboseError *osutil.VerboseError
 		switch {
 		case errors.As(err, &kernelError):
-			log.Printf("kernel error: %q / %s", kernelError.Report, kernelError.Output)
+			tracer.Log("kernel error: %q / %s", kernelError.Report, kernelError.Output)
 		case errors.As(err, &verboseError):
-			log.Printf("verbose error: %q / %s", verboseError.Title, verboseError.Output)
+			tracer.Log("verbose error: %q / %s", verboseError.Title, verboseError.Output)
 		default:
-			log.Printf("other error: %v", err)
+			tracer.Log("other error: %v", err)
 		}
 		return err
 	}
-	log.Printf("build finished successfully")
+	tracer.Log("build finished successfully")
 	// TODO: capture build logs and the compiler identity.
 	// Note: Output directory has the following structure:
 	//   |-- image