syz-cluster: upload and share build config and log

14 files changed, 146 insertions, 36 deletions

diff --git a/syz-cluster/dashboard/handler.go b/syz-cluster/dashboard/handler.go
index f6c9b3e3b..794feba83 100644
--- a/syz-cluster/dashboard/handler.go
+++ b/syz-cluster/dashboard/handler.go
@@ -22,6 +22,7 @@ import (
 
 type dashboardHandler struct {
 	title           string
+	buildRepo       *db.BuildRepository
 	seriesRepo      *db.SeriesRepository
 	sessionRepo     *db.SessionRepository
 	sessionTestRepo *db.SessionTestRepository
@@ -37,7 +38,8 @@ func newHandler(env *app.AppEnvironment) (*dashboardHandler, error) {
 	perFile := map[string]*template.Template{}
 	var err error
 	for _, name := range []string{"index.html", "series.html"} {
-		perFile[name], err = template.ParseFS(templates, "templates/base.html", "templates/"+name)
+		perFile[name], err = template.ParseFS(templates,
+			"templates/base.html", "templates/templates.html", "templates/"+name)
 		if err != nil {
 			return nil, err
 		}
@@ -46,6 +48,7 @@ func newHandler(env *app.AppEnvironment) (*dashboardHandler, error) {
 		title:           env.Config.Name,
 		templates:       perFile,
 		blobStorage:     env.BlobStorage,
+		buildRepo:       db.NewBuildRepository(env.Spanner),
 		seriesRepo:      db.NewSeriesRepository(env.Spanner),
 		sessionRepo:     db.NewSessionRepository(env.Spanner),
 		sessionTestRepo: db.NewSessionTestRepository(env.Spanner),
@@ -65,6 +68,7 @@ func (h *dashboardHandler) Mux() *http.ServeMux {
 	mux.HandleFunc("/series/{id}", errToStatus(h.seriesInfo))
 	mux.HandleFunc("/patches/{id}", errToStatus(h.patchContent))
 	mux.HandleFunc("/findings/{id}/{key}", errToStatus(h.findingInfo))
+	mux.HandleFunc("/builds/{id}/{key}", errToStatus(h.buildInfo))
 	mux.HandleFunc("/", errToStatus(h.seriesList))
 	staticFiles, err := fs.Sub(staticFs, "static")
 	if err != nil {
@@ -290,6 +294,23 @@ func (h *dashboardHandler) findingInfo(w http.ResponseWriter, r *http.Request) e
 	}
 }
 
+func (h *dashboardHandler) buildInfo(w http.ResponseWriter, r *http.Request) error {
+	build, err := h.buildRepo.GetByID(r.Context(), r.PathValue("id"))
+	if err != nil {
+		return err
+	} else if build == nil {
+		return fmt.Errorf("%w: build", errNotFound)
+	}
+	switch r.PathValue("key") {
+	case "log":
+		return h.streamBlob(w, build.LogURI)
+	case "config":
+		return h.streamBlob(w, build.ConfigURI)
+	default:
+		return fmt.Errorf("%w: unknown key value", errBadRequest)
+	}
+}
+
 func (h *dashboardHandler) sessionTestLog(w http.ResponseWriter, r *http.Request) error {
 	test, err := h.sessionTestRepo.Get(r.Context(), r.PathValue("id"), r.FormValue("name"))
 	if err != nil {
diff --git a/syz-cluster/dashboard/handler_test.go b/syz-cluster/dashboard/handler_test.go
index 9ef993b93..a1200582e 100644
--- a/syz-cluster/dashboard/handler_test.go
+++ b/syz-cluster/dashboard/handler_test.go
@@ -4,6 +4,7 @@
 package main
 
 import (
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -11,6 +12,7 @@ import (
 	"github.com/google/syzkaller/syz-cluster/pkg/api"
 	"github.com/google/syzkaller/syz-cluster/pkg/app"
 	"github.com/google/syzkaller/syz-cluster/pkg/controller"
+	"github.com/google/syzkaller/syz-cluster/pkg/db"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -24,9 +26,13 @@ func TestURLs(t *testing.T) {
 	handler, baseURL := testServer(t, env)
 	urlGen := api.NewURLGenerator(baseURL)
 
-	var urls []string
-	urls = append(urls, urlGen.Series(ids.SeriesID))
-	findings, err := handler.findingRepo.ListForSession(ctx, ids.SessionID, 0)
+	urls := []string{urlGen.Series(ids.SeriesID)}
+	for _, buildID := range []string{ids.BaseBuildID, ids.PatchedBuildID} {
+		urls = append(urls, urlGen.BuildConfig(buildID))
+		urls = append(urls, urlGen.BuildLog(buildID))
+	}
+
+	findings, err := handler.findingRepo.ListForSession(ctx, ids.SessionID, db.NoLimit)
 	require.NoError(t, err)
 	for _, finding := range findings {
 		urls = append(urls, urlGen.FindingLog(finding.ID))
@@ -36,9 +42,11 @@ func TestURLs(t *testing.T) {
 	for _, url := range urls {
 		t.Logf("checking %s", url)
 		resp, err := http.Get(url)
-		assert.NoError(t, err)
+		body, _ := io.ReadAll(resp.Body)
 		resp.Body.Close()
-		assert.Equal(t, http.StatusOK, resp.StatusCode, "%q was expected to return HTTP 200", url)
+		assert.NoError(t, err)
+		assert.Equal(t, http.StatusOK, resp.StatusCode,
+			"%q was expected to return HTTP 200, body: %s", url, string(body))
 	}
 }
 
diff --git a/syz-cluster/dashboard/templates/series.html b/syz-cluster/dashboard/templates/series.html
index 6e01360b1..35961d1ba 100644
--- a/syz-cluster/dashboard/templates/series.html
+++ b/syz-cluster/dashboard/templates/series.html
@@ -128,8 +128,8 @@
             {{range .Tests}}
                 <tr>
                     <td>{{.TestName}}</td>
-                    <td>{{if .BaseBuild}}{{.BaseBuild.CommitHash}}{{end}}</td>
-                    <td>{{if .PatchedBuild}}{{.PatchedBuild.CommitHash}}[patched]{{end}}</td>
+                    <td>{{if .BaseBuild}}{{template "build_info" .BaseBuild}}{{end}}</td>
+                    <td>{{if .PatchedBuild}}{{template "build_info" .PatchedBuild}}[patched]{{end}}</td>
                     <th>
                         {{.Result}}
                         {{if .LogURI}}
diff --git a/syz-cluster/dashboard/templates/templates.html b/syz-cluster/dashboard/templates/templates.html
new file mode 100644
index 000000000..1d3ba4cb2
--- /dev/null
+++ b/syz-cluster/dashboard/templates/templates.html
@@ -0,0 +1,9 @@
+{{define "build_info"}}
+{{.CommitHash}}
+{{if .ConfigURI}}
+<a href="/builds/{{.ID}}/config" class="modal-link-raw">[Config]</a>
+{{end}}
+{{if .LogURI}}
+<a href="/builds/{{.ID}}/log" class="modal-link-raw">[Log]</a>
+{{end}}
+{{end}}
diff --git a/syz-cluster/pkg/api/api.go b/syz-cluster/pkg/api/api.go
index 5fcd0e797..d48175199 100644
--- a/syz-cluster/pkg/api/api.go
+++ b/syz-cluster/pkg/api/api.go
@@ -57,6 +57,7 @@ type Build struct {
 	CommitDate   time.Time `json:"commit_date"`
 	ConfigName   string    `json:"config_name"`
 	SeriesID     string    `json:"series_id"`
+	Compiler     string    `json:"compiler"`
 	BuildSuccess bool      `json:"build_success"`
 }
 
diff --git a/syz-cluster/pkg/api/urls.go b/syz-cluster/pkg/api/urls.go
index 54f447b97..f8af9c27b 100644
--- a/syz-cluster/pkg/api/urls.go
+++ b/syz-cluster/pkg/api/urls.go
@@ -29,3 +29,11 @@ func (g *URLGenerator) FindingCRepro(findingID string) string {
 func (g *URLGenerator) Series(seriesID string) string {
 	return fmt.Sprintf("%s/series/%s", g.baseURL, seriesID)
 }
+
+func (g *URLGenerator) BuildConfig(buildID string) string {
+	return fmt.Sprintf("%s/builds/%s/config", g.baseURL, buildID)
+}
+
+func (g *URLGenerator) BuildLog(buildID string) string {
+	return fmt.Sprintf("%s/builds/%s/log", g.baseURL, buildID)
+}
diff --git a/syz-cluster/pkg/controller/testutil.go b/syz-cluster/pkg/controller/testutil.go
index 0a72914c7..c1ab8b6f5 100644
--- a/syz-cluster/pkg/controller/testutil.go
+++ b/syz-cluster/pkg/controller/testutil.go
@@ -39,7 +39,9 @@ func UploadTestSeries(t *testing.T, ctx context.Context,
 func UploadTestBuild(t *testing.T, ctx context.Context, client *api.Client,
 	build *api.Build) *api.UploadBuildResp {
 	ret, err := client.UploadBuild(ctx, &api.UploadBuildReq{
-		Build: *build,
+		Build:  *build,
+		Log:    []byte("build log"),
+		Config: []byte("build config"),
 	})
 	assert.NoError(t, err)
 	assert.NotEmpty(t, ret.ID)
@@ -74,6 +76,7 @@ func DummyBuild() *api.Build {
 		TreeName:   "mainline",
 		ConfigName: "config",
 		CommitHash: "abcd",
+		Compiler:   "compiler",
 	}
 }
 
@@ -92,8 +95,14 @@ func DummyFindings() []*api.NewFinding {
 	return findings
 }
 
+type SeriesWithFindingIDs struct {
+	EntityIDs
+	BaseBuildID    string
+	PatchedBuildID string
+}
+
 func FakeSeriesWithFindings(t *testing.T, ctx context.Context, env *app.AppEnvironment,
-	client *api.Client, series *api.Series) EntityIDs {
+	client *api.Client, series *api.Series) SeriesWithFindingIDs {
 	ids := UploadTestSeries(t, ctx, client, series)
 	baseBuild := UploadTestBuild(t, ctx, client, DummyBuild())
 	patchedBuild := UploadTestBuild(t, ctx, client, DummyBuild())
@@ -113,7 +122,11 @@ func FakeSeriesWithFindings(t *testing.T, ctx context.Context, env *app.AppEnvir
 		assert.NoError(t, err)
 	}
 	MarkSessionFinished(t, env, ids.SessionID)
-	return ids
+	return SeriesWithFindingIDs{
+		EntityIDs:      ids,
+		BaseBuildID:    baseBuild.ID,
+		PatchedBuildID: patchedBuild.ID,
+	}
 }
 
 func MarkSessionFinished(t *testing.T, env *app.AppEnvironment, sessionID string) {
diff --git a/syz-cluster/pkg/db/entities.go b/syz-cluster/pkg/db/entities.go
index c914d7361..9959e81b2 100644
--- a/syz-cluster/pkg/db/entities.go
+++ b/syz-cluster/pkg/db/entities.go
@@ -45,7 +45,9 @@ type Build struct {
 	Arch       string             `spanner:"Arch"`
 	ConfigName string             `spanner:"ConfigName"`
 	ConfigURI  string             `spanner:"ConfigURI"`
+	LogURI     string             `spanner:"LogURI"`
 	Status     string             `spanner:"Status"`
+	Compiler   string             `spanner:"Compiler"`
 }
 
 func (b *Build) SetSeriesID(val string) {
diff --git a/syz-cluster/pkg/db/migrations/2_extend_build.down.sql b/syz-cluster/pkg/db/migrations/2_extend_build.down.sql
new file mode 100644
index 000000000..e96c67e95
--- /dev/null
+++ b/syz-cluster/pkg/db/migrations/2_extend_build.down.sql
@@ -0,0 +1,2 @@
+ALTER TABLE Builds DROP COLUMN Compiler;
+ALTER TABLE Builds DROP COLUMN LogURI;
diff --git a/syz-cluster/pkg/db/migrations/2_extend_build.up.sql b/syz-cluster/pkg/db/migrations/2_extend_build.up.sql
new file mode 100644
index 000000000..542d7ae33
--- /dev/null
+++ b/syz-cluster/pkg/db/migrations/2_extend_build.up.sql
@@ -0,0 +1,7 @@
+-- One cannot just add a non null column in spanner.
+
+ALTER TABLE Builds ADD COLUMN Compiler STRING(512) DEFAULT('');
+ALTER TABLE Builds ALTER COLUMN Compiler STRING(512) NOT NULL;
+
+ALTER TABLE Builds ADD COLUMN LogURI STRING(512) DEFAULT('');
+ALTER TABLE Builds ALTER COLUMN LogURI STRING(512) NOT NULL;
diff --git a/syz-cluster/pkg/reporter/api_test.go b/syz-cluster/pkg/reporter/api_test.go
index 279055e5a..69ee0c604 100644
--- a/syz-cluster/pkg/reporter/api_test.go
+++ b/syz-cluster/pkg/reporter/api_test.go
@@ -43,6 +43,8 @@ func TestAPIReportFlow(t *testing.T) {
 		finding.LinkCRepro = ""
 		assert.NotEmpty(t, finding.LinkSyzRepro, "%q's LinkSyzRepro is empty", finding.Title)
 		finding.LinkSyzRepro = ""
+		assert.NotEmpty(t, finding.Build.ConfigLink, "%q's ConfigLink is empty", finding.Title)
+		finding.Build.ConfigLink = ""
 	}
 
 	assert.Equal(t, &api.SessionReport{
@@ -69,6 +71,7 @@ func TestAPIReportFlow(t *testing.T) {
 					Repo:       "mainline",
 					BaseCommit: "abcd",
 					Arch:       "amd64",
+					Compiler:   "compiler",
 				},
 			},
 			{
@@ -78,6 +81,7 @@ func TestAPIReportFlow(t *testing.T) {
 					Repo:       "mainline",
 					BaseCommit: "abcd",
 					Arch:       "amd64",
+					Compiler:   "compiler",
 				},
 			},
 		},
diff --git a/syz-cluster/pkg/service/build.go b/syz-cluster/pkg/service/build.go
index 40d716472..b6dcdb620 100644
--- a/syz-cluster/pkg/service/build.go
+++ b/syz-cluster/pkg/service/build.go
@@ -4,30 +4,38 @@
 package service
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 
 	"github.com/google/syzkaller/syz-cluster/pkg/api"
 	"github.com/google/syzkaller/syz-cluster/pkg/app"
+	"github.com/google/syzkaller/syz-cluster/pkg/blob"
 	"github.com/google/syzkaller/syz-cluster/pkg/db"
+	"github.com/google/uuid"
 )
 
 type BuildService struct {
-	buildRepo *db.BuildRepository
+	buildRepo   *db.BuildRepository
+	blobStorage blob.Storage
 }
 
 func NewBuildService(env *app.AppEnvironment) *BuildService {
 	return &BuildService{
-		buildRepo: db.NewBuildRepository(env.Spanner),
+		buildRepo:   db.NewBuildRepository(env.Spanner),
+		blobStorage: env.BlobStorage,
 	}
 }
 
 func (s *BuildService) Upload(ctx context.Context, req *api.UploadBuildReq) (*api.UploadBuildResp, error) {
 	build := &db.Build{
+		ID:         uuid.NewString(),
 		Arch:       req.Arch,
 		ConfigName: req.ConfigName,
 		TreeName:   req.TreeName,
 		CommitHash: req.CommitHash,
 		CommitDate: req.CommitDate,
+		Compiler:   req.Compiler,
 	}
 	if req.SeriesID != "" {
 		build.SetSeriesID(req.SeriesID)
@@ -37,7 +45,20 @@ func (s *BuildService) Upload(ctx context.Context, req *api.UploadBuildReq) (*ap
 	} else {
 		build.Status = db.BuildFailed
 	}
-	// TODO: upload config and log.
+	if len(req.Log) > 0 {
+		var err error
+		build.LogURI, err = s.blobStorage.Write(bytes.NewReader(req.Log), "Build", build.ID, "log")
+		if err != nil {
+			return nil, fmt.Errorf("failed to write log: %w", err)
+		}
+	}
+	if len(req.Config) > 0 {
+		var err error
+		build.ConfigURI, err = s.blobStorage.Write(bytes.NewReader(req.Config), "Build", build.ID, "config")
+		if err != nil {
+			return nil, fmt.Errorf("failed to write kernel config: %w", err)
+		}
+	}
 	err := s.buildRepo.Insert(ctx, build)
 	if err != nil {
 		return nil, err
@@ -72,11 +93,12 @@ func (s *BuildService) LastBuild(ctx context.Context, req *api.LastBuildReq) (*a
 	return resp, nil
 }
 
-func makeBuildInfo(build *db.Build) api.BuildInfo {
+func makeBuildInfo(url *api.URLGenerator, build *db.Build) api.BuildInfo {
 	return api.BuildInfo{
 		Repo:       build.TreeName, // TODO: we actually want to use repo URI here.
 		BaseCommit: build.CommitHash,
 		Arch:       build.Arch,
-		ConfigLink: "",
+		Compiler:   build.Compiler,
+		ConfigLink: url.BuildConfig(build.ID),
 	}
 }
diff --git a/syz-cluster/pkg/service/finding.go b/syz-cluster/pkg/service/finding.go
index 2f238cc7d..001860e1c 100644
--- a/syz-cluster/pkg/service/finding.go
+++ b/syz-cluster/pkg/service/finding.go
@@ -100,7 +100,7 @@ func (s *FindingService) List(ctx context.Context, sessionID string, limit int)
 		}
 		build := testPerName[item.TestName].PatchedBuild
 		if build != nil {
-			finding.Build = makeBuildInfo(build)
+			finding.Build = makeBuildInfo(s.urls, build)
 		}
 		bytes, err := blob.ReadAllBytes(s.blobStorage, item.ReportURI)
 		if err != nil {
diff --git a/syz-cluster/workflow/build-step/main.go b/syz-cluster/workflow/build-step/main.go
index 6c61912e3..3c859dc8c 100644
--- a/syz-cluster/workflow/build-step/main.go
+++ b/syz-cluster/workflow/build-step/main.go
@@ -86,25 +86,28 @@ func main() {
 			return
 		}
 	}
-	var finding *api.NewFinding
+	ret := &BuildResult{}
 	if err != nil {
 		log.Printf("failed to checkout: %v", err)
 		uploadReq.Log = []byte(err.Error())
 	} else {
-		finding, err = buildKernel(tracer, req)
+		ret, err = buildKernel(tracer, req)
 		if err != nil {
 			log.Printf("build process failed: %v", err)
 			uploadReq.Log = []byte(err.Error())
-		} else if finding == nil {
-			uploadReq.BuildSuccess = true
 		} else {
-			log.Printf("%s", output.Bytes())
-			log.Printf("failed: %s\n%s", finding.Title, finding.Report)
-			uploadReq.Log = finding.Log
+			uploadReq.Compiler = ret.Compiler
+			uploadReq.Config = ret.Config
+			if ret.Finding == nil {
+				uploadReq.BuildSuccess = true
+			} else {
+				log.Printf("%s", output.Bytes())
+				log.Printf("failed: %s\n%s", ret.Finding.Title, ret.Finding.Report)
+				uploadReq.Log = ret.Finding.Log
+			}
 		}
 	}
-	reportResults(ctx, client, req.SeriesID != "",
-		uploadReq, finding, output.Bytes())
+	reportResults(ctx, client, req.SeriesID != "", uploadReq, ret.Finding, output.Bytes())
 }
 
 func reportResults(ctx context.Context, client *api.Client, patched bool,
@@ -197,7 +200,13 @@ func checkoutKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest, serie
 	return commit, err
 }
 
-func buildKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest) (*api.NewFinding, error) {
+type BuildResult struct {
+	Config   []byte
+	Compiler string
+	Finding  *api.NewFinding
+}
+
+func buildKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest) (*BuildResult, error) {
 	kernelConfig, err := os.ReadFile(filepath.Join("/kernel-configs", req.ConfigName))
 	if err != nil {
 		return nil, fmt.Errorf("failed to read the kernel config: %w", err)
@@ -222,8 +231,13 @@ func buildKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest) (*api.Ne
 	info, err := build.Image(params)
 	tracer.Log("compiler: %q", info.CompilerID)
 	tracer.Log("signature: %q", info.Signature)
+	// We can fill this regardless of whether it succeeded.
+	ret := &BuildResult{
+		Compiler: info.CompilerID,
+	}
+	ret.Config, _ = os.ReadFile(filepath.Join(*flagOutput, "kernel.config"))
 	if err != nil {
-		finding := &api.NewFinding{
+		ret.Finding = &api.NewFinding{
 			SessionID: *flagSession,
 			TestName:  *flagTestName,
 			Title:     "kernel build error",
@@ -233,28 +247,27 @@ func buildKernel(tracer debugtracer.DebugTracer, req *api.BuildRequest) (*api.Ne
 		switch {
 		case errors.As(err, &kernelError):
 			tracer.Log("kernel error: %q / %s", kernelError.Report, kernelError.Output)
-			finding.Report = kernelError.Report
-			finding.Log = kernelError.Output
-			return finding, nil
+			ret.Finding.Report = kernelError.Report
+			ret.Finding.Log = kernelError.Output
+			return ret, nil
 		case errors.As(err, &verboseError):
 			tracer.Log("verbose error: %q / %s", verboseError.Title, verboseError.Output)
-			finding.Report = []byte(verboseError.Title)
-			finding.Log = verboseError.Output
-			return finding, nil
+			ret.Finding.Report = []byte(verboseError.Title)
+			ret.Finding.Log = verboseError.Output
+			return ret, nil
 		default:
 			tracer.Log("other error: %v", err)
 		}
 		return nil, err
 	}
 	tracer.Log("build finished successfully")
-	// TODO: capture build logs and the compiler identity.
 	// Note: Output directory has the following structure:
 	//   |-- image
 	//   |-- kernel
 	//   |-- kernel.config
 	//   `-- obj
 	//      `-- vmlinux
-	return nil, nil
+	return ret, nil
 }
 
 func ensureFlags(args ...string) {