From 76718eb73e3d1e2a43363d80ab15366706b6491f Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Fri, 11 Jul 2025 13:17:34 +0200 Subject: syz-cluster: generate web dashboard URLs for reports Take web dashboard URL from the config and use it to generate links for logs, reproducers, etc. --- syz-cluster/controller/main.go | 6 +-- syz-cluster/controller/processor_test.go | 6 +-- syz-cluster/dashboard/handler.go | 6 +-- syz-cluster/dashboard/handler_test.go | 51 ++++++++++++++++++++++ syz-cluster/overlays/gke/prod/global-config.yaml | 1 + .../overlays/gke/staging/global-config.yaml | 1 + syz-cluster/overlays/minikube/global-config.yaml | 1 + syz-cluster/pkg/api/urls.go | 31 +++++++++++++ syz-cluster/pkg/app/config.go | 16 +++++++ syz-cluster/pkg/app/env.go | 12 +++++ syz-cluster/pkg/controller/api_test.go | 20 ++++----- syz-cluster/pkg/controller/testutil.go | 26 ++++++++--- syz-cluster/pkg/reporter/api_test.go | 15 +++++-- syz-cluster/pkg/service/finding.go | 10 ++++- syz-cluster/pkg/service/report.go | 3 ++ syz-cluster/reporter-server/deployment.yaml | 7 +++ 16 files changed, 178 insertions(+), 34 deletions(-) create mode 100644 syz-cluster/dashboard/handler_test.go create mode 100644 syz-cluster/pkg/api/urls.go diff --git a/syz-cluster/controller/main.go b/syz-cluster/controller/main.go index dc6404b38..f52b4eae5 100644 --- a/syz-cluster/controller/main.go +++ b/syz-cluster/controller/main.go @@ -20,11 +20,7 @@ func main() { if err != nil { app.Fatalf("failed to set up environment: %v", err) } - cfg, err := app.Config() - if err != nil { - app.Fatalf("failed to fetch the config: %v", err) - } - sp := NewSeriesProcessor(env, cfg) + sp := NewSeriesProcessor(env, env.Config) go func() { err := sp.Loop(ctx) app.Fatalf("processor loop failed: %v", err) diff --git a/syz-cluster/controller/processor_test.go b/syz-cluster/controller/processor_test.go index 68c62ba48..147f5d6e2 100644 --- a/syz-cluster/controller/processor_test.go +++ b/syz-cluster/controller/processor_test.go @@ -97,7 +97,7 @@ func TestFinishRunningSteps(t *testing.T) { ExtID: "ext-id", Title: "title", } - _, sessionID := controller.UploadTestSeries(t, ctx, client, series) + ids := controller.UploadTestSeries(t, ctx, client, series) buildResp := controller.UploadTestBuild(t, ctx, client, &api.Build{ Arch: "amd64", TreeName: "mainline", @@ -105,7 +105,7 @@ func TestFinishRunningSteps(t *testing.T) { CommitHash: "abcd", }) err := client.UploadTestResult(ctx, &api.TestResult{ - SessionID: sessionID, + SessionID: ids.SessionID, BaseBuildID: buildResp.ID, TestName: "test", Result: api.TestRunning, @@ -119,7 +119,7 @@ func TestFinishRunningSteps(t *testing.T) { // Verify that the session test is finished. // A bit hacky, but it works. - list, err := processor.sessionTestRepo.BySessionRaw(ctx, sessionID) + list, err := processor.sessionTestRepo.BySessionRaw(ctx, ids.SessionID) assert.NoError(t, err) assert.Equal(t, api.TestError, list[0].Result) } diff --git a/syz-cluster/dashboard/handler.go b/syz-cluster/dashboard/handler.go index ea3802533..f6c9b3e3b 100644 --- a/syz-cluster/dashboard/handler.go +++ b/syz-cluster/dashboard/handler.go @@ -42,12 +42,8 @@ func newHandler(env *app.AppEnvironment) (*dashboardHandler, error) { return nil, err } } - cfg, err := app.Config() - if err != nil { - return nil, err - } return &dashboardHandler{ - title: cfg.Name, + title: env.Config.Name, templates: perFile, blobStorage: env.BlobStorage, seriesRepo: db.NewSeriesRepository(env.Spanner), diff --git a/syz-cluster/dashboard/handler_test.go b/syz-cluster/dashboard/handler_test.go new file mode 100644 index 000000000..9ef993b93 --- /dev/null +++ b/syz-cluster/dashboard/handler_test.go @@ -0,0 +1,51 @@ +// Copyright 2025 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/google/syzkaller/syz-cluster/pkg/api" + "github.com/google/syzkaller/syz-cluster/pkg/app" + "github.com/google/syzkaller/syz-cluster/pkg/controller" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestURLs(t *testing.T) { + env, ctx := app.TestEnvironment(t) + client := controller.TestServer(t, env) + testSeries := controller.DummySeries() + ids := controller.FakeSeriesWithFindings(t, ctx, env, client, testSeries) + + handler, baseURL := testServer(t, env) + urlGen := api.NewURLGenerator(baseURL) + + var urls []string + urls = append(urls, urlGen.Series(ids.SeriesID)) + findings, err := handler.findingRepo.ListForSession(ctx, ids.SessionID, 0) + require.NoError(t, err) + for _, finding := range findings { + urls = append(urls, urlGen.FindingLog(finding.ID)) + urls = append(urls, urlGen.FindingCRepro(finding.ID)) + urls = append(urls, urlGen.FindingSyzRepro(finding.ID)) + } + for _, url := range urls { + t.Logf("checking %s", url) + resp, err := http.Get(url) + assert.NoError(t, err) + resp.Body.Close() + assert.Equal(t, http.StatusOK, resp.StatusCode, "%q was expected to return HTTP 200", url) + } +} + +func testServer(t *testing.T, env *app.AppEnvironment) (*dashboardHandler, string) { + handler, err := newHandler(env) + require.NoError(t, err) + server := httptest.NewServer(handler.Mux()) + t.Cleanup(server.Close) + return handler, server.URL +} diff --git a/syz-cluster/overlays/gke/prod/global-config.yaml b/syz-cluster/overlays/gke/prod/global-config.yaml index 3b9e5e24b..03a53ab07 100644 --- a/syz-cluster/overlays/gke/prod/global-config.yaml +++ b/syz-cluster/overlays/gke/prod/global-config.yaml @@ -7,6 +7,7 @@ metadata: name: global-config data: config.yaml: | + URL: https://ci.syzbot.org parallelWorkflows: 7 loreArchives: - netdev diff --git a/syz-cluster/overlays/gke/staging/global-config.yaml b/syz-cluster/overlays/gke/staging/global-config.yaml index d9ed497f7..617bbc022 100644 --- a/syz-cluster/overlays/gke/staging/global-config.yaml +++ b/syz-cluster/overlays/gke/staging/global-config.yaml @@ -7,6 +7,7 @@ metadata: name: global-config data: config.yaml: | + URL: http://unknown parallelWorkflows: 3 loreArchives: - netdev diff --git a/syz-cluster/overlays/minikube/global-config.yaml b/syz-cluster/overlays/minikube/global-config.yaml index c0cfa0078..50bc0892d 100644 --- a/syz-cluster/overlays/minikube/global-config.yaml +++ b/syz-cluster/overlays/minikube/global-config.yaml @@ -7,6 +7,7 @@ metadata: name: global-config data: config.yaml: | + URL: http://localhost parallelWorkflows: 1 # Whatever, it's just for debugging. loreArchives: diff --git a/syz-cluster/pkg/api/urls.go b/syz-cluster/pkg/api/urls.go new file mode 100644 index 000000000..54f447b97 --- /dev/null +++ b/syz-cluster/pkg/api/urls.go @@ -0,0 +1,31 @@ +// Copyright 2025 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package api + +import "fmt" + +// URLGenerator creates URLs for accessing the web dashboard. +type URLGenerator struct { + baseURL string +} + +func NewURLGenerator(baseURL string) *URLGenerator { + return &URLGenerator{baseURL} +} + +func (g *URLGenerator) FindingLog(findingID string) string { + return fmt.Sprintf("%s/findings/%s/log", g.baseURL, findingID) +} + +func (g *URLGenerator) FindingSyzRepro(findingID string) string { + return fmt.Sprintf("%s/findings/%s/syz_repro", g.baseURL, findingID) +} + +func (g *URLGenerator) FindingCRepro(findingID string) string { + return fmt.Sprintf("%s/findings/%s/c_repro", g.baseURL, findingID) +} + +func (g *URLGenerator) Series(seriesID string) string { + return fmt.Sprintf("%s/series/%s", g.baseURL, seriesID) +} diff --git a/syz-cluster/pkg/app/config.go b/syz-cluster/pkg/app/config.go index 92a3535e0..38cea5e8a 100644 --- a/syz-cluster/pkg/app/config.go +++ b/syz-cluster/pkg/app/config.go @@ -7,6 +7,7 @@ import ( "fmt" "net/mail" "os" + "strings" "sync" "gopkg.in/yaml.v3" @@ -15,6 +16,8 @@ import ( type AppConfig struct { // The name that will be shown on the Web UI. Name string `yaml:"name"` + // Public URL of the web dashboard (without / at the end). + URL string `yaml:"URL"` // How many workflows are scheduled in parallel. ParallelWorkflows int `yaml:"parallelWorkflows"` // What Lore archives are to be polled for new patch series. @@ -108,6 +111,9 @@ func (c AppConfig) Validate() error { if c.ParallelWorkflows < 0 { return fmt.Errorf("parallelWorkflows must be non-negative") } + if err := ensureURL("url", c.URL); err != nil { + return err + } if c.EmailReporting != nil { if err := c.EmailReporting.Validate(); err != nil { return fmt.Errorf("emailReporting: %w", err) @@ -176,6 +182,16 @@ func ensureNonEmpty(name, val string) error { return nil } +func ensureURL(name, val string) error { + if err := ensureNonEmpty(name, val); err != nil { + return err + } + if strings.HasSuffix(val, "/") { + return fmt.Errorf("%v should not contain / at the end", name) + } + return nil +} + func ensureEmail(name, val string) error { if err := ensureNonEmpty(name, val); err != nil { return err diff --git a/syz-cluster/pkg/app/env.go b/syz-cluster/pkg/app/env.go index 7a5b8cca5..191513ddd 100644 --- a/syz-cluster/pkg/app/env.go +++ b/syz-cluster/pkg/app/env.go @@ -18,6 +18,8 @@ import ( type AppEnvironment struct { Spanner *spanner.Client BlobStorage blob.Storage + Config *AppConfig + URLs *api.URLGenerator } func Environment(ctx context.Context) (*AppEnvironment, error) { @@ -29,9 +31,15 @@ func Environment(ctx context.Context) (*AppEnvironment, error) { if err != nil { return nil, fmt.Errorf("failed to set up the blob storage: %w", err) } + cfg, err := Config() + if err != nil { + return nil, fmt.Errorf("failed to query the config: %w", err) + } return &AppEnvironment{ Spanner: spanner, BlobStorage: storage, + Config: cfg, + URLs: api.NewURLGenerator(cfg.URL), }, nil } @@ -40,6 +48,10 @@ func TestEnvironment(t *testing.T) (*AppEnvironment, context.Context) { return &AppEnvironment{ Spanner: client, BlobStorage: blob.NewLocalStorage(t.TempDir()), + Config: &AppConfig{ + Name: "Test", + }, + URLs: api.NewURLGenerator("http://dashboard"), }, ctx } diff --git a/syz-cluster/pkg/controller/api_test.go b/syz-cluster/pkg/controller/api_test.go index 621be5686..c086f31ce 100644 --- a/syz-cluster/pkg/controller/api_test.go +++ b/syz-cluster/pkg/controller/api_test.go @@ -16,14 +16,14 @@ import ( func TestAPIGetSeries(t *testing.T) { env, ctx := app.TestEnvironment(t) client := TestServer(t, env) - seriesID, sessionID := UploadTestSeries(t, ctx, client, testSeries) + ids := UploadTestSeries(t, ctx, client, testSeries) - ret, err := client.GetSessionSeries(ctx, sessionID) + ret, err := client.GetSessionSeries(ctx, ids.SessionID) assert.NoError(t, err) ret.ID = "" assert.Equal(t, testSeries, ret) - ret, err = client.GetSeries(ctx, seriesID) + ret, err = client.GetSeries(ctx, ids.SeriesID) assert.NoError(t, err) ret.ID = "" assert.Equal(t, testSeries, ret) @@ -47,10 +47,10 @@ func TestAPISaveFinding(t *testing.T) { env, ctx := app.TestEnvironment(t) client := TestServer(t, env) - _, sessionID := UploadTestSeries(t, ctx, client, testSeries) + ids := UploadTestSeries(t, ctx, client, testSeries) buildResp := UploadTestBuild(t, ctx, client, testBuild) err := client.UploadTestResult(ctx, &api.TestResult{ - SessionID: sessionID, + SessionID: ids.SessionID, BaseBuildID: buildResp.ID, TestName: "test", Result: api.TestRunning, @@ -60,7 +60,7 @@ func TestAPISaveFinding(t *testing.T) { t.Run("not existing test", func(t *testing.T) { err = client.UploadFinding(ctx, &api.NewFinding{ - SessionID: sessionID, + SessionID: ids.SessionID, TestName: "unknown test", }) assert.Error(t, err) @@ -68,7 +68,7 @@ func TestAPISaveFinding(t *testing.T) { t.Run("must succeed", func(t *testing.T) { finding := &api.NewFinding{ - SessionID: sessionID, + SessionID: ids.SessionID, TestName: "test", Report: []byte("report"), Log: []byte("log"), @@ -88,17 +88,17 @@ func TestAPIUploadTestArtifacts(t *testing.T) { env, ctx := app.TestEnvironment(t) client := TestServer(t, env) - _, sessionID := UploadTestSeries(t, ctx, client, testSeries) + ids := UploadTestSeries(t, ctx, client, testSeries) buildResp := UploadTestBuild(t, ctx, client, testBuild) err := client.UploadTestResult(ctx, &api.TestResult{ - SessionID: sessionID, + SessionID: ids.SessionID, BaseBuildID: buildResp.ID, TestName: "test", Result: api.TestRunning, Log: []byte("some log"), }) assert.NoError(t, err) - err = client.UploadTestArtifacts(ctx, sessionID, "test", bytes.NewReader([]byte("artifacts content"))) + err = client.UploadTestArtifacts(ctx, ids.SessionID, "test", bytes.NewReader([]byte("artifacts content"))) assert.NoError(t, err) } diff --git a/syz-cluster/pkg/controller/testutil.go b/syz-cluster/pkg/controller/testutil.go index f4d813e2f..0a72914c7 100644 --- a/syz-cluster/pkg/controller/testutil.go +++ b/syz-cluster/pkg/controller/testutil.go @@ -16,16 +16,24 @@ import ( "github.com/stretchr/testify/assert" ) +type EntityIDs struct { + SeriesID string + SessionID string +} + // UploadTestSeries returns a (series ID, session ID) tuple. func UploadTestSeries(t *testing.T, ctx context.Context, - client *api.Client, series *api.Series) (string, string) { + client *api.Client, series *api.Series) EntityIDs { retSeries, err := client.UploadSeries(ctx, series) assert.NoError(t, err) retSession, err := client.UploadSession(ctx, &api.NewSession{ ExtID: series.ExtID, }) assert.NoError(t, err) - return retSeries.ID, retSession.ID + return EntityIDs{ + SeriesID: retSeries.ID, + SessionID: retSession.ID, + } } func UploadTestBuild(t *testing.T, ctx context.Context, client *api.Client, @@ -76,18 +84,21 @@ func DummyFindings() []*api.NewFinding { Title: fmt.Sprintf("finding %d", i), TestName: "test", Report: []byte(fmt.Sprintf("report %d", i)), + Log: []byte(fmt.Sprintf("log %d", i)), + SyzRepro: []byte(fmt.Sprintf("log %d", i)), + CRepro: []byte(fmt.Sprintf("log %d", i)), }) } return findings } func FakeSeriesWithFindings(t *testing.T, ctx context.Context, env *app.AppEnvironment, - client *api.Client, series *api.Series) { - _, sessionID := UploadTestSeries(t, ctx, client, series) + client *api.Client, series *api.Series) EntityIDs { + ids := UploadTestSeries(t, ctx, client, series) baseBuild := UploadTestBuild(t, ctx, client, DummyBuild()) patchedBuild := UploadTestBuild(t, ctx, client, DummyBuild()) err := client.UploadTestResult(ctx, &api.TestResult{ - SessionID: sessionID, + SessionID: ids.SessionID, BaseBuildID: baseBuild.ID, PatchedBuildID: patchedBuild.ID, TestName: "test", @@ -97,11 +108,12 @@ func FakeSeriesWithFindings(t *testing.T, ctx context.Context, env *app.AppEnvir findings := DummyFindings() for _, finding := range findings { - finding.SessionID = sessionID + finding.SessionID = ids.SessionID err = client.UploadFinding(ctx, finding) assert.NoError(t, err) } - MarkSessionFinished(t, env, sessionID) + MarkSessionFinished(t, env, ids.SessionID) + return ids } func MarkSessionFinished(t *testing.T, env *app.AppEnvironment, sessionID string) { diff --git a/syz-cluster/pkg/reporter/api_test.go b/syz-cluster/pkg/reporter/api_test.go index b7170bc68..279055e5a 100644 --- a/syz-cluster/pkg/reporter/api_test.go +++ b/syz-cluster/pkg/reporter/api_test.go @@ -19,7 +19,7 @@ func TestAPIReportFlow(t *testing.T) { // Create series/session/test/findings. testSeries := controller.DummySeries() - controller.FakeSeriesWithFindings(t, ctx, env, client, testSeries) + ids := controller.FakeSeriesWithFindings(t, ctx, env, client, testSeries) generator := NewGenerator(env) err := generator.Process(ctx, 1) @@ -35,8 +35,19 @@ func TestAPIReportFlow(t *testing.T) { // We don't know IDs in advance. nextResp.Report.ID = "" nextResp.Report.Series.ID = "" + // For URLs, just check if they are in place. + for _, finding := range nextResp.Report.Findings { + assert.NotEmpty(t, finding.LogURL, "%q's LogURL is empty", finding.Title) + finding.LogURL = "" + assert.NotEmpty(t, finding.LinkCRepro, "%q's LinkCRepro is empty", finding.Title) + finding.LinkCRepro = "" + assert.NotEmpty(t, finding.LinkSyzRepro, "%q's LinkSyzRepro is empty", finding.Title) + finding.LinkSyzRepro = "" + } + assert.Equal(t, &api.SessionReport{ Moderation: true, + Link: env.URLs.Series(ids.SeriesID), Series: &api.Series{ ExtID: testSeries.ExtID, Title: testSeries.Title, @@ -54,7 +65,6 @@ func TestAPIReportFlow(t *testing.T) { { Title: "finding 0", Report: "report 0", - LogURL: "TODO", // TODO Build: api.BuildInfo{ Repo: "mainline", BaseCommit: "abcd", @@ -64,7 +74,6 @@ func TestAPIReportFlow(t *testing.T) { { Title: "finding 1", Report: "report 1", - LogURL: "TODO", // TODO Build: api.BuildInfo{ Repo: "mainline", BaseCommit: "abcd", diff --git a/syz-cluster/pkg/service/finding.go b/syz-cluster/pkg/service/finding.go index 5c059f8a8..2f238cc7d 100644 --- a/syz-cluster/pkg/service/finding.go +++ b/syz-cluster/pkg/service/finding.go @@ -19,6 +19,7 @@ type FindingService struct { findingRepo *db.FindingRepository sessionTestRepo *db.SessionTestRepository buildRepo *db.BuildRepository + urls *api.URLGenerator blobStorage blob.Storage } @@ -26,6 +27,7 @@ func NewFindingService(env *app.AppEnvironment) *FindingService { return &FindingService{ findingRepo: db.NewFindingRepository(env.Spanner), blobStorage: env.BlobStorage, + urls: env.URLs, buildRepo: db.NewBuildRepository(env.Spanner), sessionTestRepo: db.NewSessionTestRepository(env.Spanner), } @@ -88,7 +90,13 @@ func (s *FindingService) List(ctx context.Context, sessionID string, limit int) for _, item := range list { finding := &api.Finding{ Title: item.Title, - LogURL: "TODO", // TODO: where to take it from? + LogURL: s.urls.FindingLog(item.ID), + } + if item.SyzReproURI != "" { + finding.LinkSyzRepro = s.urls.FindingSyzRepro(item.ID) + } + if item.CReproURI != "" { + finding.LinkCRepro = s.urls.FindingCRepro(item.ID) } build := testPerName[item.TestName].PatchedBuild if build != nil { diff --git a/syz-cluster/pkg/service/report.go b/syz-cluster/pkg/service/report.go index 4b23d32f5..c92ccfa11 100644 --- a/syz-cluster/pkg/service/report.go +++ b/syz-cluster/pkg/service/report.go @@ -18,10 +18,12 @@ type ReportService struct { reportRepo *db.ReportRepository seriesService *SeriesService findingService *FindingService + urls *api.URLGenerator } func NewReportService(env *app.AppEnvironment) *ReportService { return &ReportService{ + urls: env.URLs, reportRepo: db.NewReportRepository(env.Spanner), seriesService: NewSeriesService(env), findingService: NewFindingService(env), @@ -89,6 +91,7 @@ func (rs *ReportService) Next(ctx context.Context, reporter string) (*api.NextRe ID: report.ID, Moderation: report.Moderation, Series: series, + Link: rs.urls.Series(series.ID), Findings: findings, }, }, nil diff --git a/syz-cluster/reporter-server/deployment.yaml b/syz-cluster/reporter-server/deployment.yaml index 310924f75..342fc1a08 100644 --- a/syz-cluster/reporter-server/deployment.yaml +++ b/syz-cluster/reporter-server/deployment.yaml @@ -22,6 +22,9 @@ spec: envFrom: - configMapRef: name: global-config-env + volumeMounts: + - name: config-volume + mountPath: /config ports: - containerPort: 8080 resources: @@ -31,3 +34,7 @@ spec: limits: cpu: 4 memory: 16G + volumes: + - name: config-volume + configMap: + name: global-config -- cgit mrf-deployment