syz-cluster: display statistics

Add a web dashboard page with the main statistics concerning patch series fuzzing. Improve the navigation bar on top of the page.
author: Aleksandr Nogikh <nogikh@google.com> 2025-08-13 17:32:02 +0200
committer: Aleksandr Nogikh <nogikh@google.com> 2025-08-14 11:29:47 +0000
commit: 83b914a1837f0f4de012f4dc6bdc84c9d5bab713 (patch)
tree: 4014acd61a94833bd66b17d8066979d2dee33807 /syz-cluster
parent: 028e3578dbbde98fdb5a4d170f0e1dce7a8a2ab4 (diff)
5 files changed, 283 insertions, 8 deletions
diff --git a/syz-cluster/dashboard/handler.go b/syz-cluster/dashboard/handler.go
index 794feba83..2212bd959 100644
--- a/syz-cluster/dashboard/handler.go
+++ b/syz-cluster/dashboard/handler.go
@@ -27,6 +27,7 @@ type dashboardHandler struct {
 	sessionRepo     *db.SessionRepository
 	sessionTestRepo *db.SessionTestRepository
 	findingRepo     *db.FindingRepository
+	statsRepo       *db.StatsRepository
 	blobStorage     blob.Storage
 	templates       map[string]*template.Template
 }
@@ -37,7 +38,7 @@ var templates embed.FS
 func newHandler(env *app.AppEnvironment) (*dashboardHandler, error) {
 	perFile := map[string]*template.Template{}
 	var err error
-	for _, name := range []string{"index.html", "series.html"} {
+	for _, name := range []string{"index.html", "series.html", "graphs.html"} {
 		perFile[name], err = template.ParseFS(templates,
 			"templates/base.html", "templates/templates.html", "templates/"+name)
 		if err != nil {
@@ -53,6 +54,7 @@ func newHandler(env *app.AppEnvironment) (*dashboardHandler, error) {
 		sessionRepo:     db.NewSessionRepository(env.Spanner),
 		sessionTestRepo: db.NewSessionTestRepository(env.Spanner),
 		findingRepo:     db.NewFindingRepository(env.Spanner),
+		statsRepo:       db.NewStatsRepository(env.Spanner),
 	}, nil
 }
 
@@ -69,6 +71,7 @@ func (h *dashboardHandler) Mux() *http.ServeMux {
 	mux.HandleFunc("/patches/{id}", errToStatus(h.patchContent))
 	mux.HandleFunc("/findings/{id}/{key}", errToStatus(h.findingInfo))
 	mux.HandleFunc("/builds/{id}/{key}", errToStatus(h.buildInfo))
+	mux.HandleFunc("/stats", errToStatus(h.statsPage))
 	mux.HandleFunc("/", errToStatus(h.seriesList))
 	staticFiles, err := fs.Sub(staticFs, "static")
 	if err != nil {
@@ -211,6 +214,34 @@ func (h *dashboardHandler) seriesInfo(w http.ResponseWriter, r *http.Request) er
 	return h.renderTemplate(w, "series.html", data)
 }
 
+func (h *dashboardHandler) statsPage(w http.ResponseWriter, r *http.Request) error {
+	type StatsPageData struct {
+		Processed    []*db.CountPerWeek
+		Findings     []*db.CountPerWeek
+		Delay        []*db.DelayPerWeek
+		Distribution []*db.StatusPerWeek
+	}
+	var data StatsPageData
+	var err error
+	data.Processed, err = h.statsRepo.ProcessedSeriesPerWeek(r.Context())
+	if err != nil {
+		return fmt.Errorf("failed to query processed series data: %w", err)
+	}
+	data.Findings, err = h.statsRepo.FindingsPerWeek(r.Context())
+	if err != nil {
+		return fmt.Errorf("failed to query findings data: %w", err)
+	}
+	data.Delay, err = h.statsRepo.DelayPerWeek(r.Context())
+	if err != nil {
+		return fmt.Errorf("failed to query delay data: %w", err)
+	}
+	data.Distribution, err = h.statsRepo.SessionStatusPerWeek(r.Context())
+	if err != nil {
+		return fmt.Errorf("failed to query distribution data: %w", err)
+	}
+	return h.renderTemplate(w, "graphs.html", data)
+}
+
 func groupFindings(findings []*db.Finding) map[string][]*db.Finding {
 	ret := map[string][]*db.Finding{}
 	for _, finding := range findings {
@@ -221,12 +252,14 @@ func groupFindings(findings []*db.Finding) map[string][]*db.Finding {
 
 func (h *dashboardHandler) renderTemplate(w http.ResponseWriter, name string, data any) error {
 	type page struct {
-		Title string
-		Data  any
+		Title    string
+		Template string
+		Data     any
 	}
 	return h.templates[name].ExecuteTemplate(w, "base.html", page{
-		Title: h.title,
-		Data:  data,
+		Title:    h.title,
+		Template: name,
+		Data:     data,
 	})
 }
 
diff --git a/syz-cluster/dashboard/templates/base.html b/syz-cluster/dashboard/templates/base.html
index c154de1db..519e4f74a 100644
--- a/syz-cluster/dashboard/templates/base.html
+++ b/syz-cluster/dashboard/templates/base.html
@@ -9,13 +9,16 @@
     <title>{{.Title}}</title>
   </head>
   <body>
-    <nav class="navbar navbar-light bg-light justify-content-start">
+    <nav class="navbar navbar-light navbar-expand-lg bg-light justify-content-start">
       <a class="navbar-brand px-3" href="#">{{.Title}}</a>
-      <div class="">
+      <div class="collapse navbar-collapse">
         <ul class="navbar-nav">
-          <li class="nav-item active">
+          <li class="nav-item{{if eq .Template "index.html"}} active{{end}}">
             <a class="nav-link" href="/">All Series</a>
           </li>
+          <li class="nav-item{{if eq .Template "graphs.html"}} active{{end}}">
+            <a class="nav-link" href="/stats">Statistics</a>
+          </li>
         </ul>
       </div>
     </nav>
diff --git a/syz-cluster/dashboard/templates/graphs.html b/syz-cluster/dashboard/templates/graphs.html
new file mode 100644
index 000000000..7c0e56258
--- /dev/null
+++ b/syz-cluster/dashboard/templates/graphs.html
@@ -0,0 +1,114 @@
+{{define "content"}}
+    <main class="container-fluid my-4">
+      <div class="chart-wrapper mb-5">
+        <h2 class="h4">Processed Series (weekly)</h2>
+        <p class="text-muted small">Click and drag to zoom into a time range. Right-click to reset.</p>
+        <div id="processed_chart_div" style="width: 100%; height: 400px;"></div>
+      </div>
+
+      <hr class="my-4">
+
+      <div class="chart-wrapper mb-4">
+        <h2 class="h4">Findings (weekly)</h2>
+        <p class="text-muted small">Click and drag to zoom into a time range. Right-click to reset.</p>
+        <div id="findings_chart_div" style="width: 100%; height: 400px;"></div>
+      </div>
+
+      <hr class="my-4">
+
+      <div class="chart-wrapper mb-4">
+        <h2 class="h4">Wait Time before Fuzzing (avg hours, weekly)</h2>
+        <p class="text-muted small">How many hours have passed since the moment the series was published and the moment we started fuzzing it.</p>
+        <p class="text-muted small">Click and drag to zoom into a time range. Right-click to reset.</p>
+        <div id="avg_wait_chart_div" style="width: 100%; height: 400px;"></div>
+      </div>
+
+      <hr class="my-4">
+
+      <div class="chart-wrapper mb-4">
+        <h2 class="h4">Status Distribution (weekly)</h2>
+        <p class="text-muted small">Click and drag to zoom into a time range. Right-click to reset.</p>
+        <div id="distribution_chart_div" style="width: 100%; height: 400px;"></div>
+      </div>
+    </main>
+    <script type="text/javascript" src="https://www.gstatic.com/charts/loader.js"></script>
+    <script type="text/javascript">
+      google.charts.load('current', {'packages':['corechart']});
+      google.charts.setOnLoadCallback(drawAllCharts);
+
+      const processedData = [
+        {{range .Processed}}
+          [new Date({{.Date.Format "2006-01-02"}}), {{.Count}}],
+        {{end}}
+      ];
+
+      const findingsData = [
+        {{range .Findings}}
+          [new Date({{.Date.Format "2006-01-02"}}), {{.Count}}],
+        {{end}}
+      ];
+
+      const delayData = [
+        {{range .Delay}}
+          [new Date({{.Date.Format "2006-01-02"}}), {{.DelayHours}}],
+        {{end}}
+      ];
+
+      const distributionData = [
+        {{range .Distribution}}
+          [new Date({{.Date.Format "2006-01-02"}}), {{.Finished}}, {{.Skipped}}],
+        {{end}}
+      ];
+
+      function drawAllCharts() {
+        drawChart('processed_chart_div', processedData, '#007bff');
+        drawChart('findings_chart_div', findingsData, '#dc3545');
+        drawChart('avg_wait_chart_div', delayData, 'black');
+        drawDistributionChart();
+      }
+
+      function drawChart(chartDivId, chartData, chartColor) {
+        const data = new google.visualization.DataTable();
+        data.addColumn('date', 'Date');
+        data.addColumn('number', 'Count');
+        data.addRows(chartData);
+        const options = {
+          legend: { position: 'none' },
+          hAxis: { title: 'Date', format: 'MMM dd, yyyy', gridlines: { color: 'transparent' } },
+          vAxis: { title: 'Count', minValue: 0 },
+          colors: [chartColor],
+          curveType: 'function',
+          explorer: {
+            actions: ['dragToZoom', 'rightClickToReset'],
+            axis: 'horizontal',
+            keepInBounds: true,
+            maxZoomIn: 4.0
+          }
+        };
+
+        const chart = new google.visualization.LineChart(document.getElementById(chartDivId));
+        chart.draw(data, options);
+      }
+
+      function drawDistributionChart() {
+        const data = new google.visualization.DataTable();
+        data.addColumn('date', 'Date');
+        data.addColumn('number', 'Processed');
+        data.addColumn('number', 'Skipped');
+        data.addRows(distributionData);
+        const options = {
+          isStacked: 'percent',
+          legend: { position: 'top', maxLines: 2 },
+          hAxis: { title: 'Date', format: 'MMM dd, yyyy', gridlines: { color: 'transparent' } },
+          vAxis: {
+            minValue: 0,
+            format: '#%'
+          },
+          colors: ['#28a745', '#ffc107'],
+          areaOpacity: 0.8
+        };
+        const chart = new google.visualization.AreaChart(document.getElementById('distribution_chart_div'));
+        chart.draw(data, options);
+    }
+    </script>
+{{end}}
diff --git a/syz-cluster/pkg/db/stats_repo.go b/syz-cluster/pkg/db/stats_repo.go
new file mode 100644
index 000000000..fa6c1d901
--- /dev/null
+++ b/syz-cluster/pkg/db/stats_repo.go
@@ -0,0 +1,92 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package db
+
+import (
+	"context"
+	"time"
+
+	"cloud.google.com/go/spanner"
+)
+
+type StatsRepository struct {
+	client *spanner.Client
+}
+
+func NewStatsRepository(client *spanner.Client) *StatsRepository {
+	return &StatsRepository{
+		client: client,
+	}
+}
+
+type CountPerWeek struct {
+	Date  time.Time `spanner:"Date"`
+	Count int64     `spanner:"Count"`
+}
+
+func (repo *StatsRepository) ProcessedSeriesPerWeek(ctx context.Context) (
+	[]*CountPerWeek, error) {
+	return readEntities[CountPerWeek](ctx, repo.client.Single(), spanner.Statement{
+		SQL: `SELECT
+  TIMESTAMP_TRUNC(Sessions.FinishedAt, WEEK) as Date,
+  COUNT(*) as Count
+FROM Series
+JOIN Sessions ON Sessions.ID = Series.LatestSessionID
+WHERE FinishedAt IS NOT NULL
+GROUP BY Date
+ORDER BY Date`,
+	})
+}
+
+func (repo *StatsRepository) FindingsPerWeek(ctx context.Context) (
+	[]*CountPerWeek, error) {
+	return readEntities[CountPerWeek](ctx, repo.client.Single(), spanner.Statement{
+		SQL: `SELECT
+  TIMESTAMP_TRUNC(Sessions.FinishedAt, WEEK) as Date,
+  COUNT(*) as Count
+FROM Findings
+JOIN Sessions ON Sessions.ID = Findings.SessionID
+GROUP BY Date
+ORDER BY Date`,
+	})
+}
+
+type StatusPerWeek struct {
+	Date     time.Time `spanner:"Date"`
+	Finished int64     `spanner:"Finished"`
+	Skipped  int64     `spanner:"Skipped"`
+}
+
+func (repo *StatsRepository) SessionStatusPerWeek(ctx context.Context) (
+	[]*StatusPerWeek, error) {
+	return readEntities[StatusPerWeek](ctx, repo.client.Single(), spanner.Statement{
+		SQL: `SELECT
+  TIMESTAMP_TRUNC(Sessions.FinishedAt, WEEK) as Date,
+  COUNTIF(Sessions.SkipReason IS NULL) as Finished,
+  COUNTIF(Sessions.SkipReason IS NOT NULL) as Skipped
+FROM Series
+JOIN Sessions ON Sessions.ID = Series.LatestSessionID
+WHERE FinishedAt IS NOT NULL
+GROUP BY Date
+ORDER BY Date`,
+	})
+}
+
+type DelayPerWeek struct {
+	Date       time.Time `spanner:"Date"`
+	DelayHours float64   `spanner:"AvgDelayHours"`
+}
+
+func (repo *StatsRepository) DelayPerWeek(ctx context.Context) (
+	[]*DelayPerWeek, error) {
+	return readEntities[DelayPerWeek](ctx, repo.client.Single(), spanner.Statement{
+		SQL: `SELECT
+  TIMESTAMP_TRUNC(Sessions.StartedAt, WEEK) as Date,
+  AVG(TIMESTAMP_DIFF(Sessions.StartedAt,Sessions.CreatedAt, HOUR)) as AvgDelayHours
+FROM Sessions
+WHERE StartedAt IS NOT NULL
+GROUP BY Date
+ORDER BY Date`,
+	})
+}
diff --git a/syz-cluster/pkg/db/stats_repo_test.go b/syz-cluster/pkg/db/stats_repo_test.go
new file mode 100644
index 000000000..637894edf
--- /dev/null
+++ b/syz-cluster/pkg/db/stats_repo_test.go
@@ -0,0 +1,33 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package db
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStatsSQLs(t *testing.T) {
+	// Ideally, there should be some proper tests, but for now let's at least
+	// check that the SQL queries themselves have no errors.
+	// That already brings a lot of value.
+	client, ctx := NewTransientDB(t)
+
+	// Add some data to test field decoding as well.
+	dtd := &dummyTestData{t, ctx, client}
+	session := dtd.dummySession(dtd.dummySeries())
+	dtd.startSession(session)
+	dtd.finishSession(session)
+
+	statsRepo := NewStatsRepository(client)
+	_, err := statsRepo.ProcessedSeriesPerWeek(ctx)
+	assert.NoError(t, err)
+	_, err = statsRepo.FindingsPerWeek(ctx)
+	assert.NoError(t, err)
+	_, err = statsRepo.SessionStatusPerWeek(ctx)
+	assert.NoError(t, err)
+	_, err = statsRepo.DelayPerWeek(ctx)
+	assert.NoError(t, err)
+}
author	Aleksandr Nogikh <nogikh@google.com>	2025-08-13 17:32:02 +0200
committer	Aleksandr Nogikh <nogikh@google.com>	2025-08-14 11:29:47 +0000
commit	83b914a1837f0f4de012f4dc6bdc84c9d5bab713 (patch)
tree	4014acd61a94833bd66b17d8066979d2dee33807 /syz-cluster
parent	028e3578dbbde98fdb5a4d170f0e1dce7a8a2ab4 (diff)