syz-cluster: add a boot test

Run a smoke test on the base kernel build and report back the results.

8 files changed, 254 insertions, 13 deletions

diff --git a/syz-cluster/Makefile b/syz-cluster/Makefile
index a248a4af4..c58b168bb 100644
--- a/syz-cluster/Makefile
+++ b/syz-cluster/Makefile
@@ -37,6 +37,10 @@ build-build-step-dev:
 	eval $$(minikube docker-env);\
 	docker build -t build-step-local -f ./workflow/build-step/Dockerfile ../
 
+build-boot-step-dev:
+	eval $$(minikube docker-env);\
+	docker build -t boot-step-local -f ./workflow/boot-step/Dockerfile ../
+
 build-go-tests-dev:
 	eval $$(minikube docker-env);\
 	docker build -t go-tests-local -f Dockerfile.go-tests ../
diff --git a/syz-cluster/dashboard/templates/series.html b/syz-cluster/dashboard/templates/series.html
index e3afb88a6..2b39f8a69 100644
--- a/syz-cluster/dashboard/templates/series.html
+++ b/syz-cluster/dashboard/templates/series.html
@@ -1,4 +1,3 @@
-
 {{define "content"}}
     <div class="container">
         <h2>Patch Series</h2>
@@ -82,18 +81,14 @@
                 <tr>
                   <td colspan="4">
                     <table class="table mb-0">
-                      <thead>
-                        <tr>
-                          <th scope="col">Title</th>
-                        </tr>
-                      </thead>
                       <tbody>
-                      {{range . .Findings}}
+                      {{range .Findings}}
                       <tr>
                         <td>{{.Title}}</td>
                       </tr>
                       {{end}}
                       </tbody>
+                    </table>
                   </td>
                 </tr>
                 {{end}}
diff --git a/syz-cluster/overlays/dev/workflow-roles.yaml b/syz-cluster/overlays/dev/workflow-roles.yaml
index c6382a691..e4722a2be 100644
--- a/syz-cluster/overlays/dev/workflow-roles.yaml
+++ b/syz-cluster/overlays/dev/workflow-roles.yaml
@@ -7,9 +7,19 @@ metadata:
   namespace: default
   name: argo-workflow-role
 rules:
-- apiGroups: ["argoproj.io"]
-  resources: ["workflows"]
-  verbs: ["list", "create", "delete"]
+- apiGroups:
+    - argoproj.io
+  resources:
+    - workflowartifactgctasks
+    - workflows
+  verbs:
+    - get
+    - list
+    - watch
+    - create
+    - update
+    - patch
+    - delete
 
 ---
 
diff --git a/syz-cluster/pkg/workflow/template.yaml b/syz-cluster/pkg/workflow/template.yaml
index c8ef8357d..fe4b3c35f 100644
--- a/syz-cluster/pkg/workflow/template.yaml
+++ b/syz-cluster/pkg/workflow/template.yaml
@@ -11,6 +11,10 @@ spec:
     parameters:
       - name: session-id
         value: "some-session-id"
+# TODO: there seems to be no way to pass env variables into the GC workflow.
+# Set ARGO_ARTIFACT_GC_ENABLED=0 for the local setup?
+#  artifactGC:
+#    strategy: OnWorkflowCompletion
   templates:
     - name: main
       # Don't schedule new steps if any of the previous steps failed.
@@ -53,7 +57,7 @@ spec:
               failed: true
         - - name: break-if-succeeded
             template: exit-workflow
-            when: "{{steps['run-process-fuzz'].status}} == Succeeded"
+            when: "{{=steps['run-process-fuzz'].status == Succeeded}}"
     - name: process-fuzz
       inputs:
         parameters:
@@ -64,7 +68,7 @@ spec:
             arguments:
               parameters:
                 - name: data
-                  value: "{{= jsonpath(inputs.parameters.element, '$.base')}}"
+                  value: "{{=jsonpath(inputs.parameters.element, '$.base')}}"
         - - name: base-build
             templateRef:
               name: build-step-template
@@ -100,7 +104,29 @@ spec:
                   from: "{{steps.save-patched-req.outputs.artifacts.request}}"
         - - name: continue-if-patched-build-failed
             template: exit-workflow
-            when: "{{=jsonpath(steps['patched-build'].outputs.parameters.result, '$.success')}} == false"
+            when: "{{=jsonpath(steps['patched-build'].outputs.parameters.result, '$.success') == false}}"
+        - - name: boot-test-base
+            templateRef:
+              name: boot-step-template
+              template: boot-step
+            arguments:
+              artifacts:
+                - name: kernel
+                  from: "{{steps.base-build.outputs.artifacts.kernel}}"
+              parameters:
+                - name: config
+                  value: "{{=jsonpath(inputs.parameters.element, '$.config')}}"
+                - name: base-build-id
+                  value: "{{=jsonpath(steps['base-build'].outputs.parameters.result, '$.build_id')}}"
+                - name: test-name
+                  value: "Boot test: Base"
+        - - name: break-if-base-boot-failed
+            template: exit-workflow
+            arguments:
+              parameters:
+                - name: code
+                  value: 0
+            when: "{{=jsonpath(steps['boot-test-base'].outputs.parameters.result, '$.success') == false}}"
     - name: extract-request
       inputs:
         parameters:
diff --git a/syz-cluster/workflow/boot-step/Dockerfile b/syz-cluster/workflow/boot-step/Dockerfile
new file mode 100644
index 000000000..7fa89e04a
--- /dev/null
+++ b/syz-cluster/workflow/boot-step/Dockerfile
@@ -0,0 +1,45 @@
+# syntax=docker.io/docker/dockerfile:1.7-labs
+# Copyright 2025 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+# Build syzkaller.
+FROM gcr.io/syzkaller/env as syzkaller-builder
+WORKDIR /build
+# First query the modules to facilitate caching.
+COPY go.mod go.sum ./
+RUN go mod download
+COPY --exclude=vendor --exclude=syz-cluster . .
+RUN make TARGETARCH=amd64
+
+FROM golang:1.23-alpine AS boot-step-builder
+WORKDIR /build
+
+# Copy the code and the dependencies.
+COPY go.mod go.sum ./
+RUN go mod download
+COPY pkg/ pkg/
+# TODO: get rid of the prog/ dependency?
+COPY prog/ prog/
+COPY vm/ vm/
+COPY executor/ executor/
+COPY dashboard/dashapi/ dashboard/dashapi/
+# Copying from the builder to take the `make descriptions` result.
+COPY --from=syzkaller-builder /build/sys/ sys/
+COPY syz-cluster/workflow/boot-step/*.go syz-cluster/workflow/boot-step/
+COPY syz-cluster/pkg/ syz-cluster/pkg/
+
+RUN go build -o /build/boot-step-bin /build/syz-cluster/workflow/boot-step
+
+FROM debian:bookworm
+
+RUN apt-get update && \
+    apt-get install -y qemu-system openssh-client
+
+# pkg/osutil uses syzkaller user for sandboxing.
+RUN useradd --create-home syzkaller
+
+COPY --from=syzkaller-builder /build/bin/ /syzkaller/bin/
+COPY --from=boot-step-builder /build/boot-step-bin /bin/boot-step
+COPY syz-cluster/workflow/configs/ /configs/
+
+ENTRYPOINT ["/bin/boot-tracker"]
diff --git a/syz-cluster/workflow/boot-step/main.go b/syz-cluster/workflow/boot-step/main.go
new file mode 100644
index 000000000..353bc88a6
--- /dev/null
+++ b/syz-cluster/workflow/boot-step/main.go
@@ -0,0 +1,98 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"path/filepath"
+
+	"github.com/google/syzkaller/pkg/instance"
+	"github.com/google/syzkaller/pkg/mgrconfig"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/syz-cluster/pkg/api"
+	"github.com/google/syzkaller/syz-cluster/pkg/app"
+)
+
+var (
+	flagConfig       = flag.String("config", "", "syzkaller config")
+	flagSession      = flag.String("session", "", "session ID")
+	flagTestName     = flag.String("test_name", "", "test name")
+	flagBaseBuild    = flag.String("base_build", "", "base build ID")
+	flagPatchedBuild = flag.String("patched_build", "", "patched build ID")
+	flagOutput       = flag.String("output", "", "where to store the result")
+)
+
+func main() {
+	flag.Parse()
+	if *flagConfig == "" || *flagSession == "" || *flagTestName == "" {
+		app.Fatalf("--config, --session and --test_name must be set")
+	}
+
+	ctx := context.Background()
+	client := app.DefaultClient()
+
+	testResult := &api.TestResult{
+		SessionID:      *flagSession,
+		TestName:       *flagTestName,
+		BaseBuildID:    *flagBaseBuild,
+		PatchedBuildID: *flagPatchedBuild,
+		Result:         api.TestRunning,
+	}
+	// Report that we've begun the test -- it will let us report the findings.
+	err := client.UploadTestResult(ctx, testResult)
+	if err != nil {
+		app.Fatalf("failed to upload test result: %v", err)
+	}
+
+	bootedFine, err := runTest(ctx, client)
+	if err != nil {
+		app.Fatalf("failed to run the boot test: %v", err)
+	}
+	if bootedFine {
+		testResult.Result = api.TestPassed
+	} else {
+		testResult.Result = api.TestFailed
+	}
+
+	// Report the test results.
+	err = client.UploadTestResult(ctx, testResult)
+	if err != nil {
+		app.Fatalf("failed to upload test result: %v", err)
+	}
+	if *flagOutput != "" {
+		osutil.WriteJSON(*flagOutput, &api.BootResult{
+			Success: bootedFine,
+		})
+	}
+}
+
+func runTest(ctx context.Context, client *api.Client) (bool, error) {
+	cfg, err := mgrconfig.LoadFile(filepath.Join("/configs", *flagConfig, "base.cfg"))
+	if err != nil {
+		return false, err
+	}
+	cfg.Workdir = "/tmp/test-workdir"
+	rep, err := instance.RunSmokeTest(cfg)
+	if err != nil {
+		return false, err
+	}
+	if rep != nil {
+		log.Printf("uploading the finding %q", rep.Title)
+		findingErr := client.UploadFinding(ctx, &api.Finding{
+			SessionID: *flagSession,
+			TestName:  *flagTestName,
+			Title:     rep.Title,
+			Report:    rep.Report,
+			Log:       rep.Output,
+		})
+		if findingErr != nil {
+			return false, fmt.Errorf("failed to report the finding: %w", findingErr)
+		}
+		return false, nil
+	}
+	return true, nil
+}
diff --git a/syz-cluster/workflow/boot-step/workflow-template.yaml b/syz-cluster/workflow/boot-step/workflow-template.yaml
new file mode 100644
index 000000000..25a9c25ba
--- /dev/null
+++ b/syz-cluster/workflow/boot-step/workflow-template.yaml
@@ -0,0 +1,60 @@
+# Copyright 2025 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: boot-step-template
+spec:
+  templates:
+    - name: boot-step
+      inputs:
+        parameters:
+          - name: config
+            value: ""
+          - name: base-build-id
+            value: ""
+          - name: patched-build-id
+            value: ""
+          - name: test-name
+            value: ""
+        artifacts:
+          - name: kernel
+            path: /kernel
+      container:
+        image: boot-step-local
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/boot-step"]
+        args: [
+          "--config", "{{inputs.parameters.config}}",
+          "--output", "/output/result.json",
+          "--session", "{{workflow.parameters.session-id}}",
+          "--test_name", "{{inputs.parameters.test-name}}",
+          "--base_build", "{{inputs.parameters.base-build-id}}",
+          "--patched_build", "{{inputs.parameters.patched-build-id}}"
+          ]
+        volumeMounts:
+        - name: workdir
+          mountPath: /workdir
+        - name: output
+          mountPath: /output
+        - name: dev-kvm
+          mountPath: /dev/kvm
+        # Needed for /dev/kvm.
+        # TODO: there's a "device plugin" mechanism in k8s that can share it more safely.
+        securityContext:
+          privileged: true
+      volumes:
+        - name: workdir
+          emptyDir: {}
+        - name: output
+          emptyDir: {}
+        - name: dev-kvm
+          hostPath:
+            path: /dev/kvm
+            type: CharDevice
+      outputs:
+        parameters:
+          - name: result
+            valueFrom:
+              path: /output/result.json
diff --git a/syz-cluster/workflow/build-step/workflow-template.yaml b/syz-cluster/workflow/build-step/workflow-template.yaml
index 341d22072..fa6c9c0ca 100644
--- a/syz-cluster/workflow/build-step/workflow-template.yaml
+++ b/syz-cluster/workflow/build-step/workflow-template.yaml
@@ -96,3 +96,6 @@ spec:
           - name: result
             valueFrom:
               path: /output/result.json
+        artifacts:
+          - name: kernel
+            path: /output