syz-cluster: support multiple campaigns per fuzz target

During triage, process each fuzzing campaign separately as they may have
different base kernel revisions (e.g. if the newest revisions of the
kernel no longer build/boot under the specific kernel configuration).

Refactor the representation of the fuzzing targets in api.go.

3 files changed, 74 insertions, 27 deletions

diff --git a/syz-cluster/workflow/build-step/Dockerfile b/syz-cluster/workflow/build-step/Dockerfile
index 413e1e2ed..074c91a25 100644
--- a/syz-cluster/workflow/build-step/Dockerfile
+++ b/syz-cluster/workflow/build-step/Dockerfile
@@ -23,6 +23,7 @@ RUN gzip -d /disk-images/buildroot_amd64_2024.09.gz
 # Download base kernel configs.
 RUN mkdir -p /kernel-configs
 ADD https://raw.githubusercontent.com/google/syzkaller/refs/heads/master/dashboard/config/linux/upstream-apparmor-kasan.config /kernel-configs/upstream-apparmor-kasan.config
+ADD https://raw.githubusercontent.com/google/syzkaller/refs/heads/master/dashboard/config/linux/upstream-kmsan.config /kernel-configs/upstream-kmsan.config
 
 COPY --from=build-step-builder /build/build-step-bin /bin/build-step
 
diff --git a/syz-cluster/workflow/rebuild-kernels-cron.yaml b/syz-cluster/workflow/rebuild-kernels-cron.yaml
index 653f2edb6..2c4e4a226 100644
--- a/syz-cluster/workflow/rebuild-kernels-cron.yaml
+++ b/syz-cluster/workflow/rebuild-kernels-cron.yaml
@@ -63,7 +63,9 @@ spec:
 
           data = json.loads('''{{inputs.parameters.response}}''')
           unique_kernel_configs = sorted(list(set(
-              config["kernel_config"] for config in data.get("fuzz_configs", [])
+              campaign["kernel_config"]
+              for fuzz_config in data.get("fuzz_targets", [])
+              for campaign in fuzz_config.get("campaigns", [])
           )))
           build_requests = []
           for tree in data.get("trees", []):
diff --git a/syz-cluster/workflow/triage-step/main.go b/syz-cluster/workflow/triage-step/main.go
index cecc3ada8..7e8061aef 100644
--- a/syz-cluster/workflow/triage-step/main.go
+++ b/syz-cluster/workflow/triage-step/main.go
@@ -6,6 +6,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 
@@ -36,7 +37,13 @@ func main() {
 	ctx := context.Background()
 	output := new(bytes.Buffer)
 	tracer := &debugtracer.GenericTracer{WithTime: true, TraceWriter: output}
-	verdict, err := getVerdict(ctx, tracer, client, repo)
+
+	triager := &seriesTriager{
+		DebugTracer: tracer,
+		client:      client,
+		ops:         repo,
+	}
+	verdict, err := triager.GetVerdict(ctx, *flagSession)
 	if err != nil {
 		app.Fatalf("failed to get the verdict: %v", err)
 	}
@@ -56,14 +63,19 @@ func main() {
 	// 2. What if controller does not reply? Let Argo just restart the step.
 }
 
-func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api.Client,
-	ops triage.TreeOps) (*api.TriageResult, error) {
-	series, err := client.GetSessionSeries(ctx, *flagSession)
+type seriesTriager struct {
+	debugtracer.DebugTracer
+	client *api.Client
+	ops    triage.TreeOps
+}
+
+func (triager *seriesTriager) GetVerdict(ctx context.Context, sessionID string) (*api.TriageResult, error) {
+	series, err := triager.client.GetSessionSeries(ctx, sessionID)
 	if err != nil {
 		// TODO: the workflow step must be retried.
 		return nil, fmt.Errorf("failed to query series: %w", err)
 	}
-	treesResp, err := client.GetTrees(ctx)
+	treesResp, err := triager.client.GetTrees(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to query trees: %w", err)
 	}
@@ -73,19 +85,40 @@ func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api
 			SkipReason: "no suitable base kernel trees found",
 		}, nil
 	}
-	fuzzConfig := triage.SelectFuzzConfig(series, treesResp.FuzzConfigs)
+	fuzzConfig := triage.SelectFuzzConfig(series, treesResp.FuzzTargets)
 	if fuzzConfig == nil {
 		return &api.TriageResult{
 			SkipReason: "no suitable fuzz config found",
 		}, nil
 	}
-	var triageResult *api.TriageResult
-	for _, tree := range selectedTrees {
-		tracer.Log("considering tree %q", tree.Name)
+	ret := &api.TriageResult{}
+	for _, campaign := range fuzzConfig.Campaigns {
+		fuzzTask, err := triager.prepareFuzzingTask(ctx, series, selectedTrees, campaign)
+		var skipErr *SkipTriageError
+		if errors.As(err, &skipErr) {
+			ret.SkipReason = skipErr.Reason.Error()
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+		ret.Fuzz = append(ret.Fuzz, fuzzTask)
+	}
+	if len(ret.Fuzz) > 0 {
+		// If we have prepared at least one fuzzing task, the series was not skipped.
+		ret.SkipReason = ""
+	}
+	return ret, nil
+}
+
+func (triager *seriesTriager) prepareFuzzingTask(ctx context.Context, series *api.Series, trees []*api.Tree,
+	target *api.KernelFuzzConfig) (*api.FuzzTask, error) {
+	var skipErr error
+	for _, tree := range trees {
+		triager.Log("considering tree %q", tree.Name)
 		arch := "amd64"
-		lastBuild, err := client.LastBuild(ctx, &api.LastBuildReq{
+		lastBuild, err := triager.client.LastBuild(ctx, &api.LastBuildReq{
 			Arch:       arch,
-			ConfigName: fuzzConfig.KernelConfig,
+			ConfigName: target.KernelConfig,
 			TreeName:   tree.Name,
 			Status:     api.BuildSuccess,
 		})
@@ -93,40 +126,51 @@ func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api
 			// TODO: the workflow step must be retried.
 			return nil, fmt.Errorf("failed to query the last build for %q: %w", tree.Name, err)
 		}
-		tracer.Log("%q's last build: %q", tree.Name, lastBuild)
-		selector := triage.NewCommitSelector(ops, tracer)
+		triager.Log("%q's last build: %q", tree.Name, lastBuild)
+		selector := triage.NewCommitSelector(triager.ops, triager.DebugTracer)
 		result, err := selector.Select(series, tree, lastBuild)
 		if err != nil {
 			// TODO: the workflow step must be retried.
 			return nil, fmt.Errorf("failed to run the commit selector for %q: %w", tree.Name, err)
 		} else if result.Commit == "" {
 			// If we fail to find a suitable commit for all the trees, return an error just about the first one.
-			if triageResult == nil {
-				triageResult = &api.TriageResult{
-					SkipReason: "failed to find a base commit: " + result.Reason,
-				}
+			if skipErr == nil {
+				skipErr = SkipError("failed to find a base commit: " + result.Reason)
 			}
-			tracer.Log("failed to find a base commit for %q", tree.Name)
+			triager.Log("failed to find a base commit for %q", tree.Name)
 			continue
 		}
-		tracer.Log("selected base commit: %s", result.Commit)
+		triager.Log("selected base commit: %s", result.Commit)
 		base := api.BuildRequest{
 			TreeName:   tree.Name,
 			TreeURL:    tree.URL,
-			ConfigName: fuzzConfig.KernelConfig,
+			ConfigName: target.KernelConfig,
 			CommitHash: result.Commit,
 			Arch:       arch,
 		}
 		fuzz := &api.FuzzTask{
 			Base:       base,
 			Patched:    base,
-			FuzzConfig: fuzzConfig.FuzzConfig,
+			FuzzConfig: target.FuzzConfig,
 		}
 		fuzz.Patched.SeriesID = series.ID
-		triageResult = &api.TriageResult{
-			Fuzz: []*api.FuzzTask{fuzz},
-		}
-		break
+		return fuzz, nil
 	}
-	return triageResult, nil
+	return nil, skipErr
+}
+
+type SkipTriageError struct {
+	Reason error
+}
+
+func SkipError(reason string) *SkipTriageError {
+	return &SkipTriageError{Reason: errors.New(reason)}
+}
+
+func (e *SkipTriageError) Error() string {
+	return fmt.Sprintf("series must be skipped: %s", e.Reason)
+}
+
+func (e *SkipTriageError) Unwrap() error {
+	return e.Reason
 }