syz-cluster: support multiple campaigns per fuzz target

During triage, process each fuzzing campaign separately as they may have
different base kernel revisions (e.g. if the newest revisions of the
kernel no longer build/boot under the specific kernel configuration).

Refactor the representation of the fuzzing targets in api.go.

8 files changed, 166 insertions, 95 deletions

diff --git a/syz-cluster/pkg/api/api.go b/syz-cluster/pkg/api/api.go
index a6a199431..feb06be32 100644
--- a/syz-cluster/pkg/api/api.go
+++ b/syz-cluster/pkg/api/api.go
@@ -39,10 +39,16 @@ type Tree struct {
 	EmailLists []string `json:"email_lists"`
 }
 
-// TriageFuzzConfig is a single record in the list of supported fuzz configs.
-type TriageFuzzConfig struct {
-	EmailLists   []string `json:"email_lists"`
-	KernelConfig string   `json:"kernel_config"`
+// FuzzTriageTarget is a single record in the list of supported fuzz configs.
+type FuzzTriageTarget struct {
+	EmailLists []string            `json:"email_lists"`
+	Campaigns  []*KernelFuzzConfig `json:"campaigns"`
+}
+
+// KernelFuzzConfig is a specific fuzzing assignment.
+// Based on it, the triage step will construct FuzzTasks.
+type KernelFuzzConfig struct {
+	KernelConfig string `json:"kernel_config"`
 	FuzzConfig
 }
 
@@ -234,32 +240,44 @@ const (
 const kasanTrack = "KASAN"
 
 // The list is ordered by decreasing importance.
-var FuzzConfigs = []*TriageFuzzConfig{
+var FuzzTargets = []*FuzzTriageTarget{
 	{
-		EmailLists:   []string{`kvm@vger.kernel.org`},
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `kvm`,
-			CorpusURL: allCorpusURL,
+		EmailLists: []string{`kvm@vger.kernel.org`},
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `kvm`,
+					CorpusURL: allCorpusURL,
+				},
+			},
 		},
 	},
 	{
-		EmailLists:   []string{`io-uring@vger.kernel.org`},
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `io-uring`,
-			CorpusURL: allCorpusURL,
+		EmailLists: []string{`io-uring@vger.kernel.org`},
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `io-uring`,
+					CorpusURL: allCorpusURL,
+				},
+			},
 		},
 	},
 	{
-		EmailLists:   []string{`bpf@vger.kernel.org`},
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `bpf`,
-			CorpusURL: bpfCorpusURL,
+		EmailLists: []string{`bpf@vger.kernel.org`},
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `bpf`,
+					CorpusURL: bpfCorpusURL,
+				},
+			},
 		},
 	},
 	{
@@ -268,11 +286,15 @@ var FuzzConfigs = []*TriageFuzzConfig{
 			`netfilter-devel@vger.kernel.org`,
 			`linux-wireless@vger.kernel.org`,
 		},
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `net`,
-			CorpusURL: netCorpusURL,
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `net`,
+					CorpusURL: netCorpusURL,
+				},
+			},
 		},
 	},
 	{
@@ -282,31 +304,43 @@ var FuzzConfigs = []*TriageFuzzConfig{
 			`linux-unionfs@vger.kernel.org`,
 			`linux-ext4@vger.kernel.org`,
 		},
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `fs`,
-			CorpusURL: fsCorpusURL,
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `fs`,
+					CorpusURL: fsCorpusURL,
+				},
+			},
 		},
 	},
 	{
-		EmailLists:   []string{`linux-mm@kvack.org`},
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `all`,
-			CorpusURL: allCorpusURL,
-			// Not all mm/ code is instrumented with KCOV.
-			SkipCoverCheck: true,
+		EmailLists: []string{`linux-mm@kvack.org`},
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `all`,
+					CorpusURL: allCorpusURL,
+					// Not all mm/ code is instrumented with KCOV.
+					SkipCoverCheck: true,
+				},
+			},
 		},
 	},
 	{
-		EmailLists:   nil, // A fallback option.
-		KernelConfig: `upstream-apparmor-kasan.config`,
-		FuzzConfig: FuzzConfig{
-			Track:     kasanTrack,
-			Config:    `all`,
-			CorpusURL: allCorpusURL,
+		EmailLists: nil, // A fallback option.
+		Campaigns: []*KernelFuzzConfig{
+			{
+				KernelConfig: `upstream-apparmor-kasan.config`,
+				FuzzConfig: FuzzConfig{
+					Track:     kasanTrack,
+					Config:    `all`,
+					CorpusURL: allCorpusURL,
+				},
+			},
 		},
 	},
 }
diff --git a/syz-cluster/pkg/api/client.go b/syz-cluster/pkg/api/client.go
index bd9e4466e..fb20cc4d7 100644
--- a/syz-cluster/pkg/api/client.go
+++ b/syz-cluster/pkg/api/client.go
@@ -42,7 +42,7 @@ func (client Client) UploadTriageResult(ctx context.Context, sessionID string, r
 
 type TreesResp struct {
 	Trees       []*Tree             `json:"trees"`
-	FuzzConfigs []*TriageFuzzConfig `json:"fuzz_configs"`
+	FuzzTargets []*FuzzTriageTarget `json:"fuzz_targets"`
 }
 
 func (client Client) GetTrees(ctx context.Context) (*TreesResp, error) {
diff --git a/syz-cluster/pkg/controller/api.go b/syz-cluster/pkg/controller/api.go
index 4a230a35d..7520a05f9 100644
--- a/syz-cluster/pkg/controller/api.go
+++ b/syz-cluster/pkg/controller/api.go
@@ -207,7 +207,7 @@ func (c APIServer) uploadSession(w http.ResponseWriter, r *http.Request) {
 func (c APIServer) getTrees(w http.ResponseWriter, r *http.Request) {
 	api.ReplyJSON(w, &api.TreesResp{
 		Trees:       api.DefaultTrees,
-		FuzzConfigs: api.FuzzConfigs,
+		FuzzTargets: api.FuzzTargets,
 	})
 }
 
diff --git a/syz-cluster/pkg/triage/fuzz_config.go b/syz-cluster/pkg/triage/fuzz_target.go
index 05443d704..6fecc24c0 100644
--- a/syz-cluster/pkg/triage/fuzz_config.go
+++ b/syz-cluster/pkg/triage/fuzz_target.go
@@ -9,7 +9,7 @@ import (
 	"github.com/google/syzkaller/syz-cluster/pkg/api"
 )
 
-func SelectFuzzConfig(series *api.Series, fuzzConfigs []*api.TriageFuzzConfig) *api.TriageFuzzConfig {
+func SelectFuzzConfig(series *api.Series, fuzzConfigs []*api.FuzzTriageTarget) *api.FuzzTriageTarget {
 	seriesCc := map[string]bool{}
 	for _, cc := range series.Cc {
 		seriesCc[strings.ToLower(cc)] = true
diff --git a/syz-cluster/pkg/triage/fuzz_config_test.go b/syz-cluster/pkg/triage/fuzz_target_test.go
index 04f948493..fa7039bda 100644
--- a/syz-cluster/pkg/triage/fuzz_config_test.go
+++ b/syz-cluster/pkg/triage/fuzz_target_test.go
@@ -11,33 +11,23 @@ import (
 )
 
 func TestSelectFuzzConfig(t *testing.T) {
-	configs := []*api.TriageFuzzConfig{
-		{
-			EmailLists: []string{"bpf@list"},
-			FuzzConfig: api.FuzzConfig{Config: "bpf"},
-		},
-		{
-			EmailLists: []string{"net@list"},
-			FuzzConfig: api.FuzzConfig{Config: "net"},
-		},
-		{
-			EmailLists: nil,
-			FuzzConfig: api.FuzzConfig{Config: "mainline"},
-		},
-	}
+	bpf := &api.FuzzTriageTarget{EmailLists: []string{"bpf@list"}}
+	net := &api.FuzzTriageTarget{EmailLists: []string{"net@list"}}
+	mainline := &api.FuzzTriageTarget{EmailLists: nil}
+	configs := []*api.FuzzTriageTarget{bpf, net, mainline}
 	tests := []struct {
 		testName string
-		result   string
+		result   *api.FuzzTriageTarget
 		series   *api.Series
 	}{
 		{
 			testName: "select-first",
-			result:   "bpf",
+			result:   bpf,
 			series:   &api.Series{Cc: []string{"bpf@list", "net@list"}},
 		},
 		{
 			testName: "fallback",
-			result:   "mainline",
+			result:   mainline,
 			series:   &api.Series{Cc: []string{"unknown@list"}},
 		},
 	}
@@ -45,7 +35,7 @@ func TestSelectFuzzConfig(t *testing.T) {
 	for _, test := range tests {
 		t.Run(test.testName, func(t *testing.T) {
 			ret := SelectFuzzConfig(test.series, configs)
-			assert.Equal(t, test.result, ret.Config)
+			assert.Equal(t, test.result, ret)
 		})
 	}
 }
diff --git a/syz-cluster/workflow/build-step/Dockerfile b/syz-cluster/workflow/build-step/Dockerfile
index 413e1e2ed..074c91a25 100644
--- a/syz-cluster/workflow/build-step/Dockerfile
+++ b/syz-cluster/workflow/build-step/Dockerfile
@@ -23,6 +23,7 @@ RUN gzip -d /disk-images/buildroot_amd64_2024.09.gz
 # Download base kernel configs.
 RUN mkdir -p /kernel-configs
 ADD https://raw.githubusercontent.com/google/syzkaller/refs/heads/master/dashboard/config/linux/upstream-apparmor-kasan.config /kernel-configs/upstream-apparmor-kasan.config
+ADD https://raw.githubusercontent.com/google/syzkaller/refs/heads/master/dashboard/config/linux/upstream-kmsan.config /kernel-configs/upstream-kmsan.config
 
 COPY --from=build-step-builder /build/build-step-bin /bin/build-step
 
diff --git a/syz-cluster/workflow/rebuild-kernels-cron.yaml b/syz-cluster/workflow/rebuild-kernels-cron.yaml
index 653f2edb6..2c4e4a226 100644
--- a/syz-cluster/workflow/rebuild-kernels-cron.yaml
+++ b/syz-cluster/workflow/rebuild-kernels-cron.yaml
@@ -63,7 +63,9 @@ spec:
 
           data = json.loads('''{{inputs.parameters.response}}''')
           unique_kernel_configs = sorted(list(set(
-              config["kernel_config"] for config in data.get("fuzz_configs", [])
+              campaign["kernel_config"]
+              for fuzz_config in data.get("fuzz_targets", [])
+              for campaign in fuzz_config.get("campaigns", [])
           )))
           build_requests = []
           for tree in data.get("trees", []):
diff --git a/syz-cluster/workflow/triage-step/main.go b/syz-cluster/workflow/triage-step/main.go
index cecc3ada8..7e8061aef 100644
--- a/syz-cluster/workflow/triage-step/main.go
+++ b/syz-cluster/workflow/triage-step/main.go
@@ -6,6 +6,7 @@ package main
 import (
 	"bytes"
 	"context"
+	"errors"
 	"flag"
 	"fmt"
 
@@ -36,7 +37,13 @@ func main() {
 	ctx := context.Background()
 	output := new(bytes.Buffer)
 	tracer := &debugtracer.GenericTracer{WithTime: true, TraceWriter: output}
-	verdict, err := getVerdict(ctx, tracer, client, repo)
+
+	triager := &seriesTriager{
+		DebugTracer: tracer,
+		client:      client,
+		ops:         repo,
+	}
+	verdict, err := triager.GetVerdict(ctx, *flagSession)
 	if err != nil {
 		app.Fatalf("failed to get the verdict: %v", err)
 	}
@@ -56,14 +63,19 @@ func main() {
 	// 2. What if controller does not reply? Let Argo just restart the step.
 }
 
-func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api.Client,
-	ops triage.TreeOps) (*api.TriageResult, error) {
-	series, err := client.GetSessionSeries(ctx, *flagSession)
+type seriesTriager struct {
+	debugtracer.DebugTracer
+	client *api.Client
+	ops    triage.TreeOps
+}
+
+func (triager *seriesTriager) GetVerdict(ctx context.Context, sessionID string) (*api.TriageResult, error) {
+	series, err := triager.client.GetSessionSeries(ctx, sessionID)
 	if err != nil {
 		// TODO: the workflow step must be retried.
 		return nil, fmt.Errorf("failed to query series: %w", err)
 	}
-	treesResp, err := client.GetTrees(ctx)
+	treesResp, err := triager.client.GetTrees(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("failed to query trees: %w", err)
 	}
@@ -73,19 +85,40 @@ func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api
 			SkipReason: "no suitable base kernel trees found",
 		}, nil
 	}
-	fuzzConfig := triage.SelectFuzzConfig(series, treesResp.FuzzConfigs)
+	fuzzConfig := triage.SelectFuzzConfig(series, treesResp.FuzzTargets)
 	if fuzzConfig == nil {
 		return &api.TriageResult{
 			SkipReason: "no suitable fuzz config found",
 		}, nil
 	}
-	var triageResult *api.TriageResult
-	for _, tree := range selectedTrees {
-		tracer.Log("considering tree %q", tree.Name)
+	ret := &api.TriageResult{}
+	for _, campaign := range fuzzConfig.Campaigns {
+		fuzzTask, err := triager.prepareFuzzingTask(ctx, series, selectedTrees, campaign)
+		var skipErr *SkipTriageError
+		if errors.As(err, &skipErr) {
+			ret.SkipReason = skipErr.Reason.Error()
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+		ret.Fuzz = append(ret.Fuzz, fuzzTask)
+	}
+	if len(ret.Fuzz) > 0 {
+		// If we have prepared at least one fuzzing task, the series was not skipped.
+		ret.SkipReason = ""
+	}
+	return ret, nil
+}
+
+func (triager *seriesTriager) prepareFuzzingTask(ctx context.Context, series *api.Series, trees []*api.Tree,
+	target *api.KernelFuzzConfig) (*api.FuzzTask, error) {
+	var skipErr error
+	for _, tree := range trees {
+		triager.Log("considering tree %q", tree.Name)
 		arch := "amd64"
-		lastBuild, err := client.LastBuild(ctx, &api.LastBuildReq{
+		lastBuild, err := triager.client.LastBuild(ctx, &api.LastBuildReq{
 			Arch:       arch,
-			ConfigName: fuzzConfig.KernelConfig,
+			ConfigName: target.KernelConfig,
 			TreeName:   tree.Name,
 			Status:     api.BuildSuccess,
 		})
@@ -93,40 +126,51 @@ func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api
 			// TODO: the workflow step must be retried.
 			return nil, fmt.Errorf("failed to query the last build for %q: %w", tree.Name, err)
 		}
-		tracer.Log("%q's last build: %q", tree.Name, lastBuild)
-		selector := triage.NewCommitSelector(ops, tracer)
+		triager.Log("%q's last build: %q", tree.Name, lastBuild)
+		selector := triage.NewCommitSelector(triager.ops, triager.DebugTracer)
 		result, err := selector.Select(series, tree, lastBuild)
 		if err != nil {
 			// TODO: the workflow step must be retried.
 			return nil, fmt.Errorf("failed to run the commit selector for %q: %w", tree.Name, err)
 		} else if result.Commit == "" {
 			// If we fail to find a suitable commit for all the trees, return an error just about the first one.
-			if triageResult == nil {
-				triageResult = &api.TriageResult{
-					SkipReason: "failed to find a base commit: " + result.Reason,
-				}
+			if skipErr == nil {
+				skipErr = SkipError("failed to find a base commit: " + result.Reason)
 			}
-			tracer.Log("failed to find a base commit for %q", tree.Name)
+			triager.Log("failed to find a base commit for %q", tree.Name)
 			continue
 		}
-		tracer.Log("selected base commit: %s", result.Commit)
+		triager.Log("selected base commit: %s", result.Commit)
 		base := api.BuildRequest{
 			TreeName:   tree.Name,
 			TreeURL:    tree.URL,
-			ConfigName: fuzzConfig.KernelConfig,
+			ConfigName: target.KernelConfig,
 			CommitHash: result.Commit,
 			Arch:       arch,
 		}
 		fuzz := &api.FuzzTask{
 			Base:       base,
 			Patched:    base,
-			FuzzConfig: fuzzConfig.FuzzConfig,
+			FuzzConfig: target.FuzzConfig,
 		}
 		fuzz.Patched.SeriesID = series.ID
-		triageResult = &api.TriageResult{
-			Fuzz: []*api.FuzzTask{fuzz},
-		}
-		break
+		return fuzz, nil
 	}
-	return triageResult, nil
+	return nil, skipErr
+}
+
+type SkipTriageError struct {
+	Reason error
+}
+
+func SkipError(reason string) *SkipTriageError {
+	return &SkipTriageError{Reason: errors.New(reason)}
+}
+
+func (e *SkipTriageError) Error() string {
+	return fmt.Sprintf("series must be skipped: %s", e.Reason)
+}
+
+func (e *SkipTriageError) Unwrap() error {
+	return e.Reason
 }