From 685f11d0e806c0d613da4372a60e0a933d1b1422 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Thu, 4 Sep 2025 14:45:49 +0200 Subject: syz-cluster: support multiple campaigns per fuzz target During triage, process each fuzzing campaign separately as they may have different base kernel revisions (e.g. if the newest revisions of the kernel no longer build/boot under the specific kernel configuration). Refactor the representation of the fuzzing targets in api.go. --- syz-cluster/pkg/api/api.go | 128 ++++++++++++++++--------- syz-cluster/pkg/api/client.go | 2 +- syz-cluster/pkg/controller/api.go | 2 +- syz-cluster/pkg/triage/fuzz_config.go | 28 ------ syz-cluster/pkg/triage/fuzz_config_test.go | 51 ---------- syz-cluster/pkg/triage/fuzz_target.go | 28 ++++++ syz-cluster/pkg/triage/fuzz_target_test.go | 41 ++++++++ syz-cluster/workflow/build-step/Dockerfile | 1 + syz-cluster/workflow/rebuild-kernels-cron.yaml | 4 +- syz-cluster/workflow/triage-step/main.go | 96 ++++++++++++++----- 10 files changed, 226 insertions(+), 155 deletions(-) delete mode 100644 syz-cluster/pkg/triage/fuzz_config.go delete mode 100644 syz-cluster/pkg/triage/fuzz_config_test.go create mode 100644 syz-cluster/pkg/triage/fuzz_target.go create mode 100644 syz-cluster/pkg/triage/fuzz_target_test.go diff --git a/syz-cluster/pkg/api/api.go b/syz-cluster/pkg/api/api.go index a6a199431..feb06be32 100644 --- a/syz-cluster/pkg/api/api.go +++ b/syz-cluster/pkg/api/api.go @@ -39,10 +39,16 @@ type Tree struct { EmailLists []string `json:"email_lists"` } -// TriageFuzzConfig is a single record in the list of supported fuzz configs. -type TriageFuzzConfig struct { - EmailLists []string `json:"email_lists"` - KernelConfig string `json:"kernel_config"` +// FuzzTriageTarget is a single record in the list of supported fuzz configs. +type FuzzTriageTarget struct { + EmailLists []string `json:"email_lists"` + Campaigns []*KernelFuzzConfig `json:"campaigns"` +} + +// KernelFuzzConfig is a specific fuzzing assignment. +// Based on it, the triage step will construct FuzzTasks. +type KernelFuzzConfig struct { + KernelConfig string `json:"kernel_config"` FuzzConfig } @@ -234,32 +240,44 @@ const ( const kasanTrack = "KASAN" // The list is ordered by decreasing importance. -var FuzzConfigs = []*TriageFuzzConfig{ +var FuzzTargets = []*FuzzTriageTarget{ { - EmailLists: []string{`kvm@vger.kernel.org`}, - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `kvm`, - CorpusURL: allCorpusURL, + EmailLists: []string{`kvm@vger.kernel.org`}, + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `kvm`, + CorpusURL: allCorpusURL, + }, + }, }, }, { - EmailLists: []string{`io-uring@vger.kernel.org`}, - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `io-uring`, - CorpusURL: allCorpusURL, + EmailLists: []string{`io-uring@vger.kernel.org`}, + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `io-uring`, + CorpusURL: allCorpusURL, + }, + }, }, }, { - EmailLists: []string{`bpf@vger.kernel.org`}, - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `bpf`, - CorpusURL: bpfCorpusURL, + EmailLists: []string{`bpf@vger.kernel.org`}, + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `bpf`, + CorpusURL: bpfCorpusURL, + }, + }, }, }, { @@ -268,11 +286,15 @@ var FuzzConfigs = []*TriageFuzzConfig{ `netfilter-devel@vger.kernel.org`, `linux-wireless@vger.kernel.org`, }, - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `net`, - CorpusURL: netCorpusURL, + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `net`, + CorpusURL: netCorpusURL, + }, + }, }, }, { @@ -282,31 +304,43 @@ var FuzzConfigs = []*TriageFuzzConfig{ `linux-unionfs@vger.kernel.org`, `linux-ext4@vger.kernel.org`, }, - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `fs`, - CorpusURL: fsCorpusURL, + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `fs`, + CorpusURL: fsCorpusURL, + }, + }, }, }, { - EmailLists: []string{`linux-mm@kvack.org`}, - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `all`, - CorpusURL: allCorpusURL, - // Not all mm/ code is instrumented with KCOV. - SkipCoverCheck: true, + EmailLists: []string{`linux-mm@kvack.org`}, + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `all`, + CorpusURL: allCorpusURL, + // Not all mm/ code is instrumented with KCOV. + SkipCoverCheck: true, + }, + }, }, }, { - EmailLists: nil, // A fallback option. - KernelConfig: `upstream-apparmor-kasan.config`, - FuzzConfig: FuzzConfig{ - Track: kasanTrack, - Config: `all`, - CorpusURL: allCorpusURL, + EmailLists: nil, // A fallback option. + Campaigns: []*KernelFuzzConfig{ + { + KernelConfig: `upstream-apparmor-kasan.config`, + FuzzConfig: FuzzConfig{ + Track: kasanTrack, + Config: `all`, + CorpusURL: allCorpusURL, + }, + }, }, }, } diff --git a/syz-cluster/pkg/api/client.go b/syz-cluster/pkg/api/client.go index bd9e4466e..fb20cc4d7 100644 --- a/syz-cluster/pkg/api/client.go +++ b/syz-cluster/pkg/api/client.go @@ -42,7 +42,7 @@ func (client Client) UploadTriageResult(ctx context.Context, sessionID string, r type TreesResp struct { Trees []*Tree `json:"trees"` - FuzzConfigs []*TriageFuzzConfig `json:"fuzz_configs"` + FuzzTargets []*FuzzTriageTarget `json:"fuzz_targets"` } func (client Client) GetTrees(ctx context.Context) (*TreesResp, error) { diff --git a/syz-cluster/pkg/controller/api.go b/syz-cluster/pkg/controller/api.go index 4a230a35d..7520a05f9 100644 --- a/syz-cluster/pkg/controller/api.go +++ b/syz-cluster/pkg/controller/api.go @@ -207,7 +207,7 @@ func (c APIServer) uploadSession(w http.ResponseWriter, r *http.Request) { func (c APIServer) getTrees(w http.ResponseWriter, r *http.Request) { api.ReplyJSON(w, &api.TreesResp{ Trees: api.DefaultTrees, - FuzzConfigs: api.FuzzConfigs, + FuzzTargets: api.FuzzTargets, }) } diff --git a/syz-cluster/pkg/triage/fuzz_config.go b/syz-cluster/pkg/triage/fuzz_config.go deleted file mode 100644 index 05443d704..000000000 --- a/syz-cluster/pkg/triage/fuzz_config.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2025 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package triage - -import ( - "strings" - - "github.com/google/syzkaller/syz-cluster/pkg/api" -) - -func SelectFuzzConfig(series *api.Series, fuzzConfigs []*api.TriageFuzzConfig) *api.TriageFuzzConfig { - seriesCc := map[string]bool{} - for _, cc := range series.Cc { - seriesCc[strings.ToLower(cc)] = true - } - for _, config := range fuzzConfigs { - intersects := false - for _, cc := range config.EmailLists { - intersects = intersects || seriesCc[cc] - } - if len(config.EmailLists) != 0 && !intersects { - continue - } - return config - } - return nil -} diff --git a/syz-cluster/pkg/triage/fuzz_config_test.go b/syz-cluster/pkg/triage/fuzz_config_test.go deleted file mode 100644 index 04f948493..000000000 --- a/syz-cluster/pkg/triage/fuzz_config_test.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2025 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package triage - -import ( - "testing" - - "github.com/google/syzkaller/syz-cluster/pkg/api" - "github.com/stretchr/testify/assert" -) - -func TestSelectFuzzConfig(t *testing.T) { - configs := []*api.TriageFuzzConfig{ - { - EmailLists: []string{"bpf@list"}, - FuzzConfig: api.FuzzConfig{Config: "bpf"}, - }, - { - EmailLists: []string{"net@list"}, - FuzzConfig: api.FuzzConfig{Config: "net"}, - }, - { - EmailLists: nil, - FuzzConfig: api.FuzzConfig{Config: "mainline"}, - }, - } - tests := []struct { - testName string - result string - series *api.Series - }{ - { - testName: "select-first", - result: "bpf", - series: &api.Series{Cc: []string{"bpf@list", "net@list"}}, - }, - { - testName: "fallback", - result: "mainline", - series: &api.Series{Cc: []string{"unknown@list"}}, - }, - } - - for _, test := range tests { - t.Run(test.testName, func(t *testing.T) { - ret := SelectFuzzConfig(test.series, configs) - assert.Equal(t, test.result, ret.Config) - }) - } -} diff --git a/syz-cluster/pkg/triage/fuzz_target.go b/syz-cluster/pkg/triage/fuzz_target.go new file mode 100644 index 000000000..6fecc24c0 --- /dev/null +++ b/syz-cluster/pkg/triage/fuzz_target.go @@ -0,0 +1,28 @@ +// Copyright 2025 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package triage + +import ( + "strings" + + "github.com/google/syzkaller/syz-cluster/pkg/api" +) + +func SelectFuzzConfig(series *api.Series, fuzzConfigs []*api.FuzzTriageTarget) *api.FuzzTriageTarget { + seriesCc := map[string]bool{} + for _, cc := range series.Cc { + seriesCc[strings.ToLower(cc)] = true + } + for _, config := range fuzzConfigs { + intersects := false + for _, cc := range config.EmailLists { + intersects = intersects || seriesCc[cc] + } + if len(config.EmailLists) != 0 && !intersects { + continue + } + return config + } + return nil +} diff --git a/syz-cluster/pkg/triage/fuzz_target_test.go b/syz-cluster/pkg/triage/fuzz_target_test.go new file mode 100644 index 000000000..fa7039bda --- /dev/null +++ b/syz-cluster/pkg/triage/fuzz_target_test.go @@ -0,0 +1,41 @@ +// Copyright 2025 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package triage + +import ( + "testing" + + "github.com/google/syzkaller/syz-cluster/pkg/api" + "github.com/stretchr/testify/assert" +) + +func TestSelectFuzzConfig(t *testing.T) { + bpf := &api.FuzzTriageTarget{EmailLists: []string{"bpf@list"}} + net := &api.FuzzTriageTarget{EmailLists: []string{"net@list"}} + mainline := &api.FuzzTriageTarget{EmailLists: nil} + configs := []*api.FuzzTriageTarget{bpf, net, mainline} + tests := []struct { + testName string + result *api.FuzzTriageTarget + series *api.Series + }{ + { + testName: "select-first", + result: bpf, + series: &api.Series{Cc: []string{"bpf@list", "net@list"}}, + }, + { + testName: "fallback", + result: mainline, + series: &api.Series{Cc: []string{"unknown@list"}}, + }, + } + + for _, test := range tests { + t.Run(test.testName, func(t *testing.T) { + ret := SelectFuzzConfig(test.series, configs) + assert.Equal(t, test.result, ret) + }) + } +} diff --git a/syz-cluster/workflow/build-step/Dockerfile b/syz-cluster/workflow/build-step/Dockerfile index 413e1e2ed..074c91a25 100644 --- a/syz-cluster/workflow/build-step/Dockerfile +++ b/syz-cluster/workflow/build-step/Dockerfile @@ -23,6 +23,7 @@ RUN gzip -d /disk-images/buildroot_amd64_2024.09.gz # Download base kernel configs. RUN mkdir -p /kernel-configs ADD https://raw.githubusercontent.com/google/syzkaller/refs/heads/master/dashboard/config/linux/upstream-apparmor-kasan.config /kernel-configs/upstream-apparmor-kasan.config +ADD https://raw.githubusercontent.com/google/syzkaller/refs/heads/master/dashboard/config/linux/upstream-kmsan.config /kernel-configs/upstream-kmsan.config COPY --from=build-step-builder /build/build-step-bin /bin/build-step diff --git a/syz-cluster/workflow/rebuild-kernels-cron.yaml b/syz-cluster/workflow/rebuild-kernels-cron.yaml index 653f2edb6..2c4e4a226 100644 --- a/syz-cluster/workflow/rebuild-kernels-cron.yaml +++ b/syz-cluster/workflow/rebuild-kernels-cron.yaml @@ -63,7 +63,9 @@ spec: data = json.loads('''{{inputs.parameters.response}}''') unique_kernel_configs = sorted(list(set( - config["kernel_config"] for config in data.get("fuzz_configs", []) + campaign["kernel_config"] + for fuzz_config in data.get("fuzz_targets", []) + for campaign in fuzz_config.get("campaigns", []) ))) build_requests = [] for tree in data.get("trees", []): diff --git a/syz-cluster/workflow/triage-step/main.go b/syz-cluster/workflow/triage-step/main.go index cecc3ada8..7e8061aef 100644 --- a/syz-cluster/workflow/triage-step/main.go +++ b/syz-cluster/workflow/triage-step/main.go @@ -6,6 +6,7 @@ package main import ( "bytes" "context" + "errors" "flag" "fmt" @@ -36,7 +37,13 @@ func main() { ctx := context.Background() output := new(bytes.Buffer) tracer := &debugtracer.GenericTracer{WithTime: true, TraceWriter: output} - verdict, err := getVerdict(ctx, tracer, client, repo) + + triager := &seriesTriager{ + DebugTracer: tracer, + client: client, + ops: repo, + } + verdict, err := triager.GetVerdict(ctx, *flagSession) if err != nil { app.Fatalf("failed to get the verdict: %v", err) } @@ -56,14 +63,19 @@ func main() { // 2. What if controller does not reply? Let Argo just restart the step. } -func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api.Client, - ops triage.TreeOps) (*api.TriageResult, error) { - series, err := client.GetSessionSeries(ctx, *flagSession) +type seriesTriager struct { + debugtracer.DebugTracer + client *api.Client + ops triage.TreeOps +} + +func (triager *seriesTriager) GetVerdict(ctx context.Context, sessionID string) (*api.TriageResult, error) { + series, err := triager.client.GetSessionSeries(ctx, sessionID) if err != nil { // TODO: the workflow step must be retried. return nil, fmt.Errorf("failed to query series: %w", err) } - treesResp, err := client.GetTrees(ctx) + treesResp, err := triager.client.GetTrees(ctx) if err != nil { return nil, fmt.Errorf("failed to query trees: %w", err) } @@ -73,19 +85,40 @@ func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api SkipReason: "no suitable base kernel trees found", }, nil } - fuzzConfig := triage.SelectFuzzConfig(series, treesResp.FuzzConfigs) + fuzzConfig := triage.SelectFuzzConfig(series, treesResp.FuzzTargets) if fuzzConfig == nil { return &api.TriageResult{ SkipReason: "no suitable fuzz config found", }, nil } - var triageResult *api.TriageResult - for _, tree := range selectedTrees { - tracer.Log("considering tree %q", tree.Name) + ret := &api.TriageResult{} + for _, campaign := range fuzzConfig.Campaigns { + fuzzTask, err := triager.prepareFuzzingTask(ctx, series, selectedTrees, campaign) + var skipErr *SkipTriageError + if errors.As(err, &skipErr) { + ret.SkipReason = skipErr.Reason.Error() + continue + } else if err != nil { + return nil, err + } + ret.Fuzz = append(ret.Fuzz, fuzzTask) + } + if len(ret.Fuzz) > 0 { + // If we have prepared at least one fuzzing task, the series was not skipped. + ret.SkipReason = "" + } + return ret, nil +} + +func (triager *seriesTriager) prepareFuzzingTask(ctx context.Context, series *api.Series, trees []*api.Tree, + target *api.KernelFuzzConfig) (*api.FuzzTask, error) { + var skipErr error + for _, tree := range trees { + triager.Log("considering tree %q", tree.Name) arch := "amd64" - lastBuild, err := client.LastBuild(ctx, &api.LastBuildReq{ + lastBuild, err := triager.client.LastBuild(ctx, &api.LastBuildReq{ Arch: arch, - ConfigName: fuzzConfig.KernelConfig, + ConfigName: target.KernelConfig, TreeName: tree.Name, Status: api.BuildSuccess, }) @@ -93,40 +126,51 @@ func getVerdict(ctx context.Context, tracer debugtracer.DebugTracer, client *api // TODO: the workflow step must be retried. return nil, fmt.Errorf("failed to query the last build for %q: %w", tree.Name, err) } - tracer.Log("%q's last build: %q", tree.Name, lastBuild) - selector := triage.NewCommitSelector(ops, tracer) + triager.Log("%q's last build: %q", tree.Name, lastBuild) + selector := triage.NewCommitSelector(triager.ops, triager.DebugTracer) result, err := selector.Select(series, tree, lastBuild) if err != nil { // TODO: the workflow step must be retried. return nil, fmt.Errorf("failed to run the commit selector for %q: %w", tree.Name, err) } else if result.Commit == "" { // If we fail to find a suitable commit for all the trees, return an error just about the first one. - if triageResult == nil { - triageResult = &api.TriageResult{ - SkipReason: "failed to find a base commit: " + result.Reason, - } + if skipErr == nil { + skipErr = SkipError("failed to find a base commit: " + result.Reason) } - tracer.Log("failed to find a base commit for %q", tree.Name) + triager.Log("failed to find a base commit for %q", tree.Name) continue } - tracer.Log("selected base commit: %s", result.Commit) + triager.Log("selected base commit: %s", result.Commit) base := api.BuildRequest{ TreeName: tree.Name, TreeURL: tree.URL, - ConfigName: fuzzConfig.KernelConfig, + ConfigName: target.KernelConfig, CommitHash: result.Commit, Arch: arch, } fuzz := &api.FuzzTask{ Base: base, Patched: base, - FuzzConfig: fuzzConfig.FuzzConfig, + FuzzConfig: target.FuzzConfig, } fuzz.Patched.SeriesID = series.ID - triageResult = &api.TriageResult{ - Fuzz: []*api.FuzzTask{fuzz}, - } - break + return fuzz, nil } - return triageResult, nil + return nil, skipErr +} + +type SkipTriageError struct { + Reason error +} + +func SkipError(reason string) *SkipTriageError { + return &SkipTriageError{Reason: errors.New(reason)} +} + +func (e *SkipTriageError) Error() string { + return fmt.Sprintf("series must be skipped: %s", e.Reason) +} + +func (e *SkipTriageError) Unwrap() error { + return e.Reason } -- cgit mrf-deployment