aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--executor/common_test.h20
-rw-r--r--pkg/csource/generated.go19
-rw-r--r--pkg/fuzzer/corpus.go114
-rw-r--r--pkg/fuzzer/corpus_test.go (renamed from syz-fuzzer/fuzzer_test.go)22
-rw-r--r--pkg/fuzzer/fuzzer.go350
-rw-r--r--pkg/fuzzer/fuzzer_test.go290
-rw-r--r--pkg/fuzzer/job.go401
-rw-r--r--pkg/fuzzer/prio_queue.go100
-rw-r--r--pkg/fuzzer/prio_queue_test.go44
-rw-r--r--pkg/fuzzer/stats.go26
-rw-r--r--sys/targets/targets.go13
-rw-r--r--sys/test/expressions.txt.const2
-rw-r--r--sys/test/fuzzer.txt7
-rw-r--r--syz-fuzzer/fuzzer.go448
-rw-r--r--syz-fuzzer/proc.go345
-rw-r--r--syz-fuzzer/workqueue.go131
16 files changed, 1586 insertions, 746 deletions
diff --git a/executor/common_test.h b/executor/common_test.h
index 6ef6ed82d..971108df8 100644
--- a/executor/common_test.h
+++ b/executor/common_test.h
@@ -133,3 +133,23 @@ static int do_sandbox_none(void)
return 0;
}
#endif
+
+#if SYZ_EXECUTOR || __NR_syz_test_fuzzer1
+
+static void fake_crash(const char* name)
+{
+ failmsg("crash", "{{CRASH: %s}}", name);
+ doexit(1);
+}
+
+static long syz_test_fuzzer1(volatile long a, volatile long b, volatile long c)
+{
+ // We probably want something more interesting here.
+ if (a == 1 && b == 1 && c == 1)
+ fake_crash("first bug");
+ if (a == 1 && b == 2 && c == 3)
+ fake_crash("second bug");
+ return 0;
+}
+
+#endif
diff --git a/pkg/csource/generated.go b/pkg/csource/generated.go
index 7c01243ce..2c33e7a29 100644
--- a/pkg/csource/generated.go
+++ b/pkg/csource/generated.go
@@ -12330,6 +12330,25 @@ static int do_sandbox_none(void)
}
#endif
+#if SYZ_EXECUTOR || __NR_syz_test_fuzzer1
+
+static void fake_crash(const char* name)
+{
+ failmsg("crash", "{{CRASH: %s}}", name);
+ doexit(1);
+}
+
+static long syz_test_fuzzer1(volatile long a, volatile long b, volatile long c)
+{
+ if (a == 1 && b == 1 && c == 1)
+ fake_crash("first bug");
+ if (a == 1 && b == 2 && c == 3)
+ fake_crash("second bug");
+ return 0;
+}
+
+#endif
+
#elif GOOS_windows
#include <direct.h>
diff --git a/pkg/fuzzer/corpus.go b/pkg/fuzzer/corpus.go
new file mode 100644
index 000000000..b92ab1c64
--- /dev/null
+++ b/pkg/fuzzer/corpus.go
@@ -0,0 +1,114 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+import (
+ "math/rand"
+ "sort"
+ "sync"
+
+ "github.com/google/syzkaller/pkg/hash"
+ "github.com/google/syzkaller/pkg/signal"
+ "github.com/google/syzkaller/prog"
+)
+
+type Corpus struct {
+ mu sync.RWMutex
+ progs []*prog.Prog
+ hashes map[hash.Sig]struct{}
+ sumPrios int64
+ accPrios []int64
+ signal signal.Signal // signal of inputs in corpus
+ maxSignal signal.Signal // max signal ever observed (including flakes)
+ newSignal signal.Signal
+}
+
+// CorpusStat is a snapshot of the relevant current state figures.
+type CorpusStat struct {
+ Progs int
+ Signal int
+ MaxSignal int
+}
+
+func newCorpus() *Corpus {
+ return &Corpus{
+ hashes: make(map[hash.Sig]struct{}),
+ }
+}
+
+// TODO: maybe we want to treat progs from other fuzzers exactly like
+// candidates? And even triage them?
+func (corpus *Corpus) Save(p *prog.Prog, signal signal.Signal, sig hash.Sig) {
+ corpus.mu.Lock()
+ defer corpus.mu.Unlock()
+ if _, ok := corpus.hashes[sig]; !ok {
+ corpus.progs = append(corpus.progs, p)
+ corpus.hashes[sig] = struct{}{}
+ prio := int64(len(signal))
+ if prio == 0 {
+ prio = 1
+ }
+ corpus.sumPrios += prio
+ corpus.accPrios = append(corpus.accPrios, corpus.sumPrios)
+ }
+ corpus.signal.Merge(signal)
+ corpus.maxSignal.Merge(signal)
+}
+
+// Signal that should no longer be chased after.
+func (corpus *Corpus) AddMaxSignal(sign signal.Signal) {
+ // TODO: how do we ensure occasional drop of this max cover?
+ corpus.mu.Lock()
+ defer corpus.mu.Unlock()
+ corpus.maxSignal.Merge(sign)
+}
+
+func (corpus *Corpus) AddRawMaxSignal(signal []uint32, prio uint8) signal.Signal {
+ corpus.mu.Lock()
+ defer corpus.mu.Unlock()
+ diff := corpus.maxSignal.DiffRaw(signal, prio)
+ if diff.Empty() {
+ return diff
+ }
+ corpus.maxSignal.Merge(diff)
+ corpus.newSignal.Merge(diff)
+ return diff
+}
+
+func (corpus *Corpus) chooseProgram(r *rand.Rand) *prog.Prog {
+ corpus.mu.RLock()
+ defer corpus.mu.RUnlock()
+ if len(corpus.progs) == 0 {
+ return nil
+ }
+ randVal := r.Int63n(corpus.sumPrios + 1)
+ idx := sort.Search(len(corpus.accPrios), func(i int) bool {
+ return corpus.accPrios[i] >= randVal
+ })
+ return corpus.progs[idx]
+}
+
+func (corpus *Corpus) Programs() []*prog.Prog {
+ corpus.mu.RLock()
+ defer corpus.mu.RUnlock()
+ return corpus.progs
+}
+
+func (corpus *Corpus) GrabNewSignal() signal.Signal {
+ corpus.mu.Lock()
+ defer corpus.mu.Unlock()
+ sign := corpus.newSignal
+ corpus.newSignal = nil
+ return sign
+}
+
+func (corpus *Corpus) Stat() CorpusStat {
+ corpus.mu.RLock()
+ defer corpus.mu.RUnlock()
+ return CorpusStat{
+ Progs: len(corpus.progs),
+ Signal: len(corpus.signal),
+ MaxSignal: len(corpus.maxSignal),
+ }
+}
diff --git a/syz-fuzzer/fuzzer_test.go b/pkg/fuzzer/corpus_test.go
index dae8329f7..0b62d8c5a 100644
--- a/syz-fuzzer/fuzzer_test.go
+++ b/pkg/fuzzer/corpus_test.go
@@ -1,7 +1,7 @@
-// Copyright 2019 syzkaller project authors. All rights reserved.
+// Copyright 2024 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-package main
+package fuzzer
import (
"math"
@@ -24,7 +24,7 @@ func TestChooseProgram(t *testing.T) {
rs := rand.NewSource(0)
r := rand.New(rs)
target := getTarget(t, targets.TestOS, targets.TestArch64)
- fuzzer := &Fuzzer{corpusHashes: make(map[hash.Sig]struct{})}
+ corpus := newCorpus()
const (
maxIters = 1000
@@ -39,16 +39,15 @@ func TestChooseProgram(t *testing.T) {
sizeSig = 0
}
inp := generateInput(target, rs, 10, sizeSig)
- fuzzer.addInputToCorpus(inp.p, inp.sign, inp.sig)
+ corpus.Save(inp.p, inp.sign, inp.sig)
priorities[inp.p] = int64(len(inp.sign))
}
- snapshot := fuzzer.snapshot()
counters := make(map[*prog.Prog]int)
for it := 0; it < maxIters; it++ {
- counters[snapshot.chooseProgram(r)]++
+ counters[corpus.chooseProgram(r)]++
}
for p, prio := range priorities {
- prob := float64(prio) / float64(fuzzer.sumPrios)
+ prob := float64(prio) / float64(corpus.sumPrios)
diff := math.Abs(prob*maxIters - float64(counters[p]))
if diff > eps*maxIters {
t.Fatalf("the difference (%f) is higher than %f%%", diff, eps*100)
@@ -56,9 +55,9 @@ func TestChooseProgram(t *testing.T) {
}
}
-func TestAddInputConcurrency(t *testing.T) {
+func TestCorpusSaveConcurrency(t *testing.T) {
target := getTarget(t, targets.TestOS, targets.TestArch64)
- fuzzer := &Fuzzer{corpusHashes: make(map[hash.Sig]struct{})}
+ corpus := newCorpus()
const (
routines = 10
@@ -71,9 +70,8 @@ func TestAddInputConcurrency(t *testing.T) {
r := rand.New(rs)
for it := 0; it < iters; it++ {
inp := generateInput(target, rs, 10, it)
- fuzzer.addInputToCorpus(inp.p, inp.sign, inp.sig)
- snapshot := fuzzer.snapshot()
- snapshot.chooseProgram(r).Clone()
+ corpus.Save(inp.p, inp.sign, inp.sig)
+ corpus.chooseProgram(r).Clone()
}
}()
}
diff --git a/pkg/fuzzer/fuzzer.go b/pkg/fuzzer/fuzzer.go
new file mode 100644
index 000000000..64be8325c
--- /dev/null
+++ b/pkg/fuzzer/fuzzer.go
@@ -0,0 +1,350 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+import (
+ "context"
+ "fmt"
+ "math/rand"
+ "runtime"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/syzkaller/pkg/hash"
+ "github.com/google/syzkaller/pkg/ipc"
+ "github.com/google/syzkaller/pkg/rpctype"
+ "github.com/google/syzkaller/prog"
+)
+
+type Fuzzer struct {
+ Config *Config
+ Corpus *Corpus
+ NeedCandidates chan struct{}
+
+ ctx context.Context
+ mu sync.Mutex
+ stats map[string]uint64
+ rnd *rand.Rand
+ target *prog.Target
+
+ ct *prog.ChoiceTable
+ ctProgs int
+ ctMu sync.Mutex // TODO: use RWLock.
+ ctRegenerate chan struct{}
+
+ nextExec *priorityQueue[*Request]
+ runningExecs map[*Request]time.Time
+ nextJobID atomic.Int64
+
+ queuedCandidates atomic.Int64
+ // If the source of candidates runs out of them, we risk
+ // generating too many needCandidate requests (one for
+ // each Config.MinCandidates). We prevent this with candidatesRequested.
+ candidatesRequested atomic.Bool
+}
+
+func NewFuzzer(ctx context.Context, cfg *Config, rnd *rand.Rand,
+ target *prog.Target) *Fuzzer {
+ f := &Fuzzer{
+ Config: cfg,
+ Corpus: newCorpus(),
+ NeedCandidates: make(chan struct{}, 1),
+
+ ctx: ctx,
+ stats: map[string]uint64{},
+ rnd: rnd,
+ target: target,
+
+ // We're okay to lose some of the messages -- if we are already
+ // regenerating the table, we don't want to repeat it right away.
+ ctRegenerate: make(chan struct{}),
+
+ nextExec: makePriorityQueue[*Request](),
+ runningExecs: map[*Request]time.Time{},
+ }
+ f.updateChoiceTable(nil)
+ go f.choiceTableUpdater()
+ if cfg.Debug {
+ go f.leakDetector()
+ go f.logCurrentStats()
+ }
+ return f
+}
+
+type Config struct {
+ Debug bool
+ Logf func(level int, msg string, args ...interface{})
+ Coverage bool
+ FaultInjection bool
+ Comparisons bool
+ Collide bool
+ EnabledCalls map[*prog.Syscall]bool
+ NoMutateCalls map[int]bool
+ LeakChecking bool
+ FetchRawCover bool
+ // If the number of queued candidates is less than MinCandidates,
+ // NeedCandidates is triggered.
+ MinCandidates uint
+ NewInputs chan rpctype.Input
+}
+
+type Request struct {
+ Prog *prog.Prog
+ NeedCover bool
+ NeedRawCover bool
+ NeedSignal bool
+ NeedHints bool
+ // Fields that are only relevant within pkg/fuzzer.
+ flags ProgTypes
+ stat string
+ result *Result
+ resultC chan *Result
+}
+
+type Result struct {
+ Info *ipc.ProgInfo
+ Stop bool
+}
+
+func (fuzzer *Fuzzer) Done(req *Request, res *Result) {
+ // Triage individual calls.
+ // We do it before unblocking the waiting threads because
+ // it may result it concurrent modification of req.Prog.
+ if req.NeedSignal && res.Info != nil {
+ for call, info := range res.Info.Calls {
+ fuzzer.triageProgCall(req.Prog, &info, call, req.flags)
+ }
+ fuzzer.triageProgCall(req.Prog, &res.Info.Extra, -1, req.flags)
+ }
+ // Unblock threads that wait for the result.
+ req.result = res
+ if req.resultC != nil {
+ req.resultC <- res
+ }
+ // Update stats.
+ fuzzer.mu.Lock()
+ fuzzer.stats[req.stat]++
+ delete(fuzzer.runningExecs, req)
+ fuzzer.mu.Unlock()
+}
+
+func (fuzzer *Fuzzer) triageProgCall(p *prog.Prog, info *ipc.CallInfo, call int,
+ flags ProgTypes) {
+ prio := signalPrio(p, info, call)
+ newMaxSignal := fuzzer.Corpus.AddRawMaxSignal(info.Signal, prio)
+ if newMaxSignal.Empty() {
+ return
+ }
+ fuzzer.Logf(2, "found new signal in call %d in %s", call, p)
+ fuzzer.startJob(&triageJob{
+ p: p.Clone(),
+ call: call,
+ info: *info,
+ newSignal: newMaxSignal,
+ flags: flags,
+ jobPriority: triageJobPrio(flags),
+ })
+}
+
+func signalPrio(p *prog.Prog, info *ipc.CallInfo, call int) (prio uint8) {
+ if call == -1 {
+ return 0
+ }
+ if info.Errno == 0 {
+ prio |= 1 << 1
+ }
+ if !p.Target.CallContainsAny(p.Calls[call]) {
+ prio |= 1 << 0
+ }
+ return
+}
+
+type Candidate struct {
+ Prog *prog.Prog
+ Hash hash.Sig
+ Smashed bool
+ Minimized bool
+}
+
+func (fuzzer *Fuzzer) NextInput() *Request {
+ req := fuzzer.nextInput()
+ fuzzer.mu.Lock()
+ fuzzer.runningExecs[req] = time.Now()
+ fuzzer.mu.Unlock()
+ if req.stat == statCandidate {
+ if fuzzer.queuedCandidates.Add(-1) < 0 {
+ panic("queuedCandidates is out of sync")
+ }
+ }
+ if fuzzer.NeedCandidatesNow() &&
+ !fuzzer.candidatesRequested.CompareAndSwap(false, true) {
+ select {
+ case fuzzer.NeedCandidates <- struct{}{}:
+ default:
+ }
+ }
+ return req
+}
+
+func (fuzzer *Fuzzer) nextInput() *Request {
+ nextExec := fuzzer.nextExec.tryPop()
+ if nextExec != nil {
+ return nextExec.value
+ }
+ // Either generate a new input or mutate an existing one.
+ mutateRate := 0.95
+ if !fuzzer.Config.Coverage {
+ // If we don't have real coverage signal, generate programs
+ // more frequently because fallback signal is weak.
+ mutateRate = 0.5
+ }
+ rnd := fuzzer.rand()
+ if rnd.Float64() < mutateRate {
+ req := mutateProgRequest(fuzzer, rnd)
+ if req != nil {
+ return req
+ }
+ }
+ return genProgRequest(fuzzer, rnd)
+}
+
+func (fuzzer *Fuzzer) startJob(newJob job) {
+ fuzzer.Logf(2, "started %T", newJob)
+ newJob.saveID(-fuzzer.nextJobID.Add(1))
+ go newJob.run(fuzzer)
+}
+
+func (fuzzer *Fuzzer) Logf(level int, msg string, args ...interface{}) {
+ if fuzzer.Config.Logf == nil {
+ return
+ }
+ fuzzer.Config.Logf(level, msg, args...)
+}
+
+func (fuzzer *Fuzzer) NeedCandidatesNow() bool {
+ return fuzzer.queuedCandidates.Load() < int64(fuzzer.Config.MinCandidates)
+}
+
+func (fuzzer *Fuzzer) AddCandidates(candidates []Candidate) {
+ fuzzer.queuedCandidates.Add(int64(len(candidates)))
+ for _, candidate := range candidates {
+ fuzzer.pushExec(candidateRequest(candidate), priority{candidatePrio})
+ }
+ fuzzer.candidatesRequested.Store(false)
+}
+
+func (fuzzer *Fuzzer) rand() *rand.Rand {
+ fuzzer.mu.Lock()
+ seed := fuzzer.rnd.Int63()
+ fuzzer.mu.Unlock()
+ return rand.New(rand.NewSource(seed))
+}
+
+func (fuzzer *Fuzzer) pushExec(req *Request, prio priority) {
+ if req.stat == "" {
+ panic("Request.Stat field must be set")
+ }
+ if req.NeedHints && (req.NeedCover || req.NeedSignal) {
+ panic("Request.NeedHints is mutually exclusive with other fields")
+ }
+ fuzzer.nextExec.push(&priorityQueueItem[*Request]{
+ value: req, prio: prio,
+ })
+}
+
+func (fuzzer *Fuzzer) exec(job job, req *Request) *Result {
+ req.resultC = make(chan *Result, 1)
+ fuzzer.pushExec(req, job.priority())
+ select {
+ case <-fuzzer.ctx.Done():
+ return &Result{Stop: true}
+ case res := <-req.resultC:
+ close(req.resultC)
+ return res
+ }
+}
+
+func (fuzzer *Fuzzer) leakDetector() {
+ const timeout = 20 * time.Minute
+ ticket := time.NewTicker(timeout)
+ defer ticket.Stop()
+ for {
+ select {
+ case now := <-ticket.C:
+ fuzzer.mu.Lock()
+ for req, startedTime := range fuzzer.runningExecs {
+ if now.Sub(startedTime) > timeout {
+ panic(fmt.Sprintf("execution timed out: %v", req))
+ }
+ }
+ fuzzer.mu.Unlock()
+ case <-fuzzer.ctx.Done():
+ return
+ }
+ }
+}
+
+func (fuzzer *Fuzzer) updateChoiceTable(programs []*prog.Prog) {
+ newCt := fuzzer.target.BuildChoiceTable(programs, fuzzer.Config.EnabledCalls)
+
+ fuzzer.ctMu.Lock()
+ defer fuzzer.ctMu.Unlock()
+ if len(programs) >= fuzzer.ctProgs {
+ fuzzer.ctProgs = len(programs)
+ fuzzer.ct = newCt
+ }
+}
+
+func (fuzzer *Fuzzer) choiceTableUpdater() {
+ for {
+ select {
+ case <-fuzzer.ctx.Done():
+ return
+ case <-fuzzer.ctRegenerate:
+ }
+ fuzzer.updateChoiceTable(fuzzer.Corpus.Programs())
+ }
+}
+
+func (fuzzer *Fuzzer) ChoiceTable() *prog.ChoiceTable {
+ progs := fuzzer.Corpus.Programs()
+
+ fuzzer.ctMu.Lock()
+ defer fuzzer.ctMu.Unlock()
+
+ // There were no deep ideas nor any calculations behind these numbers.
+ regenerateEveryProgs := 333
+ if len(progs) < 100 {
+ regenerateEveryProgs = 33
+ }
+ if fuzzer.ctProgs+regenerateEveryProgs < len(progs) {
+ select {
+ case fuzzer.ctRegenerate <- struct{}{}:
+ default:
+ // We're okay to lose the message.
+ // It means that we're already regenerating the table.
+ }
+ }
+ return fuzzer.ct
+}
+
+func (fuzzer *Fuzzer) logCurrentStats() {
+ for {
+ select {
+ case <-time.After(time.Minute):
+ case <-fuzzer.ctx.Done():
+ return
+ }
+
+ var m runtime.MemStats
+ runtime.ReadMemStats(&m)
+
+ fuzzer.mu.Lock()
+ str := fmt.Sprintf("exec queue size: %d, running execs: %d, heap (MB): %d",
+ fuzzer.nextExec.Len(), len(fuzzer.runningExecs), m.Alloc/1000/1000)
+ fuzzer.mu.Unlock()
+ fuzzer.Logf(0, "%s", str)
+ }
+}
diff --git a/pkg/fuzzer/fuzzer_test.go b/pkg/fuzzer/fuzzer_test.go
new file mode 100644
index 000000000..1896b2b84
--- /dev/null
+++ b/pkg/fuzzer/fuzzer_test.go
@@ -0,0 +1,290 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+import (
+ "bytes"
+ "context"
+ "fmt"
+ "hash/crc32"
+ "math/rand"
+ "os"
+ "path/filepath"
+ "regexp"
+ "runtime"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/google/syzkaller/pkg/csource"
+ "github.com/google/syzkaller/pkg/ipc"
+ "github.com/google/syzkaller/pkg/ipc/ipcconfig"
+ "github.com/google/syzkaller/pkg/rpctype"
+ "github.com/google/syzkaller/pkg/testutil"
+ "github.com/google/syzkaller/prog"
+ "github.com/google/syzkaller/sys/targets"
+ "github.com/stretchr/testify/assert"
+ "golang.org/x/sync/errgroup"
+)
+
+func TestFuzz(t *testing.T) {
+ defer checkGoroutineLeaks()
+
+ target, err := prog.GetTarget(targets.TestOS, targets.TestArch64Fuzz)
+ if err != nil {
+ t.Fatal(err)
+ }
+ executor := buildExecutor(t, target)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ fuzzer := NewFuzzer(ctx, &Config{
+ Debug: true,
+ Logf: func(level int, msg string, args ...interface{}) {
+ if level > 1 {
+ return
+ }
+ t.Logf(msg, args...)
+ },
+ Coverage: true,
+ EnabledCalls: map[*prog.Syscall]bool{
+ target.SyscallMap["syz_test_fuzzer1"]: true,
+ },
+ NewInputs: make(chan rpctype.Input),
+ }, rand.New(testutil.RandSource(t)), target)
+
+ go func() {
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case c := <-fuzzer.Config.NewInputs:
+ t.Logf("new prog:\n%s", c.Prog)
+ }
+ }
+ }()
+
+ tf := newTestFuzzer(t, fuzzer, map[string]bool{
+ "first bug": true,
+ "second bug": true,
+ }, 10000)
+
+ for i := 0; i < 2; i++ {
+ tf.registerExecutor(newProc(t, target, executor))
+ }
+ tf.wait()
+
+ t.Logf("resulting corpus:")
+ for _, p := range fuzzer.Corpus.Programs() {
+ t.Logf("-----")
+ t.Logf("%s", p.Serialize())
+ }
+
+ assert.Equal(t, len(tf.expectedCrashes), len(tf.crashes),
+ "not all expected crashes were found")
+}
+
+func BenchmarkFuzzer(b *testing.B) {
+ b.ReportAllocs()
+ target, err := prog.GetTarget(targets.TestOS, targets.TestArch64Fuzz)
+ if err != nil {
+ b.Fatal(err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ calls := map[*prog.Syscall]bool{}
+ for _, c := range target.Syscalls {
+ calls[c] = true
+ }
+ fuzzer := NewFuzzer(ctx, &Config{
+ Coverage: true,
+ EnabledCalls: calls,
+ }, rand.New(rand.NewSource(time.Now().UnixNano())), target)
+
+ b.ResetTimer()
+ b.RunParallel(func(pb *testing.PB) {
+ for pb.Next() {
+ req := fuzzer.NextInput()
+ res, _, _ := emulateExec(req)
+ fuzzer.Done(req, res)
+ }
+ })
+}
+
+// Based on the example from Go documentation.
+var crc32q = crc32.MakeTable(0xD5828281)
+
+func emulateExec(req *Request) (*Result, string, error) {
+ serializedLines := bytes.Split(req.Prog.Serialize(), []byte("\n"))
+ var info ipc.ProgInfo
+ for i, call := range req.Prog.Calls {
+ cover := uint32(call.Meta.ID*1024) +
+ crc32.Checksum(serializedLines[i], crc32q)%4
+ callInfo := ipc.CallInfo{}
+ if req.NeedCover {
+ callInfo.Cover = []uint32{cover}
+ }
+ if req.NeedSignal {
+ callInfo.Signal = []uint32{cover}
+ }
+ info.Calls = append(info.Calls, callInfo)
+ }
+ return &Result{Info: &info}, "", nil
+}
+
+type testFuzzer struct {
+ t testing.TB
+ eg errgroup.Group
+ fuzzer *Fuzzer
+ mu sync.Mutex
+ crashes map[string]int
+ expectedCrashes map[string]bool
+ iter int
+ iterLimit int
+}
+
+func newTestFuzzer(t testing.TB, fuzzer *Fuzzer, expectedCrashes map[string]bool, iterLimit int) *testFuzzer {
+ return &testFuzzer{
+ t: t,
+ fuzzer: fuzzer,
+ expectedCrashes: expectedCrashes,
+ crashes: map[string]int{},
+ iterLimit: iterLimit,
+ }
+}
+
+func (f *testFuzzer) oneMore() bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.iter++
+ if f.iter%100 == 0 {
+ stat := f.fuzzer.Corpus.Stat()
+ f.t.Logf("<iter %d>: corpus %d, signal %d, max signal %d, crash types %d",
+ f.iter, stat.Progs, stat.Signal, stat.MaxSignal, len(f.crashes))
+ }
+ return f.iter < f.iterLimit &&
+ (f.expectedCrashes == nil || len(f.crashes) != len(f.expectedCrashes))
+}
+
+func (f *testFuzzer) registerExecutor(proc *executorProc) {
+ f.eg.Go(func() error {
+ for f.oneMore() {
+ req := f.fuzzer.NextInput()
+ res, crash, err := proc.execute(req)
+ if err != nil {
+ return err
+ }
+ if crash != "" {
+ res = &Result{Stop: true}
+ if !f.expectedCrashes[crash] {
+ return fmt.Errorf("unexpected crash: %q", crash)
+ }
+ f.mu.Lock()
+ f.t.Logf("CRASH: %s", crash)
+ f.crashes[crash]++
+ f.mu.Unlock()
+ }
+ f.fuzzer.Done(req, res)
+ }
+ return nil
+ })
+}
+
+func (f *testFuzzer) wait() {
+ t := f.t
+ err := f.eg.Wait()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("crashes:")
+ for title, cnt := range f.crashes {
+ t.Logf("%s: %d", title, cnt)
+ }
+ t.Logf("stats:\n%v", f.fuzzer.GrabStats())
+}
+
+// TODO: it's already implemented in syz-fuzzer/proc.go,
+// pkg/runtest and tools/syz-execprog.
+// Looks like it's time to factor out this functionality.
+type executorProc struct {
+ env *ipc.Env
+ execOpts ipc.ExecOpts
+}
+
+func newProc(t *testing.T, target *prog.Target, executor string) *executorProc {
+ config, execOpts, err := ipcconfig.Default(target)
+ if err != nil {
+ t.Fatal(err)
+ }
+ config.Executor = executor
+ config.Flags |= ipc.FlagSignal
+ env, err := ipc.MakeEnv(config, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { env.Close() })
+ return &executorProc{
+ env: env,
+ execOpts: *execOpts,
+ }
+}
+
+var crashRe = regexp.MustCompile(`{{CRASH: (.*?)}}`)
+
+func (proc *executorProc) execute(req *Request) (*Result, string, error) {
+ execOpts := proc.execOpts
+ // TODO: it's duplicated from fuzzer.go.
+ if req.NeedSignal {
+ execOpts.Flags |= ipc.FlagCollectSignal
+ }
+ if req.NeedCover {
+ execOpts.Flags |= ipc.FlagCollectCover
+ }
+ // TODO: support req.NeedHints.
+ output, info, _, err := proc.env.Exec(&execOpts, req.Prog)
+ ret := crashRe.FindStringSubmatch(string(output))
+ if ret != nil {
+ return nil, ret[1], nil
+ } else if err != nil {
+ return nil, "", err
+ }
+ return &Result{Info: info}, "", nil
+}
+
+func buildExecutor(t *testing.T, target *prog.Target) string {
+ executor, err := csource.BuildFile(target,
+ filepath.FromSlash("../../executor/executor.cc"),
+ "-fsanitize-coverage=trace-pc", "-g",
+ )
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { os.Remove(executor) })
+ return executor
+}
+
+func checkGoroutineLeaks() {
+ // Inspired by src/net/http/main_test.go.
+ buf := make([]byte, 2<<20)
+ err := ""
+ for i := 0; i < 3; i++ {
+ buf = buf[:runtime.Stack(buf, true)]
+ err = ""
+ for _, g := range strings.Split(string(buf), "\n\n") {
+ if !strings.Contains(g, "pkg/fuzzer/fuzzer.go") {
+ continue
+ }
+ err = fmt.Sprintf("%sLeaked goroutine:\n%s", err, g)
+ }
+ if err == "" {
+ return
+ }
+ // Give ctx.Done() a chance to propagate to all goroutines.
+ time.Sleep(100 * time.Millisecond)
+ }
+ if err != "" {
+ panic(err)
+ }
+}
diff --git a/pkg/fuzzer/job.go b/pkg/fuzzer/job.go
new file mode 100644
index 000000000..5a8f610b0
--- /dev/null
+++ b/pkg/fuzzer/job.go
@@ -0,0 +1,401 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+import (
+ "fmt"
+ "math/rand"
+
+ "github.com/google/syzkaller/pkg/cover"
+ "github.com/google/syzkaller/pkg/hash"
+ "github.com/google/syzkaller/pkg/ipc"
+ "github.com/google/syzkaller/pkg/rpctype"
+ "github.com/google/syzkaller/pkg/signal"
+ "github.com/google/syzkaller/prog"
+)
+
+const (
+ smashPrio int64 = iota + 1
+ genPrio
+ triagePrio
+ candidatePrio
+ candidateTriagePrio
+)
+
+type job interface {
+ run(fuzzer *Fuzzer)
+ saveID(id int64)
+ priority() priority
+}
+
+type ProgTypes int
+
+const (
+ ProgCandidate ProgTypes = 1 << iota
+ ProgMinimized
+ ProgSmashed
+ ProgNormal ProgTypes = 0
+)
+
+type jobPriority struct {
+ prio priority
+}
+
+func newJobPriority(base int64) jobPriority {
+ prio := append(make(priority, 0, 2), base)
+ return jobPriority{prio}
+}
+
+func (jp jobPriority) priority() priority {
+ return jp.prio
+}
+
+// If we prioritize execution requests only by the base priorities of their origin
+// jobs, we risk letting 1000s of simultaneous jobs slowly progress in parallel.
+// It's better to let same-prio jobs that were started earlier finish first.
+// saveID() allows Fuzzer to attach this sub-prio at the moment of job creation.
+func (jp *jobPriority) saveID(id int64) {
+ jp.prio = append(jp.prio, id)
+}
+
+func genProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *Request {
+ p := fuzzer.target.Generate(rnd,
+ prog.RecommendedCalls,
+ fuzzer.ChoiceTable())
+ return &Request{
+ Prog: p,
+ NeedSignal: true,
+ stat: statGenerate,
+ }
+}
+
+func mutateProgRequest(fuzzer *Fuzzer, rnd *rand.Rand) *Request {
+ p := fuzzer.Corpus.chooseProgram(rnd)
+ if p == nil {
+ return nil
+ }
+ newP := p.Clone()
+ newP.Mutate(rnd,
+ prog.RecommendedCalls,
+ fuzzer.ChoiceTable(),
+ fuzzer.Config.NoMutateCalls,
+ fuzzer.Corpus.Programs(),
+ )
+ return &Request{
+ Prog: newP,
+ NeedSignal: true,
+ stat: statFuzz,
+ }
+}
+
+func candidateRequest(input Candidate) *Request {
+ flags := ProgCandidate
+ if input.Minimized {
+ flags |= ProgMinimized
+ }
+ if input.Smashed {
+ flags |= ProgSmashed
+ }
+ return &Request{
+ Prog: input.Prog,
+ NeedSignal: true,
+ stat: statCandidate,
+ flags: flags,
+ }
+}
+
+// triageJob are programs for which we noticed potential new coverage during
+// first execution. But we are not sure yet if the coverage is real or not.
+// During triage we understand if these programs in fact give new coverage,
+// and if yes, minimize them and add to corpus.
+type triageJob struct {
+ p *prog.Prog
+ call int
+ info ipc.CallInfo
+ newSignal signal.Signal
+ flags ProgTypes
+ jobPriority
+}
+
+func triageJobPrio(flags ProgTypes) jobPriority {
+ if flags&ProgCandidate > 0 {
+ return newJobPriority(candidateTriagePrio)
+ }
+ return newJobPriority(triagePrio)
+}
+
+func (job *triageJob) run(fuzzer *Fuzzer) {
+ callName := ".extra"
+ logCallName := "extra"
+ if job.call != -1 {
+ callName = job.p.Calls[job.call].Meta.Name
+ logCallName = fmt.Sprintf("call #%v %v", job.call, callName)
+ }
+ fuzzer.Logf(3, "triaging input for %v (new signal=%v)", logCallName, job.newSignal.Len())
+ // Compute input coverage and non-flaky signal for minimization.
+ info, stop := job.deflake(fuzzer)
+ if stop || info.newStableSignal.Empty() {
+ return
+ }
+ if job.flags&ProgMinimized == 0 {
+ stop = job.minimize(fuzzer, info.newStableSignal)
+ if stop {
+ return
+ }
+ }
+ data := job.p.Serialize()
+ fuzzer.Logf(2, "added new input for %q to the corpus:\n%s",
+ logCallName, string(data))
+ if job.flags&ProgSmashed == 0 {
+ fuzzer.startJob(&smashJob{
+ p: job.p.Clone(),
+ call: job.call,
+ jobPriority: newJobPriority(smashPrio),
+ })
+ }
+ fuzzer.Corpus.Save(job.p, info.stableSignal, hash.Hash(data))
+ if fuzzer.Config.NewInputs != nil {
+ select {
+ case <-fuzzer.ctx.Done():
+ case fuzzer.Config.NewInputs <- rpctype.Input{
+ Call: callName,
+ CallID: job.call,
+ Prog: data,
+ Signal: info.stableSignal.Serialize(),
+ Cover: info.cover.Serialize(),
+ RawCover: info.rawCover,
+ }:
+ }
+ }
+}
+
+type deflakedCover struct {
+ stableSignal signal.Signal
+ newStableSignal signal.Signal
+ cover cover.Cover
+ rawCover []uint32
+}
+
+func (job *triageJob) deflake(fuzzer *Fuzzer) (info deflakedCover, stop bool) {
+ const signalRuns = 3
+ var notExecuted int
+ for i := 0; i < signalRuns; i++ {
+ result := fuzzer.exec(job, &Request{
+ Prog: job.p,
+ NeedSignal: true,
+ NeedCover: true,
+ NeedRawCover: fuzzer.Config.FetchRawCover,
+ stat: statTriage,
+ })
+ if result.Stop {
+ stop = true
+ return
+ }
+ if !reexecutionSuccess(result.Info, &job.info, job.call) {
+ // The call was not executed or failed.
+ notExecuted++
+ if notExecuted >= signalRuns/2+1 {
+ stop = true
+ return // if happens too often, give up
+ }
+ continue
+ }
+ thisSignal, thisCover := getSignalAndCover(job.p, result.Info, job.call)
+ if len(info.rawCover) == 0 && fuzzer.Config.FetchRawCover {
+ info.rawCover = thisCover
+ }
+ if i == 0 {
+ info.stableSignal = thisSignal
+ info.newStableSignal = job.newSignal.Intersection(thisSignal)
+ } else {
+ info.stableSignal = info.stableSignal.Intersection(thisSignal)
+ info.newStableSignal = info.newStableSignal.Intersection(thisSignal)
+ }
+ if info.newStableSignal.Empty() {
+ return
+ }
+ info.cover.Merge(thisCover)
+ }
+ return
+}
+
+func (job *triageJob) minimize(fuzzer *Fuzzer, newSignal signal.Signal) (stop bool) {
+ const minimizeAttempts = 3
+ job.p, job.call = prog.Minimize(job.p, job.call, false,
+ func(p1 *prog.Prog, call1 int) bool {
+ if stop {
+ return false
+ }
+ for i := 0; i < minimizeAttempts; i++ {
+ result := fuzzer.exec(job, &Request{
+ Prog: p1,
+ NeedSignal: true,
+ stat: statMinimize,
+ })
+ if result.Stop {
+ stop = true
+ return false
+ }
+ info := result.Info
+ if !reexecutionSuccess(info, &job.info, call1) {
+ // The call was not executed or failed.
+ continue
+ }
+ thisSignal, _ := getSignalAndCover(p1, info, call1)
+ if newSignal.Intersection(thisSignal).Len() == newSignal.Len() {
+ return true
+ }
+ }
+ return false
+ })
+ return stop
+}
+
+func reexecutionSuccess(info *ipc.ProgInfo, oldInfo *ipc.CallInfo, call int) bool {
+ if info == nil || len(info.Calls) == 0 {
+ return false
+ }
+ if call != -1 {
+ // Don't minimize calls from successful to unsuccessful.
+ // Successful calls are much more valuable.
+ if oldInfo.Errno == 0 && info.Calls[call].Errno != 0 {
+ return false
+ }
+ return len(info.Calls[call].Signal) != 0
+ }
+ return len(info.Extra.Signal) != 0
+}
+
+func getSignalAndCover(p *prog.Prog, info *ipc.ProgInfo, call int) (signal.Signal, []uint32) {
+ inf := &info.Extra
+ if call != -1 {
+ inf = &info.Calls[call]
+ }
+ return signal.FromRaw(inf.Signal, signalPrio(p, inf, call)), inf.Cover
+}
+
+type smashJob struct {
+ p *prog.Prog
+ call int
+ jobPriority
+}
+
+func (job *smashJob) run(fuzzer *Fuzzer) {
+ fuzzer.Logf(2, "smashing the program %s (call=%d):", job.p, job.call)
+ if fuzzer.Config.Comparisons && job.call >= 0 {
+ fuzzer.startJob(&hintsJob{
+ p: job.p.Clone(),
+ call: job.call,
+ jobPriority: newJobPriority(smashPrio),
+ })
+ }
+
+ const iters = 100
+ rnd := fuzzer.rand()
+ for i := 0; i < iters; i++ {
+ p := job.p.Clone()
+ p.Mutate(rnd, prog.RecommendedCalls,
+ fuzzer.ChoiceTable(),
+ fuzzer.Config.NoMutateCalls,
+ fuzzer.Corpus.Programs())
+ result := fuzzer.exec(job, &Request{
+ Prog: p,
+ NeedSignal: true,
+ stat: statSmash,
+ })
+ if result.Stop {
+ return
+ }
+ if fuzzer.Config.Collide {
+ result := fuzzer.exec(job, &Request{
+ Prog: randomCollide(p, rnd),
+ stat: statCollide,
+ })
+ if result.Stop {
+ return
+ }
+ }
+ }
+ if fuzzer.Config.FaultInjection && job.call >= 0 {
+ job.faultInjection(fuzzer)
+ }
+}
+
+func randomCollide(origP *prog.Prog, rnd *rand.Rand) *prog.Prog {
+ if rnd.Intn(5) == 0 {
+ // Old-style collide with a 20% probability.
+ p, err := prog.DoubleExecCollide(origP, rnd)
+ if err == nil {
+ return p
+ }
+ }
+ if rnd.Intn(4) == 0 {
+ // Duplicate random calls with a 20% probability (25% * 80%).
+ p, err := prog.DupCallCollide(origP, rnd)
+ if err == nil {
+ return p
+ }
+ }
+ p := prog.AssignRandomAsync(origP, rnd)
+ if rnd.Intn(2) != 0 {
+ prog.AssignRandomRerun(p, rnd)
+ }
+ return p
+}
+
+func (job *smashJob) faultInjection(fuzzer *Fuzzer) {
+ for nth := 1; nth <= 100; nth++ {
+ fuzzer.Logf(2, "injecting fault into call %v, step %v",
+ job.call, nth)
+ newProg := job.p.Clone()
+ newProg.Calls[job.call].Props.FailNth = nth
+ result := fuzzer.exec(job, &Request{
+ Prog: job.p,
+ stat: statSmash,
+ })
+ if result.Stop {
+ return
+ }
+ info := result.Info
+ if info != nil && len(info.Calls) > job.call &&
+ info.Calls[job.call].Flags&ipc.CallFaultInjected == 0 {
+ break
+ }
+ }
+}
+
+type hintsJob struct {
+ p *prog.Prog
+ call int
+ jobPriority
+}
+
+func (job *hintsJob) run(fuzzer *Fuzzer) {
+ // First execute the original program to dump comparisons from KCOV.
+ p := job.p
+ result := fuzzer.exec(job, &Request{
+ Prog: p,
+ NeedHints: true,
+ stat: statSeed,
+ })
+ if result.Stop || result.Info == nil {
+ return
+ }
+ // Then mutate the initial program for every match between
+ // a syscall argument and a comparison operand.
+ // Execute each of such mutants to check if it gives new coverage.
+ var stop bool
+ p.MutateWithHints(job.call, result.Info.Calls[job.call].Comps,
+ func(p *prog.Prog) {
+ if stop {
+ return
+ }
+ result := fuzzer.exec(job, &Request{
+ Prog: p,
+ NeedSignal: true,
+ stat: statHint,
+ })
+ stop = stop || result.Stop
+ })
+}
diff --git a/pkg/fuzzer/prio_queue.go b/pkg/fuzzer/prio_queue.go
new file mode 100644
index 000000000..ea8f448c9
--- /dev/null
+++ b/pkg/fuzzer/prio_queue.go
@@ -0,0 +1,100 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+import (
+ "container/heap"
+ "sync"
+)
+
+type priority []int64
+
+func (p priority) greaterThan(other priority) bool {
+ for i := range p {
+ if i >= len(other) || p[i] > other[i] {
+ return true
+ }
+ if p[i] < other[i] {
+ return false
+ }
+ }
+ return false
+}
+
+type priorityQueue[T any] struct {
+ impl priorityQueueImpl[T]
+ c *sync.Cond
+}
+
+func makePriorityQueue[T any]() *priorityQueue[T] {
+ return &priorityQueue[T]{
+ c: sync.NewCond(&sync.Mutex{}),
+ }
+}
+
+func (pq *priorityQueue[T]) Len() int {
+ pq.c.L.Lock()
+ defer pq.c.L.Unlock()
+ return pq.impl.Len()
+}
+
+func (pq *priorityQueue[T]) push(item *priorityQueueItem[T]) {
+ pq.c.L.Lock()
+ defer pq.c.L.Unlock()
+ heap.Push(&pq.impl, item)
+ pq.c.Signal()
+}
+
+// pop() blocks until there's input.
+func (pq *priorityQueue[T]) pop() *priorityQueueItem[T] {
+ pq.c.L.Lock()
+ defer pq.c.L.Unlock()
+ for pq.impl.Len() == 0 {
+ pq.c.Wait()
+ }
+ return heap.Pop(&pq.impl).(*priorityQueueItem[T])
+}
+
+func (pq *priorityQueue[T]) tryPop() *priorityQueueItem[T] {
+ pq.c.L.Lock()
+ defer pq.c.L.Unlock()
+ if len(pq.impl) == 0 {
+ return nil
+ }
+ return heap.Pop(&pq.impl).(*priorityQueueItem[T])
+}
+
+// The implementation below is based on the example provided
+// by https://pkg.go.dev/container/heap.
+
+type priorityQueueItem[T any] struct {
+ value T
+ prio priority
+}
+
+type priorityQueueImpl[T any] []*priorityQueueItem[T]
+
+func (pq priorityQueueImpl[T]) Len() int { return len(pq) }
+
+func (pq priorityQueueImpl[T]) Less(i, j int) bool {
+ // We want Pop to give us the highest, not lowest,
+ // priority so we use greater than here.
+ return pq[i].prio.greaterThan(pq[j].prio)
+}
+
+func (pq priorityQueueImpl[T]) Swap(i, j int) {
+ pq[i], pq[j] = pq[j], pq[i]
+}
+
+func (pq *priorityQueueImpl[T]) Push(x any) {
+ *pq = append(*pq, x.(*priorityQueueItem[T]))
+}
+
+func (pq *priorityQueueImpl[T]) Pop() any {
+ n := len(*pq)
+ item := (*pq)[n-1]
+ (*pq)[n-1] = nil
+ *pq = (*pq)[:n-1]
+ return item
+}
diff --git a/pkg/fuzzer/prio_queue_test.go b/pkg/fuzzer/prio_queue_test.go
new file mode 100644
index 000000000..b2abdb01b
--- /dev/null
+++ b/pkg/fuzzer/prio_queue_test.go
@@ -0,0 +1,44 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+import (
+ "sync"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestPriority(t *testing.T) {
+ assert.True(t, priority{1, 2}.greaterThan(priority{1, 1}))
+ assert.True(t, priority{3, 2}.greaterThan(priority{2, 3}))
+ assert.True(t, priority{1, -5}.greaterThan(priority{1, -10}))
+}
+
+func TestPrioQueueOrder(t *testing.T) {
+ pq := makePriorityQueue[int]()
+ pq.push(&priorityQueueItem[int]{value: 1, prio: priority{1}})
+ pq.push(&priorityQueueItem[int]{value: 3, prio: priority{3}})
+ pq.push(&priorityQueueItem[int]{value: 2, prio: priority{2}})
+
+ assert.Equal(t, 3, pq.pop().value)
+ assert.Equal(t, 2, pq.pop().value)
+ assert.Equal(t, 1, pq.pop().value)
+ assert.Nil(t, pq.tryPop())
+}
+
+func TestPrioQueueWait(t *testing.T) {
+ var wg sync.WaitGroup
+ pq := makePriorityQueue[int]()
+ assert.Nil(t, pq.tryPop())
+
+ wg.Add(1)
+ go func() {
+ assert.Equal(t, 10, pq.pop().value)
+ wg.Done()
+ }()
+
+ pq.push(&priorityQueueItem[int]{value: 10, prio: priority{1}})
+ wg.Wait()
+}
diff --git a/pkg/fuzzer/stats.go b/pkg/fuzzer/stats.go
new file mode 100644
index 000000000..17bc6131c
--- /dev/null
+++ b/pkg/fuzzer/stats.go
@@ -0,0 +1,26 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package fuzzer
+
+const (
+ statGenerate = "exec gen"
+ statFuzz = "exec fuzz"
+ statCandidate = "exec candidate"
+ statTriage = "exec triage"
+ statMinimize = "exec minimize"
+ statSmash = "exec smash"
+ statHint = "exec hints"
+ statSeed = "exec seeds"
+ statCollide = "exec collide"
+ statExecTotal = "exec total"
+ statBufferTooSmall = "buffer too small"
+)
+
+func (fuzzer *Fuzzer) GrabStats() map[string]uint64 {
+ fuzzer.mu.Lock()
+ defer fuzzer.mu.Unlock()
+ ret := fuzzer.stats
+ fuzzer.stats = map[string]uint64{}
+ return ret
+}
diff --git a/sys/targets/targets.go b/sys/targets/targets.go
index 40fc40991..840c1febf 100644
--- a/sys/targets/targets.go
+++ b/sys/targets/targets.go
@@ -141,6 +141,7 @@ const (
S390x = "s390x"
RiscV64 = "riscv64"
TestArch64 = "64"
+ TestArch64Fuzz = "64_fuzz"
TestArch64Fork = "64_fork"
TestArch32Shmem = "32_shmem"
TestArch32ForkShmem = "32_fork_shmem"
@@ -189,6 +190,18 @@ var List = map[string]map[string]*Target{
ExecutorUsesForkServer: false,
},
},
+ TestArch64Fuzz: {
+ PtrSize: 8,
+ PageSize: 8 << 10,
+ // -fsanitize=address causes SIGSEGV.
+ CFlags: []string{"-no-pie"},
+ osCommon: osCommon{
+ SyscallNumbers: true,
+ SyscallPrefix: "SYS_",
+ ExecutorUsesShmem: true,
+ ExecutorUsesForkServer: true,
+ },
+ },
TestArch64Fork: {
PtrSize: 8,
PageSize: 8 << 10,
diff --git a/sys/test/expressions.txt.const b/sys/test/expressions.txt.const
index 8b3a2dae5..77e181281 100644
--- a/sys/test/expressions.txt.const
+++ b/sys/test/expressions.txt.const
@@ -1,3 +1,3 @@
-arches = 32_fork_shmem, 32_shmem, 64, 64_fork
+arches = 32_fork_shmem, 32_shmem, 64, 64_fork, 64_fuzz
FIELD_FLAG1 = 2
FIELD_FLAG2 = 4 \ No newline at end of file
diff --git a/sys/test/fuzzer.txt b/sys/test/fuzzer.txt
new file mode 100644
index 000000000..11e91c992
--- /dev/null
+++ b/sys/test/fuzzer.txt
@@ -0,0 +1,7 @@
+# Copyright 2024 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+# These definitions are used for pkg/fuzzer tests.
+# They must be in close sync with common_test.h.
+
+syz_test_fuzzer1(a int64[0:16], b int64[0:16], c int64[0:16])
diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go
index 2d9ab7cba..1adec40d4 100644
--- a/syz-fuzzer/fuzzer.go
+++ b/syz-fuzzer/fuzzer.go
@@ -4,6 +4,7 @@
package main
import (
+ "context"
"flag"
"fmt"
"math/rand"
@@ -12,12 +13,12 @@ import (
"os"
"runtime"
"runtime/debug"
- "sort"
"sync"
"sync/atomic"
"time"
"github.com/google/syzkaller/pkg/csource"
+ "github.com/google/syzkaller/pkg/fuzzer"
"github.com/google/syzkaller/pkg/hash"
"github.com/google/syzkaller/pkg/host"
"github.com/google/syzkaller/pkg/ipc"
@@ -25,90 +26,29 @@ import (
"github.com/google/syzkaller/pkg/log"
"github.com/google/syzkaller/pkg/osutil"
"github.com/google/syzkaller/pkg/rpctype"
- "github.com/google/syzkaller/pkg/signal"
"github.com/google/syzkaller/pkg/tool"
"github.com/google/syzkaller/prog"
_ "github.com/google/syzkaller/sys"
"github.com/google/syzkaller/sys/targets"
)
-type Fuzzer struct {
- name string
- outputType OutputType
- config *ipc.Config
- execOpts *ipc.ExecOpts
- procs []*Proc
- gate *ipc.Gate
- workQueue *WorkQueue
- needPoll chan struct{}
- choiceTable *prog.ChoiceTable
- noMutate map[int]bool
- // The stats field cannot unfortunately be just an uint64 array, because it
- // results in "unaligned 64-bit atomic operation" errors on 32-bit platforms.
- stats []uint64
+type FuzzerTool struct {
+ name string
+ outputType OutputType
+ config *ipc.Config
+ fuzzer *fuzzer.Fuzzer
+ procs []*Proc
+ gate *ipc.Gate
manager *rpctype.RPCClient
target *prog.Target
triagedCandidates uint32
timeouts targets.Timeouts
- faultInjectionEnabled bool
- comparisonTracingEnabled bool
- fetchRawCover bool
-
- corpusMu sync.RWMutex
- corpus []*prog.Prog
- corpusHashes map[hash.Sig]struct{}
- corpusPrios []int64
- sumPrios int64
-
- signalMu sync.RWMutex
- corpusSignal signal.Signal // signal of inputs in corpus
- maxSignal signal.Signal // max signal ever observed including flakes
- newSignal signal.Signal // diff of maxSignal since last sync with master
-
checkResult *rpctype.CheckArgs
logMu sync.Mutex
- // Let's limit the number of concurrent NewInput requests.
- parallelNewInputs chan struct{}
-
- // Experimental flags.
- resetAccState bool
-}
-
-type FuzzerSnapshot struct {
- corpus []*prog.Prog
- corpusPrios []int64
- sumPrios int64
-}
-
-type Stat int
-
-const (
- StatGenerate Stat = iota
- StatFuzz
- StatCandidate
- StatTriage
- StatMinimize
- StatSmash
- StatHint
- StatSeed
- StatCollide
- StatBufferTooSmall
- StatCount
-)
-
-var statNames = [StatCount]string{
- StatGenerate: "exec gen",
- StatFuzz: "exec fuzz",
- StatCandidate: "exec candidate",
- StatTriage: "exec triage",
- StatMinimize: "exec minimize",
- StatSmash: "exec smash",
- StatHint: "exec hints",
- StatSeed: "exec seeds",
- StatCollide: "exec collide",
- StatBufferTooSmall: "buffer too small",
+ bufferTooSmall uint64
+ resetAccState bool
}
type OutputType int
@@ -157,7 +97,8 @@ func createIPCConfig(features *host.Features, config *ipc.Config) {
// It coincides with prog.MaxPids.
const gateSize = prog.MaxPids
-// nolint: funlen
+// TODO: split into smaller methods.
+// nolint: funlen, gocyclo
func main() {
debug.SetGCPercent(50)
@@ -283,60 +224,74 @@ func main() {
runTest(target, manager, *flagName, config.Executor)
return
}
-
- needPoll := make(chan struct{}, 1)
- needPoll <- struct{}{}
- fuzzer := &Fuzzer{
- name: *flagName,
- outputType: outputType,
- config: config,
- execOpts: execOpts,
- workQueue: newWorkQueue(*flagProcs, needPoll),
- needPoll: needPoll,
- manager: manager,
- target: target,
- timeouts: timeouts,
- faultInjectionEnabled: r.CheckResult.Features[host.FeatureFault].Enabled,
- comparisonTracingEnabled: r.CheckResult.Features[host.FeatureComparisons].Enabled,
- corpusHashes: make(map[hash.Sig]struct{}),
- checkResult: r.CheckResult,
- fetchRawCover: *flagRawCover,
- noMutate: r.NoMutateCalls,
- stats: make([]uint64, StatCount),
- // Queue no more than ~3 new inputs / proc.
- parallelNewInputs: make(chan struct{}, int64(3**flagProcs)),
- resetAccState: *flagResetAccState,
- }
- gateCallback := fuzzer.useBugFrames(r, *flagProcs)
- fuzzer.gate = ipc.NewGate(gateSize, gateCallback)
-
- for needCandidates, more := true, true; more; needCandidates = false {
- more = fuzzer.poll(needCandidates, nil)
- // This loop lead to "no output" in qemu emulation, tell manager we are not dead.
- log.Logf(0, "fetching corpus: %v, signal %v/%v (executing program)",
- len(fuzzer.corpus), len(fuzzer.corpusSignal), len(fuzzer.maxSignal))
- }
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
calls := make(map[*prog.Syscall]bool)
for _, id := range r.CheckResult.EnabledCalls[sandbox] {
calls[target.Syscalls[id]] = true
}
- fuzzer.choiceTable = target.BuildChoiceTable(fuzzer.corpus, calls)
-
+ fuzzerObj := fuzzer.NewFuzzer(context.Background(), &fuzzer.Config{
+ Coverage: config.Flags&ipc.FlagSignal > 0,
+ FaultInjection: r.CheckResult.Features[host.FeatureFault].Enabled,
+ Comparisons: r.CheckResult.Features[host.FeatureComparisons].Enabled,
+ Collide: execOpts.Flags&ipc.FlagThreaded > 0,
+ EnabledCalls: calls,
+ NoMutateCalls: r.NoMutateCalls,
+ LeakChecking: r.CheckResult.Features[host.FeatureLeak].Enabled,
+ FetchRawCover: *flagRawCover,
+ MinCandidates: uint(*flagProcs * 2),
+ NewInputs: make(chan rpctype.Input),
+ }, rnd, target)
+
+ fuzzerTool := &FuzzerTool{
+ fuzzer: fuzzerObj,
+ name: *flagName,
+ outputType: outputType,
+ manager: manager,
+ target: target,
+ timeouts: timeouts,
+ config: config,
+ checkResult: r.CheckResult,
+ resetAccState: *flagResetAccState,
+ }
+ fuzzerObj.Config.Logf = func(level int, msg string, args ...interface{}) {
+ // Log 0 messages are most important: send them directly to syz-manager.
+ if level == 0 {
+ fuzzerTool.Logf(level, msg, args...)
+ }
+ // Dump log level 0 and 1 messages into syz-fuzzer output.
+ if level <= 1 {
+ fuzzerTool.logMu.Lock()
+ defer fuzzerTool.logMu.Unlock()
+ log.Logf(0, "fuzzer: "+msg, args...)
+ }
+ }
+ fuzzerTool.gate = ipc.NewGate(gateSize,
+ fuzzerTool.useBugFrames(r, *flagProcs))
+ for needCandidates, more := true, true; more; needCandidates = false {
+ more = fuzzerTool.poll(needCandidates, nil)
+ // This loop lead to "no output" in qemu emulation, tell manager we are not dead.
+ stat := fuzzerObj.Corpus.Stat()
+ log.Logf(0, "fetching corpus: %v, signal %v/%v (executing program)",
+ stat.Progs, stat.Signal, stat.MaxSignal)
+ }
if r.CoverFilterBitmap != nil {
- fuzzer.execOpts.Flags |= ipc.FlagEnableCoverageFilter
+ execOpts.Flags |= ipc.FlagEnableCoverageFilter
}
log.Logf(0, "starting %v fuzzer processes", *flagProcs)
for pid := 0; pid < *flagProcs; pid++ {
- proc, err := newProc(fuzzer, pid)
+ proc, err := newProc(fuzzerTool, execOpts, pid)
if err != nil {
log.SyzFatalf("failed to create proc: %v", err)
}
- fuzzer.procs = append(fuzzer.procs, proc)
+ fuzzerTool.procs = append(fuzzerTool.procs, proc)
go proc.loop()
}
-
- fuzzer.pollLoop()
+ // Start send input workers.
+ for i := 0; i < *flagProcs*2; i++ {
+ go fuzzerTool.sendInputsWorker()
+ }
+ fuzzerTool.pollLoop()
}
func collectMachineInfos(target *prog.Target) ([]byte, []host.KernelModule) {
@@ -352,33 +307,33 @@ func collectMachineInfos(target *prog.Target) ([]byte, []host.KernelModule) {
}
// Returns gateCallback for leak checking if enabled.
-func (fuzzer *Fuzzer) useBugFrames(r *rpctype.ConnectRes, flagProcs int) func() {
+func (tool *FuzzerTool) useBugFrames(r *rpctype.ConnectRes, flagProcs int) func() {
var gateCallback func()
if r.CheckResult.Features[host.FeatureLeak].Enabled {
- gateCallback = func() { fuzzer.gateCallback(r.MemoryLeakFrames) }
+ gateCallback = func() { tool.gateCallback(r.MemoryLeakFrames) }
}
if r.CheckResult.Features[host.FeatureKCSAN].Enabled && len(r.DataRaceFrames) != 0 {
- fuzzer.filterDataRaceFrames(r.DataRaceFrames)
+ tool.filterDataRaceFrames(r.DataRaceFrames)
}
return gateCallback
}
-func (fuzzer *Fuzzer) gateCallback(leakFrames []string) {
+func (tool *FuzzerTool) gateCallback(leakFrames []string) {
// Leak checking is very slow so we don't do it while triaging the corpus
// (otherwise it takes infinity). When we have presumably triaged the corpus
// (triagedCandidates == 1), we run leak checking bug ignore the result
// to flush any previous leaks. After that (triagedCandidates == 2)
// we do actual leak checking and report leaks.
- triagedCandidates := atomic.LoadUint32(&fuzzer.triagedCandidates)
+ triagedCandidates := atomic.LoadUint32(&tool.triagedCandidates)
if triagedCandidates == 0 {
return
}
args := append([]string{"leak"}, leakFrames...)
- timeout := fuzzer.timeouts.NoOutput * 9 / 10
- output, err := osutil.RunCmd(timeout, "", fuzzer.config.Executor, args...)
+ timeout := tool.timeouts.NoOutput * 9 / 10
+ output, err := osutil.RunCmd(timeout, "", tool.config.Executor, args...)
if err != nil && triagedCandidates == 2 {
// If we exit right away, dying executors will dump lots of garbage to console.
os.Stdout.Write(output)
@@ -387,261 +342,165 @@ func (fuzzer *Fuzzer) gateCallback(leakFrames []string) {
os.Exit(1)
}
if triagedCandidates == 1 {
- atomic.StoreUint32(&fuzzer.triagedCandidates, 2)
+ atomic.StoreUint32(&tool.triagedCandidates, 2)
}
}
-func (fuzzer *Fuzzer) filterDataRaceFrames(frames []string) {
+func (tool *FuzzerTool) filterDataRaceFrames(frames []string) {
args := append([]string{"setup_kcsan_filterlist"}, frames...)
- timeout := time.Minute * fuzzer.timeouts.Scale
- output, err := osutil.RunCmd(timeout, "", fuzzer.config.Executor, args...)
+ timeout := time.Minute * tool.timeouts.Scale
+ output, err := osutil.RunCmd(timeout, "", tool.config.Executor, args...)
if err != nil {
log.SyzFatalf("failed to set KCSAN filterlist: %v", err)
}
log.Logf(0, "%s", output)
}
-func (fuzzer *Fuzzer) pollLoop() {
+func (tool *FuzzerTool) pollLoop() {
var execTotal uint64
var lastPoll time.Time
var lastPrint time.Time
- ticker := time.NewTicker(3 * time.Second * fuzzer.timeouts.Scale).C
+ ticker := time.NewTicker(3 * time.Second * tool.timeouts.Scale).C
for {
- poll := false
+ needCandidates := false
select {
case <-ticker:
- case <-fuzzer.needPoll:
- poll = true
+ case <-tool.fuzzer.NeedCandidates:
+ needCandidates = true
}
- if fuzzer.outputType != OutputStdout && time.Since(lastPrint) > 10*time.Second*fuzzer.timeouts.Scale {
+ if tool.outputType != OutputStdout && time.Since(lastPrint) > 10*time.Second*tool.timeouts.Scale {
// Keep-alive for manager.
log.Logf(0, "alive, executed %v", execTotal)
lastPrint = time.Now()
}
- if poll || time.Since(lastPoll) > 10*time.Second*fuzzer.timeouts.Scale {
- needCandidates := fuzzer.workQueue.wantCandidates()
- if poll && !needCandidates {
- continue
- }
- stats := make(map[string]uint64)
- for _, proc := range fuzzer.procs {
- stats["exec total"] += atomic.SwapUint64(&proc.env.StatExecs, 0)
- stats["executor restarts"] += atomic.SwapUint64(&proc.env.StatRestarts, 0)
- }
- for stat := Stat(0); stat < StatCount; stat++ {
- v := atomic.SwapUint64(&fuzzer.stats[stat], 0)
- stats[statNames[stat]] = v
- execTotal += v
- }
- if !fuzzer.poll(needCandidates, stats) {
+ needCandidates = tool.fuzzer.NeedCandidatesNow()
+ if needCandidates || time.Since(lastPoll) > 10*time.Second*tool.timeouts.Scale {
+ more := tool.poll(needCandidates, tool.grabStats())
+ if !more {
lastPoll = time.Now()
}
}
}
}
-func (fuzzer *Fuzzer) poll(needCandidates bool, stats map[string]uint64) bool {
+func (tool *FuzzerTool) poll(needCandidates bool, stats map[string]uint64) bool {
+ fuzzer := tool.fuzzer
a := &rpctype.PollArgs{
- Name: fuzzer.name,
+ Name: tool.name,
NeedCandidates: needCandidates,
- MaxSignal: fuzzer.grabNewSignal().Serialize(),
+ MaxSignal: fuzzer.Corpus.GrabNewSignal().Serialize(),
Stats: stats,
}
r := &rpctype.PollRes{}
- if err := fuzzer.manager.Call("Manager.Poll", a, r); err != nil {
+ if err := tool.manager.Call("Manager.Poll", a, r); err != nil {
log.SyzFatalf("Manager.Poll call failed: %v", err)
}
maxSignal := r.MaxSignal.Deserialize()
log.Logf(1, "poll: candidates=%v inputs=%v signal=%v",
len(r.Candidates), len(r.NewInputs), maxSignal.Len())
- fuzzer.addMaxSignal(maxSignal)
+ fuzzer.Corpus.AddMaxSignal(maxSignal)
for _, inp := range r.NewInputs {
- fuzzer.addInputFromAnotherFuzzer(inp)
- }
- for _, candidate := range r.Candidates {
- fuzzer.addCandidateInput(candidate)
+ tool.inputFromOtherFuzzer(inp)
}
- if needCandidates && len(r.Candidates) == 0 && atomic.LoadUint32(&fuzzer.triagedCandidates) == 0 {
- atomic.StoreUint32(&fuzzer.triagedCandidates, 1)
+ tool.addCandidates(r.Candidates)
+ if needCandidates && len(r.Candidates) == 0 && atomic.LoadUint32(&tool.triagedCandidates) == 0 {
+ atomic.StoreUint32(&tool.triagedCandidates, 1)
}
return len(r.NewInputs) != 0 || len(r.Candidates) != 0 || maxSignal.Len() != 0
}
-func (fuzzer *Fuzzer) sendInputToManager(inp rpctype.Input) {
- fuzzer.parallelNewInputs <- struct{}{}
- go func() {
- defer func() { <-fuzzer.parallelNewInputs }()
+func (tool *FuzzerTool) sendInputsWorker() {
+ for inp := range tool.fuzzer.Config.NewInputs {
a := &rpctype.NewInputArgs{
- Name: fuzzer.name,
+ Name: tool.name,
Input: inp,
}
- if err := fuzzer.manager.Call("Manager.NewInput", a, nil); err != nil {
+ if err := tool.manager.Call("Manager.NewInput", a, nil); err != nil {
log.SyzFatalf("Manager.NewInput call failed: %v", err)
}
- }()
+ }
}
-func (fuzzer *Fuzzer) addInputFromAnotherFuzzer(inp rpctype.Input) {
- p := fuzzer.deserializeInput(inp.Prog)
- if p == nil {
- return
+func (tool *FuzzerTool) grabStats() map[string]uint64 {
+ stats := tool.fuzzer.GrabStats()
+ for _, proc := range tool.procs {
+ stats["exec total"] += atomic.SwapUint64(&proc.env.StatExecs, 0)
+ stats["executor restarts"] += atomic.SwapUint64(&proc.env.StatRestarts, 0)
}
- sig := hash.Hash(inp.Prog)
- sign := inp.Signal.Deserialize()
- fuzzer.addInputToCorpus(p, sign, sig)
+ stats["buffer too small"] = atomic.SwapUint64(&tool.bufferTooSmall, 0)
+ return stats
}
-func (fuzzer *Fuzzer) addCandidateInput(candidate rpctype.Candidate) {
- p := fuzzer.deserializeInput(candidate.Prog)
- if p == nil {
- return
+func (tool *FuzzerTool) addCandidates(candidates []rpctype.Candidate) {
+ var inputs []fuzzer.Candidate
+ for _, candidate := range candidates {
+ p := tool.deserializeInput(candidate.Prog)
+ if p == nil {
+ continue
+ }
+ inputs = append(inputs, fuzzer.Candidate{
+ Prog: p,
+ Smashed: candidate.Smashed,
+ Minimized: candidate.Minimized,
+ })
}
- flags := ProgCandidate
- if candidate.Minimized {
- flags |= ProgMinimized
+ if len(inputs) > 0 {
+ tool.fuzzer.AddCandidates(inputs)
}
- if candidate.Smashed {
- flags |= ProgSmashed
+}
+
+func (tool *FuzzerTool) inputFromOtherFuzzer(inp rpctype.Input) {
+ p := tool.deserializeInput(inp.Prog)
+ if p == nil {
+ return
}
- fuzzer.workQueue.enqueue(&WorkCandidate{
- p: p,
- flags: flags,
- })
+ tool.fuzzer.Corpus.Save(p,
+ inp.Signal.Deserialize(),
+ hash.Hash(inp.Prog))
}
-func (fuzzer *Fuzzer) deserializeInput(inp []byte) *prog.Prog {
- p, err := fuzzer.target.Deserialize(inp, prog.NonStrict)
+func (tool *FuzzerTool) deserializeInput(inp []byte) *prog.Prog {
+ p, err := tool.target.Deserialize(inp, prog.NonStrict)
if err != nil {
log.SyzFatalf("failed to deserialize prog: %v\n%s", err, inp)
}
- // We build choice table only after we received the initial corpus,
- // so we don't check the initial corpus here, we check it later in BuildChoiceTable.
- if fuzzer.choiceTable != nil {
- fuzzer.checkDisabledCalls(p)
- }
+ tool.checkDisabledCalls(p)
if len(p.Calls) > prog.MaxCalls {
return nil
}
return p
}
-func (fuzzer *Fuzzer) checkDisabledCalls(p *prog.Prog) {
+func (tool *FuzzerTool) checkDisabledCalls(p *prog.Prog) {
+ ct := tool.fuzzer.ChoiceTable()
for _, call := range p.Calls {
- if !fuzzer.choiceTable.Enabled(call.Meta.ID) {
+ if !ct.Enabled(call.Meta.ID) {
fmt.Printf("executing disabled syscall %v [%v]\n", call.Meta.Name, call.Meta.ID)
- sandbox := ipc.FlagsToSandbox(fuzzer.config.Flags)
+ sandbox := ipc.FlagsToSandbox(tool.config.Flags)
fmt.Printf("check result for sandbox=%v:\n", sandbox)
- for _, id := range fuzzer.checkResult.EnabledCalls[sandbox] {
- meta := fuzzer.target.Syscalls[id]
+ for _, id := range tool.checkResult.EnabledCalls[sandbox] {
+ meta := tool.target.Syscalls[id]
fmt.Printf(" %v [%v]\n", meta.Name, meta.ID)
}
fmt.Printf("choice table:\n")
- for i, meta := range fuzzer.target.Syscalls {
- fmt.Printf(" #%v: %v [%v]: enabled=%v\n", i, meta.Name, meta.ID, fuzzer.choiceTable.Enabled(meta.ID))
+ for i, meta := range tool.target.Syscalls {
+ fmt.Printf(" #%v: %v [%v]: enabled=%v\n", i, meta.Name, meta.ID, ct.Enabled(meta.ID))
}
panic("disabled syscall")
}
}
}
-func (fuzzer *FuzzerSnapshot) chooseProgram(r *rand.Rand) *prog.Prog {
- randVal := r.Int63n(fuzzer.sumPrios + 1)
- idx := sort.Search(len(fuzzer.corpusPrios), func(i int) bool {
- return fuzzer.corpusPrios[i] >= randVal
- })
- return fuzzer.corpus[idx]
-}
-
-func (fuzzer *Fuzzer) addInputToCorpus(p *prog.Prog, sign signal.Signal, sig hash.Sig) {
- fuzzer.corpusMu.Lock()
- if _, ok := fuzzer.corpusHashes[sig]; !ok {
- fuzzer.corpus = append(fuzzer.corpus, p)
- fuzzer.corpusHashes[sig] = struct{}{}
- prio := int64(len(sign))
- if sign.Empty() {
- prio = 1
- }
- fuzzer.sumPrios += prio
- fuzzer.corpusPrios = append(fuzzer.corpusPrios, fuzzer.sumPrios)
- }
- fuzzer.corpusMu.Unlock()
-
- if !sign.Empty() {
- fuzzer.signalMu.Lock()
- fuzzer.corpusSignal.Merge(sign)
- fuzzer.maxSignal.Merge(sign)
- fuzzer.signalMu.Unlock()
- }
-}
-
-func (fuzzer *Fuzzer) snapshot() FuzzerSnapshot {
- fuzzer.corpusMu.RLock()
- defer fuzzer.corpusMu.RUnlock()
- return FuzzerSnapshot{fuzzer.corpus, fuzzer.corpusPrios, fuzzer.sumPrios}
-}
-
-func (fuzzer *Fuzzer) addMaxSignal(sign signal.Signal) {
- if sign.Len() == 0 {
- return
- }
- fuzzer.signalMu.Lock()
- defer fuzzer.signalMu.Unlock()
- fuzzer.maxSignal.Merge(sign)
-}
-
-func (fuzzer *Fuzzer) grabNewSignal() signal.Signal {
- fuzzer.signalMu.Lock()
- defer fuzzer.signalMu.Unlock()
- sign := fuzzer.newSignal
- if sign.Empty() {
- return nil
- }
- fuzzer.newSignal = nil
- return sign
-}
-
-func (fuzzer *Fuzzer) corpusSignalDiff(sign signal.Signal) signal.Signal {
- fuzzer.signalMu.RLock()
- defer fuzzer.signalMu.RUnlock()
- return fuzzer.corpusSignal.Diff(sign)
-}
-
-func (fuzzer *Fuzzer) checkNewSignal(p *prog.Prog, info *ipc.ProgInfo) (calls []int, extra bool) {
- fuzzer.signalMu.RLock()
- defer fuzzer.signalMu.RUnlock()
- for i, inf := range info.Calls {
- if fuzzer.checkNewCallSignal(p, &inf, i) {
- calls = append(calls, i)
- }
- }
- extra = fuzzer.checkNewCallSignal(p, &info.Extra, -1)
- return
-}
-
-func (fuzzer *Fuzzer) checkNewCallSignal(p *prog.Prog, info *ipc.CallInfo, call int) bool {
- diff := fuzzer.maxSignal.DiffRaw(info.Signal, signalPrio(p, info, call))
- if diff.Empty() {
- return false
- }
- fuzzer.signalMu.RUnlock()
- fuzzer.signalMu.Lock()
- fuzzer.maxSignal.Merge(diff)
- fuzzer.newSignal.Merge(diff)
- fuzzer.signalMu.Unlock()
- fuzzer.signalMu.RLock()
- return true
-}
-
// nolint: unused
// It's only needed for debugging.
-func (fuzzer *Fuzzer) Logf(level int, msg string, args ...interface{}) {
+func (tool *FuzzerTool) Logf(level int, msg string, args ...interface{}) {
go func() {
a := &rpctype.LogMessageReq{
Level: level,
- Name: fuzzer.name,
+ Name: tool.name,
Message: fmt.Sprintf(msg, args...),
}
- if err := fuzzer.manager.Call("Manager.LogMessage", a, nil); err != nil {
+ if err := tool.manager.Call("Manager.LogMessage", a, nil); err != nil {
log.SyzFatalf("Manager.LogMessage call failed: %v", err)
}
}()
@@ -657,19 +516,6 @@ func setupPprofHandler(port int) {
}()
}
-func signalPrio(p *prog.Prog, info *ipc.CallInfo, call int) (prio uint8) {
- if call == -1 {
- return 0
- }
- if info.Errno == 0 {
- prio |= 1 << 1
- }
- if !p.Target.CallContainsAny(p.Calls[call]) {
- prio |= 1 << 0
- }
- return
-}
-
func parseOutputType(str string) OutputType {
switch str {
case "none":
diff --git a/syz-fuzzer/proc.go b/syz-fuzzer/proc.go
index 89c12a1a7..369ec5735 100644
--- a/syz-fuzzer/proc.go
+++ b/syz-fuzzer/proc.go
@@ -13,316 +13,67 @@ import (
"syscall"
"time"
- "github.com/google/syzkaller/pkg/cover"
- "github.com/google/syzkaller/pkg/hash"
+ "github.com/google/syzkaller/pkg/fuzzer"
"github.com/google/syzkaller/pkg/ipc"
"github.com/google/syzkaller/pkg/log"
- "github.com/google/syzkaller/pkg/rpctype"
- "github.com/google/syzkaller/pkg/signal"
"github.com/google/syzkaller/prog"
)
// Proc represents a single fuzzing process (executor).
type Proc struct {
- fuzzer *Fuzzer
- pid int
- env *ipc.Env
- rnd *rand.Rand
- execOpts *ipc.ExecOpts
- execOptsCollide *ipc.ExecOpts
- execOptsCover *ipc.ExecOpts
- execOptsComps *ipc.ExecOpts
+ tool *FuzzerTool
+ pid int
+ env *ipc.Env
+ execOpts *ipc.ExecOpts
}
-func newProc(fuzzer *Fuzzer, pid int) (*Proc, error) {
- env, err := ipc.MakeEnv(fuzzer.config, pid)
+func newProc(tool *FuzzerTool, execOpts *ipc.ExecOpts, pid int) (*Proc, error) {
+ env, err := ipc.MakeEnv(tool.config, pid)
if err != nil {
return nil, err
}
- rnd := rand.New(rand.NewSource(time.Now().UnixNano() + int64(pid)*1e12))
- execOptsCollide := *fuzzer.execOpts
- execOptsCollide.Flags &= ^ipc.FlagCollectSignal
- execOptsCover := *fuzzer.execOpts
- execOptsCover.Flags |= ipc.FlagCollectCover
- execOptsComps := *fuzzer.execOpts
- execOptsComps.Flags |= ipc.FlagCollectComps
proc := &Proc{
- fuzzer: fuzzer,
- pid: pid,
- env: env,
- rnd: rnd,
- execOpts: fuzzer.execOpts,
- execOptsCollide: &execOptsCollide,
- execOptsCover: &execOptsCover,
- execOptsComps: &execOptsComps,
+ tool: tool,
+ pid: pid,
+ env: env,
+ execOpts: execOpts,
}
return proc, nil
}
func (proc *Proc) loop() {
- generatePeriod := 100
- if proc.fuzzer.config.Flags&ipc.FlagSignal == 0 {
- // If we don't have real coverage signal, generate programs more frequently
- // because fallback signal is weak.
- generatePeriod = 2
- }
- for i := 0; ; i++ {
- item := proc.fuzzer.workQueue.dequeue()
- if item != nil {
- switch item := item.(type) {
- case *WorkTriage:
- proc.triageInput(item)
- case *WorkCandidate:
- proc.execute(proc.execOpts, item.p, item.flags, StatCandidate, false)
- case *WorkSmash:
- proc.smashInput(item)
- default:
- log.SyzFatalf("unknown work type: %#v", item)
- }
- continue
- }
-
- ct := proc.fuzzer.choiceTable
- fuzzerSnapshot := proc.fuzzer.snapshot()
- if len(fuzzerSnapshot.corpus) == 0 || i%generatePeriod == 0 {
- // Generate a new prog.
- p := proc.fuzzer.target.Generate(proc.rnd, prog.RecommendedCalls, ct)
- log.Logf(1, "#%v: generated", proc.pid)
- proc.executeAndCollide(proc.execOpts, p, ProgNormal, StatGenerate)
- } else {
- // Mutate an existing prog.
- p := fuzzerSnapshot.chooseProgram(proc.rnd).Clone()
- p.Mutate(proc.rnd, prog.RecommendedCalls, ct, proc.fuzzer.noMutate, fuzzerSnapshot.corpus)
- log.Logf(1, "#%v: mutated", proc.pid)
- proc.executeAndCollide(proc.execOpts, p, ProgNormal, StatFuzz)
- }
- }
-}
-
-func (proc *Proc) triageInput(item *WorkTriage) {
- log.Logf(1, "#%v: triaging type=%x", proc.pid, item.flags)
-
- prio := signalPrio(item.p, &item.info, item.call)
- inputSignal := signal.FromRaw(item.info.Signal, prio)
- newSignal := proc.fuzzer.corpusSignalDiff(inputSignal)
- if newSignal.Empty() {
- return
- }
- callName := ".extra"
- logCallName := "extra"
- if item.call != -1 {
- callName = item.p.Calls[item.call].Meta.Name
- logCallName = fmt.Sprintf("call #%v %v", item.call, callName)
- }
- log.Logf(3, "triaging input for %v (new signal=%v)", logCallName, newSignal.Len())
- var inputCover cover.Cover
- const (
- signalRuns = 3
- minimizeAttempts = 3
- )
- // Compute input coverage and non-flaky signal for minimization.
- notexecuted := 0
- rawCover := []uint32{}
- for i := 0; i < signalRuns; i++ {
- proc.resetAccState()
- info := proc.executeRaw(proc.execOptsCover, item.p, StatTriage)
- if !reexecutionSuccess(info, &item.info, item.call) {
- // The call was not executed or failed.
- notexecuted++
- if notexecuted > signalRuns/2+1 {
- return // if happens too often, give up
- }
- continue
- }
- thisSignal, thisCover := getSignalAndCover(item.p, info, item.call)
- if len(rawCover) == 0 && proc.fuzzer.fetchRawCover {
- rawCover = append([]uint32{}, thisCover...)
- }
- newSignal = newSignal.Intersection(thisSignal)
- // Without !minimized check manager starts losing some considerable amount
- // of coverage after each restart. Mechanics of this are not completely clear.
- if newSignal.Empty() && item.flags&ProgMinimized == 0 {
- return
+ rnd := rand.New(rand.NewSource(time.Now().UnixNano() + int64(proc.pid)))
+ for {
+ req := proc.tool.fuzzer.NextInput()
+ opts := *proc.execOpts
+ if !req.NeedSignal {
+ opts.Flags &= ^ipc.FlagCollectSignal
}
- inputCover.Merge(thisCover)
- }
- if item.flags&ProgMinimized == 0 {
- item.p, item.call = prog.Minimize(item.p, item.call, false,
- func(p1 *prog.Prog, call1 int) bool {
- for i := 0; i < minimizeAttempts; i++ {
- info := proc.execute(proc.execOpts, p1, ProgNormal,
- StatMinimize, i == 0)
- if !reexecutionSuccess(info, &item.info, call1) {
- // The call was not executed or failed.
- continue
- }
- thisSignal, _ := getSignalAndCover(p1, info, call1)
- if newSignal.Intersection(thisSignal).Len() == newSignal.Len() {
- return true
- }
- }
- return false
- })
- }
-
- data := item.p.Serialize()
- sig := hash.Hash(data)
-
- log.Logf(2, "added new input for %v to corpus:\n%s", logCallName, data)
- proc.fuzzer.sendInputToManager(rpctype.Input{
- Call: callName,
- CallID: item.call,
- Prog: data,
- Signal: inputSignal.Serialize(),
- Cover: inputCover.Serialize(),
- RawCover: rawCover,
- })
-
- proc.fuzzer.addInputToCorpus(item.p, inputSignal, sig)
-
- if item.flags&ProgSmashed == 0 {
- proc.fuzzer.workQueue.enqueue(&WorkSmash{item.p, item.call})
- }
-}
-
-func reexecutionSuccess(info *ipc.ProgInfo, oldInfo *ipc.CallInfo, call int) bool {
- if info == nil || len(info.Calls) == 0 {
- return false
- }
- if call != -1 {
- // Don't minimize calls from successful to unsuccessful.
- // Successful calls are much more valuable.
- if oldInfo.Errno == 0 && info.Calls[call].Errno != 0 {
- return false
+ if req.NeedCover {
+ opts.Flags |= ipc.FlagCollectCover
}
- return len(info.Calls[call].Signal) != 0
- }
- return len(info.Extra.Signal) != 0
-}
-
-func getSignalAndCover(p *prog.Prog, info *ipc.ProgInfo, call int) (signal.Signal, []uint32) {
- inf := &info.Extra
- if call != -1 {
- inf = &info.Calls[call]
- }
- return signal.FromRaw(inf.Signal, signalPrio(p, inf, call)), inf.Cover
-}
-
-func (proc *Proc) smashInput(item *WorkSmash) {
- if proc.fuzzer.faultInjectionEnabled && item.call != -1 {
- proc.failCall(item.p, item.call)
- }
- if proc.fuzzer.comparisonTracingEnabled && item.call != -1 {
- proc.executeHintSeed(item.p, item.call)
- }
- fuzzerSnapshot := proc.fuzzer.snapshot()
- for i := 0; i < 100; i++ {
- p := item.p.Clone()
- p.Mutate(proc.rnd, prog.RecommendedCalls, proc.fuzzer.choiceTable, proc.fuzzer.noMutate, fuzzerSnapshot.corpus)
- log.Logf(1, "#%v: smash mutated", proc.pid)
- proc.executeAndCollide(proc.execOpts, p, ProgNormal, StatSmash)
- }
-}
-
-func (proc *Proc) failCall(p *prog.Prog, call int) {
- for nth := 1; nth <= 100; nth++ {
- log.Logf(1, "#%v: injecting fault into call %v/%v", proc.pid, call, nth)
- newProg := p.Clone()
- newProg.Calls[call].Props.FailNth = nth
- info := proc.executeRaw(proc.execOpts, newProg, StatSmash)
- if info != nil && len(info.Calls) > call && info.Calls[call].Flags&ipc.CallFaultInjected == 0 {
- break
+ if req.NeedHints {
+ opts.Flags |= ipc.FlagCollectComps
}
- }
-}
-
-func (proc *Proc) executeHintSeed(p *prog.Prog, call int) {
- log.Logf(1, "#%v: collecting comparisons", proc.pid)
- // First execute the original program to dump comparisons from KCOV.
- info := proc.execute(proc.execOptsComps, p, ProgNormal, StatSeed, true)
- if info == nil {
- return
- }
-
- // Then mutate the initial program for every match between
- // a syscall argument and a comparison operand.
- // Execute each of such mutants to check if it gives new coverage.
- p.MutateWithHints(call, info.Calls[call].Comps, func(p *prog.Prog) {
- log.Logf(1, "#%v: executing comparison hint", proc.pid)
- proc.execute(proc.execOpts, p, ProgNormal, StatHint, false)
- })
-}
-
-func (proc *Proc) execute(execOpts *ipc.ExecOpts, p *prog.Prog, flags ProgTypes, stat Stat,
- resetState bool) *ipc.ProgInfo {
- if resetState {
- proc.resetAccState()
- }
- info := proc.executeRaw(execOpts, p, stat)
- if info == nil {
- return nil
- }
- calls, extra := proc.fuzzer.checkNewSignal(p, info)
- for _, callIndex := range calls {
- proc.enqueueCallTriage(p, flags, callIndex, info.Calls[callIndex])
- }
- if extra {
- proc.enqueueCallTriage(p, flags, -1, info.Extra)
- }
- return info
-}
-
-func (proc *Proc) enqueueCallTriage(p *prog.Prog, flags ProgTypes, callIndex int, info ipc.CallInfo) {
- // info.Signal points to the output shmem region, detach it before queueing.
- info.Signal = append([]uint32{}, info.Signal...)
- // None of the caller use Cover, so just nil it instead of detaching.
- // Note: triage input uses executeRaw to get coverage.
- info.Cover = nil
- proc.fuzzer.workQueue.enqueue(&WorkTriage{
- p: p.Clone(),
- call: callIndex,
- info: info,
- flags: flags,
- })
-}
-
-func (proc *Proc) executeAndCollide(execOpts *ipc.ExecOpts, p *prog.Prog, flags ProgTypes, stat Stat) {
- proc.execute(execOpts, p, flags, stat, true)
-
- if proc.execOptsCollide.Flags&ipc.FlagThreaded == 0 {
- // We cannot collide syscalls without being in the threaded mode.
- return
- }
- const collideIterations = 2
- for i := 0; i < collideIterations; i++ {
- proc.executeRaw(proc.execOptsCollide, proc.randomCollide(p), StatCollide)
- }
-}
-
-func (proc *Proc) randomCollide(origP *prog.Prog) *prog.Prog {
- if proc.rnd.Intn(5) == 0 {
- // Old-style collide with a 20% probability.
- p, err := prog.DoubleExecCollide(origP, proc.rnd)
- if err == nil {
- return p
+ if req.NeedRawCover {
+ opts.Flags &= ^ipc.FlagDedupCover
}
- }
- if proc.rnd.Intn(4) == 0 {
- // Duplicate random calls with a 20% probability (25% * 80%).
- p, err := prog.DupCallCollide(origP, proc.rnd)
- if err == nil {
- return p
+ // Do not let too much state accumulate.
+ const restartIn = 600
+ restart := rnd.Intn(restartIn) == 0
+ if (restart || proc.tool.resetAccState) &&
+ (req.NeedCover || req.NeedSignal || req.NeedHints) {
+ proc.env.ForceRestart()
}
+ info := proc.executeRaw(&opts, req.Prog)
+ proc.tool.fuzzer.Done(req, &fuzzer.Result{
+ Info: info,
+ })
}
- p := prog.AssignRandomAsync(origP, proc.rnd)
- if proc.rnd.Intn(2) != 0 {
- prog.AssignRandomRerun(p, proc.rnd)
- }
- return p
}
-func (proc *Proc) executeRaw(opts *ipc.ExecOpts, p *prog.Prog, stat Stat) *ipc.ProgInfo {
- proc.fuzzer.checkDisabledCalls(p)
+func (proc *Proc) executeRaw(opts *ipc.ExecOpts, p *prog.Prog) *ipc.ProgInfo {
+ proc.tool.checkDisabledCalls(p)
for try := 0; ; try++ {
var output []byte
var info *ipc.ProgInfo
@@ -332,18 +83,17 @@ func (proc *Proc) executeRaw(opts *ipc.ExecOpts, p *prog.Prog, stat Stat) *ipc.P
err := proc.env.RestartIfNeeded(p.Target)
if err == nil {
// Limit concurrency.
- ticket := proc.fuzzer.gate.Enter()
+ ticket := proc.tool.gate.Enter()
proc.logProgram(opts, p)
- atomic.AddUint64(&proc.fuzzer.stats[stat], 1)
output, info, hanged, err = proc.env.Exec(opts, p)
- proc.fuzzer.gate.Leave(ticket)
+ proc.tool.gate.Leave(ticket)
}
if err != nil {
if err == prog.ErrExecBufferTooSmall {
// It's bad if we systematically fail to serialize programs,
// but so far we don't have a better handling than counting this.
// This error is observed a lot on the seeded syz_mount_image calls.
- atomic.AddUint64(&proc.fuzzer.stats[StatBufferTooSmall], 1)
+ atomic.AddUint64(&proc.tool.bufferTooSmall, 1)
return nil
}
if try > 10 {
@@ -360,7 +110,7 @@ func (proc *Proc) executeRaw(opts *ipc.ExecOpts, p *prog.Prog, stat Stat) *ipc.P
}
func (proc *Proc) logProgram(opts *ipc.ExecOpts, p *prog.Prog) {
- if proc.fuzzer.outputType == OutputNone {
+ if proc.tool.outputType == OutputNone {
return
}
@@ -368,14 +118,14 @@ func (proc *Proc) logProgram(opts *ipc.ExecOpts, p *prog.Prog) {
// The following output helps to understand what program crashed kernel.
// It must not be intermixed.
- switch proc.fuzzer.outputType {
+ switch proc.tool.outputType {
case OutputStdout:
now := time.Now()
- proc.fuzzer.logMu.Lock()
+ proc.tool.logMu.Lock()
fmt.Printf("%02v:%02v:%02v executing program %v:\n%s\n",
now.Hour(), now.Minute(), now.Second(),
proc.pid, data)
- proc.fuzzer.logMu.Unlock()
+ proc.tool.logMu.Unlock()
case OutputDmesg:
fd, err := syscall.Open("/dev/kmsg", syscall.O_WRONLY, 0)
if err == nil {
@@ -386,19 +136,12 @@ func (proc *Proc) logProgram(opts *ipc.ExecOpts, p *prog.Prog) {
syscall.Close(fd)
}
case OutputFile:
- f, err := os.Create(fmt.Sprintf("%v-%v.prog", proc.fuzzer.name, proc.pid))
+ f, err := os.Create(fmt.Sprintf("%v-%v.prog", proc.tool.name, proc.pid))
if err == nil {
f.Write(data)
f.Close()
}
default:
- log.SyzFatalf("unknown output type: %v", proc.fuzzer.outputType)
- }
-}
-
-func (proc *Proc) resetAccState() {
- if !proc.fuzzer.resetAccState {
- return
+ log.SyzFatalf("unknown output type: %v", proc.tool.outputType)
}
- proc.env.ForceRestart()
}
diff --git a/syz-fuzzer/workqueue.go b/syz-fuzzer/workqueue.go
deleted file mode 100644
index 62648336c..000000000
--- a/syz-fuzzer/workqueue.go
+++ /dev/null
@@ -1,131 +0,0 @@
-// Copyright 2017 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package main
-
-import (
- "sync"
-
- "github.com/google/syzkaller/pkg/ipc"
- "github.com/google/syzkaller/prog"
-)
-
-// WorkQueue holds global non-fuzzing work items (see the Work* structs below).
-// WorkQueue also does prioritization among work items, for example, we want
-// to triage and send to manager new inputs before we smash programs
-// in order to not permanently lose interesting programs in case of VM crash.
-type WorkQueue struct {
- mu sync.RWMutex
- triageCandidate []*WorkTriage
- candidate []*WorkCandidate
- triage []*WorkTriage
- smash []*WorkSmash
-
- procs int
- needCandidates chan struct{}
-}
-
-type ProgTypes int
-
-const (
- ProgCandidate ProgTypes = 1 << iota
- ProgMinimized
- ProgSmashed
- ProgNormal ProgTypes = 0
-)
-
-// WorkTriage are programs for which we noticed potential new coverage during
-// first execution. But we are not sure yet if the coverage is real or not.
-// During triage we understand if these programs in fact give new coverage,
-// and if yes, minimize them and add to corpus.
-type WorkTriage struct {
- p *prog.Prog
- call int
- info ipc.CallInfo
- flags ProgTypes
-}
-
-// WorkCandidate are programs from hub.
-// We don't know yet if they are useful for this fuzzer or not.
-// A proc handles them the same way as locally generated/mutated programs.
-type WorkCandidate struct {
- p *prog.Prog
- flags ProgTypes
-}
-
-// WorkSmash are programs just added to corpus.
-// During smashing these programs receive a one-time special attention
-// (emit faults, collect comparison hints, etc).
-type WorkSmash struct {
- p *prog.Prog
- call int
-}
-
-func newWorkQueue(procs int, needCandidates chan struct{}) *WorkQueue {
- return &WorkQueue{
- procs: procs,
- needCandidates: needCandidates,
- }
-}
-
-func (wq *WorkQueue) enqueue(item interface{}) {
- wq.mu.Lock()
- defer wq.mu.Unlock()
- switch item := item.(type) {
- case *WorkTriage:
- if item.flags&ProgCandidate != 0 {
- wq.triageCandidate = append(wq.triageCandidate, item)
- } else {
- wq.triage = append(wq.triage, item)
- }
- case *WorkCandidate:
- wq.candidate = append(wq.candidate, item)
- case *WorkSmash:
- wq.smash = append(wq.smash, item)
- default:
- panic("unknown work type")
- }
-}
-
-func (wq *WorkQueue) dequeue() (item interface{}) {
- wq.mu.RLock()
- if len(wq.triageCandidate)+len(wq.candidate)+len(wq.triage)+len(wq.smash) == 0 {
- wq.mu.RUnlock()
- return nil
- }
- wq.mu.RUnlock()
- wq.mu.Lock()
- wantCandidates := false
- if len(wq.triageCandidate) != 0 {
- last := len(wq.triageCandidate) - 1
- item = wq.triageCandidate[last]
- wq.triageCandidate = wq.triageCandidate[:last]
- } else if len(wq.candidate) != 0 {
- last := len(wq.candidate) - 1
- item = wq.candidate[last]
- wq.candidate = wq.candidate[:last]
- wantCandidates = len(wq.candidate) < wq.procs
- } else if len(wq.triage) != 0 {
- last := len(wq.triage) - 1
- item = wq.triage[last]
- wq.triage = wq.triage[:last]
- } else if len(wq.smash) != 0 {
- last := len(wq.smash) - 1
- item = wq.smash[last]
- wq.smash = wq.smash[:last]
- }
- wq.mu.Unlock()
- if wantCandidates {
- select {
- case wq.needCandidates <- struct{}{}:
- default:
- }
- }
- return item
-}
-
-func (wq *WorkQueue) wantCandidates() bool {
- wq.mu.RLock()
- defer wq.mu.RUnlock()
- return len(wq.candidate) < wq.procs
-}
generated by ggit (джигит) mrf-deployment (git 2.46.0) at 2026-03-11 19:55:20 +0000