From cd74cc9cf40795144dfbd7e933dcd10d220916f6 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 17 Nov 2016 18:38:10 +0100 Subject: syz-hub: add program syz-hub is used to exchange programs between syz-managers. --- config/config.go | 5 + hash/hash.go | 35 ++++++ prog/encoding.go | 33 ++++++ prog/encoding_test.go | 95 ++++++++++++++++ rpctype/rpctype.go | 26 +++++ syz-fuzzer/fuzzer.go | 31 +++--- syz-gce/syz-gce.go | 4 + syz-hub/http.go | 152 ++++++++++++++++++++++++++ syz-hub/hub.go | 118 ++++++++++++++++++++ syz-hub/state/state.go | 259 ++++++++++++++++++++++++++++++++++++++++++++ syz-hub/state/state_test.go | 27 +++++ syz-manager/manager.go | 168 +++++++++++++++++++++++++--- syz-manager/persistent.go | 54 ++------- 13 files changed, 936 insertions(+), 71 deletions(-) create mode 100644 hash/hash.go create mode 100644 prog/encoding_test.go create mode 100644 syz-hub/http.go create mode 100644 syz-hub/hub.go create mode 100644 syz-hub/state/state.go create mode 100644 syz-hub/state/state_test.go diff --git a/config/config.go b/config/config.go index ac75221c7..bb8de5b95 100644 --- a/config/config.go +++ b/config/config.go @@ -35,6 +35,9 @@ type Config struct { Debug bool // dump all VM output to console Output string // one of stdout/dmesg/file (useful only for local VM) + Hub_Addr string + Hub_Key string + Syzkaller string // path to syzkaller checkout (syz-manager will look for binaries in bin subdir) Type string // VM type (qemu, kvm, local) Count int // number of VMs (don't secify for adb, instead specify devices) @@ -291,6 +294,8 @@ func checkUnknownFields(data []byte) (string, error) { "Bin", "Debug", "Output", + "Hub_Addr", + "Hub_Key", "Syzkaller", "Type", "Count", diff --git a/hash/hash.go b/hash/hash.go new file mode 100644 index 000000000..d640536fe --- /dev/null +++ b/hash/hash.go @@ -0,0 +1,35 @@ +// Copyright 2016 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package hash + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" +) + +type Sig [sha1.Size]byte + +func Hash(data []byte) Sig { + return Sig(sha1.Sum(data)) +} + +func (sig *Sig) String() string { + return hex.EncodeToString((*sig)[:]) +} + +func FromString(str string) (Sig, error) { + bin, err := hex.DecodeString(str) + if err != nil { + return Sig{}, fmt.Errorf("failed to decode sig '%v': %v", str, err) + } + if len(bin) != len(Sig{}) { + return Sig{}, fmt.Errorf("failed to decode sig '%v': bad len", str) + } + var sig Sig + for i, v := range bin { + sig[i] = v + } + return sig, err +} diff --git a/prog/encoding.go b/prog/encoding.go index 7afe218be..11c86fb44 100644 --- a/prog/encoding.go +++ b/prog/encoding.go @@ -516,3 +516,36 @@ func (p *parser) Ident() string { func (p *parser) failf(msg string, args ...interface{}) { p.e = fmt.Errorf("%v\nline #%v: %v", fmt.Sprintf(msg, args...), p.l, p.s) } + +// CallSet returns a set of all calls in the program. +// It does very conservative parsing and is intended to parse paste/future serialization formats. +func CallSet(data []byte) (map[string]struct{}, error) { + calls := make(map[string]struct{}) + s := bufio.NewScanner(bytes.NewReader(data)) + for s.Scan() { + ln := s.Bytes() + if len(ln) == 0 || ln[0] == '#' { + continue + } + bracket := bytes.IndexByte(ln, '(') + if bracket == -1 { + return nil, fmt.Errorf("line does not contain opening bracket") + } + call := ln[:bracket] + if eq := bytes.IndexByte(call, '='); eq != -1 { + eq++ + for eq < len(call) && call[eq] == ' ' { + eq++ + } + call = call[eq:] + } + if len(call) == 0 { + return nil, fmt.Errorf("call name is empty") + } + calls[string(call)] = struct{}{} + } + if len(calls) == 0 { + return nil, fmt.Errorf("program does not contain any calls") + } + return calls, nil +} diff --git a/prog/encoding_test.go b/prog/encoding_test.go new file mode 100644 index 000000000..f23a92cc7 --- /dev/null +++ b/prog/encoding_test.go @@ -0,0 +1,95 @@ +// Copyright 2016 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package prog + +import ( + "fmt" + "reflect" + "sort" + "testing" +) + +func setToArray(s map[string]struct{}) []string { + a := make([]string, 0, len(s)) + for c := range s { + a = append(a, c) + } + sort.Strings(a) + return a +} + +func TestCallSet(t *testing.T) { + tests := []struct { + prog string + ok bool + calls []string + }{ + { + "", + false, + []string{}, + }, + { + "r0 = (foo)", + false, + []string{}, + }, + { + "getpid()", + true, + []string{"getpid"}, + }, + { + "r11 = getpid()", + true, + []string{"getpid"}, + }, + { + "getpid()\n" + + "open(0x1, something that this package may not understand)\n" + + "getpid()\n" + + "#read()\n" + + "\n" + + "close$foo(&(0x0000) = {})\n", + true, + []string{"getpid", "open", "close$foo"}, + }, + } + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + calls, err := CallSet([]byte(test.prog)) + if err != nil && test.ok { + t.Fatalf("parsing failed: %v", err) + } + if err == nil && !test.ok { + t.Fatalf("parsing did not fail") + } + callArray := setToArray(calls) + sort.Strings(test.calls) + if !reflect.DeepEqual(callArray, test.calls) { + t.Fatalf("got call set %+v, expect %+v", callArray, test.calls) + } + }) + } +} + +func TestCallSetRandom(t *testing.T) { + rs, iters := initTest(t) + for i := 0; i < iters; i++ { + p := Generate(rs, 10, nil) + calls0 := make(map[string]struct{}) + for _, c := range p.Calls { + calls0[c.Meta.Name] = struct{}{} + } + calls1, err := CallSet(p.Serialize()) + if err != nil { + t.Fatalf("CallSet failed: %v", err) + } + callArray0 := setToArray(calls0) + callArray1 := setToArray(calls1) + if !reflect.DeepEqual(callArray0, callArray1) { + t.Fatalf("got call set:\n%+v\nexpect:\n%+v", callArray1, callArray0) + } + } +} diff --git a/rpctype/rpctype.go b/rpctype/rpctype.go index a03017fc1..31e36fbfb 100644 --- a/rpctype/rpctype.go +++ b/rpctype/rpctype.go @@ -19,6 +19,13 @@ type ConnectArgs struct { type ConnectRes struct { Prios [][]float32 EnabledCalls string + NeedCheck bool +} + +type CheckArgs struct { + Name string + Kcov bool + Calls []string } type NewInputArgs struct { @@ -35,3 +42,22 @@ type PollRes struct { Candidates [][]byte NewInputs []RpcInput } + +type HubConnectArgs struct { + Name string + Key string + Fresh bool + Calls []string + Corpus [][]byte +} + +type HubSyncArgs struct { + Name string + Key string + Add [][]byte + Del []string +} + +type HubSyncRes struct { + Inputs [][]byte +} diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go index de753c8b1..f375df344 100644 --- a/syz-fuzzer/fuzzer.go +++ b/syz-fuzzer/fuzzer.go @@ -116,6 +116,20 @@ func main() { calls := buildCallList(r.EnabledCalls) ct := prog.BuildChoiceTable(r.Prios, calls) + if r.NeedCheck { + a := &CheckArgs{Name: *flagName} + if fd, err := syscall.Open("/sys/kernel/debug/kcov", syscall.O_RDWR, 0); err == nil { + syscall.Close(fd) + a.Kcov = true + } + for c := range calls { + a.Calls = append(a.Calls, c.Name) + } + if err := manager.Call("Manager.Check", a, nil); err != nil { + panic(err) + } + } + kmemleakInit() flags, timeout, err := ipc.DefaultFlags() @@ -123,13 +137,6 @@ func main() { panic(err) } noCover = flags&ipc.FlagCover == 0 - if !noCover { - fd, err := syscall.Open("/sys/kernel/debug/kcov", syscall.O_RDWR, 0) - if err != nil { - Fatalf("BUG: /sys/kernel/debug/kcov is missing (%v). Enable CONFIG_KCOV and mount debugfs.", err) - } - syscall.Close(fd) - } leakCallback := func() { if atomic.LoadUint32(&allTriaged) != 0 { // Scan for leaks once in a while (it is damn slow). @@ -269,13 +276,11 @@ func main() { triageMu.Unlock() } } - if len(r.Candidates) == 0 { - if atomic.LoadUint32(&allTriaged) == 0 { - if *flagLeak { - kmemleakScan(false) - } - atomic.StoreUint32(&allTriaged, 1) + if len(r.Candidates) == 0 && atomic.LoadUint32(&allTriaged) == 0 { + if *flagLeak { + kmemleakScan(false) } + atomic.StoreUint32(&allTriaged, 1) } if len(r.NewInputs) == 0 && len(r.Candidates) == 0 { lastPoll = time.Now() diff --git a/syz-gce/syz-gce.go b/syz-gce/syz-gce.go index 5554006ea..1bb523012 100644 --- a/syz-gce/syz-gce.go +++ b/syz-gce/syz-gce.go @@ -46,6 +46,8 @@ var ( type Config struct { Name string + Hub_Addr string + Hub_Key string Image_Archive string Image_Path string Image_Name string @@ -248,6 +250,8 @@ func writeManagerConfig(httpPort int, file string) error { } managerCfg := &config.Config{ Name: cfg.Name, + Hub_Addr: cfg.Hub_Addr, + Hub_Key: cfg.Hub_Key, Http: fmt.Sprintf(":%v", httpPort), Rpc: ":0", Workdir: "workdir", diff --git a/syz-hub/http.go b/syz-hub/http.go new file mode 100644 index 000000000..026219be1 --- /dev/null +++ b/syz-hub/http.go @@ -0,0 +1,152 @@ +// Copyright 2016 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "html/template" + "net" + "net/http" + "sort" + "strings" + + . "github.com/google/syzkaller/log" +) + +func (hub *Hub) initHttp(addr string) { + http.HandleFunc("/", hub.httpSummary) + + ln, err := net.Listen("tcp4", addr) + if err != nil { + Fatalf("failed to listen on %v: %v", addr, err) + } + Logf(0, "serving http on http://%v", ln.Addr()) + go func() { + err := http.Serve(ln, nil) + Fatalf("failed to serve http: %v", err) + }() +} + +func (hub *Hub) httpSummary(w http.ResponseWriter, r *http.Request) { + hub.mu.Lock() + defer hub.mu.Unlock() + + data := &UISummaryData{ + Log: CachedLogOutput(), + } + total := UIManager{ + Name: "total", + Corpus: len(hub.st.Corpus), + } + for name, mgr := range hub.st.Managers { + total.Added += mgr.Added + total.Deleted += mgr.Added + total.New += mgr.New + data.Managers = append(data.Managers, UIManager{ + Name: name, + Corpus: len(mgr.Corpus), + Added: mgr.Added, + Deleted: mgr.Deleted, + New: mgr.New, + }) + } + sort.Sort(UIManagerArray(data.Managers)) + data.Managers = append([]UIManager{total}, data.Managers...) + if err := summaryTemplate.Execute(w, data); err != nil { + Logf(0, "failed to execute template: %v", err) + http.Error(w, fmt.Sprintf("failed to execute template: %v", err), http.StatusInternalServerError) + return + } +} + +func compileTemplate(html string) *template.Template { + return template.Must(template.New("").Parse(strings.Replace(html, "{{STYLE}}", htmlStyle, -1))) +} + +type UISummaryData struct { + Managers []UIManager + Log string +} + +type UIManager struct { + Name string + Corpus int + Added int + Deleted int + New int +} + +type UIManagerArray []UIManager + +func (a UIManagerArray) Len() int { return len(a) } +func (a UIManagerArray) Less(i, j int) bool { return a[i].Name < a[j].Name } +func (a UIManagerArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +var summaryTemplate = compileTemplate(` + + + + syz-hub + {{STYLE}} + + +syz-hub +

+ + + + + + + + + + + {{range $m := $.Managers}} + + + + + + + + {{end}} +
Managers:
NameCorpusAddedDeletedNew
{{$m.Name}}{{$m.Corpus}}{{$m.Added}}{{$m.Deleted}}{{$m.New}}
+

+ +Log: +
+ + + + +`) + +const htmlStyle = ` + +` diff --git a/syz-hub/hub.go b/syz-hub/hub.go new file mode 100644 index 000000000..037f01d47 --- /dev/null +++ b/syz-hub/hub.go @@ -0,0 +1,118 @@ +// Copyright 2016 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "net" + "net/rpc" + "sync" + + . "github.com/google/syzkaller/log" + . "github.com/google/syzkaller/rpctype" + "github.com/google/syzkaller/syz-hub/state" +) + +var ( + flagConfig = flag.String("config", "", "config file") + + cfg *Config +) + +type Config struct { + Http string + Rpc string + Workdir string + Managers []struct { + Name string + Key string + } +} + +type Hub struct { + mu sync.Mutex + st *state.State + keys map[string]string +} + +func main() { + flag.Parse() + cfg = readConfig(*flagConfig) + EnableLogCaching(1000, 1<<20) + + st, err := state.Make(cfg.Workdir) + if err != nil { + Fatalf("failed to load state: %v", err) + } + hub := &Hub{ + st: st, + keys: make(map[string]string), + } + for _, mgr := range cfg.Managers { + hub.keys[mgr.Name] = mgr.Key + } + + hub.initHttp(cfg.Http) + + ln, err := net.Listen("tcp", cfg.Rpc) + if err != nil { + Fatalf("failed to listen on %v: %v", cfg.Rpc, err) + } + Logf(0, "serving rpc on tcp://%v", ln.Addr()) + s := rpc.NewServer() + s.Register(hub) + s.Accept(ln) +} + +func (hub *Hub) Connect(a *HubConnectArgs, r *int) error { + if key, ok := hub.keys[a.Name]; !ok || key != a.Key { + Logf(0, "connect from unauthorized manager %v", a.Name) + return fmt.Errorf("unauthorized manager") + } + hub.mu.Lock() + defer hub.mu.Unlock() + + Logf(0, "connect from %v: fresh=%v calls=%v corpus=%v", a.Name, a.Fresh, len(a.Calls), len(a.Corpus)) + if err := hub.st.Connect(a.Name, a.Fresh, a.Calls, a.Corpus); err != nil { + Logf(0, "connect error: %v", err) + return err + } + return nil +} + +func (hub *Hub) Sync(a *HubSyncArgs, r *HubSyncRes) error { + if key, ok := hub.keys[a.Name]; !ok || key != a.Key { + Logf(0, "sync from unauthorized manager %v", a.Name) + return fmt.Errorf("unauthorized manager") + } + hub.mu.Lock() + defer hub.mu.Unlock() + + inputs, err := hub.st.Sync(a.Name, a.Add, a.Del) + if err != nil { + Logf(0, "sync error: %v", err) + return err + } + r.Inputs = inputs + Logf(0, "sync from %v: add=%v del=%v new=%v", a.Name, len(a.Add), len(a.Del), len(inputs)) + return nil +} + +func readConfig(filename string) *Config { + if filename == "" { + Fatalf("supply config in -config flag") + } + data, err := ioutil.ReadFile(filename) + if err != nil { + Fatalf("failed to read config file: %v", err) + } + cfg := new(Config) + if err := json.Unmarshal(data, cfg); err != nil { + Fatalf("failed to parse config file: %v", err) + } + return cfg +} diff --git a/syz-hub/state/state.go b/syz-hub/state/state.go new file mode 100644 index 000000000..8b970fe6a --- /dev/null +++ b/syz-hub/state/state.go @@ -0,0 +1,259 @@ +// Copyright 2016 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package state + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/google/syzkaller/hash" + . "github.com/google/syzkaller/log" + "github.com/google/syzkaller/prog" +) + +// State holds all internal syz-hub state including corpus and information about managers. +// It is persisted to and can be restored from a directory. +type State struct { + seq uint64 + dir string + Corpus map[hash.Sig]*Input + Managers map[string]*Manager +} + +// Manager represents one syz-manager instance. +type Manager struct { + name string + seq uint64 + dir string + Connected time.Time + Added int + Deleted int + New int + Calls map[string]struct{} + Corpus map[hash.Sig]bool +} + +// Input holds info about a single corpus program. +type Input struct { + seq uint64 + prog []byte +} + +// Make creates State and initializes it from dir. +func Make(dir string) (*State, error) { + st := &State{ + dir: dir, + Corpus: make(map[hash.Sig]*Input), + Managers: make(map[string]*Manager), + } + + corpusDir := filepath.Join(st.dir, "corpus") + os.MkdirAll(corpusDir, 0700) + inputs, err := ioutil.ReadDir(corpusDir) + if err != nil { + return nil, fmt.Errorf("failed to read %v dir: %v", corpusDir, err) + } + for _, inp := range inputs { + data, err := ioutil.ReadFile(filepath.Join(corpusDir, inp.Name())) + if err != nil { + return nil, err + } + if _, err := prog.CallSet(data); err != nil { + return nil, err + } + parts := strings.Split(inp.Name(), "-") + if len(parts) != 2 { + return nil, fmt.Errorf("bad file in corpus: %v", inp.Name()) + } + seq, err := strconv.ParseUint(parts[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("bad file in corpus: %v", inp.Name()) + } + sig := hash.Hash(data) + if sig.String() != parts[0] { + return nil, fmt.Errorf("bad file in corpus: %v, want hash %v", inp.Name(), sig.String()) + } + st.Corpus[sig] = &Input{ + seq: seq, + prog: data, + } + if st.seq < seq { + st.seq = seq + } + } + + managersDir := filepath.Join(st.dir, "manager") + os.MkdirAll(managersDir, 0700) + managers, err := ioutil.ReadDir(managersDir) + if err != nil { + return nil, fmt.Errorf("failed to read %v dir: %v", managersDir, err) + } + for _, manager := range managers { + mgr := &Manager{ + name: manager.Name(), + } + st.Managers[mgr.name] = mgr + mgr.dir = filepath.Join(managersDir, mgr.name) + seqStr, _ := ioutil.ReadFile(filepath.Join(mgr.dir, "seq")) + mgr.seq, _ = strconv.ParseUint(string(seqStr), 10, 64) + if st.seq < mgr.seq { + st.seq = mgr.seq + } + + mgr.Corpus = make(map[hash.Sig]bool) + corpusDir := filepath.Join(mgr.dir, "corpus") + os.MkdirAll(corpusDir, 0700) + corpus, err := ioutil.ReadDir(corpusDir) + if err != nil { + return nil, fmt.Errorf("failed to read %v dir: %v", corpusDir, err) + } + for _, input := range corpus { + sig, err := hash.FromString(input.Name()) + if err != nil { + return nil, fmt.Errorf("bad file in corpus: %v", input.Name()) + } + mgr.Corpus[sig] = true + } + } + + return st, err +} + +func (st *State) Connect(name string, fresh bool, calls []string, corpus [][]byte) error { + st.seq++ + mgr := st.Managers[name] + if mgr == nil { + mgr = new(Manager) + st.Managers[name] = mgr + mgr.dir = filepath.Join(st.dir, "manager", name) + os.MkdirAll(mgr.dir, 0700) + } + mgr.Connected = time.Now() + if fresh { + mgr.seq = 0 + } + writeFile(filepath.Join(mgr.dir, "seq"), []byte(fmt.Sprint(mgr.seq))) + + mgr.Calls = make(map[string]struct{}) + for _, c := range calls { + mgr.Calls[c] = struct{}{} + } + + corpusDir := filepath.Join(mgr.dir, "corpus") + os.RemoveAll(corpusDir) + os.MkdirAll(corpusDir, 0700) + mgr.Corpus = make(map[hash.Sig]bool) + for _, prog := range corpus { + st.addInput(mgr, prog) + } + st.purgeCorpus() + return nil +} + +func (st *State) Sync(name string, add [][]byte, del []string) ([][]byte, error) { + mgr := st.Managers[name] + if mgr == nil || mgr.Connected.IsZero() { + return nil, fmt.Errorf("unconnected manager %v", name) + } + if len(del) != 0 { + for _, h := range del { + sig, err := hash.FromString(h) + if err != nil { + Logf(0, "manager %v: bad hash: %v", h) + continue + } + delete(mgr.Corpus, sig) + } + st.purgeCorpus() + } + if len(add) != 0 { + st.seq++ + for _, prog := range add { + st.addInput(mgr, prog) + } + } + inputs, err := st.pendingInputs(mgr) + mgr.Added += len(add) + mgr.Deleted += len(del) + mgr.New += len(inputs) + return inputs, err +} + +func (st *State) pendingInputs(mgr *Manager) ([][]byte, error) { + if mgr.seq == st.seq { + return nil, nil + } + var inputs [][]byte + for sig, inp := range st.Corpus { + if mgr.seq > inp.seq || mgr.Corpus[sig] { + continue + } + calls, err := prog.CallSet(inp.prog) + if err != nil { + return nil, fmt.Errorf("failed to extract call set: %v\nprogram: %v", err, string(inp.prog)) + } + if !managerSupportsAllCalls(mgr.Calls, calls) { + continue + } + inputs = append(inputs, inp.prog) + } + mgr.seq = st.seq + writeFile(filepath.Join(mgr.dir, "seq"), []byte(fmt.Sprint(mgr.seq))) + return inputs, nil +} + +func (st *State) addInput(mgr *Manager, input []byte) { + if _, err := prog.CallSet(input); err != nil { + Logf(0, "manager %v: failed to extract call set: %v, program:\n%v", mgr.name, err, string(input)) + return + } + sig := hash.Hash(input) + mgr.Corpus[sig] = true + fname := filepath.Join(mgr.dir, "corpus", sig.String()) + writeFile(fname, nil) + if st.Corpus[sig] == nil { + st.Corpus[sig] = &Input{ + seq: st.seq, + prog: input, + } + fname := filepath.Join(st.dir, "corpus", fmt.Sprintf("%v-%v", sig.String(), st.seq)) + writeFile(fname, input) + } +} + +func writeFile(name string, data []byte) { + if err := ioutil.WriteFile(name, data, 0600); err != nil { + Logf(0, "failed to write file %v: %v", name, err) + } +} + +func (st *State) purgeCorpus() { + used := make(map[hash.Sig]bool) + for _, mgr := range st.Managers { + for sig := range mgr.Corpus { + used[sig] = true + } + } + for sig, inp := range st.Corpus { + if used[sig] { + continue + } + delete(st.Corpus, sig) + os.Remove(filepath.Join(st.dir, "corpus", fmt.Sprintf("%v-%v", sig.String(), inp.seq))) + } +} + +func managerSupportsAllCalls(mgr, prog map[string]struct{}) bool { + for c := range prog { + if _, ok := mgr[c]; !ok { + return false + } + } + return true +} diff --git a/syz-hub/state/state_test.go b/syz-hub/state/state_test.go new file mode 100644 index 000000000..0fd66da2c --- /dev/null +++ b/syz-hub/state/state_test.go @@ -0,0 +1,27 @@ +// Copyright 2016 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package state + +import ( + "io/ioutil" + "os" + "testing" +) + +func TestState(t *testing.T) { + dir, err := ioutil.TempDir("", "syz-gce-state-test") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + defer os.RemoveAll(dir) + + st, err := Make(dir) + if err != nil { + t.Fatalf("failed to make state: %v", err) + } + _, err = st.Sync("foo", nil, nil) + if err == nil { + t.Fatalf("synced with unconnected manager") + } +} diff --git a/syz-manager/manager.go b/syz-manager/manager.go index 95d4e47e2..34de9b54c 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -5,7 +5,6 @@ package main import ( "bytes" - "encoding/hex" "flag" "fmt" "io/ioutil" @@ -23,6 +22,7 @@ import ( "github.com/google/syzkaller/config" "github.com/google/syzkaller/cover" + "github.com/google/syzkaller/hash" . "github.com/google/syzkaller/log" "github.com/google/syzkaller/prog" "github.com/google/syzkaller/report" @@ -50,9 +50,12 @@ type Manager struct { firstConnect time.Time stats map[string]uint64 shutdown uint32 + vmChecked bool + fresh bool mu sync.Mutex enabledSyscalls string + enabledCalls []string // as determined by fuzzer suppressions []*regexp.Regexp candidates [][]byte // untriaged inputs @@ -61,12 +64,14 @@ type Manager struct { corpusCover []cover.Cover prios [][]float32 - fuzzers map[string]*Fuzzer + fuzzers map[string]*Fuzzer + hub *rpc.Client + hubCorpus map[hash.Sig]bool } type Fuzzer struct { - name string - input int + name string + inputs []RpcInput } func main() { @@ -107,10 +112,12 @@ func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regex suppressions: suppressions, corpusCover: make([]cover.Cover, sys.CallCount), fuzzers: make(map[string]*Fuzzer), + fresh: true, } Logf(0, "loading corpus...") mgr.persistentCorpus = newPersistentSet(filepath.Join(cfg.Workdir, "corpus"), func(data []byte) bool { + mgr.fresh = false if _, err := prog.Deserialize(data); err != nil { Logf(0, "deleting broken program: %v\n%s", err, data) return false @@ -133,8 +140,10 @@ func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regex // This program contains a disabled syscall. // We won't execute it, but remeber its hash so // it is not deleted during minimization. - h := hash(data) - mgr.disabledHashes = append(mgr.disabledHashes, hex.EncodeToString(h[:])) + // TODO: use mgr.enabledCalls which accounts for missing devices, etc. + // But it is available only after vm check. + sig := hash.Hash(data) + mgr.disabledHashes = append(mgr.disabledHashes, sig.String()) continue } mgr.candidates = append(mgr.candidates, data) @@ -202,6 +211,15 @@ func RunManager(cfg *config.Config, syscalls map[int]bool, suppressions []*regex } }() + if mgr.cfg.Hub_Addr != "" { + go func() { + for { + time.Sleep(time.Minute) + mgr.hubSync() + } + }() + } + go func() { c := make(chan os.Signal, 2) signal.Notify(c, syscall.SIGINT) @@ -293,8 +311,8 @@ func (mgr *Manager) saveCrasher(vmCfg *vm.Config, desc string, text, output []by mgr.stats["crashes"]++ mgr.mu.Unlock() - h := hash([]byte(desc)) - id := hex.EncodeToString(h[:]) + sig := hash.Hash([]byte(desc)) + id := sig.String() dir := filepath.Join(mgr.crashdir, id) os.MkdirAll(dir, 0700) if err := ioutil.WriteFile(filepath.Join(dir, "description"), []byte(desc+"\n"), 0660); err != nil { @@ -369,8 +387,8 @@ func (mgr *Manager) minimizeCorpus() { if len(mgr.candidates) == 0 { hashes := make(map[string]bool) for _, inp := range mgr.corpus { - h := hash(inp.Prog) - hashes[hex.EncodeToString(h[:])] = true + sig := hash.Hash(inp.Prog) + hashes[sig.String()] = true } for _, h := range mgr.disabledHashes { hashes[h] = true @@ -389,22 +407,50 @@ func (mgr *Manager) Connect(a *ConnectArgs, r *ConnectRes) error { } mgr.stats["vm restarts"]++ + f := &Fuzzer{ + name: a.Name, + } + mgr.fuzzers[a.Name] = f mgr.minimizeCorpus() - mgr.fuzzers[a.Name] = &Fuzzer{ - name: a.Name, - input: 0, + for _, inp := range mgr.corpus { + f.inputs = append(f.inputs, inp) } r.Prios = mgr.prios r.EnabledCalls = mgr.enabledSyscalls + r.NeedCheck = !mgr.vmChecked return nil } +func (mgr *Manager) Check(a *CheckArgs, r *int) error { + mgr.mu.Lock() + defer mgr.mu.Unlock() + + if mgr.vmChecked { + return nil + } + Logf(1, "fuzzer %v vm check: %v calls enabled", a.Name, len(a.Calls)) + if len(a.Calls) == 0 { + Fatalf("no system calls enabled") + } + if mgr.cfg.Cover && !a.Kcov { + Fatalf("/sys/kernel/debug/kcov is missing. Enable CONFIG_KCOV and mount debugfs") + } + mgr.vmChecked = true + mgr.enabledCalls = a.Calls + return nil +} + func (mgr *Manager) NewInput(a *NewInputArgs, r *int) error { Logf(2, "new input from %v for syscall %v", a.Name, a.Call) mgr.mu.Lock() defer mgr.mu.Unlock() + f := mgr.fuzzers[a.Name] + if f == nil { + Fatalf("fuzzer %v is not connected", a.Name) + } + call := sys.CallID[a.Call] if len(cover.Difference(a.Cover, mgr.corpusCover[call])) == 0 { return nil @@ -413,6 +459,12 @@ func (mgr *Manager) NewInput(a *NewInputArgs, r *int) error { mgr.corpus = append(mgr.corpus, a.RpcInput) mgr.stats["manager new inputs"]++ mgr.persistentCorpus.add(a.RpcInput.Prog) + for _, f1 := range mgr.fuzzers { + if f1 == f { + continue + } + f1.inputs = append(f1.inputs, a.RpcInput) + } return nil } @@ -430,9 +482,13 @@ func (mgr *Manager) Poll(a *PollArgs, r *PollRes) error { Fatalf("fuzzer %v is not connected", a.Name) } - for i := 0; i < 100 && f.input < len(mgr.corpus); i++ { - r.NewInputs = append(r.NewInputs, mgr.corpus[f.input]) - f.input++ + for i := 0; i < 100 && len(f.inputs) > 0; i++ { + last := len(f.inputs) - 1 + r.NewInputs = append(r.NewInputs, f.inputs[last]) + f.inputs = f.inputs[:last] + } + if len(f.inputs) == 0 { + f.inputs = nil } for i := 0; i < 10 && len(mgr.candidates) > 0; i++ { @@ -446,3 +502,83 @@ func (mgr *Manager) Poll(a *PollArgs, r *PollRes) error { return nil } + +func (mgr *Manager) hubSync() { + mgr.mu.Lock() + defer mgr.mu.Unlock() + if !mgr.vmChecked || len(mgr.candidates) != 0 { + return + } + + mgr.minimizeCorpus() + if mgr.hub == nil { + conn, err := rpc.Dial("tcp", mgr.cfg.Hub_Addr) + if err != nil { + Logf(0, "failed to connect to hub at %v: %v", mgr.cfg.Hub_Addr, err) + return + } + mgr.hub = conn + a := &HubConnectArgs{ + Name: mgr.cfg.Name, + Key: mgr.cfg.Hub_Key, + Fresh: mgr.fresh, + Calls: mgr.enabledCalls, + } + mgr.hubCorpus = make(map[hash.Sig]bool) + for _, inp := range mgr.corpus { + mgr.hubCorpus[hash.Hash(inp.Prog)] = true + a.Corpus = append(a.Corpus, inp.Prog) + } + if err := mgr.hub.Call("Hub.Connect", a, nil); err != nil { + Logf(0, "Hub.Connect rpc failed: %v", err) + mgr.hub.Close() + mgr.hub = nil + return + } + mgr.fresh = false + Logf(0, "connected to hub at %v, corpus %v", mgr.cfg.Hub_Addr, len(mgr.corpus)) + } + + a := &HubSyncArgs{ + Name: mgr.cfg.Name, + Key: mgr.cfg.Hub_Key, + } + corpus := make(map[hash.Sig]bool) + for _, inp := range mgr.corpus { + sig := hash.Hash(inp.Prog) + corpus[sig] = true + if mgr.hubCorpus[sig] { + continue + } + mgr.hubCorpus[sig] = true + a.Add = append(a.Add, inp.Prog) + } + for sig := range mgr.hubCorpus { + if corpus[sig] { + continue + } + delete(mgr.hubCorpus, sig) + a.Del = append(a.Del, sig.String()) + } + r := new(HubSyncRes) + if err := mgr.hub.Call("Hub.Sync", a, r); err != nil { + Logf(0, "Hub.Sync rpc failed: %v", err) + mgr.hub.Close() + mgr.hub = nil + return + } + dropped := 0 + for _, inp := range r.Inputs { + _, err := prog.Deserialize(inp) + if err != nil { + dropped++ + continue + } + mgr.candidates = append(mgr.candidates, inp) + } + mgr.stats["hub add"] += uint64(len(a.Add)) + mgr.stats["hub del"] += uint64(len(a.Del)) + mgr.stats["hub drop"] += uint64(dropped) + mgr.stats["hub new"] += uint64(len(r.Inputs) - dropped) + Logf(0, "hub sync: add %v, del %v, drop %v, new %v", len(a.Add), len(a.Del), dropped, len(r.Inputs)-dropped) +} diff --git a/syz-manager/persistent.go b/syz-manager/persistent.go index 5839027ab..1ef53ecff 100644 --- a/syz-manager/persistent.go +++ b/syz-manager/persistent.go @@ -4,33 +4,25 @@ package main import ( - "crypto/sha1" - "encoding/hex" - "fmt" "io/ioutil" "os" "path/filepath" + "github.com/google/syzkaller/hash" . "github.com/google/syzkaller/log" ) -type Sig [sha1.Size]byte - // PersistentSet is a set of binary blobs with a persistent mirror on disk. type PersistentSet struct { dir string - m map[Sig][]byte + m map[hash.Sig][]byte a [][]byte } -func hash(data []byte) Sig { - return Sig(sha1.Sum(data)) -} - func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet { ps := &PersistentSet{ dir: dir, - m: make(map[Sig][]byte), + m: make(map[hash.Sig][]byte), } os.MkdirAll(dir, 0770) filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { @@ -48,7 +40,7 @@ func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet Fatalf("error during file read: %v\n", err) return nil } - sig := hash(data) + sig := hash.Hash(data) if _, ok := ps.m[sig]; ok { return nil } @@ -60,20 +52,17 @@ func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet os.Remove(path) return nil } - const hexLen = 2 * sha1.Size - if len(name) > hexLen+1 && isHexString(name[:hexLen]) && name[hexLen] == '.' { - return nil // description file - } - if len(name) != hexLen || !isHexString(name) { + if _, err := hash.FromString(name); err != nil { Logf(0, "unknown file in persistent dir %v: %v", dir, name) + return nil } if verify != nil && !verify(data) { os.Remove(path) return nil } - if name != hex.EncodeToString(sig[:]) { - Logf(0, "bad hash in persistent dir %v for file %v, expect %v", dir, name, hex.EncodeToString(sig[:])) - if err := ioutil.WriteFile(filepath.Join(ps.dir, hex.EncodeToString(sig[:])), data, 0660); err != nil { + if name != sig.String() { + Logf(0, "bad hash in persistent dir %v for file %v, expect %v", dir, name, sig.String()) + if err := ioutil.WriteFile(filepath.Join(ps.dir, sig.String()), data, 0660); err != nil { Fatalf("failed to write file: %v", err) } os.Remove(path) @@ -85,43 +74,24 @@ func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet return ps } -func isHexString(s string) bool { - for _, v := range []byte(s) { - if v >= '0' && v <= '9' || v >= 'a' && v <= 'f' { - continue - } - return false - } - return true -} - func (ps *PersistentSet) add(data []byte) bool { - sig := hash(data) + sig := hash.Hash(data) if _, ok := ps.m[sig]; ok { return false } ps.m[sig] = data ps.a = append(ps.a, data) - fname := filepath.Join(ps.dir, hex.EncodeToString(sig[:])) + fname := filepath.Join(ps.dir, sig.String()) if err := ioutil.WriteFile(fname, data, 0660); err != nil { Fatalf("failed to write file: %v", err) } return true } -// addDescription creates a complementary to data file on disk. -func (ps *PersistentSet) addDescription(data []byte, desc []byte, typ string) { - sig := hash(data) - fname := filepath.Join(ps.dir, fmt.Sprintf("%v.%v", hex.EncodeToString(sig[:]), typ)) - if err := ioutil.WriteFile(fname, desc, 0660); err != nil { - Fatalf("failed to write file: %v", err) - } -} - func (ps *PersistentSet) minimize(set map[string]bool) { ps.a = nil for sig, data := range ps.m { - s := hex.EncodeToString(sig[:]) + s := sig.String() if set[s] { ps.a = append(ps.a, data) } else { -- cgit mrf-deployment