syz-hub: add program

syz-hub is used to exchange programs between syz-managers.
author: Dmitry Vyukov <dvyukov@google.com> 2016-11-17 18:38:10 +0100
committer: Dmitry Vyukov <dvyukov@google.com> 2016-11-17 18:38:10 +0100
commit: cd74cc9cf40795144dfbd7e933dcd10d220916f6 (patch)
tree: 6c1e55891acf53909da12aad34c86409be8c20a6 /syz-hub
parent: 3ad1f7a214ba9f22499ae6b2b8cc1f0b824073eb (diff)
4 files changed, 556 insertions, 0 deletions
diff --git a/syz-hub/http.go b/syz-hub/http.go
new file mode 100644
index 000000000..026219be1
--- /dev/null
+++ b/syz-hub/http.go
@@ -0,0 +1,152 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"html/template"
+	"net"
+	"net/http"
+	"sort"
+	"strings"
+
+	. "github.com/google/syzkaller/log"
+)
+
+func (hub *Hub) initHttp(addr string) {
+	http.HandleFunc("/", hub.httpSummary)
+
+	ln, err := net.Listen("tcp4", addr)
+	if err != nil {
+		Fatalf("failed to listen on %v: %v", addr, err)
+	}
+	Logf(0, "serving http on http://%v", ln.Addr())
+	go func() {
+		err := http.Serve(ln, nil)
+		Fatalf("failed to serve http: %v", err)
+	}()
+}
+
+func (hub *Hub) httpSummary(w http.ResponseWriter, r *http.Request) {
+	hub.mu.Lock()
+	defer hub.mu.Unlock()
+
+	data := &UISummaryData{
+		Log: CachedLogOutput(),
+	}
+	total := UIManager{
+		Name:   "total",
+		Corpus: len(hub.st.Corpus),
+	}
+	for name, mgr := range hub.st.Managers {
+		total.Added += mgr.Added
+		total.Deleted += mgr.Added
+		total.New += mgr.New
+		data.Managers = append(data.Managers, UIManager{
+			Name:    name,
+			Corpus:  len(mgr.Corpus),
+			Added:   mgr.Added,
+			Deleted: mgr.Deleted,
+			New:     mgr.New,
+		})
+	}
+	sort.Sort(UIManagerArray(data.Managers))
+	data.Managers = append([]UIManager{total}, data.Managers...)
+	if err := summaryTemplate.Execute(w, data); err != nil {
+		Logf(0, "failed to execute template: %v", err)
+		http.Error(w, fmt.Sprintf("failed to execute template: %v", err), http.StatusInternalServerError)
+		return
+	}
+}
+
+func compileTemplate(html string) *template.Template {
+	return template.Must(template.New("").Parse(strings.Replace(html, "{{STYLE}}", htmlStyle, -1)))
+}
+
+type UISummaryData struct {
+	Managers []UIManager
+	Log      string
+}
+
+type UIManager struct {
+	Name    string
+	Corpus  int
+	Added   int
+	Deleted int
+	New     int
+}
+
+type UIManagerArray []UIManager
+
+func (a UIManagerArray) Len() int           { return len(a) }
+func (a UIManagerArray) Less(i, j int) bool { return a[i].Name < a[j].Name }
+func (a UIManagerArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+var summaryTemplate = compileTemplate(`
+<!doctype html>
+<html>
+<head>
+	<title>syz-hub</title>
+	{{STYLE}}
+</head>
+<body>
+<b>syz-hub</b>
+<br><br>
+
+<table>
+	<caption>Managers:</caption>
+	<tr>
+		<th>Name</th>
+		<th>Corpus</th>
+		<th>Added</th>
+		<th>Deleted</th>
+		<th>New</th>
+	</tr>
+	{{range $m := $.Managers}}
+	<tr>
+		<td>{{$m.Name}}</td>
+		<td>{{$m.Corpus}}</td>
+		<td>{{$m.Added}}</td>
+		<td>{{$m.Deleted}}</td>
+		<td>{{$m.New}}</td>
+	</tr>
+	{{end}}
+</table>
+<br><br>
+
+Log:
+<br>
+<textarea id="log_textarea" readonly rows="50">
+{{.Log}}
+</textarea>
+<script>
+	var textarea = document.getElementById("log_textarea");
+	textarea.scrollTop = textarea.scrollHeight;
+</script>
+
+</body></html>
+`)
+
+const htmlStyle = `
+	<style type="text/css" media="screen">
+		table {
+			border-collapse:collapse;
+			border:1px solid;
+		}
+		table caption {
+			font-weight: bold;
+		}
+		table td {
+			border:1px solid;
+			padding: 3px;
+		}
+		table th {
+			border:1px solid;
+			padding: 3px;
+		}
+		textarea {
+			width:100%;
+		}
+	</style>
+`
diff --git a/syz-hub/hub.go b/syz-hub/hub.go
new file mode 100644
index 000000000..037f01d47
--- /dev/null
+++ b/syz-hub/hub.go
@@ -0,0 +1,118 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/rpc"
+	"sync"
+
+	. "github.com/google/syzkaller/log"
+	. "github.com/google/syzkaller/rpctype"
+	"github.com/google/syzkaller/syz-hub/state"
+)
+
+var (
+	flagConfig = flag.String("config", "", "config file")
+
+	cfg *Config
+)
+
+type Config struct {
+	Http     string
+	Rpc      string
+	Workdir  string
+	Managers []struct {
+		Name string
+		Key  string
+	}
+}
+
+type Hub struct {
+	mu   sync.Mutex
+	st   *state.State
+	keys map[string]string
+}
+
+func main() {
+	flag.Parse()
+	cfg = readConfig(*flagConfig)
+	EnableLogCaching(1000, 1<<20)
+
+	st, err := state.Make(cfg.Workdir)
+	if err != nil {
+		Fatalf("failed to load state: %v", err)
+	}
+	hub := &Hub{
+		st:   st,
+		keys: make(map[string]string),
+	}
+	for _, mgr := range cfg.Managers {
+		hub.keys[mgr.Name] = mgr.Key
+	}
+
+	hub.initHttp(cfg.Http)
+
+	ln, err := net.Listen("tcp", cfg.Rpc)
+	if err != nil {
+		Fatalf("failed to listen on %v: %v", cfg.Rpc, err)
+	}
+	Logf(0, "serving rpc on tcp://%v", ln.Addr())
+	s := rpc.NewServer()
+	s.Register(hub)
+	s.Accept(ln)
+}
+
+func (hub *Hub) Connect(a *HubConnectArgs, r *int) error {
+	if key, ok := hub.keys[a.Name]; !ok || key != a.Key {
+		Logf(0, "connect from unauthorized manager %v", a.Name)
+		return fmt.Errorf("unauthorized manager")
+	}
+	hub.mu.Lock()
+	defer hub.mu.Unlock()
+
+	Logf(0, "connect from %v: fresh=%v calls=%v corpus=%v", a.Name, a.Fresh, len(a.Calls), len(a.Corpus))
+	if err := hub.st.Connect(a.Name, a.Fresh, a.Calls, a.Corpus); err != nil {
+		Logf(0, "connect error: %v", err)
+		return err
+	}
+	return nil
+}
+
+func (hub *Hub) Sync(a *HubSyncArgs, r *HubSyncRes) error {
+	if key, ok := hub.keys[a.Name]; !ok || key != a.Key {
+		Logf(0, "sync from unauthorized manager %v", a.Name)
+		return fmt.Errorf("unauthorized manager")
+	}
+	hub.mu.Lock()
+	defer hub.mu.Unlock()
+
+	inputs, err := hub.st.Sync(a.Name, a.Add, a.Del)
+	if err != nil {
+		Logf(0, "sync error: %v", err)
+		return err
+	}
+	r.Inputs = inputs
+	Logf(0, "sync from %v: add=%v del=%v new=%v", a.Name, len(a.Add), len(a.Del), len(inputs))
+	return nil
+}
+
+func readConfig(filename string) *Config {
+	if filename == "" {
+		Fatalf("supply config in -config flag")
+	}
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		Fatalf("failed to read config file: %v", err)
+	}
+	cfg := new(Config)
+	if err := json.Unmarshal(data, cfg); err != nil {
+		Fatalf("failed to parse config file: %v", err)
+	}
+	return cfg
+}
diff --git a/syz-hub/state/state.go b/syz-hub/state/state.go
new file mode 100644
index 000000000..8b970fe6a
--- /dev/null
+++ b/syz-hub/state/state.go
@@ -0,0 +1,259 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package state
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/syzkaller/hash"
+	. "github.com/google/syzkaller/log"
+	"github.com/google/syzkaller/prog"
+)
+
+// State holds all internal syz-hub state including corpus and information about managers.
+// It is persisted to and can be restored from a directory.
+type State struct {
+	seq      uint64
+	dir      string
+	Corpus   map[hash.Sig]*Input
+	Managers map[string]*Manager
+}
+
+// Manager represents one syz-manager instance.
+type Manager struct {
+	name      string
+	seq       uint64
+	dir       string
+	Connected time.Time
+	Added     int
+	Deleted   int
+	New       int
+	Calls     map[string]struct{}
+	Corpus    map[hash.Sig]bool
+}
+
+// Input holds info about a single corpus program.
+type Input struct {
+	seq  uint64
+	prog []byte
+}
+
+// Make creates State and initializes it from dir.
+func Make(dir string) (*State, error) {
+	st := &State{
+		dir:      dir,
+		Corpus:   make(map[hash.Sig]*Input),
+		Managers: make(map[string]*Manager),
+	}
+
+	corpusDir := filepath.Join(st.dir, "corpus")
+	os.MkdirAll(corpusDir, 0700)
+	inputs, err := ioutil.ReadDir(corpusDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %v dir: %v", corpusDir, err)
+	}
+	for _, inp := range inputs {
+		data, err := ioutil.ReadFile(filepath.Join(corpusDir, inp.Name()))
+		if err != nil {
+			return nil, err
+		}
+		if _, err := prog.CallSet(data); err != nil {
+			return nil, err
+		}
+		parts := strings.Split(inp.Name(), "-")
+		if len(parts) != 2 {
+			return nil, fmt.Errorf("bad file in corpus: %v", inp.Name())
+		}
+		seq, err := strconv.ParseUint(parts[1], 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("bad file in corpus: %v", inp.Name())
+		}
+		sig := hash.Hash(data)
+		if sig.String() != parts[0] {
+			return nil, fmt.Errorf("bad file in corpus: %v, want hash %v", inp.Name(), sig.String())
+		}
+		st.Corpus[sig] = &Input{
+			seq:  seq,
+			prog: data,
+		}
+		if st.seq < seq {
+			st.seq = seq
+		}
+	}
+
+	managersDir := filepath.Join(st.dir, "manager")
+	os.MkdirAll(managersDir, 0700)
+	managers, err := ioutil.ReadDir(managersDir)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read %v dir: %v", managersDir, err)
+	}
+	for _, manager := range managers {
+		mgr := &Manager{
+			name: manager.Name(),
+		}
+		st.Managers[mgr.name] = mgr
+		mgr.dir = filepath.Join(managersDir, mgr.name)
+		seqStr, _ := ioutil.ReadFile(filepath.Join(mgr.dir, "seq"))
+		mgr.seq, _ = strconv.ParseUint(string(seqStr), 10, 64)
+		if st.seq < mgr.seq {
+			st.seq = mgr.seq
+		}
+
+		mgr.Corpus = make(map[hash.Sig]bool)
+		corpusDir := filepath.Join(mgr.dir, "corpus")
+		os.MkdirAll(corpusDir, 0700)
+		corpus, err := ioutil.ReadDir(corpusDir)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read %v dir: %v", corpusDir, err)
+		}
+		for _, input := range corpus {
+			sig, err := hash.FromString(input.Name())
+			if err != nil {
+				return nil, fmt.Errorf("bad file in corpus: %v", input.Name())
+			}
+			mgr.Corpus[sig] = true
+		}
+	}
+
+	return st, err
+}
+
+func (st *State) Connect(name string, fresh bool, calls []string, corpus [][]byte) error {
+	st.seq++
+	mgr := st.Managers[name]
+	if mgr == nil {
+		mgr = new(Manager)
+		st.Managers[name] = mgr
+		mgr.dir = filepath.Join(st.dir, "manager", name)
+		os.MkdirAll(mgr.dir, 0700)
+	}
+	mgr.Connected = time.Now()
+	if fresh {
+		mgr.seq = 0
+	}
+	writeFile(filepath.Join(mgr.dir, "seq"), []byte(fmt.Sprint(mgr.seq)))
+
+	mgr.Calls = make(map[string]struct{})
+	for _, c := range calls {
+		mgr.Calls[c] = struct{}{}
+	}
+
+	corpusDir := filepath.Join(mgr.dir, "corpus")
+	os.RemoveAll(corpusDir)
+	os.MkdirAll(corpusDir, 0700)
+	mgr.Corpus = make(map[hash.Sig]bool)
+	for _, prog := range corpus {
+		st.addInput(mgr, prog)
+	}
+	st.purgeCorpus()
+	return nil
+}
+
+func (st *State) Sync(name string, add [][]byte, del []string) ([][]byte, error) {
+	mgr := st.Managers[name]
+	if mgr == nil || mgr.Connected.IsZero() {
+		return nil, fmt.Errorf("unconnected manager %v", name)
+	}
+	if len(del) != 0 {
+		for _, h := range del {
+			sig, err := hash.FromString(h)
+			if err != nil {
+				Logf(0, "manager %v: bad hash: %v", h)
+				continue
+			}
+			delete(mgr.Corpus, sig)
+		}
+		st.purgeCorpus()
+	}
+	if len(add) != 0 {
+		st.seq++
+		for _, prog := range add {
+			st.addInput(mgr, prog)
+		}
+	}
+	inputs, err := st.pendingInputs(mgr)
+	mgr.Added += len(add)
+	mgr.Deleted += len(del)
+	mgr.New += len(inputs)
+	return inputs, err
+}
+
+func (st *State) pendingInputs(mgr *Manager) ([][]byte, error) {
+	if mgr.seq == st.seq {
+		return nil, nil
+	}
+	var inputs [][]byte
+	for sig, inp := range st.Corpus {
+		if mgr.seq > inp.seq || mgr.Corpus[sig] {
+			continue
+		}
+		calls, err := prog.CallSet(inp.prog)
+		if err != nil {
+			return nil, fmt.Errorf("failed to extract call set: %v\nprogram: %v", err, string(inp.prog))
+		}
+		if !managerSupportsAllCalls(mgr.Calls, calls) {
+			continue
+		}
+		inputs = append(inputs, inp.prog)
+	}
+	mgr.seq = st.seq
+	writeFile(filepath.Join(mgr.dir, "seq"), []byte(fmt.Sprint(mgr.seq)))
+	return inputs, nil
+}
+
+func (st *State) addInput(mgr *Manager, input []byte) {
+	if _, err := prog.CallSet(input); err != nil {
+		Logf(0, "manager %v: failed to extract call set: %v, program:\n%v", mgr.name, err, string(input))
+		return
+	}
+	sig := hash.Hash(input)
+	mgr.Corpus[sig] = true
+	fname := filepath.Join(mgr.dir, "corpus", sig.String())
+	writeFile(fname, nil)
+	if st.Corpus[sig] == nil {
+		st.Corpus[sig] = &Input{
+			seq:  st.seq,
+			prog: input,
+		}
+		fname := filepath.Join(st.dir, "corpus", fmt.Sprintf("%v-%v", sig.String(), st.seq))
+		writeFile(fname, input)
+	}
+}
+
+func writeFile(name string, data []byte) {
+	if err := ioutil.WriteFile(name, data, 0600); err != nil {
+		Logf(0, "failed to write file %v: %v", name, err)
+	}
+}
+
+func (st *State) purgeCorpus() {
+	used := make(map[hash.Sig]bool)
+	for _, mgr := range st.Managers {
+		for sig := range mgr.Corpus {
+			used[sig] = true
+		}
+	}
+	for sig, inp := range st.Corpus {
+		if used[sig] {
+			continue
+		}
+		delete(st.Corpus, sig)
+		os.Remove(filepath.Join(st.dir, "corpus", fmt.Sprintf("%v-%v", sig.String(), inp.seq)))
+	}
+}
+
+func managerSupportsAllCalls(mgr, prog map[string]struct{}) bool {
+	for c := range prog {
+		if _, ok := mgr[c]; !ok {
+			return false
+		}
+	}
+	return true
+}
diff --git a/syz-hub/state/state_test.go b/syz-hub/state/state_test.go
new file mode 100644
index 000000000..0fd66da2c
--- /dev/null
+++ b/syz-hub/state/state_test.go
@@ -0,0 +1,27 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package state
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestState(t *testing.T) {
+	dir, err := ioutil.TempDir("", "syz-gce-state-test")
+	if err != nil {
+		t.Fatalf("failed to create temp dir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	st, err := Make(dir)
+	if err != nil {
+		t.Fatalf("failed to make state: %v", err)
+	}
+	_, err = st.Sync("foo", nil, nil)
+	if err == nil {
+		t.Fatalf("synced with unconnected manager")
+	}
+}
author	Dmitry Vyukov <dvyukov@google.com>	2016-11-17 18:38:10 +0100
committer	Dmitry Vyukov <dvyukov@google.com>	2016-11-17 18:38:10 +0100
commit	cd74cc9cf40795144dfbd7e933dcd10d220916f6 (patch)
tree	6c1e55891acf53909da12aad34c86409be8c20a6 /syz-hub
parent	3ad1f7a214ba9f22499ae6b2b8cc1f0b824073eb (diff)