initial commit

author: Dmitry Vyukov <dvyukov@google.com> 2015-10-12 10:16:57 +0200
committer: Dmitry Vyukov <dvyukov@google.com> 2015-10-12 10:16:57 +0200
commit: 874c5754bb22dbf77d6b600ff91f0f4f1fc5073a (patch)
tree: 0075fbd088046ad5c86e6e972235701d68b3ce7c /master
3 files changed, 393 insertions, 0 deletions
diff --git a/master/html.go b/master/html.go
new file mode 100644
index 000000000..bb7ac1bd7
--- /dev/null
+++ b/master/html.go
@@ -0,0 +1,94 @@
+// Copyright 2015 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"io/ioutil"
+	"net/http"
+)
+
+func (m *Master) httpInfo(w http.ResponseWriter, r *http.Request) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	data := &UIData{
+		CorpusLen: len(m.corpus.m),
+	}
+	for _, mgr := range m.managers {
+		data.Managers = append(data.Managers, UIManager{
+			Name: mgr.name,
+			Http: mgr.http,
+		})
+	}
+	if err := htmlTemplate.Execute(w, data); err != nil {
+		http.Error(w, fmt.Sprintf("failed to execute template: %v", err), http.StatusInternalServerError)
+	}
+}
+
+func (m *Master) httpMinimize(w http.ResponseWriter, r *http.Request) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	corpus := make(map[string]bool)
+	for _, mgr := range m.managers {
+		resp, err := http.Get("http://" + mgr.http + "/current_corpus")
+		if err != nil {
+			http.Error(w, fmt.Sprintf("failed to query corpus from %v: %v", mgr.name, err), http.StatusInternalServerError)
+			return
+		}
+		defer resp.Body.Close()
+		data, err := ioutil.ReadAll(resp.Body)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("failed to query corpus from %v: %v", mgr.name, err), http.StatusInternalServerError)
+			return
+		}
+		var hashes []string
+		err = json.Unmarshal(data, &hashes)
+		if err != nil || len(hashes) == 0 {
+			http.Error(w, fmt.Sprintf("failed to parse corpus from %v: %v", mgr.name, err), http.StatusInternalServerError)
+			return
+		}
+		for _, hash := range hashes {
+			corpus[hash] = true
+		}
+	}
+	orig := len(m.corpus.m)
+	m.corpus.minimize(corpus)
+	fmt.Printf("minimized: %v -> %v -> %v\n", orig, len(corpus), len(m.corpus.m))
+	for _, mgr := range m.managers {
+		mgr.input = 0
+	}
+}
+
+type UIData struct {
+	CorpusLen int
+	Managers  []UIManager
+}
+
+type UIManager struct {
+	Name string
+	Http string
+}
+
+var htmlTemplate = template.Must(template.New("").Parse(`
+<!doctype html>
+<html>
+<head>
+    <title>syzkaller master</title>
+</head>
+<body>
+Corpus: {{.CorpusLen}} <br>
+{{if .Managers}}
+	Managers:<br>
+	{{range $mgr := $.Managers}}
+		<a href='http://{{$mgr.Http}}'>{{$mgr.Name}}</a><br>
+	{{end}}
+{{else}}
+	No managers connected<br>
+{{end}}
+</body></html>
+`))
diff --git a/master/master.go b/master/master.go
new file mode 100644
index 000000000..c1a427732
--- /dev/null
+++ b/master/master.go
@@ -0,0 +1,170 @@
+// Copyright 2015 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"net"
+	"net/http"
+	"net/rpc"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/google/syzkaller/prog"
+	. "github.com/google/syzkaller/rpctype"
+)
+
+var (
+	flagWorkdir = flag.String("workdir", "", "dir with persistent artifacts")
+	flagAddr    = flag.String("addr", "", "RPC listen address to connect managers")
+	flagHTTP    = flag.String("http", "", "HTTP server listen address")
+	flagV       = flag.Int("v", 0, "verbosity")
+)
+
+// Master manages persistent fuzzer state (input corpus and crashers).
+type Master struct {
+	mu        sync.Mutex
+	managers  map[string]*Manager
+	corpus    *PersistentSet
+	crashers  *PersistentSet
+	startTime time.Time
+	lastInput time.Time
+}
+
+type Manager struct {
+	name  string
+	http  string
+	input int
+}
+
+func main() {
+	flag.Parse()
+	if *flagWorkdir == "" {
+		fatalf("-workdir is not set")
+	}
+	if *flagAddr == "" {
+		fatalf("-addr is not set")
+	}
+	if *flagHTTP == "" {
+		fatalf("-http is not set")
+	}
+	ln, err := net.Listen("tcp", *flagAddr)
+	if err != nil {
+		fatalf("failed to listen: %v", err)
+	}
+
+	m := &Master{}
+	m.managers = make(map[string]*Manager)
+	m.startTime = time.Now()
+	m.lastInput = time.Now()
+	logf(0, "loading corpus...")
+	m.corpus = newPersistentSet(filepath.Join(*flagWorkdir, "corpus"), func(data []byte) bool {
+		if _, err := prog.Deserialize(data); err != nil {
+			logf(0, "deleting broken program: %v\n%s", err, data)
+			return false
+		}
+		return true
+	})
+	m.crashers = newPersistentSet(filepath.Join(*flagWorkdir, "crashers"), nil)
+
+	http.HandleFunc("/", m.httpInfo)
+	http.HandleFunc("/minimize", m.httpMinimize)
+	go func() {
+		logf(0, "serving http on http://%v", *flagHTTP)
+		panic(http.ListenAndServe(*flagHTTP, nil))
+	}()
+
+	logf(0, "serving rpc on tcp://%v", *flagAddr)
+	s := rpc.NewServer()
+	s.Register(m)
+	go s.Accept(ln)
+
+	m.loop()
+}
+
+func (m *Master) loop() {
+	for range time.NewTicker(1 * time.Second).C {
+	}
+}
+
+// Connect attaches new manager to master.
+func (m *Master) Connect(a *MasterConnectArgs, r *MasterConnectRes) error {
+	logf(1, "connect from %v (http://%v)", a.Name, a.Http)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	mgr := &Manager{
+		name: a.Name,
+		http: a.Http,
+	}
+	m.managers[a.Name] = mgr
+	r.Http = *flagHTTP
+	return nil
+}
+
+// NewInput saves new interesting input on master.
+func (m *Master) NewInput(a *NewMasterInputArgs, r *int) error {
+	p, err := prog.Deserialize(a.Prog)
+	if err != nil {
+		logf(0, "bogus new input from %v: %v\n%s\n", a.Name, err, a.Prog)
+		return fmt.Errorf("the program is bogus: %v", err)
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if !m.corpus.add(a.Prog) {
+		return nil
+	}
+	m.lastInput = time.Now()
+	logf(1, "new input from %v: %s", a.Name, p)
+	return nil
+}
+
+type NewCrasherArgs struct {
+	Name        string
+	Text        []byte
+	Suppression []byte
+	Prog        []byte
+}
+
+// NewCrasher saves new crasher input on master.
+func (m *Master) NewCrasher(a *NewCrasherArgs, r *int) error {
+	logf(0, "new crasher from %v", a.Name)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	if !m.crashers.add(a.Text) {
+		return nil // Already have this.
+	}
+	return nil
+}
+
+func (m *Master) PollInputs(a *MasterPollArgs, r *MasterPollRes) error {
+	logf(2, "poll from %v", a.Name)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+
+	mgr := m.managers[a.Name]
+	if mgr == nil {
+		return fmt.Errorf("manager is not connected")
+	}
+	for i := 0; i < 100 && mgr.input < len(m.corpus.a); i++ {
+		r.Inputs = append(r.Inputs, m.corpus.a[mgr.input])
+		mgr.input++
+	}
+	return nil
+}
+
+func logf(v int, msg string, args ...interface{}) {
+	if *flagV >= v {
+		log.Printf(msg, args...)
+	}
+}
+
+func fatalf(msg string, args ...interface{}) {
+	log.Fatalf(msg, args...)
+}
diff --git a/master/persistent.go b/master/persistent.go
new file mode 100644
index 000000000..12f4bbdc8
--- /dev/null
+++ b/master/persistent.go
@@ -0,0 +1,129 @@
+// Copyright 2015 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+)
+
+type Sig [sha1.Size]byte
+
+// PersistentSet is a set of binary blobs with a persistent mirror on disk.
+type PersistentSet struct {
+	dir string
+	m   map[Sig][]byte
+	a   [][]byte
+}
+
+func hash(data []byte) Sig {
+	return Sig(sha1.Sum(data))
+}
+
+func newPersistentSet(dir string, verify func(data []byte) bool) *PersistentSet {
+	ps := &PersistentSet{
+		dir: dir,
+		m:   make(map[Sig][]byte),
+	}
+	os.MkdirAll(dir, 0770)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			log.Fatalf("error during dir walk: %v\n", err)
+		}
+		if info.IsDir() {
+			return nil
+		}
+		data, err := ioutil.ReadFile(path)
+		if err != nil {
+			log.Fatalf("error during file read: %v\n", err)
+			return nil
+		}
+		sig := hash(data)
+		if _, ok := ps.m[sig]; ok {
+			return nil
+		}
+		name := info.Name()
+		if len(data) == 0 {
+			// This can happen is master runs on machine-under-test,
+			// and it has crashed midway.
+			log.Printf("removing empty file %v", name)
+			os.Remove(path)
+			return nil
+		}
+		const hexLen = 2 * sha1.Size
+		if len(name) > hexLen+1 && isHexString(name[:hexLen]) && name[hexLen] == '.' {
+			return nil // description file
+		}
+		if len(name) != hexLen || !isHexString(name) {
+			log.Fatalf("unknown file in persistent dir %v: %v", dir, name)
+		}
+		if verify != nil && !verify(data) {
+			os.Remove(path)
+			return nil
+		}
+		if name != hex.EncodeToString(sig[:]) {
+			log.Printf("bad hash in persistent dir %v for file %v, expect %v", dir, name, hex.EncodeToString(sig[:]))
+			if err := ioutil.WriteFile(filepath.Join(ps.dir, hex.EncodeToString(sig[:])), data, 0660); err != nil {
+				log.Fatalf("failed to write file: %v", err)
+			}
+			os.Remove(path)
+		}
+		ps.m[sig] = data
+		ps.a = append(ps.a, data)
+		return nil
+	})
+	return ps
+}
+
+func isHexString(s string) bool {
+	for _, v := range []byte(s) {
+		if v >= '0' && v <= '9' || v >= 'a' && v <= 'f' {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func (ps *PersistentSet) add(data []byte) bool {
+	sig := hash(data)
+	if _, ok := ps.m[sig]; ok {
+		return false
+	}
+	data = append([]byte{}, data...)
+	ps.m[sig] = data
+	ps.a = append(ps.a, data)
+	fname := filepath.Join(ps.dir, hex.EncodeToString(sig[:]))
+	if err := ioutil.WriteFile(fname, data, 0660); err != nil {
+		log.Fatalf("failed to write file: %v", err)
+	}
+	return true
+}
+
+// addDescription creates a complementary to data file on disk.
+func (ps *PersistentSet) addDescription(data []byte, desc []byte, typ string) {
+	sig := hash(data)
+	fname := filepath.Join(ps.dir, fmt.Sprintf("%v.%v", hex.EncodeToString(sig[:]), typ))
+	if err := ioutil.WriteFile(fname, desc, 0660); err != nil {
+		log.Fatalf("failed to write file: %v", err)
+	}
+}
+
+func (ps *PersistentSet) minimize(set map[string]bool) {
+	ps.a = nil
+	for sig, data := range ps.m {
+		s := hex.EncodeToString(sig[:])
+		if set[s] {
+			ps.a = append(ps.a, data)
+		} else {
+			delete(ps.m, sig)
+			os.Remove(filepath.Join(ps.dir, s))
+		}
+	}
+}
author	Dmitry Vyukov <dvyukov@google.com>	2015-10-12 10:16:57 +0200
committer	Dmitry Vyukov <dvyukov@google.com>	2015-10-12 10:16:57 +0200
commit	874c5754bb22dbf77d6b600ff91f0f4f1fc5073a (patch)
tree	0075fbd088046ad5c86e6e972235701d68b3ce7c /master