syz-ci: add continuous integration system

8 files changed, 922 insertions, 1 deletions

diff --git a/Makefile b/Makefile
index 424ef7073..8f75d01e6 100644
--- a/Makefile
+++ b/Makefile
@@ -6,7 +6,7 @@ ifeq ($(NOSTATIC), 0)
 	STATIC_FLAG=-static
 endif
 
-.PHONY: all format tidy clean manager fuzzer executor execprog mutate prog2c stress extract generate repro db bin/syz-extract bin/syz-sysgen
+.PHONY: all format tidy clean manager fuzzer executor execprog ci mutate prog2c stress extract generate repro db bin/syz-extract bin/syz-sysgen
 
 all:
 	go install ./syz-manager ./syz-fuzzer
@@ -36,6 +36,9 @@ fuzzer:
 execprog:
 	go build $(GOFLAGS) -o ./bin/syz-execprog github.com/google/syzkaller/tools/syz-execprog
 
+ci:
+	go build $(GOFLAGS) -o ./bin/syz-ci github.com/google/syzkaller/syz-ci
+
 repro:
 	go build $(GOFLAGS) -o ./bin/syz-repro github.com/google/syzkaller/tools/syz-repro
 
diff --git a/docs/ci.md b/docs/ci.md
new file mode 100644
index 000000000..9301e5db9
--- /dev/null
+++ b/docs/ci.md
@@ -0,0 +1,5 @@
+# Continuous integration fuzzing
+
+(syz-ci)[syz-ci/] command provides support for continuous fuzzing with syzkaller.
+It runs several syz-manager's, polls and rebuilds images for managers and polls
+and rebuilds syzkaller binaries.
diff --git a/syz-ci/config_test.go b/syz-ci/config_test.go
new file mode 100644
index 000000000..4d33f11df
--- /dev/null
+++ b/syz-ci/config_test.go
@@ -0,0 +1,14 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"testing"
+)
+
+func TestLoadConfig(t *testing.T) {
+	if _, err := loadConfig("testdata/example.cfg"); err != nil {
+		t.Fatalf("failed to load: %v", err)
+	}
+}
diff --git a/syz-ci/manager.go b/syz-ci/manager.go
new file mode 100644
index 000000000..17b09a72a
--- /dev/null
+++ b/syz-ci/manager.go
@@ -0,0 +1,324 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/google/syzkaller/dashboard"
+	"github.com/google/syzkaller/pkg/config"
+	"github.com/google/syzkaller/pkg/fileutil"
+	"github.com/google/syzkaller/pkg/git"
+	"github.com/google/syzkaller/pkg/hash"
+	"github.com/google/syzkaller/pkg/kernel"
+	. "github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/syz-manager/mgrconfig"
+)
+
+// This is especially slightly longer than syzkaller rebuild period.
+// If we set kernelRebuildPeriod = syzkallerRebuildPeriod and both are changed
+// during that period (or around that period), we can rebuild kernel, restart
+// manager and then instantly shutdown everything for syzkaller update.
+// Instead we rebuild syzkaller, restart and then rebuild kernel.
+const kernelRebuildPeriod = syzkallerRebuildPeriod + time.Hour
+
+// List of required files in kernel build (contents of latest/current dirs).
+var imageFiles = []string{
+	"kernel.tag",    // git hash of kernel checkout
+	"compiler.tag",  // compiler identity string (e.g. "gcc 7.1.1")
+	"kernel.config", // kernel config used for the build (identified with SHA1 hash of contents)
+	"tag",           // SHA1 hash of the previous 3 tags (this is what uniquely identifies the build)
+	"image",         // kernel image
+	"key",           // root ssh key for the image
+	"obj/vmlinux",   // vmlinux with debug info
+}
+
+// Manager represents a single syz-manager instance.
+// Handles kernel polling, image rebuild and manager process management.
+// As syzkaller builder, it maintains 2 builds:
+//  - latest: latest known good kernel build
+//  - current: currently used kernel build
+type Manager struct {
+	name        string
+	workDir     string
+	kernelDir   string
+	currentDir  string
+	latestDir   string
+	compilerTag string
+	configTag   string
+	cfg         *Config
+	mgrcfg      *ManagerConfig
+	cmd         *ManagerCmd
+	dash        *dashboard.Dashboard
+	stop        chan struct{}
+}
+
+func createManager(dash *dashboard.Dashboard, cfg *Config, mgrcfg *ManagerConfig, stop chan struct{}) *Manager {
+	dir := osutil.Abs(filepath.Join("managers", mgrcfg.Name))
+	if err := os.MkdirAll(dir, osutil.DefaultDirPerm); err != nil {
+		Fatal(err)
+	}
+
+	// Assume compiler and config don't change underneath us.
+	compilerTag, err := kernel.CompilerIdentity(mgrcfg.Compiler)
+	if err != nil {
+		Fatal(err)
+	}
+	configData, err := ioutil.ReadFile(mgrcfg.Kernel_Config)
+	if err != nil {
+		Fatal(err)
+	}
+
+	mgr := &Manager{
+		name:        mgrcfg.Name,
+		workDir:     filepath.Join(dir, "workdir"),
+		kernelDir:   filepath.Join(dir, "kernel"),
+		currentDir:  filepath.Join(dir, "current"),
+		latestDir:   filepath.Join(dir, "latest"),
+		compilerTag: compilerTag,
+		configTag:   hash.String(configData),
+		cfg:         cfg,
+		mgrcfg:      mgrcfg,
+		dash:        dash,
+		stop:        stop,
+	}
+	os.RemoveAll(mgr.currentDir)
+	return mgr
+}
+
+// Gates kernel builds.
+// Kernel builds take whole machine, so we don't run more than one at a time.
+// Also current image build script uses some global resources (/dev/nbd0) and can't run in parallel.
+var kernelBuildSem = make(chan struct{}, 1)
+
+func (mgr *Manager) loop() {
+	lastCommit := ""
+	nextBuildTime := time.Now()
+	var managerRestartTime time.Time
+	latestTime, latestKernelTag, latestCompilerTag, latestConfigTag := mgr.checkLatest()
+	if time.Since(latestTime) < kernelRebuildPeriod/2 {
+		// If we have a reasonably fresh build,
+		// start manager straight away and don't rebuild kernel for a while.
+		Logf(0, "%v: using latest image built on %v", mgr.name, latestKernelTag)
+		managerRestartTime = latestTime
+		nextBuildTime = time.Now().Add(kernelRebuildPeriod)
+		mgr.restartManager()
+	} else {
+		Logf(0, "%v: latest image is on %v", mgr.name, formatTag(latestKernelTag))
+	}
+
+	ticker := time.NewTicker(kernelRebuildPeriod)
+	defer ticker.Stop()
+
+loop:
+	for {
+		if time.Since(nextBuildTime) >= 0 {
+			rebuildAfter := buildRetryPeriod
+			commit, err := git.Poll(mgr.kernelDir, mgr.mgrcfg.Repo, mgr.mgrcfg.Branch)
+			if err != nil {
+				Logf(0, "%v: failed to poll: %v", mgr.name, err)
+			} else {
+				Logf(0, "%v: poll: %v", mgr.name, commit)
+				if commit != lastCommit &&
+					(commit != latestKernelTag ||
+						mgr.compilerTag != latestCompilerTag ||
+						mgr.configTag != latestConfigTag) {
+					lastCommit = commit
+					select {
+					case kernelBuildSem <- struct{}{}:
+						Logf(0, "%v: building kernel...", mgr.name)
+						if err := mgr.build(); err != nil {
+							Logf(0, "%v: %v", mgr.name, err)
+						} else {
+							Logf(0, "%v: build successful, [re]starting manager", mgr.name)
+							rebuildAfter = kernelRebuildPeriod
+							latestTime, latestKernelTag, latestCompilerTag, latestConfigTag = mgr.checkLatest()
+						}
+						<-kernelBuildSem
+					case <-mgr.stop:
+						break loop
+					}
+				}
+			}
+			nextBuildTime = time.Now().Add(rebuildAfter)
+		}
+
+		select {
+		case <-mgr.stop:
+			break loop
+		default:
+		}
+
+		if managerRestartTime != latestTime {
+			managerRestartTime = latestTime
+			mgr.restartManager()
+		}
+
+		select {
+		case <-ticker.C:
+		case <-mgr.stop:
+			break loop
+		}
+	}
+
+	if mgr.cmd != nil {
+		mgr.cmd.Close()
+		mgr.cmd = nil
+	}
+	Logf(0, "%v: stopped", mgr.name)
+}
+
+// checkLatest checks if we have a good working latest build
+// and returns its kernel/compiler/config tags.
+// If the build is missing/broken, zero mod time is returned.
+func (mgr *Manager) checkLatest() (mod time.Time, kernelTag, compilerTag, configTag string) {
+	if !osutil.FilesExist(mgr.latestDir, imageFiles) {
+		return
+	}
+	configData, err := ioutil.ReadFile(filepath.Join(mgr.latestDir, "kernel.config"))
+	if err != nil {
+		return
+	}
+	configTag = hash.String(configData)
+	compilerTag, _ = readTag(filepath.Join(mgr.latestDir, "compiler.tag"))
+	if compilerTag == "" {
+		return
+	}
+	kernelTag, mod = readTag(filepath.Join(mgr.latestDir, "kernel.tag"))
+	return
+}
+
+func (mgr *Manager) build() error {
+	kernelCommit, err := git.HeadCommit(mgr.kernelDir)
+	if err != nil {
+		return fmt.Errorf("failed to get git HEAD commit: %v", err)
+	}
+	if err := kernel.Build(mgr.kernelDir, mgr.mgrcfg.Compiler, mgr.mgrcfg.Kernel_Config); err != nil {
+		return fmt.Errorf("kernel build failed: %v", err)
+	}
+
+	// We first form the whole image in tmp dir and then rename it to latest.
+	tmpDir := mgr.latestDir + ".tmp"
+	if err := os.RemoveAll(tmpDir); err != nil {
+		return fmt.Errorf("failed to remove tmp dir: %v", err)
+	}
+	if err := os.MkdirAll(tmpDir, osutil.DefaultDirPerm); err != nil {
+		return fmt.Errorf("failed to create tmp dir: %v", err)
+	}
+
+	image := filepath.Join(tmpDir, "image")
+	key := filepath.Join(tmpDir, "key")
+	if err := kernel.CreateImage(mgr.kernelDir, mgr.mgrcfg.Userspace, image, key); err != nil {
+		return fmt.Errorf("image build failed: %v", err)
+	}
+	// TODO(dvyukov): test that the image is good (boots and we can ssh into it).
+
+	vmlinux := filepath.Join(mgr.kernelDir, "vmlinux")
+	objDir := filepath.Join(tmpDir, "obj")
+	os.MkdirAll(objDir, osutil.DefaultDirPerm)
+	if err := os.Rename(vmlinux, filepath.Join(objDir, "vmlinux")); err != nil {
+		return fmt.Errorf("failed to rename vmlinux file: %v", err)
+	}
+	kernelConfig := filepath.Join(tmpDir, "kernel.config")
+	if err := fileutil.CopyFile(mgr.mgrcfg.Kernel_Config, kernelConfig); err != nil {
+		return err
+	}
+
+	writeTagFile := func(filename, data string) error {
+		f := filepath.Join(tmpDir, filename)
+		if err := ioutil.WriteFile(f, []byte(data), osutil.DefaultFilePerm); err != nil {
+			return fmt.Errorf("failed to write tag file: %v", err)
+		}
+		return nil
+	}
+	if err := writeTagFile("kernel.tag", kernelCommit); err != nil {
+		return err
+	}
+	if err := writeTagFile("compiler.tag", mgr.compilerTag); err != nil {
+		return err
+	}
+
+	var tag []byte
+	tag = append(tag, kernelCommit...)
+	tag = append(tag, mgr.configTag...)
+	tag = append(tag, mgr.compilerTag...)
+	if err := writeTagFile("tag", hash.String(tag)); err != nil {
+		return err
+	}
+
+	// Now try to replace latest with our tmp dir as atomically as we can get on Linux.
+	if err := os.RemoveAll(mgr.latestDir); err != nil {
+		return fmt.Errorf("failed to remove latest dir: %v", err)
+	}
+	return os.Rename(tmpDir, mgr.latestDir)
+}
+
+func (mgr *Manager) restartManager() {
+	if !osutil.FilesExist(mgr.latestDir, imageFiles) {
+		Logf(0, "%v: can't start manager, image files missing", mgr.name)
+		return
+	}
+	if mgr.cmd != nil {
+		mgr.cmd.Close()
+		mgr.cmd = nil
+	}
+	if err := osutil.LinkFiles(mgr.latestDir, mgr.currentDir, imageFiles); err != nil {
+		Logf(0, "%v: failed to create current image dir: %v", mgr.name, err)
+		return
+	}
+	cfgFile, err := mgr.writeConfig()
+	if err != nil {
+		Logf(0, "%v: failed to create manager config: %v", mgr.name, err)
+		return
+	}
+	bin := filepath.FromSlash("syzkaller/current/bin/syz-manager")
+	logFile := filepath.Join(mgr.currentDir, "manager.log")
+	mgr.cmd = NewManagerCmd(mgr.name, logFile, bin, "-config", cfgFile)
+}
+
+func (mgr *Manager) writeConfig() (string, error) {
+	mgrcfg := &mgrconfig.Config{
+		Cover:     true,
+		Reproduce: true,
+		Sandbox:   "setuid",
+		Rpc:       "localhost:0",
+		Procs:     1,
+	}
+	err := config.LoadData(mgr.mgrcfg.Manager_Config, mgrcfg)
+	if err != nil {
+		return "", err
+	}
+	current := mgr.currentDir
+	// TODO(dvyukov): we use kernel.tag because dashboard does not support build info yet.
+	// Later we should use tag file because it identifies kernel+compiler+config.
+	tag, err := ioutil.ReadFile(filepath.Join(current, "kernel.tag"))
+	if err != nil {
+		return "", fmt.Errorf("failed to read tag file: %v", err)
+	}
+	mgrcfg.Name = mgr.cfg.Name + "-" + mgr.name
+	mgrcfg.Hub_Addr = mgr.cfg.Hub_Addr
+	mgrcfg.Hub_Key = mgr.cfg.Hub_Key
+	mgrcfg.Dashboard_Addr = mgr.cfg.Dashboard_Addr
+	mgrcfg.Dashboard_Key = mgr.cfg.Dashboard_Key
+	mgrcfg.Workdir = mgr.workDir
+	mgrcfg.Vmlinux = filepath.Join(current, "obj", "vmlinux")
+	mgrcfg.Tag = string(tag)
+	mgrcfg.Syzkaller = filepath.FromSlash("syzkaller/current")
+	mgrcfg.Image = filepath.Join(current, "image")
+	mgrcfg.Sshkey = filepath.Join(current, "key")
+
+	configFile := filepath.Join(current, "manager.cfg")
+	if err := config.SaveFile(configFile, mgrcfg); err != nil {
+		return "", err
+	}
+	if _, _, err := mgrconfig.LoadFile(configFile); err != nil {
+		return "", err
+	}
+	return configFile, nil
+}
diff --git a/syz-ci/managercmd.go b/syz-ci/managercmd.go
new file mode 100644
index 000000000..2f495b73f
--- /dev/null
+++ b/syz-ci/managercmd.go
@@ -0,0 +1,120 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"os"
+	"os/exec"
+	"syscall"
+	"time"
+
+	. "github.com/google/syzkaller/pkg/log"
+)
+
+// ManagerCmd encapsulates a single instance of syz-manager process.
+// It automatically restarts syz-manager if it exits unexpectedly,
+// and supports graceful shutdown via SIGINT.
+type ManagerCmd struct {
+	name    string
+	log     string
+	bin     string
+	args    []string
+	closing chan bool
+}
+
+// NewManagerCmd starts new syz-manager process.
+// name - name for logging.
+// log - manager log file with stdout/stderr.
+// bin/args - process binary/args.
+func NewManagerCmd(name, log, bin string, args ...string) *ManagerCmd {
+	mc := &ManagerCmd{
+		name:    name,
+		log:     log,
+		bin:     bin,
+		args:    args,
+		closing: make(chan bool),
+	}
+	go mc.loop()
+	return mc
+}
+
+// Close gracefully shutdowns the process and waits for its termination.
+func (mc *ManagerCmd) Close() {
+	mc.closing <- true
+	<-mc.closing
+}
+
+func (mc *ManagerCmd) loop() {
+	const (
+		restartPeriod    = time.Minute // don't restart crashing manager more frequently than that
+		interruptTimeout = time.Minute // give manager that much time to react to SIGINT
+	)
+	var (
+		cmd         *exec.Cmd
+		started     time.Time
+		interrupted time.Time
+		stopped     = make(chan error, 1)
+		closing     = mc.closing
+		ticker1     = time.NewTicker(restartPeriod)
+		ticker2     = time.NewTicker(interruptTimeout)
+	)
+	defer func() {
+		ticker1.Stop()
+		ticker2.Stop()
+	}()
+	for closing != nil || cmd != nil {
+		if cmd == nil {
+			// cmd is not running
+			// don't restart too frequently (in case it instantly exits with an error)
+			if time.Since(started) > restartPeriod {
+				started = time.Now()
+				logfile, err := os.Create(mc.log)
+				if err != nil {
+					Logf(1, "%v: failed to create manager log: %v", mc.name, err)
+				} else {
+					cmd = exec.Command(mc.bin, mc.args...)
+					cmd.Stdout = logfile
+					cmd.Stderr = logfile
+					err := cmd.Start()
+					logfile.Close()
+					if err != nil {
+						Logf(1, "%v: failed to start manager: %v", mc.name, err)
+						cmd = nil
+					} else {
+						Logf(1, "%v: started manager", mc.name)
+						go func() {
+							stopped <- cmd.Wait()
+						}()
+					}
+				}
+			}
+		} else {
+			// cmd is running
+			if closing == nil && time.Since(interrupted) > interruptTimeout {
+				Logf(1, "%v: killing manager", mc.name)
+				cmd.Process.Kill()
+				interrupted = time.Now()
+			}
+		}
+
+		select {
+		case <-closing:
+			closing = nil
+			if cmd != nil {
+				Logf(1, "%v: stopping manager", mc.name)
+				cmd.Process.Signal(syscall.SIGINT)
+				interrupted = time.Now()
+			}
+		case err := <-stopped:
+			if cmd == nil {
+				panic("spurious stop signal")
+			}
+			cmd = nil
+			Logf(1, "%v: manager exited with %v", mc.name, err)
+		case <-ticker1.C:
+		case <-ticker2.C:
+		}
+	}
+	close(mc.closing)
+}
diff --git a/syz-ci/syz-ci.go b/syz-ci/syz-ci.go
new file mode 100644
index 000000000..4f9dc256c
--- /dev/null
+++ b/syz-ci/syz-ci.go
@@ -0,0 +1,177 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// syz-ci is a continuous fuzzing system for syzkaller.
+// It runs several syz-manager's, polls and rebuilds images for managers
+// and polls and rebuilds syzkaller binaries.
+// For usage instructions see: docs/ci.md
+package main
+
+// Implementation details:
+//
+// 2 main components:
+//  - SyzUpdater: handles syzkaller updates
+//  - Manager: handles kernel build and syz-manager process (one per manager)
+// Both operate in a similar way and keep 2 builds:
+//  - latest: latest known good build (i.e. we tested it)
+//    preserved across restarts/reboots, i.e. we can start fuzzing even when
+//    current syzkaller/kernel git head is broken, or git is down, or anything else
+//  - current: currently used build (a copy of one of the latest builds)
+// Other important points:
+//  - syz-ci is always built on the same revision as the rest of syzkaller binaries,
+//    this allows us to handle e.g. changes in manager config format.
+//  - consequently, syzkaller binaries are never updated on-the-fly,
+//    instead we re-exec and then update
+//  - we understand when the latest build is fresh even after reboot,
+//    i.e. we store enough information to identify it (git hash, compiler identity, etc),
+//    so we don't rebuild unnecessary (kernel builds take time)
+//  - we generally avoid crashing the process and handle all errors gracefully
+//    (this is a continuous system), except for some severe/user errors during start
+//    (e.g. bad config file, or can't create necessary dirs)
+//
+// Directory/file structure:
+// syz-ci			: current executable
+// syz-ci.tag			: tag of the current executable (syzkaller git hash)
+// syzkaller/
+//	latest/			: latest good syzkaller build
+//	current/		: syzkaller build currently in use
+// managers/
+//	manager1/		: one dir per manager
+//		kernel/		: kernel checkout
+//		workdir/	: manager workdir (never deleted)
+//		latest/		: latest good kernel image build
+//		current/	: kernel image currently in use
+//
+// Current executable, syzkaller and kernel builds are marked with tag files.
+// Tag files uniquely identify the build (git hash, compiler identity, kernel config, etc).
+// For tag files both contents and modification time are important,
+// modification time allows us to understand if we need to rebuild after a restart.
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/google/syzkaller/dashboard"
+	"github.com/google/syzkaller/pkg/config"
+	. "github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/syz-manager/mgrconfig"
+)
+
+var flagConfig = flag.String("config", "", "config file")
+
+type Config struct {
+	Name             string
+	Http             string
+	Dashboard_Addr   string
+	Dashboard_Key    string
+	Hub_Addr         string
+	Hub_Key          string
+	Goroot           string
+	Syzkaller_Repo   string
+	Syzkaller_Branch string
+	Managers         []*ManagerConfig
+}
+
+type ManagerConfig struct {
+	Name           string
+	Repo           string
+	Branch         string
+	Compiler       string
+	Userspace      string
+	Kernel_Config  string
+	Manager_Config json.RawMessage
+}
+
+func main() {
+	flag.Parse()
+	EnableLogCaching(1000, 1<<20)
+	cfg, err := loadConfig(*flagConfig)
+	if err != nil {
+		Fatalf("failed to load config: %v", err)
+	}
+
+	shutdownPending := make(chan struct{})
+	osutil.HandleInterrupts(shutdownPending)
+
+	updater := NewSyzUpdater(cfg)
+	updater.UpdateOnStart(shutdownPending)
+	updatePending := make(chan struct{})
+	go func() {
+		updater.WaitForUpdate()
+		close(updatePending)
+	}()
+
+	stop := make(chan struct{})
+	go func() {
+		select {
+		case <-shutdownPending:
+		case <-updatePending:
+		}
+		close(stop)
+	}()
+
+	var dash *dashboard.Dashboard
+	if cfg.Dashboard_Addr != "" {
+		dash, err = dashboard.New(cfg.Name, cfg.Dashboard_Addr, cfg.Dashboard_Key)
+		if err != nil {
+			Fatalf("failed to create dashboard client: %v", err)
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(len(cfg.Managers))
+	managers := make([]*Manager, len(cfg.Managers))
+	for i, mgrcfg := range cfg.Managers {
+		managers[i] = createManager(dash, cfg, mgrcfg, stop)
+	}
+	for _, mgr := range managers {
+		mgr := mgr
+		go func() {
+			defer wg.Done()
+			mgr.loop()
+		}()
+	}
+
+	<-stop
+	wg.Wait()
+
+	select {
+	case <-shutdownPending:
+	case <-updatePending:
+		updater.UpdateAndRestart()
+	}
+}
+
+func loadConfig(filename string) (*Config, error) {
+	cfg := &Config{
+		Syzkaller_Repo:   "https://github.com/google/syzkaller.git",
+		Syzkaller_Branch: "master",
+		Goroot:           os.Getenv("GOROOT"),
+	}
+	if err := config.LoadFile(filename, cfg); err != nil {
+		return nil, err
+	}
+	if cfg.Name == "" {
+		return nil, fmt.Errorf("param 'name' is empty")
+	}
+	if cfg.Http == "" {
+		return nil, fmt.Errorf("param 'http' is empty")
+	}
+	if len(cfg.Managers) == 0 {
+		return nil, fmt.Errorf("no managers specified")
+	}
+	for i, mgr := range cfg.Managers {
+		if mgr.Name == "" {
+			return nil, fmt.Errorf("param 'managers[%v].name' is empty", i)
+		}
+		mgrcfg := new(mgrconfig.Config)
+		if err := config.LoadData(mgr.Manager_Config, mgrcfg); err != nil {
+			return nil, fmt.Errorf("manager %v: %v", mgr.Name, err)
+		}
+	}
+	return cfg, nil
+}
diff --git a/syz-ci/syzupdater.go b/syz-ci/syzupdater.go
new file mode 100644
index 000000000..ec8c09e40
--- /dev/null
+++ b/syz-ci/syzupdater.go
@@ -0,0 +1,229 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/google/syzkaller/pkg/fileutil"
+	"github.com/google/syzkaller/pkg/git"
+	. "github.com/google/syzkaller/pkg/log"
+	"github.com/google/syzkaller/pkg/osutil"
+)
+
+const (
+	syzkallerRebuildPeriod = 12 * time.Hour
+	buildRetryPeriod       = 15 * time.Minute // used for both syzkaller and kernel
+)
+
+// List of required files in syzkaller build (contents of latest/current dirs).
+var syzFiles = []string{
+	"tag",        // contains syzkaller repo git hash
+	"bin/syz-ci", // these are just copied from syzkaller dir
+	"bin/syz-manager",
+	"bin/syz-fuzzer",
+	"bin/syz-executor",
+	"bin/syz-execprog",
+}
+
+// SyzUpdater handles everything related to syzkaller updates.
+// As kernel builder, it maintains 2 builds:
+//  - latest: latest known good syzkaller build
+//  - current: currently used syzkaller build
+// Additionally it updates and restarts the current executable as necessary.
+// Current executable is always built on the same revision as the rest of syzkaller binaries.
+type SyzUpdater struct {
+	exe          string
+	repo         string
+	branch       string
+	syzkallerDir string
+	latestDir    string
+	currentDir   string
+}
+
+func NewSyzUpdater(cfg *Config) *SyzUpdater {
+	wd, err := os.Getwd()
+	if err != nil {
+		Fatalf("failed to get wd: %v", err)
+	}
+	bin := os.Args[0]
+	if !filepath.IsAbs(bin) {
+		bin = filepath.Join(wd, bin)
+	}
+	bin = filepath.Clean(bin)
+	exe := filepath.Base(bin)
+	if wd != filepath.Dir(bin) {
+		Fatalf("%v executable must be in cwd (it will be overwritten on update)", exe)
+	}
+
+	gopath := filepath.Join(wd, "gopath")
+	os.Setenv("GOPATH", gopath)
+	os.Setenv("GOROOT", cfg.Goroot)
+	os.Setenv("PATH", filepath.Join(cfg.Goroot, "bin")+
+		string(filepath.ListSeparator)+os.Getenv("PATH"))
+	syzkallerDir := filepath.Join(gopath, "src", "github.com", "google", "syzkaller")
+	os.MkdirAll(syzkallerDir, osutil.DefaultDirPerm)
+
+	return &SyzUpdater{
+		exe:          exe,
+		repo:         cfg.Syzkaller_Repo,
+		branch:       cfg.Syzkaller_Branch,
+		syzkallerDir: syzkallerDir,
+		latestDir:    filepath.Join("syzkaller", "latest"),
+		currentDir:   filepath.Join("syzkaller", "current"),
+	}
+}
+
+// UpdateOnStart does 3 things:
+//  - ensures that the current executable is fresh
+//  - ensures that we have a working syzkaller build in current
+func (upd *SyzUpdater) UpdateOnStart(shutdown chan struct{}) {
+	os.RemoveAll(upd.currentDir)
+	exeTag, exeMod := readTag(upd.exe + ".tag")
+	latestTag := upd.checkLatest()
+	if exeTag == latestTag && time.Since(exeMod) < syzkallerRebuildPeriod/2 {
+		// Have a freash up-to-date build, probably just restarted.
+		Logf(0, "current executable is up-to-date (%v)", exeTag)
+		if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, syzFiles); err != nil {
+			Fatal(err)
+		}
+		return
+	}
+	Logf(0, "current executable is on %v", formatTag(exeTag))
+	Logf(0, "latest syzkaller build is on %v", formatTag(latestTag))
+
+	// No syzkaller build or executable is stale.
+	lastCommit := ""
+	for {
+		lastCommit = upd.pollAndBuild(lastCommit)
+		latestTag := upd.checkLatest()
+		if latestTag != "" {
+			// The build was successful or we had the latest build from previous runs.
+			// Either way, use the latest build.
+			Logf(0, "using syzkaller built on %v", latestTag)
+			if err := osutil.LinkFiles(upd.latestDir, upd.currentDir, syzFiles); err != nil {
+				Fatal(err)
+			}
+			if exeTag != latestTag {
+				upd.UpdateAndRestart()
+			}
+			return
+		}
+
+		// No good build at all, try again later.
+		Logf(0, "retrying in %v", buildRetryPeriod)
+		select {
+		case <-time.After(buildRetryPeriod):
+		case <-shutdown:
+			os.Exit(0)
+		}
+	}
+}
+
+// WaitForUpdate polls and rebuilds syzkaller.
+// Returns when we have a new good build in latest.
+func (upd *SyzUpdater) WaitForUpdate() {
+	time.Sleep(syzkallerRebuildPeriod)
+	latestTag := upd.checkLatest()
+	lastCommit := latestTag
+	for {
+		lastCommit = upd.pollAndBuild(lastCommit)
+		if latestTag != upd.checkLatest() {
+			break
+		}
+		time.Sleep(buildRetryPeriod)
+	}
+	Logf(0, "syzkaller: update available, restarting")
+}
+
+// UpdateAndRestart updates and restarts the current executable.
+// Does not return.
+func (upd *SyzUpdater) UpdateAndRestart() {
+	Logf(0, "restarting executable for update")
+	latestBin := filepath.Join(upd.latestDir, "bin", upd.exe)
+	latestTag := filepath.Join(upd.latestDir, "tag")
+	if err := fileutil.CopyFile(latestBin, upd.exe); err != nil {
+		Fatal(err)
+	}
+	if err := fileutil.CopyFile(latestTag, upd.exe+".tag"); err != nil {
+		Fatal(err)
+	}
+	if err := syscall.Exec(upd.exe, os.Args, os.Environ()); err != nil {
+		Fatal(err)
+	}
+	Fatalf("not reachable")
+}
+
+func (upd *SyzUpdater) pollAndBuild(lastCommit string) string {
+	commit, err := git.Poll(upd.syzkallerDir, upd.repo, upd.branch)
+	if err != nil {
+		Logf(0, "syzkaller: failed to poll: %v", err)
+	} else {
+		Logf(0, "syzkaller: poll: %v", commit)
+		if lastCommit != commit {
+			Logf(0, "syzkaller: building ...")
+			lastCommit = commit
+			if err := upd.build(); err != nil {
+				Logf(0, "syzkaller: %v", err)
+			}
+		}
+	}
+	return lastCommit
+}
+
+func (upd *SyzUpdater) build() error {
+	commit, err := git.HeadCommit(upd.syzkallerDir)
+	if err != nil {
+		return fmt.Errorf("failed to get HEAD commit: %v", err)
+	}
+	if _, err := osutil.RunCmd(time.Hour, upd.syzkallerDir, "make", "all", "ci"); err != nil {
+		return fmt.Errorf("build failed: %v", err)
+	}
+	if _, err := osutil.RunCmd(time.Hour, upd.syzkallerDir, "go", "test", "-short", "./..."); err != nil {
+		return fmt.Errorf("tests failed: %v", err)
+	}
+	tagFile := filepath.Join(upd.syzkallerDir, "tag")
+	if err := ioutil.WriteFile(tagFile, []byte(commit), osutil.DefaultFilePerm); err != nil {
+		return fmt.Errorf("filed to write tag file: %v", err)
+	}
+	if err := osutil.CopyFiles(upd.syzkallerDir, upd.latestDir, syzFiles); err != nil {
+		return fmt.Errorf("filed to copy syzkaller: %v", err)
+	}
+	return nil
+}
+
+// checkLatest returns tag of the latest build,
+// or an empty string if latest build is missing/broken.
+func (upd *SyzUpdater) checkLatest() string {
+	if !osutil.FilesExist(upd.latestDir, syzFiles) {
+		return ""
+	}
+	tag, _ := readTag(filepath.Join(upd.latestDir, "tag"))
+	return tag
+}
+
+func readTag(file string) (tag string, mod time.Time) {
+	data, _ := ioutil.ReadFile(file)
+	tag = string(data)
+	if st, err := os.Stat(file); err == nil {
+		mod = st.ModTime()
+	}
+	if tag == "" || mod.IsZero() {
+		tag = ""
+		mod = time.Time{}
+	}
+	return
+}
+
+func formatTag(tag string) string {
+	if tag == "" {
+		return "unknown"
+	}
+	return tag
+}
diff --git a/syz-ci/testdata/example.cfg b/syz-ci/testdata/example.cfg
new file mode 100644
index 000000000..dcbb4832d
--- /dev/null
+++ b/syz-ci/testdata/example.cfg
@@ -0,0 +1,49 @@
+{
+	"name": "ci",
+	"http": ":80",
+	"dashboard_addr": "1.2.3.4:1234",
+	"dashboard_key": "111",
+	"hub_addr": "2.3.4.5:2345",
+	"hub_key": "222",
+	"goroot": "/syzkaller/goroot",
+	"managers": [
+		{
+			"name": "upstream-kasan",
+			"repo": "git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git",
+			"branch": "master",
+			"compiler": "/syzkaller/gcc/bin/gcc",
+			"userspace": "/syzkaller/wheezy",
+			"kernel_config": "/syzkaller/kasan.config",
+			"manager_config": {
+				"http": ":10000",
+				"type": "gce",
+				"sandbox": "namespace",
+				"procs": 8,
+				"vm": {
+					"count": 10,
+					"machine_type": "n1-standard-2",
+					"gcs_bucket": "syzkaller"
+				}
+			}
+		},
+		{
+			"name": "linux-next-kasan",
+			"repo": "git://git.kernel.org/pub/scm/linux/kernel/git/next/linux-next.git",
+			"branch": "master",
+			"compiler": "/syzkaller/gcc/bin/gcc",
+			"userspace": "/syzkaller/wheezy",
+			"kernel_config": "/syzkaller/kasan.config",
+			"manager_config": {
+				"http": ":10001",
+				"type": "gce",
+				"sandbox": "namespace",
+				"procs": 8,
+				"vm": {
+					"count": 10,
+					"machine_type": "n1-standard-2",
+					"gcs_bucket": "syzkaller"
+				}
+			}
+		}
+	]
+}