dashboard/app: test fix patches by request

Add initial support for testing fix patches.
A user on mailing list can request patch testing with:

and attach a patch.

11 files changed, 926 insertions, 0 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 304edcd0a..d8f1fde1f 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -32,6 +32,8 @@ var apiHandlers = map[string]APIHandler{
 	"log_error":           apiLogError,
 	"upload_build":        apiUploadBuild,
 	"builder_poll":        apiBuilderPoll,
+	"job_poll":            apiJobPoll,
+	"job_done":            apiJobDone,
 	"report_crash":        apiReportCrash,
 	"report_failed_repro": apiReportFailedRepro,
 	"need_repro":          apiNeedRepro,
@@ -157,6 +159,26 @@ loop:
 	return resp, nil
 }
 
+func apiJobPoll(c context.Context, ns string, r *http.Request) (interface{}, error) {
+	req := new(dashapi.JobPollReq)
+	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
+	}
+	if len(req.Managers) == 0 {
+		return nil, fmt.Errorf("no managers")
+	}
+	return pollPendingJobs(c, req.Managers)
+}
+
+func apiJobDone(c context.Context, ns string, r *http.Request) (interface{}, error) {
+	req := new(dashapi.JobDoneReq)
+	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
+	}
+	err := doneJob(c, req)
+	return nil, err
+}
+
 func apiUploadBuild(c context.Context, ns string, r *http.Request) (interface{}, error) {
 	req := new(dashapi.Build)
 	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index 60fb2120c..82b3c3003 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -94,6 +94,42 @@ type ReportingStateEntry struct {
 	Date int
 }
 
+// Job represent a single patch testing job for syz-ci.
+// Later we may want to extend this to other types of jobs (hense the generic name):
+//   - test of a committed fix
+//   - reproduce crash
+//   - test that crash still happens on HEAD
+//   - crash bisect
+// Job has Bug as parent entity.
+type Job struct {
+	Created   time.Time
+	User      string
+	Reporting string
+	ExtID     string // email Message-ID
+	Namespace string
+	Manager   string
+	BugTitle  string
+	CrashID   int64
+
+	// Provided by user:
+	KernelRepo   string
+	KernelBranch string
+	Patch        int64 // reference to Patch text entity
+
+	Attempts int // number of times we tried to execute this job
+	Started  time.Time
+	Finished time.Time // if set, job is finished
+
+	// Result of execution:
+	CrashTitle  string // if empty, we did not hit crash during testing
+	CrashLog    int64  // reference to CrashLog text entity
+	CrashReport int64  // reference to CrashReport text entity
+	BuildID     string
+	Error       int64 // reference to Error text entity, if set job failed
+
+	Reported bool // have we reported result back to user?
+}
+
 // Text holds text blobs (crash logs, reports, reproducers, etc).
 type Text struct {
 	Namespace string
diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml
index a981c90d6..2946e29f7 100644
--- a/dashboard/app/index.yaml
+++ b/dashboard/app/index.yaml
@@ -39,3 +39,14 @@ indexes:
     direction: desc
   - name: Time
     direction: desc
+
+- kind: Job
+  properties:
+  - name: Finished
+  - name: Attempts
+  - name: Created
+
+- kind: Job
+  properties:
+  - name: Reported
+  - name: Finished
diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go
new file mode 100644
index 000000000..ee0d1fc8b
--- /dev/null
+++ b/dashboard/app/jobs.go
@@ -0,0 +1,395 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package dash
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/syzkaller/dashboard/dashapi"
+	"github.com/google/syzkaller/pkg/email"
+	"github.com/google/syzkaller/pkg/git"
+	"golang.org/x/net/context"
+	"google.golang.org/appengine/datastore"
+	"google.golang.org/appengine/log"
+)
+
+// handleTestRequest added new job to datastore.
+// Returns empty string if job added successfully, or reason why it wasn't added.
+func handleTestRequest(c context.Context, bugID, user, extID, patch, repo, branch string) string {
+	log.Infof(c, "test request: bug=%q user=%q extID=%q patch=%v, repo=%q branch=%q",
+		bugID, user, extID, len(patch), repo, branch)
+	reply, err := addTestJob(c, bugID, user, extID, patch, repo, branch)
+	if err != nil {
+		log.Errorf(c, "test request failed: %v", err)
+		if reply == "" {
+			reply = internalError
+		}
+	}
+	return reply
+}
+
+func addTestJob(c context.Context, bugID, user, extID, patch, repo, branch string) (string, error) {
+	bug, bugKey, err := findBugByReportingID(c, bugID)
+	if err != nil {
+		return "can't find associated bug", err
+	}
+	now := timeNow(c)
+	bugReporting, _ := bugReportingByID(bug, bugID, now)
+
+	// TODO(dvyukov): find the exact crash that we reported.
+	crash, crashKey, err := findCrashForBug(c, bug)
+	if err != nil {
+		return "", err
+	}
+	if crash.ReproC == 0 && crash.ReproSyz == 0 {
+		return "This crash does not have a reproducer. I cannot test it.", nil
+	}
+
+	switch {
+	case !git.CheckRepoAddress(repo):
+		return fmt.Sprintf("%q does not look like a valid git repo address.", repo), nil
+	case !git.CheckBranch(branch):
+		return fmt.Sprintf("%q does not look like a valid git branch name.", branch), nil
+	case len(patch) == 0:
+		return "I don't see any patch attached to the request.", nil
+	case crash.ReproC == 0 && crash.ReproSyz == 0:
+		return "This crash does not have a reproducer. I cannot test it.", nil
+	case bug.Status == BugStatusFixed:
+		return "This bug is already marked as fixed. No point in testing.", nil
+	case bug.Status == BugStatusInvalid:
+		return "This bug is already marked as invalid. No point in testing.", nil
+	// TODO(dvyukov): for BugStatusDup check status of the canonical bug.
+	case !bugReporting.Closed.IsZero():
+		return "This bug is already upstreamed. Please test upstream.", nil
+	}
+
+	patchID, err := putText(c, bug.Namespace, "Patch", []byte(patch), false)
+	if err != nil {
+		return "", err
+	}
+
+	job := &Job{
+		Created:      now,
+		User:         user,
+		Reporting:    bugReporting.Name,
+		ExtID:        extID,
+		Namespace:    bug.Namespace,
+		Manager:      crash.Manager,
+		BugTitle:     bug.displayTitle(),
+		CrashID:      crashKey.IntID(),
+		KernelRepo:   repo,
+		KernelBranch: branch,
+		Patch:        patchID,
+	}
+	jobKey, err := datastore.Put(c, datastore.NewIncompleteKey(c, "Job", bugKey), job)
+	if err != nil {
+		return "", fmt.Errorf("failed to put job: %v", err)
+	}
+	jobID := extJobID(jobKey)
+
+	// Add user to bug reporting CC.
+	tx := func(c context.Context) error {
+		bug := new(Bug)
+		if err := datastore.Get(c, bugKey, bug); err != nil {
+			return err
+		}
+		bugReporting := bugReportingByName(bug, bugReporting.Name)
+		cc := strings.Split(bugReporting.CC, "|")
+		if stringInList(cc, user) {
+			return nil
+		}
+		merged := email.MergeEmailLists(cc, []string{user})
+		bugReporting.CC = strings.Join(merged, "|")
+		if _, err := datastore.Put(c, bugKey, bug); err != nil {
+			return err
+		}
+		return nil
+	}
+	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
+		// We've already stored the job, so just log the error.
+		log.Errorf(c, "job %v: failed to update bug: %v", jobID, err)
+	}
+	return "", nil
+}
+
+// pollPendingJobs returns the next job to execute for the provided list of managers.
+func pollPendingJobs(c context.Context, managers []string) (interface{}, error) {
+retry:
+	job, jobKey, err := loadPendingJob(c, managers)
+	if job == nil || err != nil {
+		return job, err
+	}
+	jobID := extJobID(jobKey)
+	patch, err := getText(c, "Patch", job.Patch)
+	if err != nil {
+		return nil, err
+	}
+	bugKey := jobKey.Parent()
+	crashKey := datastore.NewKey(c, "Crash", "", job.CrashID, bugKey)
+	crash := new(Crash)
+	if err := datastore.Get(c, crashKey, crash); err != nil {
+		return nil, fmt.Errorf("job %v: failed to get crash: %v", jobID, err)
+	}
+
+	build, err := loadBuild(c, job.Namespace, crash.BuildID)
+	if err != nil {
+		return nil, err
+	}
+	kernelConfig, err := getText(c, "KernelConfig", build.KernelConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	reproC, err := getText(c, "ReproC", crash.ReproC)
+	if err != nil {
+		return nil, err
+	}
+	reproSyz, err := getText(c, "ReproSyz", crash.ReproSyz)
+	if err != nil {
+		return nil, err
+	}
+
+	now := timeNow(c)
+	stale := false
+	tx := func(c context.Context) error {
+		stale = false
+		job := new(Job)
+		if err := datastore.Get(c, jobKey, job); err != nil {
+			return fmt.Errorf("job %v: failed to get in tx: %v", jobID, err)
+		}
+		if !job.Finished.IsZero() {
+			// This happens sometimes due to inconsistent datastore.
+			stale = true
+			return nil
+		}
+		job.Attempts++
+		job.Started = now
+		if _, err := datastore.Put(c, jobKey, job); err != nil {
+			return fmt.Errorf("job %v: failed to put: %v", jobID, err)
+		}
+		return nil
+	}
+	if err := datastore.RunInTransaction(c, tx, nil); err != nil {
+		return nil, err
+	}
+	if stale {
+		goto retry
+	}
+	resp := &dashapi.JobPollResp{
+		ID:              jobID,
+		Manager:         job.Manager,
+		KernelRepo:      job.KernelRepo,
+		KernelBranch:    job.KernelBranch,
+		KernelConfig:    kernelConfig,
+		SyzkallerCommit: build.SyzkallerCommit,
+		Patch:           patch,
+		ReproOpts:       crash.ReproOpts,
+		ReproSyz:        reproSyz,
+		ReproC:          reproC,
+	}
+	return resp, nil
+}
+
+// doneJob is called by syz-ci to mark completion of a job.
+func doneJob(c context.Context, req *dashapi.JobDoneReq) error {
+	jobID := req.ID
+	jobKey, err := jobID2Key(c, req.ID)
+	if err != nil {
+		return err
+	}
+	now := timeNow(c)
+	tx := func(c context.Context) error {
+		job := new(Job)
+		if err := datastore.Get(c, jobKey, job); err != nil {
+			return fmt.Errorf("job %v: failed to get job: %v", jobID, err)
+		}
+		if !job.Finished.IsZero() {
+			return fmt.Errorf("job %v: already finished", jobID)
+		}
+		ns := job.Namespace
+		if err := uploadBuild(c, ns, &req.Build); err != nil {
+			return err
+		}
+		if job.Error, err = putText(c, ns, "Error", req.Error, false); err != nil {
+			return err
+		}
+		if job.CrashLog, err = putText(c, ns, "CrashLog", req.CrashLog, false); err != nil {
+			return err
+		}
+		if job.CrashReport, err = putText(c, ns, "CrashReport", req.CrashReport, false); err != nil {
+			return err
+		}
+		job.BuildID = req.Build.ID
+		job.CrashTitle = req.CrashTitle
+		job.Finished = now
+		if _, err := datastore.Put(c, jobKey, job); err != nil {
+			return err
+		}
+		return nil
+	}
+	return datastore.RunInTransaction(c, tx, &datastore.TransactionOptions{XG: true, Attempts: 30})
+}
+
+func pollCompletedJobs(c context.Context, typ string) ([]*dashapi.BugReport, error) {
+	var jobs []*Job
+	keys, err := datastore.NewQuery("Job").
+		Filter("Finished>", time.Time{}).
+		Filter("Reported=", false).
+		GetAll(c, &jobs)
+	if err != nil {
+		return nil, fmt.Errorf("failed to query jobs: %v", err)
+	}
+	var reports []*dashapi.BugReport
+	for i, job := range jobs {
+		reporting := config.Namespaces[job.Namespace].ReportingByName(job.Reporting)
+		if reporting.Config.Type() != typ {
+			continue
+		}
+		rep, err := createBugReportForJob(c, job, keys[i], reporting.Config)
+		if err != nil {
+			log.Errorf(c, "failed to create report for job: %v", err)
+			continue
+		}
+		reports = append(reports, rep)
+	}
+	return reports, nil
+}
+
+func createBugReportForJob(c context.Context, job *Job, jobKey *datastore.Key, config interface{}) (*dashapi.BugReport, error) {
+	reportingConfig, err := json.Marshal(config)
+	if err != nil {
+		return nil, err
+	}
+	crashLog, err := getText(c, "CrashLog", job.CrashLog)
+	if err != nil {
+		return nil, err
+	}
+	if len(crashLog) > maxMailLogLen {
+		crashLog = crashLog[len(crashLog)-maxMailLogLen:]
+	}
+	report, err := getText(c, "CrashReport", job.CrashReport)
+	if err != nil {
+		return nil, err
+	}
+	if len(report) > maxMailReportLen {
+		report = report[:maxMailReportLen]
+	}
+	jobError, err := getText(c, "Error", job.Error)
+	if err != nil {
+		return nil, err
+	}
+	if len(jobError) > maxMailLogLen {
+		jobError = jobError[:maxMailLogLen]
+	}
+	patch, err := getText(c, "Patch", job.Patch)
+	if err != nil {
+		return nil, err
+	}
+	build, err := loadBuild(c, job.Namespace, job.BuildID)
+	if err != nil {
+		return nil, err
+	}
+	kernelConfig, err := getText(c, "KernelConfig", build.KernelConfig)
+	if err != nil {
+		return nil, err
+	}
+	bug := new(Bug)
+	if err := datastore.Get(c, jobKey.Parent(), bug); err != nil {
+		return nil, fmt.Errorf("failed to load job parent bug: %v", err)
+	}
+	bugReporting := bugReportingByName(bug, job.Reporting)
+	if bugReporting == nil {
+		return nil, fmt.Errorf("job bug has no reporting %q", job.Reporting)
+	}
+	rep := &dashapi.BugReport{
+		Namespace:    job.Namespace,
+		Config:       reportingConfig,
+		ID:           bugReporting.ID,
+		JobID:        extJobID(jobKey),
+		ExtID:        bugReporting.ExtID,
+		Title:        bug.displayTitle(),
+		Log:          crashLog,
+		Report:       report,
+		OS:           build.OS,
+		Arch:         build.Arch,
+		VMArch:       build.VMArch,
+		CompilerID:   build.CompilerID,
+		KernelRepo:   build.KernelRepo,
+		KernelBranch: build.KernelBranch,
+		KernelCommit: build.KernelCommit,
+		KernelConfig: kernelConfig,
+		CrashTitle:   job.CrashTitle,
+		Error:        jobError,
+		Patch:        patch,
+	}
+	if bugReporting.CC != "" {
+		rep.CC = strings.Split(bugReporting.CC, "|")
+	}
+	rep.CC = append(rep.CC, job.User)
+	return rep, nil
+}
+
+func jobReported(c context.Context, jobID string) error {
+	jobKey, err := jobID2Key(c, jobID)
+	if err != nil {
+		return err
+	}
+	tx := func(c context.Context) error {
+		job := new(Job)
+		if err := datastore.Get(c, jobKey, job); err != nil {
+			return fmt.Errorf("job %v: failed to get job: %v", jobID, err)
+		}
+		job.Reported = true
+		if _, err := datastore.Put(c, jobKey, job); err != nil {
+			return err
+		}
+		return nil
+	}
+	return datastore.RunInTransaction(c, tx, nil)
+}
+
+func loadPendingJob(c context.Context, managers []string) (*Job, *datastore.Key, error) {
+	var jobs []*Job
+	keys, err := datastore.NewQuery("Job").
+		Filter("Finished=", time.Time{}).
+		Order("Attempts").
+		Order("Created").
+		GetAll(c, &jobs)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to query jobs: %v", err)
+	}
+	mgrs := make(map[string]bool)
+	for _, mgr := range managers {
+		mgrs[mgr] = true
+	}
+	for i, job := range jobs {
+		if !mgrs[job.Manager] {
+			continue
+		}
+		return job, keys[i], nil
+	}
+	return nil, nil, nil
+}
+
+func extJobID(jobKey *datastore.Key) string {
+	return fmt.Sprintf("%v|%v", jobKey.Parent().StringID(), jobKey.IntID())
+}
+
+func jobID2Key(c context.Context, id string) (*datastore.Key, error) {
+	keyStr := strings.Split(id, "|")
+	if len(keyStr) != 2 {
+		return nil, fmt.Errorf("bad job id %q", id)
+	}
+	jobKeyID, err := strconv.ParseInt(keyStr[1], 10, 64)
+	if err != nil {
+		return nil, fmt.Errorf("bad job id %q", id)
+	}
+	bugKey := datastore.NewKey(c, "Bug", keyStr[0], 0, nil)
+	jobKey := datastore.NewKey(c, "Job", "", jobKeyID, bugKey)
+	return jobKey, nil
+}
diff --git a/dashboard/app/jobs_test.go b/dashboard/app/jobs_test.go
new file mode 100644
index 000000000..08b41d061
--- /dev/null
+++ b/dashboard/app/jobs_test.go
@@ -0,0 +1,201 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+// +build aetest
+
+package dash
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/google/syzkaller/dashboard/dashapi"
+)
+
+func TestJob(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	build := testBuild(1)
+	c.expectOK(c.API(client2, key2, "upload_build", build, nil))
+
+	patch := `--- a/mm/kasan/kasan.c
++++ b/mm/kasan/kasan.c
+-       current->kasan_depth++;
++       current->kasan_depth--;
+`
+
+	// Report crash without repro, check that test requests are not accepted.
+	crash := testCrash(build, 1)
+	c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
+
+	c.expectOK(c.GET("/email_poll"))
+	c.expectEQ(len(c.emailSink), 1)
+	sender := (<-c.emailSink).Sender
+
+	c.incomingEmail(sender, "#syz test: git://git.git/git.git kernel-branch\n"+patch)
+	c.expectEQ(len(c.emailSink), 1)
+	c.expectEQ(strings.Contains((<-c.emailSink).Body, "This crash does not have a reproducer"), true)
+
+	// Report crash with repro.
+	crash.Maintainers = []string{"foo@bar.com"}
+	crash.ReproOpts = []byte("repro opts")
+	crash.ReproSyz = []byte("repro syz")
+	crash.ReproC = []byte("repro C")
+	c.expectOK(c.API(client2, key2, "report_crash", crash, nil))
+
+	c.expectOK(c.GET("/email_poll"))
+	c.expectEQ(len(c.emailSink), 1)
+	c.expectEQ(strings.Contains((<-c.emailSink).Body, "syzkaller has found reproducer"), true)
+
+	c.incomingEmail(sender, "#syz test: repo")
+	c.expectEQ(len(c.emailSink), 1)
+	c.expectEQ(strings.Contains((<-c.emailSink).Body, "want 2 args"), true)
+
+	c.incomingEmail(sender, "#syz test: repo branch commit")
+	c.expectEQ(len(c.emailSink), 1)
+	c.expectEQ(strings.Contains((<-c.emailSink).Body, "want 2 args"), true)
+
+	c.incomingEmail(sender, "#syz test: repo branch")
+	c.expectEQ(len(c.emailSink), 1)
+	c.expectEQ(strings.Contains((<-c.emailSink).Body, "does not look like a valid git repo"), true)
+
+	c.incomingEmail(sender, "#syz test: git://git.git/git.git master")
+	c.expectEQ(len(c.emailSink), 1)
+	c.expectEQ(strings.Contains((<-c.emailSink).Body, "I don't see any patch attached to the request"), true)
+
+	c.incomingEmail(sender, "#syz test: git://git.git/git.git kernel-branch\n"+patch)
+	c.expectOK(c.GET("/email_poll"))
+	c.expectEQ(len(c.emailSink), 0)
+
+	pollResp := new(dashapi.JobPollResp)
+	c.expectOK(c.API(client2, key2, "job_poll", &dashapi.JobPollReq{[]string{"foobar"}}, pollResp))
+	c.expectEQ(pollResp.ID, "")
+	c.expectOK(c.API(client2, key2, "job_poll", &dashapi.JobPollReq{[]string{build.Manager}}, pollResp))
+	c.expectEQ(pollResp.ID != "", true)
+	c.expectEQ(pollResp.Manager, build.Manager)
+	c.expectEQ(pollResp.KernelRepo, "git://git.git/git.git")
+	c.expectEQ(pollResp.KernelBranch, "kernel-branch")
+	c.expectEQ(pollResp.KernelConfig, build.KernelConfig)
+	c.expectEQ(pollResp.SyzkallerCommit, build.SyzkallerCommit)
+	c.expectEQ(pollResp.Patch, []byte(patch+"\n"))
+	c.expectEQ(pollResp.ReproOpts, []byte("repro opts"))
+	c.expectEQ(pollResp.ReproSyz, []byte("repro syz"))
+	c.expectEQ(pollResp.ReproC, []byte("repro C"))
+
+	pollResp2 := new(dashapi.JobPollResp)
+	c.expectOK(c.API(client2, key2, "job_poll", &dashapi.JobPollReq{[]string{build.Manager}}, pollResp2))
+	c.expectEQ(pollResp2, pollResp)
+
+	jobDoneReq := &dashapi.JobDoneReq{
+		ID:          pollResp.ID,
+		Build:       *build,
+		CrashTitle:  "test crash title",
+		CrashLog:    []byte("test crash log"),
+		CrashReport: []byte("test crash report"),
+	}
+	c.expectOK(c.API(client2, key2, "job_done", jobDoneReq, nil))
+
+	c.expectOK(c.GET("/email_poll"))
+	c.expectEQ(len(c.emailSink), 1)
+	{
+		msg := <-c.emailSink
+		list := config.Namespaces["test2"].Reporting[0].Config.(*EmailConfig).Email
+		c.expectEQ(msg.To, []string{"default@sender.com", list})
+		c.expectEQ(msg.Subject, crash.Title)
+		c.expectEQ(len(msg.Attachments), 3)
+		c.expectEQ(msg.Attachments[0].Name, "config.txt")
+		c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+		c.expectEQ(msg.Attachments[1].Name, "patch.txt")
+		c.expectEQ(msg.Attachments[1].Data, []byte(patch+"\n"))
+		c.expectEQ(msg.Attachments[2].Name, "raw.log")
+		c.expectEQ(msg.Attachments[2].Data, jobDoneReq.CrashLog)
+		c.expectEQ(msg.Body, `Hello,
+
+syzbot has tested the proposed patch but the reproducer still triggered crash:
+test crash title
+
+test crash report
+
+Tested on commit kernel_commit1
+repo1/branch1
+compiler: compiler1
+Patch and kernel config are attached.
+Raw console output is attached.
+
+`)
+	}
+	c.incomingEmail(sender, "#syz test: git://git.git/git.git kernel-branch\n"+patch)
+	c.expectOK(c.API(client2, key2, "job_poll", &dashapi.JobPollReq{[]string{build.Manager}}, pollResp))
+	jobDoneReq = &dashapi.JobDoneReq{
+		ID:    pollResp.ID,
+		Build: *build,
+		Error: []byte("failed to apply patch"),
+	}
+	c.expectOK(c.API(client2, key2, "job_done", jobDoneReq, nil))
+	c.expectOK(c.GET("/email_poll"))
+	c.expectEQ(len(c.emailSink), 1)
+	{
+		msg := <-c.emailSink
+		c.expectEQ(len(msg.Attachments), 2)
+		c.expectEQ(msg.Attachments[0].Name, "config.txt")
+		c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+		c.expectEQ(msg.Attachments[1].Name, "patch.txt")
+		c.expectEQ(msg.Attachments[1].Data, []byte(patch+"\n"))
+		c.expectEQ(msg.Body, `Hello,
+
+syzbot tried to test the proposed patch but build failed:
+
+failed to apply patch
+
+Tested on commit kernel_commit1
+repo1/branch1
+compiler: compiler1
+Patch and kernel config are attached.
+
+
+`)
+	}
+
+	c.incomingEmail(sender, "#syz test: git://git.git/git.git kernel-branch\n"+patch)
+	c.expectOK(c.API(client2, key2, "job_poll", &dashapi.JobPollReq{[]string{build.Manager}}, pollResp))
+	jobDoneReq = &dashapi.JobDoneReq{
+		ID:    pollResp.ID,
+		Build: *build,
+	}
+	c.expectOK(c.API(client2, key2, "job_done", jobDoneReq, nil))
+	c.expectOK(c.GET("/email_poll"))
+	c.expectEQ(len(c.emailSink), 1)
+	{
+		msg := <-c.emailSink
+		c.expectEQ(len(msg.Attachments), 2)
+		c.expectEQ(msg.Attachments[0].Name, "config.txt")
+		c.expectEQ(msg.Attachments[0].Data, build.KernelConfig)
+		c.expectEQ(msg.Attachments[1].Name, "patch.txt")
+		c.expectEQ(msg.Attachments[1].Data, []byte(patch+"\n"))
+		c.expectEQ(msg.Body, `Hello,
+
+syzbot has tested the proposed patch and the reproducer did not trigger crash:
+
+Tested-by: syzbot <syzkaller@googlegroups.com>
+
+Once the fix is committed, please reply to this email with:
+#syz fix: exact-commit-title
+
+Tested on commit kernel_commit1
+repo1/branch1
+compiler: compiler1
+Patch and kernel config are attached.
+
+
+---
+There is no WARRANTY for the result, to the extent permitted by applicable law.
+Except when otherwise stated in writing syzbot provides the result "AS IS"
+without warranty of any kind, either expressed or implied, but not limited to,
+the implied warranties of merchantability and fittness for a particular purpose.
+The entire risk as to the quality of the result is with you. Should the result
+prove defective, you assume the cost of all necessary servicing, repair or
+correction.
+`)
+	}
+}
diff --git a/dashboard/app/mail_test_result.txt b/dashboard/app/mail_test_result.txt
new file mode 100644
index 000000000..210a04546
--- /dev/null
+++ b/dashboard/app/mail_test_result.txt
@@ -0,0 +1,32 @@
+Hello,
+{{if .CrashTitle}}
+syzbot has tested the proposed patch but the reproducer still triggered crash:
+{{.CrashTitle}}
+
+{{printf "%s" .Report}}
+{{else if .Error}}
+syzbot tried to test the proposed patch but build failed:
+
+{{printf "%s" .Error}}
+{{else}}
+syzbot has tested the proposed patch and the reproducer did not trigger crash:
+
+Tested-by: syzbot <syzkaller@googlegroups.com>
+
+Once the fix is committed, please reply to this email with:
+#syz fix: exact-commit-title
+{{end}}
+Tested on commit {{.KernelCommit}}
+{{.KernelRepo}}/{{.KernelBranch}}
+compiler: {{.CompilerID}}
+Patch and kernel config are attached.
+{{if .HasLog}}Raw console output is attached.{{end}}
+{{if and (not .CrashTitle) (not .Error)}}
+---
+There is no WARRANTY for the result, to the extent permitted by applicable law.
+Except when otherwise stated in writing syzbot provides the result "AS IS"
+without warranty of any kind, either expressed or implied, but not limited to,
+the implied warranties of merchantability and fittness for a particular purpose.
+The entire risk as to the quality of the result is with you. Should the result
+prove defective, you assume the cost of all necessary servicing, repair or
+correction.{{end}}
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 206fb46d2..7f93e7b02 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -29,6 +29,7 @@ func init() {
 type uiMain struct {
 	Header    *uiHeader
 	Log       []byte
+	Jobs      []*uiJob
 	BugGroups []*uiBugGroup
 }
 
@@ -74,6 +75,28 @@ type uiCrash struct {
 	KernelConfigLink string
 }
 
+type uiJob struct {
+	Created         time.Time
+	User            string
+	Reporting       string
+	Namespace       string
+	Manager         string
+	BugTitle        string
+	BugID           string
+	KernelRepo      string
+	KernelBranch    string
+	KernelCommit    string
+	PatchLink       string
+	Attempts        int
+	Started         time.Time
+	Finished        time.Time
+	CrashTitle      string
+	CrashLogLink    string
+	CrashReportLink string
+	ErrorLink       string
+	Reported        bool
+}
+
 // handleMain serves main page.
 func handleMain(c context.Context, w http.ResponseWriter, r *http.Request) error {
 	h, err := commonHeader(c)
@@ -84,6 +107,10 @@ func handleMain(c context.Context, w http.ResponseWriter, r *http.Request) error
 	if err != nil {
 		return err
 	}
+	jobs, err := loadRecentJobs(c)
+	if err != nil {
+		return err
+	}
 	groups, err := fetchBugs(c)
 	if err != nil {
 		return err
@@ -91,6 +118,7 @@ func handleMain(c context.Context, w http.ResponseWriter, r *http.Request) error
 	data := &uiMain{
 		Header:    h,
 		Log:       errorLog,
+		Jobs:      jobs,
 		BugGroups: groups,
 	}
 	return templates.ExecuteTemplate(w, "main.html", data)
@@ -257,6 +285,41 @@ func loadCrashesForBug(c context.Context, bug *Bug) ([]*uiCrash, error) {
 	return results, nil
 }
 
+func loadRecentJobs(c context.Context) ([]*uiJob, error) {
+	var jobs []*Job
+	keys, err := datastore.NewQuery("Job").
+		Order("-Created").
+		Limit(20).
+		GetAll(c, &jobs)
+	if err != nil {
+		return nil, err
+	}
+	var results []*uiJob
+	for i, job := range jobs {
+		ui := &uiJob{
+			Created:         job.Created,
+			User:            job.User,
+			Reporting:       job.Reporting,
+			Namespace:       job.Namespace,
+			Manager:         job.Manager,
+			BugTitle:        job.BugTitle,
+			BugID:           keys[i].Parent().StringID(),
+			KernelRepo:      job.KernelRepo,
+			KernelBranch:    job.KernelBranch,
+			PatchLink:       textLink("Patch", job.Patch),
+			Attempts:        job.Attempts,
+			Started:         job.Started,
+			Finished:        job.Finished,
+			CrashTitle:      job.CrashTitle,
+			CrashLogLink:    textLink("CrashLog", job.CrashLog),
+			CrashReportLink: textLink("CrashReport", job.CrashReport),
+			ErrorLink:       textLink("Error", job.Error),
+		}
+		results = append(results, ui)
+	}
+	return results, nil
+}
+
 func fetchErrorLogs(c context.Context) ([]byte, error) {
 	const (
 		minLogLevel  = 2
diff --git a/dashboard/app/main.html b/dashboard/app/main.html
index 34d85d89f..56ee285ee 100644
--- a/dashboard/app/main.html
+++ b/dashboard/app/main.html
@@ -39,6 +39,56 @@
 	</script>
 	<br><br>
 
+	<table class="list_table">
+		<caption>Recent jobs:</caption>
+		<tr>
+			<th>Created</th>
+			<th>Started</th>
+			<th>Finished</th>
+			<th>User</th>
+			<th>Bug</th>
+			<th>Patch</th>
+			<th>Reporting</th>
+			<th>Manager</th>
+			<th>Repo</th>
+			<th>Result</th>
+		</tr>
+		{{range $job := $.Jobs}}
+			<tr>
+				<td class="time">{{formatTime $job.Created}}</td>
+				<td class="time">{{formatTime $job.Started}}{{if gt $job.Attempts 1}} ({{$job.Attempts}}){{end}}</td>
+				<td class="time">{{formatTime $job.Finished}}</td>
+				<td>{{$job.User}}</td>
+				<td class="title"><a href="/bug?id={{$job.BugID}}">{{$job.BugTitle}}</a></td>
+				<td><a href="{{$job.PatchLink}}">patch</a></td>
+				<td>{{$job.Namespace}}/{{$job.Reporting}}</td>
+				<td>{{$job.Manager}}</td>
+				<td>{{$job.KernelRepo}}/{{$job.KernelBranch}}</td>
+				<td>
+					{{if $job.ErrorLink}}
+						<a href="{{$job.ErrorLink}}">error</a>
+					{{else if $job.CrashTitle}}
+						{{if $job.CrashReportLink}}
+							<a href="{{$job.CrashReportLink}}">{{$job.CrashTitle}}</a>
+						{{else}}
+							{{$job.CrashTitle}}
+						{{end}}
+						{{if $job.CrashLogLink}}
+							 (<a href="{{$job.CrashLogLink}}">log</a>)
+						{{end}}
+					{{else if formatTime $job.Finished}}
+						OK
+					{{else if formatTime $job.Started}}
+						running
+					{{else}}
+						pending
+					{{end}}
+				</td>
+			</tr>
+		{{end}}
+	</table>
+	<br><br>
+
 	{{range $g := $.BugGroups}}
 		{{template "bug_table" $g}} <br>
 	{{end}}
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 48d046d7f..2a3e704e2 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/mail"
+	"strings"
 	"text/template"
 
 	"github.com/google/syzkaller/dashboard/dashapi"
@@ -71,6 +72,11 @@ func handleEmailPoll(w http.ResponseWriter, r *http.Request) {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+	if err := emailPollJobs(c); err != nil {
+		log.Errorf(c, "job poll failed: %v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
 	w.Write([]byte("OK"))
 }
 
@@ -99,6 +105,24 @@ func emailPollBugs(c context.Context) error {
 	return nil
 }
 
+func emailPollJobs(c context.Context) error {
+	jobs, err := pollCompletedJobs(c, emailType)
+	if err != nil {
+		return err
+	}
+	for _, job := range jobs {
+		if err := emailReport(c, job, "mail_test_result.txt"); err != nil {
+			log.Errorf(c, "failed to report job: %v", err)
+			continue
+		}
+		if err := jobReported(c, job.JobID); err != nil {
+			log.Errorf(c, "failed to mark job reported: %v", err)
+			continue
+		}
+	}
+	return nil
+}
+
 func emailReport(c context.Context, rep *dashapi.BugReport, templ string) error {
 	cfg := new(EmailConfig)
 	if err := json.Unmarshal(rep.Config, cfg); err != nil {
@@ -115,6 +139,12 @@ func emailReport(c context.Context, rep *dashapi.BugReport, templ string) error
 			Data: rep.KernelConfig,
 		},
 	}
+	if len(rep.Patch) != 0 {
+		attachments = append(attachments, aemail.Attachment{
+			Name: "patch.txt",
+			Data: rep.Patch,
+		})
+	}
 	if len(rep.Log) != 0 {
 		attachments = append(attachments, aemail.Attachment{
 			Name: "raw.log",
@@ -146,7 +176,9 @@ func emailReport(c context.Context, rep *dashapi.BugReport, templ string) error
 		KernelRepo   string
 		KernelBranch string
 		KernelCommit string
+		CrashTitle   string
 		Report       []byte
+		Error        []byte
 		HasLog       bool
 		ReproSyz     bool
 		ReproC       bool
@@ -159,7 +191,9 @@ func emailReport(c context.Context, rep *dashapi.BugReport, templ string) error
 		KernelRepo:   rep.KernelRepo,
 		KernelBranch: rep.KernelBranch,
 		KernelCommit: rep.KernelCommit,
+		CrashTitle:   rep.CrashTitle,
 		Report:       rep.Report,
+		Error:        rep.Error,
 		HasLog:       len(rep.Log) != 0,
 		ReproSyz:     len(rep.ReproSyz) != 0,
 		ReproC:       len(rep.ReproC) != 0,
@@ -194,6 +228,8 @@ func incomingMail(c context.Context, r *http.Request) error {
 		log.Infof(c, "duplicate email from mailing list, ignoring")
 		return nil
 	}
+	// TODO(dvyukov): check that our mailing list is in CC
+	// (otherwise there will be no history of what hsppened with a bug).
 	cmd := &dashapi.BugUpdate{
 		ID:    msg.BugID,
 		ExtID: msg.MessageID,
@@ -219,6 +255,21 @@ func incomingMail(c context.Context, r *http.Request) error {
 		}
 		cmd.Status = dashapi.BugStatusDup
 		cmd.DupOf = msg.CommandArgs
+	case "test:":
+		// TODO(dvyukov): remember email link for jobs.
+		if !appengine.IsDevAppServer() {
+			return replyTo(c, msg, "testing is experimental", nil)
+		}
+		args := strings.Split(msg.CommandArgs, " ")
+		if len(args) != 2 {
+			return replyTo(c, msg, fmt.Sprintf("want 2 args (repo, branch), got %v", len(args)), nil)
+		}
+		reply := handleTestRequest(c, msg.BugID, email.CanonicalEmail(msg.From),
+			msg.MessageID, msg.Patch, args[0], args[1])
+		if reply != "" {
+			return replyTo(c, msg, reply, nil)
+		}
+		return nil
 	default:
 		return replyTo(c, msg, fmt.Sprintf("unknown command %q", msg.Command), nil)
 	}
diff --git a/dashboard/app/util_test.go b/dashboard/app/util_test.go
index bc0eef27c..5bfd96119 100644
--- a/dashboard/app/util_test.go
+++ b/dashboard/app/util_test.go
@@ -169,6 +169,20 @@ func (c *Ctx) httpRequest(method, url, body string) error {
 	return nil
 }
 
+func (c *Ctx) incomingEmail(to, body string) {
+	email := fmt.Sprintf(`Sender: foo@bar.com
+Date: Tue, 15 Aug 2017 14:59:00 -0700
+Message-ID: <1234>
+Subject: crash1
+From: default@sender.com
+To: %v
+Content-Type: text/plain
+
+%v
+`, to, body)
+	c.expectOK(c.POST("/_ah/mail/", email))
+}
+
 func init() {
 	// Mock time as some functionality relies on real time.
 	timeNow = func(c context.Context) time.Time {
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go
index 28a4f7bf8..7051e3f2e 100644
--- a/dashboard/dashapi/dashapi.go
+++ b/dashboard/dashapi/dashapi.go
@@ -75,6 +75,52 @@ func (dash *Dashboard) BuilderPoll(manager string) (*BuilderPollResp, error) {
 	return resp, err
 }
 
+// Jobs workflow:
+//   - syz-ci sends JobPollReq periodically to check for new jobs,
+//     request contains list of managers that this syz-ci runs.
+//   - dashboard replies with JobPollResp that contains job details,
+//     if no new jobs available ID is set to empty string.
+//   - when syz-ci finishes the job, it sends JobDoneReq which contains
+//     job execution result (Build, Crash or Error details),
+//     ID must match JobPollResp.ID.
+
+type JobPollReq struct {
+	Managers []string
+}
+
+type JobPollResp struct {
+	ID              string
+	Manager         string
+	KernelRepo      string
+	KernelBranch    string
+	KernelConfig    []byte
+	SyzkallerCommit string
+	Patch           []byte
+	ReproOpts       []byte
+	ReproSyz        []byte
+	ReproC          []byte
+}
+
+type JobDoneReq struct {
+	ID          string
+	Build       Build
+	Error       []byte
+	CrashTitle  string
+	CrashLog    []byte
+	CrashReport []byte
+}
+
+func (dash *Dashboard) JobPoll(managers []string) (*JobPollResp, error) {
+	req := &JobPollReq{Managers: managers}
+	resp := new(JobPollResp)
+	err := dash.query("job_poll", req, resp)
+	return resp, err
+}
+
+func (dash *Dashboard) JobDone(req *JobDoneReq) error {
+	return dash.query("job_done", req, nil)
+}
+
 // Crash describes a single kernel crash (potentially with repro).
 type Crash struct {
 	BuildID     string // refers to Build.ID
@@ -140,6 +186,7 @@ type BugReport struct {
 	Namespace    string
 	Config       []byte
 	ID           string
+	JobID        string
 	ExtID        string // arbitrary reporting ID forwarded from BugUpdate.ExtID
 	First        bool   // Set for first report for this bug.
 	Title        string
@@ -157,6 +204,10 @@ type BugReport struct {
 	Report       []byte
 	ReproC       []byte
 	ReproSyz     []byte
+
+	CrashTitle string // job execution crash title
+	Error      []byte // job execution error
+	Patch      []byte // testing job patch
 }
 
 type BugUpdate struct {