dashboard: introduce an emergency stop mode

Add an emergency stop button that can be used by any admin. After it's clicked two times, syzbot stops all reporting and recoding of new bugs. It's assumed that the stop mode is revoked by manually deleting an entry from the database.
author: Aleksandr Nogikh <nogikh@google.com> 2024-01-09 16:27:55 +0100
committer: Aleksandr Nogikh <nogikh@google.com> 2024-01-09 17:48:04 +0000
commit: b438bd66d6f95113d52f25c25bfef0e963c8ce8d (patch)
tree: affa3733771acfc752397dc89a1c6e712d4d622c
parent: 83b32fe8f20dc1e910866fa110b0d872df283f03 (diff)
8 files changed, 242 insertions, 5 deletions
diff --git a/dashboard/app/admin.html b/dashboard/app/admin.html
index 8ada1c64b..dddc03d46 100644
--- a/dashboard/app/admin.html
+++ b/dashboard/app/admin.html
@@ -13,6 +13,12 @@ Main page.
 </head>
 <body>
 	{{template "header" .Header}}
+	{{if $.Stopped}}
+		<div class="emergency-stopped">Syzbot is in the emergency stop state</div>
+	{{else}}
+		<div class="emergency-stop">Syzbot is reporting too many bugs? {{link $.StopLink "Emergency stop"}} [click {{$.MoreStopClicks}} more times]<br />
+		In this mode, syzbot will stop all reporting and won't record any new findings.</div>
+	{{end}}
 
 	<a class="plain" href="#log"><div id="log"><b>Error log:</b></div></a>
 	<textarea id="log_textarea" readonly rows="20" wrap=off>{{printf "%s" .Log}}</textarea>
@@ -21,7 +27,6 @@ Main page.
 		textarea.scrollTop = textarea.scrollHeight;
 	</script>
 	<br><br>
-
 	{{with $.MemcacheStats}}
 	<table class="list_table">
 		<caption><a href="https://pkg.go.dev/google.golang.org/appengine/memcache?tab=doc#Item" target="_blank">Memcache stats:</a></caption>
diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 2dce92d55..6a84c758b 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -30,6 +30,7 @@ import (
 	"google.golang.org/appengine/v2"
 	db "google.golang.org/appengine/v2/datastore"
 	"google.golang.org/appengine/v2/log"
+	"google.golang.org/appengine/v2/user"
 )
 
 func initAPIHandlers() {
@@ -363,6 +364,10 @@ func addCommitInfoToBugImpl(c context.Context, bug *Bug, com dashapi.Commit) (bo
 }
 
 func apiJobPoll(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		// The bot's operation was aborted. Don't accept new crash reports.
+		return &dashapi.JobPollResp{}, err
+	}
 	req := new(dashapi.JobPollReq)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -699,6 +704,10 @@ const (
 )
 
 func apiReportCrash(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		// The bot's operation was aborted. Don't accept new crash reports.
+		return &dashapi.ReportCrashResp{}, err
+	}
 	req := new(dashapi.Crash)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -1632,3 +1641,23 @@ func apiSaveDiscussion(c context.Context, r *http.Request, payload []byte) (inte
 	}
 	return nil, mergeDiscussion(c, d)
 }
+
+func emergentlyStopped(c context.Context) (bool, error) {
+	keys, err := db.NewQuery("EmergencyStop").
+		Limit(1).
+		KeysOnly().
+		GetAll(c, nil)
+	if err != nil {
+		return false, err
+	}
+	return len(keys) > 0, nil
+}
+
+func recordEmergencyStop(c context.Context) error {
+	key := db.NewKey(c, "EmergencyStop", "all", 0, nil)
+	_, err := db.Put(c, key, &EmergencyStop{
+		Time: timeNow(c),
+		User: user.Current(c).Email,
+	})
+	return err
+}
diff --git a/dashboard/app/api_test.go b/dashboard/app/api_test.go
index 8d63ce7a7..4a872231e 100644
--- a/dashboard/app/api_test.go
+++ b/dashboard/app/api_test.go
@@ -5,6 +5,9 @@ package main
 
 import (
 	"testing"
+	"time"
+
+	"github.com/google/syzkaller/dashboard/dashapi"
 )
 
 func TestClientSecretOK(t *testing.T) {
@@ -63,3 +66,139 @@ func TestClientNamespaceOK(t *testing.T) {
 		t.Errorf("Unexpected error %v %v", got, err)
 	}
 }
+
+func TestEmergentlyStoppedEmail(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	client.ReportCrash(crash)
+
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedReproEmail(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	client.ReportCrash(crash)
+	c.pollEmailBug()
+
+	crash2 := testCrash(build, 1)
+	crash2.ReproOpts = []byte("repro opts")
+	crash2.ReproSyz = []byte("getpid()")
+	client.ReportCrash(crash2)
+
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedExternalReport(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.client
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	client.ReportCrash(crash)
+
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	client.pollBugs(0)
+}
+
+func TestEmergentlyStoppedEmailJob(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	crash.ReproOpts = []byte("repro opts")
+	crash.ReproSyz = []byte("getpid()")
+	client.ReportCrash(crash)
+	sender := c.pollEmailBug().Sender
+	c.incomingEmail(sender, "#syz upstream\n")
+	sender = c.pollEmailBug().Sender
+
+	// Send a patch testing request.
+	c.advanceTime(time.Hour)
+	c.incomingEmail(sender, syzTestGitBranchSamplePatch,
+		EmailOptMessageID(1), EmailOptFrom("test@requester.com"),
+		EmailOptCC([]string{"somebody@else.com", "test@syzkaller.com"}))
+	c.expectNoEmail()
+
+	// Emulate a finished job.
+	pollResp := client.pollJobs(build.Manager)
+	c.expectEQ(pollResp.Type, dashapi.JobTestPatch)
+
+	c.advanceTime(time.Hour)
+	jobDoneReq := &dashapi.JobDoneReq{
+		ID:          pollResp.ID,
+		Build:       *build,
+		CrashTitle:  "test crash title",
+		CrashLog:    []byte("test crash log"),
+		CrashReport: []byte("test crash report"),
+	}
+	client.JobDone(jobDoneReq)
+
+	// Now we emergently stop syzbot.
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	// There should be no email.
+	c.advanceTime(time.Hour)
+	c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedCrashReport(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	client := c.publicClient
+	build := testBuild(1)
+	client.UploadBuild(build)
+
+	// Now we emergently stop syzbot.
+	c.advanceTime(time.Hour)
+	_, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+	c.expectOK(err)
+
+	crash := testCrash(build, 1)
+	crash.ReproOpts = []byte("repro opts")
+	crash.ReproSyz = []byte("getpid()")
+	client.ReportCrash(crash)
+
+	listResp, err := client.BugList()
+	c.expectOK(err)
+	c.expectEQ(len(listResp.List), 0)
+}
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index 659474005..71f4a92fc 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -986,6 +986,13 @@ func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) {
 	return status, nil
 }
 
+// If an entity of type EmergencyStop exists, syzbot's operation is paused until
+// a support engineer deletes it from the DB.
+type EmergencyStop struct {
+	Time time.Time
+	User string
+}
+
 func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error {
 	crash := new(Crash)
 	crashKey := db.NewKey(c, "Crash", "", crashID, bugKey)
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 8d56c0ee5..ae8e138af 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -203,6 +203,9 @@ type uiAdminPage struct {
 	CauseBisectionsLink string
 	JobOverviewLink     string
 	MemcacheStats       *memcache.Statistics
+	Stopped             bool
+	StopLink            string
+	MoreStopClicks      int
 }
 
 type uiManagerPage struct {
@@ -904,6 +907,10 @@ func handleAdmin(c context.Context, w http.ResponseWriter, r *http.Request) erro
 		}
 	case "invalidate_bisection":
 		return handleInvalidateBisection(c, w, r)
+	case "emergency_stop":
+		if err := recordEmergencyStop(c); err != nil {
+			return fmt.Errorf("failed to record an emergency stop: %w", err)
+		}
 	default:
 		return fmt.Errorf("%w: unknown action %q", ErrClientBadRequest, action)
 	}
@@ -963,15 +970,28 @@ func handleAdmin(c context.Context, w http.ResponseWriter, r *http.Request) erro
 			return err
 		})
 	}
+	alreadyStopped := false
+	g.Go(func() error {
+		var err error
+		alreadyStopped, err = emergentlyStopped(c)
+		return err
+	})
 	err = g.Wait()
 	if err != nil {
 		return err
 	}
 	data := &uiAdminPage{
-		Header:        hdr,
-		Log:           errorLog,
-		Managers:      makeManagerList(managers, hdr.Namespace),
-		MemcacheStats: memcacheStats,
+		Header:         hdr,
+		Log:            errorLog,
+		Managers:       makeManagerList(managers, hdr.Namespace),
+		MemcacheStats:  memcacheStats,
+		Stopped:        alreadyStopped,
+		MoreStopClicks: 2,
+		StopLink:       html.AmendURL("/admin", "stop_clicked", "1"),
+	}
+	if r.FormValue("stop_clicked") != "" {
+		data.MoreStopClicks = 1
+		data.StopLink = html.AmendURL("/admin", "action", "emergency_stop")
 	}
 	if r.FormValue("job_type") != "" {
 		data.TypeJobs = &uiJobList{Title: "Last jobs:", Jobs: typeJobs}
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 42fffb6f3..8eaf5e5ff 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -105,6 +105,16 @@ func (cfg *EmailConfig) Validate() error {
 // handleEmailPoll is called by cron and sends emails for new bugs, if any.
 func handleEmailPoll(w http.ResponseWriter, r *http.Request) {
 	c := appengine.NewContext(r)
+	stop, err := emergentlyStopped(c)
+	if err != nil {
+		log.Errorf(c, "emergency stop querying failed: %v", err)
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	if stop {
+		log.Errorf(c, "aborting email poll due to an emergency stop")
+		return
+	}
 	if err := emailPollJobs(c); err != nil {
 		log.Errorf(c, "job poll failed: %v", err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -476,8 +486,14 @@ func handleIncomingMail(w http.ResponseWriter, r *http.Request) {
 	}
 	log.Infof(c, "received email at %q, source %q", myEmail, source)
 	if source == dashapi.NoDiscussion {
+		if stop, err := emergentlyStopped(c); err != nil || stop {
+			log.Errorf(c, "abort email processing due to emergency stop (stop %v, err %v)",
+				stop, err)
+			return
+		}
 		err = processIncomingEmail(c, msg)
 	} else {
+		// Discussions are safe to handle even during an emergency stop.
 		err = processDiscussionEmail(c, msg, source)
 	}
 	if err != nil {
diff --git a/dashboard/app/reporting_external.go b/dashboard/app/reporting_external.go
index e6f65c738..625476918 100644
--- a/dashboard/app/reporting_external.go
+++ b/dashboard/app/reporting_external.go
@@ -19,6 +19,9 @@ import (
 // and report back bug status updates with apiReportingUpdate.
 
 func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		return &dashapi.PollBugsResponse{}, err
+	}
 	req := new(dashapi.PollBugsRequest)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -36,6 +39,9 @@ func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (i
 }
 
 func apiReportingPollNotifications(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		return &dashapi.PollNotificationsResponse{}, err
+	}
 	req := new(dashapi.PollNotificationsRequest)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -48,6 +54,9 @@ func apiReportingPollNotifications(c context.Context, r *http.Request, payload [
 }
 
 func apiReportingPollClosed(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+	if stop, err := emergentlyStopped(c); err != nil || stop {
+		return &dashapi.PollClosedResponse{}, err
+	}
 	req := new(dashapi.PollClosedRequest)
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %w", err)
diff --git a/pkg/html/pages/style.css b/pkg/html/pages/style.css
index 9dd1bca7f..156bf61ba 100644
--- a/pkg/html/pages/style.css
+++ b/pkg/html/pages/style.css
@@ -409,3 +409,15 @@ aside {
 .collapsible-show .show-icon {
 	display: none;
 }
+
+.emergency-stop {
+	background-color: yellow;
+	padding: 5pt;
+	margin-bottom: 5pt;
+}
+
+.emergency-stopped {
+	background-color: coral;
+	padding: 5pt;
+	margin-bottom: 5pt;
+}
author	Aleksandr Nogikh <nogikh@google.com>	2024-01-09 16:27:55 +0100
committer	Aleksandr Nogikh <nogikh@google.com>	2024-01-09 17:48:04 +0000
commit	b438bd66d6f95113d52f25c25bfef0e963c8ce8d (patch)
tree	affa3733771acfc752397dc89a1c6e712d4d622c
parent	83b32fe8f20dc1e910866fa110b0d872df283f03 (diff)