aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--dashboard/app/admin.html7
-rw-r--r--dashboard/app/api.go29
-rw-r--r--dashboard/app/api_test.go139
-rw-r--r--dashboard/app/entities.go7
-rw-r--r--dashboard/app/main.go28
-rw-r--r--dashboard/app/reporting_email.go16
-rw-r--r--dashboard/app/reporting_external.go9
-rw-r--r--pkg/html/pages/style.css12
8 files changed, 242 insertions, 5 deletions
diff --git a/dashboard/app/admin.html b/dashboard/app/admin.html
index 8ada1c64b..dddc03d46 100644
--- a/dashboard/app/admin.html
+++ b/dashboard/app/admin.html
@@ -13,6 +13,12 @@ Main page.
</head>
<body>
{{template "header" .Header}}
+ {{if $.Stopped}}
+ <div class="emergency-stopped">Syzbot is in the emergency stop state</div>
+ {{else}}
+ <div class="emergency-stop">Syzbot is reporting too many bugs? {{link $.StopLink "Emergency stop"}} [click {{$.MoreStopClicks}} more times]<br />
+ In this mode, syzbot will stop all reporting and won't record any new findings.</div>
+ {{end}}
<a class="plain" href="#log"><div id="log"><b>Error log:</b></div></a>
<textarea id="log_textarea" readonly rows="20" wrap=off>{{printf "%s" .Log}}</textarea>
@@ -21,7 +27,6 @@ Main page.
textarea.scrollTop = textarea.scrollHeight;
</script>
<br><br>
-
{{with $.MemcacheStats}}
<table class="list_table">
<caption><a href="https://pkg.go.dev/google.golang.org/appengine/memcache?tab=doc#Item" target="_blank">Memcache stats:</a></caption>
diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 2dce92d55..6a84c758b 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -30,6 +30,7 @@ import (
"google.golang.org/appengine/v2"
db "google.golang.org/appengine/v2/datastore"
"google.golang.org/appengine/v2/log"
+ "google.golang.org/appengine/v2/user"
)
func initAPIHandlers() {
@@ -363,6 +364,10 @@ func addCommitInfoToBugImpl(c context.Context, bug *Bug, com dashapi.Commit) (bo
}
func apiJobPoll(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+ if stop, err := emergentlyStopped(c); err != nil || stop {
+ // The bot's operation was aborted. Don't accept new crash reports.
+ return &dashapi.JobPollResp{}, err
+ }
req := new(dashapi.JobPollReq)
if err := json.Unmarshal(payload, req); err != nil {
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -699,6 +704,10 @@ const (
)
func apiReportCrash(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
+ if stop, err := emergentlyStopped(c); err != nil || stop {
+ // The bot's operation was aborted. Don't accept new crash reports.
+ return &dashapi.ReportCrashResp{}, err
+ }
req := new(dashapi.Crash)
if err := json.Unmarshal(payload, req); err != nil {
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -1632,3 +1641,23 @@ func apiSaveDiscussion(c context.Context, r *http.Request, payload []byte) (inte
}
return nil, mergeDiscussion(c, d)
}
+
+func emergentlyStopped(c context.Context) (bool, error) {
+ keys, err := db.NewQuery("EmergencyStop").
+ Limit(1).
+ KeysOnly().
+ GetAll(c, nil)
+ if err != nil {
+ return false, err
+ }
+ return len(keys) > 0, nil
+}
+
+func recordEmergencyStop(c context.Context) error {
+ key := db.NewKey(c, "EmergencyStop", "all", 0, nil)
+ _, err := db.Put(c, key, &EmergencyStop{
+ Time: timeNow(c),
+ User: user.Current(c).Email,
+ })
+ return err
+}
diff --git a/dashboard/app/api_test.go b/dashboard/app/api_test.go
index 8d63ce7a7..4a872231e 100644
--- a/dashboard/app/api_test.go
+++ b/dashboard/app/api_test.go
@@ -5,6 +5,9 @@ package main
import (
"testing"
+ "time"
+
+ "github.com/google/syzkaller/dashboard/dashapi"
)
func TestClientSecretOK(t *testing.T) {
@@ -63,3 +66,139 @@ func TestClientNamespaceOK(t *testing.T) {
t.Errorf("Unexpected error %v %v", got, err)
}
}
+
+func TestEmergentlyStoppedEmail(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ client := c.publicClient
+ build := testBuild(1)
+ client.UploadBuild(build)
+
+ crash := testCrash(build, 1)
+ client.ReportCrash(crash)
+
+ c.advanceTime(time.Hour)
+ _, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+ c.expectOK(err)
+
+ // There should be no email.
+ c.advanceTime(time.Hour)
+ c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedReproEmail(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ client := c.publicClient
+ build := testBuild(1)
+ client.UploadBuild(build)
+
+ crash := testCrash(build, 1)
+ client.ReportCrash(crash)
+ c.pollEmailBug()
+
+ crash2 := testCrash(build, 1)
+ crash2.ReproOpts = []byte("repro opts")
+ crash2.ReproSyz = []byte("getpid()")
+ client.ReportCrash(crash2)
+
+ c.advanceTime(time.Hour)
+ _, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+ c.expectOK(err)
+
+ // There should be no email.
+ c.advanceTime(time.Hour)
+ c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedExternalReport(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ client := c.client
+ build := testBuild(1)
+ client.UploadBuild(build)
+
+ crash := testCrash(build, 1)
+ client.ReportCrash(crash)
+
+ c.advanceTime(time.Hour)
+ _, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+ c.expectOK(err)
+
+ // There should be no email.
+ c.advanceTime(time.Hour)
+ client.pollBugs(0)
+}
+
+func TestEmergentlyStoppedEmailJob(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ client := c.publicClient
+ build := testBuild(1)
+ client.UploadBuild(build)
+
+ crash := testCrash(build, 1)
+ crash.ReproOpts = []byte("repro opts")
+ crash.ReproSyz = []byte("getpid()")
+ client.ReportCrash(crash)
+ sender := c.pollEmailBug().Sender
+ c.incomingEmail(sender, "#syz upstream\n")
+ sender = c.pollEmailBug().Sender
+
+ // Send a patch testing request.
+ c.advanceTime(time.Hour)
+ c.incomingEmail(sender, syzTestGitBranchSamplePatch,
+ EmailOptMessageID(1), EmailOptFrom("test@requester.com"),
+ EmailOptCC([]string{"somebody@else.com", "test@syzkaller.com"}))
+ c.expectNoEmail()
+
+ // Emulate a finished job.
+ pollResp := client.pollJobs(build.Manager)
+ c.expectEQ(pollResp.Type, dashapi.JobTestPatch)
+
+ c.advanceTime(time.Hour)
+ jobDoneReq := &dashapi.JobDoneReq{
+ ID: pollResp.ID,
+ Build: *build,
+ CrashTitle: "test crash title",
+ CrashLog: []byte("test crash log"),
+ CrashReport: []byte("test crash report"),
+ }
+ client.JobDone(jobDoneReq)
+
+ // Now we emergently stop syzbot.
+ c.advanceTime(time.Hour)
+ _, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+ c.expectOK(err)
+
+ // There should be no email.
+ c.advanceTime(time.Hour)
+ c.expectNoEmail()
+}
+
+func TestEmergentlyStoppedCrashReport(t *testing.T) {
+ c := NewCtx(t)
+ defer c.Close()
+
+ client := c.publicClient
+ build := testBuild(1)
+ client.UploadBuild(build)
+
+ // Now we emergently stop syzbot.
+ c.advanceTime(time.Hour)
+ _, err := c.AuthGET(AccessAdmin, "/admin?action=emergency_stop")
+ c.expectOK(err)
+
+ crash := testCrash(build, 1)
+ crash.ReproOpts = []byte("repro opts")
+ crash.ReproSyz = []byte("getpid()")
+ client.ReportCrash(crash)
+
+ listResp, err := client.BugList()
+ c.expectOK(err)
+ c.expectEQ(len(listResp.List), 0)
+}
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index 659474005..71f4a92fc 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -986,6 +986,13 @@ func (bug *Bug) dashapiStatus() (dashapi.BugStatus, error) {
return status, nil
}
+// If an entity of type EmergencyStop exists, syzbot's operation is paused until
+// a support engineer deletes it from the DB.
+type EmergencyStop struct {
+ Time time.Time
+ User string
+}
+
func addCrashReference(c context.Context, crashID int64, bugKey *db.Key, ref CrashReference) error {
crash := new(Crash)
crashKey := db.NewKey(c, "Crash", "", crashID, bugKey)
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 8d56c0ee5..ae8e138af 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -203,6 +203,9 @@ type uiAdminPage struct {
CauseBisectionsLink string
JobOverviewLink string
MemcacheStats *memcache.Statistics
+ Stopped bool
+ StopLink string
+ MoreStopClicks int
}
type uiManagerPage struct {
@@ -904,6 +907,10 @@ func handleAdmin(c context.Context, w http.ResponseWriter, r *http.Request) erro
}
case "invalidate_bisection":
return handleInvalidateBisection(c, w, r)
+ case "emergency_stop":
+ if err := recordEmergencyStop(c); err != nil {
+ return fmt.Errorf("failed to record an emergency stop: %w", err)
+ }
default:
return fmt.Errorf("%w: unknown action %q", ErrClientBadRequest, action)
}
@@ -963,15 +970,28 @@ func handleAdmin(c context.Context, w http.ResponseWriter, r *http.Request) erro
return err
})
}
+ alreadyStopped := false
+ g.Go(func() error {
+ var err error
+ alreadyStopped, err = emergentlyStopped(c)
+ return err
+ })
err = g.Wait()
if err != nil {
return err
}
data := &uiAdminPage{
- Header: hdr,
- Log: errorLog,
- Managers: makeManagerList(managers, hdr.Namespace),
- MemcacheStats: memcacheStats,
+ Header: hdr,
+ Log: errorLog,
+ Managers: makeManagerList(managers, hdr.Namespace),
+ MemcacheStats: memcacheStats,
+ Stopped: alreadyStopped,
+ MoreStopClicks: 2,
+ StopLink: html.AmendURL("/admin", "stop_clicked", "1"),
+ }
+ if r.FormValue("stop_clicked") != "" {
+ data.MoreStopClicks = 1
+ data.StopLink = html.AmendURL("/admin", "action", "emergency_stop")
}
if r.FormValue("job_type") != "" {
data.TypeJobs = &uiJobList{Title: "Last jobs:", Jobs: typeJobs}
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 42fffb6f3..8eaf5e5ff 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -105,6 +105,16 @@ func (cfg *EmailConfig) Validate() error {
// handleEmailPoll is called by cron and sends emails for new bugs, if any.
func handleEmailPoll(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
+ stop, err := emergentlyStopped(c)
+ if err != nil {
+ log.Errorf(c, "emergency stop querying failed: %v", err)
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if stop {
+ log.Errorf(c, "aborting email poll due to an emergency stop")
+ return
+ }
if err := emailPollJobs(c); err != nil {
log.Errorf(c, "job poll failed: %v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
@@ -476,8 +486,14 @@ func handleIncomingMail(w http.ResponseWriter, r *http.Request) {
}
log.Infof(c, "received email at %q, source %q", myEmail, source)
if source == dashapi.NoDiscussion {
+ if stop, err := emergentlyStopped(c); err != nil || stop {
+ log.Errorf(c, "abort email processing due to emergency stop (stop %v, err %v)",
+ stop, err)
+ return
+ }
err = processIncomingEmail(c, msg)
} else {
+ // Discussions are safe to handle even during an emergency stop.
err = processDiscussionEmail(c, msg, source)
}
if err != nil {
diff --git a/dashboard/app/reporting_external.go b/dashboard/app/reporting_external.go
index e6f65c738..625476918 100644
--- a/dashboard/app/reporting_external.go
+++ b/dashboard/app/reporting_external.go
@@ -19,6 +19,9 @@ import (
// and report back bug status updates with apiReportingUpdate.
func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+ if stop, err := emergentlyStopped(c); err != nil || stop {
+ return &dashapi.PollBugsResponse{}, err
+ }
req := new(dashapi.PollBugsRequest)
if err := json.Unmarshal(payload, req); err != nil {
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -36,6 +39,9 @@ func apiReportingPollBugs(c context.Context, r *http.Request, payload []byte) (i
}
func apiReportingPollNotifications(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+ if stop, err := emergentlyStopped(c); err != nil || stop {
+ return &dashapi.PollNotificationsResponse{}, err
+ }
req := new(dashapi.PollNotificationsRequest)
if err := json.Unmarshal(payload, req); err != nil {
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
@@ -48,6 +54,9 @@ func apiReportingPollNotifications(c context.Context, r *http.Request, payload [
}
func apiReportingPollClosed(c context.Context, r *http.Request, payload []byte) (interface{}, error) {
+ if stop, err := emergentlyStopped(c); err != nil || stop {
+ return &dashapi.PollClosedResponse{}, err
+ }
req := new(dashapi.PollClosedRequest)
if err := json.Unmarshal(payload, req); err != nil {
return nil, fmt.Errorf("failed to unmarshal request: %w", err)
diff --git a/pkg/html/pages/style.css b/pkg/html/pages/style.css
index 9dd1bca7f..156bf61ba 100644
--- a/pkg/html/pages/style.css
+++ b/pkg/html/pages/style.css
@@ -409,3 +409,15 @@ aside {
.collapsible-show .show-icon {
display: none;
}
+
+.emergency-stop {
+ background-color: yellow;
+ padding: 5pt;
+ margin-bottom: 5pt;
+}
+
+.emergency-stopped {
+ background-color: coral;
+ padding: 5pt;
+ margin-bottom: 5pt;
+}
generated by ggit (джигит) mrf-deployment (git 2.46.0) at 2026-03-11 19:42:19 +0000