dashboard: limit the rate of job generation on pullJob

syz-cis call the method every 10 seconds, there's no sense in executing
the heavy logic that much often.

3 files changed, 60 insertions, 0 deletions

diff --git a/dashboard/app/bisect_test.go b/dashboard/app/bisect_test.go
index 2c345ffec..2e2bb4b2d 100644
--- a/dashboard/app/bisect_test.go
+++ b/dashboard/app/bisect_test.go
@@ -91,6 +91,7 @@ func TestBisectCause(t *testing.T) {
 
 	// BisectCause #2
 	pollResp2 := pollResp
+	c.advanceTime(time.Minute)
 	pollResp = c.client2.pollJobs(build.Manager)
 	c.expectNE(pollResp.ID, pollResp2.ID)
 	c.expectEQ(pollResp.ReproOpts, []byte("repro opts 2"))
@@ -248,6 +249,7 @@ https://goo.gl/tpsmEJ#testing-patches`,
 		"bugs2@syzkaller.com",
 		"default2@maintainers.com",
 	})
+	c.advanceTime(time.Minute)
 	pollResp = c.client2.pollJobs(build.Manager)
 
 	// Bisection succeeded.
@@ -307,6 +309,7 @@ https://goo.gl/tpsmEJ#testing-patches`,
 	}
 
 	// BisectFix #2
+	c.advanceTime(time.Minute)
 	pollResp = c.client2.pollJobs(build.Manager)
 	c.expectNE(pollResp.ID, "")
 	c.expectEQ(pollResp.Type, dashapi.JobBisectFix)
@@ -320,6 +323,7 @@ https://goo.gl/tpsmEJ#testing-patches`,
 	c.expectOK(c.client2.JobDone(done))
 
 	// BisectFix #3
+	c.advanceTime(time.Minute)
 	pollResp = c.client2.pollJobs(build.Manager)
 	c.expectNE(pollResp.ID, "")
 	c.expectEQ(pollResp.Type, dashapi.JobBisectFix)
@@ -332,6 +336,7 @@ https://goo.gl/tpsmEJ#testing-patches`,
 	c.expectOK(c.client2.JobDone(done))
 
 	// BisectFix #4
+	c.advanceTime(time.Minute)
 	pollResp = c.client2.pollJobs(build.Manager)
 	c.expectNE(pollResp.ID, "")
 	c.expectEQ(pollResp.Type, dashapi.JobBisectFix)
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index 04362c4a3..2ed019b1b 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -34,6 +34,7 @@ type Manager struct {
 	FailedSyzBuildBug string
 	LastAlive         time.Time
 	CurrentUpTime     time.Duration
+	LastGeneratedJob  time.Time
 }
 
 // ManagerStats holds per-day manager runtime stats.
diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go
index f97f443ad..36afc59b2 100644
--- a/dashboard/app/jobs.go
+++ b/dashboard/app/jobs.go
@@ -238,6 +238,13 @@ func getNextJob(c context.Context, managers map[string]dashapi.ManagerJobs) (*Jo
 	if job != nil || err != nil {
 		return job, jobKey, err
 	}
+	// Each syz-ci polls dashboard every 10 seconds. At the times when there are no
+	// matching jobs, it just doesn't make much sense to execute heavy algorithms that
+	// try to generate them too often.
+	// Note that it won't affect user-created jobs as they are not auto-generated.
+	if err := throttleJobGeneration(c, managers); err != nil {
+		return nil, nil, err
+	}
 	// We need both C and syz repros, but the crazy datastore query restrictions
 	// do not allow to use ReproLevel>ReproLevelNone in the query. So we do 2 separate queries.
 	// C repros tend to be of higher reliability so maybe it's not bad.
@@ -248,6 +255,53 @@ func getNextJob(c context.Context, managers map[string]dashapi.ManagerJobs) (*Jo
 	return createBisectJob(c, managers, ReproLevelSyz)
 }
 
+const jobGenerationPeriod = time.Minute
+
+func throttleJobGeneration(c context.Context, managers map[string]dashapi.ManagerJobs) error {
+	drop := map[string]struct{}{}
+	for name := range managers {
+		// Technically the key is Namespace+Manager, so it's not guaranteed
+		// that there'll be only one.
+		// But for throttling purposes any single entity will do.
+		// Also note that we do the query outside of the transaction as
+		// datastore prohibits non-ancestor queries.
+		keys, err := db.NewQuery("Manager").
+			Filter("Name=", name).
+			Limit(1).
+			KeysOnly().
+			GetAll(c, nil)
+		if err != nil {
+			return err
+		}
+		if len(keys) == 0 {
+			drop[name] = struct{}{}
+			continue
+		}
+		tx := func(c context.Context) error {
+			manager := new(Manager)
+			if err := db.Get(c, keys[0], manager); err != nil {
+				return fmt.Errorf("failed to get %v", keys[0])
+			}
+			if timeNow(c).Sub(manager.LastGeneratedJob) < jobGenerationPeriod {
+				drop[name] = struct{}{}
+				return nil
+			}
+			manager.LastGeneratedJob = timeNow(c)
+			if _, err = db.Put(c, keys[0], manager); err != nil {
+				return fmt.Errorf("failed to put Manager: %v", err)
+			}
+			return nil
+		}
+		if err := db.RunInTransaction(c, tx, &db.TransactionOptions{}); err != nil {
+			return fmt.Errorf("failed to throttle: %v", err)
+		}
+	}
+	for name := range drop {
+		delete(managers, name)
+	}
+	return nil
+}
+
 // Ensure that for each manager there's one pending retest repro job.
 func updateRetestReproJobs(c context.Context, ns string) error {
 	if config.Obsoleting.ReproRetestPeriod == 0 {