dashboard: revoke no longer working reproducers

Reuse the existing patch testing mechanism to periodically re-check
existing reproducers. If a repro no longer works, mark it as such and
don't consider during bug obsoletion testing.

For obsoleted bugs, pass also a string argument specifying the reason
for the obsoletion. This will make email messages more user-friendly and
also let us later estimate the amount of no longer relevant bugs that we
had on the dashboard.

14 files changed, 454 insertions, 88 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 411c46440..0683802ee 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -753,6 +753,7 @@ func reportCrash(c context.Context, build *Build, req *dashapi.Crash) (*Bug, err
 		}
 		if bug.ReproLevel < reproLevel {
 			bug.ReproLevel = reproLevel
+			bug.HeadReproLevel = reproLevel
 		}
 		if len(req.Report) != 0 {
 			bug.HasReport = true
@@ -777,20 +778,27 @@ func reportCrash(c context.Context, build *Build, req *dashapi.Crash) (*Bug, err
 	return bug, nil
 }
 
-func saveCrash(c context.Context, ns string, req *dashapi.Crash, bug *Bug, bugKey *db.Key, build *Build) error {
-	// Reporting priority of this crash.
+func (crash *Crash) UpdateReportingPriority(build *Build, bug *Bug) {
 	prio := int64(kernelRepoInfo(build).ReportingPriority) * 1e6
-	if len(req.ReproC) != 0 {
-		prio += 4e12
-	} else if len(req.ReproSyz) != 0 {
-		prio += 2e12
+	divReproPrio := int64(1)
+	if crash.ReproIsRevoked {
+		divReproPrio = 10
 	}
-	if req.Title == bug.Title {
+	if crash.ReproC > 0 {
+		prio += 4e12 / divReproPrio
+	} else if crash.ReproSyz > 0 {
+		prio += 2e12 / divReproPrio
+	}
+	if crash.Title == bug.Title {
 		prio += 1e8 // prefer reporting crash that matches bug title
 	}
 	if build.Arch == targets.AMD64 {
 		prio += 1e3
 	}
+	crash.ReportLen = prio
+}
+
+func saveCrash(c context.Context, ns string, req *dashapi.Crash, bug *Bug, bugKey *db.Key, build *Build) error {
 	crash := &Crash{
 		Title:   req.Title,
 		Manager: build.Manager,
@@ -800,7 +808,6 @@ func saveCrash(c context.Context, ns string, req *dashapi.Crash, bug *Bug, bugKe
 			GetEmails(req.Recipients, dashapi.To),
 			GetEmails(req.Recipients, dashapi.Cc)),
 		ReproOpts: req.ReproOpts,
-		ReportLen: prio,
 		Flags:     int64(req.Flags),
 	}
 	var err error
@@ -819,6 +826,7 @@ func saveCrash(c context.Context, ns string, req *dashapi.Crash, bug *Bug, bugKe
 	if crash.MachineInfo, err = putText(c, ns, textMachineInfo, req.MachineInfo, true); err != nil {
 		return err
 	}
+	crash.UpdateReportingPriority(build, bug)
 	crashKey := db.NewIncompleteKey(c, "Crash", bugKey)
 	if _, err = db.Put(c, crashKey, crash); err != nil {
 		return fmt.Errorf("failed to put crash: %v", err)
@@ -1314,7 +1322,7 @@ func needReproForBug(c context.Context, bug *Bug) bool {
 	if syzErrorTitleRe.MatchString(bug.Title) {
 		bestReproLevel = ReproLevelSyz
 	}
-	if bug.ReproLevel < bestReproLevel {
+	if bug.HeadReproLevel < bestReproLevel {
 		// We have not found a best-level repro yet, try until we do.
 		return bug.NumRepro < maxReproPerBug || timeSince(c, bug.LastReproTime) >= reproRetryPeriod
 	}
@@ -1441,3 +1449,11 @@ func checkClient(conf *GlobalConfig, name0, secretPassword, oauthSubject string)
 	}
 	return "", ErrAccess
 }
+
+func handleRetestRepros(w http.ResponseWriter, r *http.Request) {
+	c := appengine.NewContext(r)
+	err := updateRetestReproJobs(c)
+	if err != nil {
+		log.Errorf(c, "failed to update retest repro jobs: %v", err)
+	}
+}
diff --git a/dashboard/app/app_test.go b/dashboard/app/app_test.go
index b4b07ddae..4c1fd32ad 100644
--- a/dashboard/app/app_test.go
+++ b/dashboard/app/app_test.go
@@ -50,6 +50,7 @@ var testConfig = &GlobalConfig{
 		MaxPeriod:         100 * 24 * time.Hour,
 		NonFinalMinPeriod: 40 * 24 * time.Hour,
 		NonFinalMaxPeriod: 60 * 24 * time.Hour,
+		ReproRetestPeriod: 100 * 24 * time.Hour,
 	},
 	DefaultNamespace: "test1",
 	Namespaces: map[string]*Config{
diff --git a/dashboard/app/config.go b/dashboard/app/config.go
index ba6a0d542..a82890547 100644
--- a/dashboard/app/config.go
+++ b/dashboard/app/config.go
@@ -114,6 +114,7 @@ type ObsoletingConfig struct {
 	MaxPeriod         time.Duration
 	NonFinalMinPeriod time.Duration
 	NonFinalMaxPeriod time.Duration
+	ReproRetestPeriod time.Duration
 }
 
 // ConfigManager describes a single syz-manager instance.
diff --git a/dashboard/app/cron.yaml b/dashboard/app/cron.yaml
index f46621a05..fca974bb5 100644
--- a/dashboard/app/cron.yaml
+++ b/dashboard/app/cron.yaml
@@ -10,6 +10,8 @@ cron:
   schedule: every 1 hours
 - url: /kcidb_poll
   schedule: every 5 minutes
+- url: /retest_repros
+  schedule: every 1 hours
 - url: /_ah/datastore_admin/backup.create?name=backup&filesystem=gs&gs_bucket_name=syzkaller-backups&kind=Bug&kind=Build&kind=Crash&kind=CrashLog&kind=CrashReport&kind=Error&kind=Job&kind=KernelConfig&kind=Manager&kind=ManagerStats&kind=Patch&kind=ReportingState&kind=ReproC&kind=ReproSyz
   schedule: every monday 00:00
   target: ah-builtin-python-bundle
diff --git a/dashboard/app/entities.go b/dashboard/app/entities.go
index e19d3a9e2..14f3fd785 100644
--- a/dashboard/app/entities.go
+++ b/dashboard/app/entities.go
@@ -79,16 +79,20 @@ type Build struct {
 }
 
 type Bug struct {
-	Namespace      string
-	Seq            int64 // sequences of the bug with the same title
-	Title          string
-	MergedTitles   []string // crash titles that we already merged into this bug
-	AltTitles      []string // alternative crash titles that we may merge into this bug
-	Status         int
-	DupOf          string
-	NumCrashes     int64
-	NumRepro       int64
+	Namespace    string
+	Seq          int64 // sequences of the bug with the same title
+	Title        string
+	MergedTitles []string // crash titles that we already merged into this bug
+	AltTitles    []string // alternative crash titles that we may merge into this bug
+	Status       int
+	StatusReason dashapi.BugStatusReason // e.g. if the bug status is "invalid", here's the reason why
+	DupOf        string
+	NumCrashes   int64
+	NumRepro     int64
+	// ReproLevel is the best ever found repro level for this bug.
+	// HeadReproLevel is best known repro level that still works on the HEAD commit.
 	ReproLevel     dashapi.ReproLevel
+	HeadReproLevel dashapi.ReproLevel `datastore:"HeadReproLevel"`
 	BisectCause    BisectStatus
 	BisectFix      BisectStatus
 	HasReport      bool
@@ -113,6 +117,29 @@ type Bug struct {
 	DailyStats  []BugDailyStats
 }
 
+func (bug *Bug) Load(ps []db.Property) error {
+	if err := db.LoadStruct(bug, ps); err != nil {
+		return err
+	}
+	headReproFound := false
+	for _, p := range ps {
+		if p.Name == "HeadReproLevel" {
+			headReproFound = true
+			break
+		}
+	}
+	if !headReproFound {
+		// The field is new, so it won't be set in all entities.
+		// Assume it to be equal to the best found repro for the bug.
+		bug.HeadReproLevel = bug.ReproLevel
+	}
+	return nil
+}
+
+func (bug *Bug) Save() ([]db.Property, error) {
+	return db.SaveStruct(bug)
+}
+
 type BugDailyStats struct {
 	Date       int // YYYYMMDD
 	CrashCount int
@@ -148,19 +175,21 @@ type BugReporting struct {
 type Crash struct {
 	// May be different from bug.Title due to AltTitles.
 	// May be empty for old bugs, in such case bug.Title is the right title.
-	Title       string
-	Manager     string
-	BuildID     string
-	Time        time.Time
-	Reported    time.Time // set if this crash was ever reported
-	Maintainers []string  `datastore:",noindex"`
-	Log         int64     // reference to CrashLog text entity
-	Flags       int64     // properties of the Crash
-	Report      int64     // reference to CrashReport text entity
-	ReproOpts   []byte    `datastore:",noindex"`
-	ReproSyz    int64     // reference to ReproSyz text entity
-	ReproC      int64     // reference to ReproC text entity
-	MachineInfo int64     // Reference to MachineInfo text entity.
+	Title           string
+	Manager         string
+	BuildID         string
+	Time            time.Time
+	Reported        time.Time // set if this crash was ever reported
+	Maintainers     []string  `datastore:",noindex"`
+	Log             int64     // reference to CrashLog text entity
+	Flags           int64     // properties of the Crash
+	Report          int64     // reference to CrashReport text entity
+	ReproOpts       []byte    `datastore:",noindex"`
+	ReproSyz        int64     // reference to ReproSyz text entity
+	ReproC          int64     // reference to ReproC text entity
+	ReproIsRevoked  bool      // the repro no longer triggers the bug on HEAD
+	LastReproRetest time.Time // the last time when the repro was re-checked
+	MachineInfo     int64     // Reference to MachineInfo text entity.
 	// Custom crash priority for reporting (greater values are higher priority).
 	// For example, a crash in mainline kernel has higher priority than a crash in a side branch.
 	// For historical reasons this is called ReportLen.
diff --git a/dashboard/app/index.yaml b/dashboard/app/index.yaml
index abe7d3920..fe249f7e6 100644
--- a/dashboard/app/index.yaml
+++ b/dashboard/app/index.yaml
@@ -41,6 +41,12 @@ indexes:
 
 - kind: Bug
   properties:
+  - name: Namespace
+  - name: Status
+  - name: LastTime
+
+- kind: Bug
+  properties:
   - name: HappenedOn
   - name: Status
 
diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go
index 8ca2d17bd..518d05396 100644
--- a/dashboard/app/jobs.go
+++ b/dashboard/app/jobs.go
@@ -41,10 +41,16 @@ func handleTestRequest(c context.Context, bugID, user, extID, link, patch, repo,
 	}
 	bugReporting, _ := bugReportingByID(bug, bugID)
 	now := timeNow(c)
+	crash, crashKey, err := findCrashForBug(c, bug)
+	if err != nil {
+		log.Errorf(c, "failed to find a crash: %v", err)
+		return ""
+	}
 	reply, err := addTestJob(c, &testJobArgs{
 		bug: bug, bugKey: bugKey, bugReporting: bugReporting,
 		user: user, extID: extID, link: link, patch: patch,
 		repo: repo, branch: branch, jobCC: jobCC,
+		crash: crash, crashKey: crashKey,
 	}, now)
 	if err != nil {
 		log.Errorf(c, "test request failed: %v", err)
@@ -79,6 +85,8 @@ func handleTestRequest(c context.Context, bugID, user, extID, link, patch, repo,
 }
 
 type testJobArgs struct {
+	crash        *Crash
+	crashKey     *db.Key
 	bug          *Bug
 	bugKey       *db.Key
 	bugReporting *BugReporting
@@ -92,15 +100,11 @@ type testJobArgs struct {
 }
 
 func addTestJob(c context.Context, args *testJobArgs, now time.Time) (string, error) {
-	crash, crashKey, err := findCrashForBug(c, args.bug)
-	if err != nil {
-		return "", err
-	}
-	if reason := checkTestJob(c, args.bug, args.bugReporting, crash,
+	if reason := checkTestJob(c, args.bug, args.bugReporting, args.crash,
 		args.repo, args.branch); reason != "" {
 		return reason, nil
 	}
-	manager := crash.Manager
+	manager := args.crash.Manager
 	for _, ns := range config.Namespaces {
 		if mgr, ok := ns.Managers[manager]; ok {
 			if mgr.RestrictedTestingRepo != "" && args.repo != mgr.RestrictedTestingRepo {
@@ -116,18 +120,22 @@ func addTestJob(c context.Context, args *testJobArgs, now time.Time) (string, er
 	if err != nil {
 		return "", err
 	}
+	reportingName := ""
+	if args.bugReporting != nil {
+		reportingName = args.bugReporting.Name
+	}
 	job := &Job{
 		Type:         JobTestPatch,
 		Created:      now,
 		User:         args.user,
 		CC:           args.jobCC,
-		Reporting:    args.bugReporting.Name,
+		Reporting:    reportingName,
 		ExtID:        args.extID,
 		Link:         args.link,
 		Namespace:    args.bug.Namespace,
 		Manager:      manager,
 		BugTitle:     args.bug.displayTitle(),
-		CrashID:      crashKey.IntID(),
+		CrashID:      args.crashKey.IntID(),
 		KernelRepo:   args.repo,
 		KernelBranch: args.branch,
 		Patch:        patchID,
@@ -139,12 +147,16 @@ func addTestJob(c context.Context, args *testJobArgs, now time.Time) (string, er
 		// We can get 2 emails for the same request: one direct and one from a mailing list.
 		// Filter out such duplicates (for dup we only need link update).
 		var jobs []*Job
-		keys, err := db.NewQuery("Job").
-			Ancestor(args.bugKey).
-			Filter("ExtID=", args.extID).
-			GetAll(c, &jobs)
-		if len(jobs) > 1 || err != nil {
-			return fmt.Errorf("failed to query jobs: jobs=%v err=%v", len(jobs), err)
+		var keys []*db.Key
+		var err error
+		if args.extID != "" {
+			keys, err = db.NewQuery("Job").
+				Ancestor(args.bugKey).
+				Filter("ExtID=", args.extID).
+				GetAll(c, &jobs)
+			if len(jobs) > 1 || err != nil {
+				return fmt.Errorf("failed to query jobs: jobs=%v err=%v", len(jobs), err)
+			}
 		}
 		if len(jobs) != 0 {
 			// The job is already present, update link.
@@ -195,7 +207,7 @@ func checkTestJob(c context.Context, bug *Bug, bugReporting *BugReporting, crash
 	case bug.Status == BugStatusInvalid:
 		return "This bug is already marked as invalid. No point in testing."
 	// TODO(dvyukov): for BugStatusDup check status of the canonical bug.
-	case !bugReporting.Closed.IsZero():
+	case bugReporting != nil && !bugReporting.Closed.IsZero():
 		return "This bug is already upstreamed. Please test upstream."
 	}
 	return ""
@@ -234,6 +246,83 @@ func getNextJob(c context.Context, managers map[string]dashapi.ManagerJobs) (*Jo
 	return createBisectJob(c, managers, ReproLevelSyz)
 }
 
+// Ensure that for each manager there's one pending retest repro job.
+func updateRetestReproJobs(c context.Context) error {
+	if config.Obsoleting.ReproRetestPeriod == 0 {
+		return nil
+	}
+	var jobs []*Job
+	_, err := db.NewQuery("Job").
+		Filter("Finished=", time.Time{}).
+		GetAll(c, &jobs)
+	if err != nil {
+		return fmt.Errorf("failed to query jobs: %w", err)
+	}
+	managerHasJob := map[string]bool{}
+	for _, job := range jobs {
+		if job.User == "" && job.Type == JobTestPatch {
+			managerHasJob[job.Manager] = true
+		}
+	}
+	// Let's save resources and only re-check repros for bugs with no recent crashes.
+	now := timeNow(c)
+	maxLastTime := now.Add(-config.Obsoleting.ReproRetestPeriod)
+	bugs, keys, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		return query.Filter("Status=", BugStatusOpen).
+			Filter("LastTime<", maxLastTime)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to query bugs: %w", err)
+	}
+	for id, bug := range bugs {
+		err := handleRetestForBug(c, now, bug, keys[id], managerHasJob)
+		if err != nil {
+			return fmt.Errorf("bug %v repro retesting failed: %w", keys[id], err)
+		}
+	}
+	return nil
+}
+
+func handleRetestForBug(c context.Context, now time.Time, bug *Bug, bugKey *db.Key,
+	managerHasJob map[string]bool) error {
+	crashes, crashKeys, err := queryCrashesForBug(c, bugKey, maxCrashes)
+	if err != nil {
+		return err
+	}
+	for crashID, crash := range crashes {
+		if crash.ReproSyz == 0 && crash.ReproC == 0 {
+			continue
+		}
+		if now.Sub(crash.LastReproRetest) < config.Obsoleting.ReproRetestPeriod {
+			continue
+		}
+		// TODO: check if the manager can do such jobs.
+		if managerHasJob[crash.Manager] {
+			continue
+		}
+		build, err := loadBuild(c, bug.Namespace, crash.BuildID)
+		if err != nil {
+			return err
+		}
+		reason, err := addTestJob(c, &testJobArgs{
+			crash:    crash,
+			crashKey: crashKeys[crashID],
+			bug:      bug,
+			bugKey:   bugKey,
+			repo:     build.KernelRepo,
+			branch:   build.KernelBranch,
+		}, now)
+		if reason != "" {
+			return fmt.Errorf("job not added for reason %s", reason)
+		}
+		if err != nil {
+			return fmt.Errorf("failed to add job: %w", err)
+		}
+		managerHasJob[crash.Manager] = true
+	}
+	return nil
+}
+
 func createBisectJob(c context.Context, managers map[string]dashapi.ManagerJobs,
 	reproLevel dashapi.ReproLevel) (*Job, *db.Key, error) {
 	causeManagers := make(map[string]bool)
@@ -472,6 +561,77 @@ func createJobResp(c context.Context, job *Job, jobKey *db.Key) (*dashapi.JobPol
 	return resp, false, nil
 }
 
+// It would be easier to just check if the User field is empty, but let's also not
+// miss the situation when some actual user sends a patch testing request without
+// patch.
+func isRetestReproJob(job *Job, build *Build) bool {
+	return job.Type == JobTestPatch &&
+		job.Patch == 0 &&
+		job.KernelRepo == build.KernelRepo &&
+		job.KernelBranch == build.KernelBranch
+}
+
+func handleRetestedRepro(c context.Context, now time.Time, job *Job, jobKey *db.Key, crashTitle string) error {
+	bugKey := jobKey.Parent()
+	crashKey := db.NewKey(c, "Crash", "", job.CrashID, bugKey)
+	crash := new(Crash)
+	if err := db.Get(c, crashKey, crash); err != nil {
+		return fmt.Errorf("failed to get crash: %v", crashKey)
+	}
+	build, err := loadBuild(c, job.Namespace, crash.BuildID)
+	if err != nil {
+		return fmt.Errorf("failed to query build: %w", err)
+	}
+	if !isRetestReproJob(job, build) {
+		return nil
+	}
+	bug := new(Bug)
+	if err := db.Get(c, bugKey, bug); err != nil {
+		return fmt.Errorf("failed to get bug: %v", bugKey)
+	}
+	// Update the crash.
+	crash.LastReproRetest = now
+	if crashTitle == "" {
+		// If there was any crash at all, the repro is still not worth discarding.
+		crash.ReproIsRevoked = true
+	}
+	crash.UpdateReportingPriority(build, bug)
+	if _, err := db.Put(c, crashKey, crash); err != nil {
+		return fmt.Errorf("failed to put crash: %v", err)
+	}
+	reproCrashes, crashKeys, err := queryCrashesForBug(c, bugKey, 2)
+	if err != nil {
+		return fmt.Errorf("failed to fetch crashes with repro: %v", err)
+	}
+	// Now we can update the bug.
+	bug.HeadReproLevel = ReproLevelNone
+	for id, bestCrash := range reproCrashes {
+		if crashKeys[id].Equal(crashKey) {
+			// In Datastore, we don't see previous writes in a transaction...
+			bestCrash = crash
+		}
+		if bestCrash.ReproIsRevoked {
+			continue
+		}
+		if bestCrash.ReproC > 0 {
+			bug.HeadReproLevel = ReproLevelC
+		} else if bug.HeadReproLevel != ReproLevelC && bestCrash.ReproSyz > 0 {
+			bug.HeadReproLevel = ReproLevelSyz
+		}
+	}
+	if crashTitle != "" {
+		// We don't want to confuse users, so only update LastTime if the generated crash
+		// really relates to the existing bug.
+		if stringInList(bug.AltTitles, crashTitle) {
+			bug.LastTime = now
+		}
+	}
+	if _, err := db.Put(c, bugKey, bug); err != nil {
+		return fmt.Errorf("failed to put bug: %v", err)
+	}
+	return nil
+}
+
 // doneJob is called by syz-ci to mark completion of a job.
 func doneJob(c context.Context, req *dashapi.JobDoneReq) error {
 	jobID := req.ID
@@ -488,6 +648,12 @@ func doneJob(c context.Context, req *dashapi.JobDoneReq) error {
 		if !job.Finished.IsZero() {
 			return fmt.Errorf("job %v: already finished", jobID)
 		}
+		if job.Type == JobTestPatch {
+			err := handleRetestedRepro(c, now, job, jobKey, req.CrashTitle)
+			if err != nil {
+				return fmt.Errorf("job %v: failed to handle retested repro, %w", jobID, err)
+			}
+		}
 		ns := job.Namespace
 		if req.Build.ID != "" {
 			if _, isNewBuild, err := uploadBuild(c, now, ns, &req.Build, BuildJob); err != nil {
@@ -570,6 +736,13 @@ func updateBugBisection(c context.Context, job *Job, jobKey *db.Key, req *dashap
 	if _, err := db.Put(c, bugKey, bug); err != nil {
 		return fmt.Errorf("failed to put bug: %v", err)
 	}
+	// The repro is not working on the HEAD commit anymore, update the repro status.
+	if job.Type == JobBisectFix && req.Error == nil && len(req.Commits) > 0 {
+		err := handleRetestedRepro(c, now, job, jobKey, req.CrashTitle)
+		if err != nil {
+			return err
+		}
+	}
 	_, bugReporting, _, _, _ := currentReporting(c, bug)
 	// The bug is either already closed or not yet reported in the current reporting,
 	// either way we don't need to report it. If it wasn't reported, it will be reported
@@ -603,7 +776,10 @@ func pollCompletedJobs(c context.Context, typ string) ([]*dashapi.BugReport, err
 	var reports []*dashapi.BugReport
 	for i, job := range jobs {
 		if job.Reporting == "" {
-			log.Criticalf(c, "no reporting for job %v", extJobID(keys[i]))
+			if job.User != "" {
+				log.Criticalf(c, "no reporting for job %v", extJobID(keys[i]))
+			}
+			// In some cases (e.g. repro retesting), it's ok not to have a reporting.
 			continue
 		}
 		reporting := config.Namespaces[job.Namespace].ReportingByName(job.Reporting)
@@ -801,6 +977,7 @@ func loadPendingJob(c context.Context, managers map[string]dashapi.ManagerJobs)
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to query jobs: %v", err)
 	}
+	// TODO: prioritize user-initiated jobs.
 	for i, job := range jobs {
 		switch job.Type {
 		case JobTestPatch:
diff --git a/dashboard/app/jobs_test.go b/dashboard/app/jobs_test.go
index f0ff7750d..31e4eafea 100644
--- a/dashboard/app/jobs_test.go
+++ b/dashboard/app/jobs_test.go
@@ -370,6 +370,90 @@ Note: testing is done by a robot and is best-effort only.
 	c.expectEQ(pollResp.ID, "")
 }
 
+func TestReproRetestJob(t *testing.T) {
+	c := NewCtx(t)
+	defer c.Close()
+
+	build := testBuild(1)
+	build.KernelRepo = "git://mygit.com/git.git"
+	build.KernelBranch = "main"
+	c.client2.UploadBuild(build)
+
+	crash := testCrash(build, 1)
+	crash.ReproOpts = []byte("repro opts")
+	crash.ReproSyz = []byte("repro syz")
+	c.client2.ReportCrash(crash)
+	sender := c.pollEmailBug().Sender
+	_, extBugID, err := email.RemoveAddrContext(sender)
+	c.expectOK(err)
+
+	crash2 := testCrash(build, 1)
+	crash2.ReproOpts = []byte("repro opts")
+	crash2.ReproSyz = []byte("repro syz")
+	crash2.ReproC = []byte("repro C")
+	c.client2.ReportCrash(crash2)
+	// Wait until the bug is upstreamed.
+	c.advanceTime(15 * 24 * time.Hour)
+	c.pollEmailBug()
+	c.pollEmailBug()
+	bug, _, _ := c.loadBug(extBugID)
+	c.expectEQ(bug.ReproLevel, ReproLevelC)
+
+	// Let's say that the C repro testing has failed.
+	c.advanceTime(config.Obsoleting.ReproRetestPeriod + time.Hour)
+	for i := 0; i < 2; i++ {
+		c.updRetestReproJobs()
+		resp := c.client2.pollSpecificJobs(build.Manager, dashapi.ManagerJobs{TestPatches: true})
+		c.expectEQ(resp.Type, dashapi.JobTestPatch)
+		c.expectEQ(resp.KernelRepo, build.KernelRepo)
+		c.expectEQ(resp.KernelBranch, build.KernelBranch)
+		c.expectEQ(resp.Patch, []uint8(nil))
+		var done *dashapi.JobDoneReq
+		if resp.ReproC == nil {
+			// Pretend that the syz repro still works.
+			done = &dashapi.JobDoneReq{
+				ID:          resp.ID,
+				CrashTitle:  crash.Title,
+				CrashLog:    []byte("test crash log"),
+				CrashReport: []byte("test crash report"),
+			}
+		} else {
+			// Pretend that the C repro fails.
+			done = &dashapi.JobDoneReq{
+				ID: resp.ID,
+			}
+		}
+		c.client2.expectOK(c.client2.JobDone(done))
+	}
+	// Expect that the repro level is no longer ReproLevelC.
+	c.expectNoEmail()
+	bug, _, _ = c.loadBug(extBugID)
+	c.expectEQ(bug.HeadReproLevel, ReproLevelSyz)
+	// Let's also deprecate the syz repro.
+	c.advanceTime(config.Obsoleting.ReproRetestPeriod + time.Hour)
+	c.updRetestReproJobs()
+	resp := c.client2.pollSpecificJobs(build.Manager, dashapi.ManagerJobs{TestPatches: true})
+	c.expectEQ(resp.Type, dashapi.JobTestPatch)
+	c.expectEQ(resp.KernelBranch, build.KernelBranch)
+	c.expectEQ(resp.ReproC, []uint8(nil))
+	done := &dashapi.JobDoneReq{
+		ID: resp.ID,
+	}
+	c.client2.expectOK(c.client2.JobDone(done))
+	// Expect that the repro level is no longer ReproLevelC.
+	bug, _, _ = c.loadBug(extBugID)
+	c.expectEQ(bug.HeadReproLevel, ReproLevelNone)
+	c.expectEQ(bug.ReproLevel, ReproLevelC)
+	// Expect that the bug gets deprecated.
+	notif := c.pollEmailBug()
+	if !strings.Contains(notif.Body, "Auto-closing this bug as obsolete") {
+		t.Fatalf("bad notification text: %q", notif.Body)
+	}
+	// Expect that the right obsoletion reason was set.
+	bug, _, _ = c.loadBug(extBugID)
+	c.expectEQ(bug.StatusReason, dashapi.InvalidatedByRevokedRepro)
+}
+
 // Test on a restricted manager.
 func TestJobRestrictedManager(t *testing.T) {
 	c := NewCtx(t)
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index 73568da16..32288b8d5 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -60,6 +60,7 @@ func initHTTPHandlers() {
 	}
 	http.HandleFunc("/cache_update", cacheUpdate)
 	http.HandleFunc("/deprecate_assets", handleDeprecateAssets)
+	http.HandleFunc("/retest_repros", handleRetestRepros)
 }
 
 type uiMainPage struct {
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 1c48d28a2..7531f676b 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -193,7 +193,6 @@ func handleReportNotif(c context.Context, typ string, bug *Bug) (*dashapi.BugNot
 	if bug.Status != BugStatusOpen || bugReporting.Reported.IsZero() {
 		return nil, nil
 	}
-
 	if reporting.moderation &&
 		reporting.Embargo != 0 &&
 		len(bug.Commits) == 0 &&
@@ -210,11 +209,12 @@ func handleReportNotif(c context.Context, typ string, bug *Bug) (*dashapi.BugNot
 		return createNotification(c, dashapi.BugNotifUpstream, true, "", bug, reporting, bugReporting)
 	}
 	if len(bug.Commits) == 0 &&
-		bug.wontBeFixBisected() &&
+		bug.canBeObsoleted() &&
 		timeSince(c, bug.LastActivity) > notifyResendPeriod &&
 		timeSince(c, bug.LastTime) > bug.obsoletePeriod() {
 		log.Infof(c, "%v: obsoleting: %v", bug.Namespace, bug.Title)
-		return createNotification(c, dashapi.BugNotifObsoleted, false, "", bug, reporting, bugReporting)
+		why := bugObsoletionReason(bug)
+		return createNotification(c, dashapi.BugNotifObsoleted, false, string(why), bug, reporting, bugReporting)
 	}
 	if len(bug.Commits) > 0 &&
 		len(bug.PatchedOn) == 0 &&
@@ -227,14 +227,21 @@ func handleReportNotif(c context.Context, typ string, bug *Bug) (*dashapi.BugNot
 	return nil, nil
 }
 
+func bugObsoletionReason(bug *Bug) dashapi.BugStatusReason {
+	if bug.HeadReproLevel == ReproLevelNone && bug.ReproLevel != ReproLevelNone {
+		return dashapi.InvalidatedByRevokedRepro
+	}
+	return dashapi.InvalidatedByNoActivity
+}
+
 // TODO: this is what we would like to do, but we need to figure out
 // KMSAN story: we don't do fix bisection on it (rebased),
 // do we want to close all old KMSAN bugs with repros?
 // For now we only enable this in tests.
 var obsoleteWhatWontBeFixBisected = false
 
-func (bug *Bug) wontBeFixBisected() bool {
-	if bug.ReproLevel == ReproLevelNone {
+func (bug *Bug) canBeObsoleted() bool {
+	if bug.HeadReproLevel == ReproLevelNone {
 		return true
 	}
 	if obsoleteWhatWontBeFixBisected {
@@ -975,6 +982,9 @@ func incomingCommandCmd(c context.Context, now time.Time, cmd *dashapi.BugUpdate
 	default:
 		return false, internalError, fmt.Errorf("unknown bug status %v", cmd.Status)
 	}
+	if cmd.StatusReason != "" {
+		bug.StatusReason = cmd.StatusReason
+	}
 	return true, "", nil
 }
 
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 15c98aa75..a091f2f40 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -159,6 +159,7 @@ func emailPollNotifications(c context.Context) error {
 
 func emailSendBugNotif(c context.Context, notif *dashapi.BugNotification) error {
 	status, body := dashapi.BugStatusOpen, ""
+	var statusReason dashapi.BugStatusReason
 	switch notif.Type {
 	case dashapi.BugNotifUpstream:
 		body = "Sending this report upstream."
@@ -173,9 +174,15 @@ func emailSendBugNotif(c context.Context, notif *dashapi.BugNotification) error
 			"new crashes with the same signature are ignored.\n",
 			notif.Text, days)
 	case dashapi.BugNotifObsoleted:
-		body = "Auto-closing this bug as obsolete.\n" +
-			"Crashes did not happen for a while, no reproducer and no activity."
+		body = "Auto-closing this bug as obsolete.\n"
+		statusReason = dashapi.BugStatusReason(notif.Text)
+		if statusReason == dashapi.InvalidatedByRevokedRepro {
+			body += "No recent activity, existing reproducers are no longer triggering the issue."
+		} else {
+			body += "Crashes did not happen for a while, no reproducer and no activity."
+		}
 		status = dashapi.BugStatusInvalid
+
 	default:
 		return fmt.Errorf("bad notification type %v", notif.Type)
 	}
@@ -198,6 +205,7 @@ func emailSendBugNotif(c context.Context, notif *dashapi.BugNotification) error
 	cmd := &dashapi.BugUpdate{
 		ID:           notif.ID,
 		Status:       status,
+		StatusReason: statusReason,
 		Notification: true,
 	}
 	ok, reason, err := incomingCommand(c, cmd)
diff --git a/dashboard/app/repro_test.go b/dashboard/app/repro_test.go
index 9b137919c..f6a98c79b 100644
--- a/dashboard/app/repro_test.go
+++ b/dashboard/app/repro_test.go
@@ -11,9 +11,9 @@ import (
 )
 
 // Normal workflow:
-//  - upload crash -> need repro
-//  - upload syz repro -> still need repro
-//  - upload C repro -> don't need repro
+//   - upload crash -> need repro
+//   - upload syz repro -> still need repro
+//   - upload C repro -> don't need repro
 func testNeedRepro1(t *testing.T, crashCtor func(c *Ctx) *dashapi.Crash, newBug bool) {
 	c := NewCtx(t)
 	defer c.Close()
@@ -233,27 +233,30 @@ func TestNeedReproIsolated(t *testing.T) {
 		{
 			// A bug without a C repro.
 			bug: &Bug{
-				Title:      "some normal bug with a syz-repro",
-				ReproLevel: ReproLevelSyz,
+				Title:          "some normal bug with a syz-repro",
+				ReproLevel:     ReproLevelSyz,
+				HeadReproLevel: ReproLevelSyz,
 			},
 			needRepro: true,
 		},
 		{
 			// A bug for which we have recently found a repro.
 			bug: &Bug{
-				Title:         "some normal recent bug",
-				ReproLevel:    ReproLevelC,
-				LastReproTime: nowTime.Add(-time.Hour * 24),
+				Title:          "some normal recent bug",
+				ReproLevel:     ReproLevelC,
+				HeadReproLevel: ReproLevelC,
+				LastReproTime:  nowTime.Add(-time.Hour * 24),
 			},
 			needRepro: false,
 		},
 		{
 			// A bug which has an old C repro.
 			bug: &Bug{
-				Title:         "some normal bug with old repro",
-				ReproLevel:    ReproLevelC,
-				NumRepro:      2 * maxReproPerBug,
-				LastReproTime: nowTime.Add(-reproStalePeriod),
+				Title:          "some normal bug with old repro",
+				ReproLevel:     ReproLevelC,
+				HeadReproLevel: ReproLevelC,
+				NumRepro:       2 * maxReproPerBug,
+				LastReproTime:  nowTime.Add(-reproStalePeriod),
 			},
 			needRepro: true,
 		},
@@ -278,36 +281,48 @@ func TestNeedReproIsolated(t *testing.T) {
 		{
 			// Make sure we try until we find a C repro, not just a syz repro.
 			bug: &Bug{
-				Title:         "too many fails, but only a syz repro",
-				ReproLevel:    ReproLevelSyz,
-				NumRepro:      maxReproPerBug,
-				LastReproTime: nowTime.Add(-24 * time.Hour),
+				Title:          "too many fails, but only a syz repro",
+				ReproLevel:     ReproLevelSyz,
+				HeadReproLevel: ReproLevelSyz,
+				NumRepro:       maxReproPerBug,
+				LastReproTime:  nowTime.Add(-24 * time.Hour),
 			},
 			needRepro: true,
 		},
 		{
 			// We don't need a C repro for SYZFATAL: bugs.
 			bug: &Bug{
-				Title:         "SYZFATAL: Manager.Check call failed",
-				ReproLevel:    ReproLevelSyz,
-				LastReproTime: nowTime.Add(-24 * time.Hour),
+				Title:          "SYZFATAL: Manager.Check call failed",
+				ReproLevel:     ReproLevelSyz,
+				HeadReproLevel: ReproLevelSyz,
+				LastReproTime:  nowTime.Add(-24 * time.Hour),
 			},
 			needRepro: false,
 		},
 		{
 			// .. and for SYZFAIL: bugs.
 			bug: &Bug{
-				Title:         "SYZFAIL: clock_gettime failed",
-				ReproLevel:    ReproLevelSyz,
-				LastReproTime: nowTime.Add(-24 * time.Hour),
+				Title:          "SYZFAIL: clock_gettime failed",
+				ReproLevel:     ReproLevelSyz,
+				HeadReproLevel: ReproLevelSyz,
+				LastReproTime:  nowTime.Add(-24 * time.Hour),
 			},
 			needRepro: false,
 		},
 		{
 			// Yet make sure that we request at least a syz repro.
 			bug: &Bug{
-				Title:      "SYZFATAL: Manager.Check call failed",
-				ReproLevel: ReproLevelNone,
+				Title: "SYZFATAL: Manager.Check call failed",
+			},
+			needRepro: true,
+		},
+		{
+			// A bug with a revoked repro.
+			bug: &Bug{
+				Title:          "some normal bug with a syz-repro",
+				ReproLevel:     ReproLevelC,
+				HeadReproLevel: ReproLevelSyz,
+				LastReproTime:  nowTime.Add(-24 * time.Hour),
 			},
 			needRepro: true,
 		},
diff --git a/dashboard/app/util_test.go b/dashboard/app/util_test.go
index 0f9dc093c..7f3eb12d4 100644
--- a/dashboard/app/util_test.go
+++ b/dashboard/app/util_test.go
@@ -344,6 +344,11 @@ func (c *Ctx) expectNoEmail() {
 	}
 }
 
+func (c *Ctx) updRetestReproJobs() {
+	_, err := c.GET("/retest_repros")
+	c.expectOK(err)
+}
+
 type apiClient struct {
 	*Ctx
 	*dashapi.Dashboard
@@ -427,14 +432,10 @@ func (client *apiClient) updateBug(extID string, status dashapi.BugStatus, dup s
 	client.expectTrue(reply.OK)
 }
 
-func (client *apiClient) pollJobs(manager string) *dashapi.JobPollResp {
+func (client *apiClient) pollSpecificJobs(manager string, jobs dashapi.ManagerJobs) *dashapi.JobPollResp {
 	req := &dashapi.JobPollReq{
 		Managers: map[string]dashapi.ManagerJobs{
-			manager: {
-				TestPatches: true,
-				BisectCause: true,
-				BisectFix:   true,
-			},
+			manager: jobs,
 		},
 	}
 	resp, err := client.JobPoll(req)
@@ -442,6 +443,14 @@ func (client *apiClient) pollJobs(manager string) *dashapi.JobPollResp {
 	return resp
 }
 
+func (client *apiClient) pollJobs(manager string) *dashapi.JobPollResp {
+	return client.pollSpecificJobs(manager, dashapi.ManagerJobs{
+		TestPatches: true,
+		BisectCause: true,
+		BisectFix:   true,
+	})
+}
+
 func (client *apiClient) pollAndFailBisectJob(manager string) {
 	resp := client.pollJobs(manager)
 	client.expectNE(resp.ID, "")
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go
index cdefb8080..d5ccc1e0f 100644
--- a/dashboard/dashapi/dashapi.go
+++ b/dashboard/dashapi/dashapi.go
@@ -433,6 +433,7 @@ type BugUpdate struct {
 	ExtID           string
 	Link            string
 	Status          BugStatus
+	StatusReason    BugStatusReason
 	ReproLevel      ReproLevel
 	DupOf           string
 	OnHold          bool     // If set for open bugs, don't upstream this bug.
@@ -608,10 +609,11 @@ func (dash *Dashboard) LoadBug(id string) (*BugReport, error) {
 }
 
 type (
-	BugStatus  int
-	BugNotif   int
-	ReproLevel int
-	ReportType int
+	BugStatus       int
+	BugStatusReason string
+	BugNotif        int
+	ReproLevel      int
+	ReportType      int
 )
 
 const (
@@ -625,6 +627,11 @@ const (
 )
 
 const (
+	InvalidatedByRevokedRepro = BugStatusReason("invalid_no_repro")
+	InvalidatedByNoActivity   = BugStatusReason("invalid_no_activity")
+)
+
+const (
 	// Upstream bug into next reporting.
 	// If the action succeeds, reporting sends BugStatusUpstream update.
 	BugNotifUpstream BugNotif = iota