dashboard: const attempts in db.RunInTransaction

Use a common number of attempts (10) everywhere in dashboard/app
for db.RunInTransaction() with the custom wrapper runInTransaction().

This should fix the issues when the ErrConcurrentTransaction occurs
after a default number of attempts (3). If the max. limit (10) is not
enough, then it should hint on problem at the tx transaction function itself
for the further debugging.

For very valuable transactions in createBugForCrash(), reportCrash(),
let's leave 30 attempts as it is currently.

Fixes: https://github.com/google/syzkaller/issues/5441

10 files changed, 53 insertions, 42 deletions

diff --git a/dashboard/app/admin.go b/dashboard/app/admin.go
index 04df13ea2..3b9fac8a1 100644
--- a/dashboard/app/admin.go
+++ b/dashboard/app/admin.go
@@ -123,7 +123,7 @@ func dropNamespaceReportingState(c context.Context, w http.ResponseWriter, ns st
 		fmt.Fprintf(w, "ReportingState: %v\n", len(state.Entries)-len(newState.Entries))
 		return nil
 	}
-	return db.RunInTransaction(c, tx, nil)
+	return runInTransaction(c, tx, nil)
 }
 
 func dropEntities(c context.Context, keys []*db.Key, dryRun bool) error {
@@ -401,7 +401,7 @@ func updateBatch[T any](c context.Context, keys []*db.Key, transform func(key *d
 			_, err := db.PutMulti(c, batchKeys, items)
 			return err
 		}
-		if err := db.RunInTransaction(c, tx, &db.TransactionOptions{XG: true}); err != nil {
+		if err := runInTransaction(c, tx, &db.TransactionOptions{XG: true}); err != nil {
 			return err
 		}
 		log.Warningf(c, "updated %v bugs", len(batchKeys))
diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index 0b64bf9f1..107add5bd 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -361,7 +361,7 @@ func addCommitInfoToBug(c context.Context, bug *Bug, bugKey *db.Key, com dashapi
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, nil)
+	return runInTransaction(c, tx, nil)
 }
 
 func addCommitInfoToBugImpl(c context.Context, bug *Bug, com dashapi.Commit) (bool, error) {
@@ -662,7 +662,7 @@ func addCommitsToBug(c context.Context, bug *Bug, manager string, managers, fixC
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, nil)
+	return runInTransaction(c, tx, nil)
 }
 
 func bugNeedsCommitUpdate(c context.Context, bug *Bug, manager string, fixCommits []string,
@@ -882,7 +882,11 @@ func reportCrash(c context.Context, build *Build, req *dashapi.Crash) (*Bug, err
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{XG: true}); err != nil {
+	if err := runInTransaction(c, tx, &db.TransactionOptions{
+		XG: true,
+		// Very valuable transaction.
+		Attempts: 30,
+	}); err != nil {
 		return nil, fmt.Errorf("bug updating failed: %w", err)
 	}
 	if save {
@@ -1088,7 +1092,7 @@ func saveFailedReproLog(c context.Context, bug *Bug, build *Build, log []byte) e
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, &db.TransactionOptions{
+	return runInTransaction(c, tx, &db.TransactionOptions{
 		XG:       true,
 		Attempts: 30,
 	})
@@ -1249,7 +1253,7 @@ func apiUpdateReport(c context.Context, ns string, r *http.Request, payload []by
 		}
 		return nil
 	}
-	return nil, db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 5})
+	return nil, runInTransaction(c, tx, nil)
 }
 
 func apiLoadBug(c context.Context, ns string, r *http.Request, payload []byte) (interface{}, error) {
@@ -1515,8 +1519,9 @@ func createBugForCrash(c context.Context, ns string, req *dashapi.Crash) (*Bug,
 			return nil
 		}
 	}
-	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{
-		XG:       true,
+	if err := runInTransaction(c, tx, &db.TransactionOptions{
+		XG: true,
+		// Very valuable transaction.
 		Attempts: 30,
 	}); err != nil {
 		return nil, err
diff --git a/dashboard/app/asset_storage.go b/dashboard/app/asset_storage.go
index 199a44733..2eb042ea1 100644
--- a/dashboard/app/asset_storage.go
+++ b/dashboard/app/asset_storage.go
@@ -49,7 +49,7 @@ func appendBuildAssets(c context.Context, ns, buildID string, assets []Asset) (*
 		log.Infof(c, "updated build: %#v", build)
 		return nil
 	}
-	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{}); err != nil {
+	if err := runInTransaction(c, tx, nil); err != nil {
 		return nil, err
 	}
 	return retBuild, nil
@@ -349,7 +349,7 @@ func (ad *buildAssetDeprecator) updateBuild(buildID string, urlsToDelete []strin
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(ad.c, tx, nil); err != nil {
+	if err := runInTransaction(ad.c, tx, nil); err != nil {
 		return fmt.Errorf("failed to update build: %w", err)
 	}
 	return nil
@@ -463,7 +463,7 @@ func (ad *crashAssetDeprecator) updateCrash(crashKey *db.Key, urlsToDelete []str
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(ad.c, tx, &db.TransactionOptions{Attempts: 10}); err != nil {
+	if err := runInTransaction(ad.c, tx, nil); err != nil {
 		return fmt.Errorf("failed to update crash: %w", err)
 	}
 	return nil
diff --git a/dashboard/app/discussion.go b/dashboard/app/discussion.go
index 5a234795a..47869341d 100644
--- a/dashboard/app/discussion.go
+++ b/dashboard/app/discussion.go
@@ -87,18 +87,16 @@ func mergeDiscussion(c context.Context, update *dashapi.Discussion) error {
 		}
 		return nil
 	}
-	err = db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 15, XG: true})
-	if err != nil {
+	if err = runInTransaction(c, tx, &db.TransactionOptions{XG: true}); err != nil {
 		return err
 	}
 	// Update individual bug statistics.
 	// We have to do it outside of the main transaction, as we might hit the "operating on
 	// too many entity groups in a single transaction." error.
 	for _, key := range d.BugKeys {
-		err := db.RunInTransaction(c, func(c context.Context) error {
+		if err := runInTransaction(c, func(c context.Context) error {
 			return mergeDiscussionSummary(c, key, d.Source, diff)
-		}, &db.TransactionOptions{Attempts: 15})
-		if err != nil {
+		}, nil); err != nil {
 			return fmt.Errorf("failed to put update summary for %s: %w", key, err)
 		}
 	}
diff --git a/dashboard/app/entities_datastore.go b/dashboard/app/entities_datastore.go
index 7fc70a4ae..43adf9356 100644
--- a/dashboard/app/entities_datastore.go
+++ b/dashboard/app/entities_datastore.go
@@ -208,7 +208,7 @@ func updateSingleBug(c context.Context, bugKey *db.Key, transform func(*Bug) err
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10})
+	return runInTransaction(c, tx, nil)
 }
 
 func (bug *Bug) hasUserSubsystems() bool {
@@ -785,7 +785,7 @@ func updateManager(c context.Context, ns, name string, fn func(mgr *Manager, sta
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 10})
+	return runInTransaction(c, tx, nil)
 }
 
 func loadAllManagers(c context.Context, ns string) ([]*Manager, []*db.Key, error) {
@@ -1167,3 +1167,17 @@ func (dl *dependencyLoader[T]) load(c context.Context) error {
 	}
 	return nil
 }
+
+type txFunc func(tc context.Context) error
+
+// runInTransaction is a wrapper around db.RunInTransaction,
+// with the common number of attempts.
+func runInTransaction(c context.Context, tx txFunc, opts *db.TransactionOptions) error {
+	if opts == nil {
+		opts = &db.TransactionOptions{}
+	}
+	if opts.Attempts == 0 {
+		opts.Attempts = 10
+	}
+	return db.RunInTransaction(c, tx, opts)
+}
diff --git a/dashboard/app/jobs.go b/dashboard/app/jobs.go
index 7796b9773..95fac0114 100644
--- a/dashboard/app/jobs.go
+++ b/dashboard/app/jobs.go
@@ -79,7 +79,7 @@ func handleTestRequest(c context.Context, args *testReqArgs) error {
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(c, tx, nil); err != nil {
+	if err := runInTransaction(c, tx, nil); err != nil {
 		// We've already stored the job, so just log the error.
 		log.Errorf(c, "failed to update bug: %v", err)
 	}
@@ -187,7 +187,7 @@ func addTestJob(c context.Context, args *testJobArgs) (*Job, *db.Key, error) {
 	if args.inTransaction {
 		err = tx(c)
 	} else {
-		err = db.RunInTransaction(c, tx, &db.TransactionOptions{XG: true, Attempts: 30})
+		err = runInTransaction(c, tx, &db.TransactionOptions{XG: true})
 	}
 	if patchID != 0 && (deletePatch || err != nil) {
 		if err := db.Delete(c, db.NewKey(c, textPatch, "", patchID, nil)); err != nil {
@@ -296,7 +296,7 @@ func invalidateBisection(c context.Context, jobKey *db.Key, restart bool) error
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{XG: true, Attempts: 10}); err != nil {
+	if err := runInTransaction(c, tx, &db.TransactionOptions{XG: true}); err != nil {
 		return fmt.Errorf("update failed: %w", err)
 	}
 
@@ -402,7 +402,7 @@ func throttleJobGeneration(c context.Context, managers map[string]dashapi.Manage
 			}
 			return nil
 		}
-		if err := db.RunInTransaction(c, tx, &db.TransactionOptions{}); err != nil {
+		if err := runInTransaction(c, tx, nil); err != nil {
 			return fmt.Errorf("failed to throttle: %w", err)
 		}
 	}
@@ -807,7 +807,7 @@ func createBisectJobForBug(c context.Context, bug0 *Bug, crash *Crash, bugKey, c
 		jobKey, err = saveJob(c, job, bugKey)
 		return err
 	}
-	if err := db.RunInTransaction(c, tx, &db.TransactionOptions{
+	if err := runInTransaction(c, tx, &db.TransactionOptions{
 		// We're accessing two different kinds in addCrashReference.
 		XG: true,
 	}); err != nil {
@@ -873,7 +873,7 @@ func createJobResp(c context.Context, job *Job, jobKey *db.Key) (*dashapi.JobPol
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(c, tx, nil); err != nil {
+	if err := runInTransaction(c, tx, nil); err != nil {
 		return nil, false, err
 	}
 	if stale {
@@ -1023,7 +1023,7 @@ func resetJobs(c context.Context, req *dashapi.JobResetReq) error {
 			}
 			return nil
 		}
-		if err := db.RunInTransaction(c, tx, nil); err != nil {
+		if err := runInTransaction(c, tx, nil); err != nil {
 			return err
 		}
 	}
@@ -1122,8 +1122,7 @@ func doneJob(c context.Context, req *dashapi.JobDoneReq) error {
 		log.Infof(c, "DONE JOB %v: reported=%v reporting=%v", jobID, job.Reported, job.Reporting)
 		return nil
 	}
-	err = db.RunInTransaction(c, tx, &db.TransactionOptions{XG: true, Attempts: 30})
-	if err != nil {
+	if err = runInTransaction(c, tx, &db.TransactionOptions{XG: true}); err != nil {
 		return err
 	}
 	return postJob(c, jobKey, job)
@@ -1409,7 +1408,7 @@ func jobReported(c context.Context, jobID string) error {
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, nil)
+	return runInTransaction(c, tx, nil)
 }
 
 func handleExternalTestRequest(c context.Context, req *dashapi.TestPatchRequest) error {
@@ -1726,7 +1725,7 @@ func commitBackported(c context.Context, jobKey *db.Key, commit Commit) error {
 		}
 		return nil
 	}
-	return db.RunInTransaction(c, tx, &db.TransactionOptions{Attempts: 5})
+	return runInTransaction(c, tx, nil)
 }
 
 type bugJobs struct {
diff --git a/dashboard/app/kcidb.go b/dashboard/app/kcidb.go
index 0a1ef9705..6fe641276 100644
--- a/dashboard/app/kcidb.go
+++ b/dashboard/app/kcidb.go
@@ -95,7 +95,7 @@ func publishKcidbBug(c context.Context, client *kcidb.Client, bug *Bug, bugKey *
 		}
 		return nil
 	}
-	if err := db.RunInTransaction(c, tx, nil); err != nil {
+	if err := runInTransaction(c, tx, nil); err != nil {
 		return false, err
 	}
 	log.Infof(c, "published bug to kcidb: %v:%v '%v'", bug.Namespace, bugKey.StringID(), bug.displayTitle())
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 134b5db30..cf292fc12 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -874,9 +874,8 @@ func incomingCommandImpl(c context.Context, cmd *dashapi.BugUpdate) (bool, strin
 		ok, reply, err = incomingCommandTx(c, now, cmd, bugKey, dupKey)
 		return err
 	}
-	err = db.RunInTransaction(c, tx, &db.TransactionOptions{
+	err = runInTransaction(c, tx, &db.TransactionOptions{
 		XG: true,
-		// Default is 3 which fails sometimes.
 		// We don't want incoming bug updates to fail,
 		// because for e.g. email we won't have an external retry.
 		Attempts: 30,
diff --git a/dashboard/app/reporting_lists.go b/dashboard/app/reporting_lists.go
index 0fe1dd8e0..473c24433 100644
--- a/dashboard/app/reporting_lists.go
+++ b/dashboard/app/reporting_lists.go
@@ -208,10 +208,7 @@ Please visit the new discussion thread.`
 		}
 		return nil
 	}
-	return reply, db.RunInTransaction(c, tx, &db.TransactionOptions{
-		XG:       true,
-		Attempts: 10,
-	})
+	return reply, runInTransaction(c, tx, &db.TransactionOptions{XG: true})
 }
 
 func findSubsystemReportByID(c context.Context, ID string) (*Subsystem,
@@ -534,7 +531,7 @@ func (sr *subsystemsRegistry) store(item *Subsystem) {
 
 func (sr *subsystemsRegistry) updatePoll(c context.Context, s *Subsystem, success bool) error {
 	key := subsystemKey(c, s)
-	return db.RunInTransaction(c, func(c context.Context) error {
+	return runInTransaction(c, func(c context.Context) error {
 		dbSubsystem := new(Subsystem)
 		err := db.Get(c, key, dbSubsystem)
 		if err == db.ErrNoSuchEntity {
@@ -593,7 +590,7 @@ func (srr *subsystemReportRegistry) store(ns, name string, item *SubsystemReport
 
 func storeSubsystemReport(c context.Context, s *Subsystem, report *SubsystemReport) error {
 	key := subsystemKey(c, s)
-	return db.RunInTransaction(c, func(c context.Context) error {
+	return runInTransaction(c, func(c context.Context) error {
 		// First close all previouly active per-subsystem reports.
 		var previous []*SubsystemReport
 		prevKeys, err := db.NewQuery("SubsystemReport").
diff --git a/dashboard/app/tree.go b/dashboard/app/tree.go
index eece1beb1..a6a9e9e30 100644
--- a/dashboard/app/tree.go
+++ b/dashboard/app/tree.go
@@ -54,8 +54,7 @@ func generateTreeOriginJobs(cGlobal context.Context, bugKey *db.Key,
 		job, jobKey = ctx.job, ctx.jobKey
 		return nil
 	}
-	if err := db.RunInTransaction(cGlobal, tx,
-		&db.TransactionOptions{XG: true, Attempts: 10}); err != nil {
+	if err := runInTransaction(cGlobal, tx, &db.TransactionOptions{XG: true}); err != nil {
 		return nil, nil, err
 	}
 	return job, jobKey, nil
@@ -92,7 +91,7 @@ func treeOriginJobDone(cGlobal context.Context, jobKey *db.Key, job *Job) error
 		}
 		return nil
 	}
-	return db.RunInTransaction(cGlobal, tx, &db.TransactionOptions{XG: true, Attempts: 10})
+	return runInTransaction(cGlobal, tx, &db.TransactionOptions{XG: true})
 }
 
 type pollTreeJobResult interface{}