datastore/app: switch more bug queries to paged queries

Update #1448

4 files changed, 53 insertions, 51 deletions

diff --git a/dashboard/app/api.go b/dashboard/app/api.go
index b488bb545..1baf0af0a 100644
--- a/dashboard/app/api.go
+++ b/dashboard/app/api.go
@@ -177,13 +177,12 @@ func apiBuilderPoll(c context.Context, ns string, r *http.Request, payload []byt
 	if err := json.Unmarshal(payload, req); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal request: %v", err)
 	}
-	var bugs []*Bug
-	_, err := db.NewQuery("Bug").
-		Filter("Namespace=", ns).
-		Filter("Status<", BugStatusFixed).
-		GetAll(c, &bugs)
+	bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		return query.Filter("Namespace=", ns).
+			Filter("Status<", BugStatusFixed)
+	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to query bugs: %v", err)
+		return nil, err
 	}
 	m := make(map[string]bool)
 loop:
@@ -521,13 +520,12 @@ func addCommitsToBugs(c context.Context, ns, manager string, titles []string, fi
 
 func addCommitsToBugsInStatus(c context.Context, status int, ns, manager string, managers []string,
 	presentCommits map[string]bool, bugFixedBy map[string][]string) error {
-	var bugs []*Bug
-	_, err := db.NewQuery("Bug").
-		Filter("Namespace=", ns).
-		Filter("Status=", status).
-		GetAll(c, &bugs)
+	bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		return query.Filter("Namespace=", ns).
+			Filter("Status=", status)
+	})
 	if err != nil {
-		return fmt.Errorf("failed to query bugs: %v", err)
+		return err
 	}
 	for _, bug := range bugs {
 		var fixCommits []string
diff --git a/dashboard/app/main.go b/dashboard/app/main.go
index d43f9d1e1..64fb6ffe9 100644
--- a/dashboard/app/main.go
+++ b/dashboard/app/main.go
@@ -527,7 +527,14 @@ func textFilename(tag string) string {
 
 func fetchNamespaceBugs(c context.Context, accessLevel AccessLevel,
 	ns, manager string) ([]*uiBugGroup, int, error) {
-	bugs, err := loadAllBugs(c, ns, manager)
+	filter := func(query *db.Query) *db.Query {
+		query = query.Filter("Namespace=", ns)
+		if manager != "" {
+			query = query.Filter("HappenedOn=", manager)
+		}
+		return query
+	}
+	bugs, err := loadAllBugs(c, filter)
 	if err != nil {
 		return nil, 0, err
 	}
@@ -616,14 +623,15 @@ func fetchNamespaceBugs(c context.Context, accessLevel AccessLevel,
 
 func fetchTerminalBugs(c context.Context, accessLevel AccessLevel,
 	ns, manager string, typ *TerminalBug) (*uiBugGroup, error) {
-	var bugs []*Bug
-	query := db.NewQuery("Bug").
-		Filter("Namespace=", ns).
-		Filter("Status=", typ.Status)
-	if manager != "" {
-		query = query.Filter("HappenedOn=", manager)
-	}
-	if _, err := query.GetAll(c, &bugs); err != nil {
+	bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		query = query.Filter("Namespace=", ns).
+			Filter("Status=", typ.Status)
+		if manager != "" {
+			query = query.Filter("HappenedOn=", manager)
+		}
+		return query
+	})
+	if err != nil {
 		return nil, err
 	}
 	state, err := loadReportingState(c)
diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 43067fe9b..417f3a950 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -44,10 +44,9 @@ func reportingPollBugs(c context.Context, typ string) []*dashapi.BugReport {
 		log.Errorf(c, "%v", err)
 		return nil
 	}
-	var bugs []*Bug
-	_, err = db.NewQuery("Bug").
-		Filter("Status<", BugStatusFixed).
-		GetAll(c, &bugs)
+	bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		return query.Filter("Status<", BugStatusFixed)
+	})
 	if err != nil {
 		log.Errorf(c, "%v", err)
 		return nil
@@ -152,10 +151,9 @@ func needReport(c context.Context, typ string, state *ReportingState, bug *Bug)
 }
 
 func reportingPollNotifications(c context.Context, typ string) []*dashapi.BugNotification {
-	var bugs []*Bug
-	_, err := db.NewQuery("Bug").
-		Filter("Status<", BugStatusFixed).
-		GetAll(c, &bugs)
+	bugs, err := loadAllBugs(c, func(query *db.Query) *db.Query {
+		return query.Filter("Status<", BugStatusFixed)
+	})
 	if err != nil {
 		log.Errorf(c, "%v", err)
 		return nil
@@ -471,9 +469,9 @@ func managersToRepos(c context.Context, ns string, managers []string) []string {
 	return repos
 }
 
-func loadAllBugs(c context.Context, ns, manager string) ([]*Bug, error) {
+func loadAllBugs(c context.Context, filter func(*db.Query) *db.Query) ([]*Bug, error) {
 	var bugs []*Bug
-	err := foreachBug(c, ns, manager, func(bug *Bug) error {
+	err := foreachBug(c, filter, func(bug *Bug, _ *db.Key) error {
 		bugs = append(bugs, bug)
 		return nil
 	})
@@ -483,41 +481,39 @@ func loadAllBugs(c context.Context, ns, manager string) ([]*Bug, error) {
 	return bugs, nil
 }
 
-func foreachBug(c context.Context, ns, manager string, fn func(bug *Bug) error) error {
+func foreachBug(c context.Context, filter func(*db.Query) *db.Query, fn func(bug *Bug, key *db.Key) error) error {
 	const batchSize = 1000
-	var cursor db.Cursor
-	for first := true; ; first = false {
+	var cursor *db.Cursor
+	for {
 		query := db.NewQuery("Bug").Limit(batchSize)
-		if ns != "" {
-			query = query.Filter("Namespace=", ns)
-			if manager != "" {
-				query = query.Filter("HappenedOn=", manager)
-			}
+		if filter != nil {
+			query = filter(query)
 		}
-		if !first {
-			query = query.Start(cursor)
+		if cursor != nil {
+			query = query.Start(*cursor)
 		}
 		iter := query.Run(c)
 		for i := 0; ; i++ {
 			bug := new(Bug)
-			_, err := iter.Next(bug)
+			key, err := iter.Next(bug)
 			if err == db.Done {
 				if i < batchSize {
 					return nil
 				}
-				cursor, err = iter.Cursor()
-				if err != nil {
-					return fmt.Errorf("cursor failed while fetching bugs: %v", err)
-				}
 				break
 			}
 			if err != nil {
 				return fmt.Errorf("failed to fetch bugs: %v", err)
 			}
-			if err := fn(bug); err != nil {
+			if err := fn(bug, key); err != nil {
 				return err
 			}
 		}
+		cur, err := iter.Cursor()
+		if err != nil {
+			return fmt.Errorf("cursor failed while fetching bugs: %v", err)
+		}
+		cursor = &cur
 	}
 }
 
@@ -528,7 +524,7 @@ func reportingPollClosed(c context.Context, ids []string) ([]string, error) {
 		idMap[id] = true
 	}
 	var closed []string
-	err := foreachBug(c, "", "", func(bug *Bug) error {
+	err := foreachBug(c, nil, func(bug *Bug, _ *db.Key) error {
 		for i := range bug.Reporting {
 			bugReporting := &bug.Reporting[i]
 			if !idMap[bugReporting.ID] {
diff --git a/dashboard/app/reporting_email.go b/dashboard/app/reporting_email.go
index 3b3df67f6..b88953b67 100644
--- a/dashboard/app/reporting_email.go
+++ b/dashboard/app/reporting_email.go
@@ -111,7 +111,7 @@ func emailPollBugs(c context.Context) error {
 	reports := reportingPollBugs(c, emailType)
 	for _, rep := range reports {
 		if err := emailSendBugReport(c, rep); err != nil {
-			log.Errorf(c, "%v", err)
+			log.Errorf(c, "emailPollBugs: %v", err)
 		}
 	}
 	return nil
@@ -147,7 +147,7 @@ func emailPollNotifications(c context.Context) error {
 	notifs := reportingPollNotifications(c, emailType)
 	for _, notif := range notifs {
 		if err := emailSendBugNotif(c, notif); err != nil {
-			log.Errorf(c, "%v", err)
+			log.Errorf(c, "emailPollNotifications: %v", err)
 		}
 	}
 	return nil
@@ -256,7 +256,7 @@ func emailReport(c context.Context, rep *dashapi.BugReport) error {
 func handleIncomingMail(w http.ResponseWriter, r *http.Request) {
 	c := appengine.NewContext(r)
 	if err := incomingMail(c, r); err != nil {
-		log.Errorf(c, "%v", err)
+		log.Errorf(c, "handleIncomingMail: %v", err)
 	}
 }