From 44e9a18ae72daeda928bb9c4c71b4460f9fdece9 Mon Sep 17 00:00:00 2001
From: Dmitry Vyukov <dvyukov@google.com>
Date: Thu, 24 Oct 2019 18:36:06 +0200
Subject: dashboard/app: switch foreachBug to using cursors

Cursors seem to be more efficient than Offset-based queries:
https://cloud.google.com/datastore/docs/concepts/queries#datastore-datastore-cursor-paging-go

Update #1448
---
 dashboard/app/reporting.go | 34 ++++++++++++++++++++++------------
 1 file changed, 22 insertions(+), 12 deletions(-)

diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go
index 83c2af24b..43067fe9b 100644
--- a/dashboard/app/reporting.go
+++ b/dashboard/app/reporting.go
@@ -485,29 +485,39 @@ func loadAllBugs(c context.Context, ns, manager string) ([]*Bug, error) {
 
 func foreachBug(c context.Context, ns, manager string, fn func(bug *Bug) error) error {
 	const batchSize = 1000
-	for offset := 0; ; offset += batchSize {
-		var bugs []*Bug
-		query := db.NewQuery("Bug")
+	var cursor db.Cursor
+	for first := true; ; first = false {
+		query := db.NewQuery("Bug").Limit(batchSize)
 		if ns != "" {
 			query = query.Filter("Namespace=", ns)
 			if manager != "" {
 				query = query.Filter("HappenedOn=", manager)
 			}
 		}
-		_, err := query.Offset(offset).
-			Limit(batchSize).
-			GetAll(c, &bugs)
-		if err != nil {
-			return fmt.Errorf("foreachBug: failed to query bugs: %v", err)
+		if !first {
+			query = query.Start(cursor)
 		}
-		for _, bug := range bugs {
+		iter := query.Run(c)
+		for i := 0; ; i++ {
+			bug := new(Bug)
+			_, err := iter.Next(bug)
+			if err == db.Done {
+				if i < batchSize {
+					return nil
+				}
+				cursor, err = iter.Cursor()
+				if err != nil {
+					return fmt.Errorf("cursor failed while fetching bugs: %v", err)
+				}
+				break
+			}
+			if err != nil {
+				return fmt.Errorf("failed to fetch bugs: %v", err)
+			}
 			if err := fn(bug); err != nil {
 				return err
 			}
 		}
-		if len(bugs) < batchSize {
-			return nil
-		}
 	}
 }
 
-- 
cgit mrf-deployment