dashboard: don't print error on high memcache contention

At peak load times, it can be expected that CAS will require too many
iterations. There's no reason to report it as an error.

2 files changed, 8 insertions, 2 deletions

diff --git a/dashboard/app/cache.go b/dashboard/app/cache.go
index 240a6a91f..5867f4782 100644
--- a/dashboard/app/cache.go
+++ b/dashboard/app/cache.go
@@ -5,6 +5,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"sort"
@@ -280,6 +281,8 @@ func (ri *RequesterInfo) Record(now time.Time, cfg ThrottleConfig) bool {
 	return len(newRequests) <= cfg.Limit
 }
 
+var ErrThrottleTooManyRetries = errors.New("all attempts to record request failed")
+
 func ThrottleRequest(c context.Context, requesterID string) (bool, error) {
 	cfg := getConfig(c).Throttle
 	if cfg.Empty() || requesterID == "" {
@@ -325,5 +328,5 @@ func ThrottleRequest(c context.Context, requesterID string) (bool, error) {
 		}
 		return ok, nil
 	}
-	return false, fmt.Errorf("all attempts to record request failed")
+	return false, ErrThrottleTooManyRetries
 }
diff --git a/dashboard/app/handler.go b/dashboard/app/handler.go
index 7162bd85c..ee937f1c6 100644
--- a/dashboard/app/handler.go
+++ b/dashboard/app/handler.go
@@ -94,7 +94,10 @@ func throttleRequest(c context.Context, w http.ResponseWriter, r *http.Request)
 		return true
 	}
 	accept, err := ThrottleRequest(c, ip)
-	if err != nil {
+	if errors.Is(err, ErrThrottleTooManyRetries) {
+		// We get these at peak QPS anyway, it's not an error.
+		log.Warningf(c, "failed to throttle: %v", err)
+	} else if err != nil {
 		log.Errorf(c, "failed to throttle: %v", err)
 	}
 	log.Infof(c, "throttling for %q: %t", ip, accept)