dashboard/app: backpressure robots

Single threaded curl and wget are Ok to use.
Current logic returns 429 client err requiring users to rate limit requests.
Let's add backpressure to guarantee acceptable request rate.

1 files changed, 30 insertions, 0 deletions

diff --git a/dashboard/app/handler.go b/dashboard/app/handler.go
index ee937f1c6..cf9bc9736 100644
--- a/dashboard/app/handler.go
+++ b/dashboard/app/handler.go
@@ -36,6 +36,7 @@ func handleContext(fn contextHandler) http.Handler {
 		if !throttleRequest(c, w, r) {
 			return
 		}
+		defer backpressureRobots(c, r)()
 		if err := fn(c, w, r); err != nil {
 			hdr := commonHeaderRaw(c, r)
 			data := &struct {
@@ -78,6 +79,35 @@ func handleContext(fn contextHandler) http.Handler {
 	})
 }
 
+func isRobot(r *http.Request) bool {
+	userAgent := strings.ToLower(strings.Join(r.Header["User-Agent"], " "))
+	if strings.HasPrefix(userAgent, "curl") ||
+		strings.HasPrefix(userAgent, "wget") {
+		return true
+	}
+	return false
+}
+
+// We don't count the request round trip time here.
+// Actual delay will be the minDelay + requestRoundTripTime.
+func backpressureRobots(c context.Context, r *http.Request) func() {
+	if !isRobot(r) {
+		return func() {}
+	}
+	cfg := getConfig(c).Throttle
+	if cfg.Empty() {
+		return func() {}
+	}
+	minDelay := cfg.Window / time.Duration(cfg.Limit)
+	delayUntil := time.Now().Add(minDelay)
+	return func() {
+		select {
+		case <-c.Done():
+		case <-time.After(time.Until(delayUntil)):
+		}
+	}
+}
+
 func throttleRequest(c context.Context, w http.ResponseWriter, r *http.Request) bool {
 	// AppEngine removes all App Engine-specific headers, which include
 	// X-Appengine-User-IP and X-Forwarded-For.