diff options
| author | Dmitry Vyukov <dvyukov@google.com> | 2026-01-09 13:01:06 +0100 |
|---|---|---|
| committer | Dmitry Vyukov <dvyukov@google.com> | 2026-01-13 15:58:13 +0000 |
| commit | 54c44ce53f9d4fda314d14c21458d3fcf6310820 (patch) | |
| tree | 073dd78501124b73fbf8d7937248a40b1ed40bba | |
| parent | b99f5fb3c93022696e64cea9303e64539da1cdf6 (diff) | |
dashboard/dashapi: retry requests on errors
API requests episodically fail due to internal datastore errors, some timeouts, etc.
Failure of some requests is especially unpleasant and leads to lots of wasted work
(uploading of syz-ci build info, job completion, etc). So we retry requests
several times. We do this always for all requests, since we don't expect any of them
to legitimately fail (we don't send malformed requests), and it won't harm for any
request types.
| -rw-r--r-- | dashboard/dashapi/dashapi.go | 29 |
1 files changed, 21 insertions, 8 deletions
diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go index 84b90eb97..22d625521 100644 --- a/dashboard/dashapi/dashapi.go +++ b/dashboard/dashapi/dashapi.go @@ -992,15 +992,28 @@ func (dash *Dashboard) Query(method string, req, reply any) error { if dash.logger != nil { dash.logger("API(%v): %#v", method, req) } - err := dash.queryImpl(method, req, reply) - if err != nil { - if dash.logger != nil { - dash.logger("API(%v): ERROR: %v", method, err) - } - if dash.errorHandler != nil { - dash.errorHandler(err) + for try := 0; ; try++ { + err := dash.queryImpl(method, req, reply) + if err != nil { + if dash.logger != nil { + dash.logger("API(%v): ERROR: %v", method, err) + } + if dash.errorHandler != nil { + dash.errorHandler(err) + } + // API requests episodically fail due to internal datastore errors, some timeouts, etc. + // Failure of some requests is especially unpleasant and leads to lots of wasted work + // (uploading of syz-ci build info, job completion, etc). So we retry requests + // several times. We do this always for all requests, since we don't expect any of them + // to legitimately fail (we don't send malformed requests), and it won't harm for any + // request types. + if try < 3 { + time.Sleep(time.Second) + continue + } + return err } - return err + break } if dash.logger != nil { dash.logger("API(%v): REPLY: %#v", method, reply) |