From 54c44ce53f9d4fda314d14c21458d3fcf6310820 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Fri, 9 Jan 2026 13:01:06 +0100 Subject: dashboard/dashapi: retry requests on errors API requests episodically fail due to internal datastore errors, some timeouts, etc. Failure of some requests is especially unpleasant and leads to lots of wasted work (uploading of syz-ci build info, job completion, etc). So we retry requests several times. We do this always for all requests, since we don't expect any of them to legitimately fail (we don't send malformed requests), and it won't harm for any request types. --- dashboard/dashapi/dashapi.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'dashboard/dashapi/dashapi.go') diff --git a/dashboard/dashapi/dashapi.go b/dashboard/dashapi/dashapi.go index 84b90eb97..22d625521 100644 --- a/dashboard/dashapi/dashapi.go +++ b/dashboard/dashapi/dashapi.go @@ -992,15 +992,28 @@ func (dash *Dashboard) Query(method string, req, reply any) error { if dash.logger != nil { dash.logger("API(%v): %#v", method, req) } - err := dash.queryImpl(method, req, reply) - if err != nil { - if dash.logger != nil { - dash.logger("API(%v): ERROR: %v", method, err) - } - if dash.errorHandler != nil { - dash.errorHandler(err) + for try := 0; ; try++ { + err := dash.queryImpl(method, req, reply) + if err != nil { + if dash.logger != nil { + dash.logger("API(%v): ERROR: %v", method, err) + } + if dash.errorHandler != nil { + dash.errorHandler(err) + } + // API requests episodically fail due to internal datastore errors, some timeouts, etc. + // Failure of some requests is especially unpleasant and leads to lots of wasted work + // (uploading of syz-ci build info, job completion, etc). So we retry requests + // several times. We do this always for all requests, since we don't expect any of them + // to legitimately fail (we don't send malformed requests), and it won't harm for any + // request types. + if try < 3 { + time.Sleep(time.Second) + continue + } + return err } - return err + break } if dash.logger != nil { dash.logger("API(%v): REPLY: %#v", method, reply) -- cgit mrf-deployment