From 1190297f0bb22bd80fe3edea697e11150d2ada40 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 7 Dec 2020 09:00:02 +0100 Subject: dashboard/app: don't report more than 3 bugs at once We started getting lots of the following errors: Dec 07 07:40: Exceeded soft memory limit of 256 MB with 340 MB after servicing 57 requests total. Consider setting a larger instance class in app.yaml. (/api) Dec 07 07:45: Exceeded soft memory limit of 256 MB with 395 MB after servicing 113 requests total. Consider setting a larger instance class in app.yaml. (/api) Dec 07 07:46: Exceeded soft memory limit of 256 MB with 270 MB after servicing 51 requests total. Consider setting a larger instance class in app.yaml. (/api) Dec 07 07:47: Exceeded soft memory limit of 256 MB with 420 MB after servicing 43 requests total. Consider setting a larger instance class in app.yaml. (/api) This is caused by a sudden spike of bugs that need to be reported. Number of reported bugs is capped by daily reporting limit, but it does not seem to be enough. Cap number of reported bugs explicitly. --- dashboard/app/reporting.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'dashboard') diff --git a/dashboard/app/reporting.go b/dashboard/app/reporting.go index 35a050fcb..9c8943993 100644 --- a/dashboard/app/reporting.go +++ b/dashboard/app/reporting.go @@ -63,6 +63,12 @@ func reportingPollBugs(c context.Context, typ string) []*dashapi.BugReport { continue } reports = append(reports, rep) + // Trying to report too many at once is known to cause OOMs. + // But new bugs appear incrementally and polling is frequent enough, + // so reporting lots of bugs at once is also not necessary. + if len(reports) == 3 { + break + } } return reports } @@ -136,8 +142,8 @@ func needReport(c context.Context, typ string, state *ReportingState, bug *Bug) // Ready to be reported. if bugReporting.Reported.IsZero() { - // This update won't be committed, but it will prevent us from - // reporting too many bugs in a single poll. + // This update won't be committed, but it is useful as a best effort measure + // so that we don't overflow the limit in a single poll. ent.Sent++ } status = fmt.Sprintf("%v: ready to report", reporting.DisplayTitle) -- cgit mrf-deployment