From 4c2028c3d628cb632f26a8a216975e8f02e9a3ca Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Tue, 13 Jun 2023 11:44:21 +0200 Subject: pkg/bisect: skip commits with too many boot/test errors If there are only a few non-crashed results among many boot/test errors, we cannot really be sure that the commit is good. It might be that the reproducer is not 100% reliable and we just needed more runs. Require > 50% of runs to be successful in order to claim that the revision is good. Otherwise skip it. --- pkg/bisect/bisect.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'pkg') diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go index b9f214337..1a20a5cab 100644 --- a/pkg/bisect/bisect.go +++ b/pkg/bisect/bisect.go @@ -603,14 +603,17 @@ func (env *env) test() (*testResult, error) { env.log("reproducer seems to be flaky") env.flaky = true } - } else if good != 0 { + } else if 2*good > len(results) { + // Require at least 50% of successful runs to say it's a good commit. + // Otherwise we just have too little data to come to any conclusion. res.verdict = vcs.BisectGood + env.log("too many neither good nor bad results, skipping this commit") } else { res.rep = &report.Report{ Title: fmt.Sprintf("failed testing reproducer on %v", current.Hash), } } - // If all runs failed with a boot/test error, we just end up with BisectSkip. + // If most runs failed with a boot/test error, we end up with BisectSkip. // TODO: when we start supporting boot/test error bisection, we need to make // processResults treat that verdit as "good". return res, nil -- cgit mrf-deployment