pkg/bisect: detect when too many instances errored

We currently skip a commit iff all 10 instances errored.
But if, say, only 9 errored we consider it as OK,
but this significnalty reduces chances of detecting flaky crashes.
So skip if more than 2/3 errored.

Update #501

1 files changed, 6 insertions, 1 deletions

diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go
index eff0601b4..dea41581f 100644
--- a/pkg/bisect/bisect.go
+++ b/pkg/bisect/bisect.go
@@ -275,7 +275,8 @@ func (env *env) test() (vcs.BisectResult, *vcs.Commit, *report.Report, error) {
 		return vcs.BisectSkip, current, nil, nil
 	}
 	testStart := time.Now()
-	results, err := env.inst.Test(10, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C)
+	const numTests = 10
+	results, err := env.inst.Test(numTests, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C)
 	env.testTime += time.Since(testStart)
 	if err != nil {
 		env.log("failed: %v", err)
@@ -285,6 +286,10 @@ func (env *env) test() (vcs.BisectResult, *vcs.Commit, *report.Report, error) {
 	res := vcs.BisectSkip
 	if bad != 0 {
 		res = vcs.BisectBad
+	} else if numTests-good-bad > numTests/3*2 {
+		// More than 2/3 of instances failed with infrastructure error,
+		// can't reliably tell that the commit is good.
+		res = vcs.BisectSkip
 	} else if good != 0 {
 		res = vcs.BisectGood
 	}