pkg/bisect: add flaky flag

On first test (original commit) bump up number of tests always.
Also bump up number of tests if observing more good results than bad.

2 files changed, 42 insertions, 7 deletions

diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go
index 129a6686e..3e64233c6 100644
--- a/pkg/bisect/bisect.go
+++ b/pkg/bisect/bisect.go
@@ -75,9 +75,10 @@ type env struct {
 	startTime    time.Time
 	buildTime    time.Duration
 	testTime     time.Duration
+	flaky        bool
 }
 
-const NumTests = 10 // number of tests we do per commit
+const MaxNumTests = 20 // number of tests we do per commit
 
 // Result describes bisection result:
 //  - if bisection is conclusive, the single cause/fix commit in Commits
@@ -159,6 +160,9 @@ func runImpl(cfg *Config, repo vcs.Repo, inst instance.Env) (*Result, error) {
 	}
 	start := time.Now()
 	res, err := env.bisect()
+	if env.flaky {
+		env.log("Reproducer flagged being flaky")
+	}
 	env.log("revisions tested: %v, total time: %v (build: %v, test: %v)",
 		env.numTests, time.Since(start), env.buildTime, env.testTime)
 	if err != nil {
@@ -441,7 +445,6 @@ func (env *env) test() (*testResult, error) {
 	if cfg.Timeout != 0 && time.Since(env.startTime) > cfg.Timeout {
 		return nil, fmt.Errorf("bisection is taking too long (>%v), aborting", cfg.Timeout)
 	}
-	env.numTests++
 	current, kernelSign, err := env.build()
 	res := &testResult{
 		verdict:    vcs.BisectSkip,
@@ -460,8 +463,18 @@ func (env *env) test() (*testResult, error) {
 		}
 		return res, nil
 	}
+
+	numTests := MaxNumTests / 2
+	if env.flaky && env.numTests == 0 {
+		// Use twice as many instances if the bug is flaky and during initial testing
+		// (as we don't know yet if it's flaky or not).
+		numTests *= 2
+	}
+	env.numTests++
+
 	testStart := time.Now()
-	results, err := env.inst.Test(NumTests, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C)
+
+	results, err := env.inst.Test(numTests, cfg.Repro.Syz, cfg.Repro.Opts, cfg.Repro.C)
 	env.testTime += time.Since(testStart)
 	if err != nil {
 		env.log("failed: %v", err)
@@ -472,7 +485,11 @@ func (env *env) test() (*testResult, error) {
 	res.verdict = vcs.BisectSkip
 	if bad != 0 {
 		res.verdict = vcs.BisectBad
-	} else if NumTests-good-bad > NumTests/3*2 {
+		if !env.flaky && bad < good {
+			env.log("reproducer seems to be flaky")
+			env.flaky = true
+		}
+	} else if len(results)-good-bad > len(results)/3*2 {
 		// More than 2/3 of instances failed with infrastructure error,
 		// can't reliably tell that the commit is good.
 		res.verdict = vcs.BisectSkip
diff --git a/pkg/bisect/bisect_test.go b/pkg/bisect/bisect_test.go
index b2b7596f1..7d0c89c75 100644
--- a/pkg/bisect/bisect_test.go
+++ b/pkg/bisect/bisect_test.go
@@ -57,7 +57,13 @@ func (env *testEnv) Test(numVMs int, reproSyz, reproOpts, reproC []byte) ([]erro
 	if (env.config == "baseline-repro" || env.config == "new-minimized-config" || env.config == "original config") &&
 		(!env.test.fix && commit >= env.test.culprit || env.test.fix &&
 			commit < env.test.culprit) {
-		return crashErrors(numVMs, "crash occurs"), nil
+		var errors []error
+		if env.test.flaky {
+			errors = crashErrors(1, numVMs-1, "crash occurs")
+		} else {
+			errors = crashErrors(numVMs, 0, "crash occurs")
+		}
+		return errors, nil
 	}
 
 	return make([]error, numVMs), nil
@@ -158,6 +164,7 @@ type BisectionTest struct {
 	expectRep    bool
 	noopChange   bool
 	isRelease    bool
+	flaky        bool
 	commitLen    int
 	oldestLatest int
 	// input and output
@@ -175,6 +182,14 @@ var bisectionTests = []BisectionTest{
 		expectRep:   true,
 		culprit:     602,
 	},
+	{
+		name:        "cause-finds-cause",
+		startCommit: 905,
+		commitLen:   1,
+		expectRep:   true,
+		flaky:       true,
+		culprit:     602,
+	},
 	// Test bisection returns correct cause with different baseline/config combinations.
 	{
 		name:            "cause-finds-cause-baseline-repro",
@@ -500,14 +515,17 @@ func checkTest(t *testing.T, test BisectionTest) {
 	}
 }
 
-func crashErrors(num int, title string) []error {
+func crashErrors(crashing, nonCrashing int, title string) []error {
 	var errors []error
-	for i := 0; i < num; i++ {
+	for i := 0; i < crashing; i++ {
 		errors = append(errors, &instance.CrashError{
 			Report: &report.Report{
 				Title: fmt.Sprintf("crashes at %v", title),
 			},
 		})
 	}
+	for i := 0; i < nonCrashing; i++ {
+		errors = append(errors, nil)
+	}
 	return errors
 }