syz-ci: gate concurrent env.Test executions

This will help reduce the number of overcommitted VMs.

6 files changed, 19 insertions, 5 deletions

diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go
index d24bb8f00..5df566581 100644
--- a/pkg/bisect/bisect.go
+++ b/pkg/bisect/bisect.go
@@ -32,6 +32,7 @@ type Config struct {
 	Repro           ReproConfig
 	Manager         *mgrconfig.Config
 	BuildSemaphore  *instance.Semaphore
+	TestSemaphore   *instance.Semaphore
 }
 
 type KernelConfig struct {
@@ -121,7 +122,7 @@ func Run(cfg *Config) (*Result, error) {
 	if err != nil {
 		return nil, err
 	}
-	inst, err := instance.NewEnv(cfg.Manager, cfg.BuildSemaphore)
+	inst, err := instance.NewEnv(cfg.Manager, cfg.BuildSemaphore, cfg.TestSemaphore)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go
index 1a504fdd7..31f75e21c 100644
--- a/pkg/instance/instance.go
+++ b/pkg/instance/instance.go
@@ -37,6 +37,7 @@ type env struct {
 	cfg           *mgrconfig.Config
 	optionalFlags bool
 	buildSem      *Semaphore
+	testSem       *Semaphore
 }
 
 type BuildKernelConfig struct {
@@ -49,7 +50,7 @@ type BuildKernelConfig struct {
 	KernelConfig []byte
 }
 
-func NewEnv(cfg *mgrconfig.Config, buildSem *Semaphore) (Env, error) {
+func NewEnv(cfg *mgrconfig.Config, buildSem, testSem *Semaphore) (Env, error) {
 	if !vm.AllowsOvercommit(cfg.Type) {
 		return nil, fmt.Errorf("test instances are not supported for %v VMs", cfg.Type)
 	}
@@ -69,6 +70,7 @@ func NewEnv(cfg *mgrconfig.Config, buildSem *Semaphore) (Env, error) {
 		cfg:           cfg,
 		optionalFlags: true,
 		buildSem:      buildSem,
+		testSem:       testSem,
 	}
 	return env, nil
 }
@@ -235,6 +237,10 @@ func (err *CrashError) Error() string {
 // TestError is returned if there is a problem with kernel/image (crash, reboot loop, etc).
 // CrashError is returned if the reproducer crashes kernel.
 func (env *env) Test(numVMs int, reproSyz, reproOpts, reproC []byte) ([]EnvTestResult, error) {
+	if env.testSem != nil {
+		env.testSem.Wait()
+		defer env.testSem.Signal()
+	}
 	if err := mgrconfig.Complete(env.cfg); err != nil {
 		return nil, err
 	}
diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go
index af1433200..fcdca9ada 100644
--- a/syz-ci/jobs.go
+++ b/syz-ci/jobs.go
@@ -488,6 +488,7 @@ func (jp *JobProcessor) bisect(job *Job, mgrcfg *mgrconfig.Config) error {
 		},
 		Manager:        mgrcfg,
 		BuildSemaphore: buildSem,
+		TestSemaphore:  testSem,
 	}
 
 	res, err := bisect.Run(cfg)
@@ -547,7 +548,7 @@ func (jp *JobProcessor) bisect(job *Job, mgrcfg *mgrconfig.Config) error {
 
 func (jp *JobProcessor) testPatch(job *Job, mgrcfg *mgrconfig.Config) error {
 	req, resp, mgr := job.req, job.resp, job.mgr
-	env, err := instance.NewEnv(mgrcfg, buildSem)
+	env, err := instance.NewEnv(mgrcfg, buildSem, testSem)
 	if err != nil {
 		return err
 	}
diff --git a/syz-ci/manager.go b/syz-ci/manager.go
index 3744944a5..077358509 100644
--- a/syz-ci/manager.go
+++ b/syz-ci/manager.go
@@ -152,6 +152,11 @@ func createManager(cfg *Config, mgrcfg *ManagerConfig, stop chan struct{},
 // Also current image build script uses some global resources (/dev/nbd0) and can't run in parallel.
 var buildSem = instance.NewSemaphore(1)
 
+// Gates tests that require extra VMs.
+// Currently we overcommit instances in such cases, so we'd like to minimize the number of
+// simultaneous env.Test calls.
+var testSem = instance.NewSemaphore(1)
+
 func (mgr *Manager) loop() {
 	lastCommit := ""
 	nextBuildTime := time.Now()
@@ -413,7 +418,7 @@ func (mgr *Manager) testImage(imageDir string, info *BuildInfo) error {
 	if !vm.AllowsOvercommit(mgrcfg.Type) {
 		return nil // No support for creating machines out of thin air.
 	}
-	env, err := instance.NewEnv(mgrcfg, buildSem)
+	env, err := instance.NewEnv(mgrcfg, buildSem, testSem)
 	if err != nil {
 		return err
 	}
diff --git a/syz-ci/syz-ci.go b/syz-ci/syz-ci.go
index 2017700f7..41564538c 100644
--- a/syz-ci/syz-ci.go
+++ b/syz-ci/syz-ci.go
@@ -236,6 +236,7 @@ func main() {
 		case <-updatePending:
 		}
 		buildSem.WaitAll() // wait for all current builds
+		testSem.WaitAll()
 		close(stop)
 		wg.Done()
 	}()
diff --git a/tools/syz-testbuild/testbuild.go b/tools/syz-testbuild/testbuild.go
index 5596ad02b..55d980818 100644
--- a/tools/syz-testbuild/testbuild.go
+++ b/tools/syz-testbuild/testbuild.go
@@ -110,7 +110,7 @@ func main() {
 	if err != nil {
 		tool.Fail(err)
 	}
-	env, err := instance.NewEnv(cfg, nil)
+	env, err := instance.NewEnv(cfg, nil, nil)
 	if err != nil {
 		tool.Fail(err)
 	}