vm/dispatcher: make pool.Run cancellable

Make the pool.Run() function take a context.Context to be able to abort
the callback passed to it or abort its scheduling if it's not yet
running.

Otherwise, if the callback is not yet started and the pool's Loop is
aborted, we risk waiting for pool.Run() forever. It prevents the normal
shutdown of repro.Run() and, consequently, the DiffFuzzer functionality.

2 files changed, 85 insertions, 13 deletions

diff --git a/vm/dispatcher/pool.go b/vm/dispatcher/pool.go
index 331bfd062..20e893499 100644
--- a/vm/dispatcher/pool.go
+++ b/vm/dispatcher/pool.go
@@ -114,7 +114,7 @@ func (p *Pool[T]) Loop(ctx context.Context) {
 func (p *Pool[T]) runInstance(ctx context.Context, inst *poolInstance[T]) {
 	p.waitUnpaused()
 	ctx, cancel := context.WithCancel(ctx)
-
+	defer cancel()
 	log.Logf(2, "pool: booting instance %d", inst.idx)
 
 	inst.reset(cancel)
@@ -187,13 +187,24 @@ func (p *Pool[T]) ReserveForRun(count int) {
 }
 
 // Run blocks until it has found an instance to execute job and until job has finished.
-func (p *Pool[T]) Run(job Runner[T]) {
-	done := make(chan struct{})
-	p.jobs <- func(ctx context.Context, inst T, upd UpdateInfo) {
-		job(ctx, inst, upd)
-		close(done)
+// Returns an error if the job was aborted by cancelling the context.
+func (p *Pool[T]) Run(ctx context.Context, job Runner[T]) error {
+	done := make(chan error)
+	// Submit the job.
+	select {
+	case p.jobs <- func(jobCtx context.Context, inst T, upd UpdateInfo) {
+		mergedCtx, cancel := mergeContextCancel(jobCtx, ctx)
+		defer cancel()
+
+		job(mergedCtx, inst, upd)
+		done <- mergedCtx.Err()
+	}:
+	case <-ctx.Done():
+		// If the loop is aborted, no one is going to pick up the job.
+		return ctx.Err()
 	}
-	<-done
+	// Await the job.
+	return <-done
 }
 
 func (p *Pool[T]) Total() int {
@@ -311,3 +322,15 @@ func (pi *poolInstance[T]) free(job Runner[T]) {
 	default:
 	}
 }
+
+func mergeContextCancel(main, monitor context.Context) (context.Context, func()) {
+	withCancel, cancel := context.WithCancel(main)
+	go func() {
+		select {
+		case <-withCancel.Done():
+		case <-monitor.Done():
+		}
+		cancel()
+	}()
+	return withCancel, cancel
+}
diff --git a/vm/dispatcher/pool_test.go b/vm/dispatcher/pool_test.go
index 452cd598a..cd85367c2 100644
--- a/vm/dispatcher/pool_test.go
+++ b/vm/dispatcher/pool_test.go
@@ -6,6 +6,7 @@ package dispatcher
 import (
 	"context"
 	"runtime"
+	"sync"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -87,7 +88,7 @@ func TestPoolSplit(t *testing.T) {
 		case <-stopRuns:
 		}
 	}
-	go mgr.Run(job)
+	go mgr.Run(ctx, job)
 
 	// So far, there are no reserved instances.
 	for i := 0; i < count; i++ {
@@ -113,7 +114,7 @@ func TestPoolSplit(t *testing.T) {
 
 	// Now let's create and finish more jobs.
 	for i := 0; i < 10; i++ {
-		go mgr.Run(job)
+		go mgr.Run(ctx, job)
 	}
 	mgr.ReserveForRun(2)
 	for i := 0; i < 10; i++ {
@@ -150,8 +151,7 @@ func TestPoolStress(t *testing.T) {
 		}
 	}()
 	for i := 0; i < 128; i++ {
-		go mgr.Run(func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {
-		})
+		go mgr.Run(ctx, func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {})
 		mgr.ReserveForRun(5 + i%5)
 	}
 
@@ -221,7 +221,7 @@ func TestPoolPause(t *testing.T) {
 	}()
 
 	run := make(chan bool, 1)
-	go mgr.Run(func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {
+	go mgr.Run(ctx, func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {
 		run <- true
 	})
 	time.Sleep(10 * time.Millisecond)
@@ -231,12 +231,61 @@ func TestPoolPause(t *testing.T) {
 	mgr.TogglePause(false)
 	<-run
 
-	mgr.Run(func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {})
+	mgr.Run(ctx, func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {})
 
 	cancel()
 	<-done
 }
 
+func TestPoolCancelRun(t *testing.T) {
+	// The test to aid the race detector.
+	mgr := NewPool[*nilInstance](
+		10,
+		func(idx int) (*nilInstance, error) {
+			return &nilInstance{}, nil
+		},
+		func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {
+			<-ctx.Done()
+		},
+	)
+	var wg sync.WaitGroup
+	wg.Add(1)
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		mgr.Loop(ctx)
+		wg.Done()
+	}()
+
+	mgr.ReserveForRun(2)
+
+	started := make(chan struct{})
+	// Schedule more jobs than could be processed simultaneously.
+	for i := 0; i < 15; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			mgr.Run(ctx, func(ctx context.Context, _ *nilInstance, _ UpdateInfo) {
+				select {
+				case <-ctx.Done():
+					return
+				case started <- struct{}{}:
+				}
+				<-ctx.Done()
+			})
+		}()
+	}
+
+	// Two can be started.
+	<-started
+	<-started
+
+	// Now stop the loop and the jbos.
+	cancel()
+
+	// Everything must really stop.
+	wg.Wait()
+}
+
 func makePool(count int) []testInstance {
 	var ret []testInstance
 	for i := 0; i < count; i++ {