pkg/runtest: fix episodic "directory not empty" failures in tests

I still see the following errors in every other run:

=== CONT  TestCover/64/1
=== NAME  TestCover/64/3
    testing.go:1232: TempDir RemoveAll cleanup: unlinkat /tmp/TestCover6431888025363/001: directory not empty
=== CONT  TestCover/64_fork/0
=== CONT  TestCover/64_fork/9
=== NAME  TestCover/64/1
    testing.go:1232: TempDir RemoveAll cleanup: unlinkat /tmp/TestCover6412810450597/001: directory not empty

We set PR_SET_PDEATHSIG for subprocesses, but we still don't wait for them to terminate.

1 files changed, 19 insertions, 1 deletions

diff --git a/pkg/runtest/run_test.go b/pkg/runtest/run_test.go
index f69173ef8..9d42e110c 100644
--- a/pkg/runtest/run_test.go
+++ b/pkg/runtest/run_test.go
@@ -10,9 +10,11 @@ import (
 	"encoding/hex"
 	"flag"
 	"fmt"
+	"os"
 	"path/filepath"
 	"runtime"
 	"testing"
+	"time"
 
 	"github.com/google/syzkaller/pkg/csource"
 	"github.com/google/syzkaller/pkg/flatrpc"
@@ -437,6 +439,10 @@ func makeComps(comps ...Comparison) []byte {
 
 func startRPCServer(t *testing.T, target *prog.Target, executor, vmArch string, source queue.Source,
 	maxSignal, coverFilter []uint64, machineChecked func(features flatrpc.Feature)) {
+	dir, err := os.MkdirTemp("", "syz-runtest")
+	if err != nil {
+		t.Fatal(err)
+	}
 	ctx, done := context.WithCancel(context.Background())
 	cfg := &rpcserver.LocalConfig{
 		Config: rpcserver.Config{
@@ -452,7 +458,7 @@ func startRPCServer(t *testing.T, target *prog.Target, executor, vmArch string,
 			Slowdown: 10, // to deflake slower tests
 		},
 		Executor:    executor,
-		Dir:         t.TempDir(),
+		Dir:         dir,
 		Context:     ctx,
 		GDB:         *flagGDB,
 		MaxSignal:   maxSignal,
@@ -473,6 +479,18 @@ func startRPCServer(t *testing.T, target *prog.Target, executor, vmArch string,
 		if err := <-errc; err != nil {
 			t.Fatal(err)
 		}
+		// We need to retry b/c we don't wait for all executor subprocesses (only set PR_SET_PDEATHSIG),
+		// so t.TempDir() leads to episodic "directory not empty" failures.
+		for i := 0; ; i++ {
+			if err := os.RemoveAll(dir); err == nil {
+				break
+			}
+			if i < 100 {
+				time.Sleep(100 * time.Millisecond)
+				continue
+			}
+			t.Fatalf("failed to remove temp dir %v: %v", dir, err)
+		}
 	})
 }