executor: better handling for hanged test processes

Currently we kill hanged processes and consider the corresponding test finished.
We don't kill/wait for the actual test subprocess (we don't know its pid to kill,
and waiting will presumably hang). This has 2 problems:
1. If the hanged process causes "task hung" report, we can't reproduce it,
since the test finished too long ago (manager thinks its finished and
discards the request).
2. The test process still consumed per-pid resources.

Explicitly detect and handle such cases:
Manager keeps these hanged tests forever,
and we assign a new proc id for future processes
(don't reuse the hanged one).

2 files changed, 28 insertions, 10 deletions

diff --git a/pkg/fuzzer/queue/queue.go b/pkg/fuzzer/queue/queue.go
index cbdb2ba19..0a56c76d3 100644
--- a/pkg/fuzzer/queue/queue.go
+++ b/pkg/fuzzer/queue/queue.go
@@ -161,7 +161,14 @@ func (r *Result) clone() *Result {
 }
 
 func (r *Result) Stop() bool {
-	return r.Status == ExecFailure || r.Status == Crashed
+	switch r.Status {
+	case Success, Restarted:
+		return false
+	case ExecFailure, Crashed, Hanged:
+		return true
+	default:
+		panic(fmt.Sprintf("unhandled status %v", r.Status))
+	}
 }
 
 type Status int
@@ -171,6 +178,7 @@ const (
 	ExecFailure        // For e.g. serialization errors.
 	Crashed            // The VM crashed holding the request.
 	Restarted          // The VM was restarted holding the request.
+	Hanged             // The program has hanged (can't be killed/waited).
 )
 
 // Executor describes the interface wanted by the producers of requests.
diff --git a/pkg/fuzzer/queue/retry.go b/pkg/fuzzer/queue/retry.go
index c59a2c048..186850b5b 100644
--- a/pkg/fuzzer/queue/retry.go
+++ b/pkg/fuzzer/queue/retry.go
@@ -3,6 +3,10 @@
 
 package queue
 
+import (
+	"fmt"
+)
+
 type retryer struct {
 	pq   *PlainQueue
 	base Source
@@ -28,16 +32,22 @@ func (r *retryer) Next() *Request {
 }
 
 func (r *retryer) done(req *Request, res *Result) bool {
-	// The input was on a restarted VM.
-	if res.Status == Restarted {
-		r.pq.Submit(req)
-		return false
-	}
-	// Retry important requests from crashed VMs once.
-	if res.Status == Crashed && req.Important && !req.onceCrashed {
-		req.onceCrashed = true
+	switch res.Status {
+	case Success, ExecFailure, Hanged:
+		return true
+	case Restarted:
+		// The input was on a restarted VM.
 		r.pq.Submit(req)
 		return false
+	case Crashed:
+		// Retry important requests from crashed VMs once.
+		if req.Important && !req.onceCrashed {
+			req.onceCrashed = true
+			r.pq.Submit(req)
+			return false
+		}
+		return true
+	default:
+		panic(fmt.Sprintf("unhandled status %v", res.Status))
 	}
-	return true
 }