prog: fix minimization bugs

Fix several nasty bugs in minimization that could lead
to almost arbitrary results. These bugs affected both
crash minimization and corpus population.
Extend the randomized test to catch these bugs.
Add additional asserts to code to catch similar bugs in future.

Reported-by @xairy

1 files changed, 26 insertions, 7 deletions

diff --git a/prog/minimization.go b/prog/minimization.go
index 5f3e9f604..fd81a7252 100644
--- a/prog/minimization.go
+++ b/prog/minimization.go
@@ -95,9 +95,21 @@ func (ctx *minimizeArgsCtx) do(arg Arg, path string) bool {
 	if ctx.triedPaths[path] {
 		return false
 	}
+	p0 := *ctx.p0
 	if arg.Type().minimize(ctx, arg, path) {
 		return true
 	}
+	if *ctx.p0 == ctx.p {
+		// If minimize committed a new program, it must return true.
+		// Otherwise *ctx.p0 and ctx.p will point to the same program
+		// and any temp mutations to ctx.p will unintentionally affect ctx.p0.
+		panic("shared program committed")
+	}
+	if *ctx.p0 != p0 {
+		// New program was committed, but we did not start iteration anew.
+		// This means we are iterating over a stale tree and any changes won't be visible.
+		panic("iterating over stale program")
+	}
 	ctx.triedPaths[path] = true
 	return false
 }
@@ -191,9 +203,10 @@ func minimizeInt(ctx *minimizeArgsCtx, arg Arg, path string) bool {
 	a.Val = def.Val
 	if ctx.pred(ctx.p, ctx.callIndex0) {
 		*ctx.p0 = ctx.p
-	} else {
-		a.Val = v0
+		ctx.triedPaths[path] = true
+		return true
 	}
+	a.Val = v0
 	return false
 }
 
@@ -210,11 +223,12 @@ func (typ *ResourceType) minimize(ctx *minimizeArgsCtx, arg Arg, path string) bo
 	a.Res, a.Val = nil, typ.Default()
 	if ctx.pred(ctx.p, ctx.callIndex0) {
 		*ctx.p0 = ctx.p
-	} else {
-		a.Res, a.Val = r0, 0
-		a.Res.uses[a] = true
+		ctx.triedPaths[path] = true
+		return true
 	}
-	return false
+	a.Res, a.Val = r0, 0
+	a.Res.uses[a] = true
+	return true
 }
 
 func (typ *BufferType) minimize(ctx *minimizeArgsCtx, arg Arg, path string) bool {
@@ -223,6 +237,7 @@ func (typ *BufferType) minimize(ctx *minimizeArgsCtx, arg Arg, path string) bool
 		return false
 	}
 	a := arg.(*DataArg)
+	len0 := len(a.Data())
 	minLen := int(typ.RangeBegin)
 	for step := len(a.Data()) - minLen; len(a.Data()) > minLen && step > 0; {
 		if len(a.Data())-step >= minLen {
@@ -239,6 +254,10 @@ func (typ *BufferType) minimize(ctx *minimizeArgsCtx, arg Arg, path string) bool
 			break
 		}
 	}
-	*ctx.p0 = ctx.p
+	if len(a.Data()) != len0 {
+		*ctx.p0 = ctx.p
+		ctx.triedPaths[path] = true
+		return true
+	}
 	return false
 }