prog: fix panic during squashing

Netbsd syzbot instance crashes trying to squash a pointer.
Pointers must not be squashed. This happens because of
recursive ucontext_t type that contains a pointer to itself.
When we assign SquashableElem recursive struct types may not be fully
generated yet, and ForeachArgType won't observe all types.
Assign SquashableElem after all types are fully generated.

5 files changed, 88 insertions, 49 deletions

diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go
index 0a2396b5b..51bbbdf65 100644
--- a/pkg/compiler/compiler_test.go
+++ b/pkg/compiler/compiler_test.go
@@ -355,3 +355,33 @@ foo$2(a ptr[in, string[str1]], b ptr[in, string[str2]])
 		}
 	}
 }
+
+func TestSquashablePtr(t *testing.T) {
+	t.Parallel()
+	// recursive must not be marked as squashable b/c it contains a pointer.
+	const input = `
+foo(a ptr[in, recursive])
+
+recursive {
+	f0	ptr[in, recursive, opt]
+	f1	int32
+	f2	array[int8]
+}
+`
+	eh := func(pos ast.Pos, msg string) {
+		t.Errorf("%v: %v", pos, msg)
+	}
+	desc := ast.Parse([]byte(input), "input", eh)
+	if desc == nil {
+		t.Fatal("failed to parse")
+	}
+	p := Compile(desc, map[string]uint64{"SYS_foo": 1}, targets.List[targets.TestOS][targets.TestArch64], eh)
+	if p == nil {
+		t.Fatal("failed to compile")
+	}
+	for _, typ := range p.Types {
+		if ptr, ok := typ.(*prog.PtrType); ok && ptr.SquashableElem {
+			t.Fatal("got squashable ptr")
+		}
+	}
+}
diff --git a/pkg/compiler/gen.go b/pkg/compiler/gen.go
index f91e2502c..1fcf8dcac 100644
--- a/pkg/compiler/gen.go
+++ b/pkg/compiler/gen.go
@@ -103,6 +103,14 @@ func (comp *compiler) genSyscalls() []*prog.Syscall {
 			calls = append(calls, comp.genSyscall(n, callArgSizes[n.CallName]))
 		}
 	}
+	// We assign SquashableElem here rather than during pointer type generation
+	// because during pointer generation recursive struct types may not be fully
+	// generated yet, thus ForeachArgType won't observe all types.
+	prog.ForeachTypePost(calls, func(typ prog.Type, ctx *prog.TypeCtx) {
+		if ptr, ok := typ.(*prog.PtrType); ok {
+			ptr.SquashableElem = isSquashableElem(ptr.Elem, ptr.ElemDir)
+		}
+	})
 	sort.Slice(calls, func(i, j int) bool {
 		return calls[i].Name < calls[j].Name
 	})
diff --git a/pkg/compiler/types.go b/pkg/compiler/types.go
index 70a0bd565..906af199c 100644
--- a/pkg/compiler/types.go
+++ b/pkg/compiler/types.go
@@ -213,10 +213,9 @@ var typePtr = &typeDesc{
 		elem := comp.genType(args[1], 0)
 		elemDir := genDir(args[0])
 		return &prog.PtrType{
-			TypeCommon:     base.TypeCommon,
-			Elem:           elem,
-			ElemDir:        elemDir,
-			SquashableElem: isSquashableElem(elem, elemDir),
+			TypeCommon: base.TypeCommon,
+			Elem:       elem,
+			ElemDir:    elemDir,
 		}
 	},
 }
diff --git a/prog/any.go b/prog/any.go
index ee898446b..c43d8dd8d 100644
--- a/prog/any.go
+++ b/prog/any.go
@@ -175,7 +175,7 @@ func (target *Target) squashPtrImpl(a Arg, elems *[]Arg) {
 	case *GroupArg:
 		target.squashGroup(arg, elems)
 	default:
-		panic("bad arg kind")
+		panic(fmt.Sprintf("bad arg kind %v (%#v) %v", a, a, a.Type()))
 	}
 	if pad != 0 {
 		elem := target.ensureDataElem(elems)
diff --git a/prog/any_test.go b/prog/any_test.go
index 333266622..8c3f3c8c1 100644
--- a/prog/any_test.go
+++ b/prog/any_test.go
@@ -5,6 +5,7 @@ package prog
 
 import (
 	"fmt"
+	"math/rand"
 	"sort"
 	"strings"
 	"testing"
@@ -13,56 +14,57 @@ import (
 )
 
 func TestIsComplexPtr(t *testing.T) {
-	target, rs, _ := initRandomTargetTest(t, "linux", "amd64")
-	allComplex := make(map[string]bool)
-	ForeachType(target.Syscalls, func(t Type, ctx *TypeCtx) {
-		if ptr, ok := t.(*PtrType); ok && ptr.SquashableElem {
-			allComplex[ptr.Elem.String()] = true
+	testEachTargetRandom(t, func(t *testing.T, target *Target, rs rand.Source, iters int) {
+		allComplex := make(map[string]bool)
+		ForeachType(target.Syscalls, func(t Type, ctx *TypeCtx) {
+			if ptr, ok := t.(*PtrType); ok && ptr.SquashableElem {
+				allComplex[ptr.Elem.String()] = true
+			}
+		})
+		var arr []string
+		for id := range allComplex {
+			arr = append(arr, id)
 		}
-	})
-	var arr []string
-	for id := range allComplex {
-		arr = append(arr, id)
-	}
-	sort.Strings(arr)
-	// Log all complex types for manual inspection.
-	t.Log("complex types:\n" + strings.Join(arr, "\n"))
-	if testing.Short() || testutil.RaceEnabled {
-		return
-	}
-	// Compare with what we can generate at runtime.
-	// We cannot guarantee that we will generate 100% of complex types
-	// (some are no_generate, and the process is random), but we should
-	// generate at least 90% or there is something wrong.
-	ct := target.DefaultChoiceTable()
-	r := newRand(target, rs)
-	generatedComplex := make(map[string]bool)
-	for _, meta := range target.Syscalls {
-		if meta.Attrs.Disabled || meta.Attrs.NoGenerate {
-			continue
+		sort.Strings(arr)
+		// Log all complex types for manual inspection.
+		t.Log("complex types:\n" + strings.Join(arr, "\n"))
+		if testing.Short() || testutil.RaceEnabled {
+			return
 		}
-		for i := 0; i < 10; i++ {
-			s := newState(target, ct, nil)
-			calls := r.generateParticularCall(s, meta)
-			p := &Prog{Target: target, Calls: calls}
-			for _, arg := range p.complexPtrs() {
-				generatedComplex[arg.arg.Res.Type().String()] = true
+		// Compare with what we can generate at runtime.
+		// We cannot guarantee that we will generate 100% of complex types
+		// (some are no_generate, and the process is random), but we should
+		// generate at least 90% or there is something wrong.
+		ct := target.DefaultChoiceTable()
+		r := newRand(target, rs)
+		generatedComplex := make(map[string]bool)
+		for _, meta := range target.Syscalls {
+			if meta.Attrs.Disabled || meta.Attrs.NoGenerate {
+				continue
+			}
+			for i := 0; i < 10; i++ {
+				s := newState(target, ct, nil)
+				calls := r.generateParticularCall(s, meta)
+				p := &Prog{Target: target, Calls: calls}
+				for _, arg := range p.complexPtrs() {
+					generatedComplex[arg.arg.Res.Type().String()] = true
+				}
 			}
 		}
-	}
-	for id := range generatedComplex {
-		if !allComplex[id] {
-			t.Errorf("generated complex %v that is not statically complex", id)
+		for id := range generatedComplex {
+			if !allComplex[id] {
+				t.Errorf("generated complex %v that is not statically complex", id)
+			}
 		}
-	}
-	for id := range allComplex {
-		if !generatedComplex[id] {
-			t.Logf("did not generate %v", id)
+		for id := range allComplex {
+			if !generatedComplex[id] {
+				t.Logf("did not generate %v", id)
+			}
 		}
-	}
-	if len(generatedComplex) < len(allComplex)*9/10 {
-		t.Errorf("generated too few complex types: %v/%v", len(generatedComplex), len(allComplex))
-	}
+		if len(generatedComplex) < len(allComplex)*9/10 {
+			t.Errorf("generated too few complex types: %v/%v", len(generatedComplex), len(allComplex))
+		}
+	})
 }
 
 func TestSquash(t *testing.T) {