sys, pkg/compiler: move padding computation to compiler

This makes types constant during execution, everything is precomputed.

6 files changed, 495 insertions, 88 deletions

diff --git a/pkg/compiler/check.go b/pkg/compiler/check.go
index f75c02766..7b0cf34aa 100644
--- a/pkg/compiler/check.go
+++ b/pkg/compiler/check.go
@@ -25,6 +25,7 @@ func (comp *compiler) check() {
 	comp.checkRecursion()
 	comp.checkLenTargets()
 	comp.checkConstructors()
+	comp.checkVarlens()
 }
 
 func (comp *compiler) checkNames() {
@@ -562,3 +563,45 @@ func checkTypeKind(t *ast.Type, kind int) (unexpected string, expect string, ok
 	}
 	return
 }
+
+func (comp *compiler) checkVarlens() {
+	for _, decl := range comp.desc.Nodes {
+		switch n := decl.(type) {
+		case *ast.Struct:
+			comp.checkVarlen(n)
+		}
+	}
+}
+
+func (comp *compiler) isVarlen(t *ast.Type) bool {
+	desc, args, base := comp.getArgsBase(t, "", sys.DirIn, false)
+	return desc.Varlen != nil && desc.Varlen(comp, t, args, base)
+}
+
+func (comp *compiler) checkVarlen(n *ast.Struct) {
+	// Non-varlen unions can't have varlen fields.
+	// Non-packed structs can't have varlen fields in the middle.
+	if n.IsUnion {
+		if varlen := comp.parseUnionAttrs(n); varlen {
+			return
+		}
+	} else {
+		if packed, _ := comp.parseStructAttrs(n); packed {
+			return
+		}
+	}
+	for i, f := range n.Fields {
+		if !n.IsUnion && i == len(n.Fields)-1 {
+			break
+		}
+		if comp.isVarlen(f.Type) {
+			if n.IsUnion {
+				comp.error(f.Pos, "variable size field %v in non-varlen union %v",
+					f.Name.Name, n.Name.Name)
+			} else {
+				comp.error(f.Pos, "variable size field %v in the middle of non-packed struct %v",
+					f.Name.Name, n.Name.Name)
+			}
+		}
+	}
+}
diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go
index f08a32987..fe9f286ad 100644
--- a/pkg/compiler/compiler.go
+++ b/pkg/compiler/compiler.go
@@ -33,11 +33,9 @@ import (
 
 // Prog is description compilation result.
 type Prog struct {
-	// Processed AST (temporal measure, remove later).
-	Desc         *ast.Description
-	Resources    []*sys.ResourceDesc
-	Syscalls     []*sys.Call
-	StructFields []*sys.StructFields
+	Resources   []*sys.ResourceDesc
+	Syscalls    []*sys.Call
+	StructDescs []*sys.KeyedStruct
 	// Set of unsupported syscalls/flags.
 	Unsupported map[string]bool
 }
@@ -48,15 +46,17 @@ func Compile(desc *ast.Description, consts map[string]uint64, ptrSize uint64, eh
 		eh = ast.LoggingHandler
 	}
 	comp := &compiler{
-		desc:        ast.Clone(desc),
-		eh:          eh,
-		ptrSize:     ptrSize,
-		unsupported: make(map[string]bool),
-		resources:   make(map[string]*ast.Resource),
-		structs:     make(map[string]*ast.Struct),
-		intFlags:    make(map[string]*ast.IntFlags),
-		strFlags:    make(map[string]*ast.StrFlags),
-		structUses:  make(map[sys.StructKey]*ast.Struct),
+		desc:         ast.Clone(desc),
+		eh:           eh,
+		ptrSize:      ptrSize,
+		unsupported:  make(map[string]bool),
+		resources:    make(map[string]*ast.Resource),
+		structs:      make(map[string]*ast.Struct),
+		intFlags:     make(map[string]*ast.IntFlags),
+		strFlags:     make(map[string]*ast.StrFlags),
+		structDescs:  make(map[sys.StructKey]*sys.StructDesc),
+		structNodes:  make(map[*sys.StructDesc]*ast.Struct),
+		structVarlen: make(map[string]bool),
 	}
 	comp.assignSyscallNumbers(consts)
 	comp.patchConsts(consts)
@@ -67,12 +67,12 @@ func Compile(desc *ast.Description, consts map[string]uint64, ptrSize uint64, eh
 	for _, w := range comp.warnings {
 		eh(w.pos, w.msg)
 	}
+	syscalls := comp.genSyscalls()
 	return &Prog{
-		Desc:         comp.desc,
-		Resources:    comp.genResources(),
-		Syscalls:     comp.genSyscalls(),
-		StructFields: comp.genStructFields(),
-		Unsupported:  comp.unsupported,
+		Resources:   comp.genResources(),
+		Syscalls:    syscalls,
+		StructDescs: comp.genStructDescs(syscalls),
+		Unsupported: comp.unsupported,
 	}
 }
 
@@ -88,7 +88,10 @@ type compiler struct {
 	structs     map[string]*ast.Struct
 	intFlags    map[string]*ast.IntFlags
 	strFlags    map[string]*ast.StrFlags
-	structUses  map[sys.StructKey]*ast.Struct
+
+	structDescs  map[sys.StructKey]*sys.StructDesc
+	structNodes  map[*sys.StructDesc]*ast.Struct
+	structVarlen map[string]bool
 }
 
 type warn struct {
@@ -162,12 +165,16 @@ func (comp *compiler) getArgsBase(t *ast.Type, field string, dir sys.Dir, isArg
 	*typeDesc, []*ast.Type, sys.IntTypeCommon) {
 	desc := comp.getTypeDesc(t)
 	args, opt := removeOpt(t)
-	com := genCommon(t.Ident, field, dir, opt)
-	base := genIntCommon(com, comp.ptrSize, 0, false)
-	if !isArg && desc.NeedBase {
-		baseType := args[len(args)-1]
-		args = args[:len(args)-1]
-		base = typeInt.Gen(comp, baseType, nil, base).(*sys.IntType).IntTypeCommon
+	size := sizeUnassigned
+	com := genCommon(t.Ident, field, size, dir, opt)
+	base := genIntCommon(com, 0, false)
+	if desc.NeedBase {
+		base.TypeSize = comp.ptrSize
+		if !isArg {
+			baseType := args[len(args)-1]
+			args = args[:len(args)-1]
+			base = typeInt.Gen(comp, baseType, nil, base).(*sys.IntType).IntTypeCommon
+		}
 	}
 	return desc, args, base
 }
diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go
index 08fb9a9b9..d6d139161 100644
--- a/pkg/compiler/compiler_test.go
+++ b/pkg/compiler/compiler_test.go
@@ -71,3 +71,33 @@ func TestFuzz(t *testing.T) {
 		}
 	}
 }
+
+func TestAlign(t *testing.T) {
+	const input = `
+foo$0(a ptr[in, s0])
+s0 {
+	f0	int8
+	f1	int16
+}
+
+foo$1(a ptr[in, s1])
+s1 {
+	f0	ptr[in, s2, opt]
+}
+s2 {
+	f1	s1
+	f2	array[s1, 2]
+	f3	array[array[s1, 2], 2]
+}
+	`
+	desc := ast.Parse([]byte(input), "input", nil)
+	if desc == nil {
+		t.Fatal("failed to parse")
+	}
+	p := Compile(desc, map[string]uint64{"__NR_foo": 1}, 8, nil)
+	if p == nil {
+		t.Fatal("failed to compile")
+	}
+	got := p.StructDescs[0].Desc
+	t.Logf("got: %#v", got)
+}
diff --git a/pkg/compiler/gen.go b/pkg/compiler/gen.go
index 9cd32fb12..6634609bd 100644
--- a/pkg/compiler/gen.go
+++ b/pkg/compiler/gen.go
@@ -11,6 +11,8 @@ import (
 	"github.com/google/syzkaller/sys"
 )
 
+const sizeUnassigned = ^uint64(0)
+
 func (comp *compiler) genResources() []*sys.ResourceDesc {
 	var resources []*sys.ResourceDesc
 	for _, decl := range comp.desc.Nodes {
@@ -69,22 +71,119 @@ func (comp *compiler) genSyscall(n *ast.Call) *sys.Call {
 	}
 }
 
-func (comp *compiler) genStructFields() []*sys.StructFields {
-	var structs []*sys.StructFields
-	generated := make(map[sys.StructKey]bool)
-	// Generating structs can produce more struct uses, so we do this in the loop.
-	// Consider, a syscall references a struct only as in,
-	// but then another struct references it as out.
-	for n := -1; n != len(generated); {
-		n = len(generated)
-		for key, n := range comp.structUses {
-			if generated[key] {
-				continue
+func (comp *compiler) genStructDescs(syscalls []*sys.Call) []*sys.KeyedStruct {
+	// Calculate struct/union/array sizes, add padding to structs and detach
+	// StructDesc's from StructType's. StructType's can be recursive so it's
+	// not possible to write them out inline as other types. To break the
+	// recursion detach them, and write StructDesc's out as separate array
+	// of KeyedStruct's. sys package will reattach them during init.
+
+	padded := make(map[interface{}]bool)
+	detach := make(map[**sys.StructDesc]bool)
+	var structs []*sys.KeyedStruct
+	var rec func(t sys.Type)
+	checkStruct := func(key sys.StructKey, descp **sys.StructDesc) bool {
+		detach[descp] = true
+		desc := *descp
+		if padded[desc] {
+			return false
+		}
+		padded[desc] = true
+		for _, f := range desc.Fields {
+			rec(f)
+			if !f.Varlen() && f.Size() == sizeUnassigned {
+				// An inner struct is not padded yet.
+				// Leave this struct for next iteration.
+				delete(padded, desc)
+				return false
+			}
+		}
+		structs = append(structs, &sys.KeyedStruct{
+			Key:  key,
+			Desc: desc,
+		})
+		return true
+	}
+	rec = func(t0 sys.Type) {
+		switch t := t0.(type) {
+		case *sys.PtrType:
+			rec(t.Type)
+		case *sys.ArrayType:
+			if padded[t] {
+				return
+			}
+			rec(t.Type)
+			if !t.Type.Varlen() && t.Type.Size() == sizeUnassigned {
+				// An inner struct is not padded yet.
+				// Leave this array for next iteration.
+				return
+			}
+			padded[t] = true
+			t.TypeSize = 0
+			if t.Kind == sys.ArrayRangeLen && t.RangeBegin == t.RangeEnd && !t.Type.Varlen() {
+				t.TypeSize = t.RangeBegin * t.Type.Size()
+			}
+		case *sys.StructType:
+			if !checkStruct(t.Key, &t.StructDesc) {
+				return
+			}
+			// Add paddings, calculate size, mark bitfields.
+			varlen := false
+			for _, f := range t.Fields {
+				if f.Varlen() {
+					varlen = true
+				}
+			}
+			comp.markBitfields(t.Fields)
+			packed, alignAttr := comp.parseStructAttrs(comp.structNodes[t.StructDesc])
+			t.Fields = comp.addAlignment(t.Fields, varlen, packed, alignAttr)
+			t.AlignAttr = alignAttr
+			t.TypeSize = 0
+			if !varlen {
+				for _, f := range t.Fields {
+					if f.BitfieldLength() == 0 || f.BitfieldLast() {
+						t.TypeSize += f.Size()
+					}
+				}
+			}
+		case *sys.UnionType:
+			if !checkStruct(t.Key, &t.StructDesc) {
+				return
+			}
+			t.TypeSize = 0
+			varlen := comp.parseUnionAttrs(comp.structNodes[t.StructDesc])
+			if !varlen {
+				for _, fld := range t.Fields {
+					if t.TypeSize < fld.Size() {
+						t.TypeSize = fld.Size()
+					}
+				}
+			}
+		}
+	}
+
+	// We have to do this in the loop until we pad nothing new
+	// due to recursive structs.
+	for {
+		start := len(padded)
+		for _, c := range syscalls {
+			for _, a := range c.Args {
+				rec(a)
+			}
+			if c.Ret != nil {
+				rec(c.Ret)
 			}
-			generated[key] = true
-			structs = append(structs, comp.genStructField(key, n))
 		}
+		if start == len(padded) {
+			break
+		}
+	}
+
+	// Detach StructDesc's from StructType's. sys will reattach them again.
+	for descp := range detach {
+		*descp = nil
 	}
+
 	sort.Slice(structs, func(i, j int) bool {
 		si, sj := structs[i], structs[j]
 		if si.Key.Name != sj.Key.Name {
@@ -95,15 +194,34 @@ func (comp *compiler) genStructFields() []*sys.StructFields {
 	return structs
 }
 
-func (comp *compiler) genStructField(key sys.StructKey, n *ast.Struct) *sys.StructFields {
-	fields := comp.genFieldArray(n.Fields, key.Dir, false)
-	if !n.IsUnion {
-		comp.markBitfields(fields)
+func (comp *compiler) genStructDesc(res *sys.StructDesc, n *ast.Struct, dir sys.Dir) {
+	// Leave node for genStructDescs to calculate size/padding.
+	comp.structNodes[res] = n
+	*res = sys.StructDesc{
+		TypeCommon: genCommon(n.Name.Name, "", sizeUnassigned, dir, false),
+		Fields:     comp.genFieldArray(n.Fields, dir, false),
+	}
+}
+
+func (comp *compiler) isStructVarlen(name string) bool {
+	if varlen, ok := comp.structVarlen[name]; ok {
+		return varlen
 	}
-	return &sys.StructFields{
-		Key:    key,
-		Fields: fields,
+	s := comp.structs[name]
+	if s.IsUnion && comp.parseUnionAttrs(s) {
+		comp.structVarlen[name] = true
+		return true
 	}
+	comp.structVarlen[name] = false // to not hang on recursive types
+	varlen := false
+	for _, fld := range s.Fields {
+		if comp.isVarlen(fld.Type) {
+			varlen = true
+			break
+		}
+	}
+	comp.structVarlen[name] = varlen
+	return varlen
 }
 
 func (comp *compiler) markBitfields(fields []sys.Type) {
@@ -127,22 +245,113 @@ func (comp *compiler) markBitfields(fields []sys.Type) {
 func setBitfieldOffset(t0 sys.Type, offset uint64, last bool) {
 	switch t := t0.(type) {
 	case *sys.IntType:
-		t.BitfieldOff = offset
-		t.BitfieldLst = last
+		t.BitfieldOff, t.BitfieldLst = offset, last
 	case *sys.ConstType:
-		t.BitfieldOff = offset
-		t.BitfieldLst = last
+		t.BitfieldOff, t.BitfieldLst = offset, last
 	case *sys.LenType:
-		t.BitfieldOff = offset
-		t.BitfieldLst = last
+		t.BitfieldOff, t.BitfieldLst = offset, last
 	case *sys.FlagsType:
-		t.BitfieldOff = offset
-		t.BitfieldLst = last
+		t.BitfieldOff, t.BitfieldLst = offset, last
 	case *sys.ProcType:
-		t.BitfieldOff = offset
-		t.BitfieldLst = last
+		t.BitfieldOff, t.BitfieldLst = offset, last
+	default:
+		panic(fmt.Sprintf("type %#v can't be a bitfield", t))
+	}
+}
+
+func (comp *compiler) addAlignment(fields []sys.Type, varlen, packed bool, alignAttr uint64) []sys.Type {
+	var newFields []sys.Type
+	if packed {
+		// If a struct is packed, statically sized and has explicitly set alignment,
+		// add a padding at the end.
+		newFields = fields
+		if !varlen && alignAttr != 0 {
+			size := uint64(0)
+			for _, f := range fields {
+				size += f.Size()
+			}
+			if tail := size % alignAttr; tail != 0 {
+				newFields = append(newFields, genPad(alignAttr-tail))
+			}
+		}
+		return newFields
+	}
+	var off uint64
+	// TODO(dvyukov): this is wrong: if alignAttr!=0, we must use it, not max
+	align := alignAttr
+	for i, f := range fields {
+		if i > 0 && (fields[i-1].BitfieldLength() == 0 || fields[i-1].BitfieldLast()) {
+			a := comp.typeAlign(f)
+			if align < a {
+				align = a
+			}
+			// Append padding if the last field is not a bitfield or it's the last bitfield in a set.
+			if off%a != 0 {
+				pad := a - off%a
+				off += pad
+				newFields = append(newFields, genPad(pad))
+			}
+		}
+		newFields = append(newFields, f)
+		if (f.BitfieldLength() == 0 || f.BitfieldLast()) && (i != len(fields)-1 || !f.Varlen()) {
+			// Increase offset if the current field is not a bitfield
+			// or it's the last bitfield in a set, except when it's
+			// the last field in a struct and has variable length.
+			off += f.Size()
+		}
+	}
+	if align != 0 && off%align != 0 && !varlen {
+		pad := align - off%align
+		off += pad
+		newFields = append(newFields, genPad(pad))
+	}
+	return newFields
+}
+
+func (comp *compiler) typeAlign(t0 sys.Type) uint64 {
+	switch t0.(type) {
+	case *sys.IntType, *sys.ConstType, *sys.LenType, *sys.FlagsType, *sys.ProcType,
+		*sys.CsumType, *sys.PtrType, *sys.VmaType, *sys.ResourceType:
+		return t0.Size()
+	case *sys.BufferType:
+		return 1
+	}
+
+	switch t := t0.(type) {
+	case *sys.ArrayType:
+		return comp.typeAlign(t.Type)
+	case *sys.StructType:
+		packed, alignAttr := comp.parseStructAttrs(comp.structNodes[t.StructDesc])
+		if alignAttr != 0 {
+			return alignAttr // overrided by user attribute
+		}
+		if packed {
+			return 1
+		}
+		align := uint64(0)
+		for _, f := range t.Fields {
+			if a := comp.typeAlign(f); align < a {
+				align = a
+			}
+		}
+		return align
+	case *sys.UnionType:
+		align := uint64(0)
+		for _, f := range t.Fields {
+			if a := comp.typeAlign(f); align < a {
+				align = a
+			}
+		}
+		return align
 	default:
-		panic(fmt.Sprintf("type %+v can't be a bitfield", t))
+		panic(fmt.Sprintf("unknown type: %#v", t))
+	}
+}
+
+func genPad(size uint64) sys.Type {
+	return &sys.ConstType{
+		IntTypeCommon: genIntCommon(genCommon("pad", "", size, sys.DirIn, false), 0, false),
+		IsPad:         true,
 	}
 }
 
@@ -163,20 +372,20 @@ func (comp *compiler) genType(t *ast.Type, field string, dir sys.Dir, isArg bool
 	return desc.Gen(comp, t, args, base)
 }
 
-func genCommon(name, field string, dir sys.Dir, opt bool) sys.TypeCommon {
+func genCommon(name, field string, size uint64, dir sys.Dir, opt bool) sys.TypeCommon {
 	return sys.TypeCommon{
 		TypeName:   name,
+		TypeSize:   size,
 		FldName:    field,
 		ArgDir:     dir,
 		IsOptional: opt,
 	}
 }
 
-func genIntCommon(com sys.TypeCommon, size, bitLen uint64, bigEndian bool) sys.IntTypeCommon {
+func genIntCommon(com sys.TypeCommon, bitLen uint64, bigEndian bool) sys.IntTypeCommon {
 	return sys.IntTypeCommon{
 		TypeCommon:  com,
 		BigEndian:   bigEndian,
-		TypeSize:    size,
 		BitfieldLen: bitLen,
 	}
 }
diff --git a/pkg/compiler/testdata/errors2.txt b/pkg/compiler/testdata/errors2.txt
index 1687a1f9b..30dd728a2 100644
--- a/pkg/compiler/testdata/errors2.txt
+++ b/pkg/compiler/testdata/errors2.txt
@@ -85,3 +85,49 @@ s300 {
 s301 {
 	f2	r105
 }
+
+# Varlen field tests.
+
+s400 {
+	f1	int32
+	f2	array[int8]
+}
+
+s401 {
+	f1	array[int8]
+	f2	array[int8]
+} [packed]
+
+s402 {
+	f1	array[int8]	### variable size field f1 in the middle of non-packed struct s402
+	f2	int32
+}
+
+u400 [
+	f1	array[int8]
+	f2	array[int16]
+] [varlen]
+
+u401 [
+	f1	filename	### variable size field f1 in non-varlen union u401
+	f2	text[x86_64]	### variable size field f2 in non-varlen union u401
+	f3	string		### variable size field f3 in non-varlen union u401
+	f4	string["foo", 10]
+	f5	string[sf400]
+	f6	string[sf401]	### variable size field f6 in non-varlen union u401
+	f7	s401		### variable size field f7 in non-varlen union u401
+]
+
+u402 [
+	f1	int32
+	f2	int32
+] [varlen]
+
+s403 {
+	f1	u400		### variable size field f1 in the middle of non-packed struct s403
+	f2	u402		### variable size field f2 in the middle of non-packed struct s403
+	f3	int32
+}
+
+sf400 = "foo", "bar", "baz"
+sf401 = "a", "b", "cd"
diff --git a/pkg/compiler/types.go b/pkg/compiler/types.go
index 9f8ec7754..92e365b3a 100644
--- a/pkg/compiler/types.go
+++ b/pkg/compiler/types.go
@@ -24,6 +24,8 @@ type typeDesc struct {
 	Args         []namedArg // type arguments
 	// Check does custom verification of the type (optional).
 	Check func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon)
+	// Varlen returns if the type is variable-length (false if not set).
+	Varlen func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) bool
 	// Gen generates corresponding sys.Type.
 	Gen func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type
 }
@@ -65,8 +67,9 @@ var typeInt = &typeDesc{
 		if len(args) > 0 {
 			kind, rangeBegin, rangeEnd = sys.IntRange, args[0].Value, args[0].Value2
 		}
+		base.TypeSize = size
 		return &sys.IntType{
-			IntTypeCommon: genIntCommon(base.TypeCommon, size, t.Value2, be),
+			IntTypeCommon: genIntCommon(base.TypeCommon, t.Value2, be),
 			Kind:          kind,
 			RangeBegin:    rangeBegin,
 			RangeEnd:      rangeEnd,
@@ -80,13 +83,12 @@ var typePtr = &typeDesc{
 	Args:     []namedArg{{"direction", typeArgDir}, {"type", typeArgType}},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
 		base.ArgDir = sys.DirIn // pointers are always in
-		size := comp.ptrSize
+		base.TypeSize = comp.ptrSize
 		if t.Ident == "ptr64" {
-			size = 8
+			base.TypeSize = 8
 		}
 		return &sys.PtrType{
 			TypeCommon: base.TypeCommon,
-			TypeSize:   size,
 			Type:       comp.genType(args[1], "", genDir(args[0]), false),
 		}
 	},
@@ -103,17 +105,30 @@ var typeArray = &typeDesc{
 			comp.error(args[1].Pos, "arrays of size 0 are not supported")
 		}
 	},
+	Varlen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) bool {
+		if comp.isVarlen(args[0]) {
+			return true
+		}
+		if len(args) > 1 {
+			return args[1].Value != args[1].Value2
+		}
+		return true
+	},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
 		elemType := comp.genType(args[0], "", base.ArgDir, false)
 		kind, begin, end := sys.ArrayRandLen, uint64(0), uint64(0)
 		if len(args) > 1 {
 			kind, begin, end = sys.ArrayRangeLen, args[1].Value, args[1].Value2
 		}
-		if it, ok := elemType.(*sys.IntType); ok && it.TypeSize == 1 {
+		if it, ok := elemType.(*sys.IntType); ok && it.Kind == sys.IntPlain && it.TypeSize == 1 {
 			// Special case: buffer is better mutated.
 			bufKind := sys.BufferBlobRand
+			base.TypeSize = 0
 			if kind == sys.ArrayRangeLen {
 				bufKind = sys.BufferBlobRange
+				if begin == end {
+					base.TypeSize = begin * elemType.Size()
+				}
 			}
 			return &sys.BufferType{
 				TypeCommon: base.TypeCommon,
@@ -122,6 +137,7 @@ var typeArray = &typeDesc{
 				RangeEnd:   end,
 			}
 		}
+		// TypeSize is assigned later in genStructDescs.
 		return &sys.ArrayType{
 			TypeCommon: base.TypeCommon,
 			Type:       elemType,
@@ -187,6 +203,7 @@ var typeFlags = &typeDesc{
 			// We can get this if all values are unsupported consts.
 			return &sys.IntType{
 				IntTypeCommon: base,
+				Kind:          sys.IntPlain,
 			}
 		}
 		return &sys.FlagsType{
@@ -209,7 +226,11 @@ var typeArgFlags = &typeArg{
 var typeFilename = &typeDesc{
 	Names:     []string{"filename"},
 	CantBeOpt: true,
+	Varlen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) bool {
+		return true
+	},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
+		base.TypeSize = 0
 		return &sys.BufferType{
 			TypeCommon: base.TypeCommon,
 			Kind:       sys.BufferFilename,
@@ -240,9 +261,9 @@ var typeVMA = &typeDesc{
 		if len(args) > 0 {
 			begin, end = args[0].Value, args[0].Value2
 		}
+		base.TypeSize = comp.ptrSize
 		return &sys.VmaType{
 			TypeCommon: base.TypeCommon,
-			TypeSize:   comp.ptrSize,
 			RangeBegin: begin,
 			RangeEnd:   end,
 		}
@@ -344,7 +365,11 @@ var typeText = &typeDesc{
 	Names:     []string{"text"},
 	CantBeOpt: true,
 	Args:      []namedArg{{"kind", typeArgTextType}},
+	Varlen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) bool {
+		return true
+	},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
+		base.TypeSize = 0
 		return &sys.BufferType{
 			TypeCommon: base.TypeCommon,
 			Kind:       sys.BufferText,
@@ -380,11 +405,12 @@ var typeBuffer = &typeDesc{
 	CanBeArg: true,
 	Args:     []namedArg{{"direction", typeArgDir}},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
+		base.TypeSize = comp.ptrSize
 		return &sys.PtrType{
 			TypeCommon: base.TypeCommon,
-			TypeSize:   comp.ptrSize,
 			Type: &sys.BufferType{
-				TypeCommon: genCommon("", "", genDir(args[0]), false),
+				// BufferBlobRand is always varlen.
+				TypeCommon: genCommon("", "", 0, genDir(args[0]), false),
 				Kind:       sys.BufferBlobRand,
 			},
 		}
@@ -411,6 +437,9 @@ var typeString = &typeDesc{
 			}
 		}
 	},
+	Varlen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) bool {
+		return comp.stringSize(args) == 0
+	},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
 		subkind := ""
 		var vals []string
@@ -433,23 +462,41 @@ var typeString = &typeDesc{
 			}
 			vals[i] = s
 		}
-		for _, s := range vals {
-			if size != 0 && size != uint64(len(s)) {
-				size = 0
-				break
-			}
-			size = uint64(len(s))
-		}
+		base.TypeSize = comp.stringSize(args)
 		return &sys.BufferType{
 			TypeCommon: base.TypeCommon,
 			Kind:       sys.BufferString,
 			SubKind:    subkind,
 			Values:     vals,
-			Length:     size,
 		}
 	},
 }
 
+// stringSize returns static string size, or 0 if it is variable length.
+func (comp *compiler) stringSize(args []*ast.Type) uint64 {
+	switch len(args) {
+	case 0:
+		return 0 // a random string
+	case 1:
+		if args[0].String != "" {
+			return uint64(len(args[0].String)) + 1 // string constant
+		}
+		var size uint64
+		for _, s := range comp.strFlags[args[0].Ident].Values {
+			s1 := uint64(len(s.Value)) + 1
+			if size != 0 && size != s1 {
+				return 0 // strings of different lengths
+			}
+			size = s1
+		}
+		return size // all strings have the same length
+	case 2:
+		return args[1].Value // have explicit length
+	default:
+		panic("too many string args")
+	}
+}
+
 var typeArgStringFlags = &typeArg{
 	Check: func(comp *compiler, t *ast.Type) {
 		if t.String == "" && t.Ident == "" {
@@ -474,33 +521,58 @@ var typeResource = &typeDesc{
 	// No Names, but compiler knows how to match it.
 	CanBeArg:     true,
 	ResourceBase: true,
-	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
+	// Gen is assigned below to avoid initialization loop.
+}
+
+func init() {
+	typeResource.Gen = func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
+		// Find and generate base type to get its size.
+		var baseType *ast.Type
+		for r := comp.resources[t.Ident]; r != nil; {
+			baseType = r.Base
+			r = comp.resources[r.Base.Ident]
+		}
+		base.TypeSize = comp.genType(baseType, "", sys.DirIn, false).Size()
 		return &sys.ResourceType{
 			TypeCommon: base.TypeCommon,
 		}
-	},
+	}
 }
 
 var typeStruct = &typeDesc{
 	// No Names, but compiler knows how to match it.
 	CantBeOpt: true,
-	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
-		s := comp.structs[base.TypeName]
-		comp.structUses[sys.StructKey{base.TypeName, base.ArgDir}] = s
+	// Varlen/Gen are assigned below due to initialization cycle.
+}
+
+func init() {
+	typeStruct.Varlen = func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) bool {
+		return comp.isStructVarlen(t.Ident)
+	}
+	typeStruct.Gen = func(comp *compiler, t *ast.Type, args []*ast.Type, base sys.IntTypeCommon) sys.Type {
+		s := comp.structs[t.Ident]
+		key := sys.StructKey{t.Ident, base.ArgDir}
+		desc := comp.structDescs[key]
+		if desc == nil {
+			// Need to assign to structDescs before calling genStructDesc to break recursion.
+			desc = new(sys.StructDesc)
+			comp.structDescs[key] = desc
+			comp.genStructDesc(desc, s, base.ArgDir)
+		}
 		if s.IsUnion {
 			return &sys.UnionType{
-				TypeCommon: base.TypeCommon,
-				IsVarlen:   comp.parseUnionAttrs(s),
+				Key:        key,
+				FldName:    base.FldName,
+				StructDesc: desc,
 			}
 		} else {
-			packed, align := comp.parseStructAttrs(s)
 			return &sys.StructType{
-				TypeCommon: base.TypeCommon,
-				IsPacked:   packed,
-				AlignAttr:  align,
+				Key:        key,
+				FldName:    base.FldName,
+				StructDesc: desc,
 			}
 		}
-	},
+	}
 }
 
 var typeArgDir = &typeArg{