prog: remove Dir from Type

Having Dir is Type is handy, but forces us to duplicate lots of types.
E.g. if a struct is referenced as both in and out, then we need to
have 2 copies and 2 copies of structs/types it includes.
If also prevents us from having the struct type as struct identity
(because we can have up to 3 of them).

Revert to the old way we used to do it: propagate Dir as we walk
syscall arguments. This moves lots of dir passing from pkg/compiler
to prog package.
Now Arg contains the dir, so once we build the tree, we can use dirs
as before.

Reduces size of sys/linux/gen/amd64.go from 6058336 to 5661150 (-6.6%).

Update #1580

4 files changed, 33 insertions, 39 deletions

diff --git a/pkg/compiler/check.go b/pkg/compiler/check.go
index e22f217fd..2c0438086 100644
--- a/pkg/compiler/check.go
+++ b/pkg/compiler/check.go
@@ -323,7 +323,7 @@ func (comp *compiler) checkLenType(t0, t *ast.Type, parents []parentDesc,
 		warned[parentName] = true
 		return
 	}
-	_, args, _ := comp.getArgsBase(t, "", prog.DirIn, isArg)
+	_, args, _ := comp.getArgsBase(t, "", isArg)
 	for i, arg := range args {
 		argDesc := desc.Args[i]
 		if argDesc.Type == typeArgLenTarget {
@@ -522,7 +522,7 @@ func (comp *compiler) collectUsedType(structs, flags, strflags map[string]bool,
 		}
 		return
 	}
-	_, args, _ := comp.getArgsBase(t, "", prog.DirIn, isArg)
+	_, args, _ := comp.getArgsBase(t, "", isArg)
 	for i, arg := range args {
 		if desc.Args[i].Type == typeArgType {
 			comp.collectUsedType(structs, flags, strflags, arg, desc.Args[i].IsArg)
@@ -603,7 +603,7 @@ func (comp *compiler) checkTypeCtors(t *ast.Type, dir prog.Dir, isArg bool,
 	if desc == typePtr {
 		dir = genDir(t.Args[0])
 	}
-	_, args, _ := comp.getArgsBase(t, "", dir, isArg)
+	_, args, _ := comp.getArgsBase(t, "", isArg)
 	for i, arg := range args {
 		if desc.Args[i].Type == typeArgType {
 			comp.checkTypeCtors(arg, dir, desc.Args[i].IsArg, ctors, checked)
@@ -684,7 +684,7 @@ func (comp *compiler) recurseField(checked map[string]bool, t *ast.Type, path []
 		comp.checkStructRecursion(checked, comp.structs[t.Ident], path)
 		return
 	}
-	_, args, base := comp.getArgsBase(t, "", prog.DirIn, false)
+	_, args, base := comp.getArgsBase(t, "", false)
 	if desc == typePtr && base.IsOptional {
 		return // optional pointers prune recursion
 	}
@@ -774,7 +774,7 @@ func (comp *compiler) checkType(ctx checkCtx, t *ast.Type, flags checkFlags) {
 		return
 	}
 	if desc.Check != nil {
-		_, args, base := comp.getArgsBase(t, "", prog.DirIn, flags&checkIsArg != 0)
+		_, args, base := comp.getArgsBase(t, "", flags&checkIsArg != 0)
 		desc.Check(comp, t, args, base)
 	}
 }
@@ -1098,12 +1098,12 @@ func (comp *compiler) checkVarlens() {
 }
 
 func (comp *compiler) isVarlen(t *ast.Type) bool {
-	desc, args, _ := comp.getArgsBase(t, "", prog.DirIn, false)
+	desc, args, _ := comp.getArgsBase(t, "", false)
 	return desc.Varlen != nil && desc.Varlen(comp, t, args)
 }
 
 func (comp *compiler) isZeroSize(t *ast.Type) bool {
-	desc, args, _ := comp.getArgsBase(t, "", prog.DirIn, false)
+	desc, args, _ := comp.getArgsBase(t, "", false)
 	return desc.ZeroSize != nil && desc.ZeroSize(comp, t, args)
 }
 
diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go
index 7316b4d4c..8e5afddde 100644
--- a/pkg/compiler/compiler.go
+++ b/pkg/compiler/compiler.go
@@ -249,14 +249,13 @@ func (comp *compiler) getTypeDesc(t *ast.Type) *typeDesc {
 	return nil
 }
 
-func (comp *compiler) getArgsBase(t *ast.Type, field string, dir prog.Dir, isArg bool) (
-	*typeDesc, []*ast.Type, prog.IntTypeCommon) {
+func (comp *compiler) getArgsBase(t *ast.Type, field string, isArg bool) (*typeDesc, []*ast.Type, prog.IntTypeCommon) {
 	desc := comp.getTypeDesc(t)
 	if desc == nil {
 		panic(fmt.Sprintf("no type desc for %#v", *t))
 	}
 	args, opt := removeOpt(t)
-	com := genCommon(t.Ident, field, sizeUnassigned, dir, opt != nil)
+	com := genCommon(t.Ident, field, sizeUnassigned, opt != nil)
 	base := genIntCommon(com, 0, false)
 	if desc.NeedBase {
 		base.TypeSize = comp.ptrSize
@@ -306,7 +305,7 @@ func (comp *compiler) foreachType(n0 ast.Node,
 
 func (comp *compiler) foreachSubType(t *ast.Type, isArg bool,
 	cb func(*ast.Type, *typeDesc, []*ast.Type, prog.IntTypeCommon)) {
-	desc, args, base := comp.getArgsBase(t, "", prog.DirIn, isArg)
+	desc, args, base := comp.getArgsBase(t, "", isArg)
 	cb(t, desc, args, base)
 	for i, arg := range args {
 		if desc.Args[i].Type == typeArgType {
diff --git a/pkg/compiler/gen.go b/pkg/compiler/gen.go
index c8255c554..61dd85eae 100644
--- a/pkg/compiler/gen.go
+++ b/pkg/compiler/gen.go
@@ -65,8 +65,8 @@ func (comp *compiler) collectCallArgSizes() map[string][]uint64 {
 			if len(argSizes) <= i {
 				argSizes = append(argSizes, comp.ptrSize)
 			}
-			desc, _, _ := comp.getArgsBase(arg.Type, arg.Name.Name, prog.DirIn, true)
-			typ := comp.genField(arg, prog.DirIn, comp.ptrSize)
+			desc, _, _ := comp.getArgsBase(arg.Type, arg.Name.Name, true)
+			typ := comp.genField(arg, comp.ptrSize)
 			// Ignore all types with base (const, flags). We don't have base in syscall args.
 			// Also ignore resources and pointers because fd can be 32-bits and pointer 64-bits,
 			// and then there is no way to fix this.
@@ -112,7 +112,7 @@ func (comp *compiler) genSyscalls() []*prog.Syscall {
 func (comp *compiler) genSyscall(n *ast.Call, argSizes []uint64) *prog.Syscall {
 	var ret prog.Type
 	if n.Ret != nil {
-		ret = comp.genType(n.Ret, "ret", prog.DirOut, comp.ptrSize)
+		ret = comp.genType(n.Ret, "ret", comp.ptrSize)
 	}
 	var attrs prog.SyscallAttrs
 	descAttrs := comp.parseAttrs(callAttrs, n, n.Attrs)
@@ -129,7 +129,7 @@ func (comp *compiler) genSyscall(n *ast.Call, argSizes []uint64) *prog.Syscall {
 		CallName:    n.CallName,
 		NR:          n.NR,
 		MissingArgs: len(argSizes) - len(n.Args),
-		Args:        comp.genFieldArray(n.Args, prog.DirIn, argSizes),
+		Args:        comp.genFieldArray(n.Args, argSizes),
 		Ret:         ret,
 		Attrs:       attrs,
 	}
@@ -237,10 +237,7 @@ func (comp *compiler) genStructDescs(syscalls []*prog.Syscall) []*prog.KeyedStru
 
 	sort.Slice(ctx.structs, func(i, j int) bool {
 		si, sj := ctx.structs[i].Key, ctx.structs[j].Key
-		if si.Name != sj.Name {
-			return si.Name < sj.Name
-		}
-		return si.Dir < sj.Dir
+		return si.Name < sj.Name
 	})
 	return ctx.structs
 }
@@ -371,14 +368,14 @@ func (ctx *structGen) walkUnion(t *prog.UnionType) {
 	}
 }
 
-func (comp *compiler) genStructDesc(res *prog.StructDesc, n *ast.Struct, dir prog.Dir, varlen bool) {
+func (comp *compiler) genStructDesc(res *prog.StructDesc, n *ast.Struct, varlen bool) {
 	// Leave node for genStructDescs to calculate size/padding.
 	comp.structNodes[res] = n
-	common := genCommon(n.Name.Name, "", sizeUnassigned, dir, false)
+	common := genCommon(n.Name.Name, "", sizeUnassigned, false)
 	common.IsVarlen = varlen
 	*res = prog.StructDesc{
 		TypeCommon: common,
-		Fields:     comp.genFieldArray(n.Fields, dir, make([]uint64, len(n.Fields))),
+		Fields:     comp.genFieldArray(n.Fields, make([]uint64, len(n.Fields))),
 	}
 }
 
@@ -570,25 +567,25 @@ func (comp *compiler) typeAlign(t0 prog.Type) uint64 {
 
 func genPad(size uint64) prog.Type {
 	return &prog.ConstType{
-		IntTypeCommon: genIntCommon(genCommon("pad", "", size, prog.DirIn, false), 0, false),
+		IntTypeCommon: genIntCommon(genCommon("pad", "", size, false), 0, false),
 		IsPad:         true,
 	}
 }
 
-func (comp *compiler) genFieldArray(fields []*ast.Field, dir prog.Dir, argSizes []uint64) []prog.Type {
+func (comp *compiler) genFieldArray(fields []*ast.Field, argSizes []uint64) []prog.Type {
 	var res []prog.Type
 	for i, f := range fields {
-		res = append(res, comp.genField(f, dir, argSizes[i]))
+		res = append(res, comp.genField(f, argSizes[i]))
 	}
 	return res
 }
 
-func (comp *compiler) genField(f *ast.Field, dir prog.Dir, argSize uint64) prog.Type {
-	return comp.genType(f.Type, f.Name.Name, dir, argSize)
+func (comp *compiler) genField(f *ast.Field, argSize uint64) prog.Type {
+	return comp.genType(f.Type, f.Name.Name, argSize)
 }
 
-func (comp *compiler) genType(t *ast.Type, field string, dir prog.Dir, argSize uint64) prog.Type {
-	desc, args, base := comp.getArgsBase(t, field, dir, argSize != 0)
+func (comp *compiler) genType(t *ast.Type, field string, argSize uint64) prog.Type {
+	desc, args, base := comp.getArgsBase(t, field, argSize != 0)
 	if desc.Gen == nil {
 		panic(fmt.Sprintf("no gen for %v %#v", field, t))
 	}
@@ -605,12 +602,11 @@ func (comp *compiler) genType(t *ast.Type, field string, dir prog.Dir, argSize u
 	return desc.Gen(comp, t, args, base)
 }
 
-func genCommon(name, field string, size uint64, dir prog.Dir, opt bool) prog.TypeCommon {
+func genCommon(name, field string, size uint64, opt bool) prog.TypeCommon {
 	return prog.TypeCommon{
 		TypeName:   name,
 		TypeSize:   size,
 		FldName:    field,
-		ArgDir:     dir,
 		IsOptional: opt,
 	}
 }
diff --git a/pkg/compiler/types.go b/pkg/compiler/types.go
index fce1153de..6f2c26a6e 100644
--- a/pkg/compiler/types.go
+++ b/pkg/compiler/types.go
@@ -156,14 +156,14 @@ var typePtr = &typeDesc{
 	CanBeTypedef: true,
 	Args:         []namedArg{{Name: "direction", Type: typeArgDir}, {Name: "type", Type: typeArgType}},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base prog.IntTypeCommon) prog.Type {
-		base.ArgDir = prog.DirIn // pointers are always in
 		base.TypeSize = comp.ptrSize
 		if t.Ident == "ptr64" {
 			base.TypeSize = 8
 		}
 		return &prog.PtrType{
 			TypeCommon: base.TypeCommon,
-			Type:       comp.genType(args[1], "", genDir(args[0]), 0),
+			Type:       comp.genType(args[1], "", 0),
+			ElemDir:    genDir(args[0]),
 		}
 	},
 }
@@ -212,7 +212,7 @@ var typeArray = &typeDesc{
 		return comp.isZeroSize(args[0])
 	},
 	Gen: func(comp *compiler, t *ast.Type, args []*ast.Type, base prog.IntTypeCommon) prog.Type {
-		elemType := comp.genType(args[0], "", base.ArgDir, 0)
+		elemType := comp.genType(args[0], "", 0)
 		kind, begin, end := prog.ArrayRandLen, uint64(0), uint64(0)
 		if len(args) > 1 {
 			kind, begin, end = prog.ArrayRangeLen, args[1].Value, args[1].Value
@@ -696,7 +696,7 @@ var typeFmt = &typeDesc{
 		{Name: "value", Type: typeArgType, IsArg: true},
 	},
 	Check: func(comp *compiler, t *ast.Type, args []*ast.Type, base prog.IntTypeCommon) {
-		desc, _, _ := comp.getArgsBase(args[1], "", base.TypeCommon.ArgDir, true)
+		desc, _, _ := comp.getArgsBase(args[1], "", true)
 		switch desc {
 		case typeResource, typeInt, typeLen, typeFlags, typeProc:
 		default:
@@ -718,7 +718,7 @@ var typeFmt = &typeDesc{
 			format = prog.FormatStrOct
 			size = 23
 		}
-		typ := comp.genType(args[1], "", base.TypeCommon.ArgDir, comp.ptrSize)
+		typ := comp.genType(args[1], "", comp.ptrSize)
 		switch t := typ.(type) {
 		case *prog.ResourceType:
 			t.ArgFormat = format
@@ -767,7 +767,7 @@ func init() {
 			baseType = r.Base
 			r = comp.resources[r.Base.Ident]
 		}
-		baseProgType := comp.genType(baseType, "", prog.DirIn, 0)
+		baseProgType := comp.genType(baseType, "", 0)
 		base.TypeSize = baseProgType.Size()
 		return &prog.ResourceType{
 			TypeCommon: base.TypeCommon,
@@ -818,14 +818,13 @@ func init() {
 		s := comp.structs[t.Ident]
 		key := prog.StructKey{
 			Name: t.Ident,
-			Dir:  base.ArgDir,
 		}
 		desc := comp.structDescs[key]
 		if desc == nil {
 			// Need to assign to structDescs before calling genStructDesc to break recursion.
 			desc = new(prog.StructDesc)
 			comp.structDescs[key] = desc
-			comp.genStructDesc(desc, s, base.ArgDir, typeStruct.Varlen(comp, t, args))
+			comp.genStructDesc(desc, s, typeStruct.Varlen(comp, t, args))
 		}
 		if s.IsUnion {
 			return &prog.UnionType{