pkg/compiler: support complex len targets

This change adds compiler support for complex path
expressions in len targets. E.g. it allows to refer
to a sibling field as len[parent_struct:field:another_field].
See the docs change for details.

This is just a compiler change.
The feature is not yet supported by the prog package.

8 files changed, 178 insertions, 67 deletions

diff --git a/pkg/ast/parser.go b/pkg/ast/parser.go
index b68a1a8a2..b8d22fd88 100644
--- a/pkg/ast/parser.go
+++ b/pkg/ast/parser.go
@@ -424,20 +424,22 @@ func (p *parser) parseType() *Type {
 		p.expect(tokInt, tokIdent, tokString)
 	}
 	p.next()
-	if allowColon && p.tryConsume(tokColon) {
-		col := &Type{
-			Pos: p.pos,
-		}
-		switch p.tok {
-		case tokInt:
-			col.Value, col.ValueFmt = p.parseIntValue()
-		case tokIdent:
-			col.Ident = p.lit
-		default:
-			p.expect(tokInt, tokIdent)
+	if allowColon {
+		for p.tryConsume(tokColon) {
+			col := &Type{
+				Pos: p.pos,
+			}
+			switch p.tok {
+			case tokInt:
+				col.Value, col.ValueFmt = p.parseIntValue()
+			case tokIdent:
+				col.Ident = p.lit
+			default:
+				p.expect(tokInt, tokIdent)
+			}
+			arg.Colon = append(arg.Colon, col)
+			p.next()
 		}
-		arg.Colon = append(arg.Colon, col)
-		p.next()
 	}
 	arg.Args = p.parseTypeList()
 	return arg
diff --git a/pkg/ast/testdata/all.txt b/pkg/ast/testdata/all.txt
index e5a1839dd..392796254 100644
--- a/pkg/ast/testdata/all.txt
+++ b/pkg/ast/testdata/all.txt
@@ -22,11 +22,12 @@ str_flags4 = "string", 42		### unexpected int, expecting string
 
 call(foo ,int32 , bar int32)		### unexpected ',', expecting int, identifier, string
 call(foo int32:"bar")			### unexpected string, expecting int, identifier
+call(a int32, b len[a:"bar"])		### unexpected string, expecting int, identifier
 
 define FOO `bar`
 define FOO `bar				### C expression is not terminated
 
-foo(x int32[1:2:3, opt])		### unexpected ':', expecting ']'
+foo(x int32[1:2:3, opt])
 foo2(x int32[1[2]:2])			### unexpected ':', expecting ']'
 
 s0 {
diff --git a/pkg/compiler/check.go b/pkg/compiler/check.go
index c5337a63b..ead5f811f 100644
--- a/pkg/compiler/check.go
+++ b/pkg/compiler/check.go
@@ -291,14 +291,29 @@ func (comp *compiler) checkLenTargets() {
 		case *ast.Call:
 			for _, arg := range n.Args {
 				checked := make(map[string]bool)
-				comp.checkLenType(arg.Type, arg.Name.Name, n.Args, nil, checked, warned, true)
+				parents := []parentDesc{{fields: n.Args}}
+				comp.checkLenType(arg.Type, arg.Type, arg.Name.Name, parents, checked, warned, true)
 			}
 		}
 	}
 }
 
-func (comp *compiler) checkLenType(t *ast.Type, name string, fields []*ast.Field,
-	parents []string, checked, warned map[string]bool, isArg bool) {
+type parentDesc struct {
+	name   string
+	fields []*ast.Field
+}
+
+func parentTargetName(s *ast.Struct) string {
+	parentName := s.Name.Name
+	if pos := strings.IndexByte(parentName, '['); pos != -1 {
+		// For template parents name is "struct_name[ARG1, ARG2]", strip the part after '['.
+		parentName = parentName[:pos]
+	}
+	return parentName
+}
+
+func (comp *compiler) checkLenType(t0, t *ast.Type, name string, parents []parentDesc,
+	checked, warned map[string]bool, isArg bool) {
 	desc := comp.getTypeDesc(t)
 	if desc == typeStruct {
 		s := comp.structs[t.Ident]
@@ -307,17 +322,14 @@ func (comp *compiler) checkLenType(t *ast.Type, name string, fields []*ast.Field
 			return
 		}
 		checked[s.Name.Name] = true
-		parentName := s.Name.Name
-		if pos := strings.IndexByte(parentName, '['); pos != -1 {
-			// For template parents name is "struct_name[ARG1, ARG2]", strip the part after '['.
-			parentName = parentName[:pos]
-		}
-		parents = append(parents, parentName)
-		if !s.IsUnion {
-			fields = s.Fields
+		fields := s.Fields
+		if s.IsUnion {
+			fields = nil
 		}
+		parentName := parentTargetName(s)
+		parents = append(parents, parentDesc{name: parentName, fields: fields})
 		for _, fld := range s.Fields {
-			comp.checkLenType(fld.Type, fld.Name.Name, fields, parents, checked, warned, false)
+			comp.checkLenType(fld.Type, fld.Type, fld.Name.Name, parents, checked, warned, false)
 		}
 		warned[parentName] = true
 		return
@@ -326,59 +338,80 @@ func (comp *compiler) checkLenType(t *ast.Type, name string, fields []*ast.Field
 	for i, arg := range args {
 		argDesc := desc.Args[i]
 		if argDesc.Type == typeArgLenTarget {
-			comp.checkLenTarget(t, name, arg.Ident, fields, parents, warned)
+			targets := append([]*ast.Type{arg}, arg.Colon...)
+			if len(targets) != 1 {
+				for _, target := range targets {
+					if target.Ident == "parent" {
+						comp.error(target.Pos, "parent can't be part of path expressions")
+						return
+					}
+				}
+			}
+			comp.checkLenTarget(t0, t, name, targets, parents, warned)
 		} else if argDesc.Type == typeArgType {
-			comp.checkLenType(arg, name, fields, parents, checked, warned, argDesc.IsArg)
+			comp.checkLenType(t0, arg, name, parents, checked, warned, argDesc.IsArg)
 		}
 	}
 }
 
-func (comp *compiler) checkLenTarget(t *ast.Type, name, target string, fields []*ast.Field,
-	parents []string, warned map[string]bool) {
-	if target == name {
-		comp.error(t.Pos, "%v target %v refer to itself", t.Ident, target)
-		return
-	}
-	if target == "parent" {
-		if len(parents) == 0 {
-			comp.error(t.Pos, "%v target %v does not exist", t.Ident, target)
-		}
+func (comp *compiler) checkLenTarget(t0, t *ast.Type, name string, targets []*ast.Type,
+	parents []parentDesc, warned map[string]bool) {
+	if len(targets) == 0 {
 		return
 	}
+	target := targets[0]
+	targets = targets[1:]
+	fields := parents[len(parents)-1].fields
 	for _, fld := range fields {
-		if target != fld.Name.Name {
+		if target.Ident != fld.Name.Name {
 			continue
 		}
-		if fld.Type == t {
-			comp.error(t.Pos, "%v target %v refer to itself", t.Ident, target)
+		if fld.Type == t0 {
+			comp.error(target.Pos, "%v target %v refers to itself", t.Ident, target.Ident)
+			return
 		}
-		if t.Ident == "len" {
-			inner := fld.Type
-			desc, args, _ := comp.getArgsBase(inner, "", prog.DirIn, false)
-			for desc == typePtr {
-				if desc != typePtr {
-					break
-				}
-				inner = args[1]
-				desc, args, _ = comp.getArgsBase(inner, "", prog.DirIn, false)
-			}
-			if desc == typeArray && comp.isVarlen(args[0]) {
-				// We can reach the same struct multiple times starting from different
-				// syscall arguments. Warn only once.
-				if len(parents) == 0 || !warned[parents[len(parents)-1]] {
-					comp.warning(t.Pos, "len target %v refer to an array with"+
-						" variable-size elements (do you mean bytesize?)", target)
+		if len(targets) == 0 {
+			if t.Ident == "len" {
+				typ, desc := comp.derefPointers(fld.Type)
+				if desc == typeArray && comp.isVarlen(typ.Args[0]) {
+					// We can reach the same struct multiple times starting from different
+					// syscall arguments. Warn only once.
+					if !warned[parents[len(parents)-1].name] {
+						comp.warning(target.Pos, "len target %v refer to an array with"+
+							" variable-size elements (do you mean bytesize?)",
+							target.Ident)
+					}
 				}
 			}
+			return
+		}
+		typ, desc := comp.derefPointers(fld.Type)
+		if desc != typeStruct {
+			comp.error(target.Pos, "%v path %v does not refer to a struct", t.Ident, target.Ident)
+			return
+		}
+		s := comp.structs[typ.Ident]
+		if s.IsUnion {
+			comp.error(target.Pos, "%v path %v does not refer to a struct", t.Ident, target.Ident)
+			return
 		}
+		parents = append(parents, parentDesc{name: parentTargetName(s), fields: s.Fields})
+		comp.checkLenTarget(t0, t, name, targets, parents, warned)
 		return
 	}
-	for _, parent := range parents {
-		if target == parent {
-			return
+	for pi := len(parents) - 1; pi >= 0; pi-- {
+		parent := parents[pi]
+		if parent.name == "" || parent.name != target.Ident && target.Ident != "parent" {
+			continue
 		}
+		if len(targets) != 0 {
+			parents1 := make([]parentDesc, pi+1)
+			copy(parents1, parents[:pi+1])
+			comp.checkLenTarget(t0, t, name, targets, parents1, warned)
+		}
+		return
 	}
-	comp.error(t.Pos, "%v target %v does not exist", t.Ident, target)
+	comp.error(target.Pos, "%v target %v does not exist", t.Ident, target.Ident)
 }
 
 func CollectUnused(desc *ast.Description, target *targets.Target, eh ast.ErrorHandler) ([]ast.Node, error) {
@@ -948,6 +981,12 @@ func (comp *compiler) checkTypeArg(t, arg *ast.Type, argDesc namedArg) {
 			comp.error(col.Pos, "unexpected ':'")
 			return
 		}
+		if desc.Kind == kindIdent {
+			if unexpected, expect, ok := checkTypeKind(col, kindIdent); !ok {
+				comp.error(arg.Pos, "unexpected %v after colon, expect %v", unexpected, expect)
+				return
+			}
+		}
 	}
 	if len(arg.Args) > desc.MaxArgs {
 		comp.error(arg.Pos, "%v argument has subargs", argDesc.Name)
diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go
index 508dc0f26..a88e44d70 100644
--- a/pkg/compiler/compiler.go
+++ b/pkg/compiler/compiler.go
@@ -285,6 +285,16 @@ func (comp *compiler) getArgsBase(t *ast.Type, field string, dir prog.Dir, isArg
 	return desc, args, base
 }
 
+func (comp *compiler) derefPointers(t *ast.Type) (*ast.Type, *typeDesc) {
+	for {
+		desc := comp.getTypeDesc(t)
+		if desc != typePtr {
+			return t, desc
+		}
+		t = t.Args[1]
+	}
+}
+
 func (comp *compiler) foreachType(n0 ast.Node,
 	cb func(*ast.Type, *typeDesc, []*ast.Type, prog.IntTypeCommon)) {
 	switch n := n0.(type) {
diff --git a/pkg/compiler/testdata/all.txt b/pkg/compiler/testdata/all.txt
index 49cc1ffb9..7acb47743 100644
--- a/pkg/compiler/testdata/all.txt
+++ b/pkg/compiler/testdata/all.txt
@@ -70,6 +70,33 @@ foo$len_templ(a ptr[in, len_templ1[int8, int16]])
 foo$len_var0(a ptr[in, array[string]], b len[a])
 foo$len_var1(a ptr[in, array[string]], b ptr[in, len[a, int32]])
 
+len_expr1 {
+	f11	len_expr2
+}
+
+len_expr2 {
+	f21	len_expr3
+	f22	len_expr4
+	f23	ptr[in, len_expr4]
+	f24	ptr[in, ptr[in, len_expr4]]
+	f25	len[f21:f31, int32]
+}
+
+len_expr3 {
+	f31	int32
+	f32	bytesize[len_expr2:f21, int32]
+	f33	bytesize[len_expr2:f22:f41, int32]
+	f34	bytesize[len_expr1:f11:f22:f41, int32]
+	f35	bytesize[len_expr2:f23:f41, int32]
+	f36	bytesize[len_expr2:f24:f41, int32]
+}
+
+len_expr4 {
+	f41	int32
+}
+
+foo$len_expr(a ptr[in, len_expr1])
+
 # Pointer type.
 
 foo$ptr(a ptr[in, int64])
diff --git a/pkg/compiler/testdata/errors.txt b/pkg/compiler/testdata/errors.txt
index e95c3be7d..8c44694bb 100644
--- a/pkg/compiler/testdata/errors.txt
+++ b/pkg/compiler/testdata/errors.txt
@@ -62,7 +62,7 @@ sf2 = "a", "b"
 sf2 = "c"			### string flags sf2 redeclared, previously declared at LOCATION
 
 resource r2[r0]: 2
-resource r3[int32:1]		###  unexpected ':', only struct fields can be bitfields
+resource r3[int32:1]		### unexpected ':', only struct fields can be bitfields
 resource r4[int32[opt]]		### resource base can't be marked as opt
 resource r5[non_existent]	### unknown type non_existent
 resource r6[int64be]		### int64be can't be resource base (int types can)
@@ -119,6 +119,9 @@ foo$61(a u6)			### u6 can't be syscall argument
 foo$62() u6			### u6 can't be syscall return
 foo$63(a int32[1[2]])		### range argument has subargs
 foo$64(a ptr[in, flags[f1[int32], int32]])	### flags argument has subargs
+foo$65(a int32, b len[1])	### unexpected int 1 for len target argument of len type, expect identifier
+foo$66(a int32, b len[a:1])	### unexpected int 1 after colon, expect identifier
+foo$67(x int32[1:2:3, opt])	### unexpected ':'
 
 opt {				### struct uses reserved name opt
 	f1	int32
diff --git a/pkg/compiler/testdata/errors2.txt b/pkg/compiler/testdata/errors2.txt
index 77db29197..6b7737ef5 100644
--- a/pkg/compiler/testdata/errors2.txt
+++ b/pkg/compiler/testdata/errors2.txt
@@ -80,8 +80,8 @@ foo$sr0(a ptr[in, use_sr])
 # Len target tests.
 
 foo$100(a int32, b len[a])
-foo$101(a len[a])		### len target a refer to itself
-foo$102(a ptr[in, len[a, int8]])	### len target a refer to itself
+foo$101(a len[a])		### len target a refers to itself
+foo$102(a ptr[in, len[a, int8]])	### len target a refers to itself
 foo$103(a int32, b len[c])	### len target c does not exist
 foo$104(a len[parent])		### len target parent does not exist
 foo$105(a ptr[in, int32], b ptr[in, array[len[a, int32]]])
@@ -109,6 +109,15 @@ s6 {
 	f2	array[int8]
 } [size[10]]			### varlen struct s6 has size attribute
 
+s7 {
+	f1	int32
+	f2	u0
+}
+
+u0 [
+	f	len[f1, int32]	### len target f1 does not exist
+]
+
 u1 [
 	f1	ptr[in, array[int8]]
 	f2	len[f1, int32]	### len target f1 does not exist
@@ -124,7 +133,26 @@ u3 [
 
 foo$200(a ptr[in, s2])
 foo$201(a ptr[in, s1])
-foo$202(a u1)
+foo$202(a ptr[in, s7])
+foo$203(a u1)
+
+foo$204(a ptr[in, slen1])
+
+slen1 {
+	f0 len[parent, int32]
+	f1 len[parent:f0, int32]	### parent can't be part of path expressions
+	f2 len[slen21:a, int32]		### len target a does not exist
+	f3 len[f0:parent, int32]	### parent can't be part of path expressions
+	f4 len[slen2:f, int32]		### len path slen2 does not refer to a struct
+	f5 len[slen22:f, int32]		### len path slen22 does not refer to a struct
+	slen2 ptr[in, array[slen2]]
+	slen21 slen2
+	slen22 array[slen2]
+}
+
+slen2 {
+	f int32
+}
 
 # Resource ctor tests.
 
diff --git a/pkg/compiler/types.go b/pkg/compiler/types.go
index 17fb1b93a..d2d3380a6 100644
--- a/pkg/compiler/types.go
+++ b/pkg/compiler/types.go
@@ -41,7 +41,7 @@ type typeArg struct {
 	Names    []string
 	Kind     int // int/ident/string
 	MaxArgs  int // maxiumum number of subargs
-	MaxColon int // max number of colons (2:3)
+	MaxColon int // max number of colons (2:3:4)
 	// Check does custom verification of the arg (optional).
 	Check       func(comp *compiler, t *ast.Type)
 	CheckConsts func(comp *compiler, t *ast.Type)
@@ -244,7 +244,8 @@ var typeConst = &typeDesc{
 }
 
 var typeArgLenTarget = &typeArg{
-	Kind: kindIdent,
+	Kind:     kindIdent,
+	MaxColon: 10,
 }
 
 var typeFlags = &typeDesc{