sys/syz-sysgen: switch to new parser

For now we just generate the old structs from the new AST.
But this allows to delete the old parser entirely.

6 files changed, 247 insertions, 451 deletions

diff --git a/pkg/ast/parser.go b/pkg/ast/parser.go
index 10f389424..fc05378b2 100644
--- a/pkg/ast/parser.go
+++ b/pkg/ast/parser.go
@@ -6,6 +6,8 @@ package ast
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
+	"path/filepath"
 	"strconv"
 	"strings"
 )
@@ -41,6 +43,35 @@ func Parse(data []byte, filename string, errorHandler func(pos Pos, msg string))
 	return
 }
 
+func ParseGlob(glob string, errorHandler func(pos Pos, msg string)) (top []interface{}, ok bool) {
+	if errorHandler == nil {
+		errorHandler = loggingHandler
+	}
+	files, err := filepath.Glob(glob)
+	if err != nil {
+		errorHandler(Pos{}, fmt.Sprintf("failed to find input files: %v", err))
+		return nil, false
+	}
+	if len(files) == 0 {
+		errorHandler(Pos{}, fmt.Sprintf("no files matched by glob %q", glob))
+		return nil, false
+	}
+	ok = true
+	for _, f := range files {
+		data, err := ioutil.ReadFile(f)
+		if err != nil {
+			errorHandler(Pos{}, fmt.Sprintf("failed to read input file: %v", err))
+			return nil, false
+		}
+		top1, ok1 := Parse(data, filepath.Base(f), errorHandler)
+		if !ok1 {
+			ok = false
+		}
+		top = append(top, top1...)
+	}
+	return
+}
+
 type parser struct {
 	s *scanner
 
diff --git a/pkg/ast/scanner.go b/pkg/ast/scanner.go
index d3e35895c..372d2df3e 100644
--- a/pkg/ast/scanner.go
+++ b/pkg/ast/scanner.go
@@ -103,9 +103,7 @@ type scanner struct {
 
 func newScanner(data []byte, filename string, errorHandler func(pos Pos, msg string)) *scanner {
 	if errorHandler == nil {
-		errorHandler = func(pos Pos, msg string) {
-			fmt.Fprintf(os.Stderr, "%v:%v:%v: %v\n", pos.File, pos.Line, pos.Col, msg)
-		}
+		errorHandler = loggingHandler
 	}
 	s := &scanner{
 		data:         data,
@@ -117,6 +115,10 @@ func newScanner(data []byte, filename string, errorHandler func(pos Pos, msg str
 	return s
 }
 
+func loggingHandler(pos Pos, msg string) {
+	fmt.Fprintf(os.Stderr, "%v:%v:%v: %v\n", pos.File, pos.Line, pos.Col, msg)
+}
+
 func (s *scanner) Scan() (tok token, lit string, pos Pos) {
 	s.skipWhitespace()
 	pos = s.pos()
diff --git a/sys/sysparser/lexer.go b/sys/sysparser/lexer.go
deleted file mode 100644
index c6a0f802e..000000000
--- a/sys/sysparser/lexer.go
+++ /dev/null
@@ -1,329 +0,0 @@
-// Copyright 2015/2016 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package sysparser
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"sort"
-	"strconv"
-	"strings"
-)
-
-type Description struct {
-	Includes  []string
-	Incdirs   []string
-	Defines   map[string]string
-	Syscalls  []Syscall
-	Structs   map[string]*Struct
-	Unnamed   map[string][]string
-	Flags     map[string][]string
-	StrFlags  map[string][]string
-	Resources map[string]Resource
-}
-
-type Syscall struct {
-	Name     string
-	CallName string
-	Args     [][]string
-	Ret      []string
-}
-
-type Struct struct {
-	Name    string
-	Flds    [][]string
-	IsUnion bool
-	Packed  bool
-	Varlen  bool
-	Align   int
-}
-
-type Resource struct {
-	Name   string
-	Base   string
-	Values []string
-}
-
-func Parse(in io.Reader) *Description {
-	p := newParser(in)
-	var includes []string
-	var incdirs []string
-	defines := make(map[string]string)
-	var syscalls []Syscall
-	structs := make(map[string]*Struct)
-	unnamed := make(map[string][]string)
-	flags := make(map[string][]string)
-	strflags := make(map[string][]string)
-	resources := make(map[string]Resource)
-	var str *Struct
-	for p.Scan() {
-		if p.EOF() || p.Char() == '#' {
-			continue
-		}
-		if str != nil {
-			// Parsing a struct.
-			if p.Char() == '}' || p.Char() == ']' {
-				p.Parse(p.Char())
-				for _, attr := range parseType1(p, unnamed, flags, "")[1:] {
-					if str.IsUnion {
-						switch attr {
-						case "varlen":
-							str.Varlen = true
-						default:
-							failf("unknown union %v attribute: %v", str.Name, attr)
-						}
-					} else {
-						switch {
-						case attr == "packed":
-							str.Packed = true
-						case strings.HasPrefix(attr, "align_ptr"):
-							str.Align = 8 // TODO: this must be target pointer size
-						case strings.HasPrefix(attr, "align_"):
-							a, err := strconv.ParseUint(attr[6:], 10, 64)
-							if err != nil {
-								failf("bad struct %v alignment %v: %v", str.Name, attr, err)
-							}
-							if a&(a-1) != 0 || a == 0 || a > 1<<30 {
-								failf("bad struct %v alignment %v: must be sane power of 2", str.Name, a)
-							}
-							str.Align = int(a)
-						default:
-							failf("unknown struct %v attribute: %v", str.Name, attr)
-						}
-					}
-				}
-				if str.IsUnion {
-					if len(str.Flds) <= 1 {
-						failf("union %v has only %v fields, need at least 2", str.Name, len(str.Flds))
-					}
-				}
-				fields := make(map[string]bool)
-				for _, f := range str.Flds {
-					if f[0] == "parent" {
-						failf("struct/union %v contains reserved field 'parent'", str.Name)
-					}
-					if fields[f[0]] {
-						failf("duplicate field %v in struct/union %v", f[0], str.Name)
-					}
-					fields[f[0]] = true
-				}
-				structs[str.Name] = str
-				str = nil
-			} else {
-				p.SkipWs()
-				fld := []string{p.Ident()}
-				fld = append(fld, parseType(p, unnamed, flags)...)
-				str.Flds = append(str.Flds, fld)
-			}
-		} else {
-			name := p.Ident()
-			if name == "include" {
-				p.Parse('<')
-				var include []byte
-				for {
-					ch := p.Char()
-					if ch == '>' {
-						break
-					}
-					p.Parse(ch)
-					include = append(include, ch)
-				}
-				p.Parse('>')
-				includes = append(includes, string(include))
-			} else if name == "define" {
-				key := p.Ident()
-				var val []byte
-				for !p.EOF() {
-					ch := p.Char()
-					p.Parse(ch)
-					val = append(val, ch)
-				}
-				if defines[key] != "" {
-					failf("%v define is defined multiple times", key)
-				}
-				defines[key] = fmt.Sprintf("(%s)", val)
-			} else if name == "resource" {
-				p.SkipWs()
-				id := p.Ident()
-				p.Parse('[')
-				base := p.Ident()
-				p.Parse(']')
-				var vals []string
-				if !p.EOF() && p.Char() == ':' {
-					p.Parse(':')
-					vals = append(vals, p.Ident())
-					for !p.EOF() {
-						p.Parse(',')
-						vals = append(vals, p.Ident())
-					}
-				}
-				if _, ok := resources[id]; ok {
-					failf("resource '%v' is defined multiple times", id)
-				}
-				if _, ok := structs[id]; ok {
-					failf("struct '%v' is redefined as resource", name)
-				}
-				resources[id] = Resource{id, base, vals}
-			} else if name == "incdir" {
-				p.Parse('"')
-				var incdir []byte
-				for {
-					ch := p.Char()
-					if ch == '"' {
-						break
-					}
-					p.Parse(ch)
-					incdir = append(incdir, ch)
-				}
-				p.Parse('"')
-				incdirs = append(incdirs, string(incdir))
-			} else {
-				switch ch := p.Char(); ch {
-				case '(':
-					// syscall
-					p.Parse('(')
-					var args [][]string
-					for p.Char() != ')' {
-						arg := []string{p.Ident()}
-						arg = append(arg, parseType(p, unnamed, flags)...)
-						args = append(args, arg)
-						if p.Char() != ')' {
-							p.Parse(',')
-						}
-					}
-					p.Parse(')')
-					var ret []string
-					if !p.EOF() {
-						ret = parseType(p, unnamed, flags)
-					}
-					callName := name
-					if idx := strings.IndexByte(callName, '$'); idx != -1 {
-						callName = callName[:idx]
-					}
-					fields := make(map[string]bool)
-					for _, a := range args {
-						if fields[a[0]] {
-							failf("duplicate arg %v in syscall %v", a[0], name)
-						}
-						fields[a[0]] = true
-					}
-					syscalls = append(syscalls, Syscall{name, callName, args, ret})
-				case '=':
-					// flag
-					p.Parse('=')
-					str := p.Char() == '"'
-					var vals []string
-					for {
-						v := p.Ident()
-						if str {
-							v = v[1 : len(v)-1]
-						}
-						vals = append(vals, v)
-						if p.EOF() {
-							break
-						}
-						p.Parse(',')
-					}
-					if str {
-						strflags[name] = vals
-					} else {
-						flags[name] = vals
-					}
-				case '{', '[':
-					p.Parse(ch)
-					if _, ok := structs[name]; ok {
-						failf("struct '%v' is defined multiple times", name)
-					}
-					if _, ok := resources[name]; ok {
-						failf("resource '%v' is redefined as struct", name)
-					}
-					str = &Struct{Name: name, IsUnion: ch == '['}
-				default:
-					failf("bad line (%v)", p.Str())
-				}
-			}
-		}
-		if !p.EOF() {
-			failf("trailing data (%v)", p.Str())
-		}
-	}
-	sort.Sort(syscallArray(syscalls))
-	return &Description{
-		Includes:  includes,
-		Incdirs:   incdirs,
-		Defines:   defines,
-		Syscalls:  syscalls,
-		Structs:   structs,
-		Unnamed:   unnamed,
-		Flags:     flags,
-		StrFlags:  strflags,
-		Resources: resources,
-	}
-}
-
-func isIdentifier(s string) bool {
-	for i, c := range s {
-		if c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || i > 0 && (c >= '0' && c <= '9') {
-			continue
-		}
-		return false
-	}
-	return true
-}
-
-func parseType(p *parser, unnamed map[string][]string, flags map[string][]string) []string {
-	return parseType1(p, unnamed, flags, p.Ident())
-}
-
-var (
-	unnamedSeq int
-	constSeq   int
-)
-
-func parseType1(p *parser, unnamed map[string][]string, flags map[string][]string, name string) []string {
-	typ := []string{name}
-	if !p.EOF() && p.Char() == '[' {
-		p.Parse('[')
-		for {
-			id := p.Ident()
-			if p.Char() == '[' {
-				inner := parseType1(p, unnamed, flags, id)
-				id = fmt.Sprintf("unnamed%v", unnamedSeq)
-				unnamedSeq++
-				unnamed[id] = inner
-			}
-			typ = append(typ, id)
-			if p.Char() == ']' {
-				break
-			}
-			p.Parse(',')
-		}
-		p.Parse(']')
-	}
-	if name == "const" && len(typ) > 1 {
-		// Create a fake flag with the const value.
-		id := fmt.Sprintf("const_flag_%v", constSeq)
-		constSeq++
-		flags[id] = typ[1:2]
-	}
-	if name == "array" && len(typ) > 2 {
-		// Create a fake flag with the const value.
-		id := fmt.Sprintf("const_flag_%v", constSeq)
-		constSeq++
-		flags[id] = typ[2:3]
-	}
-	return typ
-}
-
-type syscallArray []Syscall
-
-func (a syscallArray) Len() int           { return len(a) }
-func (a syscallArray) Less(i, j int) bool { return a[i].Name < a[j].Name }
-func (a syscallArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-
-func failf(msg string, args ...interface{}) {
-	fmt.Fprintf(os.Stderr, msg+"\n", args...)
-	os.Exit(1)
-}
diff --git a/sys/sysparser/parser.go b/sys/sysparser/parser.go
deleted file mode 100644
index d5021eee6..000000000
--- a/sys/sysparser/parser.go
+++ /dev/null
@@ -1,100 +0,0 @@
-// Copyright 2015/2016 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package sysparser
-
-import (
-	"bufio"
-	"fmt"
-	"io"
-	"os"
-)
-
-type parser struct {
-	r *bufio.Scanner
-	s string
-	i int
-	l int
-}
-
-func newParser(r io.Reader) *parser {
-	return &parser{r: bufio.NewScanner(r)}
-}
-
-func (p *parser) Scan() bool {
-	if !p.r.Scan() {
-		if err := p.r.Err(); err != nil {
-			failf("failed to read input file: %v", err)
-		}
-		return false
-	}
-	p.s = p.r.Text()
-	p.i = 0
-	p.l++
-	return true
-}
-
-func (p *parser) Str() string {
-	return p.s
-}
-
-func (p *parser) EOF() bool {
-	return p.i == len(p.s)
-}
-
-func (p *parser) Char() byte {
-	if p.EOF() {
-		p.failf("unexpected eof")
-	}
-	return p.s[p.i]
-}
-
-func (p *parser) Parse(ch byte) {
-	if p.EOF() {
-		p.failf("want %s, got EOF", string(ch))
-	}
-	if p.s[p.i] != ch {
-		p.failf("want '%v', got '%v'", string(ch), string(p.s[p.i]))
-	}
-	p.i++
-	p.SkipWs()
-}
-
-func (p *parser) SkipWs() {
-	for p.i < len(p.s) && (p.s[p.i] == ' ' || p.s[p.i] == '\t') {
-		p.i++
-	}
-}
-
-func (p *parser) Ident() string {
-	start, end := p.i, 0
-	if p.Char() == '"' {
-		p.Parse('"')
-		for p.Char() != '"' {
-			p.i++
-		}
-		end = p.i + 1
-		p.Parse('"')
-	} else {
-		for p.i < len(p.s) &&
-			(p.s[p.i] >= 'a' && p.s[p.i] <= 'z' ||
-				p.s[p.i] >= 'A' && p.s[p.i] <= 'Z' ||
-				p.s[p.i] >= '0' && p.s[p.i] <= '9' ||
-				p.s[p.i] == '_' || p.s[p.i] == '$' || // $ is for n-way syscalls (like ptrace$peek)
-				p.s[p.i] == '-' || p.s[p.i] == ':') { // : is for ranged int (like int32[-3:10])
-			p.i++
-		}
-		if start == p.i {
-			p.failf("failed to parse identifier at pos %v", start)
-		}
-		end = p.i
-	}
-	s := p.s[start:end]
-	p.SkipWs()
-	return s
-}
-
-func (p *parser) failf(msg string, args ...interface{}) {
-	fmt.Fprintf(os.Stderr, "line #%v: %v\n", p.l, p.s)
-	failf(msg, args...)
-}
diff --git a/sys/syz-sysgen/syscallnr.go b/sys/syz-sysgen/syscallnr.go
index 0b41a8634..c67f16166 100644
--- a/sys/syz-sysgen/syscallnr.go
+++ b/sys/syz-sysgen/syscallnr.go
@@ -7,8 +7,6 @@ import (
 	"bytes"
 	"sort"
 	"text/template"
-
-	. "github.com/google/syzkaller/sys/sysparser"
 )
 
 type Arch struct {
diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go
index 903361af6..a778e4819 100644
--- a/sys/syz-sysgen/sysgen.go
+++ b/sys/syz-sysgen/sysgen.go
@@ -20,7 +20,7 @@ import (
 	"strconv"
 	"strings"
 
-	. "github.com/google/syzkaller/sys/sysparser"
+	"github.com/google/syzkaller/pkg/ast"
 )
 
 var (
@@ -37,23 +37,11 @@ const (
 func main() {
 	flag.Parse()
 
-	inputFiles, err := filepath.Glob("sys/*\\.txt")
-	if err != nil {
-		failf("failed to find input files: %v", err)
-	}
-	var r io.Reader = bytes.NewReader(nil)
-	for _, f := range inputFiles {
-		inf, err := os.Open(f)
-		logf(1, "Load descriptions from file %v", f)
-		if err != nil {
-			failf("failed to open input file: %v", err)
-		}
-		defer inf.Close()
-		r = io.MultiReader(r, bufio.NewReader(inf))
+	top, ok := ast.ParseGlob("sys/*\\.txt", nil)
+	if !ok {
+		os.Exit(1)
 	}
-
-	logf(1, "Parse system call descriptions")
-	desc := Parse(r)
+	desc := astToDesc(top)
 
 	unsupportedFlags := make(map[string]int)
 	consts := make(map[string]map[string]uint64)
@@ -114,6 +102,212 @@ func main() {
 	}
 }
 
+type Description struct {
+	Syscalls  []Syscall
+	Structs   map[string]*Struct
+	Unnamed   map[string][]string
+	Flags     map[string][]string
+	StrFlags  map[string][]string
+	Resources map[string]Resource
+}
+
+type Syscall struct {
+	Name     string
+	CallName string
+	Args     [][]string
+	Ret      []string
+}
+
+type Struct struct {
+	Name    string
+	Flds    [][]string
+	IsUnion bool
+	Packed  bool
+	Varlen  bool
+	Align   int
+}
+
+type Resource struct {
+	Name   string
+	Base   string
+	Values []string
+}
+
+type syscallArray []Syscall
+
+func (a syscallArray) Len() int           { return len(a) }
+func (a syscallArray) Less(i, j int) bool { return a[i].Name < a[j].Name }
+func (a syscallArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+func astToDesc(top []interface{}) *Description {
+	// As a temporal measure we just convert the new representation to the old one.
+	// TODO: check for duplicate defines, structs, resources.
+	// TODO: check for duplicate syscall argument names.
+	desc := &Description{
+		Structs:   make(map[string]*Struct),
+		Unnamed:   make(map[string][]string),
+		Flags:     make(map[string][]string),
+		StrFlags:  make(map[string][]string),
+		Resources: make(map[string]Resource),
+	}
+	unnamedSeq := 0
+	for _, decl := range top {
+		switch n := decl.(type) {
+		case *ast.Resource:
+			var vals []string
+			for _, v := range n.Values {
+				switch {
+				case v.Ident != "":
+					vals = append(vals, v.Ident)
+				default:
+					if v.ValueHex {
+						vals = append(vals, fmt.Sprintf("0x%x", v.Value))
+					} else {
+						vals = append(vals, fmt.Sprint(v.Value))
+					}
+				}
+			}
+			desc.Resources[n.Name.Name] = Resource{
+				Name:   n.Name.Name,
+				Base:   n.Base.Name,
+				Values: vals,
+			}
+		case *ast.Call:
+			call := Syscall{
+				Name:     n.Name.Name,
+				CallName: n.CallName,
+			}
+			for _, a := range n.Args {
+				call.Args = append(call.Args, astToDescField(a, desc.Unnamed, &unnamedSeq))
+			}
+			if n.Ret != nil {
+				call.Ret = astToDescType(n.Ret, desc.Unnamed, &unnamedSeq)
+			}
+			desc.Syscalls = append(desc.Syscalls, call)
+		case *ast.Struct:
+			str := &Struct{
+				Name:    n.Name.Name,
+				IsUnion: n.IsUnion,
+			}
+			for _, f := range n.Fields {
+				str.Flds = append(str.Flds, astToDescField(f, desc.Unnamed, &unnamedSeq))
+			}
+			if n.IsUnion {
+				for _, attr := range n.Attrs {
+					switch attr.Name {
+					case "varlen":
+						str.Varlen = true
+					default:
+						failf("unknown union %v attribute: %v", str.Name, attr.Name)
+					}
+				}
+			} else {
+				for _, attr := range n.Attrs {
+					switch {
+					case attr.Name == "packed":
+						str.Packed = true
+					case attr.Name == "align_ptr":
+						str.Align = 8 // TODO: this must be target pointer size
+					case strings.HasPrefix(attr.Name, "align_"):
+						a, err := strconv.ParseUint(attr.Name[6:], 10, 64)
+						if err != nil {
+							failf("bad struct %v alignment %v: %v", str.Name, attr.Name, err)
+						}
+						if a&(a-1) != 0 || a == 0 || a > 1<<30 {
+							failf("bad struct %v alignment %v: must be sane power of 2", str.Name, a)
+						}
+						str.Align = int(a)
+					default:
+						failf("unknown struct %v attribute: %v", str.Name, attr.Name)
+					}
+				}
+			}
+			if str.IsUnion && len(str.Flds) <= 1 {
+				failf("union %v has only %v fields, need at least 2", str.Name, len(str.Flds))
+			}
+			fields := make(map[string]bool)
+			for _, f := range str.Flds {
+				if f[0] == "parent" {
+					failf("struct/union %v contains reserved field 'parent'", str.Name)
+				}
+				if fields[f[0]] {
+					failf("duplicate field %v in struct/union %v", f[0], str.Name)
+				}
+				fields[f[0]] = true
+			}
+			desc.Structs[str.Name] = str
+		case *ast.IntFlags:
+			var vals []string
+			for _, v := range n.Values {
+				switch {
+				case v.Ident != "":
+					vals = append(vals, v.Ident)
+				default:
+					if v.ValueHex {
+						vals = append(vals, fmt.Sprintf("0x%x", v.Value))
+					} else {
+						vals = append(vals, fmt.Sprint(v.Value))
+					}
+				}
+			}
+			desc.Flags[n.Name.Name] = vals
+		case *ast.StrFlags:
+			var vals []string
+			for _, v := range n.Values {
+				vals = append(vals, v.Value)
+			}
+			desc.StrFlags[n.Name.Name] = vals
+		}
+	}
+	sort.Sort(syscallArray(desc.Syscalls))
+	return desc
+}
+
+func astToDescField(n *ast.Field, unnamed map[string][]string, unnamedSeq *int) []string {
+	return append([]string{n.Name.Name}, astToDescType(n.Type, unnamed, unnamedSeq)...)
+}
+
+func astToDescType(n *ast.Type, unnamed map[string][]string, unnamedSeq *int) []string {
+	res := []string{astTypeToStr(n)}
+	for _, t := range n.Args {
+		if len(t.Args) == 0 {
+			res = append(res, astTypeToStr(t))
+			continue
+		}
+		id := fmt.Sprintf("unnamed%v", *unnamedSeq)
+		(*unnamedSeq)++
+		unnamed[id] = astToDescType(t, unnamed, unnamedSeq)
+		res = append(res, id)
+	}
+	return res
+}
+
+func astTypeToStr(n *ast.Type) string {
+	res := ""
+	switch {
+	case n.Ident != "":
+		res = n.Ident
+	case n.String != "":
+		res = fmt.Sprintf("\"%v\"", n.String)
+	default:
+		if n.ValueHex {
+			res = fmt.Sprintf("0x%x", n.Value)
+		} else {
+			res = fmt.Sprint(n.Value)
+		}
+	}
+	if n.Ident2 != "" {
+		res += ":" + n.Ident2
+	} else if n.Value2 != 0 {
+		if n.Value2Hex {
+			res += fmt.Sprintf(":0x%x", n.Value2)
+		} else {
+			res += ":" + fmt.Sprint(n.Value2)
+		}
+	}
+	return res
+}
+
 func readConsts(arch string) map[string]uint64 {
 	constFiles, err := filepath.Glob("sys/*_" + arch + ".const")
 	if err != nil {