pkg/compiler: merge const files into a single file

We now have 8 arches for Linux and .const files
produce lots of noise in PRs and lots of diffs.
If 3 .txt files are touched, the PR will have 24 .const files,
which will be intermixed with .txt files.
Frequently const values are equal across arches,
and even if they don't spreading a single value
across 8 files is inconvinient.

Merge all 8 *_arch.const files into a single .const file.
See the test for details of the new format.
The old format is still parsed for now,
we can't update all OSes at once.

For Linux this reduces number of const files/lines
from 1288/96599 to 158/11603.

Fixes #1983

4 files changed, 425 insertions, 117 deletions

diff --git a/pkg/compiler/compiler_test.go b/pkg/compiler/compiler_test.go
index 3d4ee3e64..e5c7987d1 100644
--- a/pkg/compiler/compiler_test.go
+++ b/pkg/compiler/compiler_test.go
@@ -44,7 +44,7 @@ func TestCompileAll(t *testing.T) {
 					defer func() {
 						t.Logf("\n%s", errors.Bytes())
 					}()
-					consts := DeserializeConstsGlob(filepath.Join(path, "*_"+arch+".const"), eh)
+					consts := DeserializeConstFile(filepath.Join(path, "*.const"), eh).Arch(arch)
 					if consts == nil {
 						t.Fatalf("reading consts failed")
 					}
diff --git a/pkg/compiler/const_file.go b/pkg/compiler/const_file.go
new file mode 100644
index 000000000..d64c15207
--- /dev/null
+++ b/pkg/compiler/const_file.go
@@ -0,0 +1,294 @@
+// Copyright 2020 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package compiler
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/google/syzkaller/pkg/ast"
+)
+
+// ConstFile serializes/deserializes .const files.
+type ConstFile struct {
+	arches map[string]bool
+	m      map[string]constVal
+}
+
+type constVal struct {
+	name string
+	vals map[string]uint64 // arch -> value
+}
+
+const undefined = "???"
+
+func NewConstFile() *ConstFile {
+	return &ConstFile{
+		arches: make(map[string]bool),
+		m:      make(map[string]constVal),
+	}
+}
+
+func (cf *ConstFile) AddArch(arch string, consts map[string]uint64, undeclared map[string]bool) error {
+	cf.arches[arch] = true
+	for name, val := range consts {
+		if err := cf.addConst(arch, name, val, true); err != nil {
+			return err
+		}
+	}
+	for name := range undeclared {
+		if err := cf.addConst(arch, name, 0, false); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (cf *ConstFile) addConst(arch, name string, val uint64, declared bool) error {
+	cv := cf.m[name]
+	if cv.vals == nil {
+		cv.name = name
+		cv.vals = make(map[string]uint64)
+	}
+	if val0, declared0 := cv.vals[arch]; declared && declared0 && val != val0 {
+		return fmt.Errorf("const=%v arch=%v has different values: %v[%v] vs %v[%v]",
+			name, arch, val, declared, val0, declared0)
+	}
+	if declared {
+		cv.vals[arch] = val
+	}
+	cf.m[name] = cv
+	return nil
+}
+
+func (cf *ConstFile) Arch(arch string) map[string]uint64 {
+	if cf == nil {
+		return nil
+	}
+	m := make(map[string]uint64)
+	for name, cv := range cf.m {
+		if v, ok := cv.vals[arch]; ok {
+			m[name] = v
+		}
+	}
+	return m
+}
+
+func (cf *ConstFile) Serialize() []byte {
+	if len(cf.arches) == 0 {
+		return nil
+	}
+	var arches []string
+	for arch := range cf.arches {
+		arches = append(arches, arch)
+	}
+	sort.Strings(arches)
+	var consts []constVal
+	for _, cv := range cf.m {
+		consts = append(consts, cv)
+	}
+	sort.Slice(consts, func(i, j int) bool {
+		return consts[i].name < consts[j].name
+	})
+	buf := new(bytes.Buffer)
+	fmt.Fprintf(buf, "# Code generated by syz-sysgen. DO NOT EDIT.\n")
+	fmt.Fprintf(buf, "arches = %v\n", strings.Join(arches, ", "))
+	for _, cv := range consts {
+		fmt.Fprintf(buf, "%v = ", cv.name)
+		if len(cv.vals) == 0 {
+			// Undefined for all arches.
+			fmt.Fprintf(buf, "%v\n", undefined)
+			continue
+		}
+		count := make(map[uint64]int)
+		max, dflt := 0, uint64(0)
+		for _, val := range cv.vals {
+			count[val]++
+			if count[val] > 1 && (count[val] > max || count[val] == max && val < dflt) {
+				max, dflt = count[val], val
+			}
+		}
+		if max != 0 {
+			// Have a default value.
+			fmt.Fprintf(buf, "%v", dflt)
+		}
+		handled := make([]bool, len(arches))
+		for i, arch := range arches {
+			val, ok := cv.vals[arch]
+			if ok && val == dflt || handled[i] {
+				// Default value or serialized on a previous iteration.
+				continue
+			}
+			if i != 0 || max != 0 {
+				fmt.Fprintf(buf, ", ")
+			}
+			fmt.Fprintf(buf, "%v:", arch)
+			for j := i + 1; j < len(arches); j++ {
+				// Add more arches with the same value.
+				arch1 := arches[j]
+				val1, ok1 := cv.vals[arch1]
+				if ok1 == ok && val1 == val {
+					fmt.Fprintf(buf, "%v:", arch1)
+					handled[j] = true
+				}
+			}
+			if ok {
+				fmt.Fprintf(buf, "%v", val)
+			} else {
+				fmt.Fprint(buf, undefined)
+			}
+		}
+		fmt.Fprintf(buf, "\n")
+	}
+	return buf.Bytes()
+}
+
+func DeserializeConstFile(glob string, eh ast.ErrorHandler) *ConstFile {
+	if eh == nil {
+		eh = ast.LoggingHandler
+	}
+	files, err := filepath.Glob(glob)
+	if err != nil {
+		eh(ast.Pos{}, fmt.Sprintf("failed to find const files: %v", err))
+		return nil
+	}
+	if len(files) == 0 {
+		eh(ast.Pos{}, fmt.Sprintf("no const files matched by glob %q", glob))
+		return nil
+	}
+	cf := NewConstFile()
+	oldFormat := regexp.MustCompile(`_([a-z0-9]+)\.const$`)
+	for _, f := range files {
+		data, err := ioutil.ReadFile(f)
+		if err != nil {
+			eh(ast.Pos{}, fmt.Sprintf("failed to read const file: %v", err))
+			return nil
+		}
+		// Support for old per-arch format.
+		// Remove it once we don't have any *_arch.const files anymore.
+		arch := ""
+		if match := oldFormat.FindStringSubmatch(f); match != nil {
+			arch = match[1]
+		}
+		if !cf.deserializeFile(data, filepath.Base(f), arch, eh) {
+			return nil
+		}
+	}
+	return cf
+}
+
+func (cf *ConstFile) deserializeFile(data []byte, file, arch string, eh ast.ErrorHandler) bool {
+	pos := ast.Pos{File: file, Line: 1}
+	errf := func(msg string, args ...interface{}) bool {
+		eh(pos, fmt.Sprintf(msg, args...))
+		return false
+	}
+	s := bufio.NewScanner(bytes.NewReader(data))
+	var arches []string
+	for ; s.Scan(); pos.Line++ {
+		line := s.Text()
+		if line == "" || line[0] == '#' {
+			continue
+		}
+		eq := strings.IndexByte(line, '=')
+		if eq == -1 {
+			return errf("expect '='")
+		}
+		name, val := strings.TrimSpace(line[:eq]), strings.TrimSpace(line[eq+1:])
+		if arch != "" {
+			// Old format.
+			if !cf.parseOldConst(arch, name, val, errf) {
+				return false
+			}
+			continue
+		}
+		if arch == "" && len(arches) == 0 {
+			if name != "arches" {
+				return errf("missing arches header")
+			}
+			for _, arch := range strings.Split(val, ",") {
+				arches = append(arches, strings.TrimSpace(arch))
+			}
+			continue
+		}
+		if !cf.parseConst(arches, name, val, errf) {
+			return false
+		}
+	}
+	if err := s.Err(); err != nil {
+		return errf("failed to parse: %v", err)
+	}
+	return true
+}
+
+type errft func(msg string, args ...interface{}) bool
+
+func (cf *ConstFile) parseConst(arches []string, name, line string, errf errft) bool {
+	var dflt map[string]uint64
+	for _, pair := range strings.Split(line, ",") {
+		fields := strings.Split(pair, ":")
+		if len(fields) == 1 {
+			// Default value.
+			if dflt != nil {
+				return errf("duplicate default value")
+			}
+			dflt = make(map[string]uint64)
+			valStr := strings.TrimSpace(fields[0])
+			if valStr == undefined {
+				continue
+			}
+			val, err := strconv.ParseUint(valStr, 0, 64)
+			if err != nil {
+				return errf("failed to parse int: %v", err)
+			}
+			for _, arch := range arches {
+				dflt[arch] = val
+			}
+			continue
+		}
+		if len(fields) < 2 {
+			return errf("bad value: %v", pair)
+		}
+		valStr := strings.TrimSpace(fields[len(fields)-1])
+		defined := valStr != undefined
+		var val uint64
+		if defined {
+			var err error
+			if val, err = strconv.ParseUint(valStr, 0, 64); err != nil {
+				return errf("failed to parse int: %v", err)
+			}
+		}
+		for _, arch := range fields[:len(fields)-1] {
+			arch = strings.TrimSpace(arch)
+			delete(dflt, arch)
+			if err := cf.addConst(arch, name, val, defined); err != nil {
+				return errf("%v", err)
+			}
+		}
+	}
+	for arch, val := range dflt {
+		if err := cf.addConst(arch, name, val, true); err != nil {
+			return errf("%v", err)
+		}
+	}
+	return true
+}
+
+func (cf *ConstFile) parseOldConst(arch, name, line string, errf errft) bool {
+	val, err := strconv.ParseUint(strings.TrimSpace(line), 0, 64)
+	if err != nil {
+		return errf("failed to parse int: %v", err)
+	}
+	if err := cf.addConst(arch, name, val, true); err != nil {
+		return errf("%v", err)
+	}
+	return true
+}
diff --git a/pkg/compiler/const_file_test.go b/pkg/compiler/const_file_test.go
new file mode 100644
index 000000000..51858a790
--- /dev/null
+++ b/pkg/compiler/const_file_test.go
@@ -0,0 +1,130 @@
+// Copyright 2020 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package compiler
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestConstFile(t *testing.T) {
+	type arch struct {
+		consts    map[string]uint64
+		undefined map[string]bool
+		oldFormat string
+	}
+	arches := map[string]arch{
+		"arch1": {
+			consts: map[string]uint64{
+				"CONST1_ALL_DIFFERENT":   11,
+				"CONST2_ALL_THE_SAME":    3,
+				"CONST3_SOME_UNDEFINED":  100,
+				"CONST5_SOME_UNDEFINED2": 100,
+			},
+			undefined: map[string]bool{
+				"CONST4_ALL_UNDEFINED": true,
+			},
+			oldFormat: `
+CONST1_ALL_DIFFERENT = 11
+CONST2_ALL_THE_SAME = 3
+CONST3_SOME_UNDEFINED = 100
+CONST5_SOME_UNDEFINED2 = 100
+# CONST4_ALL_UNDEFINED is not set
+`,
+		},
+		"arch2": {
+			consts: map[string]uint64{
+				"CONST1_ALL_DIFFERENT":   22,
+				"CONST2_ALL_THE_SAME":    3,
+				"CONST5_SOME_UNDEFINED2": 100,
+			},
+			undefined: map[string]bool{
+				"CONST4_ALL_UNDEFINED":  true,
+				"CONST3_SOME_UNDEFINED": true,
+			},
+			oldFormat: `
+CONST1_ALL_DIFFERENT = 22
+CONST2_ALL_THE_SAME = 3
+# CONST3_SOME_UNDEFINED is not set
+CONST5_SOME_UNDEFINED2 = 100
+# CONST4_ALL_UNDEFINED is not set
+`,
+		},
+		"arch3": {
+			consts: map[string]uint64{
+				"CONST1_ALL_DIFFERENT": 33,
+				"CONST2_ALL_THE_SAME":  3,
+			},
+			undefined: map[string]bool{
+				"CONST4_ALL_UNDEFINED":   true,
+				"CONST3_SOME_UNDEFINED":  true,
+				"CONST5_SOME_UNDEFINED2": true,
+			},
+			oldFormat: `
+CONST1_ALL_DIFFERENT = 33
+CONST2_ALL_THE_SAME = 3
+# CONST3_SOME_UNDEFINED is not set
+# CONST5_SOME_UNDEFINED2 is not set
+# CONST4_ALL_UNDEFINED is not set
+`,
+		},
+	}
+	const serialized = `# Code generated by syz-sysgen. DO NOT EDIT.
+arches = arch1, arch2, arch3
+CONST1_ALL_DIFFERENT = arch1:11, arch2:22, arch3:33
+CONST2_ALL_THE_SAME = 3
+CONST3_SOME_UNDEFINED = arch1:100, arch2:arch3:???
+CONST4_ALL_UNDEFINED = ???
+CONST5_SOME_UNDEFINED2 = 100, arch3:???
+`
+	cf := NewConstFile()
+	for name, arch := range arches {
+		cf.AddArch(name, arch.consts, arch.undefined)
+	}
+	data := cf.Serialize()
+	if diff := cmp.Diff(serialized, string(data)); diff != "" {
+		t.Fatal(diff)
+	}
+	{
+		file, err := ioutil.TempFile("", "syz-const")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		defer os.Remove(file.Name())
+		if _, err := file.Write(data); err != nil {
+			t.Fatal(err)
+		}
+		file.Close()
+		cf1 := DeserializeConstFile(file.Name(), nil)
+		for name, arch := range arches {
+			if diff := cmp.Diff(arch.consts, cf1.Arch(name)); diff != "" {
+				t.Errorf("%v: %v", name, diff)
+			}
+		}
+	}
+	{
+		dir, err := ioutil.TempDir("", "syz-const")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(dir)
+		for name, arch := range arches {
+			file := filepath.Join(dir, "consts_"+name+".const")
+			if err := ioutil.WriteFile(file, []byte(arch.oldFormat), 0600); err != nil {
+				t.Fatal(err)
+			}
+		}
+		cf1 := DeserializeConstFile(filepath.Join(dir, "*"), nil)
+		for name, arch := range arches {
+			if diff := cmp.Diff(arch.consts, cf1.Arch(name)); diff != "" {
+				t.Errorf("%v: %v", name, diff)
+			}
+		}
+	}
+}
diff --git a/pkg/compiler/consts.go b/pkg/compiler/consts.go
index e57d599cb..3e8773a03 100644
--- a/pkg/compiler/consts.go
+++ b/pkg/compiler/consts.go
@@ -4,13 +4,7 @@
 package compiler
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
-	"io/ioutil"
-	"path/filepath"
-	"sort"
-	"strconv"
 	"strings"
 
 	"github.com/google/syzkaller/pkg/ast"
@@ -294,113 +288,3 @@ func (comp *compiler) patchConst(val *uint64, id *string, consts map[string]uint
 	*val = 1
 	return false
 }
-
-func SerializeConsts(consts map[string]uint64, undeclared map[string]bool) []byte {
-	type nameValuePair struct {
-		declared bool
-		name     string
-		val      uint64
-	}
-	var nv []nameValuePair
-	for k, v := range consts {
-		nv = append(nv, nameValuePair{true, k, v})
-	}
-	for k := range undeclared {
-		nv = append(nv, nameValuePair{false, k, 0})
-	}
-	sort.Slice(nv, func(i, j int) bool {
-		return nv[i].name < nv[j].name
-	})
-
-	buf := new(bytes.Buffer)
-	fmt.Fprintf(buf, "# AUTOGENERATED FILE\n")
-	for _, x := range nv {
-		if x.declared {
-			fmt.Fprintf(buf, "%v = %v\n", x.name, x.val)
-		} else {
-			fmt.Fprintf(buf, "# %v is not set\n", x.name)
-		}
-	}
-	return buf.Bytes()
-}
-
-func DeserializeConsts(data []byte, file string, eh ast.ErrorHandler) map[string]uint64 {
-	consts := make(map[string]uint64)
-	pos := ast.Pos{
-		File: file,
-		Line: 1,
-	}
-	ok := true
-	s := bufio.NewScanner(bytes.NewReader(data))
-	for ; s.Scan(); pos.Line++ {
-		line := s.Text()
-		if line == "" || line[0] == '#' {
-			continue
-		}
-		eq := strings.IndexByte(line, '=')
-		if eq == -1 {
-			eh(pos, "expect '='")
-			ok = false
-			continue
-		}
-		name := strings.TrimSpace(line[:eq])
-		val, err := strconv.ParseUint(strings.TrimSpace(line[eq+1:]), 0, 64)
-		if err != nil {
-			eh(pos, fmt.Sprintf("failed to parse int: %v", err))
-			ok = false
-			continue
-		}
-		if _, dup := consts[name]; dup {
-			eh(pos, fmt.Sprintf("duplicate const %q", name))
-			ok = false
-			continue
-		}
-		consts[name] = val
-	}
-	if err := s.Err(); err != nil {
-		eh(pos, fmt.Sprintf("failed to parse: %v", err))
-		ok = false
-	}
-	if !ok {
-		return nil
-	}
-	return consts
-}
-
-func DeserializeConstsGlob(glob string, eh ast.ErrorHandler) map[string]uint64 {
-	if eh == nil {
-		eh = ast.LoggingHandler
-	}
-	files, err := filepath.Glob(glob)
-	if err != nil {
-		eh(ast.Pos{}, fmt.Sprintf("failed to find const files: %v", err))
-		return nil
-	}
-	if len(files) == 0 {
-		eh(ast.Pos{}, fmt.Sprintf("no const files matched by glob %q", glob))
-		return nil
-	}
-	consts := make(map[string]uint64)
-	for _, f := range files {
-		data, err := ioutil.ReadFile(f)
-		if err != nil {
-			eh(ast.Pos{}, fmt.Sprintf("failed to read const file: %v", err))
-			return nil
-		}
-		consts1 := DeserializeConsts(data, filepath.Base(f), eh)
-		if consts1 == nil {
-			consts = nil
-		}
-		if consts != nil {
-			for n, v := range consts1 {
-				if old, ok := consts[n]; ok && old != v {
-					eh(ast.Pos{}, fmt.Sprintf(
-						"different values for const %q: %v vs %v", n, v, old))
-					return nil
-				}
-				consts[n] = v
-			}
-		}
-	}
-	return consts
-}