pkg/compiler: merge const files into a single file

We now have 8 arches for Linux and .const files
produce lots of noise in PRs and lots of diffs.
If 3 .txt files are touched, the PR will have 24 .const files,
which will be intermixed with .txt files.
Frequently const values are equal across arches,
and even if they don't spreading a single value
across 8 files is inconvinient.

Merge all 8 *_arch.const files into a single .const file.
See the test for details of the new format.
The old format is still parsed for now,
we can't update all OSes at once.

For Linux this reduces number of const files/lines
from 1288/96599 to 158/11603.

Fixes #1983

1 files changed, 130 insertions, 0 deletions

diff --git a/pkg/compiler/const_file_test.go b/pkg/compiler/const_file_test.go
new file mode 100644
index 000000000..51858a790
--- /dev/null
+++ b/pkg/compiler/const_file_test.go
@@ -0,0 +1,130 @@
+// Copyright 2020 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package compiler
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+func TestConstFile(t *testing.T) {
+	type arch struct {
+		consts    map[string]uint64
+		undefined map[string]bool
+		oldFormat string
+	}
+	arches := map[string]arch{
+		"arch1": {
+			consts: map[string]uint64{
+				"CONST1_ALL_DIFFERENT":   11,
+				"CONST2_ALL_THE_SAME":    3,
+				"CONST3_SOME_UNDEFINED":  100,
+				"CONST5_SOME_UNDEFINED2": 100,
+			},
+			undefined: map[string]bool{
+				"CONST4_ALL_UNDEFINED": true,
+			},
+			oldFormat: `
+CONST1_ALL_DIFFERENT = 11
+CONST2_ALL_THE_SAME = 3
+CONST3_SOME_UNDEFINED = 100
+CONST5_SOME_UNDEFINED2 = 100
+# CONST4_ALL_UNDEFINED is not set
+`,
+		},
+		"arch2": {
+			consts: map[string]uint64{
+				"CONST1_ALL_DIFFERENT":   22,
+				"CONST2_ALL_THE_SAME":    3,
+				"CONST5_SOME_UNDEFINED2": 100,
+			},
+			undefined: map[string]bool{
+				"CONST4_ALL_UNDEFINED":  true,
+				"CONST3_SOME_UNDEFINED": true,
+			},
+			oldFormat: `
+CONST1_ALL_DIFFERENT = 22
+CONST2_ALL_THE_SAME = 3
+# CONST3_SOME_UNDEFINED is not set
+CONST5_SOME_UNDEFINED2 = 100
+# CONST4_ALL_UNDEFINED is not set
+`,
+		},
+		"arch3": {
+			consts: map[string]uint64{
+				"CONST1_ALL_DIFFERENT": 33,
+				"CONST2_ALL_THE_SAME":  3,
+			},
+			undefined: map[string]bool{
+				"CONST4_ALL_UNDEFINED":   true,
+				"CONST3_SOME_UNDEFINED":  true,
+				"CONST5_SOME_UNDEFINED2": true,
+			},
+			oldFormat: `
+CONST1_ALL_DIFFERENT = 33
+CONST2_ALL_THE_SAME = 3
+# CONST3_SOME_UNDEFINED is not set
+# CONST5_SOME_UNDEFINED2 is not set
+# CONST4_ALL_UNDEFINED is not set
+`,
+		},
+	}
+	const serialized = `# Code generated by syz-sysgen. DO NOT EDIT.
+arches = arch1, arch2, arch3
+CONST1_ALL_DIFFERENT = arch1:11, arch2:22, arch3:33
+CONST2_ALL_THE_SAME = 3
+CONST3_SOME_UNDEFINED = arch1:100, arch2:arch3:???
+CONST4_ALL_UNDEFINED = ???
+CONST5_SOME_UNDEFINED2 = 100, arch3:???
+`
+	cf := NewConstFile()
+	for name, arch := range arches {
+		cf.AddArch(name, arch.consts, arch.undefined)
+	}
+	data := cf.Serialize()
+	if diff := cmp.Diff(serialized, string(data)); diff != "" {
+		t.Fatal(diff)
+	}
+	{
+		file, err := ioutil.TempFile("", "syz-const")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer file.Close()
+		defer os.Remove(file.Name())
+		if _, err := file.Write(data); err != nil {
+			t.Fatal(err)
+		}
+		file.Close()
+		cf1 := DeserializeConstFile(file.Name(), nil)
+		for name, arch := range arches {
+			if diff := cmp.Diff(arch.consts, cf1.Arch(name)); diff != "" {
+				t.Errorf("%v: %v", name, diff)
+			}
+		}
+	}
+	{
+		dir, err := ioutil.TempDir("", "syz-const")
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer os.RemoveAll(dir)
+		for name, arch := range arches {
+			file := filepath.Join(dir, "consts_"+name+".const")
+			if err := ioutil.WriteFile(file, []byte(arch.oldFormat), 0600); err != nil {
+				t.Fatal(err)
+			}
+		}
+		cf1 := DeserializeConstFile(filepath.Join(dir, "*"), nil)
+		for name, arch := range arches {
+			if diff := cmp.Diff(arch.consts, cf1.Arch(name)); diff != "" {
+				t.Errorf("%v: %v", name, diff)
+			}
+		}
+	}
+}