pkg/compiler: merge const files into a single file

We now have 8 arches for Linux and .const files
produce lots of noise in PRs and lots of diffs.
If 3 .txt files are touched, the PR will have 24 .const files,
which will be intermixed with .txt files.
Frequently const values are equal across arches,
and even if they don't spreading a single value
across 8 files is inconvinient.

Merge all 8 *_arch.const files into a single .const file.
See the test for details of the new format.
The old format is still parsed for now,
we can't update all OSes at once.

For Linux this reduces number of const files/lines
from 1288/96599 to 158/11603.

Fixes #1983

7 files changed, 49 insertions, 49 deletions

diff --git a/sys/syz-extract/extract.go b/sys/syz-extract/extract.go
index 22cd44402..10b8b6541 100644
--- a/sys/syz-extract/extract.go
+++ b/sys/syz-extract/extract.go
@@ -101,28 +101,14 @@ func main() {
 	}
 
 	for p := 0; p < runtime.GOMAXPROCS(0); p++ {
-		go func() {
-			for job := range jobC {
-				switch j := job.(type) {
-				case *Arch:
-					infos, err := processArch(extractor, j)
-					j.err = err
-					close(j.done)
-					if j.err == nil {
-						for _, f := range j.files {
-							f.info = infos[filepath.Join("sys", j.target.OS, f.name)]
-							jobC <- f
-						}
-					}
-				case *File:
-					j.consts, j.undeclared, j.err = processFile(extractor, j.arch, j)
-					close(j.done)
-				}
-			}
-		}()
+		go worker(extractor, jobC)
 	}
 
 	failed := false
+	constFiles := make(map[string]*compiler.ConstFile)
+	for _, file := range files {
+		constFiles[file] = compiler.NewConstFile()
+	}
 	for _, arch := range arches {
 		fmt.Printf("generating %v/%v...\n", arch.target.OS, arch.target.Arch)
 		<-arch.done
@@ -138,6 +124,18 @@ func main() {
 				fmt.Printf("%v: %v\n", f.name, f.err)
 				continue
 			}
+			constFiles[f.name].AddArch(f.arch.target.Arch, f.consts, f.undeclared)
+		}
+	}
+	for file, cf := range constFiles {
+		outname := filepath.Join("sys", OS, file+".const")
+		data := cf.Serialize()
+		if len(data) == 0 {
+			os.Remove(outname)
+			continue
+		}
+		if err := osutil.WriteFile(outname, data); err != nil {
+			failf("failed to write output file: %v", err)
 		}
 	}
 
@@ -154,6 +152,26 @@ func main() {
 	}
 }
 
+func worker(extractor Extractor, jobC chan interface{}) {
+	for job := range jobC {
+		switch j := job.(type) {
+		case *Arch:
+			infos, err := processArch(extractor, j)
+			j.err = err
+			close(j.done)
+			if j.err == nil {
+				for _, f := range j.files {
+					f.info = infos[filepath.Join("sys", j.target.OS, f.name)]
+					jobC <- f
+				}
+			}
+		case *File:
+			j.consts, j.undeclared, j.err = processFile(extractor, j.arch, j)
+			close(j.done)
+		}
+	}
+}
+
 func createArches(OS string, archArray, files []string) ([]*Arch, error) {
 	var arches []*Arch
 	for _, archStr := range archArray {
@@ -291,21 +309,11 @@ func processArch(extractor Extractor, arch *Arch) (map[string]*compiler.ConstInf
 
 func processFile(extractor Extractor, arch *Arch, file *File) (map[string]uint64, map[string]bool, error) {
 	inname := filepath.Join("sys", arch.target.OS, file.name)
-	outname := strings.TrimSuffix(inname, ".txt") + "_" + arch.target.Arch + ".const"
 	if file.info == nil {
 		return nil, nil, fmt.Errorf("const info for input file %v is missing", inname)
 	}
 	if len(file.info.Consts) == 0 {
-		os.Remove(outname)
 		return nil, nil, nil
 	}
-	consts, undeclared, err := extractor.processFile(arch, file.info)
-	if err != nil {
-		return nil, nil, err
-	}
-	data := compiler.SerializeConsts(consts, undeclared)
-	if err := osutil.WriteFile(outname, data); err != nil {
-		return nil, nil, fmt.Errorf("failed to write output file: %v", err)
-	}
-	return consts, undeclared, nil
+	return extractor.processFile(arch, file.info)
 }
diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go
index 5fc2be7b7..8dfedb6a2 100644
--- a/sys/syz-sysgen/sysgen.go
+++ b/sys/syz-sysgen/sysgen.go
@@ -75,6 +75,10 @@ func main() {
 		if descriptions == nil {
 			os.Exit(1)
 		}
+		constFile := compiler.DeserializeConstFile(filepath.Join(*srcDir, "sys", OS, "*.const"), nil)
+		if constFile == nil {
+			os.Exit(1)
+		}
 		osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen"))
 
 		var archs []string
@@ -109,10 +113,7 @@ func main() {
 				eh := func(pos ast.Pos, msg string) {
 					job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg))
 				}
-				consts := compiler.DeserializeConstsGlob(filepath.Join(*srcDir, "sys", OS, "*_"+job.Target.Arch+".const"), eh)
-				if consts == nil {
-					return
-				}
+				consts := constFile.Arch(job.Target.Arch)
 				top := descriptions
 				if OS == "linux" && (job.Target.Arch == "arm" || job.Target.Arch == "riscv64") {
 					// Hack: KVM is not supported on ARM anymore. On riscv64 it
diff --git a/sys/test/test.txt.const b/sys/test/test.txt.const
new file mode 100644
index 000000000..11f548df0
--- /dev/null
+++ b/sys/test/test.txt.const
@@ -0,0 +1,6 @@
+arches = 32_fork_shmem, 32_shmem, 64, 64_fork
+IPPROTO_ICMPV6 = 58
+IPPROTO_TCP = 6
+IPPROTO_UDP = 17
+ONLY_32BITS_CONST = 32_fork_shmem:1, 32_shmem:1
+ARCH_64_SPECIFIC_CONST = 64:10
diff --git a/sys/test/test_32_fork_shmem.const b/sys/test/test_32_fork_shmem.const
deleted file mode 100644
index 938b68f89..000000000
--- a/sys/test/test_32_fork_shmem.const
+++ /dev/null
@@ -1,4 +0,0 @@
-IPPROTO_ICMPV6 = 58
-IPPROTO_TCP = 6
-IPPROTO_UDP = 17
-ONLY_32BITS_CONST = 1
diff --git a/sys/test/test_32_shmem.const b/sys/test/test_32_shmem.const
deleted file mode 100644
index 938b68f89..000000000
--- a/sys/test/test_32_shmem.const
+++ /dev/null
@@ -1,4 +0,0 @@
-IPPROTO_ICMPV6 = 58
-IPPROTO_TCP = 6
-IPPROTO_UDP = 17
-ONLY_32BITS_CONST = 1
diff --git a/sys/test/test_64.const b/sys/test/test_64.const
deleted file mode 100644
index 8b4c45cca..000000000
--- a/sys/test/test_64.const
+++ /dev/null
@@ -1,4 +0,0 @@
-IPPROTO_ICMPV6 = 58
-IPPROTO_TCP = 6
-IPPROTO_UDP = 17
-ARCH_64_SPECIFIC_CONST = 10
diff --git a/sys/test/test_64_fork.const b/sys/test/test_64_fork.const
deleted file mode 100644
index 30db9e24f..000000000
--- a/sys/test/test_64_fork.const
+++ /dev/null
@@ -1,3 +0,0 @@
-IPPROTO_ICMPV6 = 58
-IPPROTO_TCP = 6
-IPPROTO_UDP = 17