pkg/compiler: merge const files into a single file

We now have 8 arches for Linux and .const files
produce lots of noise in PRs and lots of diffs.
If 3 .txt files are touched, the PR will have 24 .const files,
which will be intermixed with .txt files.
Frequently const values are equal across arches,
and even if they don't spreading a single value
across 8 files is inconvinient.

Merge all 8 *_arch.const files into a single .const file.
See the test for details of the new format.
The old format is still parsed for now,
we can't update all OSes at once.

For Linux this reduces number of const files/lines
from 1288/96599 to 158/11603.

Fixes #1983

1 files changed, 38 insertions, 30 deletions

diff --git a/sys/syz-extract/extract.go b/sys/syz-extract/extract.go
index 22cd44402..10b8b6541 100644
--- a/sys/syz-extract/extract.go
+++ b/sys/syz-extract/extract.go
@@ -101,28 +101,14 @@ func main() {
 	}
 
 	for p := 0; p < runtime.GOMAXPROCS(0); p++ {
-		go func() {
-			for job := range jobC {
-				switch j := job.(type) {
-				case *Arch:
-					infos, err := processArch(extractor, j)
-					j.err = err
-					close(j.done)
-					if j.err == nil {
-						for _, f := range j.files {
-							f.info = infos[filepath.Join("sys", j.target.OS, f.name)]
-							jobC <- f
-						}
-					}
-				case *File:
-					j.consts, j.undeclared, j.err = processFile(extractor, j.arch, j)
-					close(j.done)
-				}
-			}
-		}()
+		go worker(extractor, jobC)
 	}
 
 	failed := false
+	constFiles := make(map[string]*compiler.ConstFile)
+	for _, file := range files {
+		constFiles[file] = compiler.NewConstFile()
+	}
 	for _, arch := range arches {
 		fmt.Printf("generating %v/%v...\n", arch.target.OS, arch.target.Arch)
 		<-arch.done
@@ -138,6 +124,18 @@ func main() {
 				fmt.Printf("%v: %v\n", f.name, f.err)
 				continue
 			}
+			constFiles[f.name].AddArch(f.arch.target.Arch, f.consts, f.undeclared)
+		}
+	}
+	for file, cf := range constFiles {
+		outname := filepath.Join("sys", OS, file+".const")
+		data := cf.Serialize()
+		if len(data) == 0 {
+			os.Remove(outname)
+			continue
+		}
+		if err := osutil.WriteFile(outname, data); err != nil {
+			failf("failed to write output file: %v", err)
 		}
 	}
 
@@ -154,6 +152,26 @@ func main() {
 	}
 }
 
+func worker(extractor Extractor, jobC chan interface{}) {
+	for job := range jobC {
+		switch j := job.(type) {
+		case *Arch:
+			infos, err := processArch(extractor, j)
+			j.err = err
+			close(j.done)
+			if j.err == nil {
+				for _, f := range j.files {
+					f.info = infos[filepath.Join("sys", j.target.OS, f.name)]
+					jobC <- f
+				}
+			}
+		case *File:
+			j.consts, j.undeclared, j.err = processFile(extractor, j.arch, j)
+			close(j.done)
+		}
+	}
+}
+
 func createArches(OS string, archArray, files []string) ([]*Arch, error) {
 	var arches []*Arch
 	for _, archStr := range archArray {
@@ -291,21 +309,11 @@ func processArch(extractor Extractor, arch *Arch) (map[string]*compiler.ConstInf
 
 func processFile(extractor Extractor, arch *Arch, file *File) (map[string]uint64, map[string]bool, error) {
 	inname := filepath.Join("sys", arch.target.OS, file.name)
-	outname := strings.TrimSuffix(inname, ".txt") + "_" + arch.target.Arch + ".const"
 	if file.info == nil {
 		return nil, nil, fmt.Errorf("const info for input file %v is missing", inname)
 	}
 	if len(file.info.Consts) == 0 {
-		os.Remove(outname)
 		return nil, nil, nil
 	}
-	consts, undeclared, err := extractor.processFile(arch, file.info)
-	if err != nil {
-		return nil, nil, err
-	}
-	data := compiler.SerializeConsts(consts, undeclared)
-	if err := osutil.WriteFile(outname, data); err != nil {
-		return nil, nil, fmt.Errorf("failed to write output file: %v", err)
-	}
-	return consts, undeclared, nil
+	return extractor.processFile(arch, file.info)
 }