From b6de93e603915b57a1eccadc8bd530efd00d28f2 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 10 Aug 2020 07:50:55 +0200 Subject: pkg/compiler: merge const files into a single file We now have 8 arches for Linux and .const files produce lots of noise in PRs and lots of diffs. If 3 .txt files are touched, the PR will have 24 .const files, which will be intermixed with .txt files. Frequently const values are equal across arches, and even if they don't spreading a single value across 8 files is inconvinient. Merge all 8 *_arch.const files into a single .const file. See the test for details of the new format. The old format is still parsed for now, we can't update all OSes at once. For Linux this reduces number of const files/lines from 1288/96599 to 158/11603. Fixes #1983 --- sys/syz-extract/extract.go | 68 ++++++++++++++++++++++----------------- sys/syz-sysgen/sysgen.go | 9 +++--- sys/test/test.txt.const | 6 ++++ sys/test/test_32_fork_shmem.const | 4 --- sys/test/test_32_shmem.const | 4 --- sys/test/test_64.const | 4 --- sys/test/test_64_fork.const | 3 -- 7 files changed, 49 insertions(+), 49 deletions(-) create mode 100644 sys/test/test.txt.const delete mode 100644 sys/test/test_32_fork_shmem.const delete mode 100644 sys/test/test_32_shmem.const delete mode 100644 sys/test/test_64.const delete mode 100644 sys/test/test_64_fork.const (limited to 'sys') diff --git a/sys/syz-extract/extract.go b/sys/syz-extract/extract.go index 22cd44402..10b8b6541 100644 --- a/sys/syz-extract/extract.go +++ b/sys/syz-extract/extract.go @@ -101,28 +101,14 @@ func main() { } for p := 0; p < runtime.GOMAXPROCS(0); p++ { - go func() { - for job := range jobC { - switch j := job.(type) { - case *Arch: - infos, err := processArch(extractor, j) - j.err = err - close(j.done) - if j.err == nil { - for _, f := range j.files { - f.info = infos[filepath.Join("sys", j.target.OS, f.name)] - jobC <- f - } - } - case *File: - j.consts, j.undeclared, j.err = processFile(extractor, j.arch, j) - close(j.done) - } - } - }() + go worker(extractor, jobC) } failed := false + constFiles := make(map[string]*compiler.ConstFile) + for _, file := range files { + constFiles[file] = compiler.NewConstFile() + } for _, arch := range arches { fmt.Printf("generating %v/%v...\n", arch.target.OS, arch.target.Arch) <-arch.done @@ -138,6 +124,18 @@ func main() { fmt.Printf("%v: %v\n", f.name, f.err) continue } + constFiles[f.name].AddArch(f.arch.target.Arch, f.consts, f.undeclared) + } + } + for file, cf := range constFiles { + outname := filepath.Join("sys", OS, file+".const") + data := cf.Serialize() + if len(data) == 0 { + os.Remove(outname) + continue + } + if err := osutil.WriteFile(outname, data); err != nil { + failf("failed to write output file: %v", err) } } @@ -154,6 +152,26 @@ func main() { } } +func worker(extractor Extractor, jobC chan interface{}) { + for job := range jobC { + switch j := job.(type) { + case *Arch: + infos, err := processArch(extractor, j) + j.err = err + close(j.done) + if j.err == nil { + for _, f := range j.files { + f.info = infos[filepath.Join("sys", j.target.OS, f.name)] + jobC <- f + } + } + case *File: + j.consts, j.undeclared, j.err = processFile(extractor, j.arch, j) + close(j.done) + } + } +} + func createArches(OS string, archArray, files []string) ([]*Arch, error) { var arches []*Arch for _, archStr := range archArray { @@ -291,21 +309,11 @@ func processArch(extractor Extractor, arch *Arch) (map[string]*compiler.ConstInf func processFile(extractor Extractor, arch *Arch, file *File) (map[string]uint64, map[string]bool, error) { inname := filepath.Join("sys", arch.target.OS, file.name) - outname := strings.TrimSuffix(inname, ".txt") + "_" + arch.target.Arch + ".const" if file.info == nil { return nil, nil, fmt.Errorf("const info for input file %v is missing", inname) } if len(file.info.Consts) == 0 { - os.Remove(outname) return nil, nil, nil } - consts, undeclared, err := extractor.processFile(arch, file.info) - if err != nil { - return nil, nil, err - } - data := compiler.SerializeConsts(consts, undeclared) - if err := osutil.WriteFile(outname, data); err != nil { - return nil, nil, fmt.Errorf("failed to write output file: %v", err) - } - return consts, undeclared, nil + return extractor.processFile(arch, file.info) } diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go index 5fc2be7b7..8dfedb6a2 100644 --- a/sys/syz-sysgen/sysgen.go +++ b/sys/syz-sysgen/sysgen.go @@ -75,6 +75,10 @@ func main() { if descriptions == nil { os.Exit(1) } + constFile := compiler.DeserializeConstFile(filepath.Join(*srcDir, "sys", OS, "*.const"), nil) + if constFile == nil { + os.Exit(1) + } osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen")) var archs []string @@ -109,10 +113,7 @@ func main() { eh := func(pos ast.Pos, msg string) { job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg)) } - consts := compiler.DeserializeConstsGlob(filepath.Join(*srcDir, "sys", OS, "*_"+job.Target.Arch+".const"), eh) - if consts == nil { - return - } + consts := constFile.Arch(job.Target.Arch) top := descriptions if OS == "linux" && (job.Target.Arch == "arm" || job.Target.Arch == "riscv64") { // Hack: KVM is not supported on ARM anymore. On riscv64 it diff --git a/sys/test/test.txt.const b/sys/test/test.txt.const new file mode 100644 index 000000000..11f548df0 --- /dev/null +++ b/sys/test/test.txt.const @@ -0,0 +1,6 @@ +arches = 32_fork_shmem, 32_shmem, 64, 64_fork +IPPROTO_ICMPV6 = 58 +IPPROTO_TCP = 6 +IPPROTO_UDP = 17 +ONLY_32BITS_CONST = 32_fork_shmem:1, 32_shmem:1 +ARCH_64_SPECIFIC_CONST = 64:10 diff --git a/sys/test/test_32_fork_shmem.const b/sys/test/test_32_fork_shmem.const deleted file mode 100644 index 938b68f89..000000000 --- a/sys/test/test_32_fork_shmem.const +++ /dev/null @@ -1,4 +0,0 @@ -IPPROTO_ICMPV6 = 58 -IPPROTO_TCP = 6 -IPPROTO_UDP = 17 -ONLY_32BITS_CONST = 1 diff --git a/sys/test/test_32_shmem.const b/sys/test/test_32_shmem.const deleted file mode 100644 index 938b68f89..000000000 --- a/sys/test/test_32_shmem.const +++ /dev/null @@ -1,4 +0,0 @@ -IPPROTO_ICMPV6 = 58 -IPPROTO_TCP = 6 -IPPROTO_UDP = 17 -ONLY_32BITS_CONST = 1 diff --git a/sys/test/test_64.const b/sys/test/test_64.const deleted file mode 100644 index 8b4c45cca..000000000 --- a/sys/test/test_64.const +++ /dev/null @@ -1,4 +0,0 @@ -IPPROTO_ICMPV6 = 58 -IPPROTO_TCP = 6 -IPPROTO_UDP = 17 -ARCH_64_SPECIFIC_CONST = 10 diff --git a/sys/test/test_64_fork.const b/sys/test/test_64_fork.const deleted file mode 100644 index 30db9e24f..000000000 --- a/sys/test/test_64_fork.const +++ /dev/null @@ -1,3 +0,0 @@ -IPPROTO_ICMPV6 = 58 -IPPROTO_TCP = 6 -IPPROTO_UDP = 17 -- cgit mrf-deployment