From 79910ad71b16dbf22e70717166c21361b5cf9bf0 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 23 Jan 2025 09:31:50 +0100 Subject: sys/syz-sysgen: serialize descriptions as gob and embed Instead of generating Go files with descriptions serialize them as gob and compress with flate. This significantly reduces build time, go vet time, and solves scalability problems with some static analysis tools. Reference times (all after rm -rf ~/.cache/go-build) before: TIME="%e %P %M" time go install ./syz-manager 48.29 577% 4824820 TIME="%e %P %M" time go test -c ./prog 56.28 380% 6973292 After: TIME="%e %P %M" time go install ./syz-manager 22.81 865% 859788 TIME="%e %P %M" time go test -c ./prog 12.74 565% 267760 syz-manager size before/after: 194712597 -> 83418407 -57% even provided we now embed all descriptions instead of just a single arch. Deflate/decoding time for a single Linux arch is ~330ms. Fixes #5542 --- sys/syz-sysgen/sysgen.go | 201 +++++++++++++++++++++++------------------------ 1 file changed, 98 insertions(+), 103 deletions(-) (limited to 'sys/syz-sysgen') diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go index 79475e819..a9b659a58 100644 --- a/sys/syz-sysgen/sysgen.go +++ b/sys/syz-sysgen/sysgen.go @@ -7,7 +7,7 @@ import ( "bytes" "flag" "fmt" - "io" + "go/format" "os" "path/filepath" "reflect" @@ -20,9 +20,9 @@ import ( "github.com/google/syzkaller/pkg/compiler" "github.com/google/syzkaller/pkg/hash" "github.com/google/syzkaller/pkg/osutil" - "github.com/google/syzkaller/pkg/serializer" "github.com/google/syzkaller/pkg/tool" "github.com/google/syzkaller/prog" + "github.com/google/syzkaller/sys/generated" "github.com/google/syzkaller/sys/targets" ) @@ -60,7 +60,8 @@ type CallPropDescription struct { Name string } -type ExecutorData struct { +type TemplateData struct { + Notice string OSes []OSData CallAttrs []string CallProps []CallPropDescription @@ -72,13 +73,34 @@ var outDir = flag.String("out", "", "path to out dir") func main() { defer tool.Init()() + // Cleanup old files in the case set of architectures has chnaged. + allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", generated.Glob())) + if err != nil { + tool.Failf("failed to glob: %v", err) + } + for _, file := range allFiles { + os.Remove(file) + } + + // Also remove old generated files since they will break build. + // TODO: remove this after some time after 2025-01-23. + oldFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", "*", "gen", "*")) + if err != nil { + tool.Failf("failed to glob: %v", err) + } + for _, file := range oldFiles { + os.Remove(file) + } + var OSList []string for OS := range targets.List { OSList = append(OSList, OS) } sort.Strings(OSList) - data := &ExecutorData{} + data := &TemplateData{ + Notice: "Automatically generated by syz-sysgen; DO NOT EDIT.", + } for _, OS := range OSList { descriptions := ast.ParseGlob(filepath.Join(*srcDir, "sys", OS, "*.txt"), nil) if descriptions == nil { @@ -88,19 +110,6 @@ func main() { if constFile == nil { os.Exit(1) } - osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen")) - - // Cleanup old files in the case set of architectures has chnaged. - allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", OS, "gen", "*.go")) - if err != nil { - tool.Failf("failed to glob: %v", err) - } - for _, file := range allFiles { - if strings.HasSuffix(file, "empty.go") { - continue - } - os.Remove(file) - } var archs []string for arch := range targets.List[OS] { @@ -178,7 +187,13 @@ func main() { }) }) - writeExecutorSyscalls(data) + sort.Slice(data.OSes, func(i, j int) bool { + return data.OSes[i].GOOS < data.OSes[j].GOOS + }) + + writeTemplate(filepath.Join(*outDir, "sys", "register.go"), registerTempl, data) + writeTemplate(filepath.Join(*outDir, "executor", "defs.h"), defsTempl, data) + writeTemplate(filepath.Join(*outDir, "executor", "syscalls.h"), syscallsTempl, data) } type Job struct { @@ -188,6 +203,7 @@ type Job struct { Unsupported map[string]bool ArchData ArchData ConstInfo map[string]*compiler.ConstInfo + Revision string } func processJob(job *Job, descriptions *ast.Description, constFile *compiler.ConstFile) { @@ -208,22 +224,37 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg)) } consts := constFile.Arch(job.Target.Arch) - prog := compiler.Compile(descriptions, consts, job.Target, eh) - if prog == nil { + constArr := make([]prog.ConstValue, 0, len(consts)) + for name, val := range consts { + constArr = append(constArr, prog.ConstValue{Name: name, Value: val}) + } + sort.Slice(constArr, func(i, j int) bool { + return constArr[i].Name < constArr[j].Name + }) + + prg := compiler.Compile(descriptions, consts, job.Target, eh) + if prg == nil { return } - for what := range prog.Unsupported { + for what := range prg.Unsupported { job.Unsupported[what] = true } - sysFile := filepath.Join(*outDir, "sys", job.Target.OS, "gen", job.Target.Arch+".go") - out := new(bytes.Buffer) - generate(job.Target, prog, consts, flags, out) - rev := hash.String(out.Bytes()) - fmt.Fprintf(out, "const revision_%v = %q\n", job.Target.Arch, rev) - writeSource(sysFile, out.Bytes()) + desc := &generated.Desc{ + Syscalls: prg.Syscalls, + Resources: prg.Resources, + Types: prg.Types, + Consts: constArr, + Flags: flags, + } + data, err := generated.Serialize(desc) + if err != nil { + tool.Fail(err) + } + sysFile := filepath.Join(*outDir, "sys", generated.FileName(job.Target.OS, job.Target.Arch)) + writeFile(sysFile, data) - job.ArchData = generateExecutorSyscalls(job.Target, prog.Syscalls, rev) + job.ArchData = generateExecutorSyscalls(job.Target, prg.Syscalls, hash.String(data)) // Don't print warnings, they are printed in syz-check. job.Errors = nil @@ -235,57 +266,6 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con job.OK = len(job.Errors) == 0 } -func generate(target *targets.Target, prg *compiler.Prog, consts map[string]uint64, flags []prog.FlagDesc, - out io.Writer) { - tag := fmt.Sprintf("syz_target,syz_os_%v,syz_arch_%v", target.OS, target.Arch) - if target.VMArch != "" { - tag += fmt.Sprintf(" syz_target,syz_os_%v,syz_arch_%v", target.OS, target.VMArch) - } - fmt.Fprintf(out, "// AUTOGENERATED FILE\n") - fmt.Fprintf(out, "// +build !codeanalysis\n") - fmt.Fprintf(out, "// +build !syz_target %v\n\n", tag) - fmt.Fprintf(out, "package gen\n\n") - fmt.Fprintf(out, "import . \"github.com/google/syzkaller/prog\"\n") - fmt.Fprintf(out, "import . \"github.com/google/syzkaller/sys/%v\"\n\n", target.OS) - - fmt.Fprintf(out, "func init() {\n") - fmt.Fprintf(out, "\tRegisterTarget(&Target{"+ - "OS: %q, Arch: %q, Revision: revision_%v, PtrSize: %v, PageSize: %v, "+ - "NumPages: %v, DataOffset: %v, BigEndian: %v, "+ - "Syscalls: syscalls_%v, Resources: resources_%v, Consts: consts_%v,"+ - "Flags: flags_%v}, types_%v, InitTarget)\n}\n\n", - target.OS, target.Arch, target.Arch, target.PtrSize, target.PageSize, - target.NumPages, target.DataOffset, target.BigEndian, - target.Arch, target.Arch, target.Arch, target.Arch, target.Arch) - - fmt.Fprintf(out, "var resources_%v = ", target.Arch) - serializer.Write(out, prg.Resources) - fmt.Fprintf(out, "\n\n") - - fmt.Fprintf(out, "var syscalls_%v = ", target.Arch) - serializer.Write(out, prg.Syscalls) - fmt.Fprintf(out, "\n\n") - - fmt.Fprintf(out, "var types_%v = ", target.Arch) - serializer.Write(out, prg.Types) - fmt.Fprintf(out, "\n\n") - - fmt.Fprintf(out, "var flags_%v = ", target.Arch) - serializer.Write(out, flags) - fmt.Fprintf(out, "\n\n") - - constArr := make([]prog.ConstValue, 0, len(consts)) - for name, val := range consts { - constArr = append(constArr, prog.ConstValue{Name: name, Value: val}) - } - sort.Slice(constArr, func(i, j int) bool { - return constArr[i].Name < constArr[j].Name - }) - fmt.Fprintf(out, "var consts_%v = ", target.Arch) - serializer.Write(out, constArr) - fmt.Fprintf(out, "\n\n") -} - func generateExecutorSyscalls(target *targets.Target, syscalls []*prog.Syscall, rev string) ArchData { data := ArchData{ Revision: rev, @@ -360,40 +340,55 @@ func newSyscallData(target *targets.Target, sc *prog.Syscall, attrs []uint64) Sy } } -func writeExecutorSyscalls(data *ExecutorData) { - osutil.MkdirAll(filepath.Join(*outDir, "executor")) - sort.Slice(data.OSes, func(i, j int) bool { - return data.OSes[i].GOOS < data.OSes[j].GOOS - }) +func writeTemplate(file string, templ *template.Template, data any) { buf := new(bytes.Buffer) - if err := defsTempl.Execute(buf, data); err != nil { - tool.Failf("failed to execute defs template: %v", err) + if err := templ.Execute(buf, data); err != nil { + tool.Failf("failed to execute template: %v", err) } - writeFile(filepath.Join(*outDir, "executor", "defs.h"), buf.Bytes()) - buf.Reset() - if err := syscallsTempl.Execute(buf, data); err != nil { - tool.Failf("failed to execute syscalls template: %v", err) + contents := buf.Bytes() + if strings.HasSuffix(file, ".go") { + var err error + contents, err = format.Source(contents) + if err != nil { + tool.Failf("failed to format generated source: %v", err) + } } - writeFile(filepath.Join(*outDir, "executor", "syscalls.h"), buf.Bytes()) + writeFile(file, contents) } -func writeSource(file string, data []byte) { - if oldSrc, err := os.ReadFile(file); err == nil && bytes.Equal(data, oldSrc) { +func writeFile(file string, data []byte) { + if current, err := os.ReadFile(file); err == nil && bytes.Equal(data, current) { return } - writeFile(file, data) + osutil.MkdirAll(filepath.Dir(file)) + if err := osutil.WriteFile(file, data); err != nil { + tool.Failf("failed to write output file: %v", err) + } } -func writeFile(file string, data []byte) { - outf, err := os.Create(file) - if err != nil { - tool.Failf("failed to create output file: %v", err) - } - defer outf.Close() - outf.Write(data) +// nolint: lll +var registerTempl = template.Must(template.New("register").Parse(`// {{.Notice}} + +package sys + +import ( + "embed" + + "github.com/google/syzkaller/sys/generated" + {{range $os := $.OSes}} + "github.com/google/syzkaller/sys/{{$os.GOOS}}"{{end}} +) + +//go:embed gen/*.gob.flate +var files embed.FS + +func init() { + {{range $os := $.OSes}}{{range $arch := $os.Archs}}generated.Register("{{$os.GOOS}}", "{{$arch.GOARCH}}", "{{$arch.Revision}}", {{$os.GOOS}}.InitTarget, files) + {{end}}{{end}} } +`)) -var defsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE +var defsTempl = template.Must(template.New("defs").Parse(`// {{.Notice}} struct call_attrs_t { {{range $attr := $.CallAttrs}} uint64_t {{$attr}};{{end}} @@ -428,7 +423,7 @@ struct call_props_t { {{range $attr := $.CallProps}} `)) // nolint: lll -var syscallsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE +var syscallsTempl = template.Must(template.New("syscalls").Parse(`// {{.Notice}} // clang-format off {{range $os := $.OSes}} #if GOOS_{{$os.GOOS}} -- cgit mrf-deployment