From 79910ad71b16dbf22e70717166c21361b5cf9bf0 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 23 Jan 2025 09:31:50 +0100 Subject: sys/syz-sysgen: serialize descriptions as gob and embed Instead of generating Go files with descriptions serialize them as gob and compress with flate. This significantly reduces build time, go vet time, and solves scalability problems with some static analysis tools. Reference times (all after rm -rf ~/.cache/go-build) before: TIME="%e %P %M" time go install ./syz-manager 48.29 577% 4824820 TIME="%e %P %M" time go test -c ./prog 56.28 380% 6973292 After: TIME="%e %P %M" time go install ./syz-manager 22.81 865% 859788 TIME="%e %P %M" time go test -c ./prog 12.74 565% 267760 syz-manager size before/after: 194712597 -> 83418407 -57% even provided we now embed all descriptions instead of just a single arch. Deflate/decoding time for a single Linux arch is ~330ms. Fixes #5542 --- sys/darwin/gen/empty.go | 6 -- sys/empty.go | 6 ++ sys/freebsd/gen/empty.go | 6 -- sys/fuchsia/gen/empty.go | 6 -- sys/generated/generated.go | 105 +++++++++++++++++++++++ sys/linux/gen/empty.go | 6 -- sys/linux/init_test.go | 2 +- sys/netbsd/gen/empty.go | 6 -- sys/netbsd/init_test.go | 2 +- sys/openbsd/gen/empty.go | 6 -- sys/openbsd/init_test.go | 2 +- sys/sys.go | 17 ---- sys/syz-sysgen/sysgen.go | 201 ++++++++++++++++++++++----------------------- sys/test/gen/empty.go | 6 -- sys/trusty/gen/empty.go | 6 -- sys/windows/gen/empty.go | 6 -- 16 files changed, 212 insertions(+), 177 deletions(-) delete mode 100644 sys/darwin/gen/empty.go create mode 100644 sys/empty.go delete mode 100644 sys/freebsd/gen/empty.go delete mode 100644 sys/fuchsia/gen/empty.go create mode 100644 sys/generated/generated.go delete mode 100644 sys/linux/gen/empty.go delete mode 100644 sys/netbsd/gen/empty.go delete mode 100644 sys/openbsd/gen/empty.go delete mode 100644 sys/sys.go delete mode 100644 sys/test/gen/empty.go delete mode 100644 sys/trusty/gen/empty.go delete mode 100644 sys/windows/gen/empty.go (limited to 'sys') diff --git a/sys/darwin/gen/empty.go b/sys/darwin/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/darwin/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/empty.go b/sys/empty.go new file mode 100644 index 000000000..53b1c25bf --- /dev/null +++ b/sys/empty.go @@ -0,0 +1,6 @@ +// Copyright 2025 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package sys + +// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/freebsd/gen/empty.go b/sys/freebsd/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/freebsd/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/fuchsia/gen/empty.go b/sys/fuchsia/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/fuchsia/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/generated/generated.go b/sys/generated/generated.go new file mode 100644 index 000000000..320902931 --- /dev/null +++ b/sys/generated/generated.go @@ -0,0 +1,105 @@ +// Copyright 2025 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package generated + +import ( + "bytes" + "compress/flate" + "embed" + "encoding/gob" + "fmt" + "path/filepath" + + "github.com/google/syzkaller/prog" + "github.com/google/syzkaller/sys/targets" +) + +type Desc struct { + Syscalls []*prog.Syscall + Resources []*prog.ResourceDesc + Consts []prog.ConstValue + Flags []prog.FlagDesc + Types []prog.Type +} + +func Register(os, arch, revision string, init func(*prog.Target), files embed.FS) { + sysTarget := targets.Get(os, arch) + target := &prog.Target{ + OS: os, + Arch: arch, + Revision: revision, + PtrSize: sysTarget.PtrSize, + PageSize: sysTarget.PageSize, + NumPages: sysTarget.NumPages, + DataOffset: sysTarget.DataOffset, + BigEndian: sysTarget.BigEndian, + } + filler := func(target *prog.Target) { + fill(target, files) + } + prog.RegisterTarget(target, filler, init) +} + +func fill(target *prog.Target, files embed.FS) { + data, err := files.ReadFile(FileName(target.OS, target.Arch)) + if err != nil { + panic(err) + } + desc := new(Desc) + if err := gob.NewDecoder(flate.NewReader(bytes.NewReader(data))).Decode(desc); err != nil { + panic(err) + } + target.Syscalls = desc.Syscalls + target.Resources = desc.Resources + target.Consts = desc.Consts + target.Flags = desc.Flags + target.Types = desc.Types +} + +func Serialize(desc *Desc) ([]byte, error) { + out := new(bytes.Buffer) + compressor, err := flate.NewWriter(out, flate.DefaultCompression) + if err != nil { + return nil, err + } + enc := gob.NewEncoder(compressor) + if err := enc.Encode(desc); err != nil { + return nil, err + } + if err := compressor.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +func FileName(os, arch string) string { + return fileName(fmt.Sprintf("%v_%v", os, arch)) +} + +func Glob() string { + return fileName("*") +} + +func fileName(name string) string { + return filepath.Join("gen", fmt.Sprintf("%v.gob.flate", name)) +} + +func init() { + gob.Register(prog.Ref(0)) + gob.Register(&prog.ResourceType{}) + gob.Register(&prog.ConstType{}) + gob.Register(&prog.IntType{}) + gob.Register(&prog.FlagsType{}) + gob.Register(&prog.LenType{}) + gob.Register(&prog.ProcType{}) + gob.Register(&prog.CsumType{}) + gob.Register(&prog.VmaType{}) + gob.Register(&prog.BufferType{}) + gob.Register(&prog.ArrayType{}) + gob.Register(&prog.PtrType{}) + gob.Register(&prog.StructType{}) + gob.Register(&prog.UnionType{}) + gob.Register(&prog.BinaryExpression{}) + gob.Register(&prog.Value{}) +} diff --git a/sys/linux/gen/empty.go b/sys/linux/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/linux/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/linux/init_test.go b/sys/linux/init_test.go index 74642f6c9..230593105 100644 --- a/sys/linux/init_test.go +++ b/sys/linux/init_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/google/syzkaller/prog" - _ "github.com/google/syzkaller/sys/linux/gen" + _ "github.com/google/syzkaller/sys" "github.com/google/syzkaller/sys/targets" ) diff --git a/sys/netbsd/gen/empty.go b/sys/netbsd/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/netbsd/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/netbsd/init_test.go b/sys/netbsd/init_test.go index 1f0abfad3..ecb85956b 100644 --- a/sys/netbsd/init_test.go +++ b/sys/netbsd/init_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/google/syzkaller/prog" - _ "github.com/google/syzkaller/sys/netbsd/gen" + _ "github.com/google/syzkaller/sys" "github.com/google/syzkaller/sys/targets" ) diff --git a/sys/openbsd/gen/empty.go b/sys/openbsd/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/openbsd/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/openbsd/init_test.go b/sys/openbsd/init_test.go index c439de565..0c895f30e 100644 --- a/sys/openbsd/init_test.go +++ b/sys/openbsd/init_test.go @@ -7,7 +7,7 @@ import ( "testing" "github.com/google/syzkaller/prog" - _ "github.com/google/syzkaller/sys/openbsd/gen" + _ "github.com/google/syzkaller/sys" "github.com/google/syzkaller/sys/targets" ) diff --git a/sys/sys.go b/sys/sys.go deleted file mode 100644 index 054358b04..000000000 --- a/sys/sys.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2017 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package sys - -import ( - // Import all targets, so that users only need to import sys. - _ "github.com/google/syzkaller/sys/darwin/gen" - _ "github.com/google/syzkaller/sys/freebsd/gen" - _ "github.com/google/syzkaller/sys/fuchsia/gen" - _ "github.com/google/syzkaller/sys/linux/gen" - _ "github.com/google/syzkaller/sys/netbsd/gen" - _ "github.com/google/syzkaller/sys/openbsd/gen" - _ "github.com/google/syzkaller/sys/test/gen" - _ "github.com/google/syzkaller/sys/trusty/gen" - _ "github.com/google/syzkaller/sys/windows/gen" -) diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go index 79475e819..a9b659a58 100644 --- a/sys/syz-sysgen/sysgen.go +++ b/sys/syz-sysgen/sysgen.go @@ -7,7 +7,7 @@ import ( "bytes" "flag" "fmt" - "io" + "go/format" "os" "path/filepath" "reflect" @@ -20,9 +20,9 @@ import ( "github.com/google/syzkaller/pkg/compiler" "github.com/google/syzkaller/pkg/hash" "github.com/google/syzkaller/pkg/osutil" - "github.com/google/syzkaller/pkg/serializer" "github.com/google/syzkaller/pkg/tool" "github.com/google/syzkaller/prog" + "github.com/google/syzkaller/sys/generated" "github.com/google/syzkaller/sys/targets" ) @@ -60,7 +60,8 @@ type CallPropDescription struct { Name string } -type ExecutorData struct { +type TemplateData struct { + Notice string OSes []OSData CallAttrs []string CallProps []CallPropDescription @@ -72,13 +73,34 @@ var outDir = flag.String("out", "", "path to out dir") func main() { defer tool.Init()() + // Cleanup old files in the case set of architectures has chnaged. + allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", generated.Glob())) + if err != nil { + tool.Failf("failed to glob: %v", err) + } + for _, file := range allFiles { + os.Remove(file) + } + + // Also remove old generated files since they will break build. + // TODO: remove this after some time after 2025-01-23. + oldFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", "*", "gen", "*")) + if err != nil { + tool.Failf("failed to glob: %v", err) + } + for _, file := range oldFiles { + os.Remove(file) + } + var OSList []string for OS := range targets.List { OSList = append(OSList, OS) } sort.Strings(OSList) - data := &ExecutorData{} + data := &TemplateData{ + Notice: "Automatically generated by syz-sysgen; DO NOT EDIT.", + } for _, OS := range OSList { descriptions := ast.ParseGlob(filepath.Join(*srcDir, "sys", OS, "*.txt"), nil) if descriptions == nil { @@ -88,19 +110,6 @@ func main() { if constFile == nil { os.Exit(1) } - osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen")) - - // Cleanup old files in the case set of architectures has chnaged. - allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", OS, "gen", "*.go")) - if err != nil { - tool.Failf("failed to glob: %v", err) - } - for _, file := range allFiles { - if strings.HasSuffix(file, "empty.go") { - continue - } - os.Remove(file) - } var archs []string for arch := range targets.List[OS] { @@ -178,7 +187,13 @@ func main() { }) }) - writeExecutorSyscalls(data) + sort.Slice(data.OSes, func(i, j int) bool { + return data.OSes[i].GOOS < data.OSes[j].GOOS + }) + + writeTemplate(filepath.Join(*outDir, "sys", "register.go"), registerTempl, data) + writeTemplate(filepath.Join(*outDir, "executor", "defs.h"), defsTempl, data) + writeTemplate(filepath.Join(*outDir, "executor", "syscalls.h"), syscallsTempl, data) } type Job struct { @@ -188,6 +203,7 @@ type Job struct { Unsupported map[string]bool ArchData ArchData ConstInfo map[string]*compiler.ConstInfo + Revision string } func processJob(job *Job, descriptions *ast.Description, constFile *compiler.ConstFile) { @@ -208,22 +224,37 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg)) } consts := constFile.Arch(job.Target.Arch) - prog := compiler.Compile(descriptions, consts, job.Target, eh) - if prog == nil { + constArr := make([]prog.ConstValue, 0, len(consts)) + for name, val := range consts { + constArr = append(constArr, prog.ConstValue{Name: name, Value: val}) + } + sort.Slice(constArr, func(i, j int) bool { + return constArr[i].Name < constArr[j].Name + }) + + prg := compiler.Compile(descriptions, consts, job.Target, eh) + if prg == nil { return } - for what := range prog.Unsupported { + for what := range prg.Unsupported { job.Unsupported[what] = true } - sysFile := filepath.Join(*outDir, "sys", job.Target.OS, "gen", job.Target.Arch+".go") - out := new(bytes.Buffer) - generate(job.Target, prog, consts, flags, out) - rev := hash.String(out.Bytes()) - fmt.Fprintf(out, "const revision_%v = %q\n", job.Target.Arch, rev) - writeSource(sysFile, out.Bytes()) + desc := &generated.Desc{ + Syscalls: prg.Syscalls, + Resources: prg.Resources, + Types: prg.Types, + Consts: constArr, + Flags: flags, + } + data, err := generated.Serialize(desc) + if err != nil { + tool.Fail(err) + } + sysFile := filepath.Join(*outDir, "sys", generated.FileName(job.Target.OS, job.Target.Arch)) + writeFile(sysFile, data) - job.ArchData = generateExecutorSyscalls(job.Target, prog.Syscalls, rev) + job.ArchData = generateExecutorSyscalls(job.Target, prg.Syscalls, hash.String(data)) // Don't print warnings, they are printed in syz-check. job.Errors = nil @@ -235,57 +266,6 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con job.OK = len(job.Errors) == 0 } -func generate(target *targets.Target, prg *compiler.Prog, consts map[string]uint64, flags []prog.FlagDesc, - out io.Writer) { - tag := fmt.Sprintf("syz_target,syz_os_%v,syz_arch_%v", target.OS, target.Arch) - if target.VMArch != "" { - tag += fmt.Sprintf(" syz_target,syz_os_%v,syz_arch_%v", target.OS, target.VMArch) - } - fmt.Fprintf(out, "// AUTOGENERATED FILE\n") - fmt.Fprintf(out, "// +build !codeanalysis\n") - fmt.Fprintf(out, "// +build !syz_target %v\n\n", tag) - fmt.Fprintf(out, "package gen\n\n") - fmt.Fprintf(out, "import . \"github.com/google/syzkaller/prog\"\n") - fmt.Fprintf(out, "import . \"github.com/google/syzkaller/sys/%v\"\n\n", target.OS) - - fmt.Fprintf(out, "func init() {\n") - fmt.Fprintf(out, "\tRegisterTarget(&Target{"+ - "OS: %q, Arch: %q, Revision: revision_%v, PtrSize: %v, PageSize: %v, "+ - "NumPages: %v, DataOffset: %v, BigEndian: %v, "+ - "Syscalls: syscalls_%v, Resources: resources_%v, Consts: consts_%v,"+ - "Flags: flags_%v}, types_%v, InitTarget)\n}\n\n", - target.OS, target.Arch, target.Arch, target.PtrSize, target.PageSize, - target.NumPages, target.DataOffset, target.BigEndian, - target.Arch, target.Arch, target.Arch, target.Arch, target.Arch) - - fmt.Fprintf(out, "var resources_%v = ", target.Arch) - serializer.Write(out, prg.Resources) - fmt.Fprintf(out, "\n\n") - - fmt.Fprintf(out, "var syscalls_%v = ", target.Arch) - serializer.Write(out, prg.Syscalls) - fmt.Fprintf(out, "\n\n") - - fmt.Fprintf(out, "var types_%v = ", target.Arch) - serializer.Write(out, prg.Types) - fmt.Fprintf(out, "\n\n") - - fmt.Fprintf(out, "var flags_%v = ", target.Arch) - serializer.Write(out, flags) - fmt.Fprintf(out, "\n\n") - - constArr := make([]prog.ConstValue, 0, len(consts)) - for name, val := range consts { - constArr = append(constArr, prog.ConstValue{Name: name, Value: val}) - } - sort.Slice(constArr, func(i, j int) bool { - return constArr[i].Name < constArr[j].Name - }) - fmt.Fprintf(out, "var consts_%v = ", target.Arch) - serializer.Write(out, constArr) - fmt.Fprintf(out, "\n\n") -} - func generateExecutorSyscalls(target *targets.Target, syscalls []*prog.Syscall, rev string) ArchData { data := ArchData{ Revision: rev, @@ -360,40 +340,55 @@ func newSyscallData(target *targets.Target, sc *prog.Syscall, attrs []uint64) Sy } } -func writeExecutorSyscalls(data *ExecutorData) { - osutil.MkdirAll(filepath.Join(*outDir, "executor")) - sort.Slice(data.OSes, func(i, j int) bool { - return data.OSes[i].GOOS < data.OSes[j].GOOS - }) +func writeTemplate(file string, templ *template.Template, data any) { buf := new(bytes.Buffer) - if err := defsTempl.Execute(buf, data); err != nil { - tool.Failf("failed to execute defs template: %v", err) + if err := templ.Execute(buf, data); err != nil { + tool.Failf("failed to execute template: %v", err) } - writeFile(filepath.Join(*outDir, "executor", "defs.h"), buf.Bytes()) - buf.Reset() - if err := syscallsTempl.Execute(buf, data); err != nil { - tool.Failf("failed to execute syscalls template: %v", err) + contents := buf.Bytes() + if strings.HasSuffix(file, ".go") { + var err error + contents, err = format.Source(contents) + if err != nil { + tool.Failf("failed to format generated source: %v", err) + } } - writeFile(filepath.Join(*outDir, "executor", "syscalls.h"), buf.Bytes()) + writeFile(file, contents) } -func writeSource(file string, data []byte) { - if oldSrc, err := os.ReadFile(file); err == nil && bytes.Equal(data, oldSrc) { +func writeFile(file string, data []byte) { + if current, err := os.ReadFile(file); err == nil && bytes.Equal(data, current) { return } - writeFile(file, data) + osutil.MkdirAll(filepath.Dir(file)) + if err := osutil.WriteFile(file, data); err != nil { + tool.Failf("failed to write output file: %v", err) + } } -func writeFile(file string, data []byte) { - outf, err := os.Create(file) - if err != nil { - tool.Failf("failed to create output file: %v", err) - } - defer outf.Close() - outf.Write(data) +// nolint: lll +var registerTempl = template.Must(template.New("register").Parse(`// {{.Notice}} + +package sys + +import ( + "embed" + + "github.com/google/syzkaller/sys/generated" + {{range $os := $.OSes}} + "github.com/google/syzkaller/sys/{{$os.GOOS}}"{{end}} +) + +//go:embed gen/*.gob.flate +var files embed.FS + +func init() { + {{range $os := $.OSes}}{{range $arch := $os.Archs}}generated.Register("{{$os.GOOS}}", "{{$arch.GOARCH}}", "{{$arch.Revision}}", {{$os.GOOS}}.InitTarget, files) + {{end}}{{end}} } +`)) -var defsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE +var defsTempl = template.Must(template.New("defs").Parse(`// {{.Notice}} struct call_attrs_t { {{range $attr := $.CallAttrs}} uint64_t {{$attr}};{{end}} @@ -428,7 +423,7 @@ struct call_props_t { {{range $attr := $.CallProps}} `)) // nolint: lll -var syscallsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE +var syscallsTempl = template.Must(template.New("syscalls").Parse(`// {{.Notice}} // clang-format off {{range $os := $.OSes}} #if GOOS_{{$os.GOOS}} diff --git a/sys/test/gen/empty.go b/sys/test/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/test/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/trusty/gen/empty.go b/sys/trusty/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/trusty/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. diff --git a/sys/windows/gen/empty.go b/sys/windows/gen/empty.go deleted file mode 100644 index 7884be456..000000000 --- a/sys/windows/gen/empty.go +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2021 syzkaller project authors. All rights reserved. -// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -package gen - -// Empty file to unbreak build while descriptions are not generated. -- cgit mrf-deployment