sys/syz-sysgen: serialize descriptions as gob and embed

Instead of generating Go files with descriptions
serialize them as gob and compress with flate.

This significantly reduces build time, go vet time,
and solves scalability problems with some static analysis tools.

Reference times (all after rm -rf ~/.cache/go-build) before:

TIME="%e %P %M" time go install ./syz-manager
48.29 577% 4824820

TIME="%e %P %M" time go test -c ./prog
56.28 380% 6973292

After:

TIME="%e %P %M" time go install ./syz-manager
22.81 865% 859788

TIME="%e %P %M" time go test -c ./prog
12.74 565% 267760

syz-manager size before/after: 194712597 -> 83418407
-57% even provided we now embed all descriptions
instead of just a single arch.

Deflate/decoding time for a single Linux arch is ~330ms.

Fixes #5542

1 files changed, 98 insertions, 103 deletions

diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go
index 79475e819..a9b659a58 100644
--- a/sys/syz-sysgen/sysgen.go
+++ b/sys/syz-sysgen/sysgen.go
@@ -7,7 +7,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"io"
+	"go/format"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -20,9 +20,9 @@ import (
 	"github.com/google/syzkaller/pkg/compiler"
 	"github.com/google/syzkaller/pkg/hash"
 	"github.com/google/syzkaller/pkg/osutil"
-	"github.com/google/syzkaller/pkg/serializer"
 	"github.com/google/syzkaller/pkg/tool"
 	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/sys/generated"
 	"github.com/google/syzkaller/sys/targets"
 )
 
@@ -60,7 +60,8 @@ type CallPropDescription struct {
 	Name string
 }
 
-type ExecutorData struct {
+type TemplateData struct {
+	Notice    string
 	OSes      []OSData
 	CallAttrs []string
 	CallProps []CallPropDescription
@@ -72,13 +73,34 @@ var outDir = flag.String("out", "", "path to out dir")
 func main() {
 	defer tool.Init()()
 
+	// Cleanup old files in the case set of architectures has chnaged.
+	allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", generated.Glob()))
+	if err != nil {
+		tool.Failf("failed to glob: %v", err)
+	}
+	for _, file := range allFiles {
+		os.Remove(file)
+	}
+
+	// Also remove old generated files since they will break build.
+	// TODO: remove this after some time after 2025-01-23.
+	oldFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", "*", "gen", "*"))
+	if err != nil {
+		tool.Failf("failed to glob: %v", err)
+	}
+	for _, file := range oldFiles {
+		os.Remove(file)
+	}
+
 	var OSList []string
 	for OS := range targets.List {
 		OSList = append(OSList, OS)
 	}
 	sort.Strings(OSList)
 
-	data := &ExecutorData{}
+	data := &TemplateData{
+		Notice: "Automatically generated by syz-sysgen; DO NOT EDIT.",
+	}
 	for _, OS := range OSList {
 		descriptions := ast.ParseGlob(filepath.Join(*srcDir, "sys", OS, "*.txt"), nil)
 		if descriptions == nil {
@@ -88,19 +110,6 @@ func main() {
 		if constFile == nil {
 			os.Exit(1)
 		}
-		osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen"))
-
-		// Cleanup old files in the case set of architectures has chnaged.
-		allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", OS, "gen", "*.go"))
-		if err != nil {
-			tool.Failf("failed to glob: %v", err)
-		}
-		for _, file := range allFiles {
-			if strings.HasSuffix(file, "empty.go") {
-				continue
-			}
-			os.Remove(file)
-		}
 
 		var archs []string
 		for arch := range targets.List[OS] {
@@ -178,7 +187,13 @@ func main() {
 		})
 	})
 
-	writeExecutorSyscalls(data)
+	sort.Slice(data.OSes, func(i, j int) bool {
+		return data.OSes[i].GOOS < data.OSes[j].GOOS
+	})
+
+	writeTemplate(filepath.Join(*outDir, "sys", "register.go"), registerTempl, data)
+	writeTemplate(filepath.Join(*outDir, "executor", "defs.h"), defsTempl, data)
+	writeTemplate(filepath.Join(*outDir, "executor", "syscalls.h"), syscallsTempl, data)
 }
 
 type Job struct {
@@ -188,6 +203,7 @@ type Job struct {
 	Unsupported map[string]bool
 	ArchData    ArchData
 	ConstInfo   map[string]*compiler.ConstInfo
+	Revision    string
 }
 
 func processJob(job *Job, descriptions *ast.Description, constFile *compiler.ConstFile) {
@@ -208,22 +224,37 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con
 		job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg))
 	}
 	consts := constFile.Arch(job.Target.Arch)
-	prog := compiler.Compile(descriptions, consts, job.Target, eh)
-	if prog == nil {
+	constArr := make([]prog.ConstValue, 0, len(consts))
+	for name, val := range consts {
+		constArr = append(constArr, prog.ConstValue{Name: name, Value: val})
+	}
+	sort.Slice(constArr, func(i, j int) bool {
+		return constArr[i].Name < constArr[j].Name
+	})
+
+	prg := compiler.Compile(descriptions, consts, job.Target, eh)
+	if prg == nil {
 		return
 	}
-	for what := range prog.Unsupported {
+	for what := range prg.Unsupported {
 		job.Unsupported[what] = true
 	}
 
-	sysFile := filepath.Join(*outDir, "sys", job.Target.OS, "gen", job.Target.Arch+".go")
-	out := new(bytes.Buffer)
-	generate(job.Target, prog, consts, flags, out)
-	rev := hash.String(out.Bytes())
-	fmt.Fprintf(out, "const revision_%v = %q\n", job.Target.Arch, rev)
-	writeSource(sysFile, out.Bytes())
+	desc := &generated.Desc{
+		Syscalls:  prg.Syscalls,
+		Resources: prg.Resources,
+		Types:     prg.Types,
+		Consts:    constArr,
+		Flags:     flags,
+	}
+	data, err := generated.Serialize(desc)
+	if err != nil {
+		tool.Fail(err)
+	}
+	sysFile := filepath.Join(*outDir, "sys", generated.FileName(job.Target.OS, job.Target.Arch))
+	writeFile(sysFile, data)
 
-	job.ArchData = generateExecutorSyscalls(job.Target, prog.Syscalls, rev)
+	job.ArchData = generateExecutorSyscalls(job.Target, prg.Syscalls, hash.String(data))
 
 	// Don't print warnings, they are printed in syz-check.
 	job.Errors = nil
@@ -235,57 +266,6 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con
 	job.OK = len(job.Errors) == 0
 }
 
-func generate(target *targets.Target, prg *compiler.Prog, consts map[string]uint64, flags []prog.FlagDesc,
-	out io.Writer) {
-	tag := fmt.Sprintf("syz_target,syz_os_%v,syz_arch_%v", target.OS, target.Arch)
-	if target.VMArch != "" {
-		tag += fmt.Sprintf(" syz_target,syz_os_%v,syz_arch_%v", target.OS, target.VMArch)
-	}
-	fmt.Fprintf(out, "// AUTOGENERATED FILE\n")
-	fmt.Fprintf(out, "// +build !codeanalysis\n")
-	fmt.Fprintf(out, "// +build !syz_target %v\n\n", tag)
-	fmt.Fprintf(out, "package gen\n\n")
-	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/prog\"\n")
-	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/sys/%v\"\n\n", target.OS)
-
-	fmt.Fprintf(out, "func init() {\n")
-	fmt.Fprintf(out, "\tRegisterTarget(&Target{"+
-		"OS: %q, Arch: %q, Revision: revision_%v, PtrSize: %v, PageSize: %v, "+
-		"NumPages: %v, DataOffset: %v, BigEndian: %v, "+
-		"Syscalls: syscalls_%v, Resources: resources_%v, Consts: consts_%v,"+
-		"Flags: flags_%v}, types_%v, InitTarget)\n}\n\n",
-		target.OS, target.Arch, target.Arch, target.PtrSize, target.PageSize,
-		target.NumPages, target.DataOffset, target.BigEndian,
-		target.Arch, target.Arch, target.Arch, target.Arch, target.Arch)
-
-	fmt.Fprintf(out, "var resources_%v = ", target.Arch)
-	serializer.Write(out, prg.Resources)
-	fmt.Fprintf(out, "\n\n")
-
-	fmt.Fprintf(out, "var syscalls_%v = ", target.Arch)
-	serializer.Write(out, prg.Syscalls)
-	fmt.Fprintf(out, "\n\n")
-
-	fmt.Fprintf(out, "var types_%v = ", target.Arch)
-	serializer.Write(out, prg.Types)
-	fmt.Fprintf(out, "\n\n")
-
-	fmt.Fprintf(out, "var flags_%v = ", target.Arch)
-	serializer.Write(out, flags)
-	fmt.Fprintf(out, "\n\n")
-
-	constArr := make([]prog.ConstValue, 0, len(consts))
-	for name, val := range consts {
-		constArr = append(constArr, prog.ConstValue{Name: name, Value: val})
-	}
-	sort.Slice(constArr, func(i, j int) bool {
-		return constArr[i].Name < constArr[j].Name
-	})
-	fmt.Fprintf(out, "var consts_%v = ", target.Arch)
-	serializer.Write(out, constArr)
-	fmt.Fprintf(out, "\n\n")
-}
-
 func generateExecutorSyscalls(target *targets.Target, syscalls []*prog.Syscall, rev string) ArchData {
 	data := ArchData{
 		Revision:   rev,
@@ -360,40 +340,55 @@ func newSyscallData(target *targets.Target, sc *prog.Syscall, attrs []uint64) Sy
 	}
 }
 
-func writeExecutorSyscalls(data *ExecutorData) {
-	osutil.MkdirAll(filepath.Join(*outDir, "executor"))
-	sort.Slice(data.OSes, func(i, j int) bool {
-		return data.OSes[i].GOOS < data.OSes[j].GOOS
-	})
+func writeTemplate(file string, templ *template.Template, data any) {
 	buf := new(bytes.Buffer)
-	if err := defsTempl.Execute(buf, data); err != nil {
-		tool.Failf("failed to execute defs template: %v", err)
+	if err := templ.Execute(buf, data); err != nil {
+		tool.Failf("failed to execute template: %v", err)
 	}
-	writeFile(filepath.Join(*outDir, "executor", "defs.h"), buf.Bytes())
-	buf.Reset()
-	if err := syscallsTempl.Execute(buf, data); err != nil {
-		tool.Failf("failed to execute syscalls template: %v", err)
+	contents := buf.Bytes()
+	if strings.HasSuffix(file, ".go") {
+		var err error
+		contents, err = format.Source(contents)
+		if err != nil {
+			tool.Failf("failed to format generated source: %v", err)
+		}
 	}
-	writeFile(filepath.Join(*outDir, "executor", "syscalls.h"), buf.Bytes())
+	writeFile(file, contents)
 }
 
-func writeSource(file string, data []byte) {
-	if oldSrc, err := os.ReadFile(file); err == nil && bytes.Equal(data, oldSrc) {
+func writeFile(file string, data []byte) {
+	if current, err := os.ReadFile(file); err == nil && bytes.Equal(data, current) {
 		return
 	}
-	writeFile(file, data)
+	osutil.MkdirAll(filepath.Dir(file))
+	if err := osutil.WriteFile(file, data); err != nil {
+		tool.Failf("failed to write output file: %v", err)
+	}
 }
 
-func writeFile(file string, data []byte) {
-	outf, err := os.Create(file)
-	if err != nil {
-		tool.Failf("failed to create output file: %v", err)
-	}
-	defer outf.Close()
-	outf.Write(data)
+// nolint: lll
+var registerTempl = template.Must(template.New("register").Parse(`// {{.Notice}}
+
+package sys
+
+import (
+	"embed"
+
+	"github.com/google/syzkaller/sys/generated"
+	{{range $os := $.OSes}}
+	"github.com/google/syzkaller/sys/{{$os.GOOS}}"{{end}}
+)
+
+//go:embed gen/*.gob.flate
+var files embed.FS
+
+func init() {
+	{{range $os := $.OSes}}{{range $arch := $os.Archs}}generated.Register("{{$os.GOOS}}", "{{$arch.GOARCH}}", "{{$arch.Revision}}", {{$os.GOOS}}.InitTarget, files)
+	{{end}}{{end}}
 }
+`))
 
-var defsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE
+var defsTempl = template.Must(template.New("defs").Parse(`// {{.Notice}}
 
 struct call_attrs_t { {{range $attr := $.CallAttrs}}
 	uint64_t {{$attr}};{{end}}
@@ -428,7 +423,7 @@ struct call_props_t { {{range $attr := $.CallProps}}
 `))
 
 // nolint: lll
-var syscallsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE
+var syscallsTempl = template.Must(template.New("syscalls").Parse(`// {{.Notice}}
 // clang-format off
 {{range $os := $.OSes}}
 #if GOOS_{{$os.GOOS}}