sys/syz-sysgen: serialize descriptions as gob and embed

Instead of generating Go files with descriptions
serialize them as gob and compress with flate.

This significantly reduces build time, go vet time,
and solves scalability problems with some static analysis tools.

Reference times (all after rm -rf ~/.cache/go-build) before:

TIME="%e %P %M" time go install ./syz-manager
48.29 577% 4824820

TIME="%e %P %M" time go test -c ./prog
56.28 380% 6973292

After:

TIME="%e %P %M" time go install ./syz-manager
22.81 865% 859788

TIME="%e %P %M" time go test -c ./prog
12.74 565% 267760

syz-manager size before/after: 194712597 -> 83418407
-57% even provided we now embed all descriptions
instead of just a single arch.

Deflate/decoding time for a single Linux arch is ~330ms.

Fixes #5542

15 files changed, 208 insertions, 173 deletions

diff --git a/sys/darwin/gen/empty.go b/sys/empty.go
index 7884be456..53b1c25bf 100644
--- a/sys/darwin/gen/empty.go
+++ b/sys/empty.go
@@ -1,6 +1,6 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
+// Copyright 2025 syzkaller project authors. All rights reserved.
 // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 
-package gen
+package sys
 
 // Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/freebsd/gen/empty.go b/sys/freebsd/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/freebsd/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/fuchsia/gen/empty.go b/sys/fuchsia/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/fuchsia/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/generated/generated.go b/sys/generated/generated.go
new file mode 100644
index 000000000..320902931
--- /dev/null
+++ b/sys/generated/generated.go
@@ -0,0 +1,105 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package generated
+
+import (
+	"bytes"
+	"compress/flate"
+	"embed"
+	"encoding/gob"
+	"fmt"
+	"path/filepath"
+
+	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/sys/targets"
+)
+
+type Desc struct {
+	Syscalls  []*prog.Syscall
+	Resources []*prog.ResourceDesc
+	Consts    []prog.ConstValue
+	Flags     []prog.FlagDesc
+	Types     []prog.Type
+}
+
+func Register(os, arch, revision string, init func(*prog.Target), files embed.FS) {
+	sysTarget := targets.Get(os, arch)
+	target := &prog.Target{
+		OS:         os,
+		Arch:       arch,
+		Revision:   revision,
+		PtrSize:    sysTarget.PtrSize,
+		PageSize:   sysTarget.PageSize,
+		NumPages:   sysTarget.NumPages,
+		DataOffset: sysTarget.DataOffset,
+		BigEndian:  sysTarget.BigEndian,
+	}
+	filler := func(target *prog.Target) {
+		fill(target, files)
+	}
+	prog.RegisterTarget(target, filler, init)
+}
+
+func fill(target *prog.Target, files embed.FS) {
+	data, err := files.ReadFile(FileName(target.OS, target.Arch))
+	if err != nil {
+		panic(err)
+	}
+	desc := new(Desc)
+	if err := gob.NewDecoder(flate.NewReader(bytes.NewReader(data))).Decode(desc); err != nil {
+		panic(err)
+	}
+	target.Syscalls = desc.Syscalls
+	target.Resources = desc.Resources
+	target.Consts = desc.Consts
+	target.Flags = desc.Flags
+	target.Types = desc.Types
+}
+
+func Serialize(desc *Desc) ([]byte, error) {
+	out := new(bytes.Buffer)
+	compressor, err := flate.NewWriter(out, flate.DefaultCompression)
+	if err != nil {
+		return nil, err
+	}
+	enc := gob.NewEncoder(compressor)
+	if err := enc.Encode(desc); err != nil {
+		return nil, err
+	}
+	if err := compressor.Close(); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}
+
+func FileName(os, arch string) string {
+	return fileName(fmt.Sprintf("%v_%v", os, arch))
+}
+
+func Glob() string {
+	return fileName("*")
+}
+
+func fileName(name string) string {
+	return filepath.Join("gen", fmt.Sprintf("%v.gob.flate", name))
+}
+
+func init() {
+	gob.Register(prog.Ref(0))
+	gob.Register(&prog.ResourceType{})
+	gob.Register(&prog.ConstType{})
+	gob.Register(&prog.IntType{})
+	gob.Register(&prog.FlagsType{})
+	gob.Register(&prog.LenType{})
+	gob.Register(&prog.ProcType{})
+	gob.Register(&prog.CsumType{})
+	gob.Register(&prog.VmaType{})
+	gob.Register(&prog.BufferType{})
+	gob.Register(&prog.ArrayType{})
+	gob.Register(&prog.PtrType{})
+	gob.Register(&prog.StructType{})
+	gob.Register(&prog.UnionType{})
+	gob.Register(&prog.BinaryExpression{})
+	gob.Register(&prog.Value{})
+}
diff --git a/sys/linux/gen/empty.go b/sys/linux/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/linux/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/linux/init_test.go b/sys/linux/init_test.go
index 74642f6c9..230593105 100644
--- a/sys/linux/init_test.go
+++ b/sys/linux/init_test.go
@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/google/syzkaller/prog"
-	_ "github.com/google/syzkaller/sys/linux/gen"
+	_ "github.com/google/syzkaller/sys"
 	"github.com/google/syzkaller/sys/targets"
 )
 
diff --git a/sys/netbsd/gen/empty.go b/sys/netbsd/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/netbsd/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/netbsd/init_test.go b/sys/netbsd/init_test.go
index 1f0abfad3..ecb85956b 100644
--- a/sys/netbsd/init_test.go
+++ b/sys/netbsd/init_test.go
@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/google/syzkaller/prog"
-	_ "github.com/google/syzkaller/sys/netbsd/gen"
+	_ "github.com/google/syzkaller/sys"
 	"github.com/google/syzkaller/sys/targets"
 )
 
diff --git a/sys/openbsd/gen/empty.go b/sys/openbsd/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/openbsd/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/openbsd/init_test.go b/sys/openbsd/init_test.go
index c439de565..0c895f30e 100644
--- a/sys/openbsd/init_test.go
+++ b/sys/openbsd/init_test.go
@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/google/syzkaller/prog"
-	_ "github.com/google/syzkaller/sys/openbsd/gen"
+	_ "github.com/google/syzkaller/sys"
 	"github.com/google/syzkaller/sys/targets"
 )
 
diff --git a/sys/sys.go b/sys/sys.go
deleted file mode 100644
index 054358b04..000000000
--- a/sys/sys.go
+++ /dev/null
@@ -1,17 +0,0 @@
-// Copyright 2017 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package sys
-
-import (
-	// Import all targets, so that users only need to import sys.
-	_ "github.com/google/syzkaller/sys/darwin/gen"
-	_ "github.com/google/syzkaller/sys/freebsd/gen"
-	_ "github.com/google/syzkaller/sys/fuchsia/gen"
-	_ "github.com/google/syzkaller/sys/linux/gen"
-	_ "github.com/google/syzkaller/sys/netbsd/gen"
-	_ "github.com/google/syzkaller/sys/openbsd/gen"
-	_ "github.com/google/syzkaller/sys/test/gen"
-	_ "github.com/google/syzkaller/sys/trusty/gen"
-	_ "github.com/google/syzkaller/sys/windows/gen"
-)
diff --git a/sys/syz-sysgen/sysgen.go b/sys/syz-sysgen/sysgen.go
index 79475e819..a9b659a58 100644
--- a/sys/syz-sysgen/sysgen.go
+++ b/sys/syz-sysgen/sysgen.go
@@ -7,7 +7,7 @@ import (
 	"bytes"
 	"flag"
 	"fmt"
-	"io"
+	"go/format"
 	"os"
 	"path/filepath"
 	"reflect"
@@ -20,9 +20,9 @@ import (
 	"github.com/google/syzkaller/pkg/compiler"
 	"github.com/google/syzkaller/pkg/hash"
 	"github.com/google/syzkaller/pkg/osutil"
-	"github.com/google/syzkaller/pkg/serializer"
 	"github.com/google/syzkaller/pkg/tool"
 	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/sys/generated"
 	"github.com/google/syzkaller/sys/targets"
 )
 
@@ -60,7 +60,8 @@ type CallPropDescription struct {
 	Name string
 }
 
-type ExecutorData struct {
+type TemplateData struct {
+	Notice    string
 	OSes      []OSData
 	CallAttrs []string
 	CallProps []CallPropDescription
@@ -72,13 +73,34 @@ var outDir = flag.String("out", "", "path to out dir")
 func main() {
 	defer tool.Init()()
 
+	// Cleanup old files in the case set of architectures has chnaged.
+	allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", generated.Glob()))
+	if err != nil {
+		tool.Failf("failed to glob: %v", err)
+	}
+	for _, file := range allFiles {
+		os.Remove(file)
+	}
+
+	// Also remove old generated files since they will break build.
+	// TODO: remove this after some time after 2025-01-23.
+	oldFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", "*", "gen", "*"))
+	if err != nil {
+		tool.Failf("failed to glob: %v", err)
+	}
+	for _, file := range oldFiles {
+		os.Remove(file)
+	}
+
 	var OSList []string
 	for OS := range targets.List {
 		OSList = append(OSList, OS)
 	}
 	sort.Strings(OSList)
 
-	data := &ExecutorData{}
+	data := &TemplateData{
+		Notice: "Automatically generated by syz-sysgen; DO NOT EDIT.",
+	}
 	for _, OS := range OSList {
 		descriptions := ast.ParseGlob(filepath.Join(*srcDir, "sys", OS, "*.txt"), nil)
 		if descriptions == nil {
@@ -88,19 +110,6 @@ func main() {
 		if constFile == nil {
 			os.Exit(1)
 		}
-		osutil.MkdirAll(filepath.Join(*outDir, "sys", OS, "gen"))
-
-		// Cleanup old files in the case set of architectures has chnaged.
-		allFiles, err := filepath.Glob(filepath.Join(*outDir, "sys", OS, "gen", "*.go"))
-		if err != nil {
-			tool.Failf("failed to glob: %v", err)
-		}
-		for _, file := range allFiles {
-			if strings.HasSuffix(file, "empty.go") {
-				continue
-			}
-			os.Remove(file)
-		}
 
 		var archs []string
 		for arch := range targets.List[OS] {
@@ -178,7 +187,13 @@ func main() {
 		})
 	})
 
-	writeExecutorSyscalls(data)
+	sort.Slice(data.OSes, func(i, j int) bool {
+		return data.OSes[i].GOOS < data.OSes[j].GOOS
+	})
+
+	writeTemplate(filepath.Join(*outDir, "sys", "register.go"), registerTempl, data)
+	writeTemplate(filepath.Join(*outDir, "executor", "defs.h"), defsTempl, data)
+	writeTemplate(filepath.Join(*outDir, "executor", "syscalls.h"), syscallsTempl, data)
 }
 
 type Job struct {
@@ -188,6 +203,7 @@ type Job struct {
 	Unsupported map[string]bool
 	ArchData    ArchData
 	ConstInfo   map[string]*compiler.ConstInfo
+	Revision    string
 }
 
 func processJob(job *Job, descriptions *ast.Description, constFile *compiler.ConstFile) {
@@ -208,22 +224,37 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con
 		job.Errors = append(job.Errors, fmt.Sprintf("%v: %v\n", pos, msg))
 	}
 	consts := constFile.Arch(job.Target.Arch)
-	prog := compiler.Compile(descriptions, consts, job.Target, eh)
-	if prog == nil {
+	constArr := make([]prog.ConstValue, 0, len(consts))
+	for name, val := range consts {
+		constArr = append(constArr, prog.ConstValue{Name: name, Value: val})
+	}
+	sort.Slice(constArr, func(i, j int) bool {
+		return constArr[i].Name < constArr[j].Name
+	})
+
+	prg := compiler.Compile(descriptions, consts, job.Target, eh)
+	if prg == nil {
 		return
 	}
-	for what := range prog.Unsupported {
+	for what := range prg.Unsupported {
 		job.Unsupported[what] = true
 	}
 
-	sysFile := filepath.Join(*outDir, "sys", job.Target.OS, "gen", job.Target.Arch+".go")
-	out := new(bytes.Buffer)
-	generate(job.Target, prog, consts, flags, out)
-	rev := hash.String(out.Bytes())
-	fmt.Fprintf(out, "const revision_%v = %q\n", job.Target.Arch, rev)
-	writeSource(sysFile, out.Bytes())
+	desc := &generated.Desc{
+		Syscalls:  prg.Syscalls,
+		Resources: prg.Resources,
+		Types:     prg.Types,
+		Consts:    constArr,
+		Flags:     flags,
+	}
+	data, err := generated.Serialize(desc)
+	if err != nil {
+		tool.Fail(err)
+	}
+	sysFile := filepath.Join(*outDir, "sys", generated.FileName(job.Target.OS, job.Target.Arch))
+	writeFile(sysFile, data)
 
-	job.ArchData = generateExecutorSyscalls(job.Target, prog.Syscalls, rev)
+	job.ArchData = generateExecutorSyscalls(job.Target, prg.Syscalls, hash.String(data))
 
 	// Don't print warnings, they are printed in syz-check.
 	job.Errors = nil
@@ -235,57 +266,6 @@ func processJob(job *Job, descriptions *ast.Description, constFile *compiler.Con
 	job.OK = len(job.Errors) == 0
 }
 
-func generate(target *targets.Target, prg *compiler.Prog, consts map[string]uint64, flags []prog.FlagDesc,
-	out io.Writer) {
-	tag := fmt.Sprintf("syz_target,syz_os_%v,syz_arch_%v", target.OS, target.Arch)
-	if target.VMArch != "" {
-		tag += fmt.Sprintf(" syz_target,syz_os_%v,syz_arch_%v", target.OS, target.VMArch)
-	}
-	fmt.Fprintf(out, "// AUTOGENERATED FILE\n")
-	fmt.Fprintf(out, "// +build !codeanalysis\n")
-	fmt.Fprintf(out, "// +build !syz_target %v\n\n", tag)
-	fmt.Fprintf(out, "package gen\n\n")
-	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/prog\"\n")
-	fmt.Fprintf(out, "import . \"github.com/google/syzkaller/sys/%v\"\n\n", target.OS)
-
-	fmt.Fprintf(out, "func init() {\n")
-	fmt.Fprintf(out, "\tRegisterTarget(&Target{"+
-		"OS: %q, Arch: %q, Revision: revision_%v, PtrSize: %v, PageSize: %v, "+
-		"NumPages: %v, DataOffset: %v, BigEndian: %v, "+
-		"Syscalls: syscalls_%v, Resources: resources_%v, Consts: consts_%v,"+
-		"Flags: flags_%v}, types_%v, InitTarget)\n}\n\n",
-		target.OS, target.Arch, target.Arch, target.PtrSize, target.PageSize,
-		target.NumPages, target.DataOffset, target.BigEndian,
-		target.Arch, target.Arch, target.Arch, target.Arch, target.Arch)
-
-	fmt.Fprintf(out, "var resources_%v = ", target.Arch)
-	serializer.Write(out, prg.Resources)
-	fmt.Fprintf(out, "\n\n")
-
-	fmt.Fprintf(out, "var syscalls_%v = ", target.Arch)
-	serializer.Write(out, prg.Syscalls)
-	fmt.Fprintf(out, "\n\n")
-
-	fmt.Fprintf(out, "var types_%v = ", target.Arch)
-	serializer.Write(out, prg.Types)
-	fmt.Fprintf(out, "\n\n")
-
-	fmt.Fprintf(out, "var flags_%v = ", target.Arch)
-	serializer.Write(out, flags)
-	fmt.Fprintf(out, "\n\n")
-
-	constArr := make([]prog.ConstValue, 0, len(consts))
-	for name, val := range consts {
-		constArr = append(constArr, prog.ConstValue{Name: name, Value: val})
-	}
-	sort.Slice(constArr, func(i, j int) bool {
-		return constArr[i].Name < constArr[j].Name
-	})
-	fmt.Fprintf(out, "var consts_%v = ", target.Arch)
-	serializer.Write(out, constArr)
-	fmt.Fprintf(out, "\n\n")
-}
-
 func generateExecutorSyscalls(target *targets.Target, syscalls []*prog.Syscall, rev string) ArchData {
 	data := ArchData{
 		Revision:   rev,
@@ -360,40 +340,55 @@ func newSyscallData(target *targets.Target, sc *prog.Syscall, attrs []uint64) Sy
 	}
 }
 
-func writeExecutorSyscalls(data *ExecutorData) {
-	osutil.MkdirAll(filepath.Join(*outDir, "executor"))
-	sort.Slice(data.OSes, func(i, j int) bool {
-		return data.OSes[i].GOOS < data.OSes[j].GOOS
-	})
+func writeTemplate(file string, templ *template.Template, data any) {
 	buf := new(bytes.Buffer)
-	if err := defsTempl.Execute(buf, data); err != nil {
-		tool.Failf("failed to execute defs template: %v", err)
+	if err := templ.Execute(buf, data); err != nil {
+		tool.Failf("failed to execute template: %v", err)
 	}
-	writeFile(filepath.Join(*outDir, "executor", "defs.h"), buf.Bytes())
-	buf.Reset()
-	if err := syscallsTempl.Execute(buf, data); err != nil {
-		tool.Failf("failed to execute syscalls template: %v", err)
+	contents := buf.Bytes()
+	if strings.HasSuffix(file, ".go") {
+		var err error
+		contents, err = format.Source(contents)
+		if err != nil {
+			tool.Failf("failed to format generated source: %v", err)
+		}
 	}
-	writeFile(filepath.Join(*outDir, "executor", "syscalls.h"), buf.Bytes())
+	writeFile(file, contents)
 }
 
-func writeSource(file string, data []byte) {
-	if oldSrc, err := os.ReadFile(file); err == nil && bytes.Equal(data, oldSrc) {
+func writeFile(file string, data []byte) {
+	if current, err := os.ReadFile(file); err == nil && bytes.Equal(data, current) {
 		return
 	}
-	writeFile(file, data)
+	osutil.MkdirAll(filepath.Dir(file))
+	if err := osutil.WriteFile(file, data); err != nil {
+		tool.Failf("failed to write output file: %v", err)
+	}
 }
 
-func writeFile(file string, data []byte) {
-	outf, err := os.Create(file)
-	if err != nil {
-		tool.Failf("failed to create output file: %v", err)
-	}
-	defer outf.Close()
-	outf.Write(data)
+// nolint: lll
+var registerTempl = template.Must(template.New("register").Parse(`// {{.Notice}}
+
+package sys
+
+import (
+	"embed"
+
+	"github.com/google/syzkaller/sys/generated"
+	{{range $os := $.OSes}}
+	"github.com/google/syzkaller/sys/{{$os.GOOS}}"{{end}}
+)
+
+//go:embed gen/*.gob.flate
+var files embed.FS
+
+func init() {
+	{{range $os := $.OSes}}{{range $arch := $os.Archs}}generated.Register("{{$os.GOOS}}", "{{$arch.GOARCH}}", "{{$arch.Revision}}", {{$os.GOOS}}.InitTarget, files)
+	{{end}}{{end}}
 }
+`))
 
-var defsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE
+var defsTempl = template.Must(template.New("defs").Parse(`// {{.Notice}}
 
 struct call_attrs_t { {{range $attr := $.CallAttrs}}
 	uint64_t {{$attr}};{{end}}
@@ -428,7 +423,7 @@ struct call_props_t { {{range $attr := $.CallProps}}
 `))
 
 // nolint: lll
-var syscallsTempl = template.Must(template.New("").Parse(`// AUTOGENERATED FILE
+var syscallsTempl = template.Must(template.New("syscalls").Parse(`// {{.Notice}}
 // clang-format off
 {{range $os := $.OSes}}
 #if GOOS_{{$os.GOOS}}
diff --git a/sys/test/gen/empty.go b/sys/test/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/test/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/trusty/gen/empty.go b/sys/trusty/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/trusty/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.
diff --git a/sys/windows/gen/empty.go b/sys/windows/gen/empty.go
deleted file mode 100644
index 7884be456..000000000
--- a/sys/windows/gen/empty.go
+++ /dev/null
@@ -1,6 +0,0 @@
-// Copyright 2021 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package gen
-
-// Empty file to unbreak build while descriptions are not generated.