From 79910ad71b16dbf22e70717166c21361b5cf9bf0 Mon Sep 17 00:00:00 2001
From: Dmitry Vyukov <dvyukov@google.com>
Date: Thu, 23 Jan 2025 09:31:50 +0100
Subject: sys/syz-sysgen: serialize descriptions as gob and embed

Instead of generating Go files with descriptions
serialize them as gob and compress with flate.

This significantly reduces build time, go vet time,
and solves scalability problems with some static analysis tools.

Reference times (all after rm -rf ~/.cache/go-build) before:

TIME="%e %P %M" time go install ./syz-manager
48.29 577% 4824820

TIME="%e %P %M" time go test -c ./prog
56.28 380% 6973292

After:

TIME="%e %P %M" time go install ./syz-manager
22.81 865% 859788

TIME="%e %P %M" time go test -c ./prog
12.74 565% 267760

syz-manager size before/after: 194712597 -> 83418407
-57% even provided we now embed all descriptions
instead of just a single arch.

Deflate/decoding time for a single Linux arch is ~330ms.

Fixes #5542
---
 sys/generated/generated.go | 105 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 sys/generated/generated.go

(limited to 'sys/generated')

diff --git a/sys/generated/generated.go b/sys/generated/generated.go
new file mode 100644
index 000000000..320902931
--- /dev/null
+++ b/sys/generated/generated.go
@@ -0,0 +1,105 @@
+// Copyright 2025 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package generated
+
+import (
+	"bytes"
+	"compress/flate"
+	"embed"
+	"encoding/gob"
+	"fmt"
+	"path/filepath"
+
+	"github.com/google/syzkaller/prog"
+	"github.com/google/syzkaller/sys/targets"
+)
+
+type Desc struct {
+	Syscalls  []*prog.Syscall
+	Resources []*prog.ResourceDesc
+	Consts    []prog.ConstValue
+	Flags     []prog.FlagDesc
+	Types     []prog.Type
+}
+
+func Register(os, arch, revision string, init func(*prog.Target), files embed.FS) {
+	sysTarget := targets.Get(os, arch)
+	target := &prog.Target{
+		OS:         os,
+		Arch:       arch,
+		Revision:   revision,
+		PtrSize:    sysTarget.PtrSize,
+		PageSize:   sysTarget.PageSize,
+		NumPages:   sysTarget.NumPages,
+		DataOffset: sysTarget.DataOffset,
+		BigEndian:  sysTarget.BigEndian,
+	}
+	filler := func(target *prog.Target) {
+		fill(target, files)
+	}
+	prog.RegisterTarget(target, filler, init)
+}
+
+func fill(target *prog.Target, files embed.FS) {
+	data, err := files.ReadFile(FileName(target.OS, target.Arch))
+	if err != nil {
+		panic(err)
+	}
+	desc := new(Desc)
+	if err := gob.NewDecoder(flate.NewReader(bytes.NewReader(data))).Decode(desc); err != nil {
+		panic(err)
+	}
+	target.Syscalls = desc.Syscalls
+	target.Resources = desc.Resources
+	target.Consts = desc.Consts
+	target.Flags = desc.Flags
+	target.Types = desc.Types
+}
+
+func Serialize(desc *Desc) ([]byte, error) {
+	out := new(bytes.Buffer)
+	compressor, err := flate.NewWriter(out, flate.DefaultCompression)
+	if err != nil {
+		return nil, err
+	}
+	enc := gob.NewEncoder(compressor)
+	if err := enc.Encode(desc); err != nil {
+		return nil, err
+	}
+	if err := compressor.Close(); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}
+
+func FileName(os, arch string) string {
+	return fileName(fmt.Sprintf("%v_%v", os, arch))
+}
+
+func Glob() string {
+	return fileName("*")
+}
+
+func fileName(name string) string {
+	return filepath.Join("gen", fmt.Sprintf("%v.gob.flate", name))
+}
+
+func init() {
+	gob.Register(prog.Ref(0))
+	gob.Register(&prog.ResourceType{})
+	gob.Register(&prog.ConstType{})
+	gob.Register(&prog.IntType{})
+	gob.Register(&prog.FlagsType{})
+	gob.Register(&prog.LenType{})
+	gob.Register(&prog.ProcType{})
+	gob.Register(&prog.CsumType{})
+	gob.Register(&prog.VmaType{})
+	gob.Register(&prog.BufferType{})
+	gob.Register(&prog.ArrayType{})
+	gob.Register(&prog.PtrType{})
+	gob.Register(&prog.StructType{})
+	gob.Register(&prog.UnionType{})
+	gob.Register(&prog.BinaryExpression{})
+	gob.Register(&prog.Value{})
+}
-- 
cgit mrf-deployment