prog: don't require preallocated buffer for exec encoding

If we send exec encoding to the fuzzer, it's not necessary to serialize
exec encoding into existing buffer (currnetly we serialize directly into shmem).
So simplify code by serializing into a new slice.

9 files changed, 37 insertions, 61 deletions

diff --git a/pkg/csource/csource.go b/pkg/csource/csource.go
index 96237e64b..3d019a59c 100644
--- a/pkg/csource/csource.go
+++ b/pkg/csource/csource.go
@@ -239,12 +239,11 @@ func (ctx *context) generateSyscallDefines() string {
 }
 
 func (ctx *context) generateProgCalls(p *prog.Prog, trace bool) ([]string, []uint64, error) {
-	exec := make([]byte, prog.ExecBufferSize)
-	progSize, err := p.SerializeForExec(exec)
+	exec, err := p.SerializeForExec()
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to serialize program: %w", err)
 	}
-	decoded, err := ctx.target.DeserializeExec(exec[:progSize], nil)
+	decoded, err := ctx.target.DeserializeExec(exec, nil)
 	if err != nil {
 		return nil, nil, err
 	}
diff --git a/pkg/ipc/ipc.go b/pkg/ipc/ipc.go
index 536b494e7..ea2e22569 100644
--- a/pkg/ipc/ipc.go
+++ b/pkg/ipc/ipc.go
@@ -185,7 +185,6 @@ func MakeEnv(config *Config, pid int) (*Env, error) {
 			}
 		}()
 	} else {
-		inmem = make([]byte, prog.ExecBufferSize)
 		outmem = make([]byte, outputSize)
 	}
 	env := &Env{
@@ -254,15 +253,15 @@ func (env *Env) Close() error {
 // hanged: program hanged and was killed
 // err0: failed to start the process or bug in executor itself.
 func (env *Env) Exec(opts *ExecOpts, p *prog.Prog) (output []byte, info *ProgInfo, hanged bool, err0 error) {
-	// Copy-in serialized program.
-	progSize, err := p.SerializeForExec(env.in)
+	progData, err := p.SerializeForExec()
 	if err != nil {
 		err0 = err
 		return
 	}
-	var progData []byte
-	if !env.config.UseShmem {
-		progData = env.in[:progSize]
+	// Copy-in serialized program.
+	if env.config.UseShmem {
+		copy(env.in, progData)
+		progData = nil
 	}
 	// Zero out the first two words (ncmd and nsig), so that we don't have garbage there
 	// if executor crashes before writing non-garbage there.
diff --git a/prog/encoding_test.go b/prog/encoding_test.go
index e5d5a2d38..bf4f708d1 100644
--- a/prog/encoding_test.go
+++ b/prog/encoding_test.go
@@ -389,39 +389,36 @@ func TestSerializeDeserialize(t *testing.T) {
 func TestSerializeDeserializeRandom(t *testing.T) {
 	testEachTargetRandom(t, func(t *testing.T, target *Target, rs rand.Source, iters int) {
 		ct := target.DefaultChoiceTable()
-		data0 := make([]byte, ExecBufferSize)
-		data1 := make([]byte, ExecBufferSize)
 		for i := 0; i < iters; i++ {
 			p0 := target.Generate(rs, 10, ct)
-			if ok, _, _ := testSerializeDeserialize(t, p0, data0, data1); ok {
+			if _, _, ok := testSerializeDeserialize(t, p0); ok {
 				continue
 			}
 			p0, _ = Minimize(p0, -1, false, func(p1 *Prog, _ int) bool {
-				ok, _, _ := testSerializeDeserialize(t, p1, data0, data1)
+				_, _, ok := testSerializeDeserialize(t, p1)
 				return !ok
 			})
-			ok, n0, n1 := testSerializeDeserialize(t, p0, data0, data1)
+			data0, data1, ok := testSerializeDeserialize(t, p0)
 			if ok {
 				t.Log("flaky?")
 			}
-			decoded0, err := target.DeserializeExec(data0[:n0], nil)
+			decoded0, err := target.DeserializeExec(data0, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
-			decoded1, err := target.DeserializeExec(data1[:n1], nil)
+			decoded1, err := target.DeserializeExec(data1, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
 			t.Logf("decoded0: %+v", decoded0)
 			t.Logf("decoded1: %+v", decoded1)
-			t.Fatalf("was: %q\ngot: %q\nprogram:\n%s",
-				data0[:n0], data1[:n1], p0.Serialize())
+			t.Fatalf("was: %q\ngot: %q\nprogram:\n%s", data0, data1, p0.Serialize())
 		}
 	})
 }
 
-func testSerializeDeserialize(t *testing.T, p0 *Prog, data0, data1 []byte) (bool, int, int) {
-	n0, err := p0.SerializeForExec(data0)
+func testSerializeDeserialize(t *testing.T, p0 *Prog) ([]byte, []byte, bool) {
+	data0, err := p0.SerializeForExec()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -430,16 +427,16 @@ func testSerializeDeserialize(t *testing.T, p0 *Prog, data0, data1 []byte) (bool
 	if err != nil {
 		t.Fatal(err)
 	}
-	n1, err := p1.SerializeForExec(data1)
+	data1, err := p1.SerializeForExec()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(data0[:n0], data1[:n1]) {
+	if !bytes.Equal(data0, data1) {
 		t.Logf("PROG0:\n%s\n", p0.Serialize())
 		t.Logf("PROG1:\n%s\n", p1.Serialize())
-		return false, n0, n1
+		return data0, data1, false
 	}
-	return true, 0, 0
+	return nil, nil, true
 }
 
 func TestSerializeCallProps(t *testing.T) {
diff --git a/prog/encodingexec.go b/prog/encodingexec.go
index b90f6a6a1..a51aa54d7 100644
--- a/prog/encodingexec.go
+++ b/prog/encodingexec.go
@@ -68,12 +68,11 @@ var ErrExecBufferTooSmall = errors.New("encodingexec: provided buffer is too sma
 // SerializeForExec serializes program p for execution by process pid into the provided buffer.
 // Returns number of bytes written to the buffer.
 // If the provided buffer is too small for the program an error is returned.
-func (p *Prog) SerializeForExec(buffer []byte) (int, error) {
+func (p *Prog) SerializeForExec() ([]byte, error) {
 	p.debugValidate()
 	w := &execContext{
 		target: p.Target,
-		buf:    buffer,
-		eof:    false,
+		buf:    make([]byte, 0, 4<<10),
 		args:   make(map[Arg]argInfo),
 	}
 	for _, c := range p.Calls {
@@ -81,10 +80,10 @@ func (p *Prog) SerializeForExec(buffer []byte) (int, error) {
 		w.serializeCall(c)
 	}
 	w.write(execInstrEOF)
-	if w.eof || w.copyoutSeq > execMaxCommands {
-		return 0, ErrExecBufferTooSmall
+	if len(w.buf) > ExecBufferSize || w.copyoutSeq > execMaxCommands {
+		return nil, ErrExecBufferTooSmall
 	}
-	return len(buffer) - len(w.buf), nil
+	return w.buf, nil
 }
 
 func (w *execContext) serializeCall(c *Call) {
@@ -122,7 +121,6 @@ func (w *execContext) serializeCall(c *Call) {
 type execContext struct {
 	target     *Target
 	buf        []byte
-	eof        bool
 	args       map[Arg]argInfo
 	copyoutSeq uint64
 	// Per-call state cached here to not pass it through all functions.
@@ -252,12 +250,7 @@ func (w *execContext) writeCopyout(c *Call) {
 }
 
 func (w *execContext) write(v uint64) {
-	if len(w.buf) < 8 {
-		w.eof = true
-		return
-	}
-	n := binary.PutVarint(w.buf, int64(v))
-	w.buf = w.buf[n:]
+	w.buf = binary.AppendVarint(w.buf, int64(v))
 }
 
 func (w *execContext) writeArg(arg Arg) {
@@ -303,12 +296,7 @@ func (w *execContext) writeArg(arg Arg) {
 			flags |= execArgDataReadable
 		}
 		w.write(flags)
-		if len(w.buf) < len(data) {
-			w.eof = true
-		} else {
-			n := copy(w.buf, data)
-			w.buf = w.buf[n:]
-		}
+		w.buf = append(w.buf, data...)
 	case *UnionArg:
 		w.writeArg(a.Option)
 	default:
diff --git a/prog/encodingexec_test.go b/prog/encodingexec_test.go
index 68e42f07e..726cbe49b 100644
--- a/prog/encodingexec_test.go
+++ b/prog/encodingexec_test.go
@@ -16,23 +16,22 @@ import (
 func TestSerializeForExecRandom(t *testing.T) {
 	target, rs, iters := initTest(t)
 	ct := target.DefaultChoiceTable()
-	buf := make([]byte, ExecBufferSize)
 	execSizes := histogram.New(1000)
 	textSizes := histogram.New(1000)
 	totalSize := 0
 	sizes := make(map[string]int)
 	for i := 0; i < iters; i++ {
 		p := target.Generate(rs, 10, ct)
-		n, err := p.SerializeForExec(buf)
+		buf, err := p.SerializeForExec()
 		if err != nil {
 			t.Fatalf("failed to serialize: %v", err)
 		}
-		_, err = target.DeserializeExec(buf[:n], sizes)
+		_, err = target.DeserializeExec(buf, sizes)
 		if err != nil {
 			t.Fatal(err)
 		}
-		totalSize += n
-		execSizes.Add(float64(n))
+		totalSize += len(buf)
+		execSizes.Add(float64(len(buf)))
 		textSizes.Add(float64(len(p.Serialize())))
 	}
 	t.Logf("exec sizes: 10%%:%v 50%%:%v 90%%:%v",
@@ -650,7 +649,6 @@ test$res1(r0)
 		},
 	}
 
-	buf := make([]byte, ExecBufferSize)
 	for i, test := range tests {
 		i, test := i, test
 		t.Run(fmt.Sprintf("%v:%v", i, test.prog), func(t *testing.T) {
@@ -658,7 +656,7 @@ test$res1(r0)
 			if err != nil {
 				t.Fatalf("failed to deserialize prog %v: %v", i, err)
 			}
-			n, err := p.SerializeForExec(buf)
+			data, err := p.SerializeForExec()
 			if err != nil {
 				t.Fatalf("failed to serialize: %v", err)
 			}
@@ -675,7 +673,6 @@ test$res1(r0)
 					t.Fatalf("unexpected elem type %T %#v", e, e)
 				}
 			}
-			data := buf[:n]
 			if !bytes.Equal(data, want) {
 				t.Logf("want: %v", test.serialized)
 				t.Logf("got:  %q", data)
@@ -749,8 +746,7 @@ func TestSerializeForExecOverflow(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			buf := make([]byte, ExecBufferSize)
-			_, err = p.SerializeForExec(buf)
+			_, err = p.SerializeForExec()
 			if test.overflow && err != ErrExecBufferTooSmall {
 				t.Fatalf("want overflow but got %v", err)
 			}
diff --git a/prog/target.go b/prog/target.go
index be2db51c5..879cffb4f 100644
--- a/prog/target.go
+++ b/prog/target.go
@@ -361,7 +361,7 @@ func (pg *Builder) Finalize() (*Prog, error) {
 	if err := pg.p.validate(); err != nil {
 		return nil, err
 	}
-	if _, err := pg.p.SerializeForExec(make([]byte, ExecBufferSize)); err != nil {
+	if _, err := pg.p.SerializeForExec(); err != nil {
 		return nil, err
 	}
 	p := pg.p
diff --git a/prog/test/fuzz.go b/prog/test/fuzz.go
index 354d6c3cd..2ed32c737 100644
--- a/prog/test/fuzz.go
+++ b/prog/test/fuzz.go
@@ -48,8 +48,8 @@ func FuzzDeserialize(data []byte) int {
 	if !bytes.Equal(data0, p3.Serialize()) {
 		panic("got different data")
 	}
-	if n, err := p0.SerializeForExec(fuzzBuffer); err == nil {
-		if _, err := fuzzTarget.DeserializeExec(fuzzBuffer[:n], nil); err != nil {
+	if prodData, err := p0.SerializeForExec(); err == nil {
+		if _, err := fuzzTarget.DeserializeExec(prodData, nil); err != nil {
 			panic(err)
 		}
 	}
@@ -64,7 +64,6 @@ func FuzzParseLog(data []byte) int {
 	return 0
 }
 
-var fuzzBuffer = make([]byte, prog.ExecBufferSize)
 var fuzzTarget, fuzzChoiceTable = func() (*prog.Target, *prog.ChoiceTable) {
 	prog.Debug()
 	target, err := prog.GetTarget(targets.TestOS, targets.TestArch64)
diff --git a/prog/test_util.go b/prog/test_util.go
index 71a52c809..eb15539ea 100644
--- a/prog/test_util.go
+++ b/prog/test_util.go
@@ -27,7 +27,6 @@ type DeserializeTest struct {
 
 func TestDeserializeHelper(t *testing.T, OS, arch string, transform func(*Target, *Prog), tests []DeserializeTest) {
 	target := InitTargetTest(t, OS, arch)
-	buf := make([]byte, ExecBufferSize)
 	for testidx, test := range tests {
 		t.Run(fmt.Sprint(testidx), func(t *testing.T) {
 			if test.StrictErr == "" {
@@ -75,7 +74,7 @@ func TestDeserializeHelper(t *testing.T, OS, arch string, transform func(*Target
 					if want != output && want != outputVerbose {
 						t.Fatalf("wrong serialized data:\n%s\nexpect:\n%s\n", outputVerbose, want)
 					}
-					p.SerializeForExec(buf)
+					p.SerializeForExec()
 				}
 			}
 		})
diff --git a/tools/syz-imagegen/imagegen.go b/tools/syz-imagegen/imagegen.go
index d71e19d76..af64f77aa 100644
--- a/tools/syz-imagegen/imagegen.go
+++ b/tools/syz-imagegen/imagegen.go
@@ -825,8 +825,7 @@ func (img *Image) generateSize() error {
 	if err != nil {
 		return fmt.Errorf("failed to deserialize resulting program: %w", err)
 	}
-	exec := make([]byte, prog.ExecBufferSize)
-	if _, err := p.SerializeForExec(exec); err != nil {
+	if _, err := p.SerializeForExec(); err != nil {
 		return fmt.Errorf("failed to serialize for execution: %w", err)
 	}