prog: don't require preallocated buffer for exec encoding

If we send exec encoding to the fuzzer, it's not necessary to serialize
exec encoding into existing buffer (currnetly we serialize directly into shmem).
So simplify code by serializing into a new slice.

6 files changed, 29 insertions, 50 deletions

diff --git a/prog/encoding_test.go b/prog/encoding_test.go
index e5d5a2d38..bf4f708d1 100644
--- a/prog/encoding_test.go
+++ b/prog/encoding_test.go
@@ -389,39 +389,36 @@ func TestSerializeDeserialize(t *testing.T) {
 func TestSerializeDeserializeRandom(t *testing.T) {
 	testEachTargetRandom(t, func(t *testing.T, target *Target, rs rand.Source, iters int) {
 		ct := target.DefaultChoiceTable()
-		data0 := make([]byte, ExecBufferSize)
-		data1 := make([]byte, ExecBufferSize)
 		for i := 0; i < iters; i++ {
 			p0 := target.Generate(rs, 10, ct)
-			if ok, _, _ := testSerializeDeserialize(t, p0, data0, data1); ok {
+			if _, _, ok := testSerializeDeserialize(t, p0); ok {
 				continue
 			}
 			p0, _ = Minimize(p0, -1, false, func(p1 *Prog, _ int) bool {
-				ok, _, _ := testSerializeDeserialize(t, p1, data0, data1)
+				_, _, ok := testSerializeDeserialize(t, p1)
 				return !ok
 			})
-			ok, n0, n1 := testSerializeDeserialize(t, p0, data0, data1)
+			data0, data1, ok := testSerializeDeserialize(t, p0)
 			if ok {
 				t.Log("flaky?")
 			}
-			decoded0, err := target.DeserializeExec(data0[:n0], nil)
+			decoded0, err := target.DeserializeExec(data0, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
-			decoded1, err := target.DeserializeExec(data1[:n1], nil)
+			decoded1, err := target.DeserializeExec(data1, nil)
 			if err != nil {
 				t.Fatal(err)
 			}
 			t.Logf("decoded0: %+v", decoded0)
 			t.Logf("decoded1: %+v", decoded1)
-			t.Fatalf("was: %q\ngot: %q\nprogram:\n%s",
-				data0[:n0], data1[:n1], p0.Serialize())
+			t.Fatalf("was: %q\ngot: %q\nprogram:\n%s", data0, data1, p0.Serialize())
 		}
 	})
 }
 
-func testSerializeDeserialize(t *testing.T, p0 *Prog, data0, data1 []byte) (bool, int, int) {
-	n0, err := p0.SerializeForExec(data0)
+func testSerializeDeserialize(t *testing.T, p0 *Prog) ([]byte, []byte, bool) {
+	data0, err := p0.SerializeForExec()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -430,16 +427,16 @@ func testSerializeDeserialize(t *testing.T, p0 *Prog, data0, data1 []byte) (bool
 	if err != nil {
 		t.Fatal(err)
 	}
-	n1, err := p1.SerializeForExec(data1)
+	data1, err := p1.SerializeForExec()
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !bytes.Equal(data0[:n0], data1[:n1]) {
+	if !bytes.Equal(data0, data1) {
 		t.Logf("PROG0:\n%s\n", p0.Serialize())
 		t.Logf("PROG1:\n%s\n", p1.Serialize())
-		return false, n0, n1
+		return data0, data1, false
 	}
-	return true, 0, 0
+	return nil, nil, true
 }
 
 func TestSerializeCallProps(t *testing.T) {
diff --git a/prog/encodingexec.go b/prog/encodingexec.go
index b90f6a6a1..a51aa54d7 100644
--- a/prog/encodingexec.go
+++ b/prog/encodingexec.go
@@ -68,12 +68,11 @@ var ErrExecBufferTooSmall = errors.New("encodingexec: provided buffer is too sma
 // SerializeForExec serializes program p for execution by process pid into the provided buffer.
 // Returns number of bytes written to the buffer.
 // If the provided buffer is too small for the program an error is returned.
-func (p *Prog) SerializeForExec(buffer []byte) (int, error) {
+func (p *Prog) SerializeForExec() ([]byte, error) {
 	p.debugValidate()
 	w := &execContext{
 		target: p.Target,
-		buf:    buffer,
-		eof:    false,
+		buf:    make([]byte, 0, 4<<10),
 		args:   make(map[Arg]argInfo),
 	}
 	for _, c := range p.Calls {
@@ -81,10 +80,10 @@ func (p *Prog) SerializeForExec(buffer []byte) (int, error) {
 		w.serializeCall(c)
 	}
 	w.write(execInstrEOF)
-	if w.eof || w.copyoutSeq > execMaxCommands {
-		return 0, ErrExecBufferTooSmall
+	if len(w.buf) > ExecBufferSize || w.copyoutSeq > execMaxCommands {
+		return nil, ErrExecBufferTooSmall
 	}
-	return len(buffer) - len(w.buf), nil
+	return w.buf, nil
 }
 
 func (w *execContext) serializeCall(c *Call) {
@@ -122,7 +121,6 @@ func (w *execContext) serializeCall(c *Call) {
 type execContext struct {
 	target     *Target
 	buf        []byte
-	eof        bool
 	args       map[Arg]argInfo
 	copyoutSeq uint64
 	// Per-call state cached here to not pass it through all functions.
@@ -252,12 +250,7 @@ func (w *execContext) writeCopyout(c *Call) {
 }
 
 func (w *execContext) write(v uint64) {
-	if len(w.buf) < 8 {
-		w.eof = true
-		return
-	}
-	n := binary.PutVarint(w.buf, int64(v))
-	w.buf = w.buf[n:]
+	w.buf = binary.AppendVarint(w.buf, int64(v))
 }
 
 func (w *execContext) writeArg(arg Arg) {
@@ -303,12 +296,7 @@ func (w *execContext) writeArg(arg Arg) {
 			flags |= execArgDataReadable
 		}
 		w.write(flags)
-		if len(w.buf) < len(data) {
-			w.eof = true
-		} else {
-			n := copy(w.buf, data)
-			w.buf = w.buf[n:]
-		}
+		w.buf = append(w.buf, data...)
 	case *UnionArg:
 		w.writeArg(a.Option)
 	default:
diff --git a/prog/encodingexec_test.go b/prog/encodingexec_test.go
index 68e42f07e..726cbe49b 100644
--- a/prog/encodingexec_test.go
+++ b/prog/encodingexec_test.go
@@ -16,23 +16,22 @@ import (
 func TestSerializeForExecRandom(t *testing.T) {
 	target, rs, iters := initTest(t)
 	ct := target.DefaultChoiceTable()
-	buf := make([]byte, ExecBufferSize)
 	execSizes := histogram.New(1000)
 	textSizes := histogram.New(1000)
 	totalSize := 0
 	sizes := make(map[string]int)
 	for i := 0; i < iters; i++ {
 		p := target.Generate(rs, 10, ct)
-		n, err := p.SerializeForExec(buf)
+		buf, err := p.SerializeForExec()
 		if err != nil {
 			t.Fatalf("failed to serialize: %v", err)
 		}
-		_, err = target.DeserializeExec(buf[:n], sizes)
+		_, err = target.DeserializeExec(buf, sizes)
 		if err != nil {
 			t.Fatal(err)
 		}
-		totalSize += n
-		execSizes.Add(float64(n))
+		totalSize += len(buf)
+		execSizes.Add(float64(len(buf)))
 		textSizes.Add(float64(len(p.Serialize())))
 	}
 	t.Logf("exec sizes: 10%%:%v 50%%:%v 90%%:%v",
@@ -650,7 +649,6 @@ test$res1(r0)
 		},
 	}
 
-	buf := make([]byte, ExecBufferSize)
 	for i, test := range tests {
 		i, test := i, test
 		t.Run(fmt.Sprintf("%v:%v", i, test.prog), func(t *testing.T) {
@@ -658,7 +656,7 @@ test$res1(r0)
 			if err != nil {
 				t.Fatalf("failed to deserialize prog %v: %v", i, err)
 			}
-			n, err := p.SerializeForExec(buf)
+			data, err := p.SerializeForExec()
 			if err != nil {
 				t.Fatalf("failed to serialize: %v", err)
 			}
@@ -675,7 +673,6 @@ test$res1(r0)
 					t.Fatalf("unexpected elem type %T %#v", e, e)
 				}
 			}
-			data := buf[:n]
 			if !bytes.Equal(data, want) {
 				t.Logf("want: %v", test.serialized)
 				t.Logf("got:  %q", data)
@@ -749,8 +746,7 @@ func TestSerializeForExecOverflow(t *testing.T) {
 			if err != nil {
 				t.Fatal(err)
 			}
-			buf := make([]byte, ExecBufferSize)
-			_, err = p.SerializeForExec(buf)
+			_, err = p.SerializeForExec()
 			if test.overflow && err != ErrExecBufferTooSmall {
 				t.Fatalf("want overflow but got %v", err)
 			}
diff --git a/prog/target.go b/prog/target.go
index be2db51c5..879cffb4f 100644
--- a/prog/target.go
+++ b/prog/target.go
@@ -361,7 +361,7 @@ func (pg *Builder) Finalize() (*Prog, error) {
 	if err := pg.p.validate(); err != nil {
 		return nil, err
 	}
-	if _, err := pg.p.SerializeForExec(make([]byte, ExecBufferSize)); err != nil {
+	if _, err := pg.p.SerializeForExec(); err != nil {
 		return nil, err
 	}
 	p := pg.p
diff --git a/prog/test/fuzz.go b/prog/test/fuzz.go
index 354d6c3cd..2ed32c737 100644
--- a/prog/test/fuzz.go
+++ b/prog/test/fuzz.go
@@ -48,8 +48,8 @@ func FuzzDeserialize(data []byte) int {
 	if !bytes.Equal(data0, p3.Serialize()) {
 		panic("got different data")
 	}
-	if n, err := p0.SerializeForExec(fuzzBuffer); err == nil {
-		if _, err := fuzzTarget.DeserializeExec(fuzzBuffer[:n], nil); err != nil {
+	if prodData, err := p0.SerializeForExec(); err == nil {
+		if _, err := fuzzTarget.DeserializeExec(prodData, nil); err != nil {
 			panic(err)
 		}
 	}
@@ -64,7 +64,6 @@ func FuzzParseLog(data []byte) int {
 	return 0
 }
 
-var fuzzBuffer = make([]byte, prog.ExecBufferSize)
 var fuzzTarget, fuzzChoiceTable = func() (*prog.Target, *prog.ChoiceTable) {
 	prog.Debug()
 	target, err := prog.GetTarget(targets.TestOS, targets.TestArch64)
diff --git a/prog/test_util.go b/prog/test_util.go
index 71a52c809..eb15539ea 100644
--- a/prog/test_util.go
+++ b/prog/test_util.go
@@ -27,7 +27,6 @@ type DeserializeTest struct {
 
 func TestDeserializeHelper(t *testing.T, OS, arch string, transform func(*Target, *Prog), tests []DeserializeTest) {
 	target := InitTargetTest(t, OS, arch)
-	buf := make([]byte, ExecBufferSize)
 	for testidx, test := range tests {
 		t.Run(fmt.Sprint(testidx), func(t *testing.T) {
 			if test.StrictErr == "" {
@@ -75,7 +74,7 @@ func TestDeserializeHelper(t *testing.T, OS, arch string, transform func(*Target
 					if want != output && want != outputVerbose {
 						t.Fatalf("wrong serialized data:\n%s\nexpect:\n%s\n", outputVerbose, want)
 					}
-					p.SerializeForExec(buf)
+					p.SerializeForExec()
 				}
 			}
 		})