prog: fix escaping of C strings

C's \xHH hex constants in strings accept any number of hex digits
(not just 2 or 4). So later non-hex escaped chars glue to the \x construct.
Use \OOO instead as it accepts at most 3 octal digits.

2 files changed, 16 insertions, 5 deletions

diff --git a/prog/encoding.go b/prog/encoding.go
index d7f25afe2..81c06313c 100644
--- a/prog/encoding.go
+++ b/prog/encoding.go
@@ -733,7 +733,7 @@ func serializeData(buf *bytes.Buffer, data []byte, readable bool) {
 		return
 	}
 	buf.WriteByte('\'')
-	encodeData(buf, data, true)
+	encodeData(buf, data, true, false)
 	buf.WriteByte('\'')
 }
 
@@ -741,10 +741,10 @@ func EncodeData(buf *bytes.Buffer, data []byte, readable bool) {
 	if !readable && isReadableData(data) {
 		readable = true
 	}
-	encodeData(buf, data, readable)
+	encodeData(buf, data, readable, true)
 }
 
-func encodeData(buf *bytes.Buffer, data []byte, readable bool) {
+func encodeData(buf *bytes.Buffer, data []byte, readable, cstr bool) {
 	for _, v := range data {
 		if !readable {
 			lo, hi := byteToHex(v)
@@ -776,8 +776,18 @@ func encodeData(buf *bytes.Buffer, data []byte, readable bool) {
 			if isPrintable(v) {
 				buf.WriteByte(v)
 			} else {
-				lo, hi := byteToHex(v)
-				buf.Write([]byte{'\\', 'x', hi, lo})
+				if cstr {
+					// We would like to use hex encoding with \x,
+					// but C's \x is hard to use: it can contain _any_ number of hex digits
+					// (not just 2 or 4), so later non-hex encoded chars will glue to \x.
+					c0 := (v>>6)&0x7 + '0'
+					c1 := (v>>3)&0x7 + '0'
+					c2 := (v>>0)&0x7 + '0'
+					buf.Write([]byte{'\\', c0, c1, c2})
+				} else {
+					lo, hi := byteToHex(v)
+					buf.Write([]byte{'\\', 'x', hi, lo})
+				}
 			}
 		}
 	}
diff --git a/sys/test/test/strings b/sys/test/test/strings
index 5b665ba5a..19af70fd7 100644
--- a/sys/test/test/strings
+++ b/sys/test/test/strings
@@ -1,3 +1,4 @@
 syz_compare(&AUTO="303100090a0d7022273a", 0xa, &AUTO=@str='01\x00\t\n\rp\"\':', AUTO)
 syz_compare(&AUTO="303100090a0d7022273a01", 0xb, &AUTO=@blob='01\x00\t\n\rp\"\':\x01', AUTO)
 syz_compare(&AUTO="303100090a0d7022273a0102", 0xc, &AUTO=@arr16be=[0x3031, 0x0009, 0x0a0d, 0x7022, 0x273a, 0x0102], AUTO)
+syz_compare(&AUTO="e91f6189591e9233614b00", 0xb, &AUTO=@str='\xe9\x1fa\x89Y\x1e\x923aK\x00', AUTO)