vm/isolated: add initial support for fuzzing chromebooks

(WIP PR)

Add support for StartupScript.
* Modify Config{} to contain PostRepairScript.
* Allow repair() to execute a startup_script after reboot. The contents
of this script execute on the DUT.

Add pstore support:
* Modify Config{} to contain Pstore.
* Modify Diagnose() to reboot the DUT and fetch pstore logs,
conditional on inst.cfg.Pstore.
* Add readPstoreContents().
* Allow clearing previous pstore logs upon Create() and after use inside
readPstoreContents().
* Fetching pstore crashlogs relies on reliably getting lost connection
on DUT reboot. Use "ServerAliveInterval=6 ServerAliveCountMax=5" ssh
options when running syz-fuzzer with Pstore support enabled.

Allow parsing pstore contents:
* Diagnose() now returns pstore contents.

Refactoring:
* Move out some reusable parts of repair() to waitRebootAndSSH().
* Have an early return inside repair() if inst.waitForSSH() fails.

3 files changed, 221 insertions, 34 deletions

diff --git a/docs/linux/setup_linux-host_isolated.md b/docs/linux/setup_linux-host_isolated.md
index ec434cd12..387ff1052 100644
--- a/docs/linux/setup_linux-host_isolated.md
+++ b/docs/linux/setup_linux-host_isolated.md
@@ -53,6 +53,17 @@ Host *
 Before fuzzing, connect to the machine and keep the connection open so all scp
 and ssh usage will reuse it.
 
+# Optional: Pstore support
+
+If the device under test (DUT) has Pstore support, it is possible to configure syzkaller to
+fetch crashlogs from /sys/fs/pstore. You can do this by setting `"pstore": true` within
+the `vm` section of the syzkaller configuration file.
+
+# Optional: Startup script
+
+To execute commands on the DUT before fuzzing (re-)starts,
+`startup_script` can be used.
+
 ## Syzkaller
 
 Build syzkaller as described [here](/docs/contributing.md).
@@ -71,6 +82,7 @@ Use the following config:
 	"type": "isolated",
 	"vm": {
 		"targets" : [ "10.0.0.1" ],
+		"pstore": false,
 		"target_dir" : "/home/user/tmp/syzkaller",
                 "target_reboot" : false
 	}
diff --git a/vm/isolated/isolated.go b/vm/isolated/isolated.go
index fa5b842fd..01528bb7b 100755
--- a/vm/isolated/isolated.go
+++ b/vm/isolated/isolated.go
@@ -4,6 +4,7 @@
 package isolated
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,16 +20,20 @@ import (
 	"github.com/google/syzkaller/vm/vmimpl"
 )
 
+const pstoreConsoleFile = "/sys/fs/pstore/console-ramoops-0"
+
 func init() {
 	vmimpl.Register("isolated", ctor, false)
 }
 
 type Config struct {
-	Host         string   `json:"host"`           // host ip addr
-	Targets      []string `json:"targets"`        // target machines: (hostname|ip)(:port)?
-	TargetDir    string   `json:"target_dir"`     // directory to copy/run on target
-	TargetReboot bool     `json:"target_reboot"`  // reboot target on repair
-	USBDevNums   []string `json:"usb_device_num"` // /sys/bus/usb/devices/
+	Host          string   `json:"host"`           // host ip addr
+	Targets       []string `json:"targets"`        // target machines: (hostname|ip)(:port)?
+	TargetDir     string   `json:"target_dir"`     // directory to copy/run on target
+	TargetReboot  bool     `json:"target_reboot"`  // reboot target on repair
+	USBDevNums    []string `json:"usb_device_num"` // /sys/bus/usb/devices/
+	StartupScript string   `json:"startup_script"` // script to execute after each startup
+	Pstore        bool     `json:"pstore"`         // use crashlogs from pstore
 }
 
 type Pool struct {
@@ -111,7 +116,7 @@ func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
 		}
 	}()
 	if err := inst.repair(); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("repair failed: %v", err)
 	}
 
 	// Remount to writable.
@@ -123,6 +128,11 @@ func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
 	// Remove temp files from previous runs.
 	inst.ssh("rm -rf '" + filepath.Join(inst.cfg.TargetDir, "*") + "'")
 
+	// Remove pstore files from previous runs.
+	if inst.cfg.Pstore {
+		inst.ssh(fmt.Sprintf("rm %v", pstoreConsoleFile))
+	}
+
 	closeInst = nil
 	return inst, nil
 }
@@ -189,39 +199,98 @@ func (inst *instance) ssh(command string) error {
 	return nil
 }
 
+func (inst *instance) waitRebootAndSSH(rebootTimeout int, sshTimeout time.Duration) error {
+	if err := inst.waitForReboot(rebootTimeout); err != nil {
+		log.Logf(2, "isolated: machine did not reboot")
+		return err
+	}
+	log.Logf(2, "isolated: rebooted wait for comeback")
+	if err := inst.waitForSSH(sshTimeout); err != nil {
+		log.Logf(2, "isolated: machine did not comeback")
+		return err
+	}
+	log.Logf(2, "isolated: reboot succeeded")
+	return nil
+}
+
+// Escapes double quotes(and nested double quote escapes). Ignores any other escapes.
+// Reference: https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html
+func escapeDoubleQuotes(inp string) string {
+	var ret strings.Builder
+	for pos := 0; pos < len(inp); pos++ {
+		// If inp[pos] is not a double quote or a backslash, just use
+		// as is.
+		if inp[pos] != '"' && inp[pos] != '\\' {
+			ret.WriteByte(inp[pos])
+			continue
+		}
+		// If it is a double quote, escape.
+		if inp[pos] == '"' {
+			ret.WriteString("\\\"")
+			continue
+		}
+		// If we detect a backslash, reescape only if what it's already escaping
+		// is a double-quotes.
+		temp := ""
+		j := pos
+		for ; j < len(inp); j++ {
+			if inp[j] == '\\' {
+				temp += string(inp[j])
+				continue
+			}
+			// If the escape corresponds to a double quotes, re-escape.
+			// Else, just use as is.
+			if inp[j] == '"' {
+				temp = temp + temp + "\\\""
+			} else {
+				temp += string(inp[j])
+			}
+			break
+		}
+		ret.WriteString(temp)
+		pos = j
+	}
+	return ret.String()
+}
+
 func (inst *instance) repair() error {
 	log.Logf(2, "isolated: trying to ssh")
-	if err := inst.waitForSSH(30 * time.Minute); err == nil {
-		if inst.cfg.TargetReboot {
-			if len(inst.cfg.USBDevNums) > 0 {
-				log.Logf(2, "isolated: trying to reboot by USB authorization")
-				usbAuth := fmt.Sprintf("%s%s%s", "/sys/bus/usb/devices/", inst.cfg.USBDevNums[inst.index], "/authorized")
-				if err := ioutil.WriteFile(usbAuth, []byte("0"), 0); err != nil {
-					log.Logf(2, "isolated: failed to turn off the device")
-					return err
-				}
-				if err := ioutil.WriteFile(usbAuth, []byte("1"), 0); err != nil {
-					log.Logf(2, "isolated: failed to turn on the device")
-					return err
-				}
-			} else {
-				log.Logf(2, "isolated: ssh succeeded, trying to reboot by ssh")
-				inst.ssh("reboot") // reboot will return an error, ignore it
+	if err := inst.waitForSSH(30 * time.Minute); err != nil {
+		log.Logf(2, "isolated: ssh failed")
+		return fmt.Errorf("SSH failed")
+	}
+	if inst.cfg.TargetReboot {
+		if len(inst.cfg.USBDevNums) > 0 {
+			log.Logf(2, "isolated: trying to reboot by USB authorization")
+			usbAuth := fmt.Sprintf("%s%s%s", "/sys/bus/usb/devices/", inst.cfg.USBDevNums[inst.index], "/authorized")
+			if err := ioutil.WriteFile(usbAuth, []byte("0"), 0); err != nil {
+				log.Logf(2, "isolated: failed to turn off the device")
+				return err
 			}
+			if err := ioutil.WriteFile(usbAuth, []byte("1"), 0); err != nil {
+				log.Logf(2, "isolated: failed to turn on the device")
+				return err
+			}
+		} else {
+			log.Logf(2, "isolated: ssh succeeded, trying to reboot by ssh")
+			inst.ssh("reboot") // reboot will return an error, ignore it
 		}
-		if err := inst.waitForReboot(5 * 60); err != nil {
-			log.Logf(2, "isolated: machine did not reboot")
-			return err
+	}
+	if err := inst.waitRebootAndSSH(5*60, 30*time.Minute); err != nil {
+		return fmt.Errorf("waitRebootAndSSH failed: %v", err)
+	}
+	if inst.cfg.StartupScript != "" {
+		log.Logf(2, "isolated: executing startup_script")
+		// Execute the contents of the StartupScript on the DUT.
+		contents, err := ioutil.ReadFile(inst.cfg.StartupScript)
+		if err != nil {
+			return fmt.Errorf("unable to read startup_script: %v", err)
 		}
-		log.Logf(2, "isolated: rebooted wait for comeback")
-		if err := inst.waitForSSH(30 * time.Minute); err != nil {
-			log.Logf(0, "isolated: machine did not comeback")
-			return err
+		c := string(contents)
+		if err := inst.ssh(fmt.Sprintf("bash -c \"%v\"", escapeDoubleQuotes(c))); err != nil {
+			return fmt.Errorf("failed to execute startup_script: %v", err)
 		}
-		log.Logf(2, "isolated: reboot succeeded")
-	} else {
-		log.Logf(2, "isolated: ssh failed")
-		return fmt.Errorf("SSH failed")
+		log.Logf(2, "isolated: done executing startup_script")
 	}
 	return nil
 }
@@ -304,6 +373,10 @@ func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command strin
 		proxy := fmt.Sprintf("%v:127.0.0.1:%v", inst.forwardPort, inst.forwardPort)
 		args = append(args, "-R", proxy)
 	}
+	if inst.cfg.Pstore {
+		args = append(args, "-o", "ServerAliveInterval=6")
+		args = append(args, "-o", "ServerAliveCountMax=5")
+	}
 	args = append(args, inst.sshUser+"@"+inst.targetAddr, "cd "+inst.cfg.TargetDir+" && exec "+command)
 	log.Logf(0, "running command: ssh %#v", args)
 	if inst.debug {
@@ -331,8 +404,37 @@ func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command strin
 	return vmimpl.Multiplex(cmd, merger, dmesg, timeout, stop, inst.closed, inst.debug)
 }
 
+func (inst *instance) readPstoreContents() ([]byte, error) {
+	log.Logf(0, "reading pstore contents")
+	args := append(vmimpl.SSHArgs(inst.debug, inst.sshKey, inst.targetPort),
+		inst.sshUser+"@"+inst.targetAddr, "cat "+pstoreConsoleFile+" && rm "+pstoreConsoleFile)
+	if inst.debug {
+		log.Logf(0, "running command: ssh %#v", args)
+	}
+	var stdout, stderr bytes.Buffer
+	cmd := osutil.Command("ssh", args...)
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		return nil, fmt.Errorf("unable to read pstore file: %v: %v", err, stderr.String())
+	}
+	return stdout.Bytes(), nil
+}
+
 func (inst *instance) Diagnose() ([]byte, bool) {
-	return nil, false
+	if !inst.cfg.Pstore {
+		return nil, false
+	}
+	log.Logf(2, "waiting for crashed DUT to come back up")
+	if err := inst.waitRebootAndSSH(5*60, 30*time.Minute); err != nil {
+		return []byte(fmt.Sprintf("unable to SSH into DUT after reboot: %v", err)), false
+	}
+	log.Logf(2, "reading contents of pstore")
+	contents, err := inst.readPstoreContents()
+	if err != nil {
+		return []byte(fmt.Sprintf("Diagnose failed: %v\n", err)), false
+	}
+	return contents, false
 }
 
 func splitTargetPort(addr string) (string, int, error) {
diff --git a/vm/isolated/isolated_test.go b/vm/isolated/isolated_test.go
new file mode 100644
index 000000000..74bbff9f1
--- /dev/null
+++ b/vm/isolated/isolated_test.go
@@ -0,0 +1,73 @@
+// Copyright 2020 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package isolated
+
+import "testing"
+
+func TestEscapeDoubleQuotes(t *testing.T) {
+	testcases := []struct {
+		inp      string
+		expected string
+	}{
+		// Input with no quoting returns the same string.
+		{
+			"",
+			"",
+		},
+		{
+			"adsf",
+			"adsf",
+		},
+		// Inputs with escaping of characters other that double
+		// quotes returns the same input.
+		{
+			"\\$\\`\\\\\n", // \$\`\\\n
+			"\\$\\`\\\\\n", // \$\`\\\n
+		},
+		// Input with double quote.
+		{
+			`"`,
+			`\"`,
+		},
+		// Input with already escaped double quote.
+		{
+			`\"`,
+			`\\\"`,
+		},
+		// Input with already escaped backtick and already
+		// double quote. Should only re-escape the
+		// double quote.
+		{
+			"\\`something\"",   // \`something"
+			"\\`something\\\"", // \`something\"
+		},
+		// Input with already escaped backtick and already
+		// escaped double quote. Should only re-escape the
+		// escaped double quote.
+		{
+			"\\`something\\\"",     // \`something\"
+			"\\`something\\\\\\\"", // \`something\\\"
+		},
+		{
+			`touch \
+    /tmp/OK
+touch '/tmp/OK2'
+touch "/tmp/OK3"
+touch /tmp/OK4
+bash -c "bash -c \"ls -al\""`,
+			`touch \
+    /tmp/OK
+touch '/tmp/OK2'
+touch \"/tmp/OK3\"
+touch /tmp/OK4
+bash -c \"bash -c \\\"ls -al\\\"\"`,
+		},
+	}
+	for i, tc := range testcases {
+		output := escapeDoubleQuotes(tc.inp)
+		if tc.expected != output {
+			t.Fatalf("%v: For input %v Expected escaped string %v got %v", i+1, tc.inp, tc.expected, output)
+		}
+	}
+}