From a6e3ac3bf259067ffd6e50fe8e4a158f097c1da5 Mon Sep 17 00:00:00 2001 From: Radoslav Gerganov Date: Wed, 14 Oct 2020 16:46:20 +0300 Subject: vm/vmware: improve kernel log collection and VM management * Collecting kernel logs with dmesg over ssh doesn't work well and sometimes we miss call traces when a crash occurs. Getting the kernel log from a virtual serial port is much more effective. * Creating linked clone VMs is faster then full clone VMs but it requires snapshot management and this will bring more complexity to syzkaller. Keep it simple and create full clone VMs for now. * Use host-only networking because the VM gets its IP faster that way --- .../setup_ubuntu-host_vmware-vm_x86-64-kernel.md | 12 +++++- docs/linux/vmw-settings.png | Bin 0 -> 87824 bytes vm/vmware/vmware.go | 42 ++++++++------------- 3 files changed, 26 insertions(+), 28 deletions(-) create mode 100644 docs/linux/vmw-settings.png diff --git a/docs/linux/setup_ubuntu-host_vmware-vm_x86-64-kernel.md b/docs/linux/setup_ubuntu-host_vmware-vm_x86-64-kernel.md index 3ac29c0c0..f1f7d8db8 100644 --- a/docs/linux/setup_ubuntu-host_vmware-vm_x86-64-kernel.md +++ b/docs/linux/setup_ubuntu-host_vmware-vm_x86-64-kernel.md @@ -46,11 +46,14 @@ Assuming you want to create the new VM in `$VMPATH`, complete the wizard as foll * Guest OS type: Linux * Virtual Machine Name and Location: select `$VMPATH` as location and "debian" as name * Processors and Memory: select as appropriate -* Network connection: NAT +* Network connection: Use host-only networking * I/O Controller Type: LSI Logic * Virtual Disk Type: IDE * Disk: select "Use an existing virtual disk" * Existing Disk File: enter the path of `disk.vmdk` created above +* Select "Cusomize Hardware..." and remove the "Printer" device if you have one. Add a new "Serial Port" device. For the serial port connection choose "Use socket (named pipe)" and enter "serial" for the socket path. At the end it should look like this: + +![Virtual Machine Settings](vmw-settings.png?raw=true) When you complete the wizard, you should have `$VMPATH/debian.vmx`. From this point onward, you no longer need the Workstation UI. @@ -69,6 +72,11 @@ SSH into the VM: ssh -i key root@ ``` +Connecting to the serial port of the VM (after it is started): +``` bash +nc -U $VMPATH/serial +``` + Stopping the VM: ``` bash vmrun stop $VMPATH/debian.vmx @@ -104,7 +112,7 @@ mkdir workdir ./bin/syz-manager -config=my.cfg ``` -Syzkaller will create linked clone VMs from the `base_vmx` VM and then use ssh to copy and execute programs in them. +Syzkaller will create full clone VMs from the `base_vmx` VM and then use ssh to copy and execute programs in them. The `base_vmx` VM will not be started and its disk will remain unmodified. If you get issues after `syz-manager` starts, consider running it with the `-debug` flag. diff --git a/docs/linux/vmw-settings.png b/docs/linux/vmw-settings.png new file mode 100644 index 000000000..816c665d2 Binary files /dev/null and b/docs/linux/vmw-settings.png differ diff --git a/vm/vmware/vmware.go b/vm/vmware/vmware.go index 05d06e0a6..25f4eb4ec 100644 --- a/vm/vmware/vmware.go +++ b/vm/vmware/vmware.go @@ -6,6 +6,7 @@ package vmware import ( "fmt" "io" + "net" "os" "os/exec" "path/filepath" @@ -38,7 +39,6 @@ type instance struct { baseVMX string vmx string ipAddr string - snapshot string closed chan bool debug bool sshuser string @@ -76,18 +76,18 @@ func (pool *Pool) Count() int { } func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) { - vmx := filepath.Join(workdir, "syzkaller.vmx") + createTime := strconv.FormatInt(time.Now().UnixNano(), 10) + vmx := filepath.Join(workdir, createTime, "syzkaller.vmx") sshkey := pool.env.SSHKey sshuser := pool.env.SSHUser inst := &instance{ - cfg: pool.cfg, - debug: pool.env.Debug, - baseVMX: pool.cfg.BaseVMX, - vmx: vmx, - sshkey: sshkey, - sshuser: sshuser, - snapshot: strconv.FormatInt(time.Now().Unix(), 10), - closed: make(chan bool), + cfg: pool.cfg, + debug: pool.env.Debug, + baseVMX: pool.cfg.BaseVMX, + vmx: vmx, + sshkey: sshkey, + sshuser: sshuser, + closed: make(chan bool), } if err := inst.clone(); err != nil { return nil, err @@ -99,17 +99,10 @@ func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) { } func (inst *instance) clone() error { - if inst.debug { - log.Logf(0, "snapshotting %v (%v)", inst.baseVMX, inst.snapshot) - } - if _, err := osutil.RunCmd(2*time.Minute, "", "vmrun", "snapshot", inst.baseVMX, inst.snapshot); err != nil { - return err - } if inst.debug { log.Logf(0, "cloning %v to %v", inst.baseVMX, inst.vmx) } - if _, err := osutil.RunCmd(2*time.Minute, "", "vmrun", "clone", inst.baseVMX, inst.vmx, "linked", - "-snapshot="+inst.snapshot); err != nil { + if _, err := osutil.RunCmd(2*time.Minute, "", "vmrun", "clone", inst.baseVMX, inst.vmx, "full"); err != nil { return err } return nil @@ -151,15 +144,11 @@ func (inst *instance) Close() { if inst.debug { log.Logf(0, "stopping %v", inst.vmx) } - osutil.RunCmd(2*time.Minute, "", "vmrun", "stop", inst.vmx) + osutil.RunCmd(2*time.Minute, "", "vmrun", "stop", inst.vmx, "hard") if inst.debug { log.Logf(0, "deleting %v", inst.vmx) } osutil.RunCmd(2*time.Minute, "", "vmrun", "deleteVM", inst.vmx) - if inst.debug { - log.Logf(0, "deleting snapshot %v", inst.snapshot) - } - osutil.RunCmd(2*time.Minute, "", "vmrun", "deleteSnapshot", inst.baseVMX, inst.snapshot) close(inst.closed) } @@ -183,8 +172,9 @@ func (inst *instance) Copy(hostSrc string) (string, error) { func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) ( <-chan []byte, <-chan error, error) { - args := append(vmimpl.SSHArgs(inst.debug, inst.sshkey, 22), inst.sshuser+"@"+inst.ipAddr) - dmesg, err := vmimpl.OpenRemoteConsole("ssh", args...) + vmxDir := filepath.Dir(inst.vmx) + serial := filepath.Join(vmxDir, "serial") + dmesg, err := net.Dial("unix", serial) if err != nil { return nil, nil, err } @@ -195,7 +185,7 @@ func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command strin return nil, nil, err } - args = vmimpl.SSHArgs(inst.debug, inst.sshkey, 22) + args := vmimpl.SSHArgs(inst.debug, inst.sshkey, 22) // Forward target port as part of the ssh connection (reverse proxy) if inst.forwardPort != 0 { proxy := fmt.Sprintf("%v:127.0.0.1:%v", inst.forwardPort, inst.forwardPort) -- cgit mrf-deployment