From 7aa6bd6859a419bfb445a3621a14124fd7cecced Mon Sep 17 00:00:00 2001 From: bobogei81123 Date: Sat, 12 Sep 2020 08:17:22 -0700 Subject: syz-manager: collect machine information * syz-manager: finish a prototype Extract machine info from /proc/cpuinfo and /sys/kvm*/parameters/* and send it from syz-fuzzer to syz-manager. Append the machine info after crash reports. * syz-manager: refactor the code - Add kvm parameters machine info. - Store the machine info in the RPCServer instead of the manager. - Store the machine info in another field instead of appending it after the original report - Save the machine info locally in machineInfo*. * syz-manager: fix coding-style problems * syz-fuzzer: improve the output from /proc/cpuinfo Improve the machine info extracted from /proc/cpuinfo by grouping lines with the same key. * syz-manager: fix race condition in runInstance * syz-fuzzer: add tests for collecting machine info - Add some tests to test collecting machine information. - Split readCPUInfo into scanCPUInfo so that we can test it. * syz-fuzzer: refactor scanCPUInfo Refactor scanCPUInfo so that no sorting is needed. * syz-fuzzer: refactor some code Fix some issue that was pointed out on Github. --- pkg/rpctype/rpctype.go | 3 +- syz-fuzzer/fuzzer.go | 10 ++- syz-fuzzer/machine_info.go | 149 ++++++++++++++++++++++++++++++++++++++++ syz-fuzzer/machine_info_test.go | 145 ++++++++++++++++++++++++++++++++++++++ syz-manager/manager.go | 50 ++++++++++---- syz-manager/rpc.go | 26 +++++-- 6 files changed, 363 insertions(+), 20 deletions(-) create mode 100644 syz-fuzzer/machine_info.go create mode 100644 syz-fuzzer/machine_info_test.go diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go index fcc01a99a..c370b0a0e 100644 --- a/pkg/rpctype/rpctype.go +++ b/pkg/rpctype/rpctype.go @@ -25,7 +25,8 @@ type RPCCandidate struct { } type ConnectArgs struct { - Name string + Name string + MachineInfo []byte } type ConnectRes struct { diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go index 876bc7c06..3218d9b88 100644 --- a/syz-fuzzer/fuzzer.go +++ b/syz-fuzzer/fuzzer.go @@ -181,13 +181,21 @@ func main() { runtime.MemProfileRate = 0 } + machineInfo, err := CollectMachineInfo() + if err != nil { + log.Fatalf("failed to collect machine information: %v", err) + } + log.Logf(0, "dialing manager at %v", *flagManager) manager, err := rpctype.NewRPCClient(*flagManager) if err != nil { log.Fatalf("failed to connect to manager: %v ", err) } log.Logf(1, "connecting to manager...") - a := &rpctype.ConnectArgs{Name: *flagName} + a := &rpctype.ConnectArgs{ + Name: *flagName, + MachineInfo: machineInfo, + } r := &rpctype.ConnectRes{} if err := manager.Call("Manager.Connect", a, r); err != nil { log.Fatalf("failed to connect to manager: %v ", err) diff --git a/syz-fuzzer/machine_info.go b/syz-fuzzer/machine_info.go new file mode 100644 index 000000000..040655e62 --- /dev/null +++ b/syz-fuzzer/machine_info.go @@ -0,0 +1,149 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" +) + +func CollectMachineInfo() ([]byte, error) { + if runtime.GOOS != "linux" { + return nil, nil + } + + type machineInfoFunc struct { + name string + fn func(*bytes.Buffer) error + } + + allMachineInfo := []machineInfoFunc{ + {"CPU Info", readCPUInfo}, + {"KVM", readKVMInfo}, + } + + buffer := new(bytes.Buffer) + + for _, pair := range allMachineInfo { + fmt.Fprintf(buffer, "[%s]\n", pair.name) + err := pair.fn(buffer) + if err != nil { + if os.IsNotExist(err) { + buffer.WriteString(err.Error() + "\n") + } else { + return nil, err + } + } + fmt.Fprintf(buffer, "-----------------------------------\n\n") + } + + return buffer.Bytes(), nil +} + +func readCPUInfo(buffer *bytes.Buffer) error { + file, err := os.Open("/proc/cpuinfo") + if err != nil { + return err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + scanCPUInfo(buffer, scanner) + return nil +} + +func scanCPUInfo(buffer *bytes.Buffer, scanner *bufio.Scanner) { + keyIndices := make(map[string]int) + type keyValues struct { + key string + values []string + } + var info []keyValues + + for scanner.Scan() { + splitted := strings.Split(scanner.Text(), ":") + if len(splitted) != 2 { + continue + } + key := strings.TrimSpace(splitted[0]) + val := strings.TrimSpace(splitted[1]) + + if idx, ok := keyIndices[key]; !ok { + idx = len(keyIndices) + keyIndices[key] = idx + info = append(info, keyValues{key, []string{val}}) + } else { + info[idx].values = append(info[idx].values, val) + } + } + + for _, kv := range info { + // It is guaranteed that len(vals) >= 1 + key := kv.key + vals := kv.values + if allEqual(vals) { + fmt.Fprintf(buffer, "%-20s: %s\n", key, vals[0]) + } else { + fmt.Fprintf(buffer, "%-20s: %s\n", key, strings.Join(vals, ", ")) + } + } +} + +func allEqual(slice []string) bool { + if len(slice) == 0 { + return true + } + for i := 1; i < len(slice); i++ { + if slice[i] != slice[0] { + return false + } + } + return true +} + +func readKVMInfo(buffer *bytes.Buffer) error { + files, err := ioutil.ReadDir("/sys/module/") + if err != nil { + return err + } + + for _, file := range files { + name := file.Name() + if !strings.HasPrefix(name, "kvm") { + continue + } + + paramPath := filepath.Join("/sys", "module", name, "parameters") + params, err := ioutil.ReadDir(paramPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return err + } + + if len(params) == 0 { + continue + } + + fmt.Fprintf(buffer, "/sys/module/%s:\n", name) + for _, key := range params { + keyName := key.Name() + data, err := ioutil.ReadFile(filepath.Join(paramPath, keyName)) + if err != nil { + return err + } + fmt.Fprintf(buffer, "\t%s: ", keyName) + buffer.Write(data) + } + buffer.WriteString("\n") + } + return nil +} diff --git a/syz-fuzzer/machine_info_test.go b/syz-fuzzer/machine_info_test.go new file mode 100644 index 000000000..71633ca5b --- /dev/null +++ b/syz-fuzzer/machine_info_test.go @@ -0,0 +1,145 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "bufio" + "bytes" + "strings" + "testing" +) + +func TestMachineInfoLinux(t *testing.T) { + result, err := CollectMachineInfo() + if err != nil { + t.Fatal(err) + } + + scanner := bufio.NewScanner(bytes.NewReader(result)) + + for scanner.Scan() { + line := scanner.Text() + + if line == "[CPU Info]" { + checkCPUInfo(t, scanner) + } + if line == "[KVM]" { + checkKVMInfo(t, scanner) + } + } +} + +func checkCPUInfo(t *testing.T, scanner *bufio.Scanner) { + keys := make(map[string]bool) + for scanner.Scan() { + line := scanner.Text() + // End of CPU Info section. + if strings.HasPrefix(line, "-----") { + break + } + splitted := strings.Split(line, ":") + if len(splitted) != 2 { + t.Fatalf("the format of line \"%s\" is not correct", line) + } + key := strings.TrimSpace(splitted[0]) + keys[key] = true + } + + importantKeys := [][]string{ + {"vendor", "vendor_id", "CPU implementer"}, + {"model", "CPU part", "cpu model"}, + {"flags", "features", "Features", "ASEs implemented", "type"}, + } + for _, possibleNames := range importantKeys { + exists := false + for _, name := range possibleNames { + if keys[name] { + exists = true + break + } + } + if !exists { + t.Fatalf("one of {%s} should exists in the output, but not found", + strings.Join(possibleNames, ", ")) + } + } +} + +func checkKVMInfo(t *testing.T, scanner *bufio.Scanner) { + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + if strings.HasPrefix(line, "-----") { + break + } + splitted := strings.Split(line, ":") + if len(splitted) != 2 { + t.Fatalf("the format of line \"%s\" is not correct", line) + } + key := strings.TrimSpace(splitted[0]) + if key == "" { + t.Fatalf("empty key") + } + if key[0] != '/' { + continue + } + + if !strings.HasPrefix(key, "/sys/module/kvm") { + t.Fatalf("the directory does not match /sys/module/kvm*") + } + } +} + +func TestScanCPUInfo(t *testing.T) { + input := `A: a +B: b + +C: c1 +D: d +C: c1 +D: d +C: c2 +D: d +` + + output := []struct { + key, val string + }{ + {"A", "a"}, + {"B", "b"}, + {"C", "c1, c1, c2"}, + {"D", "d"}, + } + scanner := bufio.NewScanner(strings.NewReader(input)) + buffer := new(bytes.Buffer) + scanCPUInfo(buffer, scanner) + result := bufio.NewScanner(strings.NewReader(buffer.String())) + + idx := 0 + for result.Scan() { + line := result.Text() + splitted := strings.Split(line, ":") + if len(splitted) != 2 { + t.Fatalf("the format of line \"%s\" is not correct", line) + } + key := strings.TrimSpace(splitted[0]) + val := strings.TrimSpace(splitted[1]) + if idx >= len(output) { + t.Fatalf("additional line \"%s: %s\"", key, val) + } + expected := output[idx] + if key != expected.key || val != expected.val { + t.Fatalf("expected \"%s: %s\", got \"%s: %s\"", + expected.key, expected.val, key, val) + } + idx++ + } + if idx < len(output) { + expected := output[idx] + t.Fatalf("expected \"%s: %s\", got end of output", + expected.key, expected.val) + } +} diff --git a/syz-manager/manager.go b/syz-manager/manager.go index ff6855bb3..8f4392d0d 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -50,7 +50,7 @@ type Manager struct { sysTarget *targets.Target reporter report.Reporter crashdir string - port int + serv *RPCServer corpusDB *db.DB startTime time.Time firstConnect time.Time @@ -109,6 +109,7 @@ type Crash struct { vmIndex int hub bool // this crash was created based on a repro from hub *report.Report + machineInfo []byte } func main() { @@ -192,7 +193,7 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, sysTarget *targets.T mgr.collectUsedFiles() // Create RPC server for fuzzers. - mgr.port, err = startRPCServer(mgr) + mgr.serv, err = startRPCServer(mgr) if err != nil { log.Fatalf("failed to create rpc server: %v", err) } @@ -266,7 +267,7 @@ func RunManager(cfg *mgrconfig.Config, target *prog.Target, sysTarget *targets.T if mgr.vmPool == nil { log.Logf(0, "no VMs started (type=none)") log.Logf(0, "you are supposed to start syz-fuzzer manually as:") - log.Logf(0, "syz-fuzzer -manager=manager.ip:%v [other flags as necessary]", mgr.port) + log.Logf(0, "syz-fuzzer -manager=manager.ip:%v [other flags as necessary]", mgr.serv.port) <-vm.Shutdown return } @@ -533,13 +534,37 @@ func checkProgram(target *prog.Target, enabled map[*prog.Syscall]bool, data []by func (mgr *Manager) runInstance(index int) (*Crash, error) { mgr.checkUsedFiles() + instanceName := fmt.Sprintf("vm-%d", index) + + rep, err := mgr.runInstanceInner(index) + + // Error that is not a VM crash. + if err != nil { + return nil, err + } + // No crash. + if rep == nil { + return nil, nil + } + + machineInfo := mgr.serv.getMachineInfo(instanceName) + crash := &Crash{ + vmIndex: index, + hub: false, + Report: rep, + machineInfo: machineInfo, + } + return crash, nil +} + +func (mgr *Manager) runInstanceInner(index int) (*report.Report, error) { inst, err := mgr.vmPool.Create(index) if err != nil { return nil, fmt.Errorf("failed to create instance: %v", err) } defer inst.Close() - fwdAddr, err := inst.Forward(mgr.port) + fwdAddr, err := inst.Forward(mgr.serv.port) if err != nil { return nil, fmt.Errorf("failed to setup port forwarding: %v", err) } @@ -570,7 +595,9 @@ func (mgr *Manager) runInstance(index int) (*Crash, error) { start := time.Now() atomic.AddUint32(&mgr.numFuzzing, 1) defer atomic.AddUint32(&mgr.numFuzzing, ^uint32(0)) - cmd := instance.FuzzerCmd(fuzzerBin, executorCmd, fmt.Sprintf("vm-%v", index), + + instanceName := fmt.Sprintf("vm-%d", index) + cmd := instance.FuzzerCmd(fuzzerBin, executorCmd, instanceName, mgr.cfg.TargetOS, mgr.cfg.TargetArch, fwdAddr, mgr.cfg.Sandbox, procs, fuzzerV, mgr.cfg.Cover, *flagDebug, false, false) outc, errc, err := inst.Run(time.Hour, mgr.vmStop, cmd) @@ -581,15 +608,9 @@ func (mgr *Manager) runInstance(index int) (*Crash, error) { rep := inst.MonitorExecution(outc, errc, mgr.reporter, vm.ExitTimeout) if rep == nil { // This is the only "OK" outcome. - log.Logf(0, "vm-%v: running for %v, restarting", index, time.Since(start)) - return nil, nil + log.Logf(0, "%s: running for %v, restarting", instanceName, time.Since(start)) } - crash := &Crash{ - vmIndex: index, - hub: false, - Report: rep, - } - return crash, nil + return rep, nil } func (mgr *Manager) emailCrash(crash *Crash) { @@ -695,6 +716,9 @@ func (mgr *Manager) saveCrash(crash *Crash) bool { if len(crash.Report.Report) > 0 { osutil.WriteFile(filepath.Join(dir, fmt.Sprintf("report%v", oldestI)), crash.Report.Report) } + if len(crash.machineInfo) > 0 { + osutil.WriteFile(filepath.Join(dir, fmt.Sprintf("machineInfo%v", oldestI)), crash.machineInfo) + } return mgr.needLocalRepro(crash) } diff --git a/syz-manager/rpc.go b/syz-manager/rpc.go index f550a2ca5..2b75dd440 100644 --- a/syz-manager/rpc.go +++ b/syz-manager/rpc.go @@ -19,6 +19,7 @@ import ( type RPCServer struct { mgr RPCManagerView + port int target *prog.Target configEnabledSyscalls []int targetEnabledSyscalls map[*prog.Syscall]bool @@ -41,6 +42,7 @@ type Fuzzer struct { inputs []rpctype.RPCInput newMaxSignal signal.Signal rotatedSignal signal.Signal + machineInfo []byte } type BugFrames struct { @@ -57,7 +59,7 @@ type RPCManagerView interface { rotateCorpus() bool } -func startRPCServer(mgr *Manager) (int, error) { +func startRPCServer(mgr *Manager) (*RPCServer, error) { serv := &RPCServer{ mgr: mgr, target: mgr.target, @@ -73,12 +75,12 @@ func startRPCServer(mgr *Manager) (int, error) { } s, err := rpctype.NewRPCServer(mgr.cfg.RPC, "Manager", serv) if err != nil { - return 0, err + return nil, err } log.Logf(0, "serving rpc on tcp://%v", s.Addr()) - port := s.Addr().(*net.TCPAddr).Port + serv.port = s.Addr().(*net.TCPAddr).Port go s.Serve() - return port, nil + return serv, nil } func (serv *RPCServer) Connect(a *rpctype.ConnectArgs, r *rpctype.ConnectRes) error { @@ -91,7 +93,8 @@ func (serv *RPCServer) Connect(a *rpctype.ConnectArgs, r *rpctype.ConnectRes) er defer serv.mu.Unlock() f := &Fuzzer{ - name: a.Name, + name: a.Name, + machineInfo: a.MachineInfo, } serv.fuzzers[a.Name] = f r.MemoryLeakFrames = bugFrames.memoryLeaks @@ -306,3 +309,16 @@ func (serv *RPCServer) Poll(a *rpctype.PollArgs, r *rpctype.PollRes) error { a.Name, len(r.Candidates), len(r.NewInputs), len(r.MaxSignal.Elems)) return nil } + +func (serv *RPCServer) getMachineInfo(name string) []byte { + serv.mu.Lock() + defer serv.mu.Unlock() + + fuzzer, ok := serv.fuzzers[name] + if !ok { + return nil + } + + serv.fuzzers[name] = nil + return fuzzer.machineInfo +} -- cgit mrf-deployment