sys/syz-extract: move from syz-extract

3 files changed, 392 insertions, 0 deletions

diff --git a/sys/extract.sh b/sys/extract.sh
new file mode 100755
index 000000000..7f8e11dea
--- /dev/null
+++ b/sys/extract.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+# Copyright 2015 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+# Assuming x86 host, you also need to install:
+# sudo apt-get install gcc-aarch64-linux-gnu gcc-powerpc64le-linux-gnu
+
+if [ "$LINUX" == "" ]; then
+	if [ "$ANDROID" == "" ]; then
+		echo "usage: make extract LINUX=/linux/checkout]"
+		echo "OR: make extract ANDROID=/linux/checkout]"
+		exit 1
+	else
+		LINUX=$ANDROID
+		BUILD_FOR_ANDROID=yes
+	fi
+else
+	BUILD_FOR_ANDROID=no
+fi
+
+UPSTREAM_FILES="sys/bpf.txt sys/dri.txt sys/fuse.txt sys/input.txt sys/ipc.txt
+		sys/key.txt sys/kvm.txt sys/loop.txt sys/perf.txt sys/random.txt
+		sys/sndcontrol.txt sys/sndseq.txt sys/sndtimer.txt
+		sys/sys.txt sys/test.txt sys/tty.txt sys/tun.txt sys/vnet.txt
+		sys/socket.txt sys/socket_alg.txt sys/socket_bluetooth.txt
+		sys/socket_inet.txt sys/socket_inet6.txt sys/socket_inet_tcp.txt
+		sys/socket_inet_udp.txt sys/socket_inet_icmp.txt
+		sys/socket_inet_sctp.txt sys/socket_inet_dccp.txt
+		sys/socket_kcm.txt sys/socket_key.txt sys/socket_netlink.txt
+		sys/socket_netrom.txt sys/socket_nfc.txt sys/socket_unix.txt
+		sys/socket_ipx.txt sys/socket_ax25.txt sys/socket_llc.txt"
+
+ANDROID_FILES="sys/tlk_device.txt sys/ion.txt"
+
+if [ "$BUILD_FOR_ANDROID" == "no" ]; then
+	FILES="$UPSTREAM_FILES"
+else
+	FILES="$ANDROID_FILES"
+fi
+
+generate_arch() {
+	echo generating arch $1...
+	echo "cd $LINUX; make defconfig"
+	OUT=`(cd $LINUX; make ARCH=$2 CROSS_COMPILE=$3-linux-gnu- defconfig 2>&1)`
+	if [ $? -ne 0 ]; then
+		echo "$OUT"
+		exit 1
+	fi
+	echo "cd $LINUX; make"
+	OUT=`(cd $LINUX; make ARCH=$2 CROSS_COMPILE=$3-linux-gnu- init/main.o 2>&1)`
+	if [ $? -ne 0 ]; then
+		echo "$OUT"
+		exit 1
+	fi
+	for F in $FILES; do
+		echo "extracting from $F"
+		bin/syz-extract -arch $1 -linux "$LINUX" -linuxbld "$LINUXBLD" $F
+		if [ $? -ne 0 ]; then
+			exit 1
+		fi
+	done
+	echo
+}
+
+generate_arch amd64 x86_64 x86_64
+generate_arch arm64 arm64 aarch64
+if [ "$BUILD_FOR_ANDROID" == "no" ]; then
+	generate_arch ppc64le powerpc powerpc64le
+fi
diff --git a/sys/syz-extract/extract.go b/sys/syz-extract/extract.go
new file mode 100644
index 000000000..344151856
--- /dev/null
+++ b/sys/syz-extract/extract.go
@@ -0,0 +1,161 @@
+// Copyright 2016 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+	"strings"
+
+	. "github.com/google/syzkaller/sys/sysparser"
+)
+
+var (
+	flagLinux    = flag.String("linux", "", "path to linux kernel source checkout")
+	flagLinuxBld = flag.String("linuxbld", "", "path to linux kernel build directory")
+	flagArch     = flag.String("arch", "", "arch to generate")
+	flagV        = flag.Int("v", 0, "verbosity")
+)
+
+type Arch struct {
+	CARCH            []string
+	KernelHeaderArch string
+	KernelInclude    string
+	CFlags           []string
+}
+
+var archs = map[string]*Arch{
+	"amd64":   {[]string{"__x86_64__"}, "x86", "asm/unistd.h", []string{"-m64"}},
+	"arm64":   {[]string{"__aarch64__"}, "arm64", "asm/unistd.h", []string{}},
+	"ppc64le": {[]string{"__ppc64__", "__PPC64__", "__powerpc64__"}, "powerpc", "asm/unistd.h", []string{"-D__powerpc64__"}},
+}
+
+func main() {
+	flag.Parse()
+	if *flagLinux == "" {
+		failf("provide path to linux kernel checkout via -linux flag (or make extract LINUX= flag)")
+	}
+	if *flagLinuxBld == "" {
+		logf(1, "No kernel build directory provided, assuming in-place build")
+		*flagLinuxBld = *flagLinux
+	}
+	if *flagArch == "" {
+		failf("-arch flag is required")
+	}
+	if archs[*flagArch] == nil {
+		failf("unknown arch %v", *flagArch)
+	}
+	if len(flag.Args()) != 1 {
+		failf("usage: syz-extract -linux=/linux/checkout -arch=arch sys/input_file.txt")
+	}
+
+	inname := flag.Args()[0]
+	outname := strings.TrimSuffix(inname, ".txt") + "_" + *flagArch + ".const"
+
+	inf, err := os.Open(inname)
+	if err != nil {
+		failf("failed to open input file: %v", err)
+	}
+	defer inf.Close()
+
+	desc := Parse(inf)
+	consts := compileConsts(archs[*flagArch], desc)
+
+	out := new(bytes.Buffer)
+	generateConsts(*flagArch, consts, out)
+	if err := ioutil.WriteFile(outname, out.Bytes(), 0660); err != nil {
+		failf("failed to write output file: %v", err)
+	}
+}
+
+func generateConsts(arch string, consts map[string]uint64, out io.Writer) {
+	var nv []NameValue
+	for k, v := range consts {
+		nv = append(nv, NameValue{k, v})
+	}
+	sort.Sort(NameValueArray(nv))
+
+	fmt.Fprintf(out, "# AUTOGENERATED FILE\n")
+	for _, x := range nv {
+		fmt.Fprintf(out, "%v = %v\n", x.name, x.val)
+	}
+}
+
+func compileConsts(arch *Arch, desc *Description) map[string]uint64 {
+	vals := make(map[string]bool)
+	for _, fvals := range desc.Flags {
+		for _, v := range fvals {
+			vals[v] = true
+		}
+	}
+	for v := range desc.Defines {
+		vals[v] = true
+	}
+	for _, sc := range desc.Syscalls {
+		if strings.HasPrefix(sc.CallName, "syz_") {
+			continue
+		}
+		name := "__NR_" + sc.CallName
+		vals[name] = true
+	}
+	for _, res := range desc.Resources {
+		for _, v := range res.Values {
+			vals[v] = true
+		}
+	}
+
+	valArr := make([]string, 0, len(vals))
+	for v := range vals {
+		if !isIdentifier(v) {
+			continue
+		}
+		valArr = append(valArr, v)
+	}
+	if len(valArr) == 0 {
+		return nil
+	}
+
+	consts, err := fetchValues(arch.KernelHeaderArch, valArr, append(desc.Includes, arch.KernelInclude), desc.Defines, arch.CFlags)
+	if err != nil {
+		failf("%v", err)
+	}
+	return consts
+}
+
+func isIdentifier(s string) bool {
+	for i, c := range s {
+		if c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || i > 0 && (c >= '0' && c <= '9') {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+type NameValue struct {
+	name string
+	val  uint64
+}
+
+type NameValueArray []NameValue
+
+func (a NameValueArray) Len() int           { return len(a) }
+func (a NameValueArray) Less(i, j int) bool { return a[i].name < a[j].name }
+func (a NameValueArray) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+
+func failf(msg string, args ...interface{}) {
+	fmt.Fprintf(os.Stderr, msg+"\n", args...)
+	os.Exit(1)
+}
+
+func logf(v int, msg string, args ...interface{}) {
+	if *flagV >= v {
+		fmt.Fprintf(os.Stderr, msg+"\n", args...)
+	}
+}
diff --git a/sys/syz-extract/fetch.go b/sys/syz-extract/fetch.go
new file mode 100644
index 000000000..79c080807
--- /dev/null
+++ b/sys/syz-extract/fetch.go
@@ -0,0 +1,162 @@
+// Copyright 2015 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+// fetchValues converts literal constants (e.g. O_APPEND) or any other C expressions
+// into their respective numeric values. It does so by builting and executing a C program
+// that prints values of the provided expressions.
+func fetchValues(arch string, vals []string, includes []string, defines map[string]string, cflags []string) (map[string]uint64, error) {
+	bin, out, err := runCompiler(arch, nil, includes, nil, cflags, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
+	}
+	os.Remove(bin)
+
+	valMap := make(map[string]bool)
+	for _, val := range vals {
+		valMap[val] = true
+	}
+
+	undeclared := make(map[string]bool)
+	bin, out, err = runCompiler(arch, vals, includes, defines, cflags, undeclared)
+	if err != nil {
+		for _, errMsg := range []string{
+			"error: ‘([a-zA-Z0-9_]+)’ undeclared",
+			"note: in expansion of macro ‘([a-zA-Z0-9_]+)’",
+		} {
+			re := regexp.MustCompile(errMsg)
+			matches := re.FindAllSubmatch(out, -1)
+			for _, match := range matches {
+				val := string(match[1])
+				if !undeclared[val] && valMap[val] {
+					logf(0, "undefined const: %v", val)
+					undeclared[val] = true
+				}
+			}
+		}
+		bin, out, err = runCompiler(arch, vals, includes, defines, cflags, undeclared)
+		if err != nil {
+			return nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
+		}
+	}
+	defer os.Remove(bin)
+
+	out, err = exec.Command(bin).CombinedOutput()
+	if err != nil {
+		return nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
+	}
+
+	flagVals := strings.Split(string(out), " ")
+	if len(flagVals) != len(vals)-len(undeclared) {
+		failf("fetched wrong number of values %v != %v - %v", len(flagVals), len(vals), len(undeclared))
+	}
+	res := make(map[string]uint64)
+	j := 0
+	for _, v := range flagVals {
+		name := vals[j]
+		j++
+		for undeclared[name] {
+			name = vals[j]
+			j++
+		}
+		n, err := strconv.ParseUint(v, 10, 64)
+		if err != nil {
+			failf("failed to parse value: %v (%v)", err, v)
+		}
+		res[name] = n
+	}
+	return res, nil
+}
+
+func runCompiler(arch string, vals []string, includes []string, defines map[string]string, cflags []string, undeclared map[string]bool) (bin string, out []byte, err error) {
+	includeText := ""
+	for _, inc := range includes {
+		includeText += fmt.Sprintf("#include <%v>\n", inc)
+	}
+	definesText := ""
+	for k, v := range defines {
+		definesText += fmt.Sprintf("#ifndef %v\n#define %v %v\n#endif\n", k, k, v)
+	}
+	valsText := ""
+	for _, v := range vals {
+		if undeclared[v] {
+			continue
+		}
+		if valsText != "" {
+			valsText += ","
+		}
+		valsText += v
+	}
+	src := strings.Replace(fetchSrc, "[[INCLUDES]]", includeText, 1)
+	src = strings.Replace(src, "[[DEFAULTS]]", definesText, 1)
+	src = strings.Replace(src, "[[VALS]]", valsText, 1)
+	binFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
+	}
+	binFile.Close()
+
+	args := []string{"-x", "c", "-", "-o", binFile.Name(), "-fmessage-length=0"}
+	args = append(args, cflags...)
+	args = append(args, []string{
+		// This would be useful to ensure that we don't include any host headers,
+		// but kernel includes at least <stdarg.h>
+		// "-nostdinc",
+		"-w",
+		"-O3", // required to get expected values for some __builtin_constant_p
+		"-I.",
+		"-D__KERNEL__",
+		"-DKBUILD_MODNAME=\"-\"",
+		"-I" + *flagLinux + "/arch/" + arch + "/include",
+		"-I" + *flagLinuxBld + "/arch/" + arch + "/include/generated/uapi",
+		"-I" + *flagLinuxBld + "/arch/" + arch + "/include/generated",
+		"-I" + *flagLinuxBld + "/include",
+		"-I" + *flagLinux + "/include",
+		"-I" + *flagLinux + "/arch/" + arch + "/include/uapi",
+		"-I" + *flagLinuxBld + "/arch/" + arch + "/include/generated/uapi",
+		"-I" + *flagLinux + "/include/uapi",
+		"-I" + *flagLinuxBld + "/include/generated/uapi",
+		"-I" + *flagLinux,
+		"-include", *flagLinux + "/include/linux/kconfig.h",
+	}...)
+
+	cmd := exec.Command("gcc", args...)
+	cmd.Stdin = strings.NewReader(src)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		os.Remove(binFile.Name())
+		return "", out, err
+	}
+	return binFile.Name(), nil, nil
+}
+
+var fetchSrc = `
+[[INCLUDES]]
+[[DEFAULTS]]
+int printf(const char *format, ...);
+unsigned long phys_base;
+#ifndef __phys_addr
+unsigned long __phys_addr(unsigned long addr) { return 0; }
+#endif
+int main() {
+	int i;
+	unsigned long vals[] = {[[VALS]]};
+	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
+		if (i != 0)
+			printf(" ");
+		printf("%lu", vals[i]);
+	}
+	return 0;
+}
+`