sys/syz-extract: factor out compilation function

Each arch duplicates significant portion of logic
to compile the extract source file.
Factor this logic into a separate function and reuse
it across all OSes.

5 files changed, 189 insertions, 291 deletions

diff --git a/sys/syz-extract/extract.go b/sys/syz-extract/extract.go
index e0717a841..523a47d64 100644
--- a/sys/syz-extract/extract.go
+++ b/sys/syz-extract/extract.go
@@ -9,11 +9,9 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 
@@ -232,35 +230,3 @@ func processFile(OS OS, arch *Arch, inname string) (map[string]bool, error) {
 	}
 	return undeclared, nil
 }
-
-func runBinaryAndParse(bin string, vals []string, undeclared map[string]bool) (map[string]uint64, error) {
-	out, err := exec.Command(bin).CombinedOutput()
-	if err != nil {
-		return nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
-	}
-	flagVals := strings.Split(string(out), " ")
-	if len(out) == 0 {
-		flagVals = nil
-	}
-	if len(flagVals) != len(vals)-len(undeclared) {
-		return nil, fmt.Errorf("fetched wrong number of values %v != %v - %v\nflagVals: %q\nvals: %q\nundeclared: %q",
-			len(flagVals), len(vals), len(undeclared),
-			flagVals, vals, undeclared)
-	}
-	res := make(map[string]uint64)
-	j := 0
-	for _, v := range flagVals {
-		name := vals[j]
-		j++
-		for undeclared[name] {
-			name = vals[j]
-			j++
-		}
-		n, err := strconv.ParseUint(v, 10, 64)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse value: %v (%v)", err, v)
-		}
-		res[name] = n
-	}
-	return res, nil
-}
diff --git a/sys/syz-extract/fetch.go b/sys/syz-extract/fetch.go
new file mode 100644
index 000000000..6b5b46675
--- /dev/null
+++ b/sys/syz-extract/fetch.go
@@ -0,0 +1,159 @@
+// Copyright 2017 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/google/syzkaller/pkg/compiler"
+)
+
+func extract(info *compiler.ConstInfo, cc string, args []string, addSource string) (map[string]uint64, map[string]bool, error) {
+	data := &CompileData{
+		AddSource: addSource,
+		Defines:   info.Defines,
+		Includes:  info.Includes,
+		Values:    info.Consts,
+	}
+	undeclared := make(map[string]bool)
+	bin, out, err := compile(cc, args, data)
+	if err != nil {
+		// Some consts and syscall numbers are not defined on some archs.
+		// Figure out from compiler output undefined consts,
+		// and try to compile again without them.
+		valMap := make(map[string]bool)
+		for _, val := range info.Consts {
+			valMap[val] = true
+		}
+		for _, errMsg := range []string{
+			"error: ‘([a-zA-Z0-9_]+)’ undeclared",
+			"note: in expansion of macro ‘([a-zA-Z0-9_]+)’",
+		} {
+			re := regexp.MustCompile(errMsg)
+			matches := re.FindAllSubmatch(out, -1)
+			for _, match := range matches {
+				val := string(match[1])
+				if valMap[val] {
+					undeclared[val] = true
+				}
+			}
+		}
+		data.Values = nil
+		for _, v := range info.Consts {
+			if undeclared[v] {
+				continue
+			}
+			data.Values = append(data.Values, v)
+		}
+		bin, out, err = compile(cc, args, data)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to run compiler: %v\n%v", err, string(out))
+		}
+	}
+	defer os.Remove(bin)
+
+	out, err = exec.Command(bin).CombinedOutput()
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to run flags binary: %v\n%v", err, string(out))
+	}
+	flagVals := strings.Split(string(out), " ")
+	if len(out) == 0 {
+		flagVals = nil
+	}
+	if len(flagVals) != len(data.Values) {
+		return nil, nil, fmt.Errorf("fetched wrong number of values %v, want != %v",
+			len(flagVals), len(data.Values))
+	}
+	res := make(map[string]uint64)
+	for i, name := range data.Values {
+		val := flagVals[i]
+		n, err := strconv.ParseUint(val, 10, 64)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to parse value: %v (%v)", err, val)
+		}
+		res[name] = n
+	}
+	return res, undeclared, nil
+}
+
+type CompileData struct {
+	AddSource string
+	Defines   map[string]string
+	Includes  []string
+	Values    []string
+}
+
+func compile(cc string, args []string, data *CompileData) (bin string, out []byte, err error) {
+	srcFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
+	}
+	srcFile.Close()
+	os.Remove(srcFile.Name())
+	srcName := srcFile.Name() + ".c"
+	defer os.Remove(srcName)
+	src := new(bytes.Buffer)
+	if err := srcTemplate.Execute(src, data); err != nil {
+		return "", nil, fmt.Errorf("failed to generate source: %v", err)
+	}
+	if err := ioutil.WriteFile(srcName, src.Bytes(), 0600); err != nil {
+		return "", nil, fmt.Errorf("failed to write source file: %v", err)
+	}
+
+	binFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
+	}
+	binFile.Close()
+
+	args = append(args, []string{
+		srcName,
+		"-o", binFile.Name(),
+		"-w",
+	}...)
+	cmd := exec.Command(cc, args...)
+	out, err = cmd.CombinedOutput()
+	if err != nil {
+		os.Remove(binFile.Name())
+		return "", out, err
+	}
+	return binFile.Name(), nil, nil
+}
+
+var srcTemplate = template.Must(template.New("").Parse(`
+{{range $incl := $.Includes}}
+#include <{{$incl}}>
+{{end}}
+
+{{range $name, $val := $.Defines}}
+#ifndef {{$name}}
+#	define {{$name}} {{$val}}
+#endif
+{{end}}
+
+{{.AddSource}}
+
+int printf(const char *format, ...);
+
+int main() {
+	int i;
+	unsigned long long vals[] = {
+		{{range $val := $.Values}}(unsigned long long){{$val}},{{end}}
+	};
+	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
+		if (i != 0)
+			printf(" ");
+		printf("%llu", vals[i]);
+	}
+	return 0;
+}
+`))
diff --git a/sys/syz-extract/fuchsia.go b/sys/syz-extract/fuchsia.go
index f3cab132f..b693f38ad 100644
--- a/sys/syz-extract/fuchsia.go
+++ b/sys/syz-extract/fuchsia.go
@@ -5,11 +5,7 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
 	"path/filepath"
-	"strings"
 
 	"github.com/google/syzkaller/pkg/compiler"
 )
@@ -28,65 +24,12 @@ func (*fuchsia) prepareArch(arch *Arch) error {
 }
 
 func (*fuchsia) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
-	bin, out, err := fuchsiaCompile(arch.sourceDir, info.Consts, info.Includes, info.Incdirs, info.Defines)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to run compiler: %v\n%v", err, string(out))
-	}
-	defer os.Remove(bin)
-	res, err := runBinaryAndParse(bin, info.Consts, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-	return res, nil, nil
-}
-
-func fuchsiaCompile(sourceDir string, vals, includes, incdirs []string, defines map[string]string) (bin string, out []byte, err error) {
-	includeText := ""
-	for _, inc := range includes {
-		includeText += fmt.Sprintf("#include <%v>\n", inc)
-	}
-	definesText := ""
-	for k, v := range defines {
-		definesText += fmt.Sprintf("#ifndef %v\n#define %v %v\n#endif\n", k, k, v)
-	}
-	valsText := strings.Join(vals, ",")
-	src := fuchsiaSrc
-	src = strings.Replace(src, "[[INCLUDES]]", includeText, 1)
-	src = strings.Replace(src, "[[DEFAULTS]]", definesText, 1)
-	src = strings.Replace(src, "[[VALS]]", valsText, 1)
-	binFile, err := ioutil.TempFile("", "")
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
-	}
-	binFile.Close()
-	compiler := filepath.Join(sourceDir, "buildtools", "linux-x64", "clang", "bin", "clang")
-	includeDir := filepath.Join(sourceDir, "out", "build-zircon", "build-zircon-pc-x86-64", "sysroot", "include")
-	args := []string{"-x", "c", "-", "-o", binFile.Name(), "-fmessage-length=0", "-w", "-I", includeDir}
-	for _, incdir := range incdirs {
-		args = append(args, "-I"+sourceDir+"/"+incdir)
-	}
-	cmd := exec.Command(compiler, args...)
-	cmd.Stdin = strings.NewReader(src)
-	out, err = cmd.CombinedOutput()
-	if err != nil {
-		os.Remove(binFile.Name())
-		return "", out, err
-	}
-	return binFile.Name(), nil, nil
-}
-
-var fuchsiaSrc = `
-[[INCLUDES]]
-[[DEFAULTS]]
-int printf(const char *format, ...);
-int main() {
-	int i;
-	unsigned long long vals[] = {[[VALS]]};
-	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
-		if (i != 0)
-			printf(" ");
-		printf("%llu", vals[i]);
-	}
-	return 0;
+	dir := arch.sourceDir
+	cc := filepath.Join(dir, "buildtools", "linux-x64", "clang", "bin", "clang")
+	includeDir := filepath.Join(dir, "out", "build-zircon", "build-zircon-pc-x86-64", "sysroot", "include")
+	args := []string{"-fmessage-length=0", "-I" + includeDir}
+	for _, incdir := range info.Incdirs {
+		args = append(args, "-I"+filepath.Join(dir, incdir))
+	}
+	return extract(info, cc, args, "")
 }
-`
diff --git a/sys/syz-extract/linux.go b/sys/syz-extract/linux.go
index 7a0a226ef..481b55dee 100644
--- a/sys/syz-extract/linux.go
+++ b/sys/syz-extract/linux.go
@@ -5,16 +5,11 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"regexp"
 	"strings"
 	"time"
 
 	"github.com/google/syzkaller/pkg/compiler"
 	"github.com/google/syzkaller/pkg/osutil"
-	"github.com/google/syzkaller/sys/targets"
 )
 
 type linux struct{}
@@ -73,133 +68,40 @@ func (*linux) prepareArch(arch *Arch) error {
 }
 
 func (*linux) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
-	vals := info.Consts
-	includes := append(info.Includes, "asm/unistd.h")
-	bin, out, err := linuxCompile(arch.target, arch.sourceDir, arch.buildDir, nil,
-		includes, info.Incdirs, nil, nil)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
-	}
-	os.Remove(bin)
-
-	valMap := make(map[string]bool)
-	for _, val := range vals {
-		valMap[val] = true
-	}
-
-	undeclared := make(map[string]bool)
-	bin, out, err = linuxCompile(arch.target, arch.sourceDir, arch.buildDir, vals,
-		includes, info.Incdirs, info.Defines, undeclared)
-	if err != nil {
-		for _, errMsg := range []string{
-			"error: ‘([a-zA-Z0-9_]+)’ undeclared",
-			"note: in expansion of macro ‘([a-zA-Z0-9_]+)’",
-		} {
-			re := regexp.MustCompile(errMsg)
-			matches := re.FindAllSubmatch(out, -1)
-			for _, match := range matches {
-				val := string(match[1])
-				if !undeclared[val] && valMap[val] {
-					undeclared[val] = true
-				}
-			}
-		}
-		bin, out, err = linuxCompile(arch.target, arch.sourceDir, arch.buildDir, vals,
-			includes, info.Incdirs, info.Defines, undeclared)
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to run gcc: %v\n%v", err, string(out))
-		}
-	}
-	defer os.Remove(bin)
-
-	res, err := runBinaryAndParse(bin, vals, undeclared)
-	if err != nil {
-		return nil, nil, err
-	}
-	return res, undeclared, nil
-}
-
-func linuxCompile(target *targets.Target, kernelDir, buildDir string, vals, includes, incdirs []string, defines map[string]string, undeclared map[string]bool) (bin string, out []byte, err error) {
-	includeText := ""
-	for _, inc := range includes {
-		includeText += fmt.Sprintf("#include <%v>\n", inc)
-	}
-	definesText := ""
-	for k, v := range defines {
-		definesText += fmt.Sprintf("#ifndef %v\n#define %v %v\n#endif\n", k, k, v)
-	}
-	valsText := ""
-	for _, v := range vals {
-		if undeclared[v] {
-			continue
-		}
-		if valsText != "" {
-			valsText += ","
-		}
-		valsText += v
-	}
-	src := strings.Replace(linuxSrc, "[[INCLUDES]]", includeText, 1)
-	src = strings.Replace(src, "[[DEFAULTS]]", definesText, 1)
-	src = strings.Replace(src, "[[VALS]]", valsText, 1)
-	binFile, err := ioutil.TempFile("", "")
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
-	}
-	binFile.Close()
-
-	arch := target.KernelHeaderArch
-	args := []string{"-x", "c", "-", "-o", binFile.Name(), "-fmessage-length=0"}
-	args = append(args, target.CFlags...)
-	args = append(args, []string{
+	headerArch := arch.target.KernelHeaderArch
+	sourceDir := arch.sourceDir
+	buildDir := arch.buildDir
+	args := []string{
 		// This would be useful to ensure that we don't include any host headers,
 		// but kernel includes at least <stdarg.h>
 		// "-nostdinc",
-		"-w",
+		"-w", "-fmessage-length=0",
 		"-O3", // required to get expected values for some __builtin_constant_p
 		"-I.",
 		"-D__KERNEL__",
 		"-DKBUILD_MODNAME=\"-\"",
-		"-I" + kernelDir + "/arch/" + arch + "/include",
-		"-I" + buildDir + "/arch/" + arch + "/include/generated/uapi",
-		"-I" + buildDir + "/arch/" + arch + "/include/generated",
+		"-I" + sourceDir + "/arch/" + headerArch + "/include",
+		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
+		"-I" + buildDir + "/arch/" + headerArch + "/include/generated",
 		"-I" + buildDir + "/include",
-		"-I" + kernelDir + "/include",
-		"-I" + kernelDir + "/arch/" + arch + "/include/uapi",
-		"-I" + buildDir + "/arch/" + arch + "/include/generated/uapi",
-		"-I" + kernelDir + "/include/uapi",
+		"-I" + sourceDir + "/include",
+		"-I" + sourceDir + "/arch/" + headerArch + "/include/uapi",
+		"-I" + buildDir + "/arch/" + headerArch + "/include/generated/uapi",
+		"-I" + sourceDir + "/include/uapi",
 		"-I" + buildDir + "/include/generated/uapi",
-		"-I" + kernelDir,
-		"-include", kernelDir + "/include/linux/kconfig.h",
-	}...)
-	for _, incdir := range incdirs {
-		args = append(args, "-I"+kernelDir+"/"+incdir)
+		"-I" + sourceDir,
+		"-include", sourceDir + "/include/linux/kconfig.h",
 	}
-	cmd := exec.Command("gcc", args...)
-	cmd.Stdin = strings.NewReader(src)
-	out, err = cmd.CombinedOutput()
-	if err != nil {
-		os.Remove(binFile.Name())
-		return "", out, err
+	args = append(args, arch.target.CFlags...)
+	for _, incdir := range info.Incdirs {
+		args = append(args, "-I"+sourceDir+"/"+incdir)
 	}
-	return binFile.Name(), nil, nil
-}
-
-var linuxSrc = `
-[[INCLUDES]]
-[[DEFAULTS]]
-int printf(const char *format, ...);
+	const addSource = `
+#include <asm/unistd.h>
 unsigned long phys_base;
 #ifndef __phys_addr
 unsigned long __phys_addr(unsigned long addr) { return 0; }
 #endif
-int main() {
-	int i;
-	unsigned long long vals[] = {[[VALS]]};
-	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
-		if (i != 0)
-			printf(" ");
-		printf("%llu", vals[i]);
-	}
-	return 0;
-}
 `
+	return extract(info, "gcc", args, addSource)
+}
diff --git a/sys/syz-extract/windows.go b/sys/syz-extract/windows.go
index 81e3f6c23..44a9cd49e 100644
--- a/sys/syz-extract/windows.go
+++ b/sys/syz-extract/windows.go
@@ -4,12 +4,6 @@
 package main
 
 import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"strings"
-
 	"github.com/google/syzkaller/pkg/compiler"
 )
 
@@ -24,71 +18,5 @@ func (*windows) prepareArch(arch *Arch) error {
 }
 
 func (*windows) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
-	bin, out, err := windowsCompile(arch.sourceDir, info.Consts, info.Includes, info.Incdirs, info.Defines)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to run compiler: %v\n%v", err, string(out))
-	}
-	defer os.Remove(bin)
-	res, err := runBinaryAndParse(bin, info.Consts, nil)
-	if err != nil {
-		return nil, nil, err
-	}
-	return res, nil, nil
-}
-
-func windowsCompile(sourceDir string, vals, includes, incdirs []string, defines map[string]string) (bin string, out []byte, err error) {
-	includeText := ""
-	for _, inc := range includes {
-		includeText += fmt.Sprintf("#include <%v>\n", inc)
-	}
-	definesText := ""
-	for k, v := range defines {
-		definesText += fmt.Sprintf("#ifndef %v\n#define %v %v\n#endif\n", k, k, v)
-	}
-	valsText := "(unsigned long long)" + strings.Join(vals, ", (unsigned long long)")
-	src := windowsSrc
-	src = strings.Replace(src, "[[INCLUDES]]", includeText, 1)
-	src = strings.Replace(src, "[[DEFAULTS]]", definesText, 1)
-	src = strings.Replace(src, "[[VALS]]", valsText, 1)
-	binFile, err := ioutil.TempFile("", "")
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
-	}
-	binFile.Close()
-
-	srcFile, err := ioutil.TempFile("", "")
-	if err != nil {
-		return "", nil, fmt.Errorf("failed to create temp file: %v", err)
-	}
-	srcFile.Close()
-	os.Remove(srcFile.Name())
-	srcName := srcFile.Name() + ".cc"
-	if err := ioutil.WriteFile(srcName, []byte(src), 0600); err != nil {
-		return "", nil, fmt.Errorf("failed to write source file: %v", err)
-	}
-	defer os.Remove(srcName)
-	args := []string{"-o", binFile.Name(), srcName}
-	cmd := exec.Command("cl", args...)
-	out, err = cmd.CombinedOutput()
-	if err != nil {
-		os.Remove(binFile.Name())
-		return "", out, err
-	}
-	return binFile.Name(), nil, nil
-}
-
-var windowsSrc = `
-#include <stdio.h>
-[[INCLUDES]]
-[[DEFAULTS]]
-int main() {
-	int i;
-	unsigned long long vals[] = {[[VALS]]};
-	for (i = 0; i < sizeof(vals)/sizeof(vals[0]); i++) {
-		if (i != 0)
-			printf(" ");
-		printf("%llu", vals[i]);
-	}
-	return 0;
+	return extract(info, "cl", nil, "")
 }
-`