sys/targets: better detection for missing/broken cross-compilers

1. Detect when compiler is present, but is not functioning
(can't build a simple program, common for Linux distros).

2. Be more strict with skipping tests due to missing/broken compilers on CI
(on CI they should work, so fail loudly if not).

3. Dedup this logic across syz-env and pkg/csource tests.

4. Add better error reporting for syz-env.

Fixes #1606

5 files changed, 80 insertions, 60 deletions

diff --git a/Makefile b/Makefile
index d8831f9db..fa6e7583c 100644
--- a/Makefile
+++ b/Makefile
@@ -24,13 +24,16 @@ define newline
 
 
 endef
-ENV := $(subst \n,$(newline),$(shell \
+ENV := $(subst \n,$(newline),$(shell TRAVIS=$(TRAVIS)\
 	SOURCEDIR=$(SOURCEDIR) HOSTOS=$(HOSTOS) HOSTARCH=$(HOSTARCH) \
 	TARGETOS=$(TARGETOS) TARGETARCH=$(TARGETARCH) TARGETVMARCH=$(TARGETVMARCH) \
 	go run tools/syz-env/env.go))
 # Uncomment in case of emergency.
 # $(info $(ENV))
 $(eval $(ENV))
+ifneq ("$(SYZERROR)", "")
+$(error $(SYZERROR))
+endif
 ifeq ("$(NCORES)", "")
 $(error syz-env failed)
 endif
@@ -111,12 +114,15 @@ target: fuzzer execprog stress executor
 executor:
 ifneq ("$(BUILDOS)", "$(NATIVEBUILDOS)")
 	$(info ************************************************************************************)
-	$(info Building executor for ${TARGETOS} is not supported on ${BUILDOS}. Executor will not be built.)
+	$(info Executor will not be built)
+	$(info Building executor for ${TARGETOS} is not supported on ${BUILDOS})
 	$(info ************************************************************************************)
 else
 ifneq ("$(NO_CROSS_COMPILER)", "")
 	$(info ************************************************************************************)
-	$(info Native cross-compiler $(CC) is missing. Executor will not be built.)
+	$(info Executor will not be built)
+	$(info Native cross-compiler is missing/broken:)
+	$(info $(NO_CROSS_COMPILER))
 	$(info ************************************************************************************)
 else
 	mkdir -p ./bin/$(TARGETOS)_$(TARGETARCH)
diff --git a/pkg/csource/build.go b/pkg/csource/build.go
index 56d5487a8..1269987e5 100644
--- a/pkg/csource/build.go
+++ b/pkg/csource/build.go
@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"runtime"
 
 	"github.com/google/syzkaller/pkg/osutil"
@@ -37,9 +36,6 @@ func BuildFile(target *prog.Target, src string) (string, error) {
 func build(target *prog.Target, src []byte, file string, warn bool) (string, error) {
 	sysTarget := targets.Get(target.OS, target.Arch)
 	compiler := sysTarget.CCompiler
-	if _, err := exec.LookPath(compiler); err != nil {
-		return "", fmt.Errorf("no target compiler %v", compiler)
-	}
 	// We call the binary syz-executor because it sometimes shows in bug titles,
 	// and we don't want 2 different bugs for when a crash is triggered during fuzzing and during repro.
 	bin, err := osutil.TempFile("syz-executor")
diff --git a/pkg/csource/csource_test.go b/pkg/csource/csource_test.go
index 59ca230b6..78145c8fa 100644
--- a/pkg/csource/csource_test.go
+++ b/pkg/csource/csource_test.go
@@ -8,7 +8,6 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"regexp"
 	"runtime"
@@ -33,26 +32,9 @@ func TestGenerate(t *testing.T) {
 			continue
 		}
 		t.Run(target.OS+"/"+target.Arch, func(t *testing.T) {
-			if target.OS == "linux" && target.Arch == "arm64" {
-				// Episodically fails on travis with:
-				// collect2: error: ld terminated with signal 11 [Segmentation fault]
-				t.Skip("broken")
-			}
-			if target.OS == "test" && target.PtrSize == 4 {
-				// The same reason as linux/32.
-				t.Skip("broken")
-			}
-			if _, err := exec.LookPath(sysTarget.CCompiler); err != nil {
-				t.Skipf("no target compiler %v", sysTarget.CCompiler)
-			}
-			bin, err := Build(target, []byte(`
-#include <stdio.h>
-int main() { printf("Hello, World!\n"); }
-`))
-			if err != nil {
+			if err := sysTarget.BrokenCrossCompiler; err != "" {
 				t.Skipf("target compiler is broken: %v", err)
 			}
-			os.Remove(bin)
 			full := !checked[target.OS]
 			checked[target.OS] = true
 			t.Parallel()
diff --git a/sys/targets/targets.go b/sys/targets/targets.go
index 698b749dd..73022ff6f 100644
--- a/sys/targets/targets.go
+++ b/sys/targets/targets.go
@@ -15,20 +15,21 @@ import (
 type Target struct {
 	init sync.Once
 	osCommon
-	OS               string
-	Arch             string
-	VMArch           string // e.g. amd64 for 386, or arm64 for arm
-	PtrSize          uint64
-	PageSize         uint64
-	NumPages         uint64
-	DataOffset       uint64
-	Int64Alignment   uint64
-	CFlags           []string
-	CrossCFlags      []string
-	CCompilerPrefix  string
-	CCompiler        string
-	KernelArch       string
-	KernelHeaderArch string
+	OS                  string
+	Arch                string
+	VMArch              string // e.g. amd64 for 386, or arm64 for arm
+	PtrSize             uint64
+	PageSize            uint64
+	NumPages            uint64
+	DataOffset          uint64
+	Int64Alignment      uint64
+	CFlags              []string
+	CrossCFlags         []string
+	CCompilerPrefix     string
+	CCompiler           string
+	KernelArch          string
+	KernelHeaderArch    string
+	BrokenCrossCompiler string
 	// NeedSyscallDefine is used by csource package to decide when to emit __NR_* defines.
 	NeedSyscallDefine func(nr uint64) bool
 }
@@ -70,9 +71,7 @@ func Get(OS, arch string) *Target {
 	if target == nil {
 		return nil
 	}
-	target.init.Do(func() {
-		checkOptionalFlags(target)
-	})
+	target.init.Do(target.lazyInit)
 	return target
 }
 
@@ -490,10 +489,17 @@ func initTarget(target *Target, OS, arch string) {
 	target.CrossCFlags = append(append([]string{}, commonCFlags...), target.CrossCFlags...)
 }
 
-func checkOptionalFlags(target *Target) {
+func (target *Target) lazyInit() {
 	if runtime.GOOS != target.BuildOS {
 		return
 	}
+	if target.OS != runtime.GOOS || !runningOnCI {
+		// On CI we want to fail loudly if cross-compilation breaks.
+		if _, err := exec.LookPath(target.CCompiler); err != nil {
+			target.BrokenCrossCompiler = fmt.Sprintf("%v is missing", target.CCompiler)
+			return
+		}
+	}
 	flags := make(map[string]*bool)
 	var wg sync.WaitGroup
 	for _, flag := range target.CrossCFlags {
@@ -516,14 +522,40 @@ func checkOptionalFlags(target *Target) {
 			i--
 		}
 	}
+	// Check that the compiler is actually functioning. It may be present, but still broken.
+	// Common for Linux distros, over time we've seen:
+	//	Error: alignment too large: 15 assumed
+	//	fatal error: asm/unistd.h: No such file or directory
+	//	fatal error: asm/errno.h: No such file or directory
+	//	collect2: error: ld terminated with signal 11 [Segmentation fault]
+	if runningOnCI {
+		return // On CI all compilers are expected to work, so we don't do the following check.
+	}
+	args := []string{"-x", "c++", "-", "-o", "/dev/null"}
+	args = append(args, target.CrossCFlags...)
+	cmd := exec.Command(target.CCompiler, args...)
+	cmd.Stdin = strings.NewReader(simpleProg)
+	if out, err := cmd.CombinedOutput(); err != nil {
+		target.BrokenCrossCompiler = string(out)
+		return
+	}
 }
 
 func checkFlagSupported(target *Target, flag string) bool {
 	cmd := exec.Command(target.CCompiler, "-x", "c", "-", "-o", "/dev/null", flag)
-	cmd.Stdin = strings.NewReader("int main(){}")
+	cmd.Stdin = strings.NewReader(simpleProg)
 	return cmd.Run() == nil
 }
 
+var runningOnCI = os.Getenv("TRAVIS") != ""
+
+// <algorithm> is included by executor, so we test is as well.
+const simpleProg = `
+#include <stdio.h>
+#include <algorithm>
+int main() { printf("Hello, World!\n"); }
+`
+
 func needSyscallDefine(nr uint64) bool {
 	return true
 }
diff --git a/tools/syz-env/env.go b/tools/syz-env/env.go
index aa0143ab9..81a40af8a 100644
--- a/tools/syz-env/env.go
+++ b/tools/syz-env/env.go
@@ -6,7 +6,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"os/exec"
 	"runtime"
 	"strconv"
 	"strings"
@@ -16,6 +15,22 @@ import (
 )
 
 func main() {
+	vars, err := impl()
+	if err != nil {
+		fmt.Printf("export SYZERROR=%v\n", err)
+		os.Exit(1)
+	}
+	for _, v := range vars {
+		fmt.Printf("export %v=%v\\n", v.Name, v.Val)
+	}
+}
+
+type Var struct {
+	Name string
+	Val  string
+}
+
+func impl() ([]Var, error) {
 	hostOS := or(os.Getenv("HOSTOS"), runtime.GOOS)
 	hostArch := or(os.Getenv("HOSTARCH"), runtime.GOARCH)
 	targetOS := or(os.Getenv("TARGETOS"), hostOS)
@@ -23,12 +38,7 @@ func main() {
 	targetVMArch := or(os.Getenv("TARGETVMARCH"), targetArch)
 	target := targets.Get(targetOS, targetArch)
 	if target == nil {
-		fmt.Printf("unknown target %v/%v\n", targetOS, targetArch)
-		os.Exit(1)
-	}
-	type Var struct {
-		Name string
-		Val  string
+		return nil, fmt.Errorf("unknown target %v/%v", targetOS, targetArch)
 	}
 	parallelism := runtime.NumCPU()
 	if mem := osutil.SystemMemorySize(); mem != 0 {
@@ -53,15 +63,9 @@ func main() {
 		{"NCORES", strconv.Itoa(parallelism)},
 		{"EXE", target.ExeExtension},
 		{"NATIVEBUILDOS", target.BuildOS},
+		{"NO_CROSS_COMPILER", target.BrokenCrossCompiler},
 	}
-	if targetOS != runtime.GOOS {
-		if _, err := exec.LookPath(target.CCompiler); err != nil {
-			vars = append(vars, Var{"NO_CROSS_COMPILER", "yes"})
-		}
-	}
-	for _, v := range vars {
-		fmt.Printf("export %v=%v\\n", v.Name, v.Val)
-	}
+	return vars, nil
 }
 
 func or(s1, s2 string) string {