pkg/build: handle OOM-killed build error

Handle SIGKILL (exit code = 137) on osutil.Run() during Linux kernel image
building and return build.InfraError without reporting.

Fixes: https://github.com/google/syzkaller/issues/5317

7 files changed, 76 insertions, 15 deletions

diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go
index 339838039..e00c54c68 100644
--- a/pkg/bisect/bisect.go
+++ b/pkg/bisect/bisect.go
@@ -134,14 +134,6 @@ type Result struct {
 	Confidence float64
 }
 
-type InfraError struct {
-	Title string
-}
-
-func (e InfraError) Error() string {
-	return e.Title
-}
-
 // Run does the bisection and returns either the Result,
 // or, if the crash is not reproduced on the start commit, an error.
 func Run(cfg *Config) (*Result, error) {
@@ -158,7 +150,7 @@ func Run(cfg *Config) (*Result, error) {
 		return nil, err
 	}
 	if _, err = repo.CheckoutBranch(cfg.Kernel.Repo, cfg.Kernel.Branch); err != nil {
-		return nil, &InfraError{Title: fmt.Sprintf("%v", err)}
+		return nil, &build.InfraError{Title: fmt.Sprintf("%v", err)}
 	}
 	return runImpl(cfg, repo, inst)
 }
@@ -692,7 +684,7 @@ func (env *env) test() (*testResult, error) {
 	if err != nil {
 		problem := fmt.Sprintf("repro testing failure: %v", err)
 		env.log(problem)
-		return res, &InfraError{Title: problem}
+		return res, &build.InfraError{Title: problem}
 	}
 	bad, good, infra, rep, types := env.processResults(current, results)
 	res.verdict, err = env.bisectionDecision(len(results), bad, good, infra)
@@ -851,7 +843,7 @@ func (env *env) bisectionDecision(total, bad, good, infra int) (vcs.BisectResult
 		// We have been unable to determine a verdict mostly because of infra errors.
 		// Abort the bisection.
 		return vcs.BisectSkip,
-			&InfraError{Title: "unable to determine the verdict because of infra errors"}
+			&build.InfraError{Title: "unable to determine the verdict because of infra errors"}
 	}
 	env.logf("unable to determine the verdict: %d good runs (wanted %d), for bad wanted %d in total, got %d",
 		good, wantGoodRuns, wantTotalRuns, good+bad)
diff --git a/pkg/bisect/bisect_test.go b/pkg/bisect/bisect_test.go
index eead42335..cb5e51c5b 100644
--- a/pkg/bisect/bisect_test.go
+++ b/pkg/bisect/bisect_test.go
@@ -651,7 +651,7 @@ var bisectionTests = []BisectionTest{
 		startCommit:   905,
 		expectRep:     false,
 		expectErr:     true,
-		expectErrType: &InfraError{},
+		expectErrType: &build.InfraError{},
 		infraErrStart: 600,
 		infraErrEnd:   800,
 		introduced:    "602",
diff --git a/pkg/build/build.go b/pkg/build/build.go
index bb6d089fd..ec47822a6 100644
--- a/pkg/build/build.go
+++ b/pkg/build/build.go
@@ -131,6 +131,18 @@ func (err *KernelError) Error() string {
 	return string(err.Report)
 }
 
+type InfraError struct {
+	Title  string
+	Output []byte
+}
+
+func (e InfraError) Error() string {
+	if len(e.Output) > 0 {
+		return fmt.Sprintf("%s: %s", e.Title, e.Output)
+	}
+	return e.Title
+}
+
 type builder interface {
 	build(params Params) (ImageDetails, error)
 	clean(kernelDir, targetArch string) error
@@ -208,6 +220,13 @@ func extractRootCause(err error, OS, kernelSrc string) error {
 	if len(reason) == 0 {
 		return err
 	}
+	// Don't report upon SIGKILL for Linux builds.
+	if OS == targets.Linux && string(reason) == "Killed" && verr.ExitCode == 137 {
+		return &InfraError{
+			Title:  string(reason),
+			Output: verr.Output,
+		}
+	}
 	kernelErr := &KernelError{
 		Report:     reason,
 		Output:     verr.Output,
@@ -313,6 +332,7 @@ var buildFailureCauses = [...]buildFailureCause{
 	{pattern: regexp.MustCompile(`^([a-zA-Z0-9_\-/.]+):[0-9]+:([0-9]+:)?.*(error|invalid|fatal|wrong)`)},
 	{pattern: regexp.MustCompile(`FAILED unresolved symbol`)},
 	{pattern: regexp.MustCompile(`No rule to make target`)},
+	{pattern: regexp.MustCompile(`^Killed$`)},
 	{weak: true, pattern: regexp.MustCompile(`: not found`)},
 	{weak: true, pattern: regexp.MustCompile(`: final link failed: `)},
 	{weak: true, pattern: regexp.MustCompile(`collect2: error: `)},
diff --git a/pkg/build/build_test.go b/pkg/build/build_test.go
index c89f63151..6aa9d9216 100644
--- a/pkg/build/build_test.go
+++ b/pkg/build/build_test.go
@@ -8,6 +8,10 @@ import (
 	"os/exec"
 	"strings"
 	"testing"
+
+	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/sys/targets"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestCompilerIdentity(t *testing.T) {
@@ -36,7 +40,7 @@ func TestCompilerIdentity(t *testing.T) {
 	}
 }
 
-func TestExtractRootCause(t *testing.T) {
+func TestExtractCauseInner(t *testing.T) {
 	for i, test := range rootCauseTests {
 		test := test
 		t.Run(fmt.Sprint(i), func(t *testing.T) {
@@ -51,6 +55,36 @@ func TestExtractRootCause(t *testing.T) {
 	}
 }
 
+func TestExtractRootCause(t *testing.T) {
+	targetOs := targets.TestOS
+	for i, test := range rootCauseTests {
+		var expected error
+		err := &osutil.VerboseError{Output: []byte(test.e)}
+		if test.reason == "" {
+			continue
+		} else if test.reason == "Killed" {
+			err.ExitCode = 137
+			targetOs = targets.Linux
+			expected = &InfraError{
+				Title:  test.reason,
+				Output: err.Output,
+			}
+		} else {
+			expected = &KernelError{
+				Report:     []byte(test.reason),
+				Output:     err.Output,
+				guiltyFile: test.file,
+			}
+		}
+		t.Run(fmt.Sprint(i), func(t *testing.T) {
+			got := extractRootCause(err, targetOs, test.src)
+			if got != nil {
+				assert.Equal(t, expected, got)
+			}
+		})
+	}
+}
+
 type RootCauseTest struct {
 	e      string
 	reason string
@@ -546,4 +580,17 @@ ninja: build stopped: subcommand failed.
 		"",
 		"",
 	},
+	{`
+  AR      built-in.a
+  AR      vmlinux.a
+  LD      vmlinux.o
+Killed
+make[1]: *** [scripts/Makefile.vmlinux_o:61: vmlinux.o] Error 137
+make[1]: *** Deleting file 'vmlinux.o'
+make: *** [Makefile:1231: vmlinux_o] Error 2
+`,
+		"Killed",
+		"",
+		"",
+	},
 }
diff --git a/pkg/osutil/osutil.go b/pkg/osutil/osutil.go
index a619f9169..465679ced 100644
--- a/pkg/osutil/osutil.go
+++ b/pkg/osutil/osutil.go
@@ -68,7 +68,7 @@ func Run(timeout time.Duration, cmd *exec.Cmd) ([]byte, error) {
 		if <-timedout {
 			text = fmt.Sprintf("timedout after %v %q", timeout, cmd.Args)
 		}
-		exitCode := 0
+		exitCode := cmd.ProcessState.ExitCode()
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) {
 			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go
index a2cd3c656..096e5df9f 100644
--- a/syz-ci/jobs.go
+++ b/syz-ci/jobs.go
@@ -524,7 +524,7 @@ func (jp *JobProcessor) bisect(job *Job, mgrcfg *mgrconfig.Config) error {
 	res, err := bisect.Run(cfg)
 	resp.Log = trace.Bytes()
 	if err != nil {
-		var infraErr *bisect.InfraError
+		var infraErr *build.InfraError
 		if errors.As(err, &infraErr) {
 			resp.Flags |= dashapi.BisectResultInfraError
 		}
diff --git a/syz-ci/manager.go b/syz-ci/manager.go
index b508cb1d8..cdbd6b40f 100644
--- a/syz-ci/manager.go
+++ b/syz-ci/manager.go
@@ -384,6 +384,8 @@ func (mgr *Manager) build(kernelCommit *vcs.Commit) error {
 		case errors.As(err, &verboseError):
 			rep.Report = []byte(verboseError.Title)
 			rep.Output = verboseError.Output
+		case errors.As(err, &build.InfraError{}):
+			return err
 		default:
 			rep.Report = []byte(err.Error())
 		}