From 49f9c2d289d546c97aae87130574abe5b963b72a Mon Sep 17 00:00:00 2001 From: Sabyrzhan Tasbolatov Date: Tue, 17 Sep 2024 01:10:32 +0500 Subject: pkg/build: handle OOM-killed build error Handle SIGKILL (exit code = 137) on osutil.Run() during Linux kernel image building and return build.InfraError without reporting. Fixes: https://github.com/google/syzkaller/issues/5317 --- pkg/bisect/bisect.go | 14 +++----------- pkg/bisect/bisect_test.go | 2 +- pkg/build/build.go | 20 +++++++++++++++++++ pkg/build/build_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++- pkg/osutil/osutil.go | 2 +- 5 files changed, 73 insertions(+), 14 deletions(-) (limited to 'pkg') diff --git a/pkg/bisect/bisect.go b/pkg/bisect/bisect.go index 339838039..e00c54c68 100644 --- a/pkg/bisect/bisect.go +++ b/pkg/bisect/bisect.go @@ -134,14 +134,6 @@ type Result struct { Confidence float64 } -type InfraError struct { - Title string -} - -func (e InfraError) Error() string { - return e.Title -} - // Run does the bisection and returns either the Result, // or, if the crash is not reproduced on the start commit, an error. func Run(cfg *Config) (*Result, error) { @@ -158,7 +150,7 @@ func Run(cfg *Config) (*Result, error) { return nil, err } if _, err = repo.CheckoutBranch(cfg.Kernel.Repo, cfg.Kernel.Branch); err != nil { - return nil, &InfraError{Title: fmt.Sprintf("%v", err)} + return nil, &build.InfraError{Title: fmt.Sprintf("%v", err)} } return runImpl(cfg, repo, inst) } @@ -692,7 +684,7 @@ func (env *env) test() (*testResult, error) { if err != nil { problem := fmt.Sprintf("repro testing failure: %v", err) env.log(problem) - return res, &InfraError{Title: problem} + return res, &build.InfraError{Title: problem} } bad, good, infra, rep, types := env.processResults(current, results) res.verdict, err = env.bisectionDecision(len(results), bad, good, infra) @@ -851,7 +843,7 @@ func (env *env) bisectionDecision(total, bad, good, infra int) (vcs.BisectResult // We have been unable to determine a verdict mostly because of infra errors. // Abort the bisection. return vcs.BisectSkip, - &InfraError{Title: "unable to determine the verdict because of infra errors"} + &build.InfraError{Title: "unable to determine the verdict because of infra errors"} } env.logf("unable to determine the verdict: %d good runs (wanted %d), for bad wanted %d in total, got %d", good, wantGoodRuns, wantTotalRuns, good+bad) diff --git a/pkg/bisect/bisect_test.go b/pkg/bisect/bisect_test.go index eead42335..cb5e51c5b 100644 --- a/pkg/bisect/bisect_test.go +++ b/pkg/bisect/bisect_test.go @@ -651,7 +651,7 @@ var bisectionTests = []BisectionTest{ startCommit: 905, expectRep: false, expectErr: true, - expectErrType: &InfraError{}, + expectErrType: &build.InfraError{}, infraErrStart: 600, infraErrEnd: 800, introduced: "602", diff --git a/pkg/build/build.go b/pkg/build/build.go index bb6d089fd..ec47822a6 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -131,6 +131,18 @@ func (err *KernelError) Error() string { return string(err.Report) } +type InfraError struct { + Title string + Output []byte +} + +func (e InfraError) Error() string { + if len(e.Output) > 0 { + return fmt.Sprintf("%s: %s", e.Title, e.Output) + } + return e.Title +} + type builder interface { build(params Params) (ImageDetails, error) clean(kernelDir, targetArch string) error @@ -208,6 +220,13 @@ func extractRootCause(err error, OS, kernelSrc string) error { if len(reason) == 0 { return err } + // Don't report upon SIGKILL for Linux builds. + if OS == targets.Linux && string(reason) == "Killed" && verr.ExitCode == 137 { + return &InfraError{ + Title: string(reason), + Output: verr.Output, + } + } kernelErr := &KernelError{ Report: reason, Output: verr.Output, @@ -313,6 +332,7 @@ var buildFailureCauses = [...]buildFailureCause{ {pattern: regexp.MustCompile(`^([a-zA-Z0-9_\-/.]+):[0-9]+:([0-9]+:)?.*(error|invalid|fatal|wrong)`)}, {pattern: regexp.MustCompile(`FAILED unresolved symbol`)}, {pattern: regexp.MustCompile(`No rule to make target`)}, + {pattern: regexp.MustCompile(`^Killed$`)}, {weak: true, pattern: regexp.MustCompile(`: not found`)}, {weak: true, pattern: regexp.MustCompile(`: final link failed: `)}, {weak: true, pattern: regexp.MustCompile(`collect2: error: `)}, diff --git a/pkg/build/build_test.go b/pkg/build/build_test.go index c89f63151..6aa9d9216 100644 --- a/pkg/build/build_test.go +++ b/pkg/build/build_test.go @@ -8,6 +8,10 @@ import ( "os/exec" "strings" "testing" + + "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/sys/targets" + "github.com/stretchr/testify/assert" ) func TestCompilerIdentity(t *testing.T) { @@ -36,7 +40,7 @@ func TestCompilerIdentity(t *testing.T) { } } -func TestExtractRootCause(t *testing.T) { +func TestExtractCauseInner(t *testing.T) { for i, test := range rootCauseTests { test := test t.Run(fmt.Sprint(i), func(t *testing.T) { @@ -51,6 +55,36 @@ func TestExtractRootCause(t *testing.T) { } } +func TestExtractRootCause(t *testing.T) { + targetOs := targets.TestOS + for i, test := range rootCauseTests { + var expected error + err := &osutil.VerboseError{Output: []byte(test.e)} + if test.reason == "" { + continue + } else if test.reason == "Killed" { + err.ExitCode = 137 + targetOs = targets.Linux + expected = &InfraError{ + Title: test.reason, + Output: err.Output, + } + } else { + expected = &KernelError{ + Report: []byte(test.reason), + Output: err.Output, + guiltyFile: test.file, + } + } + t.Run(fmt.Sprint(i), func(t *testing.T) { + got := extractRootCause(err, targetOs, test.src) + if got != nil { + assert.Equal(t, expected, got) + } + }) + } +} + type RootCauseTest struct { e string reason string @@ -546,4 +580,17 @@ ninja: build stopped: subcommand failed. "", "", }, + {` + AR built-in.a + AR vmlinux.a + LD vmlinux.o +Killed +make[1]: *** [scripts/Makefile.vmlinux_o:61: vmlinux.o] Error 137 +make[1]: *** Deleting file 'vmlinux.o' +make: *** [Makefile:1231: vmlinux_o] Error 2 +`, + "Killed", + "", + "", + }, } diff --git a/pkg/osutil/osutil.go b/pkg/osutil/osutil.go index a619f9169..465679ced 100644 --- a/pkg/osutil/osutil.go +++ b/pkg/osutil/osutil.go @@ -68,7 +68,7 @@ func Run(timeout time.Duration, cmd *exec.Cmd) ([]byte, error) { if <-timedout { text = fmt.Sprintf("timedout after %v %q", timeout, cmd.Args) } - exitCode := 0 + exitCode := cmd.ProcessState.ExitCode() var exitErr *exec.ExitError if errors.As(err, &exitErr) { if status, ok := exitErr.Sys().(syscall.WaitStatus); ok { -- cgit mrf-deployment