tools/clang: compile clang tools into the binary

Compiled clang tools into Go binaries using cgo.
This significantly simplifies building and deployment.
This also enables unit testing of clang tools.
Now raw go test for clang tools will build them, run,
and verify output.

Each clang tool is still started as a subprocess.
I've experimented with running them in-process,
but this makes stdout/stderr interception extremly complicated,
and it seems that clang tools still use unsynchronized global state,
which breaks when invoked multiple times.
Subprocesses also make it safer in the face of potential memory leaks,
or memory corruptions in clang tools.

Fixes #6645

4 files changed, 27 insertions, 17 deletions

diff --git a/pkg/aflow/tool/codesearcher/codesearcher.go b/pkg/aflow/tool/codesearcher/codesearcher.go
index 1c3e50031..50a5627dc 100644
--- a/pkg/aflow/tool/codesearcher/codesearcher.go
+++ b/pkg/aflow/tool/codesearcher/codesearcher.go
@@ -11,6 +11,7 @@ import (
 	"github.com/google/syzkaller/pkg/clangtool"
 	"github.com/google/syzkaller/pkg/codesearch"
 	"github.com/google/syzkaller/pkg/hash"
+	"github.com/google/syzkaller/tools/clang/codesearch"
 )
 
 var Tools = []aflow.Tool{
@@ -154,7 +155,7 @@ func prepare(ctx *aflow.Context, args prepareArgs) (prepareResult, error) {
 		args.KernelCommit, hash.String(args.KernelConfig), codesearch.DatabaseFormatHash)
 	dir, err := ctx.Cache("codesearch", desc, func(dir string) error {
 		cfg := &clangtool.Config{
-			ToolBin:   args.CodesearchToolBin,
+			Tool:      clangtoolimpl.Tool,
 			KernelSrc: args.KernelSrc,
 			KernelObj: args.KernelObj,
 			CacheFile: filepath.Join(dir, "index.json"),
diff --git a/pkg/clangtool/clangtool.go b/pkg/clangtool/clangtool.go
index dd7c22f0a..c0c834e2d 100644
--- a/pkg/clangtool/clangtool.go
+++ b/pkg/clangtool/clangtool.go
@@ -23,7 +23,7 @@ import (
 )
 
 type Config struct {
-	ToolBin    string
+	Tool       string // one of compiled-in tool names
 	KernelSrc  string
 	KernelObj  string
 	CacheFile  string
@@ -137,7 +137,7 @@ func (v *Verifier) Filename(file string) {
 		return
 	}
 	v.fileCache[file] = -1
-	fmt.Fprintf(&v.err, "missing file: %v\n", file)
+	fmt.Fprintf(&v.err, "missing file: %v (src dirs %+v)\n", file, v.srcDirs)
 }
 
 func (v *Verifier) LineRange(file string, start, end int) {
@@ -160,8 +160,12 @@ func runTool[Output any, OutputPtr OutputDataPtr[Output]](cfg *Config, dbFile, f
 	// version that produces more warnings.
 	// Comments are needed for codesearch tool, but may be useful for declextract
 	// in the future if we try to parse them with LLMs.
-	data, err := exec.Command(cfg.ToolBin, "-p", dbFile,
-		"--extra-arg=-w", "--extra-arg=-fparse-all-comments", file).Output()
+	cmd := exec.Command(os.Args[0], "-p", dbFile,
+		"--extra-arg=-w", "--extra-arg=-fparse-all-comments", file)
+	cmd.Dir = cfg.KernelObj
+	// This tells the C++ clang tool to execute in a constructor.
+	cmd.Env = append([]string{fmt.Sprintf("%v=%v", runToolEnv, cfg.Tool)}, os.Environ()...)
+	data, err := cmd.Output()
 	if err != nil {
 		var exitErr *exec.ExitError
 		if errors.As(err, &exitErr) {
@@ -185,6 +189,16 @@ func runTool[Output any, OutputPtr OutputDataPtr[Output]](cfg *Config, dbFile, f
 	return out, nil
 }
 
+const runToolEnv = "SYZ_RUN_CLANGTOOL"
+
+func init() {
+	// The C++ clang tool was supposed to intercept execution in a constructor,
+	// execute and exit. If we got here with the env var set, something is wrong.
+	if name := os.Getenv(runToolEnv); name != "" {
+		panic(fmt.Sprintf("clang tool %q is not compiled in", name))
+	}
+}
+
 type compileCommand struct {
 	Command   string
 	Directory string
diff --git a/pkg/clangtool/tooltest/tooltest.go b/pkg/clangtool/tooltest/tooltest.go
index 4f0b3bced..e78a85b62 100644
--- a/pkg/clangtool/tooltest/tooltest.go
+++ b/pkg/clangtool/tooltest/tooltest.go
@@ -18,16 +18,10 @@ import (
 	"github.com/google/syzkaller/pkg/testutil"
 )
 
-var (
-	FlagBin    = flag.String("bin", "", "path to the clang tool binary to use")
-	FlagUpdate = flag.Bool("update", false, "update golden files")
-)
+var FlagUpdate = flag.Bool("update", false, "update golden files")
 
-func TestClangTool[Output any, OutputPtr clangtool.OutputDataPtr[Output]](t *testing.T) {
-	if *FlagBin == "" {
-		t.Skipf("clang tool path is not specified, run with -bin=clangtool flag")
-	}
-	ForEachTestFile(t, func(t *testing.T, cfg *clangtool.Config, file string) {
+func TestClangTool[Output any, OutputPtr clangtool.OutputDataPtr[Output]](t *testing.T, tool string) {
+	ForEachTestFile(t, tool, func(t *testing.T, cfg *clangtool.Config, file string) {
 		out, err := clangtool.Run[Output, OutputPtr](cfg)
 		if err != nil {
 			t.Fatal(err)
@@ -57,7 +51,7 @@ func LoadOutput[Output any, OutputPtr clangtool.OutputDataPtr[Output]](t *testin
 	return out
 }
 
-func ForEachTestFile(t *testing.T, fn func(t *testing.T, cfg *clangtool.Config, file string)) {
+func ForEachTestFile(t *testing.T, tool string, fn func(t *testing.T, cfg *clangtool.Config, file string)) {
 	forEachTestFile(t, func(t *testing.T, file string) {
 		t.Run(filepath.Base(file), func(t *testing.T) {
 			t.Parallel()
@@ -73,7 +67,7 @@ func ForEachTestFile(t *testing.T, fn func(t *testing.T, cfg *clangtool.Config,
 				t.Fatal(err)
 			}
 			cfg := &clangtool.Config{
-				ToolBin:    *FlagBin,
+				Tool:       tool,
 				KernelSrc:  osutil.Abs("testdata"),
 				KernelObj:  buildDir,
 				CacheFile:  filepath.Join(buildDir, filepath.Base(file)+".json"),
diff --git a/pkg/codesearch/codesearch_test.go b/pkg/codesearch/codesearch_test.go
index 35671567e..5deed53bb 100644
--- a/pkg/codesearch/codesearch_test.go
+++ b/pkg/codesearch/codesearch_test.go
@@ -12,10 +12,11 @@ import (
 
 	"github.com/google/syzkaller/pkg/clangtool/tooltest"
 	"github.com/google/syzkaller/pkg/osutil"
+	"github.com/google/syzkaller/tools/clang/codesearch"
 )
 
 func TestClangTool(t *testing.T) {
-	tooltest.TestClangTool[Database](t)
+	tooltest.TestClangTool[Database](t, clangtoolimpl.Tool)
 }
 
 func TestCommands(t *testing.T) {