pkg/clangtool: make more generic

Make it possible to use pkg/clangtool with other types than declextract.Output.

8 files changed, 71 insertions, 63 deletions

diff --git a/pkg/clangtool/clangtool.go b/pkg/clangtool/clangtool.go
index bed29eb7b..8ebc40dc2 100644
--- a/pkg/clangtool/clangtool.go
+++ b/pkg/clangtool/clangtool.go
@@ -5,6 +5,7 @@ package clangtool
 
 import (
 	"bytes"
+	"crypto/sha256"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -18,7 +19,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/google/syzkaller/pkg/declextract"
 	"github.com/google/syzkaller/pkg/osutil"
 )
 
@@ -30,14 +30,21 @@ type Config struct {
 	DebugTrace io.Writer
 }
 
+type OutputDataPtr[T any] interface {
+	*T
+	Merge(*T)
+	SetSourceFile(string, func(filename string) string)
+	SortAndDedup()
+}
+
 // Run runs the clang tool on all files in the compilation database
 // in the kernel build dir and returns combined output for all files.
 // It always caches results, and optionally reuses previously cached results.
-func Run(cfg *Config) (*declextract.Output, error) {
+func Run[Output any, OutputPtr OutputDataPtr[Output]](cfg *Config) (OutputPtr, error) {
 	if cfg.CacheFile != "" {
 		data, err := os.ReadFile(cfg.CacheFile)
 		if err == nil {
-			out, err := unmarshal(data)
+			out, err := unmarshal[Output, OutputPtr](data)
 			if err == nil {
 				return out, nil
 			}
@@ -51,7 +58,7 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	}
 
 	type result struct {
-		out *declextract.Output
+		out OutputPtr
 		err error
 	}
 	results := make(chan *result, 10)
@@ -59,7 +66,7 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	for w := 0; w < runtime.NumCPU(); w++ {
 		go func() {
 			for file := range files {
-				out, err := runTool(cfg, dbFile, file)
+				out, err := runTool[Output, OutputPtr](cfg, dbFile, file)
 				results <- &result{out, err}
 			}
 		}()
@@ -69,7 +76,7 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	}
 	close(files)
 
-	out := new(declextract.Output)
+	out := OutputPtr(new(Output))
 	for range cmds {
 		res := <-results
 		if res.err != nil {
@@ -91,7 +98,7 @@ func Run(cfg *Config) (*declextract.Output, error) {
 	return out, nil
 }
 
-func runTool(cfg *Config, dbFile, file string) (*declextract.Output, error) {
+func runTool[Output any, OutputPtr OutputDataPtr[Output]](cfg *Config, dbFile, file string) (OutputPtr, error) {
 	relFile := strings.TrimPrefix(strings.TrimPrefix(strings.TrimPrefix(filepath.Clean(file),
 		cfg.KernelSrc), cfg.KernelObj), "/")
 	// Suppress warning since we may build the tool on a different clang
@@ -104,35 +111,32 @@ func runTool(cfg *Config, dbFile, file string) (*declextract.Output, error) {
 		}
 		return nil, err
 	}
-	out, err := unmarshal(data)
+	out, err := unmarshal[Output, OutputPtr](data)
 	if err != nil {
 		return nil, err
 	}
-	fixupFileNames(cfg, out, relFile)
+	// All includes in the tool output are relative to the build dir.
+	// Make them relative to the source dir.
+	out.SetSourceFile(relFile, func(filename string) string {
+		rel, err := filepath.Rel(cfg.KernelSrc, filepath.Join(cfg.KernelObj, filename))
+		if err == nil && filename != "" {
+			return rel
+		}
+		return filename
+	})
 	return out, nil
 }
 
-func unmarshal(data []byte) (*declextract.Output, error) {
+func unmarshal[Output any, OutputPtr OutputDataPtr[Output]](data []byte) (OutputPtr, error) {
 	dec := json.NewDecoder(bytes.NewReader(data))
 	dec.DisallowUnknownFields()
-	out := new(declextract.Output)
+	out := OutputPtr(new(Output))
 	if err := dec.Decode(out); err != nil {
 		return nil, fmt.Errorf("failed to unmarshal clang tool output: %w\n%s", err, data)
 	}
 	return out, nil
 }
 
-func fixupFileNames(cfg *Config, out *declextract.Output, file string) {
-	// All includes in the tool output are relative to the build dir.
-	// Make them relative to the source dir.
-	out.SetSourceFile(file, func(filename string) string {
-		if res, err := filepath.Rel(cfg.KernelSrc, filepath.Join(cfg.KernelObj, filename)); err == nil {
-			return res
-		}
-		return filename
-	})
-}
-
 type compileCommand struct {
 	Command   string
 	Directory string
@@ -170,3 +174,21 @@ func loadCompileCommands(dbFile string) ([]compileCommand, error) {
 	}
 	return cmds, nil
 }
+
+func SortAndDedupSlice[Slice ~[]E, E comparable](s Slice) Slice {
+	dedup := make(map[[sha256.Size]byte]E)
+	text := make(map[E][]byte)
+	for _, e := range s {
+		t, _ := json.Marshal(e)
+		dedup[sha256.Sum256(t)] = e
+		text[e] = t
+	}
+	s = make([]E, 0, len(dedup))
+	for _, e := range dedup {
+		s = append(s, e)
+	}
+	slices.SortFunc(s, func(a, b E) int {
+		return bytes.Compare(text[a], text[b])
+	})
+	return s
+}
diff --git a/pkg/declextract/declextract.go b/pkg/declextract/declextract.go
index 923e2440f..b22257bf8 100644
--- a/pkg/declextract/declextract.go
+++ b/pkg/declextract/declextract.go
@@ -12,6 +12,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/google/syzkaller/pkg/clangtool"
 	"github.com/google/syzkaller/pkg/cover"
 	"github.com/google/syzkaller/pkg/ifaceprobe"
 )
@@ -136,8 +137,8 @@ func (ctx *context) processConsts() map[string]string {
 			Value: fmt.Sprint(ci.Value),
 		})
 	}
-	ctx.includes = sortAndDedupSlice(ctx.includes)
-	ctx.defines = sortAndDedupSlice(ctx.defines)
+	ctx.includes = clangtool.SortAndDedupSlice(ctx.includes)
+	ctx.defines = clangtool.SortAndDedupSlice(ctx.defines)
 	// These additional includes must be at the top, because other kernel headers
 	// are broken and won't compile without these additional ones included first.
 	ctx.includes = append([]string{
@@ -194,7 +195,7 @@ func (ctx *context) processSyscalls() {
 		}
 		ctx.emitSyscall(&syscalls, call, "", "", -1, "")
 	}
-	ctx.Syscalls = sortAndDedupSlice(syscalls)
+	ctx.Syscalls = clangtool.SortAndDedupSlice(syscalls)
 }
 
 func (ctx *context) emitSyscall(syscalls *[]*Syscall, call *Syscall,
diff --git a/pkg/declextract/entity.go b/pkg/declextract/entity.go
index d0a309a0f..bd8f143d1 100644
--- a/pkg/declextract/entity.go
+++ b/pkg/declextract/entity.go
@@ -4,11 +4,9 @@
 package declextract
 
 import (
-	"bytes"
-	"crypto/sha256"
-	"encoding/json"
-	"slices"
 	"strings"
+
+	"github.com/google/syzkaller/pkg/clangtool"
 )
 
 type Output struct {
@@ -244,16 +242,16 @@ func (out *Output) Merge(other *Output) {
 }
 
 func (out *Output) SortAndDedup() {
-	out.Functions = sortAndDedupSlice(out.Functions)
-	out.Consts = sortAndDedupSlice(out.Consts)
-	out.Enums = sortAndDedupSlice(out.Enums)
-	out.Structs = sortAndDedupSlice(out.Structs)
-	out.Syscalls = sortAndDedupSlice(out.Syscalls)
-	out.FileOps = sortAndDedupSlice(out.FileOps)
-	out.Ioctls = sortAndDedupSlice(out.Ioctls)
-	out.IouringOps = sortAndDedupSlice(out.IouringOps)
-	out.NetlinkFamilies = sortAndDedupSlice(out.NetlinkFamilies)
-	out.NetlinkPolicies = sortAndDedupSlice(out.NetlinkPolicies)
+	out.Functions = clangtool.SortAndDedupSlice(out.Functions)
+	out.Consts = clangtool.SortAndDedupSlice(out.Consts)
+	out.Enums = clangtool.SortAndDedupSlice(out.Enums)
+	out.Structs = clangtool.SortAndDedupSlice(out.Structs)
+	out.Syscalls = clangtool.SortAndDedupSlice(out.Syscalls)
+	out.FileOps = clangtool.SortAndDedupSlice(out.FileOps)
+	out.Ioctls = clangtool.SortAndDedupSlice(out.Ioctls)
+	out.IouringOps = clangtool.SortAndDedupSlice(out.IouringOps)
+	out.NetlinkFamilies = clangtool.SortAndDedupSlice(out.NetlinkFamilies)
+	out.NetlinkPolicies = clangtool.SortAndDedupSlice(out.NetlinkPolicies)
 }
 
 // SetSoureFile attaches the source file to the entities that need it.
@@ -285,21 +283,3 @@ func (out *Output) SetSourceFile(file string, updatePath func(string) string) {
 		op.SourceFile = file
 	}
 }
-
-func sortAndDedupSlice[Slice ~[]E, E comparable](s Slice) Slice {
-	dedup := make(map[[sha256.Size]byte]E)
-	text := make(map[E][]byte)
-	for _, e := range s {
-		t, _ := json.Marshal(e)
-		dedup[sha256.Sum256(t)] = e
-		text[e] = t
-	}
-	s = make([]E, 0, len(dedup))
-	for _, e := range dedup {
-		s = append(s, e)
-	}
-	slices.SortFunc(s, func(a, b E) int {
-		return bytes.Compare(text[a], text[b])
-	})
-	return s
-}
diff --git a/pkg/declextract/fileops.go b/pkg/declextract/fileops.go
index 8b2ea629a..8af85189f 100644
--- a/pkg/declextract/fileops.go
+++ b/pkg/declextract/fileops.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	"github.com/google/syzkaller/pkg/ast"
+	"github.com/google/syzkaller/pkg/clangtool"
 )
 
 const (
@@ -243,7 +244,7 @@ func (ctx *context) mapFileToFops(funcs map[*Function]bool, funcToFops map[*Func
 			best = append(best, fops)
 		}
 	}
-	best = sortAndDedupSlice(best)
+	best = clangtool.SortAndDedupSlice(best)
 	// Now, filter out some excessive file_operations.
 	// An example of an excessive case is if we have 2 file_operations with just read+write,
 	// currently we emit generic read/write operations, so we would emit completly equal
diff --git a/pkg/declextract/interface.go b/pkg/declextract/interface.go
index f386f398b..73806ec4b 100644
--- a/pkg/declextract/interface.go
+++ b/pkg/declextract/interface.go
@@ -8,6 +8,7 @@ import (
 	"slices"
 	"strings"
 
+	"github.com/google/syzkaller/pkg/clangtool"
 	"github.com/google/syzkaller/pkg/cover"
 )
 
@@ -55,7 +56,7 @@ func (ctx *context) noteInterface(iface *Interface) {
 }
 
 func (ctx *context) finishInterfaces() {
-	ctx.interfaces = sortAndDedupSlice(ctx.interfaces)
+	ctx.interfaces = clangtool.SortAndDedupSlice(ctx.interfaces)
 	count := make(map[string]int)
 	for _, iface := range ctx.interfaces {
 		count[iface.Type+iface.Name]++
@@ -77,7 +78,7 @@ func (ctx *context) finishInterfaces() {
 			iface.Access = AccessUnknown
 		}
 	}
-	ctx.interfaces = sortAndDedupSlice(ctx.interfaces)
+	ctx.interfaces = clangtool.SortAndDedupSlice(ctx.interfaces)
 }
 
 func (ctx *context) processFunctions() {
diff --git a/pkg/declextract/typing.go b/pkg/declextract/typing.go
index f5fcdbdd3..e839d7bc4 100644
--- a/pkg/declextract/typing.go
+++ b/pkg/declextract/typing.go
@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"slices"
 	"strings"
+
+	"github.com/google/syzkaller/pkg/clangtool"
 )
 
 // Argument/field type inference based on data flow analysis.
@@ -346,7 +348,7 @@ func (ctx *context) inferCommandVariants(name, file string, arg int) []string {
 		visited := make(map[*typingNode]bool)
 		ctx.walkCommandVariants(n, &variants, visited, 0)
 	}
-	return sortAndDedupSlice(variants)
+	return clangtool.SortAndDedupSlice(variants)
 }
 
 func (ctx *context) collectCommandVariants(fn *Function, arg int, variants *[]string) {
diff --git a/tools/syz-declextract/declextract.go b/tools/syz-declextract/declextract.go
index 6ffca0170..d2bb505e4 100644
--- a/tools/syz-declextract/declextract.go
+++ b/tools/syz-declextract/declextract.go
@@ -151,7 +151,7 @@ func prepare(cfg *config) (*declextract.Output, *ifaceprobe.Info, []*cover.FileC
 	var out *declextract.Output
 	eg.Go(func() error {
 		var err error
-		out, err = clangtool.Run(cfg.Config)
+		out, err = clangtool.Run[declextract.Output](cfg.Config)
 		if err != nil {
 			return err
 		}
diff --git a/tools/syz-declextract/declextract_test.go b/tools/syz-declextract/declextract_test.go
index 5e4df703b..879e49707 100644
--- a/tools/syz-declextract/declextract_test.go
+++ b/tools/syz-declextract/declextract_test.go
@@ -15,6 +15,7 @@ import (
 	"github.com/google/syzkaller/pkg/ast"
 	"github.com/google/syzkaller/pkg/clangtool"
 	"github.com/google/syzkaller/pkg/compiler"
+	"github.com/google/syzkaller/pkg/declextract"
 	"github.com/google/syzkaller/pkg/ifaceprobe"
 	"github.com/google/syzkaller/pkg/osutil"
 	"github.com/google/syzkaller/pkg/testutil"
@@ -30,7 +31,7 @@ func TestClangTool(t *testing.T) {
 		t.Skipf("syz-declextract path is not specified, run with -bin=syz-declextract flag")
 	}
 	testEachFile(t, func(t *testing.T, cfg *clangtool.Config, file string) {
-		out, err := clangtool.Run(cfg)
+		out, err := clangtool.Run[declextract.Output](cfg)
 		if err != nil {
 			t.Fatal(err)
 		}