pkg/host: move glob parsing to host

Move more complex glob processing to the host (into prog package).
Make fuzzer just read and return globs if requested.

This moves us closer to #1541

9 files changed, 101 insertions, 147 deletions

diff --git a/pkg/host/machine_info.go b/pkg/host/machine_info.go
index c32a0cf3b..5e2e9e335 100644
--- a/pkg/host/machine_info.go
+++ b/pkg/host/machine_info.go
@@ -9,15 +9,6 @@ import (
 	"strings"
 )
 
-func CollectGlobsInfo(globs map[string]bool) (map[string][]string, error) {
-	if machineGlobsInfo == nil {
-		return nil, nil
-	}
-	return machineGlobsInfo(globs)
-}
-
-var machineGlobsInfo func(map[string]bool) (map[string][]string, error)
-
 type KernelModule struct {
 	Name string `json:"Name"`
 	Addr uint64 `json:"Addr"`
diff --git a/pkg/host/machine_info_linux.go b/pkg/host/machine_info_linux.go
deleted file mode 100644
index fceb3ab78..000000000
--- a/pkg/host/machine_info_linux.go
+++ /dev/null
@@ -1,78 +0,0 @@
-// Copyright 2020 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package host
-
-import (
-	"path/filepath"
-	"strings"
-)
-
-func init() {
-	machineGlobsInfo = getGlobsInfo
-}
-
-func getGlobsInfo(globs map[string]bool) (map[string][]string, error) {
-	var err error
-	files := make(map[string][]string, len(globs))
-	for glob := range globs {
-		var (
-			addglobs []string
-			subglobs []string
-			matches  []string
-		)
-		tokens := strings.Split(glob, ":")
-		for _, tok := range tokens {
-			if strings.HasPrefix(tok, "-") {
-				subglobs = append(subglobs, tok[1:])
-			} else {
-				addglobs = append(addglobs, tok)
-			}
-		}
-		for _, g := range addglobs {
-			m, err := filepath.Glob(g)
-			if err != nil {
-				return nil, err
-			}
-			matches = append(matches, m...)
-		}
-		files[glob], err = excludeGlobs(removeDupValues(matches), subglobs)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return files, nil
-}
-
-func excludeGlobs(items, subglobs []string) ([]string, error) {
-	var results []string
-	excludes := make(map[string]bool)
-	for _, glob := range subglobs {
-		matches, err := filepath.Glob(glob)
-		if err != nil {
-			return nil, err
-		}
-		for _, m := range matches {
-			excludes[m] = true
-		}
-	}
-
-	for _, item := range items {
-		if !excludes[item] {
-			results = append(results, item)
-		}
-	}
-	return results, nil
-}
-
-func removeDupValues(slice []string) []string {
-	keys := make(map[string]bool)
-	list := []string{}
-	for _, entry := range slice {
-		if !keys[entry] {
-			keys[entry] = true
-			list = append(list, entry)
-		}
-	}
-	return list
-}
diff --git a/pkg/host/machine_info_linux_test.go b/pkg/host/machine_info_linux_test.go
deleted file mode 100644
index 789f34969..000000000
--- a/pkg/host/machine_info_linux_test.go
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright 2020 syzkaller project authors. All rights reserved.
-// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
-
-package host
-
-import (
-	"path/filepath"
-	"testing"
-
-	"github.com/google/go-cmp/cmp"
-	"github.com/google/syzkaller/pkg/osutil"
-)
-
-func TestGetGlobsInfo(t *testing.T) {
-	dir := t.TempDir()
-	if err := osutil.MkdirAll(filepath.Join(dir, "a", "b", "c", "d")); err != nil {
-		t.Fatal(err)
-	}
-	if err := osutil.MkdirAll(filepath.Join(dir, "a", "b", "c", "e")); err != nil {
-		t.Fatal(err)
-	}
-	if err := osutil.MkdirAll(filepath.Join(dir, "a", "c", "d")); err != nil {
-		t.Fatal(err)
-	}
-	if err := osutil.MkdirAll(filepath.Join(dir, "a", "c", "e")); err != nil {
-		t.Fatal(err)
-	}
-
-	glob := filepath.Join(dir, "a/**/*") + ":-" + filepath.Join(dir, "a/c/e")
-	globs := map[string]bool{
-		glob: true,
-	}
-	infos, err := getGlobsInfo(globs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	want := []string{
-		filepath.Join(dir, "a/b/c"),
-		filepath.Join(dir, "a/c/d"),
-	}
-	if diff := cmp.Diff(infos[glob], want); diff != "" {
-		t.Fatal(diff)
-	}
-}
diff --git a/pkg/rpctype/rpctype.go b/pkg/rpctype/rpctype.go
index 4fd0014f5..26765213f 100644
--- a/pkg/rpctype/rpctype.go
+++ b/pkg/rpctype/rpctype.go
@@ -89,6 +89,7 @@ type ConnectRes struct {
 	Features *host.Features
 	// Fuzzer reads these files inside of the VM and returns contents in CheckArgs.Files.
 	ReadFiles []string
+	ReadGlobs []string
 }
 
 type CheckArgs struct {
@@ -97,7 +98,7 @@ type CheckArgs struct {
 	EnabledCalls  map[string][]int
 	DisabledCalls map[string][]SyscallReason
 	Features      *host.Features
-	GlobFiles     map[string][]string
+	Globs         map[string][]string
 	Files         []host.FileInfo
 }
 
diff --git a/prog/target.go b/prog/target.go
index 879cffb4f..bb49a6f5c 100644
--- a/prog/target.go
+++ b/prog/target.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"math/rand"
 	"sort"
+	"strings"
 	"sync"
 	"sync/atomic"
 )
@@ -246,30 +247,73 @@ func (target *Target) DefaultChoiceTable() *ChoiceTable {
 	return target.defaultChoiceTable
 }
 
-func (target *Target) GetGlobs() map[string]bool {
+func (target *Target) RequiredGlobs() []string {
 	globs := make(map[string]bool)
 	ForeachType(target.Syscalls, func(typ Type, ctx *TypeCtx) {
 		switch a := typ.(type) {
 		case *BufferType:
 			if a.Kind == BufferGlob {
-				globs[a.SubKind] = true
+				for _, glob := range requiredGlobs(a.SubKind) {
+					globs[glob] = true
+				}
 			}
 		}
 	})
-	return globs
+	return stringMapToSlice(globs)
 }
 
 func (target *Target) UpdateGlobs(globFiles map[string][]string) {
+	// TODO: make host.DetectSupportedSyscalls below filter out globs with no values.
+	// Also make prog package more strict with respect to generation/mutation of globs
+	// with no values (they still can appear in tests and tools). We probably should
+	// generate an empty string for these and never mutate.
 	ForeachType(target.Syscalls, func(typ Type, ctx *TypeCtx) {
 		switch a := typ.(type) {
 		case *BufferType:
 			if a.Kind == BufferGlob {
-				a.Values = globFiles[a.SubKind]
+				a.Values = populateGlob(a.SubKind, globFiles)
 			}
 		}
 	})
 }
 
+func requiredGlobs(pattern string) []string {
+	var res []string
+	for _, tok := range strings.Split(pattern, ":") {
+		if tok[0] != '-' {
+			res = append(res, tok)
+		}
+	}
+	return res
+}
+
+func populateGlob(pattern string, globFiles map[string][]string) []string {
+	files := make(map[string]bool)
+	parts := strings.Split(pattern, ":")
+	for _, tok := range parts {
+		if tok[0] != '-' {
+			for _, file := range globFiles[tok] {
+				files[file] = true
+			}
+		}
+	}
+	for _, tok := range parts {
+		if tok[0] == '-' {
+			delete(files, tok[1:])
+		}
+	}
+	return stringMapToSlice(files)
+}
+
+func stringMapToSlice(m map[string]bool) []string {
+	var res []string
+	for k := range m {
+		res = append(res, k)
+	}
+	sort.Strings(res)
+	return res
+}
+
 type Gen struct {
 	r *randGen
 	s *state
diff --git a/prog/target_test.go b/prog/target_test.go
new file mode 100644
index 000000000..4db6fdef1
--- /dev/null
+++ b/prog/target_test.go
@@ -0,0 +1,36 @@
+// Copyright 2024 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package prog
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRequiredGlobs(t *testing.T) {
+	assert.Equal(t, requiredGlobs("aa/bb"), []string{"aa/bb"})
+	assert.Equal(t, requiredGlobs("aa:bb"), []string{"aa", "bb"})
+	assert.Equal(t, requiredGlobs("aa:bb:-cc:dd"), []string{"aa", "bb", "dd"})
+}
+
+func TestPopulateGlob(t *testing.T) {
+	assert.Empty(t, populateGlob("aa", map[string][]string{
+		"bb": {"c"},
+	}))
+	assert.Equal(t, populateGlob("aa", map[string][]string{
+		"aa": {"d", "e"},
+		"bb": {"c"},
+	}), []string{"d", "e"})
+	assert.Equal(t, populateGlob("aa:cc", map[string][]string{
+		"aa": {"d", "e"},
+		"bb": {"c"},
+		"cc": {"f", "d"},
+	}), []string{"d", "e", "f"})
+	assert.Equal(t, populateGlob("aa:cc:-e", map[string][]string{
+		"aa": {"d", "e"},
+		"bb": {"c"},
+		"cc": {"f", "d"},
+	}), []string{"d", "f"})
+}
diff --git a/syz-fuzzer/fuzzer.go b/syz-fuzzer/fuzzer.go
index 6668f5235..48bdc1f96 100644
--- a/syz-fuzzer/fuzzer.go
+++ b/syz-fuzzer/fuzzer.go
@@ -9,6 +9,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 	"os"
+	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"sync"
@@ -199,6 +200,14 @@ func main() {
 	}
 	checkReq.Name = *flagName
 	checkReq.Files = host.ReadFiles(r.ReadFiles)
+	checkReq.Globs = make(map[string][]string)
+	for _, glob := range r.ReadGlobs {
+		files, err := filepath.Glob(filepath.FromSlash(glob))
+		if err != nil && checkReq.Error == "" {
+			checkReq.Error = fmt.Sprintf("failed to read glob %q: %v", glob, err)
+		}
+		checkReq.Globs[glob] = files
+	}
 	checkRes := new(rpctype.CheckRes)
 	if err := manager.Call("Manager.Check", checkReq, checkRes); err != nil {
 		log.SyzFatalf("Manager.Check call failed: %v", err)
diff --git a/syz-fuzzer/testing.go b/syz-fuzzer/testing.go
index 467d1b1a9..ceecaf76a 100644
--- a/syz-fuzzer/testing.go
+++ b/syz-fuzzer/testing.go
@@ -124,15 +124,6 @@ func checkMachine(args *checkArgs) (*rpctype.CheckArgs, error) {
 	if err := checkRevisions(args); err != nil {
 		return nil, err
 	}
-	globFiles, err := host.CollectGlobsInfo(args.target.GetGlobs())
-	if err != nil {
-		return nil, fmt.Errorf("failed to collect glob info: %w", err)
-	}
-	// TODO: make host.DetectSupportedSyscalls below filter out globs with no values.
-	// Also make prog package more strict with respect to generation/mutation of globs
-	// with no values (they still can appear in tests and tools). We probably should
-	// generate an empty string for these and never mutate.
-	args.target.UpdateGlobs(globFiles)
 	features, err := host.Check(args.target)
 	if err != nil {
 		return nil, err
@@ -161,7 +152,6 @@ func checkMachine(args *checkArgs) (*rpctype.CheckArgs, error) {
 		Features:      features,
 		EnabledCalls:  make(map[string][]int),
 		DisabledCalls: make(map[string][]rpctype.SyscallReason),
-		GlobFiles:     globFiles,
 	}
 	if err := checkCalls(args, res); err != nil {
 		return nil, err
diff --git a/syz-manager/rpc.go b/syz-manager/rpc.go
index 6ba15b103..2503353cb 100644
--- a/syz-manager/rpc.go
+++ b/syz-manager/rpc.go
@@ -27,6 +27,7 @@ import (
 type RPCServer struct {
 	mgr     RPCManagerView
 	cfg     *mgrconfig.Config
+	target  *prog.Target
 	server  *rpctype.RPCServer
 	checker *vminfo.Checker
 	port    int
@@ -94,6 +95,7 @@ func startRPCServer(mgr *Manager) (*RPCServer, error) {
 	serv := &RPCServer{
 		mgr:       mgr,
 		cfg:       mgr.cfg,
+		target:    mgr.target,
 		checker:   vminfo.New(mgr.cfg),
 		statExecs: mgr.statExecs,
 		statExecRetries: stats.Create("exec retries",
@@ -136,6 +138,9 @@ func (serv *RPCServer) Connect(a *rpctype.ConnectArgs, r *rpctype.ConnectRes) er
 	r.TargetRevision = serv.cfg.Target.Revision
 	r.Features = serv.checkFeatures
 	r.ReadFiles = serv.checker.RequiredFiles()
+	if !serv.checkDone {
+		r.ReadGlobs = serv.target.RequiredGlobs()
+	}
 	return nil
 }
 
@@ -216,7 +221,7 @@ func (serv *RPCServer) check(a *rpctype.CheckArgs, modules []host.KernelModule)
 	}
 	serv.checkFeatures = a.Features
 	serv.canonicalModules = cover.NewCanonicalizer(modules, serv.cfg.Cover)
-	serv.mgr.machineChecked(a.Features, a.GlobFiles, serv.targetEnabledSyscalls, modules)
+	serv.mgr.machineChecked(a.Features, a.Globs, serv.targetEnabledSyscalls, modules)
 	return nil
 }