pkg/ast, pkg/compiler: support per-file metadata

We have a bunch of hacks in syz-extract, syz-sysgen and syz-check
with respect to description files unsupported on some arches,
or that must not be part of make extract.

Add 2 meta attribtues to files:

meta noextract
Tells `make extract` to not extract constants for this file.
Though, `syz-extract` can still be invoked manually on this file.

meta arches["arch1", "arch2"]
Restricts this file only to the given set of architectures.
`make extract` and ``make generate` will not use it on other architectures.

Later we can potentially use meta attributes to specify git tree/commit
that must be used for extraction. Maybe something else.

Fixes #2754

2 files changed, 42 insertions, 76 deletions

diff --git a/sys/syz-extract/extract.go b/sys/syz-extract/extract.go
index 78666ac3f..117c10e58 100644
--- a/sys/syz-extract/extract.go
+++ b/sys/syz-extract/extract.go
@@ -75,17 +75,12 @@ func main() {
 	if *flagBuild && *flagBuildDir != "" {
 		tool.Failf("-build and -builddir is an invalid combination")
 	}
-
-	OS, archArray, files, err := archFileList(*flagOS, *flagArch, flag.Args())
-	if err != nil {
-		tool.Fail(err)
-	}
-
+	OS := *flagOS
 	extractor := extractors[OS]
 	if extractor == nil {
 		tool.Failf("unknown os: %v", OS)
 	}
-	arches, err := createArches(OS, archArray, files)
+	arches, err := createArches(OS, archList(OS, *flagArch), flag.Args())
 	if err != nil {
 		tool.Fail(err)
 	}
@@ -97,7 +92,7 @@ func main() {
 		tool.Fail(err)
 	}
 
-	jobC := make(chan interface{}, len(archArray)*len(files))
+	jobC := make(chan interface{}, len(arches))
 	for _, arch := range arches {
 		jobC <- arch
 	}
@@ -108,11 +103,8 @@ func main() {
 
 	failed := false
 	constFiles := make(map[string]*compiler.ConstFile)
-	for _, file := range files {
-		constFiles[file] = compiler.NewConstFile()
-	}
 	for _, arch := range arches {
-		fmt.Printf("generating %v/%v...\n", arch.target.OS, arch.target.Arch)
+		fmt.Printf("generating %v/%v...\n", OS, arch.target.Arch)
 		<-arch.done
 		if arch.err != nil {
 			failed = true
@@ -126,6 +118,9 @@ func main() {
 				fmt.Printf("%v: %v\n", f.name, f.err)
 				continue
 			}
+			if constFiles[f.name] == nil {
+				constFiles[f.name] = compiler.NewConstFile()
+			}
 			constFiles[f.name].AddArch(f.arch.target.Arch, f.consts, f.undeclared)
 		}
 	}
@@ -175,6 +170,18 @@ func worker(extractor Extractor, jobC chan interface{}) {
 }
 
 func createArches(OS string, archArray, files []string) ([]*Arch, error) {
+	errBuf := new(bytes.Buffer)
+	eh := func(pos ast.Pos, msg string) {
+		fmt.Fprintf(errBuf, "%v: %v\n", pos, msg)
+	}
+	top := ast.ParseGlob(filepath.Join("sys", OS, "*.txt"), eh)
+	if top == nil {
+		return nil, fmt.Errorf("%v", errBuf.String())
+	}
+	allFiles := compiler.FileList(top, OS, eh)
+	if allFiles == nil {
+		return nil, fmt.Errorf("%v", errBuf.String())
+	}
 	var arches []*Arch
 	for _, archStr := range archArray {
 		buildDir := ""
@@ -203,7 +210,17 @@ func createArches(OS string, archArray, files []string) ([]*Arch, error) {
 			build:       *flagBuild,
 			done:        make(chan bool),
 		}
-		for _, f := range files {
+		archFiles := files
+		if len(archFiles) == 0 {
+			for file, meta := range allFiles {
+				if meta.NoExtract || !meta.SupportsArch(archStr) {
+					continue
+				}
+				archFiles = append(archFiles, file)
+			}
+		}
+		sort.Strings(archFiles)
+		for _, f := range archFiles {
 			arch.files = append(arch.files, &File{
 				arch: arch,
 				name: f,
@@ -215,6 +232,18 @@ func createArches(OS string, archArray, files []string) ([]*Arch, error) {
 	return arches, nil
 }
 
+func archList(OS, arches string) []string {
+	if arches != "" {
+		return strings.Split(arches, ",")
+	}
+	var archArray []string
+	for arch := range targets.List[OS] {
+		archArray = append(archArray, arch)
+	}
+	sort.Strings(archArray)
+	return archArray
+}
+
 func checkUnsupportedCalls(arches []*Arch) bool {
 	supported := make(map[string]bool)
 	unsupported := make(map[string]string)
@@ -240,60 +269,6 @@ func checkUnsupportedCalls(arches []*Arch) bool {
 	return failed
 }
 
-func archFileList(os, arch string, files []string) (string, []string, []string, error) {
-	// Note: this is linux-specific and should be part of Extractor and moved to linux.go.
-	android := false
-	if os == "android" {
-		android = true
-		os = targets.Linux
-	}
-	var arches []string
-	if arch != "" {
-		arches = strings.Split(arch, ",")
-	} else {
-		for arch := range targets.List[os] {
-			arches = append(arches, arch)
-		}
-		if android {
-			arches = []string{targets.I386, targets.AMD64, targets.ARM, targets.ARM64}
-		}
-		sort.Strings(arches)
-	}
-	if len(files) == 0 {
-		matches, err := filepath.Glob(filepath.Join("sys", os, "*.txt"))
-		if err != nil || len(matches) == 0 {
-			return "", nil, nil, fmt.Errorf("failed to find sys files: %v", err)
-		}
-		manualFiles := map[string]bool{
-			// Not upstream, generated on https://github.com/multipath-tcp/mptcp_net-next
-			"vnet_mptcp.txt": true,
-			// Not upstream, generated on:
-			// https://chromium.googlesource.com/chromiumos/third_party/kernel d2a8a1eb8b86
-			"dev_bifrost.txt": true,
-			// Not upstream, generated on unknown tree.
-			"dev_img_rogue.txt": true,
-		}
-		androidFiles := map[string]bool{
-			"dev_tlk_device.txt": true,
-			// This was generated on:
-			// https://source.codeaurora.org/quic/la/kernel/msm-4.9 msm-4.9
-			"dev_video4linux.txt": true,
-			// This was generated on:
-			// https://chromium.googlesource.com/chromiumos/third_party/kernel 3a36438201f3
-			"fs_incfs.txt": true,
-		}
-		for _, f := range matches {
-			f = filepath.Base(f)
-			if manualFiles[f] || os == targets.Linux && android != androidFiles[f] {
-				continue
-			}
-			files = append(files, f)
-		}
-		sort.Strings(files)
-	}
-	return os, arches, files, nil
-}
-
 func processArch(extractor Extractor, arch *Arch) (map[string]*compiler.ConstInfo, error) {
 	errBuf := new(bytes.Buffer)
 	eh := func(pos ast.Pos, msg string) {
diff --git a/sys/syz-extract/linux.go b/sys/syz-extract/linux.go
index a07d6a968..10284d221 100644
--- a/sys/syz-extract/linux.go
+++ b/sys/syz-extract/linux.go
@@ -13,7 +13,6 @@ import (
 	"github.com/google/syzkaller/pkg/build"
 	"github.com/google/syzkaller/pkg/compiler"
 	"github.com/google/syzkaller/pkg/osutil"
-	"github.com/google/syzkaller/sys/targets"
 )
 
 type linux struct{}
@@ -117,14 +116,6 @@ func (*linux) prepareArch(arch *Arch) error {
 }
 
 func (*linux) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
-	if strings.HasSuffix(info.File, "_kvm.txt") &&
-		(arch.target.Arch == targets.ARM || arch.target.Arch == targets.RiscV64) {
-		// Hack: KVM is not supported on ARM anymore. We may want some more official support
-		// for marking descriptions arch-specific, but so far this combination is the only
-		// one. For riscv64, KVM is not supported yet but might be in the future.
-		// Note: syz-sysgen also ignores this file for arm and riscv64.
-		return nil, nil, nil
-	}
 	headerArch := arch.target.KernelHeaderArch
 	sourceDir := arch.sourceDir
 	buildDir := arch.buildDir