pkg/ast, pkg/compiler: support per-file metadata

We have a bunch of hacks in syz-extract, syz-sysgen and syz-check
with respect to description files unsupported on some arches,
or that must not be part of make extract.

Add 2 meta attribtues to files:

meta noextract
Tells `make extract` to not extract constants for this file.
Though, `syz-extract` can still be invoked manually on this file.

meta arches["arch1", "arch2"]
Restricts this file only to the given set of architectures.
`make extract` and ``make generate` will not use it on other architectures.

Later we can potentially use meta attributes to specify git tree/commit
that must be used for extraction. Maybe something else.

Fixes #2754

5 files changed, 120 insertions, 1 deletions

diff --git a/pkg/compiler/check.go b/pkg/compiler/check.go
index ae9724dfc..402e9a84e 100644
--- a/pkg/compiler/check.go
+++ b/pkg/compiler/check.go
@@ -836,11 +836,14 @@ type checkCtx struct {
 }
 
 func (comp *compiler) checkType(ctx checkCtx, t *ast.Type, flags checkFlags) {
+	comp.checkTypeImpl(ctx, t, comp.getTypeDesc(t), flags)
+}
+
+func (comp *compiler) checkTypeImpl(ctx checkCtx, t *ast.Type, desc *typeDesc, flags checkFlags) {
 	if unexpected, _, ok := checkTypeKind(t, kindIdent); !ok {
 		comp.error(t.Pos, "unexpected %v, expect type", unexpected)
 		return
 	}
-	desc := comp.getTypeDesc(t)
 	if desc == nil {
 		comp.error(t.Pos, "unknown type %v", t.Ident)
 		return
diff --git a/pkg/compiler/compiler.go b/pkg/compiler/compiler.go
index 62a102d40..b3e39be06 100644
--- a/pkg/compiler/compiler.go
+++ b/pkg/compiler/compiler.go
@@ -7,6 +7,7 @@ package compiler
 
 import (
 	"fmt"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
@@ -74,6 +75,7 @@ func createCompiler(desc *ast.Description, target *targets.Target, eh ast.ErrorH
 // Compile compiles sys description.
 func Compile(desc *ast.Description, consts map[string]uint64, target *targets.Target, eh ast.ErrorHandler) *Prog {
 	comp := createCompiler(desc.Clone(), target, eh)
+	comp.filterArch()
 	comp.typecheck()
 	// The subsequent, more complex, checks expect basic validity of the tree,
 	// in particular corrent number of type arguments. If there were errors,
@@ -151,6 +153,24 @@ func (comp *compiler) warning(pos ast.Pos, msg string, args ...interface{}) {
 	comp.warnings = append(comp.warnings, warn{pos, fmt.Sprintf(msg, args...)})
 }
 
+func (comp *compiler) filterArch() {
+	files := comp.fileList()
+	comp.desc = comp.desc.Filter(func(n ast.Node) bool {
+		pos, typ, name := n.Info()
+		meta := files[filepath.Base(pos.File)]
+		if meta.SupportsArch(comp.target.Arch) {
+			return true
+		}
+		switch n.(type) {
+		case *ast.Resource, *ast.Struct, *ast.Call, *ast.TypeDef:
+			// This is required to keep the unsupported diagnostic working,
+			// otherwise sysgen will think that these things are still supported on some arches.
+			comp.unsupported[typ+" "+name] = true
+		}
+		return false
+	})
+}
+
 func (comp *compiler) structIsVarlen(name string) bool {
 	if varlen, ok := comp.structVarlen[name]; ok {
 		return varlen
diff --git a/pkg/compiler/meta.go b/pkg/compiler/meta.go
new file mode 100644
index 000000000..f0221d0b9
--- /dev/null
+++ b/pkg/compiler/meta.go
@@ -0,0 +1,88 @@
+// Copyright 2022 syzkaller project authors. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package compiler
+
+import (
+	"path/filepath"
+
+	"github.com/google/syzkaller/pkg/ast"
+	"github.com/google/syzkaller/sys/targets"
+)
+
+type Meta struct {
+	NoExtract bool
+	Arches    map[string]bool
+}
+
+func (meta *Meta) SupportsArch(arch string) bool {
+	return len(meta.Arches) == 0 || meta.Arches[arch]
+}
+
+func FileList(desc *ast.Description, OS string, eh ast.ErrorHandler) map[string]Meta {
+	// Use any target for this OS.
+	for _, target := range targets.List[OS] {
+		return createCompiler(desc, target, eh).fileList()
+	}
+	return nil
+}
+
+func (comp *compiler) fileList() map[string]Meta {
+	files := make(map[string]Meta)
+	for _, n := range comp.desc.Nodes {
+		pos, _, _ := n.Info()
+		file := filepath.Base(pos.File)
+		if file == ast.BuiltinFile {
+			continue
+		}
+		meta := files[file]
+		switch n := n.(type) {
+		case *ast.Meta:
+			errors0 := comp.errors
+			comp.checkTypeImpl(checkCtx{}, n.Value, metaTypes[n.Value.Ident], 0)
+			if errors0 != comp.errors {
+				break
+			}
+			switch n.Value.Ident {
+			case metaNoExtract.Names[0]:
+				meta.NoExtract = true
+			case metaArches.Names[0]:
+				meta.Arches = make(map[string]bool)
+				for _, arg := range n.Value.Args {
+					meta.Arches[arg.String] = true
+				}
+			}
+		}
+		files[file] = meta
+	}
+	if comp.errors != 0 {
+		return nil
+	}
+	return files
+}
+
+var metaTypes = map[string]*typeDesc{
+	metaNoExtract.Names[0]: metaNoExtract,
+	metaArches.Names[0]:    metaArches,
+}
+
+var metaNoExtract = &typeDesc{
+	Names:     []string{"noextract"},
+	CantBeOpt: true,
+}
+
+var metaArches = &typeDesc{
+	Names:     []string{"arches"},
+	CantBeOpt: true,
+	OptArgs:   8,
+	Args:      []namedArg{metaArch, metaArch, metaArch, metaArch, metaArch, metaArch, metaArch, metaArch},
+}
+
+var metaArch = namedArg{Name: "arch", Type: &typeArg{
+	Kind: kindString,
+	Check: func(comp *compiler, t *ast.Type) {
+		if targets.List[comp.target.OS][t.String] == nil {
+			comp.error(t.Pos, "unknown arch %v", t.String)
+		}
+	},
+}}
diff --git a/pkg/compiler/testdata/all.txt b/pkg/compiler/testdata/all.txt
index c9a02f592..d03e35def 100644
--- a/pkg/compiler/testdata/all.txt
+++ b/pkg/compiler/testdata/all.txt
@@ -1,6 +1,9 @@
 # Copyright 2018 syzkaller project authors. All rights reserved.
 # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
 
+meta noextract
+meta arches["32_shmem", "32_fork_shmem", "64", "64_fork"]
+
 foo_0(a int8)
 foo_1(a int8[C1:C2])
 foo_2(a ptr[out, array[int32]])
diff --git a/pkg/compiler/testdata/errors.txt b/pkg/compiler/testdata/errors.txt
index 6d258cf50..1201fa361 100644
--- a/pkg/compiler/testdata/errors.txt
+++ b/pkg/compiler/testdata/errors.txt
@@ -3,6 +3,11 @@
 
 # Errors that happen during type checking phase.
 
+meta foobar			### unknown type foobar
+meta noextract["foo"]		### wrong number of arguments for type noextract, expect no arguments
+meta "foobar"			### unexpected string "foobar", expect type
+meta arches["z80"]		### unknown arch z80
+
 #include "something"		### confusing comment faking a directive (rephrase if it's intentional)
 #define FOO BAR			### confusing comment faking a directive (rephrase if it's intentional)
 # include "something"		### confusing comment faking a directive (rephrase if it's intentional)