diff options
| -rw-r--r-- | tools/syz-kconf/kconf.go | 376 | ||||
| -rw-r--r-- | tools/syz-kconf/parser.go | 283 |
2 files changed, 659 insertions, 0 deletions
diff --git a/tools/syz-kconf/kconf.go b/tools/syz-kconf/kconf.go new file mode 100644 index 000000000..2dcdfa5f6 --- /dev/null +++ b/tools/syz-kconf/kconf.go @@ -0,0 +1,376 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +// syz-kconf generates Linux kernel configs in dashboard/config/linux. +// See dashboard/config/linux/README.md for details. +package main + +import ( + "flag" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "github.com/google/syzkaller/pkg/kconfig" + "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/pkg/vcs" + "github.com/google/syzkaller/sys/targets" +) + +const ( + featOverride = "override" + featOptional = "optional" + featWeak = "weak" + featBaseline = "baseline" + featClang = "clang" + featAndroid = "android" +) + +func main() { + var ( + flagSourceDir = flag.String("sourcedir", "", "sourcedir") + flagConfig = flag.String("config", "", "config") + flagInstance = flag.String("instance", "", "instance") + ) + flag.Parse() + failf := func(msg string, args ...interface{}) { + fmt.Fprintf(os.Stderr, msg+"\n", args...) + os.Exit(1) + } + fail := func(err error) { + failf("%v", err) + } + if *flagSourceDir == "" { + failf("missing mandatory flag -sourcedir") + } + repo, err := vcs.NewRepo("linux", "", *flagSourceDir, vcs.OptPrecious) + if err != nil { + failf("failed to create repo: %v", err) + } + instances, err := parseMainSpec(*flagConfig) + if err != nil { + fail(err) + } + // In order to speed up the process we generate instances that use the same kernel revision in parallel. + generated := make(map[string]bool) + for _, inst := range instances { + // Find the first instance that we did not generate yet. + if *flagInstance != "" && *flagInstance != inst.Name || generated[inst.Name] { + continue + } + fmt.Printf("git checkout %v %v\n", inst.Kernel.Repo, inst.Kernel.Tag) + if _, err := repo.SwitchCommit(inst.Kernel.Tag); err != nil { + if _, err := repo.CheckoutCommit(inst.Kernel.Repo, inst.Kernel.Tag); err != nil { + failf("failed to checkout %v/%v: %v", inst.Kernel.Repo, inst.Kernel.Tag, err) + } + } + kconf, err := kconfig.Parse(filepath.Join(*flagSourceDir, "Kconfig")) + if err != nil { + fail(err) + } + releaseTag, err := repo.ReleaseTag("HEAD") + if err != nil { + fail(err) + } + fmt.Printf("kernel release %v\n", releaseTag) + // Now generate all instances that use this kernel revision in parallel (each will use own build dir). + batch := 0 + results := make(chan error) + for _, inst1 := range instances { + if *flagInstance != "" && *flagInstance != inst1.Name || generated[inst1.Name] || inst1.Kernel != inst.Kernel { + continue + } + fmt.Printf("generating %v...\n", inst1.Name) + generated[inst1.Name] = true + batch++ + ctx := &Context{ + Inst: inst1, + ConfigDir: filepath.Dir(*flagConfig), + SourceDir: *flagSourceDir, + ReleaseTag: releaseTag, + Kconf: kconf, + } + go func() { + if err := ctx.generate(); err != nil { + results <- fmt.Errorf("%v failed:\n%v", ctx.Inst.Name, err) + } + results <- nil + }() + } + for i := 0; i < batch; i++ { + if err := <-results; err != nil { + fail(err) + } + } + } + if len(generated) == 0 { + failf("unknown instance name") + } +} + +// Generation context for a single instance. +type Context struct { + Inst *Instance + Target *targets.Target + Kconf *kconfig.KConfig + ConfigDir string + BuildDir string + SourceDir string + ReleaseTag string +} + +func (ctx *Context) generate() error { + var err error + if ctx.BuildDir, err = ioutil.TempDir("", "syz-kconf"); err != nil { + return err + } + defer os.RemoveAll(ctx.BuildDir) + if err := ctx.setTarget(); err != nil { + return err + } + if err := ctx.setReleaseFeatures(); err != nil { + return err + } + if err := ctx.mrProper(); err != nil { + return err + } + if err := ctx.executeShell(); err != nil { + return err + } + configFile := filepath.Join(ctx.BuildDir, ".config") + cf, err := kconfig.ParseConfig(configFile) + if err != nil { + return err + } + // TODO: remove nousb, it is only used for upstream-kasan for transition from old configs. + if !ctx.Inst.Features[featBaseline] && !ctx.Inst.Features["nousb"] { + if err := ctx.addUSBConfigs(cf); err != nil { + return err + } + } + ctx.applyConfigs(cf) + cf.ModToYes() + original := cf.Serialize() + if err := osutil.WriteFile(configFile, original); err != nil { + return fmt.Errorf("failed to write .config file: %v", err) + } + if err := ctx.Make("oldconfig"); err != nil { + return err + } + cf, err = kconfig.ParseConfig(configFile) + if err != nil { + return err + } + verifyErr := ctx.verifyConfigs(cf) + cf.ModToNo() + config := []byte(fmt.Sprintf(`# Automatically generated by syz-kconf; DO NOT EDIT. +# Kernel: %v %v + +%s +%s +`, + ctx.Inst.Kernel.Repo, ctx.Inst.Kernel.Tag, cf.Serialize(), ctx.Inst.Verbatim)) + outputFile := filepath.Join(ctx.ConfigDir, ctx.Inst.Name+".config") + if verifyErr != nil { + // Write bad config for debugging with .tmp suffix so that it's not checked-in accidentially. + outputFile += ".tmp" + if err := osutil.WriteFile(outputFile, original); err != nil { + return fmt.Errorf("failed to write output file: %v", err) + } + return fmt.Errorf("%vsaved config to %v (run oldconfig)", verifyErr, outputFile) + } + return osutil.WriteFile(outputFile, config) +} + +func (ctx *Context) executeShell() error { + for _, shell := range ctx.Inst.Shell { + if !ctx.Inst.Features.Match(shell.Constraints) { + continue + } + args := strings.Split(shell.Cmd, " ") + if args[0] != "make" { + return fmt.Errorf("non-make shell is not supported yet") + } + if err := ctx.Make(args[1:]...); err != nil { + return err + } + } + return nil +} + +func (ctx *Context) applyConfigs(cf *kconfig.ConfigFile) { + for _, cfg := range ctx.Inst.Configs { + if !ctx.Inst.Features.Match(cfg.Constraints) { + continue + } + if cfg.Value != kconfig.No { + // If this is a choice, first unset all other options. + // If we leave 2 choice options enabled, the last one will win. + // It can make sense to move this code to kconfig in some form, + // it's needed everywhere configs are changed. + if m := ctx.Kconf.Configs[cfg.Name]; m != nil && m.Parent.Kind == kconfig.MenuChoice { + for _, choice := range m.Parent.Elems { + cf.Unset(choice.Name) + } + } + } + cf.Set(cfg.Name, cfg.Value) + } +} + +func (ctx *Context) verifyConfigs(cf *kconfig.ConfigFile) error { + errs := new(Errors) + for _, cfg := range ctx.Inst.Configs { + act := cf.Value(cfg.Name) + if act == cfg.Value || cfg.Optional || !ctx.Inst.Features.Match(cfg.Constraints) { + continue + } + if act == kconfig.No { + errs.push("%v:%v: %v is not present in the final config", cfg.File, cfg.Line, cfg.Name) + } else if cfg.Value == kconfig.No { + errs.push("%v:%v: %v is present in the final config", cfg.File, cfg.Line, cfg.Name) + } else { + errs.push("%v:%v: %v does not match final config %v vs %v", + cfg.File, cfg.Line, cfg.Name, cfg.Value, act) + } + } + return errs.err() +} + +func (ctx *Context) addUSBConfigs(cf *kconfig.ConfigFile) error { + android := "" + if ctx.Inst.Features[featAndroid] { + android = "android" + } + distroConfig := filepath.Join(ctx.ConfigDir, "distros", android+"*") + // Some USB drivers don't depend on any USB related symbols, but rather on a generic symbol + // for some input subsystem (e.g. HID), so include it as well. + // TODO: don't include HID, it does not seem to add anything relevant. + return ctx.addDependentConfigs(cf, []string{"USB_SUPPORT", "HID"}, distroConfig) +} + +func (ctx *Context) addDependentConfigs(dst *kconfig.ConfigFile, include []string, configGlob string) error { + configFiles, err := filepath.Glob(configGlob) + if err != nil { + return err + } + includes := func(a []string, b map[string]bool) bool { + for _, x := range a { + if b[x] { + return true + } + } + return false + } + selected := make(map[string]bool) + for _, cfg := range ctx.Kconf.Configs { + deps := cfg.DependsOn() + if !includes(include, deps) { + continue + } + selected[cfg.Name] = true + for dep := range deps { + selected[dep] = true + } + } + dedup := make(map[string]bool) + for _, file := range configFiles { + cf, err := kconfig.ParseConfig(file) + if err != nil { + return err + } + for _, cfg := range cf.Configs { + if cfg.Value == kconfig.No || dedup[cfg.Name] || !selected[cfg.Name] { + continue + } + dedup[cfg.Name] = true + dst.Set(cfg.Name, cfg.Value) + } + } + return nil +} + +func (ctx *Context) setTarget() error { + for _, target := range targets.List["linux"] { + if ctx.Inst.Features[target.KernelArch] { + if ctx.Target != nil { + return fmt.Errorf("arch is set twice") + } + ctx.Target = targets.GetEx("linux", target.Arch, ctx.Inst.Features[featClang]) + } + } + if ctx.Target == nil { + return fmt.Errorf("no arch feature") + } + return nil +} + +func (ctx *Context) setReleaseFeatures() error { + tag := ctx.ReleaseTag + match := regexp.MustCompile(`^v([0-9]+)\.([0-9]+)(?:\.([0-9]+))?$`).FindStringSubmatch(tag) + if match == nil { + return fmt.Errorf("bad release tag %q", tag) + } + major, err := strconv.ParseInt(match[1], 10, 32) + if err != nil { + return fmt.Errorf("bad release tag %q: %v", tag, err) + } + minor, err := strconv.ParseInt(match[2], 10, 32) + if err != nil { + return fmt.Errorf("bad release tag %q: %v", tag, err) + } + for ; major >= 2; major-- { + for ; minor >= 0; minor-- { + ctx.Inst.Features[fmt.Sprintf("v%v.%v", major, minor)] = true + } + minor = 99 + } + return nil +} + +func (ctx *Context) mrProper() error { + // Run 'make mrproper', otherwise out-of-tree build fails. + // However, it takes unreasonable amount of time, + // so first check few files and if they are missing hope for best. + files := []string{ + ".config", + "init/main.o", + "include/config", + "include/generated/compile.h", + "arch/" + ctx.Target.KernelArch + "/include/generated", + } + for _, file := range files { + if osutil.IsExist(filepath.Join(ctx.SourceDir, filepath.FromSlash(file))) { + goto clean + } + } + return nil +clean: + buildDir := ctx.BuildDir + ctx.BuildDir = ctx.SourceDir + err := ctx.Make("mrproper") + ctx.BuildDir = buildDir + return err +} + +func (ctx *Context) Make(args ...string) error { + args = append(args, + "O="+ctx.BuildDir, + "ARCH="+ctx.Target.KernelArch, + "-j", fmt.Sprint(runtime.NumCPU()), + ) + if ctx.Target.Triple != "" { + args = append(args, "CROSS_COMPILE="+ctx.Target.Triple+"-") + } + if ctx.Target.KernelCompiler != "" { + args = append(args, "CC="+ctx.Target.KernelCompiler) + } + _, err := osutil.RunCmd(10*time.Minute, ctx.SourceDir, "make", args...) + return err +} diff --git a/tools/syz-kconf/parser.go b/tools/syz-kconf/parser.go new file mode 100644 index 000000000..dad090b8a --- /dev/null +++ b/tools/syz-kconf/parser.go @@ -0,0 +1,283 @@ +// Copyright 2020 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/google/syzkaller/pkg/kconfig" + "github.com/google/syzkaller/pkg/vcs" + "gopkg.in/yaml.v3" +) + +type Instance struct { + Name string + Kernel Kernel + Verbatim []byte + Shell []Shell + Features Features + ConfigMap map[string]*Config + Configs []*Config +} + +type Config struct { + Name string + Value string + Override bool + Optional bool + Constraints []string + File string + Line int +} + +type Kernel struct { + Repo string + Tag string +} + +type Shell struct { + Cmd string + Constraints []string +} + +type Features map[string]bool + +func (features Features) Match(constraints []string) bool { + for _, feat := range constraints { + switch feat { + case featOptional, featOverride, featWeak: + continue + } + if feat[0] == '-' { + if features[feat[1:]] { + return false + } + } else if !features[feat] { + return false + } + } + return true +} + +type rawMain struct { + Instances []map[string][]string + Includes []map[string][]string +} + +type rawFile struct { + Kernel struct { + Repo string + Tag string + } + Shell []yaml.Node + Verbatim string + Config []yaml.Node +} + +func parseMainSpec(file string) ([]*Instance, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %v", err) + } + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + raw := new(rawMain) + if err := dec.Decode(raw); err != nil { + return nil, fmt.Errorf("failed to parse %v: %v", file, err) + } + var instances []*Instance + for _, inst := range raw.Instances { + for name, features := range inst { + inst, err := parseInstance(name, filepath.Dir(file), features, raw.Includes) + if err != nil { + return nil, err + } + instances = append(instances, inst) + inst, err = parseInstance(name+"-base", filepath.Dir(file), append(features, featBaseline), raw.Includes) + if err != nil { + return nil, err + } + instances = append(instances, inst) + } + } + return instances, nil +} + +func parseInstance(name, configDir string, features []string, includes []map[string][]string) (*Instance, error) { + inst := &Instance{ + Name: name, + Features: make(Features), + ConfigMap: make(map[string]*Config), + } + for _, feat := range features { + inst.Features[feat] = true + } + errs := new(Errors) + for _, include := range includes { + for file, features := range include { + if !inst.Features.Match(features) { + continue + } + raw, err := parseFile(filepath.Join(configDir, "bits", file)) + if err != nil { + return nil, err + } + mergeFile(inst, raw, file, errs) + } + } + inst.Verbatim = bytes.TrimSpace(inst.Verbatim) + return inst, errs.err() +} + +func parseFile(file string) (*rawFile, error) { + data, err := ioutil.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read %v: %v", file, err) + } + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + raw := new(rawFile) + if err := dec.Decode(raw); err != nil { + return nil, fmt.Errorf("failed to parse %v: %v", file, err) + } + return raw, nil +} + +func mergeFile(inst *Instance, raw *rawFile, file string, errs *Errors) { + if raw.Kernel.Repo != "" || raw.Kernel.Tag != "" { + if !vcs.CheckRepoAddress(raw.Kernel.Repo) { + errs.push("%v: bad kernel repo %q", file, raw.Kernel.Repo) + } + if !vcs.CheckBranch(raw.Kernel.Tag) { + errs.push("%v: bad kernel tag %q", file, raw.Kernel.Tag) + } + if inst.Kernel.Repo != "" { + errs.push("%v: kernel is set twice", file) + } + inst.Kernel = raw.Kernel + } + for _, node := range raw.Shell { + cmd, _, constraints, err := parseNode(node) + if err != nil { + errs.push("%v:%v: %v", file, node.Line, err) + } + inst.Shell = append(inst.Shell, Shell{ + Cmd: cmd, + Constraints: constraints, + }) + } + inst.Verbatim = append(append(inst.Verbatim, raw.Verbatim...), '\n') + for _, node := range raw.Config { + mergeConfig(inst, file, node, errs) + } +} + +func mergeConfig(inst *Instance, file string, node yaml.Node, errs *Errors) { + name, val, constraints, err := parseNode(node) + if err != nil { + errs.push("%v:%v: %v", file, node.Line, err) + return + } + cfg := &Config{ + Name: name, + Value: val, + File: file, + Line: node.Line, + } + for _, feat := range constraints { + switch feat { + case featOverride: + cfg.Override = true + case featOptional: + cfg.Optional = true + case featWeak: + cfg.Override, cfg.Optional = true, true + default: + cfg.Constraints = append(cfg.Constraints, feat) + } + } + if prev := inst.ConfigMap[name]; prev != nil { + if !cfg.Override { + errs.push("%v:%v: %v is already defined at %v:%v", file, node.Line, name, prev.File, prev.Line) + } + *prev = *cfg + return + } + if cfg.Override && !cfg.Optional { + errs.push("%v:%v: %v nothing to override", file, node.Line, name) + } + inst.ConfigMap[name] = cfg + inst.Configs = append(inst.Configs, cfg) +} + +func parseNode(node yaml.Node) (name, val string, constraints []string, err error) { + // Simplest case: - FOO. + val = kconfig.Yes + if node.Decode(&name) == nil { + return + } + complexVal := make(map[string]yaml.Node) + if err = node.Decode(complexVal); err != nil { + return + } + var valNode yaml.Node + for k, v := range complexVal { + name, valNode = k, v + break + } + // Case: - FOO: 42. + if intVal := 0; valNode.Decode(&intVal) == nil { + val = fmt.Sprint(intVal) + return + } + if valNode.Decode(&val) == nil { + // Case: - FOO: "string". + if valNode.Style == yaml.DoubleQuotedStyle { + val = `"` + val + `"` + return + } + // Case: - FOO: n. + if valNode.Style == 0 && val == "n" { + val = kconfig.No + return + } + err = fmt.Errorf("bad config format") + return + } + // Case: - FOO: [...]. + propsNode := []yaml.Node{} + if err = valNode.Decode(&propsNode); err != nil { + return + } + for _, propNode := range propsNode { + prop := "" + if err = propNode.Decode(&prop); err != nil { + return + } + if propNode.Style == yaml.DoubleQuotedStyle { + val = `"` + prop + `"` + } else if prop == "n" { + val = kconfig.No + } else { + constraints = append(constraints, prop) + } + } + return +} + +type Errors []byte + +func (errs *Errors) push(msg string, args ...interface{}) { + *errs = append(*errs, fmt.Sprintf(msg+"\n", args...)...) +} + +func (errs *Errors) err() error { + if len(*errs) == 0 { + return nil + } + return fmt.Errorf("%s", *errs) +} |