From 780b27623e2c0284dfb0c8e0392463d1eef00e92 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Wed, 17 Nov 2021 12:36:48 +0000 Subject: tools/syz-testbed: support per-checkout syz-manager configs Let the user of the tool specify individual syz-manager configs for checkouts. A base config is specified as previously and then individual parts of jsons are applied to that base config, e.g. <...> "checkouts": [ { "name": "first", "repo": "https://github.com/google/syzkaller.git", }, { "name": "second", "repo": "https://github.com/google/syzkaller.git", "manager_config": { "kernel_obj": "/tmp/linux-stable2" } } ], "manager_config": { "target": "linux/amd64", "kernel_obj": "/tmp/linux-stable", <...> } <...> --- pkg/config/merge.go | 57 +++++++++++++++++++++++++++++++++++++++++++ pkg/config/merge_test.go | 49 +++++++++++++++++++++++++++++++++++++ tools/syz-testbed/checkout.go | 17 +++++++------ tools/syz-testbed/instance.go | 2 +- tools/syz-testbed/testbed.go | 40 +++++++++++++++++------------- 5 files changed, 140 insertions(+), 25 deletions(-) create mode 100644 pkg/config/merge.go create mode 100644 pkg/config/merge_test.go diff --git a/pkg/config/merge.go b/pkg/config/merge.go new file mode 100644 index 000000000..49a7254de --- /dev/null +++ b/pkg/config/merge.go @@ -0,0 +1,57 @@ +// Copyright 2021 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package config + +import ( + "encoding/json" +) + +// Unfortunately, if we want to apply a JSON patch to some configuration, we cannot just unmarshal +// it twice - in that case json.RawMessage objects will be completely replaced, but not merged. +func MergeJSONData(left, right []byte) ([]byte, error) { + vLeft, err := parseFragment(left) + if err != nil { + return nil, err + } + vRight, err := parseFragment(right) + if err != nil { + return nil, err + } + merged := mergeRecursive(vLeft, vRight) + return json.Marshal(merged) +} + +func parseFragment(input []byte) (parsed interface{}, err error) { + if len(input) == 0 { + // For convenience, we allow empty strings to be passed to the function that merges JSONs. + return + } + err = json.Unmarshal(json.RawMessage(input), &parsed) + return +} + +// If one of the elements is not a map, use the new one. +// Otherwise, recursively merge map elements. +func mergeRecursive(left, right interface{}) interface{} { + if left == nil { + return right + } + if right == nil { + return left + } + mLeft, okLeft := left.(map[string]interface{}) + mRight, okRight := right.(map[string]interface{}) + if !okLeft || !okRight { + return right + } + for key, val := range mRight { + valLeft, ok := mLeft[key] + if ok { + mLeft[key] = mergeRecursive(valLeft, val) + } else { + mLeft[key] = val + } + } + return mLeft +} diff --git a/pkg/config/merge_test.go b/pkg/config/merge_test.go new file mode 100644 index 000000000..c5a45754e --- /dev/null +++ b/pkg/config/merge_test.go @@ -0,0 +1,49 @@ +// Copyright 2021 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package config_test + +import ( + "bytes" + "testing" + + "github.com/google/syzkaller/pkg/config" +) + +func TestMergeJSONData(t *testing.T) { + tests := []struct { + left string + right string + result string + }{ + { + `{"a":1,"b":2}`, + `{"b":3,"c":4}`, + `{"a":1,"b":3,"c":4}`, + }, + { + `{"a":1,"b":{"c":{"d":"nested string","e":"another string"}}}`, + `{"b":{"c":{"d":12345}}}`, + `{"a":1,"b":{"c":{"d":12345,"e":"another string"}}}`, + }, + { + `{}`, + `{"a":{"b":{"c":0}}}`, + `{"a":{"b":{"c":0}}}`, + }, + { + `{"a":{"b":{"c":0}}}`, + ``, + `{"a":{"b":{"c":0}}}`, + }, + } + for _, test := range tests { + res, err := config.MergeJSONData([]byte(test.left), []byte(test.right)) + if err != nil { + t.Errorf("unexpected error: %s", err) + } + if !bytes.Equal(res, []byte(test.result)) { + t.Errorf("expected %s, got %s", test.result, res) + } + } +} diff --git a/tools/syz-testbed/checkout.go b/tools/syz-testbed/checkout.go index fe846bbb2..91508ff99 100644 --- a/tools/syz-testbed/checkout.go +++ b/tools/syz-testbed/checkout.go @@ -10,15 +10,17 @@ import ( "time" syz_instance "github.com/google/syzkaller/pkg/instance" + "github.com/google/syzkaller/pkg/mgrconfig" "github.com/google/syzkaller/pkg/osutil" "github.com/google/syzkaller/pkg/vcs" ) type Checkout struct { - Path string - Name string - Running []*Instance - Completed []*RunResult + Path string + Name string + ManagerConfig *mgrconfig.Config + Running []*Instance + Completed []*RunResult } func (checkout *Checkout) ArchiveRunning() error { @@ -33,10 +35,11 @@ func (checkout *Checkout) ArchiveRunning() error { return nil } -func (ctx *TestbedContext) NewCheckout(config *CheckoutConfig) (*Checkout, error) { +func (ctx *TestbedContext) NewCheckout(config *CheckoutConfig, mgrConfig *mgrconfig.Config) (*Checkout, error) { checkout := &Checkout{ - Name: config.Name, - Path: filepath.Join(ctx.Config.Workdir, "checkouts", config.Name), + Name: config.Name, + Path: filepath.Join(ctx.Config.Workdir, "checkouts", config.Name), + ManagerConfig: mgrConfig, } log.Printf("[%s] Checking out", checkout.Name) if osutil.IsExist(checkout.Path) { diff --git a/tools/syz-testbed/instance.go b/tools/syz-testbed/instance.go index c51b9c597..f7293d289 100644 --- a/tools/syz-testbed/instance.go +++ b/tools/syz-testbed/instance.go @@ -108,7 +108,7 @@ func (ctx *TestbedContext) NewInstance(checkout *Checkout, mgrName string) (*Ins } log.Printf("[%s] Generating syz-manager config", name) - managerCfg := *ctx.ManagerConfig + managerCfg := *checkout.ManagerConfig managerCfg.Name = mgrName managerCfg.Workdir = workdir managerCfg.Syzkaller = checkout.Path diff --git a/tools/syz-testbed/testbed.go b/tools/syz-testbed/testbed.go index e67dd6883..dee9ffc73 100644 --- a/tools/syz-testbed/testbed.go +++ b/tools/syz-testbed/testbed.go @@ -46,14 +46,14 @@ type DurationConfig struct { } type CheckoutConfig struct { - Name string `json:"name"` - Repo string `json:"repo"` - Branch string `json:"branch"` + Name string `json:"name"` + Repo string `json:"repo"` + Branch string `json:"branch"` + ManagerConfig json.RawMessage `json:"manager_config"` // a patch to manager config } type TestbedContext struct { Config *TestbedConfig - ManagerConfig *mgrconfig.Config Checkouts []*Checkout NextRestart time.Time NextCheckoutID int @@ -76,24 +76,14 @@ func main() { if err != nil { tool.Failf("invalid config: %s", err) } - - managerCfg, err := mgrconfig.LoadPartialData(cfg.ManagerConfig) - if err != nil { - tool.Failf("failed to parse manager config: %s", err) - } - if managerCfg.HTTP == "" { - // Actually we don't care much about the specific ports of syz-managers. - managerCfg.HTTP = ":0" - } - ctx := TestbedContext{ - Config: cfg, - ManagerConfig: managerCfg, + Config: cfg, } go ctx.setupHTTPServer() for _, checkoutCfg := range cfg.Checkouts { - co, err := ctx.NewCheckout(&checkoutCfg) + mgrCfg := ctx.MakeMgrConfig(cfg.ManagerConfig, checkoutCfg.ManagerConfig) + co, err := ctx.NewCheckout(&checkoutCfg, mgrCfg) if err != nil { tool.Failf("checkout failed: %s", err) } @@ -117,6 +107,22 @@ func main() { ctx.Loop(shutdown) } +func (ctx *TestbedContext) MakeMgrConfig(base, patch json.RawMessage) *mgrconfig.Config { + mergedConfig, err := config.MergeJSONData(base, patch) + if err != nil { + tool.Failf("failed to apply a patch to the base manager config: %s", err) + } + mgrCfg, err := mgrconfig.LoadPartialData(mergedConfig) + if err != nil { + tool.Failf("failed to parse base manager config: %s", err) + } + if mgrCfg.HTTP == "" { + // Actually, we don't care much about the specific ports of syz-managers. + mgrCfg.HTTP = ":0" + } + return mgrCfg +} + func (ctx *TestbedContext) GetStatViews() ([]StatView, error) { groupsCompleted := []RunResultGroup{} groupsAll := []RunResultGroup{} -- cgit mrf-deployment