From 68ce63c46891becd752fa1c0a0c3caaa98117832 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Thu, 28 Jun 2018 12:32:30 +0200 Subject: pkg/build: support fuchsia builds --- pkg/build/build.go | 12 ++++++++++-- pkg/build/fuchsia.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ pkg/build/gvisor.go | 3 --- pkg/build/linux.go | 3 --- pkg/instance/instance.go | 36 +++++++++++++++++++++++++++++++++- pkg/vcs/fuchsia.go | 5 ----- syz-ci/jobs.go | 1 - syz-ci/manager.go | 27 +++++++++++++------------- syz-manager/manager.go | 3 +++ 9 files changed, 111 insertions(+), 29 deletions(-) create mode 100644 pkg/build/fuchsia.go diff --git a/pkg/build/build.go b/pkg/build/build.go index 6eb912e21..36e082826 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -6,6 +6,7 @@ package build import ( "fmt" + "path/filepath" "strings" "time" @@ -16,9 +17,11 @@ import ( // Kernel is taken from kernelDir, userspace system is taken from userspaceDir. // If cmdlineFile is not empty, contents of the file are appended to the kernel command line. // If sysctlFile is not empty, contents of the file are appended to the image /etc/sysctl.conf. -// Output is stored in outputDir and includes: +// Output is stored in outputDir and includes (everything except for image is optional): // - image: the image -// - key: ssh key for the image if applicable +// - key: ssh key for the image +// - kernel: kernel for injected boot +// - initrd: initrd for injected boot // - kernel.config: actual kernel config used during build // - obj/: directory with kernel object files (e.g. vmlinux for linux) func Image(targetOS, targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir, @@ -27,6 +30,9 @@ func Image(targetOS, targetArch, vmType, kernelDir, outputDir, compiler, userspa if err != nil { return err } + if err := osutil.MkdirAll(filepath.Join(outputDir, "obj")); err != nil { + return err + } return builder.build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir, cmdlineFile, sysctlFile, config) } @@ -54,6 +60,8 @@ func getBuilder(targetOS, targetArch, vmType string) (builder, error) { return gvisor{}, nil case targetOS == "linux" && targetArch == "amd64" && (vmType == "qemu" || vmType == "gce"): return linux{}, nil + case targetOS == "fuchsia" && (targetArch == "amd64" || targetArch == "arm64") && vmType == "qemu": + return fuchsia{}, nil default: return nil, fmt.Errorf("unsupported image type %v/%v/%v", targetOS, targetArch, vmType) } diff --git a/pkg/build/fuchsia.go b/pkg/build/fuchsia.go new file mode 100644 index 000000000..076fcad10 --- /dev/null +++ b/pkg/build/fuchsia.go @@ -0,0 +1,50 @@ +// Copyright 2018 syzkaller project authors. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package build + +import ( + "fmt" + "path/filepath" + "time" + + "github.com/google/syzkaller/pkg/osutil" + "github.com/google/syzkaller/sys/targets" +) + +type fuchsia struct{} + +func (fu fuchsia) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir, + cmdlineFile, sysctlFile string, config []byte) error { + sysTarget := targets.Get("fuchsia", targetArch) + if sysTarget == nil { + return fmt.Errorf("unsupported fuchsia arch %v", targetArch) + } + arch := sysTarget.KernelHeaderArch + if _, err := osutil.RunCmd(10*time.Minute, kernelDir, "scripts/fx", "set", arch, + "--packages", "garnet/packages/products/sshd"); err != nil { + return err + } + if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "full-build"); err != nil { + return err + } + for src, dst := range map[string]string{ + "out/" + arch + "/images/fvm.blk": "image", + "out/" + arch + "/ssh-keys/id_ed25519": "key", + "out/build-zircon/build-" + arch + "/zircon.elf": "obj/zircon.elf", + "out/build-zircon/build-" + arch + "/zircon.bin": "kernel", + "out/" + arch + "/bootdata-blob.bin": "initrd", + } { + fullSrc := filepath.Join(kernelDir, filepath.FromSlash(src)) + fullDst := filepath.Join(outputDir, filepath.FromSlash(dst)) + if err := osutil.CopyFile(fullSrc, fullDst); err != nil { + return fmt.Errorf("faied to copy %v: %v", src, err) + } + } + return nil +} + +func (fu fuchsia) clean(kernelDir string) error { + // Let's assume that fx always properly handles build without cleaning (until proven otherwise). + return nil +} diff --git a/pkg/build/gvisor.go b/pkg/build/gvisor.go index f222475d1..ea961d6ae 100644 --- a/pkg/build/gvisor.go +++ b/pkg/build/gvisor.go @@ -16,9 +16,6 @@ type gvisor struct{} func (gvisor gvisor) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir, cmdlineFile, sysctlFile string, config []byte) error { - if err := osutil.MkdirAll(outputDir); err != nil { - return err - } args := []string{"build", "--verbose_failures"} if strings.Contains(" "+string(config)+" ", " -race ") { args = append(args, "--features=race") diff --git a/pkg/build/linux.go b/pkg/build/linux.go index a2afc3011..ccceeec82 100644 --- a/pkg/build/linux.go +++ b/pkg/build/linux.go @@ -26,9 +26,6 @@ type linux struct{} func (linux linux) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir, cmdlineFile, sysctlFile string, config []byte) error { - if err := osutil.MkdirAll(filepath.Join(outputDir, "obj")); err != nil { - return err - } if err := linux.buildKernel(kernelDir, outputDir, compiler, config); err != nil { return err } diff --git a/pkg/instance/instance.go b/pkg/instance/instance.go index 79616f3fa..556208876 100644 --- a/pkg/instance/instance.go +++ b/pkg/instance/instance.go @@ -7,6 +7,7 @@ package instance import ( "bytes" + "encoding/json" "fmt" "net" "os" @@ -85,9 +86,42 @@ func (env *Env) BuildKernel(compilerBin, userspaceDir, cmdlineFile, sysctlFile s cmdlineFile, sysctlFile, kernelConfig); err != nil { return err } + return SetConfigImage(cfg, imageDir) +} + +func SetConfigImage(cfg *mgrconfig.Config, imageDir string) error { cfg.KernelObj = filepath.Join(imageDir, "obj") cfg.Image = filepath.Join(imageDir, "image") - cfg.SSHKey = filepath.Join(imageDir, "key") + if keyFile := filepath.Join(imageDir, "key"); osutil.IsExist(keyFile) { + cfg.SSHKey = keyFile + } + if cfg.Type == "qemu" { + kernel := filepath.Join(imageDir, "kernel") + if !osutil.IsExist(kernel) { + kernel = "" + } + initrd := filepath.Join(imageDir, "initrd") + if !osutil.IsExist(initrd) { + initrd = "" + } + if kernel != "" || initrd != "" { + qemu := make(map[string]interface{}) + if err := json.Unmarshal(cfg.VM, &qemu); err != nil { + return fmt.Errorf("failed to parse qemu config: %v", err) + } + if kernel != "" { + qemu["kernel"] = kernel + } + if initrd != "" { + qemu["initrd"] = initrd + } + vmCfg, err := json.Marshal(qemu) + if err != nil { + return fmt.Errorf("failed to serialize qemu config: %v", err) + } + cfg.VM = vmCfg + } + } return nil } diff --git a/pkg/vcs/fuchsia.go b/pkg/vcs/fuchsia.go index b8462f5ad..7b8e7335b 100644 --- a/pkg/vcs/fuchsia.go +++ b/pkg/vcs/fuchsia.go @@ -26,11 +26,6 @@ func newFuchsia(vm, dir string) *fuchsia { } } -// mkdir DIR; cd DIR -// curl -s "https://fuchsia.googlesource.com/scripts/+/master/bootstrap?format=TEXT" | base64 --decode | bash -s topaz -// (cd fuchsia && .jiri_root/bin/jiri update) -// (cd fuchsia/zircon/ && git show HEAD) - func (fu *fuchsia) Poll(repo, branch string) (*Commit, error) { if repo != "https://fuchsia.googlesource.com" || branch != "master" { // fuchsia ecosystem is hard-tailored to the main repo. diff --git a/syz-ci/jobs.go b/syz-ci/jobs.go index 0e065892f..5f944ad57 100644 --- a/syz-ci/jobs.go +++ b/syz-ci/jobs.go @@ -168,7 +168,6 @@ func (jp *JobProcessor) test(job *Job) error { mgrcfg.Name += "-job" mgrcfg.Workdir = filepath.Join(dir, "workdir") mgrcfg.KernelSrc = kernelDir - mgrcfg.KernelObj = kernelDir mgrcfg.Syzkaller = filepath.Join(dir, "gopath", "src", "github.com", "google", "syzkaller") os.RemoveAll(mgrcfg.Workdir) diff --git a/syz-ci/manager.go b/syz-ci/manager.go index 05ad42921..814a32032 100644 --- a/syz-ci/manager.go +++ b/syz-ci/manager.go @@ -31,11 +31,14 @@ const kernelRebuildPeriod = syzkallerRebuildPeriod + time.Hour // List of required files in kernel build (contents of latest/current dirs). var imageFiles = map[string]bool{ - "tag": true, // serialized BuildInfo - "kernel.config": false, // kernel config used for build - "image": true, // kernel image - "key": false, // root ssh key for the image - "obj/vmlinux": false, // vmlinux with debug info + "tag": true, // serialized BuildInfo + "kernel.config": false, // kernel config used for build + "image": true, // kernel image + "kernel": false, + "initrd": false, + "key": false, // root ssh key for the image + "obj/vmlinux": false, // Linux object file with debug info + "obj/zircon.elf": false, // Zircon object file with debug info } // Manager represents a single syz-manager instance. @@ -422,11 +425,9 @@ func (mgr *Manager) createTestConfig(imageDir string, info *BuildInfo) (*mgrconf mgrcfg.Name += "-test" mgrcfg.Tag = info.KernelCommit mgrcfg.Workdir = filepath.Join(imageDir, "workdir") - mgrcfg.Image = filepath.Join(imageDir, "image") - if keyFile := filepath.Join(imageDir, "key"); osutil.IsExist(keyFile) { - mgrcfg.SSHKey = keyFile + if err := instance.SetConfigImage(mgrcfg, imageDir); err != nil { + return nil, err } - mgrcfg.KernelObj = filepath.Join(imageDir, "obj") mgrcfg.KernelSrc = mgr.kernelDir if err := mgrconfig.Complete(mgrcfg); err != nil { return nil, fmt.Errorf("bad manager config: %v", err) @@ -450,15 +451,13 @@ func (mgr *Manager) writeConfig(buildTag string) (string, error) { } mgrcfg.Tag = buildTag mgrcfg.Workdir = mgr.workDir - mgrcfg.KernelObj = filepath.Join(mgr.currentDir, "obj") + if err := instance.SetConfigImage(mgrcfg, mgr.currentDir); err != nil { + return "", err + } // Strictly saying this is somewhat racy as builder can concurrently // update the source, or even delete and re-clone. If this causes // problems, we need to make a copy of sources after build. mgrcfg.KernelSrc = mgr.kernelDir - mgrcfg.Image = filepath.Join(mgr.currentDir, "image") - if keyFile := filepath.Join(mgr.currentDir, "key"); osutil.IsExist(keyFile) { - mgrcfg.SSHKey = keyFile - } if err := mgrconfig.Complete(mgrcfg); err != nil { return "", fmt.Errorf("bad manager config: %v", err) } diff --git a/syz-manager/manager.go b/syz-manager/manager.go index 20ec21d7f..3ead14564 100644 --- a/syz-manager/manager.go +++ b/syz-manager/manager.go @@ -1190,6 +1190,9 @@ func (mgr *Manager) collectUsedFiles() { if vmlinux := filepath.Join(cfg.KernelObj, "vmlinux"); osutil.IsExist(vmlinux) { addUsedFile(vmlinux) } + if zircon := filepath.Join(cfg.KernelObj, "zircon.elf"); osutil.IsExist(zircon) { + addUsedFile(zircon) + } if cfg.Image != "9p" { addUsedFile(cfg.Image) } -- cgit mrf-deployment