From 99d2454c57e2b4761d667f265ec24122e40cd514 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 19 Jun 2017 19:43:17 +0200 Subject: pkg/git: improve Poll Support changing repo/branch and force pushes. --- pkg/git/git.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'pkg') diff --git a/pkg/git/git.go b/pkg/git/git.go index fdea886cf..5c91c6040 100644 --- a/pkg/git/git.go +++ b/pkg/git/git.go @@ -22,16 +22,27 @@ func Poll(dir, repo, branch string) (string, error) { osutil.RunCmd(timeout, dir, "git", "reset", "--hard") origin, err := osutil.RunCmd(timeout, dir, "git", "remote", "get-url", "origin") if err != nil || strings.TrimSpace(string(origin)) != repo { + // The repo is here, but it has wrong origin (e.g. repo in config has changed), re-clone. + if err := clone(dir, repo, branch); err != nil { + return "", err + } + } + // Use origin/branch for the case the branch was force-pushed, + // in such case branch is not the same is origin/branch and we will + // stuck with the local version forever (git checkout won't fail). + if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", "origin/"+branch); err != nil { + // No such branch (e.g. branch in config has changed), re-clone. if err := clone(dir, repo, branch); err != nil { return "", err } } if _, err := osutil.RunCmd(timeout, dir, "git", "fetch", "--no-tags", "--depth", "1"); err != nil { + // Something else is wrong, re-clone. if err := clone(dir, repo, branch); err != nil { return "", err } } - if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", branch); err != nil { + if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", "origin/"+branch); err != nil { return "", err } return HeadCommit(dir) -- cgit mrf-deployment From 5cc5b2714e61b1d64d463328308fbe231ab8387f Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 19 Jun 2017 19:45:39 +0200 Subject: pkg/fileutil: improve CopyFile Make CopyFile atomic and preserve permissions. --- pkg/fileutil/fileutil.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'pkg') diff --git a/pkg/fileutil/fileutil.go b/pkg/fileutil/fileutil.go index eae069e01..dbc9366de 100644 --- a/pkg/fileutil/fileutil.go +++ b/pkg/fileutil/fileutil.go @@ -14,7 +14,7 @@ import ( "unsafe" ) -// CopyFile copies oldFile to newFile. +// CopyFile atomically copies oldFile to newFile preserving permissions and modification time. func CopyFile(oldFile, newFile string) error { oldf, err := os.Open(oldFile) if err != nil { @@ -25,7 +25,8 @@ func CopyFile(oldFile, newFile string) error { if err != nil { return err } - newf, err := os.Create(newFile) + tmpFile := newFile + ".tmp" + newf, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, stat.Mode()&os.ModePerm) if err != nil { return err } @@ -37,10 +38,10 @@ func CopyFile(oldFile, newFile string) error { if err := newf.Close(); err != nil { return err } - if err := os.Chtimes(newFile, stat.ModTime(), stat.ModTime()); err != nil { + if err := os.Chtimes(tmpFile, stat.ModTime(), stat.ModTime()); err != nil { return err } - return nil + return os.Rename(tmpFile, newFile) } // WriteTempFile writes data to a temp file and returns its name. -- cgit mrf-deployment From e39114dc0cfa7566dd2472e3e33a64586692d6cc Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 20 Jun 2017 19:56:13 +0200 Subject: pkg/kernel: move partial config functionality into a separate function Move partial config functionality into BuildWithPartConfig. It is used only for syz-gce which will be removed soon. Provide a better interface for new continuous build system. --- pkg/kernel/kernel.go | 52 ++++++++++++++++++++++++++++++---------------------- syz-gce/syz-gce.go | 14 ++++++-------- 2 files changed, 36 insertions(+), 30 deletions(-) (limited to 'pkg') diff --git a/pkg/kernel/kernel.go b/pkg/kernel/kernel.go index 77e6f6c1f..c38020b59 100644 --- a/pkg/kernel/kernel.go +++ b/pkg/kernel/kernel.go @@ -25,35 +25,43 @@ import ( "github.com/google/syzkaller/pkg/osutil" ) -func Build(dir, compiler, config string, fullConfig bool) error { +func Build(dir, compiler, config string) error { + if err := fileutil.CopyFile(config, filepath.Join(dir, ".config")); err != nil { + return fmt.Errorf("failed to write config file: %v", err) + } + return build(dir, compiler) +} + +// TODO(dvyukov): this is only for syz-gce, remove when syz-gce is deleted. +func BuildWithPartConfig(dir, compiler, config string) error { const timeout = 10 * time.Minute // default timeout for command invocations - if fullConfig { - if err := ioutil.WriteFile(filepath.Join(dir, ".config"), []byte(config), 0600); err != nil { - return fmt.Errorf("failed to write config file: %v", err) - } - } else { - os.Remove(filepath.Join(dir, ".config")) - configFile := filepath.Join(dir, "syz.config") - if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil { - return fmt.Errorf("failed to write config file: %v", err) - } - defer os.Remove(configFile) - if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil { - return err - } - if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil { - return err - } - if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil { - return err - } + os.Remove(filepath.Join(dir, ".config")) + configFile := filepath.Join(dir, "syz.config") + if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil { + return fmt.Errorf("failed to write config file: %v", err) + } + defer os.Remove(configFile) + if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil { + return err + } + if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil { + return err + } + if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil { + return err } + return build(dir, compiler) +} + +func build(dir, compiler string) error { + const timeout = 10 * time.Minute // default timeout for command invocations if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil { return err } // We build only bzImage as we currently don't use modules. // Build of a large kernel can take a while on a 1 CPU VM. - if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", strconv.Itoa(runtime.NumCPU()), "CC="+compiler); err != nil { + cpu := strconv.Itoa(runtime.NumCPU()) + if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", cpu, "CC="+compiler); err != nil { return err } return nil diff --git a/syz-gce/syz-gce.go b/syz-gce/syz-gce.go index ebe91ae6c..ec28c5e2b 100644 --- a/syz-gce/syz-gce.go +++ b/syz-gce/syz-gce.go @@ -354,16 +354,14 @@ func (a *LocalBuildAction) Build() error { } } Logf(0, "building kernel on %v...", hash) - config, full := syzconfig, false if cfg.Linux_Config != "" { - data, err := ioutil.ReadFile(cfg.Linux_Config) - if err != nil { - return fmt.Errorf("failed to read config file: %v", err) + if err := kernel.Build(dir, a.Compiler, cfg.Linux_Config); err != nil { + return fmt.Errorf("build failed: %v", err) + } + } else { + if err := kernel.BuildWithPartConfig(dir, a.Compiler, syzconfig); err != nil { + return fmt.Errorf("build failed: %v", err) } - config, full = string(data), true - } - if err := kernel.Build(dir, a.Compiler, config, full); err != nil { - return fmt.Errorf("build failed: %v", err) } Logf(0, "building image...") os.MkdirAll("image/obj", 0700) -- cgit mrf-deployment From fea266b33f5e4170b75d678d5e355f947aeb50cc Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Tue, 20 Jun 2017 19:58:09 +0200 Subject: pkg/osutil: add FilesExist/CopyFiles/LinkFiles functions Will be required by the new build system. --- pkg/osutil/osutil.go | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) (limited to 'pkg') diff --git a/pkg/osutil/osutil.go b/pkg/osutil/osutil.go index b977ecebf..3d5a18a67 100644 --- a/pkg/osutil/osutil.go +++ b/pkg/osutil/osutil.go @@ -13,6 +13,13 @@ import ( "path/filepath" "syscall" "time" + + "github.com/google/syzkaller/pkg/fileutil" +) + +const ( + DefaultDirPerm = 0755 + DefaultFilePerm = 0644 ) // RunCmd runs "bin args..." in dir with timeout and returns its output. @@ -94,3 +101,66 @@ func HandleInterrupts(shutdown chan struct{}) { os.Exit(int(syscall.SIGINT)) }() } + +// FilesExist returns true if all files exist in dir. +// Files are assumed to be relative names in slash notation. +func FilesExist(dir string, files []string) bool { + for _, f := range files { + if !IsExist(filepath.Join(dir, filepath.FromSlash(f))) { + return false + } + } + return true +} + +// CopyFiles copies files from srcDir to dstDir as atomically as possible. +// Files are assumed to be relative names in slash notation. +// All other files in dstDir are removed. +func CopyFiles(srcDir, dstDir string, files []string) error { + // Linux does not support atomic dir replace, so we copy to tmp dir first. + // Then remove dst dir and rename tmp to dst (as atomic as can get on Linux). + tmpDir := dstDir + ".tmp" + if err := os.RemoveAll(tmpDir); err != nil { + return err + } + if err := os.MkdirAll(tmpDir, DefaultDirPerm); err != nil { + return err + } + for _, f := range files { + src := filepath.Join(srcDir, filepath.FromSlash(f)) + dst := filepath.Join(tmpDir, filepath.FromSlash(f)) + if err := os.MkdirAll(filepath.Dir(dst), DefaultDirPerm); err != nil { + return err + } + if err := fileutil.CopyFile(src, dst); err != nil { + return err + } + } + if err := os.RemoveAll(dstDir); err != nil { + return err + } + return os.Rename(tmpDir, dstDir) +} + +// LinkFiles creates hard links for files from dstDir to srcDir. +// Files are assumed to be relative names in slash notation. +// All other files in dstDir are removed. +func LinkFiles(srcDir, dstDir string, files []string) error { + if err := os.RemoveAll(dstDir); err != nil { + return err + } + if err := os.MkdirAll(dstDir, DefaultDirPerm); err != nil { + return err + } + for _, f := range files { + src := filepath.Join(srcDir, filepath.FromSlash(f)) + dst := filepath.Join(dstDir, filepath.FromSlash(f)) + if err := os.MkdirAll(filepath.Dir(dst), DefaultDirPerm); err != nil { + return err + } + if err := os.Link(src, dst); err != nil { + return err + } + } + return nil +} -- cgit mrf-deployment