Merge pull request #241 from dvyukov/dvyukov-ci

syz-ci: add continuous integration system

4 files changed, 117 insertions, 27 deletions

diff --git a/pkg/fileutil/fileutil.go b/pkg/fileutil/fileutil.go
index eae069e01..dbc9366de 100644
--- a/pkg/fileutil/fileutil.go
+++ b/pkg/fileutil/fileutil.go
@@ -14,7 +14,7 @@ import (
 	"unsafe"
 )
 
-// CopyFile copies oldFile to newFile.
+// CopyFile atomically copies oldFile to newFile preserving permissions and modification time.
 func CopyFile(oldFile, newFile string) error {
 	oldf, err := os.Open(oldFile)
 	if err != nil {
@@ -25,7 +25,8 @@ func CopyFile(oldFile, newFile string) error {
 	if err != nil {
 		return err
 	}
-	newf, err := os.Create(newFile)
+	tmpFile := newFile + ".tmp"
+	newf, err := os.OpenFile(tmpFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, stat.Mode()&os.ModePerm)
 	if err != nil {
 		return err
 	}
@@ -37,10 +38,10 @@ func CopyFile(oldFile, newFile string) error {
 	if err := newf.Close(); err != nil {
 		return err
 	}
-	if err := os.Chtimes(newFile, stat.ModTime(), stat.ModTime()); err != nil {
+	if err := os.Chtimes(tmpFile, stat.ModTime(), stat.ModTime()); err != nil {
 		return err
 	}
-	return nil
+	return os.Rename(tmpFile, newFile)
 }
 
 // WriteTempFile writes data to a temp file and returns its name.
diff --git a/pkg/git/git.go b/pkg/git/git.go
index fdea886cf..5c91c6040 100644
--- a/pkg/git/git.go
+++ b/pkg/git/git.go
@@ -22,16 +22,27 @@ func Poll(dir, repo, branch string) (string, error) {
 	osutil.RunCmd(timeout, dir, "git", "reset", "--hard")
 	origin, err := osutil.RunCmd(timeout, dir, "git", "remote", "get-url", "origin")
 	if err != nil || strings.TrimSpace(string(origin)) != repo {
+		// The repo is here, but it has wrong origin (e.g. repo in config has changed), re-clone.
+		if err := clone(dir, repo, branch); err != nil {
+			return "", err
+		}
+	}
+	// Use origin/branch for the case the branch was force-pushed,
+	// in such case branch is not the same is origin/branch and we will
+	// stuck with the local version forever (git checkout won't fail).
+	if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", "origin/"+branch); err != nil {
+		// No such branch (e.g. branch in config has changed), re-clone.
 		if err := clone(dir, repo, branch); err != nil {
 			return "", err
 		}
 	}
 	if _, err := osutil.RunCmd(timeout, dir, "git", "fetch", "--no-tags", "--depth", "1"); err != nil {
+		// Something else is wrong, re-clone.
 		if err := clone(dir, repo, branch); err != nil {
 			return "", err
 		}
 	}
-	if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", branch); err != nil {
+	if _, err := osutil.RunCmd(timeout, dir, "git", "checkout", "origin/"+branch); err != nil {
 		return "", err
 	}
 	return HeadCommit(dir)
diff --git a/pkg/kernel/kernel.go b/pkg/kernel/kernel.go
index 77e6f6c1f..c38020b59 100644
--- a/pkg/kernel/kernel.go
+++ b/pkg/kernel/kernel.go
@@ -25,35 +25,43 @@ import (
 	"github.com/google/syzkaller/pkg/osutil"
 )
 
-func Build(dir, compiler, config string, fullConfig bool) error {
+func Build(dir, compiler, config string) error {
+	if err := fileutil.CopyFile(config, filepath.Join(dir, ".config")); err != nil {
+		return fmt.Errorf("failed to write config file: %v", err)
+	}
+	return build(dir, compiler)
+}
+
+// TODO(dvyukov): this is only for syz-gce, remove when syz-gce is deleted.
+func BuildWithPartConfig(dir, compiler, config string) error {
 	const timeout = 10 * time.Minute // default timeout for command invocations
-	if fullConfig {
-		if err := ioutil.WriteFile(filepath.Join(dir, ".config"), []byte(config), 0600); err != nil {
-			return fmt.Errorf("failed to write config file: %v", err)
-		}
-	} else {
-		os.Remove(filepath.Join(dir, ".config"))
-		configFile := filepath.Join(dir, "syz.config")
-		if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil {
-			return fmt.Errorf("failed to write config file: %v", err)
-		}
-		defer os.Remove(configFile)
-		if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
-			return err
-		}
-		if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
-			return err
-		}
-		if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
-			return err
-		}
+	os.Remove(filepath.Join(dir, ".config"))
+	configFile := filepath.Join(dir, "syz.config")
+	if err := ioutil.WriteFile(configFile, []byte(config), 0600); err != nil {
+		return fmt.Errorf("failed to write config file: %v", err)
+	}
+	defer os.Remove(configFile)
+	if _, err := osutil.RunCmd(timeout, dir, "make", "defconfig"); err != nil {
+		return err
+	}
+	if _, err := osutil.RunCmd(timeout, dir, "make", "kvmconfig"); err != nil {
+		return err
+	}
+	if _, err := osutil.RunCmd(timeout, dir, "scripts/kconfig/merge_config.sh", "-n", ".config", configFile); err != nil {
+		return err
 	}
+	return build(dir, compiler)
+}
+
+func build(dir, compiler string) error {
+	const timeout = 10 * time.Minute // default timeout for command invocations
 	if _, err := osutil.RunCmd(timeout, dir, "make", "olddefconfig"); err != nil {
 		return err
 	}
 	// We build only bzImage as we currently don't use modules.
 	// Build of a large kernel can take a while on a 1 CPU VM.
-	if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", strconv.Itoa(runtime.NumCPU()), "CC="+compiler); err != nil {
+	cpu := strconv.Itoa(runtime.NumCPU())
+	if _, err := osutil.RunCmd(3*time.Hour, dir, "make", "bzImage", "-j", cpu, "CC="+compiler); err != nil {
 		return err
 	}
 	return nil
diff --git a/pkg/osutil/osutil.go b/pkg/osutil/osutil.go
index b977ecebf..3d5a18a67 100644
--- a/pkg/osutil/osutil.go
+++ b/pkg/osutil/osutil.go
@@ -13,6 +13,13 @@ import (
 	"path/filepath"
 	"syscall"
 	"time"
+
+	"github.com/google/syzkaller/pkg/fileutil"
+)
+
+const (
+	DefaultDirPerm  = 0755
+	DefaultFilePerm = 0644
 )
 
 // RunCmd runs "bin args..." in dir with timeout and returns its output.
@@ -94,3 +101,66 @@ func HandleInterrupts(shutdown chan struct{}) {
 		os.Exit(int(syscall.SIGINT))
 	}()
 }
+
+// FilesExist returns true if all files exist in dir.
+// Files are assumed to be relative names in slash notation.
+func FilesExist(dir string, files []string) bool {
+	for _, f := range files {
+		if !IsExist(filepath.Join(dir, filepath.FromSlash(f))) {
+			return false
+		}
+	}
+	return true
+}
+
+// CopyFiles copies files from srcDir to dstDir as atomically as possible.
+// Files are assumed to be relative names in slash notation.
+// All other files in dstDir are removed.
+func CopyFiles(srcDir, dstDir string, files []string) error {
+	// Linux does not support atomic dir replace, so we copy to tmp dir first.
+	// Then remove dst dir and rename tmp to dst (as atomic as can get on Linux).
+	tmpDir := dstDir + ".tmp"
+	if err := os.RemoveAll(tmpDir); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(tmpDir, DefaultDirPerm); err != nil {
+		return err
+	}
+	for _, f := range files {
+		src := filepath.Join(srcDir, filepath.FromSlash(f))
+		dst := filepath.Join(tmpDir, filepath.FromSlash(f))
+		if err := os.MkdirAll(filepath.Dir(dst), DefaultDirPerm); err != nil {
+			return err
+		}
+		if err := fileutil.CopyFile(src, dst); err != nil {
+			return err
+		}
+	}
+	if err := os.RemoveAll(dstDir); err != nil {
+		return err
+	}
+	return os.Rename(tmpDir, dstDir)
+}
+
+// LinkFiles creates hard links for files from dstDir to srcDir.
+// Files are assumed to be relative names in slash notation.
+// All other files in dstDir are removed.
+func LinkFiles(srcDir, dstDir string, files []string) error {
+	if err := os.RemoveAll(dstDir); err != nil {
+		return err
+	}
+	if err := os.MkdirAll(dstDir, DefaultDirPerm); err != nil {
+		return err
+	}
+	for _, f := range files {
+		src := filepath.Join(srcDir, filepath.FromSlash(f))
+		dst := filepath.Join(dstDir, filepath.FromSlash(f))
+		if err := os.MkdirAll(filepath.Dir(dst), DefaultDirPerm); err != nil {
+			return err
+		}
+		if err := os.Link(src, dst); err != nil {
+			return err
+		}
+	}
+	return nil
+}