tools: add syz-big-env

On top of syz-env it provides akaros/fuchsia/netbsd toolchains and gcloud sdk.
With this it's possible to run dashboard/app tests on CI and locally
and test executor build and pkg/{csource,cover} for these OSes.

Update #1765

10 files changed, 136 insertions, 31 deletions

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f0677240d..e9c332cf9 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -25,12 +25,34 @@ jobs:
           key: cache
       # Run make presubmit_smoke.
       - name: run
-        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh make presubmit_smoke
-      # Upload coverage report to codecov.io.
+        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env make presubmit_smoke
+      # Upload coverage report to codecov.io. For reference see:
+      # https://github.com/codecov/codecov-action/blob/master/README.md
       - name: codecov
         uses: codecov/codecov-action@v1
         with:
           file: gopath/src/github.com/google/syzkaller/.coverage.txt
+          flags: unittests
+  dashboard:
+    runs-on: ubuntu-latest
+    needs: [smoke]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          path: gopath/src/github.com/google/syzkaller
+      - name: cache
+        uses: actions/cache@v1
+        with:
+          path: .cache
+          key: cache
+      - name: run
+        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-big-env make presubmit_big
+      - name: codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: gopath/src/github.com/google/syzkaller/.coverage.txt
+          flags: dashboard
   arch:
     runs-on: ubuntu-latest
     needs: [smoke]
@@ -45,7 +67,22 @@ jobs:
           path: .cache
           key: cache
       - name: run
-        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh make presubmit_arch
+        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-big-env make presubmit_arch
+  race:
+    runs-on: ubuntu-latest
+    needs: [smoke]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          path: gopath/src/github.com/google/syzkaller
+      - name: cache
+        uses: actions/cache@v1
+        with:
+          path: .cache
+          key: cache
+      - name: run
+        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env make presubmit_race
   fuzzit:
     runs-on: ubuntu-latest
     needs: [smoke]
@@ -61,9 +98,9 @@ jobs:
           key: cache
       - name: regression
         if: ${{ github.event_name == 'pull_request' }}
-        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh tools/fuzzit.sh local-regression
+        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env tools/fuzzit.sh local-regression
       - name: fuzzing
         if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
         env:
           FUZZIT_API_KEY: ${{ secrets.FUZZIT_API_KEY }}
-        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh tools/fuzzit.sh fuzzing
+        run: gopath/src/github.com/google/syzkaller/.github/workflows/run.sh syz-env tools/fuzzit.sh fuzzing
diff --git a/.github/workflows/run.sh b/.github/workflows/run.sh
index 82fc6fa05..697d7e366 100755
--- a/.github/workflows/run.sh
+++ b/.github/workflows/run.sh
@@ -7,5 +7,5 @@ mkdir -p .cache
 set -o pipefail
 # Run the specified command in syz-env and convert error messages to github format:
 # https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message
-gopath/src/github.com/google/syzkaller/tools/syz-env $@ | \
+gopath/src/github.com/google/syzkaller/tools/$1 "${@:2}" | \
 	sed -E 's#\s*(.+):([0-9]+):([0-9]+): (.+)#\0\n::error file=\1,line=\2,col=\3::\4#'
diff --git a/Makefile b/Makefile
index 36f6cfa10..b4572b337 100644
--- a/Makefile
+++ b/Makefile
@@ -106,6 +106,7 @@ endif
 	arch_linux_arm64_target arch_linux_arm_target arch_linux_ppc64le_target arch_linux_mips64le_target \
 	arch_freebsd_amd64_target arch_freebsd_386_target \
 	arch_netbsd_amd64_target arch_windows_amd64_target \
+	arch_akaros_target arch_fuchsia_target \
 	arch_test presubmit presubmit_parallel clean
 
 all: host target
@@ -300,6 +301,13 @@ arch_openbsd_amd64_target:
 arch_windows_amd64_target:
 	env TARGETOS=windows TARGETARCH=amd64 $(MAKE) target
 
+arch_akaros_target:
+	env TARGETOS=akaros TARGETARCH=amd64 $(MAKE) executor
+
+arch_fuchsia_target:
+	env TARGETOS=fuchsia TARGETARCH=amd64 $(MAKE) executor
+	env TARGETOS=fuchsia TARGETARCH=arm64 $(MAKE) executor
+
 arch_test:
 	env TARGETOS=test TARGETARCH=64 $(MAKE) executor
 	env TARGETOS=test TARGETARCH=64_fork $(MAKE) executor
@@ -309,6 +317,7 @@ arch_test:
 presubmit:
 	$(MAKE) presubmit_smoke
 	$(MAKE) presubmit_arch
+	$(MAKE) presubmit_race
 
 presubmit_smoke:
 	$(MAKE) generate
@@ -321,7 +330,7 @@ presubmit_build:
 	$(GO) install ./...
 	$(MAKE) lint
 
-presubmit_arch:
+presubmit_arch: descriptions
 	$(MAKE) arch_linux_amd64_host
 	$(MAKE) arch_freebsd_amd64_host
 	$(MAKE) arch_netbsd_amd64_host
@@ -338,18 +347,24 @@ presubmit_arch:
 	$(MAKE) arch_openbsd_amd64_target
 	$(MAKE) arch_darwin_amd64_host
 	$(MAKE) arch_windows_amd64_target
+	$(MAKE) arch_akaros_target
+	$(MAKE) arch_fuchsia_target
 	$(MAKE) arch_test
-	$(MAKE) test_race
 
-test: descriptions
-	$(GO) test -short -coverprofile=.coverage.txt ./...
+presubmit_big: descriptions
+	# This target runs on CI in syz-big-env,
+	# so we test packages that need GCloud SDK or OS toolchains.
+	$(GO) test -short -coverprofile=.coverage.txt ./dashboard/app ./pkg/csource ./pkg/cover
 
-test_race: descriptions
+presubmit_race: descriptions
 	# -race requires cgo
 	env CGO_ENABLED=1 $(GO) test -race; if test $$? -ne 2; then \
 	env CGO_ENABLED=1 $(GO) test -race -short -bench=.* -benchtime=.2s ./... ;\
 	fi
 
+test: descriptions
+	$(GO) test -short -coverprofile=.coverage.txt ./...
+
 clean:
 	rm -rf ./bin .descriptions sys/*/gen executor/defs.h executor/syscalls.h
 
diff --git a/dashboard/app/util_test.go b/dashboard/app/util_test.go
index 092ad3d17..e8daa19fc 100644
--- a/dashboard/app/util_test.go
+++ b/dashboard/app/util_test.go
@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"reflect"
@@ -41,8 +42,14 @@ type Ctx struct {
 	client2    *apiClient
 }
 
+var skipDevAppserverTests = func() bool {
+	_, err := exec.LookPath("dev_appserver.py")
+	// Don't silently skip tests on CI, we should have gcloud sdk installed there.
+	return err != nil && os.Getenv("SYZ_BIG_ENV") == ""
+}()
+
 func NewCtx(t *testing.T) *Ctx {
-	if _, err := exec.LookPath("dev_appserver.py"); err != nil {
+	if skipDevAppserverTests {
 		t.Skip("skipping test (no dev_appserver.py)")
 	}
 	t.Parallel()
diff --git a/docs/netbsd/README.md b/docs/netbsd/README.md
index 121fbef58..fc9c2d821 100644
--- a/docs/netbsd/README.md
+++ b/docs/netbsd/README.md
@@ -36,7 +36,7 @@ At this point you should have a NetBSD distribution in `$HOME/netbsd/dest`.
 
 3. Compile syzkaller for NetBSD.
 	```sh
-	host$ make TARGETOS=netbsd SOURCEDIR=$HOME/netbsd/src
+	host$ make TARGETOS=netbsd SOURCEDIR=$HOME/netbsd
 	```
 
 The above steps should have built the syzkaller binaries for NetBSD.
diff --git a/sys/targets/targets.go b/sys/targets/targets.go
index 7ad2865dd..47b3d8a97 100644
--- a/sys/targets/targets.go
+++ b/sys/targets/targets.go
@@ -219,9 +219,9 @@ var List = map[string]map[string]*Target{
 			CFlags: []string{
 				"-m64",
 				"-static",
-				"--sysroot", sourceDirVar + "/../dest/",
+				"--sysroot", sourceDirVar + "/dest/",
 			},
-			CCompiler: sourceDirVar + "/../tools/bin/x86_64--netbsd-g++",
+			CCompiler: sourceDirVar + "/tools/bin/x86_64--netbsd-g++",
 		},
 	},
 	"openbsd": {
@@ -270,16 +270,16 @@ var List = map[string]map[string]*Target{
 				"-ldriver",
 				"-lfdio",
 				"-lzircon",
-				"--sysroot", sourceDirVar + "/out/x64/sdk/exported/zircon_sysroot/arch/x64/sysroot",
-				"-I", sourceDirVar + "/zircon/system/ulib/ddk/include",
+				"--sysroot", sourceDirVar + "/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot",
 				"-I", sourceDirVar + "/zircon/system/ulib/fdio/include",
 				"-I", sourceDirVar + "/zircon/system/ulib/fidl/include",
-				"-I", sourceDirVar + "/out/x64/fidling/gen/zircon/system/fidl/fuchsia-device",
-				"-I", sourceDirVar + "/out/x64/fidling/gen/zircon/system/fidl/fuchsia-device-manager",
-				"-I", sourceDirVar + "/out/x64/fidling/gen/zircon/system/fidl/fuchsia-hardware-nand",
-				"-I", sourceDirVar + "/out/x64/fidling/gen/zircon/system/fidl/fuchsia-hardware-usb-peripheral",
+				"-I", sourceDirVar + "/src/lib/ddk/include",
+				"-I", sourceDirVar + "/out/x64/fidling/gen/sdk/fidl/fuchsia.device",
+				"-I", sourceDirVar + "/out/x64/fidling/gen/sdk/fidl/fuchsia.device.manager",
+				"-I", sourceDirVar + "/out/x64/fidling/gen/sdk/fidl/fuchsia.hardware.nand",
+				"-I", sourceDirVar + "/out/x64/fidling/gen/sdk/fidl/fuchsia.hardware.usb.peripheral",
+				"-I", sourceDirVar + "/out/x64/fidling/gen/zircon/vdso/zx",
 				"-L", sourceDirVar + "/out/x64/x64-shared",
-				"-L", sourceDirVar + "/out/x64/gen/zircon/public/lib/fdio",
 			},
 		},
 		"arm64": {
@@ -294,16 +294,16 @@ var List = map[string]map[string]*Target{
 				"-ldriver",
 				"-lfdio",
 				"-lzircon",
-				"--sysroot", sourceDirVar + "/out/arm64/sdk/exported/zircon_sysroot/arch/arm64/sysroot",
-				"-I", sourceDirVar + "/zircon/system/ulib/ddk/include",
+				"--sysroot", sourceDirVar + "/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot",
 				"-I", sourceDirVar + "/zircon/system/ulib/fdio/include",
 				"-I", sourceDirVar + "/zircon/system/ulib/fidl/include",
-				"-I", sourceDirVar + "/out/arm64/fidling/gen/zircon/system/fidl/fuchsia-device",
-				"-I", sourceDirVar + "/out/arm64/fidling/gen/zircon/system/fidl/fuchsia-device-manager",
-				"-I", sourceDirVar + "/out/arm64/fidling/gen/zircon/system/fidl/fuchsia-hardware-nand",
-				"-I", sourceDirVar + "/out/arm64/fidling/gen/zircon/system/fidl/fuchsia-hardware-usb-peripheral",
+				"-I", sourceDirVar + "/src/lib/ddk/include",
+				"-I", sourceDirVar + "/out/arm64/fidling/gen/sdk/fidl/fuchsia.device",
+				"-I", sourceDirVar + "/out/arm64/fidling/gen/sdk/fidl/fuchsia.device.manager",
+				"-I", sourceDirVar + "/out/arm64/fidling/gen/sdk/fidl/fuchsia.hardware.nand",
+				"-I", sourceDirVar + "/out/arm64/fidling/gen/sdk/fidl/fuchsia.hardware.usb.peripheral",
+				"-I", sourceDirVar + "/out/arm64/fidling/gen/zircon/vdso/zx",
 				"-L", sourceDirVar + "/out/arm64/arm64-shared",
-				"-L", sourceDirVar + "/out/arm64/gen/zircon/public/lib/fdio",
 			},
 		},
 	},
diff --git a/tools/docker/big-env/Dockerfile b/tools/docker/big-env/Dockerfile
new file mode 100644
index 000000000..ae39c943c
--- /dev/null
+++ b/tools/docker/big-env/Dockerfile
@@ -0,0 +1,40 @@
+# Copyright 2020 syzkaller project authors. All rights reserved.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+# The image provides dev environment suitable for syzkaller development/testing.
+# On top of syz-env it provides akaros/fuchsia/netbsd toolchains and gcloud sdk.
+
+# The image is available as gcr.io/syzkaller/big-env.
+
+# Build as:
+# docker build -t gcr.io/syzkaller/big-env tools/docker/big-env
+
+FROM gcr.io/syzkaller/env
+
+# Install OS toolchains from pre-built archives.
+# These archives were created with:
+# tar -czf akaros-toolchain.tar.gz akaros/toolchain
+# tar -czf netbsd-toolchain.tar.gz netbsd/tools netbsd/dest
+# tar -czf fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \
+#	fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \
+#	fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \
+#	fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \
+#	fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \
+#	fuchsia/out/arm64/arm64-shared/*.so
+# And then uploaded to GCS:
+# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/
+RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz
+RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz
+RUN curl https://storage.googleapis.com/syzkaller/akaros-toolchain.tar.gz | tar -C /syzkaller -xz
+ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia
+ENV SOURCEDIR_NETBSD /syzkaller/netbsd
+ENV SOURCEDIR_AKAROS /syzkaller/akaros
+
+# Install gcloud sdk for dashboard/app tests.
+RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-294.0.0-linux-x86_64.tar.gz | tar -C /usr/local -xz
+ENV PATH /usr/local/google-cloud-sdk/bin:$PATH
+RUN gcloud components install --quiet app-engine-python app-engine-go cloud-datastore-emulator
+RUN chmod 0777 /usr/local/google-cloud-sdk
+
+RUN echo "export PS1='syz-big-env🈶 '" > /syzkaller/.bashrc
+ENV SYZ_BIG_ENV yes
diff --git a/tools/Dockerfile b/tools/docker/env/Dockerfile
index fb1214e4c..f0eaa8b91 100644
--- a/tools/Dockerfile
+++ b/tools/docker/env/Dockerfile
@@ -11,7 +11,7 @@
 #	docker run -it gcr.io/syzkaller/env
 
 # To build and push new version:
-#	docker build -t gcr.io/syzkaller/env tools
+#	docker build -t gcr.io/syzkaller/env tools/docker/env
 #	gcloud auth login && gcloud auth configure-docker
 #	docker push gcr.io/syzkaller/env
 
@@ -54,5 +54,6 @@ RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \
 # The default Docker prompt is too ugly and takes the whole line:
 # I have no name!@0f3331d2fb54:~/gopath/src/github.com/google/syzkaller$
 RUN echo "export PS1='syz-env🈴 '" > /syzkaller/.bashrc
+ENV SYZ_ENV yes
 
 ENTRYPOINT ["bash"]
diff --git a/tools/syz-big-env b/tools/syz-big-env
new file mode 120000
index 000000000..f0d710eb1
--- /dev/null
+++ b/tools/syz-big-env
@@ -0,0 +1 @@
+syz-env
\ No newline at end of file
diff --git a/tools/syz-env b/tools/syz-env
index e8686cf6c..182db8752 100755
--- a/tools/syz-env
+++ b/tools/syz-env
@@ -47,6 +47,10 @@ if [ "$COMMAND" == "" ]; then
 fi
 
 SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)"
+IMAGE="env"
+if [ "$(basename -- "$0")" == "syz-big-env" ]; then
+	IMAGE="big-env"
+fi
 
 # Run everything as the host user, this is important for created/modified files.
 docker run \
@@ -62,4 +66,4 @@ docker run \
 	--env GITHUB_SHA \
 	--env CI \
 	${DOCKERARGS[@]} \
-	gcr.io/syzkaller/env -c "$COMMAND"
+	gcr.io/syzkaller/${IMAGE} -c "$COMMAND"