From 52fd7b7d77d5d922e0a2ca3c28785a6ad450b49b Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 1 Jun 2020 19:09:18 +0200 Subject: tools: add syz-big-env On top of syz-env it provides akaros/fuchsia/netbsd toolchains and gcloud sdk. With this it's possible to run dashboard/app tests on CI and locally and test executor build and pkg/{csource,cover} for these OSes. Update #1765 --- tools/Dockerfile | 58 ---------------------------------------- tools/docker/big-env/Dockerfile | 40 ++++++++++++++++++++++++++++ tools/docker/env/Dockerfile | 59 +++++++++++++++++++++++++++++++++++++++++ tools/syz-big-env | 1 + tools/syz-env | 6 ++++- 5 files changed, 105 insertions(+), 59 deletions(-) delete mode 100644 tools/Dockerfile create mode 100644 tools/docker/big-env/Dockerfile create mode 100644 tools/docker/env/Dockerfile create mode 120000 tools/syz-big-env (limited to 'tools') diff --git a/tools/Dockerfile b/tools/Dockerfile deleted file mode 100644 index fb1214e4c..000000000 --- a/tools/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2020 syzkaller project authors. All rights reserved. -# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -# The image provides dev environment suitable for syzkaller development/testing. -# It includes Go toolchain, C/C++ cross-compilers, go-fuzz and golangci-lint. - -# The image is available as gcr.io/syzkaller/env. - -# To download and run locally: -# docker pull gcr.io/syzkaller/env -# docker run -it gcr.io/syzkaller/env - -# To build and push new version: -# docker build -t gcr.io/syzkaller/env tools -# gcloud auth login && gcloud auth configure-docker -# docker push gcr.io/syzkaller/env - -FROM debian:buster - -LABEL homepage="https://github.com/google/syzkaller" - -RUN dpkg --add-architecture i386 && \ - apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y -q --no-install-recommends \ - sudo make nano git curl ca-certificates clang-format binutils g++ clang \ - g++-arm-linux-gnueabi g++-aarch64-linux-gnu g++-powerpc64le-linux-gnu g++-mips64el-linux-gnuabi64 \ - libc6-dev:i386 linux-libc-dev:i386 lib32gcc-8-dev lib32stdc++-8-dev \ - # These are needed to build Linux kernel: - flex bison bc libelf-dev libssl-dev && \ - apt-get -y autoremove && \ - apt-get clean autoclean && \ - rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* - -RUN curl https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz | tar -C /usr/local -xz -ENV PATH /usr/local/go/bin:/gopath/bin:$PATH -ENV GOPATH /gopath - -# For golangci-lint we need only the binary. -RUN GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.26.0 && \ - mv /gopath/bin/* /usr/local/bin/ && \ - rm -rf /gopath - -# For go-fuzz we also need sources (go-fuzz-dep). -RUN go get github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build - -# Pre-create dirs for syz-dock. -# This is necessary to make docker work with the current user, -# otherwise --volume will create these dirs under root and then -# the current user won't have access to them. -RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \ - mkdir -p /syzkaller/.cache && \ - chmod -R 0777 /syzkaller - -# The default Docker prompt is too ugly and takes the whole line: -# I have no name!@0f3331d2fb54:~/gopath/src/github.com/google/syzkaller$ -RUN echo "export PS1='syz-env🈴 '" > /syzkaller/.bashrc - -ENTRYPOINT ["bash"] diff --git a/tools/docker/big-env/Dockerfile b/tools/docker/big-env/Dockerfile new file mode 100644 index 000000000..ae39c943c --- /dev/null +++ b/tools/docker/big-env/Dockerfile @@ -0,0 +1,40 @@ +# Copyright 2020 syzkaller project authors. All rights reserved. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +# The image provides dev environment suitable for syzkaller development/testing. +# On top of syz-env it provides akaros/fuchsia/netbsd toolchains and gcloud sdk. + +# The image is available as gcr.io/syzkaller/big-env. + +# Build as: +# docker build -t gcr.io/syzkaller/big-env tools/docker/big-env + +FROM gcr.io/syzkaller/env + +# Install OS toolchains from pre-built archives. +# These archives were created with: +# tar -czf akaros-toolchain.tar.gz akaros/toolchain +# tar -czf netbsd-toolchain.tar.gz netbsd/tools netbsd/dest +# tar -czf fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \ +# fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \ +# fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ +# fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \ +# fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ +# fuchsia/out/arm64/arm64-shared/*.so +# And then uploaded to GCS: +# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/ +RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz +RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz +RUN curl https://storage.googleapis.com/syzkaller/akaros-toolchain.tar.gz | tar -C /syzkaller -xz +ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia +ENV SOURCEDIR_NETBSD /syzkaller/netbsd +ENV SOURCEDIR_AKAROS /syzkaller/akaros + +# Install gcloud sdk for dashboard/app tests. +RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-294.0.0-linux-x86_64.tar.gz | tar -C /usr/local -xz +ENV PATH /usr/local/google-cloud-sdk/bin:$PATH +RUN gcloud components install --quiet app-engine-python app-engine-go cloud-datastore-emulator +RUN chmod 0777 /usr/local/google-cloud-sdk + +RUN echo "export PS1='syz-big-env🈶 '" > /syzkaller/.bashrc +ENV SYZ_BIG_ENV yes diff --git a/tools/docker/env/Dockerfile b/tools/docker/env/Dockerfile new file mode 100644 index 000000000..f0eaa8b91 --- /dev/null +++ b/tools/docker/env/Dockerfile @@ -0,0 +1,59 @@ +# Copyright 2020 syzkaller project authors. All rights reserved. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +# The image provides dev environment suitable for syzkaller development/testing. +# It includes Go toolchain, C/C++ cross-compilers, go-fuzz and golangci-lint. + +# The image is available as gcr.io/syzkaller/env. + +# To download and run locally: +# docker pull gcr.io/syzkaller/env +# docker run -it gcr.io/syzkaller/env + +# To build and push new version: +# docker build -t gcr.io/syzkaller/env tools/docker/env +# gcloud auth login && gcloud auth configure-docker +# docker push gcr.io/syzkaller/env + +FROM debian:buster + +LABEL homepage="https://github.com/google/syzkaller" + +RUN dpkg --add-architecture i386 && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y -q --no-install-recommends \ + sudo make nano git curl ca-certificates clang-format binutils g++ clang \ + g++-arm-linux-gnueabi g++-aarch64-linux-gnu g++-powerpc64le-linux-gnu g++-mips64el-linux-gnuabi64 \ + libc6-dev:i386 linux-libc-dev:i386 lib32gcc-8-dev lib32stdc++-8-dev \ + # These are needed to build Linux kernel: + flex bison bc libelf-dev libssl-dev && \ + apt-get -y autoremove && \ + apt-get clean autoclean && \ + rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* + +RUN curl https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz | tar -C /usr/local -xz +ENV PATH /usr/local/go/bin:/gopath/bin:$PATH +ENV GOPATH /gopath + +# For golangci-lint we need only the binary. +RUN GO111MODULE=on go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.26.0 && \ + mv /gopath/bin/* /usr/local/bin/ && \ + rm -rf /gopath + +# For go-fuzz we also need sources (go-fuzz-dep). +RUN go get github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build + +# Pre-create dirs for syz-dock. +# This is necessary to make docker work with the current user, +# otherwise --volume will create these dirs under root and then +# the current user won't have access to them. +RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \ + mkdir -p /syzkaller/.cache && \ + chmod -R 0777 /syzkaller + +# The default Docker prompt is too ugly and takes the whole line: +# I have no name!@0f3331d2fb54:~/gopath/src/github.com/google/syzkaller$ +RUN echo "export PS1='syz-env🈴 '" > /syzkaller/.bashrc +ENV SYZ_ENV yes + +ENTRYPOINT ["bash"] diff --git a/tools/syz-big-env b/tools/syz-big-env new file mode 120000 index 000000000..f0d710eb1 --- /dev/null +++ b/tools/syz-big-env @@ -0,0 +1 @@ +syz-env \ No newline at end of file diff --git a/tools/syz-env b/tools/syz-env index e8686cf6c..182db8752 100755 --- a/tools/syz-env +++ b/tools/syz-env @@ -47,6 +47,10 @@ if [ "$COMMAND" == "" ]; then fi SCRIPT_DIR="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd -P)" +IMAGE="env" +if [ "$(basename -- "$0")" == "syz-big-env" ]; then + IMAGE="big-env" +fi # Run everything as the host user, this is important for created/modified files. docker run \ @@ -62,4 +66,4 @@ docker run \ --env GITHUB_SHA \ --env CI \ ${DOCKERARGS[@]} \ - gcr.io/syzkaller/env -c "$COMMAND" + gcr.io/syzkaller/${IMAGE} -c "$COMMAND" -- cgit mrf-deployment