From 3e6cf2cba0130e874df1d1d3af51c03d0286bc26 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Thu, 2 Jan 2025 14:12:01 +0100 Subject: tools/docker/env: download toolchains in a separate builder This allows for better caching and parallelization. --- tools/docker/env/Dockerfile | 88 ++++++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 40 deletions(-) (limited to 'tools/docker') diff --git a/tools/docker/env/Dockerfile b/tools/docker/env/Dockerfile index 42c9eaf51..e97c46562 100644 --- a/tools/docker/env/Dockerfile +++ b/tools/docker/env/Dockerfile @@ -14,8 +14,48 @@ RUN cd /tmp/ && tar -zxf Python-2.7.18.tgz RUN cd /tmp/Python-2.7.18 && ./configure --prefix=/python2/ RUN cd /tmp/Python-2.7.18 && make -j4 && make altinstall -# Now build syz-env. +# Construct a /syzkaller folder. +FROM debian:bookworm as syzkaller-folder +WORKDIR /syzkaller +RUN apt-get update --allow-releaseinfo-change +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl + +# Pre-create dirs for syz-dock. +# This is necessary to make docker work with the current user, +# otherwise --volume will create these dirs under root and then +# the current user won't have access to them. +RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \ + mkdir -p /syzkaller/.cache && \ + chmod -R 0777 /syzkaller +# Install OS toolchains from pre-built archives. +# These archives were created with: +# tar -cz --owner=0 --group=0 --mode=go=u -f netbsd-toolchain.tar.gz netbsd/tools netbsd/dest +# tar -cz --owner=0 --group=0 --mode=go=u -f fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \ +# fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \ +# fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ +# fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \ +# fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ +# fuchsia/out/arm64/arm64-shared/*.so +# +# And then uploaded to GCS with: +# gsutil mv gs://syzkaller/GOOS-toolchain.tar.gz gs://syzkaller/GOOS-toolchain.old.tar.gz +# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/ +# gsutil acl ch -g all:R gs://syzkaller/GOOS-toolchain.tar.gz +# +# NetBSD toolchain can be re-built with: +# ./build.sh -j72 -m amd64 -U -T ../tools tools +# ./build.sh -j72 -m amd64 -U -T ../tools -D ../dest distribution +# +# To build root image run: +# docker run -it --rm --privileged --device /dev/loop0 gcr.io/syzkaller/env +# mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc +# create-image.sh -a s390x -d buster + +RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz +RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz + +# Now build the actual syz-env container. FROM debian:bookworm LABEL homepage="https://github.com/google/syzkaller" @@ -44,14 +84,7 @@ RUN curl https://dl.google.com/go/go1.22.7.linux-amd64.tar.gz | tar -C /usr/loca ENV PATH /usr/local/go/bin:/gopath/bin:$PATH ENV GOPATH /gopath -# Pre-create dirs for syz-dock. -# This is necessary to make docker work with the current user, -# otherwise --volume will create these dirs under root and then -# the current user won't have access to them. -RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \ - mkdir -p /syzkaller/.cache && \ - chmod -R 0777 /syzkaller - +# Install clang. RUN apt-get install -y -q gnupg software-properties-common apt-transport-https RUN curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - RUN add-apt-repository "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main" @@ -65,37 +98,12 @@ RUN sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/b RUN apt autoremove -y -q # Install the Spanner emulator. - ARG SPANNER_EMULATOR_VERSION=1.5.28 RUN mkdir /spanner RUN curl https://storage.googleapis.com/cloud-spanner-emulator/releases/${SPANNER_EMULATOR_VERSION}/cloud-spanner-emulator_linux_amd64-${SPANNER_EMULATOR_VERSION}.tar.gz | tar -C /spanner -xz RUN chmod u+x /spanner/gateway_main /spanner/emulator_main ENV SPANNER_EMULATOR_BIN=/spanner/gateway_main -# Install OS toolchains from pre-built archives. -# These archives were created with: -# tar -cz --owner=0 --group=0 --mode=go=u -f netbsd-toolchain.tar.gz netbsd/tools netbsd/dest -# tar -cz --owner=0 --group=0 --mode=go=u -f fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \ -# fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \ -# fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ -# fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \ -# fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \ -# fuchsia/out/arm64/arm64-shared/*.so -# -# And then uploaded to GCS with: -# gsutil mv gs://syzkaller/GOOS-toolchain.tar.gz gs://syzkaller/GOOS-toolchain.old.tar.gz -# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/ -# gsutil acl ch -g all:R gs://syzkaller/GOOS-toolchain.tar.gz -# -# NetBSD toolchain can be re-built with: -# ./build.sh -j72 -m amd64 -U -T ../tools tools -# ./build.sh -j72 -m amd64 -U -T ../tools -D ../dest distribution -# -# To build root image run: -# docker run -it --rm --privileged --device /dev/loop0 gcr.io/syzkaller/env -# mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc -# create-image.sh -a s390x -d buster - RUN dpkg --add-architecture i386 && \ apt-get update --allow-releaseinfo-change && \ DEBIAN_FRONTEND=noninteractive apt-get install -y -q \ @@ -105,16 +113,16 @@ RUN dpkg --add-architecture i386 && \ apt-get clean autoclean && \ rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/* - -RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz -RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz -ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia -ENV SOURCEDIR_NETBSD /syzkaller/netbsd - # Install Python 2.7. COPY --from=python2-builder /python2/ /usr/local/ RUN ln -s /usr/local/bin/python2.7 /usr/bin/python2 +# Copy the /syzkaller folder and set the toolchain environment variables. +COPY --from=syzkaller-folder /syzkaller/ /syzkaller/ +RUN chmod 0777 /syzkaller +ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia +ENV SOURCEDIR_NETBSD /syzkaller/netbsd + # Install node to pass act jobs (https://github.com/nektos/act) RUN apt-get install -y -q nodejs -- cgit mrf-deployment