tools/docker/env: download toolchains in a separate builder

This allows for better caching and parallelization.

1 files changed, 48 insertions, 40 deletions

diff --git a/tools/docker/env/Dockerfile b/tools/docker/env/Dockerfile
index 42c9eaf51..e97c46562 100644
--- a/tools/docker/env/Dockerfile
+++ b/tools/docker/env/Dockerfile
@@ -14,8 +14,48 @@ RUN cd /tmp/ && tar -zxf Python-2.7.18.tgz
 RUN cd /tmp/Python-2.7.18 && ./configure --prefix=/python2/
 RUN cd /tmp/Python-2.7.18 && make -j4 && make altinstall
 
-# Now build syz-env.
+# Construct a /syzkaller folder.
+FROM debian:bookworm as syzkaller-folder
+WORKDIR /syzkaller
+RUN apt-get update --allow-releaseinfo-change
+RUN DEBIAN_FRONTEND=noninteractive apt-get install -y -q curl
+
+# Pre-create dirs for syz-dock.
+# This is necessary to make docker work with the current user,
+# otherwise --volume will create these dirs under root and then
+# the current user won't have access to them.
+RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \
+	mkdir -p /syzkaller/.cache && \
+	chmod -R 0777 /syzkaller
 
+# Install OS toolchains from pre-built archives.
+# These archives were created with:
+# tar -cz --owner=0 --group=0 --mode=go=u -f netbsd-toolchain.tar.gz netbsd/tools netbsd/dest
+# tar -cz --owner=0 --group=0 --mode=go=u -f fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \
+#	fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \
+#	fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \
+#	fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \
+#	fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \
+#	fuchsia/out/arm64/arm64-shared/*.so
+#
+# And then uploaded to GCS with:
+# gsutil mv gs://syzkaller/GOOS-toolchain.tar.gz gs://syzkaller/GOOS-toolchain.old.tar.gz
+# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/
+# gsutil acl ch -g all:R gs://syzkaller/GOOS-toolchain.tar.gz
+#
+# NetBSD toolchain can be re-built with:
+# ./build.sh -j72 -m amd64 -U -T ../tools tools
+# ./build.sh -j72 -m amd64 -U -T ../tools -D ../dest distribution
+#
+# To build root image run:
+#	docker run -it --rm --privileged --device /dev/loop0 gcr.io/syzkaller/env
+#	mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
+#	create-image.sh -a s390x -d buster
+
+RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz
+RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz
+
+# Now build the actual syz-env container.
 FROM debian:bookworm
 
 LABEL homepage="https://github.com/google/syzkaller"
@@ -44,14 +84,7 @@ RUN curl https://dl.google.com/go/go1.22.7.linux-amd64.tar.gz | tar -C /usr/loca
 ENV PATH /usr/local/go/bin:/gopath/bin:$PATH
 ENV GOPATH /gopath
 
-# Pre-create dirs for syz-dock.
-# This is necessary to make docker work with the current user,
-# otherwise --volume will create these dirs under root and then
-# the current user won't have access to them.
-RUN mkdir -p /syzkaller/gopath/src/github.com/google/syzkaller && \
-	mkdir -p /syzkaller/.cache && \
-	chmod -R 0777 /syzkaller
-
+# Install clang.
 RUN apt-get install -y -q gnupg software-properties-common apt-transport-https
 RUN curl https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
 RUN add-apt-repository "deb http://apt.llvm.org/bookworm/ llvm-toolchain-bookworm-15 main"
@@ -65,37 +98,12 @@ RUN sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/b
 RUN apt autoremove -y -q
 
 # Install the Spanner emulator.
-
 ARG SPANNER_EMULATOR_VERSION=1.5.28
 RUN mkdir /spanner
 RUN curl https://storage.googleapis.com/cloud-spanner-emulator/releases/${SPANNER_EMULATOR_VERSION}/cloud-spanner-emulator_linux_amd64-${SPANNER_EMULATOR_VERSION}.tar.gz | tar -C /spanner -xz
 RUN chmod u+x /spanner/gateway_main /spanner/emulator_main
 ENV SPANNER_EMULATOR_BIN=/spanner/gateway_main
 
-# Install OS toolchains from pre-built archives.
-# These archives were created with:
-# tar -cz --owner=0 --group=0 --mode=go=u -f netbsd-toolchain.tar.gz netbsd/tools netbsd/dest
-# tar -cz --owner=0 --group=0 --mode=go=u -f fuchsia-toolchain.tar.gz fuchsia/prebuilt/third_party/clang \
-#	fuchsia/zircon/system/ulib fuchsia/src/lib/ddk fuchsia/out/x64/fidling/gen \
-#	fuchsia/out/x64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \
-#	fuchsia/out/x64/x64-shared/*.so fuchsia/out/arm64/fidling/gen \
-#	fuchsia/out/arm64/zircon_toolchain/obj/zircon/public/sysroot/sysroot \
-#	fuchsia/out/arm64/arm64-shared/*.so
-#
-# And then uploaded to GCS with:
-# gsutil mv gs://syzkaller/GOOS-toolchain.tar.gz gs://syzkaller/GOOS-toolchain.old.tar.gz
-# gsutil cp GOOS-toolchain.tar.gz gs://syzkaller/
-# gsutil acl ch -g all:R gs://syzkaller/GOOS-toolchain.tar.gz
-#
-# NetBSD toolchain can be re-built with:
-# ./build.sh -j72 -m amd64 -U -T ../tools tools
-# ./build.sh -j72 -m amd64 -U -T ../tools -D ../dest distribution
-#
-# To build root image run:
-#	docker run -it --rm --privileged --device /dev/loop0 gcr.io/syzkaller/env
-#	mount binfmt_misc -t binfmt_misc /proc/sys/fs/binfmt_misc
-#	create-image.sh -a s390x -d buster
-
 RUN dpkg --add-architecture i386 && \
 	apt-get update --allow-releaseinfo-change && \
 	DEBIAN_FRONTEND=noninteractive apt-get install -y -q \
@@ -105,16 +113,16 @@ RUN dpkg --add-architecture i386 && \
 	apt-get clean autoclean && \
 	rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} /tmp/* /var/tmp/*
 
-
-RUN curl https://storage.googleapis.com/syzkaller/fuchsia-toolchain.tar.gz | tar -C /syzkaller -xz
-RUN curl https://storage.googleapis.com/syzkaller/netbsd-toolchain.tar.gz | tar -C /syzkaller -xz
-ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia
-ENV SOURCEDIR_NETBSD /syzkaller/netbsd
-
 # Install Python 2.7.
 COPY --from=python2-builder /python2/ /usr/local/
 RUN ln -s /usr/local/bin/python2.7 /usr/bin/python2
 
+# Copy the /syzkaller folder and set the toolchain environment variables.
+COPY --from=syzkaller-folder /syzkaller/ /syzkaller/
+RUN chmod 0777 /syzkaller
+ENV SOURCEDIR_FUCHSIA /syzkaller/fuchsia
+ENV SOURCEDIR_NETBSD /syzkaller/netbsd
+
 # Install node to pass act jobs (https://github.com/nektos/act)
 RUN apt-get install -y -q nodejs