From 68aa75abe16fb593ee02b5a37783371c3dfc24b2 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Mon, 24 Feb 2025 14:30:04 +0100 Subject: syz-cluster: make image prefix and tag configurable Accept IMAGE_PREFIX and IMAGE_TAG parameters that allow to reuse the Makefile and a lot of k8s configurations both for local and prod environments. Refactor Makefile: define build-* and push-* rules, use templates to avoid repetition. --- syz-cluster/Makefile | 81 +++++++++++----------- syz-cluster/README.md | 5 +- syz-cluster/controller/deployment.yaml | 2 +- syz-cluster/dashboard/deployment.yaml | 2 +- syz-cluster/overlays/dev/kustomization.yaml | 8 --- syz-cluster/reporter/deployment.yaml | 2 +- syz-cluster/run-local.sh | 2 +- syz-cluster/series-tracker/deployment.yaml | 2 +- .../workflow/boot-step/workflow-template.yaml | 2 +- .../workflow/build-step/workflow-template.yaml | 2 +- .../workflow/fuzz-step/workflow-template.yaml | 2 +- .../workflow/triage-step/workflow-template.yaml | 2 +- 12 files changed, 53 insertions(+), 59 deletions(-) diff --git a/syz-cluster/Makefile b/syz-cluster/Makefile index 1f042b064..b3227e15c 100644 --- a/syz-cluster/Makefile +++ b/syz-cluster/Makefile @@ -1,66 +1,67 @@ # Copyright 2024 syzkaller project authors. All rights reserved. # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. -build-controller-dev: - eval $$(minikube docker-env);\ - docker build -t controller-image-local -f ./controller/Dockerfile ../ +IMAGE_TAG ?= latest +IMAGE_PREFIX ?= local/ +SHELL := /bin/bash -build-series-tracker-dev: - eval $$(minikube docker-env);\ - docker build -t series-tracker-local -f ./series-tracker/Dockerfile ../ +define build_image_rules = + image_name_$(2) := $(IMAGE_PREFIX)$(2):$(IMAGE_TAG) + dockerfile_$(2) := $(1)/Dockerfile -deploy-series-tracker-dev: build-series-tracker-dev - @kubectl rollout restart deployment series-tracker + build-$(2): + @echo "Building $$(image_name_$(2)) (Dockerfile: $$(dockerfile_$(2)))" + DOCKER_BUILDKIT=1 docker build -t $$(image_name_$(2)) -f $$(dockerfile_$(2)) ../ -run-series-tracker-dev: build-series-tracker-dev - ./run-local.sh series-tracker + push-$(2): build-$(2) + @echo "Pushing $$(image_name_$(2))" + docker push $$(image_name_$(2)) -build-web-dashboard-dev: - eval $$(minikube docker-env);\ - docker build -t web-dashboard-local -f ./dashboard/Dockerfile ../ +endef -deploy-web-dashboard-dev: build-web-dashboard-dev - @kubectl rollout restart deployment web-dashboard +$(eval $(call build_image_rules,./controller,controller)) +$(eval $(call build_image_rules,./dashboard,web-dashboard)) +$(eval $(call build_image_rules,./reporter,reporter)) +$(eval $(call build_image_rules,./series-tracker,series-tracker)) +$(eval $(call build_image_rules,./db-mgmt,db-mgmt)) +$(eval $(call build_image_rules,./workflow/triage-step,triage-step)) +$(eval $(call build_image_rules,./workflow/build-step,build-step)) +$(eval $(call build_image_rules,./workflow/fuzz-step,fuzz-step)) +$(eval $(call build_image_rules,./workflow/boot-step,boot-step)) -build-reporter-dev: - eval $$(minikube docker-env);\ - docker build -t reporter-image-local -f ./reporter/Dockerfile ../ +IMAGES := controller web-dashboard reporter series-tracker db-mgmt triage-step build-step boot-step fuzz-step +BUILD_TARGETS := $(addprefix build-, $(IMAGES)) +PUSH_TARGETS := $(addprefix push-, $(IMAGES)) -install-dev-config: - minikube kubectl -- apply -f ./overlays/dev/global-config.yaml +.PHONY: build-all push-all deploy-series-tracker run-series-tracker install-dev-config build-go-tests-dev run-go-tests-dev restart-spanner kustomize-dev -build-db-mgmt-dev: - eval $$(minikube docker-env);\ - docker build -t db-mgmt-local -f ./db-mgmt/Dockerfile ../ +build-all: $(BUILD_TARGETS) -build-triage-step-dev: - eval $$(minikube docker-env);\ - docker build -t triage-step-local -f ./workflow/triage-step/Dockerfile ../ +push-all: $(PUSH_TARGETS) -build-build-step-dev: - eval $$(minikube docker-env);\ - docker build -t build-step-local -f ./workflow/build-step/Dockerfile ../ +deploy-series-tracker: build-series-tracker + @kubectl rollout restart deployment series-tracker -build-boot-step-dev: - eval $$(minikube docker-env);\ - docker build -t boot-step-local -f ./workflow/boot-step/Dockerfile ../ +run-series-tracker: build-series-tracker + ./run-local.sh series-tracker -build-fuzz-step-dev: - eval $$(minikube docker-env);\ - docker build -t fuzz-step-local -f ./workflow/fuzz-step/Dockerfile ../ +deploy-web-dashboard: build-web-dashboard + @kubectl rollout restart deployment web-dashboard + +install-dev-config: + minikube kubectl -- apply -f ./overlays/dev/global-config.yaml build-go-tests-dev: eval $$(minikube docker-env);\ docker build -t go-tests-local -f Dockerfile.go-tests ../ -build-workflow-dev: build-triage-step-dev build-build-step-dev build-boot-step-dev build-fuzz-step-dev - -all-containers: build-controller-dev build-series-tracker-dev build-db-mgmt-dev build-web-dashboard-dev build-reporter-dev build-workflow-dev - run-go-tests-dev: build-go-tests-dev ./run-local.sh go-tests -restart-spanner: build-db-mgmt-dev +restart-spanner: build-db-mgmt minikube addons disable cloud-spanner; minikube addons enable cloud-spanner; ./run-local.sh db-mgmt migrate + +k8s-config-dev: + @kubectl kustomize ./overlays/dev/ | IMAGE_PREFIX=${IMAGE_PREFIX} IMAGE_TAG=${IMAGE_TAG} envsubst diff --git a/syz-cluster/README.md b/syz-cluster/README.md index 3c5bf5b1f..9c734618d 100644 --- a/syz-cluster/README.md +++ b/syz-cluster/README.md @@ -14,13 +14,14 @@ $ minikube addons enable cloud-spanner ``` 3. Build all docker containers (might take a while): ``` -$ make all-containers +$ eval $(minikube docker-env) +$ make build-all ``` 4. Deploy the cluster: ``` $ make restart-spanner $ kubectl create namespace argo -$ kubectl apply -k ./overlays/dev/ +$ make k8s-config-dev | kubectl apply -f - ``` 5. (Optional) Pre-fetch the kernel git repository: ``` diff --git a/syz-cluster/controller/deployment.yaml b/syz-cluster/controller/deployment.yaml index c20089eae..87977e025 100644 --- a/syz-cluster/controller/deployment.yaml +++ b/syz-cluster/controller/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: controller-image - image: controller-image # The actual image name is set in overalys. + image: ${IMAGE_PREFIX}controller:${IMAGE_TAG} envFrom: - configMapRef: name: global-config diff --git a/syz-cluster/dashboard/deployment.yaml b/syz-cluster/dashboard/deployment.yaml index e33ce0ef6..766583738 100644 --- a/syz-cluster/dashboard/deployment.yaml +++ b/syz-cluster/dashboard/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: web-dashboard-image - image: web-dashboard-image + image: ${IMAGE_PREFIX}web-dashboard:${IMAGE_TAG} envFrom: - configMapRef: name: global-config diff --git a/syz-cluster/overlays/dev/kustomization.yaml b/syz-cluster/overlays/dev/kustomization.yaml index fb122b390..ff4e93d20 100644 --- a/syz-cluster/overlays/dev/kustomization.yaml +++ b/syz-cluster/overlays/dev/kustomization.yaml @@ -34,11 +34,3 @@ patches: env: - name: STORAGE_EMULATOR_HOST value: http://fake-gcs-server.default.svc.cluster.local:4443 - -images: - - name: controller-image - newName: controller-image-local:latest - - name: web-dashboard-image - newName: web-dashboard-local:latest - - name: series-tracker-image - newName: series-tracker-local:latest diff --git a/syz-cluster/reporter/deployment.yaml b/syz-cluster/reporter/deployment.yaml index e1a14690f..c7ca9a311 100644 --- a/syz-cluster/reporter/deployment.yaml +++ b/syz-cluster/reporter/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: reporter-image - image: reporter-image # The actual image name is set in overalys. + image: ${IMAGE_PREFIX}reporter:${IMAGE_TAG} envFrom: - configMapRef: name: global-config diff --git a/syz-cluster/run-local.sh b/syz-cluster/run-local.sh index 75e7867bc..f4d311a32 100755 --- a/syz-cluster/run-local.sh +++ b/syz-cluster/run-local.sh @@ -14,7 +14,7 @@ alias kubectl="minikube kubectl --" # Clean up in case the run comand was prematurely aborted. # TODO: find out how to rely on envs from overlays/dev/global-config.yaml. kubectl delete pod run-local >/dev/null 2>&1 || true -kubectl run run-local --image="$name-local" \ +kubectl run run-local --image="local/$name" \ --image-pull-policy=Never \ --restart=Never \ --env="SPANNER_EMULATOR_HOST=cloud-spanner-emulator:9010" \ diff --git a/syz-cluster/series-tracker/deployment.yaml b/syz-cluster/series-tracker/deployment.yaml index b8fbb37fd..d42c5a88d 100644 --- a/syz-cluster/series-tracker/deployment.yaml +++ b/syz-cluster/series-tracker/deployment.yaml @@ -17,7 +17,7 @@ spec: spec: containers: - name: series-tracker-image - image: series-tracker-image + image: ${IMAGE_PREFIX}series-tracker:${IMAGE_TAG} envFrom: - configMapRef: name: global-config diff --git a/syz-cluster/workflow/boot-step/workflow-template.yaml b/syz-cluster/workflow/boot-step/workflow-template.yaml index adbe2800d..241efa85c 100644 --- a/syz-cluster/workflow/boot-step/workflow-template.yaml +++ b/syz-cluster/workflow/boot-step/workflow-template.yaml @@ -24,7 +24,7 @@ spec: - name: kernel path: /base container: - image: boot-step-local + image: ${IMAGE_PREFIX}boot-step:${IMAGE_TAG} imagePullPolicy: IfNotPresent command: ["/bin/boot-step"] args: [ diff --git a/syz-cluster/workflow/build-step/workflow-template.yaml b/syz-cluster/workflow/build-step/workflow-template.yaml index 0a3282ae8..9d3f3485c 100644 --- a/syz-cluster/workflow/build-step/workflow-template.yaml +++ b/syz-cluster/workflow/build-step/workflow-template.yaml @@ -52,7 +52,7 @@ spec: capabilities: add: ["SYS_ADMIN"] container: - image: build-step-local + image: ${IMAGE_PREFIX}build-step:${IMAGE_TAG} imagePullPolicy: IfNotPresent command: ["/bin/build-step"] args: [ diff --git a/syz-cluster/workflow/fuzz-step/workflow-template.yaml b/syz-cluster/workflow/fuzz-step/workflow-template.yaml index abdb4eadc..9fd2abecd 100644 --- a/syz-cluster/workflow/fuzz-step/workflow-template.yaml +++ b/syz-cluster/workflow/fuzz-step/workflow-template.yaml @@ -23,7 +23,7 @@ spec: path: /patched timeout: 4h container: - image: fuzz-step-local + image: ${IMAGE_PREFIX}fuzz-step:${IMAGE_TAG} imagePullPolicy: IfNotPresent command: ["/bin/fuzz-step"] args: [ diff --git a/syz-cluster/workflow/triage-step/workflow-template.yaml b/syz-cluster/workflow/triage-step/workflow-template.yaml index 270c6a0fd..d55430842 100644 --- a/syz-cluster/workflow/triage-step/workflow-template.yaml +++ b/syz-cluster/workflow/triage-step/workflow-template.yaml @@ -38,7 +38,7 @@ spec: capabilities: add: ["SYS_ADMIN"] container: - image: triage-step-local + image: ${IMAGE_PREFIX}triage-step:${IMAGE_TAG} imagePullPolicy: IfNotPresent command: ["/bin/triage-step"] args: [ -- cgit mrf-deployment