From 351f479562d9d7868be8bb644f30bf047767c828 Mon Sep 17 00:00:00 2001 From: Aleksandr Nogikh Date: Mon, 19 May 2025 14:55:17 +0200 Subject: syz-cluster: split off argo workflows setup We cannot at the same time define Argo CRDs and use them because K8S needs some time to process the CRD creation. To facilitate the installation without errors, split off the step that installs Argo and create a step that waits until CRDs are installed. --- syz-cluster/Makefile | 13 +- syz-cluster/README.md | 2 + .../overlays/common/argo/kustomization.yaml | 10 ++ .../common/argo/patch-argo-controller.yaml | 13 ++ .../argo/patch-workflow-controller-configmap.yaml | 12 ++ .../overlays/common/argo/workflow-roles.yaml | 133 +++++++++++++++++++++ syz-cluster/overlays/common/kustomization.yaml | 6 - .../overlays/common/patch-argo-controller.yaml | 13 -- .../patch-workflow-controller-configmap.yaml | 12 -- syz-cluster/overlays/common/workflow-roles.yaml | 133 --------------------- syz-cluster/overlays/minikube/kustomization.yaml | 1 + 11 files changed, 183 insertions(+), 165 deletions(-) create mode 100644 syz-cluster/overlays/common/argo/kustomization.yaml create mode 100644 syz-cluster/overlays/common/argo/patch-argo-controller.yaml create mode 100644 syz-cluster/overlays/common/argo/patch-workflow-controller-configmap.yaml create mode 100644 syz-cluster/overlays/common/argo/workflow-roles.yaml delete mode 100644 syz-cluster/overlays/common/patch-argo-controller.yaml delete mode 100644 syz-cluster/overlays/common/patch-workflow-controller-configmap.yaml delete mode 100644 syz-cluster/overlays/common/workflow-roles.yaml diff --git a/syz-cluster/Makefile b/syz-cluster/Makefile index af203c1bd..9b2807d03 100644 --- a/syz-cluster/Makefile +++ b/syz-cluster/Makefile @@ -64,9 +64,20 @@ restart-spanner: build-db-mgmt minikube addons enable cloud-spanner; ./run-local.sh db-mgmt migrate - SUBSTITUTE_VARS := '$$IMAGE_PREFIX $$IMAGE_TAG $$SPANNER_DATABASE_URI $$BLOB_STORAGE_GCS_BUCKET $$WORKFLOW_ARTIFACTS_BUCKET' +# It takes some time for K8S to set up Argo CRDs, so if we `kubectl apply` everything at once, it will not +# work on the first attempt. +# The following two targets facilitate CI/CD: +# - k8s-config-argo installs Argo workflows. +# - k8s-config-argo-wait awaits untill all CRDs are ready. +k8s-config-argo: + @kubectl kustomize ./overlays/common/argo/ + +k8s-config-argo-wait: + @kubectl wait --for condition=established --timeout=60s crd/cronworkflows.argoproj.io + @kubectl wait --for condition=established --timeout=60s crd/workflows.argoproj.io + k8s-config-dev: @kubectl kustomize ./overlays/minikube/ | IMAGE_PREFIX=${IMAGE_PREFIX} IMAGE_TAG=${IMAGE_TAG} envsubst ${SUBSTITUTE_VARS} diff --git a/syz-cluster/README.md b/syz-cluster/README.md index df108ffd3..95533ef4b 100644 --- a/syz-cluster/README.md +++ b/syz-cluster/README.md @@ -21,6 +21,8 @@ $ make build-all ``` $ make restart-spanner $ kubectl create namespace argo +$ make k8s-config-argo | kubectl apply -f - +$ make k8s-config-argo-wait $ make k8s-config-dev | kubectl apply -f - ``` 5. (Optional) Pre-fetch the kernel git repository: diff --git a/syz-cluster/overlays/common/argo/kustomization.yaml b/syz-cluster/overlays/common/argo/kustomization.yaml new file mode 100644 index 000000000..f864f87e9 --- /dev/null +++ b/syz-cluster/overlays/common/argo/kustomization.yaml @@ -0,0 +1,10 @@ +# Copyright 2025 syzkaller project authors. All rights reserved. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +resources: + - https://github.com/argoproj/argo-workflows/releases/download/v3.6.2/install.yaml + - workflow-roles.yaml + +patchesStrategicMerge: + - patch-argo-controller.yaml + - patch-workflow-controller-configmap.yaml diff --git a/syz-cluster/overlays/common/argo/patch-argo-controller.yaml b/syz-cluster/overlays/common/argo/patch-argo-controller.yaml new file mode 100644 index 000000000..d79585dc1 --- /dev/null +++ b/syz-cluster/overlays/common/argo/patch-argo-controller.yaml @@ -0,0 +1,13 @@ +# Copyright 2025 syzkaller project authors. All rights reserved. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: workflow-controller + namespace: argo +spec: + template: + spec: + serviceAccountName: argo-controller-ksa diff --git a/syz-cluster/overlays/common/argo/patch-workflow-controller-configmap.yaml b/syz-cluster/overlays/common/argo/patch-workflow-controller-configmap.yaml new file mode 100644 index 000000000..0ff55f9bb --- /dev/null +++ b/syz-cluster/overlays/common/argo/patch-workflow-controller-configmap.yaml @@ -0,0 +1,12 @@ +# Copyright 2025 syzkaller project authors. All rights reserved. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: workflow-controller-configmap + namespace: argo +data: + workflowDefaults: | + spec: + serviceAccountName: argo-executor-ksa diff --git a/syz-cluster/overlays/common/argo/workflow-roles.yaml b/syz-cluster/overlays/common/argo/workflow-roles.yaml new file mode 100644 index 000000000..d1f1274db --- /dev/null +++ b/syz-cluster/overlays/common/argo/workflow-roles.yaml @@ -0,0 +1,133 @@ +# Copyright 2025 syzkaller project authors. All rights reserved. +# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-workflow-role +rules: +- apiGroups: + - argoproj.io + resources: + - workflows + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - status + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: argo-workflowtasks-role +rules: +- apiGroups: ["argoproj.io"] + resources: + - workflowtaskresults + verbs: + - create + - patch +- apiGroups: + - argoproj.io + resources: + - workflowtasksets + - workflowartifactgctasks + verbs: + - list + - watch +- apiGroups: + - argoproj.io + resources: + - workflowtasksets/status + - workflowartifactgctasks/status + verbs: + - patch + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-workflow-role-binding + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argo-workflow-role +subjects: +- kind: ServiceAccount + name: argo-executor-ksa + namespace: default +- kind: ServiceAccount + name: argo-controller-ksa + namespace: argo +- kind: ServiceAccount + name: gke-service-ksa + namespace: default + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-workflowtasks-role-binding + namespace: argo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argo-workflowtasks-role +subjects: +- kind: ServiceAccount + name: argo-executor-ksa + namespace: default +- kind: ServiceAccount + name: argo-controller-ksa + namespace: argo + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: argo-cluster-role-to-controller-binding + namespace: argo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: argo-cluster-role +subjects: +- kind: ServiceAccount + name: argo-controller-ksa + namespace: argo + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: argo-role-to-controller-binding + namespace: argo +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: argo-role +subjects: +- kind: ServiceAccount + name: argo-controller-ksa + namespace: argo + +--- + +apiVersion: v1 +kind: Secret +metadata: + annotations: + kubernetes.io/service-account.name: argo-executor-ksa + name: argo-executor-ksa.service-account-token +type: kubernetes.io/service-account-token diff --git a/syz-cluster/overlays/common/kustomization.yaml b/syz-cluster/overlays/common/kustomization.yaml index e1b31e091..d35997918 100644 --- a/syz-cluster/overlays/common/kustomization.yaml +++ b/syz-cluster/overlays/common/kustomization.yaml @@ -2,7 +2,6 @@ # Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. resources: - - https://github.com/argoproj/argo-workflows/releases/download/v3.6.2/install.yaml - ../../controller - ../../dashboard - ../../series-tracker @@ -13,11 +12,6 @@ resources: - network-policy-controller.yaml - network-policy-git-access.yaml - network-policy-web-dashboard.yaml - - workflow-roles.yaml - -patchesStrategicMerge: - - patch-argo-controller.yaml - - patch-workflow-controller-configmap.yaml patches: - target: diff --git a/syz-cluster/overlays/common/patch-argo-controller.yaml b/syz-cluster/overlays/common/patch-argo-controller.yaml deleted file mode 100644 index d79585dc1..000000000 --- a/syz-cluster/overlays/common/patch-argo-controller.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2025 syzkaller project authors. All rights reserved. -# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: workflow-controller - namespace: argo -spec: - template: - spec: - serviceAccountName: argo-controller-ksa diff --git a/syz-cluster/overlays/common/patch-workflow-controller-configmap.yaml b/syz-cluster/overlays/common/patch-workflow-controller-configmap.yaml deleted file mode 100644 index 0ff55f9bb..000000000 --- a/syz-cluster/overlays/common/patch-workflow-controller-configmap.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2025 syzkaller project authors. All rights reserved. -# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -apiVersion: v1 -kind: ConfigMap -metadata: - name: workflow-controller-configmap - namespace: argo -data: - workflowDefaults: | - spec: - serviceAccountName: argo-executor-ksa diff --git a/syz-cluster/overlays/common/workflow-roles.yaml b/syz-cluster/overlays/common/workflow-roles.yaml deleted file mode 100644 index d1f1274db..000000000 --- a/syz-cluster/overlays/common/workflow-roles.yaml +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright 2025 syzkaller project authors. All rights reserved. -# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: argo-workflow-role -rules: -- apiGroups: - - argoproj.io - resources: - - workflows - verbs: - - get - - list - - watch - - create - - update - - patch - - delete - - status - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: argo-workflowtasks-role -rules: -- apiGroups: ["argoproj.io"] - resources: - - workflowtaskresults - verbs: - - create - - patch -- apiGroups: - - argoproj.io - resources: - - workflowtasksets - - workflowartifactgctasks - verbs: - - list - - watch -- apiGroups: - - argoproj.io - resources: - - workflowtasksets/status - - workflowartifactgctasks/status - verbs: - - patch - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: argo-workflow-role-binding - namespace: default -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: argo-workflow-role -subjects: -- kind: ServiceAccount - name: argo-executor-ksa - namespace: default -- kind: ServiceAccount - name: argo-controller-ksa - namespace: argo -- kind: ServiceAccount - name: gke-service-ksa - namespace: default - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: argo-workflowtasks-role-binding - namespace: argo -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: argo-workflowtasks-role -subjects: -- kind: ServiceAccount - name: argo-executor-ksa - namespace: default -- kind: ServiceAccount - name: argo-controller-ksa - namespace: argo - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: argo-cluster-role-to-controller-binding - namespace: argo -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: argo-cluster-role -subjects: -- kind: ServiceAccount - name: argo-controller-ksa - namespace: argo - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: argo-role-to-controller-binding - namespace: argo -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: argo-role -subjects: -- kind: ServiceAccount - name: argo-controller-ksa - namespace: argo - ---- - -apiVersion: v1 -kind: Secret -metadata: - annotations: - kubernetes.io/service-account.name: argo-executor-ksa - name: argo-executor-ksa.service-account-token -type: kubernetes.io/service-account-token diff --git a/syz-cluster/overlays/minikube/kustomization.yaml b/syz-cluster/overlays/minikube/kustomization.yaml index 3e683677d..432a64f83 100644 --- a/syz-cluster/overlays/minikube/kustomization.yaml +++ b/syz-cluster/overlays/minikube/kustomization.yaml @@ -4,6 +4,7 @@ resources: - service-accounts.yaml - kernel-disk-pvc.yaml + - ../common/argo - ../common - global-config.yaml - global-config-env.yaml -- cgit mrf-deployment