From 00b94e59e07ecbbc33307884521be41661c78dae Mon Sep 17 00:00:00 2001 From: timflannagan Date: Thu, 15 Oct 2026 16:38:14 -0400 Subject: [PATCH 1/2] charts: Clean up RustFS bucket init Jobs The RustFS bucket initializer now expires five minutes after it finishes. Asset staging tolerates an initializer that Kubernetes has already removed. Previously, completed initialization Jobs and their Pods remained in the cluster indefinitely, and later staging assumed the Job was still present. Now, the Kubernetes TTL controller deletes finished Jobs and Pods, while staging waits for initialization only when the Job still exists. Signed-off-by: timflannagan --- charts/substrate/templates/rustfs.yaml | 1 + charts/substrate/tests/control_plane_test.yaml | 9 +++++++++ hack/install-microvm-deps.sh | 1 - hack/microvm-assets/stage-to-rustfs.sh | 7 ++++++- manifests/ate-install/rustfs.yaml | 1 + 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml index ab04cb1a20..73514be6d4 100644 --- a/charts/substrate/templates/rustfs.yaml +++ b/charts/substrate/templates/rustfs.yaml @@ -110,6 +110,7 @@ metadata: namespace: {{ .Release.Namespace }} spec: backoffLimit: 10 + ttlSecondsAfterFinished: 300 template: spec: restartPolicy: OnFailure diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index 44c6938fa5..c47543f6d1 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -161,3 +161,12 @@ tests: value: - protocol: TCP port: api +- it: cleans up the bucket initialization job + template: rustfs.yaml + documentSelector: + path: kind + value: Job + asserts: + - equal: + path: spec.ttlSecondsAfterFinished + value: 300 diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index e37fd313ad..cc4516952e 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -208,7 +208,6 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." - run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh index 2f8c66f7b2..b552007ee0 100755 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ b/hack/microvm-assets/stage-to-rustfs.sh @@ -72,7 +72,12 @@ done # rustfs must be serving and the bucket must exist before anything is uploaded. echo ">> Waiting for rustfs in namespace ${NAMESPACE}..." run_kubectl rollout status deploy/rustfs --timeout=300s -run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s +# The TTL controller removes the initialization Job after it finishes. Wait +# for it when it is still present; if it has expired, it already created the +# bucket successfully. +if run_kubectl get job/rustfs-bucket-init >/dev/null 2>&1; then + run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s +fi NODE="$("${ROOT}/hack/kind.sh" get nodes --name "${KIND_CLUSTER_NAME}" | head -n1)" if [[ -z "${NODE}" ]]; then diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml index 0256197359..53f2cc98bc 100644 --- a/manifests/ate-install/rustfs.yaml +++ b/manifests/ate-install/rustfs.yaml @@ -141,6 +141,7 @@ metadata: namespace: ate-system spec: backoffLimit: 10 + ttlSecondsAfterFinished: 300 template: spec: restartPolicy: OnFailure From 871abd0eae05d15e7283698d9d5e2fb272032c75 Mon Sep 17 00:00:00 2001 From: timflannagan Date: Thu, 15 Oct 2026 16:40:24 -0400 Subject: [PATCH 2/2] charts: Use a hook for RustFS bucket init The RustFS bucket initializer now runs as a post-install and post-upgrade hook with successful-hook cleanup. Previously, a TTL deleted a release-managed resource. GitOps controllers could treat that deletion as drift and recreate the Job indefinitely. Now, hook-aware installers remove successful Jobs and replace an old Job before the next run, while failed Jobs remain available for diagnosis. Signed-off-by: timflannagan --- charts/substrate/templates/rustfs.yaml | 4 +++- charts/substrate/tests/control_plane_test.yaml | 9 ++++++--- hack/microvm-assets/stage-to-rustfs.sh | 5 ++--- manifests/ate-install/rustfs.yaml | 4 +++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml index 73514be6d4..fd7443877b 100644 --- a/charts/substrate/templates/rustfs.yaml +++ b/charts/substrate/templates/rustfs.yaml @@ -108,9 +108,11 @@ kind: Job metadata: name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: backoffLimit: 10 - ttlSecondsAfterFinished: 300 template: spec: restartPolicy: OnFailure diff --git a/charts/substrate/tests/control_plane_test.yaml b/charts/substrate/tests/control_plane_test.yaml index c47543f6d1..be9d2b76ad 100644 --- a/charts/substrate/tests/control_plane_test.yaml +++ b/charts/substrate/tests/control_plane_test.yaml @@ -161,12 +161,15 @@ tests: value: - protocol: TCP port: api -- it: cleans up the bucket initialization job +- it: removes the bucket initialization job after it succeeds template: rustfs.yaml documentSelector: path: kind value: Job asserts: - equal: - path: spec.ttlSecondsAfterFinished - value: 300 + path: metadata.annotations["helm.sh/hook"] + value: post-install,post-upgrade + - equal: + path: metadata.annotations["helm.sh/hook-delete-policy"] + value: before-hook-creation,hook-succeeded diff --git a/hack/microvm-assets/stage-to-rustfs.sh b/hack/microvm-assets/stage-to-rustfs.sh index b552007ee0..43d17aeae5 100755 --- a/hack/microvm-assets/stage-to-rustfs.sh +++ b/hack/microvm-assets/stage-to-rustfs.sh @@ -72,9 +72,8 @@ done # rustfs must be serving and the bucket must exist before anything is uploaded. echo ">> Waiting for rustfs in namespace ${NAMESPACE}..." run_kubectl rollout status deploy/rustfs --timeout=300s -# The TTL controller removes the initialization Job after it finishes. Wait -# for it when it is still present; if it has expired, it already created the -# bucket successfully. +# Helm removes the initialization Job after its hook succeeds, while plain +# kubectl installs retain it. Wait for the Job only when it is still present. if run_kubectl get job/rustfs-bucket-init >/dev/null 2>&1; then run_kubectl wait --for=condition=Complete job/rustfs-bucket-init --timeout=300s fi diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml index 53f2cc98bc..dafd6ec6d9 100644 --- a/manifests/ate-install/rustfs.yaml +++ b/manifests/ate-install/rustfs.yaml @@ -139,9 +139,11 @@ kind: Job metadata: name: rustfs-bucket-init namespace: ate-system + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded spec: backoffLimit: 10 - ttlSecondsAfterFinished: 300 template: spec: restartPolicy: OnFailure