From 72f56bf39f5615e6bbeafcfc10cd9895eba86364 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Tue, 20 Oct 2026 09:26:51 -0500
Subject: [PATCH 1/4] feat: add RustFS object storage for local celld fleets

spec.rustfs.enabled installs the official RustFS chart (1.0.3,
standalone) in the release namespace and points celld at
s3://<bucket> on the in-cluster endpoint. It replaces Azurite for
local use and takes precedence over azurite and aws.

- Root keys come from a user Secret (RUSTFS_ACCESS_KEY,
  RUSTFS_SECRET_KEY); the stack copies it into the release namespace
  with AWS_* aliases, AWS_REGION and S3_ENDPOINT. No keys in manifests.
- A one-shot Job creates the fleet bucket plus spec.rustfs.buckets and
  deploys the placeholder Worker only into an empty fleet bucket, so a
  celld pod restart never replaces the deployed application (the chart
  placeholder init container redeploys on every start).
- celld renders after the bucket Job succeeds, then stays rendered.
- Usages order teardown: celld, RustFS, credentials, namespace.
- status.rustfs exposes endpoint, credentials Secret, buckets, ready.

celld stays on chart 0.4.0 and image 0.6.2, the current releases.
---
 .github/workflows/on-pr.yaml                  |   4 +-
 .github/workflows/on-push-main.yaml           |   4 +-
 Makefile                                      |   4 +-
 README.md                                     |  49 ++-
 apis/celldstacks/definition.yaml              |  61 +++-
 .../celldstacks/observed/rustfs-ready.yaml    |  70 +++++
 examples/celldstacks/rustfs.yaml              |  13 +
 functions/render/000-state-init.yaml.gotmpl   |  51 +++-
 .../render/001-state-observed.yaml.gotmpl     |  30 +-
 functions/render/010-state-status.yaml.gotmpl |  18 +-
 functions/render/120-rustfs.yaml.gotmpl       | 285 ++++++++++++++++++
 .../render/200-helm-release-celld.yaml.gotmpl |  49 +++
 functions/render/999-status.yaml.gotmpl       |   5 +
 tests/test-render/main.k                      | 182 +++++++++++
 upbound.yaml                                  |   1 +
 15 files changed, 812 insertions(+), 14 deletions(-)
 create mode 100644 examples/celldstacks/observed/rustfs-ready.yaml
 create mode 100644 examples/celldstacks/rustfs.yaml
 create mode 100644 functions/render/120-rustfs.yaml.gotmpl

diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml
index 4cb4fb2..a2f3399 100644
--- a/.github/workflows/on-pr.yaml
+++ b/.github/workflows/on-pr.yaml
@@ -35,7 +35,9 @@ jobs:
           { "example": "examples/celldstacks/standard.yaml" },
           { "example": "examples/celldstacks/azurite.yaml" },
           { "example": "examples/celldstacks/aws.yaml" },
-          { "example": "examples/celldstacks/aws-local.yaml" }
+          { "example": "examples/celldstacks/aws-local.yaml" },
+          { "example": "examples/celldstacks/rustfs.yaml" },
+          { "example": "examples/celldstacks/rustfs.yaml", "observed_resources": "examples/celldstacks/observed/rustfs-ready.yaml" }
         ]
       api_path: apis/celldstacks
       error_on_missing_schemas: true
diff --git a/.github/workflows/on-push-main.yaml b/.github/workflows/on-push-main.yaml
index 76165f3..165cea0 100644
--- a/.github/workflows/on-push-main.yaml
+++ b/.github/workflows/on-push-main.yaml
@@ -30,7 +30,9 @@ jobs:
           { "example": "examples/celldstacks/standard.yaml" },
           { "example": "examples/celldstacks/azurite.yaml" },
           { "example": "examples/celldstacks/aws.yaml" },
-          { "example": "examples/celldstacks/aws-local.yaml" }
+          { "example": "examples/celldstacks/aws-local.yaml" },
+          { "example": "examples/celldstacks/rustfs.yaml" },
+          { "example": "examples/celldstacks/rustfs.yaml", "observed_resources": "examples/celldstacks/observed/rustfs-ready.yaml" }
         ]
       api_path: apis/celldstacks
       error_on_missing_schemas: true
diff --git a/Makefile b/Makefile
index a286bda..6a433c6 100644
--- a/Makefile
+++ b/Makefile
@@ -26,7 +26,9 @@ EXAMPLES := \
     examples/celldstacks/standard.yaml:: \
     examples/celldstacks/azurite.yaml:: \
     examples/celldstacks/aws.yaml:: \
-    examples/celldstacks/aws-local.yaml::
+    examples/celldstacks/aws-local.yaml:: \
+    examples/celldstacks/rustfs.yaml:: \
+    examples/celldstacks/rustfs.yaml::examples/celldstacks/observed/rustfs-ready.yaml
 
 render\:all:
 	@tmpdir=$$(mktemp -d); \
diff --git a/README.md b/README.md
index a4f171c..91a6b6e 100644
--- a/README.md
+++ b/README.md
@@ -20,7 +20,7 @@ spec:
   clusterName: my-cluster
 ```
 
-This installs one celld node in namespace `celld`. For local, enable Azurite. For AWS, enable `spec.aws` so the stack creates a bucket and binds the celld ServiceAccount with EKS Pod Identity.
+This installs one celld node in namespace `celld`. For local, enable RustFS (S3-compatible, recommended) or Azurite. For AWS, enable `spec.aws` so the stack creates a bucket and binds the celld ServiceAccount with EKS Pod Identity.
 
 ```yaml
 spec:
@@ -29,6 +29,38 @@ spec:
     enabled: true
 ```
 
+### Stage 1b: Local S3 with RustFS
+
+```yaml
+spec:
+  clusterName: my-cluster
+  rustfs:
+    enabled: true
+    buckets:
+      - game-services          # extra application buckets; the fleet bucket is always created
+    credentialsSecretRef:
+      name: celld-rustfs-credentials
+```
+
+Create the root credentials yourself; the stack never generates or embeds keys:
+
+```bash
+kubectl -n default create secret generic celld-rustfs-credentials \
+  --from-literal=RUSTFS_ACCESS_KEY="$(openssl rand -hex 12)" \
+  --from-literal=RUSTFS_SECRET_KEY="$(openssl rand -hex 24)"
+```
+
+The stack copies that Secret into the release namespace as `<releaseName>-rustfs`
+with `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` and
+`S3_ENDPOINT` added, installs the official RustFS chart (standalone, one PVC),
+creates `celld` plus the extra buckets with a one-shot Job (which also deploys
+a fetch-only placeholder Worker once, only into an empty fleet bucket, so pod
+restarts never replace your application), and then installs
+celld against `s3://celld` on `http://<releaseName>-rustfs-svc.<namespace>.svc.cluster.local:9000`.
+Applications in the cluster can mount the same `<releaseName>-rustfs` Secret
+(`envFrom`) to reach their buckets with any S3 client. RustFS takes precedence
+over `azurite` and `aws`.
+
 ### Stage 2: Growing — AWS S3
 
 ```yaml
@@ -80,6 +112,13 @@ Not applicable. This stack installs a Helm release; it does not adopt cloud reso
 | `credentials.existingSecret` | string | — | Secret with AWS_* keys (R2 / BYO). Unused when `aws.enabled` |
 | `azurite.enabled` | boolean | `false` | Deploy in-cluster Azurite and point celld at `az://celld` (local/dev only) |
 | `azurite.container` | string | `celld` | Blob container name |
+| `rustfs.enabled` | boolean | `false` | Deploy in-cluster RustFS and point celld at it (wins over `azurite`/`aws`) |
+| `rustfs.bucket` | string | `celld` | Fleet bucket name |
+| `rustfs.buckets` | string[] | — | Extra buckets to create |
+| `rustfs.credentialsSecretRef` | object | `<name>-rustfs-credentials` in the XR namespace | Secret with `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` |
+| `rustfs.persistence.size` | string | `2Gi` | RustFS data volume |
+| `rustfs.persistence.storageClassName` | string | cluster default | RustFS storage class |
+| `rustfs.values` | object | — | RustFS chart values merged with stack defaults |
 | `aws.enabled` | boolean | `false` | Create S3 bucket and point celld at `s3://` |
 | `aws.bucketName` | string | `hops-celld-<name>` | Globally unique S3 bucket name |
 | `aws.region` | string | `us-east-2` | Bucket region |
@@ -95,11 +134,13 @@ Not applicable. This stack installs a Helm release; it does not adopt cloud reso
 
 | Field | Meaning |
 |-------|---------|
-| `ready` | Namespace, Helm release, Usage, and (when `aws.enabled`) bucket + Pod Identity are Ready |
+| `ready` | Namespace, Helm release, Usage, and (when `aws.enabled`) bucket + Pod Identity, or (when `rustfs.enabled`) RustFS + buckets are Ready |
 | `release.name` / `release.namespace` | Installed Helm release |
 | `service.name` / `service.port` | Public Worker Service (port 8080) |
 | `bucket.name` / `bucket.id` | Observed S3 bucket when `aws.enabled` |
 | `podIdentity.roleArn` / `podIdentity.associationId` | Observed Pod Identity when `aws.enabled` |
+| `rustfs.endpoint` / `rustfs.credentialsSecret` | In-cluster S3 endpoint and the Secret S3 clients mount (when `rustfs.enabled`) |
+| `rustfs.buckets` / `rustfs.ready` | Buckets the bucket Job ensured; credentials, RustFS and buckets ready |
 
 ## Composed Resources
 
@@ -110,6 +151,10 @@ Not applicable. This stack installs a Helm release; it does not adopt cloud reso
 | `<name>-s3` | `s3.aws.m.upbound.io/Bucket` (when `aws.enabled`) |
 | `<name>-celld` | `aws.hops.ops.com.ai/PodIdentity` (when `aws.enabled`) |
 | `<name>-delete-helm-celld-before-namespace` | `protection.crossplane.io/Usage` (after both are Ready) |
+| `<name>-rustfs-creds` | `kubernetes.m.crossplane.io/Object` (Secret, when `rustfs.enabled`) |
+| `<releaseName>-rustfs` | `helm.m.crossplane.io/Release` (RustFS, when `rustfs.enabled`) |
+| `<name>-rustfs-buckets-<hash>` | `kubernetes.m.crossplane.io/Object` (bucket Job, when `rustfs.enabled`) |
+| `<name>-delete-helm-celld-before-rustfs`, `-before-rustfs-creds`, `<name>-delete-helm-rustfs-before-namespace`, `-before-rustfs-creds` | `protection.crossplane.io/Usage` (RustFS teardown order) |
 | `<name>-delete-helm-celld-before-pod-identity` | `protection.crossplane.io/Usage` (after Helm + Pod Identity are Ready) |
 
 ## Development
diff --git a/apis/celldstacks/definition.yaml b/apis/celldstacks/definition.yaml
index c3e3fa9..0ae994a 100644
--- a/apis/celldstacks/definition.yaml
+++ b/apis/celldstacks/definition.yaml
@@ -147,8 +147,48 @@ spec:
                         default: 2Gi
                       storageClassName:
                         type: string
+              rustfs:
+                description: Deploy in-cluster RustFS (S3-compatible) and point celld at it. Takes precedence over azurite and aws. Local/dev and single-node use.
+                type: object
+                properties:
+                  enabled:
+                    description: When true, install RustFS (standalone), create the buckets, and configure celld for s3://<bucket> on the RustFS endpoint.
+                    type: boolean
+                    default: false
+                  bucket:
+                    description: celld fleet bucket name. Defaults to "celld".
+                    type: string
+                    pattern: '^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$'
+                  buckets:
+                    description: Additional buckets to create for applications (for example game-services blobs). The fleet bucket is always created.
+                    type: array
+                    items:
+                      type: string
+                      pattern: '^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$'
+                  credentialsSecretRef:
+                    description: Secret holding the RustFS root credentials as RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY. Copied into the release namespace with AWS_* aliases and S3_ENDPOINT. Defaults to <name>-rustfs-credentials in the XR namespace. Must not be the copied Secret itself (<releaseName>-rustfs in the release namespace). No credentials are generated or stored in the manifest.
+                    type: object
+                    properties:
+                      name:
+                        type: string
+                      namespace:
+                        type: string
+                  persistence:
+                    description: Persistent volume for RustFS object data.
+                    type: object
+                    properties:
+                      size:
+                        type: string
+                        default: 2Gi
+                      storageClassName:
+                        description: Storage class. Empty uses the cluster default.
+                        type: string
+                  values:
+                    description: Helm values merged on top of the RustFS chart defaults set by this stack.
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
               aws:
-                description: Provision an S3 bucket and grant the celld pod access. On EKS, Pod Identity is the default. Locally, disable podIdentity and reuse hops `local aws` credentials. Ignored when azurite.enabled is true.
+                description: Provision an S3 bucket and grant the celld pod access. On EKS, Pod Identity is the default. Locally, disable podIdentity and reuse hops `local aws` credentials. Ignored when azurite.enabled or rustfs.enabled is true.
                 type: object
                 properties:
                   enabled:
@@ -237,7 +277,7 @@ spec:
             type: object
             properties:
               ready:
-                description: Whether the namespace, Helm release, and deletion Usage are ready.
+                description: Whether the namespace, Helm releases, buckets, and deletion Usage are ready.
                 type: boolean
               release:
                 description: Observed Helm release.
@@ -269,6 +309,23 @@ spec:
                     type: string
                   ready:
                     type: boolean
+              rustfs:
+                description: Observed in-cluster RustFS object store.
+                type: object
+                properties:
+                  endpoint:
+                    description: In-cluster S3 endpoint URL.
+                    type: string
+                  credentialsSecret:
+                    description: Secret in the release namespace with AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION and S3_ENDPOINT for S3 clients.
+                    type: string
+                  buckets:
+                    description: Buckets the bucket Job created or found.
+                    type: array
+                    items:
+                      type: string
+                  ready:
+                    type: boolean
               podIdentity:
                 description: Observed EKS Pod Identity binding the celld ServiceAccount to the fleet bucket.
                 type: object
diff --git a/examples/celldstacks/observed/rustfs-ready.yaml b/examples/celldstacks/observed/rustfs-ready.yaml
new file mode 100644
index 0000000..83a4b4a
--- /dev/null
+++ b/examples/celldstacks/observed/rustfs-ready.yaml
@@ -0,0 +1,70 @@
+# Observed state for examples/celldstacks/rustfs.yaml once every step is up:
+# credentials copied, RustFS installed, buckets created, celld installed.
+apiVersion: kubernetes.m.crossplane.io/v1alpha1
+kind: Object
+metadata:
+  name: celld-namespace
+  namespace: default
+  annotations:
+    crossplane.io/composition-resource-name: namespace
+status:
+  conditions:
+    - type: Ready
+      status: "True"
+---
+apiVersion: kubernetes.m.crossplane.io/v1alpha1
+kind: Object
+metadata:
+  name: celld-rustfs-creds
+  namespace: default
+  uid: 0b6d3f0e-0000-4000-8000-000000000001
+  annotations:
+    crossplane.io/composition-resource-name: rustfs-creds-secret
+status:
+  conditions:
+    - type: Ready
+      status: "True"
+---
+apiVersion: helm.m.crossplane.io/v1beta1
+kind: Release
+metadata:
+  name: celld-rustfs
+  namespace: default
+  annotations:
+    crossplane.io/composition-resource-name: helm-release-rustfs
+status:
+  atProvider:
+    revision: 1
+  conditions:
+    - type: Ready
+      status: "True"
+---
+apiVersion: kubernetes.m.crossplane.io/v1alpha1
+kind: Object
+metadata:
+  name: celld-rustfs-buckets-ea80c0f0
+  namespace: default
+  annotations:
+    crossplane.io/composition-resource-name: rustfs-buckets-job
+status:
+  atProvider:
+    manifest:
+      status:
+        succeeded: 1
+  conditions:
+    - type: Ready
+      status: "True"
+---
+apiVersion: helm.m.crossplane.io/v1beta1
+kind: Release
+metadata:
+  name: celld
+  namespace: default
+  annotations:
+    crossplane.io/composition-resource-name: helm-release-celld
+status:
+  atProvider:
+    revision: 1
+  conditions:
+    - type: Ready
+      status: "True"
diff --git a/examples/celldstacks/rustfs.yaml b/examples/celldstacks/rustfs.yaml
new file mode 100644
index 0000000..6eba827
--- /dev/null
+++ b/examples/celldstacks/rustfs.yaml
@@ -0,0 +1,13 @@
+apiVersion: hops.ops.com.ai/v1alpha1
+kind: CelldStack
+metadata:
+  name: celld
+  namespace: default
+spec:
+  clusterName: my-cluster
+  rustfs:
+    enabled: true
+    buckets:
+      - game-services
+    credentialsSecretRef:
+      name: celld-rustfs-credentials
diff --git a/functions/render/000-state-init.yaml.gotmpl b/functions/render/000-state-init.yaml.gotmpl
index c26b17e..d0b902e 100644
--- a/functions/render/000-state-init.yaml.gotmpl
+++ b/functions/render/000-state-init.yaml.gotmpl
@@ -55,10 +55,16 @@
 {{- $azuritePersistence := $azurite.persistence | default dict }}
 {{- $azuritePersistenceEnabled := true }}
 {{- if hasKey $azuritePersistence "enabled" }}{{- $azuritePersistenceEnabled = $azuritePersistence.enabled }}{{- end }}
+{{- $rustfs := $spec.rustfs | default dict }}
+{{- $rustfsEnabled := false }}
+{{- if hasKey $rustfs "enabled" }}{{- $rustfsEnabled = $rustfs.enabled }}{{- end }}
+{{- if $rustfsEnabled }}
+  {{- $azuriteEnabled = false }}
+{{- end }}
 {{- $aws := $spec.aws | default dict }}
 {{- $awsEnabled := false }}
 {{- if hasKey $aws "enabled" }}{{- $awsEnabled = $aws.enabled }}{{- end }}
-{{- if $azuriteEnabled }}
+{{- if or $azuriteEnabled $rustfsEnabled }}
   {{- $awsEnabled = false }}
 {{- end }}
 {{- $awsForceDestroy := true }}
@@ -83,8 +89,27 @@
 {{- if and $awsEnabled (ne $awsCredsName "") }}
   {{- $awsCredsLocalName = printf "%s-aws" $releaseName }}
 {{- end }}
+{{- $rustfsName := printf "%s-rustfs" $releaseName }}
+{{- $rustfsBucket := $rustfs.bucket | default "celld" }}
+{{- $rustfsBuckets := list $rustfsBucket }}
+{{- range ($rustfs.buckets | default list) }}
+  {{- if not (has . $rustfsBuckets) }}
+    {{- $rustfsBuckets = append $rustfsBuckets . }}
+  {{- end }}
+{{- end }}
+{{- $rustfsCredsRef := $rustfs.credentialsSecretRef | default dict }}
+{{- $rustfsPersistence := $rustfs.persistence | default dict }}
+{{- $rustfsEndpoint := printf "http://%s-svc.%s.svc.cluster.local:9000" $rustfsName $namespace }}
+{{- $endpoint := $spec.endpoint | default "" }}
+{{- $existingSecret := $credentials.existingSecret | default "" }}
+{{- if $rustfsEnabled }}
+  {{- $endpoint = $rustfsEndpoint }}
+  {{- $existingSecret = $rustfsName }}
+{{- end }}
 {{- $bucket := $spec.bucket | default "" }}
-{{- if $azuriteEnabled }}
+{{- if $rustfsEnabled }}
+  {{- $bucket = printf "s3://%s" $rustfsBucket }}
+{{- else if $azuriteEnabled }}
   {{- $bucket = printf "az://%s" $azuriteContainer }}
 {{- else if $awsEnabled }}
   {{- $bucket = printf "s3://%s" $awsBucketName }}
@@ -114,11 +139,11 @@
   "awsProviderConfigRef" $awsProviderConfigRef
   "serviceName" $serviceName
   "bucket" $bucket
-  "endpoint" ($spec.endpoint | default "")
-  "region" $awsRegion
+  "endpoint" $endpoint
+  "region" (ternary "us-east-1" $awsRegion $rustfsEnabled)
   "xrNamespace" $xrNamespace
   "credentials" (dict
-    "existingSecret" ($credentials.existingSecret | default "")
+    "existingSecret" $existingSecret
     "sharedCredentialsSecret" $awsCredsLocalName
     "sharedCredentialsKey" "credentials"
   )
@@ -155,6 +180,22 @@
       "storageClassName" ($azuritePersistence.storageClassName | default "")
     )
   )
+  "rustfs" (dict
+    "enabled" $rustfsEnabled
+    "name" $rustfsName
+    "endpoint" $rustfsEndpoint
+    "bucket" $rustfsBucket
+    "buckets" $rustfsBuckets
+    "credentialsSecretRef" (dict
+      "name" ($rustfsCredsRef.name | default (printf "%s-rustfs-credentials" $name))
+      "namespace" ($rustfsCredsRef.namespace | default $xrNamespace)
+    )
+    "persistence" (dict
+      "size" ($rustfsPersistence.size | default "2Gi")
+      "storageClassName" ($rustfsPersistence.storageClassName | default "")
+    )
+    "values" ($rustfs.values | default dict)
+  )
   "image" (dict
     "repository" ($image.repository | default "ghcr.io/denoland/celld")
     "tag" $imageTag
diff --git a/functions/render/001-state-observed.yaml.gotmpl b/functions/render/001-state-observed.yaml.gotmpl
index 0442dd5..d14bfdb 100644
--- a/functions/render/001-state-observed.yaml.gotmpl
+++ b/functions/render/001-state-observed.yaml.gotmpl
@@ -26,6 +26,19 @@
 {{- $awsCredsResource := $awsCredsEntry.resource | default dict }}
 {{- $awsCredsUid := (($awsCredsResource.metadata | default dict).uid | default "") }}
 
+{{- $revisionOf := dict }}
+{{- range $key, $entry := $observed }}
+  {{- $atProvider := (($entry.resource | default dict).status | default dict).atProvider | default dict }}
+  {{- $revisionOf = set $revisionOf $key (gt (int ($atProvider.revision | default 0)) 0) }}
+{{- end }}
+
+{{- $rustfsCredsEntry := get $observed "rustfs-creds-secret" | default dict }}
+{{- $rustfsCredsUid := ((($rustfsCredsEntry.resource | default dict).metadata | default dict).uid | default "") }}
+
+{{- $rustfsBucketsEntry := get $observed "rustfs-buckets-job" | default dict }}
+{{- $rustfsBucketsManifest := ((($rustfsBucketsEntry.resource | default dict).status | default dict).atProvider | default dict).manifest | default dict }}
+{{- $rustfsBucketsSucceeded := gt (int (($rustfsBucketsManifest.status | default dict).succeeded | default 0)) 0 }}
+
 {{- $podIdentityEntry := get $observed "pod-identity" | default dict }}
 {{- $podIdentityResource := $podIdentityEntry.resource | default dict }}
 {{- $podIdentityStatus := ($podIdentityResource.status | default dict).podIdentity | default dict }}
@@ -34,7 +47,22 @@
 
 {{- $state = set $state "observed" (merge $state.observed (dict
   "namespace" (dict "ready" (get $readyOf "namespace" | default false))
-  "release" (dict "ready" (get $readyOf "helm-release-celld" | default false))
+  "release" (dict
+    "ready" (get $readyOf "helm-release-celld" | default false)
+    "exists" (get $revisionOf "helm-release-celld" | default false)
+  )
+  "rustfsCreds" (dict
+    "ready" (get $readyOf "rustfs-creds-secret" | default false)
+    "id" $rustfsCredsUid
+  )
+  "rustfsRelease" (dict
+    "ready" (get $readyOf "helm-release-rustfs" | default false)
+    "exists" (get $revisionOf "helm-release-rustfs" | default false)
+  )
+  "rustfsBuckets" (dict
+    "ready" (get $readyOf "rustfs-buckets-job" | default false)
+    "succeeded" $rustfsBucketsSucceeded
+  )
   "bucket" (dict
     "ready" (get $readyOf "s3-bucket" | default false)
     "id" (get $idOf "s3-bucket" | default "")
diff --git a/functions/render/010-state-status.yaml.gotmpl b/functions/render/010-state-status.yaml.gotmpl
index 9e5ee3f..2f074ab 100644
--- a/functions/render/010-state-status.yaml.gotmpl
+++ b/functions/render/010-state-status.yaml.gotmpl
@@ -28,6 +28,21 @@
   {{- end }}
 {{- end }}
 
+{{- $rustfsReady := true }}
+{{- if $state.rustfs.enabled }}
+  {{- $rustfsReady = and $obs.rustfsCreds.ready $obs.rustfsRelease.ready $obs.rustfsBuckets.succeeded }}
+{{- end }}
+
+{{- $rustfsStatus := dict "endpoint" "" "credentialsSecret" "" "buckets" (list) "ready" false }}
+{{- if $state.rustfs.enabled }}
+  {{- $rustfsStatus = dict
+    "endpoint" $state.rustfs.endpoint
+    "credentialsSecret" $state.rustfs.name
+    "buckets" (ternary $state.rustfs.buckets (list) $obs.rustfsBuckets.succeeded)
+    "ready" $rustfsReady
+  }}
+{{- end }}
+
 {{- $bucketStatus := dict "name" "" "id" "" "ready" false }}
 {{- $podIdentityStatus := dict "roleArn" "" "associationId" "" "ready" false }}
 {{- if $state.aws.enabled }}
@@ -44,7 +59,7 @@
 {{- end }}
 
 {{- $state = set $state "status" (dict
-  "ready" (and $namespaceReady $releaseReady $usageReady $awsReady)
+  "ready" (and $namespaceReady $releaseReady $usageReady $awsReady $rustfsReady)
   "release" (dict
     "name" $state.releaseName
     "namespace" $state.namespace
@@ -57,4 +72,5 @@
   )
   "bucket" $bucketStatus
   "podIdentity" $podIdentityStatus
+  "rustfs" $rustfsStatus
 ) }}
diff --git a/functions/render/120-rustfs.yaml.gotmpl b/functions/render/120-rustfs.yaml.gotmpl
new file mode 100644
index 0000000..cc955e2
--- /dev/null
+++ b/functions/render/120-rustfs.yaml.gotmpl
@@ -0,0 +1,285 @@
+# code: language=yaml
+#
+# RustFS: in-cluster S3-compatible object storage (rustfs.enabled).
+#
+# Chart: https://charts.rustfs.com (chart name: rustfs)
+# Upstream: https://github.com/rustfs/rustfs
+#
+# 1. Copy the user's RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY Secret into the
+#    release namespace, adding the AWS_* aliases and S3_ENDPOINT that celld
+#    and its bootstrap deploy init container read through envFrom.
+# 2. Install RustFS (standalone) with that Secret as its root credentials.
+# 3. Create the fleet bucket and any extra buckets with a one-shot Job, and
+#    deploy a placeholder Worker once if the fleet bucket has no deployment.
+
+{{- if $state.rustfs.enabled }}
+{{- $rustfs := $state.rustfs }}
+{{- $creds := $rustfs.credentialsSecretRef }}
+---
+apiVersion: kubernetes.m.crossplane.io/v1alpha1
+kind: Object
+metadata:
+  name: {{ $state.name }}-rustfs-creds
+  annotations:
+    {{ setResourceNameAnnotation "rustfs-creds-secret" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  managementPolicies: {{ $state.managementPolicies | toJson }}
+  forProvider:
+    manifest:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: {{ $rustfs.name }}
+        namespace: {{ $state.namespace }}
+        labels: {{ $state.labels | toJson }}
+      type: Opaque
+      data:
+        S3_ENDPOINT: {{ $rustfs.endpoint | b64enc | quote }}
+        AWS_REGION: {{ $state.region | b64enc | quote }}
+  references:
+    - patchesFrom:
+        apiVersion: v1
+        kind: Secret
+        name: {{ $creds.name }}
+        namespace: {{ $creds.namespace }}
+        fieldPath: data.RUSTFS_ACCESS_KEY
+      toFieldPath: data.RUSTFS_ACCESS_KEY
+    - patchesFrom:
+        apiVersion: v1
+        kind: Secret
+        name: {{ $creds.name }}
+        namespace: {{ $creds.namespace }}
+        fieldPath: data.RUSTFS_SECRET_KEY
+      toFieldPath: data.RUSTFS_SECRET_KEY
+    - patchesFrom:
+        apiVersion: v1
+        kind: Secret
+        name: {{ $creds.name }}
+        namespace: {{ $creds.namespace }}
+        fieldPath: data.RUSTFS_ACCESS_KEY
+      toFieldPath: data.AWS_ACCESS_KEY_ID
+    - patchesFrom:
+        apiVersion: v1
+        kind: Secret
+        name: {{ $creds.name }}
+        namespace: {{ $creds.namespace }}
+        fieldPath: data.RUSTFS_SECRET_KEY
+      toFieldPath: data.AWS_SECRET_ACCESS_KEY
+    - dependsOn:
+        apiVersion: v1
+        kind: Namespace
+        name: {{ $state.namespace }}
+  providerConfigRef:
+    name: {{ $state.kubernetesProviderConfigRef.name }}
+    kind: {{ $state.kubernetesProviderConfigRef.kind }}
+
+{{- $rustfsDefaults := dict
+  "fullnameOverride" $rustfs.name
+  "commonLabels" $state.labels
+  "replicaCount" 1
+  "mode" (dict
+    "standalone" (dict "enabled" true)
+    "distributed" (dict "enabled" false)
+  )
+  "secret" (dict "existingSecret" $rustfs.name)
+  "config" (dict "rustfs" (dict "region" $state.region))
+  "ingress" (dict "enabled" false)
+  "storageclass" (dict
+    "name" $rustfs.persistence.storageClassName
+    "dataStorageSize" $rustfs.persistence.size
+    "logStorageSize" "256Mi"
+  )
+  "resources" (dict
+    "requests" (dict "cpu" "50m" "memory" "128Mi")
+    "limits" (dict "memory" "1Gi")
+  )
+}}
+
+{{- if ne $state.observed.rustfsCreds.id "" }}
+---
+apiVersion: helm.m.crossplane.io/v1beta1
+kind: Release
+metadata:
+  name: {{ $rustfs.name }}
+  annotations:
+    {{ setResourceNameAnnotation "helm-release-rustfs" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  managementPolicies: {{ $state.managementPolicies | toJson }}
+  forProvider:
+    chart:
+      name: rustfs
+      repository: https://charts.rustfs.com
+      # renovate: datasource=helm depName=rustfs registryUrl=https://charts.rustfs.com
+      version: "1.0.3"
+    namespace: {{ $state.namespace }}
+    wait: false
+    values:
+      {{- toYaml (mergeOverwrite $rustfsDefaults $rustfs.values) | nindent 6 }}
+  rollbackLimit: 3
+  providerConfigRef:
+    name: {{ $state.helmProviderConfigRef.name }}
+    kind: {{ $state.helmProviderConfigRef.kind }}
+{{- end }}
+
+{{- if $state.observed.rustfsRelease.exists }}
+{{- /* A Job template is immutable: a new name per bucket set and celld image. */}}
+{{- $jobHash := printf "%s|%s|%s:%s" ($rustfs.buckets | join ",") $rustfs.bucket $state.image.repository $state.image.tag | sha256sum | trunc 8 }}
+{{- $jobName := printf "%s-rustfs-buckets-%s" $state.name $jobHash }}
+---
+apiVersion: kubernetes.m.crossplane.io/v1alpha1
+kind: Object
+metadata:
+  name: {{ $jobName }}
+  annotations:
+    {{ setResourceNameAnnotation "rustfs-buckets-job" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  managementPolicies: {{ $state.managementPolicies | toJson }}
+  readiness:
+    policy: DeriveFromCelQuery
+    celQuery: "has(object.status.succeeded) && object.status.succeeded > 0"
+  forProvider:
+    manifest:
+      apiVersion: batch/v1
+      kind: Job
+      metadata:
+        name: {{ $jobName }}
+        namespace: {{ $state.namespace }}
+        labels: {{ $state.labels | toJson }}
+      spec:
+        backoffLimit: 10
+        template:
+          metadata:
+            labels: {{ $state.labels | toJson }}
+          spec:
+            restartPolicy: OnFailure
+            volumes:
+              - name: work
+                emptyDir: {}
+            initContainers:
+              - name: create-buckets
+                # renovate: datasource=docker depName=amazon/aws-cli
+                image: amazon/aws-cli:2.31.13
+                command:
+                  - /bin/sh
+                  - -c
+                  - |
+                    set -eu
+                    i=0
+                    until aws --endpoint-url "$S3_ENDPOINT" s3api list-buckets >/dev/null; do
+                      i=$((i + 1))
+                      if [ "$i" -ge 60 ]; then echo "RustFS is not reachable" >&2; exit 1; fi
+                      sleep 2
+                    done
+                    for bucket in {{ $rustfs.buckets | join " " }}; do
+                      if aws --endpoint-url "$S3_ENDPOINT" s3api head-bucket --bucket "$bucket" 2>/dev/null; then
+                        echo "bucket $bucket exists"
+                      else
+                        aws --endpoint-url "$S3_ENDPOINT" s3api create-bucket --bucket "$bucket"
+                        echo "bucket $bucket created"
+                      fi
+                    done
+                    if ! aws --endpoint-url "$S3_ENDPOINT" s3api head-object --bucket {{ $rustfs.bucket }} --key deploy/current.json >/dev/null 2>&1; then
+                      touch /work/needs-deployment
+                    fi
+                envFrom:
+                  - secretRef:
+                      name: {{ $rustfs.name }}
+                volumeMounts:
+                  - name: work
+                    mountPath: /work
+                resources:
+                  requests:
+                    cpu: 10m
+                    memory: 64Mi
+                  limits:
+                    memory: 256Mi
+            containers:
+              # celld serves only after a deployment exists. Deploy a fetch-only
+              # placeholder once, into an empty fleet bucket, so a later pod
+              # restart never replaces the application deployed on top of it.
+              - name: bootstrap-deployment
+                image: {{ printf "%s:%s" $state.image.repository $state.image.tag }}
+                command:
+                  - /bin/sh
+                  - -c
+                  - |
+                    set -eu
+                    if [ ! -f /work/needs-deployment ]; then
+                      echo "s3://{{ $rustfs.bucket }} already has a deployment"
+                      exit 0
+                    fi
+                    apt-get update
+                    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates xz-utils
+                    cd /tmp
+                    curl -fsSL https://esbuild.github.io/dl/v0.25.5 | sh
+                    export PATH="/tmp:$PATH"
+                    mkdir -p /tmp/worker
+                    printf '%s\n' '{"name":"celld-placeholder","main":"index.js","compatibility_date":"2026-01-01"}' > /tmp/worker/wrangler.jsonc
+                    printf '%s\n' 'export default { fetch() { return new Response("celld ok"); } };' > /tmp/worker/index.js
+                    celld deploy /tmp/worker --bucket s3://{{ $rustfs.bucket }}
+                envFrom:
+                  - secretRef:
+                      name: {{ $rustfs.name }}
+                volumeMounts:
+                  - name: work
+                    mountPath: /work
+                resources:
+                  requests:
+                    cpu: 10m
+                    memory: 64Mi
+                  limits:
+                    memory: 512Mi
+  providerConfigRef:
+    name: {{ $state.kubernetesProviderConfigRef.name }}
+    kind: {{ $state.kubernetesProviderConfigRef.kind }}
+{{- end }}
+
+{{- if and $state.observed.namespace.ready $state.observed.rustfsRelease.ready }}
+---
+apiVersion: protection.crossplane.io/v1beta1
+kind: Usage
+metadata:
+  name: {{ $state.name }}-delete-helm-rustfs-before-namespace
+  annotations:
+    {{ setResourceNameAnnotation "usage-rustfs-namespace" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  replayDeletion: true
+  of:
+    apiVersion: kubernetes.m.crossplane.io/v1alpha1
+    kind: Object
+    resourceRef:
+      name: {{ $state.name }}-namespace
+  by:
+    apiVersion: helm.m.crossplane.io/v1beta1
+    kind: Release
+    resourceRef:
+      name: {{ $rustfs.name }}
+{{- end }}
+
+{{- if and $state.observed.rustfsCreds.ready $state.observed.rustfsRelease.ready }}
+---
+apiVersion: protection.crossplane.io/v1beta1
+kind: Usage
+metadata:
+  name: {{ $state.name }}-delete-helm-rustfs-before-rustfs-creds
+  annotations:
+    {{ setResourceNameAnnotation "usage-rustfs-creds" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  replayDeletion: true
+  of:
+    apiVersion: kubernetes.m.crossplane.io/v1alpha1
+    kind: Object
+    resourceRef:
+      name: {{ $state.name }}-rustfs-creds
+  by:
+    apiVersion: helm.m.crossplane.io/v1beta1
+    kind: Release
+    resourceRef:
+      name: {{ $rustfs.name }}
+{{- end }}
+{{- end }}
diff --git a/functions/render/200-helm-release-celld.yaml.gotmpl b/functions/render/200-helm-release-celld.yaml.gotmpl
index 37550cc..a443214 100644
--- a/functions/render/200-helm-release-celld.yaml.gotmpl
+++ b/functions/render/200-helm-release-celld.yaml.gotmpl
@@ -31,6 +31,9 @@
     {{- $renderHelm = and $renderHelm (ne $state.observed.awsCreds.id "") }}
   {{- end }}
 {{- end }}
+{{- if $state.rustfs.enabled }}
+  {{- $renderHelm = or $state.observed.release.exists $state.observed.rustfsBuckets.succeeded }}
+{{- end }}
 {{- if $renderHelm }}
 ---
 apiVersion: helm.m.crossplane.io/v1beta1
@@ -113,4 +116,50 @@ spec:
     resourceRef:
       name: {{ $state.releaseName }}
 {{- end }}
+
+{{- if and $state.rustfs.enabled $state.observed.rustfsRelease.ready $state.observed.release.ready }}
+---
+apiVersion: protection.crossplane.io/v1beta1
+kind: Usage
+metadata:
+  name: {{ $state.name }}-delete-helm-celld-before-rustfs
+  annotations:
+    {{ setResourceNameAnnotation "usage-helm-celld-rustfs" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  replayDeletion: true
+  of:
+    apiVersion: helm.m.crossplane.io/v1beta1
+    kind: Release
+    resourceRef:
+      name: {{ $state.rustfs.name }}
+  by:
+    apiVersion: helm.m.crossplane.io/v1beta1
+    kind: Release
+    resourceRef:
+      name: {{ $state.releaseName }}
+{{- end }}
+
+{{- if and $state.rustfs.enabled $state.observed.rustfsCreds.ready $state.observed.release.ready }}
+---
+apiVersion: protection.crossplane.io/v1beta1
+kind: Usage
+metadata:
+  name: {{ $state.name }}-delete-helm-celld-before-rustfs-creds
+  annotations:
+    {{ setResourceNameAnnotation "usage-helm-celld-rustfs-creds" }}
+  labels: {{ $state.labels | toJson }}
+spec:
+  replayDeletion: true
+  of:
+    apiVersion: kubernetes.m.crossplane.io/v1alpha1
+    kind: Object
+    resourceRef:
+      name: {{ $state.name }}-rustfs-creds
+  by:
+    apiVersion: helm.m.crossplane.io/v1beta1
+    kind: Release
+    resourceRef:
+      name: {{ $state.releaseName }}
+{{- end }}
 {{- end }}
diff --git a/functions/render/999-status.yaml.gotmpl b/functions/render/999-status.yaml.gotmpl
index 4dd71c5..1003b05 100644
--- a/functions/render/999-status.yaml.gotmpl
+++ b/functions/render/999-status.yaml.gotmpl
@@ -25,3 +25,8 @@ status:
     roleArn: {{ $s.podIdentity.roleArn | quote }}
     associationId: {{ $s.podIdentity.associationId | quote }}
     ready: {{ $s.podIdentity.ready }}
+  rustfs:
+    endpoint: {{ $s.rustfs.endpoint | quote }}
+    credentialsSecret: {{ $s.rustfs.credentialsSecret | quote }}
+    buckets: {{ $s.rustfs.buckets | toJson }}
+    ready: {{ $s.rustfs.ready }}
diff --git a/tests/test-render/main.k b/tests/test-render/main.k
index da8c549..14afaa8 100644
--- a/tests/test-render/main.k
+++ b/tests/test-render/main.k
@@ -386,6 +386,188 @@ _items = [
             ]
         }
     }
+
+    # ==========================================================================
+    # Test 12: rustfs.enabled first copies credentials and waits to install.
+    # ==========================================================================
+    metav1alpha1.CompositionTest {
+        metadata.name = "rustfs-copies-credentials-before-installing"
+        spec = {
+            compositionPath = "apis/celldstacks/composition.yaml"
+            xrdPath = "apis/celldstacks/definition.yaml"
+            timeoutSeconds = 60
+            validate = False
+            xr = stacksv1alpha1.CelldStack {
+                metadata.name = "celld"
+                metadata.namespace = "game"
+                spec = {
+                    clusterName = "my-cluster"
+                    rustfs = {enabled = True}
+                }
+            }
+            assertResources = [
+                {
+                    apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
+                    kind = "Object"
+                    metadata.name = "celld-rustfs-creds"
+                    spec.forProvider.manifest = {
+                        apiVersion = "v1"
+                        kind = "Secret"
+                        metadata = {name = "celld-rustfs", namespace = "celld"}
+                        data = {
+                            S3_ENDPOINT = "aHR0cDovL2NlbGxkLXJ1c3Rmcy1zdmMuY2VsbGQuc3ZjLmNsdXN0ZXIubG9jYWw6OTAwMA=="
+                            AWS_REGION = "dXMtZWFzdC0x"
+                        }
+                    }
+                }
+                {
+                    apiVersion = "hops.ops.com.ai/v1alpha1"
+                    kind = "CelldStack"
+                    status.rustfs = {
+                        endpoint = "http://celld-rustfs-svc.celld.svc.cluster.local:9000"
+                        ready = False
+                    }
+                }
+            ]
+        }
+    }
+
+    # ==========================================================================
+    # Test 13: with RustFS up and buckets created, celld targets RustFS.
+    # ==========================================================================
+    metav1alpha1.CompositionTest {
+        metadata.name = "rustfs-ready-points-celld-at-rustfs"
+        spec = {
+            compositionPath = "apis/celldstacks/composition.yaml"
+            xrdPath = "apis/celldstacks/definition.yaml"
+            observedResources = [
+                {
+                    apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
+                    kind = "Object"
+                    metadata = {
+                        name = "celld-namespace"
+                        annotations = {"crossplane.io/composition-resource-name" = "namespace"}
+                    }
+                    status.conditions = [{type = "Ready", status = "True"}]
+                }
+                {
+                    apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
+                    kind = "Object"
+                    metadata = {
+                        name = "celld-rustfs-creds"
+                        uid = "0b6d3f0e-0000-4000-8000-000000000001"
+                        annotations = {"crossplane.io/composition-resource-name" = "rustfs-creds-secret"}
+                    }
+                    status.conditions = [{type = "Ready", status = "True"}]
+                }
+                {
+                    apiVersion = "helm.m.crossplane.io/v1beta1"
+                    kind = "Release"
+                    metadata = {
+                        name = "celld-rustfs"
+                        annotations = {"crossplane.io/composition-resource-name" = "helm-release-rustfs"}
+                    }
+                    status = {
+                        atProvider.revision = 1
+                        conditions = [{type = "Ready", status = "True"}]
+                    }
+                }
+                {
+                    apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
+                    kind = "Object"
+                    metadata = {
+                        name = "celld-rustfs-buckets-ea80c0f0"
+                        annotations = {"crossplane.io/composition-resource-name" = "rustfs-buckets-job"}
+                    }
+                    status = {
+                        atProvider.manifest.status.succeeded = 1
+                        conditions = [{type = "Ready", status = "True"}]
+                    }
+                }
+                {
+                    apiVersion = "helm.m.crossplane.io/v1beta1"
+                    kind = "Release"
+                    metadata = {
+                        name = "celld"
+                        annotations = {"crossplane.io/composition-resource-name" = "helm-release-celld"}
+                    }
+                    status = {
+                        atProvider.revision = 1
+                        conditions = [{type = "Ready", status = "True"}]
+                    }
+                }
+            ]
+            timeoutSeconds = 60
+            validate = False
+            xr = stacksv1alpha1.CelldStack {
+                metadata.name = "celld"
+                metadata.namespace = "default"
+                spec = {
+                    clusterName = "my-cluster"
+                    rustfs = {
+                        enabled = True
+                        buckets = ["game-services"]
+                        credentialsSecretRef.name = "celld-rustfs-credentials"
+                    }
+                }
+            }
+            assertResources = [
+                {
+                    apiVersion = "helm.m.crossplane.io/v1beta1"
+                    kind = "Release"
+                    metadata.name = "celld-rustfs"
+                    spec.forProvider = {
+                        chart = {
+                            name = "rustfs"
+                            repository = "https://charts.rustfs.com"
+                            version = "1.0.3"
+                        }
+                        namespace = "celld"
+                        values = {
+                            fullnameOverride = "celld-rustfs"
+                            mode.standalone.enabled = True
+                            secret.existingSecret = "celld-rustfs"
+                            ingress.enabled = False
+                        }
+                    }
+                }
+                {
+                    apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
+                    kind = "Object"
+                    metadata.name = "celld-rustfs-buckets-ea80c0f0"
+                }
+                {
+                    apiVersion = "helm.m.crossplane.io/v1beta1"
+                    kind = "Release"
+                    metadata.name = "celld"
+                    spec.forProvider.values = {
+                        bootstrapPlaceholder = False
+                        celld = {
+                            bucket = "s3://celld"
+                            endpoint = "http://celld-rustfs-svc.celld.svc.cluster.local:9000"
+                            region = "us-east-1"
+                        }
+                        credentials = {existingSecret = "celld-rustfs"}
+                        azurite = {enabled = False}
+                    }
+                }
+                {
+                    apiVersion = "protection.crossplane.io/v1beta1"
+                    kind = "Usage"
+                    metadata.name = "celld-delete-helm-celld-before-rustfs"
+                }
+                {
+                    apiVersion = "hops.ops.com.ai/v1alpha1"
+                    kind = "CelldStack"
+                    status.rustfs = {
+                        credentialsSecret = "celld-rustfs"
+                        buckets = ["celld", "game-services"]
+                        ready = True
+                    }
+                }
+            ]
+        }
+    }
 ]
 
 items = _items
diff --git a/upbound.yaml b/upbound.yaml
index 8417965..6e14c37 100644
--- a/upbound.yaml
+++ b/upbound.yaml
@@ -37,6 +37,7 @@ spec:
     Composes:
     - Namespace for the release
     - celld Helm release (StatefulSet, public Service, internal headless Service)
+    - Optional in-cluster RustFS S3-compatible storage with buckets (local)
     - Optional in-cluster Azurite (local)
     - Optional AWS S3 bucket + EKS Pod Identity (aws.enabled)
   repository: ghcr.io/hops-ops/celld-stack

From ab77f04faff09b38d1e72ed3332d682167f6f3bb Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Tue, 20 Oct 2026 09:54:01 -0500
Subject: [PATCH 2/4] test: set XR namespace on RustFS observed-resource
 fixtures

Crossplane 2 render matches observed composed resources of a
namespaced XR only in the XR's namespace. The KCL fixtures had no
metadata.namespace, so the render ignored them and the test saw a
first reconcile (status.rustfs.ready false, no RustFS release, no
bucket Job). The YAML fixture already set the namespace; with the
namespace removed, up composition render reproduces the CI failure.
---
 tests/test-render/main.k | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/test-render/main.k b/tests/test-render/main.k
index 14afaa8..416feb7 100644
--- a/tests/test-render/main.k
+++ b/tests/test-render/main.k
@@ -440,12 +440,14 @@ _items = [
         spec = {
             compositionPath = "apis/celldstacks/composition.yaml"
             xrdPath = "apis/celldstacks/definition.yaml"
+            # Namespaced XR: observed MRs must carry the XR namespace to match.
             observedResources = [
                 {
                     apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
                     kind = "Object"
                     metadata = {
                         name = "celld-namespace"
+                        namespace = "default"
                         annotations = {"crossplane.io/composition-resource-name" = "namespace"}
                     }
                     status.conditions = [{type = "Ready", status = "True"}]
@@ -455,6 +457,7 @@ _items = [
                     kind = "Object"
                     metadata = {
                         name = "celld-rustfs-creds"
+                        namespace = "default"
                         uid = "0b6d3f0e-0000-4000-8000-000000000001"
                         annotations = {"crossplane.io/composition-resource-name" = "rustfs-creds-secret"}
                     }
@@ -465,6 +468,7 @@ _items = [
                     kind = "Release"
                     metadata = {
                         name = "celld-rustfs"
+                        namespace = "default"
                         annotations = {"crossplane.io/composition-resource-name" = "helm-release-rustfs"}
                     }
                     status = {
@@ -477,6 +481,7 @@ _items = [
                     kind = "Object"
                     metadata = {
                         name = "celld-rustfs-buckets-ea80c0f0"
+                        namespace = "default"
                         annotations = {"crossplane.io/composition-resource-name" = "rustfs-buckets-job"}
                     }
                     status = {
@@ -489,6 +494,7 @@ _items = [
                     kind = "Release"
                     metadata = {
                         name = "celld"
+                        namespace = "default"
                         annotations = {"crossplane.io/composition-resource-name" = "helm-release-celld"}
                     }
                     status = {

From ac456071a0e23fbcfbbbd881137dee277e056026 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Tue, 20 Oct 2026 10:08:22 -0500
Subject: [PATCH 3/4] fix: deploy the RustFS placeholder only on a 404

The bucket Job marked the fleet bucket as empty on any head-object
error, so an auth, network or 5xx failure could deploy the
placeholder Worker over a real application. Now only a 404 does; any
other error is printed and fails the Job, which retries.

Checked against RustFS 1.0.3: missing key gives (404), wrong secret
gives (403) and exits 1, an unreachable endpoint exits 1.

Also documents why the Job is gated on the RustFS release existing
rather than Ready.
---
 functions/render/120-rustfs.yaml.gotmpl | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/functions/render/120-rustfs.yaml.gotmpl b/functions/render/120-rustfs.yaml.gotmpl
index cc955e2..f1acdcf 100644
--- a/functions/render/120-rustfs.yaml.gotmpl
+++ b/functions/render/120-rustfs.yaml.gotmpl
@@ -123,6 +123,8 @@ spec:
     kind: {{ $state.helmProviderConfigRef.kind }}
 {{- end }}
 
+{{- /* Gate on sticky existence (revision > 0), not Ready: un-rendering would
+     delete the Job when Ready flaps. The Job itself waits for RustFS to answer. */}}
 {{- if $state.observed.rustfsRelease.exists }}
 {{- /* A Job template is immutable: a new name per bucket set and celld image. */}}
 {{- $jobHash := printf "%s|%s|%s:%s" ($rustfs.buckets | join ",") $rustfs.bucket $state.image.repository $state.image.tag | sha256sum | trunc 8 }}
@@ -181,8 +183,16 @@ spec:
                         echo "bucket $bucket created"
                       fi
                     done
-                    if ! aws --endpoint-url "$S3_ENDPOINT" s3api head-object --bucket {{ $rustfs.bucket }} --key deploy/current.json >/dev/null 2>&1; then
+                    # Only a 404 means "no deployment yet". Any other error (auth,
+                    # network, 5xx) fails the Job so it retries, and never deploys
+                    # the placeholder over an application.
+                    if out=$(aws --endpoint-url "$S3_ENDPOINT" s3api head-object --bucket {{ $rustfs.bucket }} --key deploy/current.json 2>&1); then
+                      echo "s3://{{ $rustfs.bucket }}/deploy/current.json exists"
+                    elif printf '%s' "$out" | grep -q '(404)'; then
                       touch /work/needs-deployment
+                    else
+                      printf '%s\n' "$out" >&2
+                      exit 1
                     fi
                 envFrom:
                   - secretRef:

From 831dcd7df4792098ab8ed519f443dd1c11918c78 Mon Sep 17 00:00:00 2001
From: Patrick Lee Scott <pat@patscott.io>
Date: Tue, 20 Oct 2026 10:19:56 -0500
Subject: [PATCH 4/4] fix: hash the full RustFS bucket Job spec into its name

A Job's pod template is immutable, and the old name hashed only the
buckets and celld image. A script, image or resource change on an
existing install would then try to update the Job in place and leave
the Object failing to sync.

The Job spec is now one named template. It is rendered once, its
sha256 goes into the Job name, and the same text is emitted. Any
change makes a new Job; old Jobs are retired and the new one is
idempotent.

Fixtures and the render test now expect celld-rustfs-buckets-f68067af.
---
 .../celldstacks/observed/rustfs-ready.yaml    |   2 +-
 functions/render/120-rustfs.yaml.gotmpl       | 194 +++++++++---------
 tests/test-render/main.k                      |   4 +-
 3 files changed, 104 insertions(+), 96 deletions(-)

diff --git a/examples/celldstacks/observed/rustfs-ready.yaml b/examples/celldstacks/observed/rustfs-ready.yaml
index 83a4b4a..4b1ee32 100644
--- a/examples/celldstacks/observed/rustfs-ready.yaml
+++ b/examples/celldstacks/observed/rustfs-ready.yaml
@@ -42,7 +42,7 @@ status:
 apiVersion: kubernetes.m.crossplane.io/v1alpha1
 kind: Object
 metadata:
-  name: celld-rustfs-buckets-ea80c0f0
+  name: celld-rustfs-buckets-f68067af
   namespace: default
   annotations:
     crossplane.io/composition-resource-name: rustfs-buckets-job
diff --git a/functions/render/120-rustfs.yaml.gotmpl b/functions/render/120-rustfs.yaml.gotmpl
index f1acdcf..8d90807 100644
--- a/functions/render/120-rustfs.yaml.gotmpl
+++ b/functions/render/120-rustfs.yaml.gotmpl
@@ -12,6 +12,104 @@
 # 3. Create the fleet bucket and any extra buckets with a one-shot Job, and
 #    deploy a placeholder Worker once if the fleet bucket has no deployment.
 
+{{- /* Bucket Job spec. Rendered once, hashed into the Job name (a Job's pod
+     template is immutable, so any change to it must create a new Job). */}}
+{{- define "celldstack.rustfsBucketJobSpec" }}
+{{- $s := .state }}
+{{- $r := .rustfs }}
+backoffLimit: 10
+template:
+  metadata:
+    labels: {{ $s.labels | toJson }}
+  spec:
+    restartPolicy: OnFailure
+    volumes:
+      - name: work
+        emptyDir: {}
+    initContainers:
+      - name: create-buckets
+        # renovate: datasource=docker depName=amazon/aws-cli
+        image: amazon/aws-cli:2.31.13
+        command:
+          - /bin/sh
+          - -c
+          - |
+            set -eu
+            i=0
+            until aws --endpoint-url "$S3_ENDPOINT" s3api list-buckets >/dev/null; do
+              i=$((i + 1))
+              if [ "$i" -ge 60 ]; then echo "RustFS is not reachable" >&2; exit 1; fi
+              sleep 2
+            done
+            for bucket in {{ $r.buckets | join " " }}; do
+              if aws --endpoint-url "$S3_ENDPOINT" s3api head-bucket --bucket "$bucket" 2>/dev/null; then
+                echo "bucket $bucket exists"
+              else
+                aws --endpoint-url "$S3_ENDPOINT" s3api create-bucket --bucket "$bucket"
+                echo "bucket $bucket created"
+              fi
+            done
+            # Only a 404 means "no deployment yet". Any other error (auth,
+            # network, 5xx) fails the Job so it retries, and never deploys
+            # the placeholder over an application.
+            if out=$(aws --endpoint-url "$S3_ENDPOINT" s3api head-object --bucket {{ $r.bucket }} --key deploy/current.json 2>&1); then
+              echo "s3://{{ $r.bucket }}/deploy/current.json exists"
+            elif printf '%s' "$out" | grep -q '(404)'; then
+              touch /work/needs-deployment
+            else
+              printf '%s\n' "$out" >&2
+              exit 1
+            fi
+        envFrom:
+          - secretRef:
+              name: {{ $r.name }}
+        volumeMounts:
+          - name: work
+            mountPath: /work
+        resources:
+          requests:
+            cpu: 10m
+            memory: 64Mi
+          limits:
+            memory: 256Mi
+    containers:
+      # celld serves only after a deployment exists. Deploy a fetch-only
+      # placeholder once, into an empty fleet bucket, so a later pod
+      # restart never replaces the application deployed on top of it.
+      - name: bootstrap-deployment
+        image: {{ printf "%s:%s" $s.image.repository $s.image.tag }}
+        command:
+          - /bin/sh
+          - -c
+          - |
+            set -eu
+            if [ ! -f /work/needs-deployment ]; then
+              echo "s3://{{ $r.bucket }} already has a deployment"
+              exit 0
+            fi
+            apt-get update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates xz-utils
+            cd /tmp
+            curl -fsSL https://esbuild.github.io/dl/v0.25.5 | sh
+            export PATH="/tmp:$PATH"
+            mkdir -p /tmp/worker
+            printf '%s\n' '{"name":"celld-placeholder","main":"index.js","compatibility_date":"2026-01-01"}' > /tmp/worker/wrangler.jsonc
+            printf '%s\n' 'export default { fetch() { return new Response("celld ok"); } };' > /tmp/worker/index.js
+            celld deploy /tmp/worker --bucket s3://{{ $r.bucket }}
+        envFrom:
+          - secretRef:
+              name: {{ $r.name }}
+        volumeMounts:
+          - name: work
+            mountPath: /work
+        resources:
+          requests:
+            cpu: 10m
+            memory: 64Mi
+          limits:
+            memory: 512Mi
+{{- end }}
+
 {{- if $state.rustfs.enabled }}
 {{- $rustfs := $state.rustfs }}
 {{- $creds := $rustfs.credentialsSecretRef }}
@@ -126,8 +224,8 @@ spec:
 {{- /* Gate on sticky existence (revision > 0), not Ready: un-rendering would
      delete the Job when Ready flaps. The Job itself waits for RustFS to answer. */}}
 {{- if $state.observed.rustfsRelease.exists }}
-{{- /* A Job template is immutable: a new name per bucket set and celld image. */}}
-{{- $jobHash := printf "%s|%s|%s:%s" ($rustfs.buckets | join ",") $rustfs.bucket $state.image.repository $state.image.tag | sha256sum | trunc 8 }}
+{{- $jobSpec := include "celldstack.rustfsBucketJobSpec" (dict "state" $state "rustfs" $rustfs) | trim }}
+{{- $jobHash := $jobSpec | sha256sum | trunc 8 }}
 {{- $jobName := printf "%s-rustfs-buckets-%s" $state.name $jobHash }}
 ---
 apiVersion: kubernetes.m.crossplane.io/v1alpha1
@@ -151,97 +249,7 @@ spec:
         namespace: {{ $state.namespace }}
         labels: {{ $state.labels | toJson }}
       spec:
-        backoffLimit: 10
-        template:
-          metadata:
-            labels: {{ $state.labels | toJson }}
-          spec:
-            restartPolicy: OnFailure
-            volumes:
-              - name: work
-                emptyDir: {}
-            initContainers:
-              - name: create-buckets
-                # renovate: datasource=docker depName=amazon/aws-cli
-                image: amazon/aws-cli:2.31.13
-                command:
-                  - /bin/sh
-                  - -c
-                  - |
-                    set -eu
-                    i=0
-                    until aws --endpoint-url "$S3_ENDPOINT" s3api list-buckets >/dev/null; do
-                      i=$((i + 1))
-                      if [ "$i" -ge 60 ]; then echo "RustFS is not reachable" >&2; exit 1; fi
-                      sleep 2
-                    done
-                    for bucket in {{ $rustfs.buckets | join " " }}; do
-                      if aws --endpoint-url "$S3_ENDPOINT" s3api head-bucket --bucket "$bucket" 2>/dev/null; then
-                        echo "bucket $bucket exists"
-                      else
-                        aws --endpoint-url "$S3_ENDPOINT" s3api create-bucket --bucket "$bucket"
-                        echo "bucket $bucket created"
-                      fi
-                    done
-                    # Only a 404 means "no deployment yet". Any other error (auth,
-                    # network, 5xx) fails the Job so it retries, and never deploys
-                    # the placeholder over an application.
-                    if out=$(aws --endpoint-url "$S3_ENDPOINT" s3api head-object --bucket {{ $rustfs.bucket }} --key deploy/current.json 2>&1); then
-                      echo "s3://{{ $rustfs.bucket }}/deploy/current.json exists"
-                    elif printf '%s' "$out" | grep -q '(404)'; then
-                      touch /work/needs-deployment
-                    else
-                      printf '%s\n' "$out" >&2
-                      exit 1
-                    fi
-                envFrom:
-                  - secretRef:
-                      name: {{ $rustfs.name }}
-                volumeMounts:
-                  - name: work
-                    mountPath: /work
-                resources:
-                  requests:
-                    cpu: 10m
-                    memory: 64Mi
-                  limits:
-                    memory: 256Mi
-            containers:
-              # celld serves only after a deployment exists. Deploy a fetch-only
-              # placeholder once, into an empty fleet bucket, so a later pod
-              # restart never replaces the application deployed on top of it.
-              - name: bootstrap-deployment
-                image: {{ printf "%s:%s" $state.image.repository $state.image.tag }}
-                command:
-                  - /bin/sh
-                  - -c
-                  - |
-                    set -eu
-                    if [ ! -f /work/needs-deployment ]; then
-                      echo "s3://{{ $rustfs.bucket }} already has a deployment"
-                      exit 0
-                    fi
-                    apt-get update
-                    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates xz-utils
-                    cd /tmp
-                    curl -fsSL https://esbuild.github.io/dl/v0.25.5 | sh
-                    export PATH="/tmp:$PATH"
-                    mkdir -p /tmp/worker
-                    printf '%s\n' '{"name":"celld-placeholder","main":"index.js","compatibility_date":"2026-01-01"}' > /tmp/worker/wrangler.jsonc
-                    printf '%s\n' 'export default { fetch() { return new Response("celld ok"); } };' > /tmp/worker/index.js
-                    celld deploy /tmp/worker --bucket s3://{{ $rustfs.bucket }}
-                envFrom:
-                  - secretRef:
-                      name: {{ $rustfs.name }}
-                volumeMounts:
-                  - name: work
-                    mountPath: /work
-                resources:
-                  requests:
-                    cpu: 10m
-                    memory: 64Mi
-                  limits:
-                    memory: 512Mi
+        {{- $jobSpec | nindent 8 }}
   providerConfigRef:
     name: {{ $state.kubernetesProviderConfigRef.name }}
     kind: {{ $state.kubernetesProviderConfigRef.kind }}
diff --git a/tests/test-render/main.k b/tests/test-render/main.k
index 416feb7..15e8a56 100644
--- a/tests/test-render/main.k
+++ b/tests/test-render/main.k
@@ -480,7 +480,7 @@ _items = [
                     apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
                     kind = "Object"
                     metadata = {
-                        name = "celld-rustfs-buckets-ea80c0f0"
+                        name = "celld-rustfs-buckets-f68067af"
                         namespace = "default"
                         annotations = {"crossplane.io/composition-resource-name" = "rustfs-buckets-job"}
                     }
@@ -540,7 +540,7 @@ _items = [
                 {
                     apiVersion = "kubernetes.m.crossplane.io/v1alpha1"
                     kind = "Object"
-                    metadata.name = "celld-rustfs-buckets-ea80c0f0"
+                    metadata.name = "celld-rustfs-buckets-f68067af"
                 }
                 {
                     apiVersion = "helm.m.crossplane.io/v1beta1"