diff --git a/.github/workflows/on-pr.yaml b/.github/workflows/on-pr.yaml index 4cb4fb2..a2f3399 100644 --- a/.github/workflows/on-pr.yaml +++ b/.github/workflows/on-pr.yaml @@ -35,7 +35,9 @@ jobs: { "example": "examples/celldstacks/standard.yaml" }, { "example": "examples/celldstacks/azurite.yaml" }, { "example": "examples/celldstacks/aws.yaml" }, - { "example": "examples/celldstacks/aws-local.yaml" } + { "example": "examples/celldstacks/aws-local.yaml" }, + { "example": "examples/celldstacks/rustfs.yaml" }, + { "example": "examples/celldstacks/rustfs.yaml", "observed_resources": "examples/celldstacks/observed/rustfs-ready.yaml" } ] api_path: apis/celldstacks error_on_missing_schemas: true diff --git a/.github/workflows/on-push-main.yaml b/.github/workflows/on-push-main.yaml index 76165f3..165cea0 100644 --- a/.github/workflows/on-push-main.yaml +++ b/.github/workflows/on-push-main.yaml @@ -30,7 +30,9 @@ jobs: { "example": "examples/celldstacks/standard.yaml" }, { "example": "examples/celldstacks/azurite.yaml" }, { "example": "examples/celldstacks/aws.yaml" }, - { "example": "examples/celldstacks/aws-local.yaml" } + { "example": "examples/celldstacks/aws-local.yaml" }, + { "example": "examples/celldstacks/rustfs.yaml" }, + { "example": "examples/celldstacks/rustfs.yaml", "observed_resources": "examples/celldstacks/observed/rustfs-ready.yaml" } ] api_path: apis/celldstacks error_on_missing_schemas: true diff --git a/Makefile b/Makefile index a286bda..6a433c6 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,9 @@ EXAMPLES := \ examples/celldstacks/standard.yaml:: \ examples/celldstacks/azurite.yaml:: \ examples/celldstacks/aws.yaml:: \ - examples/celldstacks/aws-local.yaml:: + examples/celldstacks/aws-local.yaml:: \ + examples/celldstacks/rustfs.yaml:: \ + examples/celldstacks/rustfs.yaml::examples/celldstacks/observed/rustfs-ready.yaml render\:all: @tmpdir=$$(mktemp -d); \ diff --git a/README.md b/README.md index a4f171c..91a6b6e 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ spec: clusterName: my-cluster ``` -This installs one celld node in namespace `celld`. For local, enable Azurite. For AWS, enable `spec.aws` so the stack creates a bucket and binds the celld ServiceAccount with EKS Pod Identity. +This installs one celld node in namespace `celld`. For local, enable RustFS (S3-compatible, recommended) or Azurite. For AWS, enable `spec.aws` so the stack creates a bucket and binds the celld ServiceAccount with EKS Pod Identity. ```yaml spec: @@ -29,6 +29,38 @@ spec: enabled: true ``` +### Stage 1b: Local S3 with RustFS + +```yaml +spec: + clusterName: my-cluster + rustfs: + enabled: true + buckets: + - game-services # extra application buckets; the fleet bucket is always created + credentialsSecretRef: + name: celld-rustfs-credentials +``` + +Create the root credentials yourself; the stack never generates or embeds keys: + +```bash +kubectl -n default create secret generic celld-rustfs-credentials \ + --from-literal=RUSTFS_ACCESS_KEY="$(openssl rand -hex 12)" \ + --from-literal=RUSTFS_SECRET_KEY="$(openssl rand -hex 24)" +``` + +The stack copies that Secret into the release namespace as `-rustfs` +with `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_REGION` and +`S3_ENDPOINT` added, installs the official RustFS chart (standalone, one PVC), +creates `celld` plus the extra buckets with a one-shot Job (which also deploys +a fetch-only placeholder Worker once, only into an empty fleet bucket, so pod +restarts never replace your application), and then installs +celld against `s3://celld` on `http://-rustfs-svc..svc.cluster.local:9000`. +Applications in the cluster can mount the same `-rustfs` Secret +(`envFrom`) to reach their buckets with any S3 client. RustFS takes precedence +over `azurite` and `aws`. + ### Stage 2: Growing — AWS S3 ```yaml @@ -80,6 +112,13 @@ Not applicable. This stack installs a Helm release; it does not adopt cloud reso | `credentials.existingSecret` | string | — | Secret with AWS_* keys (R2 / BYO). Unused when `aws.enabled` | | `azurite.enabled` | boolean | `false` | Deploy in-cluster Azurite and point celld at `az://celld` (local/dev only) | | `azurite.container` | string | `celld` | Blob container name | +| `rustfs.enabled` | boolean | `false` | Deploy in-cluster RustFS and point celld at it (wins over `azurite`/`aws`) | +| `rustfs.bucket` | string | `celld` | Fleet bucket name | +| `rustfs.buckets` | string[] | — | Extra buckets to create | +| `rustfs.credentialsSecretRef` | object | `-rustfs-credentials` in the XR namespace | Secret with `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` | +| `rustfs.persistence.size` | string | `2Gi` | RustFS data volume | +| `rustfs.persistence.storageClassName` | string | cluster default | RustFS storage class | +| `rustfs.values` | object | — | RustFS chart values merged with stack defaults | | `aws.enabled` | boolean | `false` | Create S3 bucket and point celld at `s3://` | | `aws.bucketName` | string | `hops-celld-` | Globally unique S3 bucket name | | `aws.region` | string | `us-east-2` | Bucket region | @@ -95,11 +134,13 @@ Not applicable. This stack installs a Helm release; it does not adopt cloud reso | Field | Meaning | |-------|---------| -| `ready` | Namespace, Helm release, Usage, and (when `aws.enabled`) bucket + Pod Identity are Ready | +| `ready` | Namespace, Helm release, Usage, and (when `aws.enabled`) bucket + Pod Identity, or (when `rustfs.enabled`) RustFS + buckets are Ready | | `release.name` / `release.namespace` | Installed Helm release | | `service.name` / `service.port` | Public Worker Service (port 8080) | | `bucket.name` / `bucket.id` | Observed S3 bucket when `aws.enabled` | | `podIdentity.roleArn` / `podIdentity.associationId` | Observed Pod Identity when `aws.enabled` | +| `rustfs.endpoint` / `rustfs.credentialsSecret` | In-cluster S3 endpoint and the Secret S3 clients mount (when `rustfs.enabled`) | +| `rustfs.buckets` / `rustfs.ready` | Buckets the bucket Job ensured; credentials, RustFS and buckets ready | ## Composed Resources @@ -110,6 +151,10 @@ Not applicable. This stack installs a Helm release; it does not adopt cloud reso | `-s3` | `s3.aws.m.upbound.io/Bucket` (when `aws.enabled`) | | `-celld` | `aws.hops.ops.com.ai/PodIdentity` (when `aws.enabled`) | | `-delete-helm-celld-before-namespace` | `protection.crossplane.io/Usage` (after both are Ready) | +| `-rustfs-creds` | `kubernetes.m.crossplane.io/Object` (Secret, when `rustfs.enabled`) | +| `-rustfs` | `helm.m.crossplane.io/Release` (RustFS, when `rustfs.enabled`) | +| `-rustfs-buckets-` | `kubernetes.m.crossplane.io/Object` (bucket Job, when `rustfs.enabled`) | +| `-delete-helm-celld-before-rustfs`, `-before-rustfs-creds`, `-delete-helm-rustfs-before-namespace`, `-before-rustfs-creds` | `protection.crossplane.io/Usage` (RustFS teardown order) | | `-delete-helm-celld-before-pod-identity` | `protection.crossplane.io/Usage` (after Helm + Pod Identity are Ready) | ## Development diff --git a/apis/celldstacks/definition.yaml b/apis/celldstacks/definition.yaml index c3e3fa9..0ae994a 100644 --- a/apis/celldstacks/definition.yaml +++ b/apis/celldstacks/definition.yaml @@ -147,8 +147,48 @@ spec: default: 2Gi storageClassName: type: string + rustfs: + description: Deploy in-cluster RustFS (S3-compatible) and point celld at it. Takes precedence over azurite and aws. Local/dev and single-node use. + type: object + properties: + enabled: + description: When true, install RustFS (standalone), create the buckets, and configure celld for s3:// on the RustFS endpoint. + type: boolean + default: false + bucket: + description: celld fleet bucket name. Defaults to "celld". + type: string + pattern: '^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$' + buckets: + description: Additional buckets to create for applications (for example game-services blobs). The fleet bucket is always created. + type: array + items: + type: string + pattern: '^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$' + credentialsSecretRef: + description: Secret holding the RustFS root credentials as RUSTFS_ACCESS_KEY and RUSTFS_SECRET_KEY. Copied into the release namespace with AWS_* aliases and S3_ENDPOINT. Defaults to -rustfs-credentials in the XR namespace. Must not be the copied Secret itself (-rustfs in the release namespace). No credentials are generated or stored in the manifest. + type: object + properties: + name: + type: string + namespace: + type: string + persistence: + description: Persistent volume for RustFS object data. + type: object + properties: + size: + type: string + default: 2Gi + storageClassName: + description: Storage class. Empty uses the cluster default. + type: string + values: + description: Helm values merged on top of the RustFS chart defaults set by this stack. + type: object + x-kubernetes-preserve-unknown-fields: true aws: - description: Provision an S3 bucket and grant the celld pod access. On EKS, Pod Identity is the default. Locally, disable podIdentity and reuse hops `local aws` credentials. Ignored when azurite.enabled is true. + description: Provision an S3 bucket and grant the celld pod access. On EKS, Pod Identity is the default. Locally, disable podIdentity and reuse hops `local aws` credentials. Ignored when azurite.enabled or rustfs.enabled is true. type: object properties: enabled: @@ -237,7 +277,7 @@ spec: type: object properties: ready: - description: Whether the namespace, Helm release, and deletion Usage are ready. + description: Whether the namespace, Helm releases, buckets, and deletion Usage are ready. type: boolean release: description: Observed Helm release. @@ -269,6 +309,23 @@ spec: type: string ready: type: boolean + rustfs: + description: Observed in-cluster RustFS object store. + type: object + properties: + endpoint: + description: In-cluster S3 endpoint URL. + type: string + credentialsSecret: + description: Secret in the release namespace with AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION and S3_ENDPOINT for S3 clients. + type: string + buckets: + description: Buckets the bucket Job created or found. + type: array + items: + type: string + ready: + type: boolean podIdentity: description: Observed EKS Pod Identity binding the celld ServiceAccount to the fleet bucket. type: object diff --git a/examples/celldstacks/observed/rustfs-ready.yaml b/examples/celldstacks/observed/rustfs-ready.yaml new file mode 100644 index 0000000..4b1ee32 --- /dev/null +++ b/examples/celldstacks/observed/rustfs-ready.yaml @@ -0,0 +1,70 @@ +# Observed state for examples/celldstacks/rustfs.yaml once every step is up: +# credentials copied, RustFS installed, buckets created, celld installed. +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: celld-namespace + namespace: default + annotations: + crossplane.io/composition-resource-name: namespace +status: + conditions: + - type: Ready + status: "True" +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: celld-rustfs-creds + namespace: default + uid: 0b6d3f0e-0000-4000-8000-000000000001 + annotations: + crossplane.io/composition-resource-name: rustfs-creds-secret +status: + conditions: + - type: Ready + status: "True" +--- +apiVersion: helm.m.crossplane.io/v1beta1 +kind: Release +metadata: + name: celld-rustfs + namespace: default + annotations: + crossplane.io/composition-resource-name: helm-release-rustfs +status: + atProvider: + revision: 1 + conditions: + - type: Ready + status: "True" +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: celld-rustfs-buckets-f68067af + namespace: default + annotations: + crossplane.io/composition-resource-name: rustfs-buckets-job +status: + atProvider: + manifest: + status: + succeeded: 1 + conditions: + - type: Ready + status: "True" +--- +apiVersion: helm.m.crossplane.io/v1beta1 +kind: Release +metadata: + name: celld + namespace: default + annotations: + crossplane.io/composition-resource-name: helm-release-celld +status: + atProvider: + revision: 1 + conditions: + - type: Ready + status: "True" diff --git a/examples/celldstacks/rustfs.yaml b/examples/celldstacks/rustfs.yaml new file mode 100644 index 0000000..6eba827 --- /dev/null +++ b/examples/celldstacks/rustfs.yaml @@ -0,0 +1,13 @@ +apiVersion: hops.ops.com.ai/v1alpha1 +kind: CelldStack +metadata: + name: celld + namespace: default +spec: + clusterName: my-cluster + rustfs: + enabled: true + buckets: + - game-services + credentialsSecretRef: + name: celld-rustfs-credentials diff --git a/functions/render/000-state-init.yaml.gotmpl b/functions/render/000-state-init.yaml.gotmpl index c26b17e..d0b902e 100644 --- a/functions/render/000-state-init.yaml.gotmpl +++ b/functions/render/000-state-init.yaml.gotmpl @@ -55,10 +55,16 @@ {{- $azuritePersistence := $azurite.persistence | default dict }} {{- $azuritePersistenceEnabled := true }} {{- if hasKey $azuritePersistence "enabled" }}{{- $azuritePersistenceEnabled = $azuritePersistence.enabled }}{{- end }} +{{- $rustfs := $spec.rustfs | default dict }} +{{- $rustfsEnabled := false }} +{{- if hasKey $rustfs "enabled" }}{{- $rustfsEnabled = $rustfs.enabled }}{{- end }} +{{- if $rustfsEnabled }} + {{- $azuriteEnabled = false }} +{{- end }} {{- $aws := $spec.aws | default dict }} {{- $awsEnabled := false }} {{- if hasKey $aws "enabled" }}{{- $awsEnabled = $aws.enabled }}{{- end }} -{{- if $azuriteEnabled }} +{{- if or $azuriteEnabled $rustfsEnabled }} {{- $awsEnabled = false }} {{- end }} {{- $awsForceDestroy := true }} @@ -83,8 +89,27 @@ {{- if and $awsEnabled (ne $awsCredsName "") }} {{- $awsCredsLocalName = printf "%s-aws" $releaseName }} {{- end }} +{{- $rustfsName := printf "%s-rustfs" $releaseName }} +{{- $rustfsBucket := $rustfs.bucket | default "celld" }} +{{- $rustfsBuckets := list $rustfsBucket }} +{{- range ($rustfs.buckets | default list) }} + {{- if not (has . $rustfsBuckets) }} + {{- $rustfsBuckets = append $rustfsBuckets . }} + {{- end }} +{{- end }} +{{- $rustfsCredsRef := $rustfs.credentialsSecretRef | default dict }} +{{- $rustfsPersistence := $rustfs.persistence | default dict }} +{{- $rustfsEndpoint := printf "http://%s-svc.%s.svc.cluster.local:9000" $rustfsName $namespace }} +{{- $endpoint := $spec.endpoint | default "" }} +{{- $existingSecret := $credentials.existingSecret | default "" }} +{{- if $rustfsEnabled }} + {{- $endpoint = $rustfsEndpoint }} + {{- $existingSecret = $rustfsName }} +{{- end }} {{- $bucket := $spec.bucket | default "" }} -{{- if $azuriteEnabled }} +{{- if $rustfsEnabled }} + {{- $bucket = printf "s3://%s" $rustfsBucket }} +{{- else if $azuriteEnabled }} {{- $bucket = printf "az://%s" $azuriteContainer }} {{- else if $awsEnabled }} {{- $bucket = printf "s3://%s" $awsBucketName }} @@ -114,11 +139,11 @@ "awsProviderConfigRef" $awsProviderConfigRef "serviceName" $serviceName "bucket" $bucket - "endpoint" ($spec.endpoint | default "") - "region" $awsRegion + "endpoint" $endpoint + "region" (ternary "us-east-1" $awsRegion $rustfsEnabled) "xrNamespace" $xrNamespace "credentials" (dict - "existingSecret" ($credentials.existingSecret | default "") + "existingSecret" $existingSecret "sharedCredentialsSecret" $awsCredsLocalName "sharedCredentialsKey" "credentials" ) @@ -155,6 +180,22 @@ "storageClassName" ($azuritePersistence.storageClassName | default "") ) ) + "rustfs" (dict + "enabled" $rustfsEnabled + "name" $rustfsName + "endpoint" $rustfsEndpoint + "bucket" $rustfsBucket + "buckets" $rustfsBuckets + "credentialsSecretRef" (dict + "name" ($rustfsCredsRef.name | default (printf "%s-rustfs-credentials" $name)) + "namespace" ($rustfsCredsRef.namespace | default $xrNamespace) + ) + "persistence" (dict + "size" ($rustfsPersistence.size | default "2Gi") + "storageClassName" ($rustfsPersistence.storageClassName | default "") + ) + "values" ($rustfs.values | default dict) + ) "image" (dict "repository" ($image.repository | default "ghcr.io/denoland/celld") "tag" $imageTag diff --git a/functions/render/001-state-observed.yaml.gotmpl b/functions/render/001-state-observed.yaml.gotmpl index 0442dd5..d14bfdb 100644 --- a/functions/render/001-state-observed.yaml.gotmpl +++ b/functions/render/001-state-observed.yaml.gotmpl @@ -26,6 +26,19 @@ {{- $awsCredsResource := $awsCredsEntry.resource | default dict }} {{- $awsCredsUid := (($awsCredsResource.metadata | default dict).uid | default "") }} +{{- $revisionOf := dict }} +{{- range $key, $entry := $observed }} + {{- $atProvider := (($entry.resource | default dict).status | default dict).atProvider | default dict }} + {{- $revisionOf = set $revisionOf $key (gt (int ($atProvider.revision | default 0)) 0) }} +{{- end }} + +{{- $rustfsCredsEntry := get $observed "rustfs-creds-secret" | default dict }} +{{- $rustfsCredsUid := ((($rustfsCredsEntry.resource | default dict).metadata | default dict).uid | default "") }} + +{{- $rustfsBucketsEntry := get $observed "rustfs-buckets-job" | default dict }} +{{- $rustfsBucketsManifest := ((($rustfsBucketsEntry.resource | default dict).status | default dict).atProvider | default dict).manifest | default dict }} +{{- $rustfsBucketsSucceeded := gt (int (($rustfsBucketsManifest.status | default dict).succeeded | default 0)) 0 }} + {{- $podIdentityEntry := get $observed "pod-identity" | default dict }} {{- $podIdentityResource := $podIdentityEntry.resource | default dict }} {{- $podIdentityStatus := ($podIdentityResource.status | default dict).podIdentity | default dict }} @@ -34,7 +47,22 @@ {{- $state = set $state "observed" (merge $state.observed (dict "namespace" (dict "ready" (get $readyOf "namespace" | default false)) - "release" (dict "ready" (get $readyOf "helm-release-celld" | default false)) + "release" (dict + "ready" (get $readyOf "helm-release-celld" | default false) + "exists" (get $revisionOf "helm-release-celld" | default false) + ) + "rustfsCreds" (dict + "ready" (get $readyOf "rustfs-creds-secret" | default false) + "id" $rustfsCredsUid + ) + "rustfsRelease" (dict + "ready" (get $readyOf "helm-release-rustfs" | default false) + "exists" (get $revisionOf "helm-release-rustfs" | default false) + ) + "rustfsBuckets" (dict + "ready" (get $readyOf "rustfs-buckets-job" | default false) + "succeeded" $rustfsBucketsSucceeded + ) "bucket" (dict "ready" (get $readyOf "s3-bucket" | default false) "id" (get $idOf "s3-bucket" | default "") diff --git a/functions/render/010-state-status.yaml.gotmpl b/functions/render/010-state-status.yaml.gotmpl index 9e5ee3f..2f074ab 100644 --- a/functions/render/010-state-status.yaml.gotmpl +++ b/functions/render/010-state-status.yaml.gotmpl @@ -28,6 +28,21 @@ {{- end }} {{- end }} +{{- $rustfsReady := true }} +{{- if $state.rustfs.enabled }} + {{- $rustfsReady = and $obs.rustfsCreds.ready $obs.rustfsRelease.ready $obs.rustfsBuckets.succeeded }} +{{- end }} + +{{- $rustfsStatus := dict "endpoint" "" "credentialsSecret" "" "buckets" (list) "ready" false }} +{{- if $state.rustfs.enabled }} + {{- $rustfsStatus = dict + "endpoint" $state.rustfs.endpoint + "credentialsSecret" $state.rustfs.name + "buckets" (ternary $state.rustfs.buckets (list) $obs.rustfsBuckets.succeeded) + "ready" $rustfsReady + }} +{{- end }} + {{- $bucketStatus := dict "name" "" "id" "" "ready" false }} {{- $podIdentityStatus := dict "roleArn" "" "associationId" "" "ready" false }} {{- if $state.aws.enabled }} @@ -44,7 +59,7 @@ {{- end }} {{- $state = set $state "status" (dict - "ready" (and $namespaceReady $releaseReady $usageReady $awsReady) + "ready" (and $namespaceReady $releaseReady $usageReady $awsReady $rustfsReady) "release" (dict "name" $state.releaseName "namespace" $state.namespace @@ -57,4 +72,5 @@ ) "bucket" $bucketStatus "podIdentity" $podIdentityStatus + "rustfs" $rustfsStatus ) }} diff --git a/functions/render/120-rustfs.yaml.gotmpl b/functions/render/120-rustfs.yaml.gotmpl new file mode 100644 index 0000000..8d90807 --- /dev/null +++ b/functions/render/120-rustfs.yaml.gotmpl @@ -0,0 +1,303 @@ +# code: language=yaml +# +# RustFS: in-cluster S3-compatible object storage (rustfs.enabled). +# +# Chart: https://charts.rustfs.com (chart name: rustfs) +# Upstream: https://github.com/rustfs/rustfs +# +# 1. Copy the user's RUSTFS_ACCESS_KEY / RUSTFS_SECRET_KEY Secret into the +# release namespace, adding the AWS_* aliases and S3_ENDPOINT that celld +# and its bootstrap deploy init container read through envFrom. +# 2. Install RustFS (standalone) with that Secret as its root credentials. +# 3. Create the fleet bucket and any extra buckets with a one-shot Job, and +# deploy a placeholder Worker once if the fleet bucket has no deployment. + +{{- /* Bucket Job spec. Rendered once, hashed into the Job name (a Job's pod + template is immutable, so any change to it must create a new Job). */}} +{{- define "celldstack.rustfsBucketJobSpec" }} +{{- $s := .state }} +{{- $r := .rustfs }} +backoffLimit: 10 +template: + metadata: + labels: {{ $s.labels | toJson }} + spec: + restartPolicy: OnFailure + volumes: + - name: work + emptyDir: {} + initContainers: + - name: create-buckets + # renovate: datasource=docker depName=amazon/aws-cli + image: amazon/aws-cli:2.31.13 + command: + - /bin/sh + - -c + - | + set -eu + i=0 + until aws --endpoint-url "$S3_ENDPOINT" s3api list-buckets >/dev/null; do + i=$((i + 1)) + if [ "$i" -ge 60 ]; then echo "RustFS is not reachable" >&2; exit 1; fi + sleep 2 + done + for bucket in {{ $r.buckets | join " " }}; do + if aws --endpoint-url "$S3_ENDPOINT" s3api head-bucket --bucket "$bucket" 2>/dev/null; then + echo "bucket $bucket exists" + else + aws --endpoint-url "$S3_ENDPOINT" s3api create-bucket --bucket "$bucket" + echo "bucket $bucket created" + fi + done + # Only a 404 means "no deployment yet". Any other error (auth, + # network, 5xx) fails the Job so it retries, and never deploys + # the placeholder over an application. + if out=$(aws --endpoint-url "$S3_ENDPOINT" s3api head-object --bucket {{ $r.bucket }} --key deploy/current.json 2>&1); then + echo "s3://{{ $r.bucket }}/deploy/current.json exists" + elif printf '%s' "$out" | grep -q '(404)'; then + touch /work/needs-deployment + else + printf '%s\n' "$out" >&2 + exit 1 + fi + envFrom: + - secretRef: + name: {{ $r.name }} + volumeMounts: + - name: work + mountPath: /work + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 256Mi + containers: + # celld serves only after a deployment exists. Deploy a fetch-only + # placeholder once, into an empty fleet bucket, so a later pod + # restart never replaces the application deployed on top of it. + - name: bootstrap-deployment + image: {{ printf "%s:%s" $s.image.repository $s.image.tag }} + command: + - /bin/sh + - -c + - | + set -eu + if [ ! -f /work/needs-deployment ]; then + echo "s3://{{ $r.bucket }} already has a deployment" + exit 0 + fi + apt-get update + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl ca-certificates xz-utils + cd /tmp + curl -fsSL https://esbuild.github.io/dl/v0.25.5 | sh + export PATH="/tmp:$PATH" + mkdir -p /tmp/worker + printf '%s\n' '{"name":"celld-placeholder","main":"index.js","compatibility_date":"2026-01-01"}' > /tmp/worker/wrangler.jsonc + printf '%s\n' 'export default { fetch() { return new Response("celld ok"); } };' > /tmp/worker/index.js + celld deploy /tmp/worker --bucket s3://{{ $r.bucket }} + envFrom: + - secretRef: + name: {{ $r.name }} + volumeMounts: + - name: work + mountPath: /work + resources: + requests: + cpu: 10m + memory: 64Mi + limits: + memory: 512Mi +{{- end }} + +{{- if $state.rustfs.enabled }} +{{- $rustfs := $state.rustfs }} +{{- $creds := $rustfs.credentialsSecretRef }} +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: {{ $state.name }}-rustfs-creds + annotations: + {{ setResourceNameAnnotation "rustfs-creds-secret" }} + labels: {{ $state.labels | toJson }} +spec: + managementPolicies: {{ $state.managementPolicies | toJson }} + forProvider: + manifest: + apiVersion: v1 + kind: Secret + metadata: + name: {{ $rustfs.name }} + namespace: {{ $state.namespace }} + labels: {{ $state.labels | toJson }} + type: Opaque + data: + S3_ENDPOINT: {{ $rustfs.endpoint | b64enc | quote }} + AWS_REGION: {{ $state.region | b64enc | quote }} + references: + - patchesFrom: + apiVersion: v1 + kind: Secret + name: {{ $creds.name }} + namespace: {{ $creds.namespace }} + fieldPath: data.RUSTFS_ACCESS_KEY + toFieldPath: data.RUSTFS_ACCESS_KEY + - patchesFrom: + apiVersion: v1 + kind: Secret + name: {{ $creds.name }} + namespace: {{ $creds.namespace }} + fieldPath: data.RUSTFS_SECRET_KEY + toFieldPath: data.RUSTFS_SECRET_KEY + - patchesFrom: + apiVersion: v1 + kind: Secret + name: {{ $creds.name }} + namespace: {{ $creds.namespace }} + fieldPath: data.RUSTFS_ACCESS_KEY + toFieldPath: data.AWS_ACCESS_KEY_ID + - patchesFrom: + apiVersion: v1 + kind: Secret + name: {{ $creds.name }} + namespace: {{ $creds.namespace }} + fieldPath: data.RUSTFS_SECRET_KEY + toFieldPath: data.AWS_SECRET_ACCESS_KEY + - dependsOn: + apiVersion: v1 + kind: Namespace + name: {{ $state.namespace }} + providerConfigRef: + name: {{ $state.kubernetesProviderConfigRef.name }} + kind: {{ $state.kubernetesProviderConfigRef.kind }} + +{{- $rustfsDefaults := dict + "fullnameOverride" $rustfs.name + "commonLabels" $state.labels + "replicaCount" 1 + "mode" (dict + "standalone" (dict "enabled" true) + "distributed" (dict "enabled" false) + ) + "secret" (dict "existingSecret" $rustfs.name) + "config" (dict "rustfs" (dict "region" $state.region)) + "ingress" (dict "enabled" false) + "storageclass" (dict + "name" $rustfs.persistence.storageClassName + "dataStorageSize" $rustfs.persistence.size + "logStorageSize" "256Mi" + ) + "resources" (dict + "requests" (dict "cpu" "50m" "memory" "128Mi") + "limits" (dict "memory" "1Gi") + ) +}} + +{{- if ne $state.observed.rustfsCreds.id "" }} +--- +apiVersion: helm.m.crossplane.io/v1beta1 +kind: Release +metadata: + name: {{ $rustfs.name }} + annotations: + {{ setResourceNameAnnotation "helm-release-rustfs" }} + labels: {{ $state.labels | toJson }} +spec: + managementPolicies: {{ $state.managementPolicies | toJson }} + forProvider: + chart: + name: rustfs + repository: https://charts.rustfs.com + # renovate: datasource=helm depName=rustfs registryUrl=https://charts.rustfs.com + version: "1.0.3" + namespace: {{ $state.namespace }} + wait: false + values: + {{- toYaml (mergeOverwrite $rustfsDefaults $rustfs.values) | nindent 6 }} + rollbackLimit: 3 + providerConfigRef: + name: {{ $state.helmProviderConfigRef.name }} + kind: {{ $state.helmProviderConfigRef.kind }} +{{- end }} + +{{- /* Gate on sticky existence (revision > 0), not Ready: un-rendering would + delete the Job when Ready flaps. The Job itself waits for RustFS to answer. */}} +{{- if $state.observed.rustfsRelease.exists }} +{{- $jobSpec := include "celldstack.rustfsBucketJobSpec" (dict "state" $state "rustfs" $rustfs) | trim }} +{{- $jobHash := $jobSpec | sha256sum | trunc 8 }} +{{- $jobName := printf "%s-rustfs-buckets-%s" $state.name $jobHash }} +--- +apiVersion: kubernetes.m.crossplane.io/v1alpha1 +kind: Object +metadata: + name: {{ $jobName }} + annotations: + {{ setResourceNameAnnotation "rustfs-buckets-job" }} + labels: {{ $state.labels | toJson }} +spec: + managementPolicies: {{ $state.managementPolicies | toJson }} + readiness: + policy: DeriveFromCelQuery + celQuery: "has(object.status.succeeded) && object.status.succeeded > 0" + forProvider: + manifest: + apiVersion: batch/v1 + kind: Job + metadata: + name: {{ $jobName }} + namespace: {{ $state.namespace }} + labels: {{ $state.labels | toJson }} + spec: + {{- $jobSpec | nindent 8 }} + providerConfigRef: + name: {{ $state.kubernetesProviderConfigRef.name }} + kind: {{ $state.kubernetesProviderConfigRef.kind }} +{{- end }} + +{{- if and $state.observed.namespace.ready $state.observed.rustfsRelease.ready }} +--- +apiVersion: protection.crossplane.io/v1beta1 +kind: Usage +metadata: + name: {{ $state.name }}-delete-helm-rustfs-before-namespace + annotations: + {{ setResourceNameAnnotation "usage-rustfs-namespace" }} + labels: {{ $state.labels | toJson }} +spec: + replayDeletion: true + of: + apiVersion: kubernetes.m.crossplane.io/v1alpha1 + kind: Object + resourceRef: + name: {{ $state.name }}-namespace + by: + apiVersion: helm.m.crossplane.io/v1beta1 + kind: Release + resourceRef: + name: {{ $rustfs.name }} +{{- end }} + +{{- if and $state.observed.rustfsCreds.ready $state.observed.rustfsRelease.ready }} +--- +apiVersion: protection.crossplane.io/v1beta1 +kind: Usage +metadata: + name: {{ $state.name }}-delete-helm-rustfs-before-rustfs-creds + annotations: + {{ setResourceNameAnnotation "usage-rustfs-creds" }} + labels: {{ $state.labels | toJson }} +spec: + replayDeletion: true + of: + apiVersion: kubernetes.m.crossplane.io/v1alpha1 + kind: Object + resourceRef: + name: {{ $state.name }}-rustfs-creds + by: + apiVersion: helm.m.crossplane.io/v1beta1 + kind: Release + resourceRef: + name: {{ $rustfs.name }} +{{- end }} +{{- end }} diff --git a/functions/render/200-helm-release-celld.yaml.gotmpl b/functions/render/200-helm-release-celld.yaml.gotmpl index 37550cc..a443214 100644 --- a/functions/render/200-helm-release-celld.yaml.gotmpl +++ b/functions/render/200-helm-release-celld.yaml.gotmpl @@ -31,6 +31,9 @@ {{- $renderHelm = and $renderHelm (ne $state.observed.awsCreds.id "") }} {{- end }} {{- end }} +{{- if $state.rustfs.enabled }} + {{- $renderHelm = or $state.observed.release.exists $state.observed.rustfsBuckets.succeeded }} +{{- end }} {{- if $renderHelm }} --- apiVersion: helm.m.crossplane.io/v1beta1 @@ -113,4 +116,50 @@ spec: resourceRef: name: {{ $state.releaseName }} {{- end }} + +{{- if and $state.rustfs.enabled $state.observed.rustfsRelease.ready $state.observed.release.ready }} +--- +apiVersion: protection.crossplane.io/v1beta1 +kind: Usage +metadata: + name: {{ $state.name }}-delete-helm-celld-before-rustfs + annotations: + {{ setResourceNameAnnotation "usage-helm-celld-rustfs" }} + labels: {{ $state.labels | toJson }} +spec: + replayDeletion: true + of: + apiVersion: helm.m.crossplane.io/v1beta1 + kind: Release + resourceRef: + name: {{ $state.rustfs.name }} + by: + apiVersion: helm.m.crossplane.io/v1beta1 + kind: Release + resourceRef: + name: {{ $state.releaseName }} +{{- end }} + +{{- if and $state.rustfs.enabled $state.observed.rustfsCreds.ready $state.observed.release.ready }} +--- +apiVersion: protection.crossplane.io/v1beta1 +kind: Usage +metadata: + name: {{ $state.name }}-delete-helm-celld-before-rustfs-creds + annotations: + {{ setResourceNameAnnotation "usage-helm-celld-rustfs-creds" }} + labels: {{ $state.labels | toJson }} +spec: + replayDeletion: true + of: + apiVersion: kubernetes.m.crossplane.io/v1alpha1 + kind: Object + resourceRef: + name: {{ $state.name }}-rustfs-creds + by: + apiVersion: helm.m.crossplane.io/v1beta1 + kind: Release + resourceRef: + name: {{ $state.releaseName }} +{{- end }} {{- end }} diff --git a/functions/render/999-status.yaml.gotmpl b/functions/render/999-status.yaml.gotmpl index 4dd71c5..1003b05 100644 --- a/functions/render/999-status.yaml.gotmpl +++ b/functions/render/999-status.yaml.gotmpl @@ -25,3 +25,8 @@ status: roleArn: {{ $s.podIdentity.roleArn | quote }} associationId: {{ $s.podIdentity.associationId | quote }} ready: {{ $s.podIdentity.ready }} + rustfs: + endpoint: {{ $s.rustfs.endpoint | quote }} + credentialsSecret: {{ $s.rustfs.credentialsSecret | quote }} + buckets: {{ $s.rustfs.buckets | toJson }} + ready: {{ $s.rustfs.ready }} diff --git a/tests/test-render/main.k b/tests/test-render/main.k index da8c549..15e8a56 100644 --- a/tests/test-render/main.k +++ b/tests/test-render/main.k @@ -386,6 +386,194 @@ _items = [ ] } } + + # ========================================================================== + # Test 12: rustfs.enabled first copies credentials and waits to install. + # ========================================================================== + metav1alpha1.CompositionTest { + metadata.name = "rustfs-copies-credentials-before-installing" + spec = { + compositionPath = "apis/celldstacks/composition.yaml" + xrdPath = "apis/celldstacks/definition.yaml" + timeoutSeconds = 60 + validate = False + xr = stacksv1alpha1.CelldStack { + metadata.name = "celld" + metadata.namespace = "game" + spec = { + clusterName = "my-cluster" + rustfs = {enabled = True} + } + } + assertResources = [ + { + apiVersion = "kubernetes.m.crossplane.io/v1alpha1" + kind = "Object" + metadata.name = "celld-rustfs-creds" + spec.forProvider.manifest = { + apiVersion = "v1" + kind = "Secret" + metadata = {name = "celld-rustfs", namespace = "celld"} + data = { + S3_ENDPOINT = "aHR0cDovL2NlbGxkLXJ1c3Rmcy1zdmMuY2VsbGQuc3ZjLmNsdXN0ZXIubG9jYWw6OTAwMA==" + AWS_REGION = "dXMtZWFzdC0x" + } + } + } + { + apiVersion = "hops.ops.com.ai/v1alpha1" + kind = "CelldStack" + status.rustfs = { + endpoint = "http://celld-rustfs-svc.celld.svc.cluster.local:9000" + ready = False + } + } + ] + } + } + + # ========================================================================== + # Test 13: with RustFS up and buckets created, celld targets RustFS. + # ========================================================================== + metav1alpha1.CompositionTest { + metadata.name = "rustfs-ready-points-celld-at-rustfs" + spec = { + compositionPath = "apis/celldstacks/composition.yaml" + xrdPath = "apis/celldstacks/definition.yaml" + # Namespaced XR: observed MRs must carry the XR namespace to match. + observedResources = [ + { + apiVersion = "kubernetes.m.crossplane.io/v1alpha1" + kind = "Object" + metadata = { + name = "celld-namespace" + namespace = "default" + annotations = {"crossplane.io/composition-resource-name" = "namespace"} + } + status.conditions = [{type = "Ready", status = "True"}] + } + { + apiVersion = "kubernetes.m.crossplane.io/v1alpha1" + kind = "Object" + metadata = { + name = "celld-rustfs-creds" + namespace = "default" + uid = "0b6d3f0e-0000-4000-8000-000000000001" + annotations = {"crossplane.io/composition-resource-name" = "rustfs-creds-secret"} + } + status.conditions = [{type = "Ready", status = "True"}] + } + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata = { + name = "celld-rustfs" + namespace = "default" + annotations = {"crossplane.io/composition-resource-name" = "helm-release-rustfs"} + } + status = { + atProvider.revision = 1 + conditions = [{type = "Ready", status = "True"}] + } + } + { + apiVersion = "kubernetes.m.crossplane.io/v1alpha1" + kind = "Object" + metadata = { + name = "celld-rustfs-buckets-f68067af" + namespace = "default" + annotations = {"crossplane.io/composition-resource-name" = "rustfs-buckets-job"} + } + status = { + atProvider.manifest.status.succeeded = 1 + conditions = [{type = "Ready", status = "True"}] + } + } + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata = { + name = "celld" + namespace = "default" + annotations = {"crossplane.io/composition-resource-name" = "helm-release-celld"} + } + status = { + atProvider.revision = 1 + conditions = [{type = "Ready", status = "True"}] + } + } + ] + timeoutSeconds = 60 + validate = False + xr = stacksv1alpha1.CelldStack { + metadata.name = "celld" + metadata.namespace = "default" + spec = { + clusterName = "my-cluster" + rustfs = { + enabled = True + buckets = ["game-services"] + credentialsSecretRef.name = "celld-rustfs-credentials" + } + } + } + assertResources = [ + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata.name = "celld-rustfs" + spec.forProvider = { + chart = { + name = "rustfs" + repository = "https://charts.rustfs.com" + version = "1.0.3" + } + namespace = "celld" + values = { + fullnameOverride = "celld-rustfs" + mode.standalone.enabled = True + secret.existingSecret = "celld-rustfs" + ingress.enabled = False + } + } + } + { + apiVersion = "kubernetes.m.crossplane.io/v1alpha1" + kind = "Object" + metadata.name = "celld-rustfs-buckets-f68067af" + } + { + apiVersion = "helm.m.crossplane.io/v1beta1" + kind = "Release" + metadata.name = "celld" + spec.forProvider.values = { + bootstrapPlaceholder = False + celld = { + bucket = "s3://celld" + endpoint = "http://celld-rustfs-svc.celld.svc.cluster.local:9000" + region = "us-east-1" + } + credentials = {existingSecret = "celld-rustfs"} + azurite = {enabled = False} + } + } + { + apiVersion = "protection.crossplane.io/v1beta1" + kind = "Usage" + metadata.name = "celld-delete-helm-celld-before-rustfs" + } + { + apiVersion = "hops.ops.com.ai/v1alpha1" + kind = "CelldStack" + status.rustfs = { + credentialsSecret = "celld-rustfs" + buckets = ["celld", "game-services"] + ready = True + } + } + ] + } + } ] items = _items diff --git a/upbound.yaml b/upbound.yaml index 8417965..6e14c37 100644 --- a/upbound.yaml +++ b/upbound.yaml @@ -37,6 +37,7 @@ spec: Composes: - Namespace for the release - celld Helm release (StatefulSet, public Service, internal headless Service) + - Optional in-cluster RustFS S3-compatible storage with buckets (local) - Optional in-cluster Azurite (local) - Optional AWS S3 bucket + EKS Pod Identity (aws.enabled) repository: ghcr.io/hops-ops/celld-stack