kind: run the rate limiters on Redis by default, as deployments do

Tiltfile

@@ -513,10 +513,12 @@ k8s_resource(
 
 # API server.
 # Per-sandbox pods need the shim wired into containerd before the first create
-# lands.
+# lands, and the rate limiters' store should answer before the first create
+# reaches them (they would admit it open, not fail it).
 SPACE_API_DEPS = ["space-postgres", "label-kind-workers", "label-kind-infra",
                   "advertise-kind-worker-network-slots", "install-space-shim",
-                  "internal-rpc-tls", "space-sandbox-network-policy", "space-db-migrate"]
+                  "internal-rpc-tls", "space-sandbox-network-policy", "space-db-migrate",
+                  "space-redis"]
 # No port_forward: localhost:8080 is the cluster's host port mapping onto the
 # space-api-entry NodePort (kind-cluster.yaml.tmpl), which spreads connections
 # across the autoscaled replicas as a deployment's load balancer does.
@@ -644,9 +646,9 @@ k8s_resource(
 )
 
 # Redis: shared store for the two API rate limiters (cluster activation
-# admission + per-client concurrency). Public docker.io image pulled directly by
-# the kind nodes (no `kind load`). Idle unless SPACE_REDIS_URL points at it; the
-# rate-limiter smoke (scripts/tests/smoke-kind-rate-limiters.sh) wires it up.
+# admission + per-client concurrency), which space-api, space-admin-api and
+# space-cluster-controller reach through SPACE_REDIS_URL. Public docker.io image
+# pulled directly by the kind nodes (no `kind load`). space-api waits for it.
 k8s_yaml(helm_release("space", "infra/space/kind/charts/space-redis-local"))
 k8s_resource(
     "space-redis",

infra/space/aws/reference/README.md

@@ -158,8 +158,9 @@ retagged image. They run in the `space` namespace, and each writes a Secret a
 SPACE deployment can read. `space-postgres` holds `POSTGRES_USER`,
 `POSTGRES_PASSWORD`, `POSTGRES_DB` and `POSTGRES_CONNECTION_STRING`, the same
 Secret and keys SPACE's services read in the Kind harness. `space-valkey`
-holds `SPACE_REDIS_URL`, the variable SPACE's API reads its rate-limiter store
-from; Kind sets that variable directly rather than from a Secret.
+holds `SPACE_REDIS_URL`, the variable SPACE's API and cluster-controller read
+their rate-limiter store from; the Kind charts render that variable as a
+literal on the API, admin API and cluster-controller rather than from a Secret.
 Terraform generates both passwords and keeps them in `addons/` state.
 
 Valkey holds SPACE's rate-limiter counters, a guardrail rather than a source

infra/space/cluster/chart/templates/validate.yaml

@@ -120,6 +120,15 @@ rejects the Kind value itself.
 {{- if or (not $cc.snapshotEndpoint) (eq (toString $cc.snapshotEndpoint) "http://object-store:8333") -}}
 {{- $fail = append $fail "space-cluster-controller-local.snapshotEndpoint is required (the Kind object store is not installed)" -}}
 {{- end -}}
+{{- /* The rate limiters' Redis is optional; this installer provides none. */ -}}
+{{- $apiRedis := toString (($api.redis | default dict).url | default "") -}}
+{{- $ccRedis := toString (($cc.redis | default dict).url | default "") -}}
+{{- if or (contains "space-redis" $apiRedis) (contains "space-redis" $ccRedis) -}}
+{{- $fail = append $fail "redis.url must not name the Kind store space-redis (this installer does not install Redis): leave it empty or name the caller's store" -}}
+{{- end -}}
+{{- if ne $apiRedis $ccRedis -}}
+{{- $fail = append $fail "space-api-local.redis.url and space-cluster-controller-local.redis.url must be the same: both consume the cluster activation bucket" -}}
+{{- end -}}
 {{- end -}}
 
 {{- /* Node services: the object store. */ -}}

infra/space/cluster/chart/values.yaml

@@ -28,6 +28,18 @@ space-api-local:
   entry:
     type: ClusterIP
   ecwHosts: ""
+  # This installer provisions no Redis, so the rate limiters' store is off
+  # unless the caller names one (values.example.yaml): an unset URL renders no
+  # SPACE_REDIS_URL and both limiters fail open, as they did before Kind turned
+  # its store on. templates/validate.yaml refuses the Kind endpoint and a
+  # URL that differs from space-cluster-controller-local's.
+  redis:
+    url: ""
+
+space-cluster-controller-local:
+  # Shares the API's cluster-activation bucket, so it names the same store.
+  redis:
+    url: ""
 
 space-data-plane-local:
   entry:

infra/space/cluster/values.example.yaml

@@ -50,6 +50,11 @@ space-api-local:
   dataPlanePublicUrl: ""
   entry:
     type: ClusterIP
+  # Optional: the rate limiters' Redis, which this installer does not provide.
+  # Leave it unset to run without rate limiting; set the same URL under
+  # space-cluster-controller-local, which shares the activation bucket.
+  # redis:
+  #   url: rediss://redis.example.com:6379
 
 space-internal-api-local:
   clusterId: *clusterId
@@ -84,6 +89,8 @@ space-cluster-controller-local:
   snapshotRegion: us-east-1
   snapshotBucket: snapshots
   snapshotCredentialsSecretName: space-object-store
+  # redis:
+  #   url: rediss://redis.example.com:6379   # the same URL as space-api-local
 
 space-sandbox-network-policy:
   # Every private range sandboxes must not reach: pod, Service and node CIDRs

infra/space/kind/README.md

@@ -995,6 +995,7 @@ kind cluster: space-local (kind-cluster.yaml.tmpl — 6 nodes)
 ├── worker3 [pool=infra,    space.pplx.net/infra=true]
 │   ├── space-postgres        (StatefulSet; PostgreSQL 17 serving TLS only, non-superuser master user)
 │   ├── space-object-store    (versitygw S3 gateway + bucket-init Job)
+│   ├── space-redis           (Deployment; the rate limiters' store, which api, admin-api and cluster-controller reach through SPACE_REDIS_URL)
 │   ├── space-api             (Deployment, 2–3 replicas by HPA; SPACE_K8S_CONTROL_PLANE=true)
 │   ├── space-admin-api       (Deployment; the same binary on the dedicated admin surface)
 │   ├── space-internal-api    (Deployment; ClusterApi — node registration + template releases)
@@ -1402,6 +1403,8 @@ asks of the hosting deployment.
 | Server logs lock waits; the application roles' `lock_timeout` is off (`SPACE_DB_APP_LOCK_TIMEOUT` `0`) | `log_lock_waits = on`; `SPACE_DB_APP_LOCK_TIMEOUT` is unset, which leaves `lock_timeout` off the same way. Other server parameters are the image's defaults rather than a managed parameter group |
 | The migration Job fails fast: one attempt, no restarts, its deadline passed to the migrator | The same `space-db-migrate` Job shape (`backoffLimit: 0`, `restartPolicy: Never`, `activeDeadlineSeconds` = `SPACE_DB_MIGRATE_JOB_DEADLINE_SECONDS`), bootstrap-roles → migrate → bootstrap-roles. `bootstrap.sh` stops on a failed Job with its logs; rerunning it replaces the Job |
 | S3 for snapshots + template artifacts                     | In-cluster versitygw (real S3 protocol, sigv4, path-style)                    |
+| Rate limiters (cluster activation admission at the binary defaults, per-client concurrency in `observe`) on a Redis-compatible store reached over TLS (`rediss://`), its URL delivered through the `space-redis` Secret (`app_deploy.tf`) | the same limiters, modes and defaults on the in-cluster `space-redis` Deployment (one in-memory replica on the infra node), reached over plaintext `redis://space-redis:6379` rendered as a literal: the workspace's Redis clients trust only the webpki roots (`tls-rustls-webpki-roots` in `Cargo.toml`), so they would refuse a CA `internal-rpc-tls.sh` could issue, and the `#insecure` URL fragment would be an environment-only switch. `local.rate_limiters_enforce_and_fail_open` shrinks the caps and scales the store to zero for its fail-open half |
+| `SPACE_REDIS_URL` on api, admin-api, cluster-controller and internal-api | on api, admin-api and cluster-controller only: the limiters that read it live in `crates/libs/api` (api and admin-api build both, the cluster-controller's public-operation dispatcher builds the activation limiter), and nothing `space-internal-api-svc` links reads it |
 | `space-credmgr-authority` Deployment whose platform KEK comes from an external intermediate-key broker, with TLS from its own certificate (`credmgr_authority_tls.tf`; the direct KMS backend is gone, `credmgr_authority_kms.tf`) | two replicas on the same broker path against the in-repo broker fixture (`space-kind-key-broker`), which derives keys instead of protecting them; TLS from a self-signed certificate `internal-rpc-tls.sh` issues per cluster |
 | Internal RPC CA and listener leaves issued by Terraform and synced by External Secrets (`internal_rpc_tls.tf`) | the same Secrets issued per cluster by `scripts/internal-rpc-tls.sh`; the ingress activation leaf names `space-api` rather than `space-ingress-activation` |
 | volume-manager credentials via credmgr STS/IRSA           | static `SPACE_OBJECT_STORE_STATIC_*` creds (the documented local escape hatch) |

infra/space/kind/charts/space-api-local/templates/deployment.yaml

@@ -197,6 +197,11 @@ spec:
               value: {{ .Values.clusterActivationAdmission.enabled | quote }}
             - name: SPACE_API_CLIENT_CONCURRENCY_MODE
               value: {{ .Values.clientConcurrency.mode | quote }}
+            {{- if .Values.redis.url }}
+            # Both limiters' shared store; see `redis` in values.yaml.
+            - name: SPACE_REDIS_URL
+              value: {{ .Values.redis.url | quote }}
+            {{- end }}
             {{- if .Values.eppo.secretName }}
             - name: EPPO_SDK_KEY
               valueFrom:

infra/space/kind/charts/space-api-local/values.yaml

@@ -170,10 +170,34 @@ dataPlanePublicUrl: ""
 # default the binary ships. Writing a value here would be a copy of a Rust
 # constant with nothing linking the two, and would silently override it the day
 # it changes. The client-concurrency mode accepts `off` as the hard off.
+#
+# The one exception is the client-concurrency mode, which deployments pin to
+# `observe` rather than inherit: it is their configured value, not a copy of
+# the binary's default, and it stays `observe` if that default ever moves.
+# Observe needs no allowlist to classify traffic: an id outside
+# SPACE_API_CLIENT_CONCURRENCY_KNOWN_CLIENTS, or none, lands in the shared
+# `unspecified` bucket, which is metered like any other
+# (`ClientKey::classify` in crates/libs/api/src/client_concurrency), so the
+# allowlist stays at its empty default.
 clusterActivationAdmission:
   enabled: ""
 clientConcurrency:
-  mode: ""
+  mode: "observe"
+
+# The store both limiters keep their state in: the space-redis Deployment
+# (charts/space-redis-local). Each limiter reads SPACE_REDIS_URL when the
+# process builds its state, above the `--surface` branch
+# (`AppState::from_pools_with_lifecycle`), so the admin API reads it too:
+# `ClusterSandboxAdmissionConfig::from_env` and
+# `ClientConcurrencyConfig::from_env`. Without it both fail open as
+# `misconfigured`. Plaintext `redis://`, unlike a deployment's
+# `rediss://`: the clients trust only the webpki roots (the
+# `tls-rustls-webpki-roots` feature in the workspace Cargo.toml), so they
+# would refuse a per-cluster CA. Must equal `redis.url` in the
+# cluster-controller chart, whose copy of the activation limiter shares the
+# bucket.
+redis:
+  url: "redis://space-redis:6379"
 
 eppo:
   secretName: ""

infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml

@@ -180,6 +180,11 @@ spec:
             # `api.clusterActivationAdmission.enabled`.
             - name: SPACE_CLUSTER_ACTIVATION_ADMISSION_ENABLED
               value: {{ .Values.clusterActivationAdmission.enabled | quote }}
+            {{- if .Values.redis.url }}
+            # The limiter's shared store; see `redis` in values.yaml.
+            - name: SPACE_REDIS_URL
+              value: {{ .Values.redis.url | quote }}
+            {{- end }}
             {{- range $name, $reconciler := .Values.reconcilers }}
             {{- range $key, $value := (default (dict) $reconciler.env) }}
             {{- if ne $key "SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS" }}

infra/space/kind/charts/space-cluster-controller-local/values.yaml

@@ -18,6 +18,16 @@ clusterId: "kind-local"
 clusterActivationAdmission:
   enabled: ""
 
+# The activation limiter's store. The public-operation dispatcher
+# (`NativePublicOperationExecutor::new`) builds its limiter from
+# `ClusterSandboxAdmissionConfig::from_env`, which reads SPACE_REDIS_URL;
+# without it this copy admits every queued activation as `misconfigured`.
+# **Must equal `redis.url` in `charts/space-api-local/values.yaml`**: both
+# copies consume from one bucket key, so a second store would double the
+# cluster's rate. This process runs no per-client concurrency limiter.
+redis:
+  url: "redis://space-redis:6379"
+
 # Telemetry defaults come from the space-kind-telemetry library.
 telemetry: {}
 

infra/space/kind/charts/space-redis-local/templates/deployment.yaml

@@ -48,9 +48,12 @@ spec:
             - name: redis
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+          # Ready once the server answers PING, not merely once its port
+          # accepts connections: the limiters fail open on any store error, so
+          # a ready endpoint should be one that serves their commands.
           readinessProbe:
-            tcpSocket:
-              port: redis
+            exec:
+              command: ["redis-cli", "-p", "{{ .Values.service.port }}", "ping"]
             initialDelaySeconds: 2
             periodSeconds: 5
           livenessProbe:

infra/space/kind/charts/space-redis-local/values.yaml

@@ -1,7 +1,7 @@
 # Redis for Kind's cluster activation and per-client concurrency limiters.
-# Both use SPACE_REDIS_URL. This chart provides a local store for
-# scripts/tests/smoke-kind-rate-limiters.sh; without a configured URL the
-# limiters fail open and this pod idles.
+# space-api, space-admin-api and space-cluster-controller reach it through
+# SPACE_REDIS_URL (their charts' `redis.url`), as deployed services reach
+# theirs, so every Kind run exercises both limiters against a real store.
 #
 # Single-node, in-memory only (no RDB/AOF persistence): the limiter state is a
 # guardrail, not a source of truth, and a wiped store just refills the buckets.
@@ -14,7 +14,7 @@ image:
 datadogTeam: team-space
 
 service:
-  # DNS name the api-server reaches via SPACE_REDIS_URL=redis://space-redis:6379.
+  # DNS name in the API and cluster-controller charts' `redis.url`.
   name: space-redis
   port: 6379
 

infra/space/kind/scripts/bootstrap.sh

@@ -374,8 +374,9 @@ log "waiting for object-store Deployment to be ready"
 kubectl --context "$KCTX" -n "$NAMESPACE" rollout status deploy/space-object-store --timeout="${TIMEOUT}s"
 
 # Redis — shared store for the API rate limiters (cluster activation admission
-# + per-client concurrency). Idle unless SPACE_REDIS_URL points at it; the
-# rate-limiter smoke (scripts/tests/smoke-kind-rate-limiters.sh) wires that up.
+# + per-client concurrency), which space-api, space-admin-api and
+# space-cluster-controller reach through SPACE_REDIS_URL. Ready before any of
+# them starts, so the first create meets the limiters, not their fail-open arm.
 helm_apply space "$KIND_DIR/charts/space-redis-local"
 log "waiting for redis Deployment to be ready"
 kubectl --context "$KCTX" -n "$NAMESPACE" rollout status deploy/space-redis --timeout="${TIMEOUT}s"

scripts/tests/lib/bootstrap-kind-stack.sh

@@ -24,9 +24,8 @@
 # consuming the timeout; a Tilt bound to a deleted cluster is restarted.
 #
 # Public entry points: `bootstrap_kind_stack`, `kind_stack_prepare_cluster`, plus
-# `kind_stack_sweep_stale_node_tombstones`, `kind_stack_psql`, and
-# `kind_stack_ensure_redis` for suites that have to repeat a bring-up step in
-# their own setup or cleanup.
+# `kind_stack_sweep_stale_node_tombstones` and `kind_stack_psql` for suites that
+# have to repeat a bring-up step in their own setup or cleanup.
 #
 # Inputs (env vars, all optional with defaults set by the caller):
 #   API_URL           — health-check URL (e.g. http://localhost:8080)
@@ -664,37 +663,6 @@ kind_stack_psql() {
         psql -U "$KIND_STACK_PG_USER" -d "$KIND_STACK_PG_DB" -tA -v ON_ERROR_STOP=1 "$@"
 }
 
-# Redis, the store behind the two API rate limiters. Part of the harness
-# (`bootstrap.sh` / the Tiltfile), so this renders the chart only when it is
-# missing — which keeps the normal path from re-applying it while still letting a
-# suite run against a stack that predates the chart. It always scales to 1 and
-# waits, so it also recovers a prior run that died with Redis scaled to zero.
-#
-# Here rather than in each suite because it is bring-up, not assertion. Two
-# suites had the same twelve lines when it was extracted; one of them
-# (`smoke-kind-node-selector-extras.sh`) dropped its Redis setup later in the same
-# change, once the token bucket turned out not to affect either of its phases — so
-# `smoke-kind-rate-limiters.sh` is the only caller today.
-kind_stack_ensure_redis() {
-    local deploy="${1:-space-redis}"
-    # Declared here rather than in the bring-up loop above, because a suite
-    # running with SKIP_BOOTSTRAP=1 reaches this function and never that loop.
-    if ! command -v helm >/dev/null 2>&1; then
-        echo "ERROR: helm is required to ensure the Redis chart is applied" >&2
-        return 1
-    fi
-    # Targeting flags last, per the repository's kubectl convention. None of
-    # these is an `exec`, where the flags would have to precede `--`.
-    local target=(-n "${NAMESPACE:-space}" --context "kind-${KIND_CLUSTER}")
-    if ! kubectl get "deploy/$deploy" "${target[@]}" >/dev/null 2>&1; then
-        helm template space "$REPO_ROOT/infra/space/kind/charts/space-redis-local" \
-            --namespace "${NAMESPACE:-space}" \
-            | kubectl apply -f - "${target[@]}" >/dev/null
-    fi
-    kubectl scale "deploy/$deploy" --replicas=1 "${target[@]}" >/dev/null 2>&1 || true
-    kubectl rollout status "deploy/$deploy" --timeout=120s "${target[@]}" >/dev/null
-}
-
 # Wait for `space-api` to report `/readyz=200`, up to a caller-chosen bound.
 # 0 when it did, 1 on timeout — the caller decides whether that is fatal.
 #

space_smoke/__main__.py

@@ -13,7 +13,7 @@
 from space_smoke.registry import Scenario
 from space_smoke.runtime import SmokeConfigurationError, execute_scenario
 from space_smoke.scenarios import REGISTRY
-from space_smoke.services import MissingToolError, SmokeServices
+from space_smoke.services import MissingToolError, SmokeServices, repair_orphaned_kind_faults
 from space_smoke.sharding import select_shard
 from space_smoke.suite import (
     RESULT_PREFIX,
@@ -91,6 +91,13 @@ def _run(scenario_id: str) -> int:
     except SmokeConfigurationError as error:
         print(f"ERROR: {error}", file=sys.stderr)
         return 2
+    # Outside the scenario's budget: the repair may wait out two rollouts.
+    try:
+        if repair_orphaned_kind_faults(os.environ):
+            print("REPAIRED: a limiter fault an earlier run left recorded", flush=True)
+    except Exception as error:  # noqa: BLE001 - a fault left up makes any verdict untrustworthy
+        print(f"ERROR: {type(error).__name__}: {error}", file=sys.stderr)
+        return 2
     result = execute_scenario(
         title=scenario.title,
         body=lambda context: scenario.run(context, services),