diff --git a/.gitleaks.toml b/.gitleaks.toml index f382902b26..e154ec98ca 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -9,13 +9,6 @@ condition = "AND" paths = ['''^infra/space/kind/charts/space-api-local/values\.yaml$'''] regexes = ['''^-----BEGIN[ ]PRIVATE KEY-----\r?\n[ \t]*MC4CAQAwBQYDK2VwBCIEIJ1hsZ3v/VpguoRK9JLsLMREScVpezJpGXA7rAMcrn9g\r?\n[ \t]*-----END[ ]PRIVATE KEY-----$'''] -[[allowlists]] -description = "Negative TokenReview fixture with a literal fake signature" -targetRules = ["jwt"] -condition = "AND" -paths = ['''^scripts/k8s/projected-tokens/02-test\.sh$'''] -regexes = ['''^eyJhbGciOiJSUzI1NiIsImtpZCI6ImJvZ3VzIn0[.]eyJzdWIiOiJmYWtlIiwiYXVkIjpbInNwYWNlLW5vZGUtYXBpIl19[.]ZmFrZXNpZ25hdHVyZQ$'''] - # Diff scans read context lines too, so edits near these fixtures surface it. [[allowlists]] description = "Synthetic credential-reference UUID shared by credential test fixtures" diff --git a/crates/libs/common/svc/src/auth/k8s.rs b/crates/libs/common/svc/src/auth/k8s.rs index 19fe03a9de..2ede2734a2 100644 --- a/crates/libs/common/svc/src/auth/k8s.rs +++ b/crates/libs/common/svc/src/auth/k8s.rs @@ -21,7 +21,7 @@ use super::{ }; // The namespace is read from the pod's service account mount. The override is a -// test hook for in-process tests and kind, which have no such mount. +// test hook for in-process tests, which have no such mount. const SPACE_K8S_TOKEN_AUTH_NAMESPACE_ENV: &str = "SPACE_K8S_TOKEN_AUTH_NAMESPACE"; const SPACE_K8S_TOKEN_AUTH_REVIEW_RATE_PER_SEC_ENV: &str = "SPACE_K8S_TOKEN_AUTH_REVIEW_RATE_PER_SEC"; diff --git a/docker/Dockerfile.rust-build-kind b/docker/Dockerfile.rust-build-kind index 34646e56b0..163b6ee660 100644 --- a/docker/Dockerfile.rust-build-kind +++ b/docker/Dockerfile.rust-build-kind @@ -50,7 +50,7 @@ COPY infra/space/identity/node-identities.sh infra/space/identity/node-identitie # space-runtime-supervisor-svc is cooked here but built by # scripts/build-runtime-supervisor.sh below, as the production worker stage does. RUN cargo chef cook --locked --release --recipe-path recipe.json \ - --features "pplx-space-node-api-svc/otel,pplx-space-node-api-svc/datadog,pplx-space-node-api-svc/k8s,pplx-space-internal-api-svc/otel,pplx-space-internal-api-svc/datadog,pplx-space-internal-api-svc/k8s,pplx-space-node-ctl-svc/otel,pplx-space-node-ctl-svc/datadog,pplx-space-node-ctl-svc/kubernetes,pplx-space-node-mgr-svc/otel,pplx-space-node-mgr-svc/datadog,pplx-space-build-mgr-svc/otel,pplx-space-build-mgr-svc/datadog,pplx-space-runtime-supervisor-svc/otel,pplx-space-runtime-supervisor-svc/datadog,pplx-space-runtime-manager-svc/otel,pplx-space-runtime-manager-svc/datadog,pplx-space-sbxmgr-svc/otel,pplx-space-sbxmgr-svc/datadog,pplx-space-spaced-host-svc/otel,pplx-space-spaced-host-svc/datadog,pplx-space-net-gateway-svc/otel,pplx-space-net-gateway-svc/datadog,pplx-space-tmplmgr-svc/otel,pplx-space-tmplmgr-svc/datadog,pplx-space-volmgr-svc/otel,pplx-space-volmgr-svc/datadog,pplx-space-api-svc/otel,pplx-space-api-svc/datadog,pplx-space-api-svc/kubernetes,pplx-space-api-svc/federated-identity,pplx-space-data-plane-svc/otel,pplx-space-data-plane-svc/datadog,pplx-space-data-plane-svc/kubernetes,pplx-space-cluster-controller/otel,pplx-space-cluster-controller/datadog,pplx-space-spaced-guest-svc/test-hooks,pplx-space-node-mgr-svc/test-hooks,pplx-space-sbxmgr-svc/test-hooks,pplx-space-tmplmgr-svc/test-hooks,pplx-space-template-mgr/test-hooks,pplx-space-internal-api-server/test-hooks,pplx-space-credmgr/test-hooks,pplx-space-common-svc/test-hooks,pplx-space-spaced-guest-server/test-hooks,pplx-space-observability/test-hooks,pplx-space-cluster-controller-server/test-hooks,pplx-space-common-core/test-hooks,pplx-space-sbx-mgr/test-hooks,pplx-space-api/test-hooks,pplx-space-build-mgr-server/test-hooks,pplx-space-node-api-server/test-hooks,pplx-space-node-mgr/test-hooks" \ + --features "pplx-space-node-api-svc/otel,pplx-space-node-api-svc/datadog,pplx-space-node-api-svc/k8s,pplx-space-internal-api-svc/otel,pplx-space-internal-api-svc/datadog,pplx-space-internal-api-svc/k8s,pplx-space-node-ctl-svc/otel,pplx-space-node-ctl-svc/datadog,pplx-space-node-ctl-svc/kubernetes,pplx-space-node-mgr-svc/otel,pplx-space-node-mgr-svc/datadog,pplx-space-build-mgr-svc/otel,pplx-space-build-mgr-svc/datadog,pplx-space-build-mgr-svc/k8s,pplx-space-runtime-supervisor-svc/otel,pplx-space-runtime-supervisor-svc/datadog,pplx-space-runtime-manager-svc/otel,pplx-space-runtime-manager-svc/datadog,pplx-space-sbxmgr-svc/otel,pplx-space-sbxmgr-svc/datadog,pplx-space-spaced-host-svc/otel,pplx-space-spaced-host-svc/datadog,pplx-space-net-gateway-svc/otel,pplx-space-net-gateway-svc/datadog,pplx-space-tmplmgr-svc/otel,pplx-space-tmplmgr-svc/datadog,pplx-space-volmgr-svc/otel,pplx-space-volmgr-svc/datadog,pplx-space-api-svc/otel,pplx-space-api-svc/datadog,pplx-space-api-svc/kubernetes,pplx-space-api-svc/federated-identity,pplx-space-data-plane-svc/otel,pplx-space-data-plane-svc/datadog,pplx-space-data-plane-svc/kubernetes,pplx-space-cluster-controller/otel,pplx-space-cluster-controller/datadog,pplx-space-spaced-guest-svc/test-hooks,pplx-space-node-mgr-svc/test-hooks,pplx-space-sbxmgr-svc/test-hooks,pplx-space-tmplmgr-svc/test-hooks,pplx-space-template-mgr/test-hooks,pplx-space-internal-api-server/test-hooks,pplx-space-credmgr/test-hooks,pplx-space-common-svc/test-hooks,pplx-space-spaced-guest-server/test-hooks,pplx-space-observability/test-hooks,pplx-space-cluster-controller-server/test-hooks,pplx-space-common-core/test-hooks,pplx-space-sbx-mgr/test-hooks,pplx-space-api/test-hooks,pplx-space-build-mgr-server/test-hooks,pplx-space-node-api-server/test-hooks,pplx-space-node-mgr/test-hooks" \ --bin spaced \ --bin spaced-svc \ --bin space-buildmgr \ @@ -100,6 +100,7 @@ RUN cargo build --locked --release \ -F pplx-space-node-ctl-svc/kubernetes \ -F pplx-space-node-mgr-svc/otel,pplx-space-node-mgr-svc/datadog \ -F pplx-space-build-mgr-svc/otel,pplx-space-build-mgr-svc/datadog \ + -F pplx-space-build-mgr-svc/k8s \ -F pplx-space-runtime-manager-svc/otel,pplx-space-runtime-manager-svc/datadog \ -F pplx-space-sbxmgr-svc/otel,pplx-space-sbxmgr-svc/datadog \ -F pplx-space-spaced-host-svc/otel,pplx-space-spaced-host-svc/datadog \ diff --git a/docs/configuration.md b/docs/configuration.md index 0b63c1c749..8b27e65a57 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -334,11 +334,52 @@ authentication is enabled, static fallback has no transfer authority. Missing authentication context is denied. Local root/operator exports remain available through template-manager's Unix socket. +A peer transfer has two callers, and each needs the grant and admission by +`SPACE_NODE_API_AUTH_ALLOWED_SERVICE_ACCOUNTS` on the node it calls. The caller +of `PullTemplate` on the target node starts it: the admin API's node +template-pull route, which only the direct substrate serves, or an operator +calling node-api directly. The target's node-api then calls +`NodeExportTemplate` on the source node with its own `SPACE_NODE_API_TOKEN_PATH` +token. List the account that starts pulls and the node-api account of every +role that can be a target. An empty list disables peer +transfers and node-api logs a startup warning. A deployment without peer +transfers may leave it empty. + Exact-digest export can use the local cache offline. An omitted digest resolves the requested catalog tag (default `latest`) and exports only that digest; a missing tag, unavailable catalog, or uncached resolved digest fails without falling back to a newer local build. +### Policy authorization + +Before enabling Kubernetes authentication or upgrading an authenticated worker +or builder, set `SPACE_NODE_API_AUTH_TRUSTED_POLICY_SERVICE_ACCOUNTS` to the +service accounts that own policy intent. It must be nonempty on both roles or +node-api refuses to start. These accounts must also be admitted by the normal +and internal surface allowlists for the RPCs they call. Policy trust is +independent of the `SpacedWaitReady` trust set. Policy-bearing pool adoption +uses the internal surface, so its caller needs an internal-audience token. + +Shared static-token fallback cannot mutate policy when Kubernetes +authentication is enabled; static-only deployments retain that authority. +Policy-bearing cold creation, internal pool adoption, and worker and builder +template updates require a trusted caller. Cold creation and pool adoption also +require policy authority for a nonzero sandbox-policy generation, even without +an override. Requests with no overrides and generation zero remain available to +otherwise admitted callers and inherit the template's policy. Guest-bridge +accounts already have separate method restrictions. + +`Local` policy is node-administrator-owned, including normal standalone use and +operator overrides on cluster-managed nodes. It requires a root or Operator Unix +socket peer. Forwarders such as node-api and node-ctl must run under their +dedicated non-root identities: UID 0 is classified as Root before the +service-account lookup. + +Forwarded imports carrying `Local` network or auto-lifecycle policy are +rejected, and retrying does not help. To retain `Local` policy, an operator must +review its ownership and perform the import through an authorized local peer; +forwarding never relabels it. + ## Database | Variable | Notes | diff --git a/infra/space/cluster/chart/values-builder.yaml b/infra/space/cluster/chart/values-builder.yaml index 276acfda66..3cc8a903b4 100644 --- a/infra/space/cluster/chart/values-builder.yaml +++ b/infra/space/cluster/chart/values-builder.yaml @@ -46,6 +46,30 @@ space-node-api-local: # kind/charts/space-node-api-local/values-builder.yaml serviceName: space-builder-node-api buildManagerEndpoint: https://space-builder-build-manager:9900 + # The builder's TokenReview allowlists, which replace the worker's + # (values.yaml), as the same file sets them; without the template-build + # grant every BuildTemplate call would be refused. + auth: + SPACE_NODE_API_AUTH_ALLOWED_SERVICE_ACCOUNTS: + - space-api + - space-cluster-controller + - space-node-api + - space-builder-node-api + SPACE_NODE_API_INTERNAL_AUTH_ALLOWED_SERVICE_ACCOUNTS: [] + SPACE_NODE_API_AUTH_TRUSTED_WAIT_READY_SERVICE_ACCOUNTS: [] + SPACE_NODE_API_AUTH_TRUSTED_POLICY_SERVICE_ACCOUNTS: + - space-api + SPACE_NODE_API_AUTH_GUEST_BRIDGE_SERVICE_ACCOUNTS: [] + SPACE_NODE_API_AUTH_TEMPLATE_TRANSFER_SERVICE_ACCOUNTS: + - space-api + - space-node-api + - space-builder-node-api + SPACE_NODE_API_AUTH_TEMPLATE_BUILD_SERVICE_ACCOUNTS: + - space-api + SPACE_BUILD_AUTH_SYSTEM_SERVICE_ACCOUNTS: + - space-api + SPACE_BUILD_AUTH_TENANT_SERVICE_ACCOUNTS: + - space-api space-build-manager-local: node: role: builder diff --git a/infra/space/kind/README.md b/infra/space/kind/README.md index a01ac4f008..9bcd2faddf 100644 --- a/infra/space/kind/README.md +++ b/infra/space/kind/README.md @@ -91,16 +91,12 @@ releases, or tags; this is unconditional and has no compatibility switch. Each node service runs under its own DaemonSet's ServiceAccount and projects its own ClusterApi token, and only build-manager's builder account (`space-builder-build-manager`) carries operator and tenant publisher authority -(`clusterApiAuth` in the internal-api chart). The Kind node-api and build-manager -listeners still accept the shared static token as trusted build authority: a -worker can submit an operator build through that route even though ClusterApi -rejects direct worker publication. - -The direct-registration tests prove ClusterApi's publisher checks, not end-to-end -worker isolation. That also requires Kubernetes authentication on the builder -listeners, with each system-build caller explicitly reviewed. Kind's static build -credentials are a local development limitation and must not be used as a secure -deployment model. +(`clusterApiAuth` in the internal-api chart). The build path before publication +is reviewed the same way: the builder node-api admits `BuildTemplate` only from +the api's account, which holds both system and tenant build authority, and +build-manager admits only the builder node-api's account. Neither listener +accepts the shared static token, so a worker cannot submit a build through +either (see [Service identities](#service-identities)). ## CI smoke partitions @@ -942,6 +938,19 @@ inside a Kind node before starting a large build. Cold Rust builds can consume substantial space; the incremental artifact build keeps its `target/` under `$HOME` instead. +### Kind apiserver timeouts during parallel builds + +Kind runs the whole control plane, apiserver and etcd, in one container that +shares the host's CPU, memory and disk with the image builds and with Tilt's +loads of every image into every node. When many of those run at once, etcd's +write latency can spike until the apiserver stops answering: Tilt or `tilt ci` +then fails on a Kind API liveness timeout, or the apiserver returns TLS +handshake timeouts. Start Tilt with `SPACE_TILT_MAX_PARALLEL_UPDATES=1` (or `2`) +in its environment to bound its parallel updates; the `Tiltfile` reads it into +`update_settings(max_parallel_updates)`, defaults to 8, and `tilt up` has no flag +for it. The bring-up is slower but keeps the control plane responsive. If the +host is under-provisioned, give it more CPU and memory. + ## Seed template The seed is built locally by the **production seed builder** @@ -1159,8 +1168,8 @@ reflector use) and `app.kubernetes.io/part-of=space-node`. | template-manager | space-tmplmgr + SPACE_TEMPLATE_MANAGER_GROUPS | non-root, no capabilities, read-only rootfs, hostNetwork; `/space/cache` HostToContainer | btrfs work delegated to node-manager (`btrfs_via_nodemgr.rs`); host network because its template-release reads are scoped by source address (below) | | sandbox-manager | space-sbxmgr + SPACE_SANDBOX_MANAGER_GROUPS | non-root, no capabilities, read-only rootfs, hostPID, hostNetwork; `/space/cache` HostToContainer, `/space/etc` read-only | gateway callers are authorized by their host pid, and Firecracker pids go to the node's shim (`service/gateway_identity.rs`, `network_runtime.rs`); host network for its node-bound ClusterApi RPCs | | node-ctl | space-nodectl + SPACE_NODECTL_GROUPS | non-root, no capabilities, read-only rootfs, hostNetwork; its own ServiceAccount with Node RBAC, fenced by `node-fence.yaml` | the Kubernetes node-local controller; host network for its node-bound ClusterApi RPCs | -| node-api | space-nodeapi + worker/builder groups | non-root, no capabilities, read-only rootfs, hostNetwork, port `grpc` 9800 | the API dials `host_ip:9800` and reads the `grpc` port; on builders it reaches build-manager through the node-local `space-builder-build-manager` Service and sleeps 5 s in preStop (below) | -| build-manager | root | privileged, hostPID, pod network behind the node-local `space-builder-build-manager` Service (`internalTrafficPolicy: Local`); `/var/run/netns` and `/space/cache` HostToContainer, the node's containerd | loop and bind mounts for OCI assembly, `nsenter --net` into build VMs through netns handles node-manager mounts after it starts, `ctr` against the node's containerd (`crates/libs/build/mgr/src`); host PID to watch the build VM's host Firecracker pid exit (`vm.rs`); no host network: SSH into build VMs runs inside `nsenter --net` | +| node-api | space-nodeapi + worker/builder groups | non-root, no capabilities, read-only rootfs, hostNetwork, port `grpc` 9800; its ServiceAccount's API credentials (TokenReview only) and projected node-api and, on builders, build-manager tokens | reviews callers' tokens; the API dials `host_ip:9800` and reads the `grpc` port; on builders it reaches build-manager through the node-local `space-builder-build-manager` Service and sleeps 5 s in preStop (below) | +| build-manager | root | privileged, hostPID, pod network behind the node-local `space-builder-build-manager` Service (`internalTrafficPolicy: Local`); `/var/run/netns` and `/space/cache` HostToContainer, the node's containerd; its ServiceAccount's API credentials (TokenReview only) | reviews the builder node-api's token; loop and bind mounts for OCI assembly, `nsenter --net` into build VMs through netns handles node-manager mounts after it starts, `ctr` against the node's containerd (`crates/libs/build/mgr/src`); host PID to watch the build VM's host Firecracker pid exit (`vm.rs`); no host network: SSH into build VMs runs inside `nsenter --net` | | space-node-debug (Kind only) | root | privileged; `/space/run`, `/space/cache` HostToContainer, `/space/bin`, `/space/vendor`, `/run/containerd`, `/var/run/netns` HostToContainer read-only | the operator shell; see below | internal-api binds every node-bound ClusterApi RPC — `PublishNodeState`, the @@ -1355,6 +1364,77 @@ broker and the authority and every replica serves every org, and so Kind's keys protect nothing. The authority runs two replicas and rolls one at a time. Agents started before the authority exit and are restarted until it answers. +**node-api and build-manager authentication.** As deployed services do, both +node-api roles and build-manager authenticate every caller by TokenReview of +its projected ServiceAccount token, with the static fallback off +(`SPACE_K8S_TOKEN_AUTH_ENABLED`, `SPACE_K8S_TOKEN_AUTH_STATIC_FALLBACK_ENABLED=false`). +The shared `SPACE_INTERNAL_TOKEN` and `SPACE_INTERNAL_CONTROL_TOKEN` are still +mounted, because the binaries require them at startup, but admit nobody. The +audiences are the code's: `space-node-api` for WorkerApi, BuilderApi and +NodeInfo, `space-node-api-internal` for WorkerApiInternal, and +`space-build-manager`. Each surface admits the accounts the code shows calling +it, set in `charts/space-node-api-local/values.yaml` (workers), +`values-builder.yaml` (builders) and `charts/space-build-manager-local/values.yaml`. +Nothing in Kind starts a peer template transfer by itself: the admin +template-pull route serves only the direct substrate and answers 501 here, as on +any Kubernetes deployment. An operator starts one with `PullTemplate` on the +target node-api and a `space-api` token, and the transfer list admits that call +and the target node-api's export request to the source: + +| Surface | Admits | +|---|---| +| worker WorkerApi, NodeInfo | `space-api` (shared and admin api), `space-cluster-controller`, `space-data-plane` (guest bridge, guest calls only), `space-internal-api`, both node-api accounts (template transfers) | +| worker WorkerApiInternal | `space-api`, `space-cluster-controller` | +| worker lifecycle-probe `SpacedWaitReady`, caller-supplied policy, template and pool mutation | `space-api`, `space-cluster-controller` | +| builder BuilderApi, NodeInfo | `space-api`, `space-cluster-controller`, both node-api accounts | +| builder `BuildTemplate` (system and tenant), policy, template deletion | `space-api` | +| peer template transfers (`PullTemplate`, `NodeExportTemplate`), both roles | `space-api` (starts a pull), both node-api accounts (the target asks the source for the export) | +| build-manager (system and tenant builds) | `space-builder-node-api` | + +Every caller mounts its tokens under `/var/run/space-auth` and names them with +`SPACE_NODE_API_TOKEN_PATH`, `SPACE_NODE_API_INTERNAL_TOKEN_PATH` (api and +cluster-controller) or `SPACE_BUILD_MANAGER_TOKEN_PATH` (builder node-api). An +explicit path makes a missing token an error instead of a fall back to the +static token. The tokens live 600 s, the builder's build-manager token 7200 s, +and node services' ClusterApi and credential-manager tokens 600 s, as deployed; +the kubelet refreshes each at 80% of its lifetime and the clients reread the +file. node-api and build-manager mount their own ServiceAccount's API +credentials, which may only create TokenReviews, and read the namespace their +allowlists name from that mount, as a release binary does: no chart sets the +`SPACE_K8S_TOKEN_AUTH_NAMESPACE` test hook. node-ctl's own authentication is +unchanged. + +The smoke harness and the `space-node-debug` CLIs call neither surface: the +harness drives the public API, and the shell's CLIs use node-local Unix +sockets. An operator who calls build-manager directly with `space-buildmgr` +does what they would on a deployed builder: mint a short-lived token for an +admitted account with the build-manager audience, and give the CLI that token +and the cluster's internal RPC CA. A job exists only in the node-local SQLite +store of the build-manager that ran it, and the CLI's +`space-builder-build-manager` Service is `internalTrafficPolicy: Local`, so ask +from each builder's shell until one knows the job; the other answers +`NotFound`. The job ID is the `job_id` the admin template build API returns: + +```bash +k() { kubectl --context kind-space-local -n space "$@"; } +job= +for builder in $(kubectl --context kind-space-local get nodes -l space.pplx.net/node-pool=builder \ + -o jsonpath='{.items[*].metadata.name}'); do + shell=$(k get pods -l app.kubernetes.io/name=space-node-debug \ + --field-selector spec.nodeName="$builder" -o jsonpath='{.items[0].metadata.name}') + k create token space-builder-node-api --audience space-build-manager --duration 10m \ + | k exec -i "$shell" -c shell -- sh -c 'cat >/tmp/build-manager.token' + k get secret space-internal-rpc-ca -o jsonpath='{.data.ca\.crt}' | base64 -d \ + | k exec -i "$shell" -c shell -- sh -c 'cat >/tmp/rpc-ca.crt' + k exec "$shell" -c shell -- env SPACE_BUILD_MANAGER_TOKEN_PATH=/tmp/build-manager.token \ + SPACE_INTERNAL_RPC_TLS_CA=/tmp/rpc-ca.crt \ + SPACE_BUILD_MANAGER_ENDPOINT=https://space-builder-build-manager:9900 \ + space-buildmgr status --job-id "$job" && echo "ran on $builder" && break +done +``` + +The public template build API needs none of this and is the supported path. + **Internal RPC TLS.** Kind runs ClusterApi, NodeApi, BuildManager, IngressActivationApi and the credential-manager authority over authenticated TLS, as deployed services do @@ -1381,8 +1461,9 @@ node-local `space-builder-build-manager` Service. The CA's key stays in the Kind-only `space-internal-rpc-ca-signing` Secret, which no workload mounts. The script restarts a listener whose identity it reissued, and every workload if the CA itself changed, because neither reloads them while running. -TLS here is server-authenticated: bearer tokens still authorize callers, and -internal-api's node-address check is unchanged. The api's ingress activation +TLS here is server-authenticated: projected ServiceAccount tokens, reviewed by +each listener, authorize callers (see node-api and build-manager authentication +above), and internal-api's node-address check is unchanged. The api's ingress activation listener admits only internal-api's ServiceAccount, by TokenReview of its projected `space-cluster-api` token with no static fallback, as [Ingress activation delegation](../../../docs/internal-rpc-tls.md#ingress-activation-delegation) @@ -1397,7 +1478,7 @@ asks of the hosting deployment. | `/space/cache` is btrfs on instance-store NVMe, reformatted on every boot | btrfs on a sparse loopback image at `/var/lib/space-cache.img` on the kind node (40 GiB, same code paths, less I/O headroom), mounted `nosuid,noatime` on the node's `/space/cache` by `space-cache.service`, which reformats it on every start of the node container, so like the NVMe it outlives pod restarts but not node restarts | | The node bootstrap installs a checksum-verified `containerd-shim-space-v2` before containerd starts, and registers the `space` runtime under `io.containerd.cri.v1.runtime` with a `CniConfDir` option | the node image registers the same runtime with the same options (in kind's version 2 config format, which containerd migrates to that plugin), so containerd loads it on its first start; the binary arrives after the cluster exists, copied onto each running sandbox worker by `install-shim.sh`, which checks its SHA-256 on the node. Installing it at boot would need the shim built and staged on a host path before `kind create` in every bring-up path | | Every node group's bootstrap sets kubelet `eventRecordQPS: 0` and `--fail-swap-on=false` | the same settings on sandbox workers and builders, baked into the node image as a kubelet `--config-dir` drop-in (`node/kubelet/50-space.conf`); the infra node runs the unmodified base image, so its kubelet keeps kind's event rate limit (`failSwapOn` is already `false` there). No sandbox pod runs on it, so none of their events pass through that limit | -| Release binaries read no test hooks | every binary is built with `test-hooks`, and the charts set these hooks on every run: sandbox-manager's on-node snapshot reaper at target fraction 0, interval 15 s, grace 60 s and idle 120 s, against release values of 0.40, 300 s, 1800 s and 86400 s (`charts/space-sandbox-manager-local/values.yaml`, `snapshot_gc.rs`); template-manager's cache GC at interval 15 s and grace 60 s, against 300 s and 1800 s, and its prefetch latest-N at 1, against 5 (`charts/space-template-manager-local/values.yaml`, `eviction.rs`, `prefetch.rs`); the disk-admission reservation below; and api's and internal-api's TokenReview namespace, set to the release namespace that release builds read from the service-account mount. Scenarios shorten further hooks, such as the snapshot-retention sweeper's minimum age and the activation-admission burst, for their own run and restore them. The public-operation dispatcher's tick, claim limit and claim staleness stay at their release values | +| Release binaries read no test hooks | every binary is built with `test-hooks`, and the charts set these hooks on every run: sandbox-manager's on-node snapshot reaper at target fraction 0, interval 15 s, grace 60 s and idle 120 s, against release values of 0.40, 300 s, 1800 s and 86400 s (`charts/space-sandbox-manager-local/values.yaml`, `snapshot_gc.rs`); template-manager's cache GC at interval 15 s and grace 60 s, against 300 s and 1800 s, and its prefetch latest-N at 1, against 5 (`charts/space-template-manager-local/values.yaml`, `eviction.rs`, `prefetch.rs`); and the disk-admission reservation below. Scenarios shorten further hooks, such as the snapshot-retention sweeper's minimum age and the activation-admission burst, for their own run and restore them. The public-operation dispatcher's tick, claim limit and claim staleness stay at their release values | | 8 GiB disk-admission reservation per cache allocation (release default) | 1 GiB through the `SPACE_NODE_DISK_ADMISSION_ALLOCATION_RESERVATION_BYTES` test hook, with the 1 GiB `MIN_AVAILABLE_BYTES` floor, so allocations fit the 40 GiB cache (`charts/space-node-service/values.yaml`) | | Postgres on RDS, secrets via ExternalSecrets / AWS Secrets Manager | In-cluster `space-postgres` StatefulSet, one pod on a 1 GiB volume, static Kubernetes Secrets | | PostgreSQL 17 | The same major version (`postgres:17`; CI takes it from the image mirror at a pinned digest) | @@ -1420,6 +1501,7 @@ asks of the hosting deployment. | ALB ingress in front of api / data-plane / ui | NodePort entry Services behind the cluster's host port mappings for api and data-plane (kube-proxy balances per connection); `kubectl port-forward` for the single-replica ui | | The runtime manager installs releases and kernels, which node-manager publishes into the root-owned store `/space/vendor/artifacts` | `/space/vendor` is a read-only host bind, so the runtime manager runs `--list-only`: listing and cache checks work, installs are off | | Operators run node-local CLIs from the node host's shell | the same CLIs run in the Kind-only `space-node-debug` root shell DaemonSet | +| Release binaries, built without `test-hooks` | binaries built with `test-hooks` (`docker/Dockerfile.rust-build-kind`), so smokes can shrink timers and flip scenario switches; no chart sets a hook for authentication, and the TokenReview listeners read their namespace from the ServiceAccount mount as release binaries do | | Build sizing, concurrency, and multipart sizes are configurable | 20 GiB default build disk, one concurrent build, and smaller multipart parts fit the 40 GiB local btrfs cache | `space-seed-publish` remains the bootstrap for `seed:latest`; it is not a diff --git a/infra/space/kind/charts/space-api-local/templates/deployment.yaml b/infra/space/kind/charts/space-api-local/templates/deployment.yaml index 79a2af3f4b..5b9a32b474 100644 --- a/infra/space/kind/charts/space-api-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-api-local/templates/deployment.yaml @@ -98,6 +98,15 @@ spec: value: space-node-api - name: SPACE_CLUSTER_API_TOKEN_PATH value: "/var/run/space-publication/cluster-api.token" + # The substrate's NodeApi credentials (SpaceBearerToken, + # crates/libs/common/svc/src/bearer_token.rs): one projected + # token per node-api surface, which node-api admits by TokenReview + # (space-node-api-local's auth allowlists). An explicit path makes a + # missing file an error rather than a fall back to the static token. + - name: SPACE_NODE_API_TOKEN_PATH + value: /var/run/space-auth/node-api.token + - name: SPACE_NODE_API_INTERNAL_TOKEN_PATH + value: /var/run/space-auth/node-api-internal.token # Datadog tracing: DD_AGENT_HOST routes via # status.hostIP to the node-local Agent selected by the backend # helper. OTLP comparisons require that receiver to be running. @@ -248,11 +257,11 @@ spec: # activation delegation"): TokenReview with audience # space-cluster-api, no static fallback, and only the route # resolvers that have already authorized the request. No other - # api surface reads these settings. + # api surface reads these settings. The allowlist's namespace is + # the pod's own, read from its ServiceAccount mount + # (auth_namespace, crates/libs/common/svc/src/auth/k8s.rs). - name: SPACE_K8S_TOKEN_AUTH_ENABLED value: "true" - - name: SPACE_K8S_TOKEN_AUTH_NAMESPACE - value: {{ .Release.Namespace | quote }} - name: SPACE_K8S_TOKEN_AUTH_STATIC_FALLBACK_ENABLED value: "false" - name: SPACE_CLUSTER_API_AUTH_ALLOWED_SERVICE_ACCOUNTS @@ -298,6 +307,9 @@ spec: - name: publication-token mountPath: /var/run/space-publication readOnly: true + - name: node-api-token + mountPath: /var/run/space-auth + readOnly: true - name: route-signing mountPath: /var/run/space-route-signing readOnly: true @@ -323,6 +335,18 @@ spec: path: cluster-api.token audience: space-cluster-api expirationSeconds: 3600 + - name: node-api-token + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: node-api.token + audience: space-node-api + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} + - serviceAccountToken: + path: node-api-internal.token + audience: space-node-api-internal + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} - name: route-signing secret: secretName: {{ .Values.spaceApiRouteToken.ed25519.signingSecretName }} diff --git a/infra/space/kind/charts/space-api-local/values.yaml b/infra/space/kind/charts/space-api-local/values.yaml index 41e7bb442e..28a9a01415 100644 --- a/infra/space/kind/charts/space-api-local/values.yaml +++ b/infra/space/kind/charts/space-api-local/values.yaml @@ -208,6 +208,10 @@ eppo: serviceAccount: name: space-api +# Lifetime of the projected node-api tokens, as deployed; the kubelet +# refreshes them at 80% of that. +nodeApiTokenExpirationSeconds: 600 + # ServiceAccounts the ingress activation listener admits: internal-api, the # only route resolver that delegates cold wakes (RemoteIngressActivation). ingressActivation: diff --git a/infra/space/kind/charts/space-build-manager-local/templates/daemonset.yaml b/infra/space/kind/charts/space-build-manager-local/templates/daemonset.yaml index 0938696843..ebec27cd10 100644 --- a/infra/space/kind/charts/space-build-manager-local/templates/daemonset.yaml +++ b/infra/space/kind/charts/space-build-manager-local/templates/daemonset.yaml @@ -21,6 +21,10 @@ {{- $db := "/space/run/build-manager/build-manager.db" -}} {{- include "space-node-service.serviceAccount" $ctx }} --- +# build-manager reviews its caller's projected token (BuildTokenReviewer, +# crates/libs/common/svc/src/auth/build/k8s.rs). +{{ include "space-node-service.tokenReviewRbac" $ctx }} +--- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -51,8 +55,13 @@ spec: spec: {{- include "space-node-service.nodeSelector" $ctx | nindent 6 }} serviceAccountName: {{ include "space-node-service.name" $ctx }} - # The ClusterApi token is projected below; no Kubernetes API access. - automountServiceAccountToken: false + # TokenReview needs the pod's own API credentials (the in-cluster + # config of traced_kube_client_from_env, + # crates/libs/common/svc/src/kube_client.rs), and the allowlists name + # accounts in the namespace read from the same mount (auth_namespace, + # crates/libs/common/svc/src/auth/k8s.rs). The account may only + # create TokenReviews (RBAC above). + automountServiceAccountToken: true # Pod network: build-manager needs nothing from the node's network # namespace. SSH into build VMs runs inside `nsenter --net` on the VM's # namespace handle (crates/libs/build/mgr/src/ssh.rs); its other peers @@ -111,7 +120,24 @@ spec: # The BuildManager listener's TLS identity # (space-builder-build-manager, docs/internal-rpc-tls.md). {{- include "space-node-service.env.rpcIdentity" . | nindent 12 }} + # Still required at startup (run, crates/libs/build/mgr/server/src/run.rs), + # but with Kubernetes auth on it admits no caller (BuildAuthenticator, + # auth/build.rs). {{- include "space-node-service.env.internalToken" . | nindent 12 }} + # TokenReview of the caller's projected token, audience + # space-build-manager, with no static fallback, as deployed + # (BuildAuthLayer::from_env, crates/libs/common/svc/src/auth/build.rs; + # BuildSurface::Manager, auth/build/k8s.rs). + - name: SPACE_K8S_TOKEN_AUTH_ENABLED + value: "true" + - name: SPACE_K8S_TOKEN_AUTH_STATIC_FALLBACK_ENABLED + value: "false" + {{- range $name, $accounts := .Values.auth }} + {{- if $accounts }} + - name: {{ $name }} + value: {{ join "," $accounts | quote }} + {{- end }} + {{- end }} {{- include "space-node-service.env.diskAdmission" . | nindent 12 }} {{- include "space-node-service.env.objectStoreCredentials" . | nindent 12 }} - name: SPACE_OBJECT_STORE_BUCKET diff --git a/infra/space/kind/charts/space-build-manager-local/values.yaml b/infra/space/kind/charts/space-build-manager-local/values.yaml index 29cd81e7f9..91043f3d9b 100644 --- a/infra/space/kind/charts/space-build-manager-local/values.yaml +++ b/infra/space/kind/charts/space-build-manager-local/values.yaml @@ -21,6 +21,22 @@ listen: 0.0.0.0:9900 probeAddr: 127.0.0.1:9900 # The node-local Service, named as build-manager's TLS leaf. serviceName: space-builder-build-manager +# The ServiceAccounts build-manager admits, rendered as the env vars of the +# same names (BuildTokenReviewer::from_env, +# crates/libs/common/svc/src/auth/build/k8s.rs). Only the builder +# node-api dials it (build_mgr_lazy, +# crates/libs/node/api/server/src/clients.rs), and it +# forwards tenant builds (with an org) and system builds alike, so it needs +# both authorities StartBuild checks (authorize_template_build, +# crates/libs/build/mgr/server/src/lib.rs). +auth: + SPACE_BUILD_MANAGER_AUTH_ALLOWED_SERVICE_ACCOUNTS: + - space-builder-node-api + SPACE_BUILD_AUTH_SYSTEM_SERVICE_ACCOUNTS: + - space-builder-node-api + SPACE_BUILD_AUTH_TENANT_SERVICE_ACCOUNTS: + - space-builder-node-api + # Kind sizing: one build at a time fits the 40 GiB local btrfs cache with the # 20 GiB default build disk. maxConcurrentBuilds: 1 diff --git a/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml b/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml index 80ec51ca72..f9b227a2fb 100644 --- a/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml @@ -55,6 +55,15 @@ spec: value: space-node-api - name: SPACE_CLUSTER_API_TOKEN_PATH value: "/var/run/space-publication/cluster-api.token" + # NodeApi credentials for the api substrate the operation + # dispatcher and warm-pool probe run, and the trigger fan-out + # (SpaceBearerToken, crates/libs/common/svc/src/bearer_token.rs): + # one projected token per node-api surface, admitted by TokenReview + # (space-node-api-local's auth allowlists). + - name: SPACE_NODE_API_TOKEN_PATH + value: /var/run/space-auth/node-api.token + - name: SPACE_NODE_API_INTERNAL_TOKEN_PATH + value: /var/run/space-auth/node-api-internal.token # Datadog tracing — see datadog-agent-local chart. DD_AGENT_HOST # routes to the node-local agent; absent the agent the tracer # drops spans silently. @@ -213,6 +222,9 @@ spec: - name: publication-token mountPath: /var/run/space-publication readOnly: true + - name: node-api-token + mountPath: /var/run/space-auth + readOnly: true - name: route-signing mountPath: /var/run/space-route-signing readOnly: true @@ -232,6 +244,18 @@ spec: path: cluster-api.token audience: space-cluster-api expirationSeconds: 3600 + - name: node-api-token + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: node-api.token + audience: space-node-api + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} + - serviceAccountToken: + path: node-api-internal.token + audience: space-node-api-internal + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} - name: route-signing secret: secretName: {{ .Values.spaceApiRouteToken.ed25519.signingSecretName }} diff --git a/infra/space/kind/charts/space-cluster-controller-local/values.yaml b/infra/space/kind/charts/space-cluster-controller-local/values.yaml index bc11fbd05c..12b6907147 100644 --- a/infra/space/kind/charts/space-cluster-controller-local/values.yaml +++ b/infra/space/kind/charts/space-cluster-controller-local/values.yaml @@ -50,6 +50,10 @@ nodeSelector: serviceAccount: name: space-cluster-controller +# Lifetime of the projected node-api tokens, as deployed; the kubelet +# refreshes them at 80% of that. +nodeApiTokenExpirationSeconds: 600 + # Same KubernetesSubstrate env block as the api-server. The cluster-controller # requires SPACE_K8S_CONTROL_PLANE=true to even start, and shares the same # substrate plumbing for orphan-sweep + warm-pool reconcilers. diff --git a/infra/space/kind/charts/space-data-plane-local/templates/deployment.yaml b/infra/space/kind/charts/space-data-plane-local/templates/deployment.yaml index 15c82c09ed..29b7fad51e 100644 --- a/infra/space/kind/charts/space-data-plane-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-data-plane-local/templates/deployment.yaml @@ -58,6 +58,12 @@ spec: value: /etc/space-rpc-ca/ca.crt - name: SPACE_NODE_API_TLS_SERVER_NAME value: space-node-api + # The guest bridge's NodeApi credential (SpaceBearerToken::NodeApi, + # crates/libs/data/plane/server/src/native/protocol.rs); node-api + # admits it by TokenReview and confines it to guest calls. The data + # plane never calls WorkerApiInternal. + - name: SPACE_NODE_API_TOKEN_PATH + value: /var/run/space-auth/node-api.token # Datadog tracing — see datadog-agent-local chart. {{- include "space-kind-telemetry.env" . | nindent 12 }} - name: SPACE_K8S_POD_NAME @@ -108,6 +114,9 @@ spec: - name: publication-token mountPath: /var/run/space-publication readOnly: true + - name: node-api-token + mountPath: /var/run/space-auth + readOnly: true volumes: - name: rpc-ca secret: @@ -121,3 +130,11 @@ spec: path: cluster-api.token audience: space-cluster-api expirationSeconds: 3600 + - name: node-api-token + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: node-api.token + audience: space-node-api + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} diff --git a/infra/space/kind/charts/space-data-plane-local/values.yaml b/infra/space/kind/charts/space-data-plane-local/values.yaml index bfb86b94b4..19076cf104 100644 --- a/infra/space/kind/charts/space-data-plane-local/values.yaml +++ b/infra/space/kind/charts/space-data-plane-local/values.yaml @@ -35,6 +35,10 @@ autoscaling: targetCPUUtilizationPercentage: 80 targetMemoryAverageValue: 384Mi +# Lifetime of the projected node-api token, as deployed; the kubelet +# refreshes it at 80% of that. +nodeApiTokenExpirationSeconds: 600 + internal: secretName: space-internal apiEndpoint: https://space-internal-api:9800 diff --git a/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml b/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml index 10131fb09d..db6bd75290 100644 --- a/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-internal-api-local/templates/deployment.yaml @@ -60,6 +60,11 @@ spec: value: /etc/space-rpc-ca/ca.crt - name: SPACE_NODE_API_TLS_SERVER_NAME value: space-node-api + # internal-api's NodeApi credential for its service-port routes + # (crates/libs/internal/api/server/src/sandbox_service_ports.rs), + # admitted by TokenReview. It never calls WorkerApiInternal. + - name: SPACE_NODE_API_TOKEN_PATH + value: /var/run/space-auth/node-api.token - name: SPACE_K8S_CONTROL_PLANE value: "true" - name: SPACE_SANDBOX_NAMESPACE @@ -76,10 +81,12 @@ spec: value: {{ .Values.k8s.sandboxNodeSelectorValue | quote }} - name: SPACE_INTERNAL_SECRET_NAME value: {{ .Values.internal.secretName | quote }} + # ClusterApi reviews callers' projected tokens with no static + # fallback. The allowlists below name accounts in the pod's own + # namespace, read from its ServiceAccount mount (auth_namespace, + # crates/libs/common/svc/src/auth/k8s.rs). - name: SPACE_K8S_TOKEN_AUTH_ENABLED value: "true" - - name: SPACE_K8S_TOKEN_AUTH_NAMESPACE - value: {{ .Release.Namespace | quote }} - name: SPACE_K8S_TOKEN_AUTH_STATIC_FALLBACK_ENABLED value: "false" # Each node service that calls ClusterApi authenticates with its @@ -200,6 +207,9 @@ spec: - name: activation-token mountPath: /var/run/space-activation readOnly: true + - name: node-api-token + mountPath: /var/run/space-auth + readOnly: true - name: route-signing mountPath: /var/run/space-route-signing readOnly: true @@ -223,6 +233,14 @@ spec: path: cluster-api.token audience: space-cluster-api expirationSeconds: 3600 + - name: node-api-token + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: node-api.token + audience: space-node-api + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} - name: route-signing secret: secretName: {{ .Values.spaceApiRouteToken.ed25519.signingSecretName }} diff --git a/infra/space/kind/charts/space-internal-api-local/values.yaml b/infra/space/kind/charts/space-internal-api-local/values.yaml index 4a6f72322c..bf4466a362 100644 --- a/infra/space/kind/charts/space-internal-api-local/values.yaml +++ b/infra/space/kind/charts/space-internal-api-local/values.yaml @@ -40,6 +40,10 @@ service: port: 9800 type: ClusterIP +# Lifetime of the projected node-api token, as deployed; the kubelet +# refreshes it at 80% of that. +nodeApiTokenExpirationSeconds: 600 + internal: secretName: space-internal diff --git a/infra/space/kind/charts/space-node-api-local/templates/daemonset.yaml b/infra/space/kind/charts/space-node-api-local/templates/daemonset.yaml index 573cc678bd..d4552e8980 100644 --- a/infra/space/kind/charts/space-node-api-local/templates/daemonset.yaml +++ b/infra/space/kind/charts/space-node-api-local/templates/daemonset.yaml @@ -5,6 +5,11 @@ {{- end -}} {{- include "space-node-service.serviceAccount" $ctx }} --- +# node-api reviews every caller's projected ServiceAccount token +# (K8sTokenReviewer, crates/libs/common/svc/src/auth/k8s.rs; and, on builders, +# the BuildTemplate reviewer in auth/build/k8s.rs). +{{ include "space-node-service.tokenReviewRbac" $ctx }} +--- apiVersion: apps/v1 kind: DaemonSet metadata: @@ -32,8 +37,13 @@ spec: spec: {{- include "space-node-service.nodeSelector" $ctx | nindent 6 }} serviceAccountName: {{ include "space-node-service.name" $ctx }} - # The ClusterApi token is projected below; no Kubernetes API access. - automountServiceAccountToken: false + # TokenReview needs the pod's own API credentials, found by the kube + # client's in-cluster config (traced_kube_client_from_env, + # crates/libs/common/svc/src/kube_client.rs), and the auth allowlists + # name accounts in the namespace read from the same mount + # (auth_namespace, crates/libs/common/svc/src/auth/k8s.rs). The + # account may only create TokenReviews (RBAC above). + automountServiceAccountToken: true # Host network: the API dials each node's node-api at host_ip:9800. # ClusterFirstWithHostNet keeps in-cluster DNS for ClusterApi and, on # builders, build-manager's node-local Service. @@ -67,9 +77,11 @@ spec: # The NodeApi listener's TLS identity, the leaf every node-api on # the cluster shares (space-node-api, docs/internal-rpc-tls.md). {{- include "space-node-service.env.rpcIdentity" . | nindent 12 }} - # Static bearer tokens node-api's listeners require - # (crates/libs/node/api/server/src/run.rs); the worker role also - # protects WorkerApiInternal with the control token. + # The static bearer tokens node-api still requires at startup + # (run, run.rs; the worker role's WorkerApiInternal control + # token). With Kubernetes auth on and the fallback off, neither + # admits a caller (NodeApiAuthenticator::authenticate, + # crates/libs/common/svc/src/auth/mod.rs). {{- include "space-node-service.env.internalToken" . | nindent 12 }} {{- if eq $role "worker" }} - name: SPACE_INTERNAL_CONTROL_TOKEN @@ -80,6 +92,35 @@ spec: {{- else }} - name: SPACE_BUILD_MANAGER_ENDPOINT value: {{ .Values.buildManagerEndpoint | quote }} + # node-api's own build-manager credential (BearerAuthInterceptor, + # crates/libs/build/mgr/client/src/lib.rs); build-manager + # admits this account by TokenReview with no static fallback. + - name: SPACE_BUILD_MANAGER_TOKEN_PATH + value: /var/run/space-auth/build-manager.token + {{- end }} + # node-api's own NodeApi credential, for the template transfers it + # pulls from a registered peer node-api of either role: it presents + # this token to the peer's NodeExportTemplate (connect_remote, + # crates/libs/node/api/server/src/transfer.rs). + - name: SPACE_NODE_API_TOKEN_PATH + value: /var/run/space-auth/node-api.token + # Kubernetes TokenReview auth on both listeners, with no static + # fallback, as deployed (NodeApiAuthConfig, auth/mod.rs; + # K8sTokenAuthConfig::from_environment, auth/k8s.rs). The + # audiences are the code's own: space-node-api for WorkerApi, + # BuilderApi and NodeInfo, space-node-api-internal for + # WorkerApiInternal (DEFAULT_NODE_API_AUDIENCE, + # DEFAULT_NODE_API_INTERNAL_AUDIENCE and K8sTokenAuthConfig::policy, + # auth/k8s.rs). + - name: SPACE_K8S_TOKEN_AUTH_ENABLED + value: "true" + - name: SPACE_K8S_TOKEN_AUTH_STATIC_FALLBACK_ENABLED + value: "false" + {{- range $name, $accounts := .Values.auth }} + {{- if $accounts }} + - name: {{ $name }} + value: {{ join "," $accounts | quote }} + {{- end }} {{- end }} {{- if eq $role "builder" }} # Leave the builder Service's endpoints before draining (values.yaml). @@ -95,6 +136,9 @@ spec: {{- include "space-node-service.mount.rpcCa" . | nindent 12 }} {{- include "space-node-service.mount.rpcIdentity" . | nindent 12 }} {{- include "space-node-service.mount.private" . | nindent 12 }} + - name: space-auth + mountPath: /var/run/space-auth + readOnly: true {{- with .Values.resources }} resources: {{- toYaml . | nindent 12 }} @@ -105,6 +149,22 @@ spec: {{- include "space-node-service.volume.rpcCa" . | nindent 8 }} {{- include "space-node-service.volume.rpcIdentity" (dict "secret" "space-node-api-tls") | nindent 8 }} {{- include "space-node-service.volume.private" . | nindent 8 }} + # Projected tokens node-api presents: to peer node-apis for + # transfers and, on builders, to build-manager. + - name: space-auth + projected: + defaultMode: 0444 + sources: + - serviceAccountToken: + path: node-api.token + audience: space-node-api + expirationSeconds: {{ .Values.nodeApiTokenExpirationSeconds | int }} + {{- if eq $role "builder" }} + - serviceAccountToken: + path: build-manager.token + audience: space-build-manager + expirationSeconds: {{ .Values.buildManagerTokenExpirationSeconds | int }} + {{- end }} --- apiVersion: v1 kind: Service diff --git a/infra/space/kind/charts/space-node-api-local/values-builder.yaml b/infra/space/kind/charts/space-node-api-local/values-builder.yaml index cbd178eb50..d2947b9288 100644 --- a/infra/space/kind/charts/space-node-api-local/values-builder.yaml +++ b/infra/space/kind/charts/space-node-api-local/values-builder.yaml @@ -1,2 +1,39 @@ serviceName: space-builder-node-api buildManagerEndpoint: https://space-builder-build-manager:9900 + +# The builder's BuilderApi and NodeInfo (audience space-node-api). The api's +# substrate dials it as space-builder-node-api:9800, from the api, admin-api +# and the cluster-controller's trigger fan-out; either role's node-api pulls +# templates it exports (connect_remote, transfer.rs). No data plane or +# internal-api call reaches a builder, and it serves no WorkerApiInternal and +# no lifecycle-probe wait. +auth: + SPACE_NODE_API_AUTH_ALLOWED_SERVICE_ACCOUNTS: + - space-api + - space-cluster-controller + - space-node-api + - space-builder-node-api + SPACE_NODE_API_INTERNAL_AUTH_ALLOWED_SERVICE_ACCOUNTS: [] + SPACE_NODE_API_AUTH_TRUSTED_WAIT_READY_SERVICE_ACCOUNTS: [] + # The api's template deletion, settings and pool calls on the builder + # (TemplateController, crates/libs/api/src/controllers/template.rs); + # required on builders too (validate_for_role, auth/k8s.rs). + SPACE_NODE_API_AUTH_TRUSTED_POLICY_SERVICE_ACCOUNTS: + - space-api + SPACE_NODE_API_AUTH_GUEST_BRIDGE_SERVICE_ACCOUNTS: [] + # A builder is a transfer target and source like a worker (values.yaml). + SPACE_NODE_API_AUTH_TEMPLATE_TRANSFER_SERVICE_ACCOUNTS: + - space-api + - space-node-api + - space-builder-node-api + # BuilderApi/BuildTemplate also passes the build reviewer + # (BuildSurface::NodeApi, crates/libs/common/svc/src/auth/build/k8s.rs): + # these accounts, if also allowed above, may submit builds; tenant builds + # (with an org) come from the api's ECW template build route, system + # builds from admin-api's template builds, both as space-api. + SPACE_NODE_API_AUTH_TEMPLATE_BUILD_SERVICE_ACCOUNTS: + - space-api + SPACE_BUILD_AUTH_SYSTEM_SERVICE_ACCOUNTS: + - space-api + SPACE_BUILD_AUTH_TENANT_SERVICE_ACCOUNTS: + - space-api diff --git a/infra/space/kind/charts/space-node-api-local/values.yaml b/infra/space/kind/charts/space-node-api-local/values.yaml index 23516f8792..594af3b67b 100644 --- a/infra/space/kind/charts/space-node-api-local/values.yaml +++ b/infra/space/kind/charts/space-node-api-local/values.yaml @@ -19,6 +19,78 @@ node: {} # sandbox traffic goes to each pod's host_ip:9800 instead. serviceName: space-worker-node-api +# Kubernetes token auth: the ServiceAccounts each node-api surface admits, +# rendered as the env vars of the same names (K8sTokenAuthConfig, +# crates/libs/common/svc/src/auth/k8s.rs; names are qualified with +# node-api's own namespace). These are the worker role's; values-builder.yaml +# replaces them for builders. An empty list renders nothing. +auth: + # WorkerApi and NodeInfo (audience space-node-api). The api's substrate, + # which the shared and admin api surfaces and the cluster-controller's + # operation dispatcher, warm-pool probe and trigger fan-out all run + # (crates/libs/api/src/controllers/substrate/mod.rs, substrate/kubernetes/, + # crates/libs/cluster/reconciler/src/trigger_fanout.rs); the data plane's + # guest calls (crates/libs/data/plane/server/src/native/protocol.rs); + # internal-api's service-port routes + # (crates/libs/internal/api/server/src/sandbox_service_ports.rs); and + # either role's node-api pulling a template it exports (connect_remote, + # crates/libs/node/api/server/src/transfer.rs). + SPACE_NODE_API_AUTH_ALLOWED_SERVICE_ACCOUNTS: + - space-api + - space-cluster-controller + - space-data-plane + - space-internal-api + - space-node-api + - space-builder-node-api + # WorkerApiInternal (audience space-node-api-internal): only the api's + # substrate dials it, from the api and the cluster-controller + # (internal_client, substrate/mod.rs; + # substrate/kubernetes/pod_ops/worker_clients.rs). + SPACE_NODE_API_INTERNAL_AUTH_ALLOWED_SERVICE_ACCOUNTS: + - space-api + - space-cluster-controller + # Lifecycle-probe SpacedWaitReady (classify_wait_ready_caller, worker.rs): + # the substrate's readiness waits and the cluster-controller's warm-pool + # probe. Required on a worker (validate_for_role, auth/k8s.rs). + SPACE_NODE_API_AUTH_TRUSTED_WAIT_READY_SERVICE_ACCOUNTS: + - space-api + - space-cluster-controller + # Caller-supplied network and auto-lifecycle policy, template and pool + # mutation (worker.rs, auth/template.rs): the substrate's creates, starts, + # wakes and policy updates. Required on both roles (validate_for_role). + SPACE_NODE_API_AUTH_TRUSTED_POLICY_SERVICE_ACCOUNTS: + - space-api + - space-cluster-controller + # The data plane is the guest bridge, confined to guest calls + # (guest_bridge_method, auth/k8s.rs). node-api adds it whatever this + # says (K8sTokenAuthConfig::from_environment); listed so the edge's + # restriction is visible. + # internal-api must not be here: its AllowPort is not a guest call. + SPACE_NODE_API_AUTH_GUEST_BRIDGE_SERVICE_ACCOUNTS: + - space-data-plane + # Peer template transfers: PullTemplate and NodeExportTemplate need this + # grant as well as admission above (require_template_transfer_authority and + # prepare_template_export, + # crates/libs/common/svc/src/auth/template_transfer.rs). With Kubernetes + # auth on, only these accounts get it (AuthPolicy::authenticated_caller, + # auth/k8s.rs), and an empty list refuses every transfer. A pull is started + # on the target as the api's account: its only in-tree caller, admin-api's + # pull_node_template (crates/libs/api/src/routes/admin/node_maintenance.rs), + # answers 501 on the Kubernetes substrate (require_admin_node_registry), so + # here a pull is an operator's direct PullTemplate call with a space-api + # token. The target's node-api, of either role, then asks the source for the + # export with its own token (connect_remote, transfer.rs). + SPACE_NODE_API_AUTH_TEMPLATE_TRANSFER_SERVICE_ACCOUNTS: + - space-api + - space-node-api + - space-builder-node-api + +# Lifetimes of the projected tokens node-api presents, as deployed: its +# NodeApi token, and on builders its build-manager token, which outlasts a +# build's longest call. The kubelet refreshes each at 80% of its lifetime. +nodeApiTokenExpirationSeconds: 600 +buildManagerTokenExpirationSeconds: 7200 + # Builder role only: build-manager's node-local Service # (space-build-manager-local), whose name is the one its TLS leaf carries # (crates/libs/build/mgr/client/src/lib.rs). diff --git a/infra/space/kind/charts/space-node-service/templates/_helpers.tpl b/infra/space/kind/charts/space-node-service/templates/_helpers.tpl index 9bab9be6af..5341159058 100644 --- a/infra/space/kind/charts/space-node-service/templates/_helpers.tpl +++ b/infra/space/kind/charts/space-node-service/templates/_helpers.tpl @@ -308,7 +308,9 @@ pod's own mounts back to the node. - serviceAccountToken: path: credmgr.token audience: space-credential-manager - expirationSeconds: 3600 + # The node services' deployed lifetime, as for the ClusterApi + # token below. + expirationSeconds: 600 - secret: name: space-credmgr-authority-trust items: @@ -335,6 +337,11 @@ a root listener reads it as owner. defaultMode: 0440 {{- end -}} +{{/* +The 600 s lifetime is the node services' deployed one; the kubelet refreshes +the file at 80% of it and the client rereads it (TOKEN_FILE_CACHE_TTL, +bearer_token.rs). +*/}} {{- define "space-node-service.volume.clusterApiToken" -}} - name: cluster-api-token projected: @@ -343,7 +350,7 @@ a root listener reads it as owner. - serviceAccountToken: path: cluster-api.token audience: space-cluster-api - expirationSeconds: 3600 + expirationSeconds: 600 {{- end -}} {{- define "space-node-service.volume.routeVerification" -}} @@ -450,6 +457,39 @@ ad.datadoghq.com/tags: '{"team":"{{ .ctx.Values.node.datadogTeam }}"}' {{ include "space-kind-telemetry.logAnnotations" .ctx }} {{- end -}} +{{/* +Lets the service's ServiceAccount create TokenReviews, which a listener that +authenticates its callers' projected tokens issues for each one +(crates/libs/common/svc/src/auth/k8s.rs, auth/build/k8s.rs). Nothing else. +*/}} +{{- define "space-node-service.tokenReviewRbac" -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "space-node-service.name" . }}-token-review + labels: + {{- include "space-node-service.labels" . | nindent 4 }} +rules: + - apiGroups: ["authentication.k8s.io"] + resources: ["tokenreviews"] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "space-node-service.name" . }}-token-review + labels: + {{- include "space-node-service.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "space-node-service.name" . }}-token-review +subjects: + - kind: ServiceAccount + name: {{ include "space-node-service.name" . }} + namespace: {{ .ctx.Release.Namespace }} +{{- end -}} + {{- define "space-node-service.serviceAccount" -}} apiVersion: v1 kind: ServiceAccount diff --git a/scripts/k8s/projected-tokens/01-apply-auth.sh b/scripts/k8s/projected-tokens/01-apply-auth.sh deleted file mode 100755 index 9d5b40d1dd..0000000000 --- a/scripts/k8s/projected-tokens/01-apply-auth.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -# Wire the projected-token (TokenReview) auth path onto a running local stack: -# - server (node-api DaemonSet): a tokenreviews/create ClusterRole bound to its SA, -# plus the SPACE_K8S_TOKEN_AUTH_* env that flips node-api into -# k8s_token_review mode (static fallback kept on). -# - client (space-api): projected ServiceAccount tokens for the normal and -# internal node-api audiences, with token paths set on both client surfaces. -# -# Idempotent: safe to re-run, including after a `tilt up` reconcile re-applies -# the charts and drops the env/volume patches. -# -# The Kind node-api is built with the `k8s` Cargo feature -# (docker/Dockerfile.rust-build-kind); without it node-api fails loud at startup -# with KubernetesAuthFeatureDisabled instead of booting k8s mode. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "${SCRIPT_DIR}/lib.sh" - -require kubectl -assert_local_context - -log "1/3 apply TokenReview RBAC (ClusterRole + binding -> ${NS}/${NODE_API_SA})" -kubectl --context "$KCTX" apply -f - </dev/null | grep -E '"node-api auth initialized"' | tail -1 \ - || warn "no auth-init line yet; check 'kubectl -n ${NS} logs '" diff --git a/scripts/k8s/projected-tokens/02-test.sh b/scripts/k8s/projected-tokens/02-test.sh deleted file mode 100755 index de7ddb7b1b..0000000000 --- a/scripts/k8s/projected-tokens/02-test.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/env bash -# Prove the projected-token (mode:k8s) path end-to-end against node-api: -# 1. read space-api's actual projected JWT, -# 2. port-forward node-api, -# 3. POSITIVE: call WorkerApi/ListSandboxes + CreateSandbox with that JWT and -# show node-api accept it via TokenReview (verified SA + audience), -# 4. NEGATIVE: call with a bogus JWT and show node-api reject it, -# 5. INTERNAL surface: mint a space-cluster-controller token and show -# WorkerApiInternal accepts it (nodeApiInternal allowlist), while a SA that -# is not on that allowlist is denied (the cluster-controller warm-pool edge). -# -# Requires `grpcurl` (brew install grpcurl) and ./01-apply-auth.sh already -# applied. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "${SCRIPT_DIR}/lib.sh" -REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" - -require kubectl -require grpcurl -assert_local_context - -POD="$(node_api_pod)"; [[ -n "$POD" ]] || die "node-api pod not found" -APIPOD="$(k get pod -l app.kubernetes.io/name="${SPACE_API_DEPLOY}" -o jsonpath='{.items[0].metadata.name}')" -[[ -n "$APIPOD" ]] || die "space-api pod not found" - -TOK="$(mktemp)"; CA="$(mktemp)" -trap 'rm -f "$TOK" "$CA"; [[ -n "${PF_PID:-}" ]] && kill "$PF_PID" 2>/dev/null || true' EXIT -# node-api serves authenticated TLS with the cluster's shared space-node-api -# identity (infra/space/kind/scripts/internal-rpc-tls.sh); verify it against -# the cluster CA under that name, since the dial address is a port-forward. -k get secret space-internal-rpc-ca -o jsonpath='{.data.ca\.crt}' | base64 -d >"$CA" -[[ -s "$CA" ]] || die "no space-internal-rpc-ca Secret (run infra/space/kind/scripts/internal-rpc-tls.sh)" -log "reading space-api projected token" -k exec "$APIPOD" -c "${SPACE_API_CONTAINER}" -- cat "${TOKEN_PATH}" > "$TOK" 2>/dev/null -[[ -s "$TOK" ]] || die "no projected token at ${TOKEN_PATH} on ${APIPOD} (run ./01-apply-auth.sh first)" - -log "port-forward node-api ${LOCAL_NODE_API_PORT}->${NODE_API_PORT}" -kubectl --context "$KCTX" -n "$NS" port-forward "$POD" "${LOCAL_NODE_API_PORT}:${NODE_API_PORT}" >/dev/null 2>&1 & -PF_PID=$! -for _ in $(seq 1 15); do nc -z localhost "${LOCAL_NODE_API_PORT}" 2>/dev/null && break; sleep 1; done - -GRPC=(grpcurl -cacert "$CA" -servername space-node-api -import-path "${REPO_ROOT}/proto" -proto node/api/v1/worker.proto) -ADDR="localhost:${LOCAL_NODE_API_PORT}" -WAPI="space.node.api.v1.WorkerApi" - -pass=0; fail=0 -ok() { printf '\033[1;32m PASS\033[0m %s\n' "$*"; pass=$((pass+1)); } -bad() { printf '\033[1;31m FAIL\033[0m %s\n' "$*"; fail=$((fail+1)); } - -echo -log "POSITIVE 1/2: WorkerApi/ListSandboxes with space-api's projected JWT" -out="$("${GRPC[@]}" -H "authorization: Bearer $(cat "$TOK")" -d '{}' "$ADDR" "${WAPI}/ListSandboxes" 2>&1 || true)" -echo "$out" | sed 's/^/ /' -echo "$out" | grep -q "Unauthenticated\|PermissionDenied" \ - && bad "ListSandboxes was rejected (projected token not accepted)" \ - || ok "ListSandboxes accepted (TokenReview validated the projected JWT)" - -echo -log "POSITIVE 2/2: WorkerApi/CreateSandbox with the projected JWT (auth runs before the handler)" -out="$("${GRPC[@]}" -H "authorization: Bearer $(cat "$TOK")" \ - -d '{"template":"seed:latest","runtime":"fc","owner":"root@local","sandbox_id":"sbx_pt_poc"}' \ - "$ADDR" "${WAPI}/CreateSandbox" 2>&1 || true)" -echo "$out" | sed 's/^/ /' | head -4 -# Auth passed if we got past it: a clean response, or a handler error that is -# NOT an auth/permission rejection of the bearer. -if echo "$out" | grep -q "Unauthenticated"; then - bad "CreateSandbox rejected at auth" -else - ok "CreateSandbox authenticated (handler ran past auth; any error here is post-auth)" -fi - -echo -log "NEGATIVE: WorkerApi/ListSandboxes with a BOGUS JWT (expect Unauthenticated)" -BOGUS="eyJhbGciOiJSUzI1NiIsImtpZCI6ImJvZ3VzIn0.eyJzdWIiOiJmYWtlIiwiYXVkIjpbInNwYWNlLW5vZGUtYXBpIl19.ZmFrZXNpZ25hdHVyZQ" -out="$("${GRPC[@]}" -H "authorization: Bearer ${BOGUS}" -d '{}' "$ADDR" "${WAPI}/ListSandboxes" 2>&1 || true)" -echo "$out" | sed 's/^/ /' | head -4 -echo "$out" | grep -q "Unauthenticated" \ - && ok "bogus token rejected (TokenReview rejected the JWT)" \ - || bad "bogus token was NOT rejected" - -# ── Internal surface (WorkerApiInternal): client-identity allowlist ── -# Mirrors the cluster-controller -> node-api/WorkerApiInternal edge (warm-pool -# WakeSandbox). node-api's nodeApiInternal allowlist (deployed by 01-apply-auth.sh -# from NODE_API_INTERNAL_ALLOWED_SAS) must accept INTERNAL_ALLOWED_SA and reject -# any SA not on it. Tokens are minted per-SA directly via the TokenRequest API. -WIAPI="space.node.api.v1.WorkerApiInternal" -mint_internal_token() { # $1=ServiceAccount -> projected JWT for the internal audience - k get sa "$1" >/dev/null 2>&1 || k create sa "$1" >/dev/null 2>&1 || true - k create token "$1" --audience="${NODE_API_INTERNAL_AUDIENCE}" 2>/dev/null -} - -echo -log "POSITIVE 3/3: WorkerApiInternal/WakeSandbox as ${INTERNAL_ALLOWED_SA} (on the nodeApiInternal allowlist)" -tok="$(mint_internal_token "${INTERNAL_ALLOWED_SA}")" -if [[ -z "$tok" ]]; then - bad "could not mint an internal-audience token for ${INTERNAL_ALLOWED_SA} (kubectl create token failed)" -else - out="$("${GRPC[@]}" -H "authorization: Bearer ${tok}" -d '{}' "$ADDR" "${WIAPI}/WakeSandbox" 2>&1 || true)" - echo "$out" | sed 's/^/ /' | head -4 - if echo "$out" | grep -qE "Unauthenticated|PermissionDenied"; then - bad "${INTERNAL_ALLOWED_SA} rejected on the internal surface (is it on the nodeApiInternal allowlist?)" - else - ok "${INTERNAL_ALLOWED_SA} authenticated on nodeApiInternal (handler ran past auth)" - fi -fi - -echo -log "NEGATIVE 2/2: WorkerApiInternal/WakeSandbox as ${INTERNAL_DENIED_SA} (NOT on the allowlist)" -tok="$(mint_internal_token "${INTERNAL_DENIED_SA}")" -if [[ -z "$tok" ]]; then - bad "could not mint an internal-audience token for ${INTERNAL_DENIED_SA} (kubectl create token failed)" -else - out="$("${GRPC[@]}" -H "authorization: Bearer ${tok}" -d '{}' "$ADDR" "${WIAPI}/WakeSandbox" 2>&1 || true)" - echo "$out" | sed 's/^/ /' | head -4 - # Require PermissionDenied specifically: the token is valid (TokenReview + - # audience OK) but the SA is not on the allowlist — that is the allowlist doing - # its job. Unauthenticated means a pre-allowlist failure (bad/empty token, wrong - # audience, TokenReview rejection) and must NOT count as "enforced". - if echo "$out" | grep -q "PermissionDenied"; then - ok "${INTERNAL_DENIED_SA} denied with PermissionDenied (allowlist enforced)" - elif echo "$out" | grep -q "Unauthenticated"; then - bad "${INTERNAL_DENIED_SA} got Unauthenticated — token/audience failed before the allowlist (inconclusive)" - else - bad "${INTERNAL_DENIED_SA} was NOT denied on the internal surface" - fi -fi - -echo -# Supplementary: the human-readable acceptance log, if a fresh (uncached) -# TokenReview happened during this run. Verdicts cache (~k8sAuth.cacheTtlSeconds), -# so a cache hit legitimately leaves no new line — the gRPC results above are the -# authoritative signal. -acc="$(k logs "$POD" --since=10m 2>/dev/null | grep '"accepted Kubernetes service account"' | tail -1 || true)" -if [[ -n "$acc" ]]; then - log "node-api also logged (most recent acceptance):" - echo "$acc" | python3 -c "import sys,json -d=json.load(sys.stdin); print(' ACCEPT sa=%s aud=%s surface=%s' % (d.get('service_account'), d.get('audience'), d.get('surface')))" 2>/dev/null || true -else - log "node-api: positive verdicts served from the TokenReview cache (no fresh log line; expected)." -fi - -echo -[[ "$fail" -eq 0 ]] && log "RESULT: all ${pass} checks passed — mode:k8s works end-to-end." \ - || die "RESULT: ${fail} check(s) failed." diff --git a/scripts/k8s/projected-tokens/03-teardown.sh b/scripts/k8s/projected-tokens/03-teardown.sh deleted file mode 100755 index 7caf952a28..0000000000 --- a/scripts/k8s/projected-tokens/03-teardown.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash -# Revert the projected-token wiring applied by 01-apply-auth.sh: drop the -# TokenReview RBAC, the node-api k8sAuth env, and the space-api projected -# token volume/env. Returns node-api to static-only auth. -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=lib.sh -source "${SCRIPT_DIR}/lib.sh" -require kubectl -assert_local_context - -log "remove TokenReview RBAC" -kubectl --context "$KCTX" delete clusterrole space-node-api-tokenreview --ignore-not-found -kubectl --context "$KCTX" delete clusterrolebinding space-node-api-tokenreview --ignore-not-found - -log "remove k8sAuth env from node-api" -k set env "daemonset/${NODE_API_DS}" -c "${NODE_API_CONTAINER}" \ - SPACE_K8S_TOKEN_AUTH_ENABLED- \ - SPACE_K8S_TOKEN_AUTH_NAMESPACE- \ - SPACE_K8S_TOKEN_AUTH_STATIC_FALLBACK_ENABLED- \ - SPACE_NODE_API_AUTH_ALLOWED_SERVICE_ACCOUNTS- \ - SPACE_NODE_API_AUTH_TRUSTED_WAIT_READY_SERVICE_ACCOUNTS- \ - SPACE_NODE_API_AUTH_TRUSTED_POLICY_SERVICE_ACCOUNTS- \ - SPACE_NODE_API_AUTH_TEMPLATE_TRANSFER_SERVICE_ACCOUNTS- \ - SPACE_NODE_API_INTERNAL_AUTH_ALLOWED_SERVICE_ACCOUNTS- || true - -log "remove projected token + env from space-api" -k set env "deploy/${SPACE_API_DEPLOY}" -c "${SPACE_API_CONTAINER}" \ - SPACE_NODE_API_TOKEN_PATH- SPACE_NODE_API_INTERNAL_TOKEN_PATH- || true -# Delete the space-auth volume AND its mount by NAME, never by positional index: -# anything (e.g. a tilt reconcile) may reorder volumes, so removing /volumes/0 -# could drop an unrelated volume, and an orphaned mount would wedge the pod. -# Strategic-merge `$patch: delete` keys on the volume name / mount path and is a -# no-op if they're already gone. -k patch "deploy/${SPACE_API_DEPLOY}" --type strategic --patch "$(cat <:` must also match the -allowlist for the requested surface. This harness enables static-token fallback -so clients without a projected token can still use `SPACE_INTERNAL_TOKEN`; -deployments can disable that fallback. - -```mermaid -flowchart LR - api["space-api pod
SA space-api"] -->|"Bearer projected JWT
aud = space-node-api"| na["node-api DaemonSet pod
SA space-node-api"] - na -->|"TokenReview(JWT, audience)"| kas["kind kube-apiserver"] - kas -->|"valid → system:serviceaccount:space:space-api"| na - na -->|"allowlist check → accept (mode:k8s)"| na - api -. "static secret (mode:static) when projected token absent" .-> na -``` - -## Why a direct RPC test - -The Kind node-api is built with the `k8s` Cargo feature -(`docker/Dockerfile.rust-build-kind`), so it has the TokenReview code; the local -charts simply run it in static mode. Enabling `k8sAuth` here switches it on. - -The full async create flow does not exercise every edge a deployment -authenticates: sandbox creation goes through the cluster-controller and a -per-sandbox Pod. So the scripts exercise the path with a **direct `WorkerApi` -RPC** carrying space-api's real projected token — the node-api auth behaviour -is identical to what space-api's own client does. - -## Prerequisites - -- A running local stack on `kind` (see the - [installation guide](../../../docs/installation.md) and `Tiltfile`). - The scripts default `KCTX` to your **current kube-context** (whatever Tilt - left selected) and refuse any non-`kind-*` context. -- `kubectl`, `grpcurl` (`brew install grpcurl`), and `nc`. - -## Usage - -```bash -cd scripts/k8s/projected-tokens - -# 1. wire the auth: TokenReview RBAC + node-api k8sAuth env + space-api -# projected token. Idempotent; re-run after any `tilt up` reconcile. -./01-apply-auth.sh - -# 2. prove it: positive (ListSandboxes + CreateSandbox accepted via TokenReview) -# and negative (bogus JWT rejected). -./02-test.sh - -# 3. (optional) revert the wiring back to static-only auth. -./03-teardown.sh -``` - -All knobs (context, namespace, SAs, audiences, allowlist) are overridable via -the environment — see `lib.sh`. `KCTX` defaults to your current kube-context; set -`KCTX=kind-` to target a specific cluster. - -## Policy authorization and rollout - -Before enabling Kubernetes auth or upgrading an authenticated worker or builder, -configure `SPACE_NODE_API_AUTH_TRUSTED_POLICY_SERVICE_ACCOUNTS` with the service -accounts that own policy intent. It must be nonempty on both roles or node-api -refuses startup. This setup supplies `NODE_API_TRUSTED_POLICY_SAS`, defaulting to -`space-api,space-cluster-controller`; override it for your deployment. These -accounts must also be admitted by the normal/internal surface allowlists for the -RPCs they call. Policy trust is independent of the WaitReady trust set. -The setup projects both normal and internal audience tokens into space-api; the -internal token is required for policy-bearing pool adoption. - -Peer template transfers require the independent -`SPACE_NODE_API_AUTH_TEMPLATE_TRANSFER_SERVICE_ACCOUNTS` allowlist. This setup -uses `NODE_API_TEMPLATE_TRANSFER_SAS`, defaulting to `space-api,space-node-api`: -the API initiates pulls and the receiving node-api requests exports from peers. -Both must also be admitted by `NODE_API_ALLOWED_SAS`. An empty transfer list -disables transfers and emits a startup warning; deployments without peer -transfers may intentionally leave it empty. - -Shared static-token fallback cannot mutate policy when Kubernetes auth is enabled. -Static-only deployments retain that authority. Policy-bearing cold creation, -internal pool adoption, and worker/builder template updates require a trusted -caller. Cold creation and pool adoption also require policy authority for a nonzero -sandbox-policy generation, even without an override. No overrides and generation -zero remain available to otherwise admitted callers and inherit template policy. -Admitted service accounts and shared fallback credentials are the relevant threat: -guest-bridge accounts already have separate method restrictions. - -`Local` means node-administrator-owned policy, including normal standalone use and -operator overrides on cluster-managed nodes. It requires a root/Operator Unix -socket peer. Forwarders such as node-api and node-ctl must run under their dedicated -non-root identities: UID 0 is classified as Root before service-account lookup. - -Forwarded imports carrying `Local` network or auto-lifecycle policy are intentionally -rejected. Retrying does not help. An operator must review the policy ownership and -perform the import through an authorized local peer when retaining Local policy; -forwarding must never silently relabel it. - -## What success looks like - -After `01-apply-auth.sh`, node-api boots: - -``` -node-api auth initialized role=Worker mode=k8s_token_review static_fallback_enabled=true -``` - -`02-test.sh` then shows node-api validating space-api's projected JWT against the -kind apiserver: - -``` -[pt] POSITIVE 1/2: ListSandboxes ... -> {} (accepted) -[pt] POSITIVE 2/2: CreateSandbox ... -> handler ran past auth -[pt] NEGATIVE: bogus JWT ... -> Unauthenticated: token validation failed -[pt] node-api TokenReview decisions: - ACCEPT sa=system:serviceaccount:space:space-api aud=space-node-api surface=normal -``` - -The `CreateSandbox` handler may fail *after* auth (no bootable guest in kind); -that's expected — the point is the `ACCEPT` line, which proves the projected -token was validated via TokenReview and matched the allowlist (`mode:k8s`). - -## Troubleshooting - -**kind API liveness timeouts during the `tilt up`/`tilt ci` build.** kind runs the -whole control plane (apiserver + etcd) inside one Docker container, sharing the -VM's CPU/RAM/disk with the image builds. Heavy *parallel* Rust image builds (plus -the per-image `kind load`) saturate that VM and starve etcd — its `wal` fsync -latency spikes, the apiserver can't reach it, and the build fails with a kind API -liveness timeout (or a wedged apiserver that returns TLS-handshake timeouts). If -you hit this: - -- Build **sequentially** or bound parallelism — start Tilt with - `SPACE_TILT_MAX_PARALLEL_UPDATES=1` (or `2`) in its environment; the Tiltfile - reads it (`tilt up` has no parallelism flag). It defaults to 8. Slower, but it keeps the control plane responsive. `tilt ci` is the most likely to trip - this since it gates on everything going green in one shot. -- Prefer a **local image registry** (`kind-with-registry`) — it removes the - `kind load` IO entirely (containerd pulls directly), which is the single biggest - reduction in pressure. Tilt warns when you're running kind without one. -- Give the **Docker VM more CPU/RAM** if it's under-provisioned. - -## Files - -| File | Purpose | -| --- | --- | -| `lib.sh` | Shared config (context, namespace, SAs, audiences, ports) + helpers | -| `01-apply-auth.sh` | Wire TokenReview RBAC + server env + client projected token | -| `02-test.sh` | Positive + negative TokenReview tests via `grpcurl` | -| `03-teardown.sh` | Revert the wiring to static-only auth | diff --git a/scripts/k8s/projected-tokens/lib.sh b/scripts/k8s/projected-tokens/lib.sh deleted file mode 100755 index 98e1885c22..0000000000 --- a/scripts/k8s/projected-tokens/lib.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC2034 # config values below are read by scripts that source this file -# Shared config + helpers for the projected-token (Kubernetes TokenReview) PoC -# on a local kind cluster. Source this from the numbered scripts. -# -# Override any value via the environment, e.g.: -# KCTX=kind-space-local NS=space ./01-apply-auth.sh -set -euo pipefail - -# kube context + namespace for the local stack. KCTX defaults to your current -# kube-context — whatever `kind create` / `tilt up` selected — so it tracks the -# cluster Tilt actually brought up regardless of its name (the Tiltfile derives -# the name from SPACE_KIND_CLUSTER, default "space-local"). The mutating -# scripts call assert_local_context, which refuses any non-kind context so this -# can never touch testing/prod. -KCTX="${KCTX:-$(kubectl config current-context 2>/dev/null || echo kind-space-local)}" -NS="${NS:-space}" - -# Server side: the worker node-api DaemonSet (space-node-api-local), container -# `node-api`, ServiceAccount `space-node-api`. -NODE_API_DS="${NODE_API_DS:-space-node-api}" -NODE_API_CONTAINER="${NODE_API_CONTAINER:-node-api}" -NODE_API_SA="${NODE_API_SA:-space-node-api}" -NODE_API_PORT="${NODE_API_PORT:-9800}" - -# Client side: space-api dials node-api. It runs as SA `space-api`, container -# `api`. -SPACE_API_DEPLOY="${SPACE_API_DEPLOY:-space-api}" -SPACE_API_CONTAINER="${SPACE_API_CONTAINER:-api}" -SPACE_API_SA="${SPACE_API_SA:-space-api}" - -# Audiences (must match the node-api server config and the client volume). -NODE_API_AUDIENCE="${NODE_API_AUDIENCE:-space-node-api}" -NODE_API_INTERNAL_AUDIENCE="${NODE_API_INTERNAL_AUDIENCE:-space-node-api-internal}" - -# Which client ServiceAccounts node-api accepts on the NodeApi surface. -# space-api is the caller under test; the controller and cluster-api entries -# keep the local wiring aligned with the deployed node-api allowlist. -NODE_API_ALLOWED_SAS="${NODE_API_ALLOWED_SAS:-space-api,space-cluster-controller,space-cluster-api,${NODE_API_SA}}" -# Exact subset authorized for tokenless SpacedWaitReady lifecycle probes. -NODE_API_TRUSTED_WAIT_READY_SAS="${NODE_API_TRUSTED_WAIT_READY_SAS:-space-api,space-cluster-controller}" -# Independent subset authorized to supply network/auto-lifecycle policy and generations. -NODE_API_TRUSTED_POLICY_SAS="${NODE_API_TRUSTED_POLICY_SAS:-space-api,space-cluster-controller}" -# API initiates pulls; node-api exports from the source on behalf of the target. -NODE_API_TEMPLATE_TRANSFER_SAS="${NODE_API_TEMPLATE_TRANSFER_SAS:-space-api,${NODE_API_SA}}" -# nodeApiInternal accepts space-api (control-plane request path) and -# space-cluster-controller (warm-pool WorkerApiInternal/WakeSandbox). Mirrors the -# node-api chart allowlist. -NODE_API_INTERNAL_ALLOWED_SAS="${NODE_API_INTERNAL_ALLOWED_SAS:-space-api,space-cluster-controller}" - -# ServiceAccount identities for the internal-surface test in 02-test.sh: one that -# is on the nodeApiInternal allowlist, one deliberately not (to prove enforcement). -INTERNAL_ALLOWED_SA="${INTERNAL_ALLOWED_SA:-space-cluster-controller}" -INTERNAL_DENIED_SA="${INTERNAL_DENIED_SA:-space-pt-unlisted}" - -# Where the client mounts its normal and internal projected NodeApi tokens. -TOKEN_MOUNT="${TOKEN_MOUNT:-/var/run/space-auth}" -TOKEN_FILE="${TOKEN_FILE:-node-api.token}" -TOKEN_PATH="${TOKEN_MOUNT}/${TOKEN_FILE}" -INTERNAL_TOKEN_FILE="${INTERNAL_TOKEN_FILE:-node-api-internal.token}" -INTERNAL_TOKEN_PATH="${TOKEN_MOUNT}/${INTERNAL_TOKEN_FILE}" - -# Local port for the node-api port-forward used by the test script. -LOCAL_NODE_API_PORT="${LOCAL_NODE_API_PORT:-19800}" - -k() { kubectl --context "$KCTX" -n "$NS" "$@"; } - -log() { printf '\033[1;34m[pt]\033[0m %s\n' "$*"; } -warn() { printf '\033[1;33m[pt] WARN:\033[0m %s\n' "$*" >&2; } -die() { printf '\033[1;31m[pt] ERROR:\033[0m %s\n' "$*" >&2; exit 1; } - -# Resolve one worker node-api pod name. -node_api_pod() { - k get pod -l app.kubernetes.io/name=space-worker,app.kubernetes.io/component=node-api \ - -o jsonpath='{.items[0].metadata.name}' 2>/dev/null -} - -require() { command -v "$1" >/dev/null 2>&1 || die "missing dependency: $1"; } - -# Refuse to run cluster-mutating steps against anything that isn't a local kind -# cluster, so the harness can never touch testing/prod. The 01/02/03 scripts call -# this. -assert_local_context() { - case "$KCTX" in - kind-*) log "targeting kube-context: $KCTX" ;; - "") die "no kube-context set; bring up the local stack or set KCTX=kind-" ;; - *) die "refusing to run against non-kind context '$KCTX' — set KCTX=kind- if intentional" ;; - esac -}