From ef9103f78df2f5a7a8dae0c2f10b45331c5777cd Mon Sep 17 00:00:00 2001 From: Abhinav Chordia Date: Wed, 14 Oct 2026 17:23:52 +0000 Subject: [PATCH] kind: run node and runtime configuration like a deployed node Kind already runs the production node layout; this makes its node and runtime settings match what deployed nodes run, so the paths those settings select are the ones Kind exercises. - The public-operation dispatcher's tick, claim limit and claim staleness and the snapshot-retention sweeper's minimum age are test hooks Kind set without needing them. The tick override made Kind pick up queued operations every second instead of on the release 25 ms tick. All four are dropped from the cluster-controller chart; the retention scenarios still shorten the minimum age for their own run. - node-manager runs with no warm network-slot pool (target, retained and minimum ready all 0), as deployed nodes do. With the binary defaults the pool kept every slot the node has (300) ready and bridged to the node's own namespace. Sandbox pods bring their own namespace, so each sandbox claim took one of those slots and rebuilt it for the pod, and the pool rebuilt it after release. Builders, whose build VMs allocate without a pod namespace, leased pooled slots. Now every slot is set up on allocation and torn down on release, and the pool reports healthy while empty. - The static network-slots capacity that bootstrap.sh and the Tiltfile seeded is removed: node-ctl already advertised node-manager's capacity over it, as it does on a deployed node. - The SPACE Kind node image carries a kubelet --config-dir drop-in with eventRecordQPS: 0 and failSwapOn: false, and registers the `space` runtime with the deployed CniConfDir option through a drop-in that kind's containerd config imports. The drop-in uses kind's version 2 format, so containerd migrates it with kind's own settings into the CRI v1 runtime plugin where a deployed node has it. A version 3 drop-in would raise the merged version and skip that migration, losing runc's systemd cgroup driver (kubelet would then run cgroupfs), base_runtime_spec and discard_unpacked_layers. The image build fails unless containerd's loaded config has the runtime, its options, the systemd driver and discard_unpacked_layers. containerd loads it on first start, so install-shim.sh no longer edits its config or restarts it; it installs the shim binary and checks its SHA-256 on the node. Both settings change the image tag, so clusters created without them are refused by check-kind-node-image.sh. - space-cache.service reformats the cache image on every boot of the node container, as deployed nodes reformat their instance-store cache, stays idempotent within a boot, and mounts it nosuid,noatime. The optional bootstrap-prod-seed.sh, whose templates live only on that cache, now re-stages a worker whose cache lost them, and says to re-run it after a node restart. The README's comparison with the EKS modules gains rows for what stays different: shim installation after cluster creation, the kubelet settings on the non-SPACE nodes, the test-hooks build (naming the reaper, template GC and prefetch hooks the charts set on every run), and the disk-admission reservation, whose chart comment no longer calls it a production setting. --- Tiltfile | 36 ++------ infra/space/kind/README.md | 77 +++++++++++------ .../templates/deployment.yaml | 4 - .../values.yaml | 15 ++-- .../space-node-ctl-local/templates/rbac.yaml | 7 +- .../templates/daemonset.yaml | 22 +++++ .../charts/space-node-service/values.yaml | 9 +- infra/space/kind/manifests/runtimeclass.yaml | 26 +++--- infra/space/kind/node/Dockerfile | 28 ++++++ .../kind/node/containerd/space-runtime.toml | 29 +++++++ infra/space/kind/node/kubelet/50-space.conf | 13 +++ infra/space/kind/node/space-kind-node.sh | 72 +++++++++++----- .../space/kind/scripts/bootstrap-prod-seed.sh | 18 +++- infra/space/kind/scripts/bootstrap.sh | 24 ++---- infra/space/kind/scripts/install-shim.sh | 86 +++++++------------ infra/space/kind/scripts/kind-node-image.sh | 2 + 16 files changed, 294 insertions(+), 174 deletions(-) create mode 100644 infra/space/kind/node/containerd/space-runtime.toml create mode 100644 infra/space/kind/node/kubelet/50-space.conf diff --git a/Tiltfile b/Tiltfile index 63adb85e42..a63670a955 100644 --- a/Tiltfile +++ b/Tiltfile @@ -328,9 +328,10 @@ custom_build( # `KubernetesSubstrate::build_sandbox_pod` pins per-sandbox pods to # `runtimeClassName: space` and `nodeSelector: node.kubernetes.io/space-enabled=true`. # Neither exists in vanilla KinD, so we materialize them before the charts -# land. RuntimeClass `space` resolves to containerd-shim-space-v2 once -# install-shim.sh has wired it into each sandbox worker's containerd (the -# `install-space-shim` bootstrap resource below, full harness only). +# land. The SPACE Kind node image registers RuntimeClass `space`'s handler with +# containerd; it resolves to containerd-shim-space-v2 once install-shim.sh has +# installed that binary on each sandbox worker (the `install-space-shim` +# bootstrap resource below, full harness only). # Namespace + RuntimeClass live in infra/space/kind/manifests/ so the scripted # bootstrap (infra/space/kind/scripts/bootstrap.sh) and tilt up share one @@ -365,25 +366,6 @@ local_resource( labels = ["bootstrap"], ) -# Advertise the network-slot extended resource on sandbox workers. Every -# sandbox pod requests `space.pplx.net/network-slots: "1"` (see -# `KubernetesSubstrate::build_sandbox_pod`), so without a node advertising -# capacity the scheduler leaves every sandbox pod Pending. Node-controller -# advertises node-manager's `network_pool_max_slots`; locally we advertise a -# static pool to exercise scheduling. JSON-patch `add` on a map key is idempotent -# across re-runs. -local_resource( - "advertise-kind-worker-network-slots", - cmd = """ -for node in $(kubectl --context kind-{cluster} get nodes -l 'space.pplx.net/node-pool=sandbox' -o name); do - kubectl --context kind-{cluster} patch "$node" --subresource=status --type=json \\ - -p '[{{"op":"add","path":"/status/capacity/space.pplx.net~1network-slots","value":"100"}}]' -done -""".format(cluster = KIND_CLUSTER), - labels = ["bootstrap"], - resource_deps = ["label-kind-workers"], -) - # Label the infra node so Postgres, space-api, and space-cluster-controller # can pin there via nodeSelector. These components must not run on sandbox # worker nodes that the drain reconciler can delete. @@ -468,9 +450,8 @@ local_resource( # required: Tilt runs nothing else beside a serial local resource, and this one # runs for minutes. It waits for the serial bootstrap resources because Tilt # holds those while anything else is building. -RUST_ARTIFACTS_AFTER = ["label-kind-workers", "advertise-kind-worker-network-slots", - "label-kind-infra", "sandbox-ssh-secret", "internal-rpc-tls", - "check-kind-node-image", "install-space-shim"] +RUST_ARTIFACTS_AFTER = ["label-kind-workers", "label-kind-infra", "sandbox-ssh-secret", + "internal-rpc-tls", "check-kind-node-image", "install-space-shim"] local_resource( "rust-artifacts", cmd = RUST_ARTIFACT_BUILD, @@ -516,9 +497,8 @@ k8s_resource( # lands, and the rate limiters' store should answer before the first create # reaches them (they would admit it open, not fail it). SPACE_API_DEPS = ["space-postgres", "label-kind-workers", "label-kind-infra", - "advertise-kind-worker-network-slots", "install-space-shim", - "internal-rpc-tls", "space-sandbox-network-policy", "space-db-migrate", - "space-redis"] + "install-space-shim", "internal-rpc-tls", "space-sandbox-network-policy", + "space-db-migrate", "space-redis"] # No port_forward: localhost:8080 is the cluster's host port mapping onto the # space-api-entry NodePort (kind-cluster.yaml.tmpl), which spreads connections # across the autoscaled replicas as a deployment's load balancer does. diff --git a/infra/space/kind/README.md b/infra/space/kind/README.md index badd3d8dc6..a01ac4f008 100644 --- a/infra/space/kind/README.md +++ b/infra/space/kind/README.md @@ -223,9 +223,8 @@ kernel is staged again, so step 2 is optional in practice. It is otherwise idempotent: it creates the kind cluster (if missing) with the SPACE Kind node image on the sandbox workers and builders (see [The node layer](#the-node-layer)), labels -nodes, advertises the `space.pplx.net/network-slots` extended resource, -installs the local observability backends when the `otel` profile is selected, installs -the containerd shim on the sandbox workers, applies the `space` and +nodes, installs the local observability backends when the `otel` profile is +selected, installs the containerd shim on the sandbox workers, applies the `space` and `sandboxes` Namespaces + `RuntimeClass` and the sandbox namespace's NetworkPolicies, builds every image and `kind load`s it (first run is the long step; the @@ -706,9 +705,10 @@ workloads: `datadog` keeps DogStatsD, `otel` selects OTLP, and `off` selects would keep emitting DogStatsD metrics through an otel comparison. Datadog mode keeps the existing node-local shim settings. -Updating the shim binary or wrapper does not restart containerd unless its -configuration also changed. Existing sandbox shims keep their launch settings; -create fresh test-owned sandboxes when changing phases. +Updating the shim binary or wrapper never restarts containerd: the node image +registers the `space` runtime, and `install-shim.sh` only replaces the files +containerd runs for each new sandbox. Existing sandbox shims keep their launch +settings; create fresh test-owned sandboxes when changing phases. `install-prereqs.sh` falls back to user-local installs (`$HOME/.local/bin`, `$HOME/.local/go`) when run without root, so it works on sudo-less hosts @@ -978,7 +978,7 @@ kind cluster: space-local (kind-cluster.yaml.tmpl — 6 nodes) │ ├── space-runtime-supervisor.service (systemd; installed by node-manager's init │ │ container; runs every Firecracker VM and │ │ each sandbox's space-netgtw-svc) -│ ├── containerd-shim-space-v2 (installed into containerd by install-shim.sh) +│ ├── containerd-shim-space-v2 (runtime registered by the node image, binary installed by install-shim.sh) │ ├── one pod per node-service DaemonSet (release `space`): │ │ ├── space-node-manager → host supervisor (root, privileged) │ │ ├── space-runtime-manager (read-only list) (space-rtmgr) @@ -1394,7 +1394,11 @@ asks of the hosting deployment. |-----------------------------------------------------------|-------------------------------------------------------------------------------| | One DaemonSet per node service, systemd watchdogs via k8s probes | One DaemonSet per node service and role from the Kind charts (`charts/space--local`), probed on each service's health socket; runtime-supervisor is the same host systemd unit node-manager's init container installs | | Node image (packer) provides `/space/cache`, the cgroup subtree, loop devices, device permissions, SPACE accounts and the `/space` skeleton before any pod starts | Sandbox workers and builders boot the SPACE Kind node image, which bakes in the same as systemd units that run before kubelet on every node start (see [The node layer](#the-node-layer)) | -| `/space/cache` is btrfs on instance-store NVMe | btrfs on a sparse loopback image at `/var/lib/space-cache.img` on the kind node (40 GiB, same code paths, less I/O headroom), mounted on the node's `/space/cache` by `space-cache.service`, so it outlives pod restarts like the NVMe does | +| `/space/cache` is btrfs on instance-store NVMe, reformatted on every boot | btrfs on a sparse loopback image at `/var/lib/space-cache.img` on the kind node (40 GiB, same code paths, less I/O headroom), mounted `nosuid,noatime` on the node's `/space/cache` by `space-cache.service`, which reformats it on every start of the node container, so like the NVMe it outlives pod restarts but not node restarts | +| The node bootstrap installs a checksum-verified `containerd-shim-space-v2` before containerd starts, and registers the `space` runtime under `io.containerd.cri.v1.runtime` with a `CniConfDir` option | the node image registers the same runtime with the same options (in kind's version 2 config format, which containerd migrates to that plugin), so containerd loads it on its first start; the binary arrives after the cluster exists, copied onto each running sandbox worker by `install-shim.sh`, which checks its SHA-256 on the node. Installing it at boot would need the shim built and staged on a host path before `kind create` in every bring-up path | +| Every node group's bootstrap sets kubelet `eventRecordQPS: 0` and `--fail-swap-on=false` | the same settings on sandbox workers and builders, baked into the node image as a kubelet `--config-dir` drop-in (`node/kubelet/50-space.conf`); the infra node runs the unmodified base image, so its kubelet keeps kind's event rate limit (`failSwapOn` is already `false` there). No sandbox pod runs on it, so none of their events pass through that limit | +| Release binaries read no test hooks | every binary is built with `test-hooks`, and the charts set these hooks on every run: sandbox-manager's on-node snapshot reaper at target fraction 0, interval 15 s, grace 60 s and idle 120 s, against release values of 0.40, 300 s, 1800 s and 86400 s (`charts/space-sandbox-manager-local/values.yaml`, `snapshot_gc.rs`); template-manager's cache GC at interval 15 s and grace 60 s, against 300 s and 1800 s, and its prefetch latest-N at 1, against 5 (`charts/space-template-manager-local/values.yaml`, `eviction.rs`, `prefetch.rs`); the disk-admission reservation below; and api's and internal-api's TokenReview namespace, set to the release namespace that release builds read from the service-account mount. Scenarios shorten further hooks, such as the snapshot-retention sweeper's minimum age and the activation-admission burst, for their own run and restore them. The public-operation dispatcher's tick, claim limit and claim staleness stay at their release values | +| 8 GiB disk-admission reservation per cache allocation (release default) | 1 GiB through the `SPACE_NODE_DISK_ADMISSION_ALLOCATION_RESERVATION_BYTES` test hook, with the 1 GiB `MIN_AVAILABLE_BYTES` floor, so allocations fit the 40 GiB cache (`charts/space-node-service/values.yaml`) | | Postgres on RDS, secrets via ExternalSecrets / AWS Secrets Manager | In-cluster `space-postgres` StatefulSet, one pod on a 1 GiB volume, static Kubernetes Secrets | | PostgreSQL 17 | The same major version (`postgres:17`; CI takes it from the image mirror at a pinned digest) | | Every client connects with `sslmode=require` | The same `sslmode=require` on every connection: the services and the migrate Job through the `space-postgres` Secret's connection string, and the harness's `psql` helpers over loopback TLS inside the pod. The certificate is self-signed per pod start, which `require` accepts because it does not verify the server. Kind's server also refuses unencrypted TCP, so no Kind client runs in plaintext and one that sets `sslmode=disable` fails. A client that omits `sslmode` still connects over TLS, because libpq and sqlx default to `prefer`, so Kind cannot catch a connection string that loses `sslmode=require` | @@ -1452,10 +1456,16 @@ infra node. The builder release selects only `pool=builder`, and the builder nodes are deliberately not `space-enabled`, so user sandbox pods cannot land there. -The two sandbox workers also advertise an extended resource -`space.pplx.net/network-slots: 100` so the kube scheduler can satisfy each -sandbox pod's `space.pplx.net/network-slots: 1` request. Bootstrap advertises -a static pool in place of node-controller's dynamic capacity updates. +Each sandbox pod requests `space.pplx.net/network-slots: 1`. As on a deployed +node, each sandbox worker's node-ctl advertises that extended resource on its +Node's `status.capacity`, from its node-manager's `network_pool_max_slots` +(`crates/libs/node/ctl/server/src/kubernetes_controller/network_slots.rs`), and +re-advertises it on every periodic tick, including after the Node object is +recreated. node-manager keeps no warm pool of slots, as deployed nodes do not +(`SPACE_NODEMGR_NETWORK_POOL_TARGET`, `_RETAIN_READY` and `_MIN_READY` are `0` +in `charts/space-node-manager-local`): each slot, a sandbox's or a build's, is +set up when it is allocated and torn down when it is released, and no slot is +kept ready for the next allocation. ## The node layer @@ -1475,7 +1485,7 @@ these units, enabled in the image: | `space-kind-devices.service` | mode 0666 on `/dev/kvm`, `/dev/vhost-vsock`, `/dev/vhost-net` | the `kvm` group; Kind-only, never on a worker | | `space-cgroup.service` | `/sys/fs/cgroup/space` with `+cpu +memory +io +pids` delegated | `space-cgroup.service` (07-sysctl.sh) | | `space-layout.service` | the SPACE groups and service accounts; the `/space/run` skeleton and dependency-drain gates; protected Firecracker copies under `/space/bin/runtime/firecracker` and `/space/bin/{firecracker,jailer}` links; `/space/etc/sandbox.toml`; `/var/run/netns` | 07-sysctl.sh accounts and directory skeleton, 04-firecracker.sh runtime installation | -| `space-cache.service` | formats `/var/lib/space-cache.img` on first boot (sparse, 40 GiB), mounts it on `/space/cache` with loop autoclear and shared propagation, then creates the cache skeleton | 01-partitions.sh / 11-nvme-cache.sh | +| `space-cache.service` | formats `/var/lib/space-cache.img` on every boot (sparse, 40 GiB), mounts it `nosuid,noatime` on `/space/cache` with loop autoclear and shared propagation, then creates the cache skeleton | 01-partitions.sh / 11-nvme-cache.sh | | `space-kind-vsock-jails.service` | long-running: mknods `/dev/vhost-vsock` into each Firecracker jail under `/space/cache/firecracker/jails` as it appears (the jailer does not) | the node image provides it; Kind-only | They are ordered `Before=kubelet` and run on every boot of the node container, @@ -1484,11 +1494,31 @@ and cgroups; all but the vhost-vsock unit are oneshots. Nothing is installed when the cluster comes up. The sandbox workers and builders run this image (`kind-cluster.yaml.tmpl`); the control-plane and infra nodes run its base unchanged, as production's non-SPACE nodes carry no SPACE node layer, so -every node runs the same Kubernetes version. The cache image sits on the node's -`/var`, which kind keeps on a Docker volume, so the cache survives pod and node -restarts. Defaults are baked like the rest of the layer; a node's -`/etc/space/kind-node.env` can override `SPACE_CACHE_SIZE_MB` and -`SPACE_LOOP_DEVICE_TARGET` for its next boot. +every node runs the same Kubernetes version. The cache survives pod restarts but +not node restarts, as the instance-store cache of a deployed node does: on every +start of the node container `space-cache.service` truncates the image on the +node's `/var` and formats it afresh, so templates, sandboxes and snapshots on the +cache are gone and the node services re-create what they need (template-manager +pulls templates again, node-manager's init container stages the guest kernel +again). What was put on the cache by hand is not re-created: templates the +optional `scripts/bootstrap-prod-seed.sh` staged must be staged again by +re-running it, which skips the workers that still have them. Re-running the +unit within one boot keeps the mounted cache. Defaults are baked like the rest +of the layer; a node's `/etc/space/kind-node.env` can +override `SPACE_CACHE_SIZE_MB` and `SPACE_LOOP_DEVICE_TARGET` for its next boot. + +The image also carries the kubelet and containerd settings a deployed node's +bootstrap applies (`infra/space/aws/eks/modules/space/templates/`): a kubelet +`--config-dir` drop-in (`node/kubelet/50-space.conf`) with no client-side event +rate limit (`eventRecordQPS: 0`) and `failSwapOn: false`, and the `space` +runtime handler with its `CniConfDir` option (`node/containerd/space-runtime.toml`). +The drop-in is written in the version 2 format of kind's own containerd config, +which imports it, so containerd migrates both into the CRI v1 runtime plugin +where a deployed node has the handler, and kind's runc settings (the systemd +cgroup driver, which kubelet adopts from containerd) still apply; the image +build fails otherwise. Check them with +`kubectl get --raw /api/v1/nodes//proxy/configz` and `docker exec +containerd config dump`. The vendor mount remains a read-only, developer-owned download cache. At boot, `space-layout.service` copies the bootstrap Firecracker 1.15.1 binaries into @@ -1525,13 +1555,10 @@ the pod and never reach the node's mount table. Check a node's view with `docker exec findmnt -o TARGET,FSTYPE,PROPAGATION`. Diagnose a node with `docker exec systemctl status 'space-*'` and -`docker exec journalctl -u space-cache`. To start a node's cache clean or -at a new size, delete the node-service DaemonSets of that node's role, run -`systemctl stop space-runtime-supervisor && umount /space/cache && rm -/var/lib/space-cache.img` on the node (adding `SPACE_CACHE_SIZE_MB=` to -`/etc/space/kind-node.env` for a new size), then `systemctl restart -space-cache` and re-apply the charts. Recreating the cluster -also starts clean. +`docker exec journalctl -u space-cache`. To start a node's cache clean, +`docker restart `; for a new size, first add `SPACE_CACHE_SIZE_MB=` +to the node's `/etc/space/kind-node.env`. Recreating the cluster also starts +clean. **Clusters created before this image, or from an older build of it, must be recreated.** A kind node keeps the image it was created from, so such a diff --git a/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml b/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml index de3c6e9281..80ec51ca72 100644 --- a/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml @@ -83,8 +83,6 @@ spec: # BYOK switch here too (same value as the api chart). - name: SPACE_BYOK_VOLUME_ENCRYPTION value: {{ .Values.byokVolumeEncryption | quote }} - - name: SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS - value: {{ .Values.reconcilers.snapshotRetentionSweeper.minAgeSeconds | quote }} - name: SPACE_INTERNAL_TOKEN valueFrom: secretKeyRef: @@ -187,12 +185,10 @@ spec: {{- end }} {{- range $name, $reconciler := .Values.reconcilers }} {{- range $key, $value := (default (dict) $reconciler.env) }} - {{- if ne $key "SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS" }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} {{- end }} - {{- end }} # Ready once the process has reached the Lease (won it, or been validly # denied by a peer), which it signals by touching this file. # diff --git a/infra/space/kind/charts/space-cluster-controller-local/values.yaml b/infra/space/kind/charts/space-cluster-controller-local/values.yaml index 90f7a10d44..bc11fbd05c 100644 --- a/infra/space/kind/charts/space-cluster-controller-local/values.yaml +++ b/infra/space/kind/charts/space-cluster-controller-local/values.yaml @@ -109,15 +109,15 @@ reconcilers: # /v1/sandboxes is POSTed under KubernetesSubstrate. It is ON by default in # the binary, so the explicit enable below is redundant but keeps local # smoke deployments independent of that default; setting it to false leaves - # CreateSandbox returning 202 with the operation `queued` forever. Tight - # ticks/concurrency keep local smoke turnaround snappy. + # CreateSandbox returning 202 with the operation `queued` forever. The tick, + # claim limit and claim staleness are test hooks + # (crates/libs/cluster/controller/server/src/public_operation_dispatcher.rs) + # left at their release values, so Kind picks up queued operations on the + # deployed 25 ms tick. Execution concurrency is a deployment setting. publicOperationDispatcher: env: SPACE_PUBLIC_OPERATION_DISPATCHER_ENABLED: "true" - SPACE_PUBLIC_OPERATION_DISPATCHER_TICK_SECS: "1" - SPACE_PUBLIC_OPERATION_DISPATCHER_CLAIM_LIMIT: "64" SPACE_PUBLIC_OPERATION_DISPATCHER_EXECUTION_CONCURRENCY: "16" - SPACE_PUBLIC_OPERATION_DISPATCHER_CLAIM_STALE_SECS: "60" # These repairs are live when enabled. Keep kind explicitly disabled until # the end-to-end and smoke suites exercise their default-on behavior. failedRestorableRepair: @@ -179,8 +179,11 @@ reconcilers: stuckSnapshotReaper: env: SPACE_STUCK_SNAPSHOT_REAPER_ENABLED: "false" + # The minimum age (SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS) is a test + # hook left at its release value, which equals the API's fork-retention grace + # (crates/libs/common/core/src/types/snapshot.rs). The retention scenarios + # shorten it for their own run and restore it afterwards. snapshotRetentionSweeper: - minAgeSeconds: "3600" env: SPACE_SNAPSHOT_RETENTION_SWEEPER_ENABLED: "true" snapshotStorageSweeper: diff --git a/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml b/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml index 588c1be6eb..188acf6475 100644 --- a/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml +++ b/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml @@ -2,10 +2,9 @@ {{- $name := include "space-node-service.name" $ctx -}} # Same RBAC shape as the prod space-node-ctl chart: node-ctl's Kubernetes # node-local controller watches its own Node and advertises the -# space.pplx.net/network-slots extended resource on status.capacity (the -# production source of the capacity bootstrap.sh also seeds statically). Only -# node-ctl's ServiceAccount holds it; no other node service talks to the -# Kubernetes API. +# space.pplx.net/network-slots extended resource on status.capacity, the only +# source of that capacity here as on a deployed node. Only node-ctl's +# ServiceAccount holds it; no other node service talks to the Kubernetes API. {{ include "space-node-service.serviceAccount" $ctx }} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml b/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml index 970a7272d9..248f37494e 100644 --- a/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml +++ b/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml @@ -258,6 +258,28 @@ spec: # Sandbox pods own their network namespaces (KubernetesSubstrate). - name: SPACE_NODEMGR_SERVICE_PORT_INGRESS_ISOLATION value: shared_host + # No warm network-slot pool, as deployed nodes run. Each slot is its + # own network namespace, bridged to the sandbox pod's namespace or, + # for an allocation without one, to the node's; claim_allocation + # (crates/libs/node/mgr/server/src/slot_pool.rs) serves only the + # latter from the pool. Sandbox pods bring their own namespace + # (KubernetesSubstrate), so their slots are set up on allocation + # and torn down on release either way. The binary defaults also + # keep max(target, retained) slots, every slot the node has, ready + # and bridged to the node (the fill loop in slot_pool/worker.rs): + # a sandbox claim then takes one of those and rebuilds it for the + # pod, and the pool rebuilds it after release. build-manager + # allocates without a namespace, so with the pool off builders set + # up their slots on demand instead of leasing pooled ones. With no + # ready minimum the pool reports healthy while empty (pool_status + # in service/network_helpers.rs). Read at startup in + # crates/libs/node/mgr/server/src/run.rs. + - name: SPACE_NODEMGR_NETWORK_POOL_TARGET + value: "0" + - name: SPACE_NODEMGR_NETWORK_POOL_RETAIN_READY + value: "0" + - name: SPACE_NODEMGR_NETWORK_POOL_MIN_READY + value: "0" {{- if ne (toString .Values.loopDeviceTarget) "" }} - name: SPACE_LOOP_DEVICE_TARGET value: {{ .Values.loopDeviceTarget | quote }} diff --git a/infra/space/kind/charts/space-node-service/values.yaml b/infra/space/kind/charts/space-node-service/values.yaml index 1155ab761e..c4fcd2f319 100644 --- a/infra/space/kind/charts/space-node-service/values.yaml +++ b/infra/space/kind/charts/space-node-service/values.yaml @@ -37,11 +37,14 @@ node: credentialsSecretName: space-object-store endpoint: http://object-store:8333 region: us-east-1 - # Disk-admission sizing for /space/cache. Production defaults assume NVMe + # Disk-admission sizing for /space/cache. Release defaults assume NVMe # (50 GiB min-available floor, 8 GiB per-sandbox allocation reservation); # the Kind cache is a 40 GiB sparse loopback btrfs image, so both byte knobs - # scale to 1 GiB through the same production config surface. Without this - # every node publishes scheduling_status=full and sandbox-manager rejects + # scale to 1 GiB. The floor is a deployment setting; the reservation is a + # test hook that only the Kind build's test-hooks feature reads, and release + # binaries always reserve 8 GiB + # (crates/libs/common/core/src/host/disk_admission.rs). Without this every + # node publishes scheduling_status=full and sandbox-manager rejects # prepare_sandbox with disk_available_below_threshold. Read by the services # that admit cache allocations: node-manager, sandbox-manager, # volume-manager and build-manager. diff --git a/infra/space/kind/manifests/runtimeclass.yaml b/infra/space/kind/manifests/runtimeclass.yaml index a715003990..d04401a1f5 100644 --- a/infra/space/kind/manifests/runtimeclass.yaml +++ b/infra/space/kind/manifests/runtimeclass.yaml @@ -1,16 +1,20 @@ # KubernetesSubstrate pins per-sandbox pods to runtimeClassName: space. -# `infra/space/kind/scripts/install-shim.sh` installs the SPACE containerd -# shim (`io.containerd.space.v2`) on each kind sandbox worker and registers -# a `space` CRI runtime that points at it. The shim hijacks the per-sandbox -# pod's pause container, reads the SPACE annotations (sandbox-id, vcpu, -# memory, template, …), and calls into the local sandbox-manager UDS to -# create a real sandbox via node-manager, which launches Firecracker through -# the node's runtime-supervisor, as in production. +# The SPACE Kind node image registers the `space` CRI runtime handler with +# containerd (infra/space/kind/node/containerd/space-runtime.toml), pointing at +# the SPACE containerd shim (`io.containerd.space.v2`), and +# `infra/space/kind/scripts/install-shim.sh` installs the shim binary on each +# kind sandbox worker. The shim hijacks the per-sandbox pod's pause container, +# reads the SPACE annotations (sandbox-id, vcpu, memory, template, …), and +# calls into the local sandbox-manager UDS to create a real sandbox via +# node-manager, which launches Firecracker through the node's +# runtime-supervisor, as in production. # -# If the shim isn't installed (e.g. on a fresh cluster before bootstrap.sh -# has run install-shim.sh), kubelet will fail pod-start with -# "runtime handler 'space' is not configured" and the api-server will report -# `proxy_unreachable`. Run install-shim.sh and recreate the affected pods. +# If the shim binary isn't installed (e.g. on a fresh cluster before +# bootstrap.sh has run install-shim.sh), containerd cannot start the pod's +# sandbox and kubelet reports FailedCreatePodSandBox. Run install-shim.sh and +# recreate the affected pods; containerd is not restarted. A node whose image +# predates the registration has no `space` handler at all; recreate the cluster +# (scripts/check-kind-node-image.sh). apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: diff --git a/infra/space/kind/node/Dockerfile b/infra/space/kind/node/Dockerfile index 11529e60e5..be2d72249f 100644 --- a/infra/space/kind/node/Dockerfile +++ b/infra/space/kind/node/Dockerfile @@ -30,6 +30,34 @@ COPY --chmod=0755 infra/space/kind/node/space-kind-node.sh /usr/local/lib/space/ COPY --chmod=0644 scripts/lib/space-loop-devices.sh /usr/local/lib/space/kind-node/space-loop-devices.sh COPY --chmod=0644 infra/space/identity/node-identities.sh /usr/local/lib/space/kind-node/node-identities.sh COPY --chmod=0644 infra/space/kind/node/systemd/ /etc/systemd/system/ +# The kubelet and containerd settings a deployed node's bootstrap applies before +# either starts (infra/space/aws/eks/modules/space/templates/ +# worker-bootstrap.sh.tftpl). Baked in rather than patched into the cluster +# config, so a cluster created without them runs another image tag and +# check-kind-node-image.sh refuses it. kind's kubelet unit takes extra flags from +# /etc/default/kubelet, and its containerd config (version 2) gains an import of +# the version 2 drop-in. The build fails unless the configuration containerd +# loads has the `space` runtime with its option and still has kind's own +# settings: runc's systemd cgroup driver, which kubelet adopts from containerd, +# and discard_unpacked_layers (see the drop-in for how a version mismatch loses +# them). The awk prints each `table|key = value` of the dump. +COPY --chmod=0644 infra/space/kind/node/kubelet/50-space.conf /etc/kubernetes/kubelet.conf.d/50-space.conf +COPY --chmod=0644 infra/space/kind/node/containerd/space-runtime.toml /etc/containerd/conf.d/space-runtime.toml +RUN sed -i 's|^KUBELET_EXTRA_ARGS=|KUBELET_EXTRA_ARGS=--config-dir=/etc/kubernetes/kubelet.conf.d |' /etc/default/kubelet \ + && grep -q '^KUBELET_EXTRA_ARGS=--config-dir=/etc/kubernetes/kubelet.conf.d ' /etc/default/kubelet \ + && sed -i '0,/^version = 2$/s||version = 2\nimports = ["/etc/containerd/conf.d/*.toml"]|' /etc/containerd/config.toml \ + && containerd config dump 2>/dev/null \ + | awk '/^ *\[/ { table = $0; gsub(/^ *\[|\] *$/, "", table); next } NF { sub(/^ */, ""); print table "|" $0 }' \ + > /tmp/containerd-config \ + && for want in \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space|runtime_type = 'io.containerd.space.v2'" \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space|pod_annotations = ['space.pplx.net/*']" \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space.options|CniConfDir = '/etc/cni/net.d'" \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options|SystemdCgroup = true" \ + "plugins.'io.containerd.cri.v1.images'|discard_unpacked_layers = true"; do \ + grep -qxF "$want" /tmp/containerd-config || { echo "containerd config lacks: $want" >&2; exit 1; }; \ + done \ + && rm /tmp/containerd-config RUN systemctl enable \ space-loop-nodes.service \ space-kind-devices.service \ diff --git a/infra/space/kind/node/containerd/space-runtime.toml b/infra/space/kind/node/containerd/space-runtime.toml new file mode 100644 index 0000000000..48fc71402b --- /dev/null +++ b/infra/space/kind/node/containerd/space-runtime.toml @@ -0,0 +1,29 @@ +# The `space` runtime handler RuntimeClass `space` names +# (infra/space/kind/manifests/runtimeclass.yaml), registered with the options a +# deployed node's bootstrap gives it (infra/space/aws/eks/modules/space/ +# templates/worker-bootstrap.sh.tftpl, `spec.containerd.config`). Baked into the +# Kind node image, whose containerd config imports this file, so containerd +# loads it when it first starts and is never restarted to pick it up. +# +# Version 2, the version of the kind config.toml that imports it. containerd +# merges an import over that config, its version included, and migrates the +# result to version 3 only when the merged version is below 3. A version 3 file +# here would make the merged config version 3 and leave kind's own settings +# (runc's SystemdCgroup and base_runtime_spec, discard_unpacked_layers, the +# test-handler runtime) unmigrated under io.containerd.grpc.v1.cri, which the version 3 +# CRI plugins do not read: containerd would report the cgroupfs driver, and +# kubelet would adopt it. Written as version 2, this table is migrated with +# kind's into io.containerd.cri.v1.runtime, where a deployed node's containerd +# has it. node/Dockerfile fails the build unless both survive. +version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.space] + # containerd runs /usr/local/bin/containerd-shim-space-v2, which + # scripts/install-shim.sh installs. + runtime_type = "io.containerd.space.v2" + # The shim reads its instructions (sandbox id, owner, template, sizing) from + # the space.pplx.net/* pod annotations in the OCI spec; containerd forwards + # only the annotations listed here. + pod_annotations = ["space.pplx.net/*"] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.space.options] + CniConfDir = "/etc/cni/net.d" diff --git a/infra/space/kind/node/kubelet/50-space.conf b/infra/space/kind/node/kubelet/50-space.conf new file mode 100644 index 0000000000..bcf3acc053 --- /dev/null +++ b/infra/space/kind/node/kubelet/50-space.conf @@ -0,0 +1,13 @@ +# Kubelet settings a deployed SPACE node sets in its bootstrap +# (infra/space/aws/eks/modules/space/templates/worker-bootstrap.sh.tftpl and +# bootstrap.sh.tftpl, `spec.kubelet`). Baked into the Kind node image, which +# points kubelet's --config-dir at this file's directory (node/Dockerfile); +# kubelet applies it over the configuration kubeadm writes at join. +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +# No client-side event rate limit. A burst of sandbox creates emits a burst of +# pod events (FailedCreatePodSandBox from the shim among them), which a limited +# kubelet delays and, once its queue is full, drops. +eventRecordQPS: 0 +# Deployed nodes pass --fail-swap-on=false. +failSwapOn: false diff --git a/infra/space/kind/node/space-kind-node.sh b/infra/space/kind/node/space-kind-node.sh index 5d77da3c50..6d72c5a333 100755 --- a/infra/space/kind/node/space-kind-node.sh +++ b/infra/space/kind/node/space-kind-node.sh @@ -25,8 +25,9 @@ # # A container's /dev, mounts and cgroups do not survive `docker restart`, which # is why these are boot units and not a one-time setup. Each step is -# idempotent: after a restart the cache image and the directories are already -# in place, and re-running a unit by hand on a live node is safe. +# idempotent within a boot, so re-running a unit by hand on a live node is +# safe. The cache is the one step that does not carry state across boots: as on +# a deployed node, every boot starts it as a fresh filesystem. # # Defaults are baked like the rest of the layer; /etc/space/kind-node.env, when # present, overrides SPACE_CACHE_SIZE_MB and SPACE_LOOP_DEVICE_TARGET. @@ -44,9 +45,13 @@ LIB_DIR=/usr/local/lib/space/kind-node STEP="${1:-}" CACHE_DIR=/space/cache -# On the node's /var, which kind keeps on a volume, so the cache outlives node -# restarts as well as pod restarts, as the instance-store disk does on EKS. +# On the node's /var, which kind keeps on a volume. The image file outlives a +# node restart; its filesystem does not (see cache below). CACHE_IMAGE=/var/lib/space-cache.img +# Marks the cache formatted and mounted for this boot. kind mounts the node's +# /run as a tmpfs, so a node start finds it absent. +CACHE_STATE_DIR=/run/space-kind-node +CACHE_BOOT_MARKER="$CACHE_STATE_DIR/cache.done" CACHE_SIZE_MB="${SPACE_CACHE_SIZE_MB:-40960}" log() { printf '%s\n' "$*"; } @@ -221,9 +226,12 @@ layout() { # ── btrfs /space/cache ─────────────────────────────────────────────────────── # sandbox-manager clones template rootfs into instance dirs with btrfs subvolume -# snapshots, so the cache must be btrfs. Production formats instance-store NVMe -# (or a loopback image on nodes without one); here it is a sparse loopback image -# on the node, formatted once and reused on every later start. +# snapshots, so the cache must be btrfs. A deployed node formats its +# instance-store NVMe on every boot (packer 11-nvme-cache.sh), so templates, +# sandboxes and snapshots on the cache never survive a node restart, and node +# services re-create what they need on it. Here the cache is a sparse loopback +# image on the node, reformatted the same way on every start of the node +# container. cache_is_mounted() { findmnt -n -t btrfs --mountpoint "$CACHE_DIR" >/dev/null 2>&1 } @@ -232,18 +240,32 @@ cache_image_is_btrfs() { [[ -s $CACHE_IMAGE ]] && [[ "$(blkid -p -s TYPE -o value "$CACHE_IMAGE" 2>/dev/null)" == btrfs ]] } +# A loop device still bound to the image (a crash before autoclear was armed) +# would keep serving the old filesystem's blocks over the new one. `losetup -j` +# matches an existing file by device and inode, but a missing one by path, and +# the host kernel's loop devices are shared by every kind node, each with an +# image at this path: without the image there is nothing of this node's to +# release, and matching by path would find the other nodes' caches. +release_cache_image() { + local loopdev + [[ -e $CACHE_IMAGE ]] || return 0 + while read -r loopdev; do + [[ -n $loopdev ]] || continue + findmnt -n --source "$loopdev" >/dev/null 2>&1 \ + && die "$loopdev (backing $CACHE_IMAGE) is still mounted; restart the node to start its cache fresh" + space_loop_detach "$loopdev" || true + done < <(losetup -l -n -O NAME -j "$CACHE_IMAGE" 2>/dev/null | tr -d '[:blank:]') + [[ -z "$(losetup -l -n -O NAME -j "$CACHE_IMAGE" 2>/dev/null)" ]] \ + || die "a loop device still holds $CACHE_IMAGE; restart the node to start its cache fresh" +} + format_cache_image() { - # A populated image that is not btrfs (a format interrupted between truncate - # and mkfs, or damage) would fail to mount on every start until someone - # deleted it; reformat it like an empty one. - if [[ -s $CACHE_IMAGE ]]; then - log "backing image $CACHE_IMAGE is not a btrfs filesystem; reformatting it" - else - log "creating btrfs backing image $CACHE_IMAGE (${CACHE_SIZE_MB}M, sparse)" - fi + log "formatting btrfs backing image $CACHE_IMAGE (${CACHE_SIZE_MB}M, sparse)" + release_cache_image # Sparse (truncate, not fallocate): the virtual size gives disk admission # production-like headroom while real blocks are only consumed as sandboxes - # write. + # write. Truncating to zero first also returns the previous boot's blocks to + # the host. truncate -s 0 "$CACHE_IMAGE" truncate -s "${CACHE_SIZE_MB}M" "$CACHE_IMAGE" mkfs.btrfs -f -L space-cache "$CACHE_IMAGE" >/dev/null @@ -268,7 +290,8 @@ mount_cache_image() { die "no loop device available for $CACHE_IMAGE" fi fi - if ! mount -o nosuid "$loopdev" "$CACHE_DIR"; then + # noatime as deployed nodes mount the cache (packer 11-nvme-cache.sh). + if ! mount -o nosuid,noatime "$loopdev" "$CACHE_DIR"; then (( reused )) || space_loop_detach "$loopdev" || true die "mount $loopdev $CACHE_DIR failed (image $CACHE_IMAGE; if this repeats, delete the image on the node)" fi @@ -298,8 +321,14 @@ cache_skeleton() { install -d -m 0755 -o 0 -g 0 "$CACHE_DIR/firecracker/jails" } +# The first run in a boot formats the image; a later run in the same boot (the +# unit re-run by hand) keeps the mounted cache, or remounts the image if it was +# unmounted, reformatting it only if it is no longer btrfs. The marker is +# written last, so a run that failed partway formats again. cache() { - install -d -m 0755 -o 0 -g 0 /space "$CACHE_DIR" + install -d -m 0755 -o 0 -g 0 /space "$CACHE_DIR" "$CACHE_STATE_DIR" + exec 9>"$CACHE_STATE_DIR/cache.lock" + flock 9 if cache_is_mounted; then log "$CACHE_DIR is already btrfs" else @@ -309,7 +338,9 @@ cache() { command -v mkfs.btrfs >/dev/null \ || die "mkfs.btrfs missing; this node is not running the SPACE Kind node image (infra/space/kind/node/Dockerfile)" space_loop__losetup_bin >/dev/null || die "losetup missing (util-linux)" - cache_image_is_btrfs || format_cache_image + if [[ ! -e $CACHE_BOOT_MARKER ]] || ! cache_image_is_btrfs; then + format_cache_image + fi mount_cache_image fi # Shared, so a pod that receives the cache HostToContainer also receives a @@ -319,9 +350,10 @@ cache() { local img_mb img_mb=$(( $(stat -c %s "$CACHE_IMAGE") / 1048576 )) if (( img_mb != CACHE_SIZE_MB )); then - log "WARNING: the cache size is ${CACHE_SIZE_MB}M (SPACE_CACHE_SIZE_MB) but the reused image is ${img_mb}M; see \"The node layer\" in infra/space/kind/README.md to resize" + log "WARNING: the cache size is ${CACHE_SIZE_MB}M (SPACE_CACHE_SIZE_MB) but the image formatted this boot is ${img_mb}M; restart the node to reformat it at the new size" fi cache_skeleton + touch "$CACHE_BOOT_MARKER" } # ── vhost-vsock nodes in Firecracker jails (Kind only) ─────────────────────── diff --git a/infra/space/kind/scripts/bootstrap-prod-seed.sh b/infra/space/kind/scripts/bootstrap-prod-seed.sh index 5015591ba8..7429e8f878 100755 --- a/infra/space/kind/scripts/bootstrap-prod-seed.sh +++ b/infra/space/kind/scripts/bootstrap-prod-seed.sh @@ -25,6 +25,13 @@ # TEMPLATE_BOOTSTRAP_NAMES — comma/space-separated. Default: seed. # # This script is idempotent: re-running it skips work that's already done. +# +# Re-run it after a sandbox worker restarts. The staged files live on the +# worker's /space/cache, which space-cache.service reformats on every boot of +# the node (infra/space/kind/node/space-kind-node.sh), and nothing else puts +# them back: template-manager cannot pull them from this bucket. A re-run +# stages the templates onto the restarted worker again and skips the workers +# that still have them on their cache. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -136,12 +143,15 @@ SQL local s3_path="${storage_uri%%\?*}" s3_path="${s3_path%/}" - # If the worker already has this hash registered, skip — idempotency. - # `space-tmplmgr list` truncates the hash to ~12 hex chars, so grep on - # the bare prefix (post-`sha256:`). + # If the worker already has this hash registered and its build directory + # is on the cache, skip — idempotency. `space-tmplmgr list` truncates the + # hash to ~12 hex chars, so grep on the bare prefix (post-`sha256:`). The + # directory check covers a worker restarted moments ago, whose catalog + # can still list the build until template-manager's startup recovery + # prunes it. local hex_prefix="${hash#sha256:}" hex_prefix="${hex_prefix:0:12}" - if kubectl -n space exec "$pod" -c shell -- bash -c "space-tmplmgr list 2>/dev/null | grep -q $hex_prefix" 2>/dev/null; then + if kubectl -n space exec "$pod" -c shell -- bash -c "space-tmplmgr list 2>/dev/null | grep -q $hex_prefix && compgen -G '/space/cache/firecracker/templates/${name}/*${hex_prefix}*' >/dev/null" 2>/dev/null; then log " $name@${hash:0:24}… already staged locally; skip" continue fi diff --git a/infra/space/kind/scripts/bootstrap.sh b/infra/space/kind/scripts/bootstrap.sh index 46974e82c3..2dceb57b57 100755 --- a/infra/space/kind/scripts/bootstrap.sh +++ b/infra/space/kind/scripts/bootstrap.sh @@ -258,26 +258,18 @@ apply_node_labels() { } apply_node_labels -# ── Step 3: extended resource — network slots ───────────────────────────────── +# ── Step 3: network-slot capacity ──────────────────────────────────────────── # Every sandbox pod requests `space.pplx.net/network-slots: "1"` (set by -# KubernetesSubstrate::build_sandbox_pod). Without a node advertising -# capacity, the kube scheduler leaves sandbox pods Pending forever. In prod -# space-node-ctl patches each Node's status.capacity from node-manager's -# `network_pool_max_slots`; locally we advertise a static pool. -log "advertising network-slots capacity on sandbox workers" -for node in $(kubectl --context "$KCTX" get nodes -l space.pplx.net/node-pool=sandbox -o name); do - # JSON-patch `add` on a map key is idempotent. ~1 in the json-pointer is - # the escape for `/` inside the key `space.pplx.net/network-slots`. - kubectl --context "$KCTX" patch "$node" --subresource=status --type=json \ - -p '[{"op":"add","path":"/status/capacity/space.pplx.net~1network-slots","value":"100"}]' \ - >/dev/null || true -done +# KubernetesSubstrate::build_sandbox_pod). Each node's space-node-ctl advertises +# its node-manager's `network_pool_max_slots` on the Node's status.capacity, as +# on a deployed node (crates/libs/node/ctl/server/src/kubernetes_controller/ +# network_slots.rs). Nothing seeds it here, as nothing does on a deployed node. # ── Step 4: install the SPACE containerd shim on each sandbox worker ───────── # Without this, RuntimeClass `space` (handler: space) is unresolvable and -# every per-sandbox pod fails to start. install-shim.sh is idempotent: it -# only builds the shim if needed and only edits containerd config.toml if -# the runtime stanza isn't already present. +# every per-sandbox pod fails to start. The node image registers the runtime +# with containerd; install-shim.sh installs the binary containerd runs for it, +# and is idempotent: it only builds the shim if needed. # Reconcile receivers and owned log collection before changing producers. # Local node-side endpoints stay at the Kind profiles' fixed ClusterIPs, so # already-running exporters can also reconnect after a backend restart. diff --git a/infra/space/kind/scripts/install-shim.sh b/infra/space/kind/scripts/install-shim.sh index db9cef4436..2f05826ae8 100755 --- a/infra/space/kind/scripts/install-shim.sh +++ b/infra/space/kind/scripts/install-shim.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -# install-shim.sh — build `containerd-shim-space-v2` and wire it into each -# kind worker's containerd, so per-sandbox pods scheduled with +# install-shim.sh — build `containerd-shim-space-v2` and install it on each +# kind sandbox worker, so per-sandbox pods scheduled with # `runtimeClassName: space` get hijacked by the SPACE shim (which delegates # to sandbox-manager → node-manager → guest runtime) instead of running as -# a plain runc container. +# a plain runc container. The node image registers the runtime with +# containerd; this installs the binary containerd runs for it. # -# Without this script, RuntimeClass `space` is unresolvable and every -# per-sandbox pod fails to start with "runtime handler not configured". +# Without the shim, every per-sandbox pod that RuntimeClass `space` routes to +# it fails to start. # # Idempotent. Safe to re-run after a kind cluster recreate. # @@ -123,61 +124,40 @@ list_workers_fallback() { done } -# ── 3. Install the shim binary + containerd config on one kind node ─────────── +# ── 3. Install the shim binary on one kind node ───────────────────────────── +# The `space` runtime itself is registered by the SPACE Kind node image +# (infra/space/kind/node/containerd/space-runtime.toml), under containerd's CRI +# v1 runtime plugin as a deployed node registers it, so containerd has it from +# its first start and is never edited or restarted here. containerd resolves +# /usr/local/bin/containerd-shim-space-v2 when each sandbox pod starts, so a +# replaced binary takes effect for the next sandbox. install_on_node() { local node=$1 log "installing shim on $node" - local prior_config - prior_config=$(docker exec "$node" sha256sum /etc/containerd/config.toml) + docker exec "$node" containerd config dump 2>/dev/null \ + | grep -F "[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space]" >/dev/null \ + || die "$node's containerd has no 'space' runtime; the node predates the SPACE Kind node image that registers it, so recreate the cluster (scripts/check-kind-node-image.sh)" docker exec "$node" mkdir -p "$(dirname "$SHIM_REAL_BIN")" - docker cp "$SHIM_BIN" "$node:${SHIM_REAL_BIN}.tmp" - docker exec "$node" sh -c \ - "chown root:root '${SHIM_REAL_BIN}.tmp' && chmod 0755 '${SHIM_REAL_BIN}.tmp' && mv -f '${SHIM_REAL_BIN}.tmp' '$SHIM_REAL_BIN'" - docker cp "$SHIM_WRAPPER" "$node:/usr/local/bin/.containerd-shim-space-v2.tmp" - docker exec "$node" sh -c \ - "chown root:root /usr/local/bin/.containerd-shim-space-v2.tmp && chmod 0755 /usr/local/bin/.containerd-shim-space-v2.tmp && mv -f /usr/local/bin/.containerd-shim-space-v2.tmp /usr/local/bin/containerd-shim-space-v2" - - # Add the `space` CRI runtime to containerd's config if not already there. - # The runtime_type is the shim's protocol name — kind's containerd looks for - # /usr/local/bin/containerd-shim--. The bash - # wrapper installed above now occupies that path and execs the compiled - # binary from /usr/local/lib/space. - # NOTE: heredoc delimiter is QUOTED ('"'"'EOF'"'"') so sh doesn't run command - # substitution on backticks inside the comment block (a previous version - # without quotes had sh trying to exec the literal word "space"). - # - # pod_annotations is the allowlist containerd uses to decide which K8s pod - # annotations get propagated into the OCI runtime spec. The SPACE shim - # reads its instructions (sandbox-id, owner, vcpu, memory, template, etc.) - # off the OCI spec, so without this allowlist every pod create fails with - # "space.pplx.net/sandbox-owner or space.pplx.net/sandbox-user is required". - # `space.pplx.net/*` matches every SPACE annotation in one rule. - docker exec "$node" sh -c ' - cfg=/etc/containerd/config.toml - grep -q "io.containerd.space.v2" "$cfg" && exit 0 - cat >>"$cfg" <<'"'"'EOF'"'"' - -# Added by infra/space/kind/scripts/install-shim.sh — kind-only. -# Provides RuntimeClass "space" -> containerd-shim-space-v2, which bridges -# kubelet pod-start into the per-node sandbox-manager UDS so per-sandbox -# pods run as real SPACE sandboxes (guest VM via node-manager) instead of -# plain runc containers. -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.space] - runtime_type = "io.containerd.space.v2" - pod_annotations = ["space.pplx.net/*"] -EOF - ' + install_file "$node" "$SHIM_BIN" "$SHIM_REAL_BIN" + install_file "$node" "$SHIM_WRAPPER" /usr/local/bin/containerd-shim-space-v2 +} - # Restart containerd inside the kind node so it picks up the new runtime. - # kind nodes run systemd, so the standard systemctl path works. (We - # tolerate failure here — on the first install the kubelet briefly loses - # the apiserver socket; it reconnects within a few seconds.) - if [[ "$(docker exec "$node" sha256sum /etc/containerd/config.toml)" != "$prior_config" ]]; then - docker exec "$node" systemctl restart containerd - else - log "containerd configuration unchanged; new sandbox shims use the updated binary" +# Copy beside the destination, check the copy against the source's SHA-256 (as +# a deployed node's bootstrap checks the shim it downloads), then rename it +# into place. +install_file() { + local node=$1 source=$2 dest=$3 want got + local staged + staged="$(dirname "$dest")/.$(basename "$dest").tmp" + want="$(sha256sum "$source" | cut -d' ' -f1)" + docker cp "$source" "$node:$staged" + got="$(docker exec "$node" sha256sum "$staged" | cut -d' ' -f1)" + if [[ "$got" != "$want" ]]; then + docker exec "$node" rm -f "$staged" || true + die "$dest on $node: copied file has SHA-256 $got, expected $want" fi + docker exec "$node" sh -c "chown root:root '$staged' && chmod 0755 '$staged' && mv -f '$staged' '$dest'" } # ── Driver ─────────────────────────────────────────────────────────────────── diff --git a/infra/space/kind/scripts/kind-node-image.sh b/infra/space/kind/scripts/kind-node-image.sh index 0201880818..0e4108cfc0 100755 --- a/infra/space/kind/scripts/kind-node-image.sh +++ b/infra/space/kind/scripts/kind-node-image.sh @@ -36,6 +36,8 @@ DOCKERFILE=infra/space/kind/node/Dockerfile INPUTS=( "${DOCKERFILE}" infra/space/kind/node/space-kind-node.sh + infra/space/kind/node/containerd/space-runtime.toml + infra/space/kind/node/kubelet/50-space.conf infra/space/kind/node/systemd/space-cache.service infra/space/kind/node/systemd/space-cgroup.service infra/space/kind/node/systemd/space-kind-devices.service