diff --git a/Tiltfile b/Tiltfile index 63adb85e42..a63670a955 100644 --- a/Tiltfile +++ b/Tiltfile @@ -328,9 +328,10 @@ custom_build( # `KubernetesSubstrate::build_sandbox_pod` pins per-sandbox pods to # `runtimeClassName: space` and `nodeSelector: node.kubernetes.io/space-enabled=true`. # Neither exists in vanilla KinD, so we materialize them before the charts -# land. RuntimeClass `space` resolves to containerd-shim-space-v2 once -# install-shim.sh has wired it into each sandbox worker's containerd (the -# `install-space-shim` bootstrap resource below, full harness only). +# land. The SPACE Kind node image registers RuntimeClass `space`'s handler with +# containerd; it resolves to containerd-shim-space-v2 once install-shim.sh has +# installed that binary on each sandbox worker (the `install-space-shim` +# bootstrap resource below, full harness only). # Namespace + RuntimeClass live in infra/space/kind/manifests/ so the scripted # bootstrap (infra/space/kind/scripts/bootstrap.sh) and tilt up share one @@ -365,25 +366,6 @@ local_resource( labels = ["bootstrap"], ) -# Advertise the network-slot extended resource on sandbox workers. Every -# sandbox pod requests `space.pplx.net/network-slots: "1"` (see -# `KubernetesSubstrate::build_sandbox_pod`), so without a node advertising -# capacity the scheduler leaves every sandbox pod Pending. Node-controller -# advertises node-manager's `network_pool_max_slots`; locally we advertise a -# static pool to exercise scheduling. JSON-patch `add` on a map key is idempotent -# across re-runs. -local_resource( - "advertise-kind-worker-network-slots", - cmd = """ -for node in $(kubectl --context kind-{cluster} get nodes -l 'space.pplx.net/node-pool=sandbox' -o name); do - kubectl --context kind-{cluster} patch "$node" --subresource=status --type=json \\ - -p '[{{"op":"add","path":"/status/capacity/space.pplx.net~1network-slots","value":"100"}}]' -done -""".format(cluster = KIND_CLUSTER), - labels = ["bootstrap"], - resource_deps = ["label-kind-workers"], -) - # Label the infra node so Postgres, space-api, and space-cluster-controller # can pin there via nodeSelector. These components must not run on sandbox # worker nodes that the drain reconciler can delete. @@ -468,9 +450,8 @@ local_resource( # required: Tilt runs nothing else beside a serial local resource, and this one # runs for minutes. It waits for the serial bootstrap resources because Tilt # holds those while anything else is building. -RUST_ARTIFACTS_AFTER = ["label-kind-workers", "advertise-kind-worker-network-slots", - "label-kind-infra", "sandbox-ssh-secret", "internal-rpc-tls", - "check-kind-node-image", "install-space-shim"] +RUST_ARTIFACTS_AFTER = ["label-kind-workers", "label-kind-infra", "sandbox-ssh-secret", + "internal-rpc-tls", "check-kind-node-image", "install-space-shim"] local_resource( "rust-artifacts", cmd = RUST_ARTIFACT_BUILD, @@ -516,9 +497,8 @@ k8s_resource( # lands, and the rate limiters' store should answer before the first create # reaches them (they would admit it open, not fail it). SPACE_API_DEPS = ["space-postgres", "label-kind-workers", "label-kind-infra", - "advertise-kind-worker-network-slots", "install-space-shim", - "internal-rpc-tls", "space-sandbox-network-policy", "space-db-migrate", - "space-redis"] + "install-space-shim", "internal-rpc-tls", "space-sandbox-network-policy", + "space-db-migrate", "space-redis"] # No port_forward: localhost:8080 is the cluster's host port mapping onto the # space-api-entry NodePort (kind-cluster.yaml.tmpl), which spreads connections # across the autoscaled replicas as a deployment's load balancer does. diff --git a/infra/space/kind/README.md b/infra/space/kind/README.md index badd3d8dc6..a01ac4f008 100644 --- a/infra/space/kind/README.md +++ b/infra/space/kind/README.md @@ -223,9 +223,8 @@ kernel is staged again, so step 2 is optional in practice. It is otherwise idempotent: it creates the kind cluster (if missing) with the SPACE Kind node image on the sandbox workers and builders (see [The node layer](#the-node-layer)), labels -nodes, advertises the `space.pplx.net/network-slots` extended resource, -installs the local observability backends when the `otel` profile is selected, installs -the containerd shim on the sandbox workers, applies the `space` and +nodes, installs the local observability backends when the `otel` profile is +selected, installs the containerd shim on the sandbox workers, applies the `space` and `sandboxes` Namespaces + `RuntimeClass` and the sandbox namespace's NetworkPolicies, builds every image and `kind load`s it (first run is the long step; the @@ -706,9 +705,10 @@ workloads: `datadog` keeps DogStatsD, `otel` selects OTLP, and `off` selects would keep emitting DogStatsD metrics through an otel comparison. Datadog mode keeps the existing node-local shim settings. -Updating the shim binary or wrapper does not restart containerd unless its -configuration also changed. Existing sandbox shims keep their launch settings; -create fresh test-owned sandboxes when changing phases. +Updating the shim binary or wrapper never restarts containerd: the node image +registers the `space` runtime, and `install-shim.sh` only replaces the files +containerd runs for each new sandbox. Existing sandbox shims keep their launch +settings; create fresh test-owned sandboxes when changing phases. `install-prereqs.sh` falls back to user-local installs (`$HOME/.local/bin`, `$HOME/.local/go`) when run without root, so it works on sudo-less hosts @@ -978,7 +978,7 @@ kind cluster: space-local (kind-cluster.yaml.tmpl — 6 nodes) │ ├── space-runtime-supervisor.service (systemd; installed by node-manager's init │ │ container; runs every Firecracker VM and │ │ each sandbox's space-netgtw-svc) -│ ├── containerd-shim-space-v2 (installed into containerd by install-shim.sh) +│ ├── containerd-shim-space-v2 (runtime registered by the node image, binary installed by install-shim.sh) │ ├── one pod per node-service DaemonSet (release `space`): │ │ ├── space-node-manager → host supervisor (root, privileged) │ │ ├── space-runtime-manager (read-only list) (space-rtmgr) @@ -1394,7 +1394,11 @@ asks of the hosting deployment. |-----------------------------------------------------------|-------------------------------------------------------------------------------| | One DaemonSet per node service, systemd watchdogs via k8s probes | One DaemonSet per node service and role from the Kind charts (`charts/space--local`), probed on each service's health socket; runtime-supervisor is the same host systemd unit node-manager's init container installs | | Node image (packer) provides `/space/cache`, the cgroup subtree, loop devices, device permissions, SPACE accounts and the `/space` skeleton before any pod starts | Sandbox workers and builders boot the SPACE Kind node image, which bakes in the same as systemd units that run before kubelet on every node start (see [The node layer](#the-node-layer)) | -| `/space/cache` is btrfs on instance-store NVMe | btrfs on a sparse loopback image at `/var/lib/space-cache.img` on the kind node (40 GiB, same code paths, less I/O headroom), mounted on the node's `/space/cache` by `space-cache.service`, so it outlives pod restarts like the NVMe does | +| `/space/cache` is btrfs on instance-store NVMe, reformatted on every boot | btrfs on a sparse loopback image at `/var/lib/space-cache.img` on the kind node (40 GiB, same code paths, less I/O headroom), mounted `nosuid,noatime` on the node's `/space/cache` by `space-cache.service`, which reformats it on every start of the node container, so like the NVMe it outlives pod restarts but not node restarts | +| The node bootstrap installs a checksum-verified `containerd-shim-space-v2` before containerd starts, and registers the `space` runtime under `io.containerd.cri.v1.runtime` with a `CniConfDir` option | the node image registers the same runtime with the same options (in kind's version 2 config format, which containerd migrates to that plugin), so containerd loads it on its first start; the binary arrives after the cluster exists, copied onto each running sandbox worker by `install-shim.sh`, which checks its SHA-256 on the node. Installing it at boot would need the shim built and staged on a host path before `kind create` in every bring-up path | +| Every node group's bootstrap sets kubelet `eventRecordQPS: 0` and `--fail-swap-on=false` | the same settings on sandbox workers and builders, baked into the node image as a kubelet `--config-dir` drop-in (`node/kubelet/50-space.conf`); the infra node runs the unmodified base image, so its kubelet keeps kind's event rate limit (`failSwapOn` is already `false` there). No sandbox pod runs on it, so none of their events pass through that limit | +| Release binaries read no test hooks | every binary is built with `test-hooks`, and the charts set these hooks on every run: sandbox-manager's on-node snapshot reaper at target fraction 0, interval 15 s, grace 60 s and idle 120 s, against release values of 0.40, 300 s, 1800 s and 86400 s (`charts/space-sandbox-manager-local/values.yaml`, `snapshot_gc.rs`); template-manager's cache GC at interval 15 s and grace 60 s, against 300 s and 1800 s, and its prefetch latest-N at 1, against 5 (`charts/space-template-manager-local/values.yaml`, `eviction.rs`, `prefetch.rs`); the disk-admission reservation below; and api's and internal-api's TokenReview namespace, set to the release namespace that release builds read from the service-account mount. Scenarios shorten further hooks, such as the snapshot-retention sweeper's minimum age and the activation-admission burst, for their own run and restore them. The public-operation dispatcher's tick, claim limit and claim staleness stay at their release values | +| 8 GiB disk-admission reservation per cache allocation (release default) | 1 GiB through the `SPACE_NODE_DISK_ADMISSION_ALLOCATION_RESERVATION_BYTES` test hook, with the 1 GiB `MIN_AVAILABLE_BYTES` floor, so allocations fit the 40 GiB cache (`charts/space-node-service/values.yaml`) | | Postgres on RDS, secrets via ExternalSecrets / AWS Secrets Manager | In-cluster `space-postgres` StatefulSet, one pod on a 1 GiB volume, static Kubernetes Secrets | | PostgreSQL 17 | The same major version (`postgres:17`; CI takes it from the image mirror at a pinned digest) | | Every client connects with `sslmode=require` | The same `sslmode=require` on every connection: the services and the migrate Job through the `space-postgres` Secret's connection string, and the harness's `psql` helpers over loopback TLS inside the pod. The certificate is self-signed per pod start, which `require` accepts because it does not verify the server. Kind's server also refuses unencrypted TCP, so no Kind client runs in plaintext and one that sets `sslmode=disable` fails. A client that omits `sslmode` still connects over TLS, because libpq and sqlx default to `prefer`, so Kind cannot catch a connection string that loses `sslmode=require` | @@ -1452,10 +1456,16 @@ infra node. The builder release selects only `pool=builder`, and the builder nodes are deliberately not `space-enabled`, so user sandbox pods cannot land there. -The two sandbox workers also advertise an extended resource -`space.pplx.net/network-slots: 100` so the kube scheduler can satisfy each -sandbox pod's `space.pplx.net/network-slots: 1` request. Bootstrap advertises -a static pool in place of node-controller's dynamic capacity updates. +Each sandbox pod requests `space.pplx.net/network-slots: 1`. As on a deployed +node, each sandbox worker's node-ctl advertises that extended resource on its +Node's `status.capacity`, from its node-manager's `network_pool_max_slots` +(`crates/libs/node/ctl/server/src/kubernetes_controller/network_slots.rs`), and +re-advertises it on every periodic tick, including after the Node object is +recreated. node-manager keeps no warm pool of slots, as deployed nodes do not +(`SPACE_NODEMGR_NETWORK_POOL_TARGET`, `_RETAIN_READY` and `_MIN_READY` are `0` +in `charts/space-node-manager-local`): each slot, a sandbox's or a build's, is +set up when it is allocated and torn down when it is released, and no slot is +kept ready for the next allocation. ## The node layer @@ -1475,7 +1485,7 @@ these units, enabled in the image: | `space-kind-devices.service` | mode 0666 on `/dev/kvm`, `/dev/vhost-vsock`, `/dev/vhost-net` | the `kvm` group; Kind-only, never on a worker | | `space-cgroup.service` | `/sys/fs/cgroup/space` with `+cpu +memory +io +pids` delegated | `space-cgroup.service` (07-sysctl.sh) | | `space-layout.service` | the SPACE groups and service accounts; the `/space/run` skeleton and dependency-drain gates; protected Firecracker copies under `/space/bin/runtime/firecracker` and `/space/bin/{firecracker,jailer}` links; `/space/etc/sandbox.toml`; `/var/run/netns` | 07-sysctl.sh accounts and directory skeleton, 04-firecracker.sh runtime installation | -| `space-cache.service` | formats `/var/lib/space-cache.img` on first boot (sparse, 40 GiB), mounts it on `/space/cache` with loop autoclear and shared propagation, then creates the cache skeleton | 01-partitions.sh / 11-nvme-cache.sh | +| `space-cache.service` | formats `/var/lib/space-cache.img` on every boot (sparse, 40 GiB), mounts it `nosuid,noatime` on `/space/cache` with loop autoclear and shared propagation, then creates the cache skeleton | 01-partitions.sh / 11-nvme-cache.sh | | `space-kind-vsock-jails.service` | long-running: mknods `/dev/vhost-vsock` into each Firecracker jail under `/space/cache/firecracker/jails` as it appears (the jailer does not) | the node image provides it; Kind-only | They are ordered `Before=kubelet` and run on every boot of the node container, @@ -1484,11 +1494,31 @@ and cgroups; all but the vhost-vsock unit are oneshots. Nothing is installed when the cluster comes up. The sandbox workers and builders run this image (`kind-cluster.yaml.tmpl`); the control-plane and infra nodes run its base unchanged, as production's non-SPACE nodes carry no SPACE node layer, so -every node runs the same Kubernetes version. The cache image sits on the node's -`/var`, which kind keeps on a Docker volume, so the cache survives pod and node -restarts. Defaults are baked like the rest of the layer; a node's -`/etc/space/kind-node.env` can override `SPACE_CACHE_SIZE_MB` and -`SPACE_LOOP_DEVICE_TARGET` for its next boot. +every node runs the same Kubernetes version. The cache survives pod restarts but +not node restarts, as the instance-store cache of a deployed node does: on every +start of the node container `space-cache.service` truncates the image on the +node's `/var` and formats it afresh, so templates, sandboxes and snapshots on the +cache are gone and the node services re-create what they need (template-manager +pulls templates again, node-manager's init container stages the guest kernel +again). What was put on the cache by hand is not re-created: templates the +optional `scripts/bootstrap-prod-seed.sh` staged must be staged again by +re-running it, which skips the workers that still have them. Re-running the +unit within one boot keeps the mounted cache. Defaults are baked like the rest +of the layer; a node's `/etc/space/kind-node.env` can +override `SPACE_CACHE_SIZE_MB` and `SPACE_LOOP_DEVICE_TARGET` for its next boot. + +The image also carries the kubelet and containerd settings a deployed node's +bootstrap applies (`infra/space/aws/eks/modules/space/templates/`): a kubelet +`--config-dir` drop-in (`node/kubelet/50-space.conf`) with no client-side event +rate limit (`eventRecordQPS: 0`) and `failSwapOn: false`, and the `space` +runtime handler with its `CniConfDir` option (`node/containerd/space-runtime.toml`). +The drop-in is written in the version 2 format of kind's own containerd config, +which imports it, so containerd migrates both into the CRI v1 runtime plugin +where a deployed node has the handler, and kind's runc settings (the systemd +cgroup driver, which kubelet adopts from containerd) still apply; the image +build fails otherwise. Check them with +`kubectl get --raw /api/v1/nodes//proxy/configz` and `docker exec +containerd config dump`. The vendor mount remains a read-only, developer-owned download cache. At boot, `space-layout.service` copies the bootstrap Firecracker 1.15.1 binaries into @@ -1525,13 +1555,10 @@ the pod and never reach the node's mount table. Check a node's view with `docker exec findmnt -o TARGET,FSTYPE,PROPAGATION`. Diagnose a node with `docker exec systemctl status 'space-*'` and -`docker exec journalctl -u space-cache`. To start a node's cache clean or -at a new size, delete the node-service DaemonSets of that node's role, run -`systemctl stop space-runtime-supervisor && umount /space/cache && rm -/var/lib/space-cache.img` on the node (adding `SPACE_CACHE_SIZE_MB=` to -`/etc/space/kind-node.env` for a new size), then `systemctl restart -space-cache` and re-apply the charts. Recreating the cluster -also starts clean. +`docker exec journalctl -u space-cache`. To start a node's cache clean, +`docker restart `; for a new size, first add `SPACE_CACHE_SIZE_MB=` +to the node's `/etc/space/kind-node.env`. Recreating the cluster also starts +clean. **Clusters created before this image, or from an older build of it, must be recreated.** A kind node keeps the image it was created from, so such a diff --git a/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml b/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml index de3c6e9281..80ec51ca72 100644 --- a/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml +++ b/infra/space/kind/charts/space-cluster-controller-local/templates/deployment.yaml @@ -83,8 +83,6 @@ spec: # BYOK switch here too (same value as the api chart). - name: SPACE_BYOK_VOLUME_ENCRYPTION value: {{ .Values.byokVolumeEncryption | quote }} - - name: SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS - value: {{ .Values.reconcilers.snapshotRetentionSweeper.minAgeSeconds | quote }} - name: SPACE_INTERNAL_TOKEN valueFrom: secretKeyRef: @@ -187,12 +185,10 @@ spec: {{- end }} {{- range $name, $reconciler := .Values.reconcilers }} {{- range $key, $value := (default (dict) $reconciler.env) }} - {{- if ne $key "SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS" }} - name: {{ $key }} value: {{ $value | quote }} {{- end }} {{- end }} - {{- end }} # Ready once the process has reached the Lease (won it, or been validly # denied by a peer), which it signals by touching this file. # diff --git a/infra/space/kind/charts/space-cluster-controller-local/values.yaml b/infra/space/kind/charts/space-cluster-controller-local/values.yaml index 90f7a10d44..bc11fbd05c 100644 --- a/infra/space/kind/charts/space-cluster-controller-local/values.yaml +++ b/infra/space/kind/charts/space-cluster-controller-local/values.yaml @@ -109,15 +109,15 @@ reconcilers: # /v1/sandboxes is POSTed under KubernetesSubstrate. It is ON by default in # the binary, so the explicit enable below is redundant but keeps local # smoke deployments independent of that default; setting it to false leaves - # CreateSandbox returning 202 with the operation `queued` forever. Tight - # ticks/concurrency keep local smoke turnaround snappy. + # CreateSandbox returning 202 with the operation `queued` forever. The tick, + # claim limit and claim staleness are test hooks + # (crates/libs/cluster/controller/server/src/public_operation_dispatcher.rs) + # left at their release values, so Kind picks up queued operations on the + # deployed 25 ms tick. Execution concurrency is a deployment setting. publicOperationDispatcher: env: SPACE_PUBLIC_OPERATION_DISPATCHER_ENABLED: "true" - SPACE_PUBLIC_OPERATION_DISPATCHER_TICK_SECS: "1" - SPACE_PUBLIC_OPERATION_DISPATCHER_CLAIM_LIMIT: "64" SPACE_PUBLIC_OPERATION_DISPATCHER_EXECUTION_CONCURRENCY: "16" - SPACE_PUBLIC_OPERATION_DISPATCHER_CLAIM_STALE_SECS: "60" # These repairs are live when enabled. Keep kind explicitly disabled until # the end-to-end and smoke suites exercise their default-on behavior. failedRestorableRepair: @@ -179,8 +179,11 @@ reconcilers: stuckSnapshotReaper: env: SPACE_STUCK_SNAPSHOT_REAPER_ENABLED: "false" + # The minimum age (SPACE_SNAPSHOT_RETENTION_SWEEPER_MIN_AGE_SECS) is a test + # hook left at its release value, which equals the API's fork-retention grace + # (crates/libs/common/core/src/types/snapshot.rs). The retention scenarios + # shorten it for their own run and restore it afterwards. snapshotRetentionSweeper: - minAgeSeconds: "3600" env: SPACE_SNAPSHOT_RETENTION_SWEEPER_ENABLED: "true" snapshotStorageSweeper: diff --git a/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml b/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml index 588c1be6eb..188acf6475 100644 --- a/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml +++ b/infra/space/kind/charts/space-node-ctl-local/templates/rbac.yaml @@ -2,10 +2,9 @@ {{- $name := include "space-node-service.name" $ctx -}} # Same RBAC shape as the prod space-node-ctl chart: node-ctl's Kubernetes # node-local controller watches its own Node and advertises the -# space.pplx.net/network-slots extended resource on status.capacity (the -# production source of the capacity bootstrap.sh also seeds statically). Only -# node-ctl's ServiceAccount holds it; no other node service talks to the -# Kubernetes API. +# space.pplx.net/network-slots extended resource on status.capacity, the only +# source of that capacity here as on a deployed node. Only node-ctl's +# ServiceAccount holds it; no other node service talks to the Kubernetes API. {{ include "space-node-service.serviceAccount" $ctx }} --- apiVersion: rbac.authorization.k8s.io/v1 diff --git a/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml b/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml index 970a7272d9..248f37494e 100644 --- a/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml +++ b/infra/space/kind/charts/space-node-manager-local/templates/daemonset.yaml @@ -258,6 +258,28 @@ spec: # Sandbox pods own their network namespaces (KubernetesSubstrate). - name: SPACE_NODEMGR_SERVICE_PORT_INGRESS_ISOLATION value: shared_host + # No warm network-slot pool, as deployed nodes run. Each slot is its + # own network namespace, bridged to the sandbox pod's namespace or, + # for an allocation without one, to the node's; claim_allocation + # (crates/libs/node/mgr/server/src/slot_pool.rs) serves only the + # latter from the pool. Sandbox pods bring their own namespace + # (KubernetesSubstrate), so their slots are set up on allocation + # and torn down on release either way. The binary defaults also + # keep max(target, retained) slots, every slot the node has, ready + # and bridged to the node (the fill loop in slot_pool/worker.rs): + # a sandbox claim then takes one of those and rebuilds it for the + # pod, and the pool rebuilds it after release. build-manager + # allocates without a namespace, so with the pool off builders set + # up their slots on demand instead of leasing pooled ones. With no + # ready minimum the pool reports healthy while empty (pool_status + # in service/network_helpers.rs). Read at startup in + # crates/libs/node/mgr/server/src/run.rs. + - name: SPACE_NODEMGR_NETWORK_POOL_TARGET + value: "0" + - name: SPACE_NODEMGR_NETWORK_POOL_RETAIN_READY + value: "0" + - name: SPACE_NODEMGR_NETWORK_POOL_MIN_READY + value: "0" {{- if ne (toString .Values.loopDeviceTarget) "" }} - name: SPACE_LOOP_DEVICE_TARGET value: {{ .Values.loopDeviceTarget | quote }} diff --git a/infra/space/kind/charts/space-node-service/values.yaml b/infra/space/kind/charts/space-node-service/values.yaml index 1155ab761e..c4fcd2f319 100644 --- a/infra/space/kind/charts/space-node-service/values.yaml +++ b/infra/space/kind/charts/space-node-service/values.yaml @@ -37,11 +37,14 @@ node: credentialsSecretName: space-object-store endpoint: http://object-store:8333 region: us-east-1 - # Disk-admission sizing for /space/cache. Production defaults assume NVMe + # Disk-admission sizing for /space/cache. Release defaults assume NVMe # (50 GiB min-available floor, 8 GiB per-sandbox allocation reservation); # the Kind cache is a 40 GiB sparse loopback btrfs image, so both byte knobs - # scale to 1 GiB through the same production config surface. Without this - # every node publishes scheduling_status=full and sandbox-manager rejects + # scale to 1 GiB. The floor is a deployment setting; the reservation is a + # test hook that only the Kind build's test-hooks feature reads, and release + # binaries always reserve 8 GiB + # (crates/libs/common/core/src/host/disk_admission.rs). Without this every + # node publishes scheduling_status=full and sandbox-manager rejects # prepare_sandbox with disk_available_below_threshold. Read by the services # that admit cache allocations: node-manager, sandbox-manager, # volume-manager and build-manager. diff --git a/infra/space/kind/manifests/runtimeclass.yaml b/infra/space/kind/manifests/runtimeclass.yaml index a715003990..d04401a1f5 100644 --- a/infra/space/kind/manifests/runtimeclass.yaml +++ b/infra/space/kind/manifests/runtimeclass.yaml @@ -1,16 +1,20 @@ # KubernetesSubstrate pins per-sandbox pods to runtimeClassName: space. -# `infra/space/kind/scripts/install-shim.sh` installs the SPACE containerd -# shim (`io.containerd.space.v2`) on each kind sandbox worker and registers -# a `space` CRI runtime that points at it. The shim hijacks the per-sandbox -# pod's pause container, reads the SPACE annotations (sandbox-id, vcpu, -# memory, template, …), and calls into the local sandbox-manager UDS to -# create a real sandbox via node-manager, which launches Firecracker through -# the node's runtime-supervisor, as in production. +# The SPACE Kind node image registers the `space` CRI runtime handler with +# containerd (infra/space/kind/node/containerd/space-runtime.toml), pointing at +# the SPACE containerd shim (`io.containerd.space.v2`), and +# `infra/space/kind/scripts/install-shim.sh` installs the shim binary on each +# kind sandbox worker. The shim hijacks the per-sandbox pod's pause container, +# reads the SPACE annotations (sandbox-id, vcpu, memory, template, …), and +# calls into the local sandbox-manager UDS to create a real sandbox via +# node-manager, which launches Firecracker through the node's +# runtime-supervisor, as in production. # -# If the shim isn't installed (e.g. on a fresh cluster before bootstrap.sh -# has run install-shim.sh), kubelet will fail pod-start with -# "runtime handler 'space' is not configured" and the api-server will report -# `proxy_unreachable`. Run install-shim.sh and recreate the affected pods. +# If the shim binary isn't installed (e.g. on a fresh cluster before +# bootstrap.sh has run install-shim.sh), containerd cannot start the pod's +# sandbox and kubelet reports FailedCreatePodSandBox. Run install-shim.sh and +# recreate the affected pods; containerd is not restarted. A node whose image +# predates the registration has no `space` handler at all; recreate the cluster +# (scripts/check-kind-node-image.sh). apiVersion: node.k8s.io/v1 kind: RuntimeClass metadata: diff --git a/infra/space/kind/node/Dockerfile b/infra/space/kind/node/Dockerfile index 11529e60e5..be2d72249f 100644 --- a/infra/space/kind/node/Dockerfile +++ b/infra/space/kind/node/Dockerfile @@ -30,6 +30,34 @@ COPY --chmod=0755 infra/space/kind/node/space-kind-node.sh /usr/local/lib/space/ COPY --chmod=0644 scripts/lib/space-loop-devices.sh /usr/local/lib/space/kind-node/space-loop-devices.sh COPY --chmod=0644 infra/space/identity/node-identities.sh /usr/local/lib/space/kind-node/node-identities.sh COPY --chmod=0644 infra/space/kind/node/systemd/ /etc/systemd/system/ +# The kubelet and containerd settings a deployed node's bootstrap applies before +# either starts (infra/space/aws/eks/modules/space/templates/ +# worker-bootstrap.sh.tftpl). Baked in rather than patched into the cluster +# config, so a cluster created without them runs another image tag and +# check-kind-node-image.sh refuses it. kind's kubelet unit takes extra flags from +# /etc/default/kubelet, and its containerd config (version 2) gains an import of +# the version 2 drop-in. The build fails unless the configuration containerd +# loads has the `space` runtime with its option and still has kind's own +# settings: runc's systemd cgroup driver, which kubelet adopts from containerd, +# and discard_unpacked_layers (see the drop-in for how a version mismatch loses +# them). The awk prints each `table|key = value` of the dump. +COPY --chmod=0644 infra/space/kind/node/kubelet/50-space.conf /etc/kubernetes/kubelet.conf.d/50-space.conf +COPY --chmod=0644 infra/space/kind/node/containerd/space-runtime.toml /etc/containerd/conf.d/space-runtime.toml +RUN sed -i 's|^KUBELET_EXTRA_ARGS=|KUBELET_EXTRA_ARGS=--config-dir=/etc/kubernetes/kubelet.conf.d |' /etc/default/kubelet \ + && grep -q '^KUBELET_EXTRA_ARGS=--config-dir=/etc/kubernetes/kubelet.conf.d ' /etc/default/kubelet \ + && sed -i '0,/^version = 2$/s||version = 2\nimports = ["/etc/containerd/conf.d/*.toml"]|' /etc/containerd/config.toml \ + && containerd config dump 2>/dev/null \ + | awk '/^ *\[/ { table = $0; gsub(/^ *\[|\] *$/, "", table); next } NF { sub(/^ */, ""); print table "|" $0 }' \ + > /tmp/containerd-config \ + && for want in \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space|runtime_type = 'io.containerd.space.v2'" \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space|pod_annotations = ['space.pplx.net/*']" \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space.options|CniConfDir = '/etc/cni/net.d'" \ + "plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options|SystemdCgroup = true" \ + "plugins.'io.containerd.cri.v1.images'|discard_unpacked_layers = true"; do \ + grep -qxF "$want" /tmp/containerd-config || { echo "containerd config lacks: $want" >&2; exit 1; }; \ + done \ + && rm /tmp/containerd-config RUN systemctl enable \ space-loop-nodes.service \ space-kind-devices.service \ diff --git a/infra/space/kind/node/containerd/space-runtime.toml b/infra/space/kind/node/containerd/space-runtime.toml new file mode 100644 index 0000000000..48fc71402b --- /dev/null +++ b/infra/space/kind/node/containerd/space-runtime.toml @@ -0,0 +1,29 @@ +# The `space` runtime handler RuntimeClass `space` names +# (infra/space/kind/manifests/runtimeclass.yaml), registered with the options a +# deployed node's bootstrap gives it (infra/space/aws/eks/modules/space/ +# templates/worker-bootstrap.sh.tftpl, `spec.containerd.config`). Baked into the +# Kind node image, whose containerd config imports this file, so containerd +# loads it when it first starts and is never restarted to pick it up. +# +# Version 2, the version of the kind config.toml that imports it. containerd +# merges an import over that config, its version included, and migrates the +# result to version 3 only when the merged version is below 3. A version 3 file +# here would make the merged config version 3 and leave kind's own settings +# (runc's SystemdCgroup and base_runtime_spec, discard_unpacked_layers, the +# test-handler runtime) unmigrated under io.containerd.grpc.v1.cri, which the version 3 +# CRI plugins do not read: containerd would report the cgroupfs driver, and +# kubelet would adopt it. Written as version 2, this table is migrated with +# kind's into io.containerd.cri.v1.runtime, where a deployed node's containerd +# has it. node/Dockerfile fails the build unless both survive. +version = 2 + +[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.space] + # containerd runs /usr/local/bin/containerd-shim-space-v2, which + # scripts/install-shim.sh installs. + runtime_type = "io.containerd.space.v2" + # The shim reads its instructions (sandbox id, owner, template, sizing) from + # the space.pplx.net/* pod annotations in the OCI spec; containerd forwards + # only the annotations listed here. + pod_annotations = ["space.pplx.net/*"] + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.space.options] + CniConfDir = "/etc/cni/net.d" diff --git a/infra/space/kind/node/kubelet/50-space.conf b/infra/space/kind/node/kubelet/50-space.conf new file mode 100644 index 0000000000..bcf3acc053 --- /dev/null +++ b/infra/space/kind/node/kubelet/50-space.conf @@ -0,0 +1,13 @@ +# Kubelet settings a deployed SPACE node sets in its bootstrap +# (infra/space/aws/eks/modules/space/templates/worker-bootstrap.sh.tftpl and +# bootstrap.sh.tftpl, `spec.kubelet`). Baked into the Kind node image, which +# points kubelet's --config-dir at this file's directory (node/Dockerfile); +# kubelet applies it over the configuration kubeadm writes at join. +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +# No client-side event rate limit. A burst of sandbox creates emits a burst of +# pod events (FailedCreatePodSandBox from the shim among them), which a limited +# kubelet delays and, once its queue is full, drops. +eventRecordQPS: 0 +# Deployed nodes pass --fail-swap-on=false. +failSwapOn: false diff --git a/infra/space/kind/node/space-kind-node.sh b/infra/space/kind/node/space-kind-node.sh index 5d77da3c50..6d72c5a333 100755 --- a/infra/space/kind/node/space-kind-node.sh +++ b/infra/space/kind/node/space-kind-node.sh @@ -25,8 +25,9 @@ # # A container's /dev, mounts and cgroups do not survive `docker restart`, which # is why these are boot units and not a one-time setup. Each step is -# idempotent: after a restart the cache image and the directories are already -# in place, and re-running a unit by hand on a live node is safe. +# idempotent within a boot, so re-running a unit by hand on a live node is +# safe. The cache is the one step that does not carry state across boots: as on +# a deployed node, every boot starts it as a fresh filesystem. # # Defaults are baked like the rest of the layer; /etc/space/kind-node.env, when # present, overrides SPACE_CACHE_SIZE_MB and SPACE_LOOP_DEVICE_TARGET. @@ -44,9 +45,13 @@ LIB_DIR=/usr/local/lib/space/kind-node STEP="${1:-}" CACHE_DIR=/space/cache -# On the node's /var, which kind keeps on a volume, so the cache outlives node -# restarts as well as pod restarts, as the instance-store disk does on EKS. +# On the node's /var, which kind keeps on a volume. The image file outlives a +# node restart; its filesystem does not (see cache below). CACHE_IMAGE=/var/lib/space-cache.img +# Marks the cache formatted and mounted for this boot. kind mounts the node's +# /run as a tmpfs, so a node start finds it absent. +CACHE_STATE_DIR=/run/space-kind-node +CACHE_BOOT_MARKER="$CACHE_STATE_DIR/cache.done" CACHE_SIZE_MB="${SPACE_CACHE_SIZE_MB:-40960}" log() { printf '%s\n' "$*"; } @@ -221,9 +226,12 @@ layout() { # ── btrfs /space/cache ─────────────────────────────────────────────────────── # sandbox-manager clones template rootfs into instance dirs with btrfs subvolume -# snapshots, so the cache must be btrfs. Production formats instance-store NVMe -# (or a loopback image on nodes without one); here it is a sparse loopback image -# on the node, formatted once and reused on every later start. +# snapshots, so the cache must be btrfs. A deployed node formats its +# instance-store NVMe on every boot (packer 11-nvme-cache.sh), so templates, +# sandboxes and snapshots on the cache never survive a node restart, and node +# services re-create what they need on it. Here the cache is a sparse loopback +# image on the node, reformatted the same way on every start of the node +# container. cache_is_mounted() { findmnt -n -t btrfs --mountpoint "$CACHE_DIR" >/dev/null 2>&1 } @@ -232,18 +240,32 @@ cache_image_is_btrfs() { [[ -s $CACHE_IMAGE ]] && [[ "$(blkid -p -s TYPE -o value "$CACHE_IMAGE" 2>/dev/null)" == btrfs ]] } +# A loop device still bound to the image (a crash before autoclear was armed) +# would keep serving the old filesystem's blocks over the new one. `losetup -j` +# matches an existing file by device and inode, but a missing one by path, and +# the host kernel's loop devices are shared by every kind node, each with an +# image at this path: without the image there is nothing of this node's to +# release, and matching by path would find the other nodes' caches. +release_cache_image() { + local loopdev + [[ -e $CACHE_IMAGE ]] || return 0 + while read -r loopdev; do + [[ -n $loopdev ]] || continue + findmnt -n --source "$loopdev" >/dev/null 2>&1 \ + && die "$loopdev (backing $CACHE_IMAGE) is still mounted; restart the node to start its cache fresh" + space_loop_detach "$loopdev" || true + done < <(losetup -l -n -O NAME -j "$CACHE_IMAGE" 2>/dev/null | tr -d '[:blank:]') + [[ -z "$(losetup -l -n -O NAME -j "$CACHE_IMAGE" 2>/dev/null)" ]] \ + || die "a loop device still holds $CACHE_IMAGE; restart the node to start its cache fresh" +} + format_cache_image() { - # A populated image that is not btrfs (a format interrupted between truncate - # and mkfs, or damage) would fail to mount on every start until someone - # deleted it; reformat it like an empty one. - if [[ -s $CACHE_IMAGE ]]; then - log "backing image $CACHE_IMAGE is not a btrfs filesystem; reformatting it" - else - log "creating btrfs backing image $CACHE_IMAGE (${CACHE_SIZE_MB}M, sparse)" - fi + log "formatting btrfs backing image $CACHE_IMAGE (${CACHE_SIZE_MB}M, sparse)" + release_cache_image # Sparse (truncate, not fallocate): the virtual size gives disk admission # production-like headroom while real blocks are only consumed as sandboxes - # write. + # write. Truncating to zero first also returns the previous boot's blocks to + # the host. truncate -s 0 "$CACHE_IMAGE" truncate -s "${CACHE_SIZE_MB}M" "$CACHE_IMAGE" mkfs.btrfs -f -L space-cache "$CACHE_IMAGE" >/dev/null @@ -268,7 +290,8 @@ mount_cache_image() { die "no loop device available for $CACHE_IMAGE" fi fi - if ! mount -o nosuid "$loopdev" "$CACHE_DIR"; then + # noatime as deployed nodes mount the cache (packer 11-nvme-cache.sh). + if ! mount -o nosuid,noatime "$loopdev" "$CACHE_DIR"; then (( reused )) || space_loop_detach "$loopdev" || true die "mount $loopdev $CACHE_DIR failed (image $CACHE_IMAGE; if this repeats, delete the image on the node)" fi @@ -298,8 +321,14 @@ cache_skeleton() { install -d -m 0755 -o 0 -g 0 "$CACHE_DIR/firecracker/jails" } +# The first run in a boot formats the image; a later run in the same boot (the +# unit re-run by hand) keeps the mounted cache, or remounts the image if it was +# unmounted, reformatting it only if it is no longer btrfs. The marker is +# written last, so a run that failed partway formats again. cache() { - install -d -m 0755 -o 0 -g 0 /space "$CACHE_DIR" + install -d -m 0755 -o 0 -g 0 /space "$CACHE_DIR" "$CACHE_STATE_DIR" + exec 9>"$CACHE_STATE_DIR/cache.lock" + flock 9 if cache_is_mounted; then log "$CACHE_DIR is already btrfs" else @@ -309,7 +338,9 @@ cache() { command -v mkfs.btrfs >/dev/null \ || die "mkfs.btrfs missing; this node is not running the SPACE Kind node image (infra/space/kind/node/Dockerfile)" space_loop__losetup_bin >/dev/null || die "losetup missing (util-linux)" - cache_image_is_btrfs || format_cache_image + if [[ ! -e $CACHE_BOOT_MARKER ]] || ! cache_image_is_btrfs; then + format_cache_image + fi mount_cache_image fi # Shared, so a pod that receives the cache HostToContainer also receives a @@ -319,9 +350,10 @@ cache() { local img_mb img_mb=$(( $(stat -c %s "$CACHE_IMAGE") / 1048576 )) if (( img_mb != CACHE_SIZE_MB )); then - log "WARNING: the cache size is ${CACHE_SIZE_MB}M (SPACE_CACHE_SIZE_MB) but the reused image is ${img_mb}M; see \"The node layer\" in infra/space/kind/README.md to resize" + log "WARNING: the cache size is ${CACHE_SIZE_MB}M (SPACE_CACHE_SIZE_MB) but the image formatted this boot is ${img_mb}M; restart the node to reformat it at the new size" fi cache_skeleton + touch "$CACHE_BOOT_MARKER" } # ── vhost-vsock nodes in Firecracker jails (Kind only) ─────────────────────── diff --git a/infra/space/kind/scripts/bootstrap-prod-seed.sh b/infra/space/kind/scripts/bootstrap-prod-seed.sh index 5015591ba8..7429e8f878 100755 --- a/infra/space/kind/scripts/bootstrap-prod-seed.sh +++ b/infra/space/kind/scripts/bootstrap-prod-seed.sh @@ -25,6 +25,13 @@ # TEMPLATE_BOOTSTRAP_NAMES — comma/space-separated. Default: seed. # # This script is idempotent: re-running it skips work that's already done. +# +# Re-run it after a sandbox worker restarts. The staged files live on the +# worker's /space/cache, which space-cache.service reformats on every boot of +# the node (infra/space/kind/node/space-kind-node.sh), and nothing else puts +# them back: template-manager cannot pull them from this bucket. A re-run +# stages the templates onto the restarted worker again and skips the workers +# that still have them on their cache. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -136,12 +143,15 @@ SQL local s3_path="${storage_uri%%\?*}" s3_path="${s3_path%/}" - # If the worker already has this hash registered, skip — idempotency. - # `space-tmplmgr list` truncates the hash to ~12 hex chars, so grep on - # the bare prefix (post-`sha256:`). + # If the worker already has this hash registered and its build directory + # is on the cache, skip — idempotency. `space-tmplmgr list` truncates the + # hash to ~12 hex chars, so grep on the bare prefix (post-`sha256:`). The + # directory check covers a worker restarted moments ago, whose catalog + # can still list the build until template-manager's startup recovery + # prunes it. local hex_prefix="${hash#sha256:}" hex_prefix="${hex_prefix:0:12}" - if kubectl -n space exec "$pod" -c shell -- bash -c "space-tmplmgr list 2>/dev/null | grep -q $hex_prefix" 2>/dev/null; then + if kubectl -n space exec "$pod" -c shell -- bash -c "space-tmplmgr list 2>/dev/null | grep -q $hex_prefix && compgen -G '/space/cache/firecracker/templates/${name}/*${hex_prefix}*' >/dev/null" 2>/dev/null; then log " $name@${hash:0:24}… already staged locally; skip" continue fi diff --git a/infra/space/kind/scripts/bootstrap.sh b/infra/space/kind/scripts/bootstrap.sh index 46974e82c3..2dceb57b57 100755 --- a/infra/space/kind/scripts/bootstrap.sh +++ b/infra/space/kind/scripts/bootstrap.sh @@ -258,26 +258,18 @@ apply_node_labels() { } apply_node_labels -# ── Step 3: extended resource — network slots ───────────────────────────────── +# ── Step 3: network-slot capacity ──────────────────────────────────────────── # Every sandbox pod requests `space.pplx.net/network-slots: "1"` (set by -# KubernetesSubstrate::build_sandbox_pod). Without a node advertising -# capacity, the kube scheduler leaves sandbox pods Pending forever. In prod -# space-node-ctl patches each Node's status.capacity from node-manager's -# `network_pool_max_slots`; locally we advertise a static pool. -log "advertising network-slots capacity on sandbox workers" -for node in $(kubectl --context "$KCTX" get nodes -l space.pplx.net/node-pool=sandbox -o name); do - # JSON-patch `add` on a map key is idempotent. ~1 in the json-pointer is - # the escape for `/` inside the key `space.pplx.net/network-slots`. - kubectl --context "$KCTX" patch "$node" --subresource=status --type=json \ - -p '[{"op":"add","path":"/status/capacity/space.pplx.net~1network-slots","value":"100"}]' \ - >/dev/null || true -done +# KubernetesSubstrate::build_sandbox_pod). Each node's space-node-ctl advertises +# its node-manager's `network_pool_max_slots` on the Node's status.capacity, as +# on a deployed node (crates/libs/node/ctl/server/src/kubernetes_controller/ +# network_slots.rs). Nothing seeds it here, as nothing does on a deployed node. # ── Step 4: install the SPACE containerd shim on each sandbox worker ───────── # Without this, RuntimeClass `space` (handler: space) is unresolvable and -# every per-sandbox pod fails to start. install-shim.sh is idempotent: it -# only builds the shim if needed and only edits containerd config.toml if -# the runtime stanza isn't already present. +# every per-sandbox pod fails to start. The node image registers the runtime +# with containerd; install-shim.sh installs the binary containerd runs for it, +# and is idempotent: it only builds the shim if needed. # Reconcile receivers and owned log collection before changing producers. # Local node-side endpoints stay at the Kind profiles' fixed ClusterIPs, so # already-running exporters can also reconnect after a backend restart. diff --git a/infra/space/kind/scripts/install-shim.sh b/infra/space/kind/scripts/install-shim.sh index db9cef4436..2f05826ae8 100755 --- a/infra/space/kind/scripts/install-shim.sh +++ b/infra/space/kind/scripts/install-shim.sh @@ -1,12 +1,13 @@ #!/usr/bin/env bash -# install-shim.sh — build `containerd-shim-space-v2` and wire it into each -# kind worker's containerd, so per-sandbox pods scheduled with +# install-shim.sh — build `containerd-shim-space-v2` and install it on each +# kind sandbox worker, so per-sandbox pods scheduled with # `runtimeClassName: space` get hijacked by the SPACE shim (which delegates # to sandbox-manager → node-manager → guest runtime) instead of running as -# a plain runc container. +# a plain runc container. The node image registers the runtime with +# containerd; this installs the binary containerd runs for it. # -# Without this script, RuntimeClass `space` is unresolvable and every -# per-sandbox pod fails to start with "runtime handler not configured". +# Without the shim, every per-sandbox pod that RuntimeClass `space` routes to +# it fails to start. # # Idempotent. Safe to re-run after a kind cluster recreate. # @@ -123,61 +124,40 @@ list_workers_fallback() { done } -# ── 3. Install the shim binary + containerd config on one kind node ─────────── +# ── 3. Install the shim binary on one kind node ───────────────────────────── +# The `space` runtime itself is registered by the SPACE Kind node image +# (infra/space/kind/node/containerd/space-runtime.toml), under containerd's CRI +# v1 runtime plugin as a deployed node registers it, so containerd has it from +# its first start and is never edited or restarted here. containerd resolves +# /usr/local/bin/containerd-shim-space-v2 when each sandbox pod starts, so a +# replaced binary takes effect for the next sandbox. install_on_node() { local node=$1 log "installing shim on $node" - local prior_config - prior_config=$(docker exec "$node" sha256sum /etc/containerd/config.toml) + docker exec "$node" containerd config dump 2>/dev/null \ + | grep -F "[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.space]" >/dev/null \ + || die "$node's containerd has no 'space' runtime; the node predates the SPACE Kind node image that registers it, so recreate the cluster (scripts/check-kind-node-image.sh)" docker exec "$node" mkdir -p "$(dirname "$SHIM_REAL_BIN")" - docker cp "$SHIM_BIN" "$node:${SHIM_REAL_BIN}.tmp" - docker exec "$node" sh -c \ - "chown root:root '${SHIM_REAL_BIN}.tmp' && chmod 0755 '${SHIM_REAL_BIN}.tmp' && mv -f '${SHIM_REAL_BIN}.tmp' '$SHIM_REAL_BIN'" - docker cp "$SHIM_WRAPPER" "$node:/usr/local/bin/.containerd-shim-space-v2.tmp" - docker exec "$node" sh -c \ - "chown root:root /usr/local/bin/.containerd-shim-space-v2.tmp && chmod 0755 /usr/local/bin/.containerd-shim-space-v2.tmp && mv -f /usr/local/bin/.containerd-shim-space-v2.tmp /usr/local/bin/containerd-shim-space-v2" - - # Add the `space` CRI runtime to containerd's config if not already there. - # The runtime_type is the shim's protocol name — kind's containerd looks for - # /usr/local/bin/containerd-shim--. The bash - # wrapper installed above now occupies that path and execs the compiled - # binary from /usr/local/lib/space. - # NOTE: heredoc delimiter is QUOTED ('"'"'EOF'"'"') so sh doesn't run command - # substitution on backticks inside the comment block (a previous version - # without quotes had sh trying to exec the literal word "space"). - # - # pod_annotations is the allowlist containerd uses to decide which K8s pod - # annotations get propagated into the OCI runtime spec. The SPACE shim - # reads its instructions (sandbox-id, owner, vcpu, memory, template, etc.) - # off the OCI spec, so without this allowlist every pod create fails with - # "space.pplx.net/sandbox-owner or space.pplx.net/sandbox-user is required". - # `space.pplx.net/*` matches every SPACE annotation in one rule. - docker exec "$node" sh -c ' - cfg=/etc/containerd/config.toml - grep -q "io.containerd.space.v2" "$cfg" && exit 0 - cat >>"$cfg" <<'"'"'EOF'"'"' - -# Added by infra/space/kind/scripts/install-shim.sh — kind-only. -# Provides RuntimeClass "space" -> containerd-shim-space-v2, which bridges -# kubelet pod-start into the per-node sandbox-manager UDS so per-sandbox -# pods run as real SPACE sandboxes (guest VM via node-manager) instead of -# plain runc containers. -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.space] - runtime_type = "io.containerd.space.v2" - pod_annotations = ["space.pplx.net/*"] -EOF - ' + install_file "$node" "$SHIM_BIN" "$SHIM_REAL_BIN" + install_file "$node" "$SHIM_WRAPPER" /usr/local/bin/containerd-shim-space-v2 +} - # Restart containerd inside the kind node so it picks up the new runtime. - # kind nodes run systemd, so the standard systemctl path works. (We - # tolerate failure here — on the first install the kubelet briefly loses - # the apiserver socket; it reconnects within a few seconds.) - if [[ "$(docker exec "$node" sha256sum /etc/containerd/config.toml)" != "$prior_config" ]]; then - docker exec "$node" systemctl restart containerd - else - log "containerd configuration unchanged; new sandbox shims use the updated binary" +# Copy beside the destination, check the copy against the source's SHA-256 (as +# a deployed node's bootstrap checks the shim it downloads), then rename it +# into place. +install_file() { + local node=$1 source=$2 dest=$3 want got + local staged + staged="$(dirname "$dest")/.$(basename "$dest").tmp" + want="$(sha256sum "$source" | cut -d' ' -f1)" + docker cp "$source" "$node:$staged" + got="$(docker exec "$node" sha256sum "$staged" | cut -d' ' -f1)" + if [[ "$got" != "$want" ]]; then + docker exec "$node" rm -f "$staged" || true + die "$dest on $node: copied file has SHA-256 $got, expected $want" fi + docker exec "$node" sh -c "chown root:root '$staged' && chmod 0755 '$staged' && mv -f '$staged' '$dest'" } # ── Driver ─────────────────────────────────────────────────────────────────── diff --git a/infra/space/kind/scripts/kind-node-image.sh b/infra/space/kind/scripts/kind-node-image.sh index 0201880818..0e4108cfc0 100755 --- a/infra/space/kind/scripts/kind-node-image.sh +++ b/infra/space/kind/scripts/kind-node-image.sh @@ -36,6 +36,8 @@ DOCKERFILE=infra/space/kind/node/Dockerfile INPUTS=( "${DOCKERFILE}" infra/space/kind/node/space-kind-node.sh + infra/space/kind/node/containerd/space-runtime.toml + infra/space/kind/node/kubelet/50-space.conf infra/space/kind/node/systemd/space-cache.service infra/space/kind/node/systemd/space-cgroup.service infra/space/kind/node/systemd/space-kind-devices.service