feat(computer): a versioned disk boot seed, and a reset an older launcher can't misread

packages/agentenv-protocol/README.md

@@ -636,6 +636,17 @@ leasing, and unverified for one with it until the probe has lease checks. The pr
 default role; the tool-access check hides a tool from a role of its own, `agentenv-probe-<random>`,
 and gives it back.
 
+### A disk's boot seed
+
+A Linux computer's disk form gets the deploy's environment at boot from a NoCloud seed, a FAT filesystem labelled
+`CIDATA` that the launcher writes and the disk's init reads. `agentenv_protocol.computer.seed` defines it. A disk
+composed by one agent-env version boots under another's launcher, so the seed carries every version its launcher
+writes: version 1 in `user-data`, a later version N in `user-data.vN`, and the oldest it carries in
+`agentenv-seed-oldest`. An init reads the newest version it knows, and stops the boot when the seed carries none of
+them. Version 1 is `#agentenv-environment`, then `NAME=<value, base64'd>` a line, in byte order; `SAMPLE` and
+`SAMPLE_USER_DATA` pin it, for a launcher's writer and an init's reader, wherever either is written, to be tested
+against. A change an older init would misread is a new version; one it ignores, such as a new file on the seed, isn't.
+
 ## Computer apps
 
 A computer app is software an agent uses on a computer, beside the driver: a desktop app, a service

packages/agentenv-protocol/src/agentenv_protocol/computer/seed.py

@@ -0,0 +1,53 @@
+"""The disk boot seed: how the launcher that boots a Linux computer's disk form hands it the deploy's environment, as a
+NoCloud seed its init reads at boot. A disk composed by one agent-env version boots under another's launcher, so the
+seed carries every version its launcher writes, and an init reads the newest it knows; a launcher that has stopped
+writing every version an init reads says so, and that init stops the boot rather than start without its environment.
+
+Version 1 is a FAT filesystem labelled ``CIDATA`` holding ``user-data``: ``HEADER``, then a line a variable,
+``NAME=<its value, base64'd>``, ordered by line, so a value of any bytes stays one line. ``meta-data`` is NoCloud's own:
+an ``instance-id`` and the guest's ``local-hostname``. A later version N goes in its own file, ``user-data.vN``, beside
+the ones before it, and ``OLDEST`` names the oldest version the seed still carries."""
+
+from __future__ import annotations
+
+import base64
+
+#: The label of the seed's FAT filesystem, which an init finds it by.
+LABEL = "CIDATA"
+#: The first line of every version's user-data.
+HEADER = "#agentenv-environment"
+#: The file holding the oldest version the seed carries, a decimal number. A seed without it carries version 1.
+OLDEST = "agentenv-seed-oldest"
+#: The versions a launcher of this protocol writes, oldest first.
+WRITTEN = (1,)
+
+
+def user_data_file(version: int) -> str:
+    """The file ``version``'s user-data is in: ``user-data``, the name NoCloud gives it, for version 1, and
+    ``user-data.v<version>`` after it."""
+    return "user-data" if version == 1 else f"user-data.v{version}"
+
+
+def user_data(environment: dict[str, str]) -> str:
+    """Version 1's user-data for ``environment``."""
+    lines = sorted(f"{name}={base64.b64encode(value.encode()).decode()}" for name, value in environment.items())
+    return "".join(f"{line}\n" for line in [HEADER, *lines])
+
+
+def read_user_data(text: str) -> dict[str, str]:
+    """The environment version 1's ``text`` holds, as an init reads it: blank and ``#`` lines skipped."""
+    return {name: base64.b64decode(value).decode() for line in text.splitlines() if line and not line.startswith("#")
+            for name, _, value in [line.partition("=")]}
+
+
+#: An environment and version 1's user-data for it, byte for byte, which a launcher's writer and an init's reader are
+#: each tested against, wherever either is implemented.
+SAMPLE = {"AGENTENV_CONTROL_TOKEN": "a value with spaces, = signs, 'quotes', $dollars and\na second line\n",
+          "ENVIRONMENT_NAME": "desk", "ENVIRONMENT_NAME2": "", "NOTE": "first line\nFAKE=1"}
+SAMPLE_USER_DATA = (
+    "#agentenv-environment\n"
+    "AGENTENV_CONTROL_TOKEN=YSB2YWx1ZSB3aXRoIHNwYWNlcywgPSBzaWducywgJ3F1b3RlcycsICRkb2xsYXJzIGFuZAphIHNlY29uZCBsaW5lCg==\n"
+    "ENVIRONMENT_NAME2=\n"
+    "ENVIRONMENT_NAME=ZGVzaw==\n"
+    "NOTE=Zmlyc3QgbGluZQpGQUtFPTE=\n"
+)

packages/agentenv-protocol/tests/test_computer_seed.py

@@ -0,0 +1,14 @@
+"""The disk boot seed's version 1, as its sample pins it: what a launcher writes for an environment and what an init
+reads back, every byte of a value kept, and where each version's user-data goes."""
+
+from agentenv_protocol.computer import seed
+
+
+def test_version_1_holds_the_sample_environment_byte_for_byte_and_reads_back_whole():
+    assert seed.user_data(seed.SAMPLE) == seed.SAMPLE_USER_DATA
+    assert seed.read_user_data(seed.SAMPLE_USER_DATA) == seed.SAMPLE
+
+
+def test_each_version_has_its_own_file_and_version_1_keeps_nocloud_s_name():
+    assert [seed.user_data_file(version) for version in (1, 2, 10)] == ["user-data", "user-data.v2", "user-data.v10"]
+    assert seed.WRITTEN[0] == 1

src/agent_env/computers/README.md

@@ -20,15 +20,19 @@ It also has a disk form, a disk a VM boots from UEFI, which is the container for
 recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs,
 and an ext4 root filesystem that is the image's own, with `/etc/agentenv/vm/image.env`, its entrypoint, command,
 user, working folder and environment, from its OCI config. The init mounts it, brings up the network, reads the deploy's
-settings from a NoCloud seed (`CIDATA`) and becomes the image's entrypoint, as its user. A release publishes it as
+settings from a NoCloud seed (`CIDATA`) and becomes the image's entrypoint, as its user. The seed is
+`agentenv_protocol.computer.seed`'s: it carries every version its launcher writes, and the init (`disk/linux/boot/`,
+its reader in `seed-env`) reads the newest it knows, so a disk keeps booting under a later agent-env's launcher, and
+stops the boot, saying to recompose the disk, when the seed carries none. A release publishes it as
 `ghcr.io/scaleapi/agentenv-ubuntu-24.04-disk:src-<hex>`, tagged by both its sources; without it, composing makes it.
 Composing a computer env makes each arch's disk from its composed container, the root filesystem rebuilt with
 `mkfs.ext4 -d` by the disk tools (`disk/linux/tools/`) on the builder's own platform and the base disk's EFI system
 partition kept, so nothing is emulated or privileged. A disk boots where `/dev/kvm` is, in the QEMU launcher
 (`disk/linux/launcher/`): gate 3 probes it there, and elsewhere it stays `unverified` until a deploy's first boot with
 KVM probes it. A deploy boots it with `env_params = { form = "disk" }`. The launcher serves each of the computer's ports
 through socat, whose listen backlog takes a burst of connections, in front of QEMU's own forward, whose backlog is 1, and
-a reset boots the disk afresh on a new overlay in the same launcher, at the same URLs. It boots on local Docker where
+a reset boots the disk afresh on a new overlay in the same launcher, at the same URLs; a container an agent-env from
+before reset in place started is refused a reset, to be redeployed. It boots on local Docker where
 Docker can pass it `/dev/kvm`, and on `modal_vm`, amd64 only, whose sandbox runs the launcher as its image under nested
 virtualization: the provider exports the disk from the image store with this machine's Docker, compressed, once for each
 composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal uploads once for each

src/agent_env/computers/disk/linux/boot/Dockerfile

@@ -14,7 +14,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
         linux-image-virtual busybox-static kmod zstd cpio systemd-boot-efi systemd-ukify python3-pefile \
     && rm -rf /var/lib/apt/lists/*
-COPY init udhcpc.script uki.sh /build/
+COPY init seed-env udhcpc.script uki.sh /build/
 RUN sh /build/uki.sh /uki
 
 FROM scratch AS uki

src/agent_env/computers/disk/linux/boot/init

@@ -9,8 +9,7 @@ mount -t sysfs sysfs /sys
 mount -t devtmpfs devtmpfs /dev
 log() { echo "agentenv-init: $*" > /dev/console; }
 fail() { log "$*"; sleep 2; poweroff -f; }
-# decode VALUE: VALUE's base64 decoded into $decoded, its trailing newlines kept, which $(...) alone would drop.
-decode() { decoded=$(echo "$1" | base64 -d; echo .); decoded=${decoded%.}; }
+. /seed-env
 while read -r module; do insmod "$module" 2>/dev/null; done < /modules.order
 root=""
 for _ in $(seq 200); do root=$(findfs LABEL=agentenv-root 2>/dev/null) && break; sleep 0.05; done
@@ -36,14 +35,7 @@ while read -r line; do
         workdir) decode "$value"; workdir=$decoded ;;
     esac
 done < "$config"
-if [ -f /seed/user-data ]; then
-    while read -r line; do
-        key=${line%%=*} value=${line#*=}
-        case "$key" in ''|\#*) continue ;; esac
-        decode "$value"
-        export "$key=$decoded"
-    done < /seed/user-data
-fi
+[ -n "$seed" ] && { seed_environment /seed || fail "$refused"; }
 hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data)
 hostname "${hostname:-computer}"
 hostname > /newroot/etc/hostname

src/agent_env/computers/disk/linux/boot/seed-env

@@ -0,0 +1,45 @@
+# seed-env, sourced by init: the deploy's environment from the launcher's NoCloud seed, as
+# agentenv_protocol.computer.seed defines it. init sources it before it exports the image's environment, and a seed may
+# name any variable, so the reader works in a subshell and hands back only exports, each name checked and each value
+# quoted, which seed_environment evaluates: no variable but those the seed names is touched.
+
+# decode VALUE: VALUE's base64 decoded into $decoded, its trailing newlines kept, which $(...) alone would drop.
+decode() { decoded=$(echo "$1" | base64 -d; echo .); decoded=${decoded%.}; }
+
+# seed_environment DIR: export each variable of the newest seed version this init reads (1) that the seed mounted at DIR
+# carries, from the one its agentenv-seed-oldest names, 1 without it; with none, return 1, the reason in $refused.
+seed_environment() { eval "$(_seed_exports "$1")"; }
+
+# _seed_exports DIR: what seed_environment evaluates: an export a variable, or the refusal.
+_seed_exports() (
+    reads=1
+    oldest=$(cat "$1/agentenv-seed-oldest" 2>/dev/null || echo 1)
+    newest=${reads##* }
+    case "$oldest" in ''|*[!0-9]*) oldest=1 ;; esac
+    if [ "$oldest" -gt "$newest" ]; then
+        echo "refused='this disk reads seed v$newest; this agent-env writes v$oldest or later;" \
+             "recompose it with agent-env env build'; false"
+        exit 0
+    fi
+    file=
+    for version in $reads; do
+        [ "$version" -ge "$oldest" ] || continue
+        name=user-data
+        [ "$version" -gt 1 ] && name=user-data.v$version
+        [ -f "$1/$name" ] && file=$1/$name
+    done
+    [ -n "$file" ] || exit 0
+    while read -r line; do
+        name=${line%%=*} value=${line#*=}
+        case "$name" in ''|\#*|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac
+        decode "$value"
+        quoted= rest=$decoded
+        while :; do
+            case "$rest" in
+                *\'*) quoted="$quoted${rest%%\'*}'\\''" rest=${rest#*\'} ;;
+                *) quoted="$quoted$rest"; break ;;
+            esac
+        done
+        printf "export %s='%s'\n" "$name" "$quoted"
+    done < "$file"
+)

src/agent_env/computers/disk/linux/boot/uki.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 # uki.sh OUT: the one file the disk form's EFI system partition holds, written to OUT under the name UEFI firmware boots
-# from removable media: a UKI of Ubuntu's kernel and an initramfs of busybox, init, and the modules a virtio guest needs
-# that the kernel doesn't build in, decompressed, since busybox's insmod reads no zstd.
+# from removable media: a UKI of Ubuntu's kernel and an initramfs of busybox, init with its seed-env, and the modules a
+# virtio guest needs that the kernel doesn't build in, decompressed, since busybox's insmod reads no zstd.
 set -eu
 out=$1
 kver=$(ls /lib/modules | sort -V | tail -1)
@@ -12,7 +12,7 @@ esac
 root=/tmp/initramfs
 mkdir -p "$root/bin" "$root/dev" "$root/proc" "$root/sys" "$root/newroot" "$root/seed" "$root/lib/modules" "$out"
 cp /bin/busybox "$root/bin/busybox"
-cp /build/init /build/udhcpc.script "$root/"
+cp /build/init /build/seed-env /build/udhcpc.script "$root/"
 chmod 755 "$root/init" "$root/udhcpc.script"
 : > "$root/modules.order"
 for module in virtio_pci virtio_mmio virtio_blk virtio_scsi virtio_net virtio_rng virtio_console ext4 vfat \

src/agent_env/computers/disk/linux/launcher/launch

@@ -82,9 +82,9 @@ boot() {
     rm -rf "$work"
     mkdir -p "$work"
     qemu-img create -q -f qcow2 -b "$disk" -F qcow2 "$work/disk.qcow2"
-    seed "$work"
+    seed "$work/seed"
     mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null
-    mcopy -i "$work/seed.img" "$work/user-data" "$work/meta-data" ::/
+    mcopy -i "$work/seed.img" "$work/seed/"* ::/
     if [ "$arch" = amd64 ]; then
         cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd"
     else

src/agent_env/computers/disk/linux/launcher/seed

@@ -1,10 +1,13 @@
 #!/bin/sh
-# seed DIR: the NoCloud seed's files for the computer launch boots, in DIR: user-data holding each VM_ENV_<NAME> variable
-# as NAME and its value, base64'd as the variable already holds it, a line each, as the disk's init reads them, and
-# meta-data naming the instance. A base64 value has no newline, so no line env prints can pass for another variable.
+# seed DIR: the NoCloud seed's files for the computer launch boots, in DIR, as agentenv_protocol.computer.seed defines
+# them: version 1's user-data, holding each VM_ENV_<NAME> variable as NAME and its value, base64'd as the variable
+# already holds it, a line each in byte order; agentenv-seed-oldest, the oldest version written; and meta-data naming
+# the instance. A base64 value has no newline, so no line env prints can pass for another variable.
 set -eu
+mkdir -p "$1"
 {
     echo "#agentenv-environment"
-    env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=\([A-Za-z0-9+/=]*\)$/\1=\2/p' | sort
+    env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=\([A-Za-z0-9+/=]*\)$/\1=\2/p' | LC_ALL=C sort
 } > "$1/user-data"
+echo 1 > "$1/agentenv-seed-oldest"
 printf 'instance-id: %s\nlocal-hostname: computer\n' "$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" > "$1/meta-data"

src/agent_env/providers/sandbox_providers/disk_launcher.py

@@ -9,8 +9,13 @@
 #: The launch script's path in the launcher image, its ENTRYPOINT, which boots the computer; a provider that clears the
 #: ENTRYPOINT runs it as the command.
 LAUNCH = "/usr/local/bin/launch"
-#: Its command that boots the computer afresh, with the environment ``reset_input`` gives on its stdin.
-RESET = (LAUNCH, "reset")
+#: Where the launcher keeps what a reset and its boot loop share: a launcher older than reset in place has none.
+CONTROL = "/var/lib/agentenv-vm/control"
+#: The status ``RESET`` ends with in a sandbox whose launcher is older than reset in place.
+UNRESETTABLE = 75
+#: The command that boots the computer afresh, with the environment ``reset_input`` gives on its stdin: the launcher's
+#: ``launch reset``, which a launcher older than it would take for a second boot, so it isn't run there.
+RESET = ("sh", "-c", f"[ -d {CONTROL} ] || exit {UNRESETTABLE}; exec {LAUNCH} reset")
 
 
 def launcher_variables(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> dict[str, str]:
@@ -25,5 +30,10 @@ def reset_input(env: dict[str, str]) -> str:
     return "".join(f"{name}={value}\n" for name, value in _computer_variables(env).items())
 
 
+def unresettable(computer: str) -> str:
+    """Why ``computer``, booted by a launcher older than reset in place, isn't reset."""
+    return f"{computer} was booted by an older agent-env, whose launcher can't reset it in place; redeploy it"
+
+
 def _computer_variables(env: dict[str, str]) -> dict[str, str]:
     return {f"{ENV_PREFIX}{name}": base64.b64encode(value.encode()).decode() for name, value in env.items()}

src/agent_env/providers/sandbox_providers/local_docker/cli.py

@@ -3,9 +3,17 @@
 import subprocess
 
 
+class DockerFailed(RuntimeError):
+    """A docker command that ended with ``returncode``."""
+
+    def __init__(self, message: str, returncode: int):
+        super().__init__(message)
+        self.returncode = returncode
+
+
 def docker(*args: str, input: str | None = None, timeout: float | None = None) -> str:
-    """``docker`` with ``args``, ``input`` on its stdin: its output, stripped. A failure raises."""
+    """``docker`` with ``args``, ``input`` on its stdin: its output, stripped. A failure raises DockerFailed."""
     done = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=timeout, input=input)
     if done.returncode:
-        raise RuntimeError(f"docker {' '.join(args)} failed: {done.stderr.strip()}")
+        raise DockerFailed(f"docker {' '.join(args)} failed: {done.stderr.strip()}", done.returncode)
     return done.stdout.strip()

src/agent_env/providers/sandbox_providers/local_docker/provider.py

@@ -39,8 +39,8 @@
 from agent_env.attribution import Attribution
 from agent_env.config.paths import state_root
 from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here
-from agent_env.providers.sandbox_providers.disk_launcher import RESET, reset_input
-from agent_env.providers.sandbox_providers.local_docker.cli import docker
+from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, reset_input, unresettable
+from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker
 from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_image
 from agent_env.providers.sandbox_providers.local_docker.relay import relay_image
 from agent_env.providers.sandbox_providers.sandbox import (
@@ -714,9 +714,15 @@ async def create_vm(
         return sandbox
 
     async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None:
-        """The launcher in the sandbox's container booting its disk afresh with ``env``."""
-        await asyncio.to_thread(docker, "exec", "-i", sandbox.container_name, *RESET, input=reset_input(env),
-                                timeout=_RESET_SECONDS)
+        """The launcher in the sandbox's container booting its disk afresh with ``env``; refused for a container an
+        agent-env older than reset in place started, before an upgrade."""
+        try:
+            await asyncio.to_thread(docker, "exec", "-i", sandbox.container_name, *RESET, input=reset_input(env),
+                                    timeout=_RESET_SECONDS)
+        except DockerFailed as e:
+            if e.returncode == UNRESETTABLE:
+                raise RuntimeError(unresettable(f"the computer in container {sandbox.container_name}")) from None
+            raise
 
     async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, str]) -> None:
         """The launcher booting ``image`` as the sandbox's container, its ports published as a container's are."""