diff --git a/packages/agentenv-protocol/README.md b/packages/agentenv-protocol/README.md index 3c4c68f22..c14c1c64e 100644 --- a/packages/agentenv-protocol/README.md +++ b/packages/agentenv-protocol/README.md @@ -636,6 +636,17 @@ leasing, and unverified for one with it until the probe has lease checks. The pr default role; the tool-access check hides a tool from a role of its own, `agentenv-probe-`, and gives it back. +### A disk's boot seed + +A Linux computer's disk form gets the deploy's environment at boot from a NoCloud seed, a FAT filesystem labelled +`CIDATA` that the launcher writes and the disk's init reads. `agentenv_protocol.computer.seed` defines it. A disk +composed by one agent-env version boots under another's launcher, so the seed carries every version its launcher +writes: version 1 in `user-data`, a later version N in `user-data.vN`, and the oldest it carries in +`agentenv-seed-oldest`. An init reads the newest version it knows, and stops the boot when the seed carries none of +them. Version 1 is `#agentenv-environment`, then `NAME=` a line, in byte order; `SAMPLE` and +`SAMPLE_USER_DATA` pin it, for a launcher's writer and an init's reader, wherever either is written, to be tested +against. A change an older init would misread is a new version; one it ignores, such as a new file on the seed, isn't. + ## Computer apps A computer app is software an agent uses on a computer, beside the driver: a desktop app, a service diff --git a/packages/agentenv-protocol/src/agentenv_protocol/computer/seed.py b/packages/agentenv-protocol/src/agentenv_protocol/computer/seed.py new file mode 100644 index 000000000..b4a5adf5c --- /dev/null +++ b/packages/agentenv-protocol/src/agentenv_protocol/computer/seed.py @@ -0,0 +1,53 @@ +"""The disk boot seed: how the launcher that boots a Linux computer's disk form hands it the deploy's environment, as a +NoCloud seed its init reads at boot. A disk composed by one agent-env version boots under another's launcher, so the +seed carries every version its launcher writes, and an init reads the newest it knows; a launcher that has stopped +writing every version an init reads says so, and that init stops the boot rather than start without its environment. + +Version 1 is a FAT filesystem labelled ``CIDATA`` holding ``user-data``: ``HEADER``, then a line a variable, +``NAME=``, ordered by line, so a value of any bytes stays one line. ``meta-data`` is NoCloud's own: +an ``instance-id`` and the guest's ``local-hostname``. A later version N goes in its own file, ``user-data.vN``, beside +the ones before it, and ``OLDEST`` names the oldest version the seed still carries.""" + +from __future__ import annotations + +import base64 + +#: The label of the seed's FAT filesystem, which an init finds it by. +LABEL = "CIDATA" +#: The first line of every version's user-data. +HEADER = "#agentenv-environment" +#: The file holding the oldest version the seed carries, a decimal number. A seed without it carries version 1. +OLDEST = "agentenv-seed-oldest" +#: The versions a launcher of this protocol writes, oldest first. +WRITTEN = (1,) + + +def user_data_file(version: int) -> str: + """The file ``version``'s user-data is in: ``user-data``, the name NoCloud gives it, for version 1, and + ``user-data.v`` after it.""" + return "user-data" if version == 1 else f"user-data.v{version}" + + +def user_data(environment: dict[str, str]) -> str: + """Version 1's user-data for ``environment``.""" + lines = sorted(f"{name}={base64.b64encode(value.encode()).decode()}" for name, value in environment.items()) + return "".join(f"{line}\n" for line in [HEADER, *lines]) + + +def read_user_data(text: str) -> dict[str, str]: + """The environment version 1's ``text`` holds, as an init reads it: blank and ``#`` lines skipped.""" + return {name: base64.b64decode(value).decode() for line in text.splitlines() if line and not line.startswith("#") + for name, _, value in [line.partition("=")]} + + +#: An environment and version 1's user-data for it, byte for byte, which a launcher's writer and an init's reader are +#: each tested against, wherever either is implemented. +SAMPLE = {"AGENTENV_CONTROL_TOKEN": "a value with spaces, = signs, 'quotes', $dollars and\na second line\n", + "ENVIRONMENT_NAME": "desk", "ENVIRONMENT_NAME2": "", "NOTE": "first line\nFAKE=1"} +SAMPLE_USER_DATA = ( + "#agentenv-environment\n" + "AGENTENV_CONTROL_TOKEN=YSB2YWx1ZSB3aXRoIHNwYWNlcywgPSBzaWducywgJ3F1b3RlcycsICRkb2xsYXJzIGFuZAphIHNlY29uZCBsaW5lCg==\n" + "ENVIRONMENT_NAME2=\n" + "ENVIRONMENT_NAME=ZGVzaw==\n" + "NOTE=Zmlyc3QgbGluZQpGQUtFPTE=\n" +) diff --git a/packages/agentenv-protocol/tests/test_computer_seed.py b/packages/agentenv-protocol/tests/test_computer_seed.py new file mode 100644 index 000000000..e823cd5b3 --- /dev/null +++ b/packages/agentenv-protocol/tests/test_computer_seed.py @@ -0,0 +1,14 @@ +"""The disk boot seed's version 1, as its sample pins it: what a launcher writes for an environment and what an init +reads back, every byte of a value kept, and where each version's user-data goes.""" + +from agentenv_protocol.computer import seed + + +def test_version_1_holds_the_sample_environment_byte_for_byte_and_reads_back_whole(): + assert seed.user_data(seed.SAMPLE) == seed.SAMPLE_USER_DATA + assert seed.read_user_data(seed.SAMPLE_USER_DATA) == seed.SAMPLE + + +def test_each_version_has_its_own_file_and_version_1_keeps_nocloud_s_name(): + assert [seed.user_data_file(version) for version in (1, 2, 10)] == ["user-data", "user-data.v2", "user-data.v10"] + assert seed.WRITTEN[0] == 1 diff --git a/src/agent_env/computers/README.md b/src/agent_env/computers/README.md index 550764fcd..b95bfbcef 100644 --- a/src/agent_env/computers/README.md +++ b/src/agent_env/computers/README.md @@ -20,7 +20,10 @@ It also has a disk form, a disk a VM boots from UEFI, which is the container for recipe in `disk/linux/`: a GPT disk of an EFI system partition holding one UKI, Ubuntu's kernel with a busybox initramfs, and an ext4 root filesystem that is the image's own, with `/etc/agentenv/vm/image.env`, its entrypoint, command, user, working folder and environment, from its OCI config. The init mounts it, brings up the network, reads the deploy's -settings from a NoCloud seed (`CIDATA`) and becomes the image's entrypoint, as its user. A release publishes it as +settings from a NoCloud seed (`CIDATA`) and becomes the image's entrypoint, as its user. The seed is +`agentenv_protocol.computer.seed`'s: it carries every version its launcher writes, and the init (`disk/linux/boot/`, +its reader in `seed-env`) reads the newest it knows, so a disk keeps booting under a later agent-env's launcher, and +stops the boot, saying to recompose the disk, when the seed carries none. A release publishes it as `ghcr.io/scaleapi/agentenv-ubuntu-24.04-disk:src-`, tagged by both its sources; without it, composing makes it. Composing a computer env makes each arch's disk from its composed container, the root filesystem rebuilt with `mkfs.ext4 -d` by the disk tools (`disk/linux/tools/`) on the builder's own platform and the base disk's EFI system @@ -28,7 +31,8 @@ partition kept, so nothing is emulated or privileged. A disk boots where `/dev/k (`disk/linux/launcher/`): gate 3 probes it there, and elsewhere it stays `unverified` until a deploy's first boot with KVM probes it. A deploy boots it with `env_params = { form = "disk" }`. The launcher serves each of the computer's ports through socat, whose listen backlog takes a burst of connections, in front of QEMU's own forward, whose backlog is 1, and -a reset boots the disk afresh on a new overlay in the same launcher, at the same URLs. It boots on local Docker where +a reset boots the disk afresh on a new overlay in the same launcher, at the same URLs; a container an agent-env from +before reset in place started is refused a reset, to be redeployed. It boots on local Docker where Docker can pass it `/dev/kvm`, and on `modal_vm`, amd64 only, whose sandbox runs the launcher as its image under nested virtualization: the provider exports the disk from the image store with this machine's Docker, compressed, once for each composed disk, into the per-user cache, and mounts it on the sandbox at create, which Modal uploads once for each diff --git a/src/agent_env/computers/disk/linux/boot/Dockerfile b/src/agent_env/computers/disk/linux/boot/Dockerfile index e91a97261..934b43af6 100644 --- a/src/agent_env/computers/disk/linux/boot/Dockerfile +++ b/src/agent_env/computers/disk/linux/boot/Dockerfile @@ -14,7 +14,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ linux-image-virtual busybox-static kmod zstd cpio systemd-boot-efi systemd-ukify python3-pefile \ && rm -rf /var/lib/apt/lists/* -COPY init udhcpc.script uki.sh /build/ +COPY init seed-env udhcpc.script uki.sh /build/ RUN sh /build/uki.sh /uki FROM scratch AS uki diff --git a/src/agent_env/computers/disk/linux/boot/init b/src/agent_env/computers/disk/linux/boot/init index ba43f8f97..886180daf 100755 --- a/src/agent_env/computers/disk/linux/boot/init +++ b/src/agent_env/computers/disk/linux/boot/init @@ -9,8 +9,7 @@ mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev log() { echo "agentenv-init: $*" > /dev/console; } fail() { log "$*"; sleep 2; poweroff -f; } -# decode VALUE: VALUE's base64 decoded into $decoded, its trailing newlines kept, which $(...) alone would drop. -decode() { decoded=$(echo "$1" | base64 -d; echo .); decoded=${decoded%.}; } +. /seed-env while read -r module; do insmod "$module" 2>/dev/null; done < /modules.order root="" for _ in $(seq 200); do root=$(findfs LABEL=agentenv-root 2>/dev/null) && break; sleep 0.05; done @@ -36,14 +35,7 @@ while read -r line; do workdir) decode "$value"; workdir=$decoded ;; esac done < "$config" -if [ -f /seed/user-data ]; then - while read -r line; do - key=${line%%=*} value=${line#*=} - case "$key" in ''|\#*) continue ;; esac - decode "$value" - export "$key=$decoded" - done < /seed/user-data -fi +[ -n "$seed" ] && { seed_environment /seed || fail "$refused"; } hostname=$( [ -f /seed/meta-data ] && sed -n 's/^local-hostname: *//p' /seed/meta-data) hostname "${hostname:-computer}" hostname > /newroot/etc/hostname diff --git a/src/agent_env/computers/disk/linux/boot/seed-env b/src/agent_env/computers/disk/linux/boot/seed-env new file mode 100644 index 000000000..ea97cffc2 --- /dev/null +++ b/src/agent_env/computers/disk/linux/boot/seed-env @@ -0,0 +1,45 @@ +# seed-env, sourced by init: the deploy's environment from the launcher's NoCloud seed, as +# agentenv_protocol.computer.seed defines it. init sources it before it exports the image's environment, and a seed may +# name any variable, so the reader works in a subshell and hands back only exports, each name checked and each value +# quoted, which seed_environment evaluates: no variable but those the seed names is touched. + +# decode VALUE: VALUE's base64 decoded into $decoded, its trailing newlines kept, which $(...) alone would drop. +decode() { decoded=$(echo "$1" | base64 -d; echo .); decoded=${decoded%.}; } + +# seed_environment DIR: export each variable of the newest seed version this init reads (1) that the seed mounted at DIR +# carries, from the one its agentenv-seed-oldest names, 1 without it; with none, return 1, the reason in $refused. +seed_environment() { eval "$(_seed_exports "$1")"; } + +# _seed_exports DIR: what seed_environment evaluates: an export a variable, or the refusal. +_seed_exports() ( + reads=1 + oldest=$(cat "$1/agentenv-seed-oldest" 2>/dev/null || echo 1) + newest=${reads##* } + case "$oldest" in ''|*[!0-9]*) oldest=1 ;; esac + if [ "$oldest" -gt "$newest" ]; then + echo "refused='this disk reads seed v$newest; this agent-env writes v$oldest or later;" \ + "recompose it with agent-env env build'; false" + exit 0 + fi + file= + for version in $reads; do + [ "$version" -ge "$oldest" ] || continue + name=user-data + [ "$version" -gt 1 ] && name=user-data.v$version + [ -f "$1/$name" ] && file=$1/$name + done + [ -n "$file" ] || exit 0 + while read -r line; do + name=${line%%=*} value=${line#*=} + case "$name" in ''|\#*|[0-9]*|*[!A-Za-z0-9_]*) continue ;; esac + decode "$value" + quoted= rest=$decoded + while :; do + case "$rest" in + *\'*) quoted="$quoted${rest%%\'*}'\\''" rest=${rest#*\'} ;; + *) quoted="$quoted$rest"; break ;; + esac + done + printf "export %s='%s'\n" "$name" "$quoted" + done < "$file" +) diff --git a/src/agent_env/computers/disk/linux/boot/uki.sh b/src/agent_env/computers/disk/linux/boot/uki.sh index c7451cd2b..2abda4523 100755 --- a/src/agent_env/computers/disk/linux/boot/uki.sh +++ b/src/agent_env/computers/disk/linux/boot/uki.sh @@ -1,7 +1,7 @@ #!/bin/sh # uki.sh OUT: the one file the disk form's EFI system partition holds, written to OUT under the name UEFI firmware boots -# from removable media: a UKI of Ubuntu's kernel and an initramfs of busybox, init, and the modules a virtio guest needs -# that the kernel doesn't build in, decompressed, since busybox's insmod reads no zstd. +# from removable media: a UKI of Ubuntu's kernel and an initramfs of busybox, init with its seed-env, and the modules a +# virtio guest needs that the kernel doesn't build in, decompressed, since busybox's insmod reads no zstd. set -eu out=$1 kver=$(ls /lib/modules | sort -V | tail -1) @@ -12,7 +12,7 @@ esac root=/tmp/initramfs mkdir -p "$root/bin" "$root/dev" "$root/proc" "$root/sys" "$root/newroot" "$root/seed" "$root/lib/modules" "$out" cp /bin/busybox "$root/bin/busybox" -cp /build/init /build/udhcpc.script "$root/" +cp /build/init /build/seed-env /build/udhcpc.script "$root/" chmod 755 "$root/init" "$root/udhcpc.script" : > "$root/modules.order" for module in virtio_pci virtio_mmio virtio_blk virtio_scsi virtio_net virtio_rng virtio_console ext4 vfat \ diff --git a/src/agent_env/computers/disk/linux/launcher/launch b/src/agent_env/computers/disk/linux/launcher/launch index 48e84eab1..3bfdf66bf 100755 --- a/src/agent_env/computers/disk/linux/launcher/launch +++ b/src/agent_env/computers/disk/linux/launcher/launch @@ -82,9 +82,9 @@ boot() { rm -rf "$work" mkdir -p "$work" qemu-img create -q -f qcow2 -b "$disk" -F qcow2 "$work/disk.qcow2" - seed "$work" + seed "$work/seed" mkfs.vfat -C -n CIDATA "$work/seed.img" 2048 >/dev/null - mcopy -i "$work/seed.img" "$work/user-data" "$work/meta-data" ::/ + mcopy -i "$work/seed.img" "$work/seed/"* ::/ if [ "$arch" = amd64 ]; then cp /usr/share/OVMF/OVMF_VARS_4M.fd "$work/vars.fd" else diff --git a/src/agent_env/computers/disk/linux/launcher/seed b/src/agent_env/computers/disk/linux/launcher/seed index 2de80326b..0334f0463 100755 --- a/src/agent_env/computers/disk/linux/launcher/seed +++ b/src/agent_env/computers/disk/linux/launcher/seed @@ -1,10 +1,13 @@ #!/bin/sh -# seed DIR: the NoCloud seed's files for the computer launch boots, in DIR: user-data holding each VM_ENV_ variable -# as NAME and its value, base64'd as the variable already holds it, a line each, as the disk's init reads them, and -# meta-data naming the instance. A base64 value has no newline, so no line env prints can pass for another variable. +# seed DIR: the NoCloud seed's files for the computer launch boots, in DIR, as agentenv_protocol.computer.seed defines +# them: version 1's user-data, holding each VM_ENV_ variable as NAME and its value, base64'd as the variable +# already holds it, a line each in byte order; agentenv-seed-oldest, the oldest version written; and meta-data naming +# the instance. A base64 value has no newline, so no line env prints can pass for another variable. set -eu +mkdir -p "$1" { echo "#agentenv-environment" - env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=\([A-Za-z0-9+/=]*\)$/\1=\2/p' | sort + env | sed -n 's/^VM_ENV_\([A-Za-z_][A-Za-z0-9_]*\)=\([A-Za-z0-9+/=]*\)$/\1=\2/p' | LC_ALL=C sort } > "$1/user-data" +echo 1 > "$1/agentenv-seed-oldest" printf 'instance-id: %s\nlocal-hostname: computer\n' "$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" > "$1/meta-data" diff --git a/src/agent_env/providers/sandbox_providers/disk_launcher.py b/src/agent_env/providers/sandbox_providers/disk_launcher.py index 248e0f69c..d0812b707 100644 --- a/src/agent_env/providers/sandbox_providers/disk_launcher.py +++ b/src/agent_env/providers/sandbox_providers/disk_launcher.py @@ -9,8 +9,13 @@ #: The launch script's path in the launcher image, its ENTRYPOINT, which boots the computer; a provider that clears the #: ENTRYPOINT runs it as the command. LAUNCH = "/usr/local/bin/launch" -#: Its command that boots the computer afresh, with the environment ``reset_input`` gives on its stdin. -RESET = (LAUNCH, "reset") +#: Where the launcher keeps what a reset and its boot loop share: a launcher older than reset in place has none. +CONTROL = "/var/lib/agentenv-vm/control" +#: The status ``RESET`` ends with in a sandbox whose launcher is older than reset in place. +UNRESETTABLE = 75 +#: The command that boots the computer afresh, with the environment ``reset_input`` gives on its stdin: the launcher's +#: ``launch reset``, which a launcher older than it would take for a second boot, so it isn't run there. +RESET = ("sh", "-c", f"[ -d {CONTROL} ] || exit {UNRESETTABLE}; exec {LAUNCH} reset") def launcher_variables(ports: list[int], env: dict[str, str], cpu: float, memory: int) -> dict[str, str]: @@ -25,5 +30,10 @@ def reset_input(env: dict[str, str]) -> str: return "".join(f"{name}={value}\n" for name, value in _computer_variables(env).items()) +def unresettable(computer: str) -> str: + """Why ``computer``, booted by a launcher older than reset in place, isn't reset.""" + return f"{computer} was booted by an older agent-env, whose launcher can't reset it in place; redeploy it" + + def _computer_variables(env: dict[str, str]) -> dict[str, str]: return {f"{ENV_PREFIX}{name}": base64.b64encode(value.encode()).decode() for name, value in env.items()} diff --git a/src/agent_env/providers/sandbox_providers/local_docker/cli.py b/src/agent_env/providers/sandbox_providers/local_docker/cli.py index 6117fc359..35b61a0c9 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/cli.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/cli.py @@ -3,9 +3,17 @@ import subprocess +class DockerFailed(RuntimeError): + """A docker command that ended with ``returncode``.""" + + def __init__(self, message: str, returncode: int): + super().__init__(message) + self.returncode = returncode + + def docker(*args: str, input: str | None = None, timeout: float | None = None) -> str: - """``docker`` with ``args``, ``input`` on its stdin: its output, stripped. A failure raises.""" + """``docker`` with ``args``, ``input`` on its stdin: its output, stripped. A failure raises DockerFailed.""" done = subprocess.run(["docker", *args], capture_output=True, text=True, timeout=timeout, input=input) if done.returncode: - raise RuntimeError(f"docker {' '.join(args)} failed: {done.stderr.strip()}") + raise DockerFailed(f"docker {' '.join(args)} failed: {done.stderr.strip()}", done.returncode) return done.stdout.strip() diff --git a/src/agent_env/providers/sandbox_providers/local_docker/provider.py b/src/agent_env/providers/sandbox_providers/local_docker/provider.py index 8ab8b04e9..590e0baad 100644 --- a/src/agent_env/providers/sandbox_providers/local_docker/provider.py +++ b/src/agent_env/providers/sandbox_providers/local_docker/provider.py @@ -39,8 +39,8 @@ from agent_env.attribution import Attribution from agent_env.config.paths import state_root from agent_env.providers.sandbox_providers.host_images import build_here, built_here, host_platform, present, pull_here -from agent_env.providers.sandbox_providers.disk_launcher import RESET, reset_input -from agent_env.providers.sandbox_providers.local_docker.cli import docker +from agent_env.providers.sandbox_providers.disk_launcher import RESET, UNRESETTABLE, reset_input, unresettable +from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed, docker from agent_env.providers.sandbox_providers.local_docker.disk import kvm_platforms, launcher_args, machine_image from agent_env.providers.sandbox_providers.local_docker.relay import relay_image from agent_env.providers.sandbox_providers.sandbox import ( @@ -714,9 +714,15 @@ async def create_vm( return sandbox async def reset_vm(self, sandbox: Sandbox, *, image: str, env: dict[str, str]) -> None: - """The launcher in the sandbox's container booting its disk afresh with ``env``.""" - await asyncio.to_thread(docker, "exec", "-i", sandbox.container_name, *RESET, input=reset_input(env), - timeout=_RESET_SECONDS) + """The launcher in the sandbox's container booting its disk afresh with ``env``; refused for a container an + agent-env older than reset in place started, before an upgrade.""" + try: + await asyncio.to_thread(docker, "exec", "-i", sandbox.container_name, *RESET, input=reset_input(env), + timeout=_RESET_SECONDS) + except DockerFailed as e: + if e.returncode == UNRESETTABLE: + raise RuntimeError(unresettable(f"the computer in container {sandbox.container_name}")) from None + raise async def _boot_disk(self, sandbox: LocalSandbox, *, image: str, env: dict[str, str]) -> None: """The launcher booting ``image`` as the sandbox's container, its ports published as a container's are.""" diff --git a/tst/unit/env/envs/test_disk_recipe.py b/tst/unit/env/envs/test_disk_recipe.py index e2aecf63a..2db19a883 100644 --- a/tst/unit/env/envs/test_disk_recipe.py +++ b/tst/unit/env/envs/test_disk_recipe.py @@ -1,6 +1,6 @@ """The Linux disk form's recipe as the init reads what it writes: the image's start (image-env, run by the disk tools) and -the deploy's environment (the launcher's NoCloud seed), each a line per value base64'd so any bytes survive; the -launcher, with QEMU and its tools stood in for, serving each port through socat in front of QEMU's forward and booting +the deploy's environment (the launcher's NoCloud seed, agentenv_protocol.computer.seed's), each a line per value +base64'd so any bytes survive, the init stopping the boot when the seed carries no version it reads; the launcher, with QEMU and its tools stood in for, serving each port through socat in front of QEMU's forward and booting its disk afresh in place on a reset; and the local provider booting a disk in the launcher, with KVM and the VM's ports, settings and size, and exporting one, compressed, once.""" @@ -9,19 +9,22 @@ import json import os import re +import shlex import subprocess import threading import time -from pathlib import Path +from pathlib import Path, PurePosixPath from types import SimpleNamespace from unittest.mock import AsyncMock import pytest +from agentenv_protocol.computer import seed as seed_protocol from agent_env.env.published_images import DISK -from agent_env.providers.sandbox_providers.disk_launcher import reset_input +from agent_env.providers.sandbox_providers.disk_launcher import CONTROL, LAUNCH, RESET, UNRESETTABLE, reset_input from agent_env.providers.sandbox_providers.local_docker import disk as local_disk from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.providers.sandbox_providers.local_docker.cli import DockerFailed from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandbox, LocalSandboxProvider from agent_env.providers.sandbox_providers.sandbox import SANDBOX_LABEL @@ -66,11 +69,54 @@ def test_the_seed_holds_each_variable_the_launcher_is_handed_by_its_name_whateve ("AGENTENV_SETTING_NOTE", forged), ("AGENTENV_SETTING_RESOLUTION", "1280x800"), ("EMPTY", "")] assert meta_data.endswith("\nlocal-hostname: computer\n") and meta_data.startswith("instance-id: ") + assert (tmp_path / seed_protocol.OLDEST).read_text() == "1\n" + + +def test_the_launchers_seed_and_the_inits_reader_hold_to_the_protocols_sample_byte_for_byte(tmp_path): + _seed(tmp_path / "written", seed_protocol.SAMPLE) + (tmp_path / "sample").mkdir() + (tmp_path / "sample" / "user-data").write_text(seed_protocol.SAMPLE_USER_DATA) + + assert (tmp_path / "written" / seed_protocol.user_data_file(1)).read_text() == seed_protocol.SAMPLE_USER_DATA + assert _seed_environment(tmp_path / "sample") == (0, seed_protocol.SAMPLE) + + +@pytest.mark.parametrize("files, read", [ + ({"user-data": seed_protocol.SAMPLE_USER_DATA}, (0, seed_protocol.SAMPLE)), + ({"user-data": seed_protocol.SAMPLE_USER_DATA, "user-data.v2": "#agentenv-environment\nV2=MQ==\n", + seed_protocol.OLDEST: "1\n"}, (0, seed_protocol.SAMPLE)), + ({"user-data.v2": "#agentenv-environment\nV2=MQ==\n", seed_protocol.OLDEST: "2\n"}, + (1, "this disk reads seed v1; this agent-env writes v2 or later; recompose it with agent-env env build")), + ({}, (0, {})), +], ids=["an-older-launchers-seed", "a-newer-launchers-seed", "a-seed-past-this-init", "no-user-data"]) +def test_the_init_reads_the_newest_seed_version_it_knows_and_refuses_a_seed_that_carries_none(tmp_path, files, read): + """An older launcher's seed has no oldest-version file; a newer one's carries later versions beside version 1.""" + for name, text in files.items(): + (tmp_path / name).write_text(text) + + assert _seed_environment(tmp_path) == read + + +def test_the_inits_reader_keeps_its_versions_and_leaves_an_images_variables_of_the_same_names_alone(tmp_path): + """init sources it, then exports the image's environment, whose variables may share a name with the reader's.""" + (tmp_path / "user-data").write_text(seed_protocol.SAMPLE_USER_DATA) + image = {name: "the image's" for name in ("reads", "oldest", "newest", "file", "version", "name", "line", "value", + "quoted", "rest", "decoded", "refused")} + + assert _seed_environment(tmp_path, image) == (0, seed_protocol.SAMPLE) + + +def test_the_inits_reader_exports_a_seed_variable_named_as_one_of_its_own_and_one_holding_quotes(tmp_path): + environment = {**seed_protocol.SAMPLE, **{name: f"{name}'s value, 'quoted'\n" for name in ( + "reads", "oldest", "file", "name", "line", "value", "quoted", "rest", "decoded", "refused", "_seed_reads")}} + (tmp_path / "user-data").write_text(seed_protocol.user_data(environment)) + + assert _seed_environment(tmp_path) == (0, environment) def test_the_inits_decoder_keeps_every_byte_of_a_start_and_a_seed_trailing_newlines_included(tmp_path): """As a container runtime would: $(...) alone drops a value's trailing newlines.""" - decoder = re.search(r"^decode\(\) .*$", (DISK / "boot" / "init").read_text(), re.MULTILINE).group(0) + decoder = re.search(r"^decode\(\) .*$", (DISK / "boot" / "seed-env").read_text(), re.MULTILINE).group(0) values = ["plain", "ends with a newline\n", "\n", f"{TRICKY}\n\n"] config = tmp_path / "config.json" config.write_text(json.dumps({"Cmd": values, "User": "user\n"})) @@ -107,8 +153,8 @@ def test_the_launcher_gets_kvm_the_vms_ports_and_size_and_its_environment_under_ "cp": 'touch "${@: -1}"', "zstd": 'echo "zstd $*" >> "$CALLS"; sleep "${ZSTD_TAKES_S:-0}"; touch "${@: -1}"', "socat": 'echo "socat $*" >> "$CALLS"; sleep "${SOCAT_EXITS_S:-30}"; exit 7', - **{qemu: f'echo "{qemu} $* seed=$(grep -vc "^#" "$VM_STATE/boot/user-data")' - f' $(sed -n 2,9p "$VM_STATE/boot/user-data" | tr "\\n" " ")" >> "$CALLS"\n' + **{qemu: f'echo "{qemu} $* seed=$(grep -vc "^#" "$VM_STATE/boot/seed/user-data")' + f' $(sed -n 2,9p "$VM_STATE/boot/seed/user-data" | tr "\\n" " ")" >> "$CALLS"\n' 'trap \'echo "stopped $$" >> "$CALLS"; kill $!; exit 0\' TERM\n' 'sleep "${QEMU_RUNS_S:-30}" & wait' for qemu in ("qemu-system-x86_64", "qemu-system-aarch64")}, } @@ -254,6 +300,19 @@ def test_a_reset_with_no_computer_running_says_so(launcher): assert reset.wait(10) == 1 and "no computer is running here" in reset.stderr.read() +def test_a_reset_isnt_run_by_a_launcher_older_than_reset_in_place(): + """Such a launcher has no control folder, and would take ``launch reset`` for a second boot of its computer.""" + assert not Path(CONTROL).exists() + + assert subprocess.run(RESET, capture_output=True).returncode == UNRESETTABLE + + +def test_the_launcher_images_entrypoint_is_the_launch_script_its_providers_run(): + lines = (DISK / "launcher" / "Dockerfile").read_text().splitlines() + + assert f'ENTRYPOINT ["{LAUNCH}"]' in lines and f"COPY launch seed {PurePosixPath(LAUNCH).parent}/" in lines + + def _seed(folder, env: dict[str, str]) -> None: """The seed of ``env`` in ``folder``, as the launcher makes it from what the local provider hands it.""" args = local_disk.launcher_args([18765], env, 2, 4096) @@ -262,6 +321,21 @@ def _seed(folder, env: dict[str, str]) -> None: env={"PATH": os.environ["PATH"], "HOME": "/root", **handed}) +def _seed_environment(folder: Path, exported: dict[str, str] | None = None) -> tuple[int, dict[str, str] | str]: + """The init's reading of the seed mounted at ``folder``, ``exported`` set after it sources the reader, as the image's + environment is: 0 and the variables it exported or changed, or 1 and why it stops.""" + image = "".join(f"export {name}={shlex.quote(value)}; " for name, value in (exported or {}).items()) + script = (f'. "{DISK / "boot" / "seed-env"}"; {image}env > "$1/before"; seed_environment "$1" || ' + '{ printf %s "$refused"; exit 1; }; env -0') + done = subprocess.run(["sh", "-c", script, "seed-env", str(folder)], capture_output=True, text=True, + env={"PATH": os.environ["PATH"]}) + if done.returncode: + return done.returncode, done.stdout + before = set((folder / "before").read_text().splitlines()) + return 0, {name: value for name, _, value in (entry.partition("=") for entry in done.stdout.split("\0")[:-1]) + if f"{name}={value}" not in before and name != "_"} + + @pytest.fixture def launched(monkeypatch): """The ``docker`` commands the local provider runs for a disk, with the launcher image it builds faked.""" @@ -291,13 +365,34 @@ async def test_a_disk_boots_in_the_launcher_with_kvm_its_ports_published_and_a_r assert first == ("run", "-d", "--name", sandbox.container_name, "--label", f"{SANDBOX_LABEL}={sandbox.sandbox_id}", "-p", "127.0.0.1:20001:18765", *local_disk.launcher_args([18765], {"ENVIRONMENT_NAME": "desk"}, 2, 4096), "agentenv-vm:0123") - assert reset == ("exec", "-i", sandbox.container_name, "/usr/local/bin/launch", "reset", - "VM_ENV_ENVIRONMENT_NAME=YWdhaW4=\n", local_sandbox._RESET_SECONDS) + assert reset == ("exec", "-i", sandbox.container_name, *RESET, "VM_ENV_ENVIRONMENT_NAME=YWdhaW4=\n", + local_sandbox._RESET_SECONDS) assert local_sandbox._RESET_SECONDS > 60 LocalSandbox.pull_images.assert_awaited_once_with(["localhost:5000/disk@sha256:" + "a" * 64]) assert sandbox.tunnel_urls == {18765: "http://127.0.0.1:20001"} and rebuilt.owns_container +@pytest.mark.asyncio +@pytest.mark.parametrize("returncode, refused", [ + (UNRESETTABLE, "the computer in container .* was booted by an older agent-env, whose launcher can't reset it in " + "place; redeploy it"), + (1, "docker exec .* failed: launch reset: no computer is running here"), +], ids=["older-launcher", "other-failure"]) +async def test_a_reset_in_a_container_an_older_launcher_runs_is_refused_naming_the_fix(launched, monkeypatch, + returncode, refused): + """An agent-env before reset in place started it, and a run resumed after an upgrade resets it.""" + provider = LocalSandboxProvider() + sandbox = await provider.create_vm(image="localhost:5000/disk@sha256:" + "a" * 64, exposed_ports=[18765], + env={"ENVIRONMENT_NAME": "desk"}, cpu=2, memory=4096) + + def docker(*args, **options): + raise DockerFailed(f"docker {' '.join(args)} failed: launch reset: no computer is running here", returncode) + + monkeypatch.setattr(local_sandbox, "docker", docker) + with pytest.raises(RuntimeError, match=refused): + await provider.reset_vm(sandbox, image="localhost:5000/disk@sha256:" + "a" * 64, env={}) + + @pytest.mark.asyncio async def test_a_boot_cancelled_while_docker_starts_the_launcher_waits_for_it_then_tears_it_down(launched, monkeypatch): started, release, order = threading.Event(), threading.Event(), [] diff --git a/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py b/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py index 91a5c5577..c7d606c6e 100644 --- a/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py +++ b/tst/unit/providers/sandbox_providers/modal_vm_sandbox_test.py @@ -416,10 +416,11 @@ async def test_terminate_sandbox_terminates_a_vm_by_id_and_counts_one_modal_no_l # ── a computer's disk form: the launcher as the sandbox's image, under nested virtualization ── +import shlex # noqa: E402 from pathlib import Path # noqa: E402 from types import SimpleNamespace # noqa: E402 -from agent_env.providers.sandbox_providers.disk_launcher import launcher_variables # noqa: E402 +from agent_env.providers.sandbox_providers.disk_launcher import RESET, launcher_variables # noqa: E402 DISK_IMAGE = "localhost:5000/agentenv-computers:" + "1" * 64 + "@sha256:" + "a" * 64 @@ -511,7 +512,7 @@ async def with_stdin(script, stdin): await ModalVmSandboxProvider().reset_vm(sandbox, image=DISK_IMAGE, env={"ENVIRONMENT_NAME": "again"}) sandbox._exec_with_stdin = AsyncMock(return_value=(1, "", "launch reset: no computer is running here\n")) - assert sent == [("/usr/local/bin/launch reset", b"VM_ENV_ENVIRONMENT_NAME=YWdhaW4=\n")] + assert sent == [(shlex.join(RESET), b"VM_ENV_ENVIRONMENT_NAME=YWdhaW4=\n")] with pytest.raises(RuntimeError, match="didn't boot its disk afresh .exit 1.: launch reset: no computer is running"): await ModalVmSandboxProvider().reset_vm(sandbox, image=DISK_IMAGE, env={})