diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 828f3b71d..73984604d 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -201,7 +201,8 @@ def _toml(self, entry: BundleEntry) -> ResolvedEntry: refs.append(ref) for site in ref_sites(refs, config, inline_pins=True): try: - name, version = parse_toml_ref(site.ref.kind, site.owner if site.version_key else site.value) + name, version = parse_toml_ref(site.ref.kind, site.owner if site.version_key else site.value, + site.ref.beside) except ValueError as e: self._problem(entry, f"{site.path}: {e}") continue diff --git a/src/agent_env/entity/refs.py b/src/agent_env/entity/refs.py index 3e4568fc0..e611ca426 100644 --- a/src/agent_env/entity/refs.py +++ b/src/agent_env/entity/refs.py @@ -53,8 +53,10 @@ class RefRole(StrEnum): class EntityRef: """One ref-holding field. ``path`` is dotted keys into the step dict or toml, ``[]`` meaning each element of a list (``env_ids[]``, ``artifacts[].id``); ``version_field`` is the key beside the - ref, in the same dict, that pins its version. ``walk``, for a shape a path cannot spell, maps - the value at the path's parent to the dicts that hold the last key, each with its path parts. + ref, in the same dict, that pins its version. ``beside`` names the keys a toml table holding the + ref may carry next to it, which its type reads (a computer env's app takes ``install``). ``walk``, + for a shape a path cannot spell, maps the value at the path's parent to the dicts that hold the + last key, each with its path parts. Declarations build one with ``EntityRef.env``, ``EntityRef.agent`` or ``EntityRef.artifact``, passing the keyword fields.""" @@ -65,6 +67,7 @@ class EntityRef: role: RefRole = RefRole.INPUT artifact_type: str | None = None env_type: str | None = None + beside: tuple[str, ...] = () walk: Walker | None = None @classmethod @@ -134,12 +137,13 @@ def default_agent_refs(refs: Iterable[EntityRef], step: dict) -> list[EntityRef] and step.get(ref.field) is None] -def parse_toml_ref(kind: EntityKind, value: Any) -> tuple[str, int | None]: +def parse_toml_ref(kind: EntityKind, value: Any, beside: tuple[str, ...] = ()) -> tuple[str, int | None]: """The id and version a toml ref names: an id, or ``{ = "", version = }`` with the - version optional. Raises ValueError naming the expected form.""" + version optional, and the keys of ``beside`` allowed in the table too. Raises ValueError naming the + expected form.""" table = isinstance(value, dict) entity_id, version = (value.get(kind.value), value.get("version")) if table else (value, None) - known_keys = not table or value.keys() <= {kind.value, "version"} + known_keys = not table or value.keys() <= {kind.value, "version", *beside} positive = version is None or (isinstance(version, int) and not isinstance(version, bool) and version >= 1) if not (known_keys and isinstance(entity_id, str) and entity_id and positive): raise ValueError(f'expected an id or {{ {kind.value} = "", version = }}, the version optional, ' diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index 06c15e09b..f9d9035fe 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -17,7 +17,7 @@ from agent_env.env.env import Env from agent_env.env.envs.computer.utils import composer, gate from agent_env.env.envs.computer.utils.composer import Composing -from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.env import APP_TYPE, ComputerEnv from agent_env.env.envs.computer.utils.install import AppInstall, folder_problems, installs from agent_env.env.envs.computer.driver_env import ComputerDriverEnv, declared from agent_env.env.store import get_env_store @@ -30,7 +30,7 @@ class ComputerAppEnv(ComputerEnv): - type: ClassVar[str] = "computer_app" + type: ClassVar[str] = APP_TYPE description = ("A computer app: software and its settings on a computer, read from its ComputerAppEnvironment " "class at put; deployed alone, a computer of its own") diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index c0eb195bc..5881e0308 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -32,12 +32,15 @@ if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext from agent_env.env.env import DeployedEnv + from agent_env.env.envs.computer.app_env import ComputerAppEnv from agent_env.providers.env_providers.env_provider import EnvironmentProvider logger = logging.getLogger(__name__) #: Where load_artifact puts files in a computer by default: one of the folders data/get returns by default. DEFAULT_DESTINATION = "Downloads" +#: The computer app env's type: an app composed into a computer env, and a computer env of its own. +APP_TYPE = "computer_app" class ComputerEnv(Env): @@ -45,14 +48,16 @@ class ComputerEnv(Env): description = "A computer: a base computer and a driver, composed at put into an image for each form and arch" env_provider_type: ClassVar[str] = "computer" toml_refs: ClassVar[tuple[EntityRef, ...]] = (EntityRef.artifact("computer", artifact_type="computer"), - EntityRef.env("driver", env_type=ComputerDriverEnv.type)) + EntityRef.env("driver", env_type=ComputerDriverEnv.type), + EntityRef.env("apps[]", env_type=APP_TYPE, beside=("install",))) def __init__(self, id: str, version: Optional[int], environment_name: str, computer: ComputerArtifact, driver: ComputerDriverEnv, display: Optional[dict] = None, network: Optional[dict] = None, - computer_settings: Optional[dict[str, SettingValue]] = None, metadata: Optional[dict[str, Any]] = None): + computer_settings: Optional[dict[str, SettingValue]] = None, metadata: Optional[dict[str, Any]] = None, + apps: Optional[list[ComputerAppEnv]] = None): super().__init__(id, version, metadata=metadata) self.environment_name = environment_name - self.computer, self.driver = computer, driver + self.computer, self.driver, self.apps = computer, driver, apps or [] self.display, self.network = display, network self.computer_settings = computer_settings or {} self._sandbox = None @@ -66,7 +71,8 @@ def to_dict(self) -> dict: "environment_name": self.environment_name, # dual-write with service_name during the rename "computer": {"id": self.computer.id, "version": self.computer.version}, "driver": {"id": self.driver.id, "version": self.driver.version}, "display": self.display, - "network": self.network, "computer_settings": self.computer_settings} + "network": self.network, "computer_settings": self.computer_settings, + **({"apps": [{"id": app.id, "version": app.version} for app in self.apps]} if self.apps else {})} @classmethod def from_dict(cls, data: dict) -> ComputerEnv: @@ -77,29 +83,33 @@ def from_dict(cls, data: dict) -> ComputerEnv: environment_name=data["environment_name"] if "environment_name" in data else data["service_name"], computer=ComputerArtifact.get(data["computer"]["id"], data["computer"]["version"]), driver=driver, display=data.get("display"), network=data.get("network"), - computer_settings=data.get("computer_settings"), metadata=data.get("metadata") or {}) + computer_settings=data.get("computer_settings"), metadata=data.get("metadata") or {}, + apps=[_app(Env.get(ref["id"], ref["version"])) for ref in data.get("apps") or []]) @classmethod def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str, apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None, compose: bool = True, composing: Composing = Composing()) -> Self: - """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it as ``composing`` - says unless ``compose`` is False. Refused, naming every mismatch, before anything is stored.""" - values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, + """Write a computer env of ``computer``, ``driver`` and ``apps``, computer app envs built into it in their + order, once gate 1 passes, then compose it as ``composing`` says unless ``compose`` is False. Refused, naming + every mismatch, before anything is stored.""" + apps = list(apps) + values, problems = gate.check(computer, driver, apps=apps, display=display, network=network, computer_settings=computer_settings or {}) if problems: raise ValueError(f"computer env {id!r}: " + "; ".join(problems)) env = get_env_store().put_document(cls( - id=id, version=None, environment_name=environment_name, - computer=computer, driver=driver, display=display, network=network, computer_settings=values)) + id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, apps=apps, + display=display, network=network, computer_settings=values)) if compose: composer.compose(env, composing=composing) return env @classmethod def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: - """Write the computer env an env.toml declares, its base and driver as the bundle wrote them or the ones they - name, its environment_name the folder's when it sets none.""" + """Write the computer env an env.toml declares, its base, driver and apps as the bundle wrote them or the ones + they name, its environment_name the folder's when it sets none. An app's ``install`` is ``"build"``, the + default: it's built into the computer.""" fields, refused = ctx.accepted(data, environment_name=str, computer=object, driver=object, apps=list, display=dict, network=dict, computer_settings=dict) missing = [ctx.config_problem(f"{key} is required: the {what}, such as {example}") for key, what, example in ( @@ -108,6 +118,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: ctx.refuse([*refused, *missing]) base, driver = ctx.artifact(fields.pop("computer"), ComputerArtifact), ctx.env(fields.pop("driver"), ComputerDriverEnv) + fields["apps"] = _listed_apps(fields.get("apps", []), ctx) try: return cls.put(id=ctx.id, computer=base, driver=driver, environment_name=fields.pop("environment_name", ctx.name), composing=ctx.composing, @@ -115,9 +126,9 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: except ValueError as e: ctx.refuse([str(e)]) - def composed_apps(self) -> list: + def composed_apps(self) -> list[ComputerAppEnv]: """The computer app envs composed into the env's computer, each installed under its prefix.""" - return [] + return self.apps def computer_artifact(self) -> ComputerArtifact: """The env's computer: its base composed with its driver, over the images built.""" @@ -217,3 +228,32 @@ async def close(self) -> None: except Exception as e: # a cancellation propagates, and keeps the sandbox for the next close() logger.warning(f"Failed to terminate sandbox {self._sandbox.sandbox_id}: {e}") self._sandbox = None + + +def _app(env: Env) -> ComputerAppEnv: + if refusal := _not_an_app(env): + raise TypeError(refusal) + return env + + +def _listed_apps(refs: list, ctx: AuthoringContext) -> list[ComputerAppEnv]: + """The computer app envs an env.toml's ``apps`` names, each a ref, or a table of one with its ``install`` beside it; + refused, naming each problem, unless every one is a computer app env installed at build.""" + apps, problems = [], [] + for i, ref in enumerate(refs): + install = ref.get("install", "build") if isinstance(ref, dict) else "build" + if install != "build": + problems.append(ctx.config_problem(f"apps[{i}]: install = {install!r}: an app installs at \"build\", into " + "the computer's image; \"boot\" isn't supported yet")) + continue + app = ctx.env({key: value for key, value in ref.items() if key != "install"} if isinstance(ref, dict) else ref) + if refusal := _not_an_app(app): + problems.append(ctx.config_problem(f"apps[{i}]: {refusal}")) + apps.append(app) + if problems: + ctx.refuse(problems) + return apps + + +def _not_an_app(env: Env) -> str | None: + return f"{env.id} is a {env.type} env, not a {APP_TYPE} env" if env.type != APP_TYPE else None diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 800f66d52..4904407ed 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -27,6 +27,7 @@ from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image from agent_env.config import get_config +from agent_env.env.envs.computer.utils import layers_in_image from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM @@ -42,12 +43,9 @@ logger = logging.getLogger(__name__) -#: What the builders make of a build context, which the build key covers in place of their source: raise it with a -#: change that alters what an image holds (its files, their modes, its labels, how it's built), and leave it for one -#: that changes only how a build gets there, so images aren't built again for nothing. -FORMAT = 1 CONTEXT, RESOLVED = "context", "resolve" RESOLVER = Path(__file__).with_name("resolve_in_base.py") +LAYERS = Path(__file__).with_name("layers_in_image.py") #: The shell side of a builder VM, its subcommands run there (``VmBuilder``). IN_VM = Path(__file__).with_name("builder_in_vm.sh") #: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged, @@ -136,6 +134,11 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: """The packages ``tag``'s build installed into the venv of the driver at ``prefix``, by name, read from the image without running it, so on any arch; none for a driver with no venv.""" + @abstractmethod + def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: + """What each of ``commands``, a RUN of ``tag``'s build, wrote in its layer of ``tag``, read from a save of the + image (``layers_in_image``).""" + @abstractmethod def push(self, tag: str, ref: str, store) -> None: """Push ``tag`` to ``ref`` in the image store ``store``, signed in with its credentials.""" @@ -323,6 +326,14 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as archive: return _packages(archive.extractfile(archive.getmembers()[0]).read()) + def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: + saved = self.work / "image.tar" + save_image(tag, saved) + try: + return layers_in_image.written(str(saved), commands) + finally: + saved.unlink(missing_ok=True) + def push(self, tag: str, ref: str, store) -> None: push_local_image(tag, ref, store) @@ -425,6 +436,15 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: listed = self._exec(f"{self.helper} installed {shlex.quote(tag)} {shlex.quote(f'{prefix}/installed.json')}") return _packages(listed.encode()) if listed.strip() else {} + def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: + """Read in a container of ``tag``, on its own Python, so the VM needs none.""" + image, script = f"{self.work}/image.tar", f"{self.work}/{LAYERS.name}" + self._write(script, LAYERS.read_bytes()) + return json.loads(self._exec( + f"docker save -o {image} {shlex.quote(tag)} && docker run --rm --network none --user root " + f"-v {image}:/image.tar:ro -v {script}:/layers.py:ro --entrypoint python3 {shlex.quote(tag)} /layers.py " + f"/image.tar {shlex.quote(json.dumps(commands))} && rm -f {image}")) + def push(self, tag: str, ref: str, store) -> None: self._login(store.auth(ref)) self._exec(f"docker tag {shlex.quote(tag)} {shlex.quote(ref)} && docker push {shlex.quote(ref)}") @@ -516,6 +536,10 @@ def _packages(listed: bytes) -> dict[str, str]: return {package["name"]: package["version"] for package in json.loads(listed)} +def save_image(tag: str, path: Path) -> None: + subprocess.run(["docker", "save", "-o", str(path), tag], check=True, capture_output=True, text=True) + + def sha256_of(path: Path) -> str: hashed = hashlib.sha256() with path.open("rb") as file: diff --git a/src/agent_env/env/envs/computer/utils/clashes.py b/src/agent_env/env/envs/computer/utils/clashes.py new file mode 100644 index 000000000..747efdd0a --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/clashes.py @@ -0,0 +1,93 @@ +"""Whether a part of a composed computer, its driver or an app, undid what an earlier part installed: what each part's +run scripts and install steps wrote, as ``layers_in_image`` lists it per layer, compared in build order. Only certain +losses are refused; any other file two parts wrote differently is only said, since it may be an edit that kept the +earlier part's change.""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +#: Where what a run script or an install step writes may be lost, as the filesystem hierarchy standard has it: +#: temporary files, runtime state, logs and caches. Never compared. +SCRATCH = re.compile(r"^/(tmp|var/tmp|var/log|var/cache|run)/|/\.cache/|/__pycache__/") +#: What each kind of entry is, to a reader: a program is a file. +_KINDS = {"file": "file", "exec": "file", "link": "link", "dir": "folder"} + + +class Entry(NamedTuple): + """A path of a layer, as ``layers_in_image`` lists it: its kind, and a file's or a program's sha256 or a link's + target.""" + + kind: str + what: str = "" + + @property + def known(self) -> bool: + """Whether its content is known: not a hard link to a file of an earlier layer, which the reader doesn't read.""" + return not self.what.startswith("=> ") + + @classmethod + def listed(cls, entry: str) -> Entry: + kind, _, what = entry.partition(" ") + return cls(kind, what) + + +def compare(layers: list[tuple[str, dict[str, str]]]) -> tuple[list[str], list[str]]: + """What each of ``layers``, a part's id and what one of its commands wrote, in build order, did to what another part + left, but under ``SCRATCH``: each certain loss (``_lost``), then each other file two parts wrote differently. A path + is compared with the last other part to write it, whoever wrote it since, until it's deleted or lost.""" + entries: dict[str, Entry] = {} + writers: dict[str, list[str]] = {} + lost, differ = [], [] + for of, wrote in layers: + listed = {path: Entry.listed(entry) for path, entry in wrote.items() if not SCRATCH.search(path)} + for path, entry in sorted(listed.items()): + if entry.kind == "opaque": + for hidden in [hidden for hidden in sorted(entries) if hidden.startswith(f"{path}/") + and hidden not in listed]: + if entries[hidden].kind not in ("dir", "deleted") and (by := _other(writers[hidden], of)): + lost.append(f"{hidden}: {of} deletes {by}'s file") + entries[hidden], writers[hidden] = Entry("deleted"), [] + entry = Entry("dir") + if entry.kind == "deleted": + for gone in [gone for gone in entries if gone.startswith(f"{path}/") and gone not in listed]: + entries[gone], writers[gone] = Entry("deleted"), [] + reason = None + if path in entries and (by := _other(writers[path], of)): + if reason := _lost(path, of, entry, by, entries[path]): + lost.append(reason) + elif _differs(entry, entries[path]): + differ.append(f"{path} ({by}, {of})") + entries[path] = entry + writers[path] = ([] if entry.kind == "deleted" else + [of] if reason else [*(writer for writer in writers.get(path, []) if writer != of), of]) + return lost, differ + + +def _other(writers: list[str], of: str) -> str | None: + """The part of ``writers``, oldest write first, that wrote a path last before ``of``.""" + return next((writer for writer in reversed(writers) if writer != of), None) + + +def _lost(path: str, of: str, entry: Entry, by: str, was: Entry) -> str | None: + """How ``of``'s ``entry`` at ``path`` certainly loses what ``by`` left there, ``was``: deleting it, making it + another kind, pointing its link elsewhere, or replacing its program; None when it doesn't.""" + if was.kind == "deleted": + return None + if entry.kind == "deleted": + return f"{path}: {of} deletes {by}'s {_KINDS[was.kind]}" + if _KINDS[entry.kind] != _KINDS[was.kind]: + return f"{path}: {of} makes {by}'s {_KINDS[was.kind]} a {_KINDS[entry.kind]}" + if was.kind == "link" and entry.what != was.what: + return f"{path}: {of} points {by}'s link elsewhere" + if was.kind == "exec" and entry.what != was.what and entry.known and was.known: + return f"{path}: {of} replaces {by}'s program" + return None + + +def _differs(entry: Entry, was: Entry) -> bool: + """Whether ``entry`` is a file that may hold other content than ``was``, a file, or a program whose content isn't + known.""" + return (entry.kind in ("file", "exec") and entry.what != was.what + and (was.kind == "file" or (was.kind == "exec" and not (entry.known and was.known)))) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index cd9f82446..dd3ce211f 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -13,7 +13,6 @@ import importlib.metadata import json import logging -import re import shlex import shutil import subprocess @@ -39,7 +38,7 @@ from agent_env.config import get_config from agent_env.entity.export import Unwritable, checked_path from agent_env.env.env import Env -from agent_env.env.envs.computer.utils import boot, builder, gate, packages +from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages from agent_env.env.envs.computer.utils.install import Install from agent_env.providers.sandbox_providers.host_images import platform_tag from agent_env.store.base import ConcurrentModificationError @@ -61,6 +60,11 @@ #: The image store repository composed computers are pushed to, tagged by their keys. REPOSITORY = "agentenv-computers" +#: What the composer and its builders make of a build's inputs beyond what its key shows of them (the base, the +#: Dockerfile, the build files with their executable bits, the lock): raise it with a change that alters what an image +#: holds some other way, its labels or how it's built, and leave it for any other, so images aren't built again for +#: nothing. +FORMAT = 1 #: Where a driver's files go in a composed computer, in a folder named for the driver; an app's, under APPS. DRIVERS = "/opt/agentenv/drivers" APPS = str(installed_apps.APPS) @@ -79,6 +83,8 @@ _PROTOCOL = "agentenv-framework-protocol" # What building the protocol from its source reads, beside src/. _PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") +#: How many of the files two parts wrote differently a progress line names. +_SHOWN = 10 @dataclass(frozen=True) @@ -235,7 +241,7 @@ def on() -> builder.Builder: key = "sha256:" + hashlib.sha256(json.dumps({ "base": base_identity, "dockerfile": dockerfile, "context": _digest(context), "lock": lock.digest() if lock is not None else None, "form": form, "arch": arch, - "composer": _composer_digest(), + "format": FORMAT, }, sort_keys=True).encode()).hexdigest() store = get_config().get_image_store_for(env.stored_id) store.ensure_repository(REPOSITORY) @@ -252,6 +258,8 @@ def on() -> builder.Builder: packages.stage(lock, building_on, env.stored_id, fetched) local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) + if reason := _compare(building_on, local, layers, pair, progress): + return _failed(env, form, arch, stage, key, reason) installed = building_on.installed(local, layer.prefix) building_on.build(f"FROM {local}\n", local, platform, { INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", @@ -279,6 +287,26 @@ def on() -> builder.Builder: return _failed(env, form, arch, stage, key, f"{type(e).__name__}: {e}") +def _compare(building_on: builder.Builder, tag: str, layers: list[_Layer], pair: str, + progress: _Progress) -> str | None: + """Why ``tag`` is refused: a part's run scripts or install steps certainly undoing what an earlier part's installed + (``clashes``); None, with any other file two parts wrote differently said to ``progress``. Read only when two or + more parts have such steps, the only ones that write outside a part's folder.""" + runs = {layer.env_id: _runs(layer) for layer in layers if _runs(layer)} + if len(runs) < 2: + return None + progress.at(pair, "comparing the files its parts' run scripts and install steps wrote") + written = building_on.written(tag, [command for commands in runs.values() for _, command in commands]) + lost, differ = clashes.compare(list(zip([env_id for env_id, commands in runs.items() for _ in commands], written))) + if lost: + return "a part undoes what an earlier one installed: " + "; ".join(lost) + if differ: + progress.at(pair, "compared", "not refused, as each may keep the earlier part's change: two parts wrote " + f"{', '.join(differ[:_SHOWN])}" + (f" and {len(differ) - _SHOWN} more" if len(differ) > _SHOWN + else "")) + return None + + def _base(base: ComputerArtifact, arch: str) -> tuple[str, str]: """What the composed image is built ``FROM`` for ``arch``, and the base's part of the key, found without building anything: a registry image pinned to its digest, or the tag a build of the base's stored build context for that @@ -294,13 +322,14 @@ def _base(base: ComputerArtifact, arch: str) -> tuple[str, str]: @dataclass(frozen=True) class _Layer: """A driver's or an app's part of a build: its folder in the build context, its prefix in the image, what it - installs, and the file of its supervisord programs. An app's venv also has the protocol's ``app`` extra, for the + installs, the file of its supervisord programs, and the id of its env. An app's venv also has the protocol's ``app`` extra, for the app's own server, which listens in a folder of the app's the session user owns.""" folder: str prefix: str install: Install conf: str + env_id: str app: bool = False served: str | None = None steps: tuple[tuple[str, str], ...] = () @@ -309,11 +338,12 @@ class _Layer: def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact, from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them, served on - its port, its folder under ``DRIVERS`` named for its id's last part.""" - install, name = driver.install[base.os.type], _name(driver.id, "driver") - layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf") - _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}, - from_store) + its port, its folder under ``DRIVERS`` named for its id's last part, its programs and their file under ``driver-`` + as an app's are under ``app-``, so neither ever takes the other's.""" + install, name = driver.install[base.os.type], gate.folder_name(driver.id, "driver") + layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-driver-{name}.conf", driver.id) + _stage(context, layer, f"driver {driver.id}", driver.files, _services(f"driver-{name}", install), + {"MCP_PORT": driver.port}, from_store) return layer @@ -321,15 +351,17 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the app's install files under ``context/apps//``, with the values of its build settings the env fixes, and its supervisord programs beside them, run under its name: its services and its own server, which serves - its class from its venv and launches the app. Its folder under ``APPS`` is named for its id's last part.""" - install, name = app.install[base.os.type], _name(app.id, "app") + its class from its venv and launches the app. Its folder under ``APPS`` is named for its environment name.""" + install, name = app.install[base.os.type], gate.app_folder(app) file, _, cls = app.module.partition(":") served = shlex.quote(f"{PurePosixPath(file).relative_to(install.files)}:{cls}") - layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app=True, served=served, + layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app.id, app=True, + served=served, steps=tuple((step.name, step.scope) for step in app.declaration.install_steps if step.os is None or base.os.type in step.os)) - programs = {f"app-{name}": f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", - **_services(f"app-{name}", install)} + programs = dict(zip(gate.programs(app, base.os.type), [ + f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", + *(service.command for service in install.services)])) _stage(context, layer, f"app {app.id}", app.files, programs, {"ENVIRONMENT_NAME": app.environment_name}, from_store) if app.build_settings: @@ -340,10 +372,6 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, return layer -def _name(env_id: str, fallback: str) -> str: - return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback - - def _services(name: str, install: Install) -> dict[str, str]: """The commands of ``install``'s services by their programs' names, each named for ``name``.""" return {f"{name}-{service.name}": service.command for service in install.services} @@ -447,11 +475,8 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build f"--format=json > {prefix}/installed.json") if layer.app: lines.append(f"RUN mkdir -p {prefix}/{SOCKET.parent} && chown {user} {prefix}/{SOCKET.parent}") - for run in layer.install.run or []: - lines += _as(run.scope, user, f"RUN --network=none cd {prefix} && {shlex.quote('./' + run.script)}") - for step, scope in layer.steps: - lines += _as(scope, user, f"RUN --network=none cd {prefix} && venv/bin/python -m " - f"agentenv_protocol.computer.run_install_step {layer.served} {step}") + for scope, command in _runs(layer): + lines += _as(scope, user, f"RUN --network=none {command}") lines.append(f"COPY {layer.conf} /etc/supervisor/conf.d/{layer.conf}") checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}"] if _venv(layer): @@ -461,6 +486,14 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build return "\n".join(lines) + "\n" +def _runs(layer: _Layer) -> list[tuple[str, str]]: + """The commands of ``layer``'s run scripts and its install steps, each with the scope it runs as: the only steps of + a build that write outside the layer's folder.""" + return [*((run.scope, f"cd {layer.prefix} && {shlex.quote('./' + run.script)}") for run in layer.install.run or []), + *((scope, f"cd {layer.prefix} && venv/bin/python -m agentenv_protocol.computer.run_install_step " + f"{layer.served} {step}") for step, scope in layer.steps)] + + def _as(scope: str, user: str, run: str) -> list[str]: """``run`` as root, or for ``session``, as ``user``.""" return [run] if scope == "system" else [f"USER {user}", run, "USER root"] @@ -533,18 +566,15 @@ def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str]) - def _digest(folder: Path) -> str: - """A digest of the files under ``folder``, by their paths and bytes.""" + """A digest of the files under ``folder``, by their paths, bytes and whether each is executable, which the composer + alone decides (umask never sets it).""" hashed = hashlib.sha256() for path in sorted(p for p in folder.rglob("*") if p.is_file()): - hashed.update(path.relative_to(folder).as_posix().encode() + b"\0" + hashlib.sha256(path.read_bytes()).digest()) + hashed.update(path.relative_to(folder).as_posix().encode() + b"\0" + hashlib.sha256(path.read_bytes()).digest() + + (b"x" if path.stat().st_mode & 0o100 else b"-")) return "sha256:" + hashed.hexdigest() -def _composer_digest() -> str: - """A digest of the composer, which says what a build makes of its inputs, and of its builders' ``FORMAT``.""" - return "sha256:" + hashlib.sha256(Path(__file__).read_bytes() + f"builder {builder.FORMAT}".encode()).hexdigest() - - def _now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index 4a8ab7440..5ec5cf8f2 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -6,6 +6,9 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any +import re + +from agentenv_protocol.computer.apps import APPS from agentenv_protocol.computer.platform import Platform, supported from jsonschema import Draft202012Validator @@ -26,10 +29,7 @@ def check(base: ComputerArtifact, driver: ComputerDriverEnv, *, apps: list, disp computer_settings: Any) -> tuple[dict[str, SettingValue], list[str]]: """The env's values of the base's settings, and every way ``base``, ``driver`` and what the env declares don't compose.""" - problems = [] - if apps: - problems.append("apps: computer app envs aren't supported yet") - problems += _platform(base, driver) + problems = _platform(base, driver) + _apps(base, driver, apps) problems += _image(base) problems += _display(base, display, computer_settings) problems += _network(network) @@ -51,22 +51,77 @@ def arches(base: ComputerArtifact, driver: ComputerDriverEnv, apps: Sequence[Com return [arch for arch in base.arch if arch in matching] -def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAppEnv) -> list[str]: - """Every way ``app`` doesn't fit a composition of ``base`` and ``driver``: a platform it doesn't support, no - architecture in common, or no install for the base's OS.""" +def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAppEnv, at: str = "app") -> list[str]: + """Every way ``app`` doesn't fit a composition of ``base`` and ``driver``, each named from ``at``: a platform it + doesn't support, no architecture in common, no install for the base's OS, or a port the driver listens on too.""" os = base.os if not _matching(base, app.declaration.supports): named = ", ".join(_shown(platform) for entry in app.declaration.supports for platform in supported(entry)) - return [f"app: {app.id} supports no platform {base.id} is ({os.type} {os.distribution} {os.version} " + return [f"{at}: {app.id} supports no platform {base.id} is ({os.type} {os.distribution} {os.version} " f"{os.display}); it supports {named or 'none'}"] problems = [] if not arches(base, driver, [app]): - problems.append(f"app: no architecture {base.id}, {driver.id} and {app.id} all run") + problems.append(f"{at}: no architecture {base.id}, {driver.id} and {app.id} all run") if os.type not in app.install: - problems.append(f"app: {app.id} declares no install for {os.type}") + problems.append(f"{at}: {app.id} declares no install for {os.type}") + elif taken := sorted(set(_ports(app, os.type)) & set(_driver_ports(driver, os.type))): + problems.append(f"{at}: {app.id} listens on {', '.join(map(str, taken))}, as {driver.id} does") + return problems + + +def folder_name(name: str, fallback: str) -> str: + """``name``'s last part, lowercased, as a slug: the name of a folder a part is installed in.""" + return re.sub(r"[^a-z0-9-]+", "-", name.rsplit("/", 1)[-1].lower()).strip("-") or fallback + + +def app_folder(app: ComputerAppEnv) -> str: + """The name of the folder ``app`` is installed in, under ``APPS``, and of its programs: its environment name's, the + one name a computer knows it by.""" + return folder_name(app.environment_name, "app") + + +def _apps(base: ComputerArtifact, driver: ComputerDriverEnv, apps: list[ComputerAppEnv]) -> list[str]: + """Every way ``apps`` don't fit together on ``base`` and ``driver``: each one's own (``check_app``), no architecture + they all run, one listed twice, and two under one environment name, in one folder, with one program name or + listening on one port, each pair of apps named for the first they'd share.""" + problems = [problem for i, app in enumerate(apps) for problem in check_app(base, driver, app, f"apps[{i}]")] + if not problems and apps and not arches(base, driver, apps): + problems.append(f"apps: no architecture {base.id}, {driver.id} and every app all run") + ids = [app.id for app in apps] + problems += [f"apps: {app_id} is listed twice" for app_id in sorted({i for i in ids if ids.count(i) > 1})] + distinct, named = list({app.id: app for app in apps}.values()), set() + for kind, keys in (("environment name", lambda app: [app.environment_name]), + ("folder", lambda app: [f"{APPS}/{app_folder(app)}"]), + ("program", lambda app: programs(app, base.os.type)), + ("port", lambda app: _ports(app, base.os.type))): + claims: dict[object, list[str]] = {} + for app in distinct: + for key in keys(app): + claims.setdefault(key, []).append(app.id) + for key, owners in claims.items(): + if len(owners) > 1 and frozenset(owners) not in named: + named.add(frozenset(owners)) + problems.append(f"apps: {' and '.join(owners)} would share the {kind} {key!r}") return problems +def programs(app: ComputerAppEnv, os_type: str) -> list[str]: + """The names of the supervisord programs ``app`` runs on ``os_type``: its own server's, then its services'.""" + server = f"app-{app_folder(app)}" + install = app.install.get(os_type) + return [server, *(f"{server}-{service.name}" for service in (install.services if install is not None else []))] + + +def _ports(app: ComputerAppEnv, os_type: str) -> list[int]: + install = app.install.get(os_type) + return list(install.ports or []) if install is not None else [] + + +def _driver_ports(driver: ComputerDriverEnv, os_type: str) -> list[int]: + install = driver.install.get(os_type) + return [driver.port, *(install.ports or [] if install is not None else [])] + + def check_app_settings(app: ComputerAppEnv, values: dict[str, Any], at: str, os_type: str) -> list[str]: """The problems with ``values``, of ``app``'s settings on ``os_type`` by name, each named from ``at``: a setting it doesn't declare, a secret's given as anything but a bare reference (or null, where it's optional), a required one diff --git a/src/agent_env/env/envs/computer/utils/layers_in_image.py b/src/agent_env/env/envs/computer/utils/layers_in_image.py new file mode 100644 index 000000000..10b33cbc0 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/layers_in_image.py @@ -0,0 +1,71 @@ +"""What RUN instructions of a saved image wrote, read by the composer after a build (``builder.Builder.written``): each +RUN, given by its command in build order, is found in the image's history from the top, the last command first, each +in the newest layer below the one found before it that a shell ran exactly that command for, so a base's layers of the +same command never stand in for the build's own; and each entry of its layer listed by path, as `` ``: +``file`` or ``exec`` (a file any can run) with its sha256, ``link`` with its target, ``dir``, ``deleted``, or ``opaque`` +for a folder whose earlier contents the layer hides. A hard link is listed as its target is in the same layer, else +with ``=> `` in place of a sha256, since an earlier layer isn't read. Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py + ``; it prints what each wrote, in order, as JSON.""" + +import hashlib +import json +import sys +import tarfile +from pathlib import PurePosixPath + +_CHUNK = 1 << 20 + + +def written(image: str, commands: list[str]) -> list[dict[str, str]]: + with tarfile.open(image) as saved: + manifest = json.load(saved.extractfile("manifest.json"))[0] + history = json.load(saved.extractfile(manifest["Config"]))["history"] + made = [entry.get("created_by", "") for entry in history if not entry.get("empty_layer")] + # Matched from the top: a base built without history has fewer entries than layers. + count = min(len(made), len(manifest["Layers"])) + layers, found, at = list(zip(made[-count:], manifest["Layers"][-count:])), [], count + for command in reversed(commands): + at = next((i for i in range(at - 1, -1, -1) if _ran(layers[i][0], command)), None) + if at is None: + raise ValueError(f"no layer of the image below the last one found was made by {command!r}") + found.append(_files(saved, layers[at][1])) + return found[::-1] + + +def _ran(made_by: str, command: str) -> bool: + """Whether the history entry ``made_by`` is a RUN of ``command`` by a shell, as BuildKit and the classic builder + record one.""" + return made_by.removesuffix(" # buildkit").endswith(f"/bin/sh -c {command}") + + +def _files(saved: tarfile.TarFile, member: str) -> dict[str, str]: + files: dict[str, str] = {} + with tarfile.open(fileobj=saved.extractfile(member), mode="r|*") as layer: + for entry in layer: + path = PurePosixPath("/" + entry.name.removeprefix("./").rstrip("/")) + kind = "exec" if entry.mode & 0o111 else "file" + if path.name == ".wh..wh..opq": + files[str(path.parent)] = "opaque" + elif path.name.startswith(".wh."): + files[str(path.with_name(path.name.removeprefix(".wh.")))] = "deleted" + elif entry.isreg(): + files[str(path)] = f"{kind} {_sha256(layer.extractfile(entry))}" + elif entry.issym(): + files[str(path)] = f"link {entry.linkname}" + elif entry.islnk(): + target = "/" + entry.linkname.removeprefix("./") + files[str(path)] = files.get(target, f"{kind} => {target}") + elif entry.isdir(): + files[str(path)] = "dir" + return files + + +def _sha256(file) -> str: + hashed = hashlib.sha256() + while chunk := file.read(_CHUNK): + hashed.update(chunk) + return hashed.hexdigest() + + +if __name__ == "__main__": + json.dump(written(sys.argv[1], json.loads(sys.argv[2])), sys.stdout) diff --git a/tst/integration/computer/app/artifacts/check_desk/run.py b/tst/integration/computer/app/artifacts/check_desk/run.py new file mode 100644 index 000000000..38db8dd25 --- /dev/null +++ b/tst/integration/computer/app/artifacts/check_desk/run.py @@ -0,0 +1,32 @@ +"""What each of the two apps composed into the desk wrote on the settings its deploy gave it, the clock's answer on its +own port, the name the computer's card gives and what its readiness says of each app, asked on loopback from inside +the computer, the apps launched since it booted, and each app's install under its own folder, its run script's output +there.""" +import json +import time +import urllib.request +from pathlib import Path + +APPS = Path("/opt/agentenv/apps") + + +def run(input): + written = [Path.home() / ".notes.json", Path.home() / ".clock.json"] + for _ in range(60): + if all(path.exists() for path in written): + break + time.sleep(0.5) + with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response: + card = json.load(response) + with urllib.request.urlopen(f"{input['env']['url']}/agentenv/ext/computer-ready/status", timeout=30) as response: + ready = json.load(response) + with urllib.request.urlopen("http://127.0.0.1:8137/", timeout=10) as response: + clock = json.load(response) + notes = json.loads(written[0].read_text()) + return {"notes": {"signature": notes["signature"], "token": notes["token"]}, + "clock": {**json.loads(written[1].read_text()), "served": clock}, "card": card["name"], + "app_ready": sorted(check["name"] for check in ready["checks"] + if check["name"].startswith("app ") and check["ok"]), + "launched": sorted(p.name for p in Path("/run/agentenv/apps").iterdir()), + "folders": sorted(p.name for p in APPS.iterdir()), "hello": (APPS / "notes" / "hello.txt").exists(), + "paris": "Paris" in (APPS / "clock" / "zones.txt").read_text().split()} diff --git a/tst/integration/computer/app/envs/clock/app.py b/tst/integration/computer/app/envs/clock/app.py new file mode 100644 index 000000000..0aec9d802 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/app.py @@ -0,0 +1,16 @@ +"""A clock app for the computer env tests, composed beside notes: its service answers on a port of its own with the +time zone its settings name, and writes it to ~/.clock.json. It installs tzdata, which its run script lists the zones +of, and declares no window.""" +from pydantic import BaseModel + +from agentenv_protocol.computer import ComputerAppEnvironment, Packages, RunScript, Service, computer_app, linux + +PORT = 8137 + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@linux(packages=Packages(apt=["tzdata"]), run=[RunScript(script="setup.sh")], + services=[Service(name="clock", command="venv/bin/python serve.py")], ports=[PORT]) +class Clock(ComputerAppEnvironment): + class Settings(BaseModel): + zone: str = "UTC" diff --git a/tst/integration/computer/app/envs/clock/env.toml b/tst/integration/computer/app/envs/clock/env.toml new file mode 100644 index 000000000..60e142353 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/env.toml @@ -0,0 +1,2 @@ +type = "computer_app" +module = "app.py:Clock" diff --git a/tst/integration/computer/app/envs/clock/serve.py b/tst/integration/computer/app/envs/clock/serve.py new file mode 100644 index 000000000..681135470 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/serve.py @@ -0,0 +1,20 @@ +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +from app import PORT, Clock + +zone = Clock().settings.zone +Path.home().joinpath(".clock.json").write_text(json.dumps({"zone": zone})) + + +class Zone(BaseHTTPRequestHandler): + def do_GET(self): + body = json.dumps({"zone": zone, "known": Path("/usr/share/zoneinfo", zone).exists()}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + + +HTTPServer(("127.0.0.1", PORT), Zone).serve_forever() diff --git a/tst/integration/computer/app/envs/clock/setup.sh b/tst/integration/computer/app/envs/clock/setup.sh new file mode 100755 index 000000000..012a6b4e9 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/setup.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Run as root once tzdata is in: the zones it installed, beside the app. +set -e +ls /usr/share/zoneinfo/Europe > zones.txt diff --git a/tst/integration/computer/app/envs/desk/env.toml b/tst/integration/computer/app/envs/desk/env.toml new file mode 100644 index 000000000..8e8759b4f --- /dev/null +++ b/tst/integration/computer/app/envs/desk/env.toml @@ -0,0 +1,4 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" +apps = ["notes", { env = "clock", install = "build" }] diff --git a/tst/integration/computer/app/tasks/together.json b/tst/integration/computer/app/tasks/together.json new file mode 100644 index 000000000..a6ff2aad5 --- /dev/null +++ b/tst/integration/computer/app/tasks/together.json @@ -0,0 +1,6 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desk", + "env_params": {"app_settings": {"notes": {"signature": "-- Ada", "token": "env:NOTES_TOKEN"}, + "clock": {"zone": "Europe/Paris"}}}}, + {"id": "together", "type": "run_code", "env_id": "desk", "script_artifact_id": "check_desk"} +] diff --git a/tst/integration/computer/builder_modal_vm_test.py b/tst/integration/computer/builder_modal_vm_test.py index 6061270af..e66ed9f08 100644 --- a/tst/integration/computer/builder_modal_vm_test.py +++ b/tst/integration/computer/builder_modal_vm_test.py @@ -1,7 +1,8 @@ """A builder VM on Modal, amd64, building for arm64 through the emulation it registers: the build context sent inline, an apt package and a requirement resolved in a container of the base, a file it wrote stored off the VM, an image built -from them with the network closed, and the packages it installed read back, each in the VM. The stores stay local, -which a compose on Modal refuses, so the builder is driven directly.""" +from them with the network closed, the packages it installed read back, and what each of its RUNs wrote read from its +layers in a container of the image, under emulation too, each in the VM. The stores stay local, which a compose on Modal +refuses, so the builder is driven directly.""" import hashlib from pathlib import Path @@ -25,6 +26,9 @@ /tmp/packages/wheels/*.whl && /opt/driver/venv/bin/pip list --format=json > /opt/driver/installed.json && \ /opt/driver/start && jq --version """ +#: Each RUN's command, as the image's history records it once the Dockerfile's continued lines are joined. +RUNS = [line.removeprefix("RUN --network=none ") for line in DOCKERFILE.replace("\\\n", "").splitlines() + if line.startswith("RUN ")] def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): @@ -50,6 +54,7 @@ def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): f"{builder.RESOLVED}/out/wheels/driver/{wheel}"}, {}) vm.build(DOCKERFILE, "agentenv-builder-test:arm64", "linux/arm64", {"dev.agentenv.test": "builder"}) installed = vm.installed("agentenv-builder-test:arm64", "/opt/driver") + packaged, venv = vm.written("agentenv-builder-test:arm64", RUNS) finally: vm.close() @@ -58,3 +63,5 @@ def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): assert {deb["arch"] for deb in result["apt"]} == {"arm64"} assert hashlib.sha256(stored.read_bytes()).hexdigest() == written[f"wheels/driver/{wheel}"].sha256 assert vm.sandbox.platform == "linux/amd64" and Path(stored).stat().st_size == written[f"wheels/driver/{wheel}"].size + assert packaged["/usr/bin/jq"].startswith("exec ") and venv["/opt/driver/installed.json"].startswith("file ") + assert venv["/opt/driver/venv/bin/python"].startswith("link ") and "/opt/driver/start" not in venv diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index fb4c09da0..6f50fab65 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -3,8 +3,11 @@ settings the deploy gave it, a secret resolved at boot, its window launched once by its server as the desktop session comes up, and the computer ready only once its server says its window and its @ready_check are; its apt packages, one from a PPA, its download, run script and install step on its build setting, fetched at put, installed offline. -Needs Docker and the local registry on localhost:5000.""" +And two apps composed into one computer env, each running on its own settings from its own folder, the files their run +scripts wrote compared before the push of this machine's arch, or the image of the same key reused, as one an earlier +run pushed is. Needs Docker and the local registry on localhost:5000.""" +import platform from pathlib import Path import pytest @@ -16,6 +19,7 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] APP = Path(__file__).parent / "app" +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_named_after_it(state, monkeypatch): @@ -35,3 +39,22 @@ def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_na "app_ready": [{"name": "app notes", "ok": True, "detail": "ready"}], "launchers": ["agentenv-app-notes.desktop"], "launched": ["notes"], "hello": "Hello, world!", "license": True, "installed": "pro"} + + +def test_two_apps_composed_into_one_computer_env_each_run_on_their_own_settings(state, monkeypatch): + monkeypatch.setenv("NOTES_TOKEN", "t0ken") + + ran = CliRunner().invoke(cli, ["run", str(APP), "--task", "together"]) + + assert ran.exit_code == 0, ran.output + context = run_context(ran.output) + (deployed,) = context["deployed_envs"] + assert deployed["env_id"].endswith("/desk") and deployed["environment_card"]["name"] == "desk" + assert context["metadata"]["script_results"]["together"] == { + "notes": {"signature": "-- Ada", "token": "t0ken"}, + "clock": {"zone": "Europe/Paris", "served": {"zone": "Europe/Paris", "known": True}}, "card": "desk", + "app_ready": ["app clock", "app notes"], "launched": ["notes"], "folders": ["clock", "notes"], "hello": True, + "paris": True} + said = [line for line in ran.output.splitlines() if line.startswith(f"envs/desk: container/{ARCH}: ")] + assert any(line.endswith(("comparing the files its parts' run scripts and install steps wrote", + "reusing the image of the same key, built earlier")) for line in said), said diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index e5825860b..d01249337 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -19,6 +19,8 @@ from agent_env.env.envs.computer.env import ComputerEnv from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from tst.unit.bundle._support import layout, local_builds +from tst.unit.bundle.computer_app_env_test import APP, TOML as APP_TOML +from tst.unit.bundle.computer_driver_env_test import LINUX UBUNTU = "@local/agentenv-framework/computers/ubuntu-24.04" REFERENCE = "@local/agentenv-framework/drivers/reference" @@ -30,6 +32,12 @@ [computer_settings] timezone = "Europe/Paris" ''' +WITH_APPS = DESKTOP.replace("display =", 'apps = ["notes", { env = "mail", install = "build" }]\ndisplay =') +#: Two app folders, mail and notes, of one app class. +APPS = {f"envs/{name}/{file}": text for name in ("mail", "notes") for file, text in ( + ("env.toml", APP_TOML), ("app.py", APP.format(supports=LINUX, skill="skill.md", linux_skill="skill-linux.md")), + ("requirements.txt", "agentenv-framework-protocol\n"), ("skill.md", "# Mail\n"), + ("skill-linux.md", "# Mail on Linux\n"))} @pytest.fixture @@ -48,10 +56,10 @@ def compose(env, pairs=None, composing=Composing()): return calls -def _bundle(root, toml=DESKTOP): +def _bundle(root, toml=DESKTOP, extra=None): return layout(root, {"envs/desktop/env.toml": toml, "tasks/t.json": json.dumps([ {"id": "desktop", "type": "load_artifact", "env_id": "desktop", "artifact_id": "notes"}]), - "artifacts/notes/notes.txt": "a note\n"}) + "artifacts/notes/notes.txt": "a note\n", **(extra or {})}) def _installed(tmp_path, package, name): @@ -137,17 +145,33 @@ def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_pa assert f"{resolved.bundle.id_root}/desktop" in {write.id for write in plan.writes} +def test_a_computer_env_folder_names_its_apps_in_order_each_written_before_it(tmp_path, composed): + plan = _plan(_bundle(tmp_path / "desk", WITH_APPS, APPS)) + + materialize(plan, build_here=local_builds(plan)) + + root = plan.bundle.bundle.id_root + env = Env.get(f"{root}/desktop") + assert [(app.id, app.type, app.environment_name) for app in env.composed_apps()] == [ + (f"{root}/notes", "computer_app", "notes"), (f"{root}/mail", "computer_app", "mail")] + assert composed == [(env.id, UBUNTU, REFERENCE, None)] + + @pytest.mark.parametrize("toml, problem", [ ('type = "computer"\ndriver = "reference"\n', 'computer is required: the base computer, such as "ubuntu-24.04"'), ('type = "computer"\ncomputer = "ubuntu-24.04"\n', 'driver is required: the driver, such as "reference"'), (DESKTOP.replace("display =", "ports = [80]\ndisplay ="), "unknown key 'ports'"), (DESKTOP.replace("1600", "0"), "display is { width = , height = }, each 1 to 32767 pixels"), -], ids=["no-computer", "no-driver", "unknown-key", "display"]) + (WITH_APPS.replace('"build"', '"boot"'), 'apps[1]: install = \'boot\': an app installs at "build", into the ' + 'computer\'s image; "boot" isn\'t supported yet'), + (WITH_APPS.replace('["notes", ', '["notes", "notes", '), "apps: @local/~/desk/notes is listed twice"), + (WITH_APPS.replace('install = "build"', 'mode = "build"'), 'apps[1].env: expected an id or { env = "", version = ' + ' }, the version optional'), +], ids=["no-computer", "no-driver", "unknown-key", "display", "boot", "twice", "key-beside"]) def test_a_computer_env_folder_that_declares_what_cant_compose_is_refused_naming_the_folder(tmp_path, composed, toml, problem): - plan = _plan(_bundle(tmp_path / "desk", toml)) - with pytest.raises(BundleError) as refused: + plan = _plan(_bundle(tmp_path / "desk", toml, APPS)) materialize(plan, build_here=local_builds(plan)) assert any(line.startswith("envs/desktop") and problem in line for line in refused.value.problems), refused.value diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 3af40b05b..06c305cad 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -1,19 +1,24 @@ """What the computer env tests share: the fake Docker and registry the composer builds with, a fake sandbox provider whose VMs build, and a base, a driver and an env of them.""" +import gzip +import hashlib +import io import json import re import shlex import subprocess +import tarfile import urllib.parse -from agentenv_protocol.computer import DriverDeclaration +from agentenv_protocol.computer import AppDeclaration, DriverDeclaration from agentenv_protocol.computer.conformance import CheckResult, ProbeReport from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config -from agent_env.env.envs.computer.utils import builder +from agent_env.env.envs.computer.utils import builder, layers_in_image +from agent_env.env.envs.computer.app_env import ComputerAppEnv from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.utils.install import Install, Service from agent_env.env.envs.computer.driver_env import ComputerDriverEnv @@ -48,6 +53,7 @@ class FakeDocker: def __init__(self, registry: Registry): self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} + self.writes, self.saved = {}, [] def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" @@ -97,6 +103,24 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ "build_settings": {path.parent.parent.name: json.loads(path.read_text()) for path in context.glob("apps/*/.agentenv/build_settings.json")}}) + def save(self, tag, path): + """``tag`` saved as ``docker save`` does: a base layer with no history, then a layer for each COPY and RUN of its + Dockerfile, a RUN's holding the files ``writes`` gives for each key its command has.""" + self.saved.append(tag) + dockerfile = next(build["dockerfile"] for build in reversed(self.builds) if build["tag"] == tag) + layers = [(None, {"etc/os-release": b"ubuntu"})] + for line in dockerfile.splitlines(): + instruction, _, rest = line.partition(" ") + command = rest.removeprefix("--network=none ") + if instruction == "RUN": + layers.append((f"RUN /bin/sh -c {command} # buildkit", { + path: content for key, files in self.writes.items() if key in command for path, content in files.items()})) + elif instruction == "COPY": + layers.append((f"{line} # buildkit", {})) + elif instruction != "FROM": + layers.append((line, None)) + saved_image(path, layers) + def push(self, local, ref, store): self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], labels=self.labels.get(local)) @@ -109,6 +133,41 @@ def _arch(self, config_digest): return json.loads(self.registry.blobs[config_digest])["architecture"] +def saved_image(path, layers): + """A tar of an image as ``docker save`` writes one: each of ``layers``, bottom first, its history's ``created_by`` + (None for a layer with no entry) and its files by path, a file's content as bytes, or with its mode as (bytes, mode), + and a link's target as text, or None for an entry that made no layer; the last layer gzipped, as some savers keep + them.""" + blobs, history, made = {}, [], [] + for i, (created_by, files) in enumerate(layers): + if created_by is not None: + history.append({"created_by": created_by, **({"empty_layer": True} if files is None else {})}) + if files is None: + continue + layer = io.BytesIO() + with tarfile.open(fileobj=layer, mode="w") as tar: + for name, content in files.items(): + entry = tarfile.TarInfo(name) + if isinstance(content, str): + entry.type, entry.linkname = tarfile.SYMTYPE, content + tar.addfile(entry) + else: + data, entry.mode = content if isinstance(content, tuple) else (content, 0o644) + entry.size = len(data) + tar.addfile(entry, io.BytesIO(data)) + data = gzip.compress(layer.getvalue()) if i == len(layers) - 1 else layer.getvalue() + made.append(digest := f"blobs/sha256/{hashlib.sha256(data).hexdigest()}") + blobs[digest] = data + config = json.dumps({"history": history}).encode() + blobs[config_path := f"blobs/sha256/{hashlib.sha256(config).hexdigest()}"] = config + blobs["manifest.json"] = json.dumps([{"Config": config_path, "Layers": made}]).encode() + with tarfile.open(path, mode="w") as tar: + for name, data in blobs.items(): + entry = tarfile.TarInfo(name) + entry.size = len(data) + tar.addfile(entry, io.BytesIO(data)) + + def _base(image, **changes): return ComputerArtifact.put(inspect=False, **{ "id": "@local/~/kit/ubuntu-24.04", "arch": ["amd64", "arm64"], "os": OS, "session": {"user": "user"}, @@ -193,7 +252,7 @@ def _line(self, line): return ran.returncode, ran.stdout + ran.stderr def _docker(self, words): - docker, cwd = self.provider.docker, None + docker, cwd, printed = self.provider.docker, None, "" while words: command, words = (words[:words.index("&&")], words[words.index("&&") + 1:]) if "&&" in words else (words, []) if command[0] == "cd": @@ -214,6 +273,13 @@ def _docker(self, words): platform=command[command.index("--platform") + 1], labels=labels) except DockerBuildError as e: return 1, str(e) + elif command[:2] == ["docker", "save"]: + docker.save(command[4], self.at(command[3])) + elif command[:2] == ["docker", "run"] and "/layers.py" in command: + saved = command[command.index("-v") + 1].split(":")[0] + printed = json.dumps(layers_in_image.written(str(self.at(saved)), json.loads(command[-1]))) + elif command[:2] == ["rm", "-f"]: + self.at(command[2]).unlink(missing_ok=True) elif command[:2] == ["docker", "tag"]: self.provider.tagged[command[3]] = command[2] elif command[:2] == ["docker", "push"]: @@ -224,7 +290,7 @@ def _docker(self, words): self.provider.uploaded.append(url) else: raise AssertionError(f"a builder VM was asked to run {command}") - return 0, "" + return 0, printed class FakeVmProvider(SandboxProvider): @@ -238,3 +304,46 @@ async def create_vm(self, **size): self.sizes.append(size) self.vms.append(vm := FakeVm(self, self.root / str(len(self.vms)))) return vm + + +MAIL_SETTINGS = {"properties": { + "signature": {"default": "", "type": "string"}, + "password": {"default": "", "type": "string", "format": "password", "writeOnly": True, "minLength": 32}, + "token": {"anyOf": [{"type": "string", "format": "password", "writeOnly": True}, {"type": "null"}], "default": None}, + "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} + + +MAIL_BUILD = {"properties": {"edition": {"default": "home", "enum": ["home", "pro"], "type": "string"}, + "locale": {"default": "en_US", "type": "string"}}, "type": "object"} + + +#: The program of the mail app's own server. +MAIL_SERVER = ("[program:app-mail]\ncommand=/opt/agentenv/apps/mail/venv/bin/python -m " + "agentenv_protocol.computer.serve_app app.py:Mail\ndirectory=/opt/agentenv/apps/mail\n" + "environment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\nredirect_stderr=true\n" + "stdout_logfile=/var/log/agentenv/app-mail.log\n") + + +def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, + extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, name="mail", **changes): + """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside + them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of + its build settings, ``linux_settings``, that of its settings on Linux, and ``name``, its id's last part and its + environment name.""" + folder.mkdir(parents=True, exist_ok=True) + (folder / "app.py").write_text("class Mail: ...\n") + (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") + for path, text in (extra or {}).items(): + (folder / path).parent.mkdir(parents=True, exist_ok=True) + (folder / path).write_text(text) + files = FileArtifactUniverse.put_bundled(f"@local/~/kit/{name}__files", files={ + path: folder / path for path in ("app.py", "requirements.txt", *(extra or {}))}) + declaration = AppDeclaration.model_validate({ + "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], + "permissions": [], "settings": settings, "build_settings": build, "skill": None, + "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], + "ready_checks": [], "os": {"linux": {"launch": launch, "settings": linux_settings, "install": install or { + "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) + return ComputerAppEnv.put(**{"id": f"@local/~/kit/{name}", "environment_name": name, "module": "app.py:Mail", + "files": files, "declaration": declaration, "computer": base, "driver": driver, + "compose": compose, **changes}) diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py index 2ebe7fada..065f6a885 100644 --- a/tst/unit/env/envs/conftest.py +++ b/tst/unit/env/envs/conftest.py @@ -23,6 +23,7 @@ def docker(local_stores, cli_routing, monkeypatch): fake = FakeDocker(registry) monkeypatch.setattr(builder, "build_image", fake.build_image) monkeypatch.setattr(builder, "push_local_image", fake.push) + monkeypatch.setattr(builder, "save_image", fake.save) monkeypatch.setattr(builder, "docker_login", lambda auth: None) monkeypatch.setattr(composer, "write_index", fake.joined) monkeypatch.setattr(composer, "pin_digest", registry.pin_digest) diff --git a/tst/unit/env/envs/test_computer_apps.py b/tst/unit/env/envs/test_computer_apps.py new file mode 100644 index 000000000..4306f6f6a --- /dev/null +++ b/tst/unit/env/envs/test_computer_apps.py @@ -0,0 +1,197 @@ +"""Computer envs of several apps: each app built into the computer after the driver, in the order the env lists them; +the apps checked against each other at put; and what their run scripts and install steps wrote compared before a push, +a part certainly undoing what an earlier one installed refused and any other file two parts wrote differently said +(``clashes``). Docker, the registry and the VMs are faked.""" + +import pytest + +from agent_env.env import Env +from agent_env.env.envs.computer.utils import clashes +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.store.base import NotFoundError +from tst.unit.env.envs._computer import MAIL_SERVER, FakeVmProvider, _app, _put + +MAIL, NOTES = "@local/~/kit/mail", "@local/~/kit/notes" +A, B, C = "@local/~/kit/a", "@local/~/kit/b", "@local/~/kit/c" + + +def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_them(base, driver, docker, tmp_path): + mail = _app(base, driver, tmp_path / "mail", compose=False) + notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", id="@local/~/other/mail", + install={"ports": [8025], "services": [{"name": "sync", "command": "venv/bin/python sync.py"}]}) + + env = _put(base, driver, apps=iter([notes, mail])) + + assert sorted(build["platform"] for build in docker.builds) == ["linux/amd64", "linux/arm64"] + dockerfile, confs = docker.builds[0]["dockerfile"], docker.builds[0]["confs"] + assert dockerfile.index("COPY driver/") < dockerfile.index("COPY apps/notes/") < dockerfile.index( + "COPY apps/mail/") < dockerfile.index("USER user") + assert [layer["name"] for layer in docker.resolved[0][1]["python"]][-2:] == ["apps-notes", "apps-mail"] + assert confs["agentenv-app-mail.conf"].startswith(MAIL_SERVER) and "[program:app-notes-sync]" in confs[ + "agentenv-app-notes.conf"] + assert sorted(confs) == ["agentenv-app-mail.conf", "agentenv-app-notes.conf", "agentenv-driver-desk.conf"] + stored = Env.get(env.id) + assert [(app.id, app.version) for app in stored.composed_apps()] == [(notes.id, 1), (mail.id, 1)] + assert stored.to_dict()["apps"] == [{"id": notes.id, "version": 1}, {"id": mail.id, "version": 1}] + assert stored.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + assert docker.saved == [] + assert _put(base, driver, apps=[mail, notes]).metadata["composed"]["container"]["arm64"]["key"] != ( + stored.metadata["composed"]["container"]["arm64"]["key"]) + + +@pytest.mark.parametrize("apps, problem", [ + (lambda mail, other: [mail, mail], "apps: @local/~/kit/mail is listed twice"), + (lambda mail, other: [mail, other(name="mail-2", environment_name="mail")], + "apps: @local/~/kit/mail and @local/~/kit/mail-2 would share the environment name 'mail'"), + (lambda mail, other: [mail, other(name="x", environment_name="Mail")], + "apps: @local/~/kit/mail and @local/~/kit/x would share the folder '/opt/agentenv/apps/mail'"), + (lambda mail, other: [other(name="a", install={"ports": [8025], "services": []}), + other(name="b", install={"ports": [8025, 9000], "services": []})], + "apps: @local/~/kit/a and @local/~/kit/b would share the port 8025"), + (lambda mail, other: [mail, other(name="desk-port", install={"ports": [18765], "services": []})], + "apps[1]: @local/~/kit/desk-port listens on 18765, as @local/~/kit/desk does"), + (lambda mail, other: [mail, other(name="arm", supports=[{"os": {"type": "linux", "display": ["x11"]}, + "arch": ["riscv64"]}])], + "apps[1]: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and @local/~/kit/arm all run"), + (lambda mail, other: [other(name=arch, supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": [arch]}]) + for arch in ("amd64", "arm64")], + "apps: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and every app all run"), + (lambda mail, other: [other(name="mail", install={"services": [{"name": "sync", "command": "venv/bin/sync"}]}), + other(name="mail-sync", install={"services": []})], + "apps: @local/~/kit/mail and @local/~/kit/mail-sync would share the program 'app-mail-sync'"), +], ids=["twice", "environment-name", "folder", "app-port", "driver-port", "arch", "no-shared-arch", "program"]) +def test_apps_that_dont_fit_together_or_on_the_computer_are_refused_before_anything_is_stored(base, driver, docker, + tmp_path, apps, problem): + def other(name="other", **changes): + return _app(None, None, tmp_path / name, compose=False, name=name, **changes) + + listed = apps(_app(base, driver, tmp_path / "mail", compose=False), other) + + with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: + _put(base, driver, apps=listed) + + assert problem in str(refused.value) and str(refused.value).count(" would share ") <= 1 + with pytest.raises(NotFoundError): + Env.get("@local/~/kit/desktop") + assert docker.builds == [] + + +def _setting_up(tmp_path, *names, scripts=("setup.sh",)): + """An app of each of ``names`` whose install runs ``scripts`` as root, put without a computer.""" + return [_app(None, None, tmp_path / name, compose=False, name=name, + install={"run": [{"script": script} for script in scripts], "services": []}, + extra={script: "#!/bin/sh\n" for script in scripts}) for name in names] + + +@pytest.mark.parametrize("layers, lost", [ + ([(A, {"/usr/local/bin/tool": "file 1"}), (B, {"/usr/local/bin/tool": "deleted"})], + [f"/usr/local/bin/tool: {B} deletes {A}'s file"]), + ([(A, {"/opt/shared/data": "file 1", "/opt/shared/sub": "dir"}), + (B, {"/opt/shared": "opaque", "/opt/shared/other": "file 2"})], + [f"/opt/shared/data: {B} deletes {A}'s file"]), + ([(A, {"/usr/bin/tool": "link /opt/a/tool"}), (B, {"/usr/bin/tool": "link /opt/b/tool"})], + [f"/usr/bin/tool: {B} points {A}'s link elsewhere"]), + ([(A, {"/usr/bin/tool": "file 1"}), (B, {"/usr/bin/tool": "link /opt/b/tool"})], + [f"/usr/bin/tool: {B} makes {A}'s file a link"]), + ([(A, {"/opt/x": "dir"}), (B, {"/opt/x": "file 1"})], [f"/opt/x: {B} makes {A}'s folder a file"]), + ([(A, {"/usr/local/bin/tool": "exec 1"}), (B, {"/usr/local/bin/tool": "exec 2"})], + [f"/usr/local/bin/tool: {B} replaces {A}'s program"]), + ([(A, {"/t": "exec 1"}), (B, {"/t": "exec 1"}), (B, {"/t": "deleted"})], [f"/t: {B} deletes {A}'s file"]), + ([(A, {"/t": "link /a"}), (B, {"/t": "link /a"}), (B, {"/t": "link /b"})], + [f"/t: {B} points {A}'s link elsewhere"]), + ([(A, {"/o/data": "file 1"}), (B, {"/o": "opaque"}), (B, {"/o/data": "file 2"})], + [f"/o/data: {B} deletes {A}'s file"]), + ([(A, {"/t": "exec 1"}), (B, {"/t": "exec 2"}), (C, {"/t": "deleted"})], + [f"/t: {B} replaces {A}'s program", f"/t: {C} deletes {B}'s file"]), + ([(A, {"/opt/s": "dir", "/opt/s/t": "exec 1"}), (B, {"/opt/s": "deleted"}), (B, {"/opt/s/t": "exec 2"})], + [f"/opt/s: {B} deletes {A}'s folder"]), +], ids=["deleted", "opaque", "link", "kind", "folder", "program", "kept-then-deleted", "kept-then-pointed", + "opaque-then-written", "after-a-loss", "a-folder-deleted"]) +def test_a_part_that_certainly_undoes_what_another_left_is_a_loss(layers, lost): + assert clashes.compare(layers) == (lost, []) + + +@pytest.mark.parametrize("layers, differ", [ + ([(A, {"/etc/passwd": "file 1"}), (B, {"/etc/passwd": "file 2"})], [f"/etc/passwd ({A}, {B})"]), + ([(A, {"/x": "file 1"}), (B, {"/x": "exec 2"})], [f"/x ({A}, {B})"]), + ([(A, {"/x": "exec 1", "/l": "link /a", "/d": "dir"}), (B, {"/x": "exec 1", "/l": "link /a", "/d": "dir"})], []), + ([(A, {path: "file 1" for path in ("/var/log/s.log", "/tmp/x", "/run/x", "/home/user/.cache/x", + "/o/__pycache__/m.pyc")}), + (B, {"/var/log/s.log": "deleted", "/tmp/x": "exec 2", "/run/x": "link /x", "/home/user/.cache/x": "dir", + "/o/__pycache__/m.pyc": "file 2"})], []), + ([(A, {"/x": "exec 1"}), (A, {"/x": "deleted"})], []), + ([(A, {"/o/d": "file 1"}), (A, {"/o/d": "deleted"}), (B, {"/o": "opaque"})], []), + ([(A, {"/t": "exec 1"}), (A, {"/t": "deleted"}), (B, {"/t": "exec 2"}), (B, {"/t": "deleted"})], []), + ([(A, {"/o/t": "exec 1"}), (A, {"/o": "opaque"}), (B, {"/o/t": "exec 2"}), (B, {"/o/t": "deleted"})], []), + ([(A, {"/a": "file 1"}), (B, {"/b": "file 2"})], []), + ([(A, {"/o/t": "exec 1"}), (A, {"/o": "deleted"}), (B, {"/o/t": "exec 2"}), (B, {"/o/t": "deleted"})], []), + ([(A, {"/usr/local/bin/tool": "exec => /usr/bin/tool"}), (B, {"/usr/local/bin/tool": "exec 9"})], + [f"/usr/local/bin/tool ({A}, {B})"]), +], ids=["edited", "made-a-program", "same", "scratch", "own", "deleted-before-opaque", "recreated", + "recreated-after-opaque", "apart", "recreated-after-its-folder-was-deleted", "hard-link-to-an-earlier-layer"]) +def test_what_isnt_certainly_lost_isnt_a_loss_and_only_a_file_two_parts_wrote_differently_is_said(layers, differ): + assert clashes.compare(layers) == ([], differ) + + +def test_a_part_that_certainly_undoes_what_an_earlier_one_installed_fails_its_pairs_at_build_before_the_push( + base, driver, docker, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ": {"usr/local/bin/tool": (b"notes'", 0o755)}} + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes")).id) + + assert sorted(docker.saved) == sorted(build["tag"] for build in docker.builds) + composed = env.metadata["composed"]["container"] + assert {arch: (r["status"], r["stage"], r["reason"]) for arch, r in composed.items()} == { + arch: ("failed", "build", f"a part undoes what an earlier one installed: /usr/local/bin/tool: {NOTES} replaces " + f"{MAIL}'s program") for arch in ("amd64", "arm64")} + assert docker.probed == [] and "computer" not in env.metadata + + +@pytest.mark.parametrize("mail, notes, told", [ + ({"etc/passwd": b"root\\nmail\\n", "usr/share/icons/hicolor/icon-theme.cache": b"\\x00mail"}, + {"etc/passwd": b"root\\nmail\\nnotes\\n", "usr/share/icons/hicolor/icon-theme.cache": b"\\x00mail notes"}, + f"/etc/passwd ({MAIL}, {NOTES}), /usr/share/icons/hicolor/icon-theme.cache ({MAIL}, {NOTES})"), + ({f"etc/conf.d/{i:02}": b"mail's" for i in range(12)}, {f"etc/conf.d/{i:02}": b"notes'" for i in range(12)}, + ", ".join(f"/etc/conf.d/{i:02} ({MAIL}, {NOTES})" for i in range(10)) + " and 2 more"), + ({"etc/x": b"same"}, {"etc/x": b"same"}, None), +], ids=["edited", "many", "same"]) +def test_files_two_parts_wrote_differently_are_said_on_a_progress_line_and_not_refused(base, driver, docker, tmp_path, + mail, notes, told): + docker.writes, said = {"apps/mail && ": mail, "apps/notes && ": notes}, [] + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), + composing=Composing(on_progress=said.append)).id) + + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + compared = [line.partition(": ")[2] for line in said if line.startswith("container/arm64: not refused")] + assert compared == ([f"not refused, as each may keep the earlier part's change: two parts wrote {told}"] + if told else []) + + +def test_one_part_with_run_scripts_has_nothing_to_compare_and_its_image_isnt_saved(base, driver, docker, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": b"mail's"}} + mail = _setting_up(tmp_path, "mail")[0] + + env = Env.get(_put(base, driver, apps=[mail, _app(None, None, tmp_path / "notes", compose=False, name="notes")]).id) + + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} and docker.saved == [] + + +def test_a_builder_vm_compares_what_each_part_wrote_in_a_container_of_the_image_before_pushing(base, driver, docker, + signing, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ": {"usr/local/bin/tool": (b"notes'", 0o755)}} + vms = FakeVmProvider(docker, tmp_path / "vms") + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), composing=Composing(vms)).id) + + composed = env.metadata["composed"]["container"] + assert {arch: (r["status"], r["reason"].partition(": ")[2]) for arch, r in composed.items()} == { + arch: ("failed", f"/usr/local/bin/tool: {NOTES} replaces {MAIL}'s program") for arch in ("amd64", "arm64")} + compared = [line for line in vms.ran if "/layers.py" in line] + assert len(compared) == 2 and all("docker run --rm --network none --user root" in line for line in compared) + assert not [line for line in vms.ran if "docker push" in line] and all(vm.terminated for vm in vms.vms) + assert not list((tmp_path / "vms").rglob("image.tar")) + + diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 8b6f94cbe..146480fa3 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -9,6 +9,7 @@ import logging import re import subprocess +import sys import tarfile import time from types import SimpleNamespace @@ -20,12 +21,12 @@ from agent_env.artifact.store import get_artifact_store from agent_env.cli import cli from agent_env.env import Env -from agent_env.env.envs.computer.utils import builder, composer, packages +from agent_env.env.envs.computer.utils import builder, composer, layers_in_image, packages from agent_env.env.envs.computer.utils.composer import Composing from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider -from tst.unit.env.envs._computer import _base, _put, FakeVm, FakeVmProvider +from tst.unit.env.envs._computer import _base, _put, FakeVm, FakeVmProvider, saved_image def test_a_stage_that_goes_on_is_said_to_go_on_and_a_failed_pair_says_why(base, driver, docker, monkeypatch): @@ -113,6 +114,37 @@ def run(argv, **kwargs): assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] +def test_what_each_run_wrote_is_read_from_the_newest_layer_a_shell_ran_its_command_for_in_a_saved_images_history( + tmp_path): + saved_image(tmp_path / "image.tar", [ + (None, {"etc/os-release": b"ubuntu"}), + ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", {"usr/local/bin/tool": b"the base's"}), + ("USER root", None), + ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", { + "usr/local/bin/tool": (b"a", 0o755), "usr/bin/tool": "/usr/local/bin/tool", "etc/.wh.gone": b"", + "./etc/a": b"a", "opt/shared/.wh..wh..opq": b""}), + ("RUN /bin/sh -c cd /opt/a && ./setup.sh.extra # buildkit", {"etc/extra": b"extra"}), + ("COPY apps/b/ /opt/b/ # buildkit", {"opt/b/x": b"x"}), + ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"usr/local/bin/tool": b"b"}), + ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"etc/again": b"again"})]) + commands = ["cd /opt/a && ./setup.sh", "cd /opt/a && ./setup.sh.extra", "cd /opt/b && ./setup.sh", + "cd /opt/b && ./setup.sh"] + + ran = subprocess.run([sys.executable, "-I", str(builder.LAYERS), str(tmp_path / "image.tar"), json.dumps(commands)], + capture_output=True, text=True) + + sha = lambda data: hashlib.sha256(data).hexdigest() + assert json.loads(ran.stdout) == layers_in_image.written(str(tmp_path / "image.tar"), commands) == [ + {"/usr/local/bin/tool": f"exec {sha(b'a')}", "/usr/bin/tool": "link /usr/local/bin/tool", "/etc/gone": "deleted", + "/etc/a": f"file {sha(b'a')}", "/opt/shared": "opaque"}, + {"/etc/extra": f"file {sha(b'extra')}"}, {"/usr/local/bin/tool": f"file {sha(b'b')}"}, + {"/etc/again": f"file {sha(b'again')}"}] + for missing in (["cd /opt/c && ./setup.sh"], ["cd /opt/b && ./setup.sh", "cd /opt/a && ./setup.sh"], + [*commands, "cd /opt/b && ./setup.sh"]): + with pytest.raises(ValueError, match="no layer of the image below the last one found was made by"): + layers_in_image.written(str(tmp_path / "image.tar"), missing) + + def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): signed = [] monkeypatch.setattr(builder, "docker_login", signed.append) @@ -172,7 +204,7 @@ def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_imag assert sorted(vms.fetched) == sorted([*driver_files, *driver_files]) assert vms.uploaded and all(f"{packages.PREFIX}/files/" in url for url in vms.uploaded) builds = {build["platform"]: build for build in docker.builds} - assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", + assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-driver-desk.conf", "driver/driver.py", "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] assert composed["amd64"]["installed"] == {"mss": "10.1.0"} assert docker.labels[f"agentenv-computer:{composed['arm64']['key'][7:23]}-arm64"]["dev.agentenv.emulated"] == "true" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 3a39a1e37..87e14ad87 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1,7 +1,8 @@ """Computer envs: gate 1 refuses a base, a driver or a declaration that can't compose, naming every problem, before anything is stored; the composer builds each form and arch, reuses an image of the same key whichever env built it, records each result as it finishes, and writes the arches built as the env's computer; a deploy, a reset and a load; -and computer apps. Where and how a pair builds is test_computer_builder. Docker and the registry are faked.""" +and computer apps. Where and how a pair builds is test_computer_builder; a computer of several apps, test_computer_apps. +Docker and the registry are faked.""" import asyncio import contextlib @@ -18,7 +19,7 @@ import httpx import pytest -from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING, AppDeclaration +from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI from click.testing import CliRunner @@ -52,7 +53,8 @@ from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from tst.unit.computer_home import Computer -from tst.unit.env.envs._computer import CONFIG, SERVE, CARD_EXTENSION, CARD, _base, _driver, _put, FakeVmProvider +from tst.unit.env.envs._computer import (CONFIG, SERVE, CARD_EXTENSION, CARD, MAIL_SETTINGS, MAIL_BUILD, MAIL_SERVER, _app, + _base, _driver, _put, FakeVmProvider) def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_its_images_make(base, driver, docker, @@ -89,11 +91,11 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ "RUN --network=none python3 -m venv /opt/agentenv/drivers/desk/venv && /opt/agentenv/drivers/desk/venv/bin/pip " "install --no-index --no-deps /tmp/agentenv-packages/wheels/driver/*.whl && /opt/agentenv/drivers/desk/venv/bin/pip list " "--format=json > /opt/agentenv/drivers/desk/installed.json && rm -rf /tmp/agentenv-packages/wheels/driver", - "COPY agentenv-desk.conf /etc/supervisor/conf.d/agentenv-desk.conf", - "RUN test -d /opt/agentenv/drivers/desk && test -f /etc/supervisor/conf.d/agentenv-desk.conf && test -x " + "COPY agentenv-driver-desk.conf /etc/supervisor/conf.d/agentenv-driver-desk.conf", + "RUN test -d /opt/agentenv/drivers/desk && test -f /etc/supervisor/conf.d/agentenv-driver-desk.conf && test -x " "/opt/agentenv/drivers/desk/venv/bin/python", "USER user", f"LABEL dev.agentenv.computer-key={composed['arm64']['key']}"] - assert arm64["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", "driver/requirements.txt", + assert arm64["files"] == ["Dockerfile", "agentenv-driver-desk.conf", "driver/driver.py", "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] assert sorted(arch for arch, _ in docker.resolved) == ["amd64", "arm64"] assert docker.timeouts == {"arm64": 3600, "amd64": 10800} @@ -101,12 +103,30 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ "requirements": "requirements.txt", "add": []}] assert composed["arm64"]["lock"].startswith("sha256:") and composed["arm64"]["lock"] != composed["amd64"]["lock"] assert arm64["conf"] == ( - "[program:desk-driver]\ncommand=/opt/agentenv/drivers/desk/venv/bin/python -m agentenv_protocol.computer.serve " + "[program:driver-desk-driver]\ncommand=/opt/agentenv/drivers/desk/venv/bin/python -m agentenv_protocol.computer.serve " 'driver.py:Desk\ndirectory=/opt/agentenv/drivers/desk\nenvironment=MCP_PORT="18765"\nautorestart=true\n' - "redirect_stderr=true\nstdout_logfile=/var/log/agentenv/desk-driver.log\n") + "redirect_stderr=true\nstdout_logfile=/var/log/agentenv/driver-desk-driver.log\n") assert ComputerEnv.from_dict(stored.to_dict()).to_dict() == stored.to_dict() +def test_a_key_covers_its_build_files_executable_bits_and_the_composers_format(base, driver, docker, tmp_path, + monkeypatch): + folder = tmp_path / "ctx" + folder.mkdir() + (folder / "run").write_bytes(b"echo\n") + plain = composer._digest(folder) + (folder / "run").chmod(0o755) + runnable = composer._digest(folder) + (folder / "run").chmod(0o640) + + first = Env.get(_put(base, driver).id).metadata["composed"]["container"]["arm64"]["key"] + monkeypatch.setattr(composer, "FORMAT", composer.FORMAT + 1) + raised = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id).metadata["composed"][ + "container"]["arm64"]["key"] + + assert runnable != plain == composer._digest(folder) and raised != first + + def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one_another_env_built(base, driver, docker, monkeypatch): auth, docker_calls = RegistryAuth("localhost:5000", "user", "token"), [] @@ -330,7 +350,6 @@ async def test_a_computer_env_with_no_ready_build_is_refused_at_deploy_naming_it @pytest.mark.parametrize("changes, problem", [ - ({"apps": [{"env": "mail"}]}, "apps: computer app envs aren't supported yet"), ({"display": {"width": 0, "height": 800}}, "display is { width = , height = }, each 1 to 32767 pixels"), ({"display": {"width": 1280}}, "display is { width = , height = }"), ({"display": {"width": 1280, "height": 800}, "computer_settings": {"resolution": "800x600"}}, @@ -342,7 +361,7 @@ async def test_a_computer_env_with_no_ready_build_is_refused_at_deploy_naming_it "computer_settings.wallpaper: @local/~/kit/ubuntu-24.04 declares no setting 'wallpaper'"), ({"computer_settings": {"timezone": 3}}, "computer_settings.timezone must be a str, like its default 'UTC', not 3"), ({"computer_settings": ["timezone"]}, "computer_settings must be a table of the computer's setting values"), -], ids=["apps", "display-zero", "display-side", "display-and-resolution", "network", "network-key", +], ids=["display-zero", "display-side", "display-and-resolution", "network", "network-key", "settings-name", "settings-type", "settings-table"]) def test_what_the_env_declares_wrong_is_refused_before_anything_is_stored(base, driver, docker, changes, problem): with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: @@ -763,49 +782,9 @@ async def test_a_load_outside_the_home_is_refused_before_anything_is_sent(base, assert computer.calls == [] -MAIL_SETTINGS = {"properties": { - "signature": {"default": "", "type": "string"}, - "password": {"default": "", "type": "string", "format": "password", "writeOnly": True, "minLength": 32}, - "token": {"anyOf": [{"type": "string", "format": "password", "writeOnly": True}, {"type": "null"}], "default": None}, - "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} - - -MAIL_BUILD = {"properties": {"edition": {"default": "home", "enum": ["home", "pro"], "type": "string"}, - "locale": {"default": "en_US", "type": "string"}}, "type": "object"} - - -#: The program of the mail app's own server. -MAIL_SERVER = ("[program:app-mail]\ncommand=/opt/agentenv/apps/mail/venv/bin/python -m " - "agentenv_protocol.computer.serve_app app.py:Mail\ndirectory=/opt/agentenv/apps/mail\n" - "environment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\nredirect_stderr=true\n" - "stdout_logfile=/var/log/agentenv/app-mail.log\n") PROTOCOL_VERSION = importlib.metadata.version("agentenv-framework-protocol") -def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, - extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, **changes): - """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside - them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of - its build settings, and ``linux_settings``, that of its settings on Linux.""" - folder.mkdir(parents=True, exist_ok=True) - (folder / "app.py").write_text("class Mail: ...\n") - (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") - for name, text in (extra or {}).items(): - (folder / name).parent.mkdir(parents=True, exist_ok=True) - (folder / name).write_text(text) - files = FileArtifactUniverse.put_bundled("@local/~/kit/mail__files", files={ - name: folder / name for name in ("app.py", "requirements.txt", *(extra or {}))}) - declaration = AppDeclaration.model_validate({ - "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], - "permissions": [], "settings": settings, "build_settings": build, "skill": None, - "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], - "ready_checks": [], "os": {"linux": {"launch": launch, "settings": linux_settings, "install": install or { - "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) - return ComputerAppEnv.put(**{"id": "@local/~/kit/mail", "environment_name": "mail", "module": "app.py:Mail", - "files": files, "declaration": declaration, "computer": base, "driver": driver, - "compose": compose, **changes}) - - def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_under_its_prefix(base, driver, docker, tmp_path): app = _app(base, driver, tmp_path / "mail")