From 1161e339055b323c69bd22c6463ba03766a8c816 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 15:46:28 -0700 Subject: [PATCH 01/14] feat(computer): a computer env composes the apps it lists, in order, at build A computer env takes apps, computer app envs built into its image after the driver in the order listed. In env.toml each is a ref, or a table with install = "build" beside it; "boot" is refused as not supported yet. Gate 1 checks each app against the base and driver, now including a port the driver listens on, and the apps against each other: one listed twice, two sharing a folder, an environment name or a port. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/resolve.py | 3 +- src/agent_env/entity/refs.py | 14 ++-- src/agent_env/env/envs/computer/app_env.py | 4 +- src/agent_env/env/envs/computer/env.py | 50 +++++++++---- .../env/envs/computer/utils/composer.py | 9 +-- src/agent_env/env/envs/computer/utils/gate.py | 56 +++++++++++--- .../computer/app/artifacts/check_desk/run.py | 30 ++++++++ .../computer/app/envs/clock/app.py | 15 ++++ .../computer/app/envs/clock/env.toml | 2 + .../computer/app/envs/clock/serve.py | 20 +++++ .../computer/app/envs/desk/env.toml | 4 + .../computer/app/tasks/together.json | 6 ++ tst/integration/computer/computer_app_test.py | 18 ++++- tst/unit/bundle/computer_env_folder_test.py | 34 +++++++-- tst/unit/env/envs/test_computer_env.py | 73 ++++++++++++++++--- 15 files changed, 284 insertions(+), 54 deletions(-) create mode 100644 tst/integration/computer/app/artifacts/check_desk/run.py create mode 100644 tst/integration/computer/app/envs/clock/app.py create mode 100644 tst/integration/computer/app/envs/clock/env.toml create mode 100644 tst/integration/computer/app/envs/clock/serve.py create mode 100644 tst/integration/computer/app/envs/desk/env.toml create mode 100644 tst/integration/computer/app/tasks/together.json diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 828f3b71d..73984604d 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -201,7 +201,8 @@ def _toml(self, entry: BundleEntry) -> ResolvedEntry: refs.append(ref) for site in ref_sites(refs, config, inline_pins=True): try: - name, version = parse_toml_ref(site.ref.kind, site.owner if site.version_key else site.value) + name, version = parse_toml_ref(site.ref.kind, site.owner if site.version_key else site.value, + site.ref.beside) except ValueError as e: self._problem(entry, f"{site.path}: {e}") continue diff --git a/src/agent_env/entity/refs.py b/src/agent_env/entity/refs.py index 3e4568fc0..e611ca426 100644 --- a/src/agent_env/entity/refs.py +++ b/src/agent_env/entity/refs.py @@ -53,8 +53,10 @@ class RefRole(StrEnum): class EntityRef: """One ref-holding field. ``path`` is dotted keys into the step dict or toml, ``[]`` meaning each element of a list (``env_ids[]``, ``artifacts[].id``); ``version_field`` is the key beside the - ref, in the same dict, that pins its version. ``walk``, for a shape a path cannot spell, maps - the value at the path's parent to the dicts that hold the last key, each with its path parts. + ref, in the same dict, that pins its version. ``beside`` names the keys a toml table holding the + ref may carry next to it, which its type reads (a computer env's app takes ``install``). ``walk``, + for a shape a path cannot spell, maps the value at the path's parent to the dicts that hold the + last key, each with its path parts. Declarations build one with ``EntityRef.env``, ``EntityRef.agent`` or ``EntityRef.artifact``, passing the keyword fields.""" @@ -65,6 +67,7 @@ class EntityRef: role: RefRole = RefRole.INPUT artifact_type: str | None = None env_type: str | None = None + beside: tuple[str, ...] = () walk: Walker | None = None @classmethod @@ -134,12 +137,13 @@ def default_agent_refs(refs: Iterable[EntityRef], step: dict) -> list[EntityRef] and step.get(ref.field) is None] -def parse_toml_ref(kind: EntityKind, value: Any) -> tuple[str, int | None]: +def parse_toml_ref(kind: EntityKind, value: Any, beside: tuple[str, ...] = ()) -> tuple[str, int | None]: """The id and version a toml ref names: an id, or ``{ = "", version = }`` with the - version optional. Raises ValueError naming the expected form.""" + version optional, and the keys of ``beside`` allowed in the table too. Raises ValueError naming the + expected form.""" table = isinstance(value, dict) entity_id, version = (value.get(kind.value), value.get("version")) if table else (value, None) - known_keys = not table or value.keys() <= {kind.value, "version"} + known_keys = not table or value.keys() <= {kind.value, "version", *beside} positive = version is None or (isinstance(version, int) and not isinstance(version, bool) and version >= 1) if not (known_keys and isinstance(entity_id, str) and entity_id and positive): raise ValueError(f'expected an id or {{ {kind.value} = "", version = }}, the version optional, ' diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index 06c15e09b..f9d9035fe 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -17,7 +17,7 @@ from agent_env.env.env import Env from agent_env.env.envs.computer.utils import composer, gate from agent_env.env.envs.computer.utils.composer import Composing -from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.env import APP_TYPE, ComputerEnv from agent_env.env.envs.computer.utils.install import AppInstall, folder_problems, installs from agent_env.env.envs.computer.driver_env import ComputerDriverEnv, declared from agent_env.env.store import get_env_store @@ -30,7 +30,7 @@ class ComputerAppEnv(ComputerEnv): - type: ClassVar[str] = "computer_app" + type: ClassVar[str] = APP_TYPE description = ("A computer app: software and its settings on a computer, read from its ComputerAppEnvironment " "class at put; deployed alone, a computer of its own") diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index c0eb195bc..2d5b8b8c9 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -32,12 +32,15 @@ if TYPE_CHECKING: from agent_env.bundle.authoring import AuthoringContext from agent_env.env.env import DeployedEnv + from agent_env.env.envs.computer.app_env import ComputerAppEnv from agent_env.providers.env_providers.env_provider import EnvironmentProvider logger = logging.getLogger(__name__) #: Where load_artifact puts files in a computer by default: one of the folders data/get returns by default. DEFAULT_DESTINATION = "Downloads" +#: The computer app env's type: an app composed into a computer env, and a computer env of its own. +APP_TYPE = "computer_app" class ComputerEnv(Env): @@ -45,14 +48,16 @@ class ComputerEnv(Env): description = "A computer: a base computer and a driver, composed at put into an image for each form and arch" env_provider_type: ClassVar[str] = "computer" toml_refs: ClassVar[tuple[EntityRef, ...]] = (EntityRef.artifact("computer", artifact_type="computer"), - EntityRef.env("driver", env_type=ComputerDriverEnv.type)) + EntityRef.env("driver", env_type=ComputerDriverEnv.type), + EntityRef.env("apps[]", env_type=APP_TYPE, beside=("install",))) def __init__(self, id: str, version: Optional[int], environment_name: str, computer: ComputerArtifact, driver: ComputerDriverEnv, display: Optional[dict] = None, network: Optional[dict] = None, - computer_settings: Optional[dict[str, SettingValue]] = None, metadata: Optional[dict[str, Any]] = None): + computer_settings: Optional[dict[str, SettingValue]] = None, metadata: Optional[dict[str, Any]] = None, + apps: Optional[list[ComputerAppEnv]] = None): super().__init__(id, version, metadata=metadata) self.environment_name = environment_name - self.computer, self.driver = computer, driver + self.computer, self.driver, self.apps = computer, driver, apps or [] self.display, self.network = display, network self.computer_settings = computer_settings or {} self._sandbox = None @@ -66,7 +71,8 @@ def to_dict(self) -> dict: "environment_name": self.environment_name, # dual-write with service_name during the rename "computer": {"id": self.computer.id, "version": self.computer.version}, "driver": {"id": self.driver.id, "version": self.driver.version}, "display": self.display, - "network": self.network, "computer_settings": self.computer_settings} + "network": self.network, "computer_settings": self.computer_settings, + **({"apps": [{"id": app.id, "version": app.version} for app in self.apps]} if self.apps else {})} @classmethod def from_dict(cls, data: dict) -> ComputerEnv: @@ -77,29 +83,32 @@ def from_dict(cls, data: dict) -> ComputerEnv: environment_name=data["environment_name"] if "environment_name" in data else data["service_name"], computer=ComputerArtifact.get(data["computer"]["id"], data["computer"]["version"]), driver=driver, display=data.get("display"), network=data.get("network"), - computer_settings=data.get("computer_settings"), metadata=data.get("metadata") or {}) + computer_settings=data.get("computer_settings"), metadata=data.get("metadata") or {}, + apps=[_app(Env.get(ref["id"], ref["version"])) for ref in data.get("apps") or []]) @classmethod def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str, apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None, compose: bool = True, composing: Composing = Composing()) -> Self: - """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it as ``composing`` - says unless ``compose`` is False. Refused, naming every mismatch, before anything is stored.""" + """Write a computer env of ``computer``, ``driver`` and ``apps``, computer app envs built into it in their + order, once gate 1 passes, then compose it as ``composing`` says unless ``compose`` is False. Refused, naming + every mismatch, before anything is stored.""" values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, computer_settings=computer_settings or {}) if problems: raise ValueError(f"computer env {id!r}: " + "; ".join(problems)) env = get_env_store().put_document(cls( - id=id, version=None, environment_name=environment_name, - computer=computer, driver=driver, display=display, network=network, computer_settings=values)) + id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, apps=list(apps), + display=display, network=network, computer_settings=values)) if compose: composer.compose(env, composing=composing) return env @classmethod def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: - """Write the computer env an env.toml declares, its base and driver as the bundle wrote them or the ones they - name, its environment_name the folder's when it sets none.""" + """Write the computer env an env.toml declares, its base, driver and apps as the bundle wrote them or the ones + they name, its environment_name the folder's when it sets none. An app's ``install`` is ``"build"``, the + default: it's built into the computer.""" fields, refused = ctx.accepted(data, environment_name=str, computer=object, driver=object, apps=list, display=dict, network=dict, computer_settings=dict) missing = [ctx.config_problem(f"{key} is required: the {what}, such as {example}") for key, what, example in ( @@ -108,6 +117,15 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: ctx.refuse([*refused, *missing]) base, driver = ctx.artifact(fields.pop("computer"), ComputerArtifact), ctx.env(fields.pop("driver"), ComputerDriverEnv) + refs = fields.get("apps", []) + fields["apps"] = [ctx.env({key: value for key, value in ref.items() if key != "install"} + if isinstance(ref, dict) else ref) for ref in refs] + if problems := [*(ctx.config_problem(f"apps[{i}]: install = {ref['install']!r}: an app installs at \"build\", " + "into the computer's image; \"boot\" isn't supported yet") + for i, ref in enumerate(refs) if isinstance(ref, dict) and ref.get("install", "build") != "build"), + *(ctx.config_problem(f"apps[{i}]: {app.id} is a {app.type} env, not a {APP_TYPE} env") + for i, app in enumerate(fields["apps"]) if app.type != APP_TYPE)]: + ctx.refuse(problems) try: return cls.put(id=ctx.id, computer=base, driver=driver, environment_name=fields.pop("environment_name", ctx.name), composing=ctx.composing, @@ -115,9 +133,9 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: except ValueError as e: ctx.refuse([str(e)]) - def composed_apps(self) -> list: + def composed_apps(self) -> list[ComputerAppEnv]: """The computer app envs composed into the env's computer, each installed under its prefix.""" - return [] + return self.apps def computer_artifact(self) -> ComputerArtifact: """The env's computer: its base composed with its driver, over the images built.""" @@ -217,3 +235,9 @@ async def close(self) -> None: except Exception as e: # a cancellation propagates, and keeps the sandbox for the next close() logger.warning(f"Failed to terminate sandbox {self._sandbox.sandbox_id}: {e}") self._sandbox = None + + +def _app(env: Env) -> ComputerAppEnv: + if env.type != APP_TYPE: + raise TypeError(f"{env.id} is a {env.type} env, not a {APP_TYPE} env") + return env diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index cd9f82446..64fb1dd0e 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -13,7 +13,6 @@ import importlib.metadata import json import logging -import re import shlex import shutil import subprocess @@ -310,7 +309,7 @@ def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifa from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them, served on its port, its folder under ``DRIVERS`` named for its id's last part.""" - install, name = driver.install[base.os.type], _name(driver.id, "driver") + install, name = driver.install[base.os.type], gate.folder_name(driver.id, "driver") layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf") _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}, from_store) @@ -322,7 +321,7 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, """Write the app's install files under ``context/apps//``, with the values of its build settings the env fixes, and its supervisord programs beside them, run under its name: its services and its own server, which serves its class from its venv and launches the app. Its folder under ``APPS`` is named for its id's last part.""" - install, name = app.install[base.os.type], _name(app.id, "app") + install, name = app.install[base.os.type], gate.folder_name(app.id, "app") file, _, cls = app.module.partition(":") served = shlex.quote(f"{PurePosixPath(file).relative_to(install.files)}:{cls}") layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app=True, served=served, @@ -340,10 +339,6 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, return layer -def _name(env_id: str, fallback: str) -> str: - return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback - - def _services(name: str, install: Install) -> dict[str, str]: """The commands of ``install``'s services by their programs' names, each named for ``name``.""" return {f"{name}-{service.name}": service.command for service in install.services} diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index 4a8ab7440..d368841dd 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -6,6 +6,9 @@ from collections.abc import Sequence from typing import TYPE_CHECKING, Any +import re + +from agentenv_protocol.computer.apps import APPS from agentenv_protocol.computer.platform import Platform, supported from jsonschema import Draft202012Validator @@ -26,10 +29,7 @@ def check(base: ComputerArtifact, driver: ComputerDriverEnv, *, apps: list, disp computer_settings: Any) -> tuple[dict[str, SettingValue], list[str]]: """The env's values of the base's settings, and every way ``base``, ``driver`` and what the env declares don't compose.""" - problems = [] - if apps: - problems.append("apps: computer app envs aren't supported yet") - problems += _platform(base, driver) + problems = _platform(base, driver) + _apps(base, driver, apps) problems += _image(base) problems += _display(base, display, computer_settings) problems += _network(network) @@ -51,22 +51,58 @@ def arches(base: ComputerArtifact, driver: ComputerDriverEnv, apps: Sequence[Com return [arch for arch in base.arch if arch in matching] -def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAppEnv) -> list[str]: - """Every way ``app`` doesn't fit a composition of ``base`` and ``driver``: a platform it doesn't support, no - architecture in common, or no install for the base's OS.""" +def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAppEnv, at: str = "app") -> list[str]: + """Every way ``app`` doesn't fit a composition of ``base`` and ``driver``, each named from ``at``: a platform it + doesn't support, no architecture in common, no install for the base's OS, or a port the driver listens on too.""" os = base.os if not _matching(base, app.declaration.supports): named = ", ".join(_shown(platform) for entry in app.declaration.supports for platform in supported(entry)) - return [f"app: {app.id} supports no platform {base.id} is ({os.type} {os.distribution} {os.version} " + return [f"{at}: {app.id} supports no platform {base.id} is ({os.type} {os.distribution} {os.version} " f"{os.display}); it supports {named or 'none'}"] problems = [] if not arches(base, driver, [app]): - problems.append(f"app: no architecture {base.id}, {driver.id} and {app.id} all run") + problems.append(f"{at}: no architecture {base.id}, {driver.id} and {app.id} all run") if os.type not in app.install: - problems.append(f"app: {app.id} declares no install for {os.type}") + problems.append(f"{at}: {app.id} declares no install for {os.type}") + elif taken := sorted(set(_ports(app, os.type)) & set(_driver_ports(driver, os.type))): + problems.append(f"{at}: {app.id} listens on {', '.join(map(str, taken))}, as {driver.id} does") + return problems + + +def folder_name(env_id: str, fallback: str) -> str: + """The name of the folder a driver or an app is installed in: its id's last part, lowercased, as a slug.""" + return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback + + +def _apps(base: ComputerArtifact, driver: ComputerDriverEnv, apps: list[ComputerAppEnv]) -> list[str]: + """Every way ``apps`` don't fit together on ``base`` and ``driver``: each one's own (``check_app``), one listed + twice, two in one folder or under one environment name, and a port two of them listen on.""" + problems = [problem for i, app in enumerate(apps) for problem in check_app(base, driver, app, f"apps[{i}]")] + ids = [app.id for app in apps] + problems += [f"apps: {app_id} is listed twice" for app_id in sorted({i for i in ids if ids.count(i) > 1})] + distinct = list({app.id: app for app in apps}.values()) + for kind, keys in (("folder", lambda app: [f"{APPS}/{folder_name(app.id, 'app')}"]), + ("environment name", lambda app: [app.environment_name]), + ("port", lambda app: _ports(app, base.os.type))): + claims: dict[object, list[str]] = {} + for app in distinct: + for key in keys(app): + claims.setdefault(key, []).append(app.id) + problems += [f"apps: {' and '.join(owners)} would share the {kind} {key!r}" for key, owners in claims.items() + if len(owners) > 1] return problems +def _ports(app: ComputerAppEnv, os_type: str) -> list[int]: + install = app.install.get(os_type) + return list(install.ports or []) if install is not None else [] + + +def _driver_ports(driver: ComputerDriverEnv, os_type: str) -> list[int]: + install = driver.install.get(os_type) + return [driver.port, *(install.ports or [] if install is not None else [])] + + def check_app_settings(app: ComputerAppEnv, values: dict[str, Any], at: str, os_type: str) -> list[str]: """The problems with ``values``, of ``app``'s settings on ``os_type`` by name, each named from ``at``: a setting it doesn't declare, a secret's given as anything but a bare reference (or null, where it's optional), a required one diff --git a/tst/integration/computer/app/artifacts/check_desk/run.py b/tst/integration/computer/app/artifacts/check_desk/run.py new file mode 100644 index 000000000..68055165e --- /dev/null +++ b/tst/integration/computer/app/artifacts/check_desk/run.py @@ -0,0 +1,30 @@ +"""What each of the two apps composed into the desk wrote on the settings its deploy gave it, the clock's answer on its +own port, the name the computer's card gives and what its readiness says of each app, asked on loopback from inside +the computer, the apps launched since it booted, and each app's install under its own folder.""" +import json +import time +import urllib.request +from pathlib import Path + +APPS = Path("/opt/agentenv/apps") + + +def run(input): + written = [Path.home() / ".notes.json", Path.home() / ".clock.json"] + for _ in range(60): + if all(path.exists() for path in written): + break + time.sleep(0.5) + with urllib.request.urlopen(f"{input['env']['url']}/.well-known/agent-env.json", timeout=10) as response: + card = json.load(response) + with urllib.request.urlopen(f"{input['env']['url']}/agentenv/ext/computer-ready/status", timeout=30) as response: + ready = json.load(response) + with urllib.request.urlopen("http://127.0.0.1:8137/", timeout=10) as response: + clock = json.load(response) + notes = json.loads(written[0].read_text()) + return {"notes": {"signature": notes["signature"], "token": notes["token"]}, + "clock": {**json.loads(written[1].read_text()), "served": clock}, "card": card["name"], + "app_ready": sorted(check["name"] for check in ready["checks"] + if check["name"].startswith("app ") and check["ok"]), + "launched": sorted(p.name for p in Path("/run/agentenv/apps").iterdir()), + "folders": sorted(p.name for p in APPS.iterdir()), "hello": (APPS / "notes" / "hello.txt").exists()} diff --git a/tst/integration/computer/app/envs/clock/app.py b/tst/integration/computer/app/envs/clock/app.py new file mode 100644 index 000000000..c1833a6f0 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/app.py @@ -0,0 +1,15 @@ +"""A clock app for the computer env tests, composed beside notes: its service answers on a port of its own with the +time zone its settings name, and writes it to ~/.clock.json. It installs tzdata, and declares no window.""" +from pydantic import BaseModel + +from agentenv_protocol.computer import ComputerAppEnvironment, Packages, Service, computer_app, linux + +PORT = 8137 + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) +@linux(packages=Packages(apt=["tzdata"]), services=[Service(name="clock", command="venv/bin/python serve.py")], + ports=[PORT]) +class Clock(ComputerAppEnvironment): + class Settings(BaseModel): + zone: str = "UTC" diff --git a/tst/integration/computer/app/envs/clock/env.toml b/tst/integration/computer/app/envs/clock/env.toml new file mode 100644 index 000000000..60e142353 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/env.toml @@ -0,0 +1,2 @@ +type = "computer_app" +module = "app.py:Clock" diff --git a/tst/integration/computer/app/envs/clock/serve.py b/tst/integration/computer/app/envs/clock/serve.py new file mode 100644 index 000000000..681135470 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/serve.py @@ -0,0 +1,20 @@ +import json +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path + +from app import PORT, Clock + +zone = Clock().settings.zone +Path.home().joinpath(".clock.json").write_text(json.dumps({"zone": zone})) + + +class Zone(BaseHTTPRequestHandler): + def do_GET(self): + body = json.dumps({"zone": zone, "known": Path("/usr/share/zoneinfo", zone).exists()}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(body) + + +HTTPServer(("127.0.0.1", PORT), Zone).serve_forever() diff --git a/tst/integration/computer/app/envs/desk/env.toml b/tst/integration/computer/app/envs/desk/env.toml new file mode 100644 index 000000000..8e8759b4f --- /dev/null +++ b/tst/integration/computer/app/envs/desk/env.toml @@ -0,0 +1,4 @@ +type = "computer" +computer = "ubuntu-24.04" +driver = "reference" +apps = ["notes", { env = "clock", install = "build" }] diff --git a/tst/integration/computer/app/tasks/together.json b/tst/integration/computer/app/tasks/together.json new file mode 100644 index 000000000..a6ff2aad5 --- /dev/null +++ b/tst/integration/computer/app/tasks/together.json @@ -0,0 +1,6 @@ +[ + {"id": "env", "type": "deploy_env", "env_id": "desk", + "env_params": {"app_settings": {"notes": {"signature": "-- Ada", "token": "env:NOTES_TOKEN"}, + "clock": {"zone": "Europe/Paris"}}}}, + {"id": "together", "type": "run_code", "env_id": "desk", "script_artifact_id": "check_desk"} +] diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index fb4c09da0..f71c08b1c 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -3,7 +3,8 @@ settings the deploy gave it, a secret resolved at boot, its window launched once by its server as the desktop session comes up, and the computer ready only once its server says its window and its @ready_check are; its apt packages, one from a PPA, its download, run script and install step on its build setting, fetched at put, installed offline. -Needs Docker and the local registry on localhost:5000.""" +And two apps composed into one computer env, each running on its own settings from its own folder. Needs Docker and the +local registry on localhost:5000.""" from pathlib import Path @@ -35,3 +36,18 @@ def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_na "app_ready": [{"name": "app notes", "ok": True, "detail": "ready"}], "launchers": ["agentenv-app-notes.desktop"], "launched": ["notes"], "hello": "Hello, world!", "license": True, "installed": "pro"} + + +def test_two_apps_composed_into_one_computer_env_each_run_on_their_own_settings(state, monkeypatch): + monkeypatch.setenv("NOTES_TOKEN", "t0ken") + + ran = CliRunner().invoke(cli, ["run", str(APP), "--task", "together"]) + + assert ran.exit_code == 0, ran.output + context = run_context(ran.output) + (deployed,) = context["deployed_envs"] + assert deployed["env_id"].endswith("/desk") and deployed["environment_card"]["name"] == "desk" + assert context["metadata"]["script_results"]["together"] == { + "notes": {"signature": "-- Ada", "token": "t0ken"}, + "clock": {"zone": "Europe/Paris", "served": {"zone": "Europe/Paris", "known": True}}, "card": "desk", + "app_ready": ["app clock", "app notes"], "launched": ["notes"], "folders": ["clock", "notes"], "hello": True} diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index e5825860b..d01249337 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -19,6 +19,8 @@ from agent_env.env.envs.computer.env import ComputerEnv from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from tst.unit.bundle._support import layout, local_builds +from tst.unit.bundle.computer_app_env_test import APP, TOML as APP_TOML +from tst.unit.bundle.computer_driver_env_test import LINUX UBUNTU = "@local/agentenv-framework/computers/ubuntu-24.04" REFERENCE = "@local/agentenv-framework/drivers/reference" @@ -30,6 +32,12 @@ [computer_settings] timezone = "Europe/Paris" ''' +WITH_APPS = DESKTOP.replace("display =", 'apps = ["notes", { env = "mail", install = "build" }]\ndisplay =') +#: Two app folders, mail and notes, of one app class. +APPS = {f"envs/{name}/{file}": text for name in ("mail", "notes") for file, text in ( + ("env.toml", APP_TOML), ("app.py", APP.format(supports=LINUX, skill="skill.md", linux_skill="skill-linux.md")), + ("requirements.txt", "agentenv-framework-protocol\n"), ("skill.md", "# Mail\n"), + ("skill-linux.md", "# Mail on Linux\n"))} @pytest.fixture @@ -48,10 +56,10 @@ def compose(env, pairs=None, composing=Composing()): return calls -def _bundle(root, toml=DESKTOP): +def _bundle(root, toml=DESKTOP, extra=None): return layout(root, {"envs/desktop/env.toml": toml, "tasks/t.json": json.dumps([ {"id": "desktop", "type": "load_artifact", "env_id": "desktop", "artifact_id": "notes"}]), - "artifacts/notes/notes.txt": "a note\n"}) + "artifacts/notes/notes.txt": "a note\n", **(extra or {})}) def _installed(tmp_path, package, name): @@ -137,17 +145,33 @@ def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_pa assert f"{resolved.bundle.id_root}/desktop" in {write.id for write in plan.writes} +def test_a_computer_env_folder_names_its_apps_in_order_each_written_before_it(tmp_path, composed): + plan = _plan(_bundle(tmp_path / "desk", WITH_APPS, APPS)) + + materialize(plan, build_here=local_builds(plan)) + + root = plan.bundle.bundle.id_root + env = Env.get(f"{root}/desktop") + assert [(app.id, app.type, app.environment_name) for app in env.composed_apps()] == [ + (f"{root}/notes", "computer_app", "notes"), (f"{root}/mail", "computer_app", "mail")] + assert composed == [(env.id, UBUNTU, REFERENCE, None)] + + @pytest.mark.parametrize("toml, problem", [ ('type = "computer"\ndriver = "reference"\n', 'computer is required: the base computer, such as "ubuntu-24.04"'), ('type = "computer"\ncomputer = "ubuntu-24.04"\n', 'driver is required: the driver, such as "reference"'), (DESKTOP.replace("display =", "ports = [80]\ndisplay ="), "unknown key 'ports'"), (DESKTOP.replace("1600", "0"), "display is { width = , height = }, each 1 to 32767 pixels"), -], ids=["no-computer", "no-driver", "unknown-key", "display"]) + (WITH_APPS.replace('"build"', '"boot"'), 'apps[1]: install = \'boot\': an app installs at "build", into the ' + 'computer\'s image; "boot" isn\'t supported yet'), + (WITH_APPS.replace('["notes", ', '["notes", "notes", '), "apps: @local/~/desk/notes is listed twice"), + (WITH_APPS.replace('install = "build"', 'mode = "build"'), 'apps[1].env: expected an id or { env = "", version = ' + ' }, the version optional'), +], ids=["no-computer", "no-driver", "unknown-key", "display", "boot", "twice", "key-beside"]) def test_a_computer_env_folder_that_declares_what_cant_compose_is_refused_naming_the_folder(tmp_path, composed, toml, problem): - plan = _plan(_bundle(tmp_path / "desk", toml)) - with pytest.raises(BundleError) as refused: + plan = _plan(_bundle(tmp_path / "desk", toml, APPS)) materialize(plan, build_here=local_builds(plan)) assert any(line.startswith("envs/desktop") and problem in line for line in refused.value.problems), refused.value diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 3a39a1e37..716657625 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -330,7 +330,6 @@ async def test_a_computer_env_with_no_ready_build_is_refused_at_deploy_naming_it @pytest.mark.parametrize("changes, problem", [ - ({"apps": [{"env": "mail"}]}, "apps: computer app envs aren't supported yet"), ({"display": {"width": 0, "height": 800}}, "display is { width = , height = }, each 1 to 32767 pixels"), ({"display": {"width": 1280}}, "display is { width = , height = }"), ({"display": {"width": 1280, "height": 800}, "computer_settings": {"resolution": "800x600"}}, @@ -342,7 +341,7 @@ async def test_a_computer_env_with_no_ready_build_is_refused_at_deploy_naming_it "computer_settings.wallpaper: @local/~/kit/ubuntu-24.04 declares no setting 'wallpaper'"), ({"computer_settings": {"timezone": 3}}, "computer_settings.timezone must be a str, like its default 'UTC', not 3"), ({"computer_settings": ["timezone"]}, "computer_settings must be a table of the computer's setting values"), -], ids=["apps", "display-zero", "display-side", "display-and-resolution", "network", "network-key", +], ids=["display-zero", "display-side", "display-and-resolution", "network", "network-key", "settings-name", "settings-type", "settings-table"]) def test_what_the_env_declares_wrong_is_refused_before_anything_is_stored(base, driver, docker, changes, problem): with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: @@ -783,25 +782,26 @@ async def test_a_load_outside_the_home_is_refused_before_anything_is_sent(base, def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, - extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, **changes): + extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, name="mail", **changes): """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of - its build settings, and ``linux_settings``, that of its settings on Linux.""" + its build settings, ``linux_settings``, that of its settings on Linux, and ``name``, its id's last part and its + environment name.""" folder.mkdir(parents=True, exist_ok=True) (folder / "app.py").write_text("class Mail: ...\n") (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") - for name, text in (extra or {}).items(): - (folder / name).parent.mkdir(parents=True, exist_ok=True) - (folder / name).write_text(text) - files = FileArtifactUniverse.put_bundled("@local/~/kit/mail__files", files={ - name: folder / name for name in ("app.py", "requirements.txt", *(extra or {}))}) + for path, text in (extra or {}).items(): + (folder / path).parent.mkdir(parents=True, exist_ok=True) + (folder / path).write_text(text) + files = FileArtifactUniverse.put_bundled(f"@local/~/kit/{name}__files", files={ + path: folder / path for path in ("app.py", "requirements.txt", *(extra or {}))}) declaration = AppDeclaration.model_validate({ "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], "permissions": [], "settings": settings, "build_settings": build, "skill": None, "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], "ready_checks": [], "os": {"linux": {"launch": launch, "settings": linux_settings, "install": install or { "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) - return ComputerAppEnv.put(**{"id": "@local/~/kit/mail", "environment_name": "mail", "module": "app.py:Mail", + return ComputerAppEnv.put(**{"id": f"@local/~/kit/{name}", "environment_name": name, "module": "app.py:Mail", "files": files, "declaration": declaration, "computer": base, "driver": driver, "compose": compose, **changes}) @@ -1052,6 +1052,59 @@ def test_build_settings_an_app_doesnt_declare_or_its_schema_refuses_are_refused_ Env.get("@local/~/kit/mail") +def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_them(base, driver, docker, tmp_path): + mail = _app(base, driver, tmp_path / "mail", compose=False) + notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", + install={"ports": [8025], "services": [{"name": "sync", "command": "venv/bin/python sync.py"}]}) + + env = _put(base, driver, apps=[notes, mail]) + + assert sorted(build["platform"] for build in docker.builds) == ["linux/amd64", "linux/arm64"] + dockerfile, confs = docker.builds[0]["dockerfile"], docker.builds[0]["confs"] + assert dockerfile.index("COPY driver/") < dockerfile.index("COPY apps/notes/") < dockerfile.index( + "COPY apps/mail/") < dockerfile.index("USER user") + assert [layer["name"] for layer in docker.resolved[0][1]["python"]][-2:] == ["apps-notes", "apps-mail"] + assert confs["agentenv-app-mail.conf"].startswith(MAIL_SERVER) and "[program:app-notes-sync]" in confs[ + "agentenv-app-notes.conf"] + stored = Env.get(env.id) + assert [(app.id, app.version) for app in stored.composed_apps()] == [(notes.id, 1), (mail.id, 1)] + assert stored.to_dict()["apps"] == [{"id": notes.id, "version": 1}, {"id": mail.id, "version": 1}] + assert stored.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + assert _put(base, driver, apps=[mail, notes]).metadata["composed"]["container"]["arm64"]["key"] != ( + stored.metadata["composed"]["container"]["arm64"]["key"]) + + +@pytest.mark.parametrize("apps, problem", [ + (lambda mail, other: [mail, mail], "apps: @local/~/kit/mail is listed twice"), + (lambda mail, other: [mail, other(name="mail-2", environment_name="mail")], + "apps: @local/~/kit/mail and @local/~/kit/mail-2 would share the environment name 'mail'"), + (lambda mail, other: [mail, other(id="@local/other/mail")], + "apps: @local/~/kit/mail and @local/other/mail would share the folder '/opt/agentenv/apps/mail'"), + (lambda mail, other: [other(name="a", install={"ports": [8025], "services": []}), + other(name="b", install={"ports": [8025, 9000], "services": []})], + "apps: @local/~/kit/a and @local/~/kit/b would share the port 8025"), + (lambda mail, other: [mail, other(name="desk-port", install={"ports": [18765], "services": []})], + "apps[1]: @local/~/kit/desk-port listens on 18765, as @local/~/kit/desk does"), + (lambda mail, other: [mail, other(name="arm", supports=[{"os": {"type": "linux", "display": ["x11"]}, + "arch": ["riscv64"]}])], + "apps[1]: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and @local/~/kit/arm all run"), +], ids=["twice", "environment-name", "folder", "app-port", "driver-port", "arch"]) +def test_apps_that_dont_fit_together_or_on_the_computer_are_refused_before_anything_is_stored(base, driver, docker, + tmp_path, apps, problem): + def other(name="other", **changes): + return _app(None, None, tmp_path / name, compose=False, name=name, **changes) + + listed = apps(_app(base, driver, tmp_path / "mail", compose=False), other) + + with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: + _put(base, driver, apps=listed) + + assert problem in str(refused.value) + with pytest.raises(NotFoundError): + Env.get("@local/~/kit/desktop") + assert docker.builds == [] + + def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker, tmp_path): _app(base, driver, tmp_path / "mail", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["arm64"]}]) From 7101303c382549fbadbcda4b3f454a53c10656a2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 15:55:47 -0700 Subject: [PATCH 02/14] feat(computer): refuse a build whose parts write one file differently Two parts of a composed computer (its driver and apps) can write the same path only from their run scripts and install steps, the steps that write outside their own folders; the later part's copy would hide the earlier's. When two or more parts have such steps, the builder saves the built image and lists what each step's layer holds, found by its command in the image's history; a path two parts wrote with different content fails the pair at its build stage, before the push, naming the path and the parts. Temporary files, logs, caches and the files installers edit in place (account databases, the linker's cache, /etc/environment, shell startup files) are left out. A builder VM reads the layers in a container of the image, on its own Python. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 28 ++++++++ .../env/envs/computer/utils/composer.py | 46 +++++++++++-- .../envs/computer/utils/layers_in_image.py | 58 ++++++++++++++++ tst/unit/env/envs/_computer.py | 69 ++++++++++++++++++- tst/unit/env/envs/conftest.py | 1 + tst/unit/env/envs/test_computer_builder.py | 26 ++++++- tst/unit/env/envs/test_computer_env.py | 63 +++++++++++++++++ 7 files changed, 279 insertions(+), 12 deletions(-) create mode 100644 src/agent_env/env/envs/computer/utils/layers_in_image.py diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 800f66d52..8b2cd3698 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -27,6 +27,7 @@ from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image from agent_env.config import get_config +from agent_env.env.envs.computer.utils import layers_in_image from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM @@ -48,6 +49,7 @@ FORMAT = 1 CONTEXT, RESOLVED = "context", "resolve" RESOLVER = Path(__file__).with_name("resolve_in_base.py") +LAYERS = Path(__file__).with_name("layers_in_image.py") #: The shell side of a builder VM, its subcommands run there (``VmBuilder``). IN_VM = Path(__file__).with_name("builder_in_vm.sh") #: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged, @@ -136,6 +138,11 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: """The packages ``tag``'s build installed into the venv of the driver at ``prefix``, by name, read from the image without running it, so on any arch; none for a driver with no venv.""" + @abstractmethod + def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: + """What each of ``commands``, a RUN of ``tag``'s build, wrote in its layer of ``tag``, read from the image saved + (``layers_in_image``).""" + @abstractmethod def push(self, tag: str, ref: str, store) -> None: """Push ``tag`` to ``ref`` in the image store ``store``, signed in with its credentials.""" @@ -323,6 +330,14 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as archive: return _packages(archive.extractfile(archive.getmembers()[0]).read()) + def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: + saved = self.work / "image.tar" + save_image(tag, saved) + try: + return layers_in_image.written(str(saved), commands) + finally: + saved.unlink(missing_ok=True) + def push(self, tag: str, ref: str, store) -> None: push_local_image(tag, ref, store) @@ -425,6 +440,15 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: listed = self._exec(f"{self.helper} installed {shlex.quote(tag)} {shlex.quote(f'{prefix}/installed.json')}") return _packages(listed.encode()) if listed.strip() else {} + def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: + """Read in a container of ``tag``, on its own Python, so the VM needs none.""" + image, script = f"{self.work}/image.tar", f"{self.work}/{LAYERS.name}" + self._write(script, LAYERS.read_bytes()) + return json.loads(self._exec( + f"docker save -o {image} {shlex.quote(tag)} && docker run --rm --network none --user root " + f"-v {image}:/image.tar:ro -v {script}:/layers.py:ro --entrypoint python3 {shlex.quote(tag)} /layers.py " + f"/image.tar {shlex.quote(json.dumps(commands))} && rm -f {image}")) + def push(self, tag: str, ref: str, store) -> None: self._login(store.auth(ref)) self._exec(f"docker tag {shlex.quote(tag)} {shlex.quote(ref)} && docker push {shlex.quote(ref)}") @@ -516,6 +540,10 @@ def _packages(listed: bytes) -> dict[str, str]: return {package["name"]: package["version"] for package in json.loads(listed)} +def save_image(tag: str, path: Path) -> None: + subprocess.run(["docker", "save", "-o", str(path), tag], check=True, capture_output=True, text=True) + + def sha256_of(path: Path) -> str: hashed = hashlib.sha256() with path.open("rb") as file: diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 64fb1dd0e..15ef66ce6 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -13,6 +13,7 @@ import importlib.metadata import json import logging +import re import shlex import shutil import subprocess @@ -78,6 +79,12 @@ _PROTOCOL = "agentenv-framework-protocol" # What building the protocol from its source reads, beside src/. _PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") +#: Paths two parts' run scripts and install steps may both write: temporary files, logs and caches, and the files +#: installers edit in place, so the later part's copy holds the earlier's change: the account databases, the linker's +#: cache, the system environment and a home's shell startup files. +SHARED = re.compile(r"^/(tmp|var/tmp|var/log|var/cache|run)/|/\.cache/|/__pycache__/|" + r"^/etc/(passwd|group|shadow|gshadow|subuid|subgid|ld\.so\.cache|environment)-?$|" + r"^/(root|home/[^/]+)/\.(bashrc|profile|bash_profile)$") @dataclass(frozen=True) @@ -251,6 +258,12 @@ def on() -> builder.Builder: packages.stage(lock, building_on, env.stored_id, fetched) local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) + if len(runs := {part.of: _runs(part) for part in layers if _runs(part)}) > 1: + progress.at(pair, "comparing the files its parts' run scripts and install steps wrote") + written = building_on.written(local, [command for commands in runs.values() for _, command in commands]) + if clashes := _clashes(runs, written): + return _failed(env, form, arch, stage, key, "its parts write a file differently, so the later " + "part's would hide the earlier's: " + "; ".join(clashes)) installed = building_on.installed(local, layer.prefix) building_on.build(f"FROM {local}\n", local, platform, { INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", @@ -300,6 +313,7 @@ class _Layer: prefix: str install: Install conf: str + of: str app: bool = False served: str | None = None steps: tuple[tuple[str, str], ...] = () @@ -310,7 +324,7 @@ def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifa """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them, served on its port, its folder under ``DRIVERS`` named for its id's last part.""" install, name = driver.install[base.os.type], gate.folder_name(driver.id, "driver") - layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf") + layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf", driver.id) _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}, from_store) return layer @@ -324,7 +338,8 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, install, name = app.install[base.os.type], gate.folder_name(app.id, "app") file, _, cls = app.module.partition(":") served = shlex.quote(f"{PurePosixPath(file).relative_to(install.files)}:{cls}") - layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app=True, served=served, + layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app.id, app=True, + served=served, steps=tuple((step.name, step.scope) for step in app.declaration.install_steps if step.os is None or base.os.type in step.os)) programs = {f"app-{name}": f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", @@ -442,11 +457,8 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build f"--format=json > {prefix}/installed.json") if layer.app: lines.append(f"RUN mkdir -p {prefix}/{SOCKET.parent} && chown {user} {prefix}/{SOCKET.parent}") - for run in layer.install.run or []: - lines += _as(run.scope, user, f"RUN --network=none cd {prefix} && {shlex.quote('./' + run.script)}") - for step, scope in layer.steps: - lines += _as(scope, user, f"RUN --network=none cd {prefix} && venv/bin/python -m " - f"agentenv_protocol.computer.run_install_step {layer.served} {step}") + for scope, command in _runs(layer): + lines += _as(scope, user, f"RUN --network=none {command}") lines.append(f"COPY {layer.conf} /etc/supervisor/conf.d/{layer.conf}") checks = [f"test -d {prefix}", f"test -f /etc/supervisor/conf.d/{layer.conf}"] if _venv(layer): @@ -456,6 +468,26 @@ def _dockerfile(source: str, layers: list[_Layer], base: ComputerArtifact, build return "\n".join(lines) + "\n" +def _runs(layer: _Layer) -> list[tuple[str, str]]: + """The commands of ``layer``'s run scripts and its install steps, each with the scope it runs as: the only steps of + a build that write outside the layer's folder.""" + return [*((run.scope, f"cd {layer.prefix} && {shlex.quote('./' + run.script)}") for run in layer.install.run or []), + *((scope, f"cd {layer.prefix} && venv/bin/python -m agentenv_protocol.computer.run_install_step " + f"{layer.served} {step}") for step, scope in layer.steps)] + + +def _clashes(runs: dict[str, list[tuple[str, str]]], written: list[dict[str, str]]) -> list[str]: + """Each path, but a ``SHARED`` one, that two parts' ``runs`` wrote with different content, as ``written`` lists what + each of their commands wrote, in order, with the parts that wrote it.""" + found, owners = iter(written), {} + for of, commands in runs.items(): + for _ in commands: + for path, content in next(found).items(): + if not SHARED.search(path): + owners.setdefault(path, {})[of] = content + return [f"{path} ({' and '.join(by)})" for path, by in sorted(owners.items()) if len(set(by.values())) > 1] + + def _as(scope: str, user: str, run: str) -> list[str]: """``run`` as root, or for ``session``, as ``user``.""" return [run] if scope == "system" else [f"USER {user}", run, "USER root"] diff --git a/src/agent_env/env/envs/computer/utils/layers_in_image.py b/src/agent_env/env/envs/computer/utils/layers_in_image.py new file mode 100644 index 000000000..963c70328 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/layers_in_image.py @@ -0,0 +1,58 @@ +"""What RUN instructions of a saved image wrote, read by the composer after a build (``builder.Builder.written``): each +RUN, given by its command, is found in the image's history, and each file and link of its layer listed by path, as the +file's sha256, ``-> `` for a link, or what its target has for a hard link; folders and deletions are left out. +Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py + ``; it prints what each wrote, in order, as JSON.""" + +import hashlib +import json +import sys +import tarfile +from pathlib import PurePosixPath + +_CHUNK = 1 << 20 + + +def written(image: str, commands: list[str]) -> list[dict[str, str]]: + with tarfile.open(image) as saved: + manifest = json.load(saved.extractfile("manifest.json"))[0] + history = json.load(saved.extractfile(manifest["Config"]))["history"] + made = [entry.get("created_by", "") for entry in history if not entry.get("empty_layer")] + # Matched from the top: a base built without history has fewer entries than layers. + layers = list(zip(reversed(made), reversed(manifest["Layers"]))) + return [_files(saved, _layer(layers, command)) for command in commands] + + +def _layer(layers: list[tuple[str, str]], command: str) -> str: + found = next((layer for made_by, layer in layers if command in made_by), None) + if found is None: + raise ValueError(f"no layer of the image was made by {command!r}") + return found + + +def _files(saved: tarfile.TarFile, member: str) -> dict[str, str]: + files: dict[str, str] = {} + with tarfile.open(fileobj=saved.extractfile(member), mode="r|*") as layer: + for entry in layer: + path = "/" + entry.name.removeprefix("./").rstrip("/") + if PurePosixPath(path).name.startswith(".wh."): + continue + if entry.isreg(): + files[path] = _sha256(layer.extractfile(entry)) + elif entry.issym(): + files[path] = f"-> {entry.linkname}" + elif entry.islnk(): + target = "/" + entry.linkname.removeprefix("./") + files[path] = files.get(target, f"=> {target}") + return files + + +def _sha256(file) -> str: + hashed = hashlib.sha256() + while chunk := file.read(_CHUNK): + hashed.update(chunk) + return hashed.hexdigest() + + +if __name__ == "__main__": + json.dump(written(sys.argv[1], json.loads(sys.argv[2])), sys.stdout) diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 3af40b05b..3b24587af 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -1,10 +1,14 @@ """What the computer env tests share: the fake Docker and registry the composer builds with, a fake sandbox provider whose VMs build, and a base, a driver and an env of them.""" +import gzip +import hashlib +import io import json import re import shlex import subprocess +import tarfile import urllib.parse from agentenv_protocol.computer import DriverDeclaration @@ -13,7 +17,7 @@ from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config -from agent_env.env.envs.computer.utils import builder +from agent_env.env.envs.computer.utils import builder, layers_in_image from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.utils.install import Install, Service from agent_env.env.envs.computer.driver_env import ComputerDriverEnv @@ -48,6 +52,7 @@ class FakeDocker: def __init__(self, registry: Registry): self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} + self.writes, self.saved = {}, [] def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" @@ -97,6 +102,24 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ "build_settings": {path.parent.parent.name: json.loads(path.read_text()) for path in context.glob("apps/*/.agentenv/build_settings.json")}}) + def save(self, tag, path): + """``tag`` saved as ``docker save`` does: a base layer with no history, then a layer for each COPY and RUN of its + Dockerfile, a RUN's holding the files ``writes`` gives for each key its command has.""" + self.saved.append(tag) + dockerfile = next(build["dockerfile"] for build in reversed(self.builds) if build["tag"] == tag) + layers = [(None, {"etc/os-release": b"ubuntu"})] + for line in dockerfile.splitlines(): + instruction, _, rest = line.partition(" ") + command = rest.removeprefix("--network=none ") + if instruction == "RUN": + layers.append((f"RUN /bin/sh -c {command} # buildkit", { + path: content for key, files in self.writes.items() if key in command for path, content in files.items()})) + elif instruction == "COPY": + layers.append((f"{line} # buildkit", {})) + elif instruction != "FROM": + layers.append((line, None)) + saved_image(path, layers) + def push(self, local, ref, store): self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], labels=self.labels.get(local)) @@ -109,6 +132,39 @@ def _arch(self, config_digest): return json.loads(self.registry.blobs[config_digest])["architecture"] +def saved_image(path, layers): + """A tar of an image as ``docker save`` writes one: each of ``layers``, bottom first, its history's ``created_by`` + (None for a layer with no entry) and its files by path, a file's content as bytes and a link's target as text, or + None for an entry that made no layer; the last layer gzipped, as some savers keep them.""" + blobs, history, made = {}, [], [] + for i, (created_by, files) in enumerate(layers): + if created_by is not None: + history.append({"created_by": created_by, **({"empty_layer": True} if files is None else {})}) + if files is None: + continue + layer = io.BytesIO() + with tarfile.open(fileobj=layer, mode="w") as tar: + for name, content in files.items(): + entry = tarfile.TarInfo(name) + if isinstance(content, bytes): + entry.size = len(content) + tar.addfile(entry, io.BytesIO(content)) + else: + entry.type, entry.linkname = tarfile.SYMTYPE, content + tar.addfile(entry) + data = gzip.compress(layer.getvalue()) if i == len(layers) - 1 else layer.getvalue() + made.append(digest := f"blobs/sha256/{hashlib.sha256(data).hexdigest()}") + blobs[digest] = data + config = json.dumps({"history": history}).encode() + blobs[config_path := f"blobs/sha256/{hashlib.sha256(config).hexdigest()}"] = config + blobs["manifest.json"] = json.dumps([{"Config": config_path, "Layers": made}]).encode() + with tarfile.open(path, mode="w") as tar: + for name, data in blobs.items(): + entry = tarfile.TarInfo(name) + entry.size = len(data) + tar.addfile(entry, io.BytesIO(data)) + + def _base(image, **changes): return ComputerArtifact.put(inspect=False, **{ "id": "@local/~/kit/ubuntu-24.04", "arch": ["amd64", "arm64"], "os": OS, "session": {"user": "user"}, @@ -193,7 +249,7 @@ def _line(self, line): return ran.returncode, ran.stdout + ran.stderr def _docker(self, words): - docker, cwd = self.provider.docker, None + docker, cwd, printed = self.provider.docker, None, "" while words: command, words = (words[:words.index("&&")], words[words.index("&&") + 1:]) if "&&" in words else (words, []) if command[0] == "cd": @@ -214,6 +270,13 @@ def _docker(self, words): platform=command[command.index("--platform") + 1], labels=labels) except DockerBuildError as e: return 1, str(e) + elif command[:2] == ["docker", "save"]: + docker.save(command[4], self.at(command[3])) + elif command[:2] == ["docker", "run"] and "/layers.py" in command: + saved = command[command.index("-v") + 1].split(":")[0] + printed = json.dumps(layers_in_image.written(str(self.at(saved)), json.loads(command[-1]))) + elif command[:2] == ["rm", "-f"]: + self.at(command[2]).unlink(missing_ok=True) elif command[:2] == ["docker", "tag"]: self.provider.tagged[command[3]] = command[2] elif command[:2] == ["docker", "push"]: @@ -224,7 +287,7 @@ def _docker(self, words): self.provider.uploaded.append(url) else: raise AssertionError(f"a builder VM was asked to run {command}") - return 0, "" + return 0, printed class FakeVmProvider(SandboxProvider): diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py index 2ebe7fada..065f6a885 100644 --- a/tst/unit/env/envs/conftest.py +++ b/tst/unit/env/envs/conftest.py @@ -23,6 +23,7 @@ def docker(local_stores, cli_routing, monkeypatch): fake = FakeDocker(registry) monkeypatch.setattr(builder, "build_image", fake.build_image) monkeypatch.setattr(builder, "push_local_image", fake.push) + monkeypatch.setattr(builder, "save_image", fake.save) monkeypatch.setattr(builder, "docker_login", lambda auth: None) monkeypatch.setattr(composer, "write_index", fake.joined) monkeypatch.setattr(composer, "pin_digest", registry.pin_digest) diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 8b6f94cbe..19fe1846b 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -9,6 +9,7 @@ import logging import re import subprocess +import sys import tarfile import time from types import SimpleNamespace @@ -20,12 +21,12 @@ from agent_env.artifact.store import get_artifact_store from agent_env.cli import cli from agent_env.env import Env -from agent_env.env.envs.computer.utils import builder, composer, packages +from agent_env.env.envs.computer.utils import builder, composer, layers_in_image, packages from agent_env.env.envs.computer.utils.composer import Composing from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider -from tst.unit.env.envs._computer import _base, _put, FakeVm, FakeVmProvider +from tst.unit.env.envs._computer import _base, _put, FakeVm, FakeVmProvider, saved_image def test_a_stage_that_goes_on_is_said_to_go_on_and_a_failed_pair_says_why(base, driver, docker, monkeypatch): @@ -113,6 +114,27 @@ def run(argv, **kwargs): assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] +def test_what_each_run_wrote_is_read_from_the_layer_its_command_made_in_a_saved_images_history(tmp_path): + saved_image(tmp_path / "image.tar", [ + (None, {"etc/os-release": b"ubuntu"}), + ("USER root", None), + ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", { + "usr/local/bin/tool": b"a", "usr/bin/tool": "/usr/local/bin/tool", "etc/.wh.gone": b"", "./etc/a": b"a"}), + ("COPY apps/b/ /opt/b/ # buildkit", {"opt/b/x": b"x"}), + ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"usr/local/bin/tool": b"b"})]) + commands = ["cd /opt/b && ./setup.sh", "cd /opt/a && ./setup.sh"] + + ran = subprocess.run([sys.executable, "-I", str(builder.LAYERS), str(tmp_path / "image.tar"), json.dumps(commands)], + capture_output=True, text=True) + + assert json.loads(ran.stdout) == layers_in_image.written(str(tmp_path / "image.tar"), commands) == [ + {"/usr/local/bin/tool": hashlib.sha256(b"b").hexdigest()}, + {"/usr/local/bin/tool": hashlib.sha256(b"a").hexdigest(), "/usr/bin/tool": "-> /usr/local/bin/tool", + "/etc/a": hashlib.sha256(b"a").hexdigest()}] + with pytest.raises(ValueError, match=r"no layer of the image was made by 'cd /opt/c && \./setup\.sh'"): + layers_in_image.written(str(tmp_path / "image.tar"), ["cd /opt/c && ./setup.sh"]) + + def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): signed = [] monkeypatch.setattr(builder, "docker_login", signed.append) diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 716657625..31f1250dc 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1070,6 +1070,7 @@ def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_t assert [(app.id, app.version) for app in stored.composed_apps()] == [(notes.id, 1), (mail.id, 1)] assert stored.to_dict()["apps"] == [{"id": notes.id, "version": 1}, {"id": mail.id, "version": 1}] assert stored.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + assert docker.saved == [] assert _put(base, driver, apps=[mail, notes]).metadata["composed"]["container"]["arm64"]["key"] != ( stored.metadata["composed"]["container"]["arm64"]["key"]) @@ -1105,6 +1106,68 @@ def other(name="other", **changes): assert docker.builds == [] +#: An install that runs one script, setup.sh, as root. +SETUP = {"run": [{"script": "setup.sh"}], "services": []} + + +def _setting_up(tmp_path, *names): + """An app of each of ``names`` whose install runs setup.sh, put without a computer.""" + return [_app(None, None, tmp_path / name, compose=False, name=name, install=SETUP, + extra={"setup.sh": "#!/bin/sh\n"}) for name in names] + + +@pytest.mark.parametrize("mail, notes, clash", [ + ({"usr/local/bin/tool": b"mail's"}, {"usr/local/bin/tool": b"notes'"}, + "/usr/local/bin/tool (@local/~/kit/mail and @local/~/kit/notes)"), + ({"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}, + "/usr/bin/tool (@local/~/kit/mail and @local/~/kit/notes)"), + ({"usr/local/bin/tool": b"same"}, {"usr/local/bin/tool": b"same"}, None), + *[({path: b"mail's"}, {path: b"notes'"}, None) for path in ( + "var/log/setup.log", "tmp/x", "home/user/.cache/pip/x", "opt/x/__pycache__/m.pyc", "etc/passwd", "etc/group-", + "etc/ld.so.cache", "root/.bashrc", "home/user/.profile")], +], ids=["file", "link", "same", "log", "tmp", "cache", "pycache", "passwd", "group-backup", "ld-cache", "root-rc", + "home-profile"]) +def test_two_apps_whose_run_scripts_write_one_file_differently_fail_at_build_naming_it(base, driver, docker, tmp_path, + mail, notes, clash): + docker.writes = {"apps/mail && ": mail, "apps/notes && ": notes} + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes")).id) + + composed = env.metadata["composed"]["container"] + assert sorted(docker.saved) == sorted(build["tag"] for build in docker.builds) + if clash is None: + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + else: + assert {arch: (r["status"], r["stage"], r["reason"]) for arch, r in composed.items()} == { + arch: ("failed", "build", "its parts write a file differently, so the later part's would hide the " + f"earlier's: {clash}") for arch in ("amd64", "arm64")} + assert docker.probed == [] and "computer" not in env.metadata + + +def test_one_part_with_run_scripts_has_nothing_to_compare_and_its_image_isnt_saved(base, driver, docker, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": b"mail's"}} + mail = _setting_up(tmp_path, "mail")[0] + + env = Env.get(_put(base, driver, apps=[mail, _app(None, None, tmp_path / "notes", compose=False, name="notes")]).id) + + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} and docker.saved == [] + + +def test_a_builder_vm_compares_what_each_part_wrote_in_a_container_of_the_image_before_pushing(base, driver, docker, + signing, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": b"mail's"}, "apps/notes && ": {"usr/local/bin/tool": b"notes'"}} + vms = FakeVmProvider(docker, tmp_path / "vms") + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), composing=Composing(vms)).id) + + assert {arch: (r["status"], r["reason"].rpartition(": ")[2]) for arch, r in env.metadata["composed"]["container"].items()} == { + arch: ("failed", "/usr/local/bin/tool (@local/~/kit/mail and @local/~/kit/notes)") for arch in ("amd64", "arm64")} + compared = [line for line in vms.ran if "/layers.py" in line] + assert len(compared) == 2 and all("docker run --rm --network none --user root" in line for line in compared) + assert not [line for line in vms.ran if "docker push" in line] and all(vm.terminated for vm in vms.vms) + assert not list((tmp_path / "vms").rglob("image.tar")) + + def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker, tmp_path): _app(base, driver, tmp_path / "mail", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["arm64"]}]) From fb2f5703b26357c15d71ef5704584db1539bef0b Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 16:07:11 -0700 Subject: [PATCH 03/14] test(computer): both apps of the two-app desk run a script, so its build compares their files Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/builder.py | 4 ++-- tst/integration/computer/app/artifacts/check_desk/run.py | 6 ++++-- tst/integration/computer/app/envs/clock/app.py | 9 +++++---- tst/integration/computer/app/envs/clock/setup.sh | 4 ++++ tst/integration/computer/computer_app_test.py | 8 +++++--- 5 files changed, 20 insertions(+), 11 deletions(-) create mode 100755 tst/integration/computer/app/envs/clock/setup.sh diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 8b2cd3698..f033a8852 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -140,8 +140,8 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]: @abstractmethod def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]: - """What each of ``commands``, a RUN of ``tag``'s build, wrote in its layer of ``tag``, read from the image saved - (``layers_in_image``).""" + """What each of ``commands``, a RUN of ``tag``'s build, wrote in its layer of ``tag``, read from a save of the + image (``layers_in_image``).""" @abstractmethod def push(self, tag: str, ref: str, store) -> None: diff --git a/tst/integration/computer/app/artifacts/check_desk/run.py b/tst/integration/computer/app/artifacts/check_desk/run.py index 68055165e..38db8dd25 100644 --- a/tst/integration/computer/app/artifacts/check_desk/run.py +++ b/tst/integration/computer/app/artifacts/check_desk/run.py @@ -1,6 +1,7 @@ """What each of the two apps composed into the desk wrote on the settings its deploy gave it, the clock's answer on its own port, the name the computer's card gives and what its readiness says of each app, asked on loopback from inside -the computer, the apps launched since it booted, and each app's install under its own folder.""" +the computer, the apps launched since it booted, and each app's install under its own folder, its run script's output +there.""" import json import time import urllib.request @@ -27,4 +28,5 @@ def run(input): "app_ready": sorted(check["name"] for check in ready["checks"] if check["name"].startswith("app ") and check["ok"]), "launched": sorted(p.name for p in Path("/run/agentenv/apps").iterdir()), - "folders": sorted(p.name for p in APPS.iterdir()), "hello": (APPS / "notes" / "hello.txt").exists()} + "folders": sorted(p.name for p in APPS.iterdir()), "hello": (APPS / "notes" / "hello.txt").exists(), + "paris": "Paris" in (APPS / "clock" / "zones.txt").read_text().split()} diff --git a/tst/integration/computer/app/envs/clock/app.py b/tst/integration/computer/app/envs/clock/app.py index c1833a6f0..0aec9d802 100644 --- a/tst/integration/computer/app/envs/clock/app.py +++ b/tst/integration/computer/app/envs/clock/app.py @@ -1,15 +1,16 @@ """A clock app for the computer env tests, composed beside notes: its service answers on a port of its own with the -time zone its settings name, and writes it to ~/.clock.json. It installs tzdata, and declares no window.""" +time zone its settings name, and writes it to ~/.clock.json. It installs tzdata, which its run script lists the zones +of, and declares no window.""" from pydantic import BaseModel -from agentenv_protocol.computer import ComputerAppEnvironment, Packages, Service, computer_app, linux +from agentenv_protocol.computer import ComputerAppEnvironment, Packages, RunScript, Service, computer_app, linux PORT = 8137 @computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}]) -@linux(packages=Packages(apt=["tzdata"]), services=[Service(name="clock", command="venv/bin/python serve.py")], - ports=[PORT]) +@linux(packages=Packages(apt=["tzdata"]), run=[RunScript(script="setup.sh")], + services=[Service(name="clock", command="venv/bin/python serve.py")], ports=[PORT]) class Clock(ComputerAppEnvironment): class Settings(BaseModel): zone: str = "UTC" diff --git a/tst/integration/computer/app/envs/clock/setup.sh b/tst/integration/computer/app/envs/clock/setup.sh new file mode 100755 index 000000000..012a6b4e9 --- /dev/null +++ b/tst/integration/computer/app/envs/clock/setup.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# Run as root once tzdata is in: the zones it installed, beside the app. +set -e +ls /usr/share/zoneinfo/Europe > zones.txt diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index f71c08b1c..6b2947d7d 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -3,8 +3,8 @@ settings the deploy gave it, a secret resolved at boot, its window launched once by its server as the desktop session comes up, and the computer ready only once its server says its window and its @ready_check are; its apt packages, one from a PPA, its download, run script and install step on its build setting, fetched at put, installed offline. -And two apps composed into one computer env, each running on its own settings from its own folder. Needs Docker and the -local registry on localhost:5000.""" +And two apps composed into one computer env, each running on its own settings from its own folder, the files their run +scripts wrote compared before the push. Needs Docker and the local registry on localhost:5000.""" from pathlib import Path @@ -50,4 +50,6 @@ def test_two_apps_composed_into_one_computer_env_each_run_on_their_own_settings( assert context["metadata"]["script_results"]["together"] == { "notes": {"signature": "-- Ada", "token": "t0ken"}, "clock": {"zone": "Europe/Paris", "served": {"zone": "Europe/Paris", "known": True}}, "card": "desk", - "app_ready": ["app clock", "app notes"], "launched": ["notes"], "folders": ["clock", "notes"], "hello": True} + "app_ready": ["app clock", "app notes"], "launched": ["notes"], "folders": ["clock", "notes"], "hello": True, + "paris": True} + assert "comparing the files its parts' run scripts and install steps wrote" in ran.output From a901ab3e4b18d4ac351b69f23162c67e5207dfd2 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 16:26:44 -0700 Subject: [PATCH 04/14] fix(computer): refuse apps sharing no arch or a program name, keep apps given as an iterator, match each run's layer exactly - Gate 1 refuses apps that each share an arch with the base and driver but share none all together, and two apps whose supervisord programs would take one name (an app mail with a service sync, and an app mail-sync). gate.programs names an app's programs, and the composer writes them by it. - ComputerEnv.put lists its apps once, so an iterator is checked and stored alike. - The layer reader finds each RUN's layer by the exact command a shell ran, the next one after the last found, so setup.sh no longer matches setup.sh.extra and a command run twice maps to two layers. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/env.py | 5 ++-- .../env/envs/computer/utils/composer.py | 5 ++-- src/agent_env/env/envs/computer/utils/gate.py | 15 ++++++++++-- .../envs/computer/utils/layers_in_image.py | 23 ++++++++++++------- tst/unit/env/envs/test_computer_builder.py | 22 +++++++++++------- tst/unit/env/envs/test_computer_env.py | 10 ++++++-- 6 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 2d5b8b8c9..1adfc3985 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -93,12 +93,13 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, """Write a computer env of ``computer``, ``driver`` and ``apps``, computer app envs built into it in their order, once gate 1 passes, then compose it as ``composing`` says unless ``compose`` is False. Refused, naming every mismatch, before anything is stored.""" - values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, + apps = list(apps) + values, problems = gate.check(computer, driver, apps=apps, display=display, network=network, computer_settings=computer_settings or {}) if problems: raise ValueError(f"computer env {id!r}: " + "; ".join(problems)) env = get_env_store().put_document(cls( - id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, apps=list(apps), + id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, apps=apps, display=display, network=network, computer_settings=values)) if compose: composer.compose(env, composing=composing) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 15ef66ce6..21903144d 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -342,8 +342,9 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, served=served, steps=tuple((step.name, step.scope) for step in app.declaration.install_steps if step.os is None or base.os.type in step.os)) - programs = {f"app-{name}": f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", - **_services(f"app-{name}", install)} + programs = dict(zip(gate.programs(app, base.os.type), [ + f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", + *(service.command for service in install.services)])) _stage(context, layer, f"app {app.id}", app.files, programs, {"ENVIRONMENT_NAME": app.environment_name}, from_store) if app.build_settings: diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index d368841dd..5668fd844 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -75,14 +75,18 @@ def folder_name(env_id: str, fallback: str) -> str: def _apps(base: ComputerArtifact, driver: ComputerDriverEnv, apps: list[ComputerAppEnv]) -> list[str]: - """Every way ``apps`` don't fit together on ``base`` and ``driver``: each one's own (``check_app``), one listed - twice, two in one folder or under one environment name, and a port two of them listen on.""" + """Every way ``apps`` don't fit together on ``base`` and ``driver``: each one's own (``check_app``), no architecture + they all run, one listed twice, and two in one folder, under one environment name, with one program name or + listening on one port.""" problems = [problem for i, app in enumerate(apps) for problem in check_app(base, driver, app, f"apps[{i}]")] + if not problems and apps and not arches(base, driver, apps): + problems.append(f"apps: no architecture {base.id}, {driver.id} and every app all run") ids = [app.id for app in apps] problems += [f"apps: {app_id} is listed twice" for app_id in sorted({i for i in ids if ids.count(i) > 1})] distinct = list({app.id: app for app in apps}.values()) for kind, keys in (("folder", lambda app: [f"{APPS}/{folder_name(app.id, 'app')}"]), ("environment name", lambda app: [app.environment_name]), + ("program", lambda app: programs(app, base.os.type)), ("port", lambda app: _ports(app, base.os.type))): claims: dict[object, list[str]] = {} for app in distinct: @@ -93,6 +97,13 @@ def _apps(base: ComputerArtifact, driver: ComputerDriverEnv, apps: list[Computer return problems +def programs(app: ComputerAppEnv, os_type: str) -> list[str]: + """The names of the supervisord programs ``app`` runs on ``os_type``: its own server's, then its services'.""" + server = f"app-{folder_name(app.id, 'app')}" + install = app.install.get(os_type) + return [server, *(f"{server}-{service.name}" for service in (install.services if install is not None else []))] + + def _ports(app: ComputerAppEnv, os_type: str) -> list[int]: install = app.install.get(os_type) return list(install.ports or []) if install is not None else [] diff --git a/src/agent_env/env/envs/computer/utils/layers_in_image.py b/src/agent_env/env/envs/computer/utils/layers_in_image.py index 963c70328..a87896c34 100644 --- a/src/agent_env/env/envs/computer/utils/layers_in_image.py +++ b/src/agent_env/env/envs/computer/utils/layers_in_image.py @@ -1,5 +1,6 @@ """What RUN instructions of a saved image wrote, read by the composer after a build (``builder.Builder.written``): each -RUN, given by its command, is found in the image's history, and each file and link of its layer listed by path, as the +RUN, given by its command in build order, is found in the image's history, the next layer a shell ran exactly that +command for, and each file and link of its layer listed by path, as the file's sha256, ``-> `` for a link, or what its target has for a hard link; folders and deletions are left out. Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py ``; it prints what each wrote, in order, as JSON.""" @@ -19,15 +20,21 @@ def written(image: str, commands: list[str]) -> list[dict[str, str]]: history = json.load(saved.extractfile(manifest["Config"]))["history"] made = [entry.get("created_by", "") for entry in history if not entry.get("empty_layer")] # Matched from the top: a base built without history has fewer entries than layers. - layers = list(zip(reversed(made), reversed(manifest["Layers"]))) - return [_files(saved, _layer(layers, command)) for command in commands] + count = min(len(made), len(manifest["Layers"])) + layers, found, at = list(zip(made[-count:], manifest["Layers"][-count:])), [], 0 + for command in commands: + at = next((i for i in range(at, len(layers)) if _ran(layers[i][0], command)), None) + if at is None: + raise ValueError(f"no layer of the image after the last one found was made by {command!r}") + found.append(_files(saved, layers[at][1])) + at += 1 + return found -def _layer(layers: list[tuple[str, str]], command: str) -> str: - found = next((layer for made_by, layer in layers if command in made_by), None) - if found is None: - raise ValueError(f"no layer of the image was made by {command!r}") - return found +def _ran(made_by: str, command: str) -> bool: + """Whether the history entry ``made_by`` is a RUN of ``command`` by a shell, as BuildKit and the classic builder + record one.""" + return made_by.removesuffix(" # buildkit").endswith(f"/bin/sh -c {command}") def _files(saved: tarfile.TarFile, member: str) -> dict[str, str]: diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 19fe1846b..aea6f75c1 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -114,25 +114,31 @@ def run(argv, **kwargs): assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] -def test_what_each_run_wrote_is_read_from_the_layer_its_command_made_in_a_saved_images_history(tmp_path): +def test_what_each_run_wrote_is_read_from_the_next_layer_a_shell_ran_its_command_for_in_a_saved_images_history( + tmp_path): saved_image(tmp_path / "image.tar", [ (None, {"etc/os-release": b"ubuntu"}), ("USER root", None), ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", { "usr/local/bin/tool": b"a", "usr/bin/tool": "/usr/local/bin/tool", "etc/.wh.gone": b"", "./etc/a": b"a"}), + ("RUN /bin/sh -c cd /opt/a && ./setup.sh.extra # buildkit", {"etc/extra": b"extra"}), ("COPY apps/b/ /opt/b/ # buildkit", {"opt/b/x": b"x"}), - ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"usr/local/bin/tool": b"b"})]) - commands = ["cd /opt/b && ./setup.sh", "cd /opt/a && ./setup.sh"] + ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"usr/local/bin/tool": b"b"}), + ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"etc/again": b"again"})]) + commands = ["cd /opt/a && ./setup.sh", "cd /opt/a && ./setup.sh.extra", "cd /opt/b && ./setup.sh", + "cd /opt/b && ./setup.sh"] ran = subprocess.run([sys.executable, "-I", str(builder.LAYERS), str(tmp_path / "image.tar"), json.dumps(commands)], capture_output=True, text=True) + sha = lambda data: hashlib.sha256(data).hexdigest() assert json.loads(ran.stdout) == layers_in_image.written(str(tmp_path / "image.tar"), commands) == [ - {"/usr/local/bin/tool": hashlib.sha256(b"b").hexdigest()}, - {"/usr/local/bin/tool": hashlib.sha256(b"a").hexdigest(), "/usr/bin/tool": "-> /usr/local/bin/tool", - "/etc/a": hashlib.sha256(b"a").hexdigest()}] - with pytest.raises(ValueError, match=r"no layer of the image was made by 'cd /opt/c && \./setup\.sh'"): - layers_in_image.written(str(tmp_path / "image.tar"), ["cd /opt/c && ./setup.sh"]) + {"/usr/local/bin/tool": sha(b"a"), "/usr/bin/tool": "-> /usr/local/bin/tool", "/etc/a": sha(b"a")}, + {"/etc/extra": sha(b"extra")}, {"/usr/local/bin/tool": sha(b"b")}, {"/etc/again": sha(b"again")}] + for missing in (["cd /opt/c && ./setup.sh"], ["cd /opt/b && ./setup.sh", "cd /opt/a && ./setup.sh"], + [*commands, "cd /opt/b && ./setup.sh"]): + with pytest.raises(ValueError, match="no layer of the image after the last one found was made by"): + layers_in_image.written(str(tmp_path / "image.tar"), missing) def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 31f1250dc..0360167ab 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1057,7 +1057,7 @@ def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_t notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", install={"ports": [8025], "services": [{"name": "sync", "command": "venv/bin/python sync.py"}]}) - env = _put(base, driver, apps=[notes, mail]) + env = _put(base, driver, apps=iter([notes, mail])) assert sorted(build["platform"] for build in docker.builds) == ["linux/amd64", "linux/arm64"] dockerfile, confs = docker.builds[0]["dockerfile"], docker.builds[0]["confs"] @@ -1089,7 +1089,13 @@ def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_t (lambda mail, other: [mail, other(name="arm", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["riscv64"]}])], "apps[1]: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and @local/~/kit/arm all run"), -], ids=["twice", "environment-name", "folder", "app-port", "driver-port", "arch"]) + (lambda mail, other: [other(name=arch, supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": [arch]}]) + for arch in ("amd64", "arm64")], + "apps: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and every app all run"), + (lambda mail, other: [other(name="mail", install={"services": [{"name": "sync", "command": "venv/bin/sync"}]}), + other(name="mail-sync", install={"services": []})], + "apps: @local/~/kit/mail and @local/~/kit/mail-sync would share the program 'app-mail-sync'"), +], ids=["twice", "environment-name", "folder", "app-port", "driver-port", "arch", "no-shared-arch", "program"]) def test_apps_that_dont_fit_together_or_on_the_computer_are_refused_before_anything_is_stored(base, driver, docker, tmp_path, apps, problem): def other(name="other", **changes): From c2559ec3d65b23401af519e3c3b9d9ebefa59df1 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 18:36:26 -0700 Subject: [PATCH 05/14] fix(computer): read each run's layer from the newest down, so a base's layer of the same command never stands in Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/layers_in_image.py | 16 ++++++++-------- tst/unit/env/envs/test_computer_builder.py | 5 +++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/layers_in_image.py b/src/agent_env/env/envs/computer/utils/layers_in_image.py index a87896c34..dddb3fc74 100644 --- a/src/agent_env/env/envs/computer/utils/layers_in_image.py +++ b/src/agent_env/env/envs/computer/utils/layers_in_image.py @@ -1,6 +1,7 @@ """What RUN instructions of a saved image wrote, read by the composer after a build (``builder.Builder.written``): each -RUN, given by its command in build order, is found in the image's history, the next layer a shell ran exactly that -command for, and each file and link of its layer listed by path, as the +RUN, given by its command in build order, is found in the image's history from the top, the last command first, each +in the newest layer below the one found before it that a shell ran exactly that command for, so a base's layers of the +same command never stand in for the build's own; and each file and link of its layer listed by path, as the file's sha256, ``-> `` for a link, or what its target has for a hard link; folders and deletions are left out. Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py ``; it prints what each wrote, in order, as JSON.""" @@ -21,14 +22,13 @@ def written(image: str, commands: list[str]) -> list[dict[str, str]]: made = [entry.get("created_by", "") for entry in history if not entry.get("empty_layer")] # Matched from the top: a base built without history has fewer entries than layers. count = min(len(made), len(manifest["Layers"])) - layers, found, at = list(zip(made[-count:], manifest["Layers"][-count:])), [], 0 - for command in commands: - at = next((i for i in range(at, len(layers)) if _ran(layers[i][0], command)), None) + layers, found, at = list(zip(made[-count:], manifest["Layers"][-count:])), [], count + for command in reversed(commands): + at = next((i for i in range(at - 1, -1, -1) if _ran(layers[i][0], command)), None) if at is None: - raise ValueError(f"no layer of the image after the last one found was made by {command!r}") + raise ValueError(f"no layer of the image below the last one found was made by {command!r}") found.append(_files(saved, layers[at][1])) - at += 1 - return found + return found[::-1] def _ran(made_by: str, command: str) -> bool: diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index aea6f75c1..c54a9d5b4 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -114,10 +114,11 @@ def run(argv, **kwargs): assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] -def test_what_each_run_wrote_is_read_from_the_next_layer_a_shell_ran_its_command_for_in_a_saved_images_history( +def test_what_each_run_wrote_is_read_from_the_newest_layer_a_shell_ran_its_command_for_in_a_saved_images_history( tmp_path): saved_image(tmp_path / "image.tar", [ (None, {"etc/os-release": b"ubuntu"}), + ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", {"usr/local/bin/tool": b"the base's"}), ("USER root", None), ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", { "usr/local/bin/tool": b"a", "usr/bin/tool": "/usr/local/bin/tool", "etc/.wh.gone": b"", "./etc/a": b"a"}), @@ -137,7 +138,7 @@ def test_what_each_run_wrote_is_read_from_the_next_layer_a_shell_ran_its_command {"/etc/extra": sha(b"extra")}, {"/usr/local/bin/tool": sha(b"b")}, {"/etc/again": sha(b"again")}] for missing in (["cd /opt/c && ./setup.sh"], ["cd /opt/b && ./setup.sh", "cd /opt/a && ./setup.sh"], [*commands, "cd /opt/b && ./setup.sh"]): - with pytest.raises(ValueError, match="no layer of the image after the last one found was made by"): + with pytest.raises(ValueError, match="no layer of the image below the last one found was made by"): layers_in_image.written(str(tmp_path / "image.tar"), missing) From 12c3838aa5d5c37e0fbb8b1af233f3cd2df0da19 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 19:24:47 -0700 Subject: [PATCH 06/14] feat(computer): refuse only what a part certainly undoes; say other files two parts wrote differently The layer reader lists each entry's kind: a file, a program (a file any can run), a link, a folder, a deletion or a folder made opaque. A build fails only when a later part certainly undoes what an earlier one installed: it deletes it, points its link elsewhere, makes it another kind, or replaces its program. Any other file two parts wrote differently may be an edit that kept the earlier part's change, such as two apps adding users or a cache rebuilt from every app's entries, so a progress line names it and the parts, and the build goes on. Only the scratch locations of the filesystem hierarchy standard are never compared; the list of files edited in place is gone. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/composer.py | 76 ++++++++++++++----- .../envs/computer/utils/layers_in_image.py | 25 +++--- tst/unit/env/envs/_computer.py | 14 ++-- tst/unit/env/envs/test_computer_builder.py | 9 ++- tst/unit/env/envs/test_computer_env.py | 71 +++++++++++------ 5 files changed, 136 insertions(+), 59 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 21903144d..24652c4de 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -79,12 +79,13 @@ _PROTOCOL = "agentenv-framework-protocol" # What building the protocol from its source reads, beside src/. _PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") -#: Paths two parts' run scripts and install steps may both write: temporary files, logs and caches, and the files -#: installers edit in place, so the later part's copy holds the earlier's change: the account databases, the linker's -#: cache, the system environment and a home's shell startup files. -SHARED = re.compile(r"^/(tmp|var/tmp|var/log|var/cache|run)/|/\.cache/|/__pycache__/|" - r"^/etc/(passwd|group|shadow|gshadow|subuid|subgid|ld\.so\.cache|environment)-?$|" - r"^/(root|home/[^/]+)/\.(bashrc|profile|bash_profile)$") +#: Where what a run script or an install step writes may be lost, as the filesystem hierarchy standard has it: +#: temporary files, runtime state, logs and caches. Never compared. +SCRATCH = re.compile(r"^/(tmp|var/tmp|var/log|var/cache|run)/|/\.cache/|/__pycache__/") +#: How many of the files two parts wrote differently a progress line names. +_SHOWN = 10 +#: What each kind of entry ``layers_in_image`` lists is, to a reader: a program is a file. +_KINDS = {"file": "file", "exec": "file", "link": "link", "dir": "folder"} @dataclass(frozen=True) @@ -261,9 +262,14 @@ def on() -> builder.Builder: if len(runs := {part.of: _runs(part) for part in layers if _runs(part)}) > 1: progress.at(pair, "comparing the files its parts' run scripts and install steps wrote") written = building_on.written(local, [command for commands in runs.values() for _, command in commands]) - if clashes := _clashes(runs, written): - return _failed(env, form, arch, stage, key, "its parts write a file differently, so the later " - "part's would hide the earlier's: " + "; ".join(clashes)) + lost, differ = _clashes(runs, written) + if lost: + return _failed(env, form, arch, stage, key, "a part undoes what an earlier one installed: " + + "; ".join(lost)) + if differ: + progress.at(pair, "compared", "not refused, as each may keep the earlier part's change: two parts " + f"wrote {', '.join(differ[:_SHOWN])}" + (f" and {len(differ) - _SHOWN} more" + if len(differ) > _SHOWN else "")) installed = building_on.installed(local, layer.prefix) building_on.build(f"FROM {local}\n", local, platform, { INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", @@ -477,16 +483,52 @@ def _runs(layer: _Layer) -> list[tuple[str, str]]: f"{layer.served} {step}") for step, scope in layer.steps)] -def _clashes(runs: dict[str, list[tuple[str, str]]], written: list[dict[str, str]]) -> list[str]: - """Each path, but a ``SHARED`` one, that two parts' ``runs`` wrote with different content, as ``written`` lists what - each of their commands wrote, in order, with the parts that wrote it.""" - found, owners = iter(written), {} +def _clashes(runs: dict[str, list[tuple[str, str]]], written: list[dict[str, str]]) -> tuple[list[str], list[str]]: + """What a part's ``runs`` did to what an earlier part's left, as ``written`` lists what each of their commands wrote + (``layers_in_image``), in order, but under ``SCRATCH``: first each certain loss (``_lost``), then each other file + the two wrote differently, which may be an edit that kept the earlier part's change.""" + found, latest, lost, differ = iter(written), {}, [], [] for of, commands in runs.items(): for _ in commands: - for path, content in next(found).items(): - if not SHARED.search(path): - owners.setdefault(path, {})[of] = content - return [f"{path} ({' and '.join(by)})" for path, by in sorted(owners.items()) if len(set(by.values())) > 1] + wrote = next(found) + for path, entry in sorted(wrote.items()): + if SCRATCH.search(path): + continue + if entry == "opaque": + lost += [f"{hidden}: {of} deletes {by}'s file" for hidden, (by, was) in sorted(latest.items()) + if hidden.startswith(f"{path}/") and by != of and hidden not in wrote and was != "dir"] + entry = "dir" + if path in latest and (earlier := latest[path])[0] != of: + if reason := _lost(path, of, entry, *earlier): + lost.append(reason) + elif _differs(entry, earlier[1]): + differ.append(f"{path} ({earlier[0]}, {of})") + latest[path] = (of, entry) + return lost, differ + + +def _lost(path: str, of: str, entry: str, by: str, was: str) -> str | None: + """How ``of``'s ``entry`` at ``path`` certainly loses what ``by`` left there, ``was``: deleting it, making it another + kind, pointing its link elsewhere, or replacing its program; None when it doesn't.""" + kind, _, what = entry.partition(" ") + kind_was, _, what_was = was.partition(" ") + if kind_was == "deleted": + return None + if kind == "deleted": + return f"{path}: {of} deletes {by}'s {_KINDS[kind_was]}" + if _KINDS[kind] != _KINDS[kind_was]: + return f"{path}: {of} makes {by}'s {_KINDS[kind_was]} a {_KINDS[kind]}" + if kind_was == "link" and what != what_was: + return f"{path}: {of} points {by}'s link elsewhere" + if kind_was == "exec" and what != what_was: + return f"{path}: {of} replaces {by}'s program" + return None + + +def _differs(entry: str, was: str) -> bool: + kind, _, what = entry.partition(" ") + kind_was, _, what_was = was.partition(" ") + return kind in ("file", "exec") and kind_was == "file" and what != what_was def _as(scope: str, user: str, run: str) -> list[str]: diff --git a/src/agent_env/env/envs/computer/utils/layers_in_image.py b/src/agent_env/env/envs/computer/utils/layers_in_image.py index dddb3fc74..2e0891b8e 100644 --- a/src/agent_env/env/envs/computer/utils/layers_in_image.py +++ b/src/agent_env/env/envs/computer/utils/layers_in_image.py @@ -1,9 +1,9 @@ """What RUN instructions of a saved image wrote, read by the composer after a build (``builder.Builder.written``): each RUN, given by its command in build order, is found in the image's history from the top, the last command first, each in the newest layer below the one found before it that a shell ran exactly that command for, so a base's layers of the -same command never stand in for the build's own; and each file and link of its layer listed by path, as the -file's sha256, ``-> `` for a link, or what its target has for a hard link; folders and deletions are left out. -Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py +same command never stand in for the build's own; and each entry of its layer listed by path, as `` ``: +``file`` or ``exec`` (a file any can run) with its sha256, ``link`` with its target, ``dir``, ``deleted``, or ``opaque`` +for a folder whose earlier contents the layer hides. A hard link is listed as its target is. Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py ``; it prints what each wrote, in order, as JSON.""" import hashlib @@ -41,16 +41,21 @@ def _files(saved: tarfile.TarFile, member: str) -> dict[str, str]: files: dict[str, str] = {} with tarfile.open(fileobj=saved.extractfile(member), mode="r|*") as layer: for entry in layer: - path = "/" + entry.name.removeprefix("./").rstrip("/") - if PurePosixPath(path).name.startswith(".wh."): - continue - if entry.isreg(): - files[path] = _sha256(layer.extractfile(entry)) + path = PurePosixPath("/" + entry.name.removeprefix("./").rstrip("/")) + kind = "exec" if entry.mode & 0o111 else "file" + if path.name == ".wh..wh..opq": + files[str(path.parent)] = "opaque" + elif path.name.startswith(".wh."): + files[str(path.with_name(path.name.removeprefix(".wh.")))] = "deleted" + elif entry.isreg(): + files[str(path)] = f"{kind} {_sha256(layer.extractfile(entry))}" elif entry.issym(): - files[path] = f"-> {entry.linkname}" + files[str(path)] = f"link {entry.linkname}" elif entry.islnk(): target = "/" + entry.linkname.removeprefix("./") - files[path] = files.get(target, f"=> {target}") + files[str(path)] = files.get(target, f"{kind} => {target}") + elif entry.isdir(): + files[str(path)] = "dir" return files diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index 3b24587af..f31da5b84 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -134,8 +134,9 @@ def _arch(self, config_digest): def saved_image(path, layers): """A tar of an image as ``docker save`` writes one: each of ``layers``, bottom first, its history's ``created_by`` - (None for a layer with no entry) and its files by path, a file's content as bytes and a link's target as text, or - None for an entry that made no layer; the last layer gzipped, as some savers keep them.""" + (None for a layer with no entry) and its files by path, a file's content as bytes, or with its mode as (bytes, mode), + and a link's target as text, or None for an entry that made no layer; the last layer gzipped, as some savers keep + them.""" blobs, history, made = {}, [], [] for i, (created_by, files) in enumerate(layers): if created_by is not None: @@ -146,12 +147,13 @@ def saved_image(path, layers): with tarfile.open(fileobj=layer, mode="w") as tar: for name, content in files.items(): entry = tarfile.TarInfo(name) - if isinstance(content, bytes): - entry.size = len(content) - tar.addfile(entry, io.BytesIO(content)) - else: + if isinstance(content, str): entry.type, entry.linkname = tarfile.SYMTYPE, content tar.addfile(entry) + else: + data, entry.mode = content if isinstance(content, tuple) else (content, 0o644) + entry.size = len(data) + tar.addfile(entry, io.BytesIO(data)) data = gzip.compress(layer.getvalue()) if i == len(layers) - 1 else layer.getvalue() made.append(digest := f"blobs/sha256/{hashlib.sha256(data).hexdigest()}") blobs[digest] = data diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index c54a9d5b4..036793c0c 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -121,7 +121,8 @@ def test_what_each_run_wrote_is_read_from_the_newest_layer_a_shell_ran_its_comma ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", {"usr/local/bin/tool": b"the base's"}), ("USER root", None), ("RUN /bin/sh -c cd /opt/a && ./setup.sh # buildkit", { - "usr/local/bin/tool": b"a", "usr/bin/tool": "/usr/local/bin/tool", "etc/.wh.gone": b"", "./etc/a": b"a"}), + "usr/local/bin/tool": (b"a", 0o755), "usr/bin/tool": "/usr/local/bin/tool", "etc/.wh.gone": b"", + "./etc/a": b"a", "opt/shared/.wh..wh..opq": b""}), ("RUN /bin/sh -c cd /opt/a && ./setup.sh.extra # buildkit", {"etc/extra": b"extra"}), ("COPY apps/b/ /opt/b/ # buildkit", {"opt/b/x": b"x"}), ("RUN /bin/sh -c cd /opt/b && ./setup.sh # buildkit", {"usr/local/bin/tool": b"b"}), @@ -134,8 +135,10 @@ def test_what_each_run_wrote_is_read_from_the_newest_layer_a_shell_ran_its_comma sha = lambda data: hashlib.sha256(data).hexdigest() assert json.loads(ran.stdout) == layers_in_image.written(str(tmp_path / "image.tar"), commands) == [ - {"/usr/local/bin/tool": sha(b"a"), "/usr/bin/tool": "-> /usr/local/bin/tool", "/etc/a": sha(b"a")}, - {"/etc/extra": sha(b"extra")}, {"/usr/local/bin/tool": sha(b"b")}, {"/etc/again": sha(b"again")}] + {"/usr/local/bin/tool": f"exec {sha(b'a')}", "/usr/bin/tool": "link /usr/local/bin/tool", "/etc/gone": "deleted", + "/etc/a": f"file {sha(b'a')}", "/opt/shared": "opaque"}, + {"/etc/extra": f"file {sha(b'extra')}"}, {"/usr/local/bin/tool": f"file {sha(b'b')}"}, + {"/etc/again": f"file {sha(b'again')}"}] for missing in (["cd /opt/c && ./setup.sh"], ["cd /opt/b && ./setup.sh", "cd /opt/a && ./setup.sh"], [*commands, "cd /opt/b && ./setup.sh"]): with pytest.raises(ValueError, match="no layer of the image below the last one found was made by"): diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 0360167ab..1c56dca6d 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1122,32 +1122,56 @@ def _setting_up(tmp_path, *names): extra={"setup.sh": "#!/bin/sh\n"}) for name in names] -@pytest.mark.parametrize("mail, notes, clash", [ - ({"usr/local/bin/tool": b"mail's"}, {"usr/local/bin/tool": b"notes'"}, - "/usr/local/bin/tool (@local/~/kit/mail and @local/~/kit/notes)"), +MAIL, NOTES = "@local/~/kit/mail", "@local/~/kit/notes" + + +@pytest.mark.parametrize("mail, notes, lost", [ + ({"usr/local/bin/tool": b"mail's"}, {"usr/local/bin/.wh.tool": b""}, f"/usr/local/bin/tool: {NOTES} deletes {MAIL}'s file"), + ({"opt/shared/data": b"mail's"}, {"opt/shared/.wh..wh..opq": b"", "opt/shared/other": b"notes'"}, + f"/opt/shared/data: {NOTES} deletes {MAIL}'s file"), ({"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}, - "/usr/bin/tool (@local/~/kit/mail and @local/~/kit/notes)"), - ({"usr/local/bin/tool": b"same"}, {"usr/local/bin/tool": b"same"}, None), - *[({path: b"mail's"}, {path: b"notes'"}, None) for path in ( - "var/log/setup.log", "tmp/x", "home/user/.cache/pip/x", "opt/x/__pycache__/m.pyc", "etc/passwd", "etc/group-", - "etc/ld.so.cache", "root/.bashrc", "home/user/.profile")], -], ids=["file", "link", "same", "log", "tmp", "cache", "pycache", "passwd", "group-backup", "ld-cache", "root-rc", - "home-profile"]) -def test_two_apps_whose_run_scripts_write_one_file_differently_fail_at_build_naming_it(base, driver, docker, tmp_path, - mail, notes, clash): + f"/usr/bin/tool: {NOTES} points {MAIL}'s link elsewhere"), + ({"usr/bin/tool": b"mail's"}, {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}, + f"/usr/bin/tool: {NOTES} makes {MAIL}'s file a link"), + ({"usr/local/bin/tool": (b"mail's", 0o755)}, {"usr/local/bin/tool": (b"notes'", 0o755)}, + f"/usr/local/bin/tool: {NOTES} replaces {MAIL}'s program"), +], ids=["deleted", "opaque", "link", "kind", "program"]) +def test_a_part_that_certainly_undoes_what_an_earlier_one_installed_fails_its_pairs_at_build_naming_it( + base, driver, docker, tmp_path, mail, notes, lost): docker.writes = {"apps/mail && ": mail, "apps/notes && ": notes} env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes")).id) - composed = env.metadata["composed"]["container"] assert sorted(docker.saved) == sorted(build["tag"] for build in docker.builds) - if clash is None: - assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} - else: - assert {arch: (r["status"], r["stage"], r["reason"]) for arch, r in composed.items()} == { - arch: ("failed", "build", "its parts write a file differently, so the later part's would hide the " - f"earlier's: {clash}") for arch in ("amd64", "arm64")} - assert docker.probed == [] and "computer" not in env.metadata + assert {arch: (r["status"], r["stage"], r["reason"]) for arch, r in env.metadata["composed"]["container"].items()} == { + arch: ("failed", "build", f"a part undoes what an earlier one installed: {lost}") for arch in ("amd64", "arm64")} + assert docker.probed == [] and "computer" not in env.metadata + + +@pytest.mark.parametrize("mail, notes, told", [ + ({"etc/passwd": b"root\nmail\n", "usr/share/icons/hicolor/icon-theme.cache": b"\x00mail"}, + {"etc/passwd": b"root\nmail\nnotes\n", "usr/share/icons/hicolor/icon-theme.cache": b"\x00mail notes"}, + f"/etc/passwd ({MAIL}, {NOTES}), /usr/share/icons/hicolor/icon-theme.cache ({MAIL}, {NOTES})"), + ({f"etc/conf.d/{i:02}": b"mail's" for i in range(12)}, {f"etc/conf.d/{i:02}": b"notes'" for i in range(12)}, + ", ".join(f"/etc/conf.d/{i:02} ({MAIL}, {NOTES})" for i in range(10)) + " and 2 more"), + ({"usr/local/bin/tool": (b"same", 0o755), "usr/bin/tool": "/opt/tool", "etc/x": b"same"}, + {"usr/local/bin/tool": (b"same", 0o755), "usr/bin/tool": "/opt/tool", "etc/x": b"same"}, None), + ({path: b"mail's" for path in ("var/log/setup.log", "tmp/x", "run/x", "home/user/.cache/pip/x", "opt/x/__pycache__/m.pyc")}, + {"var/log/.wh.setup.log": b"", "tmp/x": b"notes'", "run/x": (b"notes'", 0o755), "home/user/.cache/pip/x": "/x", + "opt/x/__pycache__/m.pyc": b"notes'"}, None), + ({"etc/notes.conf": b"mail's"}, {}, None), +], ids=["edited", "many", "same", "scratch", "apart"]) +def test_files_two_parts_wrote_differently_but_not_certainly_lost_are_said_and_not_refused(base, driver, docker, tmp_path, + mail, notes, told): + docker.writes, said = {"apps/mail && ": mail, "apps/notes && ": notes}, [] + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), + composing=Composing(on_progress=said.append)).id) + + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + compared = [line.partition(": ")[2] for line in said if line.startswith("container/arm64: not refused")] + assert compared == ([f"not refused, as each may keep the earlier part's change: two parts wrote {told}"] + if told else []) def test_one_part_with_run_scripts_has_nothing_to_compare_and_its_image_isnt_saved(base, driver, docker, tmp_path): @@ -1161,13 +1185,14 @@ def test_one_part_with_run_scripts_has_nothing_to_compare_and_its_image_isnt_sav def test_a_builder_vm_compares_what_each_part_wrote_in_a_container_of_the_image_before_pushing(base, driver, docker, signing, tmp_path): - docker.writes = {"apps/mail && ": {"usr/local/bin/tool": b"mail's"}, "apps/notes && ": {"usr/local/bin/tool": b"notes'"}} + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ": {"usr/local/bin/tool": (b"notes'", 0o755)}} vms = FakeVmProvider(docker, tmp_path / "vms") env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), composing=Composing(vms)).id) - assert {arch: (r["status"], r["reason"].rpartition(": ")[2]) for arch, r in env.metadata["composed"]["container"].items()} == { - arch: ("failed", "/usr/local/bin/tool (@local/~/kit/mail and @local/~/kit/notes)") for arch in ("amd64", "arm64")} + assert {arch: (r["status"], r["reason"].partition(": ")[2]) for arch, r in env.metadata["composed"]["container"].items()} == { + arch: ("failed", f"/usr/local/bin/tool: {NOTES} replaces {MAIL}'s program") for arch in ("amd64", "arm64")} compared = [line for line in vms.ran if "/layers.py" in line] assert len(compared) == 2 and all("docker run --rm --network none --user root" in line for line in compared) assert not [line for line in vms.ran if "docker push" in line] and all(vm.terminated for vm in vms.vms) From 21121d286040a9ebfbe4dbf53b6e51653cfb039e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 19:40:09 -0700 Subject: [PATCH 07/14] refactor(computer): key images on what they're made of, not the composer's source An image's key covered composer.py's source, so any edit to it, a progress line or a check, built every computer env again. The key now covers the base, the Dockerfile, the build files with their executable bits (the composer alone sets them), the lock, the form and the arch, and composer.FORMAT, which replaces builder.FORMAT and is raised only for a change that alters what an image holds some other way, its labels or how it's built. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 4 ---- .../env/envs/computer/utils/composer.py | 18 ++++++++++-------- tst/unit/env/envs/test_computer_env.py | 18 ++++++++++++++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index f033a8852..4904407ed 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -43,10 +43,6 @@ logger = logging.getLogger(__name__) -#: What the builders make of a build context, which the build key covers in place of their source: raise it with a -#: change that alters what an image holds (its files, their modes, its labels, how it's built), and leave it for one -#: that changes only how a build gets there, so images aren't built again for nothing. -FORMAT = 1 CONTEXT, RESOLVED = "context", "resolve" RESOLVER = Path(__file__).with_name("resolve_in_base.py") LAYERS = Path(__file__).with_name("layers_in_image.py") diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 24652c4de..c9d31312c 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -61,6 +61,11 @@ #: The image store repository composed computers are pushed to, tagged by their keys. REPOSITORY = "agentenv-computers" +#: What the composer and its builders make of a build's inputs beyond what its key shows of them (the base, the +#: Dockerfile, the build files with their executable bits, the lock): raise it with a change that alters what an image +#: holds some other way, its labels or how it's built, and leave it for any other, so images aren't built again for +#: nothing. +FORMAT = 1 #: Where a driver's files go in a composed computer, in a folder named for the driver; an app's, under APPS. DRIVERS = "/opt/agentenv/drivers" APPS = str(installed_apps.APPS) @@ -242,7 +247,7 @@ def on() -> builder.Builder: key = "sha256:" + hashlib.sha256(json.dumps({ "base": base_identity, "dockerfile": dockerfile, "context": _digest(context), "lock": lock.digest() if lock is not None else None, "form": form, "arch": arch, - "composer": _composer_digest(), + "format": FORMAT, }, sort_keys=True).encode()).hexdigest() store = get_config().get_image_store_for(env.stored_id) store.ensure_repository(REPOSITORY) @@ -603,18 +608,15 @@ def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str]) - def _digest(folder: Path) -> str: - """A digest of the files under ``folder``, by their paths and bytes.""" + """A digest of the files under ``folder``, by their paths, bytes and whether each is executable, which the composer + alone decides (umask never sets it).""" hashed = hashlib.sha256() for path in sorted(p for p in folder.rglob("*") if p.is_file()): - hashed.update(path.relative_to(folder).as_posix().encode() + b"\0" + hashlib.sha256(path.read_bytes()).digest()) + hashed.update(path.relative_to(folder).as_posix().encode() + b"\0" + hashlib.sha256(path.read_bytes()).digest() + + (b"x" if path.stat().st_mode & 0o100 else b"-")) return "sha256:" + hashed.hexdigest() -def _composer_digest() -> str: - """A digest of the composer, which says what a build makes of its inputs, and of its builders' ``FORMAT``.""" - return "sha256:" + hashlib.sha256(Path(__file__).read_bytes() + f"builder {builder.FORMAT}".encode()).hexdigest() - - def _now() -> str: return datetime.now(timezone.utc).isoformat(timespec="seconds") diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 1c56dca6d..c4c7d0d55 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -107,6 +107,24 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ assert ComputerEnv.from_dict(stored.to_dict()).to_dict() == stored.to_dict() +def test_a_key_covers_its_build_files_executable_bits_and_the_composers_format(base, driver, docker, tmp_path, + monkeypatch): + folder = tmp_path / "ctx" + folder.mkdir() + (folder / "run").write_bytes(b"echo\n") + plain = composer._digest(folder) + (folder / "run").chmod(0o755) + runnable = composer._digest(folder) + (folder / "run").chmod(0o640) + + first = Env.get(_put(base, driver).id).metadata["composed"]["container"]["arm64"]["key"] + monkeypatch.setattr(composer, "FORMAT", composer.FORMAT + 1) + raised = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id).metadata["composed"][ + "container"]["arm64"]["key"] + + assert runnable != plain == composer._digest(folder) and raised != first + + def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one_another_env_built(base, driver, docker, monkeypatch): auth, docker_calls = RegistryAuth("localhost:5000", "user", "token"), [] From 77d7728d8f52f2878c93608e0d50a0ca2c47b5db Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 19:42:57 -0700 Subject: [PATCH 08/14] feat(computer): name an app's folder, programs and conf by its environment name An app was installed in a folder named for its id's last part, so two apps whose ids end alike (@a/mail, @b/mail) couldn't share a computer, and a bundle prefix made a folder's name differ from the name everything else uses. Its folder under /opt/agentenv/apps, its supervisord programs and its conf file now take its environment name, which a computer already requires to be unique; a driver keeps its id's. Gate 1 names each pair of clashing apps once, for the first thing they'd share. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/composer.py | 4 +-- src/agent_env/env/envs/computer/utils/gate.py | 30 ++++++++++++------- tst/unit/env/envs/test_computer_env.py | 8 ++--- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index c9d31312c..54f737daa 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -345,8 +345,8 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the app's install files under ``context/apps//``, with the values of its build settings the env fixes, and its supervisord programs beside them, run under its name: its services and its own server, which serves - its class from its venv and launches the app. Its folder under ``APPS`` is named for its id's last part.""" - install, name = app.install[base.os.type], gate.folder_name(app.id, "app") + its class from its venv and launches the app. Its folder under ``APPS`` is named for its environment name.""" + install, name = app.install[base.os.type], gate.app_folder(app) file, _, cls = app.module.partition(":") served = shlex.quote(f"{PurePosixPath(file).relative_to(install.files)}:{cls}") layer = _Layer(f"apps/{name}", f"{APPS}/{name}", install, f"agentenv-app-{name}.conf", app.id, app=True, diff --git a/src/agent_env/env/envs/computer/utils/gate.py b/src/agent_env/env/envs/computer/utils/gate.py index 5668fd844..5ec5cf8f2 100644 --- a/src/agent_env/env/envs/computer/utils/gate.py +++ b/src/agent_env/env/envs/computer/utils/gate.py @@ -69,37 +69,45 @@ def check_app(base: ComputerArtifact, driver: ComputerDriverEnv, app: ComputerAp return problems -def folder_name(env_id: str, fallback: str) -> str: - """The name of the folder a driver or an app is installed in: its id's last part, lowercased, as a slug.""" - return re.sub(r"[^a-z0-9-]+", "-", env_id.rsplit("/", 1)[-1].lower()).strip("-") or fallback +def folder_name(name: str, fallback: str) -> str: + """``name``'s last part, lowercased, as a slug: the name of a folder a part is installed in.""" + return re.sub(r"[^a-z0-9-]+", "-", name.rsplit("/", 1)[-1].lower()).strip("-") or fallback + + +def app_folder(app: ComputerAppEnv) -> str: + """The name of the folder ``app`` is installed in, under ``APPS``, and of its programs: its environment name's, the + one name a computer knows it by.""" + return folder_name(app.environment_name, "app") def _apps(base: ComputerArtifact, driver: ComputerDriverEnv, apps: list[ComputerAppEnv]) -> list[str]: """Every way ``apps`` don't fit together on ``base`` and ``driver``: each one's own (``check_app``), no architecture - they all run, one listed twice, and two in one folder, under one environment name, with one program name or - listening on one port.""" + they all run, one listed twice, and two under one environment name, in one folder, with one program name or + listening on one port, each pair of apps named for the first they'd share.""" problems = [problem for i, app in enumerate(apps) for problem in check_app(base, driver, app, f"apps[{i}]")] if not problems and apps and not arches(base, driver, apps): problems.append(f"apps: no architecture {base.id}, {driver.id} and every app all run") ids = [app.id for app in apps] problems += [f"apps: {app_id} is listed twice" for app_id in sorted({i for i in ids if ids.count(i) > 1})] - distinct = list({app.id: app for app in apps}.values()) - for kind, keys in (("folder", lambda app: [f"{APPS}/{folder_name(app.id, 'app')}"]), - ("environment name", lambda app: [app.environment_name]), + distinct, named = list({app.id: app for app in apps}.values()), set() + for kind, keys in (("environment name", lambda app: [app.environment_name]), + ("folder", lambda app: [f"{APPS}/{app_folder(app)}"]), ("program", lambda app: programs(app, base.os.type)), ("port", lambda app: _ports(app, base.os.type))): claims: dict[object, list[str]] = {} for app in distinct: for key in keys(app): claims.setdefault(key, []).append(app.id) - problems += [f"apps: {' and '.join(owners)} would share the {kind} {key!r}" for key, owners in claims.items() - if len(owners) > 1] + for key, owners in claims.items(): + if len(owners) > 1 and frozenset(owners) not in named: + named.add(frozenset(owners)) + problems.append(f"apps: {' and '.join(owners)} would share the {kind} {key!r}") return problems def programs(app: ComputerAppEnv, os_type: str) -> list[str]: """The names of the supervisord programs ``app`` runs on ``os_type``: its own server's, then its services'.""" - server = f"app-{folder_name(app.id, 'app')}" + server = f"app-{app_folder(app)}" install = app.install.get(os_type) return [server, *(f"{server}-{service.name}" for service in (install.services if install is not None else []))] diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index c4c7d0d55..6e998b6c8 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1072,7 +1072,7 @@ def test_build_settings_an_app_doesnt_declare_or_its_schema_refuses_are_refused_ def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_them(base, driver, docker, tmp_path): mail = _app(base, driver, tmp_path / "mail", compose=False) - notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", + notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", id="@local/~/other/mail", install={"ports": [8025], "services": [{"name": "sync", "command": "venv/bin/python sync.py"}]}) env = _put(base, driver, apps=iter([notes, mail])) @@ -1097,8 +1097,8 @@ def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_t (lambda mail, other: [mail, mail], "apps: @local/~/kit/mail is listed twice"), (lambda mail, other: [mail, other(name="mail-2", environment_name="mail")], "apps: @local/~/kit/mail and @local/~/kit/mail-2 would share the environment name 'mail'"), - (lambda mail, other: [mail, other(id="@local/other/mail")], - "apps: @local/~/kit/mail and @local/other/mail would share the folder '/opt/agentenv/apps/mail'"), + (lambda mail, other: [mail, other(name="x", environment_name="Mail")], + "apps: @local/~/kit/mail and @local/~/kit/x would share the folder '/opt/agentenv/apps/mail'"), (lambda mail, other: [other(name="a", install={"ports": [8025], "services": []}), other(name="b", install={"ports": [8025, 9000], "services": []})], "apps: @local/~/kit/a and @local/~/kit/b would share the port 8025"), @@ -1124,7 +1124,7 @@ def other(name="other", **changes): with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: _put(base, driver, apps=listed) - assert problem in str(refused.value) + assert problem in str(refused.value) and str(refused.value).count(" would share ") <= 1 with pytest.raises(NotFoundError): Env.get("@local/~/kit/desktop") assert docker.builds == [] From 577d243022b4fddc1b2e97116dba5ec38e16ee39 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 19:56:34 -0700 Subject: [PATCH 09/14] test(computer): a builder VM reads what each RUN wrote in a container of an image it emulates Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/builder_modal_vm_test.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tst/integration/computer/builder_modal_vm_test.py b/tst/integration/computer/builder_modal_vm_test.py index 6061270af..e66ed9f08 100644 --- a/tst/integration/computer/builder_modal_vm_test.py +++ b/tst/integration/computer/builder_modal_vm_test.py @@ -1,7 +1,8 @@ """A builder VM on Modal, amd64, building for arm64 through the emulation it registers: the build context sent inline, an apt package and a requirement resolved in a container of the base, a file it wrote stored off the VM, an image built -from them with the network closed, and the packages it installed read back, each in the VM. The stores stay local, -which a compose on Modal refuses, so the builder is driven directly.""" +from them with the network closed, the packages it installed read back, and what each of its RUNs wrote read from its +layers in a container of the image, under emulation too, each in the VM. The stores stay local, which a compose on Modal +refuses, so the builder is driven directly.""" import hashlib from pathlib import Path @@ -25,6 +26,9 @@ /tmp/packages/wheels/*.whl && /opt/driver/venv/bin/pip list --format=json > /opt/driver/installed.json && \ /opt/driver/start && jq --version """ +#: Each RUN's command, as the image's history records it once the Dockerfile's continued lines are joined. +RUNS = [line.removeprefix("RUN --network=none ") for line in DOCKERFILE.replace("\\\n", "").splitlines() + if line.startswith("RUN ")] def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): @@ -50,6 +54,7 @@ def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): f"{builder.RESOLVED}/out/wheels/driver/{wheel}"}, {}) vm.build(DOCKERFILE, "agentenv-builder-test:arm64", "linux/arm64", {"dev.agentenv.test": "builder"}) installed = vm.installed("agentenv-builder-test:arm64", "/opt/driver") + packaged, venv = vm.written("agentenv-builder-test:arm64", RUNS) finally: vm.close() @@ -58,3 +63,5 @@ def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): assert {deb["arch"] for deb in result["apt"]} == {"arm64"} assert hashlib.sha256(stored.read_bytes()).hexdigest() == written[f"wheels/driver/{wheel}"].sha256 assert vm.sandbox.platform == "linux/amd64" and Path(stored).stat().st_size == written[f"wheels/driver/{wheel}"].size + assert packaged["/usr/bin/jq"].startswith("exec ") and venv["/opt/driver/installed.json"].startswith("file ") + assert venv["/opt/driver/venv/bin/python"].startswith("link ") and "/opt/driver/start" not in venv From 027c80885fef39da33ca2f2976cccdafa7389a3a Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 20:07:10 -0700 Subject: [PATCH 10/14] fix(computer): check each later command against every earlier part's write; skip files already deleted under an opaque folder A path's owner passed to whichever part wrote it last, so a part that first copied identical bytes over an earlier part's program could then delete or replace it unchecked. Every part that wrote a path is now kept, and each later command is checked against the last other part to write it. An opaque folder no longer counts files its earlier part had already deleted, and marks what it hides as deleted. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/composer.py | 30 +++++++++++----- tst/unit/env/envs/test_computer_env.py | 34 ++++++++++++++++--- 2 files changed, 51 insertions(+), 13 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 54f737daa..116f02722 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -491,8 +491,9 @@ def _runs(layer: _Layer) -> list[tuple[str, str]]: def _clashes(runs: dict[str, list[tuple[str, str]]], written: list[dict[str, str]]) -> tuple[list[str], list[str]]: """What a part's ``runs`` did to what an earlier part's left, as ``written`` lists what each of their commands wrote (``layers_in_image``), in order, but under ``SCRATCH``: first each certain loss (``_lost``), then each other file - the two wrote differently, which may be an edit that kept the earlier part's change.""" - found, latest, lost, differ = iter(written), {}, [], [] + the two wrote differently, which may be an edit that kept the earlier part's change. A path stays an earlier part's + for every later command, whoever wrote it since, until it's deleted.""" + found, entries, writers, lost, differ = iter(written), {}, {}, [], [] for of, commands in runs.items(): for _ in commands: wrote = next(found) @@ -500,18 +501,29 @@ def _clashes(runs: dict[str, list[tuple[str, str]]], written: list[dict[str, str if SCRATCH.search(path): continue if entry == "opaque": - lost += [f"{hidden}: {of} deletes {by}'s file" for hidden, (by, was) in sorted(latest.items()) - if hidden.startswith(f"{path}/") and by != of and hidden not in wrote and was != "dir"] + for hidden in [hidden for hidden in sorted(entries) if hidden.startswith(f"{path}/") + and hidden not in wrote]: + if entries[hidden] not in ("dir", "deleted") and (by := _other(writers[hidden], of)): + lost.append(f"{hidden}: {of} deletes {by}'s file") + entries[hidden] = "deleted" entry = "dir" - if path in latest and (earlier := latest[path])[0] != of: - if reason := _lost(path, of, entry, *earlier): + if path in entries and (by := _other(writers[path], of)): + if reason := _lost(path, of, entry, by, entries[path]): lost.append(reason) - elif _differs(entry, earlier[1]): - differ.append(f"{path} ({earlier[0]}, {of})") - latest[path] = (of, entry) + elif _differs(entry, entries[path]): + differ.append(f"{path} ({by}, {of})") + entries[path] = entry + writers.setdefault(path, []) + if of not in writers[path]: + writers[path].append(of) return lost, differ +def _other(writers: list[str], of: str) -> str | None: + """The part of ``writers``, in the order they first wrote a path, that wrote it last before ``of``.""" + return next((writer for writer in reversed(writers) if writer != of), None) + + def _lost(path: str, of: str, entry: str, by: str, was: str) -> str | None: """How ``of``'s ``entry`` at ``path`` certainly loses what ``by`` left there, ``was``: deleting it, making it another kind, pointing its link elsewhere, or replacing its program; None when it doesn't.""" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 6e998b6c8..c5abe15ff 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1134,10 +1134,11 @@ def other(name="other", **changes): SETUP = {"run": [{"script": "setup.sh"}], "services": []} -def _setting_up(tmp_path, *names): - """An app of each of ``names`` whose install runs setup.sh, put without a computer.""" - return [_app(None, None, tmp_path / name, compose=False, name=name, install=SETUP, - extra={"setup.sh": "#!/bin/sh\n"}) for name in names] +def _setting_up(tmp_path, *names, scripts=("setup.sh",)): + """An app of each of ``names`` whose install runs ``scripts`` as root, put without a computer.""" + return [_app(None, None, tmp_path / name, compose=False, name=name, + install={"run": [{"script": script} for script in scripts], "services": []}, + extra={script: "#!/bin/sh\n" for script in scripts}) for name in names] MAIL, NOTES = "@local/~/kit/mail", "@local/~/kit/notes" @@ -1166,6 +1167,31 @@ def test_a_part_that_certainly_undoes_what_an_earlier_one_installed_fails_its_pa assert docker.probed == [] and "computer" not in env.metadata +@pytest.mark.parametrize("writes, lost", [ + ({"apps/mail && ./setup.sh": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ./setup.sh": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ./again.sh": {"usr/local/bin/.wh.tool": b""}}, f"/usr/local/bin/tool: {NOTES} deletes {MAIL}'s file"), + ({"apps/mail && ./setup.sh": {"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, + "apps/notes && ./setup.sh": {"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, + "apps/notes && ./again.sh": {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}}, + f"/usr/bin/tool: {NOTES} points {MAIL}'s link elsewhere"), + ({"apps/mail && ./setup.sh": {"opt/shared/data": b"mail's"}, "apps/mail && ./again.sh": {"opt/shared/.wh.data": b""}, + "apps/notes && ./setup.sh": {"opt/shared/.wh..wh..opq": b"", "opt/shared/other": b"notes'"}}, None), + ({"apps/mail && ./setup.sh": {"opt/shared/data": b"mail's"}, + "apps/notes && ./setup.sh": {"opt/shared/.wh..wh..opq": b"", "opt/shared/other": b"notes'"}, + "apps/notes && ./again.sh": {"opt/shared/data": b"notes'"}}, f"/opt/shared/data: {NOTES} deletes {MAIL}'s file"), +], ids=["kept-then-deleted", "kept-then-pointed", "deleted-before-opaque", "opaque-then-written"]) +def test_each_command_of_a_part_is_checked_against_what_earlier_parts_left_whoever_wrote_it_since( + base, driver, docker, tmp_path, writes, lost): + docker.writes = writes + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes", scripts=("setup.sh", "again.sh"))).id) + + composed = env.metadata["composed"]["container"].values() + assert ({(r["status"], r["reason"]) for r in composed} == {("failed", f"a part undoes what an earlier one installed: {lost}")} + if lost else env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"}) + + @pytest.mark.parametrize("mail, notes, told", [ ({"etc/passwd": b"root\nmail\n", "usr/share/icons/hicolor/icon-theme.cache": b"\x00mail"}, {"etc/passwd": b"root\nmail\nnotes\n", "usr/share/icons/hicolor/icon-theme.cache": b"\x00mail notes"}, From 4a100a3fa6655a2f72f4517b6467db7afc592c07 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 31 Oct 2026 00:17:15 -0700 Subject: [PATCH 11/14] refactor(computer): clash rules in their own module; a later write after a deletion starts the path afresh - utils/clashes.py holds what the composer refuses or says about two parts' writes: the scratch paths, each layer entry parsed once into an Entry, and compare() over (part, what one command wrote) in build order. The composer's _compare reads the layers and words the result, so _built reads build, compare, push, probe. - A deleted path, an opaque folder's hidden one, or one a loss was recorded on forgets its earlier writers, so a part that recreates and then deletes its own file is no longer blamed for an earlier part's. - _Layer.of is env_id. ComputerEnv.from_toml reads its apps in _listed_apps, which shares from_dict's type check. - Several apps' tests move to test_computer_apps.py, the clash rules as direct tables of compare(); the shared mail app helper and its constants move to _computer.py. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/env.py | 37 ++- .../env/envs/computer/utils/clashes.py | 82 +++++++ .../env/envs/computer/utils/composer.py | 105 ++------- tst/unit/env/envs/_computer.py | 46 +++- tst/unit/env/envs/test_computer_apps.py | 191 +++++++++++++++ tst/unit/env/envs/test_computer_env.py | 222 +----------------- 6 files changed, 374 insertions(+), 309 deletions(-) create mode 100644 src/agent_env/env/envs/computer/utils/clashes.py create mode 100644 tst/unit/env/envs/test_computer_apps.py diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 1adfc3985..5881e0308 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -118,15 +118,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: ctx.refuse([*refused, *missing]) base, driver = ctx.artifact(fields.pop("computer"), ComputerArtifact), ctx.env(fields.pop("driver"), ComputerDriverEnv) - refs = fields.get("apps", []) - fields["apps"] = [ctx.env({key: value for key, value in ref.items() if key != "install"} - if isinstance(ref, dict) else ref) for ref in refs] - if problems := [*(ctx.config_problem(f"apps[{i}]: install = {ref['install']!r}: an app installs at \"build\", " - "into the computer's image; \"boot\" isn't supported yet") - for i, ref in enumerate(refs) if isinstance(ref, dict) and ref.get("install", "build") != "build"), - *(ctx.config_problem(f"apps[{i}]: {app.id} is a {app.type} env, not a {APP_TYPE} env") - for i, app in enumerate(fields["apps"]) if app.type != APP_TYPE)]: - ctx.refuse(problems) + fields["apps"] = _listed_apps(fields.get("apps", []), ctx) try: return cls.put(id=ctx.id, computer=base, driver=driver, environment_name=fields.pop("environment_name", ctx.name), composing=ctx.composing, @@ -239,6 +231,29 @@ async def close(self) -> None: def _app(env: Env) -> ComputerAppEnv: - if env.type != APP_TYPE: - raise TypeError(f"{env.id} is a {env.type} env, not a {APP_TYPE} env") + if refusal := _not_an_app(env): + raise TypeError(refusal) return env + + +def _listed_apps(refs: list, ctx: AuthoringContext) -> list[ComputerAppEnv]: + """The computer app envs an env.toml's ``apps`` names, each a ref, or a table of one with its ``install`` beside it; + refused, naming each problem, unless every one is a computer app env installed at build.""" + apps, problems = [], [] + for i, ref in enumerate(refs): + install = ref.get("install", "build") if isinstance(ref, dict) else "build" + if install != "build": + problems.append(ctx.config_problem(f"apps[{i}]: install = {install!r}: an app installs at \"build\", into " + "the computer's image; \"boot\" isn't supported yet")) + continue + app = ctx.env({key: value for key, value in ref.items() if key != "install"} if isinstance(ref, dict) else ref) + if refusal := _not_an_app(app): + problems.append(ctx.config_problem(f"apps[{i}]: {refusal}")) + apps.append(app) + if problems: + ctx.refuse(problems) + return apps + + +def _not_an_app(env: Env) -> str | None: + return f"{env.id} is a {env.type} env, not a {APP_TYPE} env" if env.type != APP_TYPE else None diff --git a/src/agent_env/env/envs/computer/utils/clashes.py b/src/agent_env/env/envs/computer/utils/clashes.py new file mode 100644 index 000000000..0ff1d0744 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/clashes.py @@ -0,0 +1,82 @@ +"""Whether a part of a composed computer, its driver or an app, undid what an earlier part installed: what each part's +run scripts and install steps wrote, as ``layers_in_image`` lists it per layer, compared in build order. Only certain +losses are refused; any other file two parts wrote differently is only said, since it may be an edit that kept the +earlier part's change.""" + +from __future__ import annotations + +import re +from typing import NamedTuple + +#: Where what a run script or an install step writes may be lost, as the filesystem hierarchy standard has it: +#: temporary files, runtime state, logs and caches. Never compared. +SCRATCH = re.compile(r"^/(tmp|var/tmp|var/log|var/cache|run)/|/\.cache/|/__pycache__/") +#: What each kind of entry is, to a reader: a program is a file. +_KINDS = {"file": "file", "exec": "file", "link": "link", "dir": "folder"} + + +class Entry(NamedTuple): + """A path of a layer, as ``layers_in_image`` lists it: its kind, and a file's or a program's sha256 or a link's + target.""" + + kind: str + what: str = "" + + @classmethod + def listed(cls, entry: str) -> Entry: + kind, _, what = entry.partition(" ") + return cls(kind, what) + + +def compare(layers: list[tuple[str, dict[str, str]]]) -> tuple[list[str], list[str]]: + """What each of ``layers``, a part's id and what one of its commands wrote, in build order, did to what another part + left, but under ``SCRATCH``: each certain loss (``_lost``), then each other file two parts wrote differently. A path + is compared with the last other part to write it, whoever wrote it since, until it's deleted or lost.""" + entries: dict[str, Entry] = {} + writers: dict[str, list[str]] = {} + lost, differ = [], [] + for of, wrote in layers: + listed = {path: Entry.listed(entry) for path, entry in wrote.items() if not SCRATCH.search(path)} + for path, entry in sorted(listed.items()): + if entry.kind == "opaque": + for hidden in [hidden for hidden in sorted(entries) if hidden.startswith(f"{path}/") + and hidden not in listed]: + if entries[hidden].kind not in ("dir", "deleted") and (by := _other(writers[hidden], of)): + lost.append(f"{hidden}: {of} deletes {by}'s file") + entries[hidden], writers[hidden] = Entry("deleted"), [] + entry = Entry("dir") + reason = None + if path in entries and (by := _other(writers[path], of)): + if reason := _lost(path, of, entry, by, entries[path]): + lost.append(reason) + elif _differs(entry, entries[path]): + differ.append(f"{path} ({by}, {of})") + entries[path] = entry + writers[path] = ([] if entry.kind == "deleted" else + [of] if reason else [*(writer for writer in writers.get(path, []) if writer != of), of]) + return lost, differ + + +def _other(writers: list[str], of: str) -> str | None: + """The part of ``writers``, oldest write first, that wrote a path last before ``of``.""" + return next((writer for writer in reversed(writers) if writer != of), None) + + +def _lost(path: str, of: str, entry: Entry, by: str, was: Entry) -> str | None: + """How ``of``'s ``entry`` at ``path`` certainly loses what ``by`` left there, ``was``: deleting it, making it + another kind, pointing its link elsewhere, or replacing its program; None when it doesn't.""" + if was.kind == "deleted": + return None + if entry.kind == "deleted": + return f"{path}: {of} deletes {by}'s {_KINDS[was.kind]}" + if _KINDS[entry.kind] != _KINDS[was.kind]: + return f"{path}: {of} makes {by}'s {_KINDS[was.kind]} a {_KINDS[entry.kind]}" + if was.kind == "link" and entry.what != was.what: + return f"{path}: {of} points {by}'s link elsewhere" + if was.kind == "exec" and entry.what != was.what: + return f"{path}: {of} replaces {by}'s program" + return None + + +def _differs(entry: Entry, was: Entry) -> bool: + return entry.kind in ("file", "exec") and was.kind == "file" and entry.what != was.what diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 116f02722..a1526239e 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -13,7 +13,6 @@ import importlib.metadata import json import logging -import re import shlex import shutil import subprocess @@ -39,7 +38,7 @@ from agent_env.config import get_config from agent_env.entity.export import Unwritable, checked_path from agent_env.env.env import Env -from agent_env.env.envs.computer.utils import boot, builder, gate, packages +from agent_env.env.envs.computer.utils import boot, builder, clashes, gate, packages from agent_env.env.envs.computer.utils.install import Install from agent_env.providers.sandbox_providers.host_images import platform_tag from agent_env.store.base import ConcurrentModificationError @@ -84,13 +83,8 @@ _PROTOCOL = "agentenv-framework-protocol" # What building the protocol from its source reads, beside src/. _PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") -#: Where what a run script or an install step writes may be lost, as the filesystem hierarchy standard has it: -#: temporary files, runtime state, logs and caches. Never compared. -SCRATCH = re.compile(r"^/(tmp|var/tmp|var/log|var/cache|run)/|/\.cache/|/__pycache__/") #: How many of the files two parts wrote differently a progress line names. _SHOWN = 10 -#: What each kind of entry ``layers_in_image`` lists is, to a reader: a program is a file. -_KINDS = {"file": "file", "exec": "file", "link": "link", "dir": "folder"} @dataclass(frozen=True) @@ -264,17 +258,8 @@ def on() -> builder.Builder: packages.stage(lock, building_on, env.stored_id, fetched) local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) - if len(runs := {part.of: _runs(part) for part in layers if _runs(part)}) > 1: - progress.at(pair, "comparing the files its parts' run scripts and install steps wrote") - written = building_on.written(local, [command for commands in runs.values() for _, command in commands]) - lost, differ = _clashes(runs, written) - if lost: - return _failed(env, form, arch, stage, key, "a part undoes what an earlier one installed: " - + "; ".join(lost)) - if differ: - progress.at(pair, "compared", "not refused, as each may keep the earlier part's change: two parts " - f"wrote {', '.join(differ[:_SHOWN])}" + (f" and {len(differ) - _SHOWN} more" - if len(differ) > _SHOWN else "")) + if reason := _compare(building_on, local, layers, pair, progress): + return _failed(env, form, arch, stage, key, reason) installed = building_on.installed(local, layer.prefix) building_on.build(f"FROM {local}\n", local, platform, { INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", @@ -302,6 +287,26 @@ def on() -> builder.Builder: return _failed(env, form, arch, stage, key, f"{type(e).__name__}: {e}") +def _compare(building_on: builder.Builder, tag: str, layers: list[_Layer], pair: str, + progress: _Progress) -> str | None: + """Why ``tag`` is refused: a part's run scripts or install steps certainly undoing what an earlier part's installed + (``clashes``); None, with any other file two parts wrote differently said to ``progress``. Read only when two or + more parts have such steps, the only ones that write outside a part's folder.""" + runs = {layer.env_id: _runs(layer) for layer in layers if _runs(layer)} + if len(runs) < 2: + return None + progress.at(pair, "comparing the files its parts' run scripts and install steps wrote") + written = building_on.written(tag, [command for commands in runs.values() for _, command in commands]) + lost, differ = clashes.compare(list(zip([env_id for env_id, commands in runs.items() for _ in commands], written))) + if lost: + return "a part undoes what an earlier one installed: " + "; ".join(lost) + if differ: + progress.at(pair, "compared", "not refused, as each may keep the earlier part's change: two parts wrote " + f"{', '.join(differ[:_SHOWN])}" + (f" and {len(differ) - _SHOWN} more" if len(differ) > _SHOWN + else "")) + return None + + def _base(base: ComputerArtifact, arch: str) -> tuple[str, str]: """What the composed image is built ``FROM`` for ``arch``, and the base's part of the key, found without building anything: a registry image pinned to its digest, or the tag a build of the base's stored build context for that @@ -317,14 +322,14 @@ def _base(base: ComputerArtifact, arch: str) -> tuple[str, str]: @dataclass(frozen=True) class _Layer: """A driver's or an app's part of a build: its folder in the build context, its prefix in the image, what it - installs, and the file of its supervisord programs. An app's venv also has the protocol's ``app`` extra, for the + installs, the file of its supervisord programs, and the id of its env. An app's venv also has the protocol's ``app`` extra, for the app's own server, which listens in a folder of the app's the session user owns.""" folder: str prefix: str install: Install conf: str - of: str + env_id: str app: bool = False served: str | None = None steps: tuple[tuple[str, str], ...] = () @@ -488,66 +493,6 @@ def _runs(layer: _Layer) -> list[tuple[str, str]]: f"{layer.served} {step}") for step, scope in layer.steps)] -def _clashes(runs: dict[str, list[tuple[str, str]]], written: list[dict[str, str]]) -> tuple[list[str], list[str]]: - """What a part's ``runs`` did to what an earlier part's left, as ``written`` lists what each of their commands wrote - (``layers_in_image``), in order, but under ``SCRATCH``: first each certain loss (``_lost``), then each other file - the two wrote differently, which may be an edit that kept the earlier part's change. A path stays an earlier part's - for every later command, whoever wrote it since, until it's deleted.""" - found, entries, writers, lost, differ = iter(written), {}, {}, [], [] - for of, commands in runs.items(): - for _ in commands: - wrote = next(found) - for path, entry in sorted(wrote.items()): - if SCRATCH.search(path): - continue - if entry == "opaque": - for hidden in [hidden for hidden in sorted(entries) if hidden.startswith(f"{path}/") - and hidden not in wrote]: - if entries[hidden] not in ("dir", "deleted") and (by := _other(writers[hidden], of)): - lost.append(f"{hidden}: {of} deletes {by}'s file") - entries[hidden] = "deleted" - entry = "dir" - if path in entries and (by := _other(writers[path], of)): - if reason := _lost(path, of, entry, by, entries[path]): - lost.append(reason) - elif _differs(entry, entries[path]): - differ.append(f"{path} ({by}, {of})") - entries[path] = entry - writers.setdefault(path, []) - if of not in writers[path]: - writers[path].append(of) - return lost, differ - - -def _other(writers: list[str], of: str) -> str | None: - """The part of ``writers``, in the order they first wrote a path, that wrote it last before ``of``.""" - return next((writer for writer in reversed(writers) if writer != of), None) - - -def _lost(path: str, of: str, entry: str, by: str, was: str) -> str | None: - """How ``of``'s ``entry`` at ``path`` certainly loses what ``by`` left there, ``was``: deleting it, making it another - kind, pointing its link elsewhere, or replacing its program; None when it doesn't.""" - kind, _, what = entry.partition(" ") - kind_was, _, what_was = was.partition(" ") - if kind_was == "deleted": - return None - if kind == "deleted": - return f"{path}: {of} deletes {by}'s {_KINDS[kind_was]}" - if _KINDS[kind] != _KINDS[kind_was]: - return f"{path}: {of} makes {by}'s {_KINDS[kind_was]} a {_KINDS[kind]}" - if kind_was == "link" and what != what_was: - return f"{path}: {of} points {by}'s link elsewhere" - if kind_was == "exec" and what != what_was: - return f"{path}: {of} replaces {by}'s program" - return None - - -def _differs(entry: str, was: str) -> bool: - kind, _, what = entry.partition(" ") - kind_was, _, what_was = was.partition(" ") - return kind in ("file", "exec") and kind_was == "file" and what != what_was - - def _as(scope: str, user: str, run: str) -> list[str]: """``run`` as root, or for ``session``, as ``user``.""" return [run] if scope == "system" else [f"USER {user}", run, "USER root"] diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py index f31da5b84..06c305cad 100644 --- a/tst/unit/env/envs/_computer.py +++ b/tst/unit/env/envs/_computer.py @@ -11,13 +11,14 @@ import tarfile import urllib.parse -from agentenv_protocol.computer import DriverDeclaration +from agentenv_protocol.computer import AppDeclaration, DriverDeclaration from agentenv_protocol.computer.conformance import CheckResult, ProbeReport from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config from agent_env.env.envs.computer.utils import builder, layers_in_image +from agent_env.env.envs.computer.app_env import ComputerAppEnv from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.utils.install import Install, Service from agent_env.env.envs.computer.driver_env import ComputerDriverEnv @@ -303,3 +304,46 @@ async def create_vm(self, **size): self.sizes.append(size) self.vms.append(vm := FakeVm(self, self.root / str(len(self.vms)))) return vm + + +MAIL_SETTINGS = {"properties": { + "signature": {"default": "", "type": "string"}, + "password": {"default": "", "type": "string", "format": "password", "writeOnly": True, "minLength": 32}, + "token": {"anyOf": [{"type": "string", "format": "password", "writeOnly": True}, {"type": "null"}], "default": None}, + "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} + + +MAIL_BUILD = {"properties": {"edition": {"default": "home", "enum": ["home", "pro"], "type": "string"}, + "locale": {"default": "en_US", "type": "string"}}, "type": "object"} + + +#: The program of the mail app's own server. +MAIL_SERVER = ("[program:app-mail]\ncommand=/opt/agentenv/apps/mail/venv/bin/python -m " + "agentenv_protocol.computer.serve_app app.py:Mail\ndirectory=/opt/agentenv/apps/mail\n" + "environment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\nredirect_stderr=true\n" + "stdout_logfile=/var/log/agentenv/app-mail.log\n") + + +def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, + extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, name="mail", **changes): + """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside + them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of + its build settings, ``linux_settings``, that of its settings on Linux, and ``name``, its id's last part and its + environment name.""" + folder.mkdir(parents=True, exist_ok=True) + (folder / "app.py").write_text("class Mail: ...\n") + (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") + for path, text in (extra or {}).items(): + (folder / path).parent.mkdir(parents=True, exist_ok=True) + (folder / path).write_text(text) + files = FileArtifactUniverse.put_bundled(f"@local/~/kit/{name}__files", files={ + path: folder / path for path in ("app.py", "requirements.txt", *(extra or {}))}) + declaration = AppDeclaration.model_validate({ + "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], + "permissions": [], "settings": settings, "build_settings": build, "skill": None, + "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], + "ready_checks": [], "os": {"linux": {"launch": launch, "settings": linux_settings, "install": install or { + "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) + return ComputerAppEnv.put(**{"id": f"@local/~/kit/{name}", "environment_name": name, "module": "app.py:Mail", + "files": files, "declaration": declaration, "computer": base, "driver": driver, + "compose": compose, **changes}) diff --git a/tst/unit/env/envs/test_computer_apps.py b/tst/unit/env/envs/test_computer_apps.py new file mode 100644 index 000000000..3ea9898a8 --- /dev/null +++ b/tst/unit/env/envs/test_computer_apps.py @@ -0,0 +1,191 @@ +"""Computer envs of several apps: each app built into the computer after the driver, in the order the env lists them; +the apps checked against each other at put; and what their run scripts and install steps wrote compared before a push, +a part certainly undoing what an earlier one installed refused and any other file two parts wrote differently said +(``clashes``). Docker, the registry and the VMs are faked.""" + +import pytest + +from agent_env.env import Env +from agent_env.env.envs.computer.utils import clashes +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.store.base import NotFoundError +from tst.unit.env.envs._computer import MAIL_SERVER, FakeVmProvider, _app, _put + +MAIL, NOTES = "@local/~/kit/mail", "@local/~/kit/notes" +A, B, C = "@local/~/kit/a", "@local/~/kit/b", "@local/~/kit/c" + + +def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_them(base, driver, docker, tmp_path): + mail = _app(base, driver, tmp_path / "mail", compose=False) + notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", id="@local/~/other/mail", + install={"ports": [8025], "services": [{"name": "sync", "command": "venv/bin/python sync.py"}]}) + + env = _put(base, driver, apps=iter([notes, mail])) + + assert sorted(build["platform"] for build in docker.builds) == ["linux/amd64", "linux/arm64"] + dockerfile, confs = docker.builds[0]["dockerfile"], docker.builds[0]["confs"] + assert dockerfile.index("COPY driver/") < dockerfile.index("COPY apps/notes/") < dockerfile.index( + "COPY apps/mail/") < dockerfile.index("USER user") + assert [layer["name"] for layer in docker.resolved[0][1]["python"]][-2:] == ["apps-notes", "apps-mail"] + assert confs["agentenv-app-mail.conf"].startswith(MAIL_SERVER) and "[program:app-notes-sync]" in confs[ + "agentenv-app-notes.conf"] + stored = Env.get(env.id) + assert [(app.id, app.version) for app in stored.composed_apps()] == [(notes.id, 1), (mail.id, 1)] + assert stored.to_dict()["apps"] == [{"id": notes.id, "version": 1}, {"id": mail.id, "version": 1}] + assert stored.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + assert docker.saved == [] + assert _put(base, driver, apps=[mail, notes]).metadata["composed"]["container"]["arm64"]["key"] != ( + stored.metadata["composed"]["container"]["arm64"]["key"]) + + +@pytest.mark.parametrize("apps, problem", [ + (lambda mail, other: [mail, mail], "apps: @local/~/kit/mail is listed twice"), + (lambda mail, other: [mail, other(name="mail-2", environment_name="mail")], + "apps: @local/~/kit/mail and @local/~/kit/mail-2 would share the environment name 'mail'"), + (lambda mail, other: [mail, other(name="x", environment_name="Mail")], + "apps: @local/~/kit/mail and @local/~/kit/x would share the folder '/opt/agentenv/apps/mail'"), + (lambda mail, other: [other(name="a", install={"ports": [8025], "services": []}), + other(name="b", install={"ports": [8025, 9000], "services": []})], + "apps: @local/~/kit/a and @local/~/kit/b would share the port 8025"), + (lambda mail, other: [mail, other(name="desk-port", install={"ports": [18765], "services": []})], + "apps[1]: @local/~/kit/desk-port listens on 18765, as @local/~/kit/desk does"), + (lambda mail, other: [mail, other(name="arm", supports=[{"os": {"type": "linux", "display": ["x11"]}, + "arch": ["riscv64"]}])], + "apps[1]: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and @local/~/kit/arm all run"), + (lambda mail, other: [other(name=arch, supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": [arch]}]) + for arch in ("amd64", "arm64")], + "apps: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and every app all run"), + (lambda mail, other: [other(name="mail", install={"services": [{"name": "sync", "command": "venv/bin/sync"}]}), + other(name="mail-sync", install={"services": []})], + "apps: @local/~/kit/mail and @local/~/kit/mail-sync would share the program 'app-mail-sync'"), +], ids=["twice", "environment-name", "folder", "app-port", "driver-port", "arch", "no-shared-arch", "program"]) +def test_apps_that_dont_fit_together_or_on_the_computer_are_refused_before_anything_is_stored(base, driver, docker, + tmp_path, apps, problem): + def other(name="other", **changes): + return _app(None, None, tmp_path / name, compose=False, name=name, **changes) + + listed = apps(_app(base, driver, tmp_path / "mail", compose=False), other) + + with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: + _put(base, driver, apps=listed) + + assert problem in str(refused.value) and str(refused.value).count(" would share ") <= 1 + with pytest.raises(NotFoundError): + Env.get("@local/~/kit/desktop") + assert docker.builds == [] + + +def _setting_up(tmp_path, *names, scripts=("setup.sh",)): + """An app of each of ``names`` whose install runs ``scripts`` as root, put without a computer.""" + return [_app(None, None, tmp_path / name, compose=False, name=name, + install={"run": [{"script": script} for script in scripts], "services": []}, + extra={script: "#!/bin/sh\n" for script in scripts}) for name in names] + + +@pytest.mark.parametrize("layers, lost", [ + ([(A, {"/usr/local/bin/tool": "file 1"}), (B, {"/usr/local/bin/tool": "deleted"})], + [f"/usr/local/bin/tool: {B} deletes {A}'s file"]), + ([(A, {"/opt/shared/data": "file 1", "/opt/shared/sub": "dir"}), + (B, {"/opt/shared": "opaque", "/opt/shared/other": "file 2"})], + [f"/opt/shared/data: {B} deletes {A}'s file"]), + ([(A, {"/usr/bin/tool": "link /opt/a/tool"}), (B, {"/usr/bin/tool": "link /opt/b/tool"})], + [f"/usr/bin/tool: {B} points {A}'s link elsewhere"]), + ([(A, {"/usr/bin/tool": "file 1"}), (B, {"/usr/bin/tool": "link /opt/b/tool"})], + [f"/usr/bin/tool: {B} makes {A}'s file a link"]), + ([(A, {"/opt/x": "dir"}), (B, {"/opt/x": "file 1"})], [f"/opt/x: {B} makes {A}'s folder a file"]), + ([(A, {"/usr/local/bin/tool": "exec 1"}), (B, {"/usr/local/bin/tool": "exec 2"})], + [f"/usr/local/bin/tool: {B} replaces {A}'s program"]), + ([(A, {"/t": "exec 1"}), (B, {"/t": "exec 1"}), (B, {"/t": "deleted"})], [f"/t: {B} deletes {A}'s file"]), + ([(A, {"/t": "link /a"}), (B, {"/t": "link /a"}), (B, {"/t": "link /b"})], + [f"/t: {B} points {A}'s link elsewhere"]), + ([(A, {"/o/data": "file 1"}), (B, {"/o": "opaque"}), (B, {"/o/data": "file 2"})], + [f"/o/data: {B} deletes {A}'s file"]), + ([(A, {"/t": "exec 1"}), (B, {"/t": "exec 2"}), (C, {"/t": "deleted"})], + [f"/t: {B} replaces {A}'s program", f"/t: {C} deletes {B}'s file"]), +], ids=["deleted", "opaque", "link", "kind", "folder", "program", "kept-then-deleted", "kept-then-pointed", + "opaque-then-written", "after-a-loss"]) +def test_a_part_that_certainly_undoes_what_another_left_is_a_loss(layers, lost): + assert clashes.compare(layers) == (lost, []) + + +@pytest.mark.parametrize("layers, differ", [ + ([(A, {"/etc/passwd": "file 1"}), (B, {"/etc/passwd": "file 2"})], [f"/etc/passwd ({A}, {B})"]), + ([(A, {"/x": "file 1"}), (B, {"/x": "exec 2"})], [f"/x ({A}, {B})"]), + ([(A, {"/x": "exec 1", "/l": "link /a", "/d": "dir"}), (B, {"/x": "exec 1", "/l": "link /a", "/d": "dir"})], []), + ([(A, {path: "file 1" for path in ("/var/log/s.log", "/tmp/x", "/run/x", "/home/user/.cache/x", + "/o/__pycache__/m.pyc")}), + (B, {"/var/log/s.log": "deleted", "/tmp/x": "exec 2", "/run/x": "link /x", "/home/user/.cache/x": "dir", + "/o/__pycache__/m.pyc": "file 2"})], []), + ([(A, {"/x": "exec 1"}), (A, {"/x": "deleted"})], []), + ([(A, {"/o/d": "file 1"}), (A, {"/o/d": "deleted"}), (B, {"/o": "opaque"})], []), + ([(A, {"/t": "exec 1"}), (A, {"/t": "deleted"}), (B, {"/t": "exec 2"}), (B, {"/t": "deleted"})], []), + ([(A, {"/o/t": "exec 1"}), (A, {"/o": "opaque"}), (B, {"/o/t": "exec 2"}), (B, {"/o/t": "deleted"})], []), + ([(A, {"/a": "file 1"}), (B, {"/b": "file 2"})], []), +], ids=["edited", "made-a-program", "same", "scratch", "own", "deleted-before-opaque", "recreated", + "recreated-after-opaque", "apart"]) +def test_what_isnt_certainly_lost_isnt_a_loss_and_only_a_file_two_parts_wrote_differently_is_said(layers, differ): + assert clashes.compare(layers) == ([], differ) + + +def test_a_part_that_certainly_undoes_what_an_earlier_one_installed_fails_its_pairs_at_build_before_the_push( + base, driver, docker, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ": {"usr/local/bin/tool": (b"notes'", 0o755)}} + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes")).id) + + assert sorted(docker.saved) == sorted(build["tag"] for build in docker.builds) + composed = env.metadata["composed"]["container"] + assert {arch: (r["status"], r["stage"], r["reason"]) for arch, r in composed.items()} == { + arch: ("failed", "build", f"a part undoes what an earlier one installed: /usr/local/bin/tool: {NOTES} replaces " + f"{MAIL}'s program") for arch in ("amd64", "arm64")} + assert docker.probed == [] and "computer" not in env.metadata + + +@pytest.mark.parametrize("mail, notes, told", [ + ({"etc/passwd": b"root\\nmail\\n", "usr/share/icons/hicolor/icon-theme.cache": b"\\x00mail"}, + {"etc/passwd": b"root\\nmail\\nnotes\\n", "usr/share/icons/hicolor/icon-theme.cache": b"\\x00mail notes"}, + f"/etc/passwd ({MAIL}, {NOTES}), /usr/share/icons/hicolor/icon-theme.cache ({MAIL}, {NOTES})"), + ({f"etc/conf.d/{i:02}": b"mail's" for i in range(12)}, {f"etc/conf.d/{i:02}": b"notes'" for i in range(12)}, + ", ".join(f"/etc/conf.d/{i:02} ({MAIL}, {NOTES})" for i in range(10)) + " and 2 more"), + ({"etc/x": b"same"}, {"etc/x": b"same"}, None), +], ids=["edited", "many", "same"]) +def test_files_two_parts_wrote_differently_are_said_on_a_progress_line_and_not_refused(base, driver, docker, tmp_path, + mail, notes, told): + docker.writes, said = {"apps/mail && ": mail, "apps/notes && ": notes}, [] + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), + composing=Composing(on_progress=said.append)).id) + + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} + compared = [line.partition(": ")[2] for line in said if line.startswith("container/arm64: not refused")] + assert compared == ([f"not refused, as each may keep the earlier part's change: two parts wrote {told}"] + if told else []) + + +def test_one_part_with_run_scripts_has_nothing_to_compare_and_its_image_isnt_saved(base, driver, docker, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": b"mail's"}} + mail = _setting_up(tmp_path, "mail")[0] + + env = Env.get(_put(base, driver, apps=[mail, _app(None, None, tmp_path / "notes", compose=False, name="notes")]).id) + + assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} and docker.saved == [] + + +def test_a_builder_vm_compares_what_each_part_wrote_in_a_container_of_the_image_before_pushing(base, driver, docker, + signing, tmp_path): + docker.writes = {"apps/mail && ": {"usr/local/bin/tool": (b"mail's", 0o755)}, + "apps/notes && ": {"usr/local/bin/tool": (b"notes'", 0o755)}} + vms = FakeVmProvider(docker, tmp_path / "vms") + + env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), composing=Composing(vms)).id) + + composed = env.metadata["composed"]["container"] + assert {arch: (r["status"], r["reason"].partition(": ")[2]) for arch, r in composed.items()} == { + arch: ("failed", f"/usr/local/bin/tool: {NOTES} replaces {MAIL}'s program") for arch in ("amd64", "arm64")} + compared = [line for line in vms.ran if "/layers.py" in line] + assert len(compared) == 2 and all("docker run --rm --network none --user root" in line for line in compared) + assert not [line for line in vms.ran if "docker push" in line] and all(vm.terminated for vm in vms.vms) + assert not list((tmp_path / "vms").rglob("image.tar")) + + diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index c5abe15ff..67bc20a26 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1,7 +1,8 @@ """Computer envs: gate 1 refuses a base, a driver or a declaration that can't compose, naming every problem, before anything is stored; the composer builds each form and arch, reuses an image of the same key whichever env built it, records each result as it finishes, and writes the arches built as the env's computer; a deploy, a reset and a load; -and computer apps. Where and how a pair builds is test_computer_builder. Docker and the registry are faked.""" +and computer apps. Where and how a pair builds is test_computer_builder; a computer of several apps, test_computer_apps. +Docker and the registry are faked.""" import asyncio import contextlib @@ -18,7 +19,7 @@ import httpx import pytest -from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING, AppDeclaration +from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI from click.testing import CliRunner @@ -52,7 +53,8 @@ from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from tst.unit.computer_home import Computer -from tst.unit.env.envs._computer import CONFIG, SERVE, CARD_EXTENSION, CARD, _base, _driver, _put, FakeVmProvider +from tst.unit.env.envs._computer import (CONFIG, SERVE, CARD_EXTENSION, CARD, MAIL_SETTINGS, MAIL_BUILD, MAIL_SERVER, _app, + _base, _driver, _put, FakeVmProvider) def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_its_images_make(base, driver, docker, @@ -780,50 +782,9 @@ async def test_a_load_outside_the_home_is_refused_before_anything_is_sent(base, assert computer.calls == [] -MAIL_SETTINGS = {"properties": { - "signature": {"default": "", "type": "string"}, - "password": {"default": "", "type": "string", "format": "password", "writeOnly": True, "minLength": 32}, - "token": {"anyOf": [{"type": "string", "format": "password", "writeOnly": True}, {"type": "null"}], "default": None}, - "theme": {"enum": ["light", "dark"], "type": "string", "default": "light"}}, "type": "object"} - - -MAIL_BUILD = {"properties": {"edition": {"default": "home", "enum": ["home", "pro"], "type": "string"}, - "locale": {"default": "en_US", "type": "string"}}, "type": "object"} - - -#: The program of the mail app's own server. -MAIL_SERVER = ("[program:app-mail]\ncommand=/opt/agentenv/apps/mail/venv/bin/python -m " - "agentenv_protocol.computer.serve_app app.py:Mail\ndirectory=/opt/agentenv/apps/mail\n" - "environment=ENVIRONMENT_NAME=\"mail\"\nautorestart=true\nredirect_stderr=true\n" - "stdout_logfile=/var/log/agentenv/app-mail.log\n") PROTOCOL_VERSION = importlib.metadata.version("agentenv-framework-protocol") -def _app(base, driver, folder, compose=True, supports=None, install=None, settings=MAIL_SETTINGS, launch=None, - extra=None, steps=(), build=MAIL_BUILD, linux_settings=None, name="mail", **changes): - """A mail app on ``base`` and ``driver``, which serves IMAP from a venv of its requirements; ``extra`` files beside - them, by name, ``steps``, its install steps by name, scope and the OS types they run on, ``build``, the schema of - its build settings, ``linux_settings``, that of its settings on Linux, and ``name``, its id's last part and its - environment name.""" - folder.mkdir(parents=True, exist_ok=True) - (folder / "app.py").write_text("class Mail: ...\n") - (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") - for path, text in (extra or {}).items(): - (folder / path).parent.mkdir(parents=True, exist_ok=True) - (folder / path).write_text(text) - files = FileArtifactUniverse.put_bundled(f"@local/~/kit/{name}__files", files={ - path: folder / path for path in ("app.py", "requirements.txt", *(extra or {}))}) - declaration = AppDeclaration.model_validate({ - "supports": supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}], - "permissions": [], "settings": settings, "build_settings": build, "skill": None, - "data": [], "install_steps": [{"name": name, "scope": scope, "os": os or None} for name, scope, *os in steps], - "ready_checks": [], "os": {"linux": {"launch": launch, "settings": linux_settings, "install": install or { - "python": "requirements.txt", "services": [{"name": "imap", "command": "venv/bin/python imap.py"}]}}}}) - return ComputerAppEnv.put(**{"id": f"@local/~/kit/{name}", "environment_name": name, "module": "app.py:Mail", - "files": files, "declaration": declaration, "computer": base, "driver": driver, - "compose": compose, **changes}) - - def test_an_app_deployed_alone_is_composed_with_its_files_venv_and_programs_under_its_prefix(base, driver, docker, tmp_path): app = _app(base, driver, tmp_path / "mail") @@ -1070,179 +1031,6 @@ def test_build_settings_an_app_doesnt_declare_or_its_schema_refuses_are_refused_ Env.get("@local/~/kit/mail") -def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_them(base, driver, docker, tmp_path): - mail = _app(base, driver, tmp_path / "mail", compose=False) - notes = _app(base, driver, tmp_path / "notes", compose=False, name="notes", id="@local/~/other/mail", - install={"ports": [8025], "services": [{"name": "sync", "command": "venv/bin/python sync.py"}]}) - - env = _put(base, driver, apps=iter([notes, mail])) - - assert sorted(build["platform"] for build in docker.builds) == ["linux/amd64", "linux/arm64"] - dockerfile, confs = docker.builds[0]["dockerfile"], docker.builds[0]["confs"] - assert dockerfile.index("COPY driver/") < dockerfile.index("COPY apps/notes/") < dockerfile.index( - "COPY apps/mail/") < dockerfile.index("USER user") - assert [layer["name"] for layer in docker.resolved[0][1]["python"]][-2:] == ["apps-notes", "apps-mail"] - assert confs["agentenv-app-mail.conf"].startswith(MAIL_SERVER) and "[program:app-notes-sync]" in confs[ - "agentenv-app-notes.conf"] - stored = Env.get(env.id) - assert [(app.id, app.version) for app in stored.composed_apps()] == [(notes.id, 1), (mail.id, 1)] - assert stored.to_dict()["apps"] == [{"id": notes.id, "version": 1}, {"id": mail.id, "version": 1}] - assert stored.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} - assert docker.saved == [] - assert _put(base, driver, apps=[mail, notes]).metadata["composed"]["container"]["arm64"]["key"] != ( - stored.metadata["composed"]["container"]["arm64"]["key"]) - - -@pytest.mark.parametrize("apps, problem", [ - (lambda mail, other: [mail, mail], "apps: @local/~/kit/mail is listed twice"), - (lambda mail, other: [mail, other(name="mail-2", environment_name="mail")], - "apps: @local/~/kit/mail and @local/~/kit/mail-2 would share the environment name 'mail'"), - (lambda mail, other: [mail, other(name="x", environment_name="Mail")], - "apps: @local/~/kit/mail and @local/~/kit/x would share the folder '/opt/agentenv/apps/mail'"), - (lambda mail, other: [other(name="a", install={"ports": [8025], "services": []}), - other(name="b", install={"ports": [8025, 9000], "services": []})], - "apps: @local/~/kit/a and @local/~/kit/b would share the port 8025"), - (lambda mail, other: [mail, other(name="desk-port", install={"ports": [18765], "services": []})], - "apps[1]: @local/~/kit/desk-port listens on 18765, as @local/~/kit/desk does"), - (lambda mail, other: [mail, other(name="arm", supports=[{"os": {"type": "linux", "display": ["x11"]}, - "arch": ["riscv64"]}])], - "apps[1]: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and @local/~/kit/arm all run"), - (lambda mail, other: [other(name=arch, supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": [arch]}]) - for arch in ("amd64", "arm64")], - "apps: no architecture @local/~/kit/ubuntu-24.04, @local/~/kit/desk and every app all run"), - (lambda mail, other: [other(name="mail", install={"services": [{"name": "sync", "command": "venv/bin/sync"}]}), - other(name="mail-sync", install={"services": []})], - "apps: @local/~/kit/mail and @local/~/kit/mail-sync would share the program 'app-mail-sync'"), -], ids=["twice", "environment-name", "folder", "app-port", "driver-port", "arch", "no-shared-arch", "program"]) -def test_apps_that_dont_fit_together_or_on_the_computer_are_refused_before_anything_is_stored(base, driver, docker, - tmp_path, apps, problem): - def other(name="other", **changes): - return _app(None, None, tmp_path / name, compose=False, name=name, **changes) - - listed = apps(_app(base, driver, tmp_path / "mail", compose=False), other) - - with pytest.raises(ValueError, match="computer env '@local/~/kit/desktop'") as refused: - _put(base, driver, apps=listed) - - assert problem in str(refused.value) and str(refused.value).count(" would share ") <= 1 - with pytest.raises(NotFoundError): - Env.get("@local/~/kit/desktop") - assert docker.builds == [] - - -#: An install that runs one script, setup.sh, as root. -SETUP = {"run": [{"script": "setup.sh"}], "services": []} - - -def _setting_up(tmp_path, *names, scripts=("setup.sh",)): - """An app of each of ``names`` whose install runs ``scripts`` as root, put without a computer.""" - return [_app(None, None, tmp_path / name, compose=False, name=name, - install={"run": [{"script": script} for script in scripts], "services": []}, - extra={script: "#!/bin/sh\n" for script in scripts}) for name in names] - - -MAIL, NOTES = "@local/~/kit/mail", "@local/~/kit/notes" - - -@pytest.mark.parametrize("mail, notes, lost", [ - ({"usr/local/bin/tool": b"mail's"}, {"usr/local/bin/.wh.tool": b""}, f"/usr/local/bin/tool: {NOTES} deletes {MAIL}'s file"), - ({"opt/shared/data": b"mail's"}, {"opt/shared/.wh..wh..opq": b"", "opt/shared/other": b"notes'"}, - f"/opt/shared/data: {NOTES} deletes {MAIL}'s file"), - ({"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}, - f"/usr/bin/tool: {NOTES} points {MAIL}'s link elsewhere"), - ({"usr/bin/tool": b"mail's"}, {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}, - f"/usr/bin/tool: {NOTES} makes {MAIL}'s file a link"), - ({"usr/local/bin/tool": (b"mail's", 0o755)}, {"usr/local/bin/tool": (b"notes'", 0o755)}, - f"/usr/local/bin/tool: {NOTES} replaces {MAIL}'s program"), -], ids=["deleted", "opaque", "link", "kind", "program"]) -def test_a_part_that_certainly_undoes_what_an_earlier_one_installed_fails_its_pairs_at_build_naming_it( - base, driver, docker, tmp_path, mail, notes, lost): - docker.writes = {"apps/mail && ": mail, "apps/notes && ": notes} - - env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes")).id) - - assert sorted(docker.saved) == sorted(build["tag"] for build in docker.builds) - assert {arch: (r["status"], r["stage"], r["reason"]) for arch, r in env.metadata["composed"]["container"].items()} == { - arch: ("failed", "build", f"a part undoes what an earlier one installed: {lost}") for arch in ("amd64", "arm64")} - assert docker.probed == [] and "computer" not in env.metadata - - -@pytest.mark.parametrize("writes, lost", [ - ({"apps/mail && ./setup.sh": {"usr/local/bin/tool": (b"mail's", 0o755)}, - "apps/notes && ./setup.sh": {"usr/local/bin/tool": (b"mail's", 0o755)}, - "apps/notes && ./again.sh": {"usr/local/bin/.wh.tool": b""}}, f"/usr/local/bin/tool: {NOTES} deletes {MAIL}'s file"), - ({"apps/mail && ./setup.sh": {"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, - "apps/notes && ./setup.sh": {"usr/bin/tool": "/opt/agentenv/apps/mail/tool"}, - "apps/notes && ./again.sh": {"usr/bin/tool": "/opt/agentenv/apps/notes/tool"}}, - f"/usr/bin/tool: {NOTES} points {MAIL}'s link elsewhere"), - ({"apps/mail && ./setup.sh": {"opt/shared/data": b"mail's"}, "apps/mail && ./again.sh": {"opt/shared/.wh.data": b""}, - "apps/notes && ./setup.sh": {"opt/shared/.wh..wh..opq": b"", "opt/shared/other": b"notes'"}}, None), - ({"apps/mail && ./setup.sh": {"opt/shared/data": b"mail's"}, - "apps/notes && ./setup.sh": {"opt/shared/.wh..wh..opq": b"", "opt/shared/other": b"notes'"}, - "apps/notes && ./again.sh": {"opt/shared/data": b"notes'"}}, f"/opt/shared/data: {NOTES} deletes {MAIL}'s file"), -], ids=["kept-then-deleted", "kept-then-pointed", "deleted-before-opaque", "opaque-then-written"]) -def test_each_command_of_a_part_is_checked_against_what_earlier_parts_left_whoever_wrote_it_since( - base, driver, docker, tmp_path, writes, lost): - docker.writes = writes - - env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes", scripts=("setup.sh", "again.sh"))).id) - - composed = env.metadata["composed"]["container"].values() - assert ({(r["status"], r["reason"]) for r in composed} == {("failed", f"a part undoes what an earlier one installed: {lost}")} - if lost else env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"}) - - -@pytest.mark.parametrize("mail, notes, told", [ - ({"etc/passwd": b"root\nmail\n", "usr/share/icons/hicolor/icon-theme.cache": b"\x00mail"}, - {"etc/passwd": b"root\nmail\nnotes\n", "usr/share/icons/hicolor/icon-theme.cache": b"\x00mail notes"}, - f"/etc/passwd ({MAIL}, {NOTES}), /usr/share/icons/hicolor/icon-theme.cache ({MAIL}, {NOTES})"), - ({f"etc/conf.d/{i:02}": b"mail's" for i in range(12)}, {f"etc/conf.d/{i:02}": b"notes'" for i in range(12)}, - ", ".join(f"/etc/conf.d/{i:02} ({MAIL}, {NOTES})" for i in range(10)) + " and 2 more"), - ({"usr/local/bin/tool": (b"same", 0o755), "usr/bin/tool": "/opt/tool", "etc/x": b"same"}, - {"usr/local/bin/tool": (b"same", 0o755), "usr/bin/tool": "/opt/tool", "etc/x": b"same"}, None), - ({path: b"mail's" for path in ("var/log/setup.log", "tmp/x", "run/x", "home/user/.cache/pip/x", "opt/x/__pycache__/m.pyc")}, - {"var/log/.wh.setup.log": b"", "tmp/x": b"notes'", "run/x": (b"notes'", 0o755), "home/user/.cache/pip/x": "/x", - "opt/x/__pycache__/m.pyc": b"notes'"}, None), - ({"etc/notes.conf": b"mail's"}, {}, None), -], ids=["edited", "many", "same", "scratch", "apart"]) -def test_files_two_parts_wrote_differently_but_not_certainly_lost_are_said_and_not_refused(base, driver, docker, tmp_path, - mail, notes, told): - docker.writes, said = {"apps/mail && ": mail, "apps/notes && ": notes}, [] - - env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), - composing=Composing(on_progress=said.append)).id) - - assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} - compared = [line.partition(": ")[2] for line in said if line.startswith("container/arm64: not refused")] - assert compared == ([f"not refused, as each may keep the earlier part's change: two parts wrote {told}"] - if told else []) - - -def test_one_part_with_run_scripts_has_nothing_to_compare_and_its_image_isnt_saved(base, driver, docker, tmp_path): - docker.writes = {"apps/mail && ": {"usr/local/bin/tool": b"mail's"}} - mail = _setting_up(tmp_path, "mail")[0] - - env = Env.get(_put(base, driver, apps=[mail, _app(None, None, tmp_path / "notes", compose=False, name="notes")]).id) - - assert env.build_statuses() == {"container/amd64": "unverified", "container/arm64": "ready"} and docker.saved == [] - - -def test_a_builder_vm_compares_what_each_part_wrote_in_a_container_of_the_image_before_pushing(base, driver, docker, - signing, tmp_path): - docker.writes = {"apps/mail && ": {"usr/local/bin/tool": (b"mail's", 0o755)}, - "apps/notes && ": {"usr/local/bin/tool": (b"notes'", 0o755)}} - vms = FakeVmProvider(docker, tmp_path / "vms") - - env = Env.get(_put(base, driver, apps=_setting_up(tmp_path, "mail", "notes"), composing=Composing(vms)).id) - - assert {arch: (r["status"], r["reason"].partition(": ")[2]) for arch, r in env.metadata["composed"]["container"].items()} == { - arch: ("failed", f"/usr/local/bin/tool: {NOTES} replaces {MAIL}'s program") for arch in ("amd64", "arm64")} - compared = [line for line in vms.ran if "/layers.py" in line] - assert len(compared) == 2 and all("docker run --rm --network none --user root" in line for line in compared) - assert not [line for line in vms.ran if "docker push" in line] and all(vm.terminated for vm in vms.vms) - assert not list((tmp_path / "vms").rglob("image.tar")) - - def test_an_app_is_composed_only_for_the_arches_it_supports(base, driver, docker, tmp_path): _app(base, driver, tmp_path / "mail", supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ["arm64"]}]) From acf76f1e85622c983c7f601070139731bcd0022c Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 31 Oct 2026 07:21:33 -0700 Subject: [PATCH 12/14] test(computer): the two-app desk compares its apps' files, or reuses an image of the same key an earlier run pushed Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/computer_app_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index 6b2947d7d..32e4b38e2 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -4,7 +4,7 @@ comes up, and the computer ready only once its server says its window and its @ready_check are; its apt packages, one from a PPA, its download, run script and install step on its build setting, fetched at put, installed offline. And two apps composed into one computer env, each running on its own settings from its own folder, the files their run -scripts wrote compared before the push. Needs Docker and the local registry on localhost:5000.""" +scripts wrote compared before the push, or the image of the same key reused, as one an earlier run pushed is. Needs Docker and the local registry on localhost:5000.""" from pathlib import Path @@ -52,4 +52,7 @@ def test_two_apps_composed_into_one_computer_env_each_run_on_their_own_settings( "clock": {"zone": "Europe/Paris", "served": {"zone": "Europe/Paris", "known": True}}, "card": "desk", "app_ready": ["app clock", "app notes"], "launched": ["notes"], "folders": ["clock", "notes"], "hello": True, "paris": True} - assert "comparing the files its parts' run scripts and install steps wrote" in ran.output + for pair in ("container/amd64", "container/arm64"): + said = [line for line in ran.output.splitlines() if line.startswith(f"envs/desk: {pair}: ")] + assert any(line.endswith(("comparing the files its parts' run scripts and install steps wrote", + "reusing the image of the same key, built earlier")) for line in said), said From b646a264eef9d9bbb999c4fee4aab924b04edc3e Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 31 Oct 2026 10:38:56 -0700 Subject: [PATCH 13/14] fix(computer): keep a driver's programs apart from apps', clear a deleted folder's contents, leave unread hard links uncertain - A driver's programs and their file are named under driver- (agentenv-driver-.conf, driver--), as an app's are under app-, so a driver whose id ends app-notes no longer shares an app notes' conf or programs. - Deleting a folder deletes what was under it for the comparison, as an opaque folder does, so a part recreating a file there and deleting its own copy isn't blamed for an earlier part's. - A hard link to a file of an earlier layer, whose content the reader doesn't read, never counts as a replaced program; it's said as a file two parts may have written differently. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/clashes.py | 15 +++++++++++++-- src/agent_env/env/envs/computer/utils/composer.py | 9 +++++---- .../env/envs/computer/utils/layers_in_image.py | 3 ++- tst/unit/env/envs/test_computer_apps.py | 10 ++++++++-- tst/unit/env/envs/test_computer_builder.py | 2 +- tst/unit/env/envs/test_computer_env.py | 10 +++++----- 6 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/clashes.py b/src/agent_env/env/envs/computer/utils/clashes.py index 0ff1d0744..747efdd0a 100644 --- a/src/agent_env/env/envs/computer/utils/clashes.py +++ b/src/agent_env/env/envs/computer/utils/clashes.py @@ -22,6 +22,11 @@ class Entry(NamedTuple): kind: str what: str = "" + @property + def known(self) -> bool: + """Whether its content is known: not a hard link to a file of an earlier layer, which the reader doesn't read.""" + return not self.what.startswith("=> ") + @classmethod def listed(cls, entry: str) -> Entry: kind, _, what = entry.partition(" ") @@ -45,6 +50,9 @@ def compare(layers: list[tuple[str, dict[str, str]]]) -> tuple[list[str], list[s lost.append(f"{hidden}: {of} deletes {by}'s file") entries[hidden], writers[hidden] = Entry("deleted"), [] entry = Entry("dir") + if entry.kind == "deleted": + for gone in [gone for gone in entries if gone.startswith(f"{path}/") and gone not in listed]: + entries[gone], writers[gone] = Entry("deleted"), [] reason = None if path in entries and (by := _other(writers[path], of)): if reason := _lost(path, of, entry, by, entries[path]): @@ -73,10 +81,13 @@ def _lost(path: str, of: str, entry: Entry, by: str, was: Entry) -> str | None: return f"{path}: {of} makes {by}'s {_KINDS[was.kind]} a {_KINDS[entry.kind]}" if was.kind == "link" and entry.what != was.what: return f"{path}: {of} points {by}'s link elsewhere" - if was.kind == "exec" and entry.what != was.what: + if was.kind == "exec" and entry.what != was.what and entry.known and was.known: return f"{path}: {of} replaces {by}'s program" return None def _differs(entry: Entry, was: Entry) -> bool: - return entry.kind in ("file", "exec") and was.kind == "file" and entry.what != was.what + """Whether ``entry`` is a file that may hold other content than ``was``, a file, or a program whose content isn't + known.""" + return (entry.kind in ("file", "exec") and entry.what != was.what + and (was.kind == "file" or (was.kind == "exec" and not (entry.known and was.known)))) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index a1526239e..dd3ce211f 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -338,11 +338,12 @@ class _Layer: def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact, from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them, served on - its port, its folder under ``DRIVERS`` named for its id's last part.""" + its port, its folder under ``DRIVERS`` named for its id's last part, its programs and their file under ``driver-`` + as an app's are under ``app-``, so neither ever takes the other's.""" install, name = driver.install[base.os.type], gate.folder_name(driver.id, "driver") - layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf", driver.id) - _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}, - from_store) + layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-driver-{name}.conf", driver.id) + _stage(context, layer, f"driver {driver.id}", driver.files, _services(f"driver-{name}", install), + {"MCP_PORT": driver.port}, from_store) return layer diff --git a/src/agent_env/env/envs/computer/utils/layers_in_image.py b/src/agent_env/env/envs/computer/utils/layers_in_image.py index 2e0891b8e..10b33cbc0 100644 --- a/src/agent_env/env/envs/computer/utils/layers_in_image.py +++ b/src/agent_env/env/envs/computer/utils/layers_in_image.py @@ -3,7 +3,8 @@ in the newest layer below the one found before it that a shell ran exactly that command for, so a base's layers of the same command never stand in for the build's own; and each entry of its layer listed by path, as `` ``: ``file`` or ``exec`` (a file any can run) with its sha256, ``link`` with its target, ``dir``, ``deleted``, or ``opaque`` -for a folder whose earlier contents the layer hides. A hard link is listed as its target is. Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py +for a folder whose earlier contents the layer hides. A hard link is listed as its target is in the same layer, else +with ``=> `` in place of a sha256, since an earlier layer isn't read. Standard library only, since a builder VM runs it on the image's own Python. Usage: ``python3 layers_in_image.py ``; it prints what each wrote, in order, as JSON.""" import hashlib diff --git a/tst/unit/env/envs/test_computer_apps.py b/tst/unit/env/envs/test_computer_apps.py index 3ea9898a8..4306f6f6a 100644 --- a/tst/unit/env/envs/test_computer_apps.py +++ b/tst/unit/env/envs/test_computer_apps.py @@ -29,6 +29,7 @@ def test_a_computer_env_builds_its_apps_after_its_driver_in_the_order_it_lists_t assert [layer["name"] for layer in docker.resolved[0][1]["python"]][-2:] == ["apps-notes", "apps-mail"] assert confs["agentenv-app-mail.conf"].startswith(MAIL_SERVER) and "[program:app-notes-sync]" in confs[ "agentenv-app-notes.conf"] + assert sorted(confs) == ["agentenv-app-mail.conf", "agentenv-app-notes.conf", "agentenv-driver-desk.conf"] stored = Env.get(env.id) assert [(app.id, app.version) for app in stored.composed_apps()] == [(notes.id, 1), (mail.id, 1)] assert stored.to_dict()["apps"] == [{"id": notes.id, "version": 1}, {"id": mail.id, "version": 1}] @@ -102,8 +103,10 @@ def _setting_up(tmp_path, *names, scripts=("setup.sh",)): [f"/o/data: {B} deletes {A}'s file"]), ([(A, {"/t": "exec 1"}), (B, {"/t": "exec 2"}), (C, {"/t": "deleted"})], [f"/t: {B} replaces {A}'s program", f"/t: {C} deletes {B}'s file"]), + ([(A, {"/opt/s": "dir", "/opt/s/t": "exec 1"}), (B, {"/opt/s": "deleted"}), (B, {"/opt/s/t": "exec 2"})], + [f"/opt/s: {B} deletes {A}'s folder"]), ], ids=["deleted", "opaque", "link", "kind", "folder", "program", "kept-then-deleted", "kept-then-pointed", - "opaque-then-written", "after-a-loss"]) + "opaque-then-written", "after-a-loss", "a-folder-deleted"]) def test_a_part_that_certainly_undoes_what_another_left_is_a_loss(layers, lost): assert clashes.compare(layers) == (lost, []) @@ -121,8 +124,11 @@ def test_a_part_that_certainly_undoes_what_another_left_is_a_loss(layers, lost): ([(A, {"/t": "exec 1"}), (A, {"/t": "deleted"}), (B, {"/t": "exec 2"}), (B, {"/t": "deleted"})], []), ([(A, {"/o/t": "exec 1"}), (A, {"/o": "opaque"}), (B, {"/o/t": "exec 2"}), (B, {"/o/t": "deleted"})], []), ([(A, {"/a": "file 1"}), (B, {"/b": "file 2"})], []), + ([(A, {"/o/t": "exec 1"}), (A, {"/o": "deleted"}), (B, {"/o/t": "exec 2"}), (B, {"/o/t": "deleted"})], []), + ([(A, {"/usr/local/bin/tool": "exec => /usr/bin/tool"}), (B, {"/usr/local/bin/tool": "exec 9"})], + [f"/usr/local/bin/tool ({A}, {B})"]), ], ids=["edited", "made-a-program", "same", "scratch", "own", "deleted-before-opaque", "recreated", - "recreated-after-opaque", "apart"]) + "recreated-after-opaque", "apart", "recreated-after-its-folder-was-deleted", "hard-link-to-an-earlier-layer"]) def test_what_isnt_certainly_lost_isnt_a_loss_and_only_a_file_two_parts_wrote_differently_is_said(layers, differ): assert clashes.compare(layers) == ([], differ) diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py index 036793c0c..146480fa3 100644 --- a/tst/unit/env/envs/test_computer_builder.py +++ b/tst/unit/env/envs/test_computer_builder.py @@ -204,7 +204,7 @@ def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_imag assert sorted(vms.fetched) == sorted([*driver_files, *driver_files]) assert vms.uploaded and all(f"{packages.PREFIX}/files/" in url for url in vms.uploaded) builds = {build["platform"]: build for build in docker.builds} - assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", + assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-driver-desk.conf", "driver/driver.py", "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] assert composed["amd64"]["installed"] == {"mss": "10.1.0"} assert docker.labels[f"agentenv-computer:{composed['arm64']['key'][7:23]}-arm64"]["dev.agentenv.emulated"] == "true" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 67bc20a26..87e14ad87 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -91,11 +91,11 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ "RUN --network=none python3 -m venv /opt/agentenv/drivers/desk/venv && /opt/agentenv/drivers/desk/venv/bin/pip " "install --no-index --no-deps /tmp/agentenv-packages/wheels/driver/*.whl && /opt/agentenv/drivers/desk/venv/bin/pip list " "--format=json > /opt/agentenv/drivers/desk/installed.json && rm -rf /tmp/agentenv-packages/wheels/driver", - "COPY agentenv-desk.conf /etc/supervisor/conf.d/agentenv-desk.conf", - "RUN test -d /opt/agentenv/drivers/desk && test -f /etc/supervisor/conf.d/agentenv-desk.conf && test -x " + "COPY agentenv-driver-desk.conf /etc/supervisor/conf.d/agentenv-driver-desk.conf", + "RUN test -d /opt/agentenv/drivers/desk && test -f /etc/supervisor/conf.d/agentenv-driver-desk.conf && test -x " "/opt/agentenv/drivers/desk/venv/bin/python", "USER user", f"LABEL dev.agentenv.computer-key={composed['arm64']['key']}"] - assert arm64["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", "driver/requirements.txt", + assert arm64["files"] == ["Dockerfile", "agentenv-driver-desk.conf", "driver/driver.py", "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] assert sorted(arch for arch, _ in docker.resolved) == ["amd64", "arm64"] assert docker.timeouts == {"arm64": 3600, "amd64": 10800} @@ -103,9 +103,9 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ "requirements": "requirements.txt", "add": []}] assert composed["arm64"]["lock"].startswith("sha256:") and composed["arm64"]["lock"] != composed["amd64"]["lock"] assert arm64["conf"] == ( - "[program:desk-driver]\ncommand=/opt/agentenv/drivers/desk/venv/bin/python -m agentenv_protocol.computer.serve " + "[program:driver-desk-driver]\ncommand=/opt/agentenv/drivers/desk/venv/bin/python -m agentenv_protocol.computer.serve " 'driver.py:Desk\ndirectory=/opt/agentenv/drivers/desk\nenvironment=MCP_PORT="18765"\nautorestart=true\n' - "redirect_stderr=true\nstdout_logfile=/var/log/agentenv/desk-driver.log\n") + "redirect_stderr=true\nstdout_logfile=/var/log/agentenv/driver-desk-driver.log\n") assert ComputerEnv.from_dict(stored.to_dict()).to_dict() == stored.to_dict() From ce03be901faf1ac6b3cd5eee4e8e563586eebf5f Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Sat, 31 Oct 2026 10:59:07 -0700 Subject: [PATCH 14/14] test(computer): the two-app desk checks its files were compared for this machine's arch only CI's amd64 runner has no arm64 emulation, so its arm64 pair fails at resolve and compares nothing; the test asserted a compare or a reuse for both arches. Co-Authored-By: Claude Opus 5.5 --- tst/integration/computer/computer_app_test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tst/integration/computer/computer_app_test.py b/tst/integration/computer/computer_app_test.py index 32e4b38e2..6f50fab65 100644 --- a/tst/integration/computer/computer_app_test.py +++ b/tst/integration/computer/computer_app_test.py @@ -4,8 +4,10 @@ comes up, and the computer ready only once its server says its window and its @ready_check are; its apt packages, one from a PPA, its download, run script and install step on its build setting, fetched at put, installed offline. And two apps composed into one computer env, each running on its own settings from its own folder, the files their run -scripts wrote compared before the push, or the image of the same key reused, as one an earlier run pushed is. Needs Docker and the local registry on localhost:5000.""" +scripts wrote compared before the push of this machine's arch, or the image of the same key reused, as one an earlier +run pushed is. Needs Docker and the local registry on localhost:5000.""" +import platform from pathlib import Path import pytest @@ -17,6 +19,7 @@ pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow] APP = Path(__file__).parent / "app" +ARCH = {"x86_64": "amd64", "amd64": "amd64", "aarch64": "arm64", "arm64": "arm64"}[platform.machine().lower()] def test_an_app_deployed_alone_runs_its_service_on_its_settings_on_a_computer_named_after_it(state, monkeypatch): @@ -52,7 +55,6 @@ def test_two_apps_composed_into_one_computer_env_each_run_on_their_own_settings( "clock": {"zone": "Europe/Paris", "served": {"zone": "Europe/Paris", "known": True}}, "card": "desk", "app_ready": ["app clock", "app notes"], "launched": ["notes"], "folders": ["clock", "notes"], "hello": True, "paris": True} - for pair in ("container/amd64", "container/arm64"): - said = [line for line in ran.output.splitlines() if line.startswith(f"envs/desk: {pair}: ")] - assert any(line.endswith(("comparing the files its parts' run scripts and install steps wrote", - "reusing the image of the same key, built earlier")) for line in said), said + said = [line for line in ran.output.splitlines() if line.startswith(f"envs/desk: container/{ARCH}: ")] + assert any(line.endswith(("comparing the files its parts' run scripts and install steps wrote", + "reusing the image of the same key, built earlier")) for line in said), said