From 585a3c4b99e739e45ae63c2585cf9ee5ebf9d0ee Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 12:19:15 -0700 Subject: [PATCH 01/11] feat(computer): compose on the put's sandbox provider, in a builder VM per arch A computer env composes on the put's sandbox provider, by default the env's. The local provider builds with this machine's Docker as before; a VM provider resolves and builds each form and arch in a VM of its own, which fetches the put's stored files by signed URL, takes the rest inline, registers QEMU for an arch it emulates, pushes to the image store and is torn down. Each arch goes to the first link that runs it natively, else one that emulates it. The index is written over the registry API, so a put needs no local Docker. --sandbox on env build and bundle register; run --sandbox composes there too. FD-4734 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/bundle/authoring.py | 9 +- src/agent_env/bundle/materialize.py | 22 +- src/agent_env/bundle/register.py | 8 +- src/agent_env/bundle/run.py | 11 +- src/agent_env/cli/bundle.py | 7 +- src/agent_env/cli/env/build.py | 12 +- src/agent_env/cli/run.py | 4 +- src/agent_env/env/envs/computer/app_env.py | 13 +- src/agent_env/env/envs/computer/env.py | 15 +- .../env/envs/computer/utils/builder.py | 500 ++++++++++++++++++ .../env/envs/computer/utils/composer.py | 205 +++---- .../env/envs/computer/utils/packages.py | 175 ++---- .../store/image_store/registry_api.py | 48 +- .../computer/builder_modal_vm_test.py | 60 +++ tst/unit/bundle/computer_env_folder_test.py | 12 +- tst/unit/env/envs/test_computer_env.py | 354 +++++++++++-- tst/unit/store/registry_api_test.py | 47 +- 18 files changed, 1154 insertions(+), 350 deletions(-) create mode 100644 src/agent_env/env/envs/computer/utils/builder.py create mode 100644 tst/integration/computer/builder_modal_vm_test.py diff --git a/AGENTS.md b/AGENTS.md index 571238995..430ac1f21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,7 +112,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch with this machine's Docker, the other arch through its emulation, tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on a configured sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference; its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU), and its pair records the provider that built it (`builder`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference; its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/src/agent_env/bundle/authoring.py b/src/agent_env/bundle/authoring.py index f981e6ec9..07680e494 100644 --- a/src/agent_env/bundle/authoring.py +++ b/src/agent_env/bundle/authoring.py @@ -11,7 +11,7 @@ from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import Any, NoReturn, TypeVar, overload +from typing import TYPE_CHECKING, Any, NoReturn, TypeVar, overload from agent_env.artifact.artifact import Artifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact @@ -27,6 +27,9 @@ from ._fs import relative, show, with_article from .parse import CONFIG_FILES, LEAVINGS, NAMED_BY, Bundle, BundleEntry, BundleError +if TYPE_CHECKING: + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider + A = TypeVar("A", bound=Artifact) E = TypeVar("E", bound=Env) @@ -48,8 +51,10 @@ class AuthoringContext: bundle: Bundle entry: BundleEntry - #: Called just before writing the entry builds an image with this machine's Docker, which a run says it does. + #: Called just before writing the entry builds an image, which a run says it does. on_build: Callable[[], None] | None = None + #: Where writing a computer env composes it; the env's sandbox provider when None. + sandbox_provider: SandboxProvider | None = None @property def id(self) -> str: diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index f8db7387c..f1557dfaf 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -20,7 +20,7 @@ from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager, nullcontext from dataclasses import dataclass -from typing import Any +from typing import TYPE_CHECKING, Any from agent_env.a2a_agent import A2AAgent from agent_env.artifact import Artifact @@ -44,6 +44,9 @@ from .plan import Plan, Write, artifact_writer, env_writer, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage, build_step +if TYPE_CHECKING: + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider + @dataclass(frozen=True) class Materialized: @@ -81,6 +84,7 @@ def materialize( on_write: Callable[[Materialized], None] | None = None, build_here: frozenset[str] = frozenset(), ledger: Ledger | None = None, + sandbox_provider: SandboxProvider | None = None, ) -> Materialization: """Write ``plan``'s entities, tasks and evals. The first write that fails stops it, and every earlier one stays: they're in the ledger, so the next run reuses them. ``on_wait`` is called when another run @@ -94,6 +98,8 @@ def materialize( Each write gets the version it would reuse or the store's next, which another run can take first. A step reading what the run would write first isn't preflighted, since the store doesn't hold it yet. + ``sandbox_provider`` composes each computer env written; the env's sandbox provider when None. + ``ledger`` keeps the rows, this run's in the ``@local`` namespace's store unless given; it also says whether each task and eval names the versions it reads (``Ledger.pins_references``) and whether each image the bundle builds is built here and pushed (``Ledger.pushes_images``), as a registration's does.""" @@ -128,7 +134,7 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: if ledger.pushes_images and isinstance(write.source, BuiltImage) and write.id not in plan.published: through_ledger(write, lambda: _write_pushed_image(plan, write, on_build)) else: - through_ledger(write, lambda: _write_entity(plan, write, here, on_build)) + through_ledger(write, lambda: _write_entity(plan, write, here, on_build, sandbox_provider)) if here and done[_key(write)].reused: _build_reused(plan, write, done[_key(write)].version, on_build) unwritten = {_key(item.write) for item in done.values() if not item.reused} if dry_run else set() @@ -150,7 +156,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked)) -def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None) -> int: +def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None, + sandbox_provider: SandboxProvider | None) -> int: if isinstance(write.source, ReferencedImage): return _write_referenced_image(plan, write) if isinstance(write.source, BuiltImage): @@ -158,7 +165,7 @@ def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Writ return _write_published_image(plan, write) return _write_build_context(plan, write, here, on_build) if write.kind is BundleKind.ENV: - return _write_env(plan, write, _before(on_build, write)) + return _write_env(plan, write, _before(on_build, write), sandbox_provider) return _WRITERS[write.kind](plan, write) @@ -292,11 +299,12 @@ def _write_agent(plan: Plan, write: Write) -> int: AuthoringContext(plan.bundle_of(entry), entry)).version -def _write_env(plan: Plan, write: Write, on_build: Callable[[], None] | None = None) -> int: +def _write_env(plan: Plan, write: Write, on_build: Callable[[], None] | None = None, + sandbox_provider: SandboxProvider | None = None) -> int: entry = write.source.entry cls = get_env_registry()[entry.type] - return cls.from_toml(_pinned(plan, write, cls.toml_refs), - AuthoringContext(plan.bundle_of(entry), entry, on_build=on_build)).version + return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext( + plan.bundle_of(entry), entry, on_build=on_build, sandbox_provider=sandbox_provider)).version def _pinned(plan: Plan, write: Write, refs: tuple[EntityRef, ...]) -> Any: diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 9c66d58c7..8ac9f6959 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -31,6 +31,7 @@ from agent_env.config import get_config from agent_env.entity.refs import RefRole, ref_sites from agent_env.plugins._bundles import AGENT_ALIASES +from agent_env.providers import build_sandbox_provider from agent_env.store.document_store import DocumentStore, LocalSqliteDocumentStore from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.image_store.oci_registry_credentials import is_loopback_host @@ -77,16 +78,19 @@ def register_bundle( dry_run: bool = False, declaring: Declaring | None = None, on_progress: Callable[[str], None] | None = None, + sandbox: str | None = None, ) -> Registration: """Register the bundle at ``root``: ``tasks`` and ``evals`` by name, or with neither every task and eval it has, or every entity when it has none, with what they reference. ``prefix`` opens each entry's registry id, the folder's name and a dash unless given. An id the store holds already takes a new version, whoever wrote it. ``on_progress`` is given a line for each write, as it's checked or written, and before each image is built. A folder with no bundle.toml that uses plugin packages is given one, or told what to declare, as ``declaring`` says: by default as a - run of the folder would (``declaring_for``). + run of the folder would (``declaring_for``). ``sandbox`` is the provider its computer envs compose on, by default + the env's. It turns namespace routing on while it runs. Raises BundleError listing every problem found before anything is written; a write that fails stops it, and every earlier one stays, in the ledger, for the next run to reuse.""" + provider = build_sandbox_provider(sandbox) if sandbox else None with namespace_routing(): bundle = parse_bundle(Path(root)) resolved = resolve_bundle(bundle) @@ -108,7 +112,7 @@ def register_bundle( registered, dry_run=dry_run, ledger=ledger, on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), on_build=lambda write: say(f" {_label(registered, write)}: building with docker and pushing"), - on_write=lambda done: say(f" {registered_line(registered, done)}")) + on_write=lambda done: say(f" {registered_line(registered, done)}"), sandbox_provider=provider) return Registration(registered, materialization, ids, tuple(_floating(registered)), identity, shared, requires) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 4dc89a175..93c943f31 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -218,7 +218,7 @@ def run_bundle( """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the user simulator and the judges run on, and ``agent`` the agent the solver runs as (``Plan.agent``), leaving the - tasks as written; ``sandbox`` is the provider that envs, + tasks as written; ``sandbox`` is the provider that computer envs compose on and that envs, agents, sandboxes and the judge deploy on. ``on_progress`` is given a line of text at each write and step. ``id_root`` roots the ids of a bundle an installed package provides (``InstalledBundle.id_root``); a folder's ids otherwise come from its path. @@ -297,14 +297,17 @@ def _write_bundle( return _written_with_infra(plan, preflight, say) -def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], None]) -> Materialization: - """Write ``plan``, then set up the infra envs its runs deploy on.""" +def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], None], + sandbox: str | None = None) -> Materialization: + """Write ``plan``, its computer envs composed on ``sandbox`` when given, then set up the infra envs its runs deploy + on.""" materialization = materialize( plan, on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), on_build=lambda write: say(f"{_label(plan, write)}: building with docker, which can take minutes"), on_write=lambda done: say(_written(plan, done)), build_here=preflight.built_here, + sandbox_provider=build_sandbox_provider(sandbox) if sandbox else None, ) if preflight.infra: ensure_default_envs(preflight.infra_kinds, say=say) @@ -366,7 +369,7 @@ def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandb agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring) -> PreparedBundle: plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played) _say_plan(plan, say, declaring) - materialization = _written_with_infra(plan, preflight, say) + materialization = _written_with_infra(plan, preflight, say, sandbox) to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan)) return PreparedBundle(plan, materialization, to_run) diff --git a/src/agent_env/cli/bundle.py b/src/agent_env/cli/bundle.py index 43d6a02ce..1420712b8 100644 --- a/src/agent_env/cli/bundle.py +++ b/src/agent_env/cli/bundle.py @@ -84,8 +84,11 @@ def _report(imported: BundleImport, folder: Path) -> None: help="What each entry's registry id opens with; by default the folder's name and a dash.") @click.option("--dry-run", is_flag=True, help="Check everything and say what would be written; write nothing.") @click.option("--yes", is_flag=True, help="Write to a store other than this machine's own without asking.") +@click.option("--sandbox", default=None, + help="The sandbox provider the bundle's computer envs compose on, by default the env's; comma-separated " + "for a chain, each arch going to the first link that runs it natively.") def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefix: str | None, - dry_run: bool, yes: bool): + dry_run: bool, yes: bool, sandbox: str | None): """Write the bundle at FOLDER into the configured stores under registry ids, so whatever runs the stores' tasks runs what the folder runs. @@ -102,7 +105,7 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi with no bundle.toml that uses plugin packages is given one. An id the store holds already takes a new version, whoever wrote it. A store other than this machine's is written only after you confirm, or with --yes. Registering again writes only what changed since.""" - options = {"tasks": tasks, "evals": evals, "prefix": prefix} + options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox} click.echo(f"Registering {folder} into:") for line in describe_target(): click.echo(f" {line}") diff --git a/src/agent_env/cli/env/build.py b/src/agent_env/cli/env/build.py index f40a06951..8c132f8b3 100644 --- a/src/agent_env/cli/env/build.py +++ b/src/agent_env/cli/env/build.py @@ -6,25 +6,29 @@ from agent_env.env import Env from agent_env.env.envs.computer.utils import composer from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.providers import build_sandbox_provider from agent_env.store.base import NotFoundError @click.command() @click.option("--id", "env_id", required=True, help="The computer env's id") @click.option("--version", type=int, default=None, help="Its version; the latest when left out") -def build(env_id: str, version: int | None): +@click.option("--sandbox", default=None, help="The sandbox provider that resolves, builds and probes, by default the " + "env's; comma-separated for a chain, each arch going to the first link that runs it natively") +def build(env_id: str, version: int | None, sandbox: str | None): """Build a computer env's forms and arches not built yet, reusing any image of the same key, and print each one's status; exits 1 when one failed.""" try: + provider = build_sandbox_provider(sandbox) if sandbox else None env = Env.get(env_id, version) - except NotFoundError as e: + except (NotFoundError, ValueError) as e: raise click.ClickException(str(e)) from None if not isinstance(env, ComputerEnv): raise click.ClickException(f"{env_id} is a {env.type} env; only a computer env is built") if env.computer is None: raise click.ClickException(f"{env_id} names no computer and driver, so it has nothing to build alone") - if (pairs := composer.pending(env)) or not composer.current(env): - composer.compose(env, pairs) + if (pairs := composer.pending(env, provider)) or not composer.current(env): + composer.compose(env, pairs, sandbox_provider=provider) failed = False for form, by_arch in env.metadata.get("composed", {}).items(): for arch, result in by_arch.items(): diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index bbb1b7786..767b86f6f 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -41,8 +41,8 @@ help="The agent the solver runs as, whatever the tasks name: an alias such as pi, an agent of this " "bundle or an installed one, or the store's.") @click.option("--sandbox", default=None, - help="The sandbox provider envs, agents, sandboxes and the judge deploy on; comma-separated for a " - "fallback chain.") + help="The sandbox provider computer envs compose on and envs, agents, sandboxes and the judge deploy " + "on; comma-separated for a fallback chain.") @click.option("--keep", is_flag=True, help="Keep the sandboxes up after the runs, print their endpoints, and tear them down on Ctrl-C.") @click.option("--dry-run", is_flag=True, diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index 87bc66bbc..ee9b41218 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -25,6 +25,7 @@ from collections.abc import Callable from agent_env.bundle.authoring import AuthoringContext + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider class ComputerAppEnv(ComputerEnv): @@ -98,10 +99,11 @@ def from_dict(cls, data: dict) -> ComputerAppEnv: def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifactUniverse, declaration: AppDeclaration, computer: Optional[ComputerArtifact] = None, driver: Optional[ComputerDriverEnv] = None, build_settings: Optional[dict[str, Any]] = None, - compose: bool = True, on_build: Optional[Callable[[], None]] = None) -> Self: + compose: bool = True, on_build: Optional[Callable[[], None]] = None, + sandbox_provider: Optional[SandboxProvider] = None) -> Self: """Write the app once what it declares and ``build_settings``, its build settings' values, check out, and with - a computer and a driver, once it fits them (gate 1); then compose it with them unless ``compose`` is False. - Refused, naming every problem, before anything is stored.""" + a computer and a driver, once it fits them (gate 1); then compose it with them on ``sandbox_provider``, by + default the env's, unless ``compose`` is False. Refused, naming every problem, before anything is stored.""" app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, declaration=declaration, computer=computer, driver=driver, build_settings=build_settings) _, problems = _installs(declaration) @@ -117,7 +119,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) app = get_env_store().put_document(app) if computer is not None and compose: - composer.compose(app, on_build=on_build) + composer.compose(app, on_build=on_build, sandbox_provider=sandbox_provider) return app @classmethod @@ -143,7 +145,8 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerAppEnv: try: return cls.put(id=ctx.id, environment_name=fields.get("environment_name", ctx.name), module=fields["module"], files=universe, declaration=declaration, computer=computer, - driver=driver, build_settings=fields.get("build_settings"), on_build=ctx.on_build) + driver=driver, build_settings=fields.get("build_settings"), on_build=ctx.on_build, + sandbox_provider=ctx.sandbox_provider) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index da811607b..d94e9895d 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -33,6 +33,7 @@ from agent_env.bundle.authoring import AuthoringContext from agent_env.env.env import DeployedEnv from agent_env.providers.env_providers.env_provider import EnvironmentProvider + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider logger = logging.getLogger(__name__) @@ -82,10 +83,11 @@ def from_dict(cls, data: dict) -> ComputerEnv: @classmethod def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str, apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None, - compose: bool = True, on_build: Optional[Callable[[], None]] = None) -> Self: - """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it unless ``compose`` is - False, calling ``on_build`` just before it builds an image. Refused, naming every mismatch, before anything is - stored.""" + compose: bool = True, on_build: Optional[Callable[[], None]] = None, + sandbox_provider: Optional[SandboxProvider] = None) -> Self: + """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it on + ``sandbox_provider``, by default the env's, unless ``compose`` is False, calling ``on_build`` just before it + builds an image. Refused, naming every mismatch, before anything is stored.""" values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, computer_settings=computer_settings or {}) if problems: @@ -94,7 +96,7 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, display=display, network=network, computer_settings=values)) if compose: - composer.compose(env, on_build=on_build) + composer.compose(env, on_build=on_build, sandbox_provider=sandbox_provider) return env @classmethod @@ -111,7 +113,8 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: ComputerDriverEnv) try: return cls.put(id=ctx.id, computer=base, driver=driver, - environment_name=fields.pop("environment_name", ctx.name), on_build=ctx.on_build, **fields) + environment_name=fields.pop("environment_name", ctx.name), on_build=ctx.on_build, + sandbox_provider=ctx.sandbox_provider, **fields) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py new file mode 100644 index 000000000..964f39b23 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -0,0 +1,500 @@ +"""Where a computer env's packages are resolved and its images built and pushed (``composer``): this machine's Docker +for the local sandbox provider, else a VM of the put's sandbox provider, which fetches the files the put stored by +signed URL, takes the rest inline, and pushes straight to the image store. A builder has a work folder of its own, which +holds the build context in ``CONTEXT`` and a resolve in ``RESOLVED``; the paths it takes are relative to it.""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import io +import json +import logging +import os +import shlex +import shutil +import subprocess +import tarfile +import uuid +from abc import ABC, abstractmethod +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING + +import httpx + +from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image +from agent_env.config import get_config +from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag +from agent_env.providers.sandbox_providers.sandbox import CURL_RETRY_FLAGS, upload_vm_file +from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM +from agent_env.store.base import ObjectAlreadyExistsError +from agent_env.store.image_store.oci_registry_credentials import ( + is_loopback_host, names_registry, registry_host_from_ref) +from agent_env.utils.docker.build import DockerBuildError, build_image + +if TYPE_CHECKING: + from agent_env.artifact.artifacts.docker_image import DockerImageArtifact + from agent_env.providers.sandbox_providers.sandbox import VmSandbox + from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider + +logger = logging.getLogger(__name__) + +CONTEXT, RESOLVED = "context", "resolve" +RESOLVER = Path(__file__).with_name("resolve_in_base.py") +#: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged, +#: since the stores keep no file modes. +RUN_ONLY = (b"#!", b"\x7fELF") +EXECUTABLE = 0o755 +#: What registers QEMU in a builder VM's kernel, for a platform no link runs natively. +BINFMT = ("docker.io/tonistiigi/binfmt:qemu-v9.2.2" + "@sha256:1b804311fe87047a4c96d38b4b3ef6f62fca8cd125265917a9e3dc3c996c39e6") +#: The BuildKit client a builder VM's Docker builds with, installed where it has none, by the VM's platform. +BUILDX = "https://github.com/docker/buildx/releases/download/v0.38.0/buildx-v0.38.0.{platform}" +BUILDX_SHA256 = {"linux/amd64": "4fe4cc38adf48169132749b6ca22a990928db0118e3407584ee553723115d287", + "linux/arm64": "38e890e1a162bfdbf32fbe91404991dc4fc28e55b0de901b34374a061bab4d2c"} +_PLUGINS = "/usr/local/lib/docker/cli-plugins" +#: A builder VM: a build runs apt, pip and an app's install steps, and holds the base, the lock's files and the image. +_VM = {"cpu": 4.0, "memory": 8192, "disk_size_gb": 40, "timeout": 7200} +_DOWNLOAD_TIMEOUT_S = 600 +_FETCHES_IN_FLIGHT = 8 +_CHUNK = 1 << 20 + + +class ResolveError(RuntimeError): + """Resolving or fetching failed; ``log`` holds what the resolver printed.""" + + def __init__(self, message: str, log: str = ""): + super().__init__(message) + self.log = log + + +class Unbuildable(RuntimeError): + """No link of the put's sandbox provider builds a platform; the message says why for each.""" + + +@dataclass(frozen=True) +class Hashed: + sha256: str + size: int + + +class Builder(ABC): + #: The provider it's from, as a pair's record names it, and the platform its Docker runs natively. + name: str + platform: str + + @abstractmethod + def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: + """Run the resolver on ``request`` in a container of ``source`` for ``arch``, as root, with ``CONTEXT`` + read-only at /ctx, writing into ``RESOLVED``/out; what it says it wrote. Raises ResolveError.""" + + @abstractmethod + def download(self, url: str, sha256: str, path: str) -> None: + """Download ``url`` to ``path``, refused (ResolveError) unless its sha256 is ``sha256``.""" + + @abstractmethod + def files(self, folder: str) -> dict[str, Hashed]: + """Each plain file under ``folder``, never a link, by its path there: a dependency's build code, run in the + resolver's container, could leave a link to one of the builder's own files instead.""" + + @abstractmethod + def store(self, path: str, store, key: str) -> None: + """Put the file at ``path`` in the object store ``store`` at ``key``.""" + + @abstractmethod + def place(self, copies: dict[str, str], fetches: dict[str, tuple[str, str]]) -> None: + """Put at each path of ``copies`` a link to the file it names, and at each of ``fetches`` the object store's + object at its URL, refused unless its sha256 is the one given.""" + + @abstractmethod + def mark_runnable(self, paths: list[str]) -> None: + """Make each of ``paths`` executable that starts as a script or a Linux binary does.""" + + @abstractmethod + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + """Build ``context`` with ``dockerfile``, written into it, as the image ``tag`` for ``platform``. Raises + DockerBuildError.""" + + @abstractmethod + def installed(self, tag: str, prefix: str) -> dict[str, str]: + """The packages ``tag``'s build installed into the venv of the driver at ``prefix``, by name, read from the + image without running it, so on any arch; none for a driver with no venv.""" + + @abstractmethod + def push(self, tag: str, ref: str, store) -> None: + """Push ``tag`` to ``ref`` in the image store ``store``, signed in with its credentials.""" + + def close(self) -> None: + pass + + +class Builds: + """The builders the put's sandbox provider gives: for each platform, from the first of its links that runs the + platform natively, else from the first that emulates it. A link builds when it's this machine's Docker, or gives VMs + that reach the stores ``store_id`` routes to and the registry ``repository`` is in.""" + + def __init__(self, provider: SandboxProvider, store_id: str, repository: str): + self.provider = provider + self._refused = [(link, _refused(link, store_id, repository)) for link in provider.links] + + def link(self, platform: str) -> SandboxProvider: + """The link that builds ``platform``. Raises Unbuildable.""" + links = [link for link, problem in self._refused if problem is None] + if native := [link for link in links if platform in link.native_platforms()]: + return native[0] + for link in links: + if not _docker_here(link) or platform in _emulated_platforms(): + return link + unemulated = f"this machine can't build {platform}: its Docker has no emulation for it" + raise Unbuildable("; ".join(problem or unemulated for _, problem in self._refused)) + + @contextlib.contextmanager + def open(self, link: SandboxProvider, platform: str, work: Path, base: DockerImageArtifact, + from_store: dict[str, tuple[str, str]]) -> Iterator[Builder]: + """A builder from ``link`` of ``platform``, able to build ``FROM`` ``base``, holding the build context the + composer staged in ``work``, the files of ``from_store`` (by path, their object URL and sha256) fetched from + the store rather than sent; torn down when the block ends.""" + if _docker_here(link): + yield LocalBuilder(type(link).__name__, work, base, platform) + return + vm = VmBuilder(link) + try: + vm.start(work, base, platform, from_store) + yield vm + finally: + vm.close() + + +def _refused(link: SandboxProvider, store_id: str, repository: str) -> str | None: + """Why ``link`` can't build, or None when it can.""" + name = type(link).__name__ + if _docker_here(link): + return None + if SANDBOX_MODE_VM not in link.SUPPORTED_SANDBOX_MODES: + return f"{name} gives no VM to build in" + if link.ON_THIS_MACHINE: + return None + config = get_config() + objects = config.get_object_store_for(store_id) + if objects.signed_get_url(objects.object_url(f"{config.get_artifact_key_prefix()}agentenv-packages")) is None: + return f"{name}'s VMs can't fetch from the object store, which signs no URLs; build on the local provider" + if is_loopback_host(registry_host_from_ref(config.get_image_store_for(store_id).image_ref(repository, "key"))): + return f"{name}'s VMs can't push to the image store's registry on this machine; build on the local provider" + return None + + +def _docker_here(link: SandboxProvider) -> bool: + """Whether ``link``'s sandboxes are containers on this machine's Docker, which then builds itself.""" + return link.ON_THIS_MACHINE and link.REACHES_HOST_AT is None + + +def _emulated_platforms() -> set[str]: + """The platforms this machine's Docker builds, its own and those it emulates.""" + inspected = subprocess.run(["docker", "buildx", "inspect", "--bootstrap"], capture_output=True, text=True) + platforms = set() + for line in inspected.stdout.splitlines(): + if line.strip().startswith("Platforms:"): + platforms |= {item.strip().rstrip("*") for item in line.split(":", 1)[1].split(",")} + return platforms + + +def _resolver(source: str, arch: str, context: str, out: str) -> list[str]: + return ["docker", "run", "--rm", "--platform", f"linux/{arch}", "--user", "root", "--entrypoint", "python3", + "-v", f"{context}:/ctx:ro", "-v", f"{out}:/out", source, "/out/resolve.py", "/out/request.json", "/out"] + + +class LocalBuilder(Builder): + """This machine's Docker, in the folder the composer staged the build context in.""" + + def __init__(self, name: str, work: Path, base: DockerImageArtifact, platform: str): + self.name, self.platform, self.work = name, host_platform(), work + if names_registry(base.image_name): + docker_login(get_config().get_image_store().auth(base.pinned_name)) + else: + build_for(base, platform) + + def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: + out = self.work / RESOLVED / "out" + out.mkdir(parents=True) + (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) + shutil.copy(RESOLVER, out / "resolve.py") + try: + ran = subprocess.run(_resolver(source, arch, str(self.work / CONTEXT), str(out)), capture_output=True, + text=True, timeout=timeout) + except subprocess.TimeoutExpired as e: + raise ResolveError(f"resolving took over {timeout} s", str(e.stdout or "")) from None + if ran.returncode != 0: + raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) + return json.loads((out / "result.json").read_text()) + + def download(self, url: str, sha256: str, path: str) -> None: + target, hashed = self.work / path, hashlib.sha256() + target.parent.mkdir(parents=True, exist_ok=True) + try: + with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response: + response.raise_for_status() + with target.open("wb") as file: + for chunk in response.iter_bytes(_CHUNK): + hashed.update(chunk) + file.write(chunk) + except httpx.HTTPError as e: + raise ResolveError(f"downloading {url} failed: {e}") from None + if hashed.hexdigest() != sha256: + raise ResolveError(f"{url} has the sha256 {hashed.hexdigest()}, not the {sha256} its download names") + + def files(self, folder: str) -> dict[str, Hashed]: + root, found = self.work / folder, {} + for directory, _, names in os.walk(root): + for name in names: + path = Path(directory) / name + if path.is_file() and not path.is_symlink(): + found[path.relative_to(root).as_posix()] = Hashed(sha256_of(path), path.stat().st_size) + return found + + def store(self, path: str, store, key: str) -> None: + store.put_file(key, str(self.work / path)) + + def place(self, copies: dict[str, str], fetches: dict[str, tuple[str, str]]) -> None: + for target, source in copies.items(): + (self.work / target).parent.mkdir(parents=True, exist_ok=True) + (self.work / target).unlink(missing_ok=True) + os.link(self.work / source, self.work / target) + for target, (url, sha256) in fetches.items(): + (self.work / target).parent.mkdir(parents=True, exist_ok=True) + get_config().get_object_store_at(url).download_to_file(url, str(self.work / target)) + if (found := sha256_of(self.work / target)) != sha256: + raise RuntimeError(f"the store's {url} isn't the file this build stages: its sha256 is {found}") + + def mark_runnable(self, paths: list[str]) -> None: + for path in paths: + with (self.work / path).open("rb") as file: + if file.read(4).startswith(RUN_ONLY): + (self.work / path).chmod(EXECUTABLE) + + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + (self.work / context).mkdir(parents=True, exist_ok=True) + (self.work / context / "Dockerfile").write_text(dockerfile) + build_image(self.work / context / "Dockerfile", self.work / context, tag, platform=platform, labels=labels) + + def installed(self, tag: str, prefix: str) -> dict[str, str]: + created = subprocess.run(["docker", "create", tag], capture_output=True, text=True, check=True).stdout.strip() + try: + copied = subprocess.run(["docker", "cp", f"{created}:{prefix}/installed.json", "-"], capture_output=True) + finally: + subprocess.run(["docker", "rm", created], capture_output=True) + if copied.returncode != 0: + return {} + with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as archive: + return _packages(archive.extractfile(archive.getmembers()[0]).read()) + + def push(self, tag: str, ref: str, store) -> None: + push_local_image(tag, ref, store) + + +class VmBuilder(Builder): + """A VM of a sandbox provider with Docker in it, building in a folder of its own there. Its calls run on an event + loop of its own, in the thread that made it.""" + + def __init__(self, link: SandboxProvider): + self.name, self._runner = type(link).__name__, asyncio.Runner() + self.work = f"/tmp/agentenv-compose-{uuid.uuid4().hex[:12]}" + try: + self.sandbox: VmSandbox = self._run(link.create_vm(exposed_ports=[], **_VM)) + except BaseException: + self._runner.close() + raise + self.platform = self.sandbox.platform + + def start(self, work: Path, base: DockerImageArtifact, platform: str, + from_store: dict[str, tuple[str, str]]) -> None: + """Give the VM's Docker BuildKit when it has none, register emulation for ``platform`` when it isn't the VM's + own, sign in to the base's registry or build the base from its build context, and put the build context here: + the files of ``from_store`` from the store, and the rest, which only the putting machine holds, inline.""" + plugin = f"{_PLUGINS}/docker-buildx" + self._exec(f"docker buildx version >/dev/null 2>&1 || {{ mkdir -p {_PLUGINS} && curl -fsSL {CURL_RETRY_FLAGS} " + f"{BUILDX.format(platform=self.platform.replace('/', '-'))} -o {plugin} && echo " + f"'{BUILDX_SHA256[self.platform]} {plugin}' | sha256sum --check --quiet && chmod 755 {plugin}; }}") + if platform != self.platform: + # The VM's own binfmt_misc first: a newer kernel gives a mount of it in a container an instance of its own. + try: + self._exec("mountpoint -q /proc/sys/fs/binfmt_misc || mount -t binfmt_misc binfmt_misc " + f"/proc/sys/fs/binfmt_misc\ndocker run --privileged --rm {BINFMT} --install " + f"{platform.rsplit('/', 1)[-1]}") + except RuntimeError as e: + raise Unbuildable(f"{self.name}'s VM couldn't register emulation for {platform}: {e}") from None + if names_registry(base.image_name): + self._login(get_config().get_image_store().auth(base.pinned_name)) + else: + self._base(base, platform) + self._write(f"{self.work}/{CONTEXT}.tar.gz", _inline(work / CONTEXT, set(from_store))) + self._exec(f"mkdir -p {self.work}/{CONTEXT} && tar -xzf {self.work}/{CONTEXT}.tar.gz --no-same-owner " + f"-C {self.work}/{CONTEXT} && rm {self.work}/{CONTEXT}.tar.gz") + self.place({}, {f"{CONTEXT}/{path}": fetch for path, fetch in from_store.items()}) + self._chmod([f"{CONTEXT}/{path}" for path in from_store if os.access(work / CONTEXT / path, os.X_OK)]) + + def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: + out = f"{self.work}/{RESOLVED}/out" + self._write(f"{out}/request.json", json.dumps({**request, "owner": [0, 0]}).encode()) + self._write(f"{out}/resolve.py", RESOLVER.read_bytes()) + code, stdout, stderr = self._output( + f"timeout {timeout} {shlex.join(_resolver(source, arch, f'{self.work}/{CONTEXT}', out))}") + if code == 124: + raise ResolveError(f"resolving took over {timeout} s", stdout + stderr) + if code != 0: + raise ResolveError("resolving the packages failed", stdout + stderr) + return json.loads(self._exec(f"cat {out}/result.json")) + + def download(self, url: str, sha256: str, path: str) -> None: + target = shlex.quote(f"{self.work}/{path}") + try: + self._exec(f"mkdir -p $(dirname {target}) && curl -fsSL {CURL_RETRY_FLAGS} {shlex.quote(url)} -o {target}") + except RuntimeError as e: + raise ResolveError(f"downloading {url} failed: {e}") from None + if (found := self._exec(f"sha256sum {target}").split()[0]) != sha256: + raise ResolveError(f"{url} has the sha256 {found}, not the {sha256} its download names") + + def files(self, folder: str) -> dict[str, Hashed]: + listed = self._exec( + f"cd {shlex.quote(f'{self.work}/{folder}')} && find . -type f -printf '%P\\0' | " + "while IFS= read -r -d '' f; do printf '%s %s %s\\0' \"$(sha256sum < \"$f\" | cut -c1-64)\" " + "\"$(stat -c %s \"$f\")\" \"$f\"; done") + found = {} + for entry in filter(None, listed.split("\0")): + sha256, size, path = entry.split(" ", 2) + found[path] = Hashed(sha256, int(size)) + return found + + def store(self, path: str, store, key: str) -> None: + self._run(upload_vm_file(self.sandbox, f"{self.work}/{path}", store, store.object_url(key))) + + def place(self, copies: dict[str, str], fetches: dict[str, tuple[str, str]]) -> None: + targets = [*copies, *fetches] + self._each([f"mkdir -p {shlex.quote(str(PurePosixPath(self.work, target).parent))}" for target in targets]) + self._each([f"ln -f {shlex.quote(f'{self.work}/{source}')} {shlex.quote(f'{self.work}/{target}')}" + for target, source in copies.items()]) + if not fetches: + return + self._run(self._fetch({f"{self.work}/{target}": url for target, (url, _) in fetches.items()})) + listed = "".join(f"{sha256} {target}\n" for target, (_, sha256) in fetches.items()) + self._write(f"{self.work}/fetched.sha256", listed.encode()) + try: + self._exec(f"cd {self.work} && sha256sum --check --quiet fetched.sha256") + except RuntimeError as e: + raise RuntimeError(f"a file the store served isn't the one this build stages: {e}") from None + + def mark_runnable(self, paths: list[str]) -> None: + self._each([f"case \"$(head -c 4 {shlex.quote(f'{self.work}/{path}')} | od -An -tx1 | tr -d ' \\n')\" in " + f"2321*|7f454c46) chmod {EXECUTABLE:o} {shlex.quote(f'{self.work}/{path}')};; esac" + for path in paths]) + + def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: + folder = f"{self.work}/{context}" + self._write(f"{folder}/Dockerfile", dockerfile.encode()) + options = " ".join(f"--label {shlex.quote(f'{name}={value}')}" for name, value in labels.items()) + code, stdout, stderr = self._output(f"cd {folder} && docker build --platform {shlex.quote(platform)} {options} " + f"-f Dockerfile -t {shlex.quote(tag)} . 2>&1") + if code != 0: + raise DockerBuildError(f"docker build of {tag} failed (exit {code}):\n{(stdout + stderr).rstrip()}") + + def installed(self, tag: str, prefix: str) -> dict[str, str]: + copied = f"{self.work}/installed.json" + listed = self._exec(f"c=$(docker create {shlex.quote(tag)}) && {{ docker cp \"$c:{prefix}/installed.json\" " + f"{copied} >/dev/null 2>&1 && cat {copied}; docker rm \"$c\" >/dev/null; }}") + return _packages(listed.encode()) if listed.strip() else {} + + def push(self, tag: str, ref: str, store) -> None: + self._login(store.auth(ref)) + self._exec(f"docker tag {shlex.quote(tag)} {shlex.quote(ref)} && docker push {shlex.quote(ref)}") + + def close(self) -> None: + """Tear the VM down; one that won't go is left to its provider's timeout, since what it built is pushed.""" + try: + self._run(self.sandbox.terminate()) + except Exception as e: + logger.warning(f"{self.name} VM {self.sandbox.sandbox_id} wasn't torn down, so its provider stops it at its " + f"timeout: {type(e).__name__}: {e}") + finally: + self._runner.close() + + def _login(self, auth) -> None: + if auth is not None: + self._exec(f"echo {shlex.quote(auth.password)} | docker login --username {shlex.quote(auth.username)} " + f"--password-stdin {shlex.quote(auth.registry)}") + + def _base(self, base: DockerImageArtifact, platform: str) -> None: + """Build ``base`` for ``platform`` from its stored build context, under the tag a local build gives it.""" + folder = f"{self.work}/base" + self._exec(f"mkdir -p {folder}") + self._run(self.sandbox.load_object_file(base.build_context_object_url, f"{folder}.tar.gz")) + code, stdout, stderr = self._output( + f"tar -xzf {folder}.tar.gz -C {folder} && cd {folder} && docker build --platform {shlex.quote(platform)} " + f"-f {shlex.quote(base.dockerfile_path)} -t {shlex.quote(platform_tag(base, platform))} . 2>&1") + if code != 0: + raise DockerBuildError(f"docker build of the base {base.image_name} failed (exit {code}):\n" + f"{(stdout + stderr).rstrip()}") + + async def _fetch(self, urls: dict[str, str]) -> None: + slots = asyncio.Semaphore(_FETCHES_IN_FLIGHT) + + async def fetch(path: str, url: str) -> None: + async with slots: + await self.sandbox.load_object_file(url, path) + + await asyncio.gather(*(fetch(path, url) for path, url in urls.items())) + + def _chmod(self, paths: list[str]) -> None: + self._each([f"chmod {EXECUTABLE:o} {shlex.quote(f'{self.work}/{path}')}" for path in paths]) + + def _each(self, commands: list[str]) -> None: + """Run ``commands`` in as few scripts as fit a script's limit.""" + script = "" + for command in commands: + if script and len(script) + len(command) > self.sandbox._WFT_CHUNK_BYTES: + self._exec(script) + script = "" + script += f"{command}\n" + if script: + self._exec(script) + + def _write(self, path: str, data: bytes) -> None: + self._run(self.sandbox.write_host_file(data, path)) + + def _exec(self, script: str) -> str: + return self._run(self.sandbox.exec_script(f"set -e\n{script}")) + + def _output(self, script: str) -> tuple[int, str, str]: + return self._run(self.sandbox.exec_with_output("sudo", "bash", "-c", script)) + + def _run(self, coroutine): + """``coroutine`` run on the builder's loop; what its provider's client raises, as a RuntimeError naming the VM.""" + try: + return self._runner.run(coroutine) + except (RuntimeError, ValueError, OSError, ObjectAlreadyExistsError): + raise + except Exception as e: + raise RuntimeError(f"{self.name}'s VM failed: {type(e).__name__}: {e}") from e + + +def _inline(context: Path, skip: set[str]) -> bytes: + """A tar.gz of the folders and files under ``context`` but those of ``skip``, with their modes.""" + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + for path in sorted(context.rglob("*")): + if (name := path.relative_to(context).as_posix()) not in skip: + archive.add(path, arcname=name, recursive=False) + return buffer.getvalue() + + +def _packages(listed: bytes) -> dict[str, str]: + return {package["name"]: package["version"] for package in json.loads(listed)} + + +def sha256_of(path: Path) -> str: + hashed = hashlib.sha256() + with path.open("rb") as file: + while chunk := file.read(_CHUNK): + hashed.update(chunk) + return hashed.hexdigest() diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index d8c516be3..910c90da5 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -1,23 +1,22 @@ -"""The composer: a computer env's base, driver and apps built into one image for each form and arch with this machine's -Docker, tagged in the env's image store by a key of everything that goes into it, so an image of that key, built for any -env, is reused. +"""The composer: a computer env's base, driver and apps built into one image for each form and arch, on a builder the +put's sandbox provider gives (``builder``), tagged in the env's image store by a key of everything that goes into it, so +an image of that key, built for any env and on any builder, is reused. Each result is recorded in the env's metadata as it finishes (``composed``), and the arches that built are written as the env's computer, a ``computer`` artifact over an index of their images.""" from __future__ import annotations import asyncio +import contextlib import copy import hashlib import importlib.metadata -import io import json import logging import re import shlex import shutil import subprocess -import tarfile import tempfile import threading import urllib.parse @@ -33,21 +32,21 @@ from agentenv_protocol.computer.conformance import run_probe from agent_env.artifact.artifacts.computer import ComputerArtifact -from agent_env.artifact.artifacts.docker_image import DockerImageArtifact, docker_login, push_local_image +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.artifacts.file import FileArtifact from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.config import get_config from agent_env.entity.export import Unwritable, checked_path from agent_env.env.env import Env -from agent_env.env.envs.computer.utils import boot, gate, packages +from agent_env.env.envs.computer.utils import boot, builder, gate, packages from agent_env.env.envs.computer.utils.install import Install -from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag +from agent_env.providers.sandbox_providers.host_images import platform_tag from agent_env.store.base import ConcurrentModificationError from agent_env.store.ids import derive_id from agent_env.store.image_store.image_contents import labels from agent_env.store.image_store.oci_registry_credentials import names_registry -from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest -from agent_env.utils.docker.build import DockerBuildError, build_image +from agent_env.store.image_store.registry_api import NoSuchImageError, pin_digest, write_index +from agent_env.utils.docker.build import DockerBuildError if TYPE_CHECKING: from agentenv_protocol.computer.conformance import ProbeReport @@ -80,19 +79,23 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, - on_build: Callable[[], None] | None = None) -> None: - """Build each of ``pairs``, by default each form the base has in each arch the env composes, in parallel, and probe - each one a configured sandbox provider runs natively (gate 3), recording each as it starts and finishes, then write - the env's computer over every pair built so far. ``on_build`` is called once, just before the first image is built - here; reusing them all calls it never.""" + on_build: Callable[[], None] | None = None, sandbox_provider: SandboxProvider | None = None) -> None: + """Build each of ``pairs``, by default each form the base has in each arch the env composes, in parallel, on + ``sandbox_provider``, by default the env's, and probe each one it runs natively (gate 3), recording each as it + starts and finishes, then write the env's computer over every pair built so far. ``on_build`` is called once, just + before the first image is built; reusing them all calls it never.""" + from agent_env.providers import get_env_sandbox_provider + base, driver = env.computer, env.driver building = _once(on_build) + builds = builder.Builds(sandbox_provider or get_env_sandbox_provider(), env.stored_id, REPOSITORY) pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver, env.composed_apps())] record = _Record(env) record.started(pairs) with ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: - futures = {pool.submit(_composed, env, base, driver, form, arch, building): (form, arch) for form, arch in pairs} + futures = {pool.submit(_composed, env, base, driver, form, arch, builds, building): (form, arch) + for form, arch in pairs} for future in as_completed(futures): form, arch = futures[future] result = future.result() @@ -111,12 +114,12 @@ def current(env: ComputerEnv) -> bool: return "computer" in env.metadata and env.metadata["computer"].get("images") == _Record(env).built() -def pending(env: ComputerEnv) -> list[tuple[str, str]]: +def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) -> list[tuple[str, str]]: """The forms and arches of ``env`` left to compose: never started, failed, left building by a put that stopped, or - built but unprobed though a configured sandbox provider runs the arch natively.""" + built but unprobed though ``sandbox_provider``, by default the env's, runs the arch natively.""" from agent_env.providers import get_env_sandbox_provider - composed, provider = env.metadata.get("composed", {}), get_env_sandbox_provider() + composed, provider = env.metadata.get("composed", {}), sandbox_provider or get_env_sandbox_provider() statuses = {(form, arch): composed.get(form, {}).get(arch, {}).get("status") for form in gate.forms(env.computer) for arch in gate.arches(env.computer, env.driver, env.composed_apps())} @@ -125,20 +128,19 @@ def pending(env: ComputerEnv) -> list[tuple[str, str]]: def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, - building: Callable[[], None]) -> dict: + builds: builder.Builds, building: Callable[[], None]) -> dict: """``form`` and ``arch`` built or reused, then probed when it built (gate 3).""" - result = _built(env, base, driver, form, arch, building) - return _probed(env, driver, form, arch, result) if result["status"] == "unverified" else result + result = _built(env, base, driver, form, arch, builds, building) + return _probed(env, driver, form, arch, result, builds.provider) if result["status"] == "unverified" else result -def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict) -> dict: - """Gate 3: the build of ``result`` booted on the first configured sandbox provider that runs ``arch`` natively, and +def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, + sandbox_provider: SandboxProvider) -> dict: + """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch`` natively, and probed at ``full``: ready, with the driver's card, or failed at the probe; each with a summary of the probe and its - report stored. Still unverified, saying why, when no configured provider runs ``arch`` natively.""" - from agent_env.providers import get_env_sandbox_provider - - if not (links := boot.native_links(get_env_sandbox_provider(), arch)): - return {**result, "reason": f"no configured sandbox provider runs linux/{arch} natively to probe it"} + report stored. Still unverified, saying why, when no link runs ``arch`` natively.""" + if not (links := boot.native_links(sandbox_provider, arch)): + return {**result, "reason": f"no link of the sandbox provider runs linux/{arch} natively to probe it"} try: report, card, provider = asyncio.run(_probe(links, env, driver, result["image"])) except TimeoutError: @@ -181,34 +183,33 @@ async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: Compute def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, - building: Callable[[], None]) -> dict: - """The status of ``form`` and ``arch`` once built and pushed, or reused, or the stage that failed and why; - ``building`` is called before anything builds.""" + builds: builder.Builds, building: Callable[[], None]) -> dict: + """The status of ``form`` and ``arch`` once built and pushed, or reused, or the stage that failed and why. A builder + from ``builds`` resolves and builds it, opened when first needed, after ``building`` is called.""" stage, key = "build", None try: - with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work: - context, fetched = Path(work) / "context", {} + with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened: + context, fetched, from_store = Path(work) / builder.CONTEXT, {}, {} context.mkdir() source, base_identity = _base(base, arch) - layer = _stage_driver(context, driver, base) - apps = [_stage_app(context, app, base) for app in env.composed_apps()] + layer = _stage_driver(context, driver, base, from_store) + apps = [_stage_app(context, app, base, from_store) for app in env.composed_apps()] layers = [layer, *apps] wants = _wants(layers, _stage_protocol(context)) - platform = f"linux/{arch}" - emulated = platform != host_platform() - unemulated = f"this machine can't build {platform}: its Docker has no emulation for it" if ( - emulated and platform not in _emulated_platforms()) else None - ready = _once(lambda: _ready_base(base, arch)) + platform, held = f"linux/{arch}", [] + + def on() -> builder.Builder: + if not held: + link = builds.link(platform) + building() + held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) + return held[0] + inputs = packages.inputs_digest(wants, context, base_identity, arch) lock = packages.stored(inputs, env.stored_id) if inputs is not None else None if inputs is not None and lock is None: - if unemulated: - return _failed(env, form, arch, "resolve", None, unemulated) stage = "resolve" - building() - ready() - lock, fetched = packages.resolve(inputs, wants, context, source, arch, env.stored_id, - Path(work) / "resolve") + lock, fetched = packages.resolve(inputs, wants, on(), source, arch, env.stored_id) stage = "build" dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) key = "sha256:" + hashlib.sha256(json.dumps({ @@ -224,28 +225,28 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, return {"status": "unverified", "key": key, "image": pinned, "reused": True, "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": _now()} - if unemulated: - return _failed(env, form, arch, "build", key, unemulated) - building() - ready() + building_on = on() if lock is not None: - packages.stage(lock, context, env.stored_id, fetched) - (context / "Dockerfile").write_text(f"{dockerfile}LABEL {KEY_LABEL}={key}\n") - local = f"agentenv-computer:{key[7:23]}-{arch}" - build_image(context / "Dockerfile", context, local, platform=platform, labels={KEY_LABEL: key}) - installed = _installed(local, layer.prefix) - _label(local, platform, {INSTALLED_LABEL: json.dumps(installed, sort_keys=True), - EMULATED_LABEL: "true" if emulated else "false"}) + packages.stage(lock, building_on, env.stored_id, fetched) + local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform + building_on.build(f"{dockerfile}LABEL {KEY_LABEL}={key}\n", local, platform, {KEY_LABEL: key}) + installed = building_on.installed(local, layer.prefix) + building_on.build(f"FROM {local}\n", local, platform, { + INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", + }, context="label") stage = "push" - push_local_image(local, ref, store) + building_on.push(local, ref, store) return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, - "emulated": emulated, "lock": inputs, "installed": installed, "built_at": _now()} - except packages.ResolveError as e: + "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, + "built_at": _now()} + except builder.Unbuildable as e: + return _failed(env, form, arch, stage, key, str(e)) + except builder.ResolveError as e: return _failed(env, form, arch, "resolve", None, str(e), log=e.log or None) except DockerBuildError as e: # Docker can list a platform it has lost the emulation for, until it restarts; the build says so. unemulated = "exec format error" in str(e) - return _failed(env, form, arch, stage, key, f"this machine can't run linux/{arch} programs: its Docker has no " + return _failed(env, form, arch, stage, key, f"the builder can't run linux/{arch} programs: its Docker has no " "emulation for it" if unemulated else "the build failed", log=str(e)) except subprocess.CalledProcessError as e: return _failed(env, form, arch, stage, key, f"{' '.join(e.cmd[:2])} failed: {(e.stderr or '').strip()}") @@ -277,16 +278,6 @@ def _base(base: ComputerArtifact, arch: str) -> tuple[str, str]: return platform_tag(image, f"linux/{arch}"), f"sources:{image.source_digest}" -def _ready_base(base: ComputerArtifact, arch: str) -> None: - """Have this machine's Docker able to build ``FROM`` the base for ``arch``: signed in to its registry with the image - store's credentials for it, or holding it built from its stored build context.""" - image = base.image() - if names_registry(image.image_name): - docker_login(get_config().get_image_store().auth(image.pinned_name)) - else: - build_for(image, f"linux/{arch}") - - @dataclass(frozen=True) class _Layer: """A driver's or an app's part of a build: its folder in the build context, its prefix in the image, what it @@ -302,16 +293,19 @@ class _Layer: steps: tuple[tuple[str, str], ...] = () -def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact) -> _Layer: +def _stage_driver(context: Path, driver: ComputerDriverEnv, base: ComputerArtifact, + from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the driver's install files under ``context/driver/`` and its supervisord programs beside them, served on its port, its folder under ``DRIVERS`` named for its id's last part.""" install, name = driver.install[base.os.type], _name(driver.id, "driver") layer = _Layer("driver", f"{DRIVERS}/{name}", install, f"agentenv-{name}.conf") - _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}) + _stage(context, layer, f"driver {driver.id}", driver.files, _services(name, install), {"MCP_PORT": driver.port}, + from_store) return layer -def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact) -> _Layer: +def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, + from_store: dict[str, tuple[str, str]]) -> _Layer: """Write the app's install files under ``context/apps//``, with the values of its build settings the env fixes, and its supervisord programs beside them, run under its name: its services and its own server, which serves its class from its venv and launches the app. Its folder under ``APPS`` is named for its id's last part.""" @@ -323,7 +317,8 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact) -> _L if step.os is None or base.os.type in step.os)) programs = {f"app-{name}": f"venv/bin/python -m agentenv_protocol.computer.serve_app {served}", **_services(f"app-{name}", install)} - _stage(context, layer, f"app {app.id}", app.files, programs, {"ENVIRONMENT_NAME": app.environment_name}) + _stage(context, layer, f"app {app.id}", app.files, programs, {"ENVIRONMENT_NAME": app.environment_name}, + from_store) if app.build_settings: (context / layer.folder / BUILD_SETTINGS).parent.mkdir(exist_ok=True) (context / layer.folder / BUILD_SETTINGS).write_text(json.dumps(app.build_settings, sort_keys=True)) @@ -340,9 +335,10 @@ def _services(name: str, install: Install) -> dict[str, str]: def _stage(context: Path, layer: _Layer, what: str, files: FileArtifactUniverse, programs: dict[str, str], - environment: dict[str, Any]) -> None: + environment: dict[str, Any], from_store: dict[str, tuple[str, str]]) -> None: """Write the ``files`` of ``what``'s folder its install names to ``layer``'s folder, a script, a binary or a file its - install runs executable, and ``programs``, commands by name, as its supervisord programs, with ``environment``.""" + install runs executable, each added to ``from_store`` by its path with its object URL and sha256, for a builder to + fetch; and ``programs``, commands by name, as its supervisord programs, with ``environment``.""" folder = PurePosixPath(layer.install.files) runs = {PurePosixPath(run.script) for run in layer.install.run or []} for name, artifact in files.get_file_artifacts().items(): @@ -354,8 +350,9 @@ def _stage(context: Path, layer: _Layer, what: str, files: FileArtifactUniverse, target = context / layer.folder / path.relative_to(folder) target.parent.mkdir(parents=True, exist_ok=True) target.write_bytes(data := artifact.load()) - if data.startswith(packages.RUN_ONLY) or path.relative_to(folder) in runs: - target.chmod(packages.EXECUTABLE) + from_store[target.relative_to(context).as_posix()] = (artifact.object_url, hashlib.sha256(data).hexdigest()) + if data.startswith(builder.RUN_ONLY) or path.relative_to(folder) in runs: + target.chmod(builder.EXECUTABLE) (context / layer.folder).mkdir(parents=True, exist_ok=True) (context / layer.conf).write_text(_programs(programs, layer.prefix, environment)) @@ -471,30 +468,6 @@ def _wants(layers: list[_Layer], protocol: bool) -> list[packages.Want]: return wants -def _installed(local: str, prefix: str) -> dict[str, str]: - """The packages ``local``'s build installed into the venv of the driver at ``prefix``, by name, read from the image - without running it, so on any arch; none for a driver with no venv.""" - created = subprocess.run(["docker", "create", local], capture_output=True, text=True, check=True).stdout.strip() - try: - copied = subprocess.run(["docker", "cp", f"{created}:{prefix}/installed.json", "-"], - capture_output=True) - finally: - subprocess.run(["docker", "rm", created], capture_output=True) - if copied.returncode != 0: - return {} - with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as archive: - listed = json.loads(archive.extractfile(archive.getmembers()[0]).read()) - return {package["name"]: package["version"] for package in listed} - - -def _label(local: str, platform: str, extra: dict[str, str]) -> None: - """Add ``extra``'s labels to this machine's image ``local``, which adds no layer.""" - with tempfile.TemporaryDirectory(prefix="agentenv-label-") as work: - dockerfile = Path(work) / "Dockerfile" - dockerfile.write_text(f"FROM {local}\n") - build_image(dockerfile, Path(work), local, platform=platform, labels=extra) - - def _served(ref: str, store) -> str | None: """``ref`` pinned to the digest the image store serves for it, or None when it serves none.""" try: @@ -503,16 +476,6 @@ def _served(ref: str, store) -> str | None: return None -def _emulated_platforms() -> set[str]: - """The platforms this machine's Docker builds, its own and those it emulates.""" - inspected = subprocess.run(["docker", "buildx", "inspect", "--bootstrap"], capture_output=True, text=True) - platforms = set() - for line in inspected.stdout.splitlines(): - if line.strip().startswith("Platforms:"): - platforms |= {item.strip().rstrip("*") for item in line.split(":", 1)[1].split(",")} - return platforms - - def _failed(env: ComputerEnv, form: str, arch: str, stage: str, key: str | None, reason: str, log: str | None = None) -> dict: result = {"status": "failed", "stage": stage, "reason": reason, "failed_at": _now()} @@ -537,8 +500,7 @@ def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str]) - store = get_config().get_image_store_for(env.stored_id) index = store.image_ref(REPOSITORY, "index-" + hashlib.sha256( ",".join(built[arch] for arch in arches).encode()).hexdigest()) - docker_login(store.auth(index)) - _joined(index, [built[arch] for arch in arches]) + write_index(index, {f"linux/{arch}": built[arch] for arch in arches}, store.auth(index)) image = DockerImageArtifact.put_ref(derive_id(env.stored_id, "computer_image"), description=( f"{env.stored_id}'s composed computer"), image_name=index) config = {name: setting.model_dump() for name, setting in base.config.items()} @@ -555,14 +517,6 @@ def _computer(env: ComputerEnv, base: ComputerArtifact, built: dict[str, str]) - return {"id": computer.id, "version": computer.version} -def _joined(index: str, images: list[str]) -> None: - """Push one index of ``images`` to ``index``, as the release joins its platforms' images.""" - joined = subprocess.run(["docker", "buildx", "imagetools", "create", "-t", index, *images], - capture_output=True, text=True) - if joined.returncode != 0: - raise RuntimeError(f"joining {', '.join(images)} into {index} failed: {joined.stderr.strip()}") - - def _digest(folder: Path) -> str: """A digest of the files under ``folder``, by their paths and bytes.""" hashed = hashlib.sha256() @@ -572,7 +526,8 @@ def _digest(folder: Path) -> str: def _composer_digest() -> str: - return "sha256:" + hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + """A digest of the composer and its builders, which say what a build makes of its inputs.""" + return "sha256:" + hashlib.sha256(Path(__file__).read_bytes() + Path(builder.__file__).read_bytes()).hexdigest() def _now() -> str: diff --git a/src/agent_env/env/envs/computer/utils/packages.py b/src/agent_env/env/envs/computer/utils/packages.py index b3016b24e..f14ed632b 100644 --- a/src/agent_env/env/envs/computer/utils/packages.py +++ b/src/agent_env/env/envs/computer/utils/packages.py @@ -9,39 +9,22 @@ import hashlib import json -import os -import shutil -import subprocess from dataclasses import dataclass -from pathlib import Path +from pathlib import Path, PurePosixPath -import httpx from pydantic import BaseModel from agent_env.config import get_config +from agent_env.env.envs.computer.utils.builder import ( + CONTEXT, RESOLVED, RESOLVER, Builder, Hashed, ResolveError, sha256_of) from agent_env.env.envs.computer.utils.install import Install from agent_env.store.base import ObjectAlreadyExistsError #: Where locks and their files are kept, under the configured key prefix, in the store the env's id routes to. PREFIX = "agentenv-packages" -RESOLVER = Path(__file__).with_name("resolve_in_base.py") #: Where a build's context holds a lock's .debs; each venv's wheels are at ``staged_wheels``. STAGED_APT = "packages/apt" _RESOLVE_TIMEOUT_S = 3600 -_DOWNLOAD_TIMEOUT_S = 600 -_CHUNK = 1 << 20 -#: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged, -#: since the stores keep no file modes. -RUN_ONLY = (b"#!", b"\x7fELF") -EXECUTABLE = 0o755 - - -class ResolveError(RuntimeError): - """Resolving or fetching failed; ``log`` holds what the resolver printed.""" - - def __init__(self, message: str, log: str = ""): - super().__init__(message) - self.log = log class LockedFile(BaseModel): @@ -98,13 +81,14 @@ def inputs_digest(wants: list[Want], context: Path, base: str, arch: str) -> str install, folder = want.install, context / want.folder layers.append({ "folder": want.folder, "apt": install.packages.apt if install.packages else [], - "sources": [{**source.model_dump(), "key": _sha256(folder / source.key)} for source in install.sources or []], - "python": _sha256(folder / install.python) if install.python is not None else None, + "sources": [{**source.model_dump(), "key": sha256_of(folder / source.key)} + for source in install.sources or []], + "python": sha256_of(folder / install.python) if install.python is not None else None, "venv": want.venv, "adds": list(want.adds), "downloads": [download.model_dump() for download in _downloads(install, arch)]}) protocol = context / "protocol" return "sha256:" + hashlib.sha256(json.dumps({ - "layers": layers, "base": base, "arch": arch, "request": _request(wants, arch), "resolver": _sha256(RESOLVER), + "layers": layers, "base": base, "arch": arch, "request": _request(wants, arch), "resolver": sha256_of(RESOLVER), "lock": _lock_shape(), "protocol": _folder_digest(protocol) if protocol.is_dir() else None, }, sort_keys=True).encode()).hexdigest() @@ -116,20 +100,27 @@ def stored(inputs: str, store_id: str) -> Lock | None: return Lock.model_validate_json(store.read(key)) if store.exists(key) else None -def resolve(inputs: str, wants: list[Want], context: Path, source: str, arch: str, store_id: str, - work: Path) -> tuple[Lock, dict[str, Path]]: - """Resolve ``wants`` for ``arch`` in a container of ``source``, the base's image, with their folders and the - protocol's source from ``context``, into ``work``; fetch their downloads; store every file and the lock. The lock, - and where each file this resolve fetched is under ``work``, by its sha256, for a build that follows to stage it from - there rather than the store.""" +def resolve(inputs: str, wants: list[Want], builder: Builder, source: str, arch: str, + store_id: str) -> tuple[Lock, dict[str, str]]: + """Resolve ``wants`` for ``arch`` with ``builder``, in a container of ``source``, the base's image, with their + folders and the protocol's source from its build context; fetch their downloads; store every file and the lock. The + lock, and where on the builder each file this resolve fetched is, by its sha256, for a build that follows to stage + it from there rather than the store.""" (store, prefix), fetched = _store(store_id), {} - result, out = _run_resolver(_request(wants, arch), context, source, arch, work) - apt = [LockedDeb(**{**deb, **_put(store, prefix, fetched, _written(out, "apt", deb["file"]))}) - for deb in result["apt"]] - wheels = {want.folder: [LockedFile(**_put(store, prefix, fetched, _written(out, "wheels", want.name, wheel))) + result = builder.resolve(_request(wants, arch), source, arch, _RESOLVE_TIMEOUT_S) + written = builder.files(f"{RESOLVED}/out") + + def put(folder: str, path: str) -> dict: + return _put(builder, store, prefix, fetched, written, folder, path) + + apt = [LockedDeb(**{**deb, **put("out", f"apt/{deb['file']}")}) for deb in result["apt"]] + wheels = {want.folder: [LockedFile(**put("out", f"wheels/{want.name}/{wheel}")) for wheel in result["wheels"][want.name]] for want in wants if want.venv} - downloads = {want.folder: [LockedDownload(url=download.url, to=download.to, **_put( - store, prefix, fetched, _fetch(download.url, download.sha256, work))) + for want in wants: + for download in _downloads(want.install, arch): + builder.download(download.url, download.sha256, f"{RESOLVED}/downloads/{download.sha256}") + written = builder.files(f"{RESOLVED}/downloads") if any(_downloads(want.install, arch) for want in wants) else {} + downloads = {want.folder: [LockedDownload(url=download.url, to=download.to, **put("downloads", download.sha256)) for download in _downloads(want.install, arch)] for want in wants if _downloads(want.install, arch)} lock = Lock(inputs=inputs, arch=arch, apt=apt, wheels=wheels, downloads=downloads) @@ -141,24 +132,22 @@ def resolve(inputs: str, wants: list[Want], context: Path, source: str, arch: st return lock, fetched -def stage(lock: Lock, context: Path, store_id: str, fetched: dict[str, Path]) -> None: - """Write the lock's files into ``context`` for a build: .debs under ``STAGED_APT``, each venv's wheels under +def stage(lock: Lock, builder: Builder, store_id: str, fetched: dict[str, str]) -> None: + """Write the lock's files into ``builder``'s build context: .debs under ``STAGED_APT``, each venv's wheels under ``staged_wheels``, and each layer's downloads at their places in its folder, a script or binary executable; each - linked from ``fetched``, the files a resolve just fetched by sha256, or else downloaded from the store and refused + linked from ``fetched``, the files a resolve just fetched by sha256, or else fetched from the store and refused unless its sha256 is the one the lock names.""" store, prefix = _store(store_id) - for deb in lock.apt: - _get(store, prefix, fetched, deb.sha256, context / STAGED_APT / deb.file) - for folder, wheels in lock.wheels.items(): - for wheel in wheels: - _get(store, prefix, fetched, wheel.sha256, context / staged_wheels(folder) / wheel.file) - for folder, downloads in lock.downloads.items(): - for download in downloads: - target = context / folder / download.to - _get(store, prefix, fetched, download.sha256, target) - with target.open("rb") as file: - if file.read(4).startswith(RUN_ONLY): - target.chmod(EXECUTABLE) + places = {f"{CONTEXT}/{STAGED_APT}/{deb.file}": deb.sha256 for deb in lock.apt} + places |= {f"{CONTEXT}/{staged_wheels(folder)}/{wheel.file}": wheel.sha256 + for folder, wheels in lock.wheels.items() for wheel in wheels} + downloads = {f"{CONTEXT}/{folder}/{download.to}": download.sha256 + for folder, downloads in lock.downloads.items() for download in downloads} + places |= downloads + builder.place({target: fetched[sha256] for target, sha256 in places.items() if sha256 in fetched}, + {target: (store.object_url(f"{prefix}/files/{sha256}"), sha256) + for target, sha256 in places.items() if sha256 not in fetched}) + builder.mark_runnable(list(downloads)) def staged_wheels(folder: str) -> str: @@ -177,80 +166,24 @@ def _request(wants: list[Want], arch: str) -> dict: return {"apt": apt, "sources": sources, "python": python} -def _run_resolver(request: dict, context: Path, source: str, arch: str, work: Path) -> tuple[dict, Path]: - """Run the resolver in a container of ``source`` for ``arch``, as root, with the build context read-only at /ctx; - what it wrote, and the folder it wrote it in.""" - out = work / "out" - out.mkdir(parents=True) - (out / "request.json").write_text(json.dumps({**request, "owner": [os.getuid(), os.getgid()]})) - shutil.copy(RESOLVER, out / "resolve.py") - try: - ran = subprocess.run(["docker", "run", "--rm", "--platform", f"linux/{arch}", "--user", "root", - "--entrypoint", "python3", "-v", f"{context}:/ctx:ro", "-v", f"{out}:/out", source, - "/out/resolve.py", "/out/request.json", "/out"], - capture_output=True, text=True, timeout=_RESOLVE_TIMEOUT_S) - except subprocess.TimeoutExpired as e: - raise ResolveError(f"resolving took over {_RESOLVE_TIMEOUT_S} s", str(e.stdout or "")) from None - if ran.returncode != 0: - raise ResolveError("resolving the packages failed", ran.stdout + ran.stderr) - return json.loads((out / "result.json").read_text()), out - - -def _fetch(url: str, sha256: str, work: Path) -> Path: - """Download ``url`` into ``work``, refused unless its sha256 is ``sha256``.""" - target, hashed = work / "downloads" / sha256, hashlib.sha256() - target.parent.mkdir(parents=True, exist_ok=True) - try: - with httpx.stream("GET", url, follow_redirects=True, timeout=_DOWNLOAD_TIMEOUT_S) as response: - response.raise_for_status() - with target.open("wb") as file: - for chunk in response.iter_bytes(_CHUNK): - hashed.update(chunk) - file.write(chunk) - except httpx.HTTPError as e: - raise ResolveError(f"downloading {url} failed: {e}") from None - if hashed.hexdigest() != sha256: - raise ResolveError(f"{url} has the sha256 {hashed.hexdigest()}, not the {sha256} its download names") - return target - - def _downloads(install: Install, arch: str) -> list: return [download for download in install.downloads or [] if download.arch in (None, arch)] -def _written(out: Path, *parts: str) -> Path: - """The file the resolver says it wrote at ``parts`` under ``out``, refused unless it's a plain file there: a - dependency's build code, run in the container, could leave a link to one of this machine's files instead.""" - path = out.joinpath(*parts) - if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(out.resolve()): - raise ResolveError(f"the resolver's {'/'.join(parts)!r} isn't a plain file it wrote") - return path - - -def _put(store, prefix: str, fetched: dict[str, Path], path: Path) -> dict: - """Store the file at ``path`` by its sha256 unless it's there, and add it to ``fetched``; its name, sha256 and - size.""" - sha256 = _sha256(path) - key = f"{prefix}/files/{sha256}" +def _put(builder: Builder, store, prefix: str, fetched: dict[str, str], written: dict[str, Hashed], folder: str, + path: str) -> dict: + """Store the file at ``path`` in the resolve's ``folder`` by its sha256 unless it's there, and add it to + ``fetched``; its name, sha256 and size. Refused unless it's one of the plain files ``written`` there.""" + if (hashed := written.get(path)) is None: + raise ResolveError(f"the resolver's {path!r} isn't a plain file it wrote") + key, at = f"{prefix}/files/{hashed.sha256}", f"{RESOLVED}/{folder}/{path}" if not store.exists(key): try: - store.put_file(key, str(path)) + builder.store(at, store, key) except ObjectAlreadyExistsError: pass - fetched[sha256] = path - return {"file": path.name, "sha256": sha256, "size": path.stat().st_size} - - -def _get(store, prefix: str, fetched: dict[str, Path], sha256: str, target: Path) -> None: - target.parent.mkdir(parents=True, exist_ok=True) - if sha256 in fetched: - target.unlink(missing_ok=True) - os.link(fetched[sha256], target) - return - key = f"{prefix}/files/{sha256}" - store.download_to_file(store.object_url(key), str(target)) - if (found := _sha256(target)) != sha256: - raise RuntimeError(f"the store's {key} isn't the file the lock names: its sha256 is {found}") + fetched[hashed.sha256] = at + return {"file": PurePosixPath(path).name, "sha256": hashed.sha256, "size": hashed.size} def _store(store_id: str): @@ -258,18 +191,10 @@ def _store(store_id: str): return config.get_object_store_for(store_id), f"{config.get_artifact_key_prefix()}{PREFIX}" -def _sha256(path: Path) -> str: - hashed = hashlib.sha256() - with path.open("rb") as file: - while chunk := file.read(_CHUNK): - hashed.update(chunk) - return hashed.hexdigest() - - def _folder_digest(folder: Path) -> str: hashed = hashlib.sha256() for path in sorted(p for p in folder.rglob("*") if p.is_file()): - hashed.update(path.relative_to(folder).as_posix().encode() + b"\0" + bytes.fromhex(_sha256(path))) + hashed.update(path.relative_to(folder).as_posix().encode() + b"\0" + bytes.fromhex(sha256_of(path))) return hashed.hexdigest() diff --git a/src/agent_env/store/image_store/registry_api.py b/src/agent_env/store/image_store/registry_api.py index 9de283c47..583716407 100644 --- a/src/agent_env/store/image_store/registry_api.py +++ b/src/agent_env/store/image_store/registry_api.py @@ -1,10 +1,11 @@ """Read an image from its registry over the OCI distribution API: its digest, to pin a tag to the image it names now, -and its manifests and blobs, to inspect what it holds without pulling it.""" +and its manifests and blobs, to inspect what it holds without pulling it; and write an index of images it holds.""" from __future__ import annotations import contextlib import hashlib +import json import re from collections.abc import Iterator from dataclasses import dataclass @@ -21,6 +22,9 @@ "application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json", ]) +_INDEXES = {"application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"} +_DOCKER_TYPES = {"application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.docker.distribution.manifest.v2+json"} # The digests Docker pulls by: the algorithm, then its full-length lowercase hex. _DIGEST = re.compile(r"^(?:sha256:[a-f0-9]{64}|sha384:[a-f0-9]{96}|sha512:[a-f0-9]{128})$") _DOCKER_HUB_API = "registry-1.docker.io" @@ -159,6 +163,32 @@ def read_blob(ref: str, digest: str, auth: RegistryAuth | None, path: Path, *, raise ValueError(f"{ref}: the blob {image.host} served for {digest} hashes to sha256:{hashed.hexdigest()}") +def write_index(ref: str, images: dict[str, str], auth: RegistryAuth | None, *, + timeout: float = _TIMEOUT_SECONDS) -> None: + """Write under ``ref``'s tag an index of ``images``, by their platforms, each pinned by its digest in ``ref``'s + repository: an image's manifest, or every manifest of one that's an index itself, as ``docker buildx imagetools + create`` joins them. A Docker manifest list when every manifest is Docker's, else an OCI index.""" + image, manifests = _parse(ref), [] + with _Repository(image, auth, timeout, ref) as repository: + for platform, pinned in images.items(): + digest = _parse(pinned).digest + response = repository.get(f"manifests/{digest}", accept=_MANIFEST_TYPES) + served = response.headers.get("content-type", "").split(";")[0].strip() + if served in _INDEXES: + manifests += response.json()["manifests"] + else: + os_name, _, architecture = platform.partition("/") + manifests.append({"mediaType": served, "digest": digest, "size": len(response.content), + "platform": {"architecture": architecture, "os": os_name}}) + media = ("application/vnd.docker.distribution.manifest.list.v2+json" + if all(manifest["mediaType"] in _DOCKER_TYPES for manifest in manifests) + else "application/vnd.oci.image.index.v1+json") + # A repository of its own, so the token its first challenge asks for is one that pushes. + with _Repository(image, auth, timeout, ref) as repository: + repository.put(f"manifests/{image.tag or 'latest'}", json.dumps( + {"schemaVersion": 2, "mediaType": media, "manifests": manifests}).encode(), media) + + class _Repository: """One image's repository on its registry, read with the Bearer token the registry's first challenge asks for.""" @@ -187,6 +217,11 @@ def get(self, path: str, *, accept: str | None = None) -> httpx.Response: self._raise_for(response) return response + def put(self, path: str, content: bytes, content_type: str) -> None: + response = self._send("PUT", path, None, stream=False, content=content, content_type=content_type) + if response.status_code not in (200, 201): + self._raise_for(response) + @contextlib.contextmanager def stream(self, path: str) -> Iterator[httpx.Response]: """``path`` as a streamed response, closed when the block ends.""" @@ -199,13 +234,16 @@ def stream(self, path: str) -> Iterator[httpx.Response]: finally: response.close() - def _send(self, method: str, path: str, accept: str | None, *, stream: bool) -> httpx.Response: + def _send(self, method: str, path: str, accept: str | None, *, stream: bool, content: bytes | None = None, + content_type: str | None = None) -> httpx.Response: try: for _ in range(2): headers = {"Accept": accept} if accept else {} + if content_type: + headers["Content-Type"] = content_type if self._token: headers["Authorization"] = f"Bearer {self._token}" - request = self._client.build_request(method, f"{self._base}/{path}", headers=headers) + request = self._client.build_request(method, f"{self._base}/{path}", headers=headers, content=content) # Basic credentials go with every request until a Bearer token replaces them. response = self._client.send(request, auth=None if self._token else self._basic, stream=stream) if response.status_code != 401 or self._token: @@ -219,7 +257,9 @@ def _send(self, method: str, path: str, accept: str | None, *, stream: bool) -> return response except httpx.HTTPError as e: thing = "its manifest" if path.startswith("manifests/") else f"the blob {path.removeprefix('blobs/')}" - raise ValueError(f"{self._what}: couldn't read {thing} from {self._host}: {type(e).__name__}: {e}") from e + verb = "write it to" if method == "PUT" else "read it from" + raise ValueError(f"{self._what}: couldn't {verb.replace('it', thing)} {self._host}: " + f"{type(e).__name__}: {e}") from e def _raise_for(self, response: httpx.Response) -> None: if response.status_code == 200: diff --git a/tst/integration/computer/builder_modal_vm_test.py b/tst/integration/computer/builder_modal_vm_test.py new file mode 100644 index 000000000..ecb125fd2 --- /dev/null +++ b/tst/integration/computer/builder_modal_vm_test.py @@ -0,0 +1,60 @@ +"""A builder VM on Modal, amd64, building for arm64 through the emulation it registers: the build context sent inline, +an apt package and a requirement resolved in a container of the base, a file it wrote stored off the VM, an image built +from them with the network closed, and the packages it installed read back, each in the VM. The stores stay local, +which a compose on Modal refuses, so the builder is driven directly.""" + +import hashlib +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from agent_env.config import get_config +from agent_env.env.envs.computer.utils import builder +from agent_env.providers import build_sandbox_provider +from tst.util.capabilities import skip_without_remote_sandbox + +pytestmark = [pytest.mark.integration, pytest.mark.int_test_slow, skip_without_remote_sandbox("modal_vm")] + +BASE = "mirror.gcr.io/library/python:3.12-slim" +DOCKERFILE = f"""FROM {BASE} +COPY packages/ /tmp/packages/ +RUN --network=none apt-get install -y --no-install-recommends /tmp/packages/apt/*.deb +COPY driver/ /opt/driver/ +RUN --network=none python3 -m venv /opt/driver/venv && /opt/driver/venv/bin/pip install --no-index --no-deps \ +/tmp/packages/wheels/*.whl && /opt/driver/venv/bin/pip list --format=json > /opt/driver/installed.json && \ +/opt/driver/start && jq --version +""" + + +def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): + driver = tmp_path / builder.CONTEXT / "driver" + driver.mkdir(parents=True) + (driver / "requirements.txt").write_text("six==1.16.0\n") + (driver / "start").write_text("#!/bin/sh\necho started\n") + (driver / "start").chmod(0o755) + vm = builder.VmBuilder(build_sandbox_provider("modal_vm")) + try: + vm.start(tmp_path, SimpleNamespace(image_name=BASE, pinned_name=BASE), "linux/arm64", {}) + result = vm.resolve({"apt": ["jq"], "sources": [], "python": [ + {"name": "driver", "folder": "/ctx/driver", "requirements": "requirements.txt", "add": []}]}, + BASE, "arm64", 1800) + written = vm.files(f"{builder.RESOLVED}/out") + (wheel,) = result["wheels"]["driver"] + store = get_config().get_object_store() + vm.store(f"{builder.RESOLVED}/out/wheels/driver/{wheel}", store, f"builder-test/{wheel}") + stored = tmp_path / "stored.whl" + store.download_to_file(store.object_url(f"builder-test/{wheel}"), str(stored)) + vm.place({f"{builder.CONTEXT}/packages/apt/{deb['file']}": f"{builder.RESOLVED}/out/apt/{deb['file']}" + for deb in result["apt"]} | {f"{builder.CONTEXT}/packages/wheels/{wheel}": + f"{builder.RESOLVED}/out/wheels/driver/{wheel}"}, {}) + vm.build(DOCKERFILE, "agentenv-builder-test:arm64", "linux/arm64", {"dev.agentenv.test": "builder"}) + installed = vm.installed("agentenv-builder-test:arm64", "/opt/driver") + finally: + vm.close() + + assert wheel == "six-1.16.0-py2.py3-none-any.whl" and installed["six"] == "1.16.0" + assert {deb["package"] for deb in result["apt"]} >= {"jq", "libjq1"} + assert {deb["arch"] for deb in result["apt"]} == {"arm64"} + assert hashlib.sha256(stored.read_bytes()).hexdigest() == written[f"wheels/driver/{wheel}"].sha256 + assert vm.sandbox.platform == "linux/amd64" and Path(stored).stat().st_size == written[f"wheels/driver/{wheel}"].size diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index fc904fb04..caeb96a23 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -15,6 +15,7 @@ from agent_env.env import Env from agent_env.env.envs.computer.utils import composer from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from tst.unit.bundle._support import layout, local_builds UBUNTU = "@local/agentenv-framework/computers/ubuntu-24.04" @@ -35,9 +36,9 @@ def composed(tmp_path, monkeypatch, local_stores, cli_routing, builds): monkeypatch.setattr("agent_env.env.published_images._registry_digest", lambda ref: None) calls = [] - def compose(env, on_build=None): + def compose(env, on_build=None, sandbox_provider=None): on_build() - calls.append((env.id, env.computer.id, env.driver.id)) + calls.append((env.id, env.computer.id, env.driver.id, sandbox_provider)) monkeypatch.setattr(composer, "compose", compose) return calls @@ -69,17 +70,18 @@ def _plan_env(root): @pytest.mark.parametrize("planned", [_plan, _plan_env], ids=["a-run", "the-env-alone"]) def test_a_computer_env_naming_shipped_aliases_is_written_over_them_and_composed(tmp_path, composed, planned): - plan = planned(_bundle(tmp_path / "desk")) + plan, provider = planned(_bundle(tmp_path / "desk")), LocalSandboxProvider() building = [] - done = materialize(plan, build_here=local_builds(plan), on_build=lambda write: building.append(write.id)) + done = materialize(plan, build_here=local_builds(plan), on_build=lambda write: building.append(write.id), + sandbox_provider=provider) env = Env.get(f"{plan.bundle.bundle.id_root}/desktop") written = {item.write.id for item in done.writes} assert isinstance(env, ComputerEnv) and {UBUNTU, REFERENCE} <= written assert (env.computer.id, env.driver.id, env.environment_name) == (UBUNTU, REFERENCE, "desktop") assert env.computer_settings == {"timezone": "Europe/Paris"} and env.display == {"width": 1600, "height": 900} - assert composed == [(env.id, UBUNTU, REFERENCE)] and env.id in building + assert composed == [(env.id, UBUNTU, REFERENCE, provider)] and env.id in building def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_path, composed, monkeypatch): diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 3fd38d252..6bd1e549c 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -11,6 +11,8 @@ import json import logging import re +import shlex +import shutil import subprocess import tarfile import urllib.parse @@ -35,7 +37,7 @@ from agent_env.config import get_config from agent_env.env import Env from agent_env.env import env as env_module -from agent_env.env.envs.computer.utils import boot, composer, gate, inside, packages, resolve_in_base +from agent_env.env.envs.computer.utils import boot, builder, composer, gate, inside, packages, resolve_in_base from agent_env.env.envs.computer.utils.composer import _probe from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.app_env import ComputerAppEnv @@ -47,6 +49,9 @@ from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider from agent_env.store.base import NotFoundError from agent_env.task import teardown from agent_env.task_step.context import TaskStepContext @@ -80,6 +85,7 @@ def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" self.resolved.append((arch, request)) out, result = work / "out", {"apt": [], "wheels": {}} + out.mkdir(parents=True, exist_ok=True) for name in (re.split("[=/]", wanted)[0] for wanted in request["apt"]): (out / "apt").mkdir(parents=True, exist_ok=True) (out / "apt" / f"{name}_1.0_{arch}.deb").write_bytes(f"{name} for {arch}".encode()) @@ -92,7 +98,12 @@ def resolve(self, request, context, source, arch, work): (out / "wheels" / layer["name"]).mkdir(parents=True) (out / "wheels" / layer["name"] / "mss-10.1.0-py3-none-any.whl").write_bytes(b"mss") result["wheels"][layer["name"]] = ["mss-10.1.0-py3-none-any.whl"] - return result, out + return result + + def resolving(self, resolve=None): + """A LocalBuilder.resolve that runs ``resolve``, by default this one's, on the builder's context and resolve.""" + return lambda on, request, source, arch, timeout: (resolve or self.resolve)( + request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) async def probe(self, links, env, driver, image): arch = self._arch(self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]) @@ -104,6 +115,9 @@ async def probe(self, links, env, driver, image): return ProbeReport.of("full", "desk", checks, [CARD_EXTENSION]), CARD, "LocalSandboxProvider" def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_args=None): + if context.name == "label": + self.labels[tag] = labels + return if platform in self.fail: raise DockerBuildError(self.fail[platform]) self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), @@ -113,18 +127,13 @@ def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_ "build_settings": {path.parent.parent.name: json.loads(path.read_text()) for path in context.glob("apps/*/.agentenv/build_settings.json")}}) - def label(self, local, platform, extra): - self.labels[local] = extra - def push(self, local, ref, store): self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], labels=self.labels.get(local)) - def joined(self, index, images): - manifests = {self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]: image.rpartition("@")[2] - for image in images} - arches = {self._arch(config): digest for config, digest in manifests.items()} - self.registry.tags[index] = self.registry.index(**arches) + def joined(self, index, images, auth): + self.registry.tags[index] = self.registry.index(**{ + platform.rsplit("/", 1)[1]: image.rpartition("@")[2] for platform, image in images.items()}) def _arch(self, config_digest): return json.loads(self.registry.blobs[config_digest])["architecture"] @@ -134,20 +143,19 @@ def _arch(self, config_digest): def docker(local_stores, cli_routing, monkeypatch): registry = Registry().serve(monkeypatch) fake = FakeDocker(registry) - monkeypatch.setattr(composer, "build_image", fake.build_image) - monkeypatch.setattr(composer, "push_local_image", fake.push) - monkeypatch.setattr(composer, "_label", fake.label) - monkeypatch.setattr(composer, "docker_login", lambda auth: None) - monkeypatch.setattr(composer, "_joined", fake.joined) + monkeypatch.setattr(builder, "build_image", fake.build_image) + monkeypatch.setattr(builder, "push_local_image", fake.push) + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + monkeypatch.setattr(composer, "write_index", fake.joined) monkeypatch.setattr(composer, "pin_digest", registry.pin_digest) monkeypatch.setattr(docker_image, "pin_digest", registry.pin_digest) - monkeypatch.setattr(composer, "host_platform", lambda: "linux/arm64") + monkeypatch.setattr(builder, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") monkeypatch.setattr(composer, "_probe", fake.probe) - monkeypatch.setattr(composer, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) - monkeypatch.setattr(composer, "_installed", lambda local, layer: {"mss": "10.1.0"}) + monkeypatch.setattr(builder, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) + monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) monkeypatch.setattr(composer, "_protocol_source", lambda: None) - monkeypatch.setattr(packages, "_run_resolver", fake.resolve) + monkeypatch.setattr(builder.LocalBuilder, "resolve", fake.resolving()) monkeypatch.setattr(LocalRegistryImageStore, "ensure_repository", lambda self, repository: None) return fake @@ -206,7 +214,7 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ composed = stored.metadata["composed"]["container"] assert {arch: (r["status"], r["reused"], r["emulated"]) for arch, r in composed.items()} == { "amd64": ("unverified", False, True), "arm64": ("ready", False, False)} - assert composed["amd64"]["reason"] == "no configured sandbox provider runs linux/amd64 natively to probe it" + assert composed["amd64"]["reason"] == "no link of the sandbox provider runs linux/amd64 natively to probe it" assert docker.probed == [("arm64", composed["arm64"]["image"])] and composed["arm64"]["card"] == CARD probe = composed["arm64"]["probe"] assert (probe["level"], probe["passed"], probe["provider"], probe["extensions"]) == ( @@ -251,9 +259,9 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one docker, monkeypatch): auth, docker_calls = RegistryAuth("localhost:5000", "user", "token"), [] monkeypatch.setattr(LocalRegistryImageStore, "auth", lambda self, ref: auth) - monkeypatch.setattr(composer, "docker_login", lambda signed_in: docker_calls.append(("login", signed_in))) - monkeypatch.setattr(composer, "_joined", lambda index, images: (docker_calls.append(("join", index)), - docker.joined(index, images))) + monkeypatch.setattr(builder, "docker_login", lambda signed_in: docker_calls.append(("login", signed_in))) + monkeypatch.setattr(composer, "write_index", lambda index, images, signed_in: ( + docker_calls.append(("join", index, signed_in)), docker.joined(index, images, signed_in))) announced = [] first = _put(base, driver, on_build=lambda: announced.append("first")) docker_calls.clear() @@ -268,7 +276,7 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one assert {arch: (r["installed"], r["emulated"]) for arch, r in reused.items()} == { "amd64": ({"mss": "10.1.0"}, True), "arm64": ({"mss": "10.1.0"}, False)} index = ComputerArtifact.get(f"{second.id}__computer").image().image_name.partition("@")[0] - assert docker_calls == [("login", auth), ("join", index)] + assert docker_calls == [("join", index, auth)] @pytest.mark.parametrize("change", ["requirements", "build-setting", "service", "display-read-at-build"]) @@ -340,7 +348,7 @@ def test_a_build_that_cant_run_the_other_archs_programs_says_its_emulation_is_mi amd64 = Env.get(_put(base, driver).id).metadata["composed"]["container"]["amd64"] - assert amd64["reason"].startswith("this machine can't run linux/amd64 programs: its Docker has no emulation for it:") + assert amd64["reason"].startswith("the builder can't run linux/amd64 programs: its Docker has no emulation for it:") assert amd64["log"] @@ -350,7 +358,7 @@ def test_a_base_built_from_its_context_is_built_here_for_each_arch_and_keyed_by_ id="base-image", description="", image_name="local/base-1234:abcd", build_context_object_url="file:///c.tar.gz", dockerfile_path="Dockerfile", source_digest="sha256:" + "c" * 64)) built_for = [] - monkeypatch.setattr(composer, "build_for", lambda image, platform: built_for.append(platform) or + monkeypatch.setattr(builder, "build_for", lambda image, platform: built_for.append(platform) or f"{image.image_name}-{platform.rsplit('/', 1)[1]}") env = Env.get(_put(_base(image), driver).id) @@ -476,7 +484,7 @@ def test_env_build_builds_what_a_put_left_and_prints_each_pair(base, driver, doc again = runner.invoke(cli, ["env", "build", "--id", env.id]) assert failed.exit_code == 1 and "container/amd64: failed (the build failed: boom)" in failed.output - assert built.exit_code == 0 and ("container/amd64: unverified (no configured sandbox provider runs linux/amd64 " + assert built.exit_code == 0 and ("container/amd64: unverified (no link of the sandbox provider runs linux/amd64 " "natively to probe it)") in built.output assert "container/arm64: ready (localhost:5000" in built.output and len(docker.builds) == 2 assert again.exit_code == 0 and len(docker.builds) == 2 @@ -563,7 +571,7 @@ def test_the_platforms_this_machine_builds_are_read_from_its_builder(monkeypatch "Platforms: linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64*\nLabels:\n") monkeypatch.setattr(subprocess, "run", lambda *a, **k: SimpleNamespace(stdout=inspected, returncode=0)) - assert composer._emulated_platforms() == {"linux/arm64", "linux/amd64", "linux/amd64/v2", "linux/riscv64"} + assert builder._emulated_platforms() == {"linux/arm64", "linux/amd64", "linux/amd64/v2", "linux/riscv64"} @pytest.mark.parametrize("direct_url, expected", [ @@ -599,7 +607,7 @@ def run(argv, **kwargs): monkeypatch.setattr(subprocess, "run", run) - assert composer._installed("agentenv-computer:abc-arm64", "reference") == { + assert builder.LocalBuilder.installed(None, "agentenv-computer:abc-arm64", "reference") == { "mss": "10.2.0", "agentenv-framework-protocol": "0.1.382"} assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] @@ -611,8 +619,9 @@ def test_a_script_or_binary_in_the_drivers_files_stays_executable_in_the_image(b (folder / "start").write_text("#!/bin/sh\nexec venv/bin/python -m agentenv_protocol.computer.serve driver.py:Desk\n") (folder / "helper").write_bytes(b"\x7fELF\x02\x01\x01") modes = {} - monkeypatch.setattr(composer, "build_image", lambda dockerfile, context, tag, **kw: modes.update( - {path.name: path.stat().st_mode & 0o111 for path in (context / "driver").iterdir()})) + monkeypatch.setattr(builder, "build_image", lambda dockerfile, context, tag, **kw: modes.update( + {path.name: path.stat().st_mode & 0o111 for path in (context / "driver").iterdir()}) if context.name == "context" + else None) files = FileArtifactUniverse.put_bundled("@local/~/kit/scripted__files", files={ name: folder / name for name in ("driver.py", "start", "helper")}) driver = ComputerDriverEnv.put( @@ -624,14 +633,241 @@ def test_a_script_or_binary_in_the_drivers_files_stays_executable_in_the_image(b assert modes == {"driver.py": 0, "start": 0o111, "helper": 0o111} -def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on_and_the_store_before_the_index(base, driver, - docker, monkeypatch): +def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): signed = [] - monkeypatch.setattr(composer, "docker_login", signed.append) + monkeypatch.setattr(builder, "docker_login", signed.append) _put(base, driver) - assert len(signed) == 3 and len(docker.builds) == 2 + assert len(signed) == 2 and len(docker.builds) == 2 + + +class FakeVm: + """A builder VM, amd64: its files under ``root``, each line of a script it's given run as a VM's shell runs it, its + Docker the fake's.""" + + platform, _WFT_CHUNK_BYTES = "linux/amd64", 96 * 1024 + + def __init__(self, provider, root): + self.provider, self.root, self.terminated = provider, root, False + + def at(self, path): + return self.root / str(path).lstrip("/") + + async def write_host_file(self, data, path): + self.at(path).parent.mkdir(parents=True, exist_ok=True) + self.at(path).write_bytes(data) + + async def load_object_file(self, url, path): + self.provider.fetched.append(url) + self.at(path).parent.mkdir(parents=True, exist_ok=True) + get_config().get_object_store_at(url).download_to_file(url, str(self.at(path))) + + async def exec_script(self, script, max_retries=0): + code, out, _ = await self.exec_with_output("sudo", "bash", "-c", script) + if code: + raise RuntimeError(f"Script failed (exit {code}):\nstdout: {out}") + return out + + async def exec_with_output(self, *argv): + printed = [] + for line in argv[-1].splitlines(): + if line.strip() and line != "set -e": + code, out = self._line(line) + printed.append(out) + if code: + return code, "".join(printed), "" + return 0, "".join(printed), "" + + async def terminate(self): + self.terminated = True + + def _line(self, line): + self.provider.ran.append(line) + if line.startswith("case "): + path = self.at(shlex.split(line.split("$(head -c 4 ", 1)[1].split(" |", 1)[0])[0]) + if path.read_bytes()[:4].startswith((b"#!", b"\x7fELF")): + path.chmod(0o755) + return 0, "" + if line.startswith("c=$(docker create"): + return 0, json.dumps([{"name": "mss", "version": "10.1.0"}]) + if line.startswith(("docker buildx version", "mountpoint ")): + return 0, "" + if "docker login" in line: + self.provider.logins.append(shlex.split(line)[-1]) + return 0, "" + if "find . -type f" in line: + folder = self.at(shlex.split(line)[1]) + return 0, "".join(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.stat().st_size} " + f"{path.relative_to(folder).as_posix()}\0" for path in sorted(folder.rglob("*")) + if path.is_file() and not path.is_symlink()) + cwd, printed = None, "" + for command in " ".join(shlex.split(line, posix=False)).split(" && "): + words = shlex.split(command) + if words[0] == "cd": + cwd = self.at(words[1]) + continue + code, printed = self._command(words, cwd) + if code: + return code, printed + return 0, printed + + def _command(self, words, cwd): + docker = self.provider.docker + if words[0] == "mkdir": + for folder in words[2:]: + self.at(folder).mkdir(parents=True, exist_ok=True) + elif words[0] == "tar": + with tarfile.open(self.at(words[2])) as archive: + archive.extractall(self.at(words[words.index("-C") + 1]), filter="data") + elif words[0] == "rm": + self.at(words[-1]).unlink() + elif words[0] == "ln": + self.at(words[3]).unlink(missing_ok=True) + self.at(words[3]).hardlink_to(self.at(words[2])) + elif words[0] == "chmod": + self.at(words[2]).chmod(0o755) + elif words[0] == "cat": + return 0, self.at(words[1]).read_text() + elif words[:2] == ["sha256sum", "--check"]: + for listed in (cwd / words[-1]).read_text().splitlines(): + sha256, path = listed.split(" ", 1) + if hashlib.sha256((cwd / path).read_bytes()).hexdigest() != sha256: + return 1, f"{path}: FAILED\n" + elif words[:2] == ["curl", "-fsSL"] and "--upload-file" in words: + url = urllib.parse.unquote(words[-1].partition("?")[2]) + get_config().get_object_store_at(url).put_file_at(url, str(self.at(words[-2]))) + self.provider.uploaded.append(url) + elif words[0] == "timeout": + out, platform = self.at(words[words.index("-v", 13) + 1].split(":")[0]), words[words.index("--platform") + 1] + context = self.at(words[words.index("-v") + 1].split(":")[0]) + result = docker.resolve(json.loads((out / "request.json").read_text()), context, words[-4], + platform.split("/")[1], out.parent) + (out / "result.json").write_text(json.dumps(result)) + elif words[:2] == ["docker", "run"]: + self.provider.binfmt.append(words[-1]) + elif words[:2] == ["docker", "build"]: + labels = dict(words[i + 1].split("=", 1) for i, word in enumerate(words) if word == "--label") + try: + docker.build_image(cwd / "Dockerfile", cwd, words[words.index("-t") + 1], + platform=words[words.index("--platform") + 1], labels=labels) + except DockerBuildError as e: + return 1, str(e) + elif words[:2] == ["docker", "tag"]: + self.provider.tagged[words[3]] = words[2] + elif words[:2] == ["docker", "push"]: + docker.push(self.provider.tagged[words[2]], words[2], None) + else: + raise AssertionError(f"a builder VM was asked to run {words}") + return 0, "" + + +class FakeVmProvider(SandboxProvider): + """A sandbox provider whose VMs run amd64, recording what each was asked to do.""" + + def __init__(self, docker, root): + self.docker, self.root, self.vms, self.sizes = docker, root, [], [] + self.ran, self.fetched, self.uploaded, self.binfmt, self.logins, self.tagged = [], [], [], [], [], {} + + async def create_vm(self, **size): + self.sizes.append(size) + self.vms.append(vm := FakeVm(self, self.root / str(len(self.vms)))) + return vm + + +@pytest.fixture +def signing(local_stores, monkeypatch): + """Stores a remote VM reaches: the object store signs URLs to its objects, and the registry isn't this machine's.""" + images = OciRegistryImageStore("registry.example") + monkeypatch.setattr(type(local_stores), "get_image_store_for", lambda self, entity_id: images) + monkeypatch.setattr(OciRegistryImageStore, "ensure_repository", lambda self, repository: None) + monkeypatch.setattr(LocalFilesystemObjectStore, "signed_get_url", lambda self, url, expires_in=3600: + f"https://signed.example/get?{urllib.parse.quote(url)}") + monkeypatch.setattr(LocalFilesystemObjectStore, "signed_put_url", lambda self, url, expires_in=3600: + f"https://signed.example/put?{urllib.parse.quote(url)}") + + +def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_images_are_reused_by_key_here( + base, driver, docker, signing, tmp_path, monkeypatch): + vms = FakeVmProvider(docker, tmp_path / "vms") + for name in ("build_image", "push_local_image", "docker_login", "_emulated_platforms"): + monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) + + composed = Env.get(_put(base, driver, sandbox_provider=vms).id).metadata["composed"]["container"] + + assert {arch: (r["status"], r["builder"], r["emulated"]) for arch, r in composed.items()} == { + "amd64": ("ready", "FakeVmProvider", False), "arm64": ("unverified", "FakeVmProvider", True)} + assert composed["arm64"]["reason"] == "no link of the sandbox provider runs linux/arm64 natively to probe it" + assert len(vms.vms) == 2 and all(vm.terminated for vm in vms.vms) and vms.binfmt == ["arm64"] + assert vms.sizes == [{"exposed_ports": [], **builder._VM}] * 2 + driver_files = {artifact.object_url for artifact in driver.files.get_file_artifacts().values()} + assert sorted(vms.fetched) == sorted([*driver_files, *driver_files]) + assert vms.uploaded and all(f"{packages.PREFIX}/files/" in url for url in vms.uploaded) + builds = {build["platform"]: build for build in docker.builds} + assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", + "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] + assert composed["amd64"]["installed"] == {"mss": "10.1.0"} + assert docker.labels[f"agentenv-computer:{composed['arm64']['key'][7:23]}-arm64"]["dev.agentenv.emulated"] == "true" + + again = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id) + + assert {arch: (r["reused"], r["key"]) for arch, r in again.metadata["composed"]["container"].items()} == { + arch: (True, r["key"]) for arch, r in composed.items()} + + +def test_a_chain_builds_each_arch_on_the_first_link_that_runs_it_natively(base, driver, docker, signing, tmp_path): + vms = FakeVmProvider(docker, tmp_path / "vms") + + composed = Env.get(_put(base, driver, sandbox_provider=ChainedSandboxProvider( + [vms, LocalSandboxProvider()])).id).metadata["composed"]["container"] + + assert {arch: (r["builder"], r["emulated"]) for arch, r in composed.items()} == { + "amd64": ("FakeVmProvider", False), "arm64": ("LocalSandboxProvider", False)} + assert len(vms.vms) == 1 and vms.binfmt == [] + + +def test_a_builder_vm_its_provider_loses_fails_its_pair_alone_and_is_torn_down(base, driver, docker, signing, tmp_path, + monkeypatch): + vms, real = FakeVmProvider(docker, tmp_path / "vms"), FakeVm._command + + class Gone(Exception): + pass + + def command(vm, words, cwd): + if words[0] == "timeout" and "linux/arm64" in words: + raise Gone("the sandbox is shutting down") + return real(vm, words, cwd) + monkeypatch.setattr(FakeVm, "_command", command) + + composed = Env.get(_put(base, driver, sandbox_provider=vms).id).metadata["composed"]["container"] + + assert (composed["arm64"]["status"], composed["arm64"]["stage"], composed["arm64"]["reason"]) == ( + "failed", "resolve", "RuntimeError: FakeVmProvider's VM failed: Gone: the sandbox is shutting down") + assert composed["amd64"]["status"] == "ready" and all(vm.terminated for vm in vms.vms) + + +class Containers(SandboxProvider): + async def create_sandbox(self, **kwargs): + raise AssertionError("a container was created") + + +@pytest.mark.parametrize("links, reason", [ + (["vms"], "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local provider"), + (["containers"], "Containers gives no VM to build in"), + (["containers", "local"], "Containers gives no VM to build in; this machine can't build linux/amd64: its Docker has " + "no emulation for it"), +]) +def test_an_arch_no_link_can_build_fails_saying_why_for_each_link(base, driver, docker, tmp_path, links, reason): + docker.emulates = set() + made = {"vms": FakeVmProvider(docker, tmp_path / "vms"), "containers": Containers(), + "local": LocalSandboxProvider()} + + composed = Env.get(_put(base, driver, sandbox_provider=ChainedSandboxProvider( + [made[link] for link in links])).id).metadata["composed"]["container"] + + assert (composed["amd64"]["status"], composed["amd64"]["stage"], composed["amd64"]["reason"]) == ( + "failed", "resolve", reason) + assert made["vms"].vms == [] def test_env_build_writes_the_computer_again_when_it_isnt_over_every_pair_built(base, driver, docker, caplog): @@ -641,8 +877,10 @@ def test_env_build_writes_the_computer_again_when_it_isnt_over_every_pair_built( stored.update_metadata({**stored.metadata, "computer": stale}) assert not composer.current(Env.get(stored.id)) - rebuilt = CliRunner().invoke(cli, ["env", "build", "--id", stored.id]) + unknown = CliRunner().invoke(cli, ["env", "build", "--id", stored.id, "--sandbox", "nosuch"]) + rebuilt = CliRunner().invoke(cli, ["env", "build", "--id", stored.id, "--sandbox", "local"]) + assert unknown.exit_code == 1 and "Unknown sandbox backend: 'nosuch'" in unknown.output assert rebuilt.exit_code == 0 and composer.current(Env.get(stored.id)) and len(docker.builds) == 2 assert Env.get(stored.id).metadata["computer"]["version"] == stale["version"] + 1 @@ -1065,7 +1303,7 @@ def test_an_app_that_requires_nothing_still_gets_a_venv_for_its_own_server(base, def test_an_apps_packages_downloads_scripts_and_steps_are_fetched_at_put_and_installed_offline(base, driver, docker, tmp_path, monkeypatch): tool = b"#!/bin/sh\necho tool\n" - monkeypatch.setattr(packages.httpx, "stream", _serving({"https://example.com/tool": tool})) + monkeypatch.setattr(builder.httpx, "stream", _serving({"https://example.com/tool": tool})) install = {"python": "requirements.txt", "packages": {"apt": ["hello", "xdotool/noble"]}, "sources": [{"apt": "https://ppa.example/ubuntu", "suite": "noble", "components": ["main"], "key": "ppa.asc"}], @@ -1106,19 +1344,19 @@ def test_a_resolve_or_download_that_fails_fails_its_arch_at_the_resolve_stage(ba monkeypatch, failing): if failing == "resolver": def refuse(request, context, source, arch, work): - raise packages.ResolveError("resolving the packages failed", "E: Unable to locate package nosuch\n") - monkeypatch.setattr(packages, "_run_resolver", refuse) + raise builder.ResolveError("resolving the packages failed", "E: Unable to locate package nosuch\n") + monkeypatch.setattr(builder.LocalBuilder, "resolve", docker.resolving(refuse)) if failing == "link": (tmp_path / "credentials").write_text("secret") def linking(request, context, source, arch, work): - result, out = docker.resolve(request, context, source, arch, work) - wheel = out / "wheels" / "driver" / "mss-10.1.0-py3-none-any.whl" + result = docker.resolve(request, context, source, arch, work) + wheel = work / "out" / "wheels" / "driver" / "mss-10.1.0-py3-none-any.whl" wheel.unlink() wheel.symlink_to(tmp_path / "credentials") - return result, out - monkeypatch.setattr(packages, "_run_resolver", linking) - monkeypatch.setattr(packages.httpx, "stream", _serving({"https://example.com/tool": b"something else"})) + return result + monkeypatch.setattr(builder.LocalBuilder, "resolve", docker.resolving(linking)) + monkeypatch.setattr(builder.httpx, "stream", _serving({"https://example.com/tool": b"something else"})) install = {"python": "requirements.txt", "services": [], "downloads": [{"url": "https://example.com/tool", "sha256": "a" * 64, "to": "tool"}]} @@ -1147,8 +1385,8 @@ def test_a_stored_file_that_isnt_the_one_its_lock_names_fails_the_build_that_sta again = Env.get(_put(base, driver, computer_settings={"theme": "dark"}).id).metadata["composed"]["container"]["arm64"] assert (again["status"], again["stage"]) == ("failed", "build") - assert again["reason"].endswith(f"agentenv-packages/files/{sha256} isn't the file the lock names: its sha256 is " - f"{hashlib.sha256(b'tampered').hexdigest()}") + assert again["reason"].endswith(f"agentenv-packages/files/{sha256} isn't the file this build stages: its sha256 " + f"is {hashlib.sha256(b'tampered').hexdigest()}") def test_a_venv_whose_requirements_resolve_to_nothing_is_made_empty(base, docker, tmp_path): @@ -1168,21 +1406,29 @@ def test_two_puts_resolving_the_same_inputs_at_once_both_get_the_first_stored_lo (folder / "requirements.txt").write_text("mss\n") wants = [packages.Want("driver", Install(python="requirements.txt", services=[Service(name="d", command="x")]), True)] inputs = packages.inputs_digest(wants, tmp_path / "ctx", "base", "arm64") - first, _ = packages.resolve(inputs, wants, tmp_path / "ctx", "base", "arm64", "@local/~/kit/desktop", tmp_path / "1") + first, _ = packages.resolve(inputs, wants, _local(tmp_path / "1", tmp_path / "ctx"), "base", "arm64", + "@local/~/kit/desktop") def newer(request, context, source, arch, work): - result, out = docker.resolve(request, context, source, arch, work) - (out / "wheels" / "driver" / "mss-10.1.0-py3-none-any.whl").write_bytes(b"mss, published again") - return result, out - monkeypatch.setattr(packages, "_run_resolver", newer) - second, _ = packages.resolve(inputs, wants, tmp_path / "ctx", "base", "arm64", "@local/~/kit/desktop", - tmp_path / "2") - monkeypatch.setattr(packages.os, "getuid", lambda: 4242) + result = docker.resolve(request, context, source, arch, work) + (work / "out" / "wheels" / "driver" / "mss-10.1.0-py3-none-any.whl").write_bytes(b"mss, published again") + return result + monkeypatch.setattr(builder.LocalBuilder, "resolve", docker.resolving(newer)) + second, _ = packages.resolve(inputs, wants, _local(tmp_path / "2", tmp_path / "ctx"), "base", "arm64", + "@local/~/kit/desktop") + monkeypatch.setattr(builder.os, "getuid", lambda: 4242) assert second == first == packages.stored(inputs, "@local/~/kit/desktop") assert packages.inputs_digest(wants, tmp_path / "ctx", "base", "arm64") == inputs +def _local(work, context): + """A builder of this machine's Docker in ``work``, its build context ``context``'s files.""" + shutil.copytree(context, work / builder.CONTEXT) + base = SimpleNamespace(image_name="ghcr.io/acme/base:1", pinned_name="ghcr.io/acme/base:1") + return builder.LocalBuilder("LocalSandboxProvider", work, base, "linux/arm64") + + def test_the_resolver_installs_the_apt_packages_it_downloads_before_building_wheels(tmp_path, monkeypatch): ran = [] monkeypatch.setattr(resolve_in_base, "KEYRINGS", tmp_path / "keyrings") diff --git a/tst/unit/store/registry_api_test.py b/tst/unit/store/registry_api_test.py index 75feaa6e4..10bd8fcde 100644 --- a/tst/unit/store/registry_api_test.py +++ b/tst/unit/store/registry_api_test.py @@ -1,8 +1,9 @@ -"""Pinning an image ref to the digest its registry serves, and reading its manifests and blobs, over a fake -registry's OCI distribution API.""" +"""Pinning an image ref to the digest its registry serves, reading its manifests and blobs, and writing an index of its +images, over a fake registry's OCI distribution API.""" import base64 import hashlib +import json import httpx import pytest @@ -272,3 +273,45 @@ def test_what_the_registry_refuses_is_raised_as_pin_digest_raises_it(fake_regist with pytest.raises(error): read_blob("localhost:5000/desk:1", DIGEST, None, tmp_path / "layer") assert str(seen[0].url) == "http://localhost:5000/v2/desk/manifests/1" + + +DOCKER_IMAGE, OCI_IMAGE = "application/vnd.docker.distribution.manifest.v2+json", "application/vnd.oci.image.manifest.v1+json" +OCI_INDEX = "application/vnd.oci.image.index.v1+json" + + +@pytest.mark.parametrize("arm64, written", [("index", OCI_INDEX), + ("docker", "application/vnd.docker.distribution.manifest.list.v2+json")]) +def test_an_index_of_images_is_written_with_a_token_that_pushes_and_an_image_that_is_an_index_keeps_its_manifests( + fake_registry, arm64, written): + amd64 = b'{"schemaVersion": 2}' + children = [{"mediaType": OCI_IMAGE, "digest": "sha256:" + "c" * 64, "size": 400, + "platform": {"architecture": "arm64", "os": "linux"}}, + {"mediaType": OCI_IMAGE, "digest": "sha256:" + "d" * 64, "size": 500, + "platform": {"architecture": "unknown", "os": "unknown"}}] + served = {DIGEST: (DOCKER_IMAGE, amd64), + OTHER: (OCI_INDEX, json.dumps({"manifests": children}).encode()) if arm64 == "index" else (DOCKER_IMAGE, amd64)} + put = [] + + def ghcr(request): + scope = "repository:org/desk:pull,push" if request.method == "PUT" else "repository:org/desk:pull" + if request.url.path == "/token": + return httpx.Response(200, json={"token": request.url.params["scope"]}) + if request.headers.get("authorization") != f"Bearer {scope}": + return httpx.Response(401, headers={"WWW-Authenticate": f'Bearer realm="https://ghcr.io/token",scope="{scope}"'}) + if request.method == "PUT": + put.append(request) + return httpx.Response(201) + media, body = served[request.url.path.rpartition("/")[2]] + return httpx.Response(200, content=body, headers={"Content-Type": media}) + + fake_registry(ghcr) + registry_api.write_index("ghcr.io/org/desk:index-1", {"linux/amd64": f"ghcr.io/org/desk:a@{DIGEST}", + "linux/arm64": f"ghcr.io/org/desk:b@{OTHER}"}, None) + + (request,) = put + assert request.url.path == "/v2/org/desk/manifests/index-1" and request.headers["content-type"] == written + amd64_entry = {"mediaType": DOCKER_IMAGE, "digest": DIGEST, "size": len(amd64), + "platform": {"architecture": "amd64", "os": "linux"}} + assert json.loads(request.content) == {"schemaVersion": 2, "mediaType": written, "manifests": [ + amd64_entry, *(children if arm64 == "index" else [{**amd64_entry, "digest": OTHER, + "platform": {"architecture": "arm64", "os": "linux"}}])]} From 56d08827d8dac5feeb2f8eae7e29373986630395 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 12:38:25 -0700 Subject: [PATCH 02/11] feat(computer): a put reports each pair's progress as it goes compose reports each pair's stages to an on_progress callback and the log: where it builds, natively or emulated, resolving, building, pushing, probing, a line every 10 min a stage goes on, and how each pair ended with its time. bundle register, run and env build print them without --verbose. on_progress replaces the on_build hook on computer env puts and AuthoringContext, whose "building with docker" line was wrong for a remote build. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/authoring.py | 4 +- src/agent_env/bundle/materialize.py | 18 ++- src/agent_env/bundle/register.py | 3 +- src/agent_env/bundle/run.py | 1 + src/agent_env/cli/env/build.py | 2 +- src/agent_env/env/envs/computer/app_env.py | 9 +- src/agent_env/env/envs/computer/env.py | 10 +- .../env/envs/computer/utils/builder.py | 5 + .../env/envs/computer/utils/composer.py | 124 +++++++++++++----- tst/unit/bundle/computer_env_folder_test.py | 12 +- tst/unit/env/envs/test_computer_env.py | 29 +++- 11 files changed, 152 insertions(+), 65 deletions(-) diff --git a/src/agent_env/bundle/authoring.py b/src/agent_env/bundle/authoring.py index 07680e494..a16f9e02d 100644 --- a/src/agent_env/bundle/authoring.py +++ b/src/agent_env/bundle/authoring.py @@ -51,8 +51,8 @@ class AuthoringContext: bundle: Bundle entry: BundleEntry - #: Called just before writing the entry builds an image, which a run says it does. - on_build: Callable[[], None] | None = None + #: Told, a line at a time, how what writing the entry does that takes long goes, such as composing a computer env. + on_progress: Callable[[str], None] | None = None #: Where writing a computer env composes it; the env's sandbox provider when None. sandbox_provider: SandboxProvider | None = None diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index f1557dfaf..634c063df 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -85,6 +85,7 @@ def materialize( build_here: frozenset[str] = frozenset(), ledger: Ledger | None = None, sandbox_provider: SandboxProvider | None = None, + on_progress: Callable[[Write, str], None] | None = None, ) -> Materialization: """Write ``plan``'s entities, tasks and evals. The first write that fails stops it, and every earlier one stays: they're in the ledger, so the next run reuses them. ``on_wait`` is called when another run @@ -98,7 +99,8 @@ def materialize( Each write gets the version it would reuse or the store's next, which another run can take first. A step reading what the run would write first isn't preflighted, since the store doesn't hold it yet. - ``sandbox_provider`` composes each computer env written; the env's sandbox provider when None. + ``sandbox_provider`` composes each computer env written, the env's sandbox provider when None, and + ``on_progress`` is told how each compose goes, a line at a time. ``ledger`` keeps the rows, this run's in the ``@local`` namespace's store unless given; it also says whether each task and eval names the versions it reads (``Ledger.pins_references``) and whether each image the bundle builds is @@ -134,7 +136,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: if ledger.pushes_images and isinstance(write.source, BuiltImage) and write.id not in plan.published: through_ledger(write, lambda: _write_pushed_image(plan, write, on_build)) else: - through_ledger(write, lambda: _write_entity(plan, write, here, on_build, sandbox_provider)) + through_ledger(write, lambda: _write_entity(plan, write, here, on_build, sandbox_provider, + on_progress)) if here and done[_key(write)].reused: _build_reused(plan, write, done[_key(write)].version, on_build) unwritten = {_key(item.write) for item in done.values() if not item.reused} if dry_run else set() @@ -157,7 +160,7 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None, - sandbox_provider: SandboxProvider | None) -> int: + sandbox_provider: SandboxProvider | None, on_progress: Callable[[Write, str], None] | None) -> int: if isinstance(write.source, ReferencedImage): return _write_referenced_image(plan, write) if isinstance(write.source, BuiltImage): @@ -165,7 +168,8 @@ def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Writ return _write_published_image(plan, write) return _write_build_context(plan, write, here, on_build) if write.kind is BundleKind.ENV: - return _write_env(plan, write, _before(on_build, write), sandbox_provider) + return _write_env(plan, write, sandbox_provider, + None if on_progress is None else lambda line: on_progress(write, line)) return _WRITERS[write.kind](plan, write) @@ -299,12 +303,12 @@ def _write_agent(plan: Plan, write: Write) -> int: AuthoringContext(plan.bundle_of(entry), entry)).version -def _write_env(plan: Plan, write: Write, on_build: Callable[[], None] | None = None, - sandbox_provider: SandboxProvider | None = None) -> int: +def _write_env(plan: Plan, write: Write, sandbox_provider: SandboxProvider | None = None, + on_progress: Callable[[str], None] | None = None) -> int: entry = write.source.entry cls = get_env_registry()[entry.type] return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext( - plan.bundle_of(entry), entry, on_build=on_build, sandbox_provider=sandbox_provider)).version + plan.bundle_of(entry), entry, on_progress=on_progress, sandbox_provider=sandbox_provider)).version def _pinned(plan: Plan, write: Write, refs: tuple[EntityRef, ...]) -> Any: diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 8ac9f6959..2e9321539 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -112,7 +112,8 @@ def register_bundle( registered, dry_run=dry_run, ledger=ledger, on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), on_build=lambda write: say(f" {_label(registered, write)}: building with docker and pushing"), - on_write=lambda done: say(f" {registered_line(registered, done)}"), sandbox_provider=provider) + on_write=lambda done: say(f" {registered_line(registered, done)}"), sandbox_provider=provider, + on_progress=lambda write, line: say(f" {_label(registered, write)}: {line}")) return Registration(registered, materialization, ids, tuple(_floating(registered)), identity, shared, requires) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 93c943f31..d1c5b00f2 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -308,6 +308,7 @@ def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], N on_write=lambda done: say(_written(plan, done)), build_here=preflight.built_here, sandbox_provider=build_sandbox_provider(sandbox) if sandbox else None, + on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}"), ) if preflight.infra: ensure_default_envs(preflight.infra_kinds, say=say) diff --git a/src/agent_env/cli/env/build.py b/src/agent_env/cli/env/build.py index 8c132f8b3..c7e38a015 100644 --- a/src/agent_env/cli/env/build.py +++ b/src/agent_env/cli/env/build.py @@ -28,7 +28,7 @@ def build(env_id: str, version: int | None, sandbox: str | None): if env.computer is None: raise click.ClickException(f"{env_id} names no computer and driver, so it has nothing to build alone") if (pairs := composer.pending(env, provider)) or not composer.current(env): - composer.compose(env, pairs, sandbox_provider=provider) + composer.compose(env, pairs, on_progress=click.echo, sandbox_provider=provider) failed = False for form, by_arch in env.metadata.get("composed", {}).items(): for arch, result in by_arch.items(): diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index ee9b41218..fec54566a 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -99,11 +99,12 @@ def from_dict(cls, data: dict) -> ComputerAppEnv: def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifactUniverse, declaration: AppDeclaration, computer: Optional[ComputerArtifact] = None, driver: Optional[ComputerDriverEnv] = None, build_settings: Optional[dict[str, Any]] = None, - compose: bool = True, on_build: Optional[Callable[[], None]] = None, + compose: bool = True, on_progress: Optional[Callable[[str], None]] = None, sandbox_provider: Optional[SandboxProvider] = None) -> Self: """Write the app once what it declares and ``build_settings``, its build settings' values, check out, and with a computer and a driver, once it fits them (gate 1); then compose it with them on ``sandbox_provider``, by - default the env's, unless ``compose`` is False. Refused, naming every problem, before anything is stored.""" + default the env's, unless ``compose`` is False, telling ``on_progress`` how it goes. Refused, naming every + problem, before anything is stored.""" app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, declaration=declaration, computer=computer, driver=driver, build_settings=build_settings) _, problems = _installs(declaration) @@ -119,7 +120,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) app = get_env_store().put_document(app) if computer is not None and compose: - composer.compose(app, on_build=on_build, sandbox_provider=sandbox_provider) + composer.compose(app, on_progress=on_progress, sandbox_provider=sandbox_provider) return app @classmethod @@ -145,7 +146,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerAppEnv: try: return cls.put(id=ctx.id, environment_name=fields.get("environment_name", ctx.name), module=fields["module"], files=universe, declaration=declaration, computer=computer, - driver=driver, build_settings=fields.get("build_settings"), on_build=ctx.on_build, + driver=driver, build_settings=fields.get("build_settings"), on_progress=ctx.on_progress, sandbox_provider=ctx.sandbox_provider) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index d94e9895d..0fd5166b1 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -83,11 +83,11 @@ def from_dict(cls, data: dict) -> ComputerEnv: @classmethod def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str, apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None, - compose: bool = True, on_build: Optional[Callable[[], None]] = None, + compose: bool = True, on_progress: Optional[Callable[[str], None]] = None, sandbox_provider: Optional[SandboxProvider] = None) -> Self: """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it on - ``sandbox_provider``, by default the env's, unless ``compose`` is False, calling ``on_build`` just before it - builds an image. Refused, naming every mismatch, before anything is stored.""" + ``sandbox_provider``, by default the env's, unless ``compose`` is False, telling ``on_progress`` how it goes. + Refused, naming every mismatch, before anything is stored.""" values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, computer_settings=computer_settings or {}) if problems: @@ -96,7 +96,7 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, display=display, network=network, computer_settings=values)) if compose: - composer.compose(env, on_build=on_build, sandbox_provider=sandbox_provider) + composer.compose(env, on_progress=on_progress, sandbox_provider=sandbox_provider) return env @classmethod @@ -113,7 +113,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: ComputerDriverEnv) try: return cls.put(id=ctx.id, computer=base, driver=driver, - environment_name=fields.pop("environment_name", ctx.name), on_build=ctx.on_build, + environment_name=fields.pop("environment_name", ctx.name), on_progress=ctx.on_progress, sandbox_provider=ctx.sandbox_provider, **fields) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 964f39b23..a60216920 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -186,6 +186,11 @@ def _refused(link: SandboxProvider, store_id: str, repository: str) -> str | Non return None +def where(link: SandboxProvider) -> str: + """Where ``link`` builds, for a person to read.""" + return "with this machine's Docker" if _docker_here(link) else f"in a {type(link).__name__} VM" + + def _docker_here(link: SandboxProvider) -> bool: """Whether ``link``'s sandboxes are containers on this machine's Docker, which then builds itself.""" return link.ON_THIS_MACHINE and link.REACHES_HOST_AT is None diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 910c90da5..fbccfc9fe 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -19,6 +19,7 @@ import subprocess import tempfile import threading +import time import urllib.parse from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass @@ -73,37 +74,36 @@ _PROBE_TTL_S = 1800 #: How long the probe at full may take on a computer that booted: about ten seconds on a healthy one. _PROBE_TIMEOUT_S = 120 +#: How often a compose says a pair is still at a stage, which can take most of an hour under emulation. +_HEARTBEAT_S = 600 _PROTOCOL = "agentenv-framework-protocol" # What building the protocol from its source reads, beside src/. _PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, - on_build: Callable[[], None] | None = None, sandbox_provider: SandboxProvider | None = None) -> None: + on_progress: Callable[[str], None] | None = None, sandbox_provider: SandboxProvider | None = None) -> None: """Build each of ``pairs``, by default each form the base has in each arch the env composes, in parallel, on ``sandbox_provider``, by default the env's, and probe each one it runs natively (gate 3), recording each as it - starts and finishes, then write the env's computer over every pair built so far. ``on_build`` is called once, just - before the first image is built; reusing them all calls it never.""" + starts and finishes, then write the env's computer over every pair built so far. Each pair's stages and how it + ended go to ``on_progress`` and the log as they happen (``_Progress``).""" from agent_env.providers import get_env_sandbox_provider - base, driver = env.computer, env.driver - building = _once(on_build) - builds = builder.Builds(sandbox_provider or get_env_sandbox_provider(), env.stored_id, REPOSITORY) + base, driver, provider = env.computer, env.driver, sandbox_provider or get_env_sandbox_provider() + builds = builder.Builds(provider, env.stored_id, REPOSITORY) pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver, env.composed_apps())] record = _Record(env) record.started(pairs) - with ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: - futures = {pool.submit(_composed, env, base, driver, form, arch, builds, building): (form, arch) + with _Progress(env, on_progress) as progress, ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: + progress.say(f"composing {', '.join(f'{form}/{arch}' for form, arch in pairs)} on " + f"{' or '.join(type(link).__name__ for link in provider.links)}") + futures = {pool.submit(_composed, env, base, driver, form, arch, builds, progress): (form, arch) for form, arch in pairs} for future in as_completed(futures): form, arch = futures[future] result = future.result() - if result["status"] == "failed": - logger.warning(f"{env.stored_id}: {form}/{arch} failed at {result['stage']}: {result['reason']}") - else: - logger.info(f"{env.stored_id}: {form}/{arch} {'reused' if result['reused'] else 'built'}, " - f"{result['status']}: {result['image']}") + progress.ended(f"{form}/{arch}", result) record.finished(form, arch, result) record.computer(_computer(env, base, record.built())) @@ -128,19 +128,22 @@ def pending(env: ComputerEnv, sandbox_provider: SandboxProvider | None = None) - def _composed(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, - builds: builder.Builds, building: Callable[[], None]) -> dict: + builds: builder.Builds, progress: _Progress) -> dict: """``form`` and ``arch`` built or reused, then probed when it built (gate 3).""" - result = _built(env, base, driver, form, arch, builds, building) - return _probed(env, driver, form, arch, result, builds.provider) if result["status"] == "unverified" else result + result = _built(env, base, driver, form, arch, builds, progress) + if result["status"] != "unverified": + return result + return _probed(env, driver, form, arch, result, builds.provider, progress) def _probed(env: ComputerEnv, driver: ComputerDriverEnv, form: str, arch: str, result: dict, - sandbox_provider: SandboxProvider) -> dict: + sandbox_provider: SandboxProvider, progress: _Progress) -> dict: """Gate 3: the build of ``result`` booted on the first link of ``sandbox_provider`` that runs ``arch`` natively, and probed at ``full``: ready, with the driver's card, or failed at the probe; each with a summary of the probe and its report stored. Still unverified, saying why, when no link runs ``arch`` natively.""" if not (links := boot.native_links(sandbox_provider, arch)): return {**result, "reason": f"no link of the sandbox provider runs linux/{arch} natively to probe it"} + progress.at(f"{form}/{arch}", "probing", f"probing on {type(links[0]).__name__}") try: report, card, provider = asyncio.run(_probe(links, env, driver, result["image"])) except TimeoutError: @@ -183,10 +186,10 @@ async def _probe(links: list[SandboxProvider], env: ComputerEnv, driver: Compute def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, form: str, arch: str, - builds: builder.Builds, building: Callable[[], None]) -> dict: - """The status of ``form`` and ``arch`` once built and pushed, or reused, or the stage that failed and why. A builder - from ``builds`` resolves and builds it, opened when first needed, after ``building`` is called.""" - stage, key = "build", None + builds: builder.Builds, progress: _Progress) -> dict: + """The status of ``form`` and ``arch`` once built and pushed, or reused, or the stage that failed and why, each + stage reported to ``progress``. A builder from ``builds`` resolves and builds it, opened when first needed.""" + stage, key, pair = "build", None, f"{form}/{arch}" try: with tempfile.TemporaryDirectory(prefix="agentenv-compose-") as work, contextlib.ExitStack() as opened: context, fetched, from_store = Path(work) / builder.CONTEXT, {}, {} @@ -201,7 +204,8 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, def on() -> builder.Builder: if not held: link = builds.link(platform) - building() + how = "natively" if platform in link.native_platforms() else "emulated, which is slow," + progress.at(pair, "starting its builder", f"building {how} {builder.where(link)}") held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) return held[0] @@ -209,7 +213,10 @@ def on() -> builder.Builder: lock = packages.stored(inputs, env.stored_id) if inputs is not None else None if inputs is not None and lock is None: stage = "resolve" - lock, fetched = packages.resolve(inputs, wants, on(), source, arch, env.stored_id) + resolving = on() + progress.at(pair, "resolving packages") + lock, fetched = packages.resolve(inputs, wants, resolving, source, arch, env.stored_id) + progress.at(pair, "resolved", f"resolved {len(fetched)} files and stored them") stage = "build" dockerfile = _dockerfile(source, layers, base, _build_settings(env, base), lock) key = "sha256:" + hashlib.sha256(json.dumps({ @@ -221,11 +228,13 @@ def on() -> builder.Builder: store.ensure_repository(REPOSITORY) ref = store.image_ref(REPOSITORY, key.removeprefix("sha256:")) if (pinned := _served(ref, store)) is not None: + progress.at(pair, "reusing", "reusing the image of the same key, built earlier") kept = labels(pinned, arch, store.auth(pinned)) return {"status": "unverified", "key": key, "image": pinned, "reused": True, "emulated": kept.get(EMULATED_LABEL) == "true", "lock": inputs, "installed": json.loads(kept.get(INSTALLED_LABEL, "{}")), "built_at": _now()} building_on = on() + progress.at(pair, "building the image") if lock is not None: packages.stage(lock, building_on, env.stored_id, fetched) local, emulated = f"agentenv-computer:{key[7:23]}-{arch}", platform != building_on.platform @@ -235,6 +244,7 @@ def on() -> builder.Builder: INSTALLED_LABEL: json.dumps(installed, sort_keys=True), EMULATED_LABEL: "true" if emulated else "false", }, context="label") stage = "push" + progress.at(pair, "pushing the image") building_on.push(local, ref, store) return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, @@ -254,18 +264,6 @@ def on() -> builder.Builder: return _failed(env, form, arch, stage, key, f"{type(e).__name__}: {e}") -def _once(fn: Callable[[], None] | None) -> Callable[[], None]: - """``fn``, called at most once however many builds call it at once; nothing when it's None.""" - lock, called = threading.Lock(), [] - - def once() -> None: - with lock: - if fn is not None and not called: - called.append(True) - fn() - return once - - def _base(base: ComputerArtifact, arch: str) -> tuple[str, str]: """What the composed image is built ``FROM`` for ``arch``, and the base's part of the key, found without building anything: a registry image pinned to its digest, or the tag a build of the base's stored build context for that @@ -577,6 +575,62 @@ def _merge(self, results: dict[tuple[str, str], dict]) -> None: self.env.metadata = Env.get(self.env.stored_id, self.env.version).metadata +class _Progress: + """What a compose reports as it goes, to ``on_progress`` and the log: a line as each pair reaches a stage, one each + ``_HEARTBEAT_S`` a stage goes on, and how each pair ended, with the time since the compose began.""" + + def __init__(self, env: ComputerEnv, on_progress: Callable[[str], None] | None): + self.env, self.on_progress, self.lock = env, on_progress, threading.Lock() + self.began, self.stages, self.stopped = time.monotonic(), {}, threading.Event() + + def __enter__(self) -> _Progress: + self.beating = threading.Thread(target=self._beat, daemon=True) + self.beating.start() + return self + + def __exit__(self, *_exc) -> None: + self.stopped.set() + self.beating.join() + + def say(self, line: str, failed: bool = False) -> None: + """``line`` to ``on_progress``, and the log; a failure as a warning when nothing else shows it.""" + logger.log(logging.WARNING if failed and self.on_progress is None else logging.INFO, + f"{self.env.stored_id}: {line}") + if self.on_progress is not None: + with self.lock: + self.on_progress(line) + + def at(self, pair: str, stage: str, line: str | None = None) -> None: + now = time.monotonic() + with self.lock: + self.stages[pair] = (stage, now, now) + self.say(f"{pair}: {line or stage}") + + def ended(self, pair: str, result: dict) -> None: + with self.lock: + self.stages.pop(pair, None) + status = f"failed at {result['stage']}" if result["status"] == "failed" else result["status"] + reason = f": {result['reason']}" if result.get("reason") else "" + self.say(f"{pair}: {'reused, ' if result.get('reused') else ''}{status}{reason} " + f"({_took(time.monotonic() - self.began)})", failed=result["status"] == "failed") + + def _beat(self) -> None: + while not self.stopped.wait(_HEARTBEAT_S / 10): + now = time.monotonic() + with self.lock: + due = [(pair, stage, since) for pair, (stage, since, said) in self.stages.items() + if now - said >= _HEARTBEAT_S] + for pair, stage, since in due: + self.stages[pair] = (stage, since, now) + for pair, stage, since in due: + self.say(f"{pair}: still {stage} ({_took(now - since)})") + + +def _took(seconds: float) -> str: + minutes, seconds = divmod(round(seconds), 60) + return f"{minutes} min {seconds} s" if minutes else f"{seconds} s" + + def _rank(result: dict) -> int: """How far a pair got: ready, then built and either unprobed or failed at its probe, then anything short of an image. A result takes a stored one's place only when it got at least as far.""" diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index caeb96a23..b4c0203d4 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -36,8 +36,8 @@ def composed(tmp_path, monkeypatch, local_stores, cli_routing, builds): monkeypatch.setattr("agent_env.env.published_images._registry_digest", lambda ref: None) calls = [] - def compose(env, on_build=None, sandbox_provider=None): - on_build() + def compose(env, on_progress=None, sandbox_provider=None): + on_progress("composing") calls.append((env.id, env.computer.id, env.driver.id, sandbox_provider)) monkeypatch.setattr(composer, "compose", compose) @@ -71,17 +71,17 @@ def _plan_env(root): @pytest.mark.parametrize("planned", [_plan, _plan_env], ids=["a-run", "the-env-alone"]) def test_a_computer_env_naming_shipped_aliases_is_written_over_them_and_composed(tmp_path, composed, planned): plan, provider = planned(_bundle(tmp_path / "desk")), LocalSandboxProvider() - building = [] + told = [] - done = materialize(plan, build_here=local_builds(plan), on_build=lambda write: building.append(write.id), - sandbox_provider=provider) + done = materialize(plan, build_here=local_builds(plan), sandbox_provider=provider, + on_progress=lambda write, line: told.append((write.id, line))) env = Env.get(f"{plan.bundle.bundle.id_root}/desktop") written = {item.write.id for item in done.writes} assert isinstance(env, ComputerEnv) and {UBUNTU, REFERENCE} <= written assert (env.computer.id, env.driver.id, env.environment_name) == (UBUNTU, REFERENCE, "desktop") assert env.computer_settings == {"timezone": "Europe/Paris"} and env.display == {"width": 1600, "height": 900} - assert composed == [(env.id, UBUNTU, REFERENCE, provider)] and env.id in building + assert composed == [(env.id, UBUNTU, REFERENCE, provider)] and told == [(env.id, "composing")] def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_path, composed, monkeypatch): diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 6bd1e549c..5998a37d3 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -15,6 +15,7 @@ import shutil import subprocess import tarfile +import time import urllib.parse from pathlib import Path from types import SimpleNamespace @@ -262,14 +263,20 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one monkeypatch.setattr(builder, "docker_login", lambda signed_in: docker_calls.append(("login", signed_in))) monkeypatch.setattr(composer, "write_index", lambda index, images, signed_in: ( docker_calls.append(("join", index, signed_in)), docker.joined(index, images, signed_in))) - announced = [] - first = _put(base, driver, on_build=lambda: announced.append("first")) + first_told, second_told = [], [] + first = _put(base, driver, on_progress=first_told.append) docker_calls.clear() second = _put(base, driver, id="@local/~/kit/other", environment_name="other", display={"width": 640, "height": 480}, - network={"mode": "allowlist"}, on_build=lambda: announced.append("second")) + network={"mode": "allowlist"}, on_progress=second_told.append) built, reused = (Env.get(env.id).metadata["composed"]["container"] for env in (first, second)) - assert len(docker.builds) == 2 and all(reused[arch]["reused"] for arch in reused) and announced == ["first"] + assert len(docker.builds) == 2 and all(reused[arch]["reused"] for arch in reused) + assert "container/arm64: building natively with this machine's Docker" in first_told + assert "container/amd64: building emulated, which is slow, with this machine's Docker" in first_told + assert not [line for line in second_told if "building" in line or "resolving" in line] + assert second_told[0] == "composing container/amd64, container/arm64 on LocalSandboxProvider" + assert "container/amd64: reusing the image of the same key, built earlier" in second_told + assert any(re.fullmatch(r"container/arm64: reused, ready \(\d+ s\)", line) for line in second_told) assert len(docker.resolved) == 2 and {arch: r["lock"] for arch, r in reused.items()} == { arch: r["lock"] for arch, r in built.items()} assert {arch: reused[arch]["image"] for arch in reused} == {arch: built[arch]["image"] for arch in built} @@ -279,6 +286,20 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one assert docker_calls == [("join", index, auth)] +def test_a_stage_that_goes_on_is_said_to_go_on_and_a_failed_pair_says_why(base, driver, docker, monkeypatch): + docker.fail["linux/amd64"] = "#7 ERROR: pip install failed" + build = docker.build_image + monkeypatch.setattr(composer, "_HEARTBEAT_S", 0.1) + monkeypatch.setattr(builder, "build_image", lambda *args, **kwargs: (time.sleep(0.5), build(*args, **kwargs))[1]) + told = [] + + _put(base, driver, on_progress=told.append) + + assert [line for line in told if line.startswith("container/arm64: still building the image (")] + assert any(re.fullmatch(r"container/amd64: failed at build: the build failed: #7 ERROR: pip install failed \(\d+ s\)", + line) for line in told) + + @pytest.mark.parametrize("change", ["requirements", "build-setting", "service", "display-read-at-build"]) def test_what_changes_the_image_changes_its_key_and_builds_it_again(base, driver, docker, tmp_path, monkeypatch, change): From 5820dae44dc6bfeced9d2c2d8b0b33a5f670ce39 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 13:21:14 -0700 Subject: [PATCH 03/11] fix(computer): an app's .agentenv/ files are refused, and generated build settings go to a VM inline An app's files under .agentenv/, the folder its own server keeps its socket and build settings in, are refused at put: a build settings file there reached the app unchecked. The build settings file the composer writes leaves the files a builder VM fetches from the store, so a VM no longer builds an app stored before with the app's own copy, under a key computed from the env's values. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/app_env.py | 16 +++++++++++++++- .../env/envs/computer/utils/composer.py | 2 ++ tst/unit/env/envs/test_computer_env.py | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index fec54566a..e84f822bd 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from agentenv_protocol.computer import AppDeclaration, Launch +from agentenv_protocol.computer.app import SOCKET from agent_env.artifact import Artifact from agent_env.artifact.artifacts.computer import ComputerArtifact @@ -22,7 +23,7 @@ from agent_env.store.ids import derive_id if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import Callable, Iterable from agent_env.bundle.authoring import AuthoringContext from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider @@ -109,6 +110,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact declaration=declaration, computer=computer, driver=driver, build_settings=build_settings) _, problems = _installs(declaration) problems += _class_beyond_reach(module, declaration) + problems += _in_server_folder(declaration, files.get_file_artifacts()) problems += gate.check_app_build_settings(app, app.build_settings, "build_settings", computer.os.type if computer is not None else None) if (computer is None) != (driver is None): @@ -181,6 +183,18 @@ def _installs(declaration: AppDeclaration) -> tuple[dict[str, AppInstall], list[ return installs({os_type: on.install for os_type, on in declaration.os.items()}, AppInstall) +def _in_server_folder(declaration: AppDeclaration, names: Iterable[str]) -> list[str]: + """The app's files its install would put in the folder its own server keeps its socket and build settings in, + which is the framework's: a build settings file there would reach the app unchecked.""" + reserved, problems = SOCKET.parent.as_posix(), [] + for os_type, on in declaration.os.items(): + folder = PurePosixPath(on.install.get("files", ".")) + problems += [f"install.{os_type}.files: {name!r} is in {reserved}/, where the app's own server keeps its socket " + "and build settings; leave it out of the app's folder" + for name in sorted(names) if PurePosixPath(name).is_relative_to(folder / reserved)] + return problems + + def _class_beyond_reach(module: str, declaration: AppDeclaration) -> list[str]: """An install that leaves out the class's file, which the app's own server serves from the app's folder.""" file = PurePosixPath(module.partition(":")[0]) diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index fbccfc9fe..1de9350eb 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -320,6 +320,8 @@ def _stage_app(context: Path, app: ComputerAppEnv, base: ComputerArtifact, if app.build_settings: (context / layer.folder / BUILD_SETTINGS).parent.mkdir(exist_ok=True) (context / layer.folder / BUILD_SETTINGS).write_text(json.dumps(app.build_settings, sort_keys=True)) + # Inline, over any the app's files hold, as an app stored before that folder was refused can. + from_store.pop(f"{layer.folder}/{BUILD_SETTINGS.as_posix()}", None) return layer diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 5998a37d3..02032eed2 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -41,6 +41,7 @@ from agent_env.env.envs.computer.utils import boot, builder, composer, gate, inside, packages, resolve_in_base from agent_env.env.envs.computer.utils.composer import _probe from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer import app_env from agent_env.env.envs.computer.app_env import ComputerAppEnv from agent_env.env.envs.computer.utils.install import Install, Service from agent_env.env.envs.computer.driver_env import ComputerDriverEnv @@ -1261,6 +1262,7 @@ def _app(base, driver, folder, compose=True, supports=None, install=None, settin (folder / "app.py").write_text("class Mail: ...\n") (folder / "requirements.txt").write_text("agentenv-framework-protocol\n") for name, text in (extra or {}).items(): + (folder / name).parent.mkdir(parents=True, exist_ok=True) (folder / name).write_text(text) files = FileArtifactUniverse.put_bundled("@local/~/kit/mail__files", files={ name: folder / name for name in ("app.py", "requirements.txt", *(extra or {}))}) @@ -1491,6 +1493,20 @@ def test_an_apps_build_settings_are_fixed_in_its_folder_and_a_change_builds_anot assert ComputerAppEnv.from_dict(stored.to_dict()).to_dict() == stored.to_dict() +def test_an_apps_file_in_its_servers_folder_is_refused_and_one_stored_before_is_built_over_by_its_build_settings( + base, driver, docker, signing, tmp_path, monkeypatch): + own = {".agentenv/build_settings.json": '{"locale": "en_GB"}'} + + with pytest.raises(ValueError, match=r"install.linux.files: '.agentenv/build_settings.json' is in .agentenv/, where " + r"the app's own server keeps its socket and build settings"): + _app(base, driver, tmp_path / "mail", extra=own, build_settings={"locale": "fr_FR"}) + monkeypatch.setattr(app_env, "_in_server_folder", lambda declaration, names: []) + stored = _app(base, driver, tmp_path / "mail", extra=own, build_settings={"locale": "fr_FR"}, compose=False) + composer.compose(stored, sandbox_provider=FakeVmProvider(docker, tmp_path / "vms")) + + assert [build["build_settings"] for build in docker.builds] == [{"mail": {"locale": "fr_FR"}}] * 2 + + @pytest.mark.parametrize("build, values, problem", [ ({**MAIL_BUILD, "required": ["locale"]}, {}, "build_settings: 'locale' is a required property"), (MAIL_BUILD, {"colour": "blue"}, "build_settings.colour: @local/~/kit/mail declares no build setting 'colour'"), From 4de6f615c5ba5ea2b537c255c0273aa41f80bdad Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 13:26:41 -0700 Subject: [PATCH 04/11] feat(bundle): a reused computer env composes what it has left on the run's provider When run or bundle register reuses an unchanged computer env, it composes the env's pending pairs (failed, stopped, or built but unprobed though the provider runs the arch natively) on its sandbox provider, as agent-env env build does, writing no new version. A dry run says what it would compose, and bundle register goes on to write when anything is left rather than stopping at "Nothing to write". Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/materialize.py | 35 +++++++++++++++++++-- src/agent_env/cli/bundle.py | 2 +- tst/unit/bundle/computer_env_folder_test.py | 29 +++++++++++++++-- 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index 634c063df..a7d3b6322 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -27,6 +27,9 @@ from agent_env.artifact.artifacts.docker_image import DockerImageArtifact, push_context_image from agent_env.artifact.registry import canonical_type, get_artifact_registry from agent_env.entity.refs import EntityKind, EntityRef, RefRole, ref_sites +from agent_env.env import Env +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils import composer from agent_env.env.registry import get_env_registry from agent_env.eval import Eval, EvalTask from agent_env.providers.sandbox_providers.host_images import build, build_staged @@ -67,6 +70,8 @@ class Materialization: writes: tuple[Materialized, ...] # in the plan's order # A dry run's steps with a preflight that read what it would write first, so the store can't check them yet. not_preflighted: tuple[tuple[Write, TaskStep], ...] = () + # The computer envs reused with pairs left to compose, which a run composes and a dry run says it would. + composing: tuple[str, ...] = () def version_of(self, store: str, id: str) -> int: for done in self.writes: @@ -99,8 +104,9 @@ def materialize( Each write gets the version it would reuse or the store's next, which another run can take first. A step reading what the run would write first isn't preflighted, since the store doesn't hold it yet. - ``sandbox_provider`` composes each computer env written, the env's sandbox provider when None, and - ``on_progress`` is told how each compose goes, a line at a time. + ``sandbox_provider`` composes each computer env written, and what one reused has left, as ``agent-env env + build`` does: the env's sandbox provider when None. ``on_progress`` is told how each compose goes, a line at a + time. ``ledger`` keeps the rows, this run's in the ``@local`` namespace's store unless given; it also says whether each task and eval names the versions it reads (``Ledger.pins_references``) and whether each image the bundle builds is @@ -114,6 +120,7 @@ def materialize( tasks = [write for write in plan.writes if write.kind is BundleKind.TASK] evals = [write for write in plan.writes if write.kind is BundleKind.EVAL] done: dict[tuple[str, str], Materialized] = {} + composing: list[str] = [] def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: with _noted(plan, write, "checking" if dry_run else "writing"): @@ -140,6 +147,9 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: on_progress)) if here and done[_key(write)].reused: _build_reused(plan, write, done[_key(write)].version, on_build) + if write.kind is BundleKind.ENV and done[_key(write)].reused and _compose_reused( + write, done[_key(write)].version, sandbox_provider, on_progress, dry_run): + composing.append(write.id) unwritten = {_key(item.write) for item in done.values() if not item.reused} if dry_run else set() written = done if ledger.pins_references else None built, problems, unchecked = {}, [], [] @@ -156,7 +166,7 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: requires=write.requires).version) for write in evals: through_ledger(write, lambda: _write_eval(plan, write, written)) - return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked)) + return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(composing)) def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None, @@ -303,6 +313,25 @@ def _write_agent(plan: Plan, write: Write) -> int: AuthoringContext(plan.bundle_of(entry), entry)).version +def _compose_reused(write: Write, version: int, sandbox_provider: SandboxProvider | None, + on_progress: Callable[[Write, str], None] | None, dry_run: bool) -> bool: + """Whether the computer env ``write`` reuses has pairs left to compose on ``sandbox_provider`` (``pending``: failed, + stopped, or built but unprobed though it runs their arch natively) or its computer to write again; composed unless + ``dry_run``, which says what it would compose.""" + env = Env.get(write.id, version) + if not isinstance(env, ComputerEnv) or env.computer is None: + return False + told = None if on_progress is None else lambda line: on_progress(write, line) + pairs = composer.pending(env, sandbox_provider) + if not pairs and composer.current(env): + return False + if not dry_run: + composer.compose(env, pairs, told, sandbox_provider) + elif told is not None: + told(f"would compose {', '.join(f'{form}/{arch}' for form, arch in pairs) or 'its computer again'}") + return True + + def _write_env(plan: Plan, write: Write, sandbox_provider: SandboxProvider | None = None, on_progress: Callable[[str], None] | None = None) -> int: entry = write.source.entry diff --git a/src/agent_env/cli/bundle.py b/src/agent_env/cli/bundle.py index 1420712b8..fb6397f74 100644 --- a/src/agent_env/cli/bundle.py +++ b/src/agent_env/cli/bundle.py @@ -125,7 +125,7 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi if dry_run: click.echo("Nothing was written: --dry-run.") return - if all(done.reused for done in planned.materialization.writes): + if all(done.reused for done in planned.materialization.writes) and not planned.materialization.composing: click.echo("Nothing to write: every version is registered already.") return if planned.shared and not yes: diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index b4c0203d4..d0637237a 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -36,9 +36,10 @@ def composed(tmp_path, monkeypatch, local_stores, cli_routing, builds): monkeypatch.setattr("agent_env.env.published_images._registry_digest", lambda ref: None) calls = [] - def compose(env, on_progress=None, sandbox_provider=None): - on_progress("composing") - calls.append((env.id, env.computer.id, env.driver.id, sandbox_provider)) + def compose(env, pairs=None, on_progress=None, sandbox_provider=None): + if on_progress is not None: + on_progress("composing") + calls.append((env.id, env.computer.id, env.driver.id, sandbox_provider) + ((pairs,) if pairs is not None else ())) monkeypatch.setattr(composer, "compose", compose) return calls @@ -84,6 +85,28 @@ def test_a_computer_env_naming_shipped_aliases_is_written_over_them_and_composed assert composed == [(env.id, UBUNTU, REFERENCE, provider)] and told == [(env.id, "composing")] +def test_a_reused_computer_env_composes_what_it_has_left_on_the_runs_provider_and_nothing_once_its_built(tmp_path, + composed): + root, provider = _bundle(tmp_path / "desk"), LocalSandboxProvider() + materialize(_plan(root), build_here=local_builds(_plan(root))) + env = Env.get(f"{_plan(root).bundle.bundle.id_root}/desktop") + told = [] + + checked = materialize(_plan(root), dry_run=True, sandbox_provider=provider, + on_progress=lambda write, line: told.append((write.id, line))) + again = materialize(_plan(root), build_here=local_builds(_plan(root)), sandbox_provider=provider, + on_progress=lambda write, line: told.append((write.id, line))) + env.update_metadata({**env.metadata, "computer": {"id": "c", "version": 1, "images": {"amd64": "a", "arm64": "b"}}, + "composed": {"container": {"amd64": {"status": "ready", "image": "a"}, + "arm64": {"status": "ready", "image": "b"}}}}) + built = materialize(_plan(root), build_here=local_builds(_plan(root)), sandbox_provider=provider) + + assert built.composing == () and checked.composing == again.composing == (env.id,) and all(item.reused for item in again.writes) + assert told == [(env.id, "would compose container/amd64, container/arm64"), (env.id, "composing")] + assert composed == [(env.id, UBUNTU, REFERENCE, None), + (env.id, UBUNTU, REFERENCE, provider, [("container", "amd64"), ("container", "arm64")])] + + def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_path, composed, monkeypatch): """A default agent two installed bundles both alias names neither, which blocks a run of a task that deploys it, but not writing an env that no task of the bundle runs for.""" From f4fc511847bb4738674dd93bb7142484a6afd079 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 13:53:15 -0700 Subject: [PATCH 05/11] feat(computer): --no-emulate skips an arch no link runs natively; emulated builds get longer limits A put can turn emulation off (--no-emulate on env build, bundle register and run; emulate=False on a put): an arch no link of the sandbox provider runs natively is recorded skipped, with each link's reason, and stays pending for a later put that runs it natively or emulates it. A builder VM gets 2 h, 6 h when it emulates the arch, and the resolver 1 h, 3 h under emulation. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- src/agent_env/bundle/authoring.py | 2 ++ src/agent_env/bundle/materialize.py | 23 ++++++++------- src/agent_env/bundle/register.py | 5 ++-- src/agent_env/bundle/run.py | 14 ++++++---- src/agent_env/cli/bundle.py | 7 +++-- src/agent_env/cli/env/build.py | 5 ++-- src/agent_env/cli/run.py | 13 +++++---- src/agent_env/env/envs/computer/app_env.py | 10 +++---- src/agent_env/env/envs/computer/env.py | 11 ++++---- .../env/envs/computer/utils/builder.py | 28 +++++++++++++------ .../env/envs/computer/utils/composer.py | 10 +++++-- .../env/envs/computer/utils/packages.py | 6 ++-- tst/unit/bundle/computer_env_folder_test.py | 2 +- tst/unit/bundle/installed_test.py | 6 ++-- tst/unit/env/envs/test_computer_env.py | 27 +++++++++++++++--- 16 files changed, 112 insertions(+), 59 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 430ac1f21..e95f6e8ec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -112,7 +112,7 @@ and deserialized through a registry. | Package | Contents | |---|---| -| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU), and its pair records the provider that built it (`builder`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference; its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | +| `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference; its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | diff --git a/src/agent_env/bundle/authoring.py b/src/agent_env/bundle/authoring.py index a16f9e02d..959a0acce 100644 --- a/src/agent_env/bundle/authoring.py +++ b/src/agent_env/bundle/authoring.py @@ -55,6 +55,8 @@ class AuthoringContext: on_progress: Callable[[str], None] | None = None #: Where writing a computer env composes it; the env's sandbox provider when None. sandbox_provider: SandboxProvider | None = None + #: Whether composing it emulates an arch no link of that provider runs natively, rather than skip it. + emulate: bool = True @property def id(self) -> str: diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index a7d3b6322..ef1a9f0c2 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -91,6 +91,7 @@ def materialize( ledger: Ledger | None = None, sandbox_provider: SandboxProvider | None = None, on_progress: Callable[[Write, str], None] | None = None, + emulate: bool = True, ) -> Materialization: """Write ``plan``'s entities, tasks and evals. The first write that fails stops it, and every earlier one stays: they're in the ledger, so the next run reuses them. ``on_wait`` is called when another run @@ -105,8 +106,8 @@ def materialize( reading what the run would write first isn't preflighted, since the store doesn't hold it yet. ``sandbox_provider`` composes each computer env written, and what one reused has left, as ``agent-env env - build`` does: the env's sandbox provider when None. ``on_progress`` is told how each compose goes, a line at a - time. + build`` does: the env's sandbox provider when None, emulating an arch no link runs natively unless ``emulate`` is + False. ``on_progress`` is told how each compose goes, a line at a time. ``ledger`` keeps the rows, this run's in the ``@local`` namespace's store unless given; it also says whether each task and eval names the versions it reads (``Ledger.pins_references``) and whether each image the bundle builds is @@ -144,11 +145,11 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: through_ledger(write, lambda: _write_pushed_image(plan, write, on_build)) else: through_ledger(write, lambda: _write_entity(plan, write, here, on_build, sandbox_provider, - on_progress)) + on_progress, emulate)) if here and done[_key(write)].reused: _build_reused(plan, write, done[_key(write)].version, on_build) if write.kind is BundleKind.ENV and done[_key(write)].reused and _compose_reused( - write, done[_key(write)].version, sandbox_provider, on_progress, dry_run): + write, done[_key(write)].version, sandbox_provider, on_progress, emulate, dry_run): composing.append(write.id) unwritten = {_key(item.write) for item in done.values() if not item.reused} if dry_run else set() written = done if ledger.pins_references else None @@ -170,7 +171,8 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None, - sandbox_provider: SandboxProvider | None, on_progress: Callable[[Write, str], None] | None) -> int: + sandbox_provider: SandboxProvider | None, on_progress: Callable[[Write, str], None] | None, + emulate: bool) -> int: if isinstance(write.source, ReferencedImage): return _write_referenced_image(plan, write) if isinstance(write.source, BuiltImage): @@ -179,7 +181,7 @@ def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Writ return _write_build_context(plan, write, here, on_build) if write.kind is BundleKind.ENV: return _write_env(plan, write, sandbox_provider, - None if on_progress is None else lambda line: on_progress(write, line)) + None if on_progress is None else lambda line: on_progress(write, line), emulate) return _WRITERS[write.kind](plan, write) @@ -314,7 +316,7 @@ def _write_agent(plan: Plan, write: Write) -> int: def _compose_reused(write: Write, version: int, sandbox_provider: SandboxProvider | None, - on_progress: Callable[[Write, str], None] | None, dry_run: bool) -> bool: + on_progress: Callable[[Write, str], None] | None, emulate: bool, dry_run: bool) -> bool: """Whether the computer env ``write`` reuses has pairs left to compose on ``sandbox_provider`` (``pending``: failed, stopped, or built but unprobed though it runs their arch natively) or its computer to write again; composed unless ``dry_run``, which says what it would compose.""" @@ -326,18 +328,19 @@ def _compose_reused(write: Write, version: int, sandbox_provider: SandboxProvide if not pairs and composer.current(env): return False if not dry_run: - composer.compose(env, pairs, told, sandbox_provider) + composer.compose(env, pairs, told, sandbox_provider, emulate) elif told is not None: told(f"would compose {', '.join(f'{form}/{arch}' for form, arch in pairs) or 'its computer again'}") return True def _write_env(plan: Plan, write: Write, sandbox_provider: SandboxProvider | None = None, - on_progress: Callable[[str], None] | None = None) -> int: + on_progress: Callable[[str], None] | None = None, emulate: bool = True) -> int: entry = write.source.entry cls = get_env_registry()[entry.type] return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext( - plan.bundle_of(entry), entry, on_progress=on_progress, sandbox_provider=sandbox_provider)).version + plan.bundle_of(entry), entry, on_progress=on_progress, sandbox_provider=sandbox_provider, + emulate=emulate)).version def _pinned(plan: Plan, write: Write, refs: tuple[EntityRef, ...]) -> Any: diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 2e9321539..9410f3925 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -79,6 +79,7 @@ def register_bundle( declaring: Declaring | None = None, on_progress: Callable[[str], None] | None = None, sandbox: str | None = None, + emulate: bool = True, ) -> Registration: """Register the bundle at ``root``: ``tasks`` and ``evals`` by name, or with neither every task and eval it has, or every entity when it has none, with what they reference. ``prefix`` opens each entry's registry id, the folder's @@ -86,7 +87,7 @@ def register_bundle( is given a line for each write, as it's checked or written, and before each image is built. A folder with no bundle.toml that uses plugin packages is given one, or told what to declare, as ``declaring`` says: by default as a run of the folder would (``declaring_for``). ``sandbox`` is the provider its computer envs compose on, by default - the env's. + the env's, emulating an arch no link runs natively unless ``emulate`` is False, which leaves it skipped. It turns namespace routing on while it runs. Raises BundleError listing every problem found before anything is written; a write that fails stops it, and every earlier one stays, in the ledger, for the next run to reuse.""" @@ -113,7 +114,7 @@ def register_bundle( on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), on_build=lambda write: say(f" {_label(registered, write)}: building with docker and pushing"), on_write=lambda done: say(f" {registered_line(registered, done)}"), sandbox_provider=provider, - on_progress=lambda write, line: say(f" {_label(registered, write)}: {line}")) + on_progress=lambda write, line: say(f" {_label(registered, write)}: {line}"), emulate=emulate) return Registration(registered, materialization, ids, tuple(_floating(registered)), identity, shared, requires) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index d1c5b00f2..9f714636c 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -214,12 +214,14 @@ def run_bundle( on_progress: Callable[[str], None] | None = None, id_root: str | None = None, keep: bool = False, + emulate: bool = True, ) -> BundleRun: """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the user simulator and the judges run on, and ``agent`` the agent the solver runs as (``Plan.agent``), leaving the tasks as written; ``sandbox`` is the provider that computer envs compose on and that envs, - agents, sandboxes and the judge deploy on. ``on_progress`` is given a line of text + agents, sandboxes and the judge deploy on; a computer env composing there emulates an arch no link runs natively + unless ``emulate`` is False, which leaves it skipped. ``on_progress`` is given a line of text at each write and step. ``id_root`` roots the ids of a bundle an installed package provides (``InstalledBundle.id_root``); a folder's ids otherwise come from its path. @@ -244,7 +246,7 @@ def run_bundle( say = _progress(on_progress) with namespace_routing(): prepared = _prepare(root, tasks, evals, sandbox, id_root, agent, False, say, - declaring_for(Path(root), installed=id_root is not None, dry_run=False)) + declaring_for(Path(root), installed=id_root is not None, dry_run=False), emulate) plan, entries = prepared.plan, [entry for entry, _ in prepared.tasks] with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) @@ -298,7 +300,7 @@ def _write_bundle( def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], None], - sandbox: str | None = None) -> Materialization: + sandbox: str | None = None, emulate: bool = True) -> Materialization: """Write ``plan``, its computer envs composed on ``sandbox`` when given, then set up the infra envs its runs deploy on.""" materialization = materialize( @@ -309,6 +311,7 @@ def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], N build_here=preflight.built_here, sandbox_provider=build_sandbox_provider(sandbox) if sandbox else None, on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}"), + emulate=emulate, ) if preflight.infra: ensure_default_envs(preflight.infra_kinds, say=say) @@ -367,10 +370,11 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandbox: str | None, id_root: str | None, - agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring) -> PreparedBundle: + agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring, + emulate: bool = True) -> PreparedBundle: plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played) _say_plan(plan, say, declaring) - materialization = _written_with_infra(plan, preflight, say, sandbox) + materialization = _written_with_infra(plan, preflight, say, sandbox, emulate) to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan)) return PreparedBundle(plan, materialization, to_run) diff --git a/src/agent_env/cli/bundle.py b/src/agent_env/cli/bundle.py index fb6397f74..a4ff9efcb 100644 --- a/src/agent_env/cli/bundle.py +++ b/src/agent_env/cli/bundle.py @@ -87,8 +87,11 @@ def _report(imported: BundleImport, folder: Path) -> None: @click.option("--sandbox", default=None, help="The sandbox provider the bundle's computer envs compose on, by default the env's; comma-separated " "for a chain, each arch going to the first link that runs it natively.") +@click.option("--no-emulate", is_flag=True, + help="Skip, rather than emulate, an arch no link of the sandbox provider runs natively; a later register " + "or env build builds it.") def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefix: str | None, - dry_run: bool, yes: bool, sandbox: str | None): + dry_run: bool, yes: bool, sandbox: str | None, no_emulate: bool): """Write the bundle at FOLDER into the configured stores under registry ids, so whatever runs the stores' tasks runs what the folder runs. @@ -105,7 +108,7 @@ def register(folder: Path, tasks: tuple[str, ...], evals: tuple[str, ...], prefi with no bundle.toml that uses plugin packages is given one. An id the store holds already takes a new version, whoever wrote it. A store other than this machine's is written only after you confirm, or with --yes. Registering again writes only what changed since.""" - options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox} + options = {"tasks": tasks, "evals": evals, "prefix": prefix, "sandbox": sandbox, "emulate": not no_emulate} click.echo(f"Registering {folder} into:") for line in describe_target(): click.echo(f" {line}") diff --git a/src/agent_env/cli/env/build.py b/src/agent_env/cli/env/build.py index c7e38a015..1491a6996 100644 --- a/src/agent_env/cli/env/build.py +++ b/src/agent_env/cli/env/build.py @@ -15,7 +15,8 @@ @click.option("--version", type=int, default=None, help="Its version; the latest when left out") @click.option("--sandbox", default=None, help="The sandbox provider that resolves, builds and probes, by default the " "env's; comma-separated for a chain, each arch going to the first link that runs it natively") -def build(env_id: str, version: int | None, sandbox: str | None): +@click.option("--no-emulate", is_flag=True, help="Skip, rather than emulate, an arch no link runs natively") +def build(env_id: str, version: int | None, sandbox: str | None, no_emulate: bool): """Build a computer env's forms and arches not built yet, reusing any image of the same key, and print each one's status; exits 1 when one failed.""" try: @@ -28,7 +29,7 @@ def build(env_id: str, version: int | None, sandbox: str | None): if env.computer is None: raise click.ClickException(f"{env_id} names no computer and driver, so it has nothing to build alone") if (pairs := composer.pending(env, provider)) or not composer.current(env): - composer.compose(env, pairs, on_progress=click.echo, sandbox_provider=provider) + composer.compose(env, pairs, on_progress=click.echo, sandbox_provider=provider, emulate=not no_emulate) failed = False for form, by_arch in env.metadata.get("composed", {}).items(): for arch, result in by_arch.items(): diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 767b86f6f..65728b990 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -47,10 +47,13 @@ help="Keep the sandboxes up after the runs, print their endpoints, and tear them down on Ctrl-C.") @click.option("--dry-run", is_flag=True, help="Check the bundle as the run does and show what it would write and run; write and run nothing.") +@click.option("--no-emulate", is_flag=True, + help="Skip, rather than emulate, a computer env's arch no link of the sandbox provider runs natively; " + "a later run or env build builds it.") @click.pass_context def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: tuple[str, ...], model: str | None, user_model: str | None, judge_model: str | None, agent: str | None, sandbox: str | None, keep: bool, - dry_run: bool): + dry_run: bool, no_emulate: bool): """Run a bundle's tasks and evals, or list the installed bundles. BUNDLE is a folder, or the name of a bundle an installed package provides (PACKAGE/NAME when several @@ -78,9 +81,9 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t exits 130 (143 for SIGTERM); a second one stops the teardown and prints what is still up. """ if bundle is None: - if tasks or evals or model or user_model or judge_model or agent or sandbox or keep or dry_run: - raise click.UsageError("--task, --eval, --model, --user-model, --judge-model, --agent, --sandbox, --keep " - "and --dry-run need a BUNDLE to run") + if tasks or evals or model or user_model or judge_model or agent or sandbox or keep or dry_run or no_emulate: + raise click.UsageError("--task, --eval, --model, --user-model, --judge-model, --agent, --sandbox, --keep, " + "--dry-run and --no-emulate need a BUNDLE to run") _list_installed() return root, id_root = locate_bundle(bundle, _installed) @@ -93,7 +96,7 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t return result = run_bundle(root, tasks=tasks, evals=evals, model=model, user_model=user_model, judge_model=judge_model, agent=agent, sandbox=sandbox, on_progress=click.echo, - id_root=id_root, keep=keep) + id_root=id_root, keep=keep, emulate=not no_emulate) except BundleError as e: _note_an_installed_namesake(e, bundle, root, id_root) if isinstance(e, UnmetRequirements) and not dry_run: diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index e84f822bd..c71d2ac71 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -101,11 +101,11 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact declaration: AppDeclaration, computer: Optional[ComputerArtifact] = None, driver: Optional[ComputerDriverEnv] = None, build_settings: Optional[dict[str, Any]] = None, compose: bool = True, on_progress: Optional[Callable[[str], None]] = None, - sandbox_provider: Optional[SandboxProvider] = None) -> Self: + sandbox_provider: Optional[SandboxProvider] = None, emulate: bool = True) -> Self: """Write the app once what it declares and ``build_settings``, its build settings' values, check out, and with a computer and a driver, once it fits them (gate 1); then compose it with them on ``sandbox_provider``, by - default the env's, unless ``compose`` is False, telling ``on_progress`` how it goes. Refused, naming every - problem, before anything is stored.""" + default the env's, emulating an arch no link runs natively unless ``emulate`` is False, unless ``compose`` is + False, telling ``on_progress`` how it goes. Refused, naming every problem, before anything is stored.""" app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, declaration=declaration, computer=computer, driver=driver, build_settings=build_settings) _, problems = _installs(declaration) @@ -122,7 +122,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) app = get_env_store().put_document(app) if computer is not None and compose: - composer.compose(app, on_progress=on_progress, sandbox_provider=sandbox_provider) + composer.compose(app, on_progress=on_progress, sandbox_provider=sandbox_provider, emulate=emulate) return app @classmethod @@ -149,7 +149,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerAppEnv: return cls.put(id=ctx.id, environment_name=fields.get("environment_name", ctx.name), module=fields["module"], files=universe, declaration=declaration, computer=computer, driver=driver, build_settings=fields.get("build_settings"), on_progress=ctx.on_progress, - sandbox_provider=ctx.sandbox_provider) + sandbox_provider=ctx.sandbox_provider, emulate=ctx.emulate) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index 0fd5166b1..b1d4d5f00 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -84,10 +84,11 @@ def from_dict(cls, data: dict) -> ComputerEnv: def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str, apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None, compose: bool = True, on_progress: Optional[Callable[[str], None]] = None, - sandbox_provider: Optional[SandboxProvider] = None) -> Self: + sandbox_provider: Optional[SandboxProvider] = None, emulate: bool = True) -> Self: """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it on - ``sandbox_provider``, by default the env's, unless ``compose`` is False, telling ``on_progress`` how it goes. - Refused, naming every mismatch, before anything is stored.""" + ``sandbox_provider``, by default the env's, emulating an arch no link runs natively unless ``emulate`` is False, + unless ``compose`` is False, telling ``on_progress`` how it goes. Refused, naming every mismatch, before + anything is stored.""" values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, computer_settings=computer_settings or {}) if problems: @@ -96,7 +97,7 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, display=display, network=network, computer_settings=values)) if compose: - composer.compose(env, on_progress=on_progress, sandbox_provider=sandbox_provider) + composer.compose(env, on_progress=on_progress, sandbox_provider=sandbox_provider, emulate=emulate) return env @classmethod @@ -114,7 +115,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: try: return cls.put(id=ctx.id, computer=base, driver=driver, environment_name=fields.pop("environment_name", ctx.name), on_progress=ctx.on_progress, - sandbox_provider=ctx.sandbox_provider, **fields) + sandbox_provider=ctx.sandbox_provider, emulate=ctx.emulate, **fields) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index a60216920..6c26595d5 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -57,7 +57,9 @@ "linux/arm64": "38e890e1a162bfdbf32fbe91404991dc4fc28e55b0de901b34374a061bab4d2c"} _PLUGINS = "/usr/local/lib/docker/cli-plugins" #: A builder VM: a build runs apt, pip and an app's install steps, and holds the base, the lock's files and the image. -_VM = {"cpu": 4.0, "memory": 8192, "disk_size_gb": 40, "timeout": 7200} +_VM = {"cpu": 4.0, "memory": 8192, "disk_size_gb": 40} +#: How long a builder VM may run, natively and when it emulates the arch, which runs some 10-20 times slower. +VM_TIMEOUT_S, EMULATED_VM_TIMEOUT_S = 7200, 21600 _DOWNLOAD_TIMEOUT_S = 600 _FETCHES_IN_FLIGHT = 8 _CHUNK = 1 << 20 @@ -75,6 +77,10 @@ class Unbuildable(RuntimeError): """No link of the put's sandbox provider builds a platform; the message says why for each.""" +class Skipped(Exception): + """No link of the put's sandbox provider runs a platform natively, and the put turned emulation off.""" + + @dataclass(frozen=True) class Hashed: sha256: str @@ -133,18 +139,21 @@ def close(self) -> None: class Builds: """The builders the put's sandbox provider gives: for each platform, from the first of its links that runs the - platform natively, else from the first that emulates it. A link builds when it's this machine's Docker, or gives VMs - that reach the stores ``store_id`` routes to and the registry ``repository`` is in.""" + platform natively, else, unless ``emulate`` is False, from the first that emulates it. A link builds when it's this + machine's Docker, or gives VMs that reach the stores ``store_id`` routes to and the registry ``repository`` is in.""" - def __init__(self, provider: SandboxProvider, store_id: str, repository: str): - self.provider = provider + def __init__(self, provider: SandboxProvider, store_id: str, repository: str, emulate: bool = True): + self.provider, self.emulate = provider, emulate self._refused = [(link, _refused(link, store_id, repository)) for link in provider.links] def link(self, platform: str) -> SandboxProvider: - """The link that builds ``platform``. Raises Unbuildable.""" + """The link that builds ``platform``. Raises Unbuildable, or Skipped when it would emulate it and mustn't.""" links = [link for link, problem in self._refused if problem is None] if native := [link for link in links if platform in link.native_platforms()]: return native[0] + if not self.emulate: + raise Skipped("emulation is off, and " + "; ".join( + problem or f"{type(link).__name__} doesn't run {platform} natively" for link, problem in self._refused)) for link in links: if not _docker_here(link) or platform in _emulated_platforms(): return link @@ -160,7 +169,7 @@ def open(self, link: SandboxProvider, platform: str, work: Path, base: DockerIma if _docker_here(link): yield LocalBuilder(type(link).__name__, work, base, platform) return - vm = VmBuilder(link) + vm = VmBuilder(link, emulating=platform not in link.native_platforms()) try: vm.start(work, base, platform, from_store) yield vm @@ -303,11 +312,12 @@ class VmBuilder(Builder): """A VM of a sandbox provider with Docker in it, building in a folder of its own there. Its calls run on an event loop of its own, in the thread that made it.""" - def __init__(self, link: SandboxProvider): + def __init__(self, link: SandboxProvider, emulating: bool): self.name, self._runner = type(link).__name__, asyncio.Runner() self.work = f"/tmp/agentenv-compose-{uuid.uuid4().hex[:12]}" + timeout = EMULATED_VM_TIMEOUT_S if emulating else VM_TIMEOUT_S try: - self.sandbox: VmSandbox = self._run(link.create_vm(exposed_ports=[], **_VM)) + self.sandbox: VmSandbox = self._run(link.create_vm(exposed_ports=[], timeout=timeout, **_VM)) except BaseException: self._runner.close() raise diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 1de9350eb..4731a6aa5 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -82,15 +82,17 @@ def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, - on_progress: Callable[[str], None] | None = None, sandbox_provider: SandboxProvider | None = None) -> None: + on_progress: Callable[[str], None] | None = None, sandbox_provider: SandboxProvider | None = None, + emulate: bool = True) -> None: """Build each of ``pairs``, by default each form the base has in each arch the env composes, in parallel, on ``sandbox_provider``, by default the env's, and probe each one it runs natively (gate 3), recording each as it - starts and finishes, then write the env's computer over every pair built so far. Each pair's stages and how it + starts and finishes, then write the env's computer over every pair built so far. An arch no link runs natively is + emulated, or with ``emulate`` False skipped, which leaves it for a later compose. Each pair's stages and how it ended go to ``on_progress`` and the log as they happen (``_Progress``).""" from agent_env.providers import get_env_sandbox_provider base, driver, provider = env.computer, env.driver, sandbox_provider or get_env_sandbox_provider() - builds = builder.Builds(provider, env.stored_id, REPOSITORY) + builds = builder.Builds(provider, env.stored_id, REPOSITORY, emulate) pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver, env.composed_apps())] record = _Record(env) @@ -249,6 +251,8 @@ def on() -> builder.Builder: return {"status": "unverified", "key": key, "image": pin_digest(ref, store.auth(ref)), "reused": False, "emulated": emulated, "builder": building_on.name, "lock": inputs, "installed": installed, "built_at": _now()} + except builder.Skipped as e: + return {"status": "skipped", "reason": str(e), "skipped_at": _now()} except builder.Unbuildable as e: return _failed(env, form, arch, stage, key, str(e)) except builder.ResolveError as e: diff --git a/src/agent_env/env/envs/computer/utils/packages.py b/src/agent_env/env/envs/computer/utils/packages.py index f14ed632b..99c7df7f6 100644 --- a/src/agent_env/env/envs/computer/utils/packages.py +++ b/src/agent_env/env/envs/computer/utils/packages.py @@ -24,7 +24,8 @@ PREFIX = "agentenv-packages" #: Where a build's context holds a lock's .debs; each venv's wheels are at ``staged_wheels``. STAGED_APT = "packages/apt" -_RESOLVE_TIMEOUT_S = 3600 +#: How long the resolver may take, natively and under emulation, which runs some 10-20 times slower. +_RESOLVE_TIMEOUT_S, _EMULATED_RESOLVE_TIMEOUT_S = 3600, 10800 class LockedFile(BaseModel): @@ -107,7 +108,8 @@ def resolve(inputs: str, wants: list[Want], builder: Builder, source: str, arch: lock, and where on the builder each file this resolve fetched is, by its sha256, for a build that follows to stage it from there rather than the store.""" (store, prefix), fetched = _store(store_id), {} - result = builder.resolve(_request(wants, arch), source, arch, _RESOLVE_TIMEOUT_S) + timeout = _RESOLVE_TIMEOUT_S if builder.platform == f"linux/{arch}" else _EMULATED_RESOLVE_TIMEOUT_S + result = builder.resolve(_request(wants, arch), source, arch, timeout) written = builder.files(f"{RESOLVED}/out") def put(folder: str, path: str) -> dict: diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index d0637237a..347d66676 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -36,7 +36,7 @@ def composed(tmp_path, monkeypatch, local_stores, cli_routing, builds): monkeypatch.setattr("agent_env.env.published_images._registry_digest", lambda ref: None) calls = [] - def compose(env, pairs=None, on_progress=None, sandbox_provider=None): + def compose(env, pairs=None, on_progress=None, sandbox_provider=None, emulate=True): if on_progress is not None: on_progress("composing") calls.append((env.id, env.computer.id, env.driver.id, sandbox_provider) + ((pairs,) if pairs is not None else ())) diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index be77271c8..6d3d58eb0 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -281,11 +281,11 @@ def test_the_cli_says_so_when_no_bundles_are_installed_and_needs_a_bundle_for_it assert CliRunner().invoke(cli, ["run"]).output == "No bundles are installed. agent-env run PATH runs a folder.\n" for option in (["--task", "t"], ["--keep"], ["--dry-run"], ["--user-model", "m"], ["--judge-model", "m"], - ["--agent", "pi"]): + ["--agent", "pi"], ["--no-emulate"]): usage = CliRunner().invoke(cli, ["run", *option]) assert usage.exit_code == 2 - assert ("--task, --eval, --model, --user-model, --judge-model, --agent, --sandbox, --keep and --dry-run need a " - "BUNDLE to run") in " ".join(usage.output.split()) + assert ("--task, --eval, --model, --user-model, --judge-model, --agent, --sandbox, --keep, --dry-run and " + "--no-emulate need a BUNDLE to run") in " ".join(usage.output.split()) def test_the_cli_runs_an_installed_bundle_by_name_with_ids_rooted_at_its_package(site, monkeypatch, quiet_logs, diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 02032eed2..5302dc5f8 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -81,7 +81,7 @@ class FakeDocker: def __init__(self, registry: Registry): self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} - self.probed, self.probe_fails, self.resolved = [], {}, [] + self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} def resolve(self, request, context, source, arch, work): """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" @@ -104,8 +104,10 @@ def resolve(self, request, context, source, arch, work): def resolving(self, resolve=None): """A LocalBuilder.resolve that runs ``resolve``, by default this one's, on the builder's context and resolve.""" - return lambda on, request, source, arch, timeout: (resolve or self.resolve)( - request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) + def resolving(on, request, source, arch, timeout): + self.timeouts[arch] = timeout + return (resolve or self.resolve)(request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) + return resolving async def probe(self, links, env, driver, image): arch = self._arch(self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]) @@ -247,6 +249,7 @@ def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_ assert arm64["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] assert sorted(arch for arch, _ in docker.resolved) == ["amd64", "arm64"] + assert docker.timeouts == {"arm64": 3600, "amd64": 10800} assert docker.resolved[0][1]["python"] == [{"name": "driver", "folder": "/ctx/driver", "requirements": "requirements.txt", "add": []}] assert composed["arm64"]["lock"].startswith("sha256:") and composed["arm64"]["lock"] != composed["amd64"]["lock"] @@ -821,7 +824,8 @@ def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_imag "amd64": ("ready", "FakeVmProvider", False), "arm64": ("unverified", "FakeVmProvider", True)} assert composed["arm64"]["reason"] == "no link of the sandbox provider runs linux/arm64 natively to probe it" assert len(vms.vms) == 2 and all(vm.terminated for vm in vms.vms) and vms.binfmt == ["arm64"] - assert vms.sizes == [{"exposed_ports": [], **builder._VM}] * 2 + assert sorted(vms.sizes, key=lambda size: size["timeout"]) == [ + {"exposed_ports": [], "timeout": 7200, **builder._VM}, {"exposed_ports": [], "timeout": 21600, **builder._VM}] driver_files = {artifact.object_url for artifact in driver.files.get_file_artifacts().values()} assert sorted(vms.fetched) == sorted([*driver_files, *driver_files]) assert vms.uploaded and all(f"{packages.PREFIX}/files/" in url for url in vms.uploaded) @@ -907,6 +911,21 @@ def test_env_build_writes_the_computer_again_when_it_isnt_over_every_pair_built( assert Env.get(stored.id).metadata["computer"]["version"] == stale["version"] + 1 +def test_a_put_that_wont_emulate_skips_an_arch_no_link_runs_natively_and_a_later_build_that_will_builds_it( + base, driver, docker, caplog): + caplog.set_level(logging.ERROR, logger=composer.__name__) + env = _put(base, driver, emulate=False) + skipped = CliRunner().invoke(cli, ["env", "build", "--id", env.id, "--no-emulate"]) + built = CliRunner().invoke(cli, ["env", "build", "--id", env.id]) + + reason = "emulation is off, and LocalSandboxProvider doesn't run linux/amd64 natively" + assert skipped.exit_code == 0 and f"container/amd64: skipped ({reason})" in skipped.output + assert [build["platform"] for build in docker.builds] == ["linux/arm64", "linux/amd64"] + assert built.exit_code == 0 and "container/amd64: building emulated, which is slow," in built.output + composed = Env.get(env.id).metadata["composed"]["container"] + assert {arch: r["status"] for arch, r in composed.items()} == {"amd64": "unverified", "arm64": "ready"} + + def test_env_build_by_an_installed_alias_composes_under_the_id_of_the_env_it_names(base, driver, docker, local_stores, tmp_path, monkeypatch, caplog): caplog.set_level(logging.ERROR, logger=composer.__name__) From 383281a2010c2fdbee479c735a9bdb99043fc5f5 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 13:54:31 -0700 Subject: [PATCH 06/11] feat(computer): an emulated build says how to avoid emulating A pair that builds emulated says, on its progress line, that --no-emulate skips it and, when this machine runs the arch and the chain has no link on its Docker, that adding local to --sandbox builds it natively here. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/builder.py | 8 ++++++++ src/agent_env/env/envs/computer/utils/composer.py | 6 ++++-- tst/unit/env/envs/test_computer_env.py | 12 +++++++++--- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 6c26595d5..dc904e6f7 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -200,6 +200,14 @@ def where(link: SandboxProvider) -> str: return "with this machine's Docker" if _docker_here(link) else f"in a {type(link).__name__} VM" +def instead_of_emulating(provider: SandboxProvider, platform: str) -> str: + """What a person can do rather than wait for ``provider`` to emulate ``platform``: skip it, or, when this machine + runs it and the chain has no link on this machine's Docker, add one.""" + local = "add local to --sandbox to build it natively with this machine's Docker, or " if ( + platform == host_platform() and not any(_docker_here(link) for link in provider.links)) else "" + return f"{local}pass --no-emulate to skip it" + + def _docker_here(link: SandboxProvider) -> bool: """Whether ``link``'s sandboxes are containers on this machine's Docker, which then builds itself.""" return link.ON_THIS_MACHINE and link.REACHES_HOST_AT is None diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 4731a6aa5..0b52b814c 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -206,8 +206,10 @@ def _built(env: ComputerEnv, base: ComputerArtifact, driver: ComputerDriverEnv, def on() -> builder.Builder: if not held: link = builds.link(platform) - how = "natively" if platform in link.native_platforms() else "emulated, which is slow," - progress.at(pair, "starting its builder", f"building {how} {builder.where(link)}") + native = platform in link.native_platforms() + progress.at(pair, "starting its builder", f"building natively {builder.where(link)}" if native else + f"building emulated, which is slow, {builder.where(link)}; " + f"{builder.instead_of_emulating(builds.provider, platform)}") held.append(opened.enter_context(builds.open(link, platform, Path(work), base.image(), from_store))) return held[0] diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 5302dc5f8..550fa9bbb 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -276,7 +276,8 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one built, reused = (Env.get(env.id).metadata["composed"]["container"] for env in (first, second)) assert len(docker.builds) == 2 and all(reused[arch]["reused"] for arch in reused) assert "container/arm64: building natively with this machine's Docker" in first_told - assert "container/amd64: building emulated, which is slow, with this machine's Docker" in first_told + assert ("container/amd64: building emulated, which is slow, with this machine's Docker; pass --no-emulate to skip " + "it") in first_told assert not [line for line in second_told if "building" in line or "resolving" in line] assert second_told[0] == "composing container/amd64, container/arm64 on LocalSandboxProvider" assert "container/amd64: reusing the image of the same key, built earlier" in second_told @@ -818,8 +819,12 @@ def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_imag for name in ("build_image", "push_local_image", "docker_login", "_emulated_platforms"): monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) - composed = Env.get(_put(base, driver, sandbox_provider=vms).id).metadata["composed"]["container"] + told = [] + composed = Env.get(_put(base, driver, sandbox_provider=vms, on_progress=told.append).id).metadata["composed"][ + "container"] + assert ("container/arm64: building emulated, which is slow, in a FakeVmProvider VM; add local to --sandbox to build " + "it natively with this machine's Docker, or pass --no-emulate to skip it") in told assert {arch: (r["status"], r["builder"], r["emulated"]) for arch, r in composed.items()} == { "amd64": ("ready", "FakeVmProvider", False), "arm64": ("unverified", "FakeVmProvider", True)} assert composed["arm64"]["reason"] == "no link of the sandbox provider runs linux/arm64 natively to probe it" @@ -921,7 +926,8 @@ def test_a_put_that_wont_emulate_skips_an_arch_no_link_runs_natively_and_a_later reason = "emulation is off, and LocalSandboxProvider doesn't run linux/amd64 natively" assert skipped.exit_code == 0 and f"container/amd64: skipped ({reason})" in skipped.output assert [build["platform"] for build in docker.builds] == ["linux/arm64", "linux/amd64"] - assert built.exit_code == 0 and "container/amd64: building emulated, which is slow," in built.output + assert built.exit_code == 0 and "container/amd64: building emulated, which is slow, with this machine's Docker; " \ + "pass --no-emulate to skip it" in built.output composed = Env.get(env.id).metadata["composed"]["container"] assert {arch: r["status"] for arch, r in composed.items()} == {"amd64": "unverified", "arm64": "ready"} From 491242438bc7612e43c25b743efd678ed06719e6 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 14:13:13 -0700 Subject: [PATCH 07/11] fix(computer): a builder VM pulls binfmt through mirror.gcr.io The same pinned digest, through Google's mirror of Docker Hub, which doesn't limit anonymous pulls by IP, as provider VMs sharing IPs could hit on Docker Hub. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/env/envs/computer/utils/builder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index dc904e6f7..b98b92965 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -48,8 +48,9 @@ #: since the stores keep no file modes. RUN_ONLY = (b"#!", b"\x7fELF") EXECUTABLE = 0o755 -#: What registers QEMU in a builder VM's kernel, for a platform no link runs natively. -BINFMT = ("docker.io/tonistiigi/binfmt:qemu-v9.2.2" +#: What registers QEMU in a builder VM's kernel, for a platform no link runs natively: Docker Hub's image through +#: Google's mirror of it, which doesn't limit anonymous pulls by IP as Docker Hub does. +BINFMT = ("mirror.gcr.io/tonistiigi/binfmt:qemu-v9.2.2" "@sha256:1b804311fe87047a4c96d38b4b3ef6f62fca8cd125265917a9e3dc3c996c39e6") #: The BuildKit client a builder VM's Docker builds with, installed where it has none, by the VM's platform. BUILDX = "https://github.com/docker/buildx/releases/download/v0.38.0/buildx-v0.38.0.{platform}" From d972072eac1c953938b799b32e4ca34b5f583527 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 14:45:07 -0700 Subject: [PATCH 08/11] fix(computer): a dry run says what it would compose, and emulation off skips only what needs it run --dry-run passes its sandbox provider, emulation choice and progress to materialize, so it says what it would compose of a reused computer env and lists those envs. With emulation off, an arch a refused link runs natively still fails, naming the refusal; only an arch no link runs natively is skipped. The Modal-gated builder test passes emulating=True to VmBuilder. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/run.py | 9 +++++++-- src/agent_env/cli/run.py | 6 +++++- src/agent_env/env/envs/computer/utils/builder.py | 8 ++++++-- tst/integration/computer/builder_modal_vm_test.py | 2 +- tst/unit/bundle/computer_env_folder_test.py | 13 +++++++++++++ tst/unit/env/envs/test_computer_env.py | 14 ++++++++++++++ 6 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 9f714636c..a87980d2a 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -327,11 +327,13 @@ def dry_run_bundle( agent: str | None = None, on_progress: Callable[[str], None] | None = None, id_root: str | None = None, + emulate: bool = True, ) -> DryRun: """Check what ``run_bundle`` with these arguments checks before its first task runs, and say what it would write and run, writing and running nothing: no entity, ledger row, lock or instance. Each write's version is the one it would reuse or the store's next, which another run writing the id first can take. ``on_progress`` - is given the line ``run_bundle`` gives for each write, as it is checked. + is given the line ``run_bundle`` gives for each write, as it is checked, and for each computer env it reuses + with pairs left, what it would compose on ``sandbox`` (``emulate`` as ``run_bundle`` takes it). It raises what ``run_bundle`` raises before its first task runs, a failed preflight included. Like ``run_bundle``, it turns namespace routing on for the whole process while it runs, so it suits the CLI and @@ -342,7 +344,10 @@ def dry_run_bundle( with namespace_routing(): plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent) _say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True)) - materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done))) + materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)), + sandbox_provider=build_sandbox_provider(sandbox) if sandbox else None, + on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}"), + emulate=emulate) runs = _to_run(plan) return DryRun(materialization, runs, _skipped(plan, runs, every=not tasks and not evals), preflight.infra, preflight.built_here) diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 65728b990..de7427b02 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -92,7 +92,7 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t if dry_run: click.echo(_DRY_RUN) _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, agent=agent, - on_progress=click.echo, id_root=id_root)) + on_progress=click.echo, id_root=id_root, emulate=not no_emulate)) return result = run_bundle(root, tasks=tasks, evals=evals, model=model, user_model=user_model, judge_model=judge_model, agent=agent, sandbox=sandbox, on_progress=click.echo, @@ -222,6 +222,10 @@ def _report_dry_run(dry: DryRun) -> None: click.echo("Would set up first:") for build in dry.infra: click.echo(f" {build}: {build.planned()}") + if materialization.composing: + click.echo("Would compose what's left of:") + for env_id in materialization.composing: + click.echo(f" {env_id}") click.echo("Would run:") for entry in dry.runs: click.echo(f" {dry.path(entry)} v{materialization.version_of('task', entry.id)}") diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index b98b92965..5108128d9 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -148,13 +148,17 @@ def __init__(self, provider: SandboxProvider, store_id: str, repository: str, em self._refused = [(link, _refused(link, store_id, repository)) for link in provider.links] def link(self, platform: str) -> SandboxProvider: - """The link that builds ``platform``. Raises Unbuildable, or Skipped when it would emulate it and mustn't.""" + """The link that builds ``platform``. Raises Unbuildable, or Skipped when only emulation would build it and the + put turned emulation off.""" links = [link for link, problem in self._refused if problem is None] if native := [link for link in links if platform in link.native_platforms()]: return native[0] if not self.emulate: + if refused := [problem for link, problem in self._refused + if problem is not None and platform in link.native_platforms()]: + raise Unbuildable("; ".join(refused)) raise Skipped("emulation is off, and " + "; ".join( - problem or f"{type(link).__name__} doesn't run {platform} natively" for link, problem in self._refused)) + f"{type(link).__name__} doesn't run {platform} natively" for link, _ in self._refused)) for link in links: if not _docker_here(link) or platform in _emulated_platforms(): return link diff --git a/tst/integration/computer/builder_modal_vm_test.py b/tst/integration/computer/builder_modal_vm_test.py index ecb125fd2..6061270af 100644 --- a/tst/integration/computer/builder_modal_vm_test.py +++ b/tst/integration/computer/builder_modal_vm_test.py @@ -33,7 +33,7 @@ def test_a_builder_vm_resolves_stores_and_builds_an_arch_it_emulates(tmp_path): (driver / "requirements.txt").write_text("six==1.16.0\n") (driver / "start").write_text("#!/bin/sh\necho started\n") (driver / "start").chmod(0o755) - vm = builder.VmBuilder(build_sandbox_provider("modal_vm")) + vm = builder.VmBuilder(build_sandbox_provider("modal_vm"), emulating=True) try: vm.start(tmp_path, SimpleNamespace(image_name=BASE, pinned_name=BASE), "linux/arm64", {}) result = vm.resolve({"apt": ["jq"], "sources": [], "python": [ diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index 347d66676..5c4873937 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -12,6 +12,7 @@ from agent_env.bundle.materialize import materialize from agent_env.bundle.plan import plan_bundle, plan_entities from agent_env.bundle.resolve import resolve_bundle +from agent_env.bundle.run import dry_run_bundle from agent_env.env import Env from agent_env.env.envs.computer.utils import composer from agent_env.env.envs.computer.env import ComputerEnv @@ -107,6 +108,18 @@ def test_a_reused_computer_env_composes_what_it_has_left_on_the_runs_provider_an (env.id, UBUNTU, REFERENCE, provider, [("container", "amd64"), ("container", "arm64")])] +def test_a_dry_run_says_what_it_would_compose_of_a_reused_computer_env(tmp_path, composed): + root = _bundle(tmp_path / "desk") + materialize(_plan(root), build_here=local_builds(_plan(root))) + told = [] + + dry = dry_run_bundle(root, sandbox="local", on_progress=told.append) + + env_id = f"{_plan(root).bundle.bundle.id_root}/desktop" + assert dry.materialization.composing == (env_id,) and len(composed) == 1 + assert "envs/desktop: would compose container/amd64, container/arm64" in told + + def test_the_env_alone_is_planned_whatever_the_bundles_tasks_would_deploy(tmp_path, composed, monkeypatch): """A default agent two installed bundles both alias names neither, which blocks a run of a task that deploys it, but not writing an env that no task of the bundle runs for.""" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 550fa9bbb..c77496d4f 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -877,6 +877,20 @@ def command(vm, words, cwd): assert composed["amd64"]["status"] == "ready" and all(vm.terminated for vm in vms.vms) +def test_with_emulation_off_an_arch_a_refused_link_runs_natively_fails_and_only_one_needing_emulation_is_skipped( + base, driver, docker, tmp_path): + vms = FakeVmProvider(docker, tmp_path / "vms") + + composed = Env.get(_put(base, driver, sandbox_provider=vms, emulate=False).id).metadata["composed"]["container"] + + assert (composed["amd64"]["status"], composed["amd64"]["reason"]) == ( + "failed", "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local " + "provider") + assert (composed["arm64"]["status"], composed["arm64"]["reason"]) == ( + "skipped", "emulation is off, and FakeVmProvider doesn't run linux/arm64 natively") + assert vms.vms == [] + + class Containers(SandboxProvider): async def create_sandbox(self, **kwargs): raise AssertionError("a container was created") From 39eec1591f7eb3492a5d068d2339c5b6779accbf Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 15:05:54 -0700 Subject: [PATCH 09/11] refactor(computer): one Composing says how a put composes, and the key covers builder.FORMAT The sandbox provider, the emulation choice and the progress callback travel as one Composing, through the puts, AuthoringContext, materialize, run and register, rather than as three arguments each. The build key covers builder.FORMAT, raised with a change that alters what an image holds, in place of builder.py's source, so a change to how a build gets there doesn't build every image again. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/authoring.py | 16 ++--- src/agent_env/bundle/materialize.py | 66 +++++++++---------- src/agent_env/bundle/register.py | 7 +- src/agent_env/bundle/run.py | 37 ++++++----- src/agent_env/cli/env/build.py | 2 +- src/agent_env/env/envs/computer/app_env.py | 17 ++--- src/agent_env/env/envs/computer/env.py | 18 ++--- .../env/envs/computer/utils/builder.py | 4 ++ .../env/envs/computer/utils/composer.py | 35 ++++++---- tst/unit/bundle/computer_env_folder_test.py | 18 ++--- tst/unit/env/envs/test_computer_env.py | 26 ++++---- 11 files changed, 124 insertions(+), 122 deletions(-) diff --git a/src/agent_env/bundle/authoring.py b/src/agent_env/bundle/authoring.py index 959a0acce..4fbf71218 100644 --- a/src/agent_env/bundle/authoring.py +++ b/src/agent_env/bundle/authoring.py @@ -7,17 +7,18 @@ import stat import tempfile import unicodedata -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Iterator, Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, NoReturn, TypeVar, overload +from typing import Any, NoReturn, TypeVar, overload from agent_env.artifact.artifact import Artifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.entity.refs import EntityKind, parse_toml_ref from agent_env.env.env import Env from agent_env.env.envs._deployment import provider_refusal +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.providers.env_providers.constants import GATEWAY_SERVICE_NAMES from agent_env.providers.env_providers.env_gateway_provider import EnvironmentGatewayProvider, environment_name_problem from agent_env.providers.env_providers.env_provider import _env_provider_class @@ -27,9 +28,6 @@ from ._fs import relative, show, with_article from .parse import CONFIG_FILES, LEAVINGS, NAMED_BY, Bundle, BundleEntry, BundleError -if TYPE_CHECKING: - from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider - A = TypeVar("A", bound=Artifact) E = TypeVar("E", bound=Env) @@ -51,12 +49,8 @@ class AuthoringContext: bundle: Bundle entry: BundleEntry - #: Told, a line at a time, how what writing the entry does that takes long goes, such as composing a computer env. - on_progress: Callable[[str], None] | None = None - #: Where writing a computer env composes it; the env's sandbox provider when None. - sandbox_provider: SandboxProvider | None = None - #: Whether composing it emulates an arch no link of that provider runs natively, rather than skip it. - emulate: bool = True + #: How writing a computer env composes it. + composing: Composing = Composing() @property def id(self) -> str: diff --git a/src/agent_env/bundle/materialize.py b/src/agent_env/bundle/materialize.py index ef1a9f0c2..9b6c6717a 100644 --- a/src/agent_env/bundle/materialize.py +++ b/src/agent_env/bundle/materialize.py @@ -19,8 +19,8 @@ import copy from collections.abc import Callable, Iterator, Mapping from contextlib import contextmanager, nullcontext -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass, replace +from typing import Any from agent_env.a2a_agent import A2AAgent from agent_env.artifact import Artifact @@ -30,6 +30,7 @@ from agent_env.env import Env from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.utils import composer +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.registry import get_env_registry from agent_env.eval import Eval, EvalTask from agent_env.providers.sandbox_providers.host_images import build, build_staged @@ -47,9 +48,6 @@ from .plan import Plan, Write, artifact_writer, env_writer, unpinned_store_refs from .resolve import BuiltImage, BundleImage, ReferencedImage, build_step -if TYPE_CHECKING: - from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider - @dataclass(frozen=True) class Materialized: @@ -89,9 +87,8 @@ def materialize( on_write: Callable[[Materialized], None] | None = None, build_here: frozenset[str] = frozenset(), ledger: Ledger | None = None, - sandbox_provider: SandboxProvider | None = None, + composing: Composing = Composing(), on_progress: Callable[[Write, str], None] | None = None, - emulate: bool = True, ) -> Materialization: """Write ``plan``'s entities, tasks and evals. The first write that fails stops it, and every earlier one stays: they're in the ledger, so the next run reuses them. ``on_wait`` is called when another run @@ -105,9 +102,8 @@ def materialize( Each write gets the version it would reuse or the store's next, which another run can take first. A step reading what the run would write first isn't preflighted, since the store doesn't hold it yet. - ``sandbox_provider`` composes each computer env written, and what one reused has left, as ``agent-env env - build`` does: the env's sandbox provider when None, emulating an arch no link runs natively unless ``emulate`` is - False. ``on_progress`` is told how each compose goes, a line at a time. + Each computer env written is composed as ``composing`` says, and so is what one reused has left, as ``agent-env + env build`` does; ``on_progress``, when given, is told how each goes, a line at a time with its write. ``ledger`` keeps the rows, this run's in the ``@local`` namespace's store unless given; it also says whether each task and eval names the versions it reads (``Ledger.pins_references``) and whether each image the bundle builds is @@ -121,7 +117,7 @@ def materialize( tasks = [write for write in plan.writes if write.kind is BundleKind.TASK] evals = [write for write in plan.writes if write.kind is BundleKind.EVAL] done: dict[tuple[str, str], Materialized] = {} - composing: list[str] = [] + left: list[str] = [] def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: with _noted(plan, write, "checking" if dry_run else "writing"): @@ -144,13 +140,13 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: if ledger.pushes_images and isinstance(write.source, BuiltImage) and write.id not in plan.published: through_ledger(write, lambda: _write_pushed_image(plan, write, on_build)) else: - through_ledger(write, lambda: _write_entity(plan, write, here, on_build, sandbox_provider, - on_progress, emulate)) + through_ledger(write, lambda: _write_entity(plan, write, here, on_build, + _composing(composing, on_progress, write))) if here and done[_key(write)].reused: _build_reused(plan, write, done[_key(write)].version, on_build) if write.kind is BundleKind.ENV and done[_key(write)].reused and _compose_reused( - write, done[_key(write)].version, sandbox_provider, on_progress, emulate, dry_run): - composing.append(write.id) + write, done[_key(write)].version, _composing(composing, on_progress, write), dry_run): + left.append(write.id) unwritten = {_key(item.write) for item in done.values() if not item.reused} if dry_run else set() written = done if ledger.pins_references else None built, problems, unchecked = {}, [], [] @@ -167,12 +163,16 @@ def through_ledger(write: Write, write_fn: Callable[[], int]) -> None: requires=write.requires).version) for write in evals: through_ledger(write, lambda: _write_eval(plan, write, written)) - return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(composing)) + return Materialization(plan, tuple(done[_key(write)] for write in plan.writes), tuple(unchecked), tuple(left)) + + +def _composing(composing: Composing, on_progress: Callable[[Write, str], None] | None, write: Write) -> Composing: + """``composing`` for ``write``, telling ``on_progress`` its lines with it when given.""" + return composing if on_progress is None else replace(composing, on_progress=lambda line: on_progress(write, line)) def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Write], None] | None, - sandbox_provider: SandboxProvider | None, on_progress: Callable[[Write, str], None] | None, - emulate: bool) -> int: + composing: Composing) -> int: if isinstance(write.source, ReferencedImage): return _write_referenced_image(plan, write) if isinstance(write.source, BuiltImage): @@ -180,8 +180,7 @@ def _write_entity(plan: Plan, write: Write, here: bool, on_build: Callable[[Writ return _write_published_image(plan, write) return _write_build_context(plan, write, here, on_build) if write.kind is BundleKind.ENV: - return _write_env(plan, write, sandbox_provider, - None if on_progress is None else lambda line: on_progress(write, line), emulate) + return _write_env(plan, write, composing) return _WRITERS[write.kind](plan, write) @@ -315,32 +314,29 @@ def _write_agent(plan: Plan, write: Write) -> int: AuthoringContext(plan.bundle_of(entry), entry)).version -def _compose_reused(write: Write, version: int, sandbox_provider: SandboxProvider | None, - on_progress: Callable[[Write, str], None] | None, emulate: bool, dry_run: bool) -> bool: - """Whether the computer env ``write`` reuses has pairs left to compose on ``sandbox_provider`` (``pending``: failed, - stopped, or built but unprobed though it runs their arch natively) or its computer to write again; composed unless - ``dry_run``, which says what it would compose.""" +def _compose_reused(write: Write, version: int, composing: Composing, dry_run: bool) -> bool: + """Whether the computer env ``write`` reuses has pairs left to compose on ``composing``'s provider (``pending``: + failed, stopped, or built but unprobed though it runs their arch natively) or its computer to write again; + composed unless ``dry_run``, which says what it would compose.""" env = Env.get(write.id, version) if not isinstance(env, ComputerEnv) or env.computer is None: return False - told = None if on_progress is None else lambda line: on_progress(write, line) - pairs = composer.pending(env, sandbox_provider) + pairs = composer.pending(env, composing.sandbox_provider) if not pairs and composer.current(env): return False if not dry_run: - composer.compose(env, pairs, told, sandbox_provider, emulate) - elif told is not None: - told(f"would compose {', '.join(f'{form}/{arch}' for form, arch in pairs) or 'its computer again'}") + composer.compose(env, pairs, composing) + elif composing.on_progress is not None: + composing.on_progress( + f"would compose {', '.join(f'{form}/{arch}' for form, arch in pairs) or 'its computer again'}") return True -def _write_env(plan: Plan, write: Write, sandbox_provider: SandboxProvider | None = None, - on_progress: Callable[[str], None] | None = None, emulate: bool = True) -> int: +def _write_env(plan: Plan, write: Write, composing: Composing = Composing()) -> int: entry = write.source.entry cls = get_env_registry()[entry.type] - return cls.from_toml(_pinned(plan, write, cls.toml_refs), AuthoringContext( - plan.bundle_of(entry), entry, on_progress=on_progress, sandbox_provider=sandbox_provider, - emulate=emulate)).version + return cls.from_toml(_pinned(plan, write, cls.toml_refs), + AuthoringContext(plan.bundle_of(entry), entry, composing=composing)).version def _pinned(plan: Plan, write: Write, refs: tuple[EntityRef, ...]) -> Any: diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 9410f3925..2fa92cf73 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -31,6 +31,7 @@ from agent_env.config import get_config from agent_env.entity.refs import RefRole, ref_sites from agent_env.plugins._bundles import AGENT_ALIASES +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.providers import build_sandbox_provider from agent_env.store.document_store import DocumentStore, LocalSqliteDocumentStore from agent_env.store.ids import LOCAL_PREFIX, is_local_id @@ -91,7 +92,7 @@ def register_bundle( It turns namespace routing on while it runs. Raises BundleError listing every problem found before anything is written; a write that fails stops it, and every earlier one stays, in the ledger, for the next run to reuse.""" - provider = build_sandbox_provider(sandbox) if sandbox else None + composing = Composing(build_sandbox_provider(sandbox) if sandbox else None, emulate) with namespace_routing(): bundle = parse_bundle(Path(root)) resolved = resolve_bundle(bundle) @@ -113,8 +114,8 @@ def register_bundle( registered, dry_run=dry_run, ledger=ledger, on_wait=lambda: say("waiting for another agent-env run to finish writing this bundle's ids"), on_build=lambda write: say(f" {_label(registered, write)}: building with docker and pushing"), - on_write=lambda done: say(f" {registered_line(registered, done)}"), sandbox_provider=provider, - on_progress=lambda write, line: say(f" {_label(registered, write)}: {line}"), emulate=emulate) + on_write=lambda done: say(f" {registered_line(registered, done)}"), composing=composing, + on_progress=lambda write, line: say(f" {_label(registered, write)}: {line}")) return Registration(registered, materialization, ids, tuple(_floating(registered)), identity, shared, requires) diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index a87980d2a..07bea46e1 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -25,6 +25,7 @@ from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.env.bootstrap import InfraSetup, ensure_default_envs +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.providers import build_sandbox_provider from agent_env.store.base import NotFoundError from agent_env.store.routing import namespace_routing, run_scope @@ -241,12 +242,11 @@ def run_bundle( written, before any task is. Building an infra env the run needs raises what the build raises, before any task runs.""" _refuse_a_running_loop() - if sandbox: - build_sandbox_provider(sandbox) + composing = _composing(sandbox, emulate) say = _progress(on_progress) with namespace_routing(): prepared = _prepare(root, tasks, evals, sandbox, id_root, agent, False, say, - declaring_for(Path(root), installed=id_root is not None, dry_run=False), emulate) + declaring_for(Path(root), installed=id_root is not None, dry_run=False), composing) plan, entries = prepared.plan, [entry for entry, _ in prepared.tasks] with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) @@ -282,9 +282,9 @@ def prepare_bundle( It doesn't turn namespace routing on: the caller keeps it on for as long as it runs the tasks (``agent_env.store.routing.namespace_routing``), which suits a service that runs nothing else. Raises what ``run_bundle`` raises before its first task runs, except for a running event loop, which it doesn't need.""" - if sandbox: - build_sandbox_provider(sandbox) - return _prepare(root, tasks, evals, sandbox, id_root, None, played, _progress(on_progress), _hint(id_root)) + composing = _composing(sandbox) + return _prepare(root, tasks, evals, sandbox, id_root, None, played, _progress(on_progress), _hint(id_root), + composing) def _write_bundle( @@ -300,8 +300,8 @@ def _write_bundle( def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], None], - sandbox: str | None = None, emulate: bool = True) -> Materialization: - """Write ``plan``, its computer envs composed on ``sandbox`` when given, then set up the infra envs its runs deploy + composing: Composing = Composing()) -> Materialization: + """Write ``plan``, its computer envs composed as ``composing`` says, then set up the infra envs its runs deploy on.""" materialization = materialize( plan, @@ -309,9 +309,8 @@ def _written_with_infra(plan: Plan, preflight: Preflight, say: Callable[[str], N on_build=lambda write: say(f"{_label(plan, write)}: building with docker, which can take minutes"), on_write=lambda done: say(_written(plan, done)), build_here=preflight.built_here, - sandbox_provider=build_sandbox_provider(sandbox) if sandbox else None, + composing=composing, on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}"), - emulate=emulate, ) if preflight.infra: ensure_default_envs(preflight.infra_kinds, say=say) @@ -338,16 +337,14 @@ def dry_run_bundle( It raises what ``run_bundle`` raises before its first task runs, a failed preflight included. Like ``run_bundle``, it turns namespace routing on for the whole process while it runs, so it suits the CLI and scripts, not a service; it runs no event loop, so a running one isn't refused.""" - if sandbox: - build_sandbox_provider(sandbox) + composing = _composing(sandbox, emulate) say = _progress(on_progress) with namespace_routing(): plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent) _say_plan(plan, say, declaring_for(Path(root), installed=id_root is not None, dry_run=True)) materialization = materialize(plan, dry_run=True, on_write=lambda done: say(_written(plan, done)), - sandbox_provider=build_sandbox_provider(sandbox) if sandbox else None, - on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}"), - emulate=emulate) + composing=composing, + on_progress=lambda write, line: say(f"{_label(plan, write)}: {line}")) runs = _to_run(plan) return DryRun(materialization, runs, _skipped(plan, runs, every=not tasks and not evals), preflight.infra, preflight.built_here) @@ -376,10 +373,10 @@ def _bundle_run(plan: Plan, materialization: Materialization, runs: tuple[TaskRu def _prepare(root: Path | str, tasks: Sequence[str], evals: Sequence[str], sandbox: str | None, id_root: str | None, agent: str | None, played: bool, say: Callable[[str], None], declaring: Declaring, - emulate: bool = True) -> PreparedBundle: + composing: Composing = Composing()) -> PreparedBundle: plan, preflight = _planned(root, tasks, evals, id_root, sandbox, agent, played=played) _say_plan(plan, say, declaring) - materialization = _written_with_infra(plan, preflight, say, sandbox, emulate) + materialization = _written_with_infra(plan, preflight, say, composing) to_run = tuple((entry, Task.get(entry.id, materialization.version_of("task", entry.id))) for entry in _to_run(plan)) return PreparedBundle(plan, materialization, to_run) @@ -396,6 +393,12 @@ def _planned(root: Path | str, tasks: Sequence[str], evals: Sequence[str], id_ro played=played) +def _composing(sandbox: str | None, emulate: bool = True) -> Composing: + """How a run composes its computer envs: on the provider ``sandbox`` names, and emulating unless ``emulate`` is + False. Raises ValueError when ``sandbox`` names no provider.""" + return Composing(build_sandbox_provider(sandbox) if sandbox else None, emulate) + + def _to_run(plan: Plan) -> tuple[BundleEntry, ...]: """The tasks selected on their own and every task a selected eval names, each once, in the plan's order.""" wanted = {entry.entry.id for entry in plan.tasks} diff --git a/src/agent_env/cli/env/build.py b/src/agent_env/cli/env/build.py index 1491a6996..ec03d85d0 100644 --- a/src/agent_env/cli/env/build.py +++ b/src/agent_env/cli/env/build.py @@ -29,7 +29,7 @@ def build(env_id: str, version: int | None, sandbox: str | None, no_emulate: boo if env.computer is None: raise click.ClickException(f"{env_id} names no computer and driver, so it has nothing to build alone") if (pairs := composer.pending(env, provider)) or not composer.current(env): - composer.compose(env, pairs, on_progress=click.echo, sandbox_provider=provider, emulate=not no_emulate) + composer.compose(env, pairs, composer.Composing(provider, not no_emulate, click.echo)) failed = False for form, by_arch in env.metadata.get("composed", {}).items(): for arch, result in by_arch.items(): diff --git a/src/agent_env/env/envs/computer/app_env.py b/src/agent_env/env/envs/computer/app_env.py index c71d2ac71..06c15e09b 100644 --- a/src/agent_env/env/envs/computer/app_env.py +++ b/src/agent_env/env/envs/computer/app_env.py @@ -16,6 +16,7 @@ from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse from agent_env.env.env import Env from agent_env.env.envs.computer.utils import composer, gate +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer.utils.install import AppInstall, folder_problems, installs from agent_env.env.envs.computer.driver_env import ComputerDriverEnv, declared @@ -23,10 +24,9 @@ from agent_env.store.ids import derive_id if TYPE_CHECKING: - from collections.abc import Callable, Iterable + from collections.abc import Iterable from agent_env.bundle.authoring import AuthoringContext - from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider class ComputerAppEnv(ComputerEnv): @@ -100,12 +100,10 @@ def from_dict(cls, data: dict) -> ComputerAppEnv: def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifactUniverse, declaration: AppDeclaration, computer: Optional[ComputerArtifact] = None, driver: Optional[ComputerDriverEnv] = None, build_settings: Optional[dict[str, Any]] = None, - compose: bool = True, on_progress: Optional[Callable[[str], None]] = None, - sandbox_provider: Optional[SandboxProvider] = None, emulate: bool = True) -> Self: + compose: bool = True, composing: Composing = Composing()) -> Self: """Write the app once what it declares and ``build_settings``, its build settings' values, check out, and with - a computer and a driver, once it fits them (gate 1); then compose it with them on ``sandbox_provider``, by - default the env's, emulating an arch no link runs natively unless ``emulate`` is False, unless ``compose`` is - False, telling ``on_progress`` how it goes. Refused, naming every problem, before anything is stored.""" + a computer and a driver, once it fits them (gate 1); then compose it with them as ``composing`` says unless + ``compose`` is False. Refused, naming every problem, before anything is stored.""" app = cls(id=id, version=None, environment_name=environment_name, module=module, files=files, declaration=declaration, computer=computer, driver=driver, build_settings=build_settings) _, problems = _installs(declaration) @@ -122,7 +120,7 @@ def put(cls, *, id: str, environment_name: str, module: str, files: FileArtifact raise ValueError(f"computer app env {id!r}: " + "; ".join(problems)) app = get_env_store().put_document(app) if computer is not None and compose: - composer.compose(app, on_progress=on_progress, sandbox_provider=sandbox_provider, emulate=emulate) + composer.compose(app, composing=composing) return app @classmethod @@ -148,8 +146,7 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerAppEnv: try: return cls.put(id=ctx.id, environment_name=fields.get("environment_name", ctx.name), module=fields["module"], files=universe, declaration=declaration, computer=computer, - driver=driver, build_settings=fields.get("build_settings"), on_progress=ctx.on_progress, - sandbox_provider=ctx.sandbox_provider, emulate=ctx.emulate) + driver=driver, build_settings=fields.get("build_settings"), composing=ctx.composing) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/env.py b/src/agent_env/env/envs/computer/env.py index b1d4d5f00..c0eb195bc 100644 --- a/src/agent_env/env/envs/computer/env.py +++ b/src/agent_env/env/envs/computer/env.py @@ -6,7 +6,6 @@ import logging import posixpath -from collections.abc import Callable from typing import TYPE_CHECKING, Any, ClassVar, Optional, Self from agent_env.artifact.artifacts.computer import ComputerArtifact, SettingValue @@ -22,6 +21,7 @@ provider_or_class, ) from agent_env.env.envs.computer.utils import composer, gate, inside +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.envs.computer.utils import params as computer_params from agent_env.env.envs.computer.driver_env import ComputerDriverEnv from agent_env.env.gateway import GatewayMode @@ -33,7 +33,6 @@ from agent_env.bundle.authoring import AuthoringContext from agent_env.env.env import DeployedEnv from agent_env.providers.env_providers.env_provider import EnvironmentProvider - from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider logger = logging.getLogger(__name__) @@ -83,12 +82,9 @@ def from_dict(cls, data: dict) -> ComputerEnv: @classmethod def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str, apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None, - compose: bool = True, on_progress: Optional[Callable[[str], None]] = None, - sandbox_provider: Optional[SandboxProvider] = None, emulate: bool = True) -> Self: - """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it on - ``sandbox_provider``, by default the env's, emulating an arch no link runs natively unless ``emulate`` is False, - unless ``compose`` is False, telling ``on_progress`` how it goes. Refused, naming every mismatch, before - anything is stored.""" + compose: bool = True, composing: Composing = Composing()) -> Self: + """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it as ``composing`` + says unless ``compose`` is False. Refused, naming every mismatch, before anything is stored.""" values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network, computer_settings=computer_settings or {}) if problems: @@ -97,7 +93,7 @@ def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, display=display, network=network, computer_settings=values)) if compose: - composer.compose(env, on_progress=on_progress, sandbox_provider=sandbox_provider, emulate=emulate) + composer.compose(env, composing=composing) return env @classmethod @@ -114,8 +110,8 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv: ComputerDriverEnv) try: return cls.put(id=ctx.id, computer=base, driver=driver, - environment_name=fields.pop("environment_name", ctx.name), on_progress=ctx.on_progress, - sandbox_provider=ctx.sandbox_provider, emulate=ctx.emulate, **fields) + environment_name=fields.pop("environment_name", ctx.name), composing=ctx.composing, + **fields) except ValueError as e: ctx.refuse([str(e)]) diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 5108128d9..8fb8a03a4 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -42,6 +42,10 @@ logger = logging.getLogger(__name__) +#: What the builders make of a build context, which the build key covers in place of their source: raise it with a +#: change that alters what an image holds (its files, their modes, its labels, how it's built), and leave it for one +#: that changes only how a build gets there, so images aren't built again for nothing. +FORMAT = 1 CONTEXT, RESOLVED = "context", "resolve" RESOLVER = Path(__file__).with_name("resolve_in_base.py") #: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged, diff --git a/src/agent_env/env/envs/computer/utils/composer.py b/src/agent_env/env/envs/computer/utils/composer.py index 0b52b814c..cd9f82446 100644 --- a/src/agent_env/env/envs/computer/utils/composer.py +++ b/src/agent_env/env/envs/computer/utils/composer.py @@ -81,23 +81,32 @@ _PROTOCOL_FILES = ("pyproject.toml", "README.md", "LICENSE", "NOTICE", "THIRD_PARTY_NOTICES.md") -def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, - on_progress: Callable[[str], None] | None = None, sandbox_provider: SandboxProvider | None = None, - emulate: bool = True) -> None: - """Build each of ``pairs``, by default each form the base has in each arch the env composes, in parallel, on - ``sandbox_provider``, by default the env's, and probe each one it runs natively (gate 3), recording each as it - starts and finishes, then write the env's computer over every pair built so far. An arch no link runs natively is - emulated, or with ``emulate`` False skipped, which leaves it for a later compose. Each pair's stages and how it - ended go to ``on_progress`` and the log as they happen (``_Progress``).""" +@dataclass(frozen=True) +class Composing: + """How a put composes a computer env: on ``sandbox_provider``, by default the env's; emulating an arch no link of + it runs natively unless ``emulate`` is False, which leaves the arch skipped for a later compose; each pair's + stages told to ``on_progress``, a line at a time.""" + + sandbox_provider: SandboxProvider | None = None + emulate: bool = True + on_progress: Callable[[str], None] | None = None + + +def compose(env: ComputerEnv, pairs: list[tuple[str, str]] | None = None, composing: Composing = Composing()) -> None: + """Build each of ``pairs``, by default each form the base has in each arch the env composes, in parallel, as + ``composing`` says, and probe each one its provider runs natively (gate 3), recording each as it starts and + finishes, then write the env's computer over every pair built so far. Each pair's stages and how it ended go to the + log too (``_Progress``).""" from agent_env.providers import get_env_sandbox_provider - base, driver, provider = env.computer, env.driver, sandbox_provider or get_env_sandbox_provider() - builds = builder.Builds(provider, env.stored_id, REPOSITORY, emulate) + base, driver = env.computer, env.driver + provider = composing.sandbox_provider or get_env_sandbox_provider() + builds = builder.Builds(provider, env.stored_id, REPOSITORY, composing.emulate) pairs = pairs if pairs is not None else [(form, arch) for form in gate.forms(base) for arch in gate.arches(base, driver, env.composed_apps())] record = _Record(env) record.started(pairs) - with _Progress(env, on_progress) as progress, ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: + with _Progress(env, composing.on_progress) as progress, ThreadPoolExecutor(max_workers=max(len(pairs), 1)) as pool: progress.say(f"composing {', '.join(f'{form}/{arch}' for form, arch in pairs)} on " f"{' or '.join(type(link).__name__ for link in provider.links)}") futures = {pool.submit(_composed, env, base, driver, form, arch, builds, progress): (form, arch) @@ -532,8 +541,8 @@ def _digest(folder: Path) -> str: def _composer_digest() -> str: - """A digest of the composer and its builders, which say what a build makes of its inputs.""" - return "sha256:" + hashlib.sha256(Path(__file__).read_bytes() + Path(builder.__file__).read_bytes()).hexdigest() + """A digest of the composer, which says what a build makes of its inputs, and of its builders' ``FORMAT``.""" + return "sha256:" + hashlib.sha256(Path(__file__).read_bytes() + f"builder {builder.FORMAT}".encode()).hexdigest() def _now() -> str: diff --git a/tst/unit/bundle/computer_env_folder_test.py b/tst/unit/bundle/computer_env_folder_test.py index 5c4873937..e5825860b 100644 --- a/tst/unit/bundle/computer_env_folder_test.py +++ b/tst/unit/bundle/computer_env_folder_test.py @@ -15,6 +15,7 @@ from agent_env.bundle.run import dry_run_bundle from agent_env.env import Env from agent_env.env.envs.computer.utils import composer +from agent_env.env.envs.computer.utils.composer import Composing from agent_env.env.envs.computer.env import ComputerEnv from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider from tst.unit.bundle._support import layout, local_builds @@ -37,10 +38,11 @@ def composed(tmp_path, monkeypatch, local_stores, cli_routing, builds): monkeypatch.setattr("agent_env.env.published_images._registry_digest", lambda ref: None) calls = [] - def compose(env, pairs=None, on_progress=None, sandbox_provider=None, emulate=True): - if on_progress is not None: - on_progress("composing") - calls.append((env.id, env.computer.id, env.driver.id, sandbox_provider) + ((pairs,) if pairs is not None else ())) + def compose(env, pairs=None, composing=Composing()): + if composing.on_progress is not None: + composing.on_progress("composing") + calls.append((env.id, env.computer.id, env.driver.id, composing.sandbox_provider) + + ((pairs,) if pairs is not None else ())) monkeypatch.setattr(composer, "compose", compose) return calls @@ -75,7 +77,7 @@ def test_a_computer_env_naming_shipped_aliases_is_written_over_them_and_composed plan, provider = planned(_bundle(tmp_path / "desk")), LocalSandboxProvider() told = [] - done = materialize(plan, build_here=local_builds(plan), sandbox_provider=provider, + done = materialize(plan, build_here=local_builds(plan), composing=Composing(provider), on_progress=lambda write, line: told.append((write.id, line))) env = Env.get(f"{plan.bundle.bundle.id_root}/desktop") @@ -93,14 +95,14 @@ def test_a_reused_computer_env_composes_what_it_has_left_on_the_runs_provider_an env = Env.get(f"{_plan(root).bundle.bundle.id_root}/desktop") told = [] - checked = materialize(_plan(root), dry_run=True, sandbox_provider=provider, + checked = materialize(_plan(root), dry_run=True, composing=Composing(provider), on_progress=lambda write, line: told.append((write.id, line))) - again = materialize(_plan(root), build_here=local_builds(_plan(root)), sandbox_provider=provider, + again = materialize(_plan(root), build_here=local_builds(_plan(root)), composing=Composing(provider), on_progress=lambda write, line: told.append((write.id, line))) env.update_metadata({**env.metadata, "computer": {"id": "c", "version": 1, "images": {"amd64": "a", "arm64": "b"}}, "composed": {"container": {"amd64": {"status": "ready", "image": "a"}, "arm64": {"status": "ready", "image": "b"}}}}) - built = materialize(_plan(root), build_here=local_builds(_plan(root)), sandbox_provider=provider) + built = materialize(_plan(root), build_here=local_builds(_plan(root)), composing=Composing(provider)) assert built.composing == () and checked.composing == again.composing == (env.id,) and all(item.reused for item in again.writes) assert told == [(env.id, "would compose container/amd64, container/arm64"), (env.id, "composing")] diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index c77496d4f..203ecccb0 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -39,7 +39,7 @@ from agent_env.env import Env from agent_env.env import env as env_module from agent_env.env.envs.computer.utils import boot, builder, composer, gate, inside, packages, resolve_in_base -from agent_env.env.envs.computer.utils.composer import _probe +from agent_env.env.envs.computer.utils.composer import Composing, _probe from agent_env.env.envs.computer.env import ComputerEnv from agent_env.env.envs.computer import app_env from agent_env.env.envs.computer.app_env import ComputerAppEnv @@ -268,10 +268,10 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one monkeypatch.setattr(composer, "write_index", lambda index, images, signed_in: ( docker_calls.append(("join", index, signed_in)), docker.joined(index, images, signed_in))) first_told, second_told = [], [] - first = _put(base, driver, on_progress=first_told.append) + first = _put(base, driver, composing=Composing(on_progress=first_told.append)) docker_calls.clear() second = _put(base, driver, id="@local/~/kit/other", environment_name="other", display={"width": 640, "height": 480}, - network={"mode": "allowlist"}, on_progress=second_told.append) + network={"mode": "allowlist"}, composing=Composing(on_progress=second_told.append)) built, reused = (Env.get(env.id).metadata["composed"]["container"] for env in (first, second)) assert len(docker.builds) == 2 and all(reused[arch]["reused"] for arch in reused) @@ -298,7 +298,7 @@ def test_a_stage_that_goes_on_is_said_to_go_on_and_a_failed_pair_says_why(base, monkeypatch.setattr(builder, "build_image", lambda *args, **kwargs: (time.sleep(0.5), build(*args, **kwargs))[1]) told = [] - _put(base, driver, on_progress=told.append) + _put(base, driver, composing=Composing(on_progress=told.append)) assert [line for line in told if line.startswith("container/arm64: still building the image (")] assert any(re.fullmatch(r"container/amd64: failed at build: the build failed: #7 ERROR: pip install failed \(\d+ s\)", @@ -820,7 +820,7 @@ def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_imag monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) told = [] - composed = Env.get(_put(base, driver, sandbox_provider=vms, on_progress=told.append).id).metadata["composed"][ + composed = Env.get(_put(base, driver, composing=Composing(vms, on_progress=told.append)).id).metadata["composed"][ "container"] assert ("container/arm64: building emulated, which is slow, in a FakeVmProvider VM; add local to --sandbox to build " @@ -849,8 +849,8 @@ def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_imag def test_a_chain_builds_each_arch_on_the_first_link_that_runs_it_natively(base, driver, docker, signing, tmp_path): vms = FakeVmProvider(docker, tmp_path / "vms") - composed = Env.get(_put(base, driver, sandbox_provider=ChainedSandboxProvider( - [vms, LocalSandboxProvider()])).id).metadata["composed"]["container"] + composed = Env.get(_put(base, driver, composing=Composing(ChainedSandboxProvider( + [vms, LocalSandboxProvider()]))).id).metadata["composed"]["container"] assert {arch: (r["builder"], r["emulated"]) for arch, r in composed.items()} == { "amd64": ("FakeVmProvider", False), "arm64": ("LocalSandboxProvider", False)} @@ -870,7 +870,7 @@ def command(vm, words, cwd): return real(vm, words, cwd) monkeypatch.setattr(FakeVm, "_command", command) - composed = Env.get(_put(base, driver, sandbox_provider=vms).id).metadata["composed"]["container"] + composed = Env.get(_put(base, driver, composing=Composing(vms)).id).metadata["composed"]["container"] assert (composed["arm64"]["status"], composed["arm64"]["stage"], composed["arm64"]["reason"]) == ( "failed", "resolve", "RuntimeError: FakeVmProvider's VM failed: Gone: the sandbox is shutting down") @@ -881,7 +881,7 @@ def test_with_emulation_off_an_arch_a_refused_link_runs_natively_fails_and_only_ base, driver, docker, tmp_path): vms = FakeVmProvider(docker, tmp_path / "vms") - composed = Env.get(_put(base, driver, sandbox_provider=vms, emulate=False).id).metadata["composed"]["container"] + composed = Env.get(_put(base, driver, composing=Composing(vms, emulate=False)).id).metadata["composed"]["container"] assert (composed["amd64"]["status"], composed["amd64"]["reason"]) == ( "failed", "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local " @@ -907,8 +907,8 @@ def test_an_arch_no_link_can_build_fails_saying_why_for_each_link(base, driver, made = {"vms": FakeVmProvider(docker, tmp_path / "vms"), "containers": Containers(), "local": LocalSandboxProvider()} - composed = Env.get(_put(base, driver, sandbox_provider=ChainedSandboxProvider( - [made[link] for link in links])).id).metadata["composed"]["container"] + composed = Env.get(_put(base, driver, composing=Composing(ChainedSandboxProvider( + [made[link] for link in links]))).id).metadata["composed"]["container"] assert (composed["amd64"]["status"], composed["amd64"]["stage"], composed["amd64"]["reason"]) == ( "failed", "resolve", reason) @@ -933,7 +933,7 @@ def test_env_build_writes_the_computer_again_when_it_isnt_over_every_pair_built( def test_a_put_that_wont_emulate_skips_an_arch_no_link_runs_natively_and_a_later_build_that_will_builds_it( base, driver, docker, caplog): caplog.set_level(logging.ERROR, logger=composer.__name__) - env = _put(base, driver, emulate=False) + env = _put(base, driver, composing=Composing(emulate=False)) skipped = CliRunner().invoke(cli, ["env", "build", "--id", env.id, "--no-emulate"]) built = CliRunner().invoke(cli, ["env", "build", "--id", env.id]) @@ -1541,7 +1541,7 @@ def test_an_apps_file_in_its_servers_folder_is_refused_and_one_stored_before_is_ _app(base, driver, tmp_path / "mail", extra=own, build_settings={"locale": "fr_FR"}) monkeypatch.setattr(app_env, "_in_server_folder", lambda declaration, names: []) stored = _app(base, driver, tmp_path / "mail", extra=own, build_settings={"locale": "fr_FR"}, compose=False) - composer.compose(stored, sandbox_provider=FakeVmProvider(docker, tmp_path / "vms")) + composer.compose(stored, composing=Composing(FakeVmProvider(docker, tmp_path / "vms"))) assert [build["build_settings"] for build in docker.builds] == [{"mail": {"locale": "fr_FR"}}] * 2 From 408769e96f9cac57dfc97232887990be9bbb12c4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 15:16:32 -0700 Subject: [PATCH 10/11] refactor(computer): a builder VM's shell steps live in one helper script builder_in_vm.sh holds what a builder VM runs in shell, as subcommands (buildx, emulate, files, link, check, runnable, download, installed), sent to the VM once. Those that don't use Docker run on Linux and macOS alike, so a unit test runs them for real, and the fake VM runs each line in bash here, faking only Docker and uploads to the store. Co-Authored-By: Claude Opus 5.5 --- .../env/envs/computer/utils/builder.py | 50 +++--- .../env/envs/computer/utils/builder_in_vm.sh | 85 ++++++++++ tst/unit/env/envs/test_computer_env.py | 158 +++++++++--------- 3 files changed, 183 insertions(+), 110 deletions(-) create mode 100644 src/agent_env/env/envs/computer/utils/builder_in_vm.sh diff --git a/src/agent_env/env/envs/computer/utils/builder.py b/src/agent_env/env/envs/computer/utils/builder.py index 8fb8a03a4..800f66d52 100644 --- a/src/agent_env/env/envs/computer/utils/builder.py +++ b/src/agent_env/env/envs/computer/utils/builder.py @@ -28,7 +28,7 @@ from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image from agent_env.config import get_config from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag -from agent_env.providers.sandbox_providers.sandbox import CURL_RETRY_FLAGS, upload_vm_file +from agent_env.providers.sandbox_providers.sandbox import upload_vm_file from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM from agent_env.store.base import ObjectAlreadyExistsError from agent_env.store.image_store.oci_registry_credentials import ( @@ -48,6 +48,8 @@ FORMAT = 1 CONTEXT, RESOLVED = "context", "resolve" RESOLVER = Path(__file__).with_name("resolve_in_base.py") +#: The shell side of a builder VM, its subcommands run there (``VmBuilder``). +IN_VM = Path(__file__).with_name("builder_in_vm.sh") #: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged, #: since the stores keep no file modes. RUN_ONLY = (b"#!", b"\x7fELF") @@ -338,23 +340,19 @@ def __init__(self, link: SandboxProvider, emulating: bool): except BaseException: self._runner.close() raise - self.platform = self.sandbox.platform + self.platform, self.helper = self.sandbox.platform, f"bash {self.work}/{IN_VM.name}" def start(self, work: Path, base: DockerImageArtifact, platform: str, from_store: dict[str, tuple[str, str]]) -> None: """Give the VM's Docker BuildKit when it has none, register emulation for ``platform`` when it isn't the VM's own, sign in to the base's registry or build the base from its build context, and put the build context here: the files of ``from_store`` from the store, and the rest, which only the putting machine holds, inline.""" - plugin = f"{_PLUGINS}/docker-buildx" - self._exec(f"docker buildx version >/dev/null 2>&1 || {{ mkdir -p {_PLUGINS} && curl -fsSL {CURL_RETRY_FLAGS} " - f"{BUILDX.format(platform=self.platform.replace('/', '-'))} -o {plugin} && echo " - f"'{BUILDX_SHA256[self.platform]} {plugin}' | sha256sum --check --quiet && chmod 755 {plugin}; }}") + self._write(f"{self.work}/{IN_VM.name}", IN_VM.read_bytes()) + self._exec(f"{self.helper} buildx {BUILDX.format(platform=self.platform.replace('/', '-'))} " + f"{BUILDX_SHA256[self.platform]} {_PLUGINS}") if platform != self.platform: - # The VM's own binfmt_misc first: a newer kernel gives a mount of it in a container an instance of its own. try: - self._exec("mountpoint -q /proc/sys/fs/binfmt_misc || mount -t binfmt_misc binfmt_misc " - f"/proc/sys/fs/binfmt_misc\ndocker run --privileged --rm {BINFMT} --install " - f"{platform.rsplit('/', 1)[-1]}") + self._exec(f"{self.helper} emulate {platform.rsplit('/', 1)[-1]} {BINFMT}") except RuntimeError as e: raise Unbuildable(f"{self.name}'s VM couldn't register emulation for {platform}: {e}") from None if names_registry(base.image_name): @@ -380,19 +378,15 @@ def resolve(self, request: dict, source: str, arch: str, timeout: int) -> dict: return json.loads(self._exec(f"cat {out}/result.json")) def download(self, url: str, sha256: str, path: str) -> None: - target = shlex.quote(f"{self.work}/{path}") try: - self._exec(f"mkdir -p $(dirname {target}) && curl -fsSL {CURL_RETRY_FLAGS} {shlex.quote(url)} -o {target}") + found = self._exec(f"{self.helper} download {shlex.quote(url)} {self._at(path)}").strip() except RuntimeError as e: raise ResolveError(f"downloading {url} failed: {e}") from None - if (found := self._exec(f"sha256sum {target}").split()[0]) != sha256: + if found != sha256: raise ResolveError(f"{url} has the sha256 {found}, not the {sha256} its download names") def files(self, folder: str) -> dict[str, Hashed]: - listed = self._exec( - f"cd {shlex.quote(f'{self.work}/{folder}')} && find . -type f -printf '%P\\0' | " - "while IFS= read -r -d '' f; do printf '%s %s %s\\0' \"$(sha256sum < \"$f\" | cut -c1-64)\" " - "\"$(stat -c %s \"$f\")\" \"$f\"; done") + listed = self._exec(f"{self.helper} files {self._at(folder)}") found = {} for entry in filter(None, listed.split("\0")): sha256, size, path = entry.split(" ", 2) @@ -403,24 +397,20 @@ def store(self, path: str, store, key: str) -> None: self._run(upload_vm_file(self.sandbox, f"{self.work}/{path}", store, store.object_url(key))) def place(self, copies: dict[str, str], fetches: dict[str, tuple[str, str]]) -> None: - targets = [*copies, *fetches] - self._each([f"mkdir -p {shlex.quote(str(PurePosixPath(self.work, target).parent))}" for target in targets]) - self._each([f"ln -f {shlex.quote(f'{self.work}/{source}')} {shlex.quote(f'{self.work}/{target}')}" - for target, source in copies.items()]) + self._each([f"{self.helper} link {self._at(source)} {self._at(target)}" for target, source in copies.items()]) if not fetches: return + self._each([f"mkdir -p {shlex.quote(str(PurePosixPath(self.work, target).parent))}" for target in fetches]) self._run(self._fetch({f"{self.work}/{target}": url for target, (url, _) in fetches.items()})) listed = "".join(f"{sha256} {target}\n" for target, (_, sha256) in fetches.items()) self._write(f"{self.work}/fetched.sha256", listed.encode()) try: - self._exec(f"cd {self.work} && sha256sum --check --quiet fetched.sha256") + self._exec(f"{self.helper} check {self.work} {self.work}/fetched.sha256") except RuntimeError as e: raise RuntimeError(f"a file the store served isn't the one this build stages: {e}") from None def mark_runnable(self, paths: list[str]) -> None: - self._each([f"case \"$(head -c 4 {shlex.quote(f'{self.work}/{path}')} | od -An -tx1 | tr -d ' \\n')\" in " - f"2321*|7f454c46) chmod {EXECUTABLE:o} {shlex.quote(f'{self.work}/{path}')};; esac" - for path in paths]) + self._each([f"{self.helper} runnable {self._at(path)}" for path in paths]) def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str], context: str = CONTEXT) -> None: folder = f"{self.work}/{context}" @@ -432,9 +422,7 @@ def build(self, dockerfile: str, tag: str, platform: str, labels: dict[str, str] raise DockerBuildError(f"docker build of {tag} failed (exit {code}):\n{(stdout + stderr).rstrip()}") def installed(self, tag: str, prefix: str) -> dict[str, str]: - copied = f"{self.work}/installed.json" - listed = self._exec(f"c=$(docker create {shlex.quote(tag)}) && {{ docker cp \"$c:{prefix}/installed.json\" " - f"{copied} >/dev/null 2>&1 && cat {copied}; docker rm \"$c\" >/dev/null; }}") + listed = self._exec(f"{self.helper} installed {shlex.quote(tag)} {shlex.quote(f'{prefix}/installed.json')}") return _packages(listed.encode()) if listed.strip() else {} def push(self, tag: str, ref: str, store) -> None: @@ -478,7 +466,11 @@ async def fetch(path: str, url: str) -> None: await asyncio.gather(*(fetch(path, url) for path, url in urls.items())) def _chmod(self, paths: list[str]) -> None: - self._each([f"chmod {EXECUTABLE:o} {shlex.quote(f'{self.work}/{path}')}" for path in paths]) + self._each([f"chmod {EXECUTABLE:o} {self._at(path)}" for path in paths]) + + def _at(self, path: str) -> str: + """``path``, in the builder's work folder, quoted for a script.""" + return shlex.quote(f"{self.work}/{path}") def _each(self, commands: list[str]) -> None: """Run ``commands`` in as few scripts as fit a script's limit.""" diff --git a/src/agent_env/env/envs/computer/utils/builder_in_vm.sh b/src/agent_env/env/envs/computer/utils/builder_in_vm.sh new file mode 100644 index 000000000..dd3586965 --- /dev/null +++ b/src/agent_env/env/envs/computer/utils/builder_in_vm.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# The shell side of a computer env's builder VM (builder.VmBuilder): sent to the VM once, then run as +# `bash builder_in_vm.sh `. Those that don't use Docker run on Linux and macOS alike. +set -euo pipefail + +retry=(--retry 5 --retry-all-errors --retry-delay 1) + +sum() { + if command -v sha256sum >/dev/null; then sha256sum "$1"; else shasum -a 256 "$1"; fi | cut -c1-64 +} + +# buildx URL SHA256 PLUGINS: give Docker BuildKit's client from URL, checked by SHA256, when it has none. +buildx() { + docker buildx version >/dev/null 2>&1 && return + mkdir -p "$3" + curl -fsSL "${retry[@]}" "$1" -o "$3/docker-buildx" + [ "$(sum "$3/docker-buildx")" = "$2" ] || { echo "$1 isn't the buildx pinned, $2" >&2; exit 1; } + chmod 755 "$3/docker-buildx" +} + +# emulate ARCH IMAGE: register QEMU for ARCH with IMAGE, the VM's own binfmt_misc mounted first, since a newer kernel +# gives a mount of it in a container an instance of its own. +emulate() { + mountpoint -q /proc/sys/fs/binfmt_misc || mount -t binfmt_misc binfmt_misc /proc/sys/fs/binfmt_misc + docker run --privileged --rm "$2" --install "$1" +} + +# files DIR: each plain file under DIR, never a link, as " ", each ending in a NUL. +files() { + cd "$1" + find . -type f -print0 | while IFS= read -r -d '' f; do + printf '%s %s %s\0' "$(sum "$f")" "$(wc -c < "$f" | tr -d ' ')" "${f#./}" + done +} + +# link SOURCE TARGET ...: each TARGET a hard link to its SOURCE. +link() { + while [ $# -gt 0 ]; do + mkdir -p "$(dirname "$2")" + ln -f "$1" "$2" + shift 2 + done +} + +# check DIR LIST: each line of LIST, " ", naming each file whose sha256 differs. +check() { + cd "$1" + local failed=0 line want path + while IFS= read -r line; do + want="${line%% *}" path="${line#* }" + if [ "$(sum "$path")" != "$want" ]; then + echo "$path: its sha256 is $(sum "$path"), not $want" + failed=1 + fi + done < "$2" + return "$failed" +} + +# runnable PATH ...: make each executable that starts as a script or a Linux binary does. +runnable() { + for f in "$@"; do + case "$(head -c 4 "$f" | od -An -tx1 | tr -d ' \n')" in + 2321* | 7f454c46) chmod 755 "$f" ;; + esac + done +} + +# download URL PATH: PATH fetched from URL; its sha256. +download() { + mkdir -p "$(dirname "$2")" + curl -fsSL "${retry[@]}" "$1" -o "$2" + sum "$2" +} + +# installed TAG FILE: FILE copied out of TAG's image without running it, on any arch; nothing when it holds none. +installed() { + local container copied + container=$(docker create "$1") + copied=$(mktemp) + if docker cp "$container:$2" "$copied" >/dev/null 2>&1; then cat "$copied"; fi + docker rm "$container" >/dev/null + rm -f "$copied" +} + +"$@" diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 203ecccb0..61e1d82da 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -669,8 +669,8 @@ def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driv class FakeVm: - """A builder VM, amd64: its files under ``root``, each line of a script it's given run as a VM's shell runs it, its - Docker the fake's.""" + """A builder VM, amd64: its files under ``root``, each line of a script it's given run by bash here, but for Docker + and uploads to the store, which the fake's Docker and the store do, and the helper's subcommands that use Docker.""" platform, _WFT_CHUNK_BYTES = "linux/amd64", 96 * 1024 @@ -710,84 +710,80 @@ async def terminate(self): def _line(self, line): self.provider.ran.append(line) - if line.startswith("case "): - path = self.at(shlex.split(line.split("$(head -c 4 ", 1)[1].split(" |", 1)[0])[0]) - if path.read_bytes()[:4].startswith((b"#!", b"\x7fELF")): - path.chmod(0o755) - return 0, "" - if line.startswith("c=$(docker create"): - return 0, json.dumps([{"name": "mss", "version": "10.1.0"}]) - if line.startswith(("docker buildx version", "mountpoint ")): - return 0, "" - if "docker login" in line: - self.provider.logins.append(shlex.split(line)[-1]) - return 0, "" - if "find . -type f" in line: - folder = self.at(shlex.split(line)[1]) - return 0, "".join(f"{hashlib.sha256(path.read_bytes()).hexdigest()} {path.stat().st_size} " - f"{path.relative_to(folder).as_posix()}\0" for path in sorted(folder.rglob("*")) - if path.is_file() and not path.is_symlink()) - cwd, printed = None, "" - for command in " ".join(shlex.split(line, posix=False)).split(" && "): - words = shlex.split(command) - if words[0] == "cd": - cwd = self.at(words[1]) - continue - code, printed = self._command(words, cwd) - if code: - return code, printed - return 0, printed - - def _command(self, words, cwd): - docker = self.provider.docker - if words[0] == "mkdir": - for folder in words[2:]: - self.at(folder).mkdir(parents=True, exist_ok=True) - elif words[0] == "tar": - with tarfile.open(self.at(words[2])) as archive: - archive.extractall(self.at(words[words.index("-C") + 1]), filter="data") - elif words[0] == "rm": - self.at(words[-1]).unlink() - elif words[0] == "ln": - self.at(words[3]).unlink(missing_ok=True) - self.at(words[3]).hardlink_to(self.at(words[2])) - elif words[0] == "chmod": - self.at(words[2]).chmod(0o755) - elif words[0] == "cat": - return 0, self.at(words[1]).read_text() - elif words[:2] == ["sha256sum", "--check"]: - for listed in (cwd / words[-1]).read_text().splitlines(): - sha256, path = listed.split(" ", 1) - if hashlib.sha256((cwd / path).read_bytes()).hexdigest() != sha256: - return 1, f"{path}: FAILED\n" - elif words[:2] == ["curl", "-fsSL"] and "--upload-file" in words: - url = urllib.parse.unquote(words[-1].partition("?")[2]) - get_config().get_object_store_at(url).put_file_at(url, str(self.at(words[-2]))) - self.provider.uploaded.append(url) - elif words[0] == "timeout": - out, platform = self.at(words[words.index("-v", 13) + 1].split(":")[0]), words[words.index("--platform") + 1] - context = self.at(words[words.index("-v") + 1].split(":")[0]) - result = docker.resolve(json.loads((out / "request.json").read_text()), context, words[-4], - platform.split("/")[1], out.parent) - (out / "result.json").write_text(json.dumps(result)) - elif words[:2] == ["docker", "run"]: - self.provider.binfmt.append(words[-1]) - elif words[:2] == ["docker", "build"]: - labels = dict(words[i + 1].split("=", 1) for i, word in enumerate(words) if word == "--label") - try: - docker.build_image(cwd / "Dockerfile", cwd, words[words.index("-t") + 1], - platform=words[words.index("--platform") + 1], labels=labels) - except DockerBuildError as e: - return 1, str(e) - elif words[:2] == ["docker", "tag"]: - self.provider.tagged[words[3]] = words[2] - elif words[:2] == ["docker", "push"]: - docker.push(self.provider.tagged[words[2]], words[2], None) - else: - raise AssertionError(f"a builder VM was asked to run {words}") + words = shlex.split(line) + if builder.IN_VM.name in line and words[2] in ("buildx", "emulate", "installed"): + if words[2] == "emulate": + self.provider.binfmt.append(words[3]) + return 0, json.dumps([{"name": "mss", "version": "10.1.0"}]) if words[2] == "installed" else "" + if "docker" in words or "curl" in words: + return self._docker(words) + ran = subprocess.run(["bash", "-c", line.replace("/tmp/agentenv-compose-", f"{self.root}/tmp/agentenv-compose-")], + capture_output=True, text=True) + return ran.returncode, ran.stdout + ran.stderr + + def _docker(self, words): + docker, cwd = self.provider.docker, None + while words: + command, words = (words[:words.index("&&")], words[words.index("&&") + 1:]) if "&&" in words else (words, []) + if command[0] == "cd": + cwd = self.at(command[1]) + elif "login" in command: + self.provider.logins.append(command[-1]) + elif command[0] == "timeout": + mounts = [i for i, word in enumerate(command) if word == "-v"] + context, out = (self.at(command[i + 1].split(":")[0]) for i in mounts) + platform = command[command.index("--platform") + 1] + result = docker.resolve(json.loads((out / "request.json").read_text()), context, + command[mounts[1] + 2], platform.split("/")[1], out.parent) + (out / "result.json").write_text(json.dumps(result)) + elif command[:2] == ["docker", "build"]: + labels = dict(command[i + 1].split("=", 1) for i, word in enumerate(command) if word == "--label") + try: + docker.build_image(cwd / "Dockerfile", cwd, command[command.index("-t") + 1], + platform=command[command.index("--platform") + 1], labels=labels) + except DockerBuildError as e: + return 1, str(e) + elif command[:2] == ["docker", "tag"]: + self.provider.tagged[command[3]] = command[2] + elif command[:2] == ["docker", "push"]: + docker.push(self.provider.tagged[command[2]], command[2], None) + elif command[0] == "curl" and "--upload-file" in command: + url = urllib.parse.unquote(command[-1].partition("?")[2]) + get_config().get_object_store_at(url).put_file_at(url, str(self.at(command[-2]))) + self.provider.uploaded.append(url) + else: + raise AssertionError(f"a builder VM was asked to run {command}") return 0, "" +def test_the_builder_vms_helper_lists_plain_files_marks_runnables_links_checks_and_downloads(tmp_path): + folder = tmp_path / "out" + (folder / "sub dir").mkdir(parents=True) + (folder / "sub dir" / "run me").write_bytes(b"#!/bin/sh\n") + (folder / "plain").write_bytes(b"data") + (folder / "elf").write_bytes(b"\x7fELF\x02") + (folder / "link").symlink_to(tmp_path / "elsewhere") + (tmp_path / "list").write_text(f"{hashlib.sha256(b'data').hexdigest()} plain\n{'0' * 64} elf\n") + + def helper(*args): + return subprocess.run(["bash", str(builder.IN_VM), *map(str, args)], capture_output=True, text=True) + + listed = helper("files", folder).stdout + helper("runnable", folder / "sub dir" / "run me", folder / "plain", folder / "elf") + helper("link", folder / "plain", tmp_path / "linked" / "plain") + checked = helper("check", folder, tmp_path / "list") + downloaded = helper("download", (folder / "plain").as_uri(), tmp_path / "fetched" / "plain") + + assert sorted(filter(None, listed.split("\0"))) == sorted( + f"{hashlib.sha256(data).hexdigest()} {len(data)} {name}" + for name, data in (("sub dir/run me", b"#!/bin/sh\n"), ("plain", b"data"), ("elf", b"\x7fELF\x02"))) + assert {name: (folder / name).stat().st_mode & 0o111 for name in ("sub dir/run me", "plain", "elf")} == { + "sub dir/run me": 0o111, "plain": 0, "elf": 0o111} + assert (tmp_path / "linked" / "plain").stat().st_ino == (folder / "plain").stat().st_ino + assert checked.returncode == 1 and checked.stdout.startswith("elf: its sha256 is ") + assert downloaded.stdout.strip() == hashlib.sha256(b"data").hexdigest() + + class FakeVmProvider(SandboxProvider): """A sandbox provider whose VMs run amd64, recording what each was asked to do.""" @@ -859,16 +855,16 @@ def test_a_chain_builds_each_arch_on_the_first_link_that_runs_it_natively(base, def test_a_builder_vm_its_provider_loses_fails_its_pair_alone_and_is_torn_down(base, driver, docker, signing, tmp_path, monkeypatch): - vms, real = FakeVmProvider(docker, tmp_path / "vms"), FakeVm._command + vms, real = FakeVmProvider(docker, tmp_path / "vms"), FakeVm._docker class Gone(Exception): pass - def command(vm, words, cwd): + def gone(vm, words): if words[0] == "timeout" and "linux/arm64" in words: raise Gone("the sandbox is shutting down") - return real(vm, words, cwd) - monkeypatch.setattr(FakeVm, "_command", command) + return real(vm, words) + monkeypatch.setattr(FakeVm, "_docker", gone) composed = Env.get(_put(base, driver, composing=Composing(vms)).id).metadata["composed"]["container"] From 93e7a1eeab8aa9600aaf7cb07c5e868193679868 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Fri, 30 Oct 2026 15:23:29 -0700 Subject: [PATCH 11/11] test(computer): where and how a pair builds gets a test file of its own The builder tests (VM puts, chains, refusals, emulation, the VM's shell helper, progress and the local builder's Docker calls) move to test_computer_builder.py; the fakes and helpers both files use to _computer.py, and their fixtures to a conftest.py beside them. No test changes what it checks. Co-Authored-By: Claude Opus 5.5 --- tst/unit/env/envs/_computer.py | 240 ++++++++++ tst/unit/env/envs/conftest.py | 64 +++ tst/unit/env/envs/test_computer_builder.py | 268 +++++++++++ tst/unit/env/envs/test_computer_env.py | 516 +-------------------- 4 files changed, 577 insertions(+), 511 deletions(-) create mode 100644 tst/unit/env/envs/_computer.py create mode 100644 tst/unit/env/envs/conftest.py create mode 100644 tst/unit/env/envs/test_computer_builder.py diff --git a/tst/unit/env/envs/_computer.py b/tst/unit/env/envs/_computer.py new file mode 100644 index 000000000..3af40b05b --- /dev/null +++ b/tst/unit/env/envs/_computer.py @@ -0,0 +1,240 @@ +"""What the computer env tests share: the fake Docker and registry the composer builds with, a fake sandbox provider +whose VMs build, and a base, a driver and an env of them.""" + +import json +import re +import shlex +import subprocess +import urllib.parse + +from agentenv_protocol.computer import DriverDeclaration +from agentenv_protocol.computer.conformance import CheckResult, ProbeReport + +from agent_env.artifact.artifacts.computer import ComputerArtifact +from agent_env.artifact.artifacts.file_artifact_universe import FileArtifactUniverse +from agent_env.config import get_config +from agent_env.env.envs.computer.utils import builder +from agent_env.env.envs.computer.env import ComputerEnv +from agent_env.env.envs.computer.utils.install import Install, Service +from agent_env.env.envs.computer.driver_env import ComputerDriverEnv +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider +from agent_env.utils.docker.build import DockerBuildError +from tst.util.oci_registry import UBUNTU_LAYER, Registry + +OS = {"type": "linux", "distribution": "ubuntu", "version": "24.04", "display": "x11"} + + +CONFIG = {"resolution": {"default": "1280x800", "phase": "deploy"}, "timezone": {"default": "UTC", "phase": "deploy"}, + "theme": {"default": "light", "phase": "build"}} + + +SERVE = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:Desk" + + +CARD_EXTENSION = "urn:agentenv:computer/v1" + + +CARD = {"name": "desktop", "capabilities": {"extensions": [{"uri": CARD_EXTENSION}]}} + + +TOOLS = ["computer_screenshot", "computer_click", "computer_type", "computer_pointer_list"] + + +class FakeDocker: + """This machine's Docker for the composer, on arm64 emulating amd64: each build recorded with its Dockerfile, its + context's files and its programs, each push landing in the fake registry as an image of its arch, and each probe + recorded by the arch and image it booted.""" + + def __init__(self, registry: Registry): + self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} + self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} + + def resolve(self, request, context, source, arch, work): + """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" + self.resolved.append((arch, request)) + out, result = work / "out", {"apt": [], "wheels": {}} + out.mkdir(parents=True, exist_ok=True) + for name in (re.split("[=/]", wanted)[0] for wanted in request["apt"]): + (out / "apt").mkdir(parents=True, exist_ok=True) + (out / "apt" / f"{name}_1.0_{arch}.deb").write_bytes(f"{name} for {arch}".encode()) + result["apt"].append({"file": f"{name}_1.0_{arch}.deb", "package": name, "version": "1.0", "arch": arch}) + for layer in request["python"]: + requirements = context / layer["folder"].removeprefix("/ctx/") / (layer["requirements"] or "-") + if not (requirements.is_file() and requirements.read_text().strip()) and not layer["add"]: + result["wheels"][layer["name"]] = [] + continue + (out / "wheels" / layer["name"]).mkdir(parents=True) + (out / "wheels" / layer["name"] / "mss-10.1.0-py3-none-any.whl").write_bytes(b"mss") + result["wheels"][layer["name"]] = ["mss-10.1.0-py3-none-any.whl"] + return result + + def resolving(self, resolve=None): + """A LocalBuilder.resolve that runs ``resolve``, by default this one's, on the builder's context and resolve.""" + def resolving(on, request, source, arch, timeout): + self.timeouts[arch] = timeout + return (resolve or self.resolve)(request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) + return resolving + + async def probe(self, links, env, driver, image): + arch = self._arch(self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]) + self.probed.append((arch, image)) + if arch in self.probe_fails: + checks = [CheckResult(check="P4", name="clicks show their effect", status="failed", detail=self.probe_fails[arch])] + else: + checks = [CheckResult(check="P0", name="the card", status="passed", extensions=[CARD_EXTENSION])] + return ProbeReport.of("full", "desk", checks, [CARD_EXTENSION]), CARD, "LocalSandboxProvider" + + def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_args=None): + if context.name == "label": + self.labels[tag] = labels + return + if platform in self.fail: + raise DockerBuildError(self.fail[platform]) + self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), + "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), + "conf": next(context.glob("agentenv-*.conf")).read_text(), + "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}, + "build_settings": {path.parent.parent.name: json.loads(path.read_text()) + for path in context.glob("apps/*/.agentenv/build_settings.json")}}) + + def push(self, local, ref, store): + self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], + labels=self.labels.get(local)) + + def joined(self, index, images, auth): + self.registry.tags[index] = self.registry.index(**{ + platform.rsplit("/", 1)[1]: image.rpartition("@")[2] for platform, image in images.items()}) + + def _arch(self, config_digest): + return json.loads(self.registry.blobs[config_digest])["architecture"] + + +def _base(image, **changes): + return ComputerArtifact.put(inspect=False, **{ + "id": "@local/~/kit/ubuntu-24.04", "arch": ["amd64", "arm64"], "os": OS, "session": {"user": "user"}, + "config": CONFIG, "forms": {"container": {"image": {"id": image.id, "version": image.version}, + "init": "supervisord", "input_channel": "env"}}, **changes}) + + +def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, install=True, extra=None): + folder.mkdir(parents=True, exist_ok=True) + (folder / "driver.py").write_text("class Desk: ...\n") + (folder / "requirements.txt").write_text(requirements) + files = FileArtifactUniverse.put_bundled("@local/~/kit/desk__files", files={ + name: folder / name for name in ("driver.py", "requirements.txt")}) + if extra: + files = FileArtifactUniverse.put(files.id, file_artifacts={**files.get_file_artifacts(), **extra}) + supports = supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] + platforms = [{"platform": {"os": "linux", "arch": arch, "display": "x11"}, "provides": [], "tools": TOOLS, + "features": {}} for arch in ("amd64", "arm64")] + services = services or [Service(name="driver", command=SERVE)] + return ComputerDriverEnv.put( + id="@local/~/kit/desk", module="driver.py:Desk", files=files, + declaration=DriverDeclaration.model_validate({"name": "desk", "supports": supports, "platforms": platforms}), + install={"linux": Install(python="requirements.txt", services=services)} if install else {}, port=18765) + + +def _put(base, driver, compose=True, **changes): + return ComputerEnv.put(**{"id": "@local/~/kit/desktop", "computer": base, "driver": driver, + "environment_name": "desktop", "compose": compose, **changes}) + + +class FakeVm: + """A builder VM, amd64: its files under ``root``, each line of a script it's given run by bash here, but for Docker + and uploads to the store, which the fake's Docker and the store do, and the helper's subcommands that use Docker.""" + + platform, _WFT_CHUNK_BYTES = "linux/amd64", 96 * 1024 + + def __init__(self, provider, root): + self.provider, self.root, self.terminated = provider, root, False + + def at(self, path): + return self.root / str(path).lstrip("/") + + async def write_host_file(self, data, path): + self.at(path).parent.mkdir(parents=True, exist_ok=True) + self.at(path).write_bytes(data) + + async def load_object_file(self, url, path): + self.provider.fetched.append(url) + self.at(path).parent.mkdir(parents=True, exist_ok=True) + get_config().get_object_store_at(url).download_to_file(url, str(self.at(path))) + + async def exec_script(self, script, max_retries=0): + code, out, _ = await self.exec_with_output("sudo", "bash", "-c", script) + if code: + raise RuntimeError(f"Script failed (exit {code}):\nstdout: {out}") + return out + + async def exec_with_output(self, *argv): + printed = [] + for line in argv[-1].splitlines(): + if line.strip() and line != "set -e": + code, out = self._line(line) + printed.append(out) + if code: + return code, "".join(printed), "" + return 0, "".join(printed), "" + + async def terminate(self): + self.terminated = True + + def _line(self, line): + self.provider.ran.append(line) + words = shlex.split(line) + if builder.IN_VM.name in line and words[2] in ("buildx", "emulate", "installed"): + if words[2] == "emulate": + self.provider.binfmt.append(words[3]) + return 0, json.dumps([{"name": "mss", "version": "10.1.0"}]) if words[2] == "installed" else "" + if "docker" in words or "curl" in words: + return self._docker(words) + ran = subprocess.run(["bash", "-c", line.replace("/tmp/agentenv-compose-", f"{self.root}/tmp/agentenv-compose-")], + capture_output=True, text=True) + return ran.returncode, ran.stdout + ran.stderr + + def _docker(self, words): + docker, cwd = self.provider.docker, None + while words: + command, words = (words[:words.index("&&")], words[words.index("&&") + 1:]) if "&&" in words else (words, []) + if command[0] == "cd": + cwd = self.at(command[1]) + elif "login" in command: + self.provider.logins.append(command[-1]) + elif command[0] == "timeout": + mounts = [i for i, word in enumerate(command) if word == "-v"] + context, out = (self.at(command[i + 1].split(":")[0]) for i in mounts) + platform = command[command.index("--platform") + 1] + result = docker.resolve(json.loads((out / "request.json").read_text()), context, + command[mounts[1] + 2], platform.split("/")[1], out.parent) + (out / "result.json").write_text(json.dumps(result)) + elif command[:2] == ["docker", "build"]: + labels = dict(command[i + 1].split("=", 1) for i, word in enumerate(command) if word == "--label") + try: + docker.build_image(cwd / "Dockerfile", cwd, command[command.index("-t") + 1], + platform=command[command.index("--platform") + 1], labels=labels) + except DockerBuildError as e: + return 1, str(e) + elif command[:2] == ["docker", "tag"]: + self.provider.tagged[command[3]] = command[2] + elif command[:2] == ["docker", "push"]: + docker.push(self.provider.tagged[command[2]], command[2], None) + elif command[0] == "curl" and "--upload-file" in command: + url = urllib.parse.unquote(command[-1].partition("?")[2]) + get_config().get_object_store_at(url).put_file_at(url, str(self.at(command[-2]))) + self.provider.uploaded.append(url) + else: + raise AssertionError(f"a builder VM was asked to run {command}") + return 0, "" + + +class FakeVmProvider(SandboxProvider): + """A sandbox provider whose VMs run amd64, recording what each was asked to do.""" + + def __init__(self, docker, root): + self.docker, self.root, self.vms, self.sizes = docker, root, [], [] + self.ran, self.fetched, self.uploaded, self.binfmt, self.logins, self.tagged = [], [], [], [], [], {} + + async def create_vm(self, **size): + self.sizes.append(size) + self.vms.append(vm := FakeVm(self, self.root / str(len(self.vms)))) + return vm diff --git a/tst/unit/env/envs/conftest.py b/tst/unit/env/envs/conftest.py new file mode 100644 index 000000000..2ebe7fada --- /dev/null +++ b/tst/unit/env/envs/conftest.py @@ -0,0 +1,64 @@ +"""The computer env tests' fixtures: this machine's Docker and registry faked for the composer, a base and a driver, +and stores a remote VM reaches.""" + +import urllib.parse + +import pytest + +from agent_env.artifact.artifacts import docker_image +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.env.envs.computer.utils import builder, composer +from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox +from agent_env.store.image_store.image_store import OciRegistryImageStore +from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore +from agent_env.store.object_store.local.store import LocalFilesystemObjectStore +from tst.util.oci_registry import UBUNTU_LAYER, Registry +from tst.unit.env.envs._computer import FakeDocker, _base, _driver + + +@pytest.fixture +def docker(local_stores, cli_routing, monkeypatch): + registry = Registry().serve(monkeypatch) + fake = FakeDocker(registry) + monkeypatch.setattr(builder, "build_image", fake.build_image) + monkeypatch.setattr(builder, "push_local_image", fake.push) + monkeypatch.setattr(builder, "docker_login", lambda auth: None) + monkeypatch.setattr(composer, "write_index", fake.joined) + monkeypatch.setattr(composer, "pin_digest", registry.pin_digest) + monkeypatch.setattr(docker_image, "pin_digest", registry.pin_digest) + monkeypatch.setattr(builder, "host_platform", lambda: "linux/arm64") + monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") + monkeypatch.setattr(composer, "_probe", fake.probe) + monkeypatch.setattr(builder, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) + monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) + monkeypatch.setattr(composer, "_protocol_source", lambda: None) + monkeypatch.setattr(builder.LocalBuilder, "resolve", fake.resolving()) + monkeypatch.setattr(LocalRegistryImageStore, "ensure_repository", lambda self, repository: None) + return fake + + +@pytest.fixture +def base(docker): + index = docker.registry.index(amd64=docker.registry.image(UBUNTU_LAYER), + arm64=docker.registry.image(UBUNTU_LAYER, arch="arm64")) + image = get_artifact_store().put_document(DockerImageArtifact(id="base-image", description="", + image_name=f"ghcr.io/acme/base:1@{index}")) + return _base(image) + + +@pytest.fixture +def driver(docker, tmp_path): + return _driver(tmp_path / "driver") + + +@pytest.fixture +def signing(local_stores, monkeypatch): + """Stores a remote VM reaches: the object store signs URLs to its objects, and the registry isn't this machine's.""" + images = OciRegistryImageStore("registry.example") + monkeypatch.setattr(type(local_stores), "get_image_store_for", lambda self, entity_id: images) + monkeypatch.setattr(OciRegistryImageStore, "ensure_repository", lambda self, repository: None) + monkeypatch.setattr(LocalFilesystemObjectStore, "signed_get_url", lambda self, url, expires_in=3600: + f"https://signed.example/get?{urllib.parse.quote(url)}") + monkeypatch.setattr(LocalFilesystemObjectStore, "signed_put_url", lambda self, url, expires_in=3600: + f"https://signed.example/put?{urllib.parse.quote(url)}") diff --git a/tst/unit/env/envs/test_computer_builder.py b/tst/unit/env/envs/test_computer_builder.py new file mode 100644 index 000000000..8b6f94cbe --- /dev/null +++ b/tst/unit/env/envs/test_computer_builder.py @@ -0,0 +1,268 @@ +"""Where and how a computer env's pairs build: this machine's Docker, natively or emulated, or a VM of the put's +sandbox provider, each arch on the first link of a chain that runs it natively; what builds when emulation is off +or no link can build; a VM the provider loses; the VM's shell helper; and what a put says as it goes. Docker, +the registry and the VMs are faked.""" + +import hashlib +import io +import json +import logging +import re +import subprocess +import tarfile +import time +from types import SimpleNamespace + +import pytest +from click.testing import CliRunner + +from agent_env.artifact.artifacts.docker_image import DockerImageArtifact +from agent_env.artifact.store import get_artifact_store +from agent_env.cli import cli +from agent_env.env import Env +from agent_env.env.envs.computer.utils import builder, composer, packages +from agent_env.env.envs.computer.utils.composer import Composing +from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider +from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider +from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider +from tst.unit.env.envs._computer import _base, _put, FakeVm, FakeVmProvider + + +def test_a_stage_that_goes_on_is_said_to_go_on_and_a_failed_pair_says_why(base, driver, docker, monkeypatch): + docker.fail["linux/amd64"] = "#7 ERROR: pip install failed" + build = docker.build_image + monkeypatch.setattr(composer, "_HEARTBEAT_S", 0.1) + monkeypatch.setattr(builder, "build_image", lambda *args, **kwargs: (time.sleep(0.5), build(*args, **kwargs))[1]) + told = [] + + _put(base, driver, composing=Composing(on_progress=told.append)) + + assert [line for line in told if line.startswith("container/arm64: still building the image (")] + assert any(re.fullmatch(r"container/amd64: failed at build: the build failed: #7 ERROR: pip install failed \(\d+ s\)", + line) for line in told) + + +def test_a_machine_that_cant_emulate_the_other_arch_fails_that_arch_alone(base, driver, docker): + docker.emulates = set() + + env = Env.get(_put(base, driver).id) + + amd64 = env.metadata["composed"]["container"]["amd64"] + assert amd64["reason"] == "this machine can't build linux/amd64: its Docker has no emulation for it" + assert env.metadata["composed"]["container"]["arm64"]["status"] == "ready" + assert [build["platform"] for build in docker.builds] == ["linux/arm64"] + + +def test_a_build_that_cant_run_the_other_archs_programs_says_its_emulation_is_missing(base, driver, docker): + docker.fail["linux/amd64"] = "#6 0.01 exec /bin/sh: exec format error\n#6 ERROR: process did not complete successfully" + + amd64 = Env.get(_put(base, driver).id).metadata["composed"]["container"]["amd64"] + + assert amd64["reason"].startswith("the builder can't run linux/amd64 programs: its Docker has no emulation for it:") + assert amd64["log"] + + +def test_a_base_built_from_its_context_is_built_here_for_each_arch_and_keyed_by_its_sources(driver, docker, + monkeypatch): + image = get_artifact_store().put_document(DockerImageArtifact( + id="base-image", description="", image_name="local/base-1234:abcd", build_context_object_url="file:///c.tar.gz", + dockerfile_path="Dockerfile", source_digest="sha256:" + "c" * 64)) + built_for = [] + monkeypatch.setattr(builder, "build_for", lambda image, platform: built_for.append(platform) or + f"{image.image_name}-{platform.rsplit('/', 1)[1]}") + + env = Env.get(_put(_base(image), driver).id) + again = Env.get(_put(_base(image), driver, id="@local/~/kit/again", environment_name="again").id) + + assert sorted(built_for) == ["linux/amd64", "linux/arm64"] + assert {r["reused"] for r in again.metadata["composed"]["container"].values()} == {True} + assert {build["dockerfile"].splitlines()[0] for build in docker.builds} == { + "FROM local/base-1234:abcd-amd64", "FROM local/base-1234:abcd-arm64"} + assert {arch: r["status"] for arch, r in env.metadata["composed"]["container"].items()} == { + "amd64": "unverified", "arm64": "ready"} + + +def test_the_platforms_this_machine_builds_are_read_from_its_builder(monkeypatch): + inspected = ("Name: desktop-linux\nDriver: docker\nNodes:\nName: desktop-linux\n" + "Platforms: linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64*\nLabels:\n") + monkeypatch.setattr(subprocess, "run", lambda *a, **k: SimpleNamespace(stdout=inspected, returncode=0)) + + assert builder._emulated_platforms() == {"linux/arm64", "linux/amd64", "linux/amd64/v2", "linux/riscv64"} + + +def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): + listed = json.dumps([{"name": "mss", "version": "10.2.0"}, + {"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode() + archive = io.BytesIO() + with tarfile.open(fileobj=archive, mode="w") as tar: + info = tarfile.TarInfo("installed.json") + info.size = len(listed) + tar.addfile(info, io.BytesIO(listed)) + calls = [] + + def run(argv, **kwargs): + calls.append(argv[:2]) + if argv[1] == "create": + return SimpleNamespace(stdout="c0ffee\n", returncode=0) + return SimpleNamespace(stdout=archive.getvalue(), returncode=0) + + monkeypatch.setattr(subprocess, "run", run) + + assert builder.LocalBuilder.installed(None, "agentenv-computer:abc-arm64", "reference") == { + "mss": "10.2.0", "agentenv-framework-protocol": "0.1.382"} + assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] + + +def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): + signed = [] + monkeypatch.setattr(builder, "docker_login", signed.append) + + _put(base, driver) + + assert len(signed) == 2 and len(docker.builds) == 2 + + +def test_the_builder_vms_helper_lists_plain_files_marks_runnables_links_checks_and_downloads(tmp_path): + folder = tmp_path / "out" + (folder / "sub dir").mkdir(parents=True) + (folder / "sub dir" / "run me").write_bytes(b"#!/bin/sh\n") + (folder / "plain").write_bytes(b"data") + (folder / "elf").write_bytes(b"\x7fELF\x02") + (folder / "link").symlink_to(tmp_path / "elsewhere") + (tmp_path / "list").write_text(f"{hashlib.sha256(b'data').hexdigest()} plain\n{'0' * 64} elf\n") + + def helper(*args): + return subprocess.run(["bash", str(builder.IN_VM), *map(str, args)], capture_output=True, text=True) + + listed = helper("files", folder).stdout + helper("runnable", folder / "sub dir" / "run me", folder / "plain", folder / "elf") + helper("link", folder / "plain", tmp_path / "linked" / "plain") + checked = helper("check", folder, tmp_path / "list") + downloaded = helper("download", (folder / "plain").as_uri(), tmp_path / "fetched" / "plain") + + assert sorted(filter(None, listed.split("\0"))) == sorted( + f"{hashlib.sha256(data).hexdigest()} {len(data)} {name}" + for name, data in (("sub dir/run me", b"#!/bin/sh\n"), ("plain", b"data"), ("elf", b"\x7fELF\x02"))) + assert {name: (folder / name).stat().st_mode & 0o111 for name in ("sub dir/run me", "plain", "elf")} == { + "sub dir/run me": 0o111, "plain": 0, "elf": 0o111} + assert (tmp_path / "linked" / "plain").stat().st_ino == (folder / "plain").stat().st_ino + assert checked.returncode == 1 and checked.stdout.startswith("elf: its sha256 is ") + assert downloaded.stdout.strip() == hashlib.sha256(b"data").hexdigest() + + +def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_images_are_reused_by_key_here( + base, driver, docker, signing, tmp_path, monkeypatch): + vms = FakeVmProvider(docker, tmp_path / "vms") + for name in ("build_image", "push_local_image", "docker_login", "_emulated_platforms"): + monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) + + told = [] + composed = Env.get(_put(base, driver, composing=Composing(vms, on_progress=told.append)).id).metadata["composed"][ + "container"] + + assert ("container/arm64: building emulated, which is slow, in a FakeVmProvider VM; add local to --sandbox to build " + "it natively with this machine's Docker, or pass --no-emulate to skip it") in told + assert {arch: (r["status"], r["builder"], r["emulated"]) for arch, r in composed.items()} == { + "amd64": ("ready", "FakeVmProvider", False), "arm64": ("unverified", "FakeVmProvider", True)} + assert composed["arm64"]["reason"] == "no link of the sandbox provider runs linux/arm64 natively to probe it" + assert len(vms.vms) == 2 and all(vm.terminated for vm in vms.vms) and vms.binfmt == ["arm64"] + assert sorted(vms.sizes, key=lambda size: size["timeout"]) == [ + {"exposed_ports": [], "timeout": 7200, **builder._VM}, {"exposed_ports": [], "timeout": 21600, **builder._VM}] + driver_files = {artifact.object_url for artifact in driver.files.get_file_artifacts().values()} + assert sorted(vms.fetched) == sorted([*driver_files, *driver_files]) + assert vms.uploaded and all(f"{packages.PREFIX}/files/" in url for url in vms.uploaded) + builds = {build["platform"]: build for build in docker.builds} + assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", + "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] + assert composed["amd64"]["installed"] == {"mss": "10.1.0"} + assert docker.labels[f"agentenv-computer:{composed['arm64']['key'][7:23]}-arm64"]["dev.agentenv.emulated"] == "true" + + again = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id) + + assert {arch: (r["reused"], r["key"]) for arch, r in again.metadata["composed"]["container"].items()} == { + arch: (True, r["key"]) for arch, r in composed.items()} + + +def test_a_chain_builds_each_arch_on_the_first_link_that_runs_it_natively(base, driver, docker, signing, tmp_path): + vms = FakeVmProvider(docker, tmp_path / "vms") + + composed = Env.get(_put(base, driver, composing=Composing(ChainedSandboxProvider( + [vms, LocalSandboxProvider()]))).id).metadata["composed"]["container"] + + assert {arch: (r["builder"], r["emulated"]) for arch, r in composed.items()} == { + "amd64": ("FakeVmProvider", False), "arm64": ("LocalSandboxProvider", False)} + assert len(vms.vms) == 1 and vms.binfmt == [] + + +def test_a_builder_vm_its_provider_loses_fails_its_pair_alone_and_is_torn_down(base, driver, docker, signing, tmp_path, + monkeypatch): + vms, real = FakeVmProvider(docker, tmp_path / "vms"), FakeVm._docker + + class Gone(Exception): + pass + + def gone(vm, words): + if words[0] == "timeout" and "linux/arm64" in words: + raise Gone("the sandbox is shutting down") + return real(vm, words) + monkeypatch.setattr(FakeVm, "_docker", gone) + + composed = Env.get(_put(base, driver, composing=Composing(vms)).id).metadata["composed"]["container"] + + assert (composed["arm64"]["status"], composed["arm64"]["stage"], composed["arm64"]["reason"]) == ( + "failed", "resolve", "RuntimeError: FakeVmProvider's VM failed: Gone: the sandbox is shutting down") + assert composed["amd64"]["status"] == "ready" and all(vm.terminated for vm in vms.vms) + + +def test_with_emulation_off_an_arch_a_refused_link_runs_natively_fails_and_only_one_needing_emulation_is_skipped( + base, driver, docker, tmp_path): + vms = FakeVmProvider(docker, tmp_path / "vms") + + composed = Env.get(_put(base, driver, composing=Composing(vms, emulate=False)).id).metadata["composed"]["container"] + + assert (composed["amd64"]["status"], composed["amd64"]["reason"]) == ( + "failed", "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local " + "provider") + assert (composed["arm64"]["status"], composed["arm64"]["reason"]) == ( + "skipped", "emulation is off, and FakeVmProvider doesn't run linux/arm64 natively") + assert vms.vms == [] + + +class Containers(SandboxProvider): + async def create_sandbox(self, **kwargs): + raise AssertionError("a container was created") + + +@pytest.mark.parametrize("links, reason", [ + (["vms"], "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local provider"), + (["containers"], "Containers gives no VM to build in"), + (["containers", "local"], "Containers gives no VM to build in; this machine can't build linux/amd64: its Docker has " + "no emulation for it"), +]) +def test_an_arch_no_link_can_build_fails_saying_why_for_each_link(base, driver, docker, tmp_path, links, reason): + docker.emulates = set() + made = {"vms": FakeVmProvider(docker, tmp_path / "vms"), "containers": Containers(), + "local": LocalSandboxProvider()} + + composed = Env.get(_put(base, driver, composing=Composing(ChainedSandboxProvider( + [made[link] for link in links]))).id).metadata["composed"]["container"] + + assert (composed["amd64"]["status"], composed["amd64"]["stage"], composed["amd64"]["reason"]) == ( + "failed", "resolve", reason) + assert made["vms"].vms == [] + + +def test_a_put_that_wont_emulate_skips_an_arch_no_link_runs_natively_and_a_later_build_that_will_builds_it( + base, driver, docker, caplog): + caplog.set_level(logging.ERROR, logger=composer.__name__) + env = _put(base, driver, composing=Composing(emulate=False)) + skipped = CliRunner().invoke(cli, ["env", "build", "--id", env.id, "--no-emulate"]) + built = CliRunner().invoke(cli, ["env", "build", "--id", env.id]) + + reason = "emulation is off, and LocalSandboxProvider doesn't run linux/amd64 natively" + assert skipped.exit_code == 0 and f"container/amd64: skipped ({reason})" in skipped.output + assert [build["platform"] for build in docker.builds] == ["linux/arm64", "linux/amd64"] + assert built.exit_code == 0 and "container/amd64: building emulated, which is slow, with this machine's Docker; " \ + "pass --no-emulate to skip it" in built.output + composed = Env.get(env.id).metadata["composed"]["container"] + assert {arch: r["status"] for arch, r in composed.items()} == {"amd64": "unverified", "arm64": "ready"} diff --git a/tst/unit/env/envs/test_computer_env.py b/tst/unit/env/envs/test_computer_env.py index 61e1d82da..3a39a1e37 100644 --- a/tst/unit/env/envs/test_computer_env.py +++ b/tst/unit/env/envs/test_computer_env.py @@ -1,21 +1,16 @@ """Computer envs: gate 1 refuses a base, a driver or a declaration that can't compose, naming every problem, before -anything is stored; the composer builds each form and arch on this machine's Docker, reuses an image of the same key -whichever env built it, records each result as it finishes, and writes the arches built as the env's computer. Docker -and the registry are faked.""" +anything is stored; the composer builds each form and arch, reuses an image of the same key whichever env built it, +records each result as it finishes, and writes the arches built as the env's computer; a deploy, a reset and a load; +and computer apps. Where and how a pair builds is test_computer_builder. Docker and the registry are faked.""" import asyncio import contextlib import hashlib import importlib.metadata -import io import json import logging import re -import shlex import shutil -import subprocess -import tarfile -import time import urllib.parse from pathlib import Path from types import SimpleNamespace @@ -23,12 +18,10 @@ import httpx import pytest -from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING, AppDeclaration, DriverDeclaration -from agentenv_protocol.computer.conformance import CheckResult, ProbeReport +from agentenv_protocol.computer import COMPUTER_DISPLAY, COMPUTER_RECORDING, AppDeclaration from agentenv_protocol.tool_access import DISABLE_TOOL_EXTENSION_URI from click.testing import CliRunner -from agent_env.artifact.artifacts import docker_image from agent_env.artifact.artifacts.computer import ComputerArtifact from agent_env.artifact.artifacts.docker_image import DockerImageArtifact from agent_env.artifact.artifacts.file import FileArtifact @@ -51,9 +44,6 @@ from agent_env.task_step.task_steps.deploy_env import DeployEnvTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from agent_env.providers.sandbox_providers.local_docker import provider as local_sandbox -from agent_env.providers.sandbox_providers.chained_sandbox_provider import ChainedSandboxProvider -from agent_env.providers.sandbox_providers.local_docker.provider import LocalSandboxProvider -from agent_env.providers.sandbox_providers.sandbox_provider import SandboxProvider from agent_env.store.base import NotFoundError from agent_env.task import teardown from agent_env.task_step.context import TaskStepContext @@ -61,151 +51,8 @@ from agent_env.store.image_store.local_registry_image_store import LocalRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import RegistryAuth from agent_env.store.object_store.local.store import LocalFilesystemObjectStore -from agent_env.utils.docker.build import DockerBuildError from tst.unit.computer_home import Computer -from tst.util.oci_registry import UBUNTU_LAYER, Registry - -OS = {"type": "linux", "distribution": "ubuntu", "version": "24.04", "display": "x11"} -CONFIG = {"resolution": {"default": "1280x800", "phase": "deploy"}, "timezone": {"default": "UTC", "phase": "deploy"}, - "theme": {"default": "light", "phase": "build"}} -SERVE = "venv/bin/python -m agentenv_protocol.computer.serve driver.py:Desk" -CARD_EXTENSION = "urn:agentenv:computer/v1" -CARD = {"name": "desktop", "capabilities": {"extensions": [{"uri": CARD_EXTENSION}]}} -TOOLS = ["computer_screenshot", "computer_click", "computer_type", "computer_pointer_list"] - - -class FakeDocker: - """This machine's Docker for the composer, on arm64 emulating amd64: each build recorded with its Dockerfile, its - context's files and its programs, each push landing in the fake registry as an image of its arch, and each probe - recorded by the arch and image it booted.""" - - def __init__(self, registry: Registry): - self.registry, self.builds, self.fail, self.emulates, self.labels = registry, [], {}, {"linux/amd64"}, {} - self.probed, self.probe_fails, self.resolved, self.timeouts = [], {}, [], {} - - def resolve(self, request, context, source, arch, work): - """The resolver in the base: each apt package as one .deb, each venv's requirements as one wheel.""" - self.resolved.append((arch, request)) - out, result = work / "out", {"apt": [], "wheels": {}} - out.mkdir(parents=True, exist_ok=True) - for name in (re.split("[=/]", wanted)[0] for wanted in request["apt"]): - (out / "apt").mkdir(parents=True, exist_ok=True) - (out / "apt" / f"{name}_1.0_{arch}.deb").write_bytes(f"{name} for {arch}".encode()) - result["apt"].append({"file": f"{name}_1.0_{arch}.deb", "package": name, "version": "1.0", "arch": arch}) - for layer in request["python"]: - requirements = context / layer["folder"].removeprefix("/ctx/") / (layer["requirements"] or "-") - if not (requirements.is_file() and requirements.read_text().strip()) and not layer["add"]: - result["wheels"][layer["name"]] = [] - continue - (out / "wheels" / layer["name"]).mkdir(parents=True) - (out / "wheels" / layer["name"] / "mss-10.1.0-py3-none-any.whl").write_bytes(b"mss") - result["wheels"][layer["name"]] = ["mss-10.1.0-py3-none-any.whl"] - return result - - def resolving(self, resolve=None): - """A LocalBuilder.resolve that runs ``resolve``, by default this one's, on the builder's context and resolve.""" - def resolving(on, request, source, arch, timeout): - self.timeouts[arch] = timeout - return (resolve or self.resolve)(request, on.work / builder.CONTEXT, source, arch, on.work / builder.RESOLVED) - return resolving - - async def probe(self, links, env, driver, image): - arch = self._arch(self.registry.manifests[image.rpartition("@")[2]]["config"]["digest"]) - self.probed.append((arch, image)) - if arch in self.probe_fails: - checks = [CheckResult(check="P4", name="clicks show their effect", status="failed", detail=self.probe_fails[arch])] - else: - checks = [CheckResult(check="P0", name="the card", status="passed", extensions=[CARD_EXTENSION])] - return ProbeReport.of("full", "desk", checks, [CARD_EXTENSION]), CARD, "LocalSandboxProvider" - - def build_image(self, dockerfile, context, tag, *, platform, labels=None, build_args=None): - if context.name == "label": - self.labels[tag] = labels - return - if platform in self.fail: - raise DockerBuildError(self.fail[platform]) - self.builds.append({"platform": platform, "tag": tag, "dockerfile": dockerfile.read_text(), - "files": sorted(p.relative_to(context).as_posix() for p in context.rglob("*") if p.is_file()), - "conf": next(context.glob("agentenv-*.conf")).read_text(), - "confs": {conf.name: conf.read_text() for conf in context.glob("agentenv-*.conf")}, - "build_settings": {path.parent.parent.name: json.loads(path.read_text()) - for path in context.glob("apps/*/.agentenv/build_settings.json")}}) - - def push(self, local, ref, store): - self.registry.tags[ref] = self.registry.image(UBUNTU_LAYER, arch=local.rsplit("-", 1)[1], - labels=self.labels.get(local)) - - def joined(self, index, images, auth): - self.registry.tags[index] = self.registry.index(**{ - platform.rsplit("/", 1)[1]: image.rpartition("@")[2] for platform, image in images.items()}) - - def _arch(self, config_digest): - return json.loads(self.registry.blobs[config_digest])["architecture"] - - -@pytest.fixture -def docker(local_stores, cli_routing, monkeypatch): - registry = Registry().serve(monkeypatch) - fake = FakeDocker(registry) - monkeypatch.setattr(builder, "build_image", fake.build_image) - monkeypatch.setattr(builder, "push_local_image", fake.push) - monkeypatch.setattr(builder, "docker_login", lambda auth: None) - monkeypatch.setattr(composer, "write_index", fake.joined) - monkeypatch.setattr(composer, "pin_digest", registry.pin_digest) - monkeypatch.setattr(docker_image, "pin_digest", registry.pin_digest) - monkeypatch.setattr(builder, "host_platform", lambda: "linux/arm64") - monkeypatch.setattr(local_sandbox, "host_platform", lambda: "linux/arm64") - monkeypatch.setattr(composer, "_probe", fake.probe) - monkeypatch.setattr(builder, "_emulated_platforms", lambda: {"linux/arm64", *fake.emulates}) - monkeypatch.setattr(builder.LocalBuilder, "installed", lambda self, tag, prefix: {"mss": "10.1.0"}) - monkeypatch.setattr(composer, "_protocol_source", lambda: None) - monkeypatch.setattr(builder.LocalBuilder, "resolve", fake.resolving()) - monkeypatch.setattr(LocalRegistryImageStore, "ensure_repository", lambda self, repository: None) - return fake - - -@pytest.fixture -def base(docker): - index = docker.registry.index(amd64=docker.registry.image(UBUNTU_LAYER), - arm64=docker.registry.image(UBUNTU_LAYER, arch="arm64")) - image = get_artifact_store().put_document(DockerImageArtifact(id="base-image", description="", - image_name=f"ghcr.io/acme/base:1@{index}")) - return _base(image) - - -@pytest.fixture -def driver(docker, tmp_path): - return _driver(tmp_path / "driver") - - -def _base(image, **changes): - return ComputerArtifact.put(inspect=False, **{ - "id": "@local/~/kit/ubuntu-24.04", "arch": ["amd64", "arm64"], "os": OS, "session": {"user": "user"}, - "config": CONFIG, "forms": {"container": {"image": {"id": image.id, "version": image.version}, - "init": "supervisord", "input_channel": "env"}}, **changes}) - - -def _driver(folder, requirements="mss>=9,<11\n", supports=None, services=None, install=True, extra=None): - folder.mkdir(parents=True, exist_ok=True) - (folder / "driver.py").write_text("class Desk: ...\n") - (folder / "requirements.txt").write_text(requirements) - files = FileArtifactUniverse.put_bundled("@local/~/kit/desk__files", files={ - name: folder / name for name in ("driver.py", "requirements.txt")}) - if extra: - files = FileArtifactUniverse.put(files.id, file_artifacts={**files.get_file_artifacts(), **extra}) - supports = supports or [{"os": {"type": "linux", "display": ["x11"]}, "arch": ["amd64", "arm64"]}] - platforms = [{"platform": {"os": "linux", "arch": arch, "display": "x11"}, "provides": [], "tools": TOOLS, - "features": {}} for arch in ("amd64", "arm64")] - services = services or [Service(name="driver", command=SERVE)] - return ComputerDriverEnv.put( - id="@local/~/kit/desk", module="driver.py:Desk", files=files, - declaration=DriverDeclaration.model_validate({"name": "desk", "supports": supports, "platforms": platforms}), - install={"linux": Install(python="requirements.txt", services=services)} if install else {}, port=18765) - - -def _put(base, driver, compose=True, **changes): - return ComputerEnv.put(**{"id": "@local/~/kit/desktop", "computer": base, "driver": driver, - "environment_name": "desktop", "compose": compose, **changes}) +from tst.unit.env.envs._computer import CONFIG, SERVE, CARD_EXTENSION, CARD, _base, _driver, _put, FakeVmProvider def test_a_computer_env_is_composed_for_each_arch_and_written_with_the_computer_its_images_make(base, driver, docker, @@ -291,20 +138,6 @@ def test_an_env_that_differs_only_in_what_doesnt_change_the_image_reuses_the_one assert docker_calls == [("join", index, auth)] -def test_a_stage_that_goes_on_is_said_to_go_on_and_a_failed_pair_says_why(base, driver, docker, monkeypatch): - docker.fail["linux/amd64"] = "#7 ERROR: pip install failed" - build = docker.build_image - monkeypatch.setattr(composer, "_HEARTBEAT_S", 0.1) - monkeypatch.setattr(builder, "build_image", lambda *args, **kwargs: (time.sleep(0.5), build(*args, **kwargs))[1]) - told = [] - - _put(base, driver, composing=Composing(on_progress=told.append)) - - assert [line for line in told if line.startswith("container/arm64: still building the image (")] - assert any(re.fullmatch(r"container/amd64: failed at build: the build failed: #7 ERROR: pip install failed \(\d+ s\)", - line) for line in told) - - @pytest.mark.parametrize("change", ["requirements", "build-setting", "service", "display-read-at-build"]) def test_what_changes_the_image_changes_its_key_and_builds_it_again(base, driver, docker, tmp_path, monkeypatch, change): @@ -358,46 +191,6 @@ def test_a_driver_file_named_outside_its_folder_fails_the_build_before_anything_ assert docker.builds == [] -def test_a_machine_that_cant_emulate_the_other_arch_fails_that_arch_alone(base, driver, docker): - docker.emulates = set() - - env = Env.get(_put(base, driver).id) - - amd64 = env.metadata["composed"]["container"]["amd64"] - assert amd64["reason"] == "this machine can't build linux/amd64: its Docker has no emulation for it" - assert env.metadata["composed"]["container"]["arm64"]["status"] == "ready" - assert [build["platform"] for build in docker.builds] == ["linux/arm64"] - - -def test_a_build_that_cant_run_the_other_archs_programs_says_its_emulation_is_missing(base, driver, docker): - docker.fail["linux/amd64"] = "#6 0.01 exec /bin/sh: exec format error\n#6 ERROR: process did not complete successfully" - - amd64 = Env.get(_put(base, driver).id).metadata["composed"]["container"]["amd64"] - - assert amd64["reason"].startswith("the builder can't run linux/amd64 programs: its Docker has no emulation for it:") - assert amd64["log"] - - -def test_a_base_built_from_its_context_is_built_here_for_each_arch_and_keyed_by_its_sources(driver, docker, - monkeypatch): - image = get_artifact_store().put_document(DockerImageArtifact( - id="base-image", description="", image_name="local/base-1234:abcd", build_context_object_url="file:///c.tar.gz", - dockerfile_path="Dockerfile", source_digest="sha256:" + "c" * 64)) - built_for = [] - monkeypatch.setattr(builder, "build_for", lambda image, platform: built_for.append(platform) or - f"{image.image_name}-{platform.rsplit('/', 1)[1]}") - - env = Env.get(_put(_base(image), driver).id) - again = Env.get(_put(_base(image), driver, id="@local/~/kit/again", environment_name="again").id) - - assert sorted(built_for) == ["linux/amd64", "linux/arm64"] - assert {r["reused"] for r in again.metadata["composed"]["container"].values()} == {True} - assert {build["dockerfile"].splitlines()[0] for build in docker.builds} == { - "FROM local/base-1234:abcd-amd64", "FROM local/base-1234:abcd-arm64"} - assert {arch: r["status"] for arch, r in env.metadata["composed"]["container"].items()} == { - "amd64": "unverified", "arm64": "ready"} - - def test_a_checkout_resolves_its_own_protocol_into_the_drivers_venv(base, driver, docker, tmp_path, monkeypatch): monkeypatch.setattr(composer, "_protocol_source", lambda: _protocol_checkout(tmp_path)) @@ -592,14 +385,6 @@ def test_a_base_and_driver_with_no_arch_in_common_and_an_image_that_cant_be_got_ "context to build it from" in str(refused.value)) -def test_the_platforms_this_machine_builds_are_read_from_its_builder(monkeypatch): - inspected = ("Name: desktop-linux\nDriver: docker\nNodes:\nName: desktop-linux\n" - "Platforms: linux/arm64, linux/amd64, linux/amd64/v2, linux/riscv64*\nLabels:\n") - monkeypatch.setattr(subprocess, "run", lambda *a, **k: SimpleNamespace(stdout=inspected, returncode=0)) - - assert builder._emulated_platforms() == {"linux/arm64", "linux/amd64", "linux/amd64/v2", "linux/riscv64"} - - @pytest.mark.parametrize("direct_url, expected", [ ('{{"url": "file://{folder}", "dir_info": {{"editable": true}}}}', "folder"), (None, None), @@ -615,29 +400,6 @@ def test_the_protocol_is_composed_from_its_source_only_when_installed_from_one(t assert composer._protocol_source() == (tmp_path if expected else None) -def test_the_packages_a_build_installed_are_read_from_its_image_without_running_it(monkeypatch): - listed = json.dumps([{"name": "mss", "version": "10.2.0"}, - {"name": "agentenv-framework-protocol", "version": "0.1.382"}]).encode() - archive = io.BytesIO() - with tarfile.open(fileobj=archive, mode="w") as tar: - info = tarfile.TarInfo("installed.json") - info.size = len(listed) - tar.addfile(info, io.BytesIO(listed)) - calls = [] - - def run(argv, **kwargs): - calls.append(argv[:2]) - if argv[1] == "create": - return SimpleNamespace(stdout="c0ffee\n", returncode=0) - return SimpleNamespace(stdout=archive.getvalue(), returncode=0) - - monkeypatch.setattr(subprocess, "run", run) - - assert builder.LocalBuilder.installed(None, "agentenv-computer:abc-arm64", "reference") == { - "mss": "10.2.0", "agentenv-framework-protocol": "0.1.382"} - assert calls == [["docker", "create"], ["docker", "cp"], ["docker", "rm"]] - - def test_a_script_or_binary_in_the_drivers_files_stays_executable_in_the_image(base, docker, tmp_path, monkeypatch): folder = tmp_path / "scripted" folder.mkdir() @@ -659,258 +421,6 @@ def test_a_script_or_binary_in_the_drivers_files_stays_executable_in_the_image(b assert modes == {"driver.py": 0, "start": 0o111, "helper": 0o111} -def test_a_registry_base_is_signed_in_to_before_its_image_is_built_on(base, driver, docker, monkeypatch): - signed = [] - monkeypatch.setattr(builder, "docker_login", signed.append) - - _put(base, driver) - - assert len(signed) == 2 and len(docker.builds) == 2 - - -class FakeVm: - """A builder VM, amd64: its files under ``root``, each line of a script it's given run by bash here, but for Docker - and uploads to the store, which the fake's Docker and the store do, and the helper's subcommands that use Docker.""" - - platform, _WFT_CHUNK_BYTES = "linux/amd64", 96 * 1024 - - def __init__(self, provider, root): - self.provider, self.root, self.terminated = provider, root, False - - def at(self, path): - return self.root / str(path).lstrip("/") - - async def write_host_file(self, data, path): - self.at(path).parent.mkdir(parents=True, exist_ok=True) - self.at(path).write_bytes(data) - - async def load_object_file(self, url, path): - self.provider.fetched.append(url) - self.at(path).parent.mkdir(parents=True, exist_ok=True) - get_config().get_object_store_at(url).download_to_file(url, str(self.at(path))) - - async def exec_script(self, script, max_retries=0): - code, out, _ = await self.exec_with_output("sudo", "bash", "-c", script) - if code: - raise RuntimeError(f"Script failed (exit {code}):\nstdout: {out}") - return out - - async def exec_with_output(self, *argv): - printed = [] - for line in argv[-1].splitlines(): - if line.strip() and line != "set -e": - code, out = self._line(line) - printed.append(out) - if code: - return code, "".join(printed), "" - return 0, "".join(printed), "" - - async def terminate(self): - self.terminated = True - - def _line(self, line): - self.provider.ran.append(line) - words = shlex.split(line) - if builder.IN_VM.name in line and words[2] in ("buildx", "emulate", "installed"): - if words[2] == "emulate": - self.provider.binfmt.append(words[3]) - return 0, json.dumps([{"name": "mss", "version": "10.1.0"}]) if words[2] == "installed" else "" - if "docker" in words or "curl" in words: - return self._docker(words) - ran = subprocess.run(["bash", "-c", line.replace("/tmp/agentenv-compose-", f"{self.root}/tmp/agentenv-compose-")], - capture_output=True, text=True) - return ran.returncode, ran.stdout + ran.stderr - - def _docker(self, words): - docker, cwd = self.provider.docker, None - while words: - command, words = (words[:words.index("&&")], words[words.index("&&") + 1:]) if "&&" in words else (words, []) - if command[0] == "cd": - cwd = self.at(command[1]) - elif "login" in command: - self.provider.logins.append(command[-1]) - elif command[0] == "timeout": - mounts = [i for i, word in enumerate(command) if word == "-v"] - context, out = (self.at(command[i + 1].split(":")[0]) for i in mounts) - platform = command[command.index("--platform") + 1] - result = docker.resolve(json.loads((out / "request.json").read_text()), context, - command[mounts[1] + 2], platform.split("/")[1], out.parent) - (out / "result.json").write_text(json.dumps(result)) - elif command[:2] == ["docker", "build"]: - labels = dict(command[i + 1].split("=", 1) for i, word in enumerate(command) if word == "--label") - try: - docker.build_image(cwd / "Dockerfile", cwd, command[command.index("-t") + 1], - platform=command[command.index("--platform") + 1], labels=labels) - except DockerBuildError as e: - return 1, str(e) - elif command[:2] == ["docker", "tag"]: - self.provider.tagged[command[3]] = command[2] - elif command[:2] == ["docker", "push"]: - docker.push(self.provider.tagged[command[2]], command[2], None) - elif command[0] == "curl" and "--upload-file" in command: - url = urllib.parse.unquote(command[-1].partition("?")[2]) - get_config().get_object_store_at(url).put_file_at(url, str(self.at(command[-2]))) - self.provider.uploaded.append(url) - else: - raise AssertionError(f"a builder VM was asked to run {command}") - return 0, "" - - -def test_the_builder_vms_helper_lists_plain_files_marks_runnables_links_checks_and_downloads(tmp_path): - folder = tmp_path / "out" - (folder / "sub dir").mkdir(parents=True) - (folder / "sub dir" / "run me").write_bytes(b"#!/bin/sh\n") - (folder / "plain").write_bytes(b"data") - (folder / "elf").write_bytes(b"\x7fELF\x02") - (folder / "link").symlink_to(tmp_path / "elsewhere") - (tmp_path / "list").write_text(f"{hashlib.sha256(b'data').hexdigest()} plain\n{'0' * 64} elf\n") - - def helper(*args): - return subprocess.run(["bash", str(builder.IN_VM), *map(str, args)], capture_output=True, text=True) - - listed = helper("files", folder).stdout - helper("runnable", folder / "sub dir" / "run me", folder / "plain", folder / "elf") - helper("link", folder / "plain", tmp_path / "linked" / "plain") - checked = helper("check", folder, tmp_path / "list") - downloaded = helper("download", (folder / "plain").as_uri(), tmp_path / "fetched" / "plain") - - assert sorted(filter(None, listed.split("\0"))) == sorted( - f"{hashlib.sha256(data).hexdigest()} {len(data)} {name}" - for name, data in (("sub dir/run me", b"#!/bin/sh\n"), ("plain", b"data"), ("elf", b"\x7fELF\x02"))) - assert {name: (folder / name).stat().st_mode & 0o111 for name in ("sub dir/run me", "plain", "elf")} == { - "sub dir/run me": 0o111, "plain": 0, "elf": 0o111} - assert (tmp_path / "linked" / "plain").stat().st_ino == (folder / "plain").stat().st_ino - assert checked.returncode == 1 and checked.stdout.startswith("elf: its sha256 is ") - assert downloaded.stdout.strip() == hashlib.sha256(b"data").hexdigest() - - -class FakeVmProvider(SandboxProvider): - """A sandbox provider whose VMs run amd64, recording what each was asked to do.""" - - def __init__(self, docker, root): - self.docker, self.root, self.vms, self.sizes = docker, root, [], [] - self.ran, self.fetched, self.uploaded, self.binfmt, self.logins, self.tagged = [], [], [], [], [], {} - - async def create_vm(self, **size): - self.sizes.append(size) - self.vms.append(vm := FakeVm(self, self.root / str(len(self.vms)))) - return vm - - -@pytest.fixture -def signing(local_stores, monkeypatch): - """Stores a remote VM reaches: the object store signs URLs to its objects, and the registry isn't this machine's.""" - images = OciRegistryImageStore("registry.example") - monkeypatch.setattr(type(local_stores), "get_image_store_for", lambda self, entity_id: images) - monkeypatch.setattr(OciRegistryImageStore, "ensure_repository", lambda self, repository: None) - monkeypatch.setattr(LocalFilesystemObjectStore, "signed_get_url", lambda self, url, expires_in=3600: - f"https://signed.example/get?{urllib.parse.quote(url)}") - monkeypatch.setattr(LocalFilesystemObjectStore, "signed_put_url", lambda self, url, expires_in=3600: - f"https://signed.example/put?{urllib.parse.quote(url)}") - - -def test_a_put_on_a_vm_provider_builds_each_arch_in_a_vm_of_its_own_and_its_images_are_reused_by_key_here( - base, driver, docker, signing, tmp_path, monkeypatch): - vms = FakeVmProvider(docker, tmp_path / "vms") - for name in ("build_image", "push_local_image", "docker_login", "_emulated_platforms"): - monkeypatch.setattr(builder, name, lambda *args, **kwargs: pytest.fail("this machine's Docker was used")) - - told = [] - composed = Env.get(_put(base, driver, composing=Composing(vms, on_progress=told.append)).id).metadata["composed"][ - "container"] - - assert ("container/arm64: building emulated, which is slow, in a FakeVmProvider VM; add local to --sandbox to build " - "it natively with this machine's Docker, or pass --no-emulate to skip it") in told - assert {arch: (r["status"], r["builder"], r["emulated"]) for arch, r in composed.items()} == { - "amd64": ("ready", "FakeVmProvider", False), "arm64": ("unverified", "FakeVmProvider", True)} - assert composed["arm64"]["reason"] == "no link of the sandbox provider runs linux/arm64 natively to probe it" - assert len(vms.vms) == 2 and all(vm.terminated for vm in vms.vms) and vms.binfmt == ["arm64"] - assert sorted(vms.sizes, key=lambda size: size["timeout"]) == [ - {"exposed_ports": [], "timeout": 7200, **builder._VM}, {"exposed_ports": [], "timeout": 21600, **builder._VM}] - driver_files = {artifact.object_url for artifact in driver.files.get_file_artifacts().values()} - assert sorted(vms.fetched) == sorted([*driver_files, *driver_files]) - assert vms.uploaded and all(f"{packages.PREFIX}/files/" in url for url in vms.uploaded) - builds = {build["platform"]: build for build in docker.builds} - assert builds["linux/amd64"]["files"] == ["Dockerfile", "agentenv-desk.conf", "driver/driver.py", - "driver/requirements.txt", "packages/wheels/driver/mss-10.1.0-py3-none-any.whl"] - assert composed["amd64"]["installed"] == {"mss": "10.1.0"} - assert docker.labels[f"agentenv-computer:{composed['arm64']['key'][7:23]}-arm64"]["dev.agentenv.emulated"] == "true" - - again = Env.get(_put(base, driver, id="@local/~/kit/other", environment_name="other").id) - - assert {arch: (r["reused"], r["key"]) for arch, r in again.metadata["composed"]["container"].items()} == { - arch: (True, r["key"]) for arch, r in composed.items()} - - -def test_a_chain_builds_each_arch_on_the_first_link_that_runs_it_natively(base, driver, docker, signing, tmp_path): - vms = FakeVmProvider(docker, tmp_path / "vms") - - composed = Env.get(_put(base, driver, composing=Composing(ChainedSandboxProvider( - [vms, LocalSandboxProvider()]))).id).metadata["composed"]["container"] - - assert {arch: (r["builder"], r["emulated"]) for arch, r in composed.items()} == { - "amd64": ("FakeVmProvider", False), "arm64": ("LocalSandboxProvider", False)} - assert len(vms.vms) == 1 and vms.binfmt == [] - - -def test_a_builder_vm_its_provider_loses_fails_its_pair_alone_and_is_torn_down(base, driver, docker, signing, tmp_path, - monkeypatch): - vms, real = FakeVmProvider(docker, tmp_path / "vms"), FakeVm._docker - - class Gone(Exception): - pass - - def gone(vm, words): - if words[0] == "timeout" and "linux/arm64" in words: - raise Gone("the sandbox is shutting down") - return real(vm, words) - monkeypatch.setattr(FakeVm, "_docker", gone) - - composed = Env.get(_put(base, driver, composing=Composing(vms)).id).metadata["composed"]["container"] - - assert (composed["arm64"]["status"], composed["arm64"]["stage"], composed["arm64"]["reason"]) == ( - "failed", "resolve", "RuntimeError: FakeVmProvider's VM failed: Gone: the sandbox is shutting down") - assert composed["amd64"]["status"] == "ready" and all(vm.terminated for vm in vms.vms) - - -def test_with_emulation_off_an_arch_a_refused_link_runs_natively_fails_and_only_one_needing_emulation_is_skipped( - base, driver, docker, tmp_path): - vms = FakeVmProvider(docker, tmp_path / "vms") - - composed = Env.get(_put(base, driver, composing=Composing(vms, emulate=False)).id).metadata["composed"]["container"] - - assert (composed["amd64"]["status"], composed["amd64"]["reason"]) == ( - "failed", "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local " - "provider") - assert (composed["arm64"]["status"], composed["arm64"]["reason"]) == ( - "skipped", "emulation is off, and FakeVmProvider doesn't run linux/arm64 natively") - assert vms.vms == [] - - -class Containers(SandboxProvider): - async def create_sandbox(self, **kwargs): - raise AssertionError("a container was created") - - -@pytest.mark.parametrize("links, reason", [ - (["vms"], "FakeVmProvider's VMs can't fetch from the object store, which signs no URLs; build on the local provider"), - (["containers"], "Containers gives no VM to build in"), - (["containers", "local"], "Containers gives no VM to build in; this machine can't build linux/amd64: its Docker has " - "no emulation for it"), -]) -def test_an_arch_no_link_can_build_fails_saying_why_for_each_link(base, driver, docker, tmp_path, links, reason): - docker.emulates = set() - made = {"vms": FakeVmProvider(docker, tmp_path / "vms"), "containers": Containers(), - "local": LocalSandboxProvider()} - - composed = Env.get(_put(base, driver, composing=Composing(ChainedSandboxProvider( - [made[link] for link in links]))).id).metadata["composed"]["container"] - - assert (composed["amd64"]["status"], composed["amd64"]["stage"], composed["amd64"]["reason"]) == ( - "failed", "resolve", reason) - assert made["vms"].vms == [] - - def test_env_build_writes_the_computer_again_when_it_isnt_over_every_pair_built(base, driver, docker, caplog): caplog.set_level(logging.ERROR, logger=composer.__name__) stored = Env.get(_put(base, driver).id) @@ -926,22 +436,6 @@ def test_env_build_writes_the_computer_again_when_it_isnt_over_every_pair_built( assert Env.get(stored.id).metadata["computer"]["version"] == stale["version"] + 1 -def test_a_put_that_wont_emulate_skips_an_arch_no_link_runs_natively_and_a_later_build_that_will_builds_it( - base, driver, docker, caplog): - caplog.set_level(logging.ERROR, logger=composer.__name__) - env = _put(base, driver, composing=Composing(emulate=False)) - skipped = CliRunner().invoke(cli, ["env", "build", "--id", env.id, "--no-emulate"]) - built = CliRunner().invoke(cli, ["env", "build", "--id", env.id]) - - reason = "emulation is off, and LocalSandboxProvider doesn't run linux/amd64 natively" - assert skipped.exit_code == 0 and f"container/amd64: skipped ({reason})" in skipped.output - assert [build["platform"] for build in docker.builds] == ["linux/arm64", "linux/amd64"] - assert built.exit_code == 0 and "container/amd64: building emulated, which is slow, with this machine's Docker; " \ - "pass --no-emulate to skip it" in built.output - composed = Env.get(env.id).metadata["composed"]["container"] - assert {arch: r["status"] for arch, r in composed.items()} == {"amd64": "unverified", "arm64": "ready"} - - def test_env_build_by_an_installed_alias_composes_under_the_id_of_the_env_it_names(base, driver, docker, local_stores, tmp_path, monkeypatch, caplog): caplog.set_level(logging.ERROR, logger=composer.__name__)