feat(computer): a computer env composes the apps it lists into its image

src/agent_env/bundle/resolve.py

@@ -201,7 +201,8 @@ def _toml(self, entry: BundleEntry) -> ResolvedEntry:
                 refs.append(ref)
         for site in ref_sites(refs, config, inline_pins=True):
             try:
-                name, version = parse_toml_ref(site.ref.kind, site.owner if site.version_key else site.value)
+                name, version = parse_toml_ref(site.ref.kind, site.owner if site.version_key else site.value,
+                                               site.ref.beside)
             except ValueError as e:
                 self._problem(entry, f"{site.path}: {e}")
                 continue

src/agent_env/entity/refs.py

@@ -53,8 +53,10 @@ class RefRole(StrEnum):
 class EntityRef:
     """One ref-holding field. ``path`` is dotted keys into the step dict or toml, ``[]`` meaning each
     element of a list (``env_ids[]``, ``artifacts[].id``); ``version_field`` is the key beside the
-    ref, in the same dict, that pins its version. ``walk``, for a shape a path cannot spell, maps
-    the value at the path's parent to the dicts that hold the last key, each with its path parts.
+    ref, in the same dict, that pins its version. ``beside`` names the keys a toml table holding the
+    ref may carry next to it, which its type reads (a computer env's app takes ``install``). ``walk``,
+    for a shape a path cannot spell, maps the value at the path's parent to the dicts that hold the
+    last key, each with its path parts.
     Declarations build one with ``EntityRef.env``, ``EntityRef.agent`` or ``EntityRef.artifact``,
     passing the keyword fields."""
 
@@ -65,6 +67,7 @@ class EntityRef:
     role: RefRole = RefRole.INPUT
     artifact_type: str | None = None
     env_type: str | None = None
+    beside: tuple[str, ...] = ()
     walk: Walker | None = None
 
     @classmethod
@@ -134,12 +137,13 @@ def default_agent_refs(refs: Iterable[EntityRef], step: dict) -> list[EntityRef]
             and step.get(ref.field) is None]
 
 
-def parse_toml_ref(kind: EntityKind, value: Any) -> tuple[str, int | None]:
+def parse_toml_ref(kind: EntityKind, value: Any, beside: tuple[str, ...] = ()) -> tuple[str, int | None]:
     """The id and version a toml ref names: an id, or ``{ <kind> = "<id>", version = <n> }`` with the
-    version optional. Raises ValueError naming the expected form."""
+    version optional, and the keys of ``beside`` allowed in the table too. Raises ValueError naming the
+    expected form."""
     table = isinstance(value, dict)
     entity_id, version = (value.get(kind.value), value.get("version")) if table else (value, None)
-    known_keys = not table or value.keys() <= {kind.value, "version"}
+    known_keys = not table or value.keys() <= {kind.value, "version", *beside}
     positive = version is None or (isinstance(version, int) and not isinstance(version, bool) and version >= 1)
     if not (known_keys and isinstance(entity_id, str) and entity_id and positive):
         raise ValueError(f'expected an id or {{ {kind.value} = "<id>", version = <n> }}, the version optional, '

src/agent_env/env/envs/computer/app_env.py

@@ -17,7 +17,7 @@
 from agent_env.env.env import Env
 from agent_env.env.envs.computer.utils import composer, gate
 from agent_env.env.envs.computer.utils.composer import Composing
-from agent_env.env.envs.computer.env import ComputerEnv
+from agent_env.env.envs.computer.env import APP_TYPE, ComputerEnv
 from agent_env.env.envs.computer.utils.install import AppInstall, folder_problems, installs
 from agent_env.env.envs.computer.driver_env import ComputerDriverEnv, declared
 from agent_env.env.store import get_env_store
@@ -30,7 +30,7 @@
 
 
 class ComputerAppEnv(ComputerEnv):
-    type: ClassVar[str] = "computer_app"
+    type: ClassVar[str] = APP_TYPE
     description = ("A computer app: software and its settings on a computer, read from its ComputerAppEnvironment "
                    "class at put; deployed alone, a computer of its own")
 

src/agent_env/env/envs/computer/env.py

@@ -32,27 +32,32 @@
 if TYPE_CHECKING:
     from agent_env.bundle.authoring import AuthoringContext
     from agent_env.env.env import DeployedEnv
+    from agent_env.env.envs.computer.app_env import ComputerAppEnv
     from agent_env.providers.env_providers.env_provider import EnvironmentProvider
 
 logger = logging.getLogger(__name__)
 
 #: Where load_artifact puts files in a computer by default: one of the folders data/get returns by default.
 DEFAULT_DESTINATION = "Downloads"
+#: The computer app env's type: an app composed into a computer env, and a computer env of its own.
+APP_TYPE = "computer_app"
 
 
 class ComputerEnv(Env):
     type: ClassVar[str] = "computer"
     description = "A computer: a base computer and a driver, composed at put into an image for each form and arch"
     env_provider_type: ClassVar[str] = "computer"
     toml_refs: ClassVar[tuple[EntityRef, ...]] = (EntityRef.artifact("computer", artifact_type="computer"),
-                                                  EntityRef.env("driver", env_type=ComputerDriverEnv.type))
+                                                  EntityRef.env("driver", env_type=ComputerDriverEnv.type),
+                                                  EntityRef.env("apps[]", env_type=APP_TYPE, beside=("install",)))
 
     def __init__(self, id: str, version: Optional[int], environment_name: str, computer: ComputerArtifact,
                  driver: ComputerDriverEnv, display: Optional[dict] = None, network: Optional[dict] = None,
-                 computer_settings: Optional[dict[str, SettingValue]] = None, metadata: Optional[dict[str, Any]] = None):
+                 computer_settings: Optional[dict[str, SettingValue]] = None, metadata: Optional[dict[str, Any]] = None,
+                 apps: Optional[list[ComputerAppEnv]] = None):
         super().__init__(id, version, metadata=metadata)
         self.environment_name = environment_name
-        self.computer, self.driver = computer, driver
+        self.computer, self.driver, self.apps = computer, driver, apps or []
         self.display, self.network = display, network
         self.computer_settings = computer_settings or {}
         self._sandbox = None
@@ -66,7 +71,8 @@ def to_dict(self) -> dict:
                 "environment_name": self.environment_name,  # dual-write with service_name during the rename
                 "computer": {"id": self.computer.id, "version": self.computer.version},
                 "driver": {"id": self.driver.id, "version": self.driver.version}, "display": self.display,
-                "network": self.network, "computer_settings": self.computer_settings}
+                "network": self.network, "computer_settings": self.computer_settings,
+                **({"apps": [{"id": app.id, "version": app.version} for app in self.apps]} if self.apps else {})}
 
     @classmethod
     def from_dict(cls, data: dict) -> ComputerEnv:
@@ -77,29 +83,33 @@ def from_dict(cls, data: dict) -> ComputerEnv:
                    environment_name=data["environment_name"] if "environment_name" in data else data["service_name"],
                    computer=ComputerArtifact.get(data["computer"]["id"], data["computer"]["version"]), driver=driver,
                    display=data.get("display"), network=data.get("network"),
-                   computer_settings=data.get("computer_settings"), metadata=data.get("metadata") or {})
+                   computer_settings=data.get("computer_settings"), metadata=data.get("metadata") or {},
+                   apps=[_app(Env.get(ref["id"], ref["version"])) for ref in data.get("apps") or []])
 
     @classmethod
     def put(cls, *, id: str, computer: ComputerArtifact, driver: ComputerDriverEnv, environment_name: str,
             apps: Any = (), display: Any = None, network: Any = None, computer_settings: Any = None,
             compose: bool = True, composing: Composing = Composing()) -> Self:
-        """Write a computer env of ``computer`` and ``driver`` once gate 1 passes, then compose it as ``composing``
-        says unless ``compose`` is False. Refused, naming every mismatch, before anything is stored."""
-        values, problems = gate.check(computer, driver, apps=list(apps), display=display, network=network,
+        """Write a computer env of ``computer``, ``driver`` and ``apps``, computer app envs built into it in their
+        order, once gate 1 passes, then compose it as ``composing`` says unless ``compose`` is False. Refused, naming
+        every mismatch, before anything is stored."""
+        apps = list(apps)
+        values, problems = gate.check(computer, driver, apps=apps, display=display, network=network,
                                       computer_settings=computer_settings or {})
         if problems:
             raise ValueError(f"computer env {id!r}: " + "; ".join(problems))
         env = get_env_store().put_document(cls(
-            id=id, version=None, environment_name=environment_name,
-            computer=computer, driver=driver, display=display, network=network, computer_settings=values))
+            id=id, version=None, environment_name=environment_name, computer=computer, driver=driver, apps=apps,
+            display=display, network=network, computer_settings=values))
         if compose:
             composer.compose(env, composing=composing)
         return env
 
     @classmethod
     def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv:
-        """Write the computer env an env.toml declares, its base and driver as the bundle wrote them or the ones they
-        name, its environment_name the folder's when it sets none."""
+        """Write the computer env an env.toml declares, its base, driver and apps as the bundle wrote them or the ones
+        they name, its environment_name the folder's when it sets none. An app's ``install`` is ``"build"``, the
+        default: it's built into the computer."""
         fields, refused = ctx.accepted(data, environment_name=str, computer=object, driver=object, apps=list,
                                        display=dict, network=dict, computer_settings=dict)
         missing = [ctx.config_problem(f"{key} is required: the {what}, such as {example}") for key, what, example in (
@@ -108,16 +118,17 @@ def from_toml(cls, data: dict, ctx: AuthoringContext) -> ComputerEnv:
             ctx.refuse([*refused, *missing])
         base, driver = ctx.artifact(fields.pop("computer"), ComputerArtifact), ctx.env(fields.pop("driver"),
                                                                                        ComputerDriverEnv)
+        fields["apps"] = _listed_apps(fields.get("apps", []), ctx)
         try:
             return cls.put(id=ctx.id, computer=base, driver=driver,
                            environment_name=fields.pop("environment_name", ctx.name), composing=ctx.composing,
                            **fields)
         except ValueError as e:
             ctx.refuse([str(e)])
 
-    def composed_apps(self) -> list:
+    def composed_apps(self) -> list[ComputerAppEnv]:
         """The computer app envs composed into the env's computer, each installed under its prefix."""
-        return []
+        return self.apps
 
     def computer_artifact(self) -> ComputerArtifact:
         """The env's computer: its base composed with its driver, over the images built."""
@@ -217,3 +228,32 @@ async def close(self) -> None:
             except Exception as e:  # a cancellation propagates, and keeps the sandbox for the next close()
                 logger.warning(f"Failed to terminate sandbox {self._sandbox.sandbox_id}: {e}")
             self._sandbox = None
+
+
+def _app(env: Env) -> ComputerAppEnv:
+    if refusal := _not_an_app(env):
+        raise TypeError(refusal)
+    return env
+
+
+def _listed_apps(refs: list, ctx: AuthoringContext) -> list[ComputerAppEnv]:
+    """The computer app envs an env.toml's ``apps`` names, each a ref, or a table of one with its ``install`` beside it;
+    refused, naming each problem, unless every one is a computer app env installed at build."""
+    apps, problems = [], []
+    for i, ref in enumerate(refs):
+        install = ref.get("install", "build") if isinstance(ref, dict) else "build"
+        if install != "build":
+            problems.append(ctx.config_problem(f"apps[{i}]: install = {install!r}: an app installs at \"build\", into "
+                                               "the computer's image; \"boot\" isn't supported yet"))
+            continue
+        app = ctx.env({key: value for key, value in ref.items() if key != "install"} if isinstance(ref, dict) else ref)
+        if refusal := _not_an_app(app):
+            problems.append(ctx.config_problem(f"apps[{i}]: {refusal}"))
+        apps.append(app)
+    if problems:
+        ctx.refuse(problems)
+    return apps
+
+
+def _not_an_app(env: Env) -> str | None:
+    return f"{env.id} is a {env.type} env, not a {APP_TYPE} env" if env.type != APP_TYPE else None

src/agent_env/env/envs/computer/utils/builder.py

@@ -27,6 +27,7 @@
 
 from agent_env.artifact.artifacts.docker_image import docker_login, push_local_image
 from agent_env.config import get_config
+from agent_env.env.envs.computer.utils import layers_in_image
 from agent_env.providers.sandbox_providers.host_images import build_for, host_platform, platform_tag
 from agent_env.providers.sandbox_providers.sandbox import upload_vm_file
 from agent_env.providers.sandbox_providers.sandbox_provider import SANDBOX_MODE_VM
@@ -42,12 +43,9 @@
 
 logger = logging.getLogger(__name__)
 
-#: What the builders make of a build context, which the build key covers in place of their source: raise it with a
-#: change that alters what an image holds (its files, their modes, its labels, how it's built), and leave it for one
-#: that changes only how a build gets there, so images aren't built again for nothing.
-FORMAT = 1
 CONTEXT, RESOLVED = "context", "resolve"
 RESOLVER = Path(__file__).with_name("resolve_in_base.py")
+LAYERS = Path(__file__).with_name("layers_in_image.py")
 #: The shell side of a builder VM, its subcommands run there (``VmBuilder``).
 IN_VM = Path(__file__).with_name("builder_in_vm.sh")
 #: What a file starts with when it's only ever run: a script or a Linux binary, made executable where it's staged,
@@ -136,6 +134,11 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]:
         """The packages ``tag``'s build installed into the venv of the driver at ``prefix``, by name, read from the
         image without running it, so on any arch; none for a driver with no venv."""
 
+    @abstractmethod
+    def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]:
+        """What each of ``commands``, a RUN of ``tag``'s build, wrote in its layer of ``tag``, read from a save of the
+        image (``layers_in_image``)."""
+
     @abstractmethod
     def push(self, tag: str, ref: str, store) -> None:
         """Push ``tag`` to ``ref`` in the image store ``store``, signed in with its credentials."""
@@ -323,6 +326,14 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]:
         with tarfile.open(fileobj=io.BytesIO(copied.stdout)) as archive:
             return _packages(archive.extractfile(archive.getmembers()[0]).read())
 
+    def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]:
+        saved = self.work / "image.tar"
+        save_image(tag, saved)
+        try:
+            return layers_in_image.written(str(saved), commands)
+        finally:
+            saved.unlink(missing_ok=True)
+
     def push(self, tag: str, ref: str, store) -> None:
         push_local_image(tag, ref, store)
 
@@ -425,6 +436,15 @@ def installed(self, tag: str, prefix: str) -> dict[str, str]:
         listed = self._exec(f"{self.helper} installed {shlex.quote(tag)} {shlex.quote(f'{prefix}/installed.json')}")
         return _packages(listed.encode()) if listed.strip() else {}
 
+    def written(self, tag: str, commands: list[str]) -> list[dict[str, str]]:
+        """Read in a container of ``tag``, on its own Python, so the VM needs none."""
+        image, script = f"{self.work}/image.tar", f"{self.work}/{LAYERS.name}"
+        self._write(script, LAYERS.read_bytes())
+        return json.loads(self._exec(
+            f"docker save -o {image} {shlex.quote(tag)} && docker run --rm --network none --user root "
+            f"-v {image}:/image.tar:ro -v {script}:/layers.py:ro --entrypoint python3 {shlex.quote(tag)} /layers.py "
+            f"/image.tar {shlex.quote(json.dumps(commands))} && rm -f {image}"))
+
     def push(self, tag: str, ref: str, store) -> None:
         self._login(store.auth(ref))
         self._exec(f"docker tag {shlex.quote(tag)} {shlex.quote(ref)} && docker push {shlex.quote(ref)}")
@@ -516,6 +536,10 @@ def _packages(listed: bytes) -> dict[str, str]:
     return {package["name"]: package["version"] for package in json.loads(listed)}
 
 
+def save_image(tag: str, path: Path) -> None:
+    subprocess.run(["docker", "save", "-o", str(path), tag], check=True, capture_output=True, text=True)
+
+
 def sha256_of(path: Path) -> str:
     hashed = hashlib.sha256()
     with path.open("rb") as file: