From 47c2da8b843c874f8ade925e333fe2b656ef2b7d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:36:27 -0800 Subject: [PATCH 01/13] feat(bundle): run a bundle from the Hugging Face Hub with agent-env run hf:// Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/agent_env/bundle/hub.py | 236 ++++++++++++++ src/agent_env/bundle/importer.py | 8 +- src/agent_env/bundle/installed.py | 51 ++- src/agent_env/bundle/ledger.py | 47 +-- src/agent_env/bundle/parse.py | 19 +- src/agent_env/bundle/plan.py | 12 +- src/agent_env/bundle/register.py | 2 +- src/agent_env/bundle/resolve.py | 10 +- src/agent_env/bundle/run.py | 12 +- src/agent_env/cli/_plugin_changes.py | 2 +- src/agent_env/cli/run.py | 37 ++- src/agent_env/config/runtime.py | 54 ++-- src/agent_env/openenv/app.py | 2 +- .../store/document_store/document_store.py | 13 +- src/agent_env/store/ids.py | 32 +- src/agent_env/store/routing.py | 195 +++++++----- tst/unit/bundle/hub_test.py | 296 ++++++++++++++++++ tst/unit/bundle/installed_test.py | 22 +- tst/unit/bundle/parse_test.py | 12 + tst/unit/bundle/resolve_test.py | 10 + tst/unit/explorer/explorer_app_test.py | 6 +- tst/unit/store/ids_test.py | 16 + tst/unit/store/local_id_acceptance_test.py | 11 +- tst/unit/store/routing_test.py | 41 ++- 25 files changed, 958 insertions(+), 189 deletions(-) create mode 100644 src/agent_env/bundle/hub.py create mode 100644 tst/unit/bundle/hub_test.py diff --git a/pyproject.toml b/pyproject.toml index fc88efdd8..2a841acc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ "click>=8.0.0", "cryptography>=42", # the local transfer CA "httpx>=0.27.0", + "huggingface_hub>=1.0", # agent-env run hf:// resolves and downloads a Hub bundle; 1.0 moved it onto httpx "jsonschema>=4.20", # checks an app's settings against its Settings' schema before a deploy "litellm>=1.96.0,<1.97", "mcp>=1.25,<2", # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py new file mode 100644 index 000000000..243e3265a --- /dev/null +++ b/src/agent_env/bundle/hub.py @@ -0,0 +1,236 @@ +"""Bundles run straight from a Hugging Face Hub dataset repo, named ``hf:///[@][/]``. + +The revision, a branch, tag or commit (the main branch by default), resolves to a commit, and the bundle is picked from +that commit's file list before anything is downloaded, so a results dataset's recordings never are. Only the bundle's +files and the repo's card are downloaded, into a plain folder for the commit: the Hub's own cache keeps files as links +into its blob store, and the bundle parser refuses a link that leaves the bundle. A commit never changes, so what was +downloaded for it is reused; a reference pinned to the commit then needs no network at all. The bundle's ids are rooted +at ``@hf//``, plus ``/`` for a bundle under ``bundles/``, a local namespace of its own: the id says the +bundle came from the Hub, and a rerun of the same commit reuses what the last one wrote. A gated or private repo is read +with the Hub library's own login (``hf auth login`` or ``HF_TOKEN``). +""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import unquote + +import httpx +from huggingface_hub import HfApi +from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError +from huggingface_hub.repocard import metadata_load + +from agent_env.plugins._bundles import ALIASES_FILE +from agent_env.plugins._requirements import unmet_problems +from agent_env.store.ids import HF_PREFIX, validate_local_id + +from .parse import BUNDLE_TOML, BundleError, BundleKind +from .requires import UnmetRequirements + +SCHEME = "hf://" +BUNDLES = "bundles" +CARD = "README.md" +# The key a run's context records where its bundle came from under, in its metadata. +SOURCE_KEY = "bundle_source" + +_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +_SHA = re.compile(r"[0-9a-f]{40}") +_KINDS = frozenset(kind.value for kind in BundleKind) +_FORM = "hf:///[@][/]" + + +@dataclass(frozen=True) +class HubReference: + repo: str # / + revision: str | None # None for the main branch + path: str | None # the bundle's folder in the repo, when named + + +@dataclass(frozen=True) +class HubBundle: + """A bundle downloaded from the Hub: its folder, the id root it runs under, where it came from, and what the repo's + card says it needs.""" + + root: Path + id_root: str + source: Mapping[str, str] # hub, repo, revision, sha, path: what each run's context records + plugins: tuple[str, ...] = () # the card's, for a bundle without a bundle.toml + setup: str | None = None # the card's command that sets the bundle up, shown when a run fails, never run + + def check_plugins(self) -> None: + """Refuse the run when a plugin the card names isn't installed, naming the command that installs it. Raises + UnmetRequirements.""" + problems, unmet = unmet_problems(self.plugins, f"{self.source['repo']}'s card") + if problems: + raise UnmetRequirements(problems, unmet) if unmet else BundleError(problems) + + +@dataclass +class _Commit: + """What this machine holds of a commit, kept in its folder: the commit's file list, and the bundles downloaded.""" + + files: list[str] + downloaded: list[str] = field(default_factory=list) + + @staticmethod + def path(folder: Path) -> Path: + return folder / ".cache" / "agentenv" / "commit.json" + + @classmethod + def read(cls, folder: Path) -> _Commit | None: + try: + return cls(**json.loads(cls.path(folder).read_text())) + except FileNotFoundError: + return None + + def save(self, folder: Path) -> None: + path = self.path(folder) + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_name(f"{path.name}.{os.getpid()}") + partial.write_text(json.dumps({"files": self.files, "downloaded": self.downloaded})) + os.replace(partial, path) + + +def is_hub_reference(text: str) -> bool: + return text.startswith(SCHEME) + + +def parse_reference(text: str) -> HubReference: + """``text``, ``hf:///[@][/]``, as its parts. The revision runs to the next ``/``, so one + holding a ``/`` is written percent-encoded, as the Hub's own ``hf://`` paths take it (``@refs%2Fpr%2F3``). Raises + BundleError.""" + org, _, rest = text.removeprefix(SCHEME).partition("/") + name, _, path = rest.partition("/") + name, at, revision = name.partition("@") + parts = [part for part in path.split("/") if part] + if not (_NAME.fullmatch(org) and _NAME.fullmatch(name)) or (at and not revision): + raise BundleError([f"{text}: name a Hub dataset's bundle as {_FORM}"]) + if any(part in (".", "..") for part in parts): + raise BundleError([f"{text}: the path in the repo can't hold '.' or '..'"]) + return HubReference(f"{org}/{name}", unquote(revision) or None, "/".join(parts) or None) + + +def cache_root() -> Path: + """Where downloaded commits are kept: ``$XDG_CACHE_HOME/agentenv/hf`` when that is an absolute path, else + ``~/.cache/agentenv/hf``.""" + base = os.environ.get("XDG_CACHE_HOME", "") + return (Path(base) if os.path.isabs(base) else Path.home() / ".cache") / "agentenv" / "hf" + + +def fetch_bundle(text: str, *, api: HfApi | None = None, on_progress: Callable[[str], None] | None = None) -> HubBundle: + """The bundle ``text`` names, downloaded unless this machine has it. ``on_progress`` is given the line that says + which repo, commit and bundle it is, before anything is downloaded. Raises BundleError.""" + reference = parse_reference(text) + api = api or HfApi() + try: + return _fetch(reference, api, on_progress or (lambda line: None)) + except (HfHubHTTPError, httpx.TransportError) as e: + error = BundleError([f"{text}: {str(e).strip() or type(e).__name__}"]) + if isinstance(e, RepositoryNotFoundError): + error.add_note("a private or gated repo needs a token with access: log in with `hf auth login`, or set " + "HF_TOKEN") + raise error from e + + +def describe(source: Mapping[str, str]) -> str: + """Where a bundle came from, as a run prints it: ``/ at , bundle ``.""" + return f"{source['repo']} at {source['sha'][:12]}, bundle {source['path'] or 'at its root'}" + + +def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> HubBundle: + sha = reference.revision if reference.revision and _SHA.fullmatch(reference.revision) else api.repo_info( + reference.repo, repo_type="dataset", revision=reference.revision).sha + folder = cache_root() / reference.repo / sha + commit = _Commit.read(folder) + if commit is None: + commit = _Commit(api.list_repo_files(reference.repo, repo_type="dataset", revision=sha)) + if CARD in commit.files: + api.hf_hub_download(reference.repo, CARD, repo_type="dataset", revision=sha, local_dir=folder) + commit.save(folder) + card = _card(folder) + path = _pick(reference, sha, commit.files, card) + source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, "path": path} + say(describe(source)) + if path not in commit.downloaded: + api.snapshot_download(reference.repo, repo_type="dataset", revision=sha, local_dir=folder, + allow_patterns=_patterns(path)) + commit.downloaded.append(path) + commit.save(folder) + name = path.rpartition("/")[2] or reference.repo.partition("/")[2] + needs = (card.get("bundles") or {}).get(name) or {} + declared = _within(path, BUNDLE_TOML) in commit.files + return HubBundle(folder / path, _id_root(reference, path), source, + plugins=() if declared else tuple(needs.get("plugins") or ()), setup=needs.get("setup")) + + +def _card(folder: Path) -> dict[str, Any]: + """The ``agentenv`` table of the card's metadata, ``{}`` without one.""" + readme = folder / CARD + table = (metadata_load(readme) or {}).get("agentenv") if readme.is_file() else None + return table if isinstance(table, dict) else {} + + +def _pick(reference: HubReference, sha: str, files: Sequence[str], card: Mapping[str, Any]) -> str: + """The folder in the repo of the bundle ``reference`` names, ``""`` for the root: its path; else the only bundle + under ``bundles/``, or the one the card names its default; else the root when it is a bundle. ``bundles/`` comes + first, since a dataset can keep a ``tasks/`` of data files at its root.""" + where = f"{reference.repo} at {sha[:12]}" + if reference.path is not None: + if not _is_bundle(files, reference.path): + raise BundleError([f"{where}: {reference.path}/ isn't a bundle: it has none of {_kinds()}"]) + return reference.path + names = sorted({parts[1] for file in files if len(parts := file.split("/")) > 2 and parts[0] == BUNDLES}) + if len(names) == 1: + return _within(BUNDLES, names[0]) + if card.get("default") in names: + return _within(BUNDLES, card["default"]) + if names: + revision = f"@{reference.revision}" if reference.revision else "" + runs = ", ".join(f"{SCHEME}{reference.repo}{revision}/{BUNDLES}/{name}" for name in names) + raise BundleError([f"{where} holds several bundles under {BUNDLES}/ and its card names no default " + f"(agentenv: default); run one of {runs}"]) + if _is_bundle(files, ""): + return "" + raise BundleError([f"{where} holds no bundle: it has no {BUNDLES}// folder, and none of {_kinds()} at its " + "root"]) + + +def _is_bundle(files: Sequence[str], path: str) -> bool: + """Whether the folder at ``path`` in the repo holds a kind folder, as a bundle does.""" + prefix = _within(path, "") + return any(file.startswith(prefix) and file[len(prefix):].split("/", 1)[0] in _KINDS + and "/" in file[len(prefix):] for file in files) + + +def _patterns(path: str) -> list[str]: + """What to download of the bundle at ``path``: its folder, or for the repo's root its kind folders and files.""" + if path: + return [f"{path}/*"] + return [*(f"{kind}/*" for kind in sorted(_KINDS)), BUNDLE_TOML, ALIASES_FILE] + + +def _id_root(reference: HubReference, path: str) -> str: + """``@hf//``, then the bundle's name for one under ``bundles/``, or its path for one elsewhere.""" + parts = path.split("/") if path else [] + if len(parts) == 2 and parts[0] == BUNDLES: + parts = parts[1:] + id_root = "/".join([f"{HF_PREFIX}{reference.repo}", *parts]) + try: + validate_local_id(id_root) + except ValueError as e: + raise BundleError([f"{SCHEME}{reference.repo}/{path}: can't be an id root: {e}"]) from None + return id_root + + +def _within(folder: str, name: str) -> str: + return f"{folder}/{name}" if folder else name + + +def _kinds() -> str: + return ", ".join(f"{kind.value}/" for kind in BundleKind) diff --git a/src/agent_env/bundle/importer.py b/src/agent_env/bundle/importer.py index 731c9c187..712941c95 100644 --- a/src/agent_env/bundle/importer.py +++ b/src/agent_env/bundle/importer.py @@ -62,7 +62,7 @@ from agent_env.eval import Eval from agent_env.plugins import ARTIFACTS, ENVS, TASK_STEPS from agent_env.store.base import NotFoundError -from agent_env.store.ids import LOCAL_PREFIX +from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.store.image_store.registry_api import pin_digest from agent_env.task.task import Task @@ -608,15 +608,15 @@ def _fetch(url: str, dest: Path) -> None: def _output_names(sites: list[RefSite]) -> dict[tuple[EntityKind, str], str]: - """The name each output takes: as written, or an ``@local`` id's last segment, numbered when it is taken.""" + """The name each output takes: as written, or a local id's last segment, numbered when it is taken.""" written = {_key(site) for site in sites if site.ref.role is RefRole.OUTPUT} - taken = {_key(site) for site in sites if _key(site) not in written or not site.value.startswith(LOCAL_PREFIX)} + taken = {_key(site) for site in sites if _key(site) not in written or not is_local_id(site.value)} names: dict[tuple[EntityKind, str], str] = {} for site in sites: if site.ref.role is not RefRole.OUTPUT or _key(site) in names: continue name = site.value - if name.startswith(LOCAL_PREFIX): + if is_local_id(name): base = name = name.rsplit("/", 1)[-1] n = 1 while (site.ref.kind, _nfc(name)) in taken: diff --git a/src/agent_env/bundle/installed.py b/src/agent_env/bundle/installed.py index b6d723a0d..b158373a2 100644 --- a/src/agent_env/bundle/installed.py +++ b/src/agent_env/bundle/installed.py @@ -7,32 +7,46 @@ import difflib import os +import unicodedata from collections.abc import Callable from pathlib import Path +from typing import NamedTuple from packaging.utils import canonicalize_name from agent_env.plugins._bundles import BUNDLES, CORE, InstalledBundle, installed_bundles +from .hub import HubBundle, fetch_bundle, is_hub_reference from .materialize import materialize from .parse import Bundle, BundleError, BundleKind, parse_bundle from .plan import check_bundle, plan_entities from .resolve import resolve_bundle -__all__ = ["BUNDLES", "CORE", "InstalledBundle", "checked", "find_bundle", "installed_bundles", "run_name", - "write_installed"] +__all__ = ["BUNDLES", "CORE", "InstalledBundle", "Located", "checked", "find_bundle", "installed_bundles", + "locate_bundle", "run_name", "write_installed"] + + +class Located(NamedTuple): + root: Path + id_root: str | None # an installed or Hub bundle's; a folder's ids come from its path + hub: HubBundle | None # what a Hub bundle was downloaded as def locate_bundle( - text: str, installed: Callable[[], tuple[InstalledBundle, ...]] = installed_bundles, -) -> tuple[Path, str | None]: - """The folder ``text`` names, and the id root of an installed bundle's, as ``agent-env run BUNDLE`` reads it: an - existing folder wins, and text that starts like a path is always one; anything else names a bundle ``installed`` - lists. Raises BundleError.""" - if path_like(text) or os.path.isdir(text): - return Path(text), None + text: str, installed: Callable[[], tuple[InstalledBundle, ...]] = installed_bundles, *, + on_progress: Callable[[str], None] | None = None, +) -> Located: + """The folder ``text`` names, as ``agent-env run BUNDLE`` reads it: ``hf://`` names a bundle in a Hugging Face Hub + dataset, downloaded first (``on_progress`` is told which); an existing folder spelled exactly as written wins, and + text that starts like a path is always one; anything else names a bundle ``installed`` lists. Raises + BundleError.""" + if is_hub_reference(text): + hub = fetch_bundle(text, on_progress=on_progress) + return Located(hub.root, hub.id_root, hub) + if path_like(text) or _spelled_folder(text): + return Located(Path(text), None, None) found = find_bundle(text, installed()) - return found.root, found.id_root + return Located(found.root, found.id_root, None) def path_like(text: str) -> bool: @@ -40,6 +54,23 @@ def path_like(text: str) -> bool: return text.startswith((".", "/", "~")) +def _spelled_folder(text: str) -> bool: + """Whether ``text`` names a folder each of whose names is spelled as written: on a case-insensitive filesystem + ``desktop`` finds ``Desktop``, which isn't the folder meant.""" + if not os.path.isdir(text): + return False + above = Path() + for part in Path(text).parts: + if part not in (".", "..") and _nfc(part) not in {_nfc(name) for name in os.listdir(above)}: + return False + above /= part + return True + + +def _nfc(name: str) -> str: + return unicodedata.normalize("NFC", name) + + def find_bundle(text: str, bundles: tuple[InstalledBundle, ...] | None = None) -> InstalledBundle: """The installed bundle ``text`` names, as ```` or ``/``. Raises BundleError.""" bundles = installed_bundles() if bundles is None else bundles diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index a491a84e9..ed6c827ed 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -2,8 +2,8 @@ A row per written version records a digest of the write's inputs and one digest per input, so a new version can say why it was written ("files changed: server.py"). A row is keyed by the bundle the write comes from, so -an entity a run pulls in from an installed bundle has one row however many bundles use it. The ledger lives in the ``@local`` -namespace's own store, beside the entities it describes, and is never read through the router, so it +an entity a run pulls in from an installed bundle has one row however many bundles use it. A row lives in its entity's local +namespace's own store, ``@local``'s or ``@hf``'s, beside the entity it describes, and is never read through the router, so it never reaches a configured store. Reading it never creates the store. """ @@ -28,6 +28,7 @@ from agent_env.eval.store import EVALS_COLLECTION from agent_env.store import Filter, Sort from agent_env.store.document_store import DocumentStore, LocalSqliteDocumentStore, expecting_version +from agent_env.store.ids import local_namespace from agent_env.store.local_state import holding_locks from agent_env.task.store import TASKS_COLLECTION @@ -77,7 +78,8 @@ def next_version(self) -> int: class Ledger: - """One plan's rows in the ``@local`` namespace's store.""" + """One plan's rows, each in the store ``store_of`` gives for its entity's id: for a run, that id's local + namespace's.""" collection = LEDGER_COLLECTION # Whether a task or eval is written naming the versions of what it references, so is hashed with them. @@ -89,13 +91,14 @@ class Ledger: # store, rather than written as its build context for each deploy to build. pushes_images = False - def __init__(self, store: DocumentStore, plan: Plan): - self._store = store + def __init__(self, store_of: Callable[[str], DocumentStore], plan: Plan): + self._store_of = store_of self._plan = plan @classmethod def for_plan(cls, plan: Plan) -> Ledger: - return cls(get_config().local_namespace_document_store(), plan) + config = get_config() + return cls(lambda entity_id: config.local_namespace_document_store(local_namespace(entity_id)), plan) def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: """Compare ``write`` with the latest version the ledger records for it. ``needs`` holds, by (store, id), @@ -159,28 +162,29 @@ def record(self, check: Check, write: Callable[[], int]) -> int: A pending row marks the attempt until it's done, with its version once ``write`` returns. One an interrupted run of this bundle left is kept by ``check`` when its version can be, and dropped here.""" key = {"store": check.write.kind.store, "id": check.write.id, "bundle": self._bundle(check.write)} - self._store.ensure_index(self.collection, ["store", "id", "status"]) - while self._store.delete(self.collection, Filter.of(**key, status="pending")): + store = self._store_of(check.write.id) + store.ensure_index(self.collection, ["store", "id", "status"]) + while store.delete(self.collection, Filter.of(**key, status="pending")): pass pending = {**key, "status": "pending", "scheme": SCHEME, "digest": check.digest.value if check.digest else None, "inputs": check.digest.inputs if check.digest else None, "at": _now(), **self.row_fields()} - self._store.insert(self.collection, pending) + store.insert(self.collection, pending) try: with expecting_version(check.write.id, check.next_version) if self.strict else nullcontext(): version = write() except Exception: # It wrote no version this run knows of, so the row marks nothing for the next run to keep. - self._store.delete(self.collection, Filter.of(**key, status="pending")) + store.delete(self.collection, Filter.of(**key, status="pending")) raise # Stamped before the check below, which can take seconds, so a run interrupted during it leaves the version # this attempt wrote, and the next run keeps it rather than writing it again. - self._store.replace(self.collection, Filter.of(**key, status="pending"), {**pending, "version": version}) + store.replace(self.collection, Filter.of(**key, status="pending"), {**pending, "version": version}) done = {**pending, "status": "done", "version": version, "at": _now()} if check.digest is not None and self.digest(check.write, check.needs) != check.digest: # A file changed during the write, so what the version holds is unknown: the next run writes it again. done.update(digest=None, inputs=None) - self._store.replace(self.collection, Filter.of(**key, status="pending"), done, upsert=True) + store.replace(self.collection, Filter.of(**key, status="pending"), done, upsert=True) return version def adopt(self, check: Check) -> None: @@ -188,7 +192,7 @@ def adopt(self, check: Check) -> None: key = {"store": check.write.kind.store, "id": check.write.id, "bundle": self._bundle(check.write)} done = {**key, "status": "done", "scheme": SCHEME, "digest": check.digest.value, "inputs": check.digest.inputs, "version": check.version, "at": _now(), **self.row_fields()} - self._store.replace(self.collection, Filter.of(**key, status="pending"), done, upsert=True) + self._store_of(check.write.id).replace(self.collection, Filter.of(**key, status="pending"), done, upsert=True) def row_fields(self) -> dict[str, Any]: """What each row records beyond the write: nothing here.""" @@ -197,8 +201,8 @@ def row_fields(self) -> dict[str, Any]: def _orphan(self, write: Write, stored: int | None, digest: Digest) -> int | None: """The version an interrupted run of this bundle wrote and didn't record, when it's still the store's latest and was made from what ``write`` is now made from.""" - pending = self._find(self.collection, Filter.of(store=write.kind.store, id=write.id, - bundle=self._bundle(write), status="pending")) + pending = self._find(write.id, self.collection, Filter.of(store=write.kind.store, id=write.id, + bundle=self._bundle(write), status="pending")) if pending is None or pending.get("version") is None or pending["version"] != stored: return None return stored if (pending["scheme"], pending["digest"]) == (SCHEME, digest.value) else None @@ -208,17 +212,18 @@ def _bundle(self, write: Write) -> str: return self._plan.bundle_of(write.source.entry).id_root def _latest(self, store: str, id: str) -> dict | None: - return self._find(self.collection, Filter.of(store=store, id=id, status="done")) + return self._find(id, self.collection, Filter.of(store=store, id=id, status="done")) def _stored_version(self, write: Write) -> int | None: - doc = self._find(_COLLECTIONS[write.kind.store], Filter.of(id=write.id)) + doc = self._find(write.id, _COLLECTIONS[write.kind.store], Filter.of(id=write.id)) return doc["version"] if doc else None - def _find(self, collection: str, filter: Filter) -> dict | None: - """The highest version matching ``filter``, read without creating a local store.""" - if isinstance(self._store, LocalSqliteDocumentStore) and not self._store.path.exists(): + def _find(self, entity_id: str, collection: str, filter: Filter) -> dict | None: + """The highest version matching ``filter`` in ``entity_id``'s store, read without creating a local store.""" + store = self._store_of(entity_id) + if isinstance(store, LocalSqliteDocumentStore) and not store.path.exists(): return None - return self._store.find_one(collection, filter, sort=Sort.by("version")) + return store.find_one(collection, filter, sort=Sort.by("version")) @contextmanager diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index a31ba1e1e..70b287afe 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -26,7 +26,8 @@ from agent_env.bundle._fs import Unreadable, fold, home, on_disk, read_regular, show, with_article from agent_env.entity.refs import EntityKind from agent_env.plugins._bundles import AGENT_ALIASES, ALIASES_FILE, ARTIFACT_ALIASES, ENV_ALIASES, read_aliases -from agent_env.store.ids import LOCAL_PREFIX, MAX_AUTHORED_LOCAL_ID_BYTES, validate_local_id +from agent_env.store.ids import (HF_NAMESPACE, HF_PREFIX, LOCAL_PREFIX, MAX_AUTHORED_LOCAL_ID_BYTES, is_local_id, + local_namespace, validate_local_id) from agent_env.utils.paths import LEAVINGS, os_reason BUNDLE_TOML = "bundle.toml" @@ -115,7 +116,8 @@ def is_ignored(name: str) -> bool: def parse_bundle(root: Path, *, id_root: str | None = None, name: str | None = None) -> Bundle: """Parse the bundle at ``root``. An entry-point bundle passes its ``@local//`` - root and name; otherwise both come from the folder's canonical path. Raises BundleError.""" + root and name, and a bundle downloaded from the Hugging Face Hub its ``@hf//[/]`` root; + otherwise both come from the folder's canonical path. Raises BundleError.""" return _Parser(_canonical(Path(root)), id_root, name).parse() @@ -472,9 +474,13 @@ def _entity_id(self, kind: BundleKind, name: str, path: Path, declared: Any, dec return entity_id def _check_declared(self, entity_id: Any, where: Path, *, what: str = "id", artifact: bool = False) -> None: - if not isinstance(entity_id, str) or not entity_id.startswith(LOCAL_PREFIX): + if not isinstance(entity_id, str) or not is_local_id(entity_id): self._skip(where, f"{what} {_brief(entity_id)}: ids declared in a bundle must start with " f"{LOCAL_PREFIX!r}, since a bare id names an entity in a shared store") + if foreign_hub_id(entity_id, self.id_root): + self._skip(where, f"{what} {_brief(entity_id)}: an {HF_PREFIX!r} id names an entity of a bundle run from " + "the Hugging Face Hub, so a bundle declares one only under its own id root, which " + "agent-env run hf:// gives it") size = _size(entity_id) if size > MAX_AUTHORED_LOCAL_ID_BYTES: limit = MAX_AUTHORED_LOCAL_ID_BYTES @@ -615,6 +621,13 @@ def _rel(self, path) -> str: return show(text) +def foreign_hub_id(entity_id: str, id_root: str) -> bool: + """Whether ``entity_id`` is an ``@hf`` id outside ``id_root``. Only ``agent-env run hf://`` roots a bundle in the + ``@hf`` namespace, so an id there says the entity came from the Hub; a bundle may declare one only under its own + root.""" + return local_namespace(entity_id) == HF_NAMESPACE and entity_id != id_root and not entity_id.startswith(f"{id_root}/") + + def root_problems(root: Path) -> list[str]: """Why a bundle at ``root``, which needn't exist yet, would have no id; empty when it would have one.""" parent = _canonical(Path(root).expanduser().parent) diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index bdea42432..c9f153fbe 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -53,7 +53,7 @@ from agent_env.providers.env_providers.env_provider import _env_provider_class from agent_env.store import Filter, Sort from agent_env.store.base import NotFoundError -from agent_env.store.ids import LOCAL_PREFIX +from agent_env.store.ids import LOCAL_PREFIX, is_local_id from agent_env.store.routing import namespace_routing_enabled from agent_env.task.task import Task from agent_env.task_step.registry import get_task_step_registry @@ -413,7 +413,7 @@ def _alias_use(self, kind: EntityKind, id: str, version: int | None) -> AliasUse (its latest version, or None, cached in ``alias_stored``), recorded in ``alias_uses``; why it can't be named so; or None when it's no alias, or names the store's as any store reference does: pinned, or with the alias refused.""" - if id.startswith(LOCAL_PREFIX): + if is_local_id(id): return None table, uses, stored = self.alias_tables[kind], self.alias_uses[kind], self.alias_stored[kind] refusal, named = table.refusal(id), table.target(id) @@ -720,10 +720,10 @@ def _check_store_refs( return tuple(checked.values()), latest def _bundle_ref_problem(self, ref: Reference) -> str | None: - """What makes an ``@local`` id wrong without reading a store: it names one of this bundle's outputs by id, + """What makes a local id wrong without reading a store: it names one of this bundle's outputs by id, one of its entities of another kind, or an id under its root that it doesn't write. Another bundle's id is read like any store id.""" - if not ref.id.startswith(LOCAL_PREFIX): + if not is_local_id(ref.id): return None kind, key = ref.kind.value, _nfc(ref.id) if (kind, key) in self.outputs: @@ -739,8 +739,8 @@ def _bundle_ref_problem(self, ref: Reference) -> str | None: return None def _store_ref_problem(self, ref: Reference, read: dict) -> str | None: - if ref.id.startswith(LOCAL_PREFIX) and not namespace_routing_enabled(): - raise RuntimeError("reading an @local id from the store needs namespace routing, which the agent-env CLI " + if is_local_id(ref.id) and not namespace_routing_enabled(): + raise RuntimeError("reading a local id from the store needs namespace routing, which the agent-env CLI " "turns on; call plan_bundle inside agent_env.store.routing.namespace_routing()") kind = ref.kind.value lookup = (ref.kind, ref.id, ref.version) diff --git a/src/agent_env/bundle/register.py b/src/agent_env/bundle/register.py index 2fa92cf73..c65abfaa1 100644 --- a/src/agent_env/bundle/register.py +++ b/src/agent_env/bundle/register.py @@ -307,7 +307,7 @@ class RegistrationLedger(Ledger): pushes_images = True def __init__(self, store: DocumentStore, plan: Plan, identity: str, commit: str | None, dirty: bool | None): - super().__init__(store, plan) + super().__init__(lambda _: store, plan) self.identity, self.commit, self.dirty = identity, commit, dirty def _bundle(self, write: Write) -> str: diff --git a/src/agent_env/bundle/resolve.py b/src/agent_env/bundle/resolve.py index 73984604d..e85d7ac82 100644 --- a/src/agent_env/bundle/resolve.py +++ b/src/agent_env/bundle/resolve.py @@ -26,14 +26,14 @@ from agent_env.eval.eval import Eval from agent_env.plugins import _registration from agent_env.plugins._requirements import unmet_problems -from agent_env.store.ids import LOCAL_PREFIX, derive_id, validate_local_id +from agent_env.store.ids import HF_PREFIX, derive_id, is_local_id, validate_local_id from agent_env.store.image_store.oci_registry_credentials import names_registry from agent_env.task_step.registry import get_task_step_registry from agent_env.task_step.task_step import TaskStep, attach_retry_config, dependencies from ._fs import fold, relative, show from .authoring import AuthoringContext, default_dockerfile -from .parse import BUNDLE_TOML, DOCKERFILE, NAMED_BY, Bundle, BundleEntry, BundleError, BundleKind +from .parse import BUNDLE_TOML, DOCKERFILE, NAMED_BY, Bundle, BundleEntry, BundleError, BundleKind, foreign_hub_id from .requires import UnmetRequirements # Step keys that hold step ids, never entity ids. @@ -379,12 +379,16 @@ def _outputs(self, entry: BundleEntry, declared: list[tuple[int, dict, type]]) - self._problem(entry, f"{where}: {name!r} is also written by step " f"{outputs[kind, name].step_id!r}; give each output its own name") continue - output_id = name if name.startswith(LOCAL_PREFIX) else f"{entry.id}/{name}" + output_id = name if is_local_id(name) else f"{entry.id}/{name}" try: validate_local_id(output_id) except ValueError as e: self._problem(entry, f"{where}: {e}") continue + if foreign_hub_id(output_id, self.bundle.id_root): + self._problem(entry, f"{where}: {output_id!r} is an {HF_PREFIX!r} id outside this bundle's id " + "root, which only a bundle run from the Hugging Face Hub writes under its own") + continue site.rewrite(output_id) outputs[kind, name] = Output(kind, output_id, index, step.get("id"), site.ref.artifact_type) return outputs diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 9cb4207bc..3af4eca0f 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -35,6 +35,7 @@ from agent_env.task.teardown import TeardownReport, teardown_run from agent_env.task_step.context import TaskStepContext, sandbox_overrides +from .hub import SOURCE_KEY from .installed import installed_bundles from .materialize import Materialization, Materialized, materialize from .parse import BundleEntry, BundleError, BundleKind, parse_bundle @@ -217,6 +218,7 @@ def run_bundle( id_root: str | None = None, keep: bool = False, emulate: bool = True, + source: Mapping[str, str] | None = None, ) -> BundleRun: """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the @@ -225,7 +227,9 @@ def run_bundle( agents, sandboxes and the judge deploy on; a computer env composing there emulates an arch no link runs natively unless ``emulate`` is False, which leaves it skipped. ``on_progress`` is given a line of text at each write and step. ``id_root`` roots the ids of a bundle an installed package provides - (``InstalledBundle.id_root``); a folder's ids otherwise come from its path. + (``InstalledBundle.id_root``) or one downloaded from the Hub (``HubBundle.id_root``); a folder's ids otherwise come + from its path. ``source`` says where the bundle came from, a Hub bundle's ``HubBundle.source``, which each run's + context records under ``metadata["bundle_source"]``. Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. @@ -252,7 +256,8 @@ def run_bundle( with Interrupts() as interrupts: overrides = _overrides(sandbox, user_model, judge_model) runs = interrupts.run(_run_all(plan, list(prepared.tasks), model, overrides, - solver_params(plan, prepared.materialization), keep, say, interrupts)) + solver_params(plan, prepared.materialization), keep, say, interrupts, + source)) _mark_cancelled(runs, interrupts.reason) result = _bundle_run(plan, prepared.materialization, runs, _skipped(plan, entries, every=not tasks and not evals)) @@ -482,6 +487,7 @@ def _overrides(sandbox: str | None, user_model: str | None, judge_model: str | N async def _run_all( plan: Plan, to_run: list[tuple[BundleEntry, Task]], model: str | None, overrides: dict[str, str], step_params: Mapping[str, dict], keep: bool, say: Callable[[str], None], interrupts: Interrupts, + source: Mapping[str, str] | None, ) -> tuple[TaskRun, ...]: semaphore = asyncio.Semaphore(MAX_CONCURRENT_RUNS) run_group_id = uuid.uuid4().hex @@ -500,7 +506,7 @@ async def settle(context: TaskStepContext, cancelled: bool) -> TeardownReport | async def run_one(entry: BundleEntry, task: Task) -> TaskRun: tag = f"[{plan.path(entry)}]" - metadata = {"run_group_id": run_group_id, "task_id": task.id} + metadata = {"run_group_id": run_group_id, "task_id": task.id, **({SOURCE_KEY: dict(source)} if source else {})} user_overrides = {**overrides, **({"step_params": step_params[entry.id]} if entry.id in step_params else {})} if user_overrides: metadata["user_overrides"] = user_overrides diff --git a/src/agent_env/cli/_plugin_changes.py b/src/agent_env/cli/_plugin_changes.py index ecc001073..03bcbcc14 100644 --- a/src/agent_env/cli/_plugin_changes.py +++ b/src/agent_env/cli/_plugin_changes.py @@ -886,7 +886,7 @@ def _config_owner(path: Optional[Path], names: list[str], tools: Tools) -> Optio def _local_namespace_documents_exist(config) -> bool: store = config.get_document_store() - return isinstance(store, RoutingDocumentStore) and store.local.path.is_file() + return isinstance(store, RoutingDocumentStore) and any(local.path.is_file() for local in store.namespaces.values()) def _stored_usage(installed: Snapshot, targets: list[str], *, check_remote: bool) -> list[str]: diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index de7427b02..71847b661 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -1,4 +1,5 @@ -"""``agent-env run``: run the tasks and evals of a bundle, a folder or one an installed package provides.""" +"""``agent-env run``: run the tasks and evals of a bundle, a folder, one an installed package provides or one in a +Hugging Face Hub dataset.""" from __future__ import annotations @@ -12,6 +13,7 @@ from agent_env.bundle import (BundleError, BundleKind, BundleRun, DryRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle) +from agent_env.bundle.hub import SOURCE_KEY, HubBundle, describe from agent_env.bundle.installed import (InstalledBundle, checked, find_bundle, installed_bundles, locate_bundle, path_like, run_name) from agent_env.bundle.requires import UnmetRequirements @@ -56,8 +58,18 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t dry_run: bool, no_emulate: bool): """Run a bundle's tasks and evals, or list the installed bundles. - BUNDLE is a folder, or the name of a bundle an installed package provides (PACKAGE/NAME when several - packages provide that name). Without BUNDLE, it lists the installed bundles. + BUNDLE is a folder, the name of a bundle an installed package provides (PACKAGE/NAME when several + packages provide that name), or hf://ORG/REPO[@REVISION][/PATH], a bundle in a Hugging Face Hub dataset. Without + BUNDLE, it lists the installed bundles. + + A Hub bundle is downloaded at the commit REVISION names (a branch, tag or commit, percent-encoded when it holds a + /; the main branch by default) into a folder for that commit under $XDG_CACHE_HOME/agentenv/hf, and runs as a + folder does, with its ids under @hf/ORG/REPO. PATH names its folder in the repo; without it, the run takes the + only bundle under bundles/, or the one the dataset card's agentenv: default names, else the repo's root. Only + the bundle and the card are downloaded, and a commit already downloaded is reused. The run first prints the repo, + commit and bundle. A bundle with no bundle.toml requires the plugins the card's agentenv: bundles: NAME: plugins + lists, and a run that fails shows the card's setup command, which it never runs. A gated or private repo needs + hf auth login or HF_TOKEN. A bundle's bundle.toml names the plugin packages it requires, and a run refuses to start without them; in a terminal it offers to install them with agent-env plugin add. A folder with no bundle.toml that uses plugin @@ -86,9 +98,11 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t "--dry-run and --no-emulate need a BUNDLE to run") _list_installed() return - root, id_root = locate_bundle(bundle, _installed) + root, id_root, hub = locate_bundle(bundle, _installed, on_progress=click.echo) verbose = ctx.find_root().params.get("verbose") try: + if hub is not None: + hub.check_plugins() if dry_run: click.echo(_DRY_RUN) _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, agent=agent, @@ -96,9 +110,11 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t return result = run_bundle(root, tasks=tasks, evals=evals, model=model, user_model=user_model, judge_model=judge_model, agent=agent, sandbox=sandbox, on_progress=click.echo, - id_root=id_root, keep=keep, emulate=not no_emulate) + id_root=id_root, keep=keep, emulate=not no_emulate, source=hub.source if hub else None) except BundleError as e: _note_an_installed_namesake(e, bundle, root, id_root) + if setup := _setup(hub): + e.add_note(setup) if isinstance(e, UnmetRequirements) and not dry_run: offer_to_install(e) raise @@ -113,9 +129,18 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t result = _hold(result, interrupts) _report_teardown(result) if result.failed: + if setup := _setup(hub): + click.echo(setup) raise SystemExit(1) +def _setup(hub: HubBundle | None) -> str | None: + """The note a failed run of a Hub bundle ends with, naming the command the dataset card sets the bundle up with.""" + if hub is None or not hub.setup: + return None + return f"{hub.source['repo']}'s card sets this bundle up with: {hub.setup}" + + def _note_an_installed_namesake(error: BundleError, text: str, root: Path, id_root: str | None) -> None: """A folder that isn't a bundle can hide the installed bundle the user meant. The note names it by its qualified name, which only helps when that isn't what was typed and isn't a folder too.""" @@ -183,6 +208,8 @@ def _count(entries, kind: BundleKind, noun: str) -> str: def _summarize(result: BundleRun, verbose: bool) -> None: click.echo() + if source := next((run.context.metadata.get(SOURCE_KEY) for run in result.runs), None): + click.echo(f"From {describe(source)}") click.echo("Tasks:") for run in result.runs: click.echo(click.style(f" {_described(result, run)}", fg=_COLORS[run.outcome])) diff --git a/src/agent_env/config/runtime.py b/src/agent_env/config/runtime.py index 5a037771a..a54337d6d 100644 --- a/src/agent_env/config/runtime.py +++ b/src/agent_env/config/runtime.py @@ -500,7 +500,7 @@ def db(self) -> "Database": def get_document_store(self) -> "DocumentStore": """The control-plane document store (cached): an explicitly-set backend, else the one from ``AGENT_ENV_DOCUMENT_STORE`` / config.toml (``local`` → SQLite default). With namespace - routing on, ``@local`` documents and ``@local`` runs' records go to the ``@local`` namespace's + routing on, a local namespace's documents and its runs' records go to that namespace's own local store (``agent_env.store.routing``).""" from agent_env.store import routing @@ -629,14 +629,14 @@ def _document_alias(self, name: str) -> dict: def _local_document_store_path(self) -> str: return str(state_root() / "document_store" / "documents.db") - def local_namespace_document_store(self) -> "LocalSqliteDocumentStore": - """The ``@local`` namespace's own document store, whether or not routing is on. Records about - ``@local`` entities, like the bundle ledger, live beside them here and never reach a configured - store. Building it creates nothing; its first write creates the file.""" - return self._local_store("document") + def local_namespace_document_store(self, namespace: str = "local") -> "LocalSqliteDocumentStore": + """A local namespace's own document store, ``@local``'s unless ``namespace`` names another, whether or not + routing is on. Records about its entities, like the bundle ledger, live beside them here and never reach a + configured store. Building it creates nothing; its first write creates the file.""" + return self._local_store("document", namespace) - def _local_namespace_document_store_path(self) -> str: - return str(state_root() / "document_store" / "local.db") + def _local_namespace_document_store_path(self, namespace: str = "local") -> str: + return str(state_root() / "document_store" / f"{namespace}.db") def _document(self) -> config_snapshot.Snapshot: """The document this Config reads, resolved on first use and kept. @@ -717,9 +717,9 @@ def check_object_url(self, entity_id: str, object_url: str) -> None: return store = routing.object_store_holding(object_url, configured, local) if is_local_id(entity_id) and store is not local: - raise ValueError(f"{entity_id!r} is an @local id, so its objects go to the local object store, not {object_url!r}") + raise ValueError(f"{entity_id!r} is a local id, so its objects go to the local object store, not {object_url!r}") if not is_local_id(entity_id) and store is local: - raise ValueError(f"{entity_id!r} is not an @local id, so its objects go to the configured object store, not {object_url!r}") + raise ValueError(f"{entity_id!r} is not a local id, so its objects go to the configured object store, not {object_url!r}") if store is configured: routing.refuse_in_local_run(f"a write to {object_url!r}") @@ -832,27 +832,33 @@ def _image_alias(self, name: str) -> dict: "hosted registry (and unset AGENT_ENV_IMAGE_STORE, which overrides it)" ) - def _local_store(self, kind: str) -> Any: - """The store of ``kind`` the ``@local`` namespace lives in. Documents get a file of their own - beside the default local store's, so neither ever holds the other's documents; objects and + def _local_store(self, kind: str, namespace: str = "local") -> Any: + """The store of ``kind`` the local ``namespace`` lives in. Each namespace's documents get a file of their own + beside the default local store's, so none ever holds another's documents; objects and images use the per-user object store and local registry, whose urls and repositories already say whose they are.""" from agent_env.store.document_store import DocumentStore from agent_env.store.image_store import ImageStore from agent_env.store.object_store import ObjectStore + key = f"document:{namespace}" if kind == "document" else kind with self._routing_lock: - if kind not in self._local_stores: + if key not in self._local_stores: if kind == "document": section, abc = { "impl": "agent_env.store.routing:LocalNamespaceDocumentStore", - "config": {"path": self._local_namespace_document_store_path()}, + "config": {"path": self._local_namespace_document_store_path(namespace), "namespace": namespace}, }, DocumentStore else: alias, abc = {"object": (self._object_alias, ObjectStore), "image": (self._image_alias, ImageStore)}[kind] section = alias(_STORE_BACKEND_LOCAL) - self._local_stores[kind] = config_loader.build_store(section, abc, secret_resolver=self._resolve_secret) - return self._local_stores[kind] + self._local_stores[key] = config_loader.build_store(section, abc, secret_resolver=self._resolve_secret) + return self._local_stores[key] + + def _namespace_document_stores(self) -> dict[str, Any]: + from agent_env.store.ids import LOCAL_NAMESPACES + + return {namespace: self._local_store("document", namespace) for namespace in LOCAL_NAMESPACES} def _namespace_stores(self, kind: str) -> tuple[Any, Any]: """The configured store of ``kind`` and, under namespace routing, the ``@local`` namespace's @@ -882,7 +888,8 @@ def _compare_local_store(self, kind: str, configured: Any) -> Any: local = self._local_store(kind) if kind == "document": - same = isinstance(configured, LocalSqliteDocumentStore) and configured.path.resolve() == local.path.resolve() + same = isinstance(configured, LocalSqliteDocumentStore) and configured.path.resolve() in { + store.path.resolve() for store in self._namespace_document_stores().values()} elif kind == "object": same = isinstance(configured, LocalFilesystemObjectStore) and configured.root.resolve() == local.root.resolve() else: @@ -893,22 +900,21 @@ def _compare_local_store(self, kind: str, configured: Any) -> Any: def _routed(self, kind: str, configured: Any) -> Any: """``configured`` wrapped by namespace routing, or ``configured`` itself when it already is - the ``@local`` namespace's store.""" + the local namespaces' store.""" from agent_env.store import routing local = self._separate_local_store(kind, configured) if local is None: if kind == "document": - raise ConfigError( - f"{self._local_namespace_document_store_path()} is kept for @local documents; " - "point [stores.document] somewhere else" - ) + namespace, kept = next((namespace, store) for namespace, store in self._namespace_document_stores().items() + if store.path.resolve() == configured.path.resolve()) + raise ConfigError(f"{kept.path} is kept for @{namespace} documents; point [stores.document] somewhere else") return configured with self._routing_lock: routed = self._routed_stores.get(kind) if routed is None or routed.configured is not configured: if kind == "document": - routed = routing.RoutingDocumentStore(configured, local, local.path) + routed = routing.RoutingDocumentStore(configured, self._namespace_document_stores()) elif kind == "object": routed = routing.LocalRunObjectStore(configured, local) else: diff --git a/src/agent_env/openenv/app.py b/src/agent_env/openenv/app.py index 11acaa64f..56211c9c6 100644 --- a/src/agent_env/openenv/app.py +++ b/src/agent_env/openenv/app.py @@ -103,7 +103,7 @@ def _sources(bundle: _Source | Mapping[str, _Source]) -> dict[str | None, _Sourc def _prepared(source: _Source, sandbox: str | None) -> PreparedBundle: """``source`` prepared with every task and eval selected, so ``all`` holds every task, evals or not.""" - root, id_root = (source, None) if isinstance(source, Path) else locate_bundle(source) + root, id_root = (source, None) if isinstance(source, Path) else locate_bundle(source)[:2] entries = parse_bundle(Path(root), id_root=id_root).entries return prepare_bundle( root, tasks=[entry.name for entry in entries if entry.kind is BundleKind.TASK], diff --git a/src/agent_env/store/document_store/document_store.py b/src/agent_env/store/document_store/document_store.py index 9c2f56a11..ac33d83c8 100644 --- a/src/agent_env/store/document_store/document_store.py +++ b/src/agent_env/store/document_store/document_store.py @@ -12,7 +12,7 @@ from typing import Any, Callable, Generic, Optional, TypeVar, Union from agent_env.store.base import ConcurrentModificationError -from agent_env.store.ids import is_local_id +from agent_env.store.ids import local_namespace logger = logging.getLogger(__name__) @@ -404,7 +404,8 @@ def ensure_index( def check_id(self, entity_id: str) -> None: """Raise ValueError unless an entity with ``entity_id`` may be written here. No store takes an id - opening with the reserved ``@`` but the ``@local`` namespace's own, which takes only ``@local`` ids.""" + opening with the reserved ``@`` but a local namespace's own, ``@local``'s or ``@hf``'s, which takes only its + own ids.""" reject_reserved_id(entity_id) @@ -413,15 +414,15 @@ def check_id(self, entity_id: str) -> None: def reject_reserved_id(entity_id: object) -> None: """Refuse an id claiming the ``@`` prefix, which opens a registry-qualified - ``@namespace/id``. Only the ``@local`` namespace's store holds such ids, its own. + ``@namespace/id``. Only a local namespace's store, ``@local``'s or ``@hf``'s, holds such ids, its own. Reserved while agent-env is still private: once third parties have created ids, taking the prefix back stops being free. """ - if isinstance(entity_id, str) and is_local_id(entity_id): + if isinstance(entity_id, str) and (namespace := local_namespace(entity_id)) is not None: raise ValueError( - f"id {entity_id!r} is an @local id, which only the @local namespace's store holds; " - "the agent-env CLI writes @local ids there" + f"id {entity_id!r} is an @{namespace} id, which only the @{namespace} namespace's store holds; " + f"the agent-env CLI writes @{namespace} ids there" ) if isinstance(entity_id, str) and entity_id.startswith(RESERVED_ID_PREFIX): raise ValueError( diff --git a/src/agent_env/store/ids.py b/src/agent_env/store/ids.py index de933b3d5..98c8b5705 100644 --- a/src/agent_env/store/ids.py +++ b/src/agent_env/store/ids.py @@ -1,7 +1,8 @@ """Namespaced entity ids, and the object keys, image repositories and filenames derived from them. -An id opening ``@/`` names the store it lives in; ``@local/…`` is the only namespace -today. Documents keep an id verbatim. Everything derived from an ``@local`` id uses one bounded +An id opening ``@/`` names the store it lives in. The local namespaces, ``@local/…`` and ``@hf/…`` +(a bundle ``agent-env run hf://`` downloaded from the Hugging Face Hub), are this machine's local stores; no other +exists today. Documents keep an id verbatim. Everything derived from a local id uses one bounded segment, ``local/-``; an object key or image repository takes a bare id over 200 bytes as ``-``; every other id passes through byte-identical. The encoders never raise: they expect ids that ``validate_local_id`` accepts, and still return a @@ -19,6 +20,9 @@ LOCAL_NAMESPACE = "local" LOCAL_PREFIX = f"@{LOCAL_NAMESPACE}/" +HF_NAMESPACE = "hf" +HF_PREFIX = f"@{HF_NAMESPACE}/" +LOCAL_NAMESPACES = (LOCAL_NAMESPACE, HF_NAMESPACE) MAX_LOCAL_ID_BYTES = 4096 # A filesystem path component holds at most 255 bytes, so a bare id over this many gets a bounded segment. MAX_BARE_SEGMENT_BYTES = 200 @@ -44,29 +48,37 @@ def parse_namespace(entity_id: str) -> Optional[str]: def is_local_id(entity_id: str) -> bool: - return entity_id.startswith(LOCAL_PREFIX) + """Whether ``entity_id`` is in a local namespace, ``@local`` or ``@hf``, which only this machine's stores hold.""" + return local_namespace(entity_id) is not None + + +def local_namespace(entity_id: str) -> Optional[str]: + """The local namespace ``entity_id`` is in, or None for a bare id or another namespace's.""" + return next((namespace for namespace in LOCAL_NAMESPACES if entity_id.startswith(f"@{namespace}/")), None) def validate_local_id(entity_id: str) -> None: - """Raise ValueError unless ``entity_id`` is ``@local/`` followed by non-empty path segments, + """Raise ValueError unless ``entity_id`` is ``@local/`` or ``@hf/`` followed by non-empty path segments, none of them ``.`` or ``..`` or opening or closing with a space, made only of letters, digits, spaces and ``. _ - ~ / @ ( ) + , &``, and at most 4096 bytes of UTF-8 in all (Linux's PATH_MAX). Only the leading ``@`` names a namespace, so one inside a segment is an ordinary character. Letters include every script's, with its combining marks, as long as each mark follows a letter, digit or mark. The invisible (default-ignorable) letters and marks are refused.""" - if not entity_id.startswith(LOCAL_PREFIX): - raise ValueError(f"local id {entity_id!r} must start with {LOCAL_PREFIX!r}") + namespace = local_namespace(entity_id) + if namespace is None: + raise ValueError(f"local id {entity_id!r} must start with {LOCAL_PREFIX!r} or {HF_PREFIX!r}") + prefix = f"@{namespace}/" size = len(entity_id.encode("utf-8", "surrogatepass")) if size > MAX_LOCAL_ID_BYTES: raise ValueError(f"local id is {size} bytes; the limit is {MAX_LOCAL_ID_BYTES}: {entity_id[:64]!r}…") - disallowed = sorted({c for c in entity_id[len(LOCAL_PREFIX):] if not _allowed_in_id(c)}) + disallowed = sorted({c for c in entity_id[len(prefix):] if not _allowed_in_id(c)}) if disallowed: raise ValueError( f"local id {entity_id!r} contains {ascii(''.join(disallowed))}; " "a local id may contain only letters, digits, spaces and . _ - ~ / @ ( ) + , &" ) - segments = entity_id[len(LOCAL_PREFIX):].split("/") + segments = entity_id[len(prefix):].split("/") if any(segment in ("", ".", "..") for segment in segments): raise ValueError(f"local id {entity_id!r} has an empty, '.' or '..' path segment") if any(segment != segment.strip(" ") for segment in segments): @@ -110,7 +122,7 @@ def key_segment(entity_id: str) -> str: def image_repository(entity_id: str) -> str: - """The image repository for ``entity_id``; an ``@local`` id yields a valid OCI repository path.""" + """The image repository for ``entity_id``; a local id yields a valid OCI repository path.""" return key_segment(entity_id) @@ -121,7 +133,7 @@ def local_image_repository(entity_id: str) -> str: def fs_safe(entity_id: str) -> str: - """A filename for an ``@local`` id: its key segment with no ``/``. Any other id is returned + """A filename for a local id: its key segment with no ``/``. Any other id is returned unchanged, so it is only as filename-safe as it already was.""" return _local_segment(entity_id).replace("/", "-", 1) if is_local_id(entity_id) else entity_id diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index c9f360fab..c49852148 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -1,18 +1,18 @@ -"""Namespace-addressed stores: an ``@local/…`` id lives in the ``@local`` namespace's local stores, -every other id in the configured ones. +"""Namespace-addressed stores: an id in a local namespace, ``@local/…`` or ``@hf/…`` (a bundle run from the Hugging +Face Hub), lives in that namespace's local stores, every other id in the configured ones. Routing runs only once a process turns it on (the CLI does, for the length of a command). Everywhere else each getter returns the configured store exactly as before, so a service keeps rejecting every -``@`` id. The ``@local`` namespace keeps its documents in a file of its own beside the default local +``@`` id. Each local namespace keeps its documents in a file of its own beside the default local store's, so each file only ever holds one namespace's documents; objects and images share the per-user object store and local registry, whose urls and repositories say whose they are. Entity documents route by their id. What a task run records (instances, journals, state, runs, reviews, conversations, snapshots) and the objects it writes under keys of its own follow the run -scope ``Task.run()`` sets from its task id. A read that neither rule places reads both stores, the -reader's own namespace first: an ``@local`` run sees its records before the registry's, and anyone -else sees the registry's first. While an ``@local`` task runs, a write that would reach a configured -store is refused (the write fence), and a bare entity may never name an ``@local`` id (references +scope ``Task.run()`` sets from its task id. A read that neither rule places reads every store, the +reader's own namespace first: a local run sees its records before the registry's, and anyone +else sees the registry's first. While a local task runs, a write that would reach a configured +store is refused (the write fence), and a bare entity may never name a local id (references point outward only). """ @@ -26,7 +26,6 @@ from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime, timezone -from pathlib import Path from typing import Any, BinaryIO, Callable, Optional, TypeVar from agent_env.config.errors import ConfigError @@ -41,7 +40,14 @@ is_entity_collection, ) from agent_env.store.document_store.sqlite_document_store import LocalSqliteDocumentStore -from agent_env.store.ids import LOCAL_NAMESPACE, LOCAL_PREFIX, is_local_id, parse_namespace, validate_local_id +from agent_env.store.ids import ( + LOCAL_NAMESPACE, + LOCAL_NAMESPACES, + is_local_id, + local_namespace, + parse_namespace, + validate_local_id, +) from agent_env.store.image_store import ImageStore, OciRegistryImageStore from agent_env.store.image_store.oci_registry_credentials import ( RegistryAuth, @@ -57,7 +63,7 @@ UploadPolicy, ) -# The fields a run-time document names its owner by: one holding an @local id places it locally. +# The fields a run-time document names its owner by: one holding a local id places it in that id's namespace. _OWNER_FIELDS = ("id", "instance_id", "task_id", "task_instance_id", "env_id", "agent_id") _LOCAL_REPOSITORY_PREFIX = f"{LOCAL_NAMESPACE}/" _TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}") @@ -70,15 +76,15 @@ class LocalRunWriteError(ValueError): - """A write an ``@local`` task run may not make, since it would land in a configured store.""" + """A write a local task run may not make, since it would land in a configured store.""" class OutwardReferenceError(ValueError): - """A write that would point outside the ``@local`` namespace's stores at an ``@local`` id.""" + """A write that would point outside the local namespaces' stores at a local id.""" def enable_namespace_routing() -> None: - """Route ``@local`` ids to the per-user local stores in this process. The CLI turns this on; a + """Route local ids to the per-user local stores in this process. The CLI turns this on; a service never does, so its stores stay exactly the configured ones.""" global _enabled _enabled = True @@ -110,34 +116,41 @@ def namespace_routing() -> Iterator[None]: @contextmanager def run_scope(owner_id: str) -> Iterator[None]: """The run in progress, a task's or a deployment the CLI makes: what it records follows its owner id's - namespace. A run started inside an ``@local`` run stays local, so a nested registry task can't reopen - the configured stores.""" + namespace. A run started inside a local run stays in that run's namespace unless its own id is local, so a + nested registry task can't reopen the configured stores.""" outer = _run_namespace.get() - token = _run_namespace.set(LOCAL_NAMESPACE if outer == LOCAL_NAMESPACE else parse_namespace(owner_id) or "") + inherited = outer if outer in LOCAL_NAMESPACES else parse_namespace(owner_id) or "" + token = _run_namespace.set(local_namespace(owner_id) or inherited) try: yield finally: _run_namespace.reset(token) +def local_run_namespace() -> Optional[str]: + """The local namespace of the task running, which its records go to; None outside a local run.""" + namespace = _run_namespace.get() + return namespace if namespace_routing_enabled() and namespace in LOCAL_NAMESPACES else None + + def in_local_run() -> bool: - return namespace_routing_enabled() and _run_namespace.get() == LOCAL_NAMESPACE + return local_run_namespace() is not None def refuse_in_local_run(what: str) -> None: - if in_local_run(): + if (namespace := local_run_namespace()) is not None: raise LocalRunWriteError( - f"an @local task run writes only to the local stores, and {what} would reach a configured store" + f"an @{namespace} task run writes only to the local stores, and {what} would reach a configured store" ) def refuse_local_references(entity_id: object, value: Any) -> None: - """Refuse a bare entity naming an ``@local`` id anywhere in ``value``: a shared store must + """Refuse a bare entity naming a local id anywhere in ``value``: a shared store must never point into one person's local stores.""" path = _local_reference(value, "") if path is not None: raise OutwardReferenceError( - f"{entity_id!r} is not an @local id, so it can't reference one: {path or 'its value'} names an @local id" + f"{entity_id!r} is not a local id, so it can't reference one: {path or 'its value'} names a local id" ) @@ -214,38 +227,47 @@ def _filter_owner(filter: Filter) -> dict: return {path: preds[0].value for path, preds in filter.conditions.items() if len(preds) == 1 and isinstance(preds[0], Eq)} -def _filter_is_local(filter: Filter) -> Optional[bool]: - """Whether the ids a filter pins are all ``@local`` (True) or all bare (False); None when it pins - none, or both kinds.""" +def _filter_namespace(filter: Filter) -> Optional[str]: + """The local namespace the ids a filter pins are all in, or ``""`` when they are all bare; None when it pins + none, or ids of more than one namespace.""" values: list[Any] = [] for pred in filter.conditions.get("id", ()): if isinstance(pred, Eq): values.append(pred.value) elif isinstance(pred, In): values.extend(pred.values) - kinds = {is_local_id(value) for value in values if isinstance(value, str)} - return kinds.pop() if len(kinds) == 1 else None + namespaces = {local_namespace(value) or "" for value in values if isinstance(value, str)} + return namespaces.pop() if len(namespaces) == 1 else None class LocalNamespaceDocumentStore(LocalSqliteDocumentStore): - """The ``@local`` namespace's documents: its entities, and what its runs record.""" + """A local namespace's documents, ``@local``'s or ``@hf``'s: its entities, and what its runs record.""" + + def __init__(self, path: str, namespace: str = LOCAL_NAMESPACE) -> None: + super().__init__(path) + self.namespace = namespace def check_id(self, entity_id: str) -> None: - if not is_local_id(entity_id): - raise ValueError(f"id {entity_id!r} isn't an @local id, and the @local namespace's store holds only those") + if local_namespace(entity_id) != self.namespace: + raise ValueError(f"id {entity_id!r} isn't an @{self.namespace} id, and the @{self.namespace} namespace's " + "store holds only those") validate_local_id(entity_id) class RoutingDocumentStore(DocumentStore): - """The configured document store, with ``@local`` documents and ``@local`` runs' records kept in - the ``@local`` namespace's store. Reads never create that store; writes open it with its indexes.""" + """The configured document store, with each local namespace's documents and its runs' records kept in + that namespace's own store, ``namespaces`` by name. Reads never create one; writes open it with its indexes.""" - def __init__(self, configured: DocumentStore, local: DocumentStore, local_path: Path) -> None: + def __init__(self, configured: DocumentStore, namespaces: Mapping[str, LocalNamespaceDocumentStore]) -> None: self.configured = configured - self.local = local - self._local_path = local_path + self.namespaces = dict(namespaces) self._lock = threading.Lock() - self._pending_local_indexes: Optional[list[tuple]] = [] + self._pending_local_indexes: dict[str, Optional[list[tuple]]] = {namespace: [] for namespace in namespaces} + + @property + def local(self) -> LocalNamespaceDocumentStore: + """The ``@local`` namespace's store.""" + return self.namespaces[LOCAL_NAMESPACE] def find_one(self, collection: str, filter: Filter, sort: Optional[Sort] = None) -> Optional[dict]: found = [ @@ -441,80 +463,92 @@ def ensure_index( ttl_seconds: Optional[int] = None, ) -> None: self.configured.ensure_index(collection, fields, unique, ttl_seconds) - with self._lock: - if self._pending_local_indexes is not None: - self._pending_local_indexes.append((collection, fields, unique, ttl_seconds)) - return - self.local.ensure_index(collection, fields, unique, ttl_seconds) + for namespace, store in self.namespaces.items(): + with self._lock: + pending = self._pending_local_indexes[namespace] + if pending is not None: + pending.append((collection, fields, unique, ttl_seconds)) + continue + store.ensure_index(collection, fields, unique, ttl_seconds) def check_id(self, entity_id: str) -> None: - if is_local_id(entity_id): - self.local.check_id(entity_id) + namespace = local_namespace(entity_id) + if namespace is not None: + self.namespaces[namespace].check_id(entity_id) return - if LOCAL_PREFIX in entity_id: + if any(f"@{namespace}/" in entity_id for namespace in self.namespaces): raise ValueError( - f"id {entity_id!r} contains an @local id but isn't one: an id derived from an @local id " - "has to keep the @local namespace" + f"id {entity_id!r} contains a local id but isn't one: an id derived from a local id " + "has to keep its namespace" ) self.configured.check_id(entity_id) - def _placed(self, collection: str, filter: Filter) -> Optional[DocumentStore]: - if not is_entity_collection(collection): - return None - local = _filter_is_local(filter) - return None if local is None else self.local if local else self.configured + def _placed(self, collection: str, filter: Filter) -> Optional[str]: + """The local namespace whose store holds ``filter``'s match, ``""`` for the configured store; None when it + can be in any.""" + return _filter_namespace(filter) if is_entity_collection(collection) else None def _readers(self, collection: str, filter: Filter) -> list[DocumentStore]: - """The stores a read consults, the reader's own namespace first: a key can be in both, when - an ``@local`` run and a registry run each recorded it.""" + """The stores a read consults, the reader's own namespace first: a key can be in several, when + a local run and a registry run each recorded it.""" placed = self._placed(collection, filter) - if placed is self.configured: + if placed == "": return [self.configured] - local = [self.local] if self._local_path.exists() else [] - if placed is self.local: - return local - return [*local, self.configured] if in_local_run() else [self.configured, *local] + if placed is not None: + return self._existing([placed]) + own = local_run_namespace() + local = self._existing(self._own_first(own)) + return [*local, self.configured] if own else [self.configured, *local] + + def _own_first(self, own: Optional[str]) -> list[str]: + return sorted(self.namespaces, key=lambda namespace: namespace != own) + + def _existing(self, namespaces: list[str]) -> list[LocalNamespaceDocumentStore]: + return [self.namespaces[namespace] for namespace in namespaces if self.namespaces[namespace].path.exists()] def _read(self, store: DocumentStore, read: Callable[[], _T]) -> _T: - if store is not self.local: + if store is self.configured: return read() try: return read() except (sqlite3.DatabaseError, OSError) as e: raise ConfigError( - f"the @local store at {self._local_path} can't be read ({e}); move it aside to start a new one" + f"the @{store.namespace} store at {store.path} can't be read ({e}); move it aside to start a new one" ) from e def _writer(self, collection: str, owner: Mapping, payload: Any) -> DocumentStore: if is_entity_collection(collection): if isinstance(owner.get("id"), str): self.check_id(owner["id"]) - local = isinstance(owner.get("id"), str) and is_local_id(owner["id"]) + namespace = local_namespace(owner["id"]) if isinstance(owner.get("id"), str) else None else: - local = in_local_run() or any(isinstance(owner.get(f), str) and is_local_id(owner[f]) for f in _OWNER_FIELDS) - if local: - return self._open_local() + namespace = local_run_namespace() or next( + (found for field in _OWNER_FIELDS + if isinstance(owner.get(field), str) and (found := local_namespace(owner[field]))), None) + if namespace is not None: + return self._open(namespace) self._check_configured_write(collection, owner.get("id"), payload) return self.configured def _updaters(self, collection: str, filter: Filter, payload: Any) -> Iterator[DocumentStore]: """The stores an update of ``filter``'s match tries, in order: the one its id places it in, - else the caller's own namespace's first. Inside an ``@local`` run only the local store is + else the caller's own namespace's first. Inside a local run only the local stores are written, and a match that lives only in the configured store is refused rather than missed.""" placed = self._placed(collection, filter) - local = placed is not self.configured and self._local_path.exists() - if placed is None and not in_local_run(): + own = local_run_namespace() + if placed is None and own is None: self._check_configured_write(collection, _filter_owner(filter).get("id"), payload) yield self.configured - if local: - yield self._open_local() + for store in self._existing(list(self.namespaces)): + yield self._open(store.namespace) return - if local: - yield self._open_local() - if placed is self.configured: + if placed == "": self._check_configured_write(collection, _filter_owner(filter).get("id"), payload) yield self.configured - elif placed is None and self.configured.find_one(collection, filter) is not None: + return + for store in self._existing([placed] if placed is not None else self._own_first(own)): + yield self._open(store.namespace) + if placed is None and self.configured.find_one(collection, filter) is not None: refuse_in_local_run(f"updating {collection!r} (its record is in the configured store)") def _check_configured_write(self, collection: str, entity_id: object, payload: Any) -> None: @@ -522,12 +556,13 @@ def _check_configured_write(self, collection: str, entity_id: object, payload: A if is_entity_collection(collection) and payload is not None: refuse_local_references(entity_id, payload) - def _open_local(self) -> DocumentStore: + def _open(self, namespace: str) -> LocalNamespaceDocumentStore: + store = self.namespaces[namespace] with self._lock: - for spec in self._pending_local_indexes or (): - self._read(self.local, lambda: self.local.ensure_index(*spec)) - self._pending_local_indexes = None - return self.local + for spec in self._pending_local_indexes[namespace] or (): + self._read(store, lambda: store.ensure_index(*spec)) + self._pending_local_indexes[namespace] = None + return store def object_store_holding(object_url: str, configured: ObjectStore, local: ObjectStore) -> ObjectStore: @@ -550,7 +585,7 @@ def image_store_holding(ref: str, configured: ImageStore, local: ImageStore) -> class LocalRunObjectStore(ObjectStore): - """The object store an ``@local`` task run sees: keys go to the local store, an object_url to the + """The object store a local task run sees: keys go to the local store, an object_url to the store it addresses, and a write that would reach the configured store is refused.""" def __init__(self, configured: ObjectStore, local: ObjectStore) -> None: @@ -621,7 +656,7 @@ def signed_put_url(self, object_url: str, expires_in: int = 3600) -> str | None: return self._writing(object_url).signed_put_url(object_url, expires_in) def shared_credentials_env(self) -> dict[str, str]: - """None: an ``@local`` run's objects reach its sandboxes through grants, staging or pushed bytes, so the + """None: a local run's objects reach its sandboxes through grants, staging or pushed bytes, so the configured store's credentials, the caller's own, stay with the caller.""" return {} @@ -649,7 +684,7 @@ def _writing(self, object_url: str) -> ObjectStore: class LocalRunImageStore(ImageStore): - """The image store an ``@local`` task run sees: ``@local`` repositories are the local registry's, + """The image store a local task run sees: local ids' repositories are the local registry's, pull auth comes from the store a ref names, and creating a configured repository is refused.""" def __init__(self, configured: ImageStore, local: ImageStore) -> None: diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py new file mode 100644 index 000000000..9a073e8cc --- /dev/null +++ b/tst/unit/bundle/hub_test.py @@ -0,0 +1,296 @@ +"""Bundles run from a Hugging Face Hub dataset: ``hf://`` references, the bundle picked from a commit's file list, the +download kept per commit, the card's requirements, and the ids and record of ``agent-env run hf://``.""" + +import fnmatch +import json +import logging +import shutil +from types import SimpleNamespace + +import httpx +import pytest +from click.testing import CliRunner +from huggingface_hub.errors import RepositoryNotFoundError + +from agent_env.bundle import BundleError +from agent_env.bundle import hub as hub_module +from agent_env.bundle.hub import HubReference, fetch_bundle, parse_reference +from agent_env.bundle.ledger import LEDGER_COLLECTION +from agent_env.cli import cli +from agent_env.config import get_config +from agent_env.config.runtime import Config +from agent_env.store import Filter +from agent_env.store.routing import namespace_routing +from agent_env.task.store import task_instances +from agent_env.task_step.task_step import TaskStep + +SHA = "0123456789abcdef0123456789abcdef01234567" +TASK = json.dumps([{"id": "check", "type": "noop_hub_test"}]) +SOURCE = {"hub": "hf", "repo": "me/ds", "revision": "main", "sha": SHA, "path": "bundles/hello"} + + +class _Noop(TaskStep): + type = "noop_hub_test" + entity_refs = () + + @classmethod + def from_dict(cls, data): + return cls(**cls._base_from_dict(data)) + + async def execute(self, context): + return context + + +@pytest.fixture(autouse=True) +def registries(monkeypatch): + steps = {**Config().task_step_registry(), _Noop.type: _Noop} + monkeypatch.setattr(Config, "task_step_registry", lambda self: steps) + + +@pytest.fixture +def quiet_logs(): + """pytest's live logging swaps its own stdout back in to print a record, so CliRunner loses whatever is + echoed after the first one.""" + logging.disable(logging.CRITICAL) + yield + logging.disable(logging.NOTSET) + + +class FakeHub: + """Serves the dataset ``me/ds`` from the folder ``repo``, every revision at SHA, recording each call.""" + + def __init__(self, repo): + self.repo = repo + self.calls = [] + + def repo_info(self, repo_id, *, repo_type, revision=None): + self.calls.append("repo_info") + assert repo_type == "dataset" + if repo_id != "me/ds": + response = httpx.Response(404, request=httpx.Request("GET", f"http://hub.test/api/datasets/{repo_id}")) + raise RepositoryNotFoundError("404 Client Error. Repository Not Found", response=response) + return SimpleNamespace(sha=SHA) + + def list_repo_files(self, repo_id, *, repo_type, revision): + self.calls.append("list_repo_files") + assert (repo_type, revision) == ("dataset", SHA) + return self._files() + + def hf_hub_download(self, repo_id, filename, *, repo_type, revision, local_dir): + self.calls.append(f"hf_hub_download {filename}") + self._copy([filename], local_dir) + + def snapshot_download(self, repo_id, *, repo_type, revision, local_dir, allow_patterns): + self.calls.append(f"snapshot_download {' '.join(allow_patterns)}") + assert (repo_type, revision) == ("dataset", SHA) + self._copy([file for file in self._files() if any(fnmatch.fnmatch(file, p) for p in allow_patterns)], local_dir) + + def layout(self, files): + for rel, text in files.items(): + (self.repo / rel).parent.mkdir(parents=True, exist_ok=True) + (self.repo / rel).write_text(text) + + def card(self, agentenv=None): + meta = f"agentenv: {json.dumps(agentenv)}\n" if agentenv is not None else "" + self.layout({"README.md": f"---\ntags:\n- agentenv\n{meta}---\n\n# A dataset\n"}) + + def _files(self): + return sorted(path.relative_to(self.repo).as_posix() for path in self.repo.rglob("*") if path.is_file()) + + def _copy(self, files, local_dir): + for file in files: + (local_dir / file).parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(self.repo / file, local_dir / file) + + +@pytest.fixture +def hub(tmp_path, monkeypatch): + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + fake = FakeHub(tmp_path / "repo") + monkeypatch.setattr(hub_module, "HfApi", lambda: fake) + return fake + + +def _commit(tmp_path): + return tmp_path / "cache" / "agentenv" / "hf" / "me" / "ds" / SHA + + +def _downloaded(tmp_path): + folder = _commit(tmp_path) + return sorted(path.relative_to(folder).as_posix() for path in folder.rglob("*") + if path.is_file() and path.relative_to(folder).parts[0] != ".cache") + + +@pytest.mark.parametrize("text, reference", [ + ("hf://me/ds", HubReference("me/ds", None, None)), + ("hf://me/ds@v1.0", HubReference("me/ds", "v1.0", None)), + ("hf://me/ds/bundles/hello/", HubReference("me/ds", None, "bundles/hello")), + (f"hf://Me-2/ds_v1.2@{SHA}/suite/x", HubReference("Me-2/ds_v1.2", SHA, "suite/x")), + ("hf://me/ds@refs%2Fpr%2F3/bundles/hello", HubReference("me/ds", "refs/pr/3", "bundles/hello")), +]) +def test_a_reference_names_a_repo_then_maybe_a_revision_and_a_path(text, reference): + assert parse_reference(text) == reference + + +@pytest.mark.parametrize("text, problem", [ + ("hf://me", "name a Hub dataset's bundle as hf:///[@][/]"), + ("hf:///ds", "name a Hub dataset's bundle as"), + ("hf://me/ds@", "name a Hub dataset's bundle as"), + ("hf://me/d s", "name a Hub dataset's bundle as"), + ("hf://me/.ds", "name a Hub dataset's bundle as"), + ("hf://me/ds/bundles/../../x", "the path in the repo can't hold '.' or '..'"), +]) +def test_a_malformed_reference_is_refused(text, problem): + with pytest.raises(BundleError) as caught: + parse_reference(text) + (refused,) = caught.value.problems + assert refused.startswith(f"{text}: ") and problem in refused + + +def test_the_only_bundle_under_bundles_wins_over_a_root_of_data_files_and_only_it_and_the_card_are_downloaded( + hub, tmp_path): + hub.layout({"bundles/hello/tasks/t.json": TASK, "tasks/train.parquet": "rows", "runs/recording.mp4": "frames"}) + hub.card() + said = [] + + fetched = fetch_bundle("hf://me/ds", on_progress=said.append) + + assert said == ["me/ds at 0123456789ab, bundle bundles/hello"] + assert fetched.root == _commit(tmp_path) / "bundles" / "hello" + assert fetched.id_root == "@hf/me/ds/hello" + assert fetched.source == SOURCE + assert hub.calls == ["repo_info", "list_repo_files", "hf_hub_download README.md", + "snapshot_download bundles/hello/*"] + assert _downloaded(tmp_path) == ["README.md", "bundles/hello/tasks/t.json"] + + +def test_several_bundles_need_a_path_or_the_cards_default(hub, tmp_path): + hub.layout({"bundles/a/tasks/t.json": TASK, "bundles/b/tasks/t.json": TASK}) + hub.card() + + with pytest.raises(BundleError) as caught: + fetch_bundle("hf://me/ds@v1") + named = fetch_bundle("hf://me/ds/bundles/a") + shutil.rmtree(tmp_path / "cache") + hub.card({"default": "b"}) + by_default = fetch_bundle("hf://me/ds") + + assert caught.value.problems == ( + "me/ds at 0123456789ab holds several bundles under bundles/ and its card names no default (agentenv: " + "default); run one of hf://me/ds@v1/bundles/a, hf://me/ds@v1/bundles/b",) + assert (named.source["path"], named.id_root) == ("bundles/a", "@hf/me/ds/a") + assert (by_default.source["path"], by_default.id_root) == ("bundles/b", "@hf/me/ds/b") + + +def test_a_path_names_any_bundle_folder_and_its_ids_follow_it(hub): + hub.layout({"suites/hello/tasks/t.json": TASK, "suites/notes.md": "not a bundle"}) + + fetched = fetch_bundle("hf://me/ds/suites/hello") + with pytest.raises(BundleError) as caught: + fetch_bundle("hf://me/ds/suites") + + assert (fetched.id_root, fetched.source["path"]) == ("@hf/me/ds/suites/hello", "suites/hello") + assert caught.value.problems == ("me/ds at 0123456789ab: suites/ isn't a bundle: it has none of envs/, agents/, " + "artifacts/, skills/, tasks/, evals/",) + + +def test_the_root_is_the_bundle_when_nothing_is_under_bundles_and_its_other_folders_stay_on_the_hub(hub, tmp_path): + hub.layout({"tasks/t.json": TASK, "bundle.toml": "requires = []\n", "data/rows.parquet": "rows"}) + + fetched = fetch_bundle("hf://me/ds") + + assert (fetched.root, fetched.id_root, fetched.source["path"]) == (_commit(tmp_path), "@hf/me/ds", "") + assert _downloaded(tmp_path) == ["bundle.toml", "tasks/t.json"] + + +def test_a_repo_without_a_bundle_is_refused_before_any_download(hub): + hub.layout({"data/rows.parquet": "rows", "bundles/README.md": "nothing here"}) + + with pytest.raises(BundleError) as caught: + fetch_bundle("hf://me/ds") + + assert caught.value.problems == ("me/ds at 0123456789ab holds no bundle: it has no bundles// folder, and none " + "of envs/, agents/, artifacts/, skills/, tasks/, evals/ at its root",) + assert hub.calls == ["repo_info", "list_repo_files"] + + +def test_a_commit_already_downloaded_is_reused_and_a_pinned_one_needs_no_network(hub): + hub.layout({"bundles/a/tasks/t.json": TASK, "bundles/b/tasks/t.json": TASK}) + + first = fetch_bundle(f"hf://me/ds@{SHA}/bundles/a") + hub.calls.clear() + again = fetch_bundle(f"hf://me/ds@{SHA}/bundles/a") + by_branch = fetch_bundle("hf://me/ds/bundles/a") + reused, hub.calls[:] = list(hub.calls), [] + other = fetch_bundle(f"hf://me/ds@{SHA}/bundles/b") + + assert first.root == again.root == by_branch.root + assert again.source["revision"] == SHA and by_branch.source["revision"] == "main" + assert reused == ["repo_info"] + assert hub.calls == ["snapshot_download bundles/b/*"] + assert other.root.parent == first.root.parent + + +def test_a_repo_the_hub_wont_serve_names_how_to_log_in(hub): + with pytest.raises(BundleError) as caught: + fetch_bundle("hf://someone/private") + + assert caught.value.problems == ("hf://someone/private: 404 Client Error. Repository Not Found",) + assert "log in with `hf auth login`, or set HF_TOKEN" in caught.value.__notes__[0] + + +def test_a_hub_bundle_runs_under_its_hf_id_root_and_each_run_records_where_it_came_from(hub, local_stores, quiet_logs): + hub.layout({"bundles/hello/tasks/t.json": TASK}) + + first = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + second = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + + assert first.exit_code == 0, first.output + assert first.output.startswith("me/ds at 0123456789ab, bundle bundles/hello\n") + assert "From me/ds at 0123456789ab, bundle bundles/hello\nTasks:\n tasks/t.json v1: unscored" in first.output + assert "instance @hf/me/ds/hello/t-" in first.output + assert "tasks/t.json: v1, unchanged" in second.output + with namespace_routing(): + runs = task_instances("@hf/me/ds/hello/t") + assert [(run.task_version, run.context["metadata"]["bundle_source"]) for run in runs] == [(1, SOURCE)] * 2 + rows = local_stores.local_namespace_document_store("hf").query(LEDGER_COLLECTION, Filter()) + assert {row["id"] for row in rows} == {"@hf/me/ds/hello/t"} + assert not local_stores.local_namespace_document_store().path.exists() + + +def test_a_dry_run_downloads_and_checks_but_runs_and_writes_nothing(hub, local_stores, quiet_logs, tmp_path): + hub.layout({"bundles/hello/tasks/t.json": TASK}) + + result = CliRunner().invoke(cli, ["run", "hf://me/ds", "--dry-run"]) + + assert result.exit_code == 0, result.output + assert result.output.startswith("me/ds at 0123456789ab, bundle bundles/hello\nDry run: nothing is written or run.\n") + assert "Would run:\n tasks/t.json v1\n" in result.output + assert _downloaded(tmp_path) == ["bundles/hello/tasks/t.json"] + assert not local_stores.local_namespace_document_store("hf").path.exists() + + +def test_a_bundle_without_a_bundle_toml_needs_the_plugins_its_card_names(hub, local_stores, quiet_logs): + hub.layout({"bundles/hello/tasks/t.json": TASK}) + hub.card({"bundles": {"hello": {"plugins": ["agentenv-not-installed>=1"]}}}) + + refused = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + hub.layout({"bundles/hello/bundle.toml": "requires = []\n"}) + shutil.rmtree(hub_module.cache_root()) + declared = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + + assert refused.exit_code == 1 + assert ("me/ds's card: requires 'agentenv-not-installed>=1', which isn't installed; install it with: agent-env " + "plugin add 'agentenv-not-installed>=1'") in refused.output + assert "Tasks:" not in refused.output + assert declared.exit_code == 0, declared.output + + +def test_a_failed_run_shows_the_cards_setup_command_and_never_runs_it(hub, local_stores, quiet_logs): + hub.layout({"bundles/hello/tasks/t.json": json.dumps([{"id": "x", "type": "a_step_only_setup_registers"}])}) + hub.card({"bundles": {"hello": {"setup": "agent-env demo setup"}}}) + + result = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + + assert result.exit_code == 1 + assert "me/ds's card sets this bundle up with: agent-env demo setup" in result.output diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index 035defa36..88bfdded1 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -318,9 +318,9 @@ def test_a_folder_wins_over_an_installed_name_and_a_folder_that_isnt_a_bundle_na _registered(monkeypatch, _ep("hello", "demo_bundles")) monkeypatch.chdir(tmp_path) (tmp_path / "hello").mkdir() - assert locate_bundle("hello") == (Path("hello"), None) - assert locate_bundle("./elsewhere") == (Path("./elsewhere"), None) - assert locate_bundle("demo-bundles/hello") == (site / "demo_bundles" / "hello", "@local/demo-bundles/hello") + assert locate_bundle("hello") == (Path("hello"), None, None) + assert locate_bundle("./elsewhere") == (Path("./elsewhere"), None, None) + assert locate_bundle("demo-bundles/hello") == (site / "demo_bundles" / "hello", "@local/demo-bundles/hello", None) result = CliRunner().invoke(cli, ["run", "hello"]) @@ -330,6 +330,22 @@ def test_a_folder_wins_over_an_installed_name_and_a_folder_that_isnt_a_bundle_na assert CliRunner().invoke(cli, ["run", "demo-bundles/hello"]).exit_code == 0 +def test_a_folder_wins_only_when_spelled_as_written_so_desktop_isnt_a_case_insensitive_desktop_folder( + site, monkeypatch, tmp_path): + _package(site, "demo_bundles", ["desktop"]) + _registered(monkeypatch, _ep("desktop", "demo_bundles")) + monkeypatch.chdir(tmp_path) + (tmp_path / "Desktop").mkdir() + (tmp_path / "demo-bundles" / "Desktop").mkdir(parents=True) + exact = os.path.isdir + monkeypatch.setattr(os.path, "isdir", lambda path: exact(path) or exact(str(path).replace("desktop", "Desktop"))) + + assert locate_bundle("desktop").root == site / "demo_bundles" / "desktop" + assert locate_bundle("demo-bundles/desktop").root == site / "demo_bundles" / "desktop" + assert locate_bundle("Desktop") == (Path("Desktop"), None, None) + assert locate_bundle("demo-bundles/Desktop") == (Path("demo-bundles/Desktop"), None, None) + + def test_the_cli_reports_installed_metadata_it_cant_read(monkeypatch): def unreadable(*, group): raise TypeError("Pair.__new__() missing 1 required positional argument: 'value'") diff --git a/tst/unit/bundle/parse_test.py b/tst/unit/bundle/parse_test.py index 699c551c9..347959c75 100644 --- a/tst/unit/bundle/parse_test.py +++ b/tst/unit/bundle/parse_test.py @@ -532,6 +532,18 @@ def test_refused(self, tmp_path, declared, message): root = build(tmp_path / "b", {"envs/t/env.toml": f"id = {declared}\n"}) assert message in problems(root)[0] + def test_an_hf_id_is_declared_only_under_a_hub_bundles_own_root(self, tmp_path): + root = build(tmp_path / "b", {"envs/t/env.toml": 'id = "@hf/acme/ds/x/tickets"\n', + "envs/u/env.toml": 'id = "@local/acme/u"\n'}) + hub = parse_bundle(root, id_root="@hf/acme/ds/x") + assert [entry.id for entry in hub.entries] == ["@hf/acme/ds/x/tickets", "@local/acme/u"] + for refused_in in ({}, {"id_root": "@hf/acme/ds/y"}, {"id_root": "@local/acme/ds/x"}): + (problem,) = problems(root, **refused_in) + assert problem.startswith("envs/t/env.toml: id '@hf/acme/ds/x/tickets': an '@hf/' id names an entity of " + "a bundle run from the Hugging Face Hub") + assert parse_bundle(build(tmp_path / "c", {"envs/t/Dockerfile": ""}), + id_root="@hf/acme/ds").entries[0].id == "@hf/acme/ds/t" + def test_artifact_leaf_may_not_contain_double_underscore(self, tmp_path): root = build(tmp_path / "b", { "artifacts/x/artifact.toml": 'id = "@local/b/pdf__files"\n', "artifacts/x/a": "", diff --git a/tst/unit/bundle/resolve_test.py b/tst/unit/bundle/resolve_test.py index 7505fa9b4..23cec79f7 100644 --- a/tst/unit/bundle/resolve_test.py +++ b/tst/unit/bundle/resolve_test.py @@ -165,6 +165,16 @@ def test_an_output_id_derived_from_another_bundles_entity_is_named_under_this_ta assert [output.id for output in entry.outputs] == [entry.config[0]["cli_artifact_id"]] == [f"{ROOT}/t/crm__cli"] +def test_an_output_named_by_an_hf_id_is_refused_outside_a_hub_bundles_own_root(make): + folder = make(tasks={"t": [{"id": "snap", "type": "snapshot_env", "env_id": "tickets", + "snapshot_id": "@hf/acme/ds/t/snap"}]}) + assert problems(folder) == ("tasks/t.json: step 'snap': snapshot_id: '@hf/acme/ds/t/snap' is an '@hf/' id outside " + "this bundle's id root, which only a bundle run from the Hugging Face Hub writes " + "under its own",) + hub = resolved(parse_bundle(folder.root, id_root="@hf/acme/ds"), "t") + assert [output.id for output in hub.outputs] == ["@hf/acme/ds/t/snap"] + + def test_a_resolved_task_still_builds_its_steps(make): entry = resolved(make(tasks={"t": [ {"id": "deploy", "type": "deploy_env", "env_id": "tickets"}, diff --git a/tst/unit/explorer/explorer_app_test.py b/tst/unit/explorer/explorer_app_test.py index 86f55b9e1..797010a09 100644 --- a/tst/unit/explorer/explorer_app_test.py +++ b/tst/unit/explorer/explorer_app_test.py @@ -23,7 +23,7 @@ from agent_env.runner.runner import RunRecord, RunStatus from agent_env.store.object_store.local.store import LocalFilesystemObjectStore from agent_env.store.document_store.sqlite_document_store import LocalSqliteDocumentStore -from agent_env.store.routing import RoutingDocumentStore +from agent_env.store.routing import LocalNamespaceDocumentStore, RoutingDocumentStore from agent_env.task import Task from agent_env.explorer.routers import objects as objects_router from agent_env.explorer.routers import triggers as triggers_router @@ -436,8 +436,8 @@ def counted_list_runs(*args, **kwargs): def test_run_group_list_preserves_routed_instance_precedence(client, tmp_path): configured = get_config().get_document_store() - local = LocalSqliteDocumentStore(str(tmp_path / "local.db")) - router = RoutingDocumentStore(configured, local, tmp_path / "local.db") + local = LocalNamespaceDocumentStore(str(tmp_path / "local.db")) + router = RoutingDocumentStore(configured, {"local": local}) configured.insert("task_instances", {"instance_id": "shared", "current_step": 2, "total_steps": 4}) local.insert("task_instances", {"instance_id": "shared", "current_step": 7, "total_steps": 9}) run_store.insert_run(RunRecord( diff --git a/tst/unit/store/ids_test.py b/tst/unit/store/ids_test.py index 5482a69b6..985bfd3b2 100644 --- a/tst/unit/store/ids_test.py +++ b/tst/unit/store/ids_test.py @@ -106,6 +106,9 @@ def test_a_bare_id_over_200_bytes_encodes_to_a_frozen_segment_that_is_one_path_c ("tickets", None, False), ("user@example.com", None, False), ("@local/~/a", "local", True), + ("@hf/acme/ds", "hf", True), + ("@hf", "hf", False), + ("@huggingface/acme", "huggingface", False), ("@local", "local", False), ("@acme/tickets", "acme", False), ("@", "", False), @@ -132,6 +135,9 @@ def test_a_malformed_local_id_is_not_encoded_as_one(): "@local/~/Dropbox (Personal)/triage copy (2)", "@local/~/C++/R&D/a,b+c", "@local/~/node_modules/@types/x", + "@hf/acme/ds", + "@hf/Acme-Labs/ds_v1.2/bundles-x/" + "y" * 96, + "@hf/" + "x" * 4092, ]) def test_well_formed_local_ids_validate(entity_id): validate_local_id(entity_id) @@ -140,6 +146,9 @@ def test_well_formed_local_ids_validate(entity_id): @pytest.mark.parametrize("entity_id, reason", [ ("@local", "must start with"), ("@acme/x", "must start with"), + ("@hf", "must start with"), + ("@hf/a/../b", "'..'"), + ("@hf/" + "x" * 4093, "4097 bytes; the limit is 4096"), ("tickets", "must start with"), ("@local/", "empty"), ("@local//a", "empty"), @@ -185,6 +194,13 @@ def test_a_derived_id_keeps_its_base_namespace(): assert derive_id("tickets", "files") == "tickets__files" +def test_an_hf_id_encodes_to_a_local_segment_of_its_own(): + assert key_segment("@hf/acme/ds/hello") == image_repository("@hf/acme/ds/hello") == "local/hf-acme-ds-hello-" + ( + hashlib.sha256(b"@hf/acme/ds/hello").hexdigest()[:12]) + assert key_segment("@hf/acme/ds/hello") != key_segment("@local/hf/acme/ds/hello") + assert fs_safe("@hf/acme/ds/hello") == key_segment("@hf/acme/ds/hello").replace("/", "-", 1) + + def test_authored_cap_leaves_room_for_the_longest_derived_id(): at_cap = LOCAL_PREFIX + "a" * (MAX_AUTHORED_LOCAL_ID_BYTES - len(LOCAL_PREFIX)) validate_local_id(derive_id(derive_id(at_cap, "files"), "0" * 16)) diff --git a/tst/unit/store/local_id_acceptance_test.py b/tst/unit/store/local_id_acceptance_test.py index 49b6e602d..6f9664ede 100644 --- a/tst/unit/store/local_id_acceptance_test.py +++ b/tst/unit/store/local_id_acceptance_test.py @@ -61,10 +61,17 @@ def test_only_the_local_namespaces_store_takes_local_ids(tmp_path): with pytest.raises(ValueError, match="only the @local namespace's store holds"): store.check_id(LOCAL_ID) for refused, reason in (("registry-env", "isn't an @local id"), ("@other/x", "isn't an @local id"), - ("@local/a/../b", "'..' path segment")): + ("@hf/acme/ds/x", "isn't an @local id"), ("@local/a/../b", "'..' path segment")): with pytest.raises(ValueError, match=reason): namespace.check_id(refused) - assert not (tmp_path / "local.db").exists() + hub = LocalNamespaceDocumentStore(str(tmp_path / "hf.db"), namespace="hf") + hub.check_id("@hf/acme/ds/x") + with pytest.raises(ValueError, match="isn't an @hf id"): + hub.check_id(LOCAL_ID) + for store in (LocalSqliteDocumentStore(str(tmp_path / "documents.db")), FakeDocumentStore()): + with pytest.raises(ValueError, match="only the @hf namespace's store holds"): + store.check_id("@hf/acme/ds/x") + assert not (tmp_path / "local.db").exists() and not (tmp_path / "hf.db").exists() @pytest.mark.parametrize("entity_id", [ diff --git a/tst/unit/store/routing_test.py b/tst/unit/store/routing_test.py index 2d20b48a7..196b5925e 100644 --- a/tst/unit/store/routing_test.py +++ b/tst/unit/store/routing_test.py @@ -37,6 +37,8 @@ LOCAL_TASK = "@local/~/bundle/tasks/t" LOCAL_ENV = "@local/~/bundle/envs/e" +HUB_TASK = "@hf/acme/ds/hello/t" +HUB_ENV = "@hf/acme/ds/hello/e" # Building a store from a config table isn't a call on a store, so a wrapper has nothing to route. _NOT_ROUTED = {"from_config"} @@ -112,6 +114,37 @@ def test_outside_a_run_a_record_owned_by_an_local_entity_is_local(stores): assert _ids(configured, "runs", "run_id") == {"local-2"} +def test_hub_ids_and_runs_keep_to_the_hf_namespaces_own_file_inside_the_write_fence(stores): + router, configured, local = stores + hub = router.namespaces["hf"] + router.insert("envs", {"id": HUB_ENV, "version": 1}) + router.insert("envs", {"id": LOCAL_ENV, "version": 1}) + router.insert("runs", {"run_id": "hub-1", "task_id": HUB_TASK}) + configured.insert("artifacts", {"id": "shared", "version": 1}) + with run_scope(HUB_TASK): + router.insert("env_instances", {"instance_id": "rocket-1a2b", "env_id": "rocket"}) + with pytest.raises(LocalRunWriteError, match="an @hf task run writes only to the local stores"): + router.insert("artifacts", {"id": "cli-rocket", "version": 1}) + with pytest.raises(LocalRunWriteError): + router.update("artifacts", Filter.of(id="shared", version=1), UpdateSpec(set={"x": 1})) + with run_scope("registry-task"): + assert routing.local_run_namespace() == "hf" + with run_scope(LOCAL_TASK): + assert routing.local_run_namespace() == "local" + with pytest.raises(OutwardReferenceError, match="names a local id"): + router.insert("tasks", {"id": "t", "version": 1, "steps": [{"env_id": HUB_ENV}]}) + with pytest.raises(ValueError, match="has to keep its namespace"): + router.insert("envs", {"id": f"copy-of-{HUB_ENV}", "version": 1}) + + assert hub.path.name == "hf.db" and hub.path.parent == local.path.parent + assert _ids(hub, "envs") == {HUB_ENV} and _ids(local, "envs") == {LOCAL_ENV} + assert _ids(hub, "runs", "run_id") == {"hub-1"} + assert _ids(hub, "env_instances", "instance_id") == {"rocket-1a2b"} + assert _ids(router, "envs") == {HUB_ENV, LOCAL_ENV} + assert router.find_one("envs", Filter.of(id=HUB_ENV))["id"] == HUB_ENV + assert configured.count("env_instances", Filter()) == 0 and _ids(configured, "artifacts") == {"shared"} + + def test_a_run_started_inside_an_local_run_stays_local(cli_routing): with run_scope(LOCAL_TASK): with run_scope("registry-task"): @@ -282,6 +315,7 @@ def test_objects_route_by_their_entity_and_by_url(object_stores): assert config.get_object_store() is configured assert local is not configured and local.root == state_root() / "object_store" assert config.get_object_store_for("rocket") is configured + assert config.get_object_store_for(HUB_ENV) is local assert config.get_object_store_at(local.object_url("k")) is local assert config.get_object_store_at(configured.object_url("k")) is configured @@ -495,9 +529,10 @@ def test_latest_version_lookup_uses_the_routed_namespace(stores): assert versioned.get(local_id)["version"] == 3 -def test_the_local_namespace_file_cant_be_the_configured_store(tmp_path, cli_routing): - configure(document_store=LocalSqliteDocumentStore(str(state_root() / "document_store" / "local.db"))) - with pytest.raises(ConfigError, match="kept for @local documents"): +@pytest.mark.parametrize("namespace", ["local", "hf"]) +def test_a_local_namespace_file_cant_be_the_configured_store(tmp_path, cli_routing, namespace): + configure(document_store=LocalSqliteDocumentStore(str(state_root() / "document_store" / f"{namespace}.db"))) + with pytest.raises(ConfigError, match=f"kept for @{namespace} documents"): get_config().get_document_store() From c249de40ae0ab30999bb1e8677b270306f003048 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 16:56:52 -0800 Subject: [PATCH 02/13] feat(bundle): lock huggingface_hub, document hf:// runs, and test one against a pinned public dataset Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- README.md | 17 +++++++++++++++ src/agent_env/bundle/hub.py | 18 ++++++++-------- src/agent_env/store/routing.py | 6 +++--- tst/integration/cli/run_bundle_test.py | 29 ++++++++++++++++++++++++++ tst/unit/bundle/hub_test.py | 1 - tst/unit/bundle/plan_test.py | 2 +- tst/unit/store/routing_test.py | 2 +- uv.lock | 2 ++ 9 files changed, 64 insertions(+), 17 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index bf0168d1a..a4a1ea046 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -119,14 +119,14 @@ and deserialized through a registry. | `env/` | `Env` base class: a subclass declares `type`, implements `from_dict` and `async deploy(...) -> DeployedEnv`, and may override `reset`, `close` and `load_file_artifact_universe`; teardown closes each env a run deployed before terminating its sandboxes by id. Built-ins in `env/envs/`: `MCPServerEnv` (`mcp_server`), `MultiEnv` (`multi`, MCP servers plus websites), `ServiceDBEnv` (`service_db`, PostgreSQL), `WebsiteEnv` (`website`), `GatewayEnv` (`gateway_server`), `ComputerDriverEnv` (`computer_driver`, `env/envs/computer/driver_env.py`: a computer driver's files and what its class declares for each platform it supports, read at put in a process of its own, `agentenv_protocol.computer._declare`, where the driver's own dependencies needn't be installed, with the `port` it serves on and its install by OS, `[install.]` (`env/envs/computer/utils/install.py`): files, apt `packages` with extra `sources`, a `python` requirements file for a venv, `downloads` by URL and sha256, `run` scripts, the services the base's init runs, and `ports`; it deploys only as a computer env's driver), `ComputerEnv` (`computer`, in `env/envs/computer/`: a base computer and a driver, matched at put by gate 1 before anything is stored, then composed by `composer.py` into an image for each form and arch on a builder of the put's sandbox provider, by default the env's (`utils/builder.py`; `--sandbox` on `env build`, `bundle register` and `run`): this machine's Docker for `local`, else a VM of the provider, which fetches the put's stored files by signed URL, takes the rest inline and pushes to the image store itself, so it builds only with stores a remote VM reaches; each arch goes to the first link that runs it natively, else one that emulates it (a VM registers QEMU; `--no-emulate` leaves it `skipped` instead, for a later put), and its pair records the provider that built it (`builder`); a run or register that reuses a computer env composes what it has left (`pending`); tagged `agentenv-computers:` in the image store by a key of the image's Dockerfile, build context and lock so any env with the same inputs reuses it; every layer's packages, requirements and downloads are resolved at put in a container of the base for each arch (`utils/packages.py`, `utils/resolve_in_base.py`), stored in the object store by sha256 under `agentenv-packages/`, and listed in a lock stored by a digest of its inputs, which any env with the same inputs reuses, and builds install only the lock's files, every install step `RUN --network=none`; each pair's status goes into its metadata, `composed`, the arches built are written as its computer, a `computer` artifact over one index, and `agent-env env build` resumes what a put left; gate 3 then boots each native build on the put's sandbox provider and probes it at `full`, making it `ready` or `failed`, and a build no provider runs natively stays `unverified`. Its deploy, through the `computer` env provider (`providers/env_providers/env_computer_provider.py`), boots its first `ready` build on a sandbox provider that runs its arch natively (`native_platforms()`), with no gateway in front and the driver's card as the env card, then checks its `display`; `deploy_env` `env_params` take a `form`, `computer_settings` (deploy-phase values of the base's settings; a secret one, here or on the env, only as a bare `secret:`/`env:` reference) and `driver_settings` (the driver's `ComputerSettings`, such as its default coordinate space); a `network` that allows nothing runs it on local Docker's internal network behind a relay (`providers/sandbox_providers/local_docker/relay/`); `reset` and `close` have the driver end its recordings first (`computer-recording/v1` `end_all`, each uploading what's left and its manifest under the grant it started with), and `reset` then boots a fresh container in its place (`reset_container`), keeping the computer's address but not an agent's MCP session to the old driver; its `load_file_artifact_universe` writes files under the session user's home through the driver's `data/add`, `Downloads` by default, each file's bytes through a read grant when the store's grants reach it and its network allows everything, else inline (`env/envs/computer/utils/inside.py`)), `ComputerAppEnv` (`computer_app`, `env/envs/computer/app_env.py`: an app on a computer, a `ComputerAppEnvironment` class its env.toml's `module` names, read at put by the same `_declare`, which declares each OS with its own decorator (`@linux(...)`, or `@install(os_type, spec)` for any OS type) holding its install, its launch and, optionally, a skill and settings and build settings that add to the class's there, and install steps and ready checks for every OS or those their `os=` names; everything that reads its settings, launch or steps asks for the computer's OS type; deployed alone it is a one-app computer env, a `ComputerEnv` of the computer and driver its env.toml names, the composer installing it under `/opt/agentenv/apps//` with its services as supervisord programs beside its own server (`agentenv_protocol.computer.serve_app`, in a venv with the protocol's `app` extra, on `.agentenv/server.sock` in its folder), which adds its launcher, launches a `boot` app once per boot and says through `computer-app/v1` whether it's ready, as the driver's `computer-ready` asks, its card named after it, and `env_params` `app_settings` handing it values of its `Settings` in `AGENTENV_APP_SETTINGS`, a secret one only as a bare reference; its env.toml's `build_settings`, values of its `BuildSettings` checked at put, are written to `.agentenv/build_settings.json` in its folder, so they're part of the image's key, its install steps read them, and a deploy can't change them). An env type whose services take environment data defines `load_environment_artifact` and `load_environment_universe_artifact`; `load_artifact` refuses an environment artifact aimed at a type that doesn't, at save and before it loads. An env built on the protocol SDK can serve agents a skill (`@agent_skill`, `skill/v1`), which `add_skills` `env_ids` installs under its environment name (behind a gateway, each server's under that server's name); a skill with files beside its SKILL.md goes as a bundle stored once per content under `agentenv-env-skills/` in the object store. `env/gateway/gateway.py` is the env-side gateway server, container-only code. While its clock is armed it serves agents `get_time` (`env/gateway/get_time.py`), which answers in UTC and, given an IANA `time_zone`, also in that zone; executors' working `hours` (`env/gateway/triggers.py`) take a fixed `utc_offset` or an IANA `time_zone`. | | `artifact/` | Immutable versioned artifacts; every `put()` creates a new version. The document goes to the document store, binary payloads to the object store. Every artifact but an environment universe, whose `metadata` key holds its legacy metadata file ids, carries a `metadata` dict outside its versioned fields, changed by compare-and-swap (`merge_metadata`). Built-ins: `CliArtifact` (`cli`), `ComputerArtifact` (`computer`, a base computer: its architectures, OS, session user, settings and container form, whose image `put` reads from its registry into `metadata.validated_images`), `FileArtifact` (`file`), `FileArtifactUniverse` (`file_artifact_universe`), `DockerImageArtifact` (`docker_image`), `EnvironmentArtifact` (`environment`: its environment's files by path, stored as the one wrapped `file_artifact_ref` every environment artifact had when it has one file at its filename, else as `file_artifact_refs`; `get_file_artifacts()` reads either, and a load hands each file to the env's `data/add` with its path, which a pre-v1 load, taking one file, refuses), `EnvironmentUniverseArtifact` (`environment_universe`), `SkillArtifact` (`skill`); `[artifacts] type_aliases` maps another stored spelling of a type onto one of them. A `DockerImageArtifact` holds a tar.gz of its image, or, written by `put_ref` (`--image-ref` on `env mcp-server put` and `a2a-agent put`), only a registry reference with its tag pinned to a digest, which sandboxes pull, or a published image's tag with its `image_digest` and its build context, which sandboxes pull by the digest and build from the context when they can't (the infra images, and a bundle agent built from a shipped agent's sources, `Plan.published`), or, written by `put_context`, only its build context, which a VM sandbox builds before running it (`VmSandbox.load_docker_images`) and Modal builds before creating its container (`ModalSandboxProvider.prepare_image`, outside a fallback chain's deadline); anything else that runs images by name (a container on the local provider, a lone server on any provider but Modal) refuses such an image. A build context (`utils/docker/context.py`) is what `docker build` would send, `.dockerignore` applied, written so equal files give equal bytes and an equal `source_digest`. | | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, `start_computer_recording` and `end_computer_recording`, and more). On an env whose card serves computer-exec/v1 (a computer), `run_code` with `env_id` and `run_container_unit_tests_verifier` with `env_id` run inside it as its session user through the driver: files staged with `data/add` and read with `data/get`, commands run with `run`; `env_outcome_verifier` hands a `verify` that takes an `env_url` keyword the env's address. A computer's framework-only routes (everything but MCP and the card) take the per-deploy control token its provider boots it with (`DeployedEnv.control_token`, `AGENTENV_CONTROL_TOKEN`); framework calls send `deployed.control_headers`, `Task.run` runs every step under `protocol_v1.authorized` with each deployed env's (a `verify` included), and a `run_code` script inside the computer gets `input["env"]["headers"]`. `teardown_run` and the `teardown_sandboxes` step share one core (`task/teardown.py` `tear_down`): the recordings the run left open end with fresh grants, each env still up closes (`Env.close`; a computer env ends any recording still open), what stayed open is registered from its namespace, then every sandbox is terminated by id. `task/episode/` runs a task with some of its agents played from outside, as a training loop does (`EpisodeSpec`). | -| `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the `@local` namespace: while the CLI or a `build_app` server runs, an `@local/…` id and what an `@local` task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | +| `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB, DynamoDB, Firestore with MongoDB compatibility), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. `store/routing.py` is the local namespaces, `@local` and `@hf` (`is_local_id`), each with a documents file of its own: while the CLI or a `build_app` server runs, a local id and what a local task's run records go to local stores beside the configured ones, and that run's writes to the configured stores are refused. A new backend must pass the conformance kits in `tst/store/`. | | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. | | `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `smol_vm` (the `smol` extra), `tensorlake` (the `tensorlake` extra), `tenki` (the `tenki` extra), `vercel` (the `vercel` extra), `boxd` (the `boxd` extra), `coreweave` (the `coreweave` extra), `daytona` (the `daytona` extra), `boat` (the `boat` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store), `EnvironmentGatewayProvider` (`gateway`, an env's default), which renders a docker-compose for the gateway and its MCP servers inside the sandbox, and `EnvironmentServerProvider` (`server`), one MCP server in its own container with no gateway in front and no state store; `providers/env_state/` holds env-state providers (`local_postgres` built in). | | `a2a_agent/` | The `A2AAgent` entity (`a2a_agent`), its stores and the validator steps. The protocol package provides the agent-side framework. | | `openenv/` | `build_app(bundle)`: the [OpenEnv](https://github.com/openenv/OpenEnv) server for a bundle's tasks, each session playing one episode through the Episode API, with the envs' MCP tools, an added `end_turn` tool and the task's verifier score as the reward. It prepares bundles as `agent-env run` does (`prepare_bundle(played=True)`, without the agent the client plays) and keeps namespace routing on for the app's life. `openenv` comes with the `openenv` extra, a floor with no cap; this package and `training/` are the only ones that import it. Each task's description in the Task API names its `toolset` (the played agent's envs, role, and whether it gets `bash`). With `token`, every request and WebSocket but `/health` must carry `Authorization: Bearer `, for a server others reach. | | `training/` | One module per outside trainer, each behind its own extra; `_openenv.py`, the session on a server `openenv/` makes that each helper plays an episode through (`Session`: a background start, tool calls, the turns left open ended until scored), with the Task API reads, the per-toolset probe, `tasks_and_tools` (what the row writers read: the tasks, the split names checked, each toolset's tools) and an in-process `Server`; and `_tools.py`, which turns an env's tools into typed, documented Python methods from their JSON Schema, as transformers' `get_json_schema` reads them. `trl.py` (`trl` extra): `AgentEnv(url)` over a server `openenv/` makes, with `dataset` (a row per task) and `factory` (TRL's `environment_factory`: a generated class per toolset whose methods are the tools, a background `reset` and an owned `get_reward`); `AgentEnv.local(bundle)` serves the bundle in process. `nemo_gym/` (`nemo-gym` extra): `python -m agent_env.training.nemo_gym SOURCE ROOT` writes a bundle's NeMo Gym resources server into a Gym root, `NEMO_GYM_EXTRA_ROOTS`'s or a Gym checkout's (`app.py`; a `pyproject.toml`, since uv applies the nearest one's `[tool.uv]` and a checkout's `exclude-dependencies` would drop `cryptography`; a config with `simple_agent` and the `legacy_agent` environment server, and `data/.jsonl` rows with each task's tools); `server.AgentEnvResourcesServer` is that server, run in the venv Gym builds: a row's seed starts its episode, a catch-all route calls its tools, and `verify` ends the turns left open with the model's last reply and returns the reward, masking a rollout the environment failed; with `reward_components` it also returns each score by step id, their sum the reward, for NeMo RL's GDPO. `verl/` (`verl` extra, installed into verl's own environment): `python -m agent_env.training.verl URL OUT` writes a server's tasks as verl's `.parquet` rows (the task's prompt, `agent_name: agentenv`, the row's tools in `extra_info`) and `agent_loop.yaml`; `loop.AgentEnvAgentLoop` is verl's `ToolAgentLoop` with the row's tools, each call a call on the row's episode, the last reply ending the open turn and the episode's reward its `reward_score`, playing a rollout the env failed again on a new episode up to `attempts` times within `reset_timeout` of its start and raising `RolloutFailed` for one that fails every time, which verl's V1 trainer (the default) leaves out of the loss, the group's baseline and the eval reward; it hands verl's parsers typed tool schemas (`qwen3_coder` converts arguments by them), raises on a first prompt over `rollout.prompt_length`, which verl would cut from the left, and on an episode that ends unscored without failing, and warns once a tool of an answer verl cuts to `max_tool_response_length`. The writer reads a server that asks for a token with the one `--token-var` names, which the loop reads in each worker, and with `--tokenizer` counts each prompt with its tools as verl renders it (`--chat-template-kwargs`), refusing rows over `--max-prompt-length` before writing. `skyrl/` (`skyrl` extra, installed into SkyRL's own environment): `python -m agent_env.training.skyrl URL OUT --tokenizer MODEL` writes a server's tasks as SkyRL's `.parquet` rows (`env_class: agentenv`, the episode's split, index, task id and tools, the server's settings), each with its tools rendered into its system prompt by the model's chat template, checked to render as `tools=` would, since SkyRL's generator passes the template no tools, and refuses rows over `--max-prompt-length` counted as SkyRL's filter and generator count them; `calls.parse` reads a reply's tool calls in the `hermes` or `qwen3_coder` format, typed by the row's schema; `env.AgentEnvTextEnv` (registered as `agentenv`) is skyrl-gym's `BaseTextEnv` on the row's episode, run on the event loop the sessions share (`_openenv.on_session_loop`), its steps running the reply's calls and a reply without one ending the open turn, raising `EpisodeFailed` when the env fails it, with scores, tool calls and unparsed calls as its metrics; `generator.AgentEnvGenerator` is SkyRL's `SkyRLGymGenerator` playing a failed rollout again on a new episode up to `attempts` times within `reset_timeout`, and masking the group of one that fails every attempt in training (an eval scores it 0); `python -m agent_env.training.skyrl.train` (or `trainer.override_entrypoint=agent_env.training.skyrl.train`) runs SkyRL's trainer with both, its fully async one with `trainer.fully_async.enabled`, passing the rows' token variables to the trainer's Ray task. `miles/` (`miles` extra, installed into Miles' own environment): `python -m agent_env.training.miles URL OUT` writes a server's tasks as Miles' prompt data, `.jsonl` rows of chat `messages` and `metadata` (the episode's split, index, task id and tools, and the server it plays on with its deadlines and turn cap); `agent.run` is Miles' agent function (`--custom-agent-function-path`), which plays the row's episode, the model chatting through the sample's session URL with the row's tools, answers a call whose arguments aren't a JSON object as the model's error, and returns the reward, or flags an episode the env or the session failed, whose model missed the row's `model_timeout` or that ended without a reward, which isn't played again since the session holds its turns; `agent.reward` is the reward hook (`--custom-rm-path`) and `agent.keep_scored_groups` the dynamic-sampling filter that leaves out the group of a flagged episode, of a sample Miles aborted, or of one whose agent function never returned. It imports nothing of Miles. | | `runner/` | The `[runner]` seam: `Runner.submit()` returns a `RunHandle` (`run_id`, `instance_id`) before the run starts; `LocalRunner` is built in. `runner/store.py` is the run index every runner writes and the explorer reads. | -| `bundle/` | A bundle is a folder of entities (`envs/`, `agents/`, `artifacts/`, `skills/`, `tasks/.json`, `evals/.toml`) that runs without a registry: `agent-env run` plans it, writes it into the `@local` namespace (reusing each version whose inputs haven't changed) and runs its evals or tasks; `agent-env bundle init` writes a stored task or eval as one, with every env, agent and artifact it uses in its own folder (each image as its build context, pruned to what its Dockerfile reads and pinned to the Python and npm versions of the stored image, or, given `--image-ref ID=REF`, as that registry image pinned by digest with its context beside it, `image = { ref, dockerfile }`, which a run pulls and builds only when it can't; `task_images` lists the images an import writes, downloading nothing, so an export can push them first) so it runs with no store; `agent-env bundle register` writes a folder into the configured stores under registry ids (``), each task and eval naming the versions it reads and each image it builds pushed, through a ledger in the target store so registering again writes only what changed. | +| `bundle/` | A bundle is a folder of entities (`envs/`, `agents/`, `artifacts/`, `skills/`, `tasks/.json`, `evals/.toml`) that runs without a registry: `agent-env run` plans it, writes it into the `@local` namespace (reusing each version whose inputs haven't changed) and runs its evals or tasks; `agent-env run hf:///[@][/]` (`bundle/hub.py`) runs one from a Hugging Face Hub dataset, picked from the commit's file list (the path, else `bundles/`, else the root) and downloaded alone with the card into a folder per commit, under ids rooted at `@hf//`, which only that path produces, each run's context recording the commit (`metadata["bundle_source"]`); `agent-env bundle init` writes a stored task or eval as one, with every env, agent and artifact it uses in its own folder (each image as its build context, pruned to what its Dockerfile reads and pinned to the Python and npm versions of the stored image, or, given `--image-ref ID=REF`, as that registry image pinned by digest with its context beside it, `image = { ref, dockerfile }`, which a run pulls and builds only when it can't; `task_images` lists the images an import writes, downloading nothing, so an export can push them first) so it runs with no store; `agent-env bundle register` writes a folder into the configured stores under registry ids (``), each task and eval naming the versions it reads and each image it builds pushed, through a ledger in the target store so registering again writes only what changed. | | `eval/` | `Eval` (`eval`), a versioned list of task references, and its store. | | `plugins/` | Entry-point plugins (see Extension points): the group names, `inventory()`, which `agent-env plugin list/show/check` report, `load_failures` and `settings`. The top level is public; the underscored modules are internal. | | `explorer/` | Optional local web UI: `agent-env up`, binds loopback `:8234`. | diff --git a/README.md b/README.md index 93e469d7b..4afda26b9 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,23 @@ uv sync --extra dev source .venv/bin/activate ``` +## Run a bundle from the Hub + +`agent-env run` also takes a bundle published in a [Hugging Face Hub](https://huggingface.co/datasets) dataset: + +```bash +agent-env run hf:/// # the main branch +agent-env run hf:///@v1.0/bundles/hello # a tag, branch or commit, and the bundle's folder +``` + +The revision resolves to a commit, and only that commit's bundle and the dataset card are downloaded, into +`$XDG_CACHE_HOME/agentenv/hf///` (`~/.cache` by default), where a later run of the commit finds +them. Without a path, the run takes the only bundle under `bundles/`, or the one the card's `agentenv: default` names, +else the repo's root. It prints the repo, commit and bundle, then runs as a folder does, with ids under +`@hf//`, so a rerun of the same commit reuses what the last one wrote, and records the commit with each run. +A bundle without a `bundle.toml` needs the plugins the card's `agentenv: bundles: : plugins` lists. A gated or +private dataset needs `hf auth login` or `HF_TOKEN`. + ## Plugins A package of your own can add env types, providers, task steps and stores to agent-env, and `agent-env plugin` lists, checks, adds and removes installed plugins. The [plugin docs](https://www.agentenvframework.com/docs/plugins) show how. diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 243e3265a..5fae84353 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -23,7 +23,7 @@ import httpx from huggingface_hub import HfApi -from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError +from huggingface_hub.errors import RepositoryNotFoundError from huggingface_hub.repocard import metadata_load from agent_env.plugins._bundles import ALIASES_FILE @@ -60,8 +60,8 @@ class HubBundle: root: Path id_root: str source: Mapping[str, str] # hub, repo, revision, sha, path: what each run's context records - plugins: tuple[str, ...] = () # the card's, for a bundle without a bundle.toml - setup: str | None = None # the card's command that sets the bundle up, shown when a run fails, never run + plugins: tuple[str, ...] # the card's, for a bundle without a bundle.toml + setup: str | None # the card's command that sets the bundle up, shown when a run fails, never run def check_plugins(self) -> None: """Refuse the run when a plugin the card names isn't installed, naming the command that installs it. Raises @@ -123,14 +123,13 @@ def cache_root() -> Path: return (Path(base) if os.path.isabs(base) else Path.home() / ".cache") / "agentenv" / "hf" -def fetch_bundle(text: str, *, api: HfApi | None = None, on_progress: Callable[[str], None] | None = None) -> HubBundle: +def fetch_bundle(text: str, *, on_progress: Callable[[str], None] | None = None) -> HubBundle: """The bundle ``text`` names, downloaded unless this machine has it. ``on_progress`` is given the line that says which repo, commit and bundle it is, before anything is downloaded. Raises BundleError.""" reference = parse_reference(text) - api = api or HfApi() try: - return _fetch(reference, api, on_progress or (lambda line: None)) - except (HfHubHTTPError, httpx.TransportError) as e: + return _fetch(reference, HfApi(), on_progress or (lambda line: None)) + except httpx.HTTPError as e: error = BundleError([f"{text}: {str(e).strip() or type(e).__name__}"]) if isinstance(e, RepositoryNotFoundError): error.add_note("a private or gated repo needs a token with access: log in with `hf auth login`, or set " @@ -155,6 +154,7 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H commit.save(folder) card = _card(folder) path = _pick(reference, sha, commit.files, card) + id_root = _id_root(reference, path) source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, "path": path} say(describe(source)) if path not in commit.downloaded: @@ -165,8 +165,8 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H name = path.rpartition("/")[2] or reference.repo.partition("/")[2] needs = (card.get("bundles") or {}).get(name) or {} declared = _within(path, BUNDLE_TOML) in commit.files - return HubBundle(folder / path, _id_root(reference, path), source, - plugins=() if declared else tuple(needs.get("plugins") or ()), setup=needs.get("setup")) + return HubBundle(folder / path, id_root, source, plugins=() if declared else tuple(needs.get("plugins") or ()), + setup=needs.get("setup")) def _card(folder: Path) -> dict[str, Any]: diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index c49852148..1f66b08a8 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -476,10 +476,10 @@ def check_id(self, entity_id: str) -> None: if namespace is not None: self.namespaces[namespace].check_id(entity_id) return - if any(f"@{namespace}/" in entity_id for namespace in self.namespaces): + if (inside := next((namespace for namespace in self.namespaces if f"@{namespace}/" in entity_id), None)): raise ValueError( - f"id {entity_id!r} contains a local id but isn't one: an id derived from a local id " - "has to keep its namespace" + f"id {entity_id!r} contains an @{inside} id but isn't one: an id derived from an @{inside} id " + f"has to keep the @{inside} namespace" ) self.configured.check_id(entity_id) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 7d95138c7..4c7ad22e3 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -107,6 +107,35 @@ def test_the_shipped_hello_stays_local_when_config_defaults_to_another_provider( assert "tasks/hello.json v1: passed (hello: 1)" in _run("hello") +def test_a_hub_bundle_runs_by_its_hf_reference_at_a_pinned_commit_and_a_rerun_reuses_its_download(bundle, monkeypatch, + tmp_path, quiet_logs): + """The fixture is a public dataset holding the hello example under bundles/hello, another bundle that needs a + plugin under bundles/v3, and its recorded runs as tables at its root.""" + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "cache")) + repo, sha = "earakely-scale/agentenv-hf-smoke", "5fbd709207a73e2aeb73a77e272a61003d1be768" + + several = CliRunner().invoke(cli, ["run", f"hf://{repo}@{sha}"]) + first = _run(f"hf://{repo}@{sha}/bundles/hello") + second = _run(f"hf://{repo}@{sha}/bundles/hello") + + assert several.exit_code == 1 + assert (f"run one of hf://{repo}@{sha}/bundles/hello, hf://{repo}@{sha}/bundles/v3") in several.output + for output in (first, second): + assert output.startswith(f"{repo} at {sha[:12]}, bundle bundles/hello\n") + assert f"From {repo} at {sha[:12]}, bundle bundles/hello\nTasks:\n tasks/hello.json v1: passed" in output + assert "tasks/hello.json: v1, unchanged" in second + folder = tmp_path / "cache" / "agentenv" / "hf" / repo / sha + assert sorted(path.relative_to(folder).as_posix() for path in folder.rglob("*") + if path.is_file() and path.relative_to(folder).parts[0] != ".cache") == [ + "README.md", "bundles/hello/README.md", "bundles/hello/artifacts/greeting/check.sh", + "bundles/hello/artifacts/greeting/hello.txt", "bundles/hello/tasks/hello.json"] + (instance_id,) = re.findall(rf"instance (@hf/{repo}/hello/hello-\S+)", second) + with namespace_routing(): + instance = Task.get_instance(instance_id) + assert instance.context["metadata"]["bundle_source"] == { + "hub": "hf", "repo": repo, "revision": sha, "sha": sha, "path": "bundles/hello"} + + def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bundle, quiet_logs): first = _run(bundle) assert "tasks/hello.json v1: passed (hello: 1)" in first diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 9a073e8cc..60ff9d05e 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -17,7 +17,6 @@ from agent_env.bundle.hub import HubReference, fetch_bundle, parse_reference from agent_env.bundle.ledger import LEDGER_COLLECTION from agent_env.cli import cli -from agent_env.config import get_config from agent_env.config.runtime import Config from agent_env.store import Filter from agent_env.store.routing import namespace_routing diff --git a/tst/unit/bundle/plan_test.py b/tst/unit/bundle/plan_test.py index bce8ff7bc..b7d1288fe 100644 --- a/tst/unit/bundle/plan_test.py +++ b/tst/unit/bundle/plan_test.py @@ -311,7 +311,7 @@ def read(*_): def test_planning_a_read_of_another_bundles_local_id_without_namespace_routing_is_refused(make): - with pytest.raises(RuntimeError, match="reading an @local id from the store needs namespace routing"): + with pytest.raises(RuntimeError, match="reading a local id from the store needs namespace routing"): plan_bundle(make(tasks={"t": [deploy("@local/~/other/crm")]})) diff --git a/tst/unit/store/routing_test.py b/tst/unit/store/routing_test.py index 196b5925e..569d34f55 100644 --- a/tst/unit/store/routing_test.py +++ b/tst/unit/store/routing_test.py @@ -133,7 +133,7 @@ def test_hub_ids_and_runs_keep_to_the_hf_namespaces_own_file_inside_the_write_fe assert routing.local_run_namespace() == "local" with pytest.raises(OutwardReferenceError, match="names a local id"): router.insert("tasks", {"id": "t", "version": 1, "steps": [{"env_id": HUB_ENV}]}) - with pytest.raises(ValueError, match="has to keep its namespace"): + with pytest.raises(ValueError, match="has to keep the @hf namespace"): router.insert("envs", {"id": f"copy-of-{HUB_ENV}", "version": 1}) assert hub.path.name == "hf.db" and hub.path.parent == local.path.parent diff --git a/uv.lock b/uv.lock index f317ad5d4..d5ff7fadf 100644 --- a/uv.lock +++ b/uv.lock @@ -57,6 +57,7 @@ dependencies = [ { name = "e2b" }, { name = "fastapi" }, { name = "httpx" }, + { name = "huggingface-hub" }, { name = "jsonschema" }, { name = "litellm" }, { name = "mcp" }, @@ -221,6 +222,7 @@ requires-dist = [ { name = "griffelib", marker = "extra == 'dev'", specifier = "==2.3.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "httpx2", marker = "extra == 'vercel'", specifier = ">=2.13.1,<3" }, + { name = "huggingface-hub", specifier = ">=1.0" }, { name = "jsonschema", specifier = ">=4.20" }, { name = "litellm", specifier = ">=1.96.0,<1.97" }, { name = "mcp", specifier = ">=1.25,<2" }, From db7898ff02c4bae1cd3448cc540477039bcc11b4 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:18:45 -0800 Subject: [PATCH 03/13] fix(bundle): report the Hub's HTTP errors by huggingface_hub's own class, which 2.x bases on httpx2 Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- src/agent_env/bundle/hub.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2a841acc6..e1558a234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ "click>=8.0.0", "cryptography>=42", # the local transfer CA "httpx>=0.27.0", - "huggingface_hub>=1.0", # agent-env run hf:// resolves and downloads a Hub bundle; 1.0 moved it onto httpx + "huggingface_hub>=1.0", # agent-env run hf:// resolves and downloads a Hub bundle; checked on 1.0, 1.33 and 2.4 "jsonschema>=4.20", # checks an app's settings against its Settings' schema before a deploy "litellm>=1.96.0,<1.97", "mcp>=1.25,<2", # same floor as env/gateway/requirements.txt and agentenv-protocol; 2.0.0 removed mcp.server.fastmcp, imported at module scope by env/gateway diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 5fae84353..69275efdd 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -21,9 +21,8 @@ from typing import Any from urllib.parse import unquote -import httpx from huggingface_hub import HfApi -from huggingface_hub.errors import RepositoryNotFoundError +from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError from huggingface_hub.repocard import metadata_load from agent_env.plugins._bundles import ALIASES_FILE @@ -129,7 +128,7 @@ def fetch_bundle(text: str, *, on_progress: Callable[[str], None] | None = None) reference = parse_reference(text) try: return _fetch(reference, HfApi(), on_progress or (lambda line: None)) - except httpx.HTTPError as e: + except HfHubHTTPError as e: error = BundleError([f"{text}: {str(e).strip() or type(e).__name__}"]) if isinstance(e, RepositoryNotFoundError): error.add_note("a private or gated repo needs a token with access: log in with `hf auth login`, or set " From 7560afa3af9be4d1fb27f67760b2d447a91a4351 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 17:22:12 -0800 Subject: [PATCH 04/13] fix(bundle): lock a commit's download, refuse a card of the wrong shape, check a Hub bundle's plugins in OpenEnv, and take an unlistable parent as written Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/hub.py | 59 ++++++++++++++++++++----------- src/agent_env/bundle/installed.py | 12 ++++++- src/agent_env/openenv/app.py | 9 +++-- tst/unit/bundle/hub_test.py | 27 ++++++++++++++ tst/unit/bundle/installed_test.py | 7 ++++ 5 files changed, 90 insertions(+), 24 deletions(-) diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 69275efdd..72f01fbea 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -28,6 +28,7 @@ from agent_env.plugins._bundles import ALIASES_FILE from agent_env.plugins._requirements import unmet_problems from agent_env.store.ids import HF_PREFIX, validate_local_id +from agent_env.store.local_state import holding_locks from .parse import BUNDLE_TOML, BundleError, BundleKind from .requires import UnmetRequirements @@ -145,27 +146,27 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H sha = reference.revision if reference.revision and _SHA.fullmatch(reference.revision) else api.repo_info( reference.repo, repo_type="dataset", revision=reference.revision).sha folder = cache_root() / reference.repo / sha - commit = _Commit.read(folder) - if commit is None: - commit = _Commit(api.list_repo_files(reference.repo, repo_type="dataset", revision=sha)) - if CARD in commit.files: - api.hf_hub_download(reference.repo, CARD, repo_type="dataset", revision=sha, local_dir=folder) - commit.save(folder) - card = _card(folder) - path = _pick(reference, sha, commit.files, card) - id_root = _id_root(reference, path) - source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, "path": path} - say(describe(source)) - if path not in commit.downloaded: - api.snapshot_download(reference.repo, repo_type="dataset", revision=sha, local_dir=folder, - allow_patterns=_patterns(path)) - commit.downloaded.append(path) - commit.save(folder) - name = path.rpartition("/")[2] or reference.repo.partition("/")[2] - needs = (card.get("bundles") or {}).get(name) or {} + with holding_locks([f"{SCHEME}{reference.repo}@{sha}"]): + commit = _Commit.read(folder) + if commit is None: + commit = _Commit(api.list_repo_files(reference.repo, repo_type="dataset", revision=sha)) + if CARD in commit.files: + api.hf_hub_download(reference.repo, CARD, repo_type="dataset", revision=sha, local_dir=folder) + commit.save(folder) + card = _card(folder) + path = _pick(reference, sha, commit.files, card) + id_root = _id_root(reference, path) + plugins, setup = _needs(reference.repo, card, path.rpartition("/")[2] or reference.repo.partition("/")[2]) + source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, + "path": path} + say(describe(source)) + if path not in commit.downloaded: + api.snapshot_download(reference.repo, repo_type="dataset", revision=sha, local_dir=folder, + allow_patterns=_patterns(path)) + commit.downloaded.append(path) + commit.save(folder) declared = _within(path, BUNDLE_TOML) in commit.files - return HubBundle(folder / path, id_root, source, plugins=() if declared else tuple(needs.get("plugins") or ()), - setup=needs.get("setup")) + return HubBundle(folder / path, id_root, source, plugins=() if declared else plugins, setup=setup) def _card(folder: Path) -> dict[str, Any]: @@ -175,6 +176,24 @@ def _card(folder: Path) -> dict[str, Any]: return table if isinstance(table, dict) else {} +def _needs(repo: str, card: Mapping[str, Any], name: str) -> tuple[tuple[str, ...], str | None]: + """What the card's ``agentenv: bundles: `` table says the bundle needs: the plugins it requires, and the + command that sets it up. Raises BundleError for a value of the wrong shape.""" + where = f"{repo}'s card: agentenv: bundles" + bundles = card.get("bundles") or {} + if not isinstance(bundles, Mapping): + raise BundleError([f"{where} must map each bundle's name to what it needs"]) + needs = bundles.get(name) or {} + if not isinstance(needs, Mapping): + raise BundleError([f"{where}: {name} must be a table of plugins and setup"]) + plugins, setup = needs.get("plugins") or [], needs.get("setup") + if not isinstance(plugins, list) or not all(isinstance(plugin, str) for plugin in plugins): + raise BundleError([f"{where}: {name}: plugins must be a list of pip requirements"]) + if setup is not None and not isinstance(setup, str): + raise BundleError([f"{where}: {name}: setup must be a command"]) + return tuple(plugins), setup + + def _pick(reference: HubReference, sha: str, files: Sequence[str], card: Mapping[str, Any]) -> str: """The folder in the repo of the bundle ``reference`` names, ``""`` for the root: its path; else the only bundle under ``bundles/``, or the one the card names its default; else the root when it is a bundle. ``bundles/`` comes diff --git a/src/agent_env/bundle/installed.py b/src/agent_env/bundle/installed.py index b158373a2..3435fd8e8 100644 --- a/src/agent_env/bundle/installed.py +++ b/src/agent_env/bundle/installed.py @@ -61,12 +61,22 @@ def _spelled_folder(text: str) -> bool: return False above = Path() for part in Path(text).parts: - if part not in (".", "..") and _nfc(part) not in {_nfc(name) for name in os.listdir(above)}: + if part not in (".", "..") and not _lists(above, part): return False above /= part return True +def _lists(folder: Path, name: str) -> bool: + """Whether ``folder`` lists ``name`` as spelled; a folder that can be passed through but not read takes it as + written.""" + try: + names = os.listdir(folder) + except OSError: + return True + return _nfc(name) in {_nfc(listed) for listed in names} + + def _nfc(name: str) -> str: return unicodedata.normalize("NFC", name) diff --git a/src/agent_env/openenv/app.py b/src/agent_env/openenv/app.py index 56211c9c6..7999867f5 100644 --- a/src/agent_env/openenv/app.py +++ b/src/agent_env/openenv/app.py @@ -38,8 +38,9 @@ def build_app( max_concurrent_envs: int = 8, token: str | None = None, ) -> FastAPI: - """The OpenEnv server for ``bundle``'s tasks: a folder or an installed bundle's name, as ``agent-env run BUNDLE`` - reads it, served as the split ``all`` and a split per eval; or a mapping of split name to bundle. Each session + """The OpenEnv server for ``bundle``'s tasks: a folder, an installed bundle's name or an ``hf://`` reference, as + ``agent-env run BUNDLE`` reads it, served as the split ``all`` and a split per eval; or a mapping of split name to + bundle. Each session plays one episode of a task at a time, deploying where ``sandbox`` names, as ``agent-env run --sandbox`` does. ``reward`` makes the reward of a task's scores: a step id, or a function of the scores by step id; without it a @@ -103,7 +104,9 @@ def _sources(bundle: _Source | Mapping[str, _Source]) -> dict[str | None, _Sourc def _prepared(source: _Source, sandbox: str | None) -> PreparedBundle: """``source`` prepared with every task and eval selected, so ``all`` holds every task, evals or not.""" - root, id_root = (source, None) if isinstance(source, Path) else locate_bundle(source)[:2] + root, id_root, hub = (source, None, None) if isinstance(source, Path) else locate_bundle(source) + if hub is not None: + hub.check_plugins() entries = parse_bundle(Path(root), id_root=id_root).entries return prepare_bundle( root, tasks=[entry.name for entry in entries if entry.kind is BundleKind.TASK], diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 60ff9d05e..54d71576f 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -16,8 +16,10 @@ from agent_env.bundle import hub as hub_module from agent_env.bundle.hub import HubReference, fetch_bundle, parse_reference from agent_env.bundle.ledger import LEDGER_COLLECTION +from agent_env.bundle.requires import UnmetRequirements from agent_env.cli import cli from agent_env.config.runtime import Config +from agent_env.openenv import build_app from agent_env.store import Filter from agent_env.store.routing import namespace_routing from agent_env.task.store import task_instances @@ -285,6 +287,31 @@ def test_a_bundle_without_a_bundle_toml_needs_the_plugins_its_card_names(hub, lo assert declared.exit_code == 0, declared.output +@pytest.mark.parametrize("bundles, problem", [ + (["hello"], "me/ds's card: agentenv: bundles must map each bundle's name to what it needs"), + ({"hello": "agentenv-x"}, "me/ds's card: agentenv: bundles: hello must be a table of plugins and setup"), + ({"hello": {"plugins": "agentenv-x"}}, "me/ds's card: agentenv: bundles: hello: plugins must be a list of pip " + "requirements"), + ({"hello": {"setup": ["agent-env", "x"]}}, "me/ds's card: agentenv: bundles: hello: setup must be a command"), +]) +def test_a_card_that_says_what_a_bundle_needs_in_the_wrong_shape_is_refused(hub, bundles, problem): + hub.layout({"bundles/hello/tasks/t.json": TASK}) + hub.card({"bundles": bundles}) + + with pytest.raises(BundleError) as caught: + fetch_bundle("hf://me/ds") + + assert caught.value.problems == (problem,) + + +def test_an_openenv_server_of_a_hub_bundle_needs_the_plugins_its_card_names_too(hub): + hub.layout({"bundles/hello/tasks/t.json": TASK}) + hub.card({"bundles": {"hello": {"plugins": ["agentenv-not-installed>=1"]}}}) + + with pytest.raises(UnmetRequirements, match="agent-env plugin add 'agentenv-not-installed>=1'"): + build_app("hf://me/ds") + + def test_a_failed_run_shows_the_cards_setup_command_and_never_runs_it(hub, local_stores, quiet_logs): hub.layout({"bundles/hello/tasks/t.json": json.dumps([{"id": "x", "type": "a_step_only_setup_registers"}])}) hub.card({"bundles": {"hello": {"setup": "agent-env demo setup"}}}) diff --git a/tst/unit/bundle/installed_test.py b/tst/unit/bundle/installed_test.py index 88bfdded1..a006092b5 100644 --- a/tst/unit/bundle/installed_test.py +++ b/tst/unit/bundle/installed_test.py @@ -344,6 +344,13 @@ def test_a_folder_wins_only_when_spelled_as_written_so_desktop_isnt_a_case_insen assert locate_bundle("demo-bundles/desktop").root == site / "demo_bundles" / "desktop" assert locate_bundle("Desktop") == (Path("Desktop"), None, None) assert locate_bundle("demo-bundles/Desktop") == (Path("demo-bundles/Desktop"), None, None) + listed = os.listdir + monkeypatch.setattr(os, "listdir", lambda path: _refused() if Path(path) == Path("demo-bundles") else listed(path)) + assert locate_bundle("demo-bundles/Desktop").root == Path("demo-bundles/Desktop") + + +def _refused(): + raise PermissionError("a folder that can be passed through but not listed") def test_the_cli_reports_installed_metadata_it_cant_read(monkeypatch): From beb6c8c86f0ab8624b0fc8a548fdf8f581a60d53 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 18:54:21 -0800 Subject: [PATCH 05/13] test(verifiers): pass --no-push only to a vf-eval that has it verifiers main removed the eval's platform push (PrimeIntellect-ai/verifiers#3046), and with it the --no-push flag, which its config now rejects as an extra input. 0.4.0 still pushes by default, so the taskset test passes --no-push only when vf-eval --help lists it: both the pinned and the main jobs run the same test. Co-Authored-By: Claude Opus 5.5 --- tst/verifiers/taskset_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tst/verifiers/taskset_test.py b/tst/verifiers/taskset_test.py index c3fbc9e28..3e86089de 100644 --- a/tst/verifiers/taskset_test.py +++ b/tst/verifiers/taskset_test.py @@ -31,6 +31,8 @@ def test_vf_eval_scores_a_played_row_and_fails_a_masked_one(serve, tmp_path, tea row = json.loads((folder / "data" / "all.jsonl").read_text().splitlines()[0]) (tmp_path / "rows.jsonl").write_text(json.dumps(row) + "\n" + json.dumps({**row, "task_id": "b"}) + "\n") port, model = _free_port(), _model() + vf_help = subprocess.run([VF_EVAL, "--help"], capture_output=True, text=True, timeout=60) + no_push = ["--no-push"] if "--no-push" in vf_help.stdout + vf_help.stderr else [] served = subprocess.Popen([sys.executable, "-m", "agent_env.training.nemo_gym", "serve", str(folder), "--port", str(port), "--masked-as-error"], @@ -38,7 +40,7 @@ def test_vf_eval_scores_a_played_row_and_fails_a_masked_one(serve, tmp_path, tea try: url = _up(served, f"http://127.0.0.1:{port}") (tmp_path / "eval.toml").write_text(_config(tmp_path, url, model.server_address[1])) - evaluated = subprocess.run([VF_EVAL, "@", str(tmp_path / "eval.toml"), "-n", "2", "-r", "1", "--no-push"], + evaluated = subprocess.run([VF_EVAL, "@", str(tmp_path / "eval.toml"), "-n", "2", "-r", "1", *no_push], env={**os.environ, "STAND_IN_API_KEY": "unused"}, capture_output=True, text=True, timeout=600) finally: From e751ec98e03154222c01cabf3f97e59111074f64 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:08:54 -0800 Subject: [PATCH 06/13] fix(bundle): check a Hub bundle's requirements before downloading it, show the card's setup only for what setup fixes, and group missing store entities The plan raises NotInStore (a BundleError carrying the missing agents, envs and artifacts) when the store lacks one, and preflight does for a missing default agent; the run attaches the card's setup command only to that or to UnmetRequirements. Each missing entity is one problem at its first use, with how many more use it. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 ++- src/agent_env/bundle/hub.py | 39 +++++++++++++++++-- src/agent_env/bundle/parse.py | 64 +++++++++++++++++-------------- src/agent_env/bundle/plan.py | 32 ++++++++++++++-- src/agent_env/bundle/preflight.py | 6 ++- src/agent_env/cli/run.py | 27 +++++++------ tst/unit/bundle/hub_test.py | 58 ++++++++++++++++++++++------ tst/unit/bundle/plan_test.py | 19 ++++++++- 8 files changed, 183 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 4afda26b9..c808221ae 100644 --- a/README.md +++ b/README.md @@ -58,8 +58,9 @@ The revision resolves to a commit, and only that commit's bundle and the dataset them. Without a path, the run takes the only bundle under `bundles/`, or the one the card's `agentenv: default` names, else the repo's root. It prints the repo, commit and bundle, then runs as a folder does, with ids under `@hf//`, so a rerun of the same commit reuses what the last one wrote, and records the commit with each run. -A bundle without a `bundle.toml` needs the plugins the card's `agentenv: bundles: : plugins` lists. A gated or -private dataset needs `hf auth login` or `HF_TOKEN`. +A bundle without a `bundle.toml` needs the plugins the card's `agentenv: bundles: : plugins` lists; either is +checked before the bundle is downloaded, and a run refused for a missing plugin, agent, env or artifact names the +card's `setup` command. A gated or private dataset needs `hf auth login` or `HF_TOKEN`. ## Plugins diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 72f01fbea..1493d9798 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -30,7 +30,7 @@ from agent_env.store.ids import HF_PREFIX, validate_local_id from agent_env.store.local_state import holding_locks -from .parse import BUNDLE_TOML, BundleError, BundleKind +from .parse import BUNDLE_TOML, BundleError, BundleKind, read_requires from .requires import UnmetRequirements SCHEME = "hf://" @@ -61,7 +61,12 @@ class HubBundle: id_root: str source: Mapping[str, str] # hub, repo, revision, sha, path: what each run's context records plugins: tuple[str, ...] # the card's, for a bundle without a bundle.toml - setup: str | None # the card's command that sets the bundle up, shown when a run fails, never run + setup: str | None # the card's command that sets the bundle up, shown when setting up could fix a run, never run + + @property + def setup_note(self) -> str | None: + """What a run refused for something setting the bundle up provides ends with: the card's command for it.""" + return f"{self.source['repo']}'s card sets this bundle up with: {self.setup}" if self.setup else None def check_plugins(self) -> None: """Refuse the run when a plugin the card names isn't installed, naming the command that installs it. Raises @@ -159,14 +164,40 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H plugins, setup = _needs(reference.repo, card, path.rpartition("/")[2] or reference.repo.partition("/")[2]) source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, "path": path} + declared = _within(path, BUNDLE_TOML) in commit.files + hub = HubBundle(folder / path, id_root, source, plugins=() if declared else plugins, setup=setup) say(describe(source)) if path not in commit.downloaded: + if declared: + api.hf_hub_download(reference.repo, _within(path, BUNDLE_TOML), repo_type="dataset", revision=sha, + local_dir=folder) + _refuse_unmet(hub, declared) api.snapshot_download(reference.repo, repo_type="dataset", revision=sha, local_dir=folder, allow_patterns=_patterns(path)) commit.downloaded.append(path) commit.save(folder) - declared = _within(path, BUNDLE_TOML) in commit.files - return HubBundle(folder / path, id_root, source, plugins=() if declared else plugins, setup=setup) + return hub + + +def _refuse_unmet(hub: HubBundle, declared: bool) -> None: + """Refuse the run before the bundle is downloaded when what it requires isn't installed: its bundle.toml's + requires, by a folder's rules, once that file alone is here, else the plugins the card names. The refusal ends + with the card's setup command. Raises UnmetRequirements, or BundleError for a bundle.toml a folder's run refuses.""" + if declared: + requires, refused = read_requires(hub.root / BUNDLE_TOML) + if refused: + raise BundleError([f"{BUNDLE_TOML}: {problem}" for problem in refused]) + problems, unmet = unmet_problems(requires, BUNDLE_TOML) + else: + problems, unmet = unmet_problems(hub.plugins, f"{hub.source['repo']}'s card") + if not problems: + return + if not unmet: + raise BundleError(problems) + error = UnmetRequirements(problems, unmet) + if hub.setup_note: + error.add_note(hub.setup_note) + raise error def _card(folder: Path) -> dict[str, Any]: diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index 70b287afe..e399d0e35 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -201,35 +201,10 @@ def _requires(self) -> tuple[str, ...] | None: version, not by URL; None when it has no bundle.toml.""" if self.bundle_toml is None: return None - try: - data = tomllib.loads(self.bundle_toml.read_text(encoding="utf-8")) - except OSError as e: - self._problem(self.bundle_toml, f"can't be read: {os_reason(e)}") - return () - except (UnicodeDecodeError, tomllib.TOMLDecodeError) as e: - self._problem(self.bundle_toml, f"isn't valid TOML: {e}") - return () - for key in sorted(data.keys() - {"requires"}): - self._problem(self.bundle_toml, f"{key!r} isn't a key it takes; it takes requires") - requires = data.get("requires", []) - if not isinstance(requires, list) or not all(isinstance(line, str) for line in requires): - self._problem(self.bundle_toml, 'requires must be a list of pip requirements, such as ' - '["agentenv-browser>=1.2"]') - return () - for line in requires: - if not line.isprintable(): - self._problem(self.bundle_toml, f"requires: {line!r} holds a character that can't be printed, so it " - "can't be shown as written before it's installed") - continue - try: - requirement = Requirement(line) - except InvalidRequirement as e: - self._problem(self.bundle_toml, f"requires: {line!r} isn't a pip requirement: {e}") - continue - if requirement.url: - self._problem(self.bundle_toml, f"requires: {line!r} names a URL; name the package and the versions " - "it needs, which install from wherever whoever runs the bundle installs from") - return tuple(requires) + requires, problems = read_requires(self.bundle_toml) + for problem in problems: + self._problem(self.bundle_toml, problem) + return requires def _derive_id_root(self) -> str: id_root, problems = _id_root(self.root) @@ -621,6 +596,37 @@ def _rel(self, path) -> str: return show(text) +def read_requires(path: Path) -> tuple[tuple[str, ...], list[str]]: + """The plugin packages the bundle.toml at ``path`` requires, and each problem with it: a requirement must name a + package by name and version, not by URL.""" + problems = [] + try: + data = tomllib.loads(path.read_text(encoding="utf-8")) + except OSError as e: + return (), [f"can't be read: {os_reason(e)}"] + except (UnicodeDecodeError, tomllib.TOMLDecodeError) as e: + return (), [f"isn't valid TOML: {e}"] + for key in sorted(data.keys() - {"requires"}): + problems.append(f"{key!r} isn't a key it takes; it takes requires") + requires = data.get("requires", []) + if not isinstance(requires, list) or not all(isinstance(line, str) for line in requires): + return (), [*problems, 'requires must be a list of pip requirements, such as ["agentenv-browser>=1.2"]'] + for line in requires: + if not line.isprintable(): + problems.append(f"requires: {line!r} holds a character that can't be printed, so it can't be shown as " + "written before it's installed") + continue + try: + requirement = Requirement(line) + except InvalidRequirement as e: + problems.append(f"requires: {line!r} isn't a pip requirement: {e}") + continue + if requirement.url: + problems.append(f"requires: {line!r} names a URL; name the package and the versions it needs, which " + "install from wherever whoever runs the bundle installs from") + return tuple(requires), problems + + def foreign_hub_id(entity_id: str, id_root: str) -> bool: """Whether ``entity_id`` is an ``@hf`` id outside ``id_root``. Only ``agent-env run hf://`` roots a bundle in the ``@hf`` namespace, so an id there says the entity came from the Hub; a bundle may declare one only under its own diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index c9f153fbe..f513abff2 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -19,7 +19,7 @@ import copy import heapq import unicodedata -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, field, replace from pathlib import Path from typing import TYPE_CHECKING, Any @@ -104,6 +104,17 @@ def _stored_env(id: str, version: int | None) -> Env: _ENV_FROM_FOLDERS = (ComputerDriverEnv, ComputerAppEnv) _NOTHING_TO_RUN = "this bundle has no tasks or evals to run" +# The kinds whose absence from the store is a gap whatever sets the bundle up can fill. +_SET_UP = (EntityKind.ENV, EntityKind.AGENT, EntityKind.ARTIFACT) + + +class NotInStore(BundleError): + """A plan whose references name agents, envs or artifacts the store doesn't hold, with every other problem found; + ``missing`` are those, as (kind, id).""" + + def __init__(self, problems: Iterable[str], missing: Iterable[tuple[str, str]]): + super().__init__(problems) + self.missing = tuple(missing) @dataclass(frozen=True) @@ -248,6 +259,7 @@ def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, self.others: tuple[InstalledBundle, ...] | None = None # the installed bundles but this one, once listed self.pullable: dict[str, _Pullable | str] = {} # each installed bundle read, by id root, or why it can't be self.unpulled: dict[tuple[EntityKind, str, int | None], str] = {} # why an installed bundle's id isn't pulled + self.missing: dict[tuple[str, str], None] = {} # each agent, env and artifact the store doesn't hold, as read # By kind: its installed aliases, once a plan reads them; each of them the plan names, by alias; and the store's # latest version of each alias's id, as read. self.alias_tables: dict[EntityKind, InstalledAliases] = {} @@ -402,6 +414,7 @@ def _unstored(self, ref: Reference) -> str | None: try: STORE_GETTERS[EntityKind.AGENT](ref.id, None) except NotFoundError: + self.missing[EntityKind.AGENT.value, ref.id] = None hint = table.hint(ref.id) if (table := self.alias_tables.get(EntityKind.AGENT)) is not None else "" return f"there is no agent {ref.id!r} in this bundle, an installed bundle or the store{hint}" except (ValueError, KeyError, TypeError) as e: @@ -548,7 +561,7 @@ def _planned(self, roots: list[ResolvedEntry], selected_tasks: list[ResolvedEntr self._check_environment_names(closure, store_latest) self._check_loaded_environments(closure, store_latest) if self.problems: - raise BundleError(self.problems) + raise NotInStore(self.problems, self.missing) if self.missing else BundleError(self.problems) writes = self._ordered(closure) return Plan(self.resolved, writes, tuple(selected_tasks), tuple(selected_evals), store_refs, store_latest, dict(self.origins), self._published(writes), @@ -697,10 +710,12 @@ def _closure(self, roots: list[ResolvedEntry]) -> set[_Key]: def _check_store_refs( self, closure: set[_Key], *, stores: bool = True ) -> tuple[tuple[Reference, ...], dict[tuple[EntityKind, str], int]]: - """Check each store id the closure references; without ``stores``, only what needs no store read.""" + """Check each store id the closure references; without ``stores``, only what needs no store read. An agent, env + or artifact the store doesn't hold is one problem, at the first of the places that use it.""" read: dict[tuple[EntityKind, str, int | None], Any] = {} checked: dict[tuple[EntityKind, str, int | None], Reference] = {} latest: dict[tuple[EntityKind, str], int] = {} + gaps: dict[str, list[str]] = {} # each problem of a missing agent, env or artifact, with where it's used for key in sorted(closure, key=self.rank.__getitem__): source = self.nodes[key] if isinstance(source, BundleImage): @@ -708,15 +723,24 @@ def _check_store_refs( for ref in source.references: if ref.local is not None: continue + where = f"{self._path(source.entry)}: {ref.where}" problem = self.unpulled.get((ref.kind, ref.id, ref.version)) or self._bundle_ref_problem(ref) if problem is None and stores: problem = self._store_ref_problem(ref, read) + if problem and ref.kind in _SET_UP and read[ref.kind, ref.id, ref.version] is None: + self.missing[ref.kind.value, ref.id] = None + gaps.setdefault(problem, []).append(where) + continue if problem: - self.problems.append(f"{self._path(source.entry)}: {ref.where}: {problem}") + self.problems.append(f"{where}: {problem}") elif stores: checked.setdefault((ref.kind, ref.id, ref.version), ref) if ref.version is None: latest[ref.kind, ref.id] = read[ref.kind, ref.id, None].version + for problem, wheres in gaps.items(): + first, *others = wheres + more = f" (and {len(others)} more use{'s' if len(others) > 1 else ''})" if others else "" + self.problems.append(f"{first}{more}: {problem}") return tuple(checked.values()), latest def _bundle_ref_problem(self, ref: Reference) -> str | None: diff --git a/src/agent_env/bundle/preflight.py b/src/agent_env/bundle/preflight.py index b4c8e30db..dc60fd407 100644 --- a/src/agent_env/bundle/preflight.py +++ b/src/agent_env/bundle/preflight.py @@ -80,7 +80,7 @@ from agent_env.utils.docker.build import DEFAULT_BUILD_PLATFORM, docker_plugin_missing, docker_unreachable from .parse import BundleError, BundleKind -from .plan import AgentOverride, Plan, solver_steps +from .plan import AgentOverride, NotInStore, Plan, solver_steps from .resolve import BuiltImage, Reference, ReferencedImage, ResolvedEntry, build_step, played_deploys _SHOWN = 3 @@ -133,6 +133,7 @@ def __init__(self, plan: Plan, sandbox: str | None): self.plan, self.sandbox = plan, sandbox self.problems: dict[str, dict[str, None]] = {} # each deploy's problem, and the deploys it's found at self.run_problems: list[str] = [] # the run's as a whole + self.missing: list[tuple[str, str]] = [] # each agent the deploys need that the store doesn't hold self.infra: set[str] = set() # the infra kinds a deploy on the local provider needs self.docker_users: list[str] = [] self.compose_users: list[str] = [] # gateway deploys whose stack runs on this machine @@ -203,7 +204,7 @@ def finish(self) -> Preflight: more = f" (and {len(others)} more deploy{'s' if len(others) > 1 else ''})" if others else "" problems.append(f"{first}{more}: {problem}") if problems: - raise BundleError(problems) + raise NotInStore(problems, self.missing) if self.missing else BundleError(problems) return Preflight(frozenset(self.infra), tuple(infra), built_here) # Deploys, each resolving its provider as its step does when it runs @@ -502,6 +503,7 @@ def _default_agent(self) -> None: A2AAgent.get(agent_id) return except NotFoundError: + self.missing.append((EntityKind.AGENT.value, agent_id)) problem = (f"there is no agent {agent_id!r} in the store; register one under that id, or point " "[agents] default_a2a_agent_id at an agent that is") except (ValueError, KeyError, TypeError) as e: diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 71847b661..40894a17c 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -13,9 +13,10 @@ from agent_env.bundle import (BundleError, BundleKind, BundleRun, DryRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle) -from agent_env.bundle.hub import SOURCE_KEY, HubBundle, describe +from agent_env.bundle.hub import SOURCE_KEY, describe from agent_env.bundle.installed import (InstalledBundle, checked, find_bundle, installed_bundles, locate_bundle, path_like, run_name) +from agent_env.bundle.plan import NotInStore from agent_env.bundle.requires import UnmetRequirements from agent_env.cli._install_prompt import offer_to_install from agent_env.plugins._bundles import ALIAS_TABLES, installed_aliases @@ -68,8 +69,9 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t only bundle under bundles/, or the one the dataset card's agentenv: default names, else the repo's root. Only the bundle and the card are downloaded, and a commit already downloaded is reused. The run first prints the repo, commit and bundle. A bundle with no bundle.toml requires the plugins the card's agentenv: bundles: NAME: plugins - lists, and a run that fails shows the card's setup command, which it never runs. A gated or private repo needs - hf auth login or HF_TOKEN. + lists, checked, as a bundle.toml's requires are, before the bundle is downloaded. A run refused for a missing + plugin, or for an agent, env or artifact the store doesn't hold, shows the card's setup command, which it never + runs. A gated or private repo needs hf auth login or HF_TOKEN. A bundle's bundle.toml names the plugin packages it requires, and a run refuses to start without them; in a terminal it offers to install them with agent-env plugin add. A folder with no bundle.toml that uses plugin @@ -98,7 +100,12 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t "--dry-run and --no-emulate need a BUNDLE to run") _list_installed() return - root, id_root, hub = locate_bundle(bundle, _installed, on_progress=click.echo) + try: + root, id_root, hub = locate_bundle(bundle, _installed, on_progress=click.echo) + except UnmetRequirements as e: + if not dry_run: + offer_to_install(e) + raise verbose = ctx.find_root().params.get("verbose") try: if hub is not None: @@ -113,8 +120,8 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t id_root=id_root, keep=keep, emulate=not no_emulate, source=hub.source if hub else None) except BundleError as e: _note_an_installed_namesake(e, bundle, root, id_root) - if setup := _setup(hub): - e.add_note(setup) + if isinstance(e, (UnmetRequirements, NotInStore)) and hub is not None and hub.setup_note: + e.add_note(hub.setup_note) if isinstance(e, UnmetRequirements) and not dry_run: offer_to_install(e) raise @@ -129,17 +136,9 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t result = _hold(result, interrupts) _report_teardown(result) if result.failed: - if setup := _setup(hub): - click.echo(setup) raise SystemExit(1) -def _setup(hub: HubBundle | None) -> str | None: - """The note a failed run of a Hub bundle ends with, naming the command the dataset card sets the bundle up with.""" - if hub is None or not hub.setup: - return None - return f"{hub.source['repo']}'s card sets this bundle up with: {hub.setup}" - def _note_an_installed_namesake(error: BundleError, text: str, root: Path, id_root: str | None) -> None: """A folder that isn't a bundle can hide the installed bundle the user meant. The note names it by its diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 54d71576f..426dd086f 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -271,22 +271,46 @@ def test_a_dry_run_downloads_and_checks_but_runs_and_writes_nothing(hub, local_s assert not local_stores.local_namespace_document_store("hf").path.exists() -def test_a_bundle_without_a_bundle_toml_needs_the_plugins_its_card_names(hub, local_stores, quiet_logs): +def test_a_bundle_without_a_bundle_toml_needs_the_plugins_its_card_names_before_it_is_downloaded(hub, local_stores, + quiet_logs): hub.layout({"bundles/hello/tasks/t.json": TASK}) - hub.card({"bundles": {"hello": {"plugins": ["agentenv-not-installed>=1"]}}}) + hub.card({"bundles": {"hello": {"plugins": ["agentenv-not-installed>=1"], "setup": "agent-env demo setup"}}}) refused = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + calls = list(hub.calls) hub.layout({"bundles/hello/bundle.toml": "requires = []\n"}) shutil.rmtree(hub_module.cache_root()) declared = CliRunner().invoke(cli, ["run", "hf://me/ds"]) assert refused.exit_code == 1 assert ("me/ds's card: requires 'agentenv-not-installed>=1', which isn't installed; install it with: agent-env " - "plugin add 'agentenv-not-installed>=1'") in refused.output - assert "Tasks:" not in refused.output + "plugin add 'agentenv-not-installed>=1'\nme/ds's card sets this bundle up with: agent-env demo setup") in ( + refused.output) + assert calls == ["repo_info", "list_repo_files", "hf_hub_download README.md"] assert declared.exit_code == 0, declared.output +@pytest.mark.parametrize("requires, problem, noted", [ + ('["agentenv-not-installed>=1"]', "bundle.toml: requires 'agentenv-not-installed>=1', which isn't installed; " + "install it with: agent-env plugin add 'agentenv-not-installed>=1'", True), + ('["agentenv-x @ git+https://example.invalid/x"]', "bundle.toml: requires: 'agentenv-x @ " + "git+https://example.invalid/x' names a URL; name the package and the versions it needs, which install from " + "wherever whoever runs the bundle installs from", False), +]) +def test_a_bundle_toml_is_downloaded_alone_and_checked_as_a_folders_is_before_the_rest(hub, requires, problem, noted): + hub.layout({"bundles/hello/tasks/t.json": TASK, "bundles/hello/bundle.toml": f"requires = {requires}\n"}) + hub.card({"bundles": {"hello": {"plugins": ["agentenv-the-card-names"], "setup": "agent-env demo setup"}}}) + + with pytest.raises(BundleError) as caught: + fetch_bundle("hf://me/ds") + + assert caught.value.problems == (problem,) + assert getattr(caught.value, "__notes__", []) == (["me/ds's card sets this bundle up with: agent-env demo setup"] + if noted else []) + assert hub.calls == ["repo_info", "list_repo_files", "hf_hub_download README.md", + "hf_hub_download bundles/hello/bundle.toml"] + + @pytest.mark.parametrize("bundles, problem", [ (["hello"], "me/ds's card: agentenv: bundles must map each bundle's name to what it needs"), ({"hello": "agentenv-x"}, "me/ds's card: agentenv: bundles: hello must be a table of plugins and setup"), @@ -304,19 +328,31 @@ def test_a_card_that_says_what_a_bundle_needs_in_the_wrong_shape_is_refused(hub, assert caught.value.problems == (problem,) -def test_an_openenv_server_of_a_hub_bundle_needs_the_plugins_its_card_names_too(hub): +def test_an_openenv_server_of_a_hub_bundle_needs_the_plugins_its_card_names_before_it_is_downloaded_too(hub): hub.layout({"bundles/hello/tasks/t.json": TASK}) hub.card({"bundles": {"hello": {"plugins": ["agentenv-not-installed>=1"]}}}) with pytest.raises(UnmetRequirements, match="agent-env plugin add 'agentenv-not-installed>=1'"): build_app("hf://me/ds") + assert hub.calls == ["repo_info", "list_repo_files", "hf_hub_download README.md"] -def test_a_failed_run_shows_the_cards_setup_command_and_never_runs_it(hub, local_stores, quiet_logs): - hub.layout({"bundles/hello/tasks/t.json": json.dumps([{"id": "x", "type": "a_step_only_setup_registers"}])}) - hub.card({"bundles": {"hello": {"setup": "agent-env demo setup"}}}) - result = CliRunner().invoke(cli, ["run", "hf://me/ds"]) +def test_the_cards_setup_command_is_shown_only_when_setting_up_could_fix_the_run(hub, local_stores, quiet_logs): + note = "me/ds's card sets this bundle up with: agent-env demo setup" + gap = {"id": "deploy", "type": "deploy_env", "env_id": "env-setup-writes"} + other = {"id": "other", "type": "deploy_env", "env_id": "@hf/me/ds/hello/missing"} + hub.card({"bundles": {"hello": {"setup": "agent-env demo setup"}}}) + hub.layout({"bundles/hello/tasks/t.json": json.dumps([gap, other])}) - assert result.exit_code == 1 - assert "me/ds's card sets this bundle up with: agent-env demo setup" in result.output + mixed = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + hub.layout({"bundles/hello/tasks/t.json": json.dumps([other])}) + shutil.rmtree(hub_module.cache_root()) + neither = CliRunner().invoke(cli, ["run", "hf://me/ds"]) + + assert mixed.exit_code == neither.exit_code == 1 + assert "step 'deploy': env_id: there is no env 'env-setup-writes' in the store" in mixed.output + assert "step 'other': env_id: '@hf/me/ds/hello/missing' isn't in this bundle" in mixed.output + assert note in mixed.output + assert "step 'other': env_id: '@hf/me/ds/hello/missing' isn't in this bundle" in neither.output + assert note not in neither.output diff --git a/tst/unit/bundle/plan_test.py b/tst/unit/bundle/plan_test.py index b7d1288fe..4b2362989 100644 --- a/tst/unit/bundle/plan_test.py +++ b/tst/unit/bundle/plan_test.py @@ -11,7 +11,7 @@ from agent_env.bundle import BundleError, BundleKind, parse_bundle from agent_env.bundle import plan as plan_module from agent_env.bundle import resolve as resolve_module -from agent_env.bundle.plan import check_bundle, plan_bundle +from agent_env.bundle.plan import NotInStore, check_bundle, plan_bundle from agent_env.bundle.resolve import resolve_bundle from agent_env.config import configure from agent_env.entity.refs import EntityKind, EntityRef @@ -278,6 +278,23 @@ def test_a_store_id_must_exist(make, steps, problem): assert problems(make(tasks={"t": steps})) == (f"tasks/t.json: {problem}",) +def test_an_agent_the_store_doesnt_hold_is_one_problem_at_its_first_use_and_says_its_a_store_gap(make): + agent = {"id": "agent", "type": "deploy_agent", "env_ids": ["tickets"], "a2a_agent_id": "ghost"} + other = deploy(f"{ROOT}/tickts") + + with pytest.raises(NotInStore) as mixed: + plan_bundle(make(tasks={"a": [agent], "b": [agent], "c": [agent, other]})) + with pytest.raises(BundleError) as neither: + plan_bundle(make(tasks={"a": [other], "b": [other], "c": [other]})) + + assert mixed.value.problems == ( + "tasks/a.json: step 'agent': a2a_agent_id (and 2 more uses): there is no agent 'ghost' in the store", + f"tasks/c.json: step 'deploy': env_id: '{ROOT}/tickts' isn't in this bundle", + ) + assert mixed.value.missing == (("agent", "ghost"),) + assert not isinstance(neither.value, NotInStore) + + def test_another_bundles_local_id_is_read_from_the_local_namespace_like_a_store_id(make, local_stores, cli_routing): put_env("@local/~/other/crm") FileArtifact.put_bytes("@local/~/other/golden", description="golden", filename="g.txt", content=b"g", From 990b2d19156ab0f41ddfdbccb5c9ac5ba40abf6d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 21:42:17 -0800 Subject: [PATCH 07/13] test(verifiers): run the taskset test through vf-run where verifiers main removed vf-eval verifiers main removed vf-eval (PrimeIntellect-ai/verifiers#2610) for vf-run, which takes the same config, saves the traces and exits non-zero when an episode isn't ok, as the masked row is. The test uses vf-eval when it's on PATH, as on 0.4.0, with --no-push after the config when its help names it, and vf-run otherwise, then judges both by the traces; the workflow puts whichever of the two the installed verifiers has on PATH. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/verifiers.yml | 9 +++++---- tst/verifiers/taskset_test.py | 26 +++++++++++++++++--------- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/.github/workflows/verifiers.yml b/.github/workflows/verifiers.yml index 5b12bb423..6a2c00aa6 100644 --- a/.github/workflows/verifiers.yml +++ b/.github/workflows/verifiers.yml @@ -2,8 +2,8 @@ name: verifiers contract # Rollouts through verifiers' NeMo Gym taskset, the one prime-rl trains through, against the resources server # agent_env.training.nemo_gym serves for a bundle (tst/verifiers), on verifiers 0.4.0 and on its main branch, which -# prime-rl follows. verifiers pins mcp 2 and the framework mcp 1, so verifiers gets a venv of its own and only vf-eval -# goes on PATH. It runs on pull requests that touch the helper, the OpenEnv server it plays or the tier, weekly, and on +# prime-rl follows. verifiers pins mcp 2 and the framework mcp 1, so verifiers gets a venv of its own and only its CLI goes +# on PATH: vf-eval on 0.4.0, and vf-run on main, which removed vf-eval. It runs on pull requests that touch the helper, the OpenEnv server it plays or the tier, weekly, and on # demand; it isn't a required check. on: @@ -53,12 +53,13 @@ jobs: run: | python -m pip install --quiet -r .github/ci-requirements.txt uv sync --locked --extra dev --extra nemo-gym --python "$(command -v python)" - - name: Install verifiers in its own venv, with vf-eval on PATH + - name: Install verifiers in its own venv, with its CLI on PATH run: | uv venv "$RUNNER_TEMP/verifiers" --python "$(command -v python)" uv pip install --python "$RUNNER_TEMP/verifiers/bin/python" "${{ matrix.requirement }}" mkdir -p "$RUNNER_TEMP/vf-bin" - ln -s "$RUNNER_TEMP/verifiers/bin/vf-eval" "$RUNNER_TEMP/vf-bin/vf-eval" + script=$([ -e "$RUNNER_TEMP/verifiers/bin/vf-eval" ] && echo vf-eval || echo vf-run) + ln -s "$RUNNER_TEMP/verifiers/bin/$script" "$RUNNER_TEMP/vf-bin/$script" echo "$RUNNER_TEMP/vf-bin" >> "$GITHUB_PATH" "$RUNNER_TEMP/verifiers/bin/python" -c "from importlib.metadata import version; print('verifiers', version('verifiers'))" - name: verifiers tier diff --git a/tst/verifiers/taskset_test.py b/tst/verifiers/taskset_test.py index 3e86089de..725724a07 100644 --- a/tst/verifiers/taskset_test.py +++ b/tst/verifiers/taskset_test.py @@ -1,7 +1,8 @@ """verifiers' NeMo Gym taskset, prime-rl's, plays a written server's rows against ``serve`` with the null harness: ``vf-eval`` seeds each row, calls its tools natively, and takes the reward from verify. A rollout the environment fails is masked, and with ``--masked-as-error`` it comes back failed, which prime-rl leaves out of training, rather -than scored 0. The model is a streamed stand-in. CI runs it on verifiers 0.4.0 and on verifiers' main branch.""" +than scored 0. The model is a streamed stand-in. CI runs it on verifiers 0.4.0, through ``vf-eval``, and on verifiers' main +branch, through ``vf-run``, which replaced it there and exits non-zero when an episode isn't ok.""" import json import os @@ -21,8 +22,8 @@ pytest.importorskip("agent_env.training.nemo_gym.server", reason=missing_capability_reason(NEMO_GYM), exc_type=ImportError) -VF_EVAL = shutil.which("vf-eval") -pytestmark = pytest.mark.skipif(VF_EVAL is None, reason=missing_capability_reason(VERIFIERS)) +VF_EVAL, VF_RUN = shutil.which("vf-eval"), shutil.which("vf-run") +pytestmark = pytest.mark.skipif(VF_EVAL is None and VF_RUN is None, reason=missing_capability_reason(VERIFIERS)) def test_vf_eval_scores_a_played_row_and_fails_a_masked_one(serve, tmp_path, teardowns): @@ -31,8 +32,6 @@ def test_vf_eval_scores_a_played_row_and_fails_a_masked_one(serve, tmp_path, tea row = json.loads((folder / "data" / "all.jsonl").read_text().splitlines()[0]) (tmp_path / "rows.jsonl").write_text(json.dumps(row) + "\n" + json.dumps({**row, "task_id": "b"}) + "\n") port, model = _free_port(), _model() - vf_help = subprocess.run([VF_EVAL, "--help"], capture_output=True, text=True, timeout=60) - no_push = ["--no-push"] if "--no-push" in vf_help.stdout + vf_help.stderr else [] served = subprocess.Popen([sys.executable, "-m", "agent_env.training.nemo_gym", "serve", str(folder), "--port", str(port), "--masked-as-error"], @@ -40,16 +39,15 @@ def test_vf_eval_scores_a_played_row_and_fails_a_masked_one(serve, tmp_path, tea try: url = _up(served, f"http://127.0.0.1:{port}") (tmp_path / "eval.toml").write_text(_config(tmp_path, url, model.server_address[1])) - evaluated = subprocess.run([VF_EVAL, "@", str(tmp_path / "eval.toml"), "-n", "2", "-r", "1", *no_push], - env={**os.environ, "STAND_IN_API_KEY": "unused"}, capture_output=True, text=True, - timeout=600) + evaluated = subprocess.run(_command(tmp_path / "eval.toml"), env={**os.environ, "STAND_IN_API_KEY": "unused"}, + capture_output=True, text=True, timeout=600) finally: served.terminate() served.wait(timeout=30) model.shutdown() bundle_server.stop() - assert evaluated.returncode == 0, evaluated.stdout + evaluated.stderr + assert evaluated.returncode == (0 if VF_EVAL else 1), evaluated.stdout + evaluated.stderr traces = {trace["task"]["data"]["name"]: trace for trace in _traces(tmp_path / "out")} played, masked = traces["rows:0"], traces["rows:1"] assert (played["ok"], played.get("dropped"), played["rewards"]["nemo_gym"]["score"]) == (True, None, 1.0) @@ -58,6 +56,16 @@ def test_vf_eval_scores_a_played_row_and_fails_a_masked_one(serve, tmp_path, tea assert len(teardowns.finished) == 3 +def _command(config): + """Two rows of ``config``, once each: through ``vf-eval``, with ``--no-push`` after the config when its help names + it, or else through ``vf-run``, which only saves its traces.""" + rows = ["@", str(config), "-n", "2", "-r", "1"] + if VF_EVAL is None: + return [VF_RUN, *rows] + vf_help = subprocess.run([VF_EVAL, "--help"], capture_output=True, text=True, timeout=60) + return [VF_EVAL, *rows, *(["--no-push"] if "--no-push" in vf_help.stdout + vf_help.stderr else [])] + + def _config(tmp_path, url, model_port): return f"""model = "stand-in" output_dir = "{tmp_path / 'out'}" From 9d97eddc59a2c36ca6fdd26156dc72a5f3856dcb Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:34:17 -0800 Subject: [PATCH 08/13] refactor(store): namespace_prefix for '@/', say what a bundle may declare, type the ledger's namespace Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/ledger.py | 4 ++-- src/agent_env/bundle/parse.py | 9 +++++++-- src/agent_env/store/ids.py | 9 +++++++-- src/agent_env/store/routing.py | 4 +++- tst/unit/bundle/parse_test.py | 2 ++ 5 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/agent_env/bundle/ledger.py b/src/agent_env/bundle/ledger.py index ed6c827ed..be1407ceb 100644 --- a/src/agent_env/bundle/ledger.py +++ b/src/agent_env/bundle/ledger.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from datetime import date, datetime, time, timezone from pathlib import Path -from typing import Any +from typing import Any, cast from agent_env.a2a_agent.store import A2A_AGENTS_COLLECTION from agent_env.artifact.registry import canonical_type, get_artifact_registry @@ -98,7 +98,7 @@ def __init__(self, store_of: Callable[[str], DocumentStore], plan: Plan): @classmethod def for_plan(cls, plan: Plan) -> Ledger: config = get_config() - return cls(lambda entity_id: config.local_namespace_document_store(local_namespace(entity_id)), plan) + return cls(lambda entity_id: config.local_namespace_document_store(cast(str, local_namespace(entity_id))), plan) def check(self, write: Write, needs: Mapping[tuple[str, str], int]) -> Check: """Compare ``write`` with the latest version the ledger records for it. ``needs`` holds, by (store, id), diff --git a/src/agent_env/bundle/parse.py b/src/agent_env/bundle/parse.py index e399d0e35..fcca905d8 100644 --- a/src/agent_env/bundle/parse.py +++ b/src/agent_env/bundle/parse.py @@ -450,8 +450,8 @@ def _entity_id(self, kind: BundleKind, name: str, path: Path, declared: Any, dec def _check_declared(self, entity_id: Any, where: Path, *, what: str = "id", artifact: bool = False) -> None: if not isinstance(entity_id, str) or not is_local_id(entity_id): - self._skip(where, f"{what} {_brief(entity_id)}: ids declared in a bundle must start with " - f"{LOCAL_PREFIX!r}, since a bare id names an entity in a shared store") + self._skip(where, f"{what} {_brief(entity_id)}: ids declared in this bundle must start with " + f"{self._declarable()}, since a bare id names an entity in a shared store") if foreign_hub_id(entity_id, self.id_root): self._skip(where, f"{what} {_brief(entity_id)}: an {HF_PREFIX!r} id names an entity of a bundle run from " "the Hugging Face Hub, so a bundle declares one only under its own id root, which " @@ -469,6 +469,11 @@ def _check_declared(self, entity_id: Any, where: Path, *, what: str = "id", arti if artifact and "__" in entity_id.rsplit("/", 1)[1]: self._skip(where, f'{what} {_brief(entity_id)}: "__" is reserved for derived artifact ids') + def _declarable(self) -> str: + """What an id this bundle declares may start with: ``@local/``, and for a Hub bundle its own root.""" + own = f" or {self.id_root + '/'!r}" if local_namespace(self.id_root) == HF_NAMESPACE else "" + return f"{LOCAL_PREFIX!r}{own}" + # Across entities def _check_unique(self, entries: list[BundleEntry]) -> None: diff --git a/src/agent_env/store/ids.py b/src/agent_env/store/ids.py index 98c8b5705..fdf721f54 100644 --- a/src/agent_env/store/ids.py +++ b/src/agent_env/store/ids.py @@ -54,7 +54,12 @@ def is_local_id(entity_id: str) -> bool: def local_namespace(entity_id: str) -> Optional[str]: """The local namespace ``entity_id`` is in, or None for a bare id or another namespace's.""" - return next((namespace for namespace in LOCAL_NAMESPACES if entity_id.startswith(f"@{namespace}/")), None) + return next((namespace for namespace in LOCAL_NAMESPACES if entity_id.startswith(namespace_prefix(namespace))), None) + + +def namespace_prefix(namespace: str) -> str: + """What every id in ``namespace`` opens with: ``@/``.""" + return f"@{namespace}/" def validate_local_id(entity_id: str) -> None: @@ -68,7 +73,7 @@ def validate_local_id(entity_id: str) -> None: namespace = local_namespace(entity_id) if namespace is None: raise ValueError(f"local id {entity_id!r} must start with {LOCAL_PREFIX!r} or {HF_PREFIX!r}") - prefix = f"@{namespace}/" + prefix = namespace_prefix(namespace) size = len(entity_id.encode("utf-8", "surrogatepass")) if size > MAX_LOCAL_ID_BYTES: raise ValueError(f"local id is {size} bytes; the limit is {MAX_LOCAL_ID_BYTES}: {entity_id[:64]!r}…") diff --git a/src/agent_env/store/routing.py b/src/agent_env/store/routing.py index 1f66b08a8..4f917e504 100644 --- a/src/agent_env/store/routing.py +++ b/src/agent_env/store/routing.py @@ -45,6 +45,7 @@ LOCAL_NAMESPACES, is_local_id, local_namespace, + namespace_prefix, parse_namespace, validate_local_id, ) @@ -476,7 +477,8 @@ def check_id(self, entity_id: str) -> None: if namespace is not None: self.namespaces[namespace].check_id(entity_id) return - if (inside := next((namespace for namespace in self.namespaces if f"@{namespace}/" in entity_id), None)): + if (inside := next((namespace for namespace in self.namespaces if namespace_prefix(namespace) in entity_id), + None)): raise ValueError( f"id {entity_id!r} contains an @{inside} id but isn't one: an id derived from an @{inside} id " f"has to keep the @{inside} namespace" diff --git a/tst/unit/bundle/parse_test.py b/tst/unit/bundle/parse_test.py index 347959c75..cd925e650 100644 --- a/tst/unit/bundle/parse_test.py +++ b/tst/unit/bundle/parse_test.py @@ -543,6 +543,8 @@ def test_an_hf_id_is_declared_only_under_a_hub_bundles_own_root(self, tmp_path): "a bundle run from the Hugging Face Hub") assert parse_bundle(build(tmp_path / "c", {"envs/t/Dockerfile": ""}), id_root="@hf/acme/ds").entries[0].id == "@hf/acme/ds/t" + bare = build(tmp_path / "d", {"envs/t/env.toml": 'id = "bare"\n'}) + assert "must start with '@local/' or '@hf/acme/ds/x/'" in problems(bare, id_root="@hf/acme/ds/x")[0] def test_artifact_leaf_may_not_contain_double_underscore(self, tmp_path): root = build(tmp_path / "b", { From a472309803176eed10fde39d4239e29e09f9725d Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:34:42 -0800 Subject: [PATCH 09/13] refactor(config): cache_root beside state_root, the per-user cache Hub commits are kept under Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/hub.py | 10 ++-------- src/agent_env/config/paths.py | 7 +++++++ tst/unit/bundle/hub_test.py | 5 +++-- tst/unit/config/test_paths.py | 13 +++++++++++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 1493d9798..d0e9da2d6 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -25,6 +25,7 @@ from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError from huggingface_hub.repocard import metadata_load +from agent_env.config.paths import cache_root from agent_env.plugins._bundles import ALIASES_FILE from agent_env.plugins._requirements import unmet_problems from agent_env.store.ids import HF_PREFIX, validate_local_id @@ -121,13 +122,6 @@ def parse_reference(text: str) -> HubReference: return HubReference(f"{org}/{name}", unquote(revision) or None, "/".join(parts) or None) -def cache_root() -> Path: - """Where downloaded commits are kept: ``$XDG_CACHE_HOME/agentenv/hf`` when that is an absolute path, else - ``~/.cache/agentenv/hf``.""" - base = os.environ.get("XDG_CACHE_HOME", "") - return (Path(base) if os.path.isabs(base) else Path.home() / ".cache") / "agentenv" / "hf" - - def fetch_bundle(text: str, *, on_progress: Callable[[str], None] | None = None) -> HubBundle: """The bundle ``text`` names, downloaded unless this machine has it. ``on_progress`` is given the line that says which repo, commit and bundle it is, before anything is downloaded. Raises BundleError.""" @@ -150,7 +144,7 @@ def describe(source: Mapping[str, str]) -> str: def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> HubBundle: sha = reference.revision if reference.revision and _SHA.fullmatch(reference.revision) else api.repo_info( reference.repo, repo_type="dataset", revision=reference.revision).sha - folder = cache_root() / reference.repo / sha + folder = cache_root() / "hf" / reference.repo / sha with holding_locks([f"{SCHEME}{reference.repo}@{sha}"]): commit = _Commit.read(folder) if commit is None: diff --git a/src/agent_env/config/paths.py b/src/agent_env/config/paths.py index 69ab7b11d..1f3f7313e 100644 --- a/src/agent_env/config/paths.py +++ b/src/agent_env/config/paths.py @@ -23,3 +23,10 @@ def state_root() -> Path: if sys.platform == "win32" and os.path.isabs(local_app_data): return Path(local_app_data) / _APP_DIR return Path.home() / ".local" / "state" / _APP_DIR + + +def cache_root() -> Path: + """The per-user cache root, for what can be fetched again: ``$XDG_CACHE_HOME/agentenv`` when that is an absolute + path, else ``~/.cache/agentenv``. A path only; nothing is created.""" + xdg = os.environ.get("XDG_CACHE_HOME", "") + return (Path(xdg) if os.path.isabs(xdg) else Path.home() / ".cache") / "agentenv" diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 426dd086f..443c9c72a 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -18,6 +18,7 @@ from agent_env.bundle.ledger import LEDGER_COLLECTION from agent_env.bundle.requires import UnmetRequirements from agent_env.cli import cli +from agent_env.config.paths import cache_root from agent_env.config.runtime import Config from agent_env.openenv import build_app from agent_env.store import Filter @@ -279,7 +280,7 @@ def test_a_bundle_without_a_bundle_toml_needs_the_plugins_its_card_names_before_ refused = CliRunner().invoke(cli, ["run", "hf://me/ds"]) calls = list(hub.calls) hub.layout({"bundles/hello/bundle.toml": "requires = []\n"}) - shutil.rmtree(hub_module.cache_root()) + shutil.rmtree(cache_root()) declared = CliRunner().invoke(cli, ["run", "hf://me/ds"]) assert refused.exit_code == 1 @@ -347,7 +348,7 @@ def test_the_cards_setup_command_is_shown_only_when_setting_up_could_fix_the_run mixed = CliRunner().invoke(cli, ["run", "hf://me/ds"]) hub.layout({"bundles/hello/tasks/t.json": json.dumps([other])}) - shutil.rmtree(hub_module.cache_root()) + shutil.rmtree(cache_root()) neither = CliRunner().invoke(cli, ["run", "hf://me/ds"]) assert mixed.exit_code == neither.exit_code == 1 diff --git a/tst/unit/config/test_paths.py b/tst/unit/config/test_paths.py index 4c41b0069..e8879ebc0 100644 --- a/tst/unit/config/test_paths.py +++ b/tst/unit/config/test_paths.py @@ -1,10 +1,10 @@ -"""The per-user state root: XDG first, then the platform default.""" +"""The per-user state and cache roots: XDG first, then the platform default.""" import sys import pytest -from agent_env.config.paths import state_root +from agent_env.config.paths import cache_root, state_root @pytest.fixture @@ -34,6 +34,15 @@ def test_windows_uses_local_app_data_unless_xdg_state_home_is_set(home, tmp_path assert state_root() == tmp_path / "xdg" / "agent-env" +@pytest.mark.parametrize("value", ["", "relative/cache"]) +def test_the_cache_root_is_an_absolute_xdg_cache_homes_else_the_home_default(home, tmp_path, monkeypatch, value): + monkeypatch.setenv("XDG_CACHE_HOME", value) + assert cache_root() == home / ".cache" / "agentenv" + monkeypatch.setenv("XDG_CACHE_HOME", str(tmp_path / "xdg")) + assert cache_root() == tmp_path / "xdg" / "agentenv" + + def test_resolving_creates_nothing(home): state_root() + cache_root() assert not home.exists() From 464a14824764052e7dda5386f298bd8ac6e69fa9 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:35:26 -0800 Subject: [PATCH 10/13] refactor(bundle): a frozen HubSource for where a Hub bundle came from, stored with each run as before Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/hub.py | 40 ++++++++++++++++++-------- src/agent_env/bundle/run.py | 13 +++++---- src/agent_env/cli/run.py | 4 +-- tst/integration/cli/run_bundle_test.py | 4 +-- tst/unit/bundle/hub_test.py | 18 ++++++------ 5 files changed, 49 insertions(+), 30 deletions(-) diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index d0e9da2d6..1e80cb32e 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -53,6 +53,28 @@ class HubReference: path: str | None # the bundle's folder in the repo, when named +@dataclass(frozen=True) +class HubSource: + """Where a Hub bundle came from: its repo, the revision named (``main`` by default), its commit, and its folder in + the repo, ``""`` for the root. Each run's context records it under ``SOURCE_KEY``.""" + + repo: str + revision: str + sha: str + path: str + + @classmethod + def from_metadata(cls, data: Mapping[str, str]) -> HubSource: + return cls(data["repo"], data["revision"], data["sha"], data["path"]) + + def to_metadata(self) -> dict[str, str]: + return {"hub": "hf", "repo": self.repo, "revision": self.revision, "sha": self.sha, "path": self.path} + + def describe(self) -> str: + """As a run prints it: ``/ at , bundle ``.""" + return f"{self.repo} at {self.sha[:12]}, bundle {self.path or 'at its root'}" + + @dataclass(frozen=True) class HubBundle: """A bundle downloaded from the Hub: its folder, the id root it runs under, where it came from, and what the repo's @@ -60,19 +82,19 @@ class HubBundle: root: Path id_root: str - source: Mapping[str, str] # hub, repo, revision, sha, path: what each run's context records + source: HubSource plugins: tuple[str, ...] # the card's, for a bundle without a bundle.toml setup: str | None # the card's command that sets the bundle up, shown when setting up could fix a run, never run @property def setup_note(self) -> str | None: """What a run refused for something setting the bundle up provides ends with: the card's command for it.""" - return f"{self.source['repo']}'s card sets this bundle up with: {self.setup}" if self.setup else None + return f"{self.source.repo}'s card sets this bundle up with: {self.setup}" if self.setup else None def check_plugins(self) -> None: """Refuse the run when a plugin the card names isn't installed, naming the command that installs it. Raises UnmetRequirements.""" - problems, unmet = unmet_problems(self.plugins, f"{self.source['repo']}'s card") + problems, unmet = unmet_problems(self.plugins, f"{self.source.repo}'s card") if problems: raise UnmetRequirements(problems, unmet) if unmet else BundleError(problems) @@ -136,11 +158,6 @@ def fetch_bundle(text: str, *, on_progress: Callable[[str], None] | None = None) raise error from e -def describe(source: Mapping[str, str]) -> str: - """Where a bundle came from, as a run prints it: ``/ at , bundle ``.""" - return f"{source['repo']} at {source['sha'][:12]}, bundle {source['path'] or 'at its root'}" - - def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> HubBundle: sha = reference.revision if reference.revision and _SHA.fullmatch(reference.revision) else api.repo_info( reference.repo, repo_type="dataset", revision=reference.revision).sha @@ -156,11 +173,10 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H path = _pick(reference, sha, commit.files, card) id_root = _id_root(reference, path) plugins, setup = _needs(reference.repo, card, path.rpartition("/")[2] or reference.repo.partition("/")[2]) - source = {"hub": "hf", "repo": reference.repo, "revision": reference.revision or "main", "sha": sha, - "path": path} + source = HubSource(reference.repo, reference.revision or "main", sha, path) declared = _within(path, BUNDLE_TOML) in commit.files hub = HubBundle(folder / path, id_root, source, plugins=() if declared else plugins, setup=setup) - say(describe(source)) + say(source.describe()) if path not in commit.downloaded: if declared: api.hf_hub_download(reference.repo, _within(path, BUNDLE_TOML), repo_type="dataset", revision=sha, @@ -183,7 +199,7 @@ def _refuse_unmet(hub: HubBundle, declared: bool) -> None: raise BundleError([f"{BUNDLE_TOML}: {problem}" for problem in refused]) problems, unmet = unmet_problems(requires, BUNDLE_TOML) else: - problems, unmet = unmet_problems(hub.plugins, f"{hub.source['repo']}'s card") + problems, unmet = unmet_problems(hub.plugins, f"{hub.source.repo}'s card") if not problems: return if not unmet: diff --git a/src/agent_env/bundle/run.py b/src/agent_env/bundle/run.py index 3af4eca0f..c3d953c2c 100644 --- a/src/agent_env/bundle/run.py +++ b/src/agent_env/bundle/run.py @@ -35,7 +35,7 @@ from agent_env.task.teardown import TeardownReport, teardown_run from agent_env.task_step.context import TaskStepContext, sandbox_overrides -from .hub import SOURCE_KEY +from .hub import SOURCE_KEY, HubSource from .installed import installed_bundles from .materialize import Materialization, Materialized, materialize from .parse import BundleEntry, BundleError, BundleKind, parse_bundle @@ -218,7 +218,7 @@ def run_bundle( id_root: str | None = None, keep: bool = False, emulate: bool = True, - source: Mapping[str, str] | None = None, + source: HubSource | None = None, ) -> BundleRun: """Write what the tasks and evals selected in ``root`` need, as ``plan_bundle`` selects them, then run each of their tasks once. ``model``, ``user_model`` and ``judge_model`` replace the models the solver, the @@ -228,8 +228,8 @@ def run_bundle( unless ``emulate`` is False, which leaves it skipped. ``on_progress`` is given a line of text at each write and step. ``id_root`` roots the ids of a bundle an installed package provides (``InstalledBundle.id_root``) or one downloaded from the Hub (``HubBundle.id_root``); a folder's ids otherwise come - from its path. ``source`` says where the bundle came from, a Hub bundle's ``HubBundle.source``, which each run's - context records under ``metadata["bundle_source"]``. + from its path. ``source`` says where a Hub bundle came from (``HubBundle.source``), which each run's context + records under ``metadata["bundle_source"]``. Each run's sandboxes are torn down as it ends, so the contexts returned record sandboxes that are gone. ``keep`` leaves them up instead, for the caller to use and then remove with ``BundleRun.teardown()``. @@ -487,7 +487,7 @@ def _overrides(sandbox: str | None, user_model: str | None, judge_model: str | N async def _run_all( plan: Plan, to_run: list[tuple[BundleEntry, Task]], model: str | None, overrides: dict[str, str], step_params: Mapping[str, dict], keep: bool, say: Callable[[str], None], interrupts: Interrupts, - source: Mapping[str, str] | None, + source: HubSource | None, ) -> tuple[TaskRun, ...]: semaphore = asyncio.Semaphore(MAX_CONCURRENT_RUNS) run_group_id = uuid.uuid4().hex @@ -506,7 +506,8 @@ async def settle(context: TaskStepContext, cancelled: bool) -> TeardownReport | async def run_one(entry: BundleEntry, task: Task) -> TaskRun: tag = f"[{plan.path(entry)}]" - metadata = {"run_group_id": run_group_id, "task_id": task.id, **({SOURCE_KEY: dict(source)} if source else {})} + metadata = {"run_group_id": run_group_id, "task_id": task.id, + **({SOURCE_KEY: source.to_metadata()} if source else {})} user_overrides = {**overrides, **({"step_params": step_params[entry.id]} if entry.id in step_params else {})} if user_overrides: metadata["user_overrides"] = user_overrides diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 40894a17c..1fff81edf 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -13,7 +13,7 @@ from agent_env.bundle import (BundleError, BundleKind, BundleRun, DryRun, Outcome, RunInterrupted, TaskRun, dry_run_bundle, run_bundle) -from agent_env.bundle.hub import SOURCE_KEY, describe +from agent_env.bundle.hub import SOURCE_KEY, HubSource from agent_env.bundle.installed import (InstalledBundle, checked, find_bundle, installed_bundles, locate_bundle, path_like, run_name) from agent_env.bundle.plan import NotInStore @@ -208,7 +208,7 @@ def _count(entries, kind: BundleKind, noun: str) -> str: def _summarize(result: BundleRun, verbose: bool) -> None: click.echo() if source := next((run.context.metadata.get(SOURCE_KEY) for run in result.runs), None): - click.echo(f"From {describe(source)}") + click.echo(f"From {HubSource.from_metadata(source).describe()}") click.echo("Tasks:") for run in result.runs: click.echo(click.style(f" {_described(result, run)}", fg=_COLORS[run.outcome])) diff --git a/tst/integration/cli/run_bundle_test.py b/tst/integration/cli/run_bundle_test.py index 4c7ad22e3..5d0cf87de 100644 --- a/tst/integration/cli/run_bundle_test.py +++ b/tst/integration/cli/run_bundle_test.py @@ -21,6 +21,7 @@ import agent_env from agent_env.artifact import FileArtifactUniverse from agent_env.artifact.store import reset_artifact_store +from agent_env.bundle.hub import HubSource from agent_env.bundle.installed import find_bundle from agent_env.cli import cli from agent_env.config import configure, reset_config @@ -132,8 +133,7 @@ def test_a_hub_bundle_runs_by_its_hf_reference_at_a_pinned_commit_and_a_rerun_re (instance_id,) = re.findall(rf"instance (@hf/{repo}/hello/hello-\S+)", second) with namespace_routing(): instance = Task.get_instance(instance_id) - assert instance.context["metadata"]["bundle_source"] == { - "hub": "hf", "repo": repo, "revision": sha, "sha": sha, "path": "bundles/hello"} + assert instance.context["metadata"]["bundle_source"] == HubSource(repo, sha, sha, "bundles/hello").to_metadata() def test_a_bundle_runs_from_its_folder_and_a_rerun_writes_only_what_changed(bundle, quiet_logs): diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 443c9c72a..6ec2eb42d 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -14,7 +14,7 @@ from agent_env.bundle import BundleError from agent_env.bundle import hub as hub_module -from agent_env.bundle.hub import HubReference, fetch_bundle, parse_reference +from agent_env.bundle.hub import HubReference, HubSource, fetch_bundle, parse_reference from agent_env.bundle.ledger import LEDGER_COLLECTION from agent_env.bundle.requires import UnmetRequirements from agent_env.cli import cli @@ -28,7 +28,7 @@ SHA = "0123456789abcdef0123456789abcdef01234567" TASK = json.dumps([{"id": "check", "type": "noop_hub_test"}]) -SOURCE = {"hub": "hf", "repo": "me/ds", "revision": "main", "sha": SHA, "path": "bundles/hello"} +SOURCE = HubSource("me/ds", "main", SHA, "bundles/hello") class _Noop(TaskStep): @@ -180,8 +180,8 @@ def test_several_bundles_need_a_path_or_the_cards_default(hub, tmp_path): assert caught.value.problems == ( "me/ds at 0123456789ab holds several bundles under bundles/ and its card names no default (agentenv: " "default); run one of hf://me/ds@v1/bundles/a, hf://me/ds@v1/bundles/b",) - assert (named.source["path"], named.id_root) == ("bundles/a", "@hf/me/ds/a") - assert (by_default.source["path"], by_default.id_root) == ("bundles/b", "@hf/me/ds/b") + assert (named.source.path, named.id_root) == ("bundles/a", "@hf/me/ds/a") + assert (by_default.source.path, by_default.id_root) == ("bundles/b", "@hf/me/ds/b") def test_a_path_names_any_bundle_folder_and_its_ids_follow_it(hub): @@ -191,7 +191,7 @@ def test_a_path_names_any_bundle_folder_and_its_ids_follow_it(hub): with pytest.raises(BundleError) as caught: fetch_bundle("hf://me/ds/suites") - assert (fetched.id_root, fetched.source["path"]) == ("@hf/me/ds/suites/hello", "suites/hello") + assert (fetched.id_root, fetched.source.path) == ("@hf/me/ds/suites/hello", "suites/hello") assert caught.value.problems == ("me/ds at 0123456789ab: suites/ isn't a bundle: it has none of envs/, agents/, " "artifacts/, skills/, tasks/, evals/",) @@ -201,7 +201,7 @@ def test_the_root_is_the_bundle_when_nothing_is_under_bundles_and_its_other_fold fetched = fetch_bundle("hf://me/ds") - assert (fetched.root, fetched.id_root, fetched.source["path"]) == (_commit(tmp_path), "@hf/me/ds", "") + assert (fetched.root, fetched.id_root, fetched.source.path) == (_commit(tmp_path), "@hf/me/ds", "") assert _downloaded(tmp_path) == ["bundle.toml", "tasks/t.json"] @@ -227,7 +227,7 @@ def test_a_commit_already_downloaded_is_reused_and_a_pinned_one_needs_no_network other = fetch_bundle(f"hf://me/ds@{SHA}/bundles/b") assert first.root == again.root == by_branch.root - assert again.source["revision"] == SHA and by_branch.source["revision"] == "main" + assert again.source.revision == SHA and by_branch.source.revision == "main" assert reused == ["repo_info"] assert hub.calls == ["snapshot_download bundles/b/*"] assert other.root.parent == first.root.parent @@ -254,7 +254,9 @@ def test_a_hub_bundle_runs_under_its_hf_id_root_and_each_run_records_where_it_ca assert "tasks/t.json: v1, unchanged" in second.output with namespace_routing(): runs = task_instances("@hf/me/ds/hello/t") - assert [(run.task_version, run.context["metadata"]["bundle_source"]) for run in runs] == [(1, SOURCE)] * 2 + assert [(run.task_version, run.context["metadata"]["bundle_source"]) for run in runs] == [ + (1, {"hub": "hf", "repo": "me/ds", "revision": "main", "sha": SHA, "path": "bundles/hello"})] * 2 + assert HubSource.from_metadata(runs[0].context["metadata"]["bundle_source"]) == SOURCE rows = local_stores.local_namespace_document_store("hf").query(LEDGER_COLLECTION, Filter()) assert {row["id"] for row in rows} == {"@hf/me/ds/hello/t"} assert not local_stores.local_namespace_document_store().path.exists() From 62d87db2331e4d571293600e31e7d4e2ed72adcc Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:35:42 -0800 Subject: [PATCH 11/13] fix(bundle): one name for a Hub bundle, keying both its card table and its id root A bundle outside bundles/ was looked up in the card by its last folder name, so /a/x and /b/x shared a table while their ids differed; both now use its path. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/hub.py | 16 ++++++++++------ tst/unit/bundle/hub_test.py | 10 ++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 1e80cb32e..b69e9da25 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -172,7 +172,7 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H card = _card(folder) path = _pick(reference, sha, commit.files, card) id_root = _id_root(reference, path) - plugins, setup = _needs(reference.repo, card, path.rpartition("/")[2] or reference.repo.partition("/")[2]) + plugins, setup = _needs(reference.repo, card, _bundle_name(reference, path)) source = HubSource(reference.repo, reference.revision or "main", sha, path) declared = _within(path, BUNDLE_TOML) in commit.files hub = HubBundle(folder / path, id_root, source, plugins=() if declared else plugins, setup=setup) @@ -275,11 +275,8 @@ def _patterns(path: str) -> list[str]: def _id_root(reference: HubReference, path: str) -> str: - """``@hf//``, then the bundle's name for one under ``bundles/``, or its path for one elsewhere.""" - parts = path.split("/") if path else [] - if len(parts) == 2 and parts[0] == BUNDLES: - parts = parts[1:] - id_root = "/".join([f"{HF_PREFIX}{reference.repo}", *parts]) + """``@hf//``, then for a bundle below the root its name.""" + id_root = f"{HF_PREFIX}{reference.repo}" + (f"/{_bundle_name(reference, path)}" if path else "") try: validate_local_id(id_root) except ValueError as e: @@ -287,6 +284,13 @@ def _id_root(reference: HubReference, path: str) -> str: return id_root +def _bundle_name(reference: HubReference, path: str) -> str: + """The bundle's name, which keys its table in the card and ends its id root: its folder under ``bundles/``, its + path for a folder elsewhere, or the repo's name for the root.""" + parts = path.split("/") + return parts[1] if len(parts) == 2 and parts[0] == BUNDLES else path or reference.repo.partition("/")[2] + + def _within(folder: str, name: str) -> str: return f"{folder}/{name}" if folder else name diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 6ec2eb42d..00b6379bc 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -196,6 +196,16 @@ def test_a_path_names_any_bundle_folder_and_its_ids_follow_it(hub): "artifacts/, skills/, tasks/, evals/",) +def test_a_bundle_is_named_by_its_path_outside_bundles_so_two_folders_of_one_name_keep_apart(hub): + hub.layout({"a/x/tasks/t.json": TASK, "b/x/tasks/t.json": TASK, "bundles/x/tasks/t.json": TASK}) + hub.card({"bundles": {"a/x": {"setup": "set up a/x"}, "x": {"setup": "set up bundles/x"}}}) + + a, b, x = (fetch_bundle(f"hf://me/ds/{path}") for path in ("a/x", "b/x", "bundles/x")) + + assert [(bundle.id_root, bundle.setup) for bundle in (a, b, x)] == [ + ("@hf/me/ds/a/x", "set up a/x"), ("@hf/me/ds/b/x", None), ("@hf/me/ds/x", "set up bundles/x")] + + def test_the_root_is_the_bundle_when_nothing_is_under_bundles_and_its_other_folders_stay_on_the_hub(hub, tmp_path): hub.layout({"tasks/t.json": TASK, "bundle.toml": "requires = []\n", "data/rows.parquet": "rows"}) From 702b6c551e6757138125d3c1c8e7b8eaa76e3cc0 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:37:03 -0800 Subject: [PATCH 12/13] refactor(bundle): split a Hub fetch into the commit, a pure selection and the download, checking requirements on every fetch fetch_bundle now checks what the bundle requires on every call, a cached commit included, and adds the card's setup note itself, so HubBundle.check_plugins and its two callers go, and agent-env run has one offer_to_install path with locate_bundle inside its try. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/bundle/hub.py | 79 ++++++++++++++++++++---------------- src/agent_env/cli/run.py | 15 +++---- src/agent_env/openenv/app.py | 4 +- tst/unit/bundle/hub_test.py | 38 ++++++++++++++++- 4 files changed, 88 insertions(+), 48 deletions(-) diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index b69e9da25..3b314fc4a 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -91,13 +91,6 @@ def setup_note(self) -> str | None: """What a run refused for something setting the bundle up provides ends with: the card's command for it.""" return f"{self.source.repo}'s card sets this bundle up with: {self.setup}" if self.setup else None - def check_plugins(self) -> None: - """Refuse the run when a plugin the card names isn't installed, naming the command that installs it. Raises - UnmetRequirements.""" - problems, unmet = unmet_problems(self.plugins, f"{self.source.repo}'s card") - if problems: - raise UnmetRequirements(problems, unmet) if unmet else BundleError(problems) - @dataclass class _Commit: @@ -163,36 +156,54 @@ def _fetch(reference: HubReference, api: HfApi, say: Callable[[str], None]) -> H reference.repo, repo_type="dataset", revision=reference.revision).sha folder = cache_root() / "hf" / reference.repo / sha with holding_locks([f"{SCHEME}{reference.repo}@{sha}"]): - commit = _Commit.read(folder) - if commit is None: - commit = _Commit(api.list_repo_files(reference.repo, repo_type="dataset", revision=sha)) - if CARD in commit.files: - api.hf_hub_download(reference.repo, CARD, repo_type="dataset", revision=sha, local_dir=folder) - commit.save(folder) - card = _card(folder) - path = _pick(reference, sha, commit.files, card) - id_root = _id_root(reference, path) - plugins, setup = _needs(reference.repo, card, _bundle_name(reference, path)) - source = HubSource(reference.repo, reference.revision or "main", sha, path) - declared = _within(path, BUNDLE_TOML) in commit.files - hub = HubBundle(folder / path, id_root, source, plugins=() if declared else plugins, setup=setup) - say(source.describe()) - if path not in commit.downloaded: - if declared: - api.hf_hub_download(reference.repo, _within(path, BUNDLE_TOML), repo_type="dataset", revision=sha, - local_dir=folder) - _refuse_unmet(hub, declared) - api.snapshot_download(reference.repo, repo_type="dataset", revision=sha, local_dir=folder, - allow_patterns=_patterns(path)) - commit.downloaded.append(path) - commit.save(folder) + commit, card = _commit(api, reference, sha, folder) + hub = _select(reference, sha, commit.files, card, folder) + say(hub.source.describe()) + _download(api, hub, commit, folder) return hub -def _refuse_unmet(hub: HubBundle, declared: bool) -> None: - """Refuse the run before the bundle is downloaded when what it requires isn't installed: its bundle.toml's - requires, by a folder's rules, once that file alone is here, else the plugins the card names. The refusal ends - with the card's setup command. Raises UnmetRequirements, or BundleError for a bundle.toml a folder's run refuses.""" +def _commit(api: HfApi, reference: HubReference, sha: str, folder: Path) -> tuple[_Commit, dict[str, Any]]: + """What this machine holds of the commit, and its card's ``agentenv`` table: listed, with the card downloaded, + the first time.""" + commit = _Commit.read(folder) + if commit is None: + commit = _Commit(api.list_repo_files(reference.repo, repo_type="dataset", revision=sha)) + if CARD in commit.files: + api.hf_hub_download(reference.repo, CARD, repo_type="dataset", revision=sha, local_dir=folder) + commit.save(folder) + return commit, _card(folder) + + +def _select(reference: HubReference, sha: str, files: Sequence[str], card: Mapping[str, Any], folder: Path) -> HubBundle: + """The bundle ``reference`` names in the commit of ``files``, kept under ``folder``: picked, with its id root, + where it came from and what the card says it needs.""" + path = _pick(reference, sha, files, card) + plugins, setup = _needs(reference.repo, card, _bundle_name(reference, path)) + source = HubSource(reference.repo, reference.revision or "main", sha, path) + declared = _within(path, BUNDLE_TOML) in files + return HubBundle(folder / path, _id_root(reference, path), source, plugins=() if declared else plugins, setup=setup) + + +def _download(api: HfApi, hub: HubBundle, commit: _Commit, folder: Path) -> None: + """Check what the bundle requires, its bundle.toml downloaded alone first when it has one, then download the + bundle unless the commit's record has it.""" + path = hub.source.path + declared, downloaded = _within(path, BUNDLE_TOML) in commit.files, path in commit.downloaded + if declared and not downloaded: + api.hf_hub_download(hub.source.repo, _within(path, BUNDLE_TOML), repo_type="dataset", revision=hub.source.sha, + local_dir=folder) + _check_requirements(hub, declared) + if not downloaded: + api.snapshot_download(hub.source.repo, repo_type="dataset", revision=hub.source.sha, local_dir=folder, + allow_patterns=_patterns(path)) + commit.downloaded.append(path) + commit.save(folder) + + +def _check_requirements(hub: HubBundle, declared: bool) -> None: + """Refuse a bundle whose bundle.toml (by a folder's rules) or, without one, card names a plugin not installed, + the refusal ending with the card's setup command. Raises UnmetRequirements, or BundleError.""" if declared: requires, refused = read_requires(hub.root / BUNDLE_TOML) if refused: diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 1fff81edf..0afcd80e5 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -100,16 +100,10 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t "--dry-run and --no-emulate need a BUNDLE to run") _list_installed() return - try: - root, id_root, hub = locate_bundle(bundle, _installed, on_progress=click.echo) - except UnmetRequirements as e: - if not dry_run: - offer_to_install(e) - raise verbose = ctx.find_root().params.get("verbose") + root = id_root = hub = None try: - if hub is not None: - hub.check_plugins() + root, id_root, hub = locate_bundle(bundle, _installed, on_progress=click.echo) if dry_run: click.echo(_DRY_RUN) _report_dry_run(dry_run_bundle(root, tasks=tasks, evals=evals, sandbox=sandbox, agent=agent, @@ -119,8 +113,9 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t judge_model=judge_model, agent=agent, sandbox=sandbox, on_progress=click.echo, id_root=id_root, keep=keep, emulate=not no_emulate, source=hub.source if hub else None) except BundleError as e: - _note_an_installed_namesake(e, bundle, root, id_root) - if isinstance(e, (UnmetRequirements, NotInStore)) and hub is not None and hub.setup_note: + if root is not None: + _note_an_installed_namesake(e, bundle, root, id_root) + if isinstance(e, NotInStore) and hub is not None and hub.setup_note: e.add_note(hub.setup_note) if isinstance(e, UnmetRequirements) and not dry_run: offer_to_install(e) diff --git a/src/agent_env/openenv/app.py b/src/agent_env/openenv/app.py index 7999867f5..869b64abf 100644 --- a/src/agent_env/openenv/app.py +++ b/src/agent_env/openenv/app.py @@ -104,9 +104,7 @@ def _sources(bundle: _Source | Mapping[str, _Source]) -> dict[str | None, _Sourc def _prepared(source: _Source, sandbox: str | None) -> PreparedBundle: """``source`` prepared with every task and eval selected, so ``all`` holds every task, evals or not.""" - root, id_root, hub = (source, None, None) if isinstance(source, Path) else locate_bundle(source) - if hub is not None: - hub.check_plugins() + root, id_root = (source, None) if isinstance(source, Path) else locate_bundle(source)[:2] entries = parse_bundle(Path(root), id_root=id_root).entries return prepare_bundle( root, tasks=[entry.name for entry in entries if entry.kind is BundleKind.TASK], diff --git a/tst/unit/bundle/hub_test.py b/tst/unit/bundle/hub_test.py index 00b6379bc..afdac2425 100644 --- a/tst/unit/bundle/hub_test.py +++ b/tst/unit/bundle/hub_test.py @@ -14,7 +14,7 @@ from agent_env.bundle import BundleError from agent_env.bundle import hub as hub_module -from agent_env.bundle.hub import HubReference, HubSource, fetch_bundle, parse_reference +from agent_env.bundle.hub import HubBundle, HubReference, HubSource, _select, fetch_bundle, parse_reference from agent_env.bundle.ledger import LEDGER_COLLECTION from agent_env.bundle.requires import UnmetRequirements from agent_env.cli import cli @@ -25,6 +25,7 @@ from agent_env.store.routing import namespace_routing from agent_env.task.store import task_instances from agent_env.task_step.task_step import TaskStep +from tst.unit.bundle._support import fake_plugins SHA = "0123456789abcdef0123456789abcdef01234567" TASK = json.dumps([{"id": "check", "type": "noop_hub_test"}]) @@ -351,6 +352,41 @@ def test_an_openenv_server_of_a_hub_bundle_needs_the_plugins_its_card_names_befo assert hub.calls == ["repo_info", "list_repo_files", "hf_hub_download README.md"] +def test_a_cached_commit_is_refused_while_a_plugin_its_card_names_is_missing_by_the_cli_and_openenv_alike( + hub, monkeypatch, local_stores, quiet_logs): + hub.layout({"bundles/hello/tasks/t.json": TASK}) + hub.card({"bundles": {"hello": {"plugins": ["agentenv-gone>=1"], "setup": "agent-env demo setup"}}}) + fake_plugins(monkeypatch, versions={"agentenv-gone": "1.0"}) + fetch_bundle(f"hf://me/ds@{SHA}") + fake_plugins(monkeypatch) + hub.calls.clear() + + refused = CliRunner().invoke(cli, ["run", f"hf://me/ds@{SHA}"]) + with pytest.raises(UnmetRequirements) as served: + build_app(f"hf://me/ds@{SHA}") + + assert refused.exit_code == 1 + assert ("plugin add 'agentenv-gone>=1'\nme/ds's card sets this bundle up with: agent-env demo setup\n" + in refused.output) + assert refused.output.count("sets this bundle up with") == 1 + assert served.value.__notes__ == ["me/ds's card sets this bundle up with: agent-env demo setup"] + assert hub.calls == [] + + +def test_selecting_a_bundle_from_a_commits_files_and_card_reads_nothing(tmp_path): + files = ["README.md", "tasks/train.parquet", "bundles/a/bundle.toml", "bundles/a/tasks/t.json", + "bundles/b/tasks/t.json"] + card = {"default": "b", "bundles": {"a": {"plugins": ["agentenv-a"]}, "b": {"plugins": ["agentenv-b"], + "setup": "set up b"}}} + + default = _select(parse_reference("hf://me/ds@v1"), SHA, files, card, tmp_path) + declared = _select(parse_reference("hf://me/ds/bundles/a"), SHA, files, card, tmp_path) + + assert default == HubBundle(tmp_path / "bundles/b", "@hf/me/ds/b", HubSource("me/ds", "v1", SHA, "bundles/b"), + plugins=("agentenv-b",), setup="set up b") + assert (declared.id_root, declared.plugins, declared.setup) == ("@hf/me/ds/a", (), None) + + def test_the_cards_setup_command_is_shown_only_when_setting_up_could_fix_the_run(hub, local_stores, quiet_logs): note = "me/ds's card sets this bundle up with: agent-env demo setup" gap = {"id": "deploy", "type": "deploy_env", "env_id": "env-setup-writes"} From 4602fbd905d8c0792a361bd1808c1430b86f1387 Mon Sep 17 00:00:00 2001 From: Edgar Arakelyan Date: Mon, 2 Nov 2026 22:37:51 -0800 Subject: [PATCH 13/13] docs(bundle): the Hub bundle rules once, in the README and _pick; no field comments, shorter docstrings Co-Authored-By: Claude Opus 5.5 --- README.md | 24 ++++++++++++++++++------ src/agent_env/bundle/hub.py | 29 +++++++++++------------------ src/agent_env/bundle/installed.py | 13 +++++++------ src/agent_env/bundle/plan.py | 4 ++-- src/agent_env/cli/run.py | 13 ++++--------- 5 files changed, 42 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index c808221ae..8932bee01 100644 --- a/README.md +++ b/README.md @@ -53,14 +53,26 @@ agent-env run hf:/// # the main branch agent-env run hf:///@v1.0/bundles/hello # a tag, branch or commit, and the bundle's folder ``` -The revision resolves to a commit, and only that commit's bundle and the dataset card are downloaded, into +The revision (a branch, tag or commit, percent-encoded when it holds a `/`; the main branch by default) resolves to a +commit, and only that commit's bundle and the dataset card are downloaded, into `$XDG_CACHE_HOME/agentenv/hf///` (`~/.cache` by default), where a later run of the commit finds them. Without a path, the run takes the only bundle under `bundles/`, or the one the card's `agentenv: default` names, -else the repo's root. It prints the repo, commit and bundle, then runs as a folder does, with ids under -`@hf//`, so a rerun of the same commit reuses what the last one wrote, and records the commit with each run. -A bundle without a `bundle.toml` needs the plugins the card's `agentenv: bundles: : plugins` lists; either is -checked before the bundle is downloaded, and a run refused for a missing plugin, agent, env or artifact names the -card's `setup` command. A gated or private dataset needs `hf auth login` or `HF_TOKEN`. +else the repo's root when it has a kind folder (`envs/`, `tasks/`, ...). It prints the repo, commit and bundle, then +runs as a folder does, with ids under `@hf///`, so a rerun of the same commit reuses what the last one +wrote, and records the commit with each run. A bundle's `` is its folder under `bundles/`, or its path for a folder +elsewhere; the root's ids are `@hf//`, and its name is the repo's. The card says what each bundle needs: + +```yaml +agentenv: + default: hello # the bundle a run without a path takes, of several under bundles/ + bundles: + hello: # the bundle's , or the repo's name for the root + plugins: ["agentenv-example>=1.2"] # what a bundle without a bundle.toml requires + setup: agent-env example setup # shown when a missing plugin, agent, env or artifact stops a run +``` + +Each run checks the bundle's `bundle.toml`, else the card's plugins, the first one before the bundle is downloaded; the +`setup` command is never run. A gated or private dataset needs `hf auth login` or `HF_TOKEN`. ## Plugins diff --git a/src/agent_env/bundle/hub.py b/src/agent_env/bundle/hub.py index 3b314fc4a..e448ef70c 100644 --- a/src/agent_env/bundle/hub.py +++ b/src/agent_env/bundle/hub.py @@ -1,14 +1,6 @@ -"""Bundles run straight from a Hugging Face Hub dataset repo, named ``hf:///[@][/]``. - -The revision, a branch, tag or commit (the main branch by default), resolves to a commit, and the bundle is picked from -that commit's file list before anything is downloaded, so a results dataset's recordings never are. Only the bundle's -files and the repo's card are downloaded, into a plain folder for the commit: the Hub's own cache keeps files as links -into its blob store, and the bundle parser refuses a link that leaves the bundle. A commit never changes, so what was -downloaded for it is reused; a reference pinned to the commit then needs no network at all. The bundle's ids are rooted -at ``@hf//``, plus ``/`` for a bundle under ``bundles/``, a local namespace of its own: the id says the -bundle came from the Hub, and a rerun of the same commit reuses what the last one wrote. A gated or private repo is read -with the Hub library's own login (``hf auth login`` or ``HF_TOKEN``). -""" +"""Bundles run straight from a Hugging Face Hub dataset repo, named ``hf:///[@][/]``. Each +commit's bundle is downloaded once into a plain folder, since the Hub's own cache keeps files as links that would leave +the bundle, and its ids are rooted in the ``@hf`` namespace, which only this path produces.""" from __future__ import annotations @@ -37,7 +29,6 @@ SCHEME = "hf://" BUNDLES = "bundles" CARD = "README.md" -# The key a run's context records where its bundle came from under, in its metadata. SOURCE_KEY = "bundle_source" _NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") @@ -48,9 +39,11 @@ @dataclass(frozen=True) class HubReference: - repo: str # / - revision: str | None # None for the main branch - path: str | None # the bundle's folder in the repo, when named + """An ``hf://`` reference's parts: ``/``, the revision named, and the bundle's folder in the repo.""" + + repo: str + revision: str | None + path: str | None @dataclass(frozen=True) @@ -78,13 +71,13 @@ def describe(self) -> str: @dataclass(frozen=True) class HubBundle: """A bundle downloaded from the Hub: its folder, the id root it runs under, where it came from, and what the repo's - card says it needs.""" + card says it needs: the plugins it requires when it has no bundle.toml, and the command that sets it up.""" root: Path id_root: str source: HubSource - plugins: tuple[str, ...] # the card's, for a bundle without a bundle.toml - setup: str | None # the card's command that sets the bundle up, shown when setting up could fix a run, never run + plugins: tuple[str, ...] + setup: str | None @property def setup_note(self) -> str | None: diff --git a/src/agent_env/bundle/installed.py b/src/agent_env/bundle/installed.py index 3435fd8e8..753ff73d3 100644 --- a/src/agent_env/bundle/installed.py +++ b/src/agent_env/bundle/installed.py @@ -27,19 +27,20 @@ class Located(NamedTuple): + """A bundle's folder, the id root of an installed or Hub bundle (a folder's ids come from its path), and what a Hub + bundle was downloaded as.""" + root: Path - id_root: str | None # an installed or Hub bundle's; a folder's ids come from its path - hub: HubBundle | None # what a Hub bundle was downloaded as + id_root: str | None + hub: HubBundle | None def locate_bundle( text: str, installed: Callable[[], tuple[InstalledBundle, ...]] = installed_bundles, *, on_progress: Callable[[str], None] | None = None, ) -> Located: - """The folder ``text`` names, as ``agent-env run BUNDLE`` reads it: ``hf://`` names a bundle in a Hugging Face Hub - dataset, downloaded first (``on_progress`` is told which); an existing folder spelled exactly as written wins, and - text that starts like a path is always one; anything else names a bundle ``installed`` lists. Raises - BundleError.""" + """The bundle ``text`` names, as ``agent-env run BUNDLE`` reads it: an ``hf://`` reference, fetched first; else a folder, + when it starts like a path or exists spelled as written; else a bundle ``installed`` lists. Raises BundleError.""" if is_hub_reference(text): hub = fetch_bundle(text, on_progress=on_progress) return Located(hub.root, hub.id_root, hub) diff --git a/src/agent_env/bundle/plan.py b/src/agent_env/bundle/plan.py index f513abff2..4dda85f85 100644 --- a/src/agent_env/bundle/plan.py +++ b/src/agent_env/bundle/plan.py @@ -259,7 +259,7 @@ def __init__(self, resolved: ResolvedBundle, installed: Installed | None = None, self.others: tuple[InstalledBundle, ...] | None = None # the installed bundles but this one, once listed self.pullable: dict[str, _Pullable | str] = {} # each installed bundle read, by id root, or why it can't be self.unpulled: dict[tuple[EntityKind, str, int | None], str] = {} # why an installed bundle's id isn't pulled - self.missing: dict[tuple[str, str], None] = {} # each agent, env and artifact the store doesn't hold, as read + self.missing: dict[tuple[str, str], None] = {} # By kind: its installed aliases, once a plan reads them; each of them the plan names, by alias; and the store's # latest version of each alias's id, as read. self.alias_tables: dict[EntityKind, InstalledAliases] = {} @@ -715,7 +715,7 @@ def _check_store_refs( read: dict[tuple[EntityKind, str, int | None], Any] = {} checked: dict[tuple[EntityKind, str, int | None], Reference] = {} latest: dict[tuple[EntityKind, str], int] = {} - gaps: dict[str, list[str]] = {} # each problem of a missing agent, env or artifact, with where it's used + gaps: dict[str, list[str]] = {} for key in sorted(closure, key=self.rank.__getitem__): source = self.nodes[key] if isinstance(source, BundleImage): diff --git a/src/agent_env/cli/run.py b/src/agent_env/cli/run.py index 0afcd80e5..c66cd289c 100644 --- a/src/agent_env/cli/run.py +++ b/src/agent_env/cli/run.py @@ -63,15 +63,10 @@ def run(ctx: click.Context, bundle: str | None, tasks: tuple[str, ...], evals: t packages provide that name), or hf://ORG/REPO[@REVISION][/PATH], a bundle in a Hugging Face Hub dataset. Without BUNDLE, it lists the installed bundles. - A Hub bundle is downloaded at the commit REVISION names (a branch, tag or commit, percent-encoded when it holds a - /; the main branch by default) into a folder for that commit under $XDG_CACHE_HOME/agentenv/hf, and runs as a - folder does, with its ids under @hf/ORG/REPO. PATH names its folder in the repo; without it, the run takes the - only bundle under bundles/, or the one the dataset card's agentenv: default names, else the repo's root. Only - the bundle and the card are downloaded, and a commit already downloaded is reused. The run first prints the repo, - commit and bundle. A bundle with no bundle.toml requires the plugins the card's agentenv: bundles: NAME: plugins - lists, checked, as a bundle.toml's requires are, before the bundle is downloaded. A run refused for a missing - plugin, or for an agent, env or artifact the store doesn't hold, shows the card's setup command, which it never - runs. A gated or private repo needs hf auth login or HF_TOKEN. + A Hub bundle is downloaded at the commit REVISION names (the main branch by default), PATH being its folder in the + repo, and runs as a folder does, with its ids under @hf/ORG/REPO; the run first prints the repo, commit and bundle. + The README's "Run a bundle from the Hub" says which bundle a run without PATH takes, where it's kept, and what the + dataset card can say it needs. A bundle's bundle.toml names the plugin packages it requires, and a run refuses to start without them; in a terminal it offers to install them with agent-env plugin add. A folder with no bundle.toml that uses plugin