From 709a585b30a1c2ba9e5da28070c078fa23308b2d Mon Sep 17 00:00:00 2001 From: Jaideep <67646710+jdchawla29@users.noreply.github.com> Date: Wed, 19 Aug 2026 22:31:22 -0700 Subject: [PATCH] feat(builds): add canonical context identity --- docs/v6/reference/cli.mdx | 7 +- hud/build_context.py | 117 ++++++++++++++++++++++ hud/cli/deploy.py | 33 +++++-- hud/cli/tests/test_deploy.py | 50 ++++++++++ hud/cli/utils/context.py | 145 +++++++++++++++++----------- hud/cli/utils/source.py | 46 +-------- hud/cli/utils/tests/test_context.py | 32 ++++-- hud/cli/utils/tests/test_source.py | 30 ------ hud/integrations/harbor/adapt.py | 32 +++--- 9 files changed, 326 insertions(+), 166 deletions(-) create mode 100644 hud/build_context.py diff --git a/docs/v6/reference/cli.mdx b/docs/v6/reference/cli.mdx index dd7659443..a4a736c8f 100644 --- a/docs/v6/reference/cli.mdx +++ b/docs/v6/reference/cli.mdx @@ -46,7 +46,11 @@ hud serve env.py -p 9000 Build **and** publish to HUD infra in one step. The environment's name comes from the `Environment(...)` declaration in code; deploying the same name again -rebuilds that environment. +targets that environment. The CLI creates one canonical manifest for the exact +files placed in the build-context archive, and the platform verifies the +uploaded archive against it. An identical in-flight deploy joins the existing +build; an identical completed deploy reuses its artifact while every immutable +component still exists. The upload currently happens before this reuse check. ```bash hud deploy @@ -57,6 +61,7 @@ hud deploy | `--all`, `-a` | Deploy all environments in the directory. | | `--env`, `-e` | Env var `KEY=VALUE` (repeatable). | | `--env-file` | Path to a `.env` file. | +| `--no-cache` | Force a new build version and bypass both whole-build reuse and persistent build cache inputs. | ## Evaluate diff --git a/hud/build_context.py b/hud/build_context.py new file mode 100644 index 000000000..1201565d3 --- /dev/null +++ b/hud/build_context.py @@ -0,0 +1,117 @@ +"""Canonical filesystem identity for HUD build contexts.""" + +from __future__ import annotations + +import hashlib +import json +import stat +from typing import TYPE_CHECKING, Literal, Self + +from pydantic import BaseModel, ConfigDict, Field, model_validator + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + +BUILD_CONTEXT_MANIFEST_VERSION = 1 + + +class BuildContextEntry(BaseModel): + """One build-semantic filesystem entry.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + path: str = Field(min_length=1) + type: Literal["file", "symlink"] + mode: int = Field(ge=0, le=0o7777) + size: int | None = Field(default=None, ge=0) + content_digest: str | None = None + target: str | None = None + + @model_validator(mode="after") + def _validate_shape(self) -> Self: + if ( + self.path.startswith(("/", "\\")) + or "\\" in self.path + or any(part in {"", ".", ".."} for part in self.path.split("/")) + ): + raise ValueError(f"Build context path must be relative and normalized: {self.path!r}") + if self.type == "file": + if self.size is None or self.content_digest is None or self.target is not None: + raise ValueError("File entries require size and content_digest only") + if len(self.content_digest) != 64 or any( + character not in "0123456789abcdef" for character in self.content_digest + ): + raise ValueError("File content_digest must be a lowercase SHA-256 digest") + elif self.target is None or self.size is not None or self.content_digest is not None: + raise ValueError("Symlink entries require target only") + return self + + +class BuildContextManifest(BaseModel): + """Canonical manifest shared by build-context archiving and identity.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + version: Literal[1] = BUILD_CONTEXT_MANIFEST_VERSION + entries: tuple[BuildContextEntry, ...] + + @model_validator(mode="after") + def _validate_order(self) -> Self: + paths = [entry.path for entry in self.entries] + if paths != sorted(paths) or len(paths) != len(set(paths)): + raise ValueError("Build context entries must have unique paths in sorted order") + return self + + @classmethod + def from_paths(cls, root: Path, paths: Iterable[Path]) -> Self: + resolved_root = root.resolve() + entries = tuple( + _entry_from_path(resolved_root, path) + for path in sorted(paths, key=lambda path: path.relative_to(resolved_root).as_posix()) + ) + return cls(entries=entries) + + @classmethod + def from_directory(cls, root: Path) -> Self: + resolved_root = root.resolve() + return cls.from_paths( + resolved_root, + (path for path in resolved_root.rglob("*") if path.is_symlink() or path.is_file()), + ) + + def digest(self) -> str: + payload = json.dumps( + self.model_dump(mode="json"), + sort_keys=True, + separators=(",", ":"), + ).encode() + return hashlib.sha256(payload).hexdigest() + + +def _entry_from_path(root: Path, path: Path) -> BuildContextEntry: + relative_path = path.relative_to(root).as_posix() + metadata = path.lstat() + mode = stat.S_IMODE(metadata.st_mode) + if path.is_symlink(): + return BuildContextEntry( + path=relative_path, + type="symlink", + mode=mode, + target=path.readlink().as_posix(), + ) + if not stat.S_ISREG(metadata.st_mode): + raise ValueError(f"Unsupported build context entry: {relative_path}") + + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return BuildContextEntry( + path=relative_path, + type="file", + mode=mode, + size=metadata.st_size, + content_digest=digest.hexdigest(), + ) diff --git a/hud/cli/deploy.py b/hud/cli/deploy.py index dc9e26d6c..b63df26ab 100644 --- a/hud/cli/deploy.py +++ b/hud/cli/deploy.py @@ -9,7 +9,7 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import httpx import typer @@ -18,7 +18,7 @@ from hud.cli.utils.build_display import display_build_summary from hud.cli.utils.build_logs import poll_build_status, stream_build_logs from hud.cli.utils.config import parse_env_file, parse_key_value -from hud.cli.utils.context import create_build_context_tarball, format_size +from hud.cli.utils.context import BuildContextArchive, create_build_context_tarball, format_size from hud.cli.utils.registry import get_registry_environment from hud.cli.utils.source import EnvironmentSource from hud.eval.runtime import ComposeProject, RuntimeConfig @@ -27,6 +27,9 @@ from hud.utils.naming import normalize_environment_name from hud.utils.platform import PlatformClient +if TYPE_CHECKING: + from hud.build_context import BuildContextManifest + LOGGER = logging.getLogger(__name__) _VALID_RUNTIMES = {"hud", "modal"} _COMPOSE_RECIPE_NAMES = ( @@ -349,10 +352,15 @@ def _collect_build_secrets( return secrets -def _create_tarball(env_dir: Path, *, verbose: bool, console: HUDConsole) -> Path: +def _create_tarball( + env_dir: Path, + *, + verbose: bool, + console: HUDConsole, +) -> BuildContextArchive: console.progress_message("Creating build context tarball...") try: - tarball_path, tarball_size, file_count, tarball_duration = create_build_context_tarball( + archive = create_build_context_tarball( env_dir, verbose=verbose, ) @@ -361,10 +369,10 @@ def _create_tarball(env_dir: Path, *, verbose: bool, console: HUDConsole) -> Pat raise typer.Exit(1) from e console.success( - f"Created tarball: {format_size(tarball_size)} ({file_count} files) " - f"[{tarball_duration:.1f}s]" + f"Created tarball: {format_size(archive.size_bytes)} ({archive.file_count} files) " + f"[{archive.duration_seconds:.1f}s]" ) - return tarball_path + return archive def _prepare_deploy_plan( @@ -499,11 +507,12 @@ def deploy_environment( platform=platform, console=hud_console, ) - tarball_path = _create_tarball(env_dir, verbose=verbose, console=hud_console) + archive = _create_tarball(env_dir, verbose=verbose, console=hud_console) try: result = asyncio.run( _deploy_async( - tarball_path=tarball_path, + tarball_path=archive.path, + context_manifest=archive.manifest, no_cache=no_cache, plan=plan, platform=platform, @@ -512,7 +521,7 @@ def deploy_environment( ) ) finally: - tarball_path.unlink(missing_ok=True) + archive.path.unlink(missing_ok=True) if not result.success: raise typer.Exit(1) @@ -541,6 +550,7 @@ async def _trigger_build( platform: PlatformClient, *, build_id: str, + context_manifest: BuildContextManifest, plan: _DeployPlan, no_cache: bool, ) -> tuple[str, str]: @@ -549,6 +559,7 @@ async def _trigger_build( "build_id": build_id, "name": plan.name, "no_cache": no_cache, + "context_manifest": context_manifest.model_dump(mode="json"), } payload.update( { @@ -575,6 +586,7 @@ async def _trigger_build( async def _deploy_async( tarball_path: Path, + context_manifest: BuildContextManifest, no_cache: bool, plan: _DeployPlan, platform: PlatformClient, @@ -619,6 +631,7 @@ async def _deploy_async( build_id, registry_id = await _trigger_build( platform, build_id=reserved_id, + context_manifest=context_manifest, plan=plan, no_cache=no_cache, ) diff --git a/hud/cli/tests/test_deploy.py b/hud/cli/tests/test_deploy.py index e915092fe..a8475c29b 100644 --- a/hud/cli/tests/test_deploy.py +++ b/hud/cli/tests/test_deploy.py @@ -30,6 +30,52 @@ def test_normalize_runtime_rejects_internal_provider_name() -> None: _normalize_runtime("ec2", HUDConsole()) +@pytest.mark.asyncio +async def test_trigger_sends_canonical_context_manifest() -> None: + from hud.build_context import BuildContextEntry, BuildContextManifest + from hud.cli.deploy import _DeployPlan, _trigger_build + + manifest = BuildContextManifest( + entries=( + BuildContextEntry( + path="Dockerfile", + type="file", + mode=0o644, + size=12, + content_digest="a" * 64, + ), + ), + ) + plan = _DeployPlan( + name="test-env", + registry_id=None, + runtime=None, + runtime_config=None, + env_vars={}, + build_args={}, + build_secrets={}, + ) + platform = PlatformClient("https://api.example", "key") + + with patch.object( + PlatformClient, + "apost", + AsyncMock(return_value={"id": "existing-build", "registry_id": "registry"}), + ) as post: + result = await _trigger_build( + platform, + build_id="reserved-upload", + context_manifest=manifest, + plan=plan, + no_cache=False, + ) + + assert result == ("existing-build", "registry") + call = post.await_args + assert call is not None + assert call.kwargs["json"]["context_manifest"] == manifest.model_dump(mode="json") + + class TestResolveEnvironmentName: """Tests for code-authoritative environment name resolution.""" @@ -435,6 +481,7 @@ class TestDeployAsync: @pytest.mark.asyncio async def test_upload_url_failure(self) -> None: """Test handling of upload URL failure.""" + from hud.build_context import BuildContextManifest from hud.cli.deploy import _deploy_async, _DeployPlan from hud.utils.exceptions import HudRequestError from hud.utils.hud_console import HUDConsole @@ -446,6 +493,7 @@ async def test_upload_url_failure(self) -> None: with patch("hud.utils.platform.make_request", AsyncMock(side_effect=error)): result = await _deploy_async( tarball_path=Path("test.tar.gz"), + context_manifest=BuildContextManifest(entries=()), no_cache=False, plan=_DeployPlan( name="test-env", @@ -465,6 +513,7 @@ async def test_upload_url_failure(self) -> None: @pytest.mark.asyncio async def test_upload_url_network_error(self) -> None: """Test handling of network error during upload URL fetch.""" + from hud.build_context import BuildContextManifest from hud.cli.deploy import _deploy_async, _DeployPlan from hud.utils.hud_console import HUDConsole from hud.utils.platform import PlatformClient @@ -477,6 +526,7 @@ async def test_upload_url_network_error(self) -> None: ): result = await _deploy_async( tarball_path=Path("test.tar.gz"), + context_manifest=BuildContextManifest(entries=()), no_cache=False, plan=_DeployPlan( name="test-env", diff --git a/hud/cli/utils/context.py b/hud/cli/utils/context.py index aa199afed..09d786b28 100644 --- a/hud/cli/utils/context.py +++ b/hud/cli/utils/context.py @@ -3,12 +3,15 @@ from __future__ import annotations import fnmatch +import gzip import os import tarfile import tempfile import time +from dataclasses import dataclass from pathlib import Path +from hud.build_context import BuildContextManifest from hud.utils.hud_console import HUDConsole @@ -25,16 +28,12 @@ def parse_ignore_file(ignore_path: Path) -> list[str]: if not ignore_path.exists(): return patterns - try: - with open(ignore_path) as f: - for line in f: - # Strip whitespace and skip comments/empty lines - line = line.strip() - if not line or line.startswith("#"): - continue - patterns.append(line) - except Exception: # noqa: S110 - pass # Best effort - ignore parse errors + with ignore_path.open(encoding="utf-8") as file: + for line in file: + line = line.strip() + if not line or line.startswith("#"): + continue + patterns.append(line) return patterns @@ -165,26 +164,20 @@ def should_ignore( ] -def create_build_context_tarball( +@dataclass(frozen=True, slots=True) +class BuildContextArchive: + path: Path + manifest: BuildContextManifest + size_bytes: int + file_count: int + duration_seconds: float + + +def _build_context_paths( directory: Path, dockerignore_path: Path | None = None, verbose: bool = False, -) -> tuple[Path, int, int, float]: - """Create a gzipped tarball of the build context. - - Respects .dockerignore and .gitignore patterns, and always excludes - common sensitive files like .env and .git directories. - - Args: - directory: Directory to create tarball from - dockerignore_path: Optional path to .dockerignore file. - If None, looks for .dockerignore in directory. - verbose: Whether to print verbose output - - Returns: - Tuple of (tarball_path, size_bytes, file_count, duration_seconds) - """ - start_time = time.time() +) -> list[Path]: hud_console = HUDConsole() directory = directory.resolve() @@ -210,7 +203,45 @@ def create_build_context_tarball( if verbose and loaded_sources: hud_console.info(f"Loaded ignore patterns from: {', '.join(loaded_sources)}") - # Create temporary file for tarball + paths: list[Path] = [] + for root, dirs, files in os.walk(directory): + root_path = Path(root) + retained_dirs: list[str] = [] + for name in sorted(dirs): + path = root_path / name + if should_ignore(path, directory, ignore_patterns): + if verbose: + hud_console.debug(f"Skipping: {path.relative_to(directory)}") + continue + if path.is_symlink(): + paths.append(path) + else: + retained_dirs.append(name) + dirs[:] = retained_dirs + + for name in sorted(files): + path = root_path / name + if should_ignore(path, directory, ignore_patterns): + if verbose: + hud_console.debug(f"Skipping: {path.relative_to(directory)}") + continue + paths.append(path) + + return paths + + +def create_build_context_tarball( + directory: Path, + dockerignore_path: Path | None = None, + verbose: bool = False, +) -> BuildContextArchive: + """Create a tarball and canonical manifest from one selected file set.""" + start_time = time.time() + directory = directory.resolve() + manifest = BuildContextManifest.from_paths( + directory, + _build_context_paths(directory, dockerignore_path, verbose), + ) temp_file = tempfile.NamedTemporaryFile( # noqa: SIM115 suffix=".tar.gz", delete=False, @@ -219,37 +250,41 @@ def create_build_context_tarball( temp_path = Path(temp_file.name) temp_file.close() - file_count = 0 - try: - with tarfile.open(temp_path, "w:gz") as tar: - for root, dirs, files in os.walk(directory): - root_path = Path(root) - - # Filter directories in-place to skip ignored ones - dirs[:] = [ - d for d in dirs if not should_ignore(root_path / d, directory, ignore_patterns) - ] - - for file in files: - file_path = root_path / file - - if should_ignore(file_path, directory, ignore_patterns): - if verbose: - hud_console.debug(f"Skipping: {file_path.relative_to(directory)}") - continue - - # Add file to tarball with relative path - arcname = str(file_path.relative_to(directory)) - tar.add(file_path, arcname=arcname) - file_count += 1 - - if verbose: - hud_console.debug(f"Added: {arcname}") + with ( + temp_path.open("wb") as raw_archive, + gzip.GzipFile( + filename="", + fileobj=raw_archive, + mode="wb", + mtime=0, + ) as compressed, + tarfile.open(fileobj=compressed, mode="w") as tar, + ): + for entry in manifest.entries: + info = tarfile.TarInfo(entry.path) + info.mode = entry.mode + info.mtime = 0 + info.uid = 0 + info.gid = 0 + if entry.type == "symlink": + info.type = tarfile.SYMTYPE + info.linkname = entry.target or "" + tar.addfile(info) + else: + info.size = entry.size or 0 + with (directory / entry.path).open("rb") as source: + tar.addfile(info, source) size_bytes = temp_path.stat().st_size duration = time.time() - start_time - return temp_path, size_bytes, file_count, duration + return BuildContextArchive( + path=temp_path, + manifest=manifest, + size_bytes=size_bytes, + file_count=len(manifest.entries), + duration_seconds=duration, + ) except Exception: # Clean up temp file on error diff --git a/hud/cli/utils/source.py b/hud/cli/utils/source.py index 33fcbaac3..8e32c0882 100644 --- a/hud/cli/utils/source.py +++ b/hud/cli/utils/source.py @@ -3,7 +3,6 @@ from __future__ import annotations import ast -import hashlib import json import logging import os @@ -11,10 +10,7 @@ import tomllib from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any, ClassVar, Self - -if TYPE_CHECKING: - from collections.abc import Iterator +from typing import Any, ClassVar, Self LOGGER = logging.getLogger(__name__) @@ -51,8 +47,6 @@ class EnvironmentSource: CONFIG_FILENAME: ClassVar[str] = "config.json" LEGACY_CONFIG_FILENAME: ClassVar[str] = "deploy.json" - SOURCE_INCLUDE_FILES: ClassVar[set[str]] = {"Dockerfile", "Dockerfile.hud", "pyproject.toml"} - SOURCE_INCLUDE_DIRS: ClassVar[set[str]] = {"server", "mcp", "controller", "environment"} SOURCE_EXCLUDE_DIRS: ClassVar[set[str]] = { ".git", ".venv", @@ -64,8 +58,6 @@ class EnvironmentSource: ".pytest_cache", ".ruff_cache", } - SOURCE_EXCLUDE_FILES: ClassVar[set[str]] = {"hud.lock.yaml"} - SOURCE_EXCLUDE_SUFFIXES: ClassVar[set[str]] = {".pyc", ".log"} @classmethod def open(cls, directory: str | Path = ".") -> Self: @@ -210,42 +202,6 @@ def taskset_id(self) -> str | None: value = self.load_config().get("tasksetId") return value if isinstance(value, str) else None - def iter_source_files(self) -> Iterator[Path]: - for name in self.SOURCE_INCLUDE_FILES: - path = self.root / name - if path.is_file(): - yield path - - for directory in self.SOURCE_INCLUDE_DIRS: - source_dir = self.root / directory - if not source_dir.exists(): - continue - for dirpath, dirnames, filenames in os.walk(source_dir): - dirnames[:] = [name for name in dirnames if name not in self.SOURCE_EXCLUDE_DIRS] - for filename in filenames: - if filename in self.SOURCE_EXCLUDE_FILES: - continue - if any(filename.endswith(suffix) for suffix in self.SOURCE_EXCLUDE_SUFFIXES): - continue - yield Path(dirpath) / filename - - def source_files(self) -> list[Path]: - files = list(self.iter_source_files()) - files.sort(key=self.relative_path) - return files - - def source_file_refs(self) -> list[str]: - return [self.relative_path(path) for path in self.source_files()] - - def source_hash(self) -> str: - hasher = hashlib.sha256() - for path in self.source_files(): - hasher.update(self.relative_path(path).encode("utf-8")) - with path.open("rb") as file: - for chunk in iter(lambda: file.read(8192), b""): - hasher.update(chunk) - return hasher.hexdigest() - def relative_path(self, path: Path) -> str: return str(path.resolve().relative_to(self.root)).replace("\\", "/") diff --git a/hud/cli/utils/tests/test_context.py b/hud/cli/utils/tests/test_context.py index aebf59f45..b685651d0 100644 --- a/hud/cli/utils/tests/test_context.py +++ b/hud/cli/utils/tests/test_context.py @@ -59,16 +59,34 @@ def test_create_build_context_tarball_excludes_secrets(tmp_path: Path) -> None: git.mkdir() (git / "config").write_text("x", encoding="utf-8") - tarball, size, count, duration = create_build_context_tarball(ctx) + archive = create_build_context_tarball(ctx) try: - assert tarball.exists() - assert size > 0 - assert duration >= 0 - with tarfile.open(tarball) as tar: + assert archive.path.exists() + assert archive.size_bytes > 0 + assert archive.duration_seconds >= 0 + with tarfile.open(archive.path) as tar: names = tar.getnames() assert "main.py" in names assert ".env" not in names assert not any(n.startswith(".git") for n in names) - assert count == 1 + assert archive.file_count == 1 + assert [entry.path for entry in archive.manifest.entries] == ["main.py"] finally: - tarball.unlink(missing_ok=True) + archive.path.unlink(missing_ok=True) + + +def test_manifest_identity_ignores_archive_metadata(tmp_path: Path) -> None: + context = tmp_path / "context" + context.mkdir() + source = context / "main.py" + source.write_text("print('hi')\n", encoding="utf-8") + + first = create_build_context_tarball(context) + source.touch() + second = create_build_context_tarball(context) + try: + assert first.manifest.digest() == second.manifest.digest() + assert first.path.read_bytes() == second.path.read_bytes() + finally: + first.path.unlink(missing_ok=True) + second.path.unlink(missing_ok=True) diff --git a/hud/cli/utils/tests/test_source.py b/hud/cli/utils/tests/test_source.py index d29c7df9b..cceb6e79f 100644 --- a/hud/cli/utils/tests/test_source.py +++ b/hud/cli/utils/tests/test_source.py @@ -69,36 +69,6 @@ def test_base_image_without_dockerfile_is_none(tmp_path: Path) -> None: assert EnvironmentSource.open(tmp_path).base_image() is None -# ─── source files / hash ─────────────────────────────────────────────── - - -def test_source_hash_changes_with_content(tmp_path: Path) -> None: - env = tmp_path / "env" - env.mkdir() - (env / "Dockerfile").write_text("FROM python:3.11") - (env / "pyproject.toml").write_text("[tool.hud]\n") - (env / "server").mkdir() - (env / "server" / "main.py").write_text("print('hi')\n") - - source = EnvironmentSource.open(env) - h1 = source.source_hash() - (env / "server" / "main.py").write_text("print('bye')\n") - h2 = source.source_hash() - assert h1 != h2 - - -def test_source_files_sorted(tmp_path: Path) -> None: - env = tmp_path / "env" - env.mkdir() - (env / "Dockerfile").write_text("FROM python:3.11") - (env / "environment").mkdir() - (env / "environment" / "a.py").write_text("a") - (env / "environment" / "b.py").write_text("b") - - source = EnvironmentSource.open(env) - assert source.source_file_refs() == ["Dockerfile", "environment/a.py", "environment/b.py"] - - # ─── Environment("name") references ──────────────────────────────────── diff --git a/hud/integrations/harbor/adapt.py b/hud/integrations/harbor/adapt.py index f6a632905..2472d4332 100644 --- a/hud/integrations/harbor/adapt.py +++ b/hud/integrations/harbor/adapt.py @@ -6,7 +6,6 @@ import json import logging import math -import os import re import shlex import shutil @@ -17,6 +16,7 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator, model_validator +from hud.build_context import BuildContextManifest from hud.capabilities import Capability from hud.environment.egress import BRIDGE_PORT, VISITOR_PORT from hud.eval import Task, Taskset @@ -245,15 +245,8 @@ class HarborTask: resources: RuntimeResources | None -def _tree_hash(root: Path) -> str: - digest = hashlib.sha256() - for entry in sorted(root.rglob("*")): - relative_path = entry.relative_to(root).as_posix().encode() - if entry.is_symlink(): - digest.update(relative_path + b"\0symlink\0" + os.readlink(entry).encode()) - elif entry.is_file(): - digest.update(relative_path + b"\0" + entry.read_bytes()) - return digest.hexdigest()[:16] +def _context_digest(root: Path) -> str: + return BuildContextManifest.from_directory(root).digest() def _runtime_resources(environment: EnvironmentConfig) -> RuntimeResources | None: @@ -443,7 +436,7 @@ def add(code: str, message: str) -> None: "Compose main build escapes environment", ) if dockerfile.is_file(): - base_image = f"hud-harbor-base:{_tree_hash(environment_dir)}" + base_image = f"hud-harbor-base:{_context_digest(environment_dir)}" elif build is not None: add( "harbor.invalid.missing_compose_main_dockerfile", @@ -455,7 +448,7 @@ def add(code: str, message: str) -> None: "Compose main has neither image nor build", ) elif dockerfile.is_file(): - base_image = f"hud-harbor-base:{_tree_hash(environment_dir)}" + base_image = f"hud-harbor-base:{_context_digest(environment_dir)}" elif base_image is None: add( "harbor.invalid.environment_recipe", @@ -541,7 +534,9 @@ def add(code: str, message: str) -> None: path=task_dir, config=config, instruction=instruction.read_text("utf-8"), - environment_hash=_tree_hash(environment_dir) if environment_dir.exists() else "missing", + environment_hash=( + _context_digest(environment_dir) if environment_dir.exists() else "missing" + ), compose=compose, dockerfile=dockerfile, base_image=base_image, @@ -608,7 +603,7 @@ def adapt( rows = [] base_name = normalize_environment_name(dataset.name, default="harbor") for group_key, group in sorted(grouped.items()): - digest = hashlib.sha256("\0".join(group_key).encode()).hexdigest()[:12] + digest = hashlib.sha256("\0".join(group_key).encode()).hexdigest() name = f"{base_name}-{digest}" source = group[0] environment = source.config.environment @@ -621,7 +616,7 @@ def adapt( if service_name != "main" and service.build is not None and service.image is None: sidecar_tag = hashlib.sha256( f"{source.environment_hash}\0{service_name}".encode() - ).hexdigest()[:16] + ).hexdigest() compose_project.services[service_name] = service.model_copy( update={"image": f"hud-harbor-sidecar:{sidecar_tag}"} ) @@ -634,7 +629,7 @@ def adapt( verifier_image = base_image if separate: verifier_dockerfile = source.path / "tests" / "Dockerfile" - verifier_image = f"hud-harbor-verifier:{name}-{_tree_hash(verifier_dockerfile.parent)}" + verifier_image = f"hud-harbor-verifier:{_context_digest(verifier_dockerfile.parent)}" peers = [] healthy_services = [] @@ -758,8 +753,9 @@ def adapt( shutil.copy2(wheel, payload / "packages" / wheel.name) requirement = f"{HUD_ROOT}/packages/{wheel.name}" - tag = _tree_hash(payload) - image = f"hud-harbor:{name}-{tag}" + payload_digest = _context_digest(payload) + tag = hashlib.sha256(f"{name}\0{payload_digest}".encode()).hexdigest() + image = f"hud-harbor:{tag}" group_service_access = bool(healthy_services) or any( item.service != "main" for task in group