fix(cache): hold the Cargo maximum around every compiling gate command

CHANGELOG.md

@@ -191,6 +191,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Every gate command that compiles now enforces the shared Cargo target's
+  maximum before its first step and again after its last, including after a
+  failed run, and bounded direct Cargo commands enforce it after the command
+  as well as before. Only the host-build fragment enforced it, and only before
+  building, so clippy in `fast-test`, the static coverage build, `smoke`,
+  `pack-initrd` and other compiling commands grew the target unchecked; it
+  reached 237 GB against its 180 GiB maximum and a focused test run died with
+  ENOSPC.
+
 - `capsem doctor` no longer fails a boot stage because the host was busy. Each
   capsem-init stage now records the steal time accrued during it (time the
   host kept a runnable vCPU off a physical CPU), and the 500 ms per-stage

build_system/builder/gate/boundedlease.py

@@ -69,15 +69,23 @@ def leased(
     )
     with held(lock):
         _enforce_cargo_cache(root, command)
-        yield lock.environment()
+        try:
+            yield lock.environment()
+        finally:
+            _enforce_cargo_cache(root, command, refuse=False)
 
 
-def _enforce_cargo_cache(root: Path, command: Sequence[str]) -> None:
-    """Apply the shared Cargo target contract before direct machine work.
+def _enforce_cargo_cache(root: Path, command: Sequence[str], *, refuse: bool = True) -> None:
+    """Apply the shared Cargo target contract around direct machine work.
 
-    Gate plans expose the same operation as a timed prerequisite. Direct Cargo
-    has no plan, so the mandatory bounded-command choke point owns this half of
-    the invariant while it holds the same machine lock as a gate.
+    Gate commands hold the same bound as `CargoCacheBound`. Direct Cargo has no
+    plan, so the mandatory bounded-command choke point owns this half of the
+    invariant while it holds the same machine lock as a gate: before, and again
+    after, because the compile itself is what grows the target.
+
+    Only the check before refuses. Afterwards the command has run and its exit
+    status is the answer the caller asked for; a bound that cannot be restored
+    is reported, and the next compile's check refuses it.
     """
     policy = load_policy(root)
     paths = load_paths(root)
@@ -87,8 +95,10 @@ def _enforce_cargo_cache(root: Path, command: Sequence[str]) -> None:
         "cargo",
         reason=f"bounded direct command: {shlex.join(command)}",
     )
-    if result.violations:
+    if result.violations and refuse:
         raise GateError("; ".join(result.violations))
+    if result.violations:
+        _to_stderr("Cargo cache above its contract after the command: " + "; ".join(result.violations))
     if result.pruned:
         _to_stderr(_maintenance_notice(result))
 

build_system/builder/gate/cachecontrol.py

@@ -7,6 +7,7 @@
 from . import cachelayout
 from .config import for_root
 from .errors import GateError
+from .lifecycle import Resource
 from .proc import Runner
 from .sourcecommit import SourceCommit
 
@@ -99,3 +100,26 @@ def enforce(self, cache_id: str, label: str | None = None) -> None:
         """Enforce one cache owner's maximum before expensive work."""
         reason = f"gate cache enforcement for {label or cache_id}"
         self._run("enforce", cache_id, "--reason", reason)
+
+
+class CargoCacheBound(Resource, name="cargo-cache"):
+    """Hold the shared Cargo target to its contract around a compiling command.
+
+    Acquired before the first step and released after the last, on every path.
+    Enforcing only in front of one fragment's build left every other compiling
+    step unbounded and every compile's growth standing until some later run
+    happened to look, so worktree-salted units reached 237 GB against 180 GiB.
+    """
+
+    def __init__(self, runner: Runner) -> None:
+        self._runner = runner
+
+    def _enforce(self, moment: str) -> None:
+        if not self._runner.observing:
+            CacheControl(self._runner).enforce("cargo", f"{moment} compilation")
+
+    def acquire(self) -> None:
+        self._enforce("before")
+
+    def release(self) -> None:
+        self._enforce("after")

build_system/builder/gate/candidateprepare.py

@@ -102,13 +102,7 @@ def prepare(
     built_harness = phase.add(harness, after=(checked,))
     fit = phase.add(fitness, after=(built_harness,))
     dependencies = packagepreflight.fragment(plan, config, after=(fit,))
-    return runtimeprepare.prepare(
-        plan,
-        config,
-        after=(dependencies,),
-        permission=permission,
-        cache_already_enforced=True,
-    )
+    return runtimeprepare.prepare(plan, config, after=(dependencies,), permission=permission)
 
 
 def _enforce_cache(config: GateConfig) -> Call:

build_system/builder/gate/command.py

@@ -260,6 +260,7 @@ def execute(self) -> None:
                 self._config, runner, self.name,
                 exclusive=self.exclusive,
                 declared=egress.for_command(self, runner),
+                plan=plan,
             )
             with held(*acquiring) as acquired:
                 from .egress import guarded_runner_of

build_system/builder/gate/hostbuild.py

@@ -1,12 +1,10 @@
-"""The bounded host-binary build fragment and its Cargo cache prerequisite."""
+"""The host-binary build fragment."""
 
 from __future__ import annotations
 
-from .actions import Call, Run
-from .cachecontrol import CacheControl
+from .actions import Run
 from .config import GateConfig
 from .execution import Kind, Needs, Speed, Step, step
-from .opacity import CallJustification, Effect, OpaqueKind, machine_effects
 from .phase import Phase
 from .plan import Plan
 
@@ -35,42 +33,18 @@ def _build_step(config: GateConfig, *, label: str = "build-binaries") -> Step:
     )
 
 
-def _cargo_cache_step(config: GateConfig) -> Step:
-    return step(
-        "cargo-cache-enforcement",
-        Call(
-            "enforce the Cargo cache maximum before host compilation",
-            lambda ctx: CacheControl(ctx.runner).enforce("cargo", "host compilation"),
-            justification=CallJustification(
-                kind=OpaqueKind.RUNTIME_DERIVED,
-                reason="the typed cache owner inventories shared compilation units at runtime",
-                effects=machine_effects(Effect.PROCESS, Effect.FILESYSTEM, Effect.HOST_STATE),
-            ),
-        ),
-        contends=(config.exclusive("workspace_binaries"),),
-        kind=Kind.PACKAGE,
-        needs=frozenset({Needs.DISK}),
-        speed=Speed.FAST,
-    )
-
-
 def add(
     owner: Plan | Phase,
     config: GateConfig,
     *,
     after: tuple[Step, ...] = (),
     label: str = "build-binaries",
-    cache_already_enforced: bool = False,
 ) -> Step:
-    """Add the one host build path, with visible cache enforcement first.
+    """Add the one host build path.
 
-    Complete qualification already enforces every configured cache before it
-    reaches this fragment. Every focused composer takes the default and gets a
-    separate timed prerequisite, so the shared Cargo target cannot silently
-    grow past its contract again.
+    The shared Cargo target's maximum is held by the command, not here:
+    `CargoCacheBound` enforces it before the first and after the last step of
+    every plan that compiles, which a prerequisite of this one fragment could
+    not do for clippy, coverage, or any other compile beside it.
     """
-    dependencies = after
-    if not cache_already_enforced:
-        bounded = owner.add(_cargo_cache_step(config), after=after)
-        dependencies = (bounded,)
-    return owner.add(_build_step(config, label=label), after=dependencies)
+    return owner.add(_build_step(config, label=label), after=after)

build_system/builder/gate/preflight.py

@@ -15,8 +15,10 @@
 import os
 from collections.abc import Iterator
 from contextlib import contextmanager
+from typing import TYPE_CHECKING
 
 from . import snapshot
+from .cachecontrol import CargoCacheBound
 from .cachetooling import CompilerCache
 from .cargotarget import CheckoutBuildRoot
 from .config import GateConfig
@@ -26,6 +28,9 @@
 from .proc import Runner
 from .reaper import StaleProcesses
 
+if TYPE_CHECKING:  # pragma: no cover - imported for typing only
+    from .plan import Plan
+
 
 def refuse_inside_a_run(config: GateConfig, name: str, *, exclusive: bool) -> None:
     """Refuse to take a lock this process tree is already holding.
@@ -78,18 +83,35 @@ def holdings(
     *,
     exclusive: bool,
     declared: tuple[Resource, ...],
+    plan: Plan,
 ) -> tuple[Resource, ...]:
-    """Tooling and declared resources, inside the machine lock's outer scope."""
+    """Tooling and declared resources, inside the machine lock's outer scope.
+
+    The Cargo bound sits outside every declared resource, so its release
+    enforces after the last step and after whatever those resources tear down.
+    """
     if not exclusive:
         return declared
+    bound = (CargoCacheBound(runner),) if compiles(plan, config) else ()
     return (
         StaleProcesses(config, runner),
         CheckoutBuildRoot(config, runner),
         CompilerCache(config, runner),
+        *bound,
         *declared,
     )
 
 
+def compiles(plan: Plan, config: GateConfig) -> bool:
+    """Whether any step drives Cargo, by the claim every such step must take.
+
+    `tests/citadel/test_step_actions_are_atomic.py` holds that claim: a step
+    that reaches Cargo without `workspace_binaries` fails there.
+    """
+    claim = config.exclusive("workspace_binaries").name
+    return any(held.name == claim for step in plan.steps for held in step.contends)
+
+
 def purpose(name: str) -> str:
     """What contention should call this, for whoever arrives next."""
     return f"capsem-gate {name}"

build_system/builder/gate/runtimeprepare.py

@@ -29,7 +29,6 @@ def prepare(
     permission: RebuildPermission = DEFAULT_PERMISSION,
     build_label: str = "build-binaries",
     sign_label: str = "sign",
-    cache_already_enforced: bool = False,
 ) -> Preparation:
     """Build one self-contained runtime, optionally including VM inputs."""
     phase = plan.phase("prepare")
@@ -46,13 +45,7 @@ def prepare(
         previous = (packed,)
 
     materialized = phase.add(materialize_config_step(config), after=previous)
-    built = hostbuild.add(
-        phase,
-        config,
-        after=(materialized,),
-        label=build_label,
-        cache_already_enforced=cache_already_enforced,
-    )
+    built = hostbuild.add(phase, config, after=(materialized,), label=build_label)
     ready = phase.add(hostpackage.sign_step(config, label=sign_label), after=(built,))
     return Preparation(ready=ready, profile_content=materialized)
 

build_system/tests/gate/test_bounded_machine_lease.py

@@ -178,12 +178,42 @@ def test_direct_cargo_enforces_its_cache_contract_inside_the_machine_lease(
     monkeypatch.setattr(
         boundedlease,
         "_enforce_cargo_cache",
-        lambda root, command: enforced.append((root, tuple(command))),
+        lambda root, command, **_: enforced.append((root, tuple(command))),
     )
     monkeypatch.setenv("HOME", str(tmp_path))
 
-    with boundedlease.leased(("cargo", "test", "-p", "capsem-core"), ROOT, {}):
-        assert enforced == [(ROOT, ("cargo", "test", "-p", "capsem-core"))]
+    command = ("cargo", "test", "-p", "capsem-core")
+    with boundedlease.leased(command, ROOT, {}):
+        assert enforced == [(ROOT, command)]
+    # After as well as before: the compile itself is what grows the target,
+    # and a bound checked only on the way in leaves it above its maximum for
+    # every worktree that arrives next.
+    assert enforced == [(ROOT, command), (ROOT, command)]
+
+
+def test_only_the_check_before_the_command_refuses(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    """After the command, its own exit status is the answer; a bound that
+    cannot be restored is reported and refused by the next compile instead."""
+    from capsem_builder.cache.enforcement import EnforcementResult
+    from capsem_builder.gate import boundedlease
+    from capsem_builder.gate.errors import GateError
+
+    stuck = EnforcementResult(
+        cache_id="cargo",
+        before_size_bytes=2,
+        after_size_bytes=2,
+        pruned=False,
+        reclaim_bytes=0,
+        action_count=0,
+        violations=("cargo remains 2 bytes above max size 1",),
+    )
+    monkeypatch.setattr(boundedlease, "enforce_repository", lambda *_, **__: stuck)
+    with pytest.raises(GateError, match="above max size"):
+        boundedlease._enforce_cargo_cache(ROOT, ("cargo", "build"))
+    boundedlease._enforce_cargo_cache(ROOT, ("cargo", "build"), refuse=False)
+    assert "above its contract after the command" in capsys.readouterr().err
 
 
 def test_zero_byte_expiry_does_not_claim_cargo_exceeded_its_limit() -> None:

build_system/tests/gate/test_build_assets_profile.py

@@ -431,8 +431,7 @@ def test_runtime_recipes_materialize_generated_config_before_service() -> None:
     # that exact signed runtime exists.
     for command in ("ensure-service", "shell", "exec"):
         plan = _command(command, guest_command="true")._describe()
-        assert plan.after_of("prepare.cargo-cache-enforcement") == {"prepare.materialize-config"}
-        assert plan.after_of("prepare.build-binaries") == {"prepare.cargo-cache-enforcement"}
+        assert plan.after_of("prepare.build-binaries") == {"prepare.materialize-config"}
         assert plan.after_of("prepare.sign") == {"prepare.build-binaries"}
         assert plan.after_of("prepare") == {"prepare.sign"}