From eb9e8f298d422d2475bca2c709e9c0a1f6f1ab2e Mon Sep 17 00:00:00 2001 From: Commitrail Probe Date: Tue, 29 Sep 2026 08:47:20 +0100 Subject: [PATCH 1/6] feat(artifacts): bind checker outputs to verified storage custody --- .ci/auth-boundaries/TEST_STRUCTURE_DEBT.json | 14 +- .ci/behavior-ownership/partition.v1.json | 18 +- .../private-edge-debt.v1.json | 12 - .commitrail/INDEX.md | 12 +- .../initiatives/WS-ARCH-001/OVERVIEW.md | 3 +- .../WS-ARCH-001/WS-ARCH-001-04B2.md | 216 ++++++++ .../WS-ARCH-001/planning/CHUNK_MAP.md | 10 +- .../initiatives/WS-ARCH-001/planning/PLAN.md | 23 +- ...001-04B-art-post-submit-materialization.md | 36 +- .../WS-ARCH-001-04C-checker-current-result.md | 20 +- .../initiatives/WS-ART-001/OVERVIEW.md | 11 +- .../initiatives/WS-AUTH-001/OVERVIEW.md | 6 +- .../WS-AUTH-001/planning/CHUNK_MAP.md | 2 +- .../initiatives/WS-AUTH-001/planning/PLAN.md | 4 +- .../initiatives/WS-AUTH-003/OVERVIEW.md | 4 +- .../initiatives/WS-CON-001/OVERVIEW.md | 4 +- .../initiatives/WS-POL-003/OVERVIEW.md | 4 +- .../WS-POL-003/planning/CHUNK_MAP.md | 7 +- .../initiatives/WS-POL-003/planning/PLAN.md | 3 +- README.md | 8 +- backend/alembic/env.py | 4 +- .../versions/0007_checker_output_custody.py | 367 ++++++++++++++ backend/app/adapters/artifacts/__init__.py | 21 + backend/app/interfaces/artifact_operations.py | 52 +- .../artifacts/checker_output_bindings.py | 174 +++++++ .../artifacts/checker_output_custody.py | 215 ++++++++ .../app/modules/artifacts/checker_outputs.py | 253 ++++++++++ backend/app/modules/artifacts/models.py | 57 ++- backend/app/modules/artifacts/preparation.py | 13 +- backend/app/modules/artifacts/repository.py | 43 -- backend/app/modules/artifacts/schemas.py | 37 +- backend/app/modules/artifacts/service.py | 140 ++++-- .../modules/checkers/api/output_custody.py | 89 ++++ backend/scripts/behavior_ownership.py | 7 + backend/scripts/test_lane_catalogue.py | 2 + .../tests/checker_output_admission_helpers.py | 223 +++++++++ .../tests/checker_output_custody_helpers.py | 403 +++++++++++++++ backend/tests/conftest.py | 3 +- backend/tests/test_alembic.py | 2 +- backend/tests/test_artifact_admission.py | 171 +++---- backend/tests/test_artifact_architecture.py | 19 +- backend/tests/test_artifact_preparation.py | 71 ++- backend/tests/test_artifact_recovery.py | 16 +- backend/tests/test_behavior_ownership.py | 18 + backend/tests/test_checker_output_custody.py | 384 ++++++++++++++ backend/tests/test_checker_output_storage.py | 473 ++++++++++++++++++ backend/tests/test_ci_lane_catalogue.py | 2 + backend/tests/test_coverage_contract.py | 2 +- docs/architecture_checker_framework.md | 19 +- docs/architecture_data_model.md | 27 +- .../authorization_activation_custody.md | 10 +- docs/operations_project_operating_manual.md | 4 +- docs/roadmap_status.md | 44 +- docs/spec_artifact_storage_service.md | 97 ++-- 54 files changed, 3494 insertions(+), 385 deletions(-) create mode 100644 .commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md create mode 100644 backend/alembic/versions/0007_checker_output_custody.py create mode 100644 backend/app/modules/artifacts/checker_output_bindings.py create mode 100644 backend/app/modules/artifacts/checker_output_custody.py create mode 100644 backend/app/modules/artifacts/checker_outputs.py create mode 100644 backend/app/modules/checkers/api/output_custody.py create mode 100644 backend/tests/checker_output_admission_helpers.py create mode 100644 backend/tests/checker_output_custody_helpers.py create mode 100644 backend/tests/test_checker_output_custody.py create mode 100644 backend/tests/test_checker_output_storage.py diff --git a/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json b/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json index f8dd57a45..ff7be2786 100644 --- a/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json +++ b/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json @@ -110,11 +110,11 @@ }, { "capability": "oversized_authorization_structure", - "content_sha256": "6552c4f594bc0ad05a231f1d9c202b78e5b4e31a0ce05ea32e7eb48b67d440ba", - "end_line": 2864, + "content_sha256": "ef1302419840b5659d01e45efd5806b904507ad8f9c0e457287597ac3ecbfb6f", + "end_line": 2847, "hard_limit": 1200, "kind": "test_file", - "observed_lines": 2864, + "observed_lines": 2847, "path": "backend/tests/test_artifact_admission.py", "qualified_symbol": null, "removal_chunk": "WS-AUTH-003-CLOSE", @@ -171,26 +171,26 @@ { "capability": "oversized_authorization_structure", "content_sha256": "2fac78c747fd82908991bde192ba874d3847b3ad105a827575dfc6afd73614be", - "end_line": 2277, + "end_line": 2281, "hard_limit": 120, "kind": "test_function", "observed_lines": 194, "path": "backend/tests/test_artifact_admission.py", "qualified_symbol": "test_guide_admission_consumes_real_project_manager_prep_atomically", "removal_chunk": "WS-AUTH-003-CLOSE", - "start_line": 2084 + "start_line": 2088 }, { "capability": "oversized_authorization_structure", "content_sha256": "12f1dd1e0632ec3e5fb80ea19eb01bcce2b9a14015a04e120fb304f9b343b828", - "end_line": 2081, + "end_line": 2085, "hard_limit": 120, "kind": "test_function", "observed_lines": 132, "path": "backend/tests/test_artifact_admission.py", "qualified_symbol": "test_guide_admission_derives_three_scopes_without_provider_evidence", "removal_chunk": "WS-AUTH-003-CLOSE", - "start_line": 1950 + "start_line": 1954 }, { "capability": "oversized_authorization_structure", diff --git a/.ci/behavior-ownership/partition.v1.json b/.ci/behavior-ownership/partition.v1.json index 5bdf6758a..e3ad9ceb0 100644 --- a/.ci/behavior-ownership/partition.v1.json +++ b/.ci/behavior-ownership/partition.v1.json @@ -268,6 +268,18 @@ "group": "artifacts", "target": "backend/app/modules/artifacts/authorization.py" }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/checker_output_bindings.py" + }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/checker_output_custody.py" + }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/checker_outputs.py" + }, { "group": "artifacts", "target": "backend/app/modules/artifacts/guide_document_access.py" @@ -724,6 +736,10 @@ "group": "artifacts", "target": "backend/app/modules/checkers/api/history.py" }, + { + "group": "artifacts", + "target": "backend/app/modules/checkers/api/output_custody.py" + }, { "group": "artifacts", "target": "backend/app/modules/checkers/api/policy_compilation.py" @@ -1473,7 +1489,7 @@ "target": "backend/scripts/validate_test_lane_evidence.py" } ], - "authority_digest": "45c8605a1e6b8a29cd5c9fb59771f423f8e87f04ed7da07833139cc1f657e22b", + "authority_digest": "4cb652590f801da4066a1179a5a4ed7df33721a7f685a0b6300d4f46580ac7d0", "protected_base_commit": "7676ce4347db0c9694962a9b587a20765e16eac6", "schema": "workstream.behavior-ownership-partition.v1" } diff --git a/.ci/module-boundaries/private-edge-debt.v1.json b/.ci/module-boundaries/private-edge-debt.v1.json index 2b961e53b..9a47e2df4 100644 --- a/.ci/module-boundaries/private-edge-debt.v1.json +++ b/.ci/module-boundaries/private-edge-debt.v1.json @@ -205,24 +205,12 @@ "imported_private_path": "app.modules.tasks.models", "repair_owner": "WS-ARCH-001-03" }, - { - "source_file": "backend/app/modules/artifacts/repository.py", - "target_module": "checkers", - "imported_private_path": "app.modules.checkers.models", - "repair_owner": "WS-ARCH-001-04" - }, { "source_file": "backend/app/modules/artifacts/repository.py", "target_module": "projects", "imported_private_path": "app.modules.projects.models", "repair_owner": "WS-ARCH-001-03" }, - { - "source_file": "backend/app/modules/artifacts/repository.py", - "target_module": "tasks", - "imported_private_path": "app.modules.tasks.models", - "repair_owner": "WS-ARCH-001-03" - }, { "source_file": "backend/app/modules/artifacts/service.py", "target_module": "actors", diff --git a/.commitrail/INDEX.md b/.commitrail/INDEX.md index 051d526d1..5b4555b3b 100644 --- a/.commitrail/INDEX.md +++ b/.commitrail/INDEX.md @@ -8,12 +8,12 @@ for current product capability. |---|---|---| | [WS-DB-002](initiatives/WS-DB-002/OVERVIEW.md) | Complete | Shared UUIDv7 record generation, native-UUID relationships and fresh v0.1 baseline; natural-owner retry custody and aligned CI/local setup | | [WS-MCP-002](initiatives/WS-MCP-002/OVERVIEW.md) | Planned | Three self-service tools delivered through WS-MCP-002-02; 24 tools remain and WS-MCP-002-03 administrative reads are next | -| [WS-ARCH-001](initiatives/WS-ARCH-001/OVERVIEW.md) | Planned | Canonical ARCH-04A delivered; Automatic unified setup delivered; POL-05/06 manager review and separate approvals delivered; POL-07B internal checker phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [WS-ART-001](initiatives/WS-ART-001/OVERVIEW.md) | Planned | Hidden exact post-submit input materialization delivered; ARCH-04B2 output custody next | -| [WS-AUTH-001](initiatives/WS-AUTH-001/OVERVIEW.md) | Planned | Shared dispatcher authority and phase custody after delivered CP05; POL-04B consumes completed finalization authority; AUTH-12F4 supplies proposal authority; POL-05B public composition delivered; AUTH-12G post-policy authority delivered; POL-06B public composition delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [WS-CON-001](initiatives/WS-CON-001/OVERVIEW.md) | Planned | CP06 selected-policy validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [WS-AUTH-003](initiatives/WS-AUTH-003/OVERVIEW.md) | Planned | TASK/checker canonical history authority delivered and alternate gate removed; AUTH-18 public manager activation and ARCH-03D hidden intake delivered; ARCH-04B hidden input delivered; continue boundary recovery with ARCH-04B2 output custody next | -| [WS-POL-003](initiatives/WS-POL-003/OVERVIEW.md) | Planned | Unified setup, separate draft proposals and guide document intake delivered; Hidden proposal review/correction/pre-policy approval custody delivered; AUTH-12F4 proposal authority delivered; POL-05B public review/approval/manual correction dispatch delivered; POL-06A hidden post-policy projection/read/approval/correction delivered; AUTH-12G live authority delivered; POL-06B public access and automatic derivation delivered; POL-07A ART pre-submit attempt recovery delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-ARCH-001](initiatives/WS-ARCH-001/OVERVIEW.md) | Planned | Public guide activation and task reads, hidden approved-guide intake, exact post-submit input materialization and hidden ARCH-04B2 checker-output custody delivered; ARCH-04C durable execution is next | +| [WS-ART-001](initiatives/WS-ART-001/OVERVIEW.md) | Planned | Hidden exact post-submit input materialization and ARCH-04B2 output custody delivered; ARCH-04C durable execution next | +| [WS-AUTH-001](initiatives/WS-AUTH-001/OVERVIEW.md) | Planned | Manager guide activation, public task authority, dispatcher mechanics and hidden intake are delivered; ARCH-04B input and ARCH-04B2 output custody exist with deny-only/unavailable production authority; ARCH-04C durable execution is next | +| [WS-CON-001](initiatives/WS-CON-001/OVERVIEW.md) | Planned | ContributionPolicy administration and guide binding plus hidden intake, post-submit input and ARCH-04B2 output custody delivered; ARCH-04C durable execution precedes shared FinalAcceptance, submitter ContributionRecord and applicable awards | +| [WS-AUTH-003](initiatives/WS-AUTH-003/OVERVIEW.md) | Planned | TASK/checker canonical history authority delivered and alternate gate removed; AUTH-18 public manager activation and ARCH-03D hidden intake delivered; ARCH-04B hidden input and ARCH-04B2 output custody delivered; continue boundary recovery with ARCH-04C durable execution next | +| [WS-POL-003](initiatives/WS-POL-003/OVERVIEW.md) | Planned | Unified setup, public manager policy operations, phase composition, public guide activation, hidden intake, post-submit input and ARCH-04B2 output custody delivered; ARCH-04C durable execution is next | | [WS-REV-001](initiatives/WS-REV-001/OVERVIEW.md) | Planned | Shared acceptance/source and existing fence foundations; human hidden review work remains independently dependency-gated | | [WS-QUAL-002](initiatives/WS-QUAL-002/OVERVIEW.md) | Planned | Populate subsystem ownership before changed-line mutation work | | [WS-QUAL-003](initiatives/WS-QUAL-003/OVERVIEW.md) | Planned | Audit and prune test proof, add missing safety cases, decompose oversized test modules | diff --git a/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md b/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md index eeae50a29..7c4bec26c 100644 --- a/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md @@ -14,7 +14,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), - Current boundary: one CHECKERS catalogue, compiler/parser and implementation per checker ID serve active policy consumers; production post-submit phase execution remains unavailable. -- Next usable boundary: checker output custody (ARCH-04B2), following delivered hidden input materialization (ARCH-04B). Production execution and live materialization authority remain unavailable. +- Delivered storage boundary: hidden [ARCH-04B2 checker-output custody](WS-ARCH-001-04B2.md), following hidden input materialization (ARCH-04B). Typed store, byte-free recovery and flush-only verified binding exist; production CHECKERS reservation and authority remain unavailable. +- Next usable boundary: ARCH-04C durable execution and results, including the current structural catalogue's valid empty output set. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation using the existing CP07 operation. [CP05A](WS-ARCH-001-CP05A.md) supplies public Finance policy administration and recoverable draft selectors. diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md new file mode 100644 index 000000000..54d0fc004 --- /dev/null +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -0,0 +1,216 @@ +# ARCH-04B2 — Verified checker output custody + +- Initiative: `WS-ARCH-001` +- Durable disposition: `Complete` +- Intended merge outcome: Hidden ART storage and flush-only verified binding of exact checker-run output, with production access unavailable until CHECKERS reservations and AUTH activation exist. + +## Intent + +Complete the next storage boundary on the path to a usable contributor journey: +claim -> upload ZIP -> pre-submit feedback or immutable Submission -> automatic +post-submit evaluation -> outcome. The first complete no-human-review journey must +include shared FinalAcceptance, the submitter ContributionRecord and applicable +policy awards before live human review/revision becomes a dependency. Human review +packet foundations may proceed independently in another worktree; shared acceptance +is one operation used by both triggers, never a second automated acceptance engine. +Contributor skip and assignment expiry remain deferred. + +## Current behavior + +Main `9c292100` includes merged ARCH-04B exact verified input and bounded scratch. +`CheckerArtifactOutputPort.store` and `ArtifactBindingPort.bind_checker_output` +are unused declarations. Generic ART preparation, quota admission, durable put, +unknown-outcome recovery and independent verification already exist. Checker +output admission still reads private CheckerRun rows from ART's repository. +Existing CHECKERS rows do not persist 04A request/generation/worker reservation +facts. Current structural checker definitions permit zero generated output bytes +and no required output roles. This change must not invent evaluator outputs or +claim the current structural catalogue supports them. + +## Design and decisions + +1. Replace the unused output/binding declarations with closed typed requests and + results; no compatibility alias or second storage engine. A CHECKERS-owned + public reservation-facts port supplies exact run/request/Submission/policy, worker-lease generation and + owner-declared output-slot/budget facts. Each slot has a globally unique bounded + key within its run, media type and byte limit. Its production implementation is explicitly + unavailable until ARCH-04C; hidden proof uses a controlled owner fixture, + clearly distinguished from live reservation/currentness proof. +2. The public store request contains selector plus byte source, never PREP or raw + AuthorizationContext. ART action-specific preflight denies before protected reads, byte-source iteration, scratch or + provider access. Exact reservation facts must match the complete validated + evaluation request and output selector. Role/budget comes from CHECKERS's + reservation, never arbitrary contributor input. No production log/output role is introduced. The current structural plan has + zero slots; 04C must accept that exact empty output set. Controlled nonempty + test slots prove storage mechanics only, not a live evaluator capability. +3. Reuse canonical bounded scratch to commit exact output bytes. Add a per-call + byte ceiling bounded by the existing manager maximum; over-cap input stops + before writing excess bytes and cleans the reservation. Re-resolve owner + facts and authority in a short transaction before generic quota admission and + durable intent. Charge deployment/project/task/fixed producer, not contributor. + Bind exact evaluation/run/output identity into the existing put request digest; + persist Submission identity/version and a narrow checker-request digest on that + attempt, and verified receipt/put + ancestry on the immutable checker binding. Do not copy the evaluation envelope + or add a competing CHECKERS reservation store. This receipt is material lineage, never + independent CHECKERS currentness or authorization. + Operation identity remains (run, globally unique slot). The request digest + binds evaluation request ID/hash/generation, run, immutable owner lineage, + slot key/media/ceiling, commitment, namespace and quota scopes. It excludes + volatile worker lease ID/generation: every store/replay/bind separately resolves + and authorizes the active lease. A replacement worker recovers the same logical + output/put; an old lease cannot publish. Changed immutable facts or bytes conflict. +4. Release transaction/locks/PREP before provider I/O. Reuse existing committed + put, observation and verification operations. Unknown outcomes retain the + original attempt; absent unreproducible bytes cannot be fabricated or + regenerated. Always clean private scratch. A verified reference is returned + only for the exact attempt/replica/independent verification receipt. + `recover(selector)` retrieves the stable operation after a lost response without + a byte source. It reuses observation and verification, never provider put or + regeneration. `store` still commits supplied bytes and rejects changed replay + content. The narrow checker-request digest permits recovery/binding comparisons + without reconstructing historical quota configuration inside the full digest. +5. A separate flush-only binding participant runs inside the caller's transaction. + Caller supplies fresh CHECKERS reservation/fence facts and exact authority + before ART locks. Match persisted intent and full verified chain, then create + or replay one immutable run/role binding. Changed bytes, foreign intent, + incomplete verification, stale owner facts or late revoked authority deny. + Authority custody is prepared before CHECKERS reservation/currentness locks, + followed by ART scope/row locks. ART never locks CHECKERS/TASK privately. + Verify using immutable terminal receipt ancestry or the verifier's compatible + job -> replica -> attempt -> content lock order; do not introduce the inverse. + Generic bindings receive checker-only put_attempt_id and verification_receipt_id. + An INSERT guard verifies exact receipt -> job -> attempt -> replica/content + ancestry, verified state and project/run/slot ownership. Reuse the existing + binding immutability trigger; do not invent a common receipt FK across distinct + direct-put and observation receipt tables. ARCH-04C later composes this participant + with its final result and outbox; ARCH-04D owns live service activation. +6. Replace the raw-AuthorizationContext checker admission path with a mandatory + action-specific authority participant and owner reservation facts. Default + admission without that participant denies before mutation. Replace the affected + private checker admission/recovery lookup with owner facts. + Trace remaining consumers explicitly; preserve retained records. Do not rewrite + unrelated owner behavior or add compatibility constructors. + +## Bounded change + +### Allowed + +- `backend/app/interfaces/artifact_operations.py` output requests/results and ports. +- `backend/app/modules/artifacts/checker_outputs.py`, + `backend/app/modules/artifacts/checker_output_custody.py` and + `backend/app/modules/artifacts/checker_output_bindings.py` (new ART owners). +- `backend/app/modules/artifacts/preparation.py` for the bounded per-call cap. +- `backend/app/modules/artifacts/models.py`, `schemas.py`, `service.py`, + `repository.py` only for affected output custody and traced lookup consumers. + The separate Operator resource-to-project CHECKERS lookup remains explicitly + outside this mutation boundary; it is not a second output admission path. +- `backend/app/modules/checkers/api/output_custody.py` (new closed owner facts), + `backend/app/modules/checkers/api/__init__.py`; composition in existing + `backend/app/adapters/artifacts/__init__.py`, `backend/app/adapters/checkers/`. +- One ART-owned migration `backend/alembic/versions/0007_checker_output_custody.py` after `0006_history_read_authority` for immutable exact + output intent/binding custody; no CHECKERS run writer or retained-data deletion. +- New `backend/tests/test_checker_output_custody.py`, + `backend/tests/test_checker_output_storage.py`, + `backend/tests/checker_output_custody_helpers.py`, + `backend/tests/checker_output_admission_helpers.py`, and + existing `test_artifact_admission.py`, `test_artifact_recovery.py`, + `test_artifact_architecture.py`, `test_artifact_preparation.py`, + `test_artifact_authorization.py`, `backend/tests/conftest.py` (exact migrated + fingerprint and protected-table reset inventory), `backend/alembic/env.py`, + `test_alembic.py`, `test_coverage_contract.py` for exact migration-head registration, module boundary + tests and migration tests for traced consumers and retained behavior. +- Existing lane/ownership/module/test-structure inventories and their tests only + to register changed owners and keep existing gates intact. +- This record, linked ARCH output skeleton, current ARCH/ART/AUTH/POL/CON navigation, + `.commitrail/INDEX.md`, `README.md`, `docs/roadmap_status.md`, artifact/checker/data + model specifications, operating manual and authorization custody page for affected claims. + +### Not allowed + +Public routes, live grants, checker reservation/result/currentness writers, +TASK transitions, policy/catalogue expansion, inference, reviewer workflows, +acceptance implementation, alternate scratch/store/factory, data deletion, +compatibility paths, CI weakening or contributor quota charges for system outputs. + +## Acceptance criteria + +- Default composition fails before any protected or external access. +- Controlled exact reservation plus real PostgreSQL/Local and MinIO storage proves + preparation, exact byte commitment, independent verification and immutable binding. +- Valid foreign stored lineages and mixed reservation/intent/binding selectors deny; + positive controls succeed. Requests cannot select a role or budget outside the + owner-issued reservation. Invalid requests never invoke the byte source/provider. +- Same run/role/bytes replay produces the same intent and binding; changed request + or bytes reject. Concurrent publication and caller rollback preserve one binding + and no partially committed effect. +- Database rejects intent/binding identity mutation and mismatched stored ancestry. + Migration refuses retained checker attempts lacking provable evaluation custody; + it never invents the missing checker-request digest or deletes retained data. +- Unknown put outcome resumes observation of the same intent without regenerating + output; missing unreproducible bytes remain unavailable. Existing generic recovery + proof is reused rather than copied. +- Quota/deadline/cancellation cleanup and fresh authority after I/O are demonstrated; + independent sessions prove no row lock spans provider I/O. +- No claim of live CHECKERS request/lease custody: that remains ARCH-04C proof. + +## Risk and review routing + +- Risk class: L1 (untrusted bytes, immutable evidence, storage/schema boundary). +- Required reviewers: architecture/reuse, security, QA/test delta, docs/product; + CI integrity for affected lane/inventory registrations. +- Human review focus: exact run/request/role binding, unavailable production boundary, + shared storage reuse, retained evidence and automated acceptance sequencing. +- Plan review resolved the missing producer with an unavailable owner seam, not + a new runtime fixture; current structural outputs remain empty. Replay binds + immutable attempt facts while fresh authority fences the changing worker lease. + +## Evidence + +The focused proof lives in `test_checker_output_custody.py` and +`test_checker_output_storage.py`: + +- Default composition denies before protected access; valid stored foreign and + mixed selectors reject before source/provider use after two valid controls. +- Local/MinIO store -> independent verify -> binding preserves exact persisted + identities; binding rollback and concurrent replay preserve one result. +- Lost acknowledgement uses typed observation and byte-free recovery; + worker takeover preserves logical output identity and denies the old lease. +- Slot caps stop iteration and clean scratch; revocation during provider I/O + denies the returned reference, with independent-session lock checks. +- Direct SQL rejects mixed binding ancestry and immutable intent changes; + migration refuses retained output lacking provable request custody. + +Affected admission/recovery tests retain quota, service-identity, relationship, +namespace precedence and transaction guarantees under the new closed request. +Existing preparation tests cover deadline, cancellation and aggregate quotas; +new per-slot cases extend that same canonical path. + +Verification uses the existing isolated PostgreSQL/MinIO runner, Ruff, +structure/module-boundary validators, markdown links, stale wording, +Commitrail checks and complete hosted lanes. Exact command results and +review targets belong in the PR. Guard-removal probes must reach the behavioral +assertion after valid controls, not fail during fixture setup. Hidden controlled +reservation/action-authority fixtures prove ART mechanics only; actual CHECKERS +lease custody and AUTH activation remain subsequent work. + +## Plan review disposition + +The focused plan review found the original store PREP lifetime, +per-slot scratch ceiling, missing reservation producer, and binding ancestry +underspecified. The design above incorporates those repairs. The reviewed stable/volatile split keeps worker leases out of logical output +identity while authorizing the caller's exact claimed active lease on every phase. +Existing binding +immutability is reused; no generated-output catalogue expansion or duplicate +custody aggregate is authorized. + +## Reconciliation + +- Current-source reconciliation: merged ARCH-04B on main; no overlapping open + product PR. AUTH provisioning proof and deferred lease planning are separate. +- Next usable boundary: ARCH-04C durable execution/results, then exact AUTH and + automatic routing; shared acceptance and compensation participants precede + no-human-review activation; remediation and public intake complete the first + contributor milestone before live human review/revision. +- Remaining risks: CHECKERS reservation producer and live authority deliberately + absent; only controlled hidden storage proof is claimed by this child. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md index 0fe4ed520..55cc95b8a 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md @@ -4,7 +4,7 @@ Use the [current dependency contract](PLAN.md#current-dependency-contract). The [preserved map](../pre-cutover/CHUNK_MAP.md) retains the complete original work accounting. Foundations through 02H and CP04B are complete; none restart. AUTH-18 public manager activation/context and ARCH-03D hidden approved-guide intake are delivered. -ARCH-04B hidden exact post-submit input is delivered. ARCH-04B2 output custody is next; +ARCH-04B hidden exact post-submit input and ARCH-04B2 output custody are delivered. ARCH-04C durable execution is next; public intake remains deferred to ARCH-02I. | Boundary | Owner outcome | Risk | Current dependency | @@ -32,15 +32,15 @@ public intake remains deferred to ARCH-02I. | [WS-ARCH-001-03C4](../WS-ARCH-001-03C4.md) | Exact-authorized public task queues | L1 | Complete; contributor, manager and operational projections with signed pagination | | [WS-ARCH-001-03C5](../WS-ARCH-001-03C5.md) | Exact-authorized Contributor and Manager detail and requirements | L1 | Complete; canonical projections, historical policy custody and atomic read evidence | | [WS-ARCH-001-03C6](../WS-ARCH-001-03C6.md) | Distinct exact-authorized locked-context reads | L1 | Complete; bounded Audit Authority history access delivered by 03C7 | -| [WS-ARCH-001-03C7](../WS-ARCH-001-03C7.md) | Exact-authorized bounded task history | L1 | Complete; AUTH-18 public guide activation delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-ARCH-001-03C7](../WS-ARCH-001-03C7.md) | Exact-authorized bounded task history | L1 | Complete; AUTH-18 public guide activation, ARCH-03D hidden approved-guide intake, hidden exact post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next | | [WS-ARCH-001-03C](chunks/WS-ARCH-001-03C-auth-task-readiness.md) | Exact task/assignment public activation and integrated readiness proof | L1 | Complete through 03C7 audit history, 03C4 queues, 03C5 detail/requirements and 03C6 locked-context reads; AUTH-18 public guide activation delivered; ARCH-03D hidden intake delivered; public intake cutover remains separate | | [WS-ARCH-001-04A](../WS-ARCH-001-04A.md) | CHECKER post-submit contract and registered evaluator conformance | L1 | Complete canonical catalogue, phase facts and structural conformance; consumed by delivered POL-04B and POL-07B | | [WS-ARCH-001-04B](../WS-ARCH-001-04B.md) | ART exact verified Submission materialization | L1 | Complete hidden verified input with async scoped reads; production authority remains deny-only until 04D | -| [WS-ARCH-001-04B2](chunks/WS-ARCH-001-04B-art-post-submit-materialization.md#arch-04b2--separate-art-output-custody-child) | ART generated-output/log custody and verified binding | L1 | 04A public request/run facts plus merged ART foundations; no CHECKERS private lookup | -| [WS-ARCH-001-04C](chunks/WS-ARCH-001-04C-checker-current-result.md) | CHECKER hidden durable current output and supersession behavior | L1 | Planned after 04A/04B/04B2; production remains deny-only | +| [WS-ARCH-001-04B2](../WS-ARCH-001-04B2.md) | ART checker-output custody and verified binding | L1 | Complete hidden typed store, byte-free recovery and flush-only binding; production reservation and authority unavailable | +| [WS-ARCH-001-04C](chunks/WS-ARCH-001-04C-checker-current-result.md) | CHECKER hidden durable current output and supersession behavior | L1 | Planned after delivered 04A/04B/04B2; must support the structural catalogue's empty output set; production remains deny-only | | [WS-ARCH-001-04D](chunks/WS-ARCH-001-04D-auth-post-submit-activation.md) | AUTH exact fixed-service post-submit activation (replaces XINT-06B) | L1 | Planned after 04B/04C evidence | | [WS-ARCH-001-04E](chunks/WS-ARCH-001-04E-canonical-allow-review.md) | TASK current routing: true to canonical `allow_review`, false/pass to shared acceptance | L1 | Source 04E1A -> hidden handlers 04E1B -> AUTH 04E2 -> live 04E3, plus 04D/OUTBOX-02; false consumes shared REV/CON/fence proof and activation also requires 04F remediation | -| [WS-ARCH-001-03D](../WS-ARCH-001-03D.md) | Exact activated historical guide through hidden durable intake; obsolete lookup removed | L1 | Complete; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next, public cutover remains deferred | +| [WS-ARCH-001-03D](../WS-ARCH-001-03D.md) | Exact activated historical guide through hidden durable intake; obsolete lookup removed | L1 | Complete; hidden exact post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next, public cutover remains deferred | | [WS-ARCH-001-04F](chunks/WS-ARCH-001-04F-checker-remediation.md) | Contributor-correctable checker failures and same-lineage admission-backed replacement Submission | L1 | Planned after 04E; replaces XINT-05C, required before public 02I, not before REV begins from `allow_review` | CP09, 04E and 04F are coordination parents, not permission for multi-owner PRs. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md index c0aa70b4a..43599d603 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md @@ -44,13 +44,13 @@ checker-remediation boundary before public Submission cutover. | [ARCH-03C4](../WS-ARCH-001-03C4.md) | ARCH-03C3 | Complete: exact-authorized contributor, manager and operational public queues | | [ARCH-03C5](../WS-ARCH-001-03C5.md) | ARCH-03C4 | Complete: exact-authorized Contributor/Manager detail and requirements | | [ARCH-03C6](../WS-ARCH-001-03C6.md) | ARCH-03C5 | Complete: distinct exact-authorized locked-context reads | -| [ARCH-03C7](../WS-ARCH-001-03C7.md) | ARCH-03C6 and hidden TASK owner contracts | Complete: bounded Audit Authority history access; public guide activation is delivered by AUTH-18; ARCH-03D hidden intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [CP05A](../WS-ARCH-001-CP05A.md) | CP05, existing policy owners | Complete: public Finance policy administration and selector recovery; AUTH-18 public manager activation/context delivered; ARCH-03D hidden intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [ARCH-03C7](../WS-ARCH-001-03C7.md) | ARCH-03C6 and hidden TASK owner contracts | Complete: bounded Audit Authority history access; public guide activation, ARCH-03D hidden intake, hidden post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next | +| [CP05A](../WS-ARCH-001-CP05A.md) | CP05, existing policy owners | Complete: public Finance policy administration and selector recovery; public manager activation, ARCH-03D hidden intake, hidden post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next | | CP09 (later cleanup coordination) | All legacy consumers replaced, including CHECKER and public 02I path | Physical economic deletion; not on the allow_review critical path | | [ARCH-03D](../WS-ARCH-001-03D.md) | AUTH-18, ARCH-03A, CP08 and merged ART preparation | Complete: hidden durable intake uses exact historical TASK/PROJECTS ports; public cutover remains ARCH-02I | | ARCH-04B | ARCH-04A, POL-07, ARCH-03C, merged ARCH-02H | Complete: ART hidden verified Submission input; live authority remains deferred | -| ARCH-04B2 | ARCH-04A and merged ART admission/verification/binding foundations | ART bounded checker output/log ingestion and verified binding, no routing | -| ARCH-04C | ARCH-04A, ARCH-04B, ARCH-04B2, POL-07 | CHECKERS durable execution/result/currentness and worker recovery | +| [ARCH-04B2](../WS-ARCH-001-04B2.md) | ARCH-04A and merged ART admission/verification/binding foundations | Complete: hidden bounded checker-output store/recovery and verified binding, no routing or live reservation | +| ARCH-04C | ARCH-04A, ARCH-04B, delivered ARCH-04B2, POL-07 | CHECKERS durable execution/result/currentness and worker recovery, including valid empty output sets | | ARCH-04D | ARCH-04B, ARCH-04C | AUTH post-submit materialization/result activation | | AUTH-OUTBOX-01 | Merged shared outbox persistence and AUTH service/PREP foundations | Complete: planned dispatcher identity/action/matrix and unavailable typed authority contract | | CON-02B | AUTH-OUTBOX-01 | Complete: shared hidden dispatcher/claim fencing, typed handlers and recovery | @@ -69,6 +69,11 @@ the usable ARCH-04F failure route. The true routing foundation may ship first with false still unavailable; neither route invents a new acceptance subsystem. Shared REV acceptance persistence/CON participation can precede live human queues or decisions, so this extension adds no REV-admission dependency cycle. +The first complete contributor milestone includes public claim/upload/pre-check +feedback, immutable Submission, automatic checking, failure remediation and, for +locked false, shared FinalAcceptance plus the submitter ContributionRecord and +applicable awards. Complete it before live human review/revision; independent +review foundations may proceed in another worktree without blocking that path. CP05 and ARCH-04A have independent prerequisites. POL-04B consumes the corrected ARCH-04A catalogue/schema before producing approval-eligible generations. Owners may @@ -82,14 +87,14 @@ ARCH-03C1 completes exact reconciler authority and decision-bound receipts. ARCH-03C2 supplies originating-transaction-only per-assignment producer events and first handler registration with enforced prefork topology; it must never backfill or dispatch retained invalidation rows. Public TASK activation is complete through ARCH-03C7. AUTH-18 delivers public -manager guide activation/context; ARCH-03D hidden intake is delivered; hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. Subsequent +manager guide activation/context; ARCH-03D hidden intake, hidden exact post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C is next. Subsequent PR-sized contracts name exact files, public types, current migration head and runnable proof before implementation; they refine this design, not create a new permission requirement. ### Supporting foundations required by automatic routing -Current supporting contracts are [ARCH-04B2](chunks/WS-ARCH-001-04B-art-post-submit-materialization.md#arch-04b2--separate-art-output-custody-child), +Current supporting contracts are delivered [ARCH-04B2](../WS-ARCH-001-04B2.md), [AUTH-OUTBOX-01/02](../../WS-AUTH-001/planning/PLAN.md#ws-auth-001-outbox-01--unavailable-dispatcher-contract), [CON-02B](../../WS-CON-001/OVERVIEW.md#con-02b-current-dispatcher-contract), and [ARCH-04E1A/04E1B/04E2/04E3](chunks/WS-ARCH-001-04E-canonical-allow-review.md#current-bounded-sequence). @@ -175,8 +180,8 @@ owning implementation, not with planning prose or fake database claims. ### Named proof targets for the remaining design ARCH-04B input proof is delivered in `test_post_submit_materialization.py` and -`test_post_submit_selection.py`, including deny-before-I/O composition. Output -custody and durable/live execution proof remain with ARCH-04B2/04C/04D below. +`test_post_submit_selection.py`, including deny-before-I/O composition. ARCH-04B2 +delivers output custody proof; durable/live execution proof remains with 04C/04D below. These are required future implementation tests, not tests claimed present or executed by this planning PR. Each owner's bounded record fixes the final @@ -193,7 +198,7 @@ claims require their real custody, not unit substitutes. | ARCH-04A/POL-07 | `test_registered_evaluator_rejects_invalid_work`, `test_checker_facade_delegates_once` | Actual registered evaluator fixtures and typed composition; presence-only mutant must fail | | CP06/CP07/AUTH-12H | `test_activate_without_legacy_payment_or_task`, `test_activation_requires_exact_selected_policy`, `test_activation_rejects_missing_review_revision_config` | PostgreSQL atomic command plus full response serialization; foreign/retired/incomplete new binding denies | | CP08/ARCH-03A/03B/03C | `test_ready_preserves_screening_policy_lock`, `test_claim_copies_policy_without_current_lookup` | PostgreSQL and actual AUTH/owner composition; later publication leaves existing attempt unchanged | -| ARCH-04B2/04C/04D | `test_late_revocation_cannot_publish_result`, `test_unfinished_checker_recovery_reuses_attempt`, `test_terminal_retry_requires_operator_and_new_attempt` | Local/MinIO, real worker/provider contract, PostgreSQL races; independent sessions and staged/final state | +| ARCH-04C/04D | `test_late_revocation_cannot_publish_result`, `test_unfinished_checker_recovery_reuses_attempt`, `test_terminal_retry_requires_operator_and_new_attempt` | Consume delivered 04B2 custody; Local/MinIO, real worker/provider contract and PostgreSQL races; independent sessions and staged/final state | | ARCH-04E | `test_submission_to_current_allow_review`, `test_superseded_run_cannot_route`, `test_duplicate_dispatch_has_one_manifest`; false tests in the shared acceptance contract | Real DB/worker/storage path, exact authority-event references; true creates no acceptance, false creates the shared atomic acceptance with no human Review | Owner-local schema names and migrations are chosen from the then-current diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md index bb515bf3f..69f460c3d 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md @@ -2,25 +2,28 @@ ARCH-04B implementation follows the [current bounded change record](../../WS-ARCH-001-04B.md). It replaces this input skeleton with exact files, predicates and verification. -The output child below remains separate and Planned. +The separate output child is delivered by the +[current ARCH-04B2 record](../../WS-ARCH-001-04B2.md). ## ARCH-04B2 — Separate ART output-custody child -Input materialization does not implement output persistence. Keep a separate -ART-owned bounded change for `CheckerArtifactOutputPort.store` and -`ArtifactBindingPort.bind_checker_output`, using 04A's exact immutable request, -attempt/run and policy facts. Reuse the generic admission, put-intent, -verification, quota and binding infrastructure. Replace the current ART -repository's private CheckerRun lookup with the canonical public-fact boundary; -do not import CHECKERS models in ART behavior. Existing relational run foreign -keys may remain and use controlled foreign-row fixtures for hidden proof. +Input materialization does not implement output persistence. ARCH-04B2 now +supplies typed `CheckerArtifactOutputPort.store`, byte-free +`recover(selector)`, and `ArtifactBindingPort.bind_checker_output` using 04A's +exact immutable request and owner reservation facts. It reuses generic +admission, put intent, observation, verification, quota and binding +infrastructure. ART behavior no longer reads CheckerRun through its private +repository; the separate Operator resource-to-project lookup remains explicit. -Runtime CHECKERS reserves its run before output I/O. ART admits bounded output -and logs against project/task/fixed-service/deployment quotas, never contributor -quota, independently rereads/verifies bytes, then supplies a flush-only binding -participant. No row lock or PREP survives storage I/O. 04C later composes final -binding publication with final CHECKER result and completion outbox in one -transaction, after fresh exact action authority supplied by 04D. +Runtime CHECKERS will reserve its run before output I/O. ART admits each +owner-declared slot against project/task/fixed-service/deployment quotas, never +contributor quota, independently rereads/verifies bytes, then supplies a +flush-only binding participant. Authority is fresh for each phase and no row +lock or PREP survives storage I/O. Production reservation and authority remain +unavailable. The current structural catalogue declares zero slots; controlled +nonempty fixtures prove ART mechanics only. 04C must support the empty set and +later composes any reserved bindings with final CHECKER result and completion +outbox in one transaction, after fresh exact action authority supplied by 04D. The output child proves Local/MinIO parity, non-reproducible/unknown outcome handling without fabricated bytes, crash cleanup, deadline/quota races, @@ -28,4 +31,5 @@ foreign-request denial and fixed-service attribution. 04C/04D separately prove the integrated run/binding/result transaction. Neither child owns TASK routing, REV records or contributor-blame decisions. Expand each owner-local child into its exact implementation record rather than combining ART and CHECKERS writes -in one implementation PR. +in one implementation PR. The [ARCH-04B2 record](../../WS-ARCH-001-04B2.md) is +the delivered source; this skeleton no longer assigns the next change. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md index 40113b1b3..794974ed5 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md @@ -1,7 +1,7 @@ # Chunk Contract: WS-ARCH-001-04C CHECKER Current Result Persistence -Disposition: Planned. Dependencies: 04A, 04B, its ART-owned output-custody -child and POL-07. Risk: L1. Outcome: +Disposition: Planned. Dependencies: 04A, 04B, delivered +[ARCH-04B2 output custody](../../WS-ARCH-001-04B2.md) and POL-07. Risk: L1. Outcome: CHECKERS installs hidden, deny-only execution of the exact post-submit plan and persists one immutable final result with explicit supersession/currentness and routing recommendation. @@ -68,12 +68,16 @@ proves that capability; absent it, remain blocked rather than invoking again. Prove slow-worker/takeover completion in both orders and no duplicate provider call or final output from stale leases. -Required logs/generated outputs pass the ART-owned output admission and -independent verification path before final publication. Compose verified -checker-output bindings and CHECKERS final completion in one caller transaction -through owner public ports. Missing, stale, unverified or mismatched required -output prevents final-current result/routing. Do not implement ART writes here, -persist provider locations as evidence, or charge output bytes to a contributor. +The current structural catalogue declares zero output slots, and a completed +evaluation with that exact empty set must finalize without inventing logs or +generated artifacts. When CHECKERS reserves nonempty slots, every produced +output passes the delivered ART-owned admission and independent verification +path before final publication. Compose those verified bindings and CHECKERS +final completion in one caller transaction through owner public ports. Missing, +stale, unverified or mismatched owner-required output prevents final-current +result/routing. Do not implement ART writes here, persist provider locations as +evidence, charge output bytes to a contributor, or turn ARCH-04B2's controlled +nonempty fixtures into catalogue requirements. The final multi-participant transaction must declare one lock order consistent with AUTH PREP: prepare required service authority custody before feature locks, diff --git a/.commitrail/initiatives/WS-ART-001/OVERVIEW.md b/.commitrail/initiatives/WS-ART-001/OVERVIEW.md index 580993951..6cc29f0b9 100644 --- a/.commitrail/initiatives/WS-ART-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-ART-001/OVERVIEW.md @@ -8,8 +8,10 @@ and the [capability ledger](../../../docs/roadmap_status.md). storage, and bounded private processing scratch. - Current boundary: ready-admission publication and hidden preparation, consumption, and binding dependencies are merged through ARCH-02H. -- Next usable boundary: ARCH-04B2 checker output custody. ARCH-04B hidden input - materialization is delivered; live authority and execution cutover remain later. +- Completed boundary: ARCH-04B hidden input materialization and hidden + [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md). + Production reservation and authority remain unavailable. +- Next usable boundary: ARCH-04C durable execution and results. - Governing sources: artifact specifications, `ArtifactStore`, `ArtifactScratchManager`, code, migrations, and artifact tests. - Preserve: SHA-256/byte-count identity, reread verification, isolation, @@ -32,8 +34,9 @@ ART retains the merged default-plus-project intake compiler/executor; POL-07 is a facade, not a replacement or second precheck run. New unified generations must prove exact approved lineage at preparation, consumption and binding. -1. ARCH-04B hidden exact Submission materialization is delivered. Continue with - ARCH-04B2 output custody, then durable execution and fixed-service authority. +1. ARCH-04B hidden exact Submission materialization and ARCH-04B2 hidden output + custody are delivered. Continue with ARCH-04C durable execution, then + fixed-service authority. 2. ARCH-04F owns checker-remediation resubmission using existing ART ports; later reviewer-requested revision remains a separate REV boundary. Add those dependencies before public Submission cutover. diff --git a/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md b/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md index b68e3f3cf..b5e39aff9 100644 --- a/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md @@ -23,8 +23,8 @@ Historical pre-cutover work records: [`STATUS.md`](pre-cutover/STATUS.md), - Public post-policy composition: POL-06B delivered using existing AUTH-12G. - Completed activation boundary: AUTH-12H exact-project manager authority for CP07 complete-guide activation/binding, with live-authority replay. -- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden - ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04C durable post-submit evaluation after delivered + hidden ARCH-04B input materialization and [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), then live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published @@ -77,5 +77,5 @@ manager proposal review, pre-submit approval and manual correction dispatch. AUTH-OUTBOX-01/02 bracket hidden CON-02B dispatch; ARCH-04E2 activates only the proven TASK routing handler before ARCH-04E3 live composition. TASK queue/read exposure is complete through ARCH-03C7. AUTH-18 public - manager guide activation/context is delivered; ARCH-03D hidden intake is delivered; hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. + manager guide activation/context is delivered; ARCH-03D hidden intake, hidden exact post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C is next. Remaining work must use its exact owner, not the superseded broad designs. diff --git a/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md index 6c86ed494..d9770933d 100644 --- a/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md @@ -13,7 +13,7 @@ work and historical proposals. | [AUTH-12H](../WS-AUTH-001-12H.md) | Complete: exact manager authority for CP07; AUTH-18 exposes manager activation and selections publicly | | CP05 | Complete: exact five policy actions; public Finance exposure delivered by CP05A | | ARCH-03C | Complete through 03C7: task/assignment authority, public queues, projections and audit reads; replaces broad AUTH-13 | -| [AUTH-18](../WS-AUTH-001-18.md) | Complete: public manager activation/context over CP07 and AUTH-12H; ARCH-03D completes hidden approved-guide intake; ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next | +| [AUTH-18](../WS-AUTH-001-18.md) | Complete: public manager activation/context over CP07 and AUTH-12H; ARCH-03D hidden intake, ARCH-04B input and ARCH-04B2 output custody are delivered; ARCH-04C durable execution is next | | ARCH-04D | AUTH materialization/output plus CHECKERS execute/finalize activation after ARCH-04B/04B2/04C; replaces AUTH-14/XINT-06B | | [AUTH-OUTBOX-01](PLAN.md#ws-auth-001-outbox-01--unavailable-dispatcher-contract) | Complete: unavailable exact dispatcher identity/action/phase contract; CON-02B and AUTH-OUTBOX-02 mechanics complete; feature authority/registration remain separate | | [AUTH-OUTBOX-02](PLAN.md#ws-auth-001-outbox-02--exact-dispatcher-activation) | Complete: exact dispatcher mechanics activation, phase audit custody and bounded prefork delivery; ARCH-03C2 subsequently registers assignment invalidation, while future handlers require their own exact authority | diff --git a/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md b/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md index 76edeed5b..aa8bf777e 100644 --- a/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md @@ -25,7 +25,7 @@ AUTH-13/14 cutovers are not additional implementation work. public TASK activation is complete through ARCH-03C7. - [AUTH-18](../WS-AUTH-001-18.md) delivers public manager activation and exact selection discovery over CP07/AUTH-12H. ARCH-03D completes hidden approved-guide - intake; ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. + intake; ARCH-04B hidden exact post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C durable execution is next. - CP05 owns exact ContributionPolicy-action activation after merged CP04B. - CP08 delivered the minimal lineage writers. ARCH-03B8 hidden task audit evidence and 03B9 hidden assignment invalidation are complete. ARCH-03C delivered @@ -53,7 +53,7 @@ and public JSON-packet Submission creation. Admission-backed creation stays hidd ARCH-03B/03C reuse these exact actions and command owners; public TASK access and authority are delivered through 03C7, with invalidation wiring in 03C2. CP08 delivered ContributionPolicyVersion lineage. ARCH-03D completes hidden approved-guide intake. ARCH-04B hidden exact post-submit -input is delivered; ARCH-04B2 output custody is next; public intake remains deferred to ARCH-02I. Do not restore eligibility +input and ARCH-04B2 output custody are delivered; ARCH-04C durable execution is next; public intake remains deferred to ARCH-02I. Do not restore eligibility or register replacement aliases. ## WS-AUTH-001-OUTBOX-01 — unavailable dispatcher contract diff --git a/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md b/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md index e228dabb8..581aba1d8 100644 --- a/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md +++ b/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md @@ -8,8 +8,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), - Completed boundary: recovery foundation and [TASK/checker authorization cleanup](WS-AUTH-003-TASKCHECKER.md). - Intent: route public authorization capability through `authorization.api` and remove cross-module repository/model coupling. -- Next usable boundary: ARCH-04B2 output custody after delivered ARCH-04B hidden input, - ARCH-03D hidden intake and +- Next usable boundary: ARCH-04C durable execution after delivered ARCH-04B + hidden input, [ARCH-04B2 output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), ARCH-03D hidden intake and [AUTH-18 public manager activation](../WS-AUTH-001/WS-AUTH-001-18.md). Submission/checker history uses canonical authority; the alternate gate lifecycle is removed. Continue shrinking the canonical import ledger as implementation diff --git a/.commitrail/initiatives/WS-CON-001/OVERVIEW.md b/.commitrail/initiatives/WS-CON-001/OVERVIEW.md index afb315fff..627008146 100644 --- a/.commitrail/initiatives/WS-CON-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-CON-001/OVERVIEW.md @@ -11,8 +11,8 @@ and the [capability ledger](../../../docs/roadmap_status.md). immutable ContributionRecords and optional project-policy-driven compensation awards without coupling lifecycle truth to an economic provider. -- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden - ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04C durable post-submit evaluation after delivered + hidden ARCH-04B input materialization and [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), then live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published diff --git a/.commitrail/initiatives/WS-POL-003/OVERVIEW.md b/.commitrail/initiatives/WS-POL-003/OVERVIEW.md index a7511f3e2..842cc5be5 100644 --- a/.commitrail/initiatives/WS-POL-003/OVERVIEW.md +++ b/.commitrail/initiatives/WS-POL-003/OVERVIEW.md @@ -27,8 +27,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), Production post-submit phase execution remains unavailable. - Intent: compile one locked guide and its policies into authoritative, versioned project behavior without circular subsystem authority. -- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden - ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04C durable post-submit evaluation after delivered + hidden ARCH-04B input materialization and [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), then live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published diff --git a/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md index 69ef63219..85141bfd1 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md @@ -29,7 +29,7 @@ post-task requirement. Historical split parents 05/06 are not extra PRs. | `WS-AUTH-001-12G` | Activate exact fixed-service projection plus PM approval/correction authority for the hidden 06A manifest. | 06A | | `WS-POL-003-06B` | Live deterministic post-submit projection/approval cutover with zero additional inference. | 06A + AUTH-12G | | `WS-POL-003-07` | One typed facade over existing ART pre and CHECKER post contracts; no post-result persistence. | 06B + ARCH-04A registered capability proof + merged ART-04B1-04B3 | -| `WS-AUTH-001-12H` / [AUTH-18](../../WS-AUTH-001/WS-AUTH-001-18.md) | Complete: exact guide activation authority and public manager activation/context over the approved unified chain. CP08 lineage and ARCH-03 task authority/projections are delivered; ARCH-03D completes hidden approved-guide intake. ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next; public intake remains ARCH-02I. CP09 remains later. | POL-07 + corrected AUTH-12B2 + CP05 active ContributionPolicy behavior + CP06 validation + CP07 ProjectGuide binding | +| `WS-AUTH-001-12H` / [AUTH-18](../../WS-AUTH-001/WS-AUTH-001-18.md) | Complete: exact guide activation authority and public manager activation/context over the approved unified chain. CP08 lineage and ARCH-03 task authority/projections are delivered; ARCH-03D hidden intake, ARCH-04B input and ARCH-04B2 output custody are delivered; ARCH-04C is next and public intake remains ARCH-02I. CP09 remains later. | POL-07 + corrected AUTH-12B2 + CP05 active ContributionPolicy behavior + CP06 validation + CP07 ProjectGuide binding | | `WS-POL-003-08` | Supplementary visibility; separate remaining cleanup is parked and handled within each affected module. Essential review/correction belongs to 05A/05B and 06A/06B. | Planned after 07 + AUTH-12H + canonical WS-ARCH-001-04E manifest; any separately authorized retained-data change requires CP09's inventory, mapping and readability/recoverability proof for affected facts; no cleanup prerequisite for 04B | The 04E manifest proves canonical routing, not legacy-history preservation. @@ -54,7 +54,8 @@ mixed-generation chains deny through the existing locked-lineage checks. ARCH-04A contracts/capability conformance precede POL-07 and guide activation. ARCH-04B delivers hidden exact ART post-submit input with deny-only production -authority. ARCH-04B2 output custody is next, followed by ARCH-04C durable CHECKER -execution/results. +authority. ARCH-04B2 hidden output custody is delivered with unavailable +reservation/authority composition, followed by ARCH-04C durable CHECKER +execution/results with valid empty-output support. WS-ARCH-001-04D is the later replacement activation gate. Historical XINT-06B and AUTH-14 contracts are superseded/non-executable. diff --git a/.commitrail/initiatives/WS-POL-003/planning/PLAN.md b/.commitrail/initiatives/WS-POL-003/planning/PLAN.md index 7d41f450c..0b9664c87 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/PLAN.md +++ b/.commitrail/initiatives/WS-POL-003/planning/PLAN.md @@ -56,7 +56,8 @@ aliases or fallbacks. AUTH-12F4 and AUTH-12G supply the delivered pre/post approval authority. AUTH-12H supplies exact guide activation authority; AUTH-18 exposes manager activation and selection discovery publicly. ARCH-03D completes hidden approved-guide intake integration. ARCH-04B hidden exact -post-submit materialization is delivered; ARCH-04B2 output custody is next; public intake remains deferred to ARCH-02I. +post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C +durable execution is next; public intake remains deferred to ARCH-02I. The sequence through POL-04B is complete; POL-04B1 supplies automatic request custody for the delivered live cutover: diff --git a/README.md b/README.md index e6dc2f38d..8562e8cf1 100644 --- a/README.md +++ b/README.md @@ -651,9 +651,11 @@ history only to covered Audit Authority and removes the old payload-bearing task-only audit route. AUTH-18 delivers public manager guide activation; ARCH-03D connects hidden intake through final durable intent to canonical TASK/PROJECTS ports, preserving historical guide selection. Public intake remains deferred. ARCH-04B supplies hidden exact verified Submission -input with scoped async file access and cleanup. Production materialization -authority remains deny-only; ARCH-04B2 output custody, ARCH-04C durable execution -and ARCH-04D live authority remain separate steps. +input with scoped async file access and cleanup. ARCH-04B2 adds hidden typed +checker-output storage, byte-free recovery and flush-only verified binding over +the generic ART put/verification path. CHECKERS reservation and output authority +remain unavailable in production; ARCH-04C durable execution and ARCH-04D live +authority are the next separate steps. ## v0.1 Success Standard diff --git a/backend/alembic/env.py b/backend/alembic/env.py index f8258a08a..b6840322b 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -21,7 +21,7 @@ target_metadata = Base.metadata _BASELINE_REVISION = "0001_uuid7_v01" -_CURRENT_HEAD_REVISION = "0006_history_read_authority" +_CURRENT_HEAD_REVISION = "0007_checker_output_custody" _RECREATE_GUIDANCE = ( "Workstream v0.1 requires a fresh database; recreate this database before " "running the 0001_uuid7_v01 migration" @@ -52,7 +52,7 @@ def do_run_migrations(connection: Connection) -> None: .scalars() .all() ) - if revisions not in ((), (_BASELINE_REVISION,), ("0002_task_queue_authority",), ("0003_task_read_authority",), ("0004_task_context_authority",), ("0005_task_evidence_authority",), (_CURRENT_HEAD_REVISION,)): + if revisions not in ((), (_BASELINE_REVISION,), ("0002_task_queue_authority",), ("0003_task_read_authority",), ("0004_task_context_authority",), ("0005_task_evidence_authority",), ("0006_history_read_authority",), (_CURRENT_HEAD_REVISION,)): raise RuntimeError(_RECREATE_GUIDANCE) # The read-only preflight autobegins a SQLAlchemy transaction. End that # transaction before Alembic establishes the migration transaction; diff --git a/backend/alembic/versions/0007_checker_output_custody.py b/backend/alembic/versions/0007_checker_output_custody.py new file mode 100644 index 000000000..2343bdbbf --- /dev/null +++ b/backend/alembic/versions/0007_checker_output_custody.py @@ -0,0 +1,367 @@ +"""Bind checker output intent and publication to exact verified ART custody.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0007_checker_output_custody" +down_revision = "0006_history_read_authority" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Install fail-closed checker output intent and binding custody.""" + op.execute( + "lock table artifact_bindings, artifact_contents, artifact_operation_receipts, " + "artifact_put_attempts, artifact_put_observation_receipts, artifact_replicas, " + "artifact_verification_jobs, artifact_verification_receipts, checker_runs, " + "submissions, workstream_tasks in access exclusive mode" + ) + + op.add_column( + "artifact_put_attempts", + sa.Column("submission_id", sa.Uuid(), nullable=True), + ) + op.add_column( + "artifact_put_attempts", + sa.Column("submission_version", sa.Integer(), nullable=True), + ) + op.add_column( + "artifact_put_attempts", + sa.Column("checker_request_digest", sa.String(length=71), nullable=True), + ) + op.add_column( + "artifact_bindings", + sa.Column("put_attempt_id", sa.Uuid(), nullable=True), + ) + op.add_column( + "artifact_bindings", + sa.Column("verification_receipt_id", sa.Uuid(), nullable=True), + ) + + _refuse_unprovable_retained_checker_output() + + op.execute( + "alter table artifact_put_attempts drop constraint " + "ck_artifact_put_attempts_producer_reference" + ) + op.execute( + "alter table artifact_put_attempts add constraint " + "ck_artifact_put_attempts_producer_reference check (" + "(producer_request_type = 'guide' and guide_source_item_id is not null " + "and checker_run_id is null and task_id is null and submission_id is null " + "and submission_version is null and logical_role is null) or " + "(producer_request_type = 'checker_output' and guide_source_item_id is null " + "and checker_run_id is not null and task_id is not null " + "and submission_id is not null and submission_version is not null " + "and octet_length(logical_role) between 1 and 100) or " + "(producer_request_type = 'submission_bundle' and guide_source_item_id is null " + "and checker_run_id is null and task_id is not null and submission_id is null " + "and submission_version is null and logical_role is null))" + ) + op.execute( + "alter table artifact_put_attempts add constraint " + "ck_artifact_put_attempts_checker_request_digest check (" + "(producer_request_type = 'checker_output' and checker_request_digest is not null " + "and checker_request_digest ~ '^sha256:[0-9a-f]{64}$') or " + "(producer_request_type <> 'checker_output' and checker_request_digest is null))" + ) + op.create_foreign_key( + "fk_artifact_put_attempts_task_project", + "artifact_put_attempts", + "workstream_tasks", + ["task_id", "project_id"], + ["id", "project_id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + "fk_artifact_put_attempts_checker_run_ownership", + "artifact_put_attempts", + "checker_runs", + ["checker_run_id", "task_id", "submission_id"], + ["id", "task_id", "submission_id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + "fk_artifact_put_attempts_submission_version", + "artifact_put_attempts", + "submissions", + ["submission_id", "task_id", "submission_version"], + ["id", "task_id", "version"], + ondelete="RESTRICT", + ) + op.create_index( + "ix_artifact_put_attempts_submission_id", + "artifact_put_attempts", + ["submission_id"], + ) + + op.create_foreign_key( + "fk_artifact_bindings_checker_put_attempt", + "artifact_bindings", + "artifact_put_attempts", + ["put_attempt_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + "fk_artifact_bindings_checker_verification_receipt", + "artifact_bindings", + "artifact_verification_receipts", + ["verification_receipt_id"], + ["id"], + ondelete="RESTRICT", + ) + op.execute( + "alter table artifact_bindings add constraint " + "ck_artifact_bindings_checker_output_lineage check (" + "(resource_type = 'checker_run' and scope_version = 1 " + "and put_attempt_id is not null and verification_receipt_id is not null) or " + "(resource_type <> 'checker_run' and put_attempt_id is null " + "and verification_receipt_id is null))" + ) + op.create_index( + "ix_artifact_bindings_put_attempt_id", + "artifact_bindings", + ["put_attempt_id"], + ) + op.create_index( + "ix_artifact_bindings_verification_receipt_id", + "artifact_bindings", + ["verification_receipt_id"], + ) + + _install_checker_output_attempt_custody() + _install_checker_output_binding_guard() + + +def downgrade() -> None: + """Reject destructive downgrade of immutable checker output evidence.""" + raise RuntimeError("Workstream v0.1 migrations cannot be downgraded; recreate the database") + + +def _refuse_unprovable_retained_checker_output() -> None: + """Never invent the absent evaluation/slot digest for retained checker output.""" + op.execute( + """ + DO $$ BEGIN + IF EXISTS ( + SELECT 1 + FROM artifact_put_attempts attempt + LEFT JOIN checker_runs run + ON run.id=attempt.checker_run_id AND run.task_id=attempt.task_id + LEFT JOIN submissions submission + ON submission.id=run.submission_id + AND submission.task_id=run.task_id + AND submission.version=run.submission_version + LEFT JOIN workstream_tasks task + ON task.id=run.task_id AND task.project_id=attempt.project_id + WHERE attempt.producer_request_type='checker_output' + AND (run.id IS NULL OR submission.id IS NULL OR task.id IS NULL) + ) THEN + RAISE EXCEPTION 'retained checker output attempt ownership is inconsistent' + USING ERRCODE='23514'; + END IF; + IF EXISTS ( + SELECT 1 FROM artifact_put_attempts + WHERE producer_request_type='checker_output' + ) THEN + RAISE EXCEPTION + 'retained checker output request custody is unprovable; evaluation and slot digest was not persisted' + USING ERRCODE='23514'; + END IF; + IF EXISTS ( + SELECT 1 FROM artifact_bindings WHERE resource_type='checker_run' + ) THEN + RAISE EXCEPTION + 'retained checker output binding ancestry is unprovable' + USING ERRCODE='23514'; + END IF; + END $$ + """ + ) + + +def _install_checker_output_attempt_custody() -> None: + op.execute( + """ + CREATE FUNCTION guard_checker_output_put_attempt_custody() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN + IF OLD.producer_request_type='checker_output' THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN OLD; + END IF; + IF OLD.producer_request_type='checker_output' + OR NEW.producer_request_type='checker_output' THEN + IF ROW( + NEW.id, NEW.producer_request_type, NEW.producer_type, NEW.producer_ref, + NEW.project_id, NEW.task_id, NEW.guide_source_item_id, NEW.checker_run_id, + NEW.submission_id, NEW.submission_version, NEW.logical_role, + NEW.sha256, NEW.byte_count, NEW.media_type, NEW.storage_namespace_id, + NEW.namespace_fingerprint, NEW.canonical_target, NEW.operation_identity, + NEW.request_digest, NEW.checker_request_digest, NEW.maximum_observations, + NEW.prepared_at, NEW.created_at + ) IS DISTINCT FROM ROW( + OLD.id, OLD.producer_request_type, OLD.producer_type, OLD.producer_ref, + OLD.project_id, OLD.task_id, OLD.guide_source_item_id, OLD.checker_run_id, + OLD.submission_id, OLD.submission_version, OLD.logical_role, + OLD.sha256, OLD.byte_count, OLD.media_type, OLD.storage_namespace_id, + OLD.namespace_fingerprint, OLD.canonical_target, OLD.operation_identity, + OLD.request_digest, OLD.checker_request_digest, OLD.maximum_observations, + OLD.prepared_at, OLD.created_at + ) THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_put_attempt_custody " + "BEFORE DELETE OR UPDATE ON artifact_put_attempts FOR EACH ROW " + "EXECUTE FUNCTION guard_checker_output_put_attempt_custody()" + ) + op.execute( + """ + CREATE FUNCTION guard_checker_output_put_attempt_truncate() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM artifact_put_attempts + WHERE producer_request_type='checker_output' + ) THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NULL; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_put_attempt_no_truncate BEFORE TRUNCATE " + "ON artifact_put_attempts FOR EACH STATEMENT " + "EXECUTE FUNCTION guard_checker_output_put_attempt_truncate()" + ) + + +def _install_checker_output_binding_guard() -> None: + op.execute( + """ + CREATE FUNCTION guard_checker_output_binding_insert() RETURNS trigger + LANGUAGE plpgsql AS $$ + DECLARE + verification artifact_verification_receipts%ROWTYPE; + job artifact_verification_jobs%ROWTYPE; + replica artifact_replicas%ROWTYPE; + attempt artifact_put_attempts%ROWTYPE; + content artifact_contents%ROWTYPE; + write_receipt_matches integer; + BEGIN + IF NEW.resource_type <> 'checker_run' THEN + RETURN NEW; + END IF; + + SELECT receipt.* INTO verification + FROM artifact_verification_receipts receipt + WHERE receipt.id=NEW.verification_receipt_id; + IF NOT FOUND THEN + RAISE EXCEPTION 'checker output binding verified ancestry mismatch' + USING ERRCODE='23514'; + END IF; + + SELECT value.* INTO job + FROM artifact_verification_jobs value + WHERE value.id=verification.verification_job_id + FOR SHARE; + SELECT value.* INTO replica + FROM artifact_replicas value + WHERE value.id=job.replica_id + FOR SHARE; + SELECT value.* INTO attempt + FROM artifact_put_attempts value + WHERE value.id=job.originating_put_attempt_id + FOR SHARE; + SELECT value.* INTO content + FROM artifact_contents value + WHERE value.id=replica.content_id + FOR SHARE; + + SELECT count(*) INTO write_receipt_matches + FROM ( + SELECT receipt.id + FROM artifact_operation_receipts receipt + WHERE attempt.receipt_id IS NOT NULL + AND receipt.id=attempt.receipt_id + AND receipt.put_attempt_id=attempt.id + AND receipt.replica_id=replica.id + AND receipt.checker_run_id=attempt.checker_run_id + AND receipt.logical_role=attempt.logical_role + AND receipt.request_digest=attempt.request_digest + AND receipt.provider_object_ref=replica.provider_object_ref + AND receipt.outcome='stored_pending_verification' + UNION ALL + SELECT receipt.id + FROM artifact_put_observation_receipts receipt + WHERE attempt.receipt_id IS NULL + AND receipt.put_attempt_id=attempt.id + AND receipt.execution_generation=attempt.execution_generation + AND receipt.outcome='observed_confirmed' + AND receipt.expected_sha256=attempt.sha256 + AND receipt.observed_sha256=attempt.sha256 + AND receipt.expected_byte_count=attempt.byte_count + AND receipt.observed_byte_count=attempt.byte_count + ) successful_write; + + IF job.id IS NULL OR replica.id IS NULL OR attempt.id IS NULL OR content.id IS NULL + OR NEW.put_attempt_id IS DISTINCT FROM attempt.id + OR NEW.content_id IS DISTINCT FROM content.id + OR NEW.project_id IS DISTINCT FROM attempt.project_id + OR NEW.resource_id IS DISTINCT FROM attempt.checker_run_id::text + OR NEW.logical_role IS DISTINCT FROM attempt.logical_role + OR NEW.actor_id IS DISTINCT FROM attempt.producer_ref + OR NEW.attribution_type IS DISTINCT FROM 'service_identity' + OR NEW.scope_version <> 1 + OR NEW.supersedes_binding_id IS NOT NULL + OR attempt.producer_request_type <> 'checker_output' + OR attempt.producer_type <> 'service_identity' + OR attempt.submission_id IS NULL OR attempt.submission_version IS NULL + OR attempt.checker_request_digest IS NULL + OR attempt.status <> 'object_confirmed' + OR attempt.replica_id IS DISTINCT FROM replica.id + OR attempt.storage_namespace_id IS DISTINCT FROM replica.storage_namespace_id + OR attempt.namespace_fingerprint IS DISTINCT FROM replica.namespace_fingerprint + OR attempt.canonical_target IS DISTINCT FROM replica.provider_object_ref + OR attempt.sha256 IS DISTINCT FROM content.sha256 + OR attempt.byte_count IS DISTINCT FROM content.byte_count + OR attempt.media_type IS DISTINCT FROM content.media_type + OR job.originating_put_attempt_id IS DISTINCT FROM attempt.id + OR job.replica_id IS DISTINCT FROM replica.id + OR job.status <> 'verified' + OR job.terminal_result_code <> 'verified' + OR job.terminal_at IS NULL + OR verification.outcome <> 'verified' + OR verification.execution_generation IS DISTINCT FROM job.execution_generation + OR verification.observed_sha256 IS DISTINCT FROM attempt.sha256 + OR verification.observed_byte_count IS DISTINCT FROM attempt.byte_count + OR replica.verification_state <> 'verified' + OR replica.availability_state <> 'available' + OR replica.integrity_state <> 'valid' + OR write_receipt_matches <> 1 THEN + RAISE EXCEPTION 'checker output binding verified ancestry mismatch' + USING ERRCODE='23514'; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_binding_insert BEFORE INSERT ON artifact_bindings " + "FOR EACH ROW EXECUTE FUNCTION guard_checker_output_binding_insert()" + ) diff --git a/backend/app/adapters/artifacts/__init__.py b/backend/app/adapters/artifacts/__init__.py index e6f97ed1f..90181d8a2 100644 --- a/backend/app/adapters/artifacts/__init__.py +++ b/backend/app/adapters/artifacts/__init__.py @@ -511,3 +511,24 @@ def post_submission_materialization(*, sessions, store, namespace, preparation, preparation=preparation, inspector=inspector, authority=DenyPostSubmissionMaterializationAuthority(), ) + + +def checker_output_storage(*, sessions, store, namespace, preparation, settings): + """Compose hidden output custody; live producer and write authority remain absent.""" + from app.modules.artifacts.checker_outputs import CheckerArtifactOutputService, DenyCheckerOutputWriteAuthority + from app.modules.artifacts.schemas import DenyArtifactInternalAuthority + from app.modules.checkers.api.output_custody import UnavailableCheckerOutputReservation + return CheckerArtifactOutputService( + sessions=sessions, store=store, namespace=namespace, preparation=preparation, settings=settings, + reservations=lambda session: UnavailableCheckerOutputReservation(), + authority=lambda session: DenyCheckerOutputWriteAuthority(), + internal_authority=lambda session: DenyArtifactInternalAuthority(), + ) + + +def checker_output_binding(session, *, namespace): + """Compose a deny-only caller-transaction binding participant.""" + from app.modules.artifacts.checker_output_bindings import CheckerOutputBindingService, DenyCheckerOutputBindingAuthority + from app.modules.checkers.api.output_custody import UnavailableCheckerOutputReservation + return CheckerOutputBindingService(session, namespace_fingerprint=namespace.namespace_fingerprint, + reservations=UnavailableCheckerOutputReservation(), authority=DenyCheckerOutputBindingAuthority()) diff --git a/backend/app/interfaces/artifact_operations.py b/backend/app/interfaces/artifact_operations.py index 92ab1c2e0..8f02e9c20 100644 --- a/backend/app/interfaces/artifact_operations.py +++ b/backend/app/interfaces/artifact_operations.py @@ -9,6 +9,7 @@ from app.modules.authorization.prepared import PreparedAuthorizationHandle from app.modules.authorization.runtime import AuthorizationContext +from app.modules.checkers.api.output_custody import CheckerOutputSelector __all__ = ( "ArtifactAuditResourceType", @@ -20,6 +21,8 @@ "CheckerArtifactOutputPort", "CheckerOutputBindingRequest", "CheckerOutputArtifactRequest", + "CheckerOutputArtifactResult", + "CheckerOutputBindingResult", "GuideArtifactIngestPort", "GuideArtifactIngestCommand", "GuideArtifactIngestRequest", @@ -75,29 +78,43 @@ class GuideArtifactIngestResult: @dataclass(frozen=True, slots=True) class CheckerOutputBindingRequest: - """Verified checker output and its exact CheckerRun owner.""" + """Bind one exact independently verified output in the caller transaction.""" - prepared_authorization: PreparedAuthorizationHandle - project_id: UUID - task_id: UUID - submission_id: UUID - checker_run_id: UUID - logical_role: str - verified_content_ids: tuple[UUID, ...] + selector: CheckerOutputSelector + put_attempt_id: UUID + verification_receipt_id: UUID @dataclass(frozen=True, slots=True) class CheckerOutputArtifactRequest: - """Generated checker bytes bound to one fixed service execution.""" + """Bounded generated bytes; action authority is freshly prepared by ART.""" - prepared_authorization: PreparedAuthorizationHandle - task_id: UUID - submission_id: UUID - checker_run_id: UUID - logical_role: str + selector: CheckerOutputSelector byte_source: AsyncIterable[bytes] +@dataclass(frozen=True, slots=True) +class CheckerOutputArtifactResult: + """Durable put identity and verified custody, without provider coordinates.""" + + put_attempt_id: UUID + status: str + content_id: UUID | None + verification_receipt_id: UUID | None + replayed: bool + + +@dataclass(frozen=True, slots=True) +class CheckerOutputBindingResult: + """Immutable binding participating in the caller's final-result transaction.""" + + binding_id: UUID + content_id: UUID + put_attempt_id: UUID + verification_receipt_id: UUID + replayed: bool + + @dataclass(frozen=True, slots=True) class ArtifactRecoveryRequest: """Reason-bound Operator retry of one exact verification job.""" @@ -140,16 +157,19 @@ async def ingest( class ArtifactBindingPort(Protocol): """Create exact action-bound bindings from verified content.""" - async def bind_checker_output(self, request: CheckerOutputBindingRequest) -> object: + async def bind_checker_output(self, request: CheckerOutputBindingRequest) -> CheckerOutputBindingResult: """Bind verified checker output under the checker binding action.""" class CheckerArtifactOutputPort(Protocol): """Store generated output for one fixed checker execution.""" - async def store(self, request: CheckerOutputArtifactRequest) -> object: + async def store(self, request: CheckerOutputArtifactRequest) -> CheckerOutputArtifactResult: """Store one generated checker artifact.""" + async def recover(self, selector: CheckerOutputSelector) -> CheckerOutputArtifactResult | None: + """Recover a lost output response without a byte source or regeneration.""" + class ArtifactOperatorReadPort(Protocol): """Expose bounded Operator reads without provider references.""" diff --git a/backend/app/modules/artifacts/checker_output_bindings.py b/backend/app/modules/artifacts/checker_output_bindings.py new file mode 100644 index 000000000..547d50a78 --- /dev/null +++ b/backend/app/modules/artifacts/checker_output_bindings.py @@ -0,0 +1,174 @@ +"""Verified checker-output bindings in the caller's final-result transaction.""" + +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.identifiers import new_record_id +from app.interfaces.artifact_operations import ( + CheckerOutputBindingRequest, + CheckerOutputBindingResult, +) +from app.modules.actors.api import ServiceIdentity +from app.modules.artifacts.checker_output_custody import ( + CheckerOutputStoredFacts, + select_checker_output, +) +from app.modules.artifacts.models import ArtifactBinding +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputReservationPort, + CheckerOutputSelector, + CheckerOutputUnavailable, +) + + +class CheckerOutputBindingAuthority(Protocol): + """Prepare authority before CHECKERS locks, consume exact ART custody afterward.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Deny before any owner or artifact lookup.""" + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Stabilize exact service authority before feature row locks.""" + + async def consume( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Consume binding action authority in this caller-owned transaction.""" + + def discard(self) -> None: + """Release process-local authority without committing or rolling back.""" + + +class DenyCheckerOutputBindingAuthority: + """No production binding until exact AUTH and CHECKERS custody are implemented.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Reject before protected reads.""" + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Reject binding authority preparation.""" + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + + async def consume( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Reject publication authority.""" + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + + def discard(self) -> None: + """No prepared authority exists.""" + + +class CheckerOutputBindingService: + """Flush one immutable binding; never commit the caller's result transaction.""" + + def __init__( + self, + session: AsyncSession, + *, + reservations: CheckerOutputReservationPort, + authority: CheckerOutputBindingAuthority, + namespace_fingerprint: str, + ) -> None: + self._session, self._reservations, self._authority = session, reservations, authority + self._namespace_fingerprint = namespace_fingerprint + + async def bind_checker_output( + self, request: CheckerOutputBindingRequest + ) -> CheckerOutputBindingResult: + """Fence current worker facts before acquiring ART binding/verification locks.""" + if ( + type(request) is not CheckerOutputBindingRequest + or not self._session.in_transaction() + or self._session.in_nested_transaction() + or type(request.put_attempt_id) is not UUID + or type(request.verification_receipt_id) is not UUID + ): + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + selector = CheckerOutputSelector.model_validate(request.selector) + try: + await self._authority.preflight(selector) + await self._authority.prepare(selector) + reservation = await self._reservations.resolve(selector) + reservation.select(selector) + await self._session.execute( + text("select pg_advisory_xact_lock(hashtextextended(:key, 0))"), + { + "key": f"checker-output:{selector.evaluation.project_id}:{selector.checker_run_id}:{selector.slot_key}", + }, + ) + facts = await select_checker_output( + self._session, + selector=selector, + reservation=reservation, + namespace_fingerprint=self._namespace_fingerprint, + put_attempt_id=request.put_attempt_id, + verification_receipt_id=request.verification_receipt_id, + lock_verified=True, + ) + if ( + facts is None + or facts.status != "verified" + or facts.verification_receipt_id != request.verification_receipt_id + ): + raise CheckerOutputUnavailable("checker_output_verification_unavailable") + await self._authority.consume(reservation, facts) + binding = await self._session.scalar( + select(ArtifactBinding) + .where( + ArtifactBinding.project_id == str(selector.evaluation.project_id), + ArtifactBinding.resource_type == "checker_run", + ArtifactBinding.resource_id == str(selector.checker_run_id), + ArtifactBinding.logical_role == selector.slot_key, + ArtifactBinding.scope_version == 1, + ) + .with_for_update() + .execution_options(populate_existing=True) + ) + replayed = binding is not None + if binding is not None: + if ( + binding.content_id, + binding.put_attempt_id, + binding.verification_receipt_id, + binding.actor_id, + binding.attribution_type, + ) != ( + str(facts.content_id), + str(facts.put_attempt_id), + str(facts.verification_receipt_id), + ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + "service_identity", + ): + raise CheckerOutputUnavailable("checker_output_binding_conflict") + else: + binding = ArtifactBinding( + id=str(new_record_id()), + content_id=str(facts.content_id), + project_id=str(selector.evaluation.project_id), + resource_type="checker_run", + resource_id=str(selector.checker_run_id), + logical_role=selector.slot_key, + scope_version=1, + actor_id=ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + attribution_type="service_identity", + supersedes_binding_id=None, + put_attempt_id=str(facts.put_attempt_id), + verification_receipt_id=str(facts.verification_receipt_id), + ) + self._session.add(binding) + await self._session.flush() + return CheckerOutputBindingResult( + UUID(binding.id), + UUID(binding.content_id), + request.put_attempt_id, + request.verification_receipt_id, + replayed, + ) + finally: + self._authority.discard() diff --git a/backend/app/modules/artifacts/checker_output_custody.py b/backend/app/modules/artifacts/checker_output_custody.py new file mode 100644 index 000000000..1079f6449 --- /dev/null +++ b/backend/app/modules/artifacts/checker_output_custody.py @@ -0,0 +1,215 @@ +"""Exact ART-owned output selection shared by storage recovery and binding.""" + +from dataclasses import dataclass +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.hashing import canonical_json_hash +from app.modules.artifacts.models import ( + ArtifactContent, + ArtifactPutAttempt, + ArtifactReplica, + ArtifactStorageNamespace, + ArtifactVerificationJob, + ArtifactVerificationReceipt, +) +from app.modules.artifacts.schemas import checker_output_request_digest_facts +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSelector, + CheckerOutputUnavailable, +) + + +@dataclass(frozen=True, slots=True) +class CheckerOutputStoredFacts: + """Exact storage facts consumed by fresh output action authority.""" + + put_attempt_id: UUID + checker_request_digest: str + sha256: str + byte_count: int + media_type: str + status: str + content_id: UUID | None + replica_id: UUID | None + verification_receipt_id: UUID | None + + +def output_operation_identity(selector: CheckerOutputSelector) -> str: + """Use one logical output identity across worker-lease replacement.""" + return canonical_json_hash( + { + "request_type": "checker_output", + "checker_run_id": str(selector.checker_run_id), + "logical_role": selector.slot_key, + } + ) + + +async def select_checker_output( + session: AsyncSession, + *, + selector: CheckerOutputSelector, + reservation: CheckerOutputReservation, + namespace_fingerprint: str, + put_attempt_id: UUID | None = None, + verification_receipt_id: UUID | None = None, + lock_verified: bool = False, +) -> CheckerOutputStoredFacts | None: + """Match immutable request and verified ancestry; never read foreign owners.""" + slot = reservation.select(selector) + expected_digest = canonical_json_hash( + checker_output_request_digest_facts( + reservation=reservation, + slot=slot, + ) + ) + attempt = await session.scalar( + select(ArtifactPutAttempt) + .where( + ArtifactPutAttempt.operation_identity == output_operation_identity(selector), + ) + .execution_options(populate_existing=True) + ) + if attempt is None: + return None + evaluation = selector.evaluation + if ( + ( + attempt.producer_request_type, + attempt.project_id, + attempt.task_id, + attempt.submission_id, + attempt.submission_version, + attempt.checker_run_id, + attempt.logical_role, + attempt.checker_request_digest, + attempt.media_type, + attempt.namespace_fingerprint, + ) + != ( + "checker_output", + str(evaluation.project_id), + str(evaluation.task_id), + str(evaluation.submission_id), + evaluation.submission_version, + str(selector.checker_run_id), + slot.key, + expected_digest, + slot.media_type, + namespace_fingerprint, + ) + or attempt.byte_count > slot.maximum_bytes + or (put_attempt_id is not None and attempt.id != str(put_attempt_id)) + ): + raise CheckerOutputUnavailable("checker_output_identity_mismatch") + facts = CheckerOutputStoredFacts( + UUID(attempt.id), + expected_digest, + attempt.sha256, + attempt.byte_count, + attempt.media_type, + attempt.status, + None, + None, + None, + ) + if attempt.status != "object_confirmed": + return facts + return await _verified_output_facts( + session, + attempt=attempt, + facts=facts, + namespace_fingerprint=namespace_fingerprint, + verification_receipt_id=verification_receipt_id, + lock_verified=lock_verified, + ) + + +async def _verified_output_facts( + session: AsyncSession, + *, + attempt: ArtifactPutAttempt, + facts: CheckerOutputStoredFacts, + namespace_fingerprint: str, + verification_receipt_id: UUID | None, + lock_verified: bool, +) -> CheckerOutputStoredFacts: + """Resolve and, for publication, lock the exact independent verification chain.""" + query = ( + select( + ArtifactVerificationReceipt, + ArtifactVerificationJob, + ArtifactReplica, + ArtifactContent, + ArtifactStorageNamespace, + ) + .select_from(ArtifactVerificationReceipt) + .join( + ArtifactVerificationJob, + ArtifactVerificationJob.id == ArtifactVerificationReceipt.verification_job_id, + ) + .join(ArtifactReplica, ArtifactReplica.id == ArtifactVerificationJob.replica_id) + .join(ArtifactContent, ArtifactContent.id == ArtifactReplica.content_id) + .join( + ArtifactStorageNamespace, + ArtifactStorageNamespace.id == ArtifactReplica.storage_namespace_id, + ) + .where( + ArtifactVerificationJob.originating_put_attempt_id == attempt.id, + ArtifactVerificationReceipt.outcome == "verified", + ) + .order_by(ArtifactVerificationReceipt.id) + .execution_options(populate_existing=True) + ) + if verification_receipt_id is not None: + query = query.where(ArtifactVerificationReceipt.id == str(verification_receipt_id)) + row = (await session.execute(query.limit(1))).one_or_none() + if row is None: + return facts + receipt, job, replica, content, namespace = row + if lock_verified: + # Match verifier completion's lock order; CHECKERS/authority are already held. + for model, record_id in ( + (ArtifactVerificationJob, job.id), + (ArtifactReplica, replica.id), + (ArtifactPutAttempt, attempt.id), + (ArtifactContent, content.id), + ): + await session.scalar( + select(model) + .where(model.id == record_id) + .with_for_update() + .execution_options(populate_existing=True) + ) + if not ( + attempt.status == "object_confirmed" + and job.status == "verified" + and receipt.execution_generation == job.execution_generation + and job.replica_id == replica.id == attempt.replica_id + and (replica.verification_state, replica.availability_state, replica.integrity_state) + == ("verified", "available", "valid") + and receipt.observed_sha256 == content.sha256 == attempt.sha256 + and receipt.observed_byte_count == content.byte_count == attempt.byte_count + and content.media_type == attempt.media_type + and replica.storage_namespace_id == attempt.storage_namespace_id + and replica.namespace_fingerprint + == namespace.namespace_fingerprint + == namespace_fingerprint + and replica.provider_object_ref == attempt.canonical_target + ): + raise CheckerOutputUnavailable("checker_output_verification_unavailable") + return CheckerOutputStoredFacts( + UUID(attempt.id), + facts.checker_request_digest, + attempt.sha256, + attempt.byte_count, + attempt.media_type, + "verified", + UUID(content.id), + UUID(replica.id), + UUID(receipt.id), + ) diff --git a/backend/app/modules/artifacts/checker_outputs.py b/backend/app/modules/artifacts/checker_outputs.py new file mode 100644 index 000000000..6a81a9382 --- /dev/null +++ b/backend/app/modules/artifacts/checker_outputs.py @@ -0,0 +1,253 @@ +"""Bounded checker bytes through the existing durable ART storage workflow.""" + +from collections.abc import Callable +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.core.config import Settings +from app.interfaces.artifact_operations import ( + CheckerOutputArtifactRequest, + CheckerOutputArtifactResult, +) +from app.interfaces.artifacts import ArtifactStore +from app.modules.artifacts.checker_output_custody import ( + CheckerOutputStoredFacts, + select_checker_output, +) +from app.modules.artifacts.models import ArtifactVerificationJob +from app.modules.artifacts.preparation import ArtifactPreparationService +from app.modules.artifacts.schemas import ( + ArtifactInternalAuthority, + CheckerOutputArtifactAdmissionRequest, +) +from app.modules.artifacts.service import ( + ArtifactAdmissionService, + ArtifactStorageOrchestrator, + ArtifactStorageNamespaceSpec, +) +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputReservationPort, + CheckerOutputSelector, + CheckerOutputUnavailable, +) + + +class CheckerOutputWriteAuthority(Protocol): + """Fresh write authority; no prepared handle survives a storage operation.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Deny before owner facts, source iteration or scratch access.""" + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Stabilize service authority before CHECKERS and ART transaction locks.""" + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + """Bind the exact active worker, slot and byte commitment before admission.""" + + async def consume_existing( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Authorize exact stored output recovery with the current worker lease.""" + + def discard(self) -> None: + """Dispose transaction-local authority after each phase.""" + + +class DenyCheckerOutputWriteAuthority: + """Keep production output storage unavailable until ARCH-04D activation.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Reject before any protected access.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Reject transaction authority preparation.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + """Reject durable output admission.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + async def consume_existing( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Reject recovery of protected output facts.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + def discard(self) -> None: + """No authority was retained.""" + + +class CheckerArtifactOutputService: + """Own preparation lifetime; reuse put/observation and independent verification.""" + + def __init__( + self, + *, + sessions: async_sessionmaker[AsyncSession], + reservations: Callable[[AsyncSession], CheckerOutputReservationPort], + authority: Callable[[AsyncSession], CheckerOutputWriteAuthority], + internal_authority: Callable[[AsyncSession], ArtifactInternalAuthority], + preparation: ArtifactPreparationService, + store: ArtifactStore, + namespace: ArtifactStorageNamespaceSpec, + settings: Settings, + ) -> None: + self._sessions, self._reservations, self._authority = sessions, reservations, authority + self._internal_authority, self._preparation = internal_authority, preparation + self._store, self._namespace, self._settings = store, namespace, settings + + async def store(self, request: CheckerOutputArtifactRequest) -> CheckerOutputArtifactResult: + """Commit checked output bytes before I/O, never a CHECKERS completion.""" + if type(request) is not CheckerOutputArtifactRequest: + raise TypeError("invalid checker output request") + selector = CheckerOutputSelector.model_validate(request.selector) + prepared = None + async with self._sessions() as session: + authority = self._authority(session) + try: + await authority.preflight(selector) + async with session.begin(): + await authority.prepare(selector) + reservation = await self._reservations(session).resolve(selector) + slot = reservation.select(selector) + authority.discard() + prepared = await self._preparation.prepare( + request.byte_source, + media_type=slot.media_type, + maximum_bytes=slot.maximum_bytes, + ) + async with session.begin(): + await authority.prepare(selector) + current = await self._reservations(session).resolve(selector) + if current.select(selector) != slot: + raise CheckerOutputUnavailable("checker_output_slot_changed") + admission = await ArtifactAdmissionService( + session, self._settings, self._namespace + ).admit( + CheckerOutputArtifactAdmissionRequest( + reservation=current, + slot_key=slot.key, + source=prepared.committed_source, + ), + checker_output_authority=authority, + existing_transaction=True, + ) + authority.discard() + storage = self._storage(session) + if admission.status == "object_confirmed": + status = admission.status + elif admission.replayed: + status = await storage.resume_committed_put( + attempt_id=admission.attempt_id, + source=prepared.committed_source, + ) + else: + status = await storage.execute_committed_put( + attempt_id=admission.attempt_id, + source=prepared.committed_source, + ) + await self._verify(session, storage, admission.attempt_id, status) + return await self._result(session, authority, selector, replayed=admission.replayed) + finally: + authority.discard() + if prepared is not None: + await prepared.close() + + async def recover(self, selector: CheckerOutputSelector) -> CheckerOutputArtifactResult | None: + """Recover a lost response by logical identity without regenerating bytes.""" + selector = CheckerOutputSelector.model_validate(selector) + async with self._sessions() as session: + authority = self._authority(session) + try: + await authority.preflight(selector) + async with session.begin(): + await authority.prepare(selector) + reservation = await self._reservations(session).resolve(selector) + reservation.select(selector) + facts = await select_checker_output( + session, + selector=selector, + reservation=reservation, + namespace_fingerprint=self._namespace.namespace_fingerprint, + ) + if facts is None: + return None + await authority.consume_existing(reservation, facts) + authority.discard() + storage = self._storage(session) + status = facts.status + if status not in {"object_confirmed", "verified"}: + status = await storage.resolve_put_attempt(facts.put_attempt_id) + await self._verify(session, storage, facts.put_attempt_id, status) + return await self._result(session, authority, selector, replayed=True) + finally: + authority.discard() + + def _storage(self, session: AsyncSession) -> ArtifactStorageOrchestrator: + """Use the shared put, observation and verification implementation.""" + return ArtifactStorageOrchestrator( + session, self._store, self._namespace, self._settings, self._internal_authority(session) + ) + + async def _verify( + self, + session: AsyncSession, + storage: ArtifactStorageOrchestrator, + attempt_id: UUID, + status: str, + ) -> None: + """Independently verify confirmed bytes through the existing ART worker.""" + if status not in { + "object_confirmed", + "verified", + "stored_pending_verification", + "observed_confirmed", + }: + return + async with session.begin(): + job_id = await session.scalar( + select(ArtifactVerificationJob.id) + .where( + ArtifactVerificationJob.originating_put_attempt_id == str(attempt_id), + ArtifactVerificationJob.status == "pending", + ) + .order_by(ArtifactVerificationJob.id) + .limit(1) + ) + if job_id is not None: + await storage.verify_object(UUID(job_id)) + + async def _result( + self, + session: AsyncSession, + authority: CheckerOutputWriteAuthority, + selector: CheckerOutputSelector, + *, + replayed: bool, + ) -> CheckerOutputArtifactResult: + """Recheck current owner custody and authority after provider operations.""" + async with session.begin(): + await authority.prepare(selector) + reservation = await self._reservations(session).resolve(selector) + reservation.select(selector) + facts = await select_checker_output( + session, + selector=selector, + reservation=reservation, + namespace_fingerprint=self._namespace.namespace_fingerprint, + ) + if facts is None: + raise CheckerOutputUnavailable("checker_output_unavailable") + await authority.consume_existing(reservation, facts) + return CheckerOutputArtifactResult( + facts.put_attempt_id, + facts.status, + facts.content_id, + facts.verification_receipt_id, + replayed, + ) diff --git a/backend/app/modules/artifacts/models.py b/backend/app/modules/artifacts/models.py index 47d82e738..0d6db1134 100644 --- a/backend/app/modules/artifacts/models.py +++ b/backend/app/modules/artifacts/models.py @@ -396,6 +396,13 @@ class ArtifactBinding(Base): "(scope_version > 1 and supersedes_binding_id is not null)", name="scope_version_predecessor", ), + CheckConstraint( + "(resource_type = 'checker_run' and scope_version = 1 " + "and put_attempt_id is not null and verification_receipt_id is not null) or " + "(resource_type <> 'checker_run' and put_attempt_id is null " + "and verification_receipt_id is null)", + name="checker_output_lineage", + ), ) id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True) @@ -411,6 +418,22 @@ class ArtifactBinding(Base): scope_version: Mapped[int] = mapped_column(Integer, nullable=False) actor_id: Mapped[str] = mapped_column(String(100), nullable=False) attribution_type: Mapped[str] = mapped_column(String(30), nullable=False) + put_attempt_id: Mapped[str | None] = mapped_column( + ForeignKey( + "artifact_put_attempts.id", + ondelete="RESTRICT", + name="fk_artifact_bindings_checker_put_attempt", + ), + index=True, + ) + verification_receipt_id: Mapped[str | None] = mapped_column( + ForeignKey( + "artifact_verification_receipts.id", + ondelete="RESTRICT", + name="fk_artifact_bindings_checker_verification_receipt", + ), + index=True, + ) supersedes_binding_id: Mapped[str | None] = mapped_column( ForeignKey("artifact_bindings.id", ondelete="RESTRICT"), index=True ) @@ -969,6 +992,24 @@ class ArtifactPutAttempt(Base): ondelete="RESTRICT", name="fk_artifact_put_attempts_namespace_fingerprint", ), + ForeignKeyConstraint( + ["task_id", "project_id"], + ["workstream_tasks.id", "workstream_tasks.project_id"], + ondelete="RESTRICT", + name="fk_artifact_put_attempts_task_project", + ), + ForeignKeyConstraint( + ["checker_run_id", "task_id", "submission_id"], + ["checker_runs.id", "checker_runs.task_id", "checker_runs.submission_id"], + ondelete="RESTRICT", + name="fk_artifact_put_attempts_checker_run_ownership", + ), + ForeignKeyConstraint( + ["submission_id", "task_id", "submission_version"], + ["submissions.id", "submissions.task_id", "submissions.version"], + ondelete="RESTRICT", + name="fk_artifact_put_attempts_submission_version", + ), UniqueConstraint("operation_identity", name="uq_artifact_put_attempt_operation"), CheckConstraint( "producer_request_type in ('guide', 'checker_output', 'submission_bundle')", @@ -1005,6 +1046,12 @@ class ArtifactPutAttempt(Base): SHA256_CHECK.format(column="request_digest"), name="request_digest_shape", ), + CheckConstraint( + "(producer_request_type = 'checker_output' and checker_request_digest is not null " + "and checker_request_digest ~ '^sha256:[0-9a-f]{64}$') or " + "(producer_request_type <> 'checker_output' and checker_request_digest is null)", + name="checker_request_digest", + ), CheckConstraint( "status in ('prepared', 'put_in_flight', 'acknowledgement_unknown', " "'object_confirmed', 'absent_replay_required', 'integrity_mismatch', " @@ -1046,14 +1093,17 @@ class ArtifactPutAttempt(Base): ), CheckConstraint( "(producer_request_type = 'guide' and guide_source_item_id is not null " - "and checker_run_id is null and task_id is null " + "and checker_run_id is null and task_id is null and submission_id is null " + "and submission_version is null " "and logical_role is null) or " "(producer_request_type = 'checker_output' and guide_source_item_id is null " "and checker_run_id is not null and task_id is not null " + "and submission_id is not null and submission_version is not null " "and octet_length(logical_role) between 1 and 100) or " "(producer_request_type = 'submission_bundle' " "and guide_source_item_id is null and checker_run_id is null " - "and task_id is not null and logical_role is null)", + "and task_id is not null and submission_id is null " + "and submission_version is null and logical_role is null)", name="producer_reference", ), ) @@ -1074,6 +1124,8 @@ class ArtifactPutAttempt(Base): checker_run_id: Mapped[str | None] = mapped_column( ForeignKey("checker_runs.id", ondelete="RESTRICT"), index=True ) + submission_id: Mapped[str | None] = mapped_column(Uuid(as_uuid=False), index=True) + submission_version: Mapped[int | None] = mapped_column(Integer) logical_role: Mapped[str | None] = mapped_column(String(100)) sha256: Mapped[str] = mapped_column(String(71), nullable=False) byte_count: Mapped[int] = mapped_column(BigInteger, nullable=False) @@ -1083,6 +1135,7 @@ class ArtifactPutAttempt(Base): canonical_target: Mapped[str] = mapped_column(String(1024), nullable=False) operation_identity: Mapped[str] = mapped_column(String(71), nullable=False) request_digest: Mapped[str] = mapped_column(String(71), nullable=False) + checker_request_digest: Mapped[str | None] = mapped_column(String(71)) status: Mapped[str] = mapped_column(String(40), nullable=False, default="prepared", index=True) next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) executor_id: Mapped[str | None] = mapped_column(Uuid(as_uuid=False)) diff --git a/backend/app/modules/artifacts/preparation.py b/backend/app/modules/artifacts/preparation.py index 99b465e71..620380b89 100644 --- a/backend/app/modules/artifacts/preparation.py +++ b/backend/app/modules/artifacts/preparation.py @@ -1481,6 +1481,7 @@ async def prepare( media_type: str, expected_sha256: str | None = None, expected_size: int | None = None, + maximum_bytes: int | None = None, ) -> PreparedArtifact: """Hash and count one complete source before any future provider call.""" self._validate_client_commitment( @@ -1488,6 +1489,16 @@ async def prepare( expected_sha256=expected_sha256, expected_size=expected_size, ) + if maximum_bytes is None: + source_maximum_bytes = self._manager.limits.maximum_source_bytes + else: + if ( + type(maximum_bytes) is not int + or maximum_bytes <= 0 + or maximum_bytes > self._manager.limits.maximum_source_bytes + ): + raise ValueError("artifact preparation per-call byte limit is invalid") + source_maximum_bytes = maximum_bytes loop = asyncio.get_running_loop() deadline = loop.time() + self._manager.limits.total_deadline_seconds reservation: _ScratchReservation | None = None @@ -1506,7 +1517,7 @@ async def prepare( for offset in range(0, len(view), self._manager.limits.stream_buffer_bytes): chunk = view[offset : offset + self._manager.limits.stream_buffer_bytes] byte_count += len(chunk) - if byte_count > self._manager.limits.maximum_source_bytes: + if byte_count > source_maximum_bytes: raise ArtifactLimitExceededError( "artifact source exceeds maximum bytes" ) diff --git a/backend/app/modules/artifacts/repository.py b/backend/app/modules/artifacts/repository.py index 5c369533b..aaafc0b16 100644 --- a/backend/app/modules/artifacts/repository.py +++ b/backend/app/modules/artifacts/repository.py @@ -30,7 +30,6 @@ SubmissionBundleDurableIntent, ) from app.modules.artifacts.schemas import VERIFICATION_PRODUCERS -from app.modules.checkers.models import CheckerRun from app.modules.projects.models import ( GuideSourceArtifactIngest, GuideSourceSnapshot, @@ -38,7 +37,6 @@ Project, ProjectGuide, ) -from app.modules.tasks.models import Submission, WorkstreamTask class GuideSourceIngestConflict(ValueError): @@ -72,15 +70,6 @@ class GuideLineageFacts: media_type: str | None -@dataclass(frozen=True, slots=True) -class CheckerOutputAdmissionFacts: - """Authoritative project/task ownership for one checker run.""" - - checker_run_id: str - project_id: str - task_id: str - - class ArtifactRepository: """Persist artifact state transitions under caller-owned transactions.""" @@ -95,12 +84,6 @@ async def database_now(self) -> datetime: raise RuntimeError("PostgreSQL clock did not return a timestamp") return value - async def lock_checker_run(self, checker_run_id: str) -> CheckerRun | None: - """Lock one checker run for canonical recovery resource derivation.""" - return await self._session.scalar( - select(CheckerRun).where(CheckerRun.id == checker_run_id).with_for_update() - ) - async def get_guide_admission_facts( self, guide_source_item_id: str ) -> GuideAdmissionFacts | None: @@ -237,32 +220,6 @@ async def get_guide_lineage(self, guide_source_item_id: str) -> GuideLineageFact media_type=lineage.media_type, ) - async def get_checker_output_admission_facts( - self, checker_run_id: str - ) -> CheckerOutputAdmissionFacts | None: - """Load canonical project/task ownership for one checker run.""" - row = ( - await self._session.execute( - select(CheckerRun.id, Submission.task_id, WorkstreamTask.project_id) - .join( - Submission, - (Submission.id == CheckerRun.submission_id) - & (Submission.version == CheckerRun.submission_version) - & (Submission.task_id == CheckerRun.task_id), - ) - .join(WorkstreamTask, WorkstreamTask.id == Submission.task_id) - .where(CheckerRun.id == checker_run_id) - .with_for_update(of=(CheckerRun, Submission, WorkstreamTask)) - ) - ).one_or_none() - if row is None: - return None - return CheckerOutputAdmissionFacts( - checker_run_id=row.id, - project_id=row.project_id, - task_id=row.task_id, - ) - async def ensure_and_lock_admission_scopes( self, scopes: Sequence[tuple[str, str, int]], diff --git a/backend/app/modules/artifacts/schemas.py b/backend/app/modules/artifacts/schemas.py index e173dee83..28bcb9acd 100644 --- a/backend/app/modules/artifacts/schemas.py +++ b/backend/app/modules/artifacts/schemas.py @@ -8,6 +8,10 @@ from uuid import UUID from app.modules.artifacts.sources import CommittedArtifactSource +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSlot, +) from app.modules.authorization.runtime import AuthorizationContext from app.modules.authorization.catalogue import ActionId from app.modules.actors.api import ServiceIdentity @@ -32,12 +36,39 @@ class GuideArtifactAdmissionRequest: class CheckerOutputArtifactAdmissionRequest: """One prepared checker output admitted under its exact checker run.""" - authorization_context: AuthorizationContext - checker_run_id: UUID - logical_role: str + reservation: CheckerOutputReservation + slot_key: str source: CommittedArtifactSource +def checker_output_request_digest_facts( + *, + reservation: CheckerOutputReservation, + slot: CheckerOutputSlot, +) -> dict[str, object]: + """Return stable owner facts shared by checker admission and binding.""" + evaluation = reservation.evaluation + return { + "evaluation_request_id": str(evaluation.evaluation_request_id), + "evaluation_request_sha256": evaluation.request_sha256, + "evaluation_generation": evaluation.evaluation_generation, + "checker_run_id": str(reservation.checker_run_id), + "project_id": str(evaluation.project_id), + "task_id": str(evaluation.task_id), + "submission_id": str(evaluation.submission_id), + "submission_version": evaluation.submission_version, + "slot_key": slot.key, + "media_type": slot.media_type, + "maximum_bytes": slot.maximum_bytes, + } + + +class CheckerOutputAdmissionAuthority(Protocol): + """Consume caller-prepared authority for one exact output reservation.""" + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: ... + + @final @dataclass(frozen=True, slots=True) class SubmissionBundleArtifactAdmissionRequest: diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index 41af3efaa..8b7d92cdd 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -92,23 +92,29 @@ ArtifactRecoveryIneligibleError, ArtifactRecoveryNotFoundError, DenyArtifactInternalAuthority, + CheckerOutputAdmissionAuthority, CheckerOutputArtifactAdmissionRequest, GuideArtifactAdmissionRequest, GuideArtifactIngestAuthorityFacts, SubmissionBundleArtifactAdmissionRequest, SubmissionBundleDurableIntentAuthorityFacts, + checker_output_request_digest_facts, ) from app.modules.artifacts.submission_authorization import ( SubmissionBundlePreparedAuthorization, ) from app.modules.artifacts.sources import CommittedArtifactSource, PreparedArtifact +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSelector, + CheckerOutputUnavailable, +) from app.modules.authorization.runtime import ( ActorKind, ActorStatus, AuthorizationContext, HumanAuthorizationContext, IdentityLinkStatus, - ServiceAuthorizationContext, ) from app.modules.authorization.prepared import PreparedAuthorizationHandle from app.modules.authorization.catalogue import ActionId, PermissionId @@ -181,6 +187,10 @@ class _AdmissionFacts: guide_source_snapshot_id: str | None checker_run_id: str | None logical_role: str | None + submission_id: str | None + submission_version: int | None + checker_request_digest: str | None + checker_request_digest_facts: dict[str, object] | None pre_submit_evidence_set_id: str | None operation_identity: str @@ -1602,12 +1612,7 @@ async def _create_locked( if self._is_exact_replay(existing, request, digest): return self._result(existing, replayed=True) raise ArtifactRecoveryConflictError("artifact recovery source is already owned") - checker_run = ( - await self._repo.lock_checker_run(put_attempt.checker_run_id) - if put_attempt.checker_run_id is not None - else None - ) - canonical_submission_id = checker_run.submission_id if checker_run is not None else None + canonical_submission_id = put_attempt.submission_id canonical_project_id = UUID(put_attempt.project_id) canonical_task_id = UUID(put_attempt.task_id) canonical_submission_uuid = ( @@ -1906,16 +1911,27 @@ async def admit( submission_prepared_authorization: SubmissionBundlePreparedAuthorization | None = None, submission_task_contexts: TaskSubmissionContextPort | None = None, submission_project_contexts: ProjectLockedPolicyContextPort | None = None, + checker_output_authority: CheckerOutputAdmissionAuthority | None = None, prepared_authorization: PreparedAuthorizationHandle | None = None, existing_transaction: bool = False, ) -> ArtifactAdmissionResult: """Reserve every derived scope and persist one prepared attempt atomically.""" + if ( + type(request) is CheckerOutputArtifactAdmissionRequest + and checker_output_authority is None + ): + raise ArtifactAuthorityDeniedError( + "checker output admission authority is unavailable" + ) self._validate_request_boundary(request) commitment = request.source.commitment async with _artifact_admission_transaction( self._session, existing=existing_transaction, ): + if type(request) is CheckerOutputArtifactAdmissionRequest: + assert checker_output_authority is not None + await checker_output_authority.consume(request) if type(request) is GuideArtifactAdmissionRequest: if ( guide_prepared_authorization is None @@ -2040,6 +2056,9 @@ async def admit( "guide_source_item_id": facts.guide_source_item_id, "checker_run_id": facts.checker_run_id, "logical_role": facts.logical_role, + "submission_id": facts.submission_id, + "submission_version": facts.submission_version, + "checker_output_request": facts.checker_request_digest_facts, "pre_submit_evidence_set_id": facts.pre_submit_evidence_set_id, "sha256": commitment.sha256, "byte_count": commitment.byte_count, @@ -2103,6 +2122,9 @@ async def admit( guide_source_item_id=facts.guide_source_item_id, checker_run_id=facts.checker_run_id, logical_role=facts.logical_role, + submission_id=facts.submission_id, + submission_version=facts.submission_version, + checker_request_digest=facts.checker_request_digest, sha256=commitment.sha256, byte_count=commitment.byte_count, media_type=commitment.media_type, @@ -2149,7 +2171,22 @@ def _validate_request_boundary(request: ArtifactAdmissionRequest) -> None: if type(request.source) is not CommittedArtifactSource: raise TypeError("invalid artifact admission source") if type(request) is CheckerOutputArtifactAdmissionRequest: - ArtifactAdmissionService._validate_logical_role(request.logical_role) + if type(request.reservation) is not CheckerOutputReservation: + raise TypeError("invalid checker output reservation") + try: + request.reservation.select( + CheckerOutputSelector( + evaluation=request.reservation.evaluation, + checker_run_id=request.reservation.checker_run_id, + worker_lease_id=request.reservation.worker_lease_id, + worker_lease_generation=request.reservation.worker_lease_generation, + slot_key=request.slot_key, + ) + ) + except (CheckerOutputUnavailable, ValueError) as exc: + raise ArtifactAdmissionRelationshipError( + "checker output reservation is unavailable" + ) from exc if type(request) is GuideArtifactAdmissionRequest: lineage_claims = ( request.project_id, @@ -2163,14 +2200,7 @@ def _validate_request_boundary(request: ArtifactAdmissionRequest) -> None: return if type(request) is SubmissionBundleArtifactAdmissionRequest: return - context = request.authorization_context - if type(context) not in {HumanAuthorizationContext, ServiceAuthorizationContext}: - raise TypeError("invalid artifact admission authorization context") - if ( - context.actor_status is not ActorStatus.ACTIVE - or context.identity_link_status is not IdentityLinkStatus.ACTIVE - ): - raise ArtifactAdmissionRelationshipError("artifact admission actor is not active") + return async def _derive_admission_facts(self, request: ArtifactAdmissionRequest) -> _AdmissionFacts: """Load every product and producer relationship from authoritative rows.""" @@ -2208,6 +2238,10 @@ async def _guide_facts(self, request: GuideArtifactAdmissionRequest) -> _Admissi guide_source_snapshot_id=row.guide_source_snapshot_id, checker_run_id=None, logical_role=None, + submission_id=None, + submission_version=None, + checker_request_digest=None, + checker_request_digest_facts=None, pre_submit_evidence_set_id=None, operation_identity=operation_identity, ) @@ -2215,33 +2249,33 @@ async def _guide_facts(self, request: GuideArtifactAdmissionRequest) -> _Admissi async def _checker_output_facts( self, request: CheckerOutputArtifactAdmissionRequest ) -> _AdmissionFacts: - """Bind committed bytes to one run and fixed checker service actor.""" - context = request.authorization_context - if context.actor_kind is not ActorKind.SERVICE: - raise ArtifactAdmissionRelationshipError( - "checker output producer must be a service actor" + """Bind committed bytes to one owner-issued run slot and fixed service.""" + reservation = CheckerOutputReservation.model_validate(request.reservation) + try: + slot = reservation.select( + CheckerOutputSelector( + evaluation=reservation.evaluation, + checker_run_id=reservation.checker_run_id, + worker_lease_id=reservation.worker_lease_id, + worker_lease_generation=reservation.worker_lease_generation, + slot_key=request.slot_key, + ) ) - logical_role = request.logical_role - service_actor = await self._actors.lock_admission_proof( - context.actor_profile_id, - context.identity_link_id, - ) + except (CheckerOutputUnavailable, ValueError) as exc: + raise ArtifactAdmissionRelationshipError( + "checker output reservation is unavailable" + ) from exc + commitment = request.source.commitment if ( - service_actor is None - or service_actor.actor_kind != "service" - or service_actor.actor_status != "active" - or service_actor.identity_link_id != str(context.identity_link_id) - or service_actor.identity_link_subject_kind != "service" - or service_actor.identity_link_status != "active" - or service_actor.service_identity != ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + commitment.media_type != slot.media_type + or commitment.byte_count > slot.maximum_bytes ): raise ArtifactAdmissionRelationshipError( - "checker output service identity is unavailable" + "checker output does not match reserved slot" ) - checker_run_id = str(request.checker_run_id) - row = await self._repo.get_checker_output_admission_facts(checker_run_id) - if row is None: - raise ArtifactAdmissionRelationshipError("checker run relationship is unavailable") + evaluation = reservation.evaluation + checker_run_id = str(reservation.checker_run_id) + logical_role = slot.key operation_identity = canonical_json_hash( { "request_type": "checker_output", @@ -2249,17 +2283,25 @@ async def _checker_output_facts( "logical_role": logical_role, } ) + digest_facts = checker_output_request_digest_facts( + reservation=reservation, + slot=slot, + ) return _AdmissionFacts( request_type="checker_output", producer_type="service_identity", producer_ref=ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, - project_id=row.project_id, + project_id=str(evaluation.project_id), guide_id=None, - task_id=row.task_id, + task_id=str(evaluation.task_id), guide_source_item_id=None, guide_source_snapshot_id=None, checker_run_id=checker_run_id, logical_role=logical_role, + submission_id=str(evaluation.submission_id), + submission_version=evaluation.submission_version, + checker_request_digest=canonical_json_hash(digest_facts), + checker_request_digest_facts=digest_facts, pre_submit_evidence_set_id=None, operation_identity=operation_identity, ) @@ -2519,6 +2561,10 @@ async def _submission_bundle_facts( guide_source_snapshot_id=evidence.source_snapshot_id, checker_run_id=None, logical_role=None, + submission_id=None, + submission_version=None, + checker_request_digest=None, + checker_request_digest_facts=None, pre_submit_evidence_set_id=evidence.id, operation_identity=operation_identity, ) @@ -2542,20 +2588,6 @@ async def _require_active_human_actor(self, context: AuthorizationContext) -> No "artifact admission human identity is unavailable" ) - @staticmethod - def _validate_logical_role(value: str) -> str: - """Require one bounded printable checker-output role.""" - if ( - not isinstance(value, str) - or value != value.strip() - or not value - or not value.isascii() - or len(value) > 100 - or any(ord(character) < 32 or ord(character) == 127 for character in value) - ): - raise ArtifactAdmissionRelationshipError("checker output logical role is invalid") - return value - def _derive_scopes(self, facts: _AdmissionFacts) -> tuple[_AdmissionScopeSpec, ...]: """Derive the complete closed scope set without caller participation.""" limits = self._configured_limits() diff --git a/backend/app/modules/checkers/api/output_custody.py b/backend/app/modules/checkers/api/output_custody.py new file mode 100644 index 000000000..4f65cdbda --- /dev/null +++ b/backend/app/modules/checkers/api/output_custody.py @@ -0,0 +1,89 @@ +"""CHECKERS-owned output reservation facts; production reservations are unavailable.""" + +from typing import Annotated, Literal, Protocol, Self + +from pydantic import Field, StrictInt, model_validator + +from app.modules.checkers.api.post_submit import PostSubmissionEvaluationRequest +from app.modules.checkers.api.post_submit_catalogue import ( + Identifier, + PostSubmitValue, + ResourceId, + VersionNumber, +) + + +class CheckerOutputUnavailable(RuntimeError): + """Conceal missing, stale or unauthorized checker output custody.""" + + +class CheckerOutputSelector(PostSubmitValue): + """Select exact immutable evaluation and the caller's claimed execution lease.""" + + evaluation: PostSubmissionEvaluationRequest + checker_run_id: ResourceId + worker_lease_id: ResourceId + worker_lease_generation: VersionNumber + slot_key: Identifier + + +class CheckerOutputSlot(PostSubmitValue): + """One globally unique run slot issued by CHECKERS, not by a contributor.""" + + key: Identifier + media_type: Literal["application/json", "application/octet-stream", "text/plain"] + maximum_bytes: Annotated[StrictInt, Field(ge=1, le=536_870_912)] + + +class CheckerOutputReservation(PostSubmitValue): + """Detached owner facts, never a substitute for fresh action authority.""" + + evaluation: PostSubmissionEvaluationRequest + checker_run_id: ResourceId + worker_lease_id: ResourceId + worker_lease_generation: VersionNumber + slots: tuple[CheckerOutputSlot, ...] = Field(max_length=64) + + @model_validator(mode="after") + def unique_slots(self) -> Self: + """Keep generic ART binding's run/role scope unambiguous.""" + if len({slot.key for slot in self.slots}) != len(self.slots): + raise ValueError("checker output slots repeat") + return self + + def select(self, selector: CheckerOutputSelector) -> CheckerOutputSlot: + """Match the caller's claimed lease as well as immutable evaluation facts.""" + current = CheckerOutputReservation.model_validate(self) + selector = CheckerOutputSelector.model_validate(selector) + if ( + current.evaluation, + current.checker_run_id, + current.worker_lease_id, + current.worker_lease_generation, + ) != ( + selector.evaluation, + selector.checker_run_id, + selector.worker_lease_id, + selector.worker_lease_generation, + ): + raise CheckerOutputUnavailable("checker_output_reservation_unavailable") + for slot in current.slots: + if slot.key == selector.slot_key: + return slot + raise CheckerOutputUnavailable("checker_output_slot_unavailable") + + +class CheckerOutputReservationPort(Protocol): + """Resolve current owner custody before ART locks; CHECKERS owns lease fencing.""" + + async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReservation: + """Return exact current facts under the caller's CHECKERS transaction locks.""" + + +class UnavailableCheckerOutputReservation: + """ARCH-04C must install the real reservation producer before activation.""" + + async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReservation: + """Deny without reading protected facts or fabricating a reservation.""" + del selector + raise CheckerOutputUnavailable("checker_output_reservation_unavailable") diff --git a/backend/scripts/behavior_ownership.py b/backend/scripts/behavior_ownership.py index 5d537f926..d86699f8d 100644 --- a/backend/scripts/behavior_ownership.py +++ b/backend/scripts/behavior_ownership.py @@ -146,6 +146,12 @@ "backend/app/modules/authorization/api/outbox_dispatch.py", } ) +ARCH_04B2_OUTPUT_TARGETS = frozenset({ + "backend/app/modules/artifacts/checker_outputs.py", + "backend/app/modules/artifacts/checker_output_custody.py", + "backend/app/modules/artifacts/checker_output_bindings.py", + "backend/app/modules/checkers/api/output_custody.py", +}) ARCH_04B_MATERIALIZATION_TARGETS = frozenset({ "backend/app/modules/artifacts/api/submission_materialization.py", "backend/app/modules/artifacts/post_submit_materialization.py", @@ -705,6 +711,7 @@ def _validate_additive_partition_transition( | ARCH_CP03B_ADAPTER_BINDING_AUTH_TARGETS | ARCH_CP04A_CONTRIBUTION_POLICY_TARGETS | ARCH_CP04B_CONTRIBUTION_POLICY_TARGETS + | ARCH_04B2_OUTPUT_TARGETS | ARCH_04B_MATERIALIZATION_TARGETS | ARCH_04A_POST_SUBMIT_TARGETS | ARCH_CP05_POLICY_AUTH_TARGETS diff --git a/backend/scripts/test_lane_catalogue.py b/backend/scripts/test_lane_catalogue.py index db9e5c97f..70781575e 100644 --- a/backend/scripts/test_lane_catalogue.py +++ b/backend/scripts/test_lane_catalogue.py @@ -421,6 +421,8 @@ class TestLane: "tests/test_approved_guide_intake.py", "tests/test_post_submit_materialization.py", "tests/test_post_submit_selection.py", + "tests/test_checker_output_custody.py", + "tests/test_checker_output_storage.py", "tests/test_pre_submit_attempt_recovery.py", "tests/test_pre_submit_attempt_contracts.py", "tests/test_pre_submit_attempt_authority_integration.py", diff --git a/backend/tests/checker_output_admission_helpers.py b/backend/tests/checker_output_admission_helpers.py new file mode 100644 index 000000000..4e1deb985 --- /dev/null +++ b/backend/tests/checker_output_admission_helpers.py @@ -0,0 +1,223 @@ +"""Controlled complete CHECKERS owner facts for ART output-custody tests.""" + +from uuid import UUID + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from app.core.identifiers import new_record_id +from app.modules.actors.api import ServiceIdentity +from app.modules.actors.service import ActorService +from app.modules.artifacts.schemas import ( + ArtifactAuthorityDeniedError, + CheckerOutputArtifactAdmissionRequest, +) +from app.modules.artifacts.models import ( + ArtifactAdmissionCharge, + ArtifactAdmissionScope, + ArtifactContent, + ArtifactOperationReceipt, + ArtifactPutAttempt, + ArtifactPutAttemptCharge, + ArtifactReplica, + ArtifactStorageNamespace, +) +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSlot, +) +from app.modules.checkers.models import CheckerRun +from app.modules.checkers.post_submit_contracts import make_post_submit_request +from app.modules.tasks.models import Submission, WorkstreamTask +from tests.checkers.post_submit.support import request as checker_evaluation_request + + +async def make_checker_output_reservation( + session, + checker_run_id: str | UUID, + *, + slot_key: str = "platform-review", + media_type: str = "application/octet-stream", + maximum_bytes: int = 1024, + worker_lease_id: UUID | None = None, + worker_lease_generation: int = 1, +) -> CheckerOutputReservation: + """Derive hash-valid public owner facts from one actual seeded run.""" + checker_run = await session.get(CheckerRun, str(checker_run_id)) + assert checker_run is not None + submission = await session.scalar( + select(Submission).where( + Submission.id == checker_run.submission_id, + Submission.version == checker_run.submission_version, + Submission.task_id == checker_run.task_id, + ) + ) + task = await session.get(WorkstreamTask, checker_run.task_id) + assert submission is not None and task is not None + seed = checker_evaluation_request(project_id=UUID(task.project_id)) + fields = seed.model_dump(exclude={"request_sha256"}) + fields.update( + task_id=UUID(task.id), + assignment_id=UUID(submission.task_assignment_id), + submission_id=UUID(submission.id), + submission_version=submission.version, + ) + evaluation = make_post_submit_request(**fields) + return CheckerOutputReservation( + evaluation=evaluation, + checker_run_id=UUID(checker_run.id), + worker_lease_id=worker_lease_id or new_record_id(), + worker_lease_generation=worker_lease_generation, + slots=( + CheckerOutputSlot( + key=slot_key, + media_type=media_type, + maximum_bytes=maximum_bytes, + ), + ), + ) + + +async def make_checker_output_admission_request( + session, + checker_run_id: str, + source, + *, + slot_key: str = "platform-review", + worker_lease_id: UUID | None = None, + worker_lease_generation: int = 1, +) -> CheckerOutputArtifactAdmissionRequest: + """Build detached owner facts without rolling back or expiring caller state.""" + factory = async_sessionmaker(session.bind, expire_on_commit=False) + async with factory() as reservation_session: + reservation = await make_checker_output_reservation( + reservation_session, + checker_run_id, + slot_key=slot_key, + media_type=source.commitment.media_type, + maximum_bytes=max(1, source.commitment.byte_count), + worker_lease_id=worker_lease_id, + worker_lease_generation=worker_lease_generation, + ) + return CheckerOutputArtifactAdmissionRequest( + reservation=reservation, + slot_key=slot_key, + source=source, + ) + + +_ADMISSION_BASELINE_MODELS = ( + ArtifactStorageNamespace, + ArtifactAdmissionScope, + ArtifactAdmissionCharge, + ArtifactPutAttempt, + ArtifactContent, + ArtifactReplica, + ArtifactOperationReceipt, +) + + +async def checker_admission_baseline(session) -> dict[type, int]: + """Snapshot every ART row family asserted by checker admission proofs.""" + return { + model: int(await session.scalar(select(func.count()).select_from(model)) or 0) + for model in _ADMISSION_BASELINE_MODELS + } + + +async def assert_checker_admission_unchanged(session, baseline: dict[type, int]) -> None: + """Prove a denied checker request created no admission-owned rows.""" + for model in ( + ArtifactStorageNamespace, + ArtifactAdmissionScope, + ArtifactAdmissionCharge, + ArtifactPutAttempt, + ): + assert await session.scalar(select(func.count()).select_from(model)) == baseline[model] + + +async def assert_exact_checker_admission( + session, + *, + result, + replay, + takeover, + request, + authority, + project_id: str, + task_id: str, + checker_run_id: str, + baseline: dict[type, int], +) -> None: + """Retain the complete fixed-service, quota and replay assertion set.""" + attempt = await session.get(ArtifactPutAttempt, str(result.attempt_id)) + scopes = (await session.scalars(select(ArtifactAdmissionScope).order_by( + ArtifactAdmissionScope.scope_type, ArtifactAdmissionScope.scope_id, + ))).all() + links = (await session.scalars(select(ArtifactPutAttemptCharge).where( + ArtifactPutAttemptCharge.attempt_id == str(result.attempt_id), + ))).all() + assert attempt is not None + assert replay.attempt_id == result.attempt_id + assert replay.charge_ids == result.charge_ids + assert takeover.attempt_id == result.attempt_id + assert takeover.replayed is True + assert attempt.status == "prepared" + assert attempt.producer_request_type == "checker_output" + assert attempt.producer_type == "service_identity" + assert attempt.producer_ref == ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + assert attempt.project_id == project_id + assert attempt.task_id == task_id + assert attempt.checker_run_id == checker_run_id + assert attempt.logical_role == "platform-review" + assert attempt.submission_id == str(request.reservation.evaluation.submission_id) + assert attempt.submission_version == request.reservation.evaluation.submission_version + assert attempt.checker_request_digest is not None + assert authority.calls == 3 + assert attempt.executor_id is None + assert attempt.lease_expires_at is None + assert attempt.next_run_at is None + assert attempt.execution_generation == 0 + assert {scope.scope_type for scope in scopes} == { + "deployment", "producer", "project", "task", + } + assert len(result.charge_ids) == 4 + assert len(links) == 4 + assert await session.scalar(select(func.count()).select_from(ArtifactPutAttempt)) == ( + baseline[ArtifactPutAttempt] + 1 + ) + assert await session.scalar(select(func.count()).select_from(ArtifactAdmissionCharge)) == ( + baseline[ArtifactAdmissionCharge] + 4 + ) + for model in (ArtifactContent, ArtifactReplica, ArtifactOperationReceipt): + assert await session.scalar(select(func.count()).select_from(model)) == baseline[model] + + +class ControlledCheckerOutputAdmissionAuthority: + """Hidden active fixed-service proof; never a production activation.""" + + def __init__(self, session, actor_id: UUID, identity_link_id: UUID) -> None: + self._session = session + self._actor_id = actor_id + self._identity_link_id = identity_link_id + self.calls = 0 + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + self.calls += 1 + proof = await ActorService(self._session).lock_admission_proof( + self._actor_id, + self._identity_link_id, + ) + if ( + proof is None + or proof.actor_kind != "service" + or proof.actor_status != "active" + or proof.identity_link_id != str(self._identity_link_id) + or proof.identity_link_subject_kind != "service" + or proof.identity_link_status != "active" + or proof.service_identity + != ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + ): + raise ArtifactAuthorityDeniedError( + "checker output service identity is unavailable" + ) diff --git a/backend/tests/checker_output_custody_helpers.py b/backend/tests/checker_output_custody_helpers.py new file mode 100644 index 000000000..c881a2092 --- /dev/null +++ b/backend/tests/checker_output_custody_helpers.py @@ -0,0 +1,403 @@ +"""Controlled owner facts and real storage harness for checker-output custody.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.adapters.artifacts import create_artifact_store_bootstrap +from app.core.identifiers import new_record_id +from app.interfaces.artifact_operations import CheckerOutputArtifactRequest +from app.interfaces.artifacts import ( + ArtifactByteRange, + ArtifactStoreNamespaceClaim, + ArtifactStoreUnavailableError, +) +from app.modules.actors.api import ServiceIdentity +from app.modules.actors.models import ActorIdentityLink, ActorProfile +from app.modules.artifacts.checker_output_bindings import ( + CheckerOutputBindingService, +) +from app.modules.artifacts.checker_output_custody import CheckerOutputStoredFacts +from app.modules.artifacts.checker_outputs import CheckerArtifactOutputService +from app.modules.artifacts.preparation import ( + ArtifactPreparationService, + ArtifactScratchManager, +) +from app.modules.artifacts.schemas import CheckerOutputArtifactAdmissionRequest +from app.modules.artifacts.service import artifact_storage_namespace_spec +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSelector, + CheckerOutputUnavailable, +) +from app.modules.checkers.models import CheckerRun +from tests.artifact_store_helpers import ( + artifact_admission_limit_settings, + artifact_preparation_limits, +) +from tests.checker_output_admission_helpers import ( + ControlledCheckerOutputAdmissionAuthority, + make_checker_output_reservation, +) +from tests.projects.unified_policy_fixtures import create_standalone_unified_policy +from tests.test_artifact_admission import ( + _AllowArtifactAuthority, + _add_checker_output_actor, + _seed_checker_output_relationships, + _settings, +) + + +def selector_for(reservation: CheckerOutputReservation) -> CheckerOutputSelector: + """Select the sole controlled output slot under the reservation's current lease.""" + assert len(reservation.slots) == 1 + return CheckerOutputSelector( + evaluation=reservation.evaluation, + checker_run_id=reservation.checker_run_id, + worker_lease_id=reservation.worker_lease_id, + worker_lease_generation=reservation.worker_lease_generation, + slot_key=reservation.slots[0].key, + ) + + +async def byte_stream(*chunks: bytes, observed: list[bytes] | None = None) -> AsyncIterator[bytes]: + """Yield explicit generated bytes and optionally record source iteration.""" + for chunk in chunks: + if observed is not None: + observed.append(chunk) + yield chunk + + +@dataclass(slots=True) +class ControlledReservationState: + """Mutable test control around one immutable owner-issued reservation.""" + + reservation: CheckerOutputReservation + resolved: list[CheckerOutputSelector] = field(default_factory=list) + revoked: bool = False + preflights: list[CheckerOutputSelector] = field(default_factory=list) + prepares: list[CheckerOutputSelector] = field(default_factory=list) + admissions: list[CheckerOutputArtifactAdmissionRequest] = field(default_factory=list) + recoveries: list[tuple[CheckerOutputReservation, CheckerOutputStoredFacts]] = field( + default_factory=list + ) + bindings: list[tuple[CheckerOutputReservation, CheckerOutputStoredFacts]] = field( + default_factory=list + ) + + def require_authority(self) -> None: + """Model one fresh action-authority revocation without product activation.""" + if self.revoked: + raise CheckerOutputUnavailable("checker_output_test_authority_revoked") + + +class ControlledReservationPort: + """Lock a real run, then return controlled detached CHECKERS owner facts.""" + + def __init__(self, session: AsyncSession, state: ControlledReservationState) -> None: + self._session = session + self._state = state + + async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReservation: + """Fence the selected real run before ART acquires any artifact row lock.""" + checker_run = await self._session.scalar( + select(CheckerRun) + .where(CheckerRun.id == str(selector.checker_run_id)) + .with_for_update() + .execution_options(populate_existing=True) + ) + if checker_run is None: + raise CheckerOutputUnavailable("checker_output_reservation_unavailable") + self._state.resolved.append(selector) + return self._state.reservation + + +class ControlledWriteAuthority: + """Test-only exact action authority plus real fixed-service admission proof.""" + + def __init__( + self, + session: AsyncSession, + state: ControlledReservationState, + actor_id: UUID, + identity_link_id: UUID, + ) -> None: + self._state = state + self._admission = ControlledCheckerOutputAdmissionAuthority( + session, + actor_id, + identity_link_id, + ) + + async def preflight(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.preflights.append(selector) + + async def prepare(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.prepares.append(selector) + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + self._state.require_authority() + assert request.reservation == self._state.reservation + await self._admission.consume(request) + self._state.admissions.append(request) + + async def consume_existing( + self, + reservation: CheckerOutputReservation, + facts: CheckerOutputStoredFacts, + ) -> None: + self._state.require_authority() + assert reservation == self._state.reservation + self._state.recoveries.append((reservation, facts)) + + def discard(self) -> None: + """The controlled seam retains no prepared capability.""" + + +class ControlledBindingAuthority: + """Test-only binding authority that records exact verified custody.""" + + def __init__(self, state: ControlledReservationState) -> None: + self._state = state + + async def preflight(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.preflights.append(selector) + + async def prepare(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.prepares.append(selector) + + async def consume( + self, + reservation: CheckerOutputReservation, + facts: CheckerOutputStoredFacts, + ) -> None: + self._state.require_authority() + assert reservation == self._state.reservation + self._state.bindings.append((reservation, facts)) + + def discard(self) -> None: + """The controlled seam retains no prepared capability.""" + + +class ObservedStore: + """Count provider calls and optionally pause one put at the provider boundary.""" + + def __init__(self, wrapped: Any) -> None: + self.wrapped = wrapped + self.identity = wrapped.identity + self.puts = 0 + self.opens = 0 + self.put_entered: asyncio.Event | None = None + self.put_release: asyncio.Event | None = None + self.lose_put_acknowledgement = False + + async def put(self, source): + self.puts += 1 + if self.put_entered is not None: + self.put_entered.set() + if self.put_release is not None: + await self.put_release.wait() + result = await self.wrapped.put(source) + if self.lose_put_acknowledgement: + raise ArtifactStoreUnavailableError("controlled lost provider acknowledgement") + return result + + async def observe_put_result(self, commitment): + return await self.wrapped.observe_put_result(commitment) + + def open( + self, + provider_object_ref: str, + byte_range: ArtifactByteRange | None = None, + ): + self.opens += 1 + return self.wrapped.open(provider_object_ref, byte_range) + + async def head(self, provider_object_ref: str): + return await self.wrapped.head(provider_object_ref) + + +@dataclass(slots=True) +class OutputCustodyHarness: + """One real database/provider path with controlled unavailable owner seams.""" + + factory: async_sessionmaker[AsyncSession] + engine: Any + bootstrap: Any + store: ObservedStore + namespace: Any + settings: Any + manager: ArtifactScratchManager + preparation: ArtifactPreparationService + service: CheckerArtifactOutputService + state: ControlledReservationState + actor_id: UUID + identity_link_id: UUID + policy_bundle: Any + + @property + def selector(self) -> CheckerOutputSelector: + return selector_for(self.state.reservation) + + def request( + self, + *chunks: bytes, + selector: CheckerOutputSelector | None = None, + observed: list[bytes] | None = None, + ) -> CheckerOutputArtifactRequest: + return CheckerOutputArtifactRequest( + selector=selector or self.selector, + byte_source=byte_stream(*chunks, observed=observed), + ) + + def binding_service(self, session: AsyncSession) -> CheckerOutputBindingService: + return CheckerOutputBindingService( + session, + reservations=ControlledReservationPort(session, self.state), + authority=ControlledBindingAuthority(self.state), + namespace_fingerprint=self.namespace.namespace_fingerprint, + ) + + async def seed_reservation( + self, + *, + maximum_bytes: int = 1024, + ) -> CheckerOutputReservation: + """Create a second coherent lineage without changing the active test reservation.""" + async with self.factory() as session: + _, _, checker_run_id = await _seed_checker_output_relationships( + session, + self.namespace, + policy_bundle=self.policy_bundle, + ) + reservation = await make_checker_output_reservation( + session, + checker_run_id, + maximum_bytes=maximum_bytes, + ) + await session.rollback() + return reservation + + +@asynccontextmanager +async def output_custody_harness( + tmp_path: Path, + database_url: str, + *, + provider: str = "local", + maximum_bytes: int = 1024, +) -> AsyncIterator[OutputCustodyHarness]: + """Compose real Local/MinIO storage and controlled exact output authority.""" + engine = create_async_engine(database_url) + factory = async_sessionmaker(engine, expire_on_commit=False) + if provider == "minio": + from tests.test_s3_artifact_store import minio_settings + + settings = minio_settings(private_prefix=f"checker-output/{new_record_id()}").model_copy( + update={ + **artifact_admission_limit_settings(1024), + "artifact_scratch_root": tmp_path / "configured-scratch", + "artifact_scratch_minimum_free_bytes": 0, + } + ) + else: + settings = _settings(tmp_path, maximum_bytes=1024) + bootstrap = create_artifact_store_bootstrap(settings) + namespace = artifact_storage_namespace_spec(settings, bootstrap) + raw_store = bootstrap.initialize_after_namespace_claim( + ArtifactStoreNamespaceClaim( + bootstrap.identity, + bootstrap.namespace_identity, + namespace.namespace_fingerprint, + ) + ) + store = ObservedStore(raw_store) + manager = ArtifactScratchManager( + root=tmp_path / "output-scratch", + limits=artifact_preparation_limits(), + ) + preparation = ArtifactPreparationService(manager) + try: + policy_bundle = await create_standalone_unified_policy(factory, namespace) + async with factory() as session: + _, _, checker_run_id = await _seed_checker_output_relationships( + session, + namespace, + policy_bundle=policy_bundle, + ) + existing = ( + await session.execute( + select(ActorProfile.id, ActorIdentityLink.id) + .join(ActorIdentityLink, ActorIdentityLink.actor_profile_id == ActorProfile.id) + .where( + ActorProfile.service_identity + == ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + ) + ) + ).one_or_none() + if existing is None: + actor_id, identity_link_id = new_record_id(), new_record_id() + _add_checker_output_actor( + session, + actor_id, + identity_link_id, + subject=f"checker-output-{actor_id}", + ) + await session.commit() + else: + actor_id, identity_link_id = (UUID(value) for value in existing) + await session.rollback() + reservation = await make_checker_output_reservation( + session, + checker_run_id, + maximum_bytes=maximum_bytes, + ) + await session.rollback() + state = ControlledReservationState(reservation) + service = CheckerArtifactOutputService( + sessions=factory, + reservations=lambda session: ControlledReservationPort(session, state), + authority=lambda session: ControlledWriteAuthority( + session, + state, + actor_id, + identity_link_id, + ), + internal_authority=lambda session: _AllowArtifactAuthority(), + preparation=preparation, + store=store, + namespace=namespace, + settings=settings, + ) + yield OutputCustodyHarness( + factory=factory, + engine=engine, + bootstrap=bootstrap, + store=store, + namespace=namespace, + settings=settings, + manager=manager, + preparation=preparation, + service=service, + state=state, + actor_id=actor_id, + identity_link_id=identity_link_id, + policy_bundle=policy_bundle, + ) + finally: + manager.close() + bootstrap.close() + await engine.dispose() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 3174546a3..5cfc28a9e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -23,7 +23,7 @@ DDL_LOCK_DIRECTORY = Path("/tmp") # Match the PostgreSQL 16 engine used by Backend CI. Catalog identity rendering # differs across major versions; regenerate only after comparing actual objects. -EXPECTED_PUBLIC_SCHEMA_SHA256 = "48ad2cc4307b52524c1343311866a7d7e39ff6c9471f37001ea84192ffb86767" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "f41e86b57c564169cec756f80407a0c8e7e0f63b9c8a87be4190b58d73e8ca89" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", @@ -119,6 +119,7 @@ "workstream_tasks", ) TRUNCATE_GUARDED_TABLES = ( + "artifact_put_attempts", "checker_runs", "checker_results", "task_command_receipts", diff --git a/backend/tests/test_alembic.py b/backend/tests/test_alembic.py index 039e80129..8319cf4b0 100644 --- a/backend/tests/test_alembic.py +++ b/backend/tests/test_alembic.py @@ -79,7 +79,7 @@ def test_v01_graph_has_one_root_and_head() -> None: script = ScriptDirectory.from_config(config) revisions = list(script.walk_revisions()) - assert [revision.revision for revision in revisions] == [HEAD_REVISION, "0005_task_evidence_authority", "0004_task_context_authority", "0003_task_read_authority", "0002_task_queue_authority", BASELINE_REVISION] + assert [revision.revision for revision in revisions] == [HEAD_REVISION, "0006_history_read_authority", "0005_task_evidence_authority", "0004_task_context_authority", "0003_task_read_authority", "0002_task_queue_authority", BASELINE_REVISION] assert revisions[-1].down_revision is None assert script.get_heads() == [HEAD_REVISION] diff --git a/backend/tests/test_artifact_admission.py b/backend/tests/test_artifact_admission.py index b40687ce1..e8c2bce74 100644 --- a/backend/tests/test_artifact_admission.py +++ b/backend/tests/test_artifact_admission.py @@ -62,7 +62,6 @@ from app.modules.artifacts.schemas import ( ArtifactAuthorityDeniedError, ArtifactInternalResourceType, - CheckerOutputArtifactAdmissionRequest, GuideArtifactAdmissionRequest, ) from app.modules.artifacts.service import ( @@ -101,6 +100,13 @@ artifact_admission_limit_settings, minted_source, ) +from tests.checker_output_admission_helpers import ( + ControlledCheckerOutputAdmissionAuthority, + assert_checker_admission_unchanged, + assert_exact_checker_admission, + checker_admission_baseline, + make_checker_output_admission_request as _checker_output_request, +) class _AllowArtifactAuthority: @@ -521,15 +527,13 @@ async def _admit_checker_output(session, settings, namespace, source, *, policy_ session, actor_id, link_id, subject=f"checker-output-{actor_id}") else: actor_id, link_id = (UUID(value) for value in existing) - context = _context(actor_profile_id=actor_id, identity_link_id=link_id, actor_kind=ActorKind.SERVICE) await session.commit() + request = await _checker_output_request(session, checker_run_id, source) result = await ArtifactAdmissionService(session, settings, namespace).admit( - CheckerOutputArtifactAdmissionRequest( - authorization_context=context, - checker_run_id=UUID(checker_run_id), - logical_role="platform-review", - source=source, - ) + request, + checker_output_authority=ControlledCheckerOutputAdmissionAuthority( + session, actor_id, link_id + ), ) return project_id, task_id, checker_run_id, result @@ -2416,11 +2420,6 @@ async def test_checker_output_requires_exact_active_fixed_service_identity( namespace = _namespace(settings) actor_id = new_record_id() link_id = new_record_id() - context = _context( - actor_profile_id=actor_id, - identity_link_id=link_id, - actor_kind=ActorKind.SERVICE, - ) engine = create_async_engine(admission_database_env) factory = async_sessionmaker(engine, expire_on_commit=False) try: @@ -2439,75 +2438,61 @@ async def test_checker_output_requires_exact_active_fixed_service_identity( await session.commit() await session.rollback() - baseline = {model: await _count(session, model) for model in ( - ArtifactPutAttempt, ArtifactContent, ArtifactReplica, ArtifactOperationReceipt, - )} + baseline = await checker_admission_baseline(session) await session.rollback() async with minted_source(tmp_path / "scratch-source", b"checker") as source: service = ArtifactAdmissionService(session, settings, namespace) - forged = context.model_copy(update={"identity_link_id": new_record_id()}) + request = await _checker_output_request( + session, checker_run_id, source + ) with pytest.raises( - ArtifactAdmissionRelationshipError, + ArtifactAuthorityDeniedError, + match="admission authority is unavailable", + ): + await service.admit(request) + await assert_checker_admission_unchanged(session, baseline) + await session.rollback() + + with pytest.raises( + ArtifactAuthorityDeniedError, match="service identity is unavailable", ): await service.admit( - CheckerOutputArtifactAdmissionRequest( - authorization_context=forged, - checker_run_id=UUID(checker_run_id), - logical_role="platform-review", - source=source, - ) + request, + checker_output_authority=ControlledCheckerOutputAdmissionAuthority( + session, actor_id, new_record_id() + ), ) - assert await _count(session, ArtifactStorageNamespace) == 1 - assert await _count(session, ArtifactAdmissionScope) == 0 - assert await _count(session, ArtifactAdmissionCharge) == 0 - assert await _count(session, ArtifactPutAttempt) == baseline[ArtifactPutAttempt] + await assert_checker_admission_unchanged(session, baseline) await session.rollback() - - request = CheckerOutputArtifactAdmissionRequest( - authorization_context=context, - checker_run_id=UUID(checker_run_id), - logical_role="platform-review", - source=source, + authority = ControlledCheckerOutputAdmissionAuthority( + session, actor_id, link_id + ) + result = await service.admit( + request, checker_output_authority=authority + ) + replay = await service.admit( + request, checker_output_authority=authority + ) + takeover_request = replace( + request, + reservation=request.reservation.model_copy( + update={ + "worker_lease_id": new_record_id(), + "worker_lease_generation": 2, + } + ), + ) + takeover = await service.admit( + takeover_request, + checker_output_authority=authority, ) - result = await service.admit(request) - replay = await service.admit(request) - attempt = await session.get(ArtifactPutAttempt, str(result.attempt_id)) - scopes = (await session.scalars(select(ArtifactAdmissionScope).order_by( - ArtifactAdmissionScope.scope_type, ArtifactAdmissionScope.scope_id, - ))).all() - links = (await session.scalars(select(ArtifactPutAttemptCharge).where( - ArtifactPutAttemptCharge.attempt_id == str(result.attempt_id), - ))).all() - assert attempt is not None - assert replay.attempt_id == result.attempt_id - assert replay.charge_ids == result.charge_ids - assert attempt.status == "prepared" - assert attempt.producer_request_type == "checker_output" - assert attempt.producer_type == "service_identity" - assert attempt.producer_ref == ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value - assert attempt.project_id == project_id - assert attempt.task_id == task_id - assert attempt.checker_run_id == checker_run_id - assert attempt.logical_role == "platform-review" - assert attempt.executor_id is None - assert attempt.lease_expires_at is None - assert attempt.next_run_at is None - assert attempt.execution_generation == 0 - assert {scope.scope_type for scope in scopes} == { - "deployment", - "producer", - "project", - "task", - } - assert len(result.charge_ids) == 4 - assert len(links) == 4 - assert await _count(session, ArtifactPutAttempt) == baseline[ArtifactPutAttempt] + 1 - assert await _count(session, ArtifactAdmissionCharge) == 4 - assert await _count(session, ArtifactContent) == baseline[ArtifactContent] - assert await _count(session, ArtifactReplica) == baseline[ArtifactReplica] - assert await _count(session, ArtifactOperationReceipt) == baseline[ArtifactOperationReceipt] + await assert_exact_checker_admission( + session, result=result, replay=replay, takeover=takeover, + request=request, authority=authority, project_id=project_id, + task_id=task_id, checker_run_id=checker_run_id, baseline=baseline, + ) finally: await engine.dispose() @@ -2703,7 +2688,7 @@ async def test_checker_output_put_observation_terminal_outcomes( await engine.dispose() -async def test_invalid_checker_role_precedes_namespace_drift( +async def test_invalid_checker_slot_precedes_namespace_drift( admission_database_env: str, tmp_path: Path, ) -> None: @@ -2713,34 +2698,32 @@ async def test_invalid_checker_role_precedes_namespace_drift( factory = async_sessionmaker(engine, expire_on_commit=False) try: async with factory() as session: - session.add( - ArtifactStorageNamespace( - id="primary", - backend=namespace.backend, - adapter=namespace.adapter, - provider_profile=namespace.provider_profile, - namespace_descriptor=namespace.namespace_descriptor, - namespace_fingerprint="sha256:" + "f" * 64, - ) + project_id, task_id, checker_run_id = await _seed_checker_output_relationships( + session, namespace + ) + del project_id, task_id + baseline = await checker_admission_baseline(session) + await session.rollback() + drifted_namespace = replace( + namespace, + namespace_fingerprint="sha256:" + "f" * 64, ) - await session.commit() async with minted_source(tmp_path / "scratch-source", b"checker") as source: + valid = await _checker_output_request( + session, checker_run_id, source + ) + invalid = replace(valid, slot_key="missing-slot") with pytest.raises( ArtifactAdmissionRelationshipError, - match="logical role is invalid", + match="reservation is unavailable", ): - await ArtifactAdmissionService(session, settings, namespace).admit( - CheckerOutputArtifactAdmissionRequest( - authorization_context=_context(actor_kind=ActorKind.SERVICE), - checker_run_id=new_record_id(), - logical_role="é" * 100, - source=source, - ) + await ArtifactAdmissionService( + session, settings, drifted_namespace + ).admit( + invalid, + checker_output_authority=AsyncMock(), ) - assert await _count(session, ArtifactStorageNamespace) == 1 - assert await _count(session, ArtifactAdmissionScope) == 0 - assert await _count(session, ArtifactAdmissionCharge) == 0 - assert await _count(session, ArtifactPutAttempt) == 0 + await assert_checker_admission_unchanged(session, baseline) finally: await engine.dispose() diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index d97128560..4f45b9a1e 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -40,15 +40,15 @@ } CANONICAL_RESULTS = { "GuideArtifactIngestResult", + "CheckerOutputArtifactResult", + "CheckerOutputBindingResult", } CANONICAL_TYPE_ALIASES = { "ArtifactAuditResourceType", "ArtifactBindingResourceType", } CANONICAL_VALUE_TYPES = set() -PREPARED_MUTATION_REQUESTS = CANONICAL_REQUESTS - { - "ArtifactRecoveryRequest", -} +PREPARED_MUTATION_REQUESTS = {"GuideArtifactIngestRequest"} PREPARED_HANDLE_FORBIDDEN_ROOTS = ( APP_ROOT / "adapters", APP_ROOT / "api", @@ -469,7 +469,7 @@ def test_artifact_operations_exports_only_canonical_closed_contracts() -> None: } -def test_durable_artifact_mutation_ports_require_process_local_prepared_authority() -> None: +def test_artifact_ports_keep_prepared_authority_at_the_transaction_boundary() -> None: tree = _tree(ARTIFACT_OPERATIONS) request_classes = { node.name: node @@ -490,6 +490,12 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit assert all("AuthorizationContext" not in types for types in fields.values()), name assert {"action_id", "resource_context", "facts"}.isdisjoint(fields), name + # Output services own fresh PREP per phase; a public handle cannot span their I/O. + for name in ("CheckerOutputArtifactRequest", "CheckerOutputBindingRequest"): + node = next(item for item in tree.body if isinstance(item, ast.ClassDef) and item.name == name) + annotations = _declared_annotation_names(node) + assert {"PreparedAuthorizationHandle", "AuthorizationContext"}.isdisjoint(annotations) + assert "CheckerOutputSelector" in annotations source = ARTIFACT_OPERATIONS.read_text(encoding="utf-8") assert "upload_session" not in source assert "ContributorArtifactUploadPort" not in source @@ -501,12 +507,13 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit "ArtifactBindingPort": { "bind_checker_output", }, - "CheckerArtifactOutputPort": {"store"}, + "CheckerArtifactOutputPort": {"store", "recover"}, } expected_request_by_method = { "ingest": "GuideArtifactIngestRequest", "bind_checker_output": "CheckerOutputBindingRequest", "store": "CheckerOutputArtifactRequest", + "recover": "CheckerOutputSelector", } protocols = { node.name: node @@ -524,7 +531,7 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue assert node.args.posonlyargs == [] - assert [argument.arg for argument in node.args.args] == ["self", "request"] + assert [argument.arg for argument in node.args.args] == ["self", "selector" if node.name == "recover" else "request"] assert node.args.kwonlyargs == [] assert node.args.vararg is None assert node.args.kwarg is None diff --git a/backend/tests/test_artifact_preparation.py b/backend/tests/test_artifact_preparation.py index d17c73767..2098c7a42 100644 --- a/backend/tests/test_artifact_preparation.py +++ b/backend/tests/test_artifact_preparation.py @@ -365,13 +365,46 @@ async def future_provider_call(_: CommittedArtifactSource) -> None: @pytest.mark.asyncio -async def test_preparation_rejects_oversize_and_non_byte_sources(tmp_path: Path) -> None: - """Enforce the source ceiling and byte-only stream contract.""" - limits = preparation_limits(maximum_source_bytes=4) +async def test_preparation_enforces_source_caps_and_byte_only_streams( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Enforce manager and per-call ceilings before excess bytes reach scratch.""" + limits = preparation_limits(maximum_source_bytes=8) manager = ArtifactScratchManager(root=tmp_path / "scratch", limits=limits) service = ArtifactPreparationService(manager) + prepared = await service.prepare( + byte_stream(b"ab", b"cd"), + media_type="text/plain", + maximum_bytes=4, + ) + assert prepared.commitment.byte_count == 4 + await prepared.close() with pytest.raises(ArtifactLimitExceededError): - await service.prepare(byte_stream(b"abcde"), media_type="text/plain") + await service.prepare(byte_stream(b"123456789"), media_type="text/plain") + + written = bytearray() + yielded: list[bytes] = [] + original_write_chunk = service._write_chunk + + async def track_write(descriptor: int, chunk: memoryview) -> None: + written.extend(chunk) + await original_write_chunk(descriptor, chunk) + + async def over_cap_stream() -> AsyncIterator[bytes]: + for chunk in (b"abcd", b"e", b"must-not-be-read"): + yielded.append(chunk) + yield chunk + + monkeypatch.setattr(service, "_write_chunk", track_write) + with pytest.raises(ArtifactLimitExceededError): + await service.prepare( + over_cap_stream(), + media_type="text/plain", + maximum_bytes=4, + ) + assert written == b"abcd" + assert yielded == [b"abcd", b"e"] async def invalid_stream() -> AsyncIterator[bytes]: yield "not-bytes" # type: ignore[misc] @@ -382,6 +415,36 @@ async def invalid_stream() -> AsyncIterator[bytes]: manager.close() +@pytest.mark.asyncio +@pytest.mark.parametrize("maximum_bytes", [True, 0, -1, 65]) +async def test_preparation_rejects_invalid_per_call_caps_before_iteration( + tmp_path: Path, + maximum_bytes: object, +) -> None: + """Reject malformed or manager-exceeding per-call limits before reserving scratch.""" + manager = ArtifactScratchManager( + root=tmp_path / str(maximum_bytes), + limits=preparation_limits(maximum_source_bytes=64), + ) + iterated = False + + async def tracked_stream() -> AsyncIterator[bytes]: + nonlocal iterated + iterated = True + yield b"data" + + with pytest.raises(ValueError, match="per-call byte limit"): + await ArtifactPreparationService(manager).prepare( + tracked_stream(), + media_type="text/plain", + maximum_bytes=maximum_bytes, # type: ignore[arg-type] + ) + + assert not iterated + assert (await manager.usage()).reservation_count == 0 + manager.close() + + @pytest.mark.asyncio async def test_cross_process_ledger_prevents_concurrent_oversubscription( tmp_path: Path, diff --git a/backend/tests/test_artifact_recovery.py b/backend/tests/test_artifact_recovery.py index 99ff22070..e701b3d2a 100644 --- a/backend/tests/test_artifact_recovery.py +++ b/backend/tests/test_artifact_recovery.py @@ -27,9 +27,11 @@ ArtifactStoreUnavailableError, ) from app.modules.artifacts.models import ( + ArtifactPutAttempt, ArtifactRecoveryAttempt, ArtifactVerificationJob, ) +from app.modules.artifacts.repository import ArtifactRepository from app.modules.artifacts.schemas import ( ArtifactRecoveryAuthorizationEvidence, ArtifactRecoveryConflictError, @@ -42,7 +44,6 @@ ArtifactStorageOrchestrator, artifact_storage_namespace_spec, ) -from app.modules.checkers.models import CheckerRun from app.modules.actors.models import ActorIdentityLink, ActorProfile from app.modules.authorization.runtime import ( ActorKind, @@ -186,7 +187,7 @@ async def _exhausted_job(session, settings, tmp_path, context): async with minted_source( tmp_path / "checker-output", b"recover checker output", limits=limits, ) as source: - project_id, task_id, checker_run_id, admission = await _admit_checker_output( + project_id, task_id, _checker_run_id, admission = await _admit_checker_output( session, settings, namespace, source, policy_bundle=policy_bundle) await _seed_recovery_actor(session, context) await session.commit() @@ -209,14 +210,19 @@ async def _exhausted_job(session, settings, tmp_path, context): await orchestrator.verify_object(UUID(job_id)) job = await session.get(ArtifactVerificationJob, job_id) assert job is not None - checker_run = await session.get(CheckerRun, checker_run_id) - assert checker_run is not None - submission_id = checker_run.submission_id + attempt = await session.get(ArtifactPutAttempt, str(admission.attempt_id)) + assert attempt is not None + submission_id = attempt.submission_id + assert submission_id is not None await session.refresh(job) await session.commit() return project_id, task_id, submission_id, job, orchestrator, bootstrap +def test_recovery_repository_has_no_checker_run_lookup() -> None: + assert not hasattr(ArtifactRepository, "lock_checker_run") + + def _request( context: HumanAuthorizationContext, project_id: str, diff --git a/backend/tests/test_behavior_ownership.py b/backend/tests/test_behavior_ownership.py index df94a0f42..dc04f5d69 100644 --- a/backend/tests/test_behavior_ownership.py +++ b/backend/tests/test_behavior_ownership.py @@ -2139,3 +2139,21 @@ def test_post_submit_materialization_partition_additions_are_exact(): ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets, forbidden])), trusted) with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): ownership._validate_additive_partition_transition(_partition(sorted(targets)), trusted) + + +def test_checker_output_partition_additions_are_exact(): + targets = { + "backend/app/modules/artifacts/checker_outputs.py", + "backend/app/modules/artifacts/checker_output_custody.py", + "backend/app/modules/artifacts/checker_output_bindings.py", + "backend/app/modules/checkers/api/output_custody.py", + } + assert ownership.ARCH_04B2_OUTPUT_TARGETS == targets + retained = "backend/app/core/config.py" + trusted = _partition([retained]) + ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets])), trusted) + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition( + _partition(sorted([retained, *targets, "backend/app/modules/artifacts/arbitrary_output.py"])), trusted) + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition(_partition(sorted(targets)), trusted) diff --git a/backend/tests/test_checker_output_custody.py b/backend/tests/test_checker_output_custody.py new file mode 100644 index 000000000..1f752c349 --- /dev/null +++ b/backend/tests/test_checker_output_custody.py @@ -0,0 +1,384 @@ +"""Hidden end-to-end proof for exact verified checker-output custody.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace +from uuid import UUID + +import pytest +from sqlalchemy import func, select + +from app.adapters.artifacts import checker_output_binding, checker_output_storage +from app.core.identifiers import new_record_id +from app.interfaces.artifact_operations import ( + CheckerOutputArtifactRequest, + CheckerOutputBindingRequest, +) +from app.interfaces.artifacts import ArtifactLimitExceededError +from app.modules.actors.api import ServiceIdentity +from app.modules.artifacts.models import ( + ArtifactBinding, + ArtifactOperationReceipt, + ArtifactPutObservationReceipt, + ArtifactPutAttempt, + ArtifactVerificationReceipt, +) +from app.modules.artifacts.service import ArtifactAdmissionConflictError +from app.modules.checkers.api.output_custody import ( + CheckerOutputSelector, + CheckerOutputUnavailable, +) +from app.modules.checkers.models import CheckerRun +from tests.checker_output_custody_helpers import ( + byte_stream, + output_custody_harness, + selector_for, +) +from tests.checkers.post_submit.support import request as post_submit_request + + +async def test_default_composition_denies_before_owner_source_or_storage() -> None: + """Keep both production output capabilities unavailable before protected access.""" + + class Forbidden: + def __getattr__(self, name): + pytest.fail(f"protected access: {name}") + + @asynccontextmanager + async def session_scope(): + yield object() + + iterated = False + + async def source(): + nonlocal iterated + iterated = True + yield b"must-not-be-read" + + selector = CheckerOutputSelector( + evaluation=post_submit_request(), + checker_run_id=new_record_id(), + worker_lease_id=new_record_id(), + worker_lease_generation=1, + slot_key="platform-review", + ) + forbidden = Forbidden() + storage = checker_output_storage( + sessions=session_scope, + store=forbidden, + namespace=forbidden, + preparation=forbidden, + settings=forbidden, + ) + with pytest.raises(CheckerOutputUnavailable, match="write_unavailable"): + await storage.store(CheckerOutputArtifactRequest(selector, source())) + assert not iterated + + class Transaction: + def in_transaction(self): + return True + + def in_nested_transaction(self): + return False + + binding = checker_output_binding( + Transaction(), + namespace=SimpleNamespace(namespace_fingerprint="sha256:" + "0" * 64), + ) + with pytest.raises(CheckerOutputUnavailable, match="binding_unavailable"): + await binding.bind_checker_output( + CheckerOutputBindingRequest(selector, new_record_id(), new_record_id()) + ) + + +@pytest.mark.parametrize("provider", ["local", "minio"]) +async def test_real_store_verification_and_binding( + tmp_path, + isolated_database_env, + provider, +) -> None: + """Store, independently verify and bind one exact output through real providers.""" + if provider == "minio": + from tests.test_s3_artifact_store import provision_minio_bucket + + await provision_minio_bucket.__wrapped__() + async with output_custody_harness( + tmp_path, + isolated_database_env, + provider=provider, + ) as harness: + result = await harness.service.store(harness.request(b"verified checker output")) + assert result.status == "verified" + assert result.content_id is not None + assert result.verification_receipt_id is not None + assert not result.replayed + assert harness.store.puts == 1 + assert harness.store.opens == 1 + + async with harness.factory() as session: + async with session.begin(): + bound = await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + harness.selector, + result.put_attempt_id, + result.verification_receipt_id, + ) + ) + row = await session.get(ArtifactBinding, str(bound.binding_id)) + assert row is not None + assert ( + row.resource_type, + row.resource_id, + row.logical_role, + row.actor_id, + row.attribution_type, + row.put_attempt_id, + row.verification_receipt_id, + ) == ( + "checker_run", + str(harness.selector.checker_run_id), + harness.selector.slot_key, + ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + "service_identity", + str(result.put_attempt_id), + str(result.verification_receipt_id), + ) + assert harness.state.admissions + assert harness.state.recoveries + assert harness.state.bindings + + +async def test_foreign_lineage_and_changed_bytes_fail_before_provider( + tmp_path, + isolated_database_env, +) -> None: + """Reject coherent foreign ownership before bytes and immutable replay drift before put.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + original_reservation = harness.state.reservation + original_selector = harness.selector + foreign = await harness.seed_reservation() + foreign_selector = selector_for(foreign) + + original = await harness.service.store( + harness.request(b"stable output", selector=original_selector) + ) + assert original.status == "verified" + harness.state.reservation = foreign + foreign_result = await harness.service.store( + harness.request(b"foreign control", selector=foreign_selector) + ) + assert foreign_result.status == "verified" + assert foreign_result.verification_receipt_id is not None + assert harness.store.puts == 2 + + harness.state.reservation = original_reservation + for wrong_selector in ( + foreign_selector, + original_selector.model_copy(update={"evaluation": foreign.evaluation}), + ): + observed: list[bytes] = [] + with pytest.raises(CheckerOutputUnavailable, match="reservation_unavailable"): + await harness.service.store( + harness.request( + b"mismatched bytes", + selector=wrong_selector, + observed=observed, + ) + ) + assert observed == [] + assert harness.store.puts == 2 + assert (await harness.manager.usage()).reservation_count == 0 + + with pytest.raises(ArtifactAdmissionConflictError): + await harness.service.store( + harness.request(b"changed output", selector=original_selector) + ) + assert harness.store.puts == 2 + assert (await harness.manager.usage()).reservation_count == 0 + + +async def test_binding_rollback_and_concurrent_replay_are_atomic( + tmp_path, + isolated_database_env, +) -> None: + """Keep caller rollback empty and serialize concurrent binding to one row.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + stored = await harness.service.store(harness.request(b"binding output")) + assert stored.verification_receipt_id is not None + request = CheckerOutputBindingRequest( + harness.selector, + stored.put_attempt_id, + stored.verification_receipt_id, + ) + + async with harness.factory() as session: + transaction = await session.begin() + rolled_back = await harness.binding_service(session).bind_checker_output(request) + await transaction.rollback() + async with harness.factory() as session: + assert await session.get(ArtifactBinding, str(rolled_back.binding_id)) is None + + async def bind_once(): + async with harness.factory() as session: + async with session.begin(): + return await harness.binding_service(session).bind_checker_output(request) + + first, second = await asyncio.gather(bind_once(), bind_once()) + assert first.binding_id == second.binding_id + assert sorted((first.replayed, second.replayed)) == [False, True] + async with harness.factory() as session: + count = await session.scalar(select(func.count()).select_from(ArtifactBinding)) + assert count == 1 + + +async def test_unknown_put_is_observed_recovered_and_bound_without_bytes( + tmp_path, + isolated_database_env, +) -> None: + """Recover lost acknowledgement through typed observation evidence and bind it.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + harness.store.lose_put_acknowledgement = True + unknown = await harness.service.store(harness.request(b"observed output")) + assert unknown.status == "acknowledgement_unknown" + assert unknown.content_id is None + assert unknown.verification_receipt_id is None + assert harness.store.puts == 1 + + recovered = await harness.service.recover(harness.selector) + assert recovered is not None + assert recovered.put_attempt_id == unknown.put_attempt_id + assert recovered.status == "verified" + assert recovered.content_id is not None + assert recovered.verification_receipt_id is not None + assert recovered.replayed + assert harness.store.puts == 1 + + async with harness.factory() as session: + attempt = await session.get(ArtifactPutAttempt, str(recovered.put_attempt_id)) + observation = await session.scalar( + select(ArtifactPutObservationReceipt).where( + ArtifactPutObservationReceipt.put_attempt_id == str(recovered.put_attempt_id) + ) + ) + direct_receipt = await session.scalar( + select(ArtifactOperationReceipt).where( + ArtifactOperationReceipt.put_attempt_id == str(recovered.put_attempt_id) + ) + ) + assert attempt is not None and attempt.receipt_id is None + assert observation is not None and observation.outcome == "observed_confirmed" + assert direct_receipt is None + await session.rollback() + async with session.begin(): + bound = await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + harness.selector, + recovered.put_attempt_id, + recovered.verification_receipt_id, + ) + ) + assert bound.content_id == recovered.content_id + + +async def test_worker_takeover_recovers_same_attempt_without_bytes( + tmp_path, + isolated_database_env, +) -> None: + """Fence the old lease and recover exact stored custody under its replacement.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + old_selector = harness.selector + stored = await harness.service.store(harness.request(b"takeover output")) + puts = harness.store.puts + harness.state.reservation = harness.state.reservation.model_copy( + update={ + "worker_lease_id": new_record_id(), + "worker_lease_generation": old_selector.worker_lease_generation + 1, + } + ) + + with pytest.raises(CheckerOutputUnavailable, match="reservation_unavailable"): + await harness.service.recover(old_selector) + recovered = await harness.service.recover(harness.selector) + assert recovered is not None + assert recovered.put_attempt_id == stored.put_attempt_id + assert recovered.verification_receipt_id == stored.verification_receipt_id + assert recovered.replayed + assert harness.store.puts == puts + + +async def test_authority_revocation_during_put_prevents_return_and_binding( + tmp_path, + isolated_database_env, +) -> None: + """Recheck authority after provider I/O and deny publication after revocation.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + harness.store.put_entered = asyncio.Event() + harness.store.put_release = asyncio.Event() + operation = asyncio.create_task(harness.service.store(harness.request(b"revoked output"))) + await asyncio.wait_for(harness.store.put_entered.wait(), timeout=10) + async with harness.factory() as probe, probe.begin(): + locked_run = await probe.scalar( + select(CheckerRun) + .where(CheckerRun.id == str(harness.selector.checker_run_id)) + .with_for_update(nowait=True) + ) + locked_attempt = await probe.scalar( + select(ArtifactPutAttempt) + .where(ArtifactPutAttempt.checker_run_id == str(harness.selector.checker_run_id)) + .with_for_update(nowait=True) + ) + assert locked_run is not None and locked_attempt is not None + harness.state.revoked = True + harness.store.put_release.set() + with pytest.raises(CheckerOutputUnavailable, match="authority_revoked"): + await operation + + async with harness.factory() as session: + attempt = await session.scalar(select(ArtifactPutAttempt)) + receipt = await session.scalar(select(ArtifactVerificationReceipt)) + assert attempt is not None and receipt is not None + attempt_id, receipt_id = UUID(attempt.id), UUID(receipt.id) + await session.rollback() + with pytest.raises(CheckerOutputUnavailable, match="authority_revoked"): + async with session.begin(): + await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + harness.selector, + attempt_id, + receipt_id, + ) + ) + binding_count = await session.scalar(select(func.count()).select_from(ArtifactBinding)) + assert binding_count == 0 + + +async def test_per_slot_cap_stops_source_and_cleans_scratch( + tmp_path, + isolated_database_env, +) -> None: + """Apply the owner-issued cap before excess bytes reach durable intent or provider.""" + async with output_custody_harness( + tmp_path, + isolated_database_env, + maximum_bytes=4, + ) as harness: + observed: list[bytes] = [] + request = CheckerOutputArtifactRequest( + harness.selector, + byte_stream(b"abcd", b"e", b"must-not-be-read", observed=observed), + ) + with pytest.raises(ArtifactLimitExceededError): + await harness.service.store(request) + assert observed == [b"abcd", b"e"] + assert harness.store.puts == 0 + assert (await harness.manager.usage()).reservation_count == 0 + assert list((tmp_path / "output-scratch" / "files").iterdir()) == [] + async with harness.factory() as session: + attempts = await session.scalar( + select(func.count()) + .select_from(ArtifactPutAttempt) + .where(ArtifactPutAttempt.producer_request_type == "checker_output") + ) + assert attempts == 0 diff --git a/backend/tests/test_checker_output_storage.py b/backend/tests/test_checker_output_storage.py new file mode 100644 index 000000000..f40b46332 --- /dev/null +++ b/backend/tests/test_checker_output_storage.py @@ -0,0 +1,473 @@ +"""Direct PostgreSQL proofs for exact checker output intent and binding custody.""" + +from __future__ import annotations + +from contextlib import asynccontextmanager +from pathlib import Path +import runpy +from types import SimpleNamespace +from uuid import UUID + +from alembic.migration import MigrationContext +from alembic.operations import Operations +import pytest +from sqlalchemy import select, text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.identifiers import new_record_id +from app.modules.actors.api import ServiceIdentity +from app.modules.artifacts.models import ( + ArtifactPutAttempt, + ArtifactVerificationJob, + ArtifactVerificationReceipt, +) +from app.modules.artifacts.service import ArtifactStorageOrchestrator +from projects.unified_policy_fixtures import create_standalone_unified_policy +from tests.artifact_store_helpers import minted_source +from tests.test_artifact_admission import ( + _AllowArtifactAuthority, + _admit_checker_output, + _local_store, + _namespace, + _seed_checker_output_relationships, + _settings, +) + + +async def _store_verified_output( + *, + factory, + settings, + namespace, + store, + policy_bundle, + source_path: Path, + payload: bytes, +): + """Store and independently verify one output using shared test infrastructure.""" + async with factory() as session: + async with minted_source(source_path, payload, media_type="text/plain") as source: + project_id, task_id, checker_run_id, admission = await _admit_checker_output( + session, + settings, + namespace, + source, + policy_bundle=policy_bundle, + ) + orchestrator = ArtifactStorageOrchestrator( + session, + store, + namespace, + settings, + _AllowArtifactAuthority(), + ) + await orchestrator.ensure_storage_namespace() + assert await orchestrator.execute_committed_put( + attempt_id=admission.attempt_id, + source=source, + ) == "stored_pending_verification" + job_id = await session.scalar( + select(ArtifactVerificationJob.id).where( + ArtifactVerificationJob.originating_put_attempt_id + == str(admission.attempt_id) + ) + ) + assert job_id is not None + await session.rollback() + assert await orchestrator.verify_object(UUID(job_id)) == "verified" + receipt_id = await session.scalar( + select(ArtifactVerificationReceipt.id).where( + ArtifactVerificationReceipt.verification_job_id == job_id, + ArtifactVerificationReceipt.outcome == "verified", + ) + ) + attempt = await session.get(ArtifactPutAttempt, str(admission.attempt_id)) + assert attempt is not None and receipt_id is not None + assert attempt.replica_id is not None + logical_role = attempt.logical_role + content_id = await session.scalar( + text("select content_id from artifact_replicas where id=:replica_id"), + {"replica_id": attempt.replica_id}, + ) + assert content_id is not None + await session.rollback() + return SimpleNamespace( + factory=factory, + project_id=project_id, + task_id=task_id, + checker_run_id=checker_run_id, + put_attempt_id=str(admission.attempt_id), + verification_receipt_id=receipt_id, + content_id=str(content_id), + logical_role=logical_role, + namespace=namespace, + policy_bundle=policy_bundle, + ) + + +@asynccontextmanager +async def _verified_output(database_url: str, tmp_path: Path): + """Create one real local put and independently verified checker output.""" + settings = _settings(tmp_path) + namespace = _namespace(settings) + engine = create_async_engine(database_url) + factory = async_sessionmaker(engine, expire_on_commit=False) + bootstrap, store = _local_store(settings, namespace) + policy_bundle = await create_standalone_unified_policy(factory, namespace) + try: + case = await _store_verified_output( + factory=factory, + settings=settings, + namespace=namespace, + store=store, + policy_bundle=policy_bundle, + source_path=tmp_path / "checker-output", + payload=b"exact checker output", + ) + case.settings = settings + case.store = store + yield case + finally: + bootstrap.close() + await engine.dispose() + + +def _binding_values(case, **changes): + values = { + "id": str(new_record_id()), + "content_id": case.content_id, + "project_id": case.project_id, + "resource_type": "checker_run", + "resource_id": case.checker_run_id, + "logical_role": case.logical_role, + "scope_version": 1, + "actor_id": ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + "attribution_type": "service_identity", + "put_attempt_id": case.put_attempt_id, + "verification_receipt_id": case.verification_receipt_id, + "supersedes_binding_id": None, + } + values.update(changes) + return values + + +_INSERT_BINDING = text( + """insert into artifact_bindings( + id,content_id,project_id,resource_type,resource_id,logical_role,scope_version, + actor_id,attribution_type,put_attempt_id,verification_receipt_id,supersedes_binding_id) + values(:id,:content_id,:project_id,:resource_type,:resource_id,:logical_role, + :scope_version,:actor_id,:attribution_type,:put_attempt_id, + :verification_receipt_id,:supersedes_binding_id)""" +) + + +@pytest.mark.asyncio +async def test_exact_verified_checker_output_binding_and_generic_binding_remain_valid( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + async with case.factory() as session, session.begin(): + await session.execute(_INSERT_BINDING, _binding_values(case)) + await session.execute( + _INSERT_BINDING, + _binding_values( + case, + id=str(new_record_id()), + resource_type="task", + resource_id=case.task_id, + logical_role="diagnostic", + actor_id="test-actor", + attribution_type="human", + put_attempt_id=None, + verification_receipt_id=None, + ), + ) + async with case.factory() as session: + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where resource_type in ('checker_run','task')" + ) + ) == 2 + + +@pytest.mark.asyncio +async def test_checker_binding_rejects_mixed_or_foreign_ancestry( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + other = await _store_verified_output( + factory=case.factory, + settings=case.settings, + namespace=case.namespace, + store=case.store, + policy_bundle=case.policy_bundle, + source_path=tmp_path / "other-checker-output", + payload=b"distinct independently verified checker output", + ) + assert case.put_attempt_id != other.put_attempt_id + assert case.verification_receipt_id != other.verification_receipt_id + assert case.content_id != other.content_id + assert case.namespace.namespace_fingerprint == other.namespace.namespace_fingerprint + + async with case.factory() as session, session.begin(): + control = await session.begin_nested() + await session.execute(_INSERT_BINDING, _binding_values(case)) + await session.execute(_INSERT_BINDING, _binding_values(other)) + assert await session.scalar( + text("select count(*) from artifact_bindings where resource_type='checker_run'") + ) == 2 + await control.rollback() + assert await session.scalar( + text("select count(*) from artifact_bindings where resource_type='checker_run'") + ) == 0 + + mixed_lineages = ( + { + "put_attempt_id": other.put_attempt_id, + "verification_receipt_id": other.verification_receipt_id, + "content_id": other.content_id, + }, + {"put_attempt_id": other.put_attempt_id}, + {"verification_receipt_id": other.verification_receipt_id}, + {"content_id": other.content_id}, + ) + for mixed in mixed_lineages: + with pytest.raises( + DBAPIError, + match="checker output binding verified ancestry mismatch", + ): + async with case.factory() as session, session.begin(): + await session.execute( + _INSERT_BINDING, + _binding_values(case, **mixed), + ) + async with case.factory() as session: + assert await session.scalar( + text("select count(*) from artifact_bindings where resource_type='checker_run'") + ) == 0 + + +@pytest.mark.asyncio +async def test_checker_attempt_static_custody_allows_only_execution_lifecycle( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + for assignment in ( + "checker_request_digest='sha256:' || repeat('b',64)", + "request_digest='sha256:' || repeat('b',64)", + "sha256='sha256:' || repeat('b',64)", + "submission_id='00000000-0000-7000-8000-000000000001'::uuid", + "logical_role='substituted-slot'", + "namespace_fingerprint='sha256:' || repeat('b',64)", + ): + with pytest.raises( + DBAPIError, + match="checker output put attempt custody is immutable", + ): + async with case.factory() as session, session.begin(): + await session.execute( + text( + f"update artifact_put_attempts set {assignment} where id=:id" + ), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="checker output put attempt custody is immutable", + ): + async with case.factory() as session, session.begin(): + await session.execute( + text("delete from artifact_put_attempts where id=:id"), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="checker output put attempt custody is immutable", + ): + async with case.factory() as session, session.begin(): + await session.execute(text("truncate artifact_put_attempts cascade")) + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_put_attempts set cas_version=cas_version+1, " + "updated_at=clock_timestamp() where id=:id" + ), + {"id": case.put_attempt_id}, + ) + + +@pytest.mark.asyncio +async def test_binding_guard_uses_only_artifact_ancestry_and_exact_owner_fks( + isolated_database_env: str, +) -> None: + engine = create_async_engine(isolated_database_env) + try: + async with engine.connect() as connection: + definition = await connection.scalar( + text( + "select pg_get_functiondef(" + "'guard_checker_output_binding_insert()'::regprocedure)" + ) + ) + assert definition is not None + assert "checker_runs" not in definition + assert "submissions" not in definition + assert "workstream_tasks" not in definition + constraints = dict( + (await connection.execute(text( + "select conname,pg_get_constraintdef(oid) from pg_constraint " + "where conrelid='artifact_put_attempts'::regclass and conname in " + "('fk_artifact_put_attempts_checker_run_ownership'," + "'fk_artifact_put_attempts_submission_version'," + "'fk_artifact_put_attempts_task_project')" + ))).all() + ) + assert set(constraints) == { + "fk_artifact_put_attempts_checker_run_ownership", + "fk_artifact_put_attempts_submission_version", + "fk_artifact_put_attempts_task_project", + } + assert "FOREIGN KEY (checker_run_id, task_id, submission_id)" in constraints[ + "fk_artifact_put_attempts_checker_run_ownership" + ] + assert "FOREIGN KEY (submission_id, task_id, submission_version)" in constraints[ + "fk_artifact_put_attempts_submission_version" + ] + assert "FOREIGN KEY (task_id, project_id)" in constraints[ + "fk_artifact_put_attempts_task_project" + ] + finally: + await engine.dispose() + + +async def _restore_pre_0007_schema(connection) -> None: + """Remove only 0007 custody inside the caller's rollback-only transaction.""" + for table, trigger in ( + ("artifact_bindings", "checker_output_binding_insert"), + ("artifact_put_attempts", "checker_output_put_attempt_custody"), + ("artifact_put_attempts", "checker_output_put_attempt_no_truncate"), + ): + await connection.execute(text(f"drop trigger {trigger} on {table}")) + for function in ( + "guard_checker_output_binding_insert()", + "guard_checker_output_put_attempt_custody()", + "guard_checker_output_put_attempt_truncate()", + ): + await connection.execute(text(f"drop function {function}")) + for constraint in ( + "ck_artifact_bindings_checker_output_lineage", + "fk_artifact_bindings_checker_put_attempt", + "fk_artifact_bindings_checker_verification_receipt", + ): + await connection.execute( + text(f"alter table artifact_bindings drop constraint {constraint}") + ) + for index in ( + "ix_artifact_bindings_put_attempt_id", + "ix_artifact_bindings_verification_receipt_id", + ): + await connection.execute(text(f"drop index {index}")) + await connection.execute( + text( + "alter table artifact_bindings drop column put_attempt_id, " + "drop column verification_receipt_id" + ) + ) + for constraint in ( + "fk_artifact_put_attempts_task_project", + "fk_artifact_put_attempts_checker_run_ownership", + "fk_artifact_put_attempts_submission_version", + "ck_artifact_put_attempts_checker_request_digest", + "ck_artifact_put_attempts_producer_reference", + ): + await connection.execute( + text(f"alter table artifact_put_attempts drop constraint {constraint}") + ) + await connection.execute(text("drop index ix_artifact_put_attempts_submission_id")) + await connection.execute( + text( + "alter table artifact_put_attempts add constraint " + "ck_artifact_put_attempts_producer_reference check (" + "(producer_request_type='guide' and guide_source_item_id is not null " + "and checker_run_id is null and task_id is null and logical_role is null) or " + "(producer_request_type='checker_output' and guide_source_item_id is null " + "and checker_run_id is not null and task_id is not null " + "and octet_length(logical_role) between 1 and 100) or " + "(producer_request_type='submission_bundle' and guide_source_item_id is null " + "and checker_run_id is null and task_id is not null and logical_role is null))" + ) + ) + await connection.execute( + text( + "alter table artifact_put_attempts drop column submission_id, " + "drop column submission_version, drop column checker_request_digest" + ) + ) + + +def _run_0007_upgrade(connection) -> None: + module = runpy.run_path( + str( + Path(__file__).resolve().parents[1] + / "alembic/versions/0007_checker_output_custody.py" + ) + ) + with Operations.context(MigrationContext.configure(connection)): + module["upgrade"]() + + +@pytest.mark.asyncio +async def test_migration_refuses_unprovable_or_inconsistent_retained_checker_attempts( + isolated_database_env: str, + tmp_path: Path, +) -> None: + """The old schema lacks evaluation/slot custody, so retained attempts fail closed.""" + async with _verified_output(isolated_database_env, tmp_path / "retained") as case: + async with case.factory() as seed_session: + _, other_task_id, _ = await _seed_checker_output_relationships( + seed_session, + case.namespace, + policy_bundle=case.policy_bundle, + ) + async with case.factory() as session: + connection = await session.connection() + await _restore_pre_0007_schema(connection) + before = await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="retained checker output request custody is unprovable", + ): + async with connection.begin_nested(): + await connection.run_sync(_run_0007_upgrade) + assert await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) == before + + await session.execute( + text("update artifact_put_attempts set task_id=:task where id=:id"), + {"id": case.put_attempt_id, "task": other_task_id}, + ) + mismatched = await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="retained checker output attempt ownership is inconsistent", + ): + async with connection.begin_nested(): + await connection.run_sync(_run_0007_upgrade) + assert await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) == mismatched + await session.rollback() diff --git a/backend/tests/test_ci_lane_catalogue.py b/backend/tests/test_ci_lane_catalogue.py index 6b374327e..a2a45a389 100644 --- a/backend/tests/test_ci_lane_catalogue.py +++ b/backend/tests/test_ci_lane_catalogue.py @@ -231,6 +231,8 @@ def test_measured_hotspots_have_explicit_semantic_owners() -> None: "tests/test_approved_guide_intake.py", "tests/test_post_submit_materialization.py", "tests/test_post_submit_selection.py", + "tests/test_checker_output_custody.py", + "tests/test_checker_output_storage.py", "tests/test_pre_submit_attempt_recovery.py", "tests/test_pre_submit_attempt_contracts.py", "tests/test_pre_submit_attempt_authority_integration.py", diff --git a/backend/tests/test_coverage_contract.py b/backend/tests/test_coverage_contract.py index f4f0fa9bb..b79a09230 100644 --- a/backend/tests/test_coverage_contract.py +++ b/backend/tests/test_coverage_contract.py @@ -12,7 +12,7 @@ sys.path.insert(0, str(SCRIPTS)) import coverage_policy as policy # noqa: E402 -HEAD = "0006_history_read_authority" +HEAD = "0007_checker_output_custody" SHA = "a" * 40 PEP695_INVALID = sys.version_info < (3, 12) diff --git a/docs/architecture_checker_framework.md b/docs/architecture_checker_framework.md index bdb4ed86c..05e9dc234 100644 --- a/docs/architecture_checker_framework.md +++ b/docs/architecture_checker_framework.md @@ -331,9 +331,13 @@ Production explicitly uses `UnavailablePostSubmissionExecution`; this does not install durable post-submit execution, authorize material reads, or prove attempt and currentness ownership. ARCH-04B supplies the hidden ART input port: exact consumed Submission bytes, rebuilt manifest, async scoped file access and cleanup, followed -by a fresh material-selection check. Production composition denies materialization; -ARCH-04B2 output custody, ARCH-04C durable execution, ARCH-04D authority and ARCH-04E -routing remain the execution cutover. A returned evaluation value is not a stored +by a fresh material-selection check. Production composition denies materialization. +ARCH-04B2 now supplies hidden typed output storage, byte-free recovery and +flush-only verified binding while its CHECKERS reservation and authority adapters +remain unavailable. The current structural catalogue reserves zero output slots; +controlled nonempty slots prove ART mechanics only. ARCH-04C must support that +empty output set and owns durable execution, followed by ARCH-04D authority and +ARCH-04E routing. A returned evaluation value is not a stored current result or acceptance. Retained run and submission history now use canonical AUTH and separate fixed contributor/manager projections. The alternate execution service and Celery worker are @@ -520,8 +524,9 @@ severities. The following is the intended end-to-end lifecycle. Pre-submit intake and retained-history reads are implemented. Canonical durable post-submit execution, -result routing and recovery remain unavailable pending ARCH-04B2/04C/04D/04E/04F; -ARCH-04B hidden input is delivered with deny-only production authority; +result routing and recovery remain unavailable pending ARCH-04C/04D/04E/04F; +ARCH-04B hidden input and ARCH-04B2 hidden output custody are delivered with +deny-only/unavailable production composition; the flow below is not a claim that those jobs or transitions are live. ```text @@ -602,8 +607,8 @@ The checker run records: - warning count - completion timestamp -With ARCH-04B input delivered, ARCH-04B2 output custody, ARCH-04C result custody, -ARCH-04D activation and +With ARCH-04B input and ARCH-04B2 output custody delivered, ARCH-04C result +custody, ARCH-04D activation and ARCH-04E routing integration, this gives reviewers proof that they are reviewing the same immutable binding and manifest that passed automated checks; legacy caller-owned manifest fields are not authority. diff --git a/docs/architecture_data_model.md b/docs/architecture_data_model.md index 0a0772211..31d36fbdf 100644 --- a/docs/architecture_data_model.md +++ b/docs/architecture_data_model.md @@ -318,8 +318,12 @@ Submission's three ART references remain all-null during transaction staging or all-present. The production creation command consumes the exact ART admission and fills all three before its root transaction commits. This is separate from the required initial assignment identity. Hidden exact ART-to-CHECKERS input materialization is delivered by ARCH-04B. -Output custody (ARCH-04B2) and durable execution/results (ARCH-04C) remain pending; -production access remains deny-only until ARCH-04D. Retained payment columns on Submission and CheckerRun +ARCH-04B2 adds checker-output attempt custody with exact Submission version and +checker-request digest plus immutable verified put/receipt ancestry on each +run/slot binding. Its migration refuses retained checker attempts or bindings +whose missing evaluation digest or verified ancestry cannot be proven; it does +not invent or delete retained data. Durable execution/results (ARCH-04C) remain +pending, and production access remains deny-only until ARCH-04D. Retained payment columns on Submission and CheckerRun are nullable, so new unified-guide work requires no invented economic configuration. CP09 owns physical economic-schema removal after its remaining consumers change. @@ -949,10 +953,10 @@ cannot reproduce the authoritative result. At the later public cutover, without receiving scratch paths or rerunning the pre-submit plan. Canonical admission-backed creation remains hidden until then. -Before that cutover, hidden ART-04B2 establishes the execution boundary without +Before that cutover, hidden pre-submit materialization establishes the execution boundary without exposing a route. The fixed materializer authorizes before any prepared-byte read or workspace reservation. One callback-scoped sealed tree is checked -against the server commitment and semantic manifest. ART-04B3 extends that same +against the server commitment and semantic manifest. The locked project-policy phase extends that same callback to execute the locked project-policy phase and normalize every platform and project result into one typed envelope. The tree is cleaned before the transaction-bound evidence service reloads the actor, identity link, task, @@ -1647,13 +1651,18 @@ Fields: - `locked_revision_policy_id` - `locked_revision_policy_generation` - `locked_revision_policy_hash` -- `artifact_binding_id` (target after ARCH-04B2/04C custody) -- `submission_bundle_manifest_id` (target after ARCH-04B2/04C custody) -- `package_hash` (legacy; replacement custody in ARCH-04B2/04C) -- `artifact_hash_manifest` (legacy; replacement custody in ARCH-04B2/04C) -- `artifact_manifest_hash` (legacy; replacement custody in ARCH-04B2/04C) +- `artifact_binding_id` (planned canonical input reference; exact 04C shape pending) +- `submission_bundle_manifest_id` (planned canonical input reference; exact 04C shape pending) +- `package_hash` (retained current field; 04C must replace its authority use) +- `artifact_hash_manifest` (retained current field; 04C must replace its authority use) +- `artifact_manifest_hash` (retained current field; 04C must replace its authority use) - `summary` +ARCH-04B2 does not populate or authorize this retained run writer. It stores +checker-output custody in ART attempts and bindings only. ARCH-04C owns the +current CheckerRun/result schema and must select exact canonical input +references without treating these retained caller-manifest fields as authority. + Status: - queued diff --git a/docs/engineering/authorization_activation_custody.md b/docs/engineering/authorization_activation_custody.md index 6e60baed8..3a244f8ac 100644 --- a/docs/engineering/authorization_activation_custody.md +++ b/docs/engineering/authorization_activation_custody.md @@ -58,9 +58,13 @@ The table retains historical planning-custody labels, not a literal mapping of every typed runtime `ActionOwner`. XINT-06B groups runtime `WS-AUTH-001-ART-06A` post-submit materialization and `WS-AUTH-001-ART-06B` output write/binding. ARCH-04D is their current replacement -activation boundary after delivered ARCH-04B hidden input, planned ARCH-04B2 -output custody and ARCH-04C durable execution/results. Production materialization -remains deny-only; the typed catalogue is unchanged by this reconciliation. Do not implement an additional XINT-06B lane. +activation boundary after delivered ARCH-04B hidden input and ARCH-04B2 hidden +output custody, followed by ARCH-04C durable execution/results. ARCH-04B2 owns +fresh authority participants internally for each store, recovery and binding +phase; public requests carry selectors and byte sources, never PREP handles. +Default output reservation and authority composition remains unavailable, and +production materialization remains deny-only. The typed catalogue is unchanged +by this reconciliation. Do not implement an additional XINT-06B lane. Runtime owner `WS-XINT-002-07` retains catalogue custody. The only approved v0.1 availability transition is 07A packet materialization. Evidence binding diff --git a/docs/operations_project_operating_manual.md b/docs/operations_project_operating_manual.md index 601216f9f..2b00ae459 100644 --- a/docs/operations_project_operating_manual.md +++ b/docs/operations_project_operating_manual.md @@ -262,7 +262,9 @@ authority and decision-bound receipts. ARCH-03C2 delivers atomic producer publication and first handler registration with enforced prefork topology. Public manager activation and ARCH-03D hidden approved-guide intake are delivered. Hidden exact post-submit input (ARCH-04B) is delivered with deny-only production -authority. ARCH-04B2 output custody is next. Public intake remains deferred +authority. Hidden ARCH-04B2 output storage, recovery and verified binding are +delivered with production reservations and authority unavailable. ARCH-04C durable +execution is next. Public intake remains deferred to ARCH-02I after evaluation and remediation prerequisites. The intended unified flow uses one compilation result for sufficiency and diff --git a/docs/roadmap_status.md b/docs/roadmap_status.md index 75d38136c..8cff89898 100644 --- a/docs/roadmap_status.md +++ b/docs/roadmap_status.md @@ -66,8 +66,15 @@ creation defaults true and omitted replacements inherit the predecessor. Versioned hashing preserves old policy hashes and locks. False remains blocked by the current activation service. See the [implementation record](../.commitrail/changes/pre-review-plan-reconciliation.md#delivered-policy-setting-implementation). Enabling false follows shared final-acceptance, CON and exact AUTH integration proof, not live human-review infrastructure. -This allows an automated end-to-end milestone first; human review/revision -still belongs to the complete v0.1 release. See the +The first complete contributor milestone is claim -> upload ZIP -> pre-submit +feedback or immutable Submission -> automatic post-submit checking -> outcome. +For an eligible project with locked `human_review_required=false`, success must +include shared FinalAcceptance, the submitter ContributionRecord and applicable +policy awards. Failure/remediation and public intake belong to that milestone; +reaching hidden checker success alone does not complete it. Deliver this journey +before live human review/revision. Independent review foundations may proceed in +a separate worktree, but cannot add a human-queue or review-lease dependency to +automated acceptance. Human review/revision still belongs to the complete v0.1 release. See the [product-builder handoff](../.commitrail/changes/pre-review-plan-reconciliation.md#product-builder-handoff-implement-the-setting-next). | Status | Meaning | @@ -95,8 +102,11 @@ adds exact-authorized Contributor/Manager detail and requirements. ARCH-03C6 add separate exact-authorized locked-context reads. ARCH-03C7 adds bounded public Audit Authority history access. ARCH-03D connects hidden intake through durable intent to the activated historical guide using canonical owner ports. ARCH-04B adds hidden verified Submission input -with scoped async access; production materialization authority remains deny-only. -Checker output custody and durable evaluation follow; public intake remains deferred to their cutover prerequisites. Required success +with scoped async access. ARCH-04B2 adds hidden typed checker-output storage, +byte-free recovery and flush-only verified binding over generic ART +put/verification. Production reservation and authority remain unavailable; +durable evaluation follows in ARCH-04C, and public intake remains deferred to +the later cutover prerequisites. Required success then branches on the locked ReviewPolicy: true routes to human `allow_review`; false invokes shared authorized acceptance without a human Review. Both routing integrations remain planned. Human review/revision, contribution and conditional @@ -150,7 +160,7 @@ cannot be reused as post-submission review-gate evidence. See the | Contributor artifact preparation | **Hidden and proven** | One outer ZIP; bounded scratch inspection; canonical manifest; platform and project prechecks; unchanged-work rejection; durable put intent; verification; capacity-charged ready admission; hidden final handoff validates the exact activated historical guide through owner ports | Complete the later public admission-only cutover | | Pre-submission intake checking | **Hidden with approved-guide lineage** | Separate versioned pre-submission catalogue, locked effective-plan compilation, platform/project checks during continuous preparation, blocking feedback before Submission creation, and one internal phase command covering execution/replay with the JSON precheck removed; ARCH-03D connects approved-guide lineage through the final durable handoff | Complete the canonical public cutover after evaluation/remediation prerequisites; passing intake must never substitute for post-submit evaluation | | Immutable Submission creation | **Hidden foundation; public packet creation retired** | Contributor preparation authority; durable pre-submit reservation and exact completed-evidence recovery without rerunning checks; atomic admission consumption; TASK-owned admission-backed creation with exact assignment ContributionPolicyVersion and locked policy lineage; fixed-service artifact binding; replay/concurrency/rollback proof | Finish downstream evaluation and the canonical public integration. The retained submission-list GET is not a usable creation POST | -| Post-submission evaluation and `allow_review` | **Planned; immediate integration milestone** | One canonical CHECKER post-submit catalogue/compiler used by existing consumers, internal phase service with explicitly unavailable post execution, hidden value contracts and structural-handler conformance; ARCH-04B hidden exact verified Submission materialization with deny-only production authority | Add ARCH-04B2 output custody; connect the unavailable phase port to durable execution; evaluate the exact Submission against its locked policy; persist one durable current superseding result; activate fixed services; automatically dispatch it and publish the canonical `allow_review` manifest | +| Post-submission evaluation and `allow_review` | **Planned; immediate integration milestone** | One canonical CHECKER post-submit catalogue/compiler used by existing consumers, internal phase service with explicitly unavailable post execution, hidden value contracts and structural-handler conformance; ARCH-04B hidden exact verified Submission materialization; ARCH-04B2 hidden typed output store/recovery and verified binding, with production reservation and authority unavailable | Add ARCH-04C durable execution with valid empty-output support; evaluate the exact Submission against its locked policy; persist one durable current superseding result; activate fixed services; automatically dispatch it and publish the canonical `allow_review` manifest | | Review queue and lease | **Hidden persistence foundation** | Queue/admission idempotency and ReviewLease/preference persistence; complete unavailable REV action/principal catalogue and typed AUTH contracts | Packet-membership contract and manifest; Review schema; canonical admission from `allow_review`; claim/lease/packet authority; lease copies the Submission-stamped policy version with no CON lookup | | Review decision and revision | **Planned** | Review/revision policy identities and mutation authority; approved same-task revision-rebase semantics | Immutable findings and decisions; `accept`, `needs_revision`, and `reject`; complete-context revision preparation; finding responses; replacement contributor rules; replay and recovery | | Contribution and compensation truth | **Schema foundations plus public policy administration** | ContributionPolicyVersion persistence; lifecycle-audit participant; adapter bindings; public Finance policy administration | Persist ContributionRecord/CompensationAward and one shared FinalAcceptance/submitter operation for human accept or authorized false/pass routing. Only actual Reviews create reviewer records. Evaluate frozen actor rules into zero, one or two awards | @@ -438,8 +448,9 @@ The next dependency-safe product sequence is: data and downstream TASK consumers remain until their scoped cutover; deleting retained data is not authorized by code cleanup. 2. **Produce current post-submit evidence and policy-governed routing.** ARCH-04B's - hidden exact input materialization is delivered. Next add ARCH-04B2 output custody, - execute the locked post-submit plan, persist one current result, + hidden exact input materialization and ARCH-04B2's hidden output custody are + delivered. Next execute the locked post-submit plan through ARCH-04C, support + the current structural catalogue's empty output set, persist one current result, activate only its fixed services, and automatically publish an exact human `allow_review` manifest on true when no blocking failure exists. CHECKERS owns durable execution/currentness; the shared facade does not @@ -523,6 +534,13 @@ read port and ART's consumed admission. Local/MinIO input is independently rerea verified and projected through canonical bounded scratch. Production access remains deny-only; it does not activate durable checker execution or public intake. +ARCH-04B2 adds hidden typed checker-output `store` and byte-free +`recover(selector)` operations, per-slot preparation caps, generic put and +verification reuse, and flush-only immutable verified binding. Controlled +nonempty slots prove ART mechanics only; the current structural catalogue has +zero slots. CHECKERS reservation/currentness and live output authority remain +unavailable, so this does not activate production execution or public intake. + ## Critical Dependency Map ```text @@ -533,6 +551,7 @@ Delivered foundations (not a claim of full public integration) task/assignment/Submission lineage + hidden intake/creation ARCH-03D exact approved historical guide through durable intake handoff ARCH-04B hidden verified Submission input (production authority deny-only) + ARCH-04B2 hidden checker output store/recovery/binding (reservation and authority unavailable) shared dispatcher + exact-authorized hidden assignment invalidation ARCH-03C2 invalidation producer + first handler registration ARCH-03C3 exact manager task create/screen/release + replay @@ -543,7 +562,7 @@ Delivered foundations (not a claim of full public integration) | v Remaining integration - checker output custody + durable evaluation + live authority + remediation + durable evaluation + live authority + remediation -> public intake and immutable admitted Submission cutover -> durable current post-submit result + required checks pass | @@ -653,10 +672,11 @@ remaining trace sequence is: removes the private TASK/PROJECTS lookup at final durable handoff. No public preparation or Submission endpoint is activated. - Post-submit admission: after delivered `POL-07B` and `ARCH-03C`, delivered hidden - [ARCH-04B input materialization](../.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md) - leads to `ARCH-04B2 -> 04C -> 04D -> 04E` for output custody, durable results, - live authority and routing. - An ART-owned output/log custody child precedes `04C` final completion. + [ARCH-04B input materialization](../.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B.md) + and [ARCH-04B2 output custody](../.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md) + lead to `04C -> 04D -> 04E` for durable results, live authority and routing. + ARCH-04C must accept the exact empty output set from the current structural + catalogue; nonempty 04B2 test slots do not claim a live evaluator capability. AUTH-OUTBOX-02 delivers shared live authority, phase audit custody and Celery delivery/recovery scans over CON-02B. Delivery termination is bounded by a 300-second hard limit under prefork. diff --git a/docs/spec_artifact_storage_service.md b/docs/spec_artifact_storage_service.md index 70cfc778e..c7276495f 100644 --- a/docs/spec_artifact_storage_service.md +++ b/docs/spec_artifact_storage_service.md @@ -17,7 +17,7 @@ or Flow Node. FastAPI and Celery composition roots -> ExternalServiceAdapterFactory[ArtifactStoreBootstrap] -> PostgreSQL exact namespace claim --> initialized ArtifactStore v2 +-> initialized ArtifactStore -> LocalStorageAdapter -> S3CompatibleArtifactStore -> MinIO integration @@ -319,7 +319,7 @@ changes. A subsequently exhausted retry job may be the source of a new chained attempt. The earlier source can never be reused, including after the first attempt succeeds or fails and regardless of a different idempotency key. -## ArtifactStore v2 Port +## ArtifactStore Port The asynchronous port is: @@ -358,17 +358,20 @@ GuideDocumentGrant.open(opaque_document_handle) ArtifactBindingPort.bind_checker_output(CheckerOutputBindingRequest) PostSubmissionMaterializationPort.materialize(PostSubmissionEvaluationRequest, consumer) CheckerArtifactOutputPort.store(CheckerOutputArtifactRequest) +CheckerArtifactOutputPort.recover(CheckerOutputSelector) ArtifactOperatorReadPort.list_bindings/list_replicas/list_receipts/ get_verification_job/get_recovery_attempt/list_audit_events/admission_usage(...) ArtifactOperatorRecoveryPort.retry_verification(ArtifactRecoveryRequest) ``` -Every protected durable mutation in this port family consumes one opaque, -process-local `PreparedAuthorizationHandle` at its transaction boundary. The -hidden HTTP submission-bundle preparation request carries immutable +Every protected durable mutation in this port family obtains fresh, +action-specific authority at its protected transaction boundary. Opaque +`PreparedAuthorizationHandle` values remain process-local implementation +custody; checker-output store/recovery requests do not carry them. The hidden +HTTP submission-bundle preparation request carries immutable `ActorIdentityFacts`, request and correlation IDs, contributor selectors, packet metadata, and an asynchronous byte source rather than a prepared handle; -ART obtains separate prepared handles +ART obtains separate prepared handles internally immediately before materialization and durable put intent. The typed methods fix their expected actions, and handles never enter route schemas, outbox/Celery payloads, provider interfaces, or serialized contracts. @@ -402,14 +405,23 @@ admission and admitted verified replica through its own records plus the TASK-ow an async consumer receives a scoped read-only file view. The typed result carries material custody plus the existing closed evaluation value; it is not a durable checker result. Production composition denies before protected reads until ARCH-04D. -`CheckerOutputArtifactRequest` contains the fixed -service's prepared authority, task/submission/checker-run IDs, logical role, -and generated byte source. The Submission binding is instead the terminal result of the typed +`CheckerOutputArtifactRequest` contains a CHECKERS-owned reservation selector +and generated byte source. `store` obtains fresh authority for each phase, +resolves the exact evaluation/run/checker-worker-lease/output-slot facts, applies the +owner-declared media type and byte ceiling, and uses generic ART put and +independent verification. `recover(selector)` uses the same current owner and +authority facts to recover the stable operation without a byte source; it never +puts or regenerates bytes. The current production composition supplies an +unavailable reservation port and deny-only authority, so neither method is a +live evaluator capability. The Submission binding is instead the terminal result of the typed `SubmissionAdmissionConsumptionPort`: it consumes one ready admission against the exact TASK-supplied Submission identity and locked lineage. The remaining -checker-output binding request contains its required verified content ID and -exact CheckerRun selector. Each typed method fixes its own authority; no request -selects the action. +checker-output binding request contains the same exact selector plus the put +attempt and independent verification receipt identities. A flush-only ART +participant re-resolves current reservation and authority, verifies complete +receipt/job/attempt/replica/content ancestry, and creates or replays one +immutable run/slot binding in the caller's transaction. Each typed method fixes +its own authority; no request selects the action. `ArtifactRecoveryRequest` contains the authenticated Operator context, exact source verification-job ID, reason, client idempotency key, expected source-job CAS version, and canonical authorization resource facts. Reason-bound Operator @@ -548,7 +560,7 @@ activation evidence. Cloudflare R2 has no v0.1 runtime mode, credential issuer, sidecar, secret contract, configuration profile, or deployment proof. Any future provider -initiative must perform current discovery and pass the same ArtifactStore v2 +initiative must perform current discovery and pass the same `ArtifactStore` conformance suite before any runtime configuration is introduced. Production rejects `local`, plaintext HTTP, loopback/local endpoints, invalid @@ -731,7 +743,7 @@ and the AWS live readiness profile. ## LocalStorage Adapter -LocalStorage implements the same v2 port and conformance vectors. It uses a +LocalStorage implements the same port and conformance vectors. It uses a pre-provisioned private configured root, opaque content-derived paths, exclusive no-overwrite publication, bounded off-event-loop file I/O, no-follow path handling, private permissions, full read verification, and sanitized @@ -753,8 +765,8 @@ must then restart and revalidate the complete layout. Any different name, owner, type, link count, or mode is an integrity incident and must not be repaired by this procedure. -The v1 local provider metadata for retain/release/provider receipts is removed -in the v2 clean cut. No compatibility adapter or dual format remains. +The initial v0.1 provider contract has no retain/release methods or +provider-owned receipt metadata. No compatibility adapter or dual format exists. ## Ingest Choreography @@ -892,7 +904,8 @@ approved operational change. | `WORKSTREAM_ARTIFACT_SUBMISSION_ZIP_MAXIMUM_COMPRESSION_RATIO` | `100` | `10000` | Maximum expanded-to-compressed ratio for one file. | | `WORKSTREAM_ARTIFACT_SUBMISSION_ZIP_MAXIMUM_INSPECTION_SECONDS` | `300` | `1800` | Complete synchronous inspector deadline inside the preparation deadline. | -04B2 uses the same ledger to reserve a submission workspace's complete expanded +The delivered post-submit materializer uses the same ledger to reserve a +submission workspace's complete expanded byte count and entry count before projection. Workspace reservations participate in aggregate byte/free-space checks but do not become artifact storage. Cleanup uses the workspace's own bounded entry reservation rather than the live @@ -904,7 +917,7 @@ the already-inspected manifest into one callback-scoped sealed tree using descriptor-relative creation. Regular files use fixed `0400` or normalized executable `0500`; directories use `0500`. Checker adapters receive no scratch path, shell, subprocess, or execution primitive. The tree is destroyed first; -only then do bounded, path-redacted platform/default results return. 04B2 neither runs +only then do bounded, path-redacted platform/default results return. ARCH-04B neither runs project-policy primitives nor persists final checker evidence. Pre-submission checker catalogue configuration is separate from ZIP inspection @@ -995,8 +1008,9 @@ and the owning durable source can be regenerated, the original caller may prepare it again and compare the complete digest/size. Identical bytes may replay the original operation after admission capacity is reacquired. Changed guide bytes require a new guide version with its own document declaration, -internal snapshot and setup; changed checker output requires a new checker-run -attempt. Neither reuses the old operation, snapshot, +internal snapshot and setup. Changed checker-output bytes conflict with the +existing run/slot operation; a distinct logical run or owner-issued slot is +required. Neither reuses the old operation, snapshot, or binding identity. A generator that cannot reproduce exact bytes fails its old infrastructure attempt instead of fabricating replay. Bytes are never placed in PostgreSQL, Redis, Celery payloads, logs, or audit. @@ -1475,6 +1489,21 @@ implementation identity. Pre-submit evidence and post-submit execution name the same verified admission and exact binding. Checker logs and generated outputs become artifact bindings. +ARCH-04B2 supplies the hidden checker-output custody boundary. The typed +`store` and byte-free `recover(selector)` paths enforce one owner-issued slot +ceiling, reuse generic admission, put observation and verification, and return +only exact verified identities. The flush-only binding participant preserves +the full verified ancestry and never privately reads CHECKERS rows. Store and +recovery resolve owner facts only through the typed CHECKERS port; the separate +Operator resource-to-project lookup remains an explicit read concern and is not +an output-admission path. Production +composition remains unavailable because CHECKERS reservation/currentness and +live output authority are intentionally deferred. The current structural +catalogue declares zero output slots; controlled nonempty test reservations +prove ART mechanics only. ARCH-04C must complete a valid evaluation with an +empty output set and compose bindings only for slots actually reserved by its +owner. + Transient post-submit storage unavailability leaves the task in `evaluation_pending` and uses checker retry infrastructure. A provider object confirmed missing after its content was bound is a terminal artifact incident @@ -1506,12 +1535,10 @@ forbidden in production. ## Migration And Removal -Implementation is a clean cut: +The initial v0.1 implementation is a clean cut: -- committed-source preparation and LocalStorage private refactoring land first - without changing the active v1 port; -- ArtifactStore v1 methods, active callers, LocalStorage's public adapter - surface, and schema then migrate atomically in the v2 clean-cut chunk; +- committed-source preparation and the provider-neutral `ArtifactStore` are the + only writable storage path; - the Flow Node backend is removed from configuration without an alias; - obsolete caller-owned URI/hash and storage-scheme fields are removed in their owning guide/submission cutovers; @@ -1520,9 +1547,9 @@ Implementation is a clean cut: normal submission creation still enters evaluation automatically; - no dual write, nullable shadow field, fake verified backfill, fallback adapter, compatibility constructor, or second factory remains; -- the v0.1 baseline contains no v1 artifact schema or fabricated backfill; - pre-v0.1 development databases/storage namespaces are reprovisioned and - authoritative bytes are reingested through v2; +- the v0.1 baseline contains only the current artifact schema and no fabricated + backfill; pre-v0.1 development databases/storage namespaces are reprovisioned + and authoritative bytes are reingested through the current port; - the v0.1 baseline installs the durable admission ledger and prepared put-attempt tables; - the v0.1 baseline includes polymorphic contract-v2 operation @@ -1543,6 +1570,12 @@ Implementation is a clean cut: The baseline proves fresh installation, fail-closed refusal of old/nonempty databases, and no artifact bytes in PostgreSQL. It is not downgradable. +The checker-output custody migration adds exact Submission version and narrow +checker-request digest fields to checker-output attempts plus verified +put/receipt ancestry on checker bindings. It refuses any retained checker-output +attempt or binding whose missing evaluation/slot digest or verified ancestry +cannot be proven; it neither invents those facts nor deletes retained data. + ## Verification Strategy ### Remaining v0.1 dependency order @@ -1550,8 +1583,8 @@ databases, and no artifact bytes in PostgreSQL. It is not downgradable. The pre-submit checker materializer is a mandatory part of preparation, so its fixed-service AUTH activation must merge after hidden ART-04B1-04B3 and before contributor preparation is activated. ARCH-04B delivers hidden exact post-submit -input with deny-only production authority. ARCH-04B2 checker-output custody is -next; ARCH-04C owns durable CHECKERS execution/results, and ARCH-04D owns their +input with deny-only production authority. ARCH-04B2 hidden checker-output +custody is delivered; ARCH-04C now owns durable CHECKERS execution/results, and ARCH-04D owns their exact live activation. ARCH-04E separately owns TASK routing. Historical ART-06A/06B labels do not open duplicate implementation lanes. This ordering @@ -1597,14 +1630,14 @@ instances or an external deployment barrier. ## Deferred Flow Node Adapter `FN-ART-002` is a separate inactive initiative. It may later implement the same -ArtifactStore v2 port and conformance vectors. It cannot change product records +`ArtifactStore` port and conformance vectors. It cannot change product records or services, cannot introduce a runtime fallback, and requires an explicit maintenance cutover after v0.1 is proven. ## Deferred R2 Adapter Cloudflare R2 is outside the v0.1 dependency graph. Any later provider -initiative must perform current provider discovery, implement the same ArtifactStore v2 +initiative must perform current provider discovery, implement the same `ArtifactStore` contract, pass its own conformance and security proof, and use an explicit no-fallback maintenance cutover. From 987887d165d2b31f8a48423c0bf93c8699c924cc Mon Sep 17 00:00:00 2001 From: Commitrail Probe Date: Tue, 29 Sep 2026 09:06:48 +0100 Subject: [PATCH 2/6] fix(artifacts): protect binding truncation and prove mixed selector rejection --- .../WS-ARCH-001/WS-ARCH-001-04B2.md | 16 +++- .../versions/0007_checker_output_custody.py | 5 ++ .../tests/checker_output_custody_helpers.py | 39 +++++++++- backend/tests/conftest.py | 3 +- backend/tests/test_checker_output_custody.py | 74 +++++++++++++++++++ backend/tests/test_checker_output_storage.py | 62 ++++++++++++++++ .../authorization_activation_custody.md | 4 +- docs/spec_authorization_service.md | 5 +- 8 files changed, 201 insertions(+), 7 deletions(-) diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md index 54d0fc004..04c8f08cd 100644 --- a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -124,7 +124,8 @@ claim the current structural catalogue supports them. to register changed owners and keep existing gates intact. - This record, linked ARCH output skeleton, current ARCH/ART/AUTH/POL/CON navigation, `.commitrail/INDEX.md`, `README.md`, `docs/roadmap_status.md`, artifact/checker/data - model specifications, operating manual and authorization custody page for affected claims. + model specifications, authorization specification, operating manual and + authorization custody page for affected claims. ### Not allowed @@ -204,6 +205,19 @@ Existing binding immutability is reused; no generated-output catalogue expansion or duplicate custody aggregate is authorized. +## Review findings + +- Immutable binding custody must cover statement-level deletion as well as row + mutation. Reuse the existing ART fact-mutation rejection function for a binding + no-TRUNCATE trigger; prove direct and cascading truncation rejects and preserves + a real verified binding. The canonical isolated test reset disables/restores + this guard and pins the resulting schema fingerprint. +- Service-level binding proof must substitute valid foreign put/receipt identities + after both valid controls, independently of the database insert guard. The + requested-attempt filter removal must make the negative service test fail. +- Current authorization specifications must show delivered input/output custody, + then ARCH-04C execution/results, then ARCH-04D activation. + ## Reconciliation - Current-source reconciliation: merged ARCH-04B on main; no overlapping open diff --git a/backend/alembic/versions/0007_checker_output_custody.py b/backend/alembic/versions/0007_checker_output_custody.py index 2343bdbbf..c8010fdf2 100644 --- a/backend/alembic/versions/0007_checker_output_custody.py +++ b/backend/alembic/versions/0007_checker_output_custody.py @@ -365,3 +365,8 @@ def _install_checker_output_binding_guard() -> None: "CREATE TRIGGER checker_output_binding_insert BEFORE INSERT ON artifact_bindings " "FOR EACH ROW EXECUTE FUNCTION guard_checker_output_binding_insert()" ) + op.execute( + "CREATE TRIGGER trg_artifact_bindings_no_truncate BEFORE TRUNCATE " + "ON artifact_bindings FOR EACH STATEMENT " + "EXECUTE FUNCTION reject_artifact_fact_mutation()" + ) diff --git a/backend/tests/checker_output_custody_helpers.py b/backend/tests/checker_output_custody_helpers.py index c881a2092..3e2c7de89 100644 --- a/backend/tests/checker_output_custody_helpers.py +++ b/backend/tests/checker_output_custody_helpers.py @@ -15,7 +15,12 @@ from app.adapters.artifacts import create_artifact_store_bootstrap from app.core.identifiers import new_record_id -from app.interfaces.artifact_operations import CheckerOutputArtifactRequest +from app.interfaces.artifact_operations import ( + CheckerOutputArtifactRequest, + CheckerOutputArtifactResult, + CheckerOutputBindingRequest, + CheckerOutputBindingResult, +) from app.interfaces.artifacts import ( ArtifactByteRange, ArtifactStoreNamespaceClaim, @@ -28,6 +33,7 @@ ) from app.modules.artifacts.checker_output_custody import CheckerOutputStoredFacts from app.modules.artifacts.checker_outputs import CheckerArtifactOutputService +from app.modules.artifacts.models import ArtifactBinding from app.modules.artifacts.preparation import ( ArtifactPreparationService, ArtifactScratchManager, @@ -271,6 +277,37 @@ def binding_service(self, session: AsyncSession) -> CheckerOutputBindingService: namespace_fingerprint=self.namespace.namespace_fingerprint, ) + async def bind( + self, + selector: CheckerOutputSelector, + stored: CheckerOutputArtifactResult, + ) -> CheckerOutputBindingResult: + """Bind one successful stored-output control in its own transaction.""" + assert stored.verification_receipt_id is not None + async with self.factory() as session: + async with session.begin(): + return await self.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + selector, + stored.put_attempt_id, + stored.verification_receipt_id, + ) + ) + + async def binding_rows(self) -> list[tuple[str, str, str, str, str]]: + """Snapshot exact immutable binding identities for denial-side-effect proof.""" + async with self.factory() as session: + rows = await session.execute( + select( + ArtifactBinding.id, + ArtifactBinding.content_id, + ArtifactBinding.resource_id, + ArtifactBinding.put_attempt_id, + ArtifactBinding.verification_receipt_id, + ).order_by(ArtifactBinding.id) + ) + return [tuple(row) for row in rows] + async def seed_reservation( self, *, diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 5cfc28a9e..959d894f1 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -23,7 +23,7 @@ DDL_LOCK_DIRECTORY = Path("/tmp") # Match the PostgreSQL 16 engine used by Backend CI. Catalog identity rendering # differs across major versions; regenerate only after comparing actual objects. -EXPECTED_PUBLIC_SCHEMA_SHA256 = "f41e86b57c564169cec756f80407a0c8e7e0f63b9c8a87be4190b58d73e8ca89" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "e2af78293f8dfef88b8a185eb52961ed337957332617c62b342f3e3c857e43f6" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", @@ -119,6 +119,7 @@ "workstream_tasks", ) TRUNCATE_GUARDED_TABLES = ( + "artifact_bindings", "artifact_put_attempts", "checker_runs", "checker_results", diff --git a/backend/tests/test_checker_output_custody.py b/backend/tests/test_checker_output_custody.py index 1f752c349..4bd3e5f0f 100644 --- a/backend/tests/test_checker_output_custody.py +++ b/backend/tests/test_checker_output_custody.py @@ -199,6 +199,80 @@ async def test_foreign_lineage_and_changed_bytes_fail_before_provider( assert (await harness.manager.usage()).reservation_count == 0 +async def test_binding_rejects_mixed_stored_lineage_before_consumption_or_mutation( + tmp_path, + isolated_database_env, +) -> None: + """Bind two valid controls, then reject every original/foreign custody mix.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + original_reservation = harness.state.reservation + original_selector = harness.selector + foreign_reservation = await harness.seed_reservation() + foreign_selector = selector_for(foreign_reservation) + + original = await harness.service.store(harness.request(b"original binding control")) + harness.state.reservation = foreign_reservation + foreign = await harness.service.store( + harness.request(b"foreign binding control", selector=foreign_selector) + ) + assert original.content_id is not None and original.verification_receipt_id is not None + assert foreign.content_id is not None and foreign.verification_receipt_id is not None + + harness.state.reservation = original_reservation + original_bound = await harness.bind(original_selector, original) + harness.state.reservation = foreign_reservation + foreign_bound = await harness.bind(foreign_selector, foreign) + assert ( + original_bound.content_id, + original_bound.put_attempt_id, + original_bound.verification_receipt_id, + original_bound.replayed, + ) == ( + original.content_id, + original.put_attempt_id, + original.verification_receipt_id, + False, + ) + assert ( + foreign_bound.content_id, + foreign_bound.put_attempt_id, + foreign_bound.verification_receipt_id, + foreign_bound.replayed, + ) == ( + foreign.content_id, + foreign.put_attempt_id, + foreign.verification_receipt_id, + False, + ) + + binding_baseline = await harness.binding_rows() + assert {UUID(row[0]) for row in binding_baseline} == { + original_bound.binding_id, + foreign_bound.binding_id, + } + consumed_baseline = len(harness.state.bindings) + assert consumed_baseline == 2 + + harness.state.reservation = original_reservation + for put_attempt_id, receipt_id, failure in ( + (foreign.put_attempt_id, original.verification_receipt_id, "identity_mismatch"), + (original.put_attempt_id, foreign.verification_receipt_id, "verification_unavailable"), + (foreign.put_attempt_id, foreign.verification_receipt_id, "identity_mismatch"), + ): + async with harness.factory() as session: + with pytest.raises(CheckerOutputUnavailable, match=failure): + async with session.begin(): + await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + original_selector, + put_attempt_id, + receipt_id, + ) + ) + assert len(harness.state.bindings) == consumed_baseline + assert await harness.binding_rows() == binding_baseline + + async def test_binding_rollback_and_concurrent_replay_are_atomic( tmp_path, isolated_database_env, diff --git a/backend/tests/test_checker_output_storage.py b/backend/tests/test_checker_output_storage.py index f40b46332..3d228bf28 100644 --- a/backend/tests/test_checker_output_storage.py +++ b/backend/tests/test_checker_output_storage.py @@ -301,6 +301,67 @@ async def test_checker_attempt_static_custody_allows_only_execution_lifecycle( ) +@pytest.mark.asyncio +async def test_artifact_binding_truncate_custody_blocks_direct_and_cascade_deletion( + isolated_database_env: str, + tmp_path: Path, +) -> None: + """Table-wide deletion cannot bypass immutable artifact binding custody.""" + async with _verified_output(isolated_database_env, tmp_path) as case: + binding_id = str(new_record_id()) + async with case.factory() as session, session.begin(): + await session.execute( + _INSERT_BINDING, + _binding_values(case, id=binding_id), + ) + + for statement in ( + "truncate artifact_bindings", + "truncate artifact_contents cascade", + ): + with pytest.raises(DBAPIError, match="artifact_bindings rows are immutable"): + async with case.factory() as session, session.begin(): + await session.execute(text(statement)) + async with case.factory() as session: + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 1 + + async with case.factory() as session: + transaction = await session.begin() + try: + await session.execute( + text( + "alter table artifact_bindings disable trigger " + "trg_artifact_bindings_no_truncate" + ) + ) + await session.execute(text("truncate artifact_bindings")) + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 0 + finally: + await transaction.rollback() + + async with case.factory() as session: + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 1 + trigger_definition = await session.scalar( + text( + "select pg_get_triggerdef(oid) from pg_trigger " + "where tgrelid='artifact_bindings'::regclass " + "and tgname='trg_artifact_bindings_no_truncate'" + ) + ) + assert trigger_definition is not None + assert "BEFORE TRUNCATE" in trigger_definition + assert "EXECUTE FUNCTION reject_artifact_fact_mutation()" in trigger_definition + + @pytest.mark.asyncio async def test_binding_guard_uses_only_artifact_ancestry_and_exact_owner_fks( isolated_database_env: str, @@ -349,6 +410,7 @@ async def _restore_pre_0007_schema(connection) -> None: """Remove only 0007 custody inside the caller's rollback-only transaction.""" for table, trigger in ( ("artifact_bindings", "checker_output_binding_insert"), + ("artifact_bindings", "trg_artifact_bindings_no_truncate"), ("artifact_put_attempts", "checker_output_put_attempt_custody"), ("artifact_put_attempts", "checker_output_put_attempt_no_truncate"), ): diff --git a/docs/engineering/authorization_activation_custody.md b/docs/engineering/authorization_activation_custody.md index 3a244f8ac..88f86b1a8 100644 --- a/docs/engineering/authorization_activation_custody.md +++ b/docs/engineering/authorization_activation_custody.md @@ -58,8 +58,8 @@ The table retains historical planning-custody labels, not a literal mapping of every typed runtime `ActionOwner`. XINT-06B groups runtime `WS-AUTH-001-ART-06A` post-submit materialization and `WS-AUTH-001-ART-06B` output write/binding. ARCH-04D is their current replacement -activation boundary after delivered ARCH-04B hidden input and ARCH-04B2 hidden -output custody, followed by ARCH-04C durable execution/results. ARCH-04B2 owns +activation boundary. ARCH-04B hidden input and ARCH-04B2 hidden output custody +are delivered; ARCH-04C durable execution/results is next, followed by ARCH-04D. ARCH-04B2 owns fresh authority participants internally for each store, recovery and binding phase; public requests carry selectors and byte sources, never PREP handles. Default output reservation and authority composition remains unavailable, and diff --git a/docs/spec_authorization_service.md b/docs/spec_authorization_service.md index d5d274dc7..d5e6c49b1 100644 --- a/docs/spec_authorization_service.md +++ b/docs/spec_authorization_service.md @@ -537,8 +537,9 @@ runtime `ActionOwner` values or the current implementation boundaries. In particular, the XINT-06B grouping corresponds to runtime `WS-AUTH-001-ART-06A` for post-submit materialization and `WS-AUTH-001-ART-06B` for checker-output write/binding. The current replacement -activation contract is ARCH-04D, after delivered ARCH-04B hidden input and -remaining ARCH-04B2 output custody/ARCH-04C durable execution. It does not reopen XINT-06B as a parallel implementation lane. Likewise ARCH-02G/02H +activation contract is ARCH-04D. ARCH-04B hidden input and ARCH-04B2 output +custody are delivered; ARCH-04C durable execution/results is next, followed by +ARCH-04D activation. This does not reopen XINT-06B as a parallel implementation lane. Likewise ARCH-02G/02H are replacement implementation boundaries, not automatic registry renames. Read exact runtime ownership from the typed catalogue. No planning-only change may promote or reassign an action. From ea402207757d571cfd66d0291af85d3dae22263c Mon Sep 17 00:00:00 2001 From: Commitrail Probe Date: Tue, 29 Sep 2026 09:31:52 +0100 Subject: [PATCH 3/6] test(artifacts): reconcile migration and quota fixtures with custody --- .../WS-ARCH-001/WS-ARCH-001-04B2.md | 13 ++++++-- .../submission_history/test_migration.py | 32 +++++++++++++++++-- backend/tests/test_artifact_authorization.py | 8 ++++- docs/architecture_checker_framework.md | 12 ++++--- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md index 04c8f08cd..83b8fbe3b 100644 --- a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -119,7 +119,9 @@ claim the current structural catalogue supports them. `test_artifact_authorization.py`, `backend/tests/conftest.py` (exact migrated fingerprint and protected-table reset inventory), `backend/alembic/env.py`, `test_alembic.py`, `test_coverage_contract.py` for exact migration-head registration, module boundary - tests and migration tests for traced consumers and retained behavior. + tests and migration tests, including + `backend/tests/authorization/submission_history/test_migration.py` for exact + predecessor-schema restoration across the new dependent foreign key. - Existing lane/ownership/module/test-structure inventories and their tests only to register changed owners and keep existing gates intact. - This record, linked ARCH output skeleton, current ARCH/ART/AUTH/POL/CON navigation, @@ -216,7 +218,14 @@ custody aggregate is authorized. after both valid controls, independently of the database insert guard. The requested-attempt filter removal must make the negative service test fail. - Current authorization specifications must show delivered input/output custody, - then ARCH-04C execution/results, then ARCH-04D activation. + then ARCH-04C execution/results, then ARCH-04D activation. ARCH-04F gates + remediation/public intake and enabling false, not the earlier true routing branch. +- Retained-data migration probes must remove the exact new dependent ART foreign + key when reconstructing the predecessor checker schema and restore it after + committed concurrency probes; no CASCADE shortcut or weakened assertion. The + existing guide quota-reconciliation fixture supplies explicit null checker + custody fields under the expanded internal admission facts; its rollback and + configured-limit assertions remain intact. ## Reconciliation diff --git a/backend/tests/authorization/submission_history/test_migration.py b/backend/tests/authorization/submission_history/test_migration.py index 8b197e093..6ac5a7f72 100644 --- a/backend/tests/authorization/submission_history/test_migration.py +++ b/backend/tests/authorization/submission_history/test_migration.py @@ -68,8 +68,20 @@ async def probe(before): async def _before_custody(connection): - """Restore only this unmerged migration's predecessor checker schema in a transaction.""" + """Restore the history migration's predecessor checker schema in a transaction.""" from sqlalchemy import text + dependent_fk = "fk_artifact_put_attempts_checker_run_ownership" + dependent_fk_definition = await connection.scalar( + text( + "select pg_get_constraintdef(oid) from pg_constraint " + "where conname=:name and conrelid='artifact_put_attempts'::regclass" + ), + {"name": dependent_fk}, + ) + assert isinstance(dependent_fk_definition, str) + await connection.execute( + text(f"alter table artifact_put_attempts drop constraint {dependent_fk}") + ) for table, owner in (("checker_runs", "run"), ("checker_results", "result")): for suffix in ("custody", "no_truncate"): await connection.execute(text(f"drop trigger checker_{owner}_{suffix} on {table}")) @@ -85,6 +97,7 @@ async def _before_custody(connection): ("checker_results", "submission_id", "submissions"), ): await connection.execute(text(f"alter table {table} add constraint fk_{table}_{column}_{parent} foreign key ({column}) references {parent}(id)")) + return dependent_fk_definition def _upgrade_custody(connection): @@ -137,7 +150,7 @@ async def test_checker_migration_locks_out_coherent_rebinding(task_client, monke factory = db_session.get_session_factory() async with factory() as prepare: audit_definition = await prepare.scalar(text("select pg_get_constraintdef(oid) from pg_constraint where conname=:name"), {"name": CONSTRAINT}) - await _before_custody(await prepare.connection()) + dependent_fk_definition = await _before_custody(await prepare.connection()) await prepare.commit() preflight = asyncio.Event() original_execute = op.execute @@ -189,4 +202,19 @@ async def rebind(): assert str(row.task_id) == case[1] and str(row.submission_id) == case[2] await session.execute(text(f"alter table audit_events drop constraint {CONSTRAINT}")) await session.execute(text(f"alter table audit_events add constraint {CONSTRAINT} {audit_definition}")) + await session.execute( + text( + "alter table artifact_put_attempts add constraint " + "fk_artifact_put_attempts_checker_run_ownership " + f"{dependent_fk_definition}" + ) + ) + restored_fk_definition = await session.scalar( + text( + "select pg_get_constraintdef(oid) from pg_constraint " + "where conname='fk_artifact_put_attempts_checker_run_ownership' " + "and conrelid='artifact_put_attempts'::regclass" + ) + ) + assert restored_fk_definition == dependent_fk_definition await session.commit() diff --git a/backend/tests/test_artifact_authorization.py b/backend/tests/test_artifact_authorization.py index 01b722abd..390248719 100644 --- a/backend/tests/test_artifact_authorization.py +++ b/backend/tests/test_artifact_authorization.py @@ -148,7 +148,9 @@ def test_admission_metrics_are_bounded_and_classified() -> None: def test_readiness_is_static_and_aws_never_active() -> None: - disabled = artifact_provider_readiness(Settings()) + disabled = artifact_provider_readiness( + Settings.model_construct(artifact_store_backend="disabled") + ) assert disabled["status"] == "inactive_disabled" assert disabled["active"] is False @@ -192,6 +194,10 @@ async def test_quota_reconciliation_is_configuration_driven_and_rollback_safe() guide_source_snapshot_id=None, checker_run_id=None, logical_role=None, + submission_id=None, + submission_version=None, + checker_request_digest=None, + checker_request_digest_facts=None, pre_submit_evidence_set_id=None, operation_identity="sha256:" + "a" * 64, ) diff --git a/docs/architecture_checker_framework.md b/docs/architecture_checker_framework.md index 05e9dc234..146fa130b 100644 --- a/docs/architecture_checker_framework.md +++ b/docs/architecture_checker_framework.md @@ -523,11 +523,13 @@ severities. ## Checker Run Flow — Target Contract The following is the intended end-to-end lifecycle. Pre-submit intake and -retained-history reads are implemented. Canonical durable post-submit execution, -result routing and recovery remain unavailable pending ARCH-04C/04D/04E/04F; -ARCH-04B hidden input and ARCH-04B2 hidden output custody are delivered with -deny-only/unavailable production composition; -the flow below is not a claim that those jobs or transitions are live. +retained-history reads are implemented. Canonical durable post-submit execution +and result routing remain unavailable pending ARCH-04C/04D/04E. ARCH-04F adds +contributor-correctable remediation and gates public intake and enabling the +false-policy acceptance path; the true `allow_review` route may ship before +ARCH-04F. ARCH-04B hidden input and ARCH-04B2 hidden output custody are delivered +with deny-only/unavailable production composition. The flow below is not a claim +that those jobs or transitions are live. ```text Draft packet From ac3836bede3e4726625a58b2947fad461473f19e Mon Sep 17 00:00:00 2001 From: Commitrail Probe Date: Tue, 29 Sep 2026 12:02:08 +0100 Subject: [PATCH 4/6] Fix checker custody boundaries and seal verified ancestry --- .ci/behavior-ownership/partition.v1.json | 10 +- .../WS-ARCH-001/WS-ARCH-001-04B2.md | 60 ++- .../initiatives/WS-ARCH-001/planning/PLAN.md | 19 +- .../WS-ARCH-001-04C-checker-current-result.md | 9 + .../versions/0007_checker_output_custody.py | 201 ++++++++- backend/app/adapters/artifacts/__init__.py | 13 +- backend/app/interfaces/artifact_operations.py | 64 --- backend/app/modules/artifacts/api/__init__.py | 12 - .../artifacts/checker_output_bindings.py | 12 +- .../artifacts/checker_output_custody.py | 19 +- .../app/modules/artifacts/checker_outputs.py | 12 +- backend/app/modules/artifacts/models.py | 9 + .../artifacts/post_submit_materialization.py | 2 +- .../artifacts/post_submit_selection.py | 2 +- backend/app/modules/artifacts/schemas.py | 12 + backend/app/modules/artifacts/service.py | 9 +- .../api/materialization.py} | 5 +- .../modules/checkers/api/output_custody.py | 67 ++- backend/scripts/behavior_ownership.py | 7 +- .../architecture/test_module_boundaries.py | 47 +++ .../tests/checker_output_custody_helpers.py | 2 +- backend/tests/conftest.py | 2 +- backend/tests/test_artifact_architecture.py | 138 +++++- backend/tests/test_behavior_ownership.py | 13 +- backend/tests/test_checker_output_custody.py | 177 +++++++- backend/tests/test_checker_output_storage.py | 397 +++++++++++++++++- .../tests/test_post_submit_materialization.py | 2 +- backend/tests/test_post_submit_selection.py | 2 +- docs/architecture_data_model.md | 5 +- docs/spec_artifact_storage_service.md | 15 +- 30 files changed, 1168 insertions(+), 176 deletions(-) rename backend/app/modules/{artifacts/api/submission_materialization.py => checkers/api/materialization.py} (94%) diff --git a/.ci/behavior-ownership/partition.v1.json b/.ci/behavior-ownership/partition.v1.json index e3ad9ceb0..5cb0a507d 100644 --- a/.ci/behavior-ownership/partition.v1.json +++ b/.ci/behavior-ownership/partition.v1.json @@ -256,10 +256,6 @@ "group": "artifacts", "target": "backend/app/modules/artifacts/api/submission_admission.py" }, - { - "group": "artifacts", - "target": "backend/app/modules/artifacts/api/submission_materialization.py" - }, { "group": "artifacts", "target": "backend/app/modules/artifacts/api/submission_preparation.py" @@ -736,6 +732,10 @@ "group": "artifacts", "target": "backend/app/modules/checkers/api/history.py" }, + { + "group": "artifacts", + "target": "backend/app/modules/checkers/api/materialization.py" + }, { "group": "artifacts", "target": "backend/app/modules/checkers/api/output_custody.py" @@ -1489,7 +1489,7 @@ "target": "backend/scripts/validate_test_lane_evidence.py" } ], - "authority_digest": "4cb652590f801da4066a1179a5a4ed7df33721a7f685a0b6300d4f46580ac7d0", + "authority_digest": "c836827deba926d6b40689de2d7a723c9640fe209b11edb49d78dacb12b5744e", "protected_base_commit": "7676ce4347db0c9694962a9b587a20765e16eac6", "schema": "workstream.behavior-ownership-partition.v1" } diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md index 83b8fbe3b..d00801674 100644 --- a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -18,8 +18,8 @@ Contributor skip and assignment expiry remain deferred. ## Current behavior Main `9c292100` includes merged ARCH-04B exact verified input and bounded scratch. -`CheckerArtifactOutputPort.store` and `ArtifactBindingPort.bind_checker_output` -are unused declarations. Generic ART preparation, quota admission, durable put, +The output-store and output-binding capabilities begin as unused declarations; +this change gives them CHECKERS-owned consumer contracts and ART implementations. Generic ART preparation, quota admission, durable put, unknown-outcome recovery and independent verification already exist. Checker output admission still reads private CheckerRun rows from ART's repository. Existing CHECKERS rows do not persist 04A request/generation/worker reservation @@ -29,8 +29,11 @@ claim the current structural catalogue supports them. ## Design and decisions -1. Replace the unused output/binding declarations with closed typed requests and - results; no compatibility alias or second storage engine. A CHECKERS-owned +1. Replace the unused output/binding declarations with CHECKERS-owned consumer + requests, results and ports in its public API; ART implements them through + composition injection. The input-materialization port uses the same dependency + direction. Delete the replaced declaration paths; no compatibility alias or + second storage engine. A CHECKERS-owned public reservation-facts port supplies exact run/request/Submission/policy, worker-lease generation and owner-declared output-slot/budget facts. Each slot has a globally unique bounded key within its run, media type and byte limit. Its production implementation is explicitly @@ -81,8 +84,9 @@ claim the current structural catalogue supports them. job -> replica -> attempt -> content lock order; do not introduce the inverse. Generic bindings receive checker-only put_attempt_id and verification_receipt_id. An INSERT guard verifies exact receipt -> job -> attempt -> replica/content - ancestry, verified state and project/run/slot ownership. Reuse the existing - binding immutability trigger; do not invent a common receipt FK across distinct + ancestry, verified state and project/run/slot ownership. Seal terminal + attempt/job facts and replica identity atomically with that binding; preserve + mutable replica health. Reuse the existing binding immutability trigger; do not invent a common receipt FK across distinct direct-put and observation receipt tables. ARCH-04C later composes this participant with its final result and outbox; ARCH-04D owns live service activation. 6. Replace the raw-AuthorizationContext checker admission path with a mandatory @@ -147,7 +151,9 @@ compatibility paths, CI weakening or contributor quota charges for system output - Same run/role/bytes replay produces the same intent and binding; changed request or bytes reject. Concurrent publication and caller rollback preserve one binding and no partially committed effect. -- Database rejects intent/binding identity mutation and mismatched stored ancestry. +- Database rejects intent/binding identity mutation, mismatched stored ancestry, + and subsequent changes to sealed verified ancestry, including concurrent writes + under READ COMMITTED and REPEATABLE READ. Migration refuses retained checker attempts lacking provable evaluation custody; it never invents the missing checker-request digest or deletes retained data. - Unknown put outcome resumes observation of the same intent without regenerating @@ -237,3 +243,43 @@ custody aggregate is authorized. contributor milestone before live human review/revision. - Remaining risks: CHECKERS reservation producer and live authority deliberately absent; only controlled hidden storage proof is claimed by this child. + +## External review repair scope + +The output capability must have a typed acyclic consumer/implementation seam; +placing its declarations in shared `app.interfaces` does not remove the +ART-to-CHECKERS dependency. CHECKERS owns the consumed output and materialization +ports in its public API; ART implements them and composition injects them. Move +the existing ART materialization contract to that consumer API and delete its +old path. ARCH-04C imports its own consumer ports, never ART API/private owners. +The existing pre-submit archive-execution private ART dependency remains a +separately tracked consumer; it is not a new post-submit dependency or fallback. Replace that affected declaration path, update all +callers and boundary proof, and make ARCH-04C consume the declared seam without +importing ART implementation. No compatibility re-export is allowed. + +Verified binding custody must survive subsequent SQL mutations, not merely pass +an insertion check. Protect the ancestry facts used by the inserted binding, +including put execution state and references, while preserving legitimate +unbound put/observation recovery. Prove post-bind mutations fail and leave the +binding and its exact chain unchanged; include the competing-transaction case. + +Add focused service proofs for identical `store()` replay without another put +and cancellation while provider I/O is paused with scratch cleanup. Share the +canonical run/slot operation identity instead of rebuilding its hash. Reconcile +the current ARCH plan's stale missing-storage sentence. These are repairs within +the existing L1 boundary and require fresh architecture/reuse, security, +QA/test-delta, documentation/product and CI-integrity review. Existing prohibited +changes and human review focus remain unchanged. The affected owner API, +architecture boundary tests and ARCH-04C contract are included in allowed scope. + +The ancestry repair seals the terminal attempt and verification job plus replica +identity in the successful binding transaction. A monotonic row-local seal is +needed because a transaction using an older PostgreSQL snapshot could miss a +newly inserted binding if protection relied only on cross-table existence checks. +Binding takes the existing job -> replica -> attempt -> content order with update +locks on the sealed ancestors, validates the exact chain, and sets seals atomically. +The attempt/job terminal facts and replica identity cannot change afterward; +replica health and availability remain operational facts that may change. Failed +or rolled-back publication must not leave seals behind. Real PostgreSQL proof +covers both READ COMMITTED and REPEATABLE READ interleavings, not only sequential +mutation. No separate custody aggregate or public capability is introduced. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md index 43599d603..ad412c883 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md @@ -119,15 +119,18 @@ checker finalization or task transitions. The implementation contract must name those feature action/resource manifests, not infer authority from the event type. Lost delivery/redelivery cannot create a new logical evaluation. -ART checker-output storage is also missing, not implicit in CHECKERS result -persistence. An ART-owned child of 04B supplies bounded generated-output/log +Hidden ARCH-04B2 ART checker-output storage and binding are delivered, not +implicit in CHECKERS result persistence. They supply bounded generated-output ingestion, generic quota admission attributed to the fixed service, independent -reread verification and exact checker-output binding. Its controlled hidden -fixtures use 04A public request/run facts, not a future TASK dispatch row or -private CHECKERS import. 04C composes the resulting verified binding references -with final-result persistence; 04D activates the exact ART write/binding and -CHECKERS completion surfaces. External byte I/O occurs before the final caller -transaction; binding publication and final result become visible atomically. +reread verification and exact checker-output binding. Production CHECKERS +reservation/currentness and output authority remain unavailable, and the current +structural catalogue reserves zero output slots; controlled nonempty fixtures +prove ART mechanics only. ARCH-04C is next and must accept that empty output set; +when owner-declared slots exist it composes their verified binding references +with final-result persistence. ARCH-04D then activates the exact ART write/binding +and CHECKERS completion surfaces. In that planned flow, external byte I/O occurs +before the final caller transaction; binding publication and final result become +visible atomically. Failed storage never becomes contributor blame or an `allow_review` result. No second artifact store, quota ledger or historical ART-06B implementation lane is introduced. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md index 794974ed5..89773157e 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md @@ -17,6 +17,13 @@ Execution must reuse the POL-003 single checker-service port and or caller-triggered execution path is allowed. Production remains fail-closed until 04D activates the exact fixed-service boundaries. +CHECKERS owns the consumer contracts for post-submit materialization and output +custody in its public `api.materialization` and `api.output_custody` modules. +04C consumes only those CHECKERS-owned ports; application composition injects +the ART implementations. 04C must not import `app.interfaces.artifact_operations`, +the ART public API, or private ART modules. This preserves the one-way ART +implementation-to-CHECKERS-API dependency and prevents a public module cycle. + This is the only owner of the durable post-submit attempt, member-result, final-result, supersession/currentness and worker-recovery writes. POL-07 declares the facade contract, not a competing persistence implementation. @@ -78,6 +85,8 @@ stale, unverified or mismatched owner-required output prevents final-current result/routing. Do not implement ART writes here, persist provider locations as evidence, charge output bytes to a contributor, or turn ARCH-04B2's controlled nonempty fixtures into catalogue requirements. +This contract relocation does not change the current zero-slot catalogue or +activate output production. The final multi-participant transaction must declare one lock order consistent with AUTH PREP: prepare required service authority custody before feature locks, diff --git a/backend/alembic/versions/0007_checker_output_custody.py b/backend/alembic/versions/0007_checker_output_custody.py index c8010fdf2..deb70574a 100644 --- a/backend/alembic/versions/0007_checker_output_custody.py +++ b/backend/alembic/versions/0007_checker_output_custody.py @@ -31,6 +31,33 @@ def upgrade() -> None: "artifact_put_attempts", sa.Column("checker_request_digest", sa.String(length=71), nullable=True), ) + op.add_column( + "artifact_put_attempts", + sa.Column( + "checker_output_custody_sealed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "artifact_verification_jobs", + sa.Column( + "checker_output_custody_sealed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "artifact_replicas", + sa.Column( + "checker_output_custody_sealed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) op.add_column( "artifact_bindings", sa.Column("put_attempt_id", sa.Uuid(), nullable=True), @@ -133,6 +160,7 @@ def upgrade() -> None: ) _install_checker_output_attempt_custody() + _install_checker_output_ancestor_custody() _install_checker_output_binding_guard() @@ -190,12 +218,37 @@ def _install_checker_output_attempt_custody() -> None: LANGUAGE plpgsql AS $$ BEGIN IF TG_OP='DELETE' THEN - IF OLD.producer_request_type='checker_output' THEN + IF OLD.producer_request_type='checker_output' + OR OLD.checker_output_custody_sealed THEN RAISE EXCEPTION 'checker output put attempt custody is immutable' USING ERRCODE='55000'; END IF; RETURN OLD; END IF; + IF OLD.checker_output_custody_sealed THEN + IF NOT NEW.checker_output_custody_sealed + OR (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF NEW.checker_output_custody_sealed THEN + IF (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') + OR NOT EXISTS ( + SELECT 1 FROM artifact_bindings binding + WHERE binding.resource_type='checker_run' + AND binding.put_attempt_id=OLD.id + ) THEN + RAISE EXCEPTION 'checker output put attempt seal requires binding' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; IF OLD.producer_request_type='checker_output' OR NEW.producer_request_type='checker_output' THEN IF ROW( @@ -251,6 +304,111 @@ def _install_checker_output_attempt_custody() -> None: ) +def _install_checker_output_ancestor_custody() -> None: + op.execute( + """ + CREATE FUNCTION guard_checker_output_verification_job_custody() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN + IF OLD.checker_output_custody_sealed THEN + RAISE EXCEPTION 'checker output verification job custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN OLD; + END IF; + IF OLD.checker_output_custody_sealed THEN + IF NOT NEW.checker_output_custody_sealed + OR (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') THEN + RAISE EXCEPTION 'checker output verification job custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF NEW.checker_output_custody_sealed THEN + IF (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') + OR NOT EXISTS ( + SELECT 1 + FROM artifact_bindings binding + JOIN artifact_verification_receipts receipt + ON receipt.id=binding.verification_receipt_id + WHERE binding.resource_type='checker_run' + AND receipt.verification_job_id=OLD.id + ) THEN + RAISE EXCEPTION 'checker output verification job seal requires binding' + USING ERRCODE='55000'; + END IF; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_verification_job_custody " + "BEFORE DELETE OR UPDATE ON artifact_verification_jobs FOR EACH ROW " + "EXECUTE FUNCTION guard_checker_output_verification_job_custody()" + ) + op.execute( + """ + CREATE FUNCTION guard_checker_output_replica_custody() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN + IF OLD.checker_output_custody_sealed THEN + RAISE EXCEPTION 'checker output replica custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN OLD; + END IF; + IF OLD.checker_output_custody_sealed THEN + IF NOT NEW.checker_output_custody_sealed + OR ROW( + NEW.id, NEW.content_id, NEW.storage_namespace_id, + NEW.namespace_fingerprint, NEW.adapter, + NEW.provider_profile, NEW.provider_object_ref + ) IS DISTINCT FROM ROW( + OLD.id, OLD.content_id, OLD.storage_namespace_id, + OLD.namespace_fingerprint, OLD.adapter, + OLD.provider_profile, OLD.provider_object_ref + ) THEN + RAISE EXCEPTION 'checker output replica custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF NEW.checker_output_custody_sealed THEN + IF (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') + OR NOT EXISTS ( + SELECT 1 + FROM artifact_bindings binding + JOIN artifact_verification_receipts receipt + ON receipt.id=binding.verification_receipt_id + JOIN artifact_verification_jobs job + ON job.id=receipt.verification_job_id + WHERE binding.resource_type='checker_run' + AND job.replica_id=OLD.id + ) THEN + RAISE EXCEPTION 'checker output replica seal requires binding' + USING ERRCODE='55000'; + END IF; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_replica_custody " + "BEFORE DELETE OR UPDATE ON artifact_replicas FOR EACH ROW " + "EXECUTE FUNCTION guard_checker_output_replica_custody()" + ) + + def _install_checker_output_binding_guard() -> None: op.execute( """ @@ -279,15 +437,15 @@ def _install_checker_output_binding_guard() -> None: SELECT value.* INTO job FROM artifact_verification_jobs value WHERE value.id=verification.verification_job_id - FOR SHARE; + FOR UPDATE; SELECT value.* INTO replica FROM artifact_replicas value WHERE value.id=job.replica_id - FOR SHARE; + FOR UPDATE; SELECT value.* INTO attempt FROM artifact_put_attempts value WHERE value.id=job.originating_put_attempt_id - FOR SHARE; + FOR UPDATE; SELECT value.* INTO content FROM artifact_contents value WHERE value.id=replica.content_id @@ -365,6 +523,41 @@ def _install_checker_output_binding_guard() -> None: "CREATE TRIGGER checker_output_binding_insert BEFORE INSERT ON artifact_bindings " "FOR EACH ROW EXECUTE FUNCTION guard_checker_output_binding_insert()" ) + op.execute( + """ + CREATE FUNCTION seal_checker_output_binding_ancestry() RETURNS trigger + LANGUAGE plpgsql AS $$ + DECLARE + verification artifact_verification_receipts%ROWTYPE; + job artifact_verification_jobs%ROWTYPE; + BEGIN + IF NEW.resource_type <> 'checker_run' THEN + RETURN NEW; + END IF; + SELECT receipt.* INTO STRICT verification + FROM artifact_verification_receipts receipt + WHERE receipt.id=NEW.verification_receipt_id; + SELECT value.* INTO STRICT job + FROM artifact_verification_jobs value + WHERE value.id=verification.verification_job_id; + + UPDATE artifact_verification_jobs + SET checker_output_custody_sealed=true + WHERE id=job.id AND NOT checker_output_custody_sealed; + UPDATE artifact_replicas + SET checker_output_custody_sealed=true + WHERE id=job.replica_id AND NOT checker_output_custody_sealed; + UPDATE artifact_put_attempts + SET checker_output_custody_sealed=true + WHERE id=NEW.put_attempt_id AND NOT checker_output_custody_sealed; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_binding_seal AFTER INSERT ON artifact_bindings " + "FOR EACH ROW EXECUTE FUNCTION seal_checker_output_binding_ancestry()" + ) op.execute( "CREATE TRIGGER trg_artifact_bindings_no_truncate BEFORE TRUNCATE " "ON artifact_bindings FOR EACH STATEMENT " diff --git a/backend/app/adapters/artifacts/__init__.py b/backend/app/adapters/artifacts/__init__.py index 90181d8a2..59c38b8f9 100644 --- a/backend/app/adapters/artifacts/__init__.py +++ b/backend/app/adapters/artifacts/__init__.py @@ -19,7 +19,12 @@ from app.db.session import get_db_session from app.interfaces.artifact_operations import GuideArtifactIngestCommand from app.modules.projects.api.guide_documents import GuideDocumentUploadTargetPort -from app.modules.artifacts.api import SubmissionBundlePreparationCommand, PostSubmissionMaterializationPort +from app.modules.artifacts.api import SubmissionBundlePreparationCommand +from app.modules.checkers.api.materialization import PostSubmissionMaterializationPort +from app.modules.checkers.api.output_custody import ( + CheckerArtifactOutputPort, + CheckerOutputBindingPort, +) from app.interfaces.artifacts import ( ARTIFACT_STORE_CAPABILITY_KEY, ArtifactConfigurationError, @@ -513,7 +518,9 @@ def post_submission_materialization(*, sessions, store, namespace, preparation, ) -def checker_output_storage(*, sessions, store, namespace, preparation, settings): +def checker_output_storage( + *, sessions, store, namespace, preparation, settings +) -> CheckerArtifactOutputPort: """Compose hidden output custody; live producer and write authority remain absent.""" from app.modules.artifacts.checker_outputs import CheckerArtifactOutputService, DenyCheckerOutputWriteAuthority from app.modules.artifacts.schemas import DenyArtifactInternalAuthority @@ -526,7 +533,7 @@ def checker_output_storage(*, sessions, store, namespace, preparation, settings) ) -def checker_output_binding(session, *, namespace): +def checker_output_binding(session, *, namespace) -> CheckerOutputBindingPort: """Compose a deny-only caller-transaction binding participant.""" from app.modules.artifacts.checker_output_bindings import CheckerOutputBindingService, DenyCheckerOutputBindingAuthority from app.modules.checkers.api.output_custody import UnavailableCheckerOutputReservation diff --git a/backend/app/interfaces/artifact_operations.py b/backend/app/interfaces/artifact_operations.py index 8f02e9c20..4342692e5 100644 --- a/backend/app/interfaces/artifact_operations.py +++ b/backend/app/interfaces/artifact_operations.py @@ -9,20 +9,12 @@ from app.modules.authorization.prepared import PreparedAuthorizationHandle from app.modules.authorization.runtime import AuthorizationContext -from app.modules.checkers.api.output_custody import CheckerOutputSelector - __all__ = ( "ArtifactAuditResourceType", "ArtifactBindingResourceType", - "ArtifactBindingPort", "ArtifactOperatorReadPort", "ArtifactOperatorRecoveryPort", "ArtifactRecoveryRequest", - "CheckerArtifactOutputPort", - "CheckerOutputBindingRequest", - "CheckerOutputArtifactRequest", - "CheckerOutputArtifactResult", - "CheckerOutputBindingResult", "GuideArtifactIngestPort", "GuideArtifactIngestCommand", "GuideArtifactIngestRequest", @@ -76,45 +68,6 @@ class GuideArtifactIngestResult: replayed: bool -@dataclass(frozen=True, slots=True) -class CheckerOutputBindingRequest: - """Bind one exact independently verified output in the caller transaction.""" - - selector: CheckerOutputSelector - put_attempt_id: UUID - verification_receipt_id: UUID - - -@dataclass(frozen=True, slots=True) -class CheckerOutputArtifactRequest: - """Bounded generated bytes; action authority is freshly prepared by ART.""" - - selector: CheckerOutputSelector - byte_source: AsyncIterable[bytes] - - -@dataclass(frozen=True, slots=True) -class CheckerOutputArtifactResult: - """Durable put identity and verified custody, without provider coordinates.""" - - put_attempt_id: UUID - status: str - content_id: UUID | None - verification_receipt_id: UUID | None - replayed: bool - - -@dataclass(frozen=True, slots=True) -class CheckerOutputBindingResult: - """Immutable binding participating in the caller's final-result transaction.""" - - binding_id: UUID - content_id: UUID - put_attempt_id: UUID - verification_receipt_id: UUID - replayed: bool - - @dataclass(frozen=True, slots=True) class ArtifactRecoveryRequest: """Reason-bound Operator retry of one exact verification job.""" @@ -154,23 +107,6 @@ async def ingest( """Prepare authority before delegating to durable byte ingestion.""" -class ArtifactBindingPort(Protocol): - """Create exact action-bound bindings from verified content.""" - - async def bind_checker_output(self, request: CheckerOutputBindingRequest) -> CheckerOutputBindingResult: - """Bind verified checker output under the checker binding action.""" - - -class CheckerArtifactOutputPort(Protocol): - """Store generated output for one fixed checker execution.""" - - async def store(self, request: CheckerOutputArtifactRequest) -> CheckerOutputArtifactResult: - """Store one generated checker artifact.""" - - async def recover(self, selector: CheckerOutputSelector) -> CheckerOutputArtifactResult | None: - """Recover a lost output response without a byte source or regeneration.""" - - class ArtifactOperatorReadPort(Protocol): """Expose bounded Operator reads without provider references.""" diff --git a/backend/app/modules/artifacts/api/__init__.py b/backend/app/modules/artifacts/api/__init__.py index bb1c7602f..47813c349 100644 --- a/backend/app/modules/artifacts/api/__init__.py +++ b/backend/app/modules/artifacts/api/__init__.py @@ -31,15 +31,3 @@ "SubmissionAdmissionConsumptionResult", "SubmissionAdmissionConsumptionStatus", ) - -from app.modules.artifacts.api.submission_materialization import ( - PostSubmissionMaterialConsumer, PostSubmissionMaterializationPort, - PostSubmissionMaterializationResult, PostSubmissionMaterializationUnavailable, - SubmissionMaterialEntry, SubmissionMaterialView, -) - -__all__ += ( - "PostSubmissionMaterialConsumer", "PostSubmissionMaterializationPort", - "PostSubmissionMaterializationResult", "PostSubmissionMaterializationUnavailable", - "SubmissionMaterialEntry", "SubmissionMaterialView", -) diff --git a/backend/app/modules/artifacts/checker_output_bindings.py b/backend/app/modules/artifacts/checker_output_bindings.py index 547d50a78..23285287b 100644 --- a/backend/app/modules/artifacts/checker_output_bindings.py +++ b/backend/app/modules/artifacts/checker_output_bindings.py @@ -7,9 +7,13 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.identifiers import new_record_id -from app.interfaces.artifact_operations import ( +from app.modules.checkers.api.output_custody import ( CheckerOutputBindingRequest, CheckerOutputBindingResult, + CheckerOutputReservation, + CheckerOutputReservationPort, + CheckerOutputSelector, + CheckerOutputUnavailable, ) from app.modules.actors.api import ServiceIdentity from app.modules.artifacts.checker_output_custody import ( @@ -17,12 +21,6 @@ select_checker_output, ) from app.modules.artifacts.models import ArtifactBinding -from app.modules.checkers.api.output_custody import ( - CheckerOutputReservation, - CheckerOutputReservationPort, - CheckerOutputSelector, - CheckerOutputUnavailable, -) class CheckerOutputBindingAuthority(Protocol): diff --git a/backend/app/modules/artifacts/checker_output_custody.py b/backend/app/modules/artifacts/checker_output_custody.py index 1079f6449..beff8fdb2 100644 --- a/backend/app/modules/artifacts/checker_output_custody.py +++ b/backend/app/modules/artifacts/checker_output_custody.py @@ -15,7 +15,10 @@ ArtifactVerificationJob, ArtifactVerificationReceipt, ) -from app.modules.artifacts.schemas import checker_output_request_digest_facts +from app.modules.artifacts.schemas import ( + checker_output_operation_identity, + checker_output_request_digest_facts, +) from app.modules.checkers.api.output_custody import ( CheckerOutputReservation, CheckerOutputSelector, @@ -38,16 +41,6 @@ class CheckerOutputStoredFacts: verification_receipt_id: UUID | None -def output_operation_identity(selector: CheckerOutputSelector) -> str: - """Use one logical output identity across worker-lease replacement.""" - return canonical_json_hash( - { - "request_type": "checker_output", - "checker_run_id": str(selector.checker_run_id), - "logical_role": selector.slot_key, - } - ) - async def select_checker_output( session: AsyncSession, @@ -70,7 +63,9 @@ async def select_checker_output( attempt = await session.scalar( select(ArtifactPutAttempt) .where( - ArtifactPutAttempt.operation_identity == output_operation_identity(selector), + ArtifactPutAttempt.operation_identity == checker_output_operation_identity( + checker_run_id=selector.checker_run_id, slot_key=selector.slot_key + ), ) .execution_options(populate_existing=True) ) diff --git a/backend/app/modules/artifacts/checker_outputs.py b/backend/app/modules/artifacts/checker_outputs.py index 6a81a9382..c2a9ce0d7 100644 --- a/backend/app/modules/artifacts/checker_outputs.py +++ b/backend/app/modules/artifacts/checker_outputs.py @@ -8,9 +8,13 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from app.core.config import Settings -from app.interfaces.artifact_operations import ( +from app.modules.checkers.api.output_custody import ( CheckerOutputArtifactRequest, CheckerOutputArtifactResult, + CheckerOutputReservation, + CheckerOutputReservationPort, + CheckerOutputSelector, + CheckerOutputUnavailable, ) from app.interfaces.artifacts import ArtifactStore from app.modules.artifacts.checker_output_custody import ( @@ -28,12 +32,6 @@ ArtifactStorageOrchestrator, ArtifactStorageNamespaceSpec, ) -from app.modules.checkers.api.output_custody import ( - CheckerOutputReservation, - CheckerOutputReservationPort, - CheckerOutputSelector, - CheckerOutputUnavailable, -) class CheckerOutputWriteAuthority(Protocol): diff --git a/backend/app/modules/artifacts/models.py b/backend/app/modules/artifacts/models.py index 0d6db1134..9215b89e3 100644 --- a/backend/app/modules/artifacts/models.py +++ b/backend/app/modules/artifacts/models.py @@ -1136,6 +1136,9 @@ class ArtifactPutAttempt(Base): operation_identity: Mapped[str] = mapped_column(String(71), nullable=False) request_digest: Mapped[str] = mapped_column(String(71), nullable=False) checker_request_digest: Mapped[str | None] = mapped_column(String(71)) + checker_output_custody_sealed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) status: Mapped[str] = mapped_column(String(40), nullable=False, default="prepared", index=True) next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) executor_id: Mapped[str | None] = mapped_column(Uuid(as_uuid=False)) @@ -1365,6 +1368,9 @@ class ArtifactReplica(Base): adapter: Mapped[str] = mapped_column(String(50), nullable=False) provider_profile: Mapped[str] = mapped_column(String(100), nullable=False) provider_object_ref: Mapped[str] = mapped_column(String(1024), nullable=False) + checker_output_custody_sealed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) verification_state: Mapped[str] = mapped_column(String(30), nullable=False) availability_state: Mapped[str] = mapped_column(String(30), nullable=False) integrity_state: Mapped[str] = mapped_column(String(30), nullable=False) @@ -1506,6 +1512,9 @@ class ArtifactVerificationJob(Base): replica_id: Mapped[str] = mapped_column( ForeignKey("artifact_replicas.id", ondelete="RESTRICT"), nullable=False, index=True ) + checker_output_custody_sealed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) status: Mapped[str] = mapped_column(String(40), nullable=False, default="pending", index=True) attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) maximum_attempts: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/backend/app/modules/artifacts/post_submit_materialization.py b/backend/app/modules/artifacts/post_submit_materialization.py index 897f291e8..6d84db997 100644 --- a/backend/app/modules/artifacts/post_submit_materialization.py +++ b/backend/app/modules/artifacts/post_submit_materialization.py @@ -8,7 +8,7 @@ from app.core.cancellation import await_cancellation_resistant from app.interfaces.artifacts import ArtifactStore -from app.modules.artifacts.api.submission_materialization import ( +from app.modules.checkers.api.materialization import ( PostSubmissionMaterialConsumer, PostSubmissionMaterializationResult, PostSubmissionMaterializationUnavailable, SubmissionMaterialEntry, ) diff --git a/backend/app/modules/artifacts/post_submit_selection.py b/backend/app/modules/artifacts/post_submit_selection.py index a637d0c76..4e01cada7 100644 --- a/backend/app/modules/artifacts/post_submit_selection.py +++ b/backend/app/modules/artifacts/post_submit_selection.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.interfaces.artifacts import ArtifactStore, artifact_provider_object_ref -from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.checkers.api.materialization import PostSubmissionMaterializationUnavailable from app.modules.artifacts.models import ( ArtifactBinding, ArtifactContent, ArtifactReplica, ArtifactStorageNamespace, ArtifactVerificationJob, ArtifactVerificationReceipt, PreSubmitEvidenceSet, diff --git a/backend/app/modules/artifacts/schemas.py b/backend/app/modules/artifacts/schemas.py index 28bcb9acd..b5a012813 100644 --- a/backend/app/modules/artifacts/schemas.py +++ b/backend/app/modules/artifacts/schemas.py @@ -7,6 +7,7 @@ from typing import Protocol, TypeAlias, final from uuid import UUID +from app.core.hashing import canonical_json_hash from app.modules.artifacts.sources import CommittedArtifactSource from app.modules.checkers.api.output_custody import ( CheckerOutputReservation, @@ -41,6 +42,17 @@ class CheckerOutputArtifactAdmissionRequest: source: CommittedArtifactSource +def checker_output_operation_identity(*, checker_run_id: UUID, slot_key: str) -> str: + """Keep one logical output identity across worker-lease replacement.""" + return canonical_json_hash( + { + "request_type": "checker_output", + "checker_run_id": str(checker_run_id), + "logical_role": slot_key, + } + ) + + def checker_output_request_digest_facts( *, reservation: CheckerOutputReservation, diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index 8b7d92cdd..e0117cbff 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -98,6 +98,7 @@ GuideArtifactIngestAuthorityFacts, SubmissionBundleArtifactAdmissionRequest, SubmissionBundleDurableIntentAuthorityFacts, + checker_output_operation_identity, checker_output_request_digest_facts, ) from app.modules.artifacts.submission_authorization import ( @@ -2276,12 +2277,8 @@ async def _checker_output_facts( evaluation = reservation.evaluation checker_run_id = str(reservation.checker_run_id) logical_role = slot.key - operation_identity = canonical_json_hash( - { - "request_type": "checker_output", - "checker_run_id": checker_run_id, - "logical_role": logical_role, - } + operation_identity = checker_output_operation_identity( + checker_run_id=reservation.checker_run_id, slot_key=logical_role ) digest_facts = checker_output_request_digest_facts( reservation=reservation, diff --git a/backend/app/modules/artifacts/api/submission_materialization.py b/backend/app/modules/checkers/api/materialization.py similarity index 94% rename from backend/app/modules/artifacts/api/submission_materialization.py rename to backend/app/modules/checkers/api/materialization.py index d6b918f6e..1a109bc75 100644 --- a/backend/app/modules/artifacts/api/submission_materialization.py +++ b/backend/app/modules/checkers/api/materialization.py @@ -4,7 +4,10 @@ from typing import Literal, Protocol from uuid import UUID -from app.modules.checkers.api import PostSubmissionEvaluationRequest, PostSubmissionEvaluationResult +from app.modules.checkers.api.post_submit import ( + PostSubmissionEvaluationRequest, + PostSubmissionEvaluationResult, +) class PostSubmissionMaterializationUnavailable(RuntimeError): diff --git a/backend/app/modules/checkers/api/output_custody.py b/backend/app/modules/checkers/api/output_custody.py index 4f65cdbda..5e0b6f599 100644 --- a/backend/app/modules/checkers/api/output_custody.py +++ b/backend/app/modules/checkers/api/output_custody.py @@ -1,6 +1,9 @@ -"""CHECKERS-owned output reservation facts; production reservations are unavailable.""" +"""CHECKERS-owned output custody contracts; production reservations are unavailable.""" +from collections.abc import AsyncIterable +from dataclasses import dataclass from typing import Annotated, Literal, Protocol, Self +from uuid import UUID from pydantic import Field, StrictInt, model_validator @@ -80,6 +83,68 @@ async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReserva """Return exact current facts under the caller's CHECKERS transaction locks.""" +@dataclass(frozen=True, slots=True) +class CheckerOutputBindingRequest: + """Bind one exact independently verified output in the caller transaction.""" + + selector: CheckerOutputSelector + put_attempt_id: UUID + verification_receipt_id: UUID + + +@dataclass(frozen=True, slots=True) +class CheckerOutputArtifactRequest: + """Bounded generated bytes; action authority is freshly prepared by ART.""" + + selector: CheckerOutputSelector + byte_source: AsyncIterable[bytes] + + +@dataclass(frozen=True, slots=True) +class CheckerOutputArtifactResult: + """Durable put identity and verified custody, without provider coordinates.""" + + put_attempt_id: UUID + status: str + content_id: UUID | None + verification_receipt_id: UUID | None + replayed: bool + + +@dataclass(frozen=True, slots=True) +class CheckerOutputBindingResult: + """Immutable binding participating in the caller's final-result transaction.""" + + binding_id: UUID + content_id: UUID + put_attempt_id: UUID + verification_receipt_id: UUID + replayed: bool + + +class CheckerOutputBindingPort(Protocol): + """Create exact action-bound bindings from verified content.""" + + async def bind_checker_output( + self, request: CheckerOutputBindingRequest + ) -> CheckerOutputBindingResult: + """Bind verified checker output under the checker binding action.""" + + +class CheckerArtifactOutputPort(Protocol): + """Store generated output for one fixed checker execution.""" + + async def store( + self, request: CheckerOutputArtifactRequest + ) -> CheckerOutputArtifactResult: + """Store one generated checker artifact.""" + + async def recover( + self, selector: CheckerOutputSelector + ) -> CheckerOutputArtifactResult | None: + """Recover a lost output response without a byte source or regeneration.""" + + class UnavailableCheckerOutputReservation: """ARCH-04C must install the real reservation producer before activation.""" diff --git a/backend/scripts/behavior_ownership.py b/backend/scripts/behavior_ownership.py index d86699f8d..d8158a33d 100644 --- a/backend/scripts/behavior_ownership.py +++ b/backend/scripts/behavior_ownership.py @@ -153,12 +153,15 @@ "backend/app/modules/checkers/api/output_custody.py", }) ARCH_04B_MATERIALIZATION_TARGETS = frozenset({ - "backend/app/modules/artifacts/api/submission_materialization.py", + "backend/app/modules/checkers/api/materialization.py", "backend/app/modules/artifacts/post_submit_materialization.py", "backend/app/modules/artifacts/post_submit_selection.py", "backend/app/modules/tasks/api/submitted_bundle.py", "backend/app/modules/tasks/submitted_bundle.py", }) +ARCH_04B_SEAM_REMOVED_TARGETS = frozenset({ + "backend/app/modules/artifacts/api/submission_materialization.py", +}) ARCH_04A_POST_SUBMIT_TARGETS = frozenset( { "backend/app/modules/checkers/api/post_submit.py", @@ -666,7 +669,7 @@ def _validate_additive_partition_transition( ] if ( trusted_targets != sorted(trusted_targets) - or removed - (ARCH_03D_REMOVED_TARGETS | TASK_CHECKER_CLEANUP_REMOVED_TARGETS | ARCH_03C2_REMOVED_TARGETS | OUTBOX_IDENTITY_REMOVED_TARGETS | V01_BASELINE_REMOVED_TARGETS | POL_03B_REMOVED_TARGETS | POL_04B_REMOVED_TARGETS | POL_05A_REMOVED_TARGETS | ARCH_03A_GUIDE_CONTEXT_REMOVED_TARGETS) + or removed - (ARCH_03D_REMOVED_TARGETS | TASK_CHECKER_CLEANUP_REMOVED_TARGETS | ARCH_03C2_REMOVED_TARGETS | OUTBOX_IDENTITY_REMOVED_TARGETS | V01_BASELINE_REMOVED_TARGETS | POL_03B_REMOVED_TARGETS | POL_04B_REMOVED_TARGETS | POL_05A_REMOVED_TARGETS | ARCH_03A_GUIDE_CONTEXT_REMOVED_TARGETS | ARCH_04B_SEAM_REMOVED_TARGETS) or [current_by_target[item["target"]] for item in retained_trusted] != retained_trusted ): diff --git a/backend/tests/architecture/test_module_boundaries.py b/backend/tests/architecture/test_module_boundaries.py index 649eb20a9..e5615a3d1 100644 --- a/backend/tests/architecture/test_module_boundaries.py +++ b/backend/tests/architecture/test_module_boundaries.py @@ -413,6 +413,53 @@ def test_cyclic_public_dependencies_fail_closed() -> None: boundary._validate_acyclic(graph) # noqa: SLF001 - architecture proof +def test_art_implements_checker_consumer_ports_without_a_public_cycle() -> None: + """The delivered seam is ART implementation to CHECKERS-owned public ports.""" + registry = boundary.load_registry(REGISTRY) + private, graph, _ = boundary.scan(ROOT, registry) + assert "checkers" in graph["artifacts"] + assert "artifacts" not in graph["checkers"] + assert { + (edge.source_file, edge.imported_private_path) + for edge in private + if edge.source_file.startswith("backend/app/modules/checkers/") + and edge.target_module == "artifacts" + } == { + ( + "backend/app/modules/checkers/pre_submit_execution.py", + "app.modules.artifacts.sources", + ), + ( + "backend/app/modules/checkers/pre_submit_execution.py", + "app.modules.artifacts.submission_archive", + ), + ( + "backend/app/modules/checkers/pre_submit_execution.py", + "app.modules.artifacts.submission_manifest", + ), + } + boundary._validate_acyclic(graph) # noqa: SLF001 - architecture proof + + +def test_checker_to_art_public_import_would_close_a_cycle(tmp_path: Path) -> None: + """A concrete future CHECKERS import of ART public API fails closed.""" + registry_path = tmp_path / "registry.json" + _registry(registry_path) + _write( + tmp_path / "backend/app/modules/artifacts/implementation.py", + "from app.modules.checkers.api.output_custody import CheckerArtifactOutputPort\n", + ) + _write( + tmp_path / "backend/app/modules/checkers/future_execution.py", + "from app.modules.artifacts.api import SubmissionBundlePreparationCommand\n", + ) + _, graph, _ = boundary.scan(tmp_path, boundary.load_registry(registry_path)) + assert graph["artifacts"] == {"checkers"} + assert graph["checkers"] == {"artifacts"} + with pytest.raises(boundary.ModuleBoundaryError, match="cyclic_public_dependency"): + boundary._validate_acyclic(graph) # noqa: SLF001 - architecture proof + + def test_general_ledger_rejects_copied_authorization_edges(tmp_path: Path) -> None: """AUTH debt remains exclusively owned by the AUTH-003 ledger.""" document = { diff --git a/backend/tests/checker_output_custody_helpers.py b/backend/tests/checker_output_custody_helpers.py index 3e2c7de89..d238eb566 100644 --- a/backend/tests/checker_output_custody_helpers.py +++ b/backend/tests/checker_output_custody_helpers.py @@ -15,7 +15,7 @@ from app.adapters.artifacts import create_artifact_store_bootstrap from app.core.identifiers import new_record_id -from app.interfaces.artifact_operations import ( +from app.modules.checkers.api.output_custody import ( CheckerOutputArtifactRequest, CheckerOutputArtifactResult, CheckerOutputBindingRequest, diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 959d894f1..809063ff5 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -23,7 +23,7 @@ DDL_LOCK_DIRECTORY = Path("/tmp") # Match the PostgreSQL 16 engine used by Backend CI. Catalog identity rendering # differs across major versions; regenerate only after comparing actual objects. -EXPECTED_PUBLIC_SCHEMA_SHA256 = "e2af78293f8dfef88b8a185eb52961ed337957332617c62b342f3e3c857e43f6" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "1f23100623e232c0e231fa20f9b2ad9599c2e2b8db9c0b42732d189f95ae5a0c" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index 4f45b9a1e..cf25148f5 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -20,6 +20,15 @@ APP_ROOT / "modules" / "artifacts" / "api" / "submission_preparation.py" ) SUBMISSION_ADMISSION_API = APP_ROOT / "modules" / "artifacts" / "api" / "submission_admission.py" +CHECKER_OUTPUT_CUSTODY_API = ( + APP_ROOT / "modules" / "checkers" / "api" / "output_custody.py" +) +CHECKER_MATERIALIZATION_API = ( + APP_ROOT / "modules" / "checkers" / "api" / "materialization.py" +) +RETIRED_ARTIFACT_MATERIALIZATION_API = ( + APP_ROOT / "modules" / "artifacts" / "api" / "submission_materialization.py" +) COMPOSITION_ROOT = APP_ROOT / "adapters" / "artifacts" / "__init__.py" AGENT_COMPOSITION_ROOT = APP_ROOT / "adapters/project_agents/__init__.py" AGENT_ADAPTER_MODULE = "app.adapters.project_agents.openai_agent_sdk" @@ -27,21 +36,15 @@ CLOSED_PORTS = { "GuideArtifactIngestCommand", "GuideArtifactIngestPort", - "ArtifactBindingPort", - "CheckerArtifactOutputPort", "ArtifactOperatorReadPort", "ArtifactOperatorRecoveryPort", } CANONICAL_REQUESTS = { "GuideArtifactIngestRequest", - "CheckerOutputBindingRequest", - "CheckerOutputArtifactRequest", "ArtifactRecoveryRequest", } CANONICAL_RESULTS = { "GuideArtifactIngestResult", - "CheckerOutputArtifactResult", - "CheckerOutputBindingResult", } CANONICAL_TYPE_ALIASES = { "ArtifactAuditResourceType", @@ -490,12 +493,6 @@ def test_artifact_ports_keep_prepared_authority_at_the_transaction_boundary() -> assert all("AuthorizationContext" not in types for types in fields.values()), name assert {"action_id", "resource_context", "facts"}.isdisjoint(fields), name - # Output services own fresh PREP per phase; a public handle cannot span their I/O. - for name in ("CheckerOutputArtifactRequest", "CheckerOutputBindingRequest"): - node = next(item for item in tree.body if isinstance(item, ast.ClassDef) and item.name == name) - annotations = _declared_annotation_names(node) - assert {"PreparedAuthorizationHandle", "AuthorizationContext"}.isdisjoint(annotations) - assert "CheckerOutputSelector" in annotations source = ARTIFACT_OPERATIONS.read_text(encoding="utf-8") assert "upload_session" not in source assert "ContributorArtifactUploadPort" not in source @@ -504,16 +501,9 @@ def test_artifact_ports_keep_prepared_authority_at_the_transaction_boundary() -> expected_methods = { "GuideArtifactIngestPort": {"ingest"}, - "ArtifactBindingPort": { - "bind_checker_output", - }, - "CheckerArtifactOutputPort": {"store", "recover"}, } expected_request_by_method = { "ingest": "GuideArtifactIngestRequest", - "bind_checker_output": "CheckerOutputBindingRequest", - "store": "CheckerOutputArtifactRequest", - "recover": "CheckerOutputSelector", } protocols = { node.name: node @@ -542,6 +532,116 @@ def test_artifact_ports_keep_prepared_authority_at_the_transaction_boundary() -> assert "AuthorizationContext" not in _declared_annotation_names(node) +def test_checker_output_contract_has_one_canonical_consumer_owner() -> None: + """CHECKERS owns the ports that ART implements for future durable execution.""" + tree = _tree(CHECKER_OUTPUT_CUSTODY_API) + classes = { + node.name: node for node in tree.body if isinstance(node, ast.ClassDef) + } + assert { + "CheckerOutputArtifactRequest", + "CheckerOutputArtifactResult", + "CheckerOutputBindingRequest", + "CheckerOutputBindingResult", + "CheckerOutputBindingPort", + "CheckerArtifactOutputPort", + } <= set(classes) + assert "ArtifactBindingPort" not in classes + for name in ("CheckerOutputArtifactRequest", "CheckerOutputBindingRequest"): + annotations = _declared_annotation_names(classes[name]) + assert {"PreparedAuthorizationHandle", "AuthorizationContext"}.isdisjoint( + annotations + ) + assert "CheckerOutputSelector" in annotations + + expected_methods = { + "CheckerOutputBindingPort": {"bind_checker_output"}, + "CheckerArtifactOutputPort": {"store", "recover"}, + } + for name, methods in expected_methods.items(): + assert { + item.name + for item in classes[name].body + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + } == methods + + artifact_source = ARTIFACT_OPERATIONS.read_text(encoding="utf-8") + assert "app.modules.checkers" not in artifact_source + assert not RETIRED_ARTIFACT_MATERIALIZATION_API.exists() + assert "ArtifactBindingPort" not in "\n".join( + path.read_text(encoding="utf-8") for path in _python_files(APP_ROOT) + ) + + +def test_checker_materialization_contract_has_one_canonical_consumer_owner() -> None: + """The complete material callback contract resides in CHECKERS public API.""" + tree = _tree(CHECKER_MATERIALIZATION_API) + classes = { + node.name: node for node in tree.body if isinstance(node, ast.ClassDef) + } + assert set(classes) == { + "PostSubmissionMaterializationUnavailable", + "SubmissionMaterialEntry", + "SubmissionMaterialView", + "PostSubmissionMaterialConsumer", + "PostSubmissionMaterializationResult", + "PostSubmissionMaterializationPort", + } + assert { + node.name + for node in classes["PostSubmissionMaterializationPort"].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } == {"materialize"} + artifact_api = APP_ROOT / "modules" / "artifacts" / "api" / "__init__.py" + assert "Materialization" not in artifact_api.read_text(encoding="utf-8") + + +def test_art_implementations_are_the_exact_checker_contract_consumers() -> None: + """Prevent a reverse dependency or an extra consumer of CHECKERS custody ports.""" + expected = { + "app.modules.checkers.api.output_custody": { + "app/adapters/artifacts/__init__.py", + "app/modules/artifacts/checker_output_bindings.py", + "app/modules/artifacts/checker_output_custody.py", + "app/modules/artifacts/checker_outputs.py", + "app/modules/artifacts/schemas.py", + "app/modules/artifacts/service.py", + }, + "app.modules.checkers.api.materialization": { + "app/adapters/artifacts/__init__.py", + "app/modules/artifacts/post_submit_materialization.py", + "app/modules/artifacts/post_submit_selection.py", + }, + } + actual = {module: set() for module in expected} + for path in _python_files(APP_ROOT): + modules = { + node.module + for node in ast.walk(_tree(path)) + if isinstance(node, ast.ImportFrom) and node.module in expected + } + for module in modules: + actual[module].add(path.relative_to(BACKEND_ROOT).as_posix()) + assert actual == expected + + composition = _tree(COMPOSITION_ROOT) + returns = { + node.name: _annotation_names(node.returns) + for node in composition.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name in { + "post_submission_materialization", + "checker_output_storage", + "checker_output_binding", + } + } + assert returns == { + "post_submission_materialization": {"PostSubmissionMaterializationPort"}, + "checker_output_storage": {"CheckerArtifactOutputPort"}, + "checker_output_binding": {"CheckerOutputBindingPort"}, + } + + def test_submission_preparation_http_request_never_carries_prepared_authority() -> None: tree = _tree(SUBMISSION_PREPARATION_API) request_class = next( diff --git a/backend/tests/test_behavior_ownership.py b/backend/tests/test_behavior_ownership.py index dc04f5d69..7b39d2611 100644 --- a/backend/tests/test_behavior_ownership.py +++ b/backend/tests/test_behavior_ownership.py @@ -2124,16 +2124,27 @@ def test_approved_guide_intake_removal_is_exact(): def test_post_submit_materialization_partition_additions_are_exact(): targets = { - "backend/app/modules/artifacts/api/submission_materialization.py", + "backend/app/modules/checkers/api/materialization.py", "backend/app/modules/artifacts/post_submit_materialization.py", "backend/app/modules/artifacts/post_submit_selection.py", "backend/app/modules/tasks/api/submitted_bundle.py", "backend/app/modules/tasks/submitted_bundle.py", } assert ownership.ARCH_04B_MATERIALIZATION_TARGETS == targets + assert ownership.ARCH_04B_SEAM_REMOVED_TARGETS == { + "backend/app/modules/artifacts/api/submission_materialization.py" + } retained = "backend/app/core/config.py" + retired = "backend/app/modules/artifacts/api/submission_materialization.py" trusted = _partition([retained]) ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets])), trusted) + prior_targets = (targets - {"backend/app/modules/checkers/api/materialization.py"}) | { + retired + } + ownership._validate_additive_partition_transition( + _partition(sorted([retained, *targets])), + _partition(sorted([retained, *prior_targets])), + ) for forbidden in ("backend/app/modules/artifacts/download.py", "backend/app/modules/tasks/other_material.py"): with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets, forbidden])), trusted) diff --git a/backend/tests/test_checker_output_custody.py b/backend/tests/test_checker_output_custody.py index 4bd3e5f0f..c6ec4d3a4 100644 --- a/backend/tests/test_checker_output_custody.py +++ b/backend/tests/test_checker_output_custody.py @@ -12,10 +12,6 @@ from app.adapters.artifacts import checker_output_binding, checker_output_storage from app.core.identifiers import new_record_id -from app.interfaces.artifact_operations import ( - CheckerOutputArtifactRequest, - CheckerOutputBindingRequest, -) from app.interfaces.artifacts import ArtifactLimitExceededError from app.modules.actors.api import ServiceIdentity from app.modules.artifacts.models import ( @@ -23,10 +19,13 @@ ArtifactOperationReceipt, ArtifactPutObservationReceipt, ArtifactPutAttempt, + ArtifactVerificationJob, ArtifactVerificationReceipt, ) from app.modules.artifacts.service import ArtifactAdmissionConflictError from app.modules.checkers.api.output_custody import ( + CheckerOutputArtifactRequest, + CheckerOutputBindingRequest, CheckerOutputSelector, CheckerOutputUnavailable, ) @@ -150,6 +149,89 @@ async def test_real_store_verification_and_binding( assert harness.state.bindings +async def test_identical_store_replay_reuses_verified_custody_without_provider_work( + tmp_path, + isolated_database_env, +) -> None: + """Replay exact bytes through the stable attempt without another put or verify.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + first = await harness.service.store(harness.request(b"stable replay output")) + assert first.status == "verified" + assert first.content_id is not None + assert first.verification_receipt_id is not None + assert first.replayed is False + provider_counts = (harness.store.puts, harness.store.opens) + assert provider_counts == (1, 1) + bound = await harness.bind(harness.selector, first) + assert ( + bound.content_id, + bound.put_attempt_id, + bound.verification_receipt_id, + bound.replayed, + ) == ( + first.content_id, + first.put_attempt_id, + first.verification_receipt_id, + False, + ) + + replay = await harness.service.store(harness.request(b"stable replay output")) + + assert ( + replay.put_attempt_id, + replay.content_id, + replay.verification_receipt_id, + replay.status, + replay.replayed, + ) == ( + first.put_attempt_id, + first.content_id, + first.verification_receipt_id, + "verified", + True, + ) + assert (harness.store.puts, harness.store.opens) == provider_counts + assert (await harness.manager.usage()).reservation_count == 0 + assert list((tmp_path / "output-scratch" / "files").iterdir()) == [] + async with harness.factory() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactPutAttempt) + .where( + ArtifactPutAttempt.checker_run_id == str(harness.selector.checker_run_id) + ) + ) + == 1 + ) + assert ( + await session.get( + ArtifactVerificationReceipt, + str(first.verification_receipt_id), + ) + is not None + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactVerificationJob) + .where( + ArtifactVerificationJob.originating_put_attempt_id + == str(first.put_attempt_id) + ) + ) + == 1 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactOperationReceipt) + .where(ArtifactOperationReceipt.put_attempt_id == str(first.put_attempt_id)) + ) + == 1 + ) + + async def test_foreign_lineage_and_changed_bytes_fail_before_provider( tmp_path, isolated_database_env, @@ -428,6 +510,93 @@ async def test_authority_revocation_during_put_prevents_return_and_binding( assert binding_count == 0 +async def test_cancellation_during_provider_put_cleans_scratch_and_retains_uncertainty( + tmp_path, + isolated_database_env, +) -> None: + """Preserve cancellation while retaining only the durable in-flight attempt.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + harness.store.put_entered = asyncio.Event() + harness.store.put_release = asyncio.Event() + operation = asyncio.create_task(harness.service.store(harness.request(b"cancelled output"))) + try: + await asyncio.wait_for(harness.store.put_entered.wait(), timeout=10) + assert harness.store.puts == 1 + live_usage = await harness.manager.usage() + assert live_usage.reservation_count == 1 + assert live_usage.reserved_bytes > 0 + + operation.cancel("cancelled checker output provider put") + with pytest.raises( + asyncio.CancelledError, + match="cancelled checker output provider put", + ): + await operation + finally: + harness.store.put_release.set() + if not operation.done(): + operation.cancel() + await asyncio.gather(operation, return_exceptions=True) + + assert harness.store.puts == 1 + assert harness.store.opens == 0 + assert (await harness.manager.usage()).reservation_count == 0 + assert list((tmp_path / "output-scratch" / "files").iterdir()) == [] + async with harness.factory() as session: + attempt = await session.scalar( + select(ArtifactPutAttempt).where( + ArtifactPutAttempt.checker_run_id == str(harness.selector.checker_run_id) + ) + ) + assert attempt is not None + assert attempt.status == "put_in_flight" + assert attempt.execution_mode == "caller_put" + assert attempt.executor_id is not None + assert attempt.lease_expires_at is not None + assert attempt.replica_id is None + assert attempt.receipt_id is None + assert attempt.terminal_result_code is None + assert attempt.terminal_at is None + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactOperationReceipt) + .where( + ArtifactOperationReceipt.checker_run_id + == str(harness.selector.checker_run_id) + ) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactPutObservationReceipt) + .where(ArtifactPutObservationReceipt.put_attempt_id == attempt.id) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactVerificationJob) + .where(ArtifactVerificationJob.originating_put_attempt_id == attempt.id) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactBinding) + .where( + ArtifactBinding.resource_type == "checker_run", + ArtifactBinding.resource_id == str(harness.selector.checker_run_id), + ) + ) + == 0 + ) + + async def test_per_slot_cap_stops_source_and_cleans_scratch( tmp_path, isolated_database_env, diff --git a/backend/tests/test_checker_output_storage.py b/backend/tests/test_checker_output_storage.py index 3d228bf28..77b12a9b2 100644 --- a/backend/tests/test_checker_output_storage.py +++ b/backend/tests/test_checker_output_storage.py @@ -2,7 +2,9 @@ from __future__ import annotations +import asyncio from contextlib import asynccontextmanager +from dataclasses import replace from pathlib import Path import runpy from types import SimpleNamespace @@ -24,7 +26,7 @@ ) from app.modules.artifacts.service import ArtifactStorageOrchestrator from projects.unified_policy_fixtures import create_standalone_unified_policy -from tests.artifact_store_helpers import minted_source +from tests.artifact_store_helpers import artifact_preparation_limits, minted_source from tests.test_artifact_admission import ( _AllowArtifactAuthority, _admit_checker_output, @@ -47,7 +49,17 @@ async def _store_verified_output( ): """Store and independently verify one output using shared test infrastructure.""" async with factory() as session: - async with minted_source(source_path, payload, media_type="text/plain") as source: + limits = replace( + artifact_preparation_limits(), + reservation_ttl_seconds=360, + total_deadline_seconds=300, + ) + async with minted_source( + source_path, + payload, + media_type="text/plain", + limits=limits, + ) as source: project_id, task_id, checker_run_id, admission = await _admit_checker_output( session, settings, @@ -86,9 +98,10 @@ async def _store_verified_output( assert attempt is not None and receipt_id is not None assert attempt.replica_id is not None logical_role = attempt.logical_role + replica_id = attempt.replica_id content_id = await session.scalar( text("select content_id from artifact_replicas where id=:replica_id"), - {"replica_id": attempt.replica_id}, + {"replica_id": replica_id}, ) assert content_id is not None await session.rollback() @@ -98,6 +111,8 @@ async def _store_verified_output( task_id=task_id, checker_run_id=checker_run_id, put_attempt_id=str(admission.attempt_id), + verification_job_id=str(job_id), + replica_id=str(replica_id), verification_receipt_id=receipt_id, content_id=str(content_id), logical_role=logical_role, @@ -127,6 +142,7 @@ async def _verified_output(database_url: str, tmp_path: Path): ) case.settings = settings case.store = store + case.engine = engine yield case finally: bootstrap.close() @@ -161,6 +177,216 @@ def _binding_values(case, **changes): :verification_receipt_id,:supersedes_binding_id)""" ) +_ANCESTOR_MUTATIONS = { + "attempt": ( + "artifact_put_attempts", + "put_attempt_id", + "cas_version=cas_version+1", + "checker_output_put_attempt_custody", + "checker output put attempt custody is immutable", + ), + "job": ( + "artifact_verification_jobs", + "verification_job_id", + "cas_version=cas_version+1", + "checker_output_verification_job_custody", + "checker output verification job custody is immutable", + ), + "replica": ( + "artifact_replicas", + "replica_id", + "provider_profile=provider_profile || '-changed'", + "checker_output_replica_custody", + "checker output replica custody is immutable", + ), +} + + +async def _sealed_chain(session, case) -> tuple[bool, bool, bool]: + values = [] + for table, attribute in ( + ("artifact_verification_jobs", "verification_job_id"), + ("artifact_replicas", "replica_id"), + ("artifact_put_attempts", "put_attempt_id"), + ): + value = await session.scalar( + text( + f"select checker_output_custody_sealed from {table} " + "where id=:id" + ), + {"id": getattr(case, attribute)}, + ) + values.append(bool(value)) + return tuple(values) + + +async def _wait_for_lock(session, backend_pid: int) -> None: + for _ in range(500): + waiting = await session.scalar( + text( + "select exists(select 1 from pg_locks " + "where pid=:pid and not granted)" + ), + {"pid": backend_pid}, + ) + if waiting: + return + await asyncio.sleep(0.01) + raise AssertionError("competing ancestry transaction did not wait") + + +async def _transaction(connection, isolation: str): + transaction = await connection.begin() + await connection.execute(text(f"set transaction isolation level {isolation}")) + return transaction + + +def _ancestor_update(case, ancestor: str): + table, attribute, assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + return text(f"update {table} set {assignment} where id=:id"), { + "id": getattr(case, attribute) + } + + +def _invalid_ancestor_update(case, ancestor: str): + table, attribute, _assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + assignment = { + "attempt": ( + "status='conflict', terminal_result_code='conflict', " + "terminal_at=clock_timestamp(), cas_version=cas_version+1" + ), + "job": ( + "status='conflict', terminal_result_code='conflict', " + "terminal_at=clock_timestamp(), cas_version=cas_version+1" + ), + "replica": "provider_object_ref=provider_object_ref || '-moved'", + }[ancestor] + return text(f"update {table} set {assignment} where id=:id"), { + "id": getattr(case, attribute) + } + + +async def _ancestor_without_seal(session, case, ancestor: str): + table, attribute, _assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + return await session.scalar( + text( + f"select to_jsonb(value) - 'checker_output_custody_sealed' " + f"from {table} value where id=:id" + ), + {"id": getattr(case, attribute)}, + ) + + +def _assert_concurrent_rejection( + error: DBAPIError, + *, + isolation: str, + expected: str, +) -> None: + message = str(error) + if isolation == "REPEATABLE READ": + assert expected in message or "could not serialize access" in message + else: + assert expected in message + + +async def _binding_first_race(case, ancestor: str, isolation: str) -> None: + async with case.factory() as session: + before = await _ancestor_without_seal(session, case, ancestor) + binder = await case.engine.connect() + updater = await case.engine.connect() + binding_transaction = await _transaction(binder, isolation) + update_transaction = await _transaction(updater, isolation) + pending_update = None + try: + updater_pid = await updater.scalar(text("select pg_backend_pid()")) + assert await _sealed_chain(updater, case) == (False, False, False) + assert await _ancestor_without_seal(updater, case, ancestor) == before + await binder.execute(_INSERT_BINDING, _binding_values(case)) + statement, parameters = _ancestor_update(case, ancestor) + pending_update = asyncio.create_task(updater.execute(statement, parameters)) + async with case.engine.connect() as observer: + await _wait_for_lock(observer, updater_pid) + await binding_transaction.commit() + with pytest.raises(DBAPIError) as rejected: + await pending_update + _assert_concurrent_rejection( + rejected.value, + isolation=isolation, + expected=_ANCESTOR_MUTATIONS[ancestor][4], + ) + await update_transaction.rollback() + finally: + if pending_update is not None and not pending_update.done(): + pending_update.cancel() + await asyncio.gather(pending_update, return_exceptions=True) + if binding_transaction.is_active: + await binding_transaction.rollback() + if update_transaction.is_active: + await update_transaction.rollback() + await binder.close() + await updater.close() + async with case.factory() as session: + assert await _sealed_chain(session, case) == (True, True, True) + assert await _ancestor_without_seal(session, case, ancestor) == before + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 1 + + +async def _update_first_race(case, ancestor: str, isolation: str) -> None: + async with case.factory() as session: + before = await _ancestor_without_seal(session, case, ancestor) + updater = await case.engine.connect() + binder = await case.engine.connect() + update_transaction = await _transaction(updater, isolation) + binding_transaction = await _transaction(binder, isolation) + pending_binding = None + try: + binder_pid = await binder.scalar(text("select pg_backend_pid()")) + statement, parameters = _invalid_ancestor_update(case, ancestor) + await updater.execute(statement, parameters) + assert await _sealed_chain(binder, case) == (False, False, False) + assert await _ancestor_without_seal(binder, case, ancestor) == before + pending_binding = asyncio.create_task( + binder.execute(_INSERT_BINDING, _binding_values(case)) + ) + async with case.engine.connect() as observer: + await _wait_for_lock(observer, binder_pid) + await update_transaction.commit() + with pytest.raises(DBAPIError) as rejected: + await pending_binding + _assert_concurrent_rejection( + rejected.value, + isolation=isolation, + expected="checker output binding verified ancestry mismatch", + ) + await binding_transaction.rollback() + finally: + if pending_binding is not None and not pending_binding.done(): + pending_binding.cancel() + await asyncio.gather(pending_binding, return_exceptions=True) + if update_transaction.is_active: + await update_transaction.rollback() + if binding_transaction.is_active: + await binding_transaction.rollback() + await updater.close() + await binder.close() + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await _ancestor_without_seal(session, case, ancestor) != before + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 0 + @pytest.mark.asyncio async def test_exact_verified_checker_output_binding_and_generic_binding_remain_valid( @@ -301,6 +527,150 @@ async def test_checker_attempt_static_custody_allows_only_execution_lifecycle( ) +@pytest.mark.asyncio +async def test_binding_seals_exact_ancestry_but_preserves_replica_health( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + for ancestor in _ANCESTOR_MUTATIONS: + table, attribute, _assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + with pytest.raises(DBAPIError, match="seal requires binding"): + async with case.factory() as session, session.begin(): + await session.execute( + text( + f"update {table} set checker_output_custody_sealed=true " + "where id=:id" + ), + {"id": getattr(case, attribute)}, + ) + + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_put_attempts set cas_version=cas_version+1 " + "where id=:id" + ), + {"id": case.put_attempt_id}, + ) + await session.execute( + text( + "update artifact_verification_jobs set cas_version=cas_version+1 " + "where id=:id" + ), + {"id": case.verification_job_id}, + ) + await session.execute( + text( + "update artifact_replicas set last_reconciled_at=clock_timestamp(), " + "updated_at=clock_timestamp() where id=:id" + ), + {"id": case.replica_id}, + ) + await session.execute(_INSERT_BINDING, _binding_values(case)) + assert await _sealed_chain(session, case) == (True, True, True) + + for ancestor in _ANCESTOR_MUTATIONS: + table, attribute, assignment, _trigger, error = _ANCESTOR_MUTATIONS[ancestor] + with pytest.raises(DBAPIError, match=error): + async with case.factory() as session, session.begin(): + await session.execute( + text(f"update {table} set {assignment} where id=:id"), + {"id": getattr(case, attribute)}, + ) + with pytest.raises(DBAPIError, match=error): + async with case.factory() as session, session.begin(): + await session.execute( + text(f"delete from {table} where id=:id"), + {"id": getattr(case, attribute)}, + ) + + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_replicas set verification_state='missing', " + "availability_state='unavailable', integrity_state='invalid', " + "last_reconciled_at=clock_timestamp(), updated_at=clock_timestamp() " + "where id=:id" + ), + {"id": case.replica_id}, + ) + assert await _sealed_chain(session, case) == (True, True, True) + + for ancestor in _ANCESTOR_MUTATIONS: + table, attribute, assignment, trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + async with case.factory() as session: + before = await session.scalar( + text(f"select to_jsonb(value) from {table} value where id=:id"), + {"id": getattr(case, attribute)}, + ) + transaction = await session.begin_nested() + await session.execute(text(f"alter table {table} disable trigger {trigger}")) + await session.execute( + text(f"update {table} set {assignment} where id=:id"), + {"id": getattr(case, attribute)}, + ) + after = await session.scalar( + text(f"select to_jsonb(value) from {table} value where id=:id"), + {"id": getattr(case, attribute)}, + ) + assert after != before + await transaction.rollback() + assert await session.scalar( + text(f"select to_jsonb(value) from {table} value where id=:id"), + {"id": getattr(case, attribute)}, + ) == before + await session.rollback() + + +@pytest.mark.asyncio +async def test_binding_rollback_removes_all_ancestry_seals( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + binding_id = str(new_record_id()) + async with case.factory() as session: + transaction = await session.begin() + await session.execute( + _INSERT_BINDING, + _binding_values(case, id=binding_id), + ) + assert await _sealed_chain(session, case) == (True, True, True) + await transaction.rollback() + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 0 + + +@pytest.mark.parametrize("ancestor", tuple(_ANCESTOR_MUTATIONS)) +@pytest.mark.parametrize("isolation", ("READ COMMITTED", "REPEATABLE READ")) +@pytest.mark.asyncio +async def test_binding_and_ancestor_updates_serialize_fail_closed( + isolated_database_env: str, + tmp_path: Path, + ancestor: str, + isolation: str, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + await _binding_first_race(case, ancestor, isolation) + + other = await _store_verified_output( + factory=case.factory, + settings=case.settings, + namespace=case.namespace, + store=case.store, + policy_bundle=case.policy_bundle, + source_path=tmp_path / f"update-first-{ancestor}", + payload=f"update-first-{isolation}-{ancestor}".encode(), + ) + other.engine = case.engine + await _update_first_race(other, ancestor, isolation) + + @pytest.mark.asyncio async def test_artifact_binding_truncate_custody_blocks_direct_and_cascade_deletion( isolated_database_env: str, @@ -409,14 +779,20 @@ async def test_binding_guard_uses_only_artifact_ancestry_and_exact_owner_fks( async def _restore_pre_0007_schema(connection) -> None: """Remove only 0007 custody inside the caller's rollback-only transaction.""" for table, trigger in ( + ("artifact_bindings", "checker_output_binding_seal"), ("artifact_bindings", "checker_output_binding_insert"), ("artifact_bindings", "trg_artifact_bindings_no_truncate"), + ("artifact_verification_jobs", "checker_output_verification_job_custody"), + ("artifact_replicas", "checker_output_replica_custody"), ("artifact_put_attempts", "checker_output_put_attempt_custody"), ("artifact_put_attempts", "checker_output_put_attempt_no_truncate"), ): await connection.execute(text(f"drop trigger {trigger} on {table}")) for function in ( + "seal_checker_output_binding_ancestry()", "guard_checker_output_binding_insert()", + "guard_checker_output_verification_job_custody()", + "guard_checker_output_replica_custody()", "guard_checker_output_put_attempt_custody()", "guard_checker_output_put_attempt_truncate()", ): @@ -467,7 +843,20 @@ async def _restore_pre_0007_schema(connection) -> None: await connection.execute( text( "alter table artifact_put_attempts drop column submission_id, " - "drop column submission_version, drop column checker_request_digest" + "drop column submission_version, drop column checker_request_digest, " + "drop column checker_output_custody_sealed" + ) + ) + await connection.execute( + text( + "alter table artifact_verification_jobs " + "drop column checker_output_custody_sealed" + ) + ) + await connection.execute( + text( + "alter table artifact_replicas " + "drop column checker_output_custody_sealed" ) ) diff --git a/backend/tests/test_post_submit_materialization.py b/backend/tests/test_post_submit_materialization.py index 3869e8e0f..ac10d6d52 100644 --- a/backend/tests/test_post_submit_materialization.py +++ b/backend/tests/test_post_submit_materialization.py @@ -10,7 +10,7 @@ from app.adapters.artifacts import post_submission_materialization from app.core.identifiers import new_record_id from app.interfaces.artifacts import ArtifactInputMismatchError -from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.checkers.api.materialization import PostSubmissionMaterializationUnavailable from tests.checkers.post_submit.support import change_request from tests.checkers.post_submit.test_result_contract import result from tests.post_submit_materialization_helpers import material_fixture diff --git a/backend/tests/test_post_submit_selection.py b/backend/tests/test_post_submit_selection.py index 5001e1405..eecfa4075 100644 --- a/backend/tests/test_post_submit_selection.py +++ b/backend/tests/test_post_submit_selection.py @@ -7,7 +7,7 @@ from app.adapters.tasks import submitted_bundle_port from app.core.identifiers import new_record_id -from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.checkers.api.materialization import PostSubmissionMaterializationUnavailable from app.modules.tasks.api.submitted_bundle import SubmittedBundleRequest, SubmittedBundleUnavailable from tests.checkers.post_submit.support import change_request from tests.post_submit_materialization_helpers import material_fixture diff --git a/docs/architecture_data_model.md b/docs/architecture_data_model.md index 31d36fbdf..fb6bf39dc 100644 --- a/docs/architecture_data_model.md +++ b/docs/architecture_data_model.md @@ -320,7 +320,10 @@ fills all three before its root transaction commits. This is separate from the required initial assignment identity. Hidden exact ART-to-CHECKERS input materialization is delivered by ARCH-04B. ARCH-04B2 adds checker-output attempt custody with exact Submission version and checker-request digest plus immutable verified put/receipt ancestry on each -run/slot binding. Its migration refuses retained checker attempts or bindings +run/slot binding. Binding publication atomically seals the terminal put attempt, +verification job and replica identity. Later replica health changes remain +possible without rewriting the historical verified ancestry. Its migration +refuses retained checker attempts or bindings whose missing evaluation digest or verified ancestry cannot be proven; it does not invent or delete retained data. Durable execution/results (ARCH-04C) remain pending, and production access remains deny-only until ARCH-04D. Retained payment columns on Submission and CheckerRun diff --git a/docs/spec_artifact_storage_service.md b/docs/spec_artifact_storage_service.md index c7276495f..19de5379c 100644 --- a/docs/spec_artifact_storage_service.md +++ b/docs/spec_artifact_storage_service.md @@ -355,7 +355,7 @@ SubmissionBundlePreparationCommand.prepare(SubmissionBundlePreparationRequest) SubmissionAdmissionConsumptionPort.consume(SubmissionAdmissionConsumptionRequest) GuideDocumentManifestPort.load(GuideDocumentManifestRequest) GuideDocumentGrant.open(opaque_document_handle) -ArtifactBindingPort.bind_checker_output(CheckerOutputBindingRequest) +CheckerOutputBindingPort.bind_checker_output(CheckerOutputBindingRequest) PostSubmissionMaterializationPort.materialize(PostSubmissionEvaluationRequest, consumer) CheckerArtifactOutputPort.store(CheckerOutputArtifactRequest) CheckerArtifactOutputPort.recover(CheckerOutputSelector) @@ -376,6 +376,12 @@ immediately before materialization and durable put intent. The typed methods fix their expected actions, and handles never enter route schemas, outbox/Celery payloads, provider interfaces, or serialized contracts. +CHECKERS owns the post-submit materialization and output-custody consumer ports +in `app.modules.checkers.api.materialization` and +`app.modules.checkers.api.output_custody`. ART imports those public contracts and +supplies their concrete custody implementations through composition. CHECKERS +execution does not import ART APIs or private ART modules to consume them. + `GuideArtifactIngestRequest` contains prepared authority, exact project, guide, guide-source snapshot and item IDs, logical role, and authorized byte source. The service resolves the server-bound media type from the locked snapshot item. @@ -1572,7 +1578,12 @@ databases, and no artifact bytes in PostgreSQL. It is not downgradable. The checker-output custody migration adds exact Submission version and narrow checker-request digest fields to checker-output attempts plus verified -put/receipt ancestry on checker bindings. It refuses any retained checker-output +put/receipt ancestry on checker bindings. A successful binding atomically seals +the terminal attempt/job and replica identity under the existing +job -> replica -> attempt -> content lock order. The database rejects later +ancestry changes, including writers using an older transaction snapshot; replica +health and availability remain mutable. Rolling back publication also rolls back +the seals. It refuses any retained checker-output attempt or binding whose missing evaluation/slot digest or verified ancestry cannot be proven; it neither invents those facts nor deletes retained data. From a4e72dd76879e5ee7be4fc0b3a97ff99847bb8f9 Mon Sep 17 00:00:00 2001 From: Commitrail Probe Date: Tue, 29 Sep 2026 12:42:01 +0100 Subject: [PATCH 5/6] Reject incomplete verification and scope checker admission digests --- .../WS-ARCH-001/WS-ARCH-001-04B2.md | 12 ++ ...001-04B-art-post-submit-materialization.md | 2 +- .../versions/0007_checker_output_custody.py | 2 +- backend/app/modules/artifacts/service.py | 12 +- backend/scripts/test_lane_catalogue.py | 1 + backend/tests/conftest.py | 2 +- .../tests/test_artifact_admission_digest.py | 127 ++++++++++++++++++ backend/tests/test_checker_output_storage.py | 65 +++++++++ 8 files changed, 217 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_artifact_admission_digest.py diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md index d00801674..31d80064b 100644 --- a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -116,6 +116,7 @@ claim the current structural catalogue supports them. output intent/binding custody; no CHECKERS run writer or retained-data deletion. - New `backend/tests/test_checker_output_custody.py`, `backend/tests/test_checker_output_storage.py`, + `backend/tests/test_artifact_admission_digest.py` for unaffected guide replay, `backend/tests/checker_output_custody_helpers.py`, `backend/tests/checker_output_admission_helpers.py`, and existing `test_artifact_admission.py`, `test_artifact_recovery.py`, @@ -283,3 +284,14 @@ replica health and availability remain operational facts that may change. Failed or rolled-back publication must not leave seals behind. Real PostgreSQL proof covers both READ COMMITTED and REPEATABLE READ interleavings, not only sequential mutation. No separate custody aggregate or public capability is introduced. + +Follow-up review identified two additional bounded repairs. The binding guard +must reject a null terminal verification result explicitly: SQL's nullable +comparison cannot establish verified ancestry. Direct-SQL proof supplies an +otherwise valid chain and checks rejection without a binding or seals. Checker +lineage fields belong only in the checker-output admission digest; unrelated +guide and submission producers retain their canonical digest inputs. Guide +replay must recover its existing attempt without a conflicting digest. Submission +bundle replay already returns through its owner replay port before this digest; +no submission replay failure is claimed. These changes add no compatibility +path and do not rewrite retained attempts. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md index 69f460c3d..c469217f8 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md @@ -9,7 +9,7 @@ The separate output child is delivered by the Input materialization does not implement output persistence. ARCH-04B2 now supplies typed `CheckerArtifactOutputPort.store`, byte-free -`recover(selector)`, and `ArtifactBindingPort.bind_checker_output` using 04A's +`recover(selector)`, and `CheckerOutputBindingPort.bind_checker_output` using 04A's exact immutable request and owner reservation facts. It reuses generic admission, put intent, observation, verification, quota and binding infrastructure. ART behavior no longer reads CheckerRun through its private diff --git a/backend/alembic/versions/0007_checker_output_custody.py b/backend/alembic/versions/0007_checker_output_custody.py index deb70574a..a67971218 100644 --- a/backend/alembic/versions/0007_checker_output_custody.py +++ b/backend/alembic/versions/0007_checker_output_custody.py @@ -502,7 +502,7 @@ def _install_checker_output_binding_guard() -> None: OR job.originating_put_attempt_id IS DISTINCT FROM attempt.id OR job.replica_id IS DISTINCT FROM replica.id OR job.status <> 'verified' - OR job.terminal_result_code <> 'verified' + OR job.terminal_result_code IS DISTINCT FROM 'verified' OR job.terminal_at IS NULL OR verification.outcome <> 'verified' OR verification.execution_generation IS DISTINCT FROM job.execution_generation diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index e0117cbff..69c579354 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -2057,9 +2057,15 @@ async def admit( "guide_source_item_id": facts.guide_source_item_id, "checker_run_id": facts.checker_run_id, "logical_role": facts.logical_role, - "submission_id": facts.submission_id, - "submission_version": facts.submission_version, - "checker_output_request": facts.checker_request_digest_facts, + **( + { + "submission_id": facts.submission_id, + "submission_version": facts.submission_version, + "checker_output_request": facts.checker_request_digest_facts, + } + if facts.request_type == "checker_output" + else {} + ), "pre_submit_evidence_set_id": facts.pre_submit_evidence_set_id, "sha256": commitment.sha256, "byte_count": commitment.byte_count, diff --git a/backend/scripts/test_lane_catalogue.py b/backend/scripts/test_lane_catalogue.py index 70781575e..c292d6e14 100644 --- a/backend/scripts/test_lane_catalogue.py +++ b/backend/scripts/test_lane_catalogue.py @@ -217,6 +217,7 @@ class TestLane: "tests/authorization/contribution_policies/test_scope_locks.py", "tests/test_behavior_ownership.py", "tests/test_artifact_admission.py", + "tests/test_artifact_admission_digest.py", "tests/test_submission_bundle_admission.py", "tests/test_submission_bundle_preparation_recovery.py", "tests/checkers/test_phase_service.py", diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 809063ff5..154eb4d6f 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -23,7 +23,7 @@ DDL_LOCK_DIRECTORY = Path("/tmp") # Match the PostgreSQL 16 engine used by Backend CI. Catalog identity rendering # differs across major versions; regenerate only after comparing actual objects. -EXPECTED_PUBLIC_SCHEMA_SHA256 = "1f23100623e232c0e231fa20f9b2ad9599c2e2b8db9c0b42732d189f95ae5a0c" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "f7bbe6bcb1c6298f79f612d6627ca47900fa84d5df3c94addc37c94904861504" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", diff --git a/backend/tests/test_artifact_admission_digest.py b/backend/tests/test_artifact_admission_digest.py new file mode 100644 index 000000000..5716d580f --- /dev/null +++ b/backend/tests/test_artifact_admission_digest.py @@ -0,0 +1,127 @@ +"""Producer-scoped request-digest replay through the ART admission owner.""" + +from dataclasses import replace +from pathlib import Path + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.hashing import canonical_json_hash +from app.modules.artifacts.models import ArtifactPutAttempt +from app.modules.artifacts.repository import ArtifactRepository +from app.modules.artifacts.service import ArtifactAdmissionService +from tests.artifact_store_helpers import artifact_preparation_limits, minted_source +from tests.test_artifact_admission import ( + _AllowGuidePreparedAuthorization, + _context, + _guide_admission_request, + _namespace, + _seed_guide, + _settings, +) + + +async def test_guide_replay_preserves_canonical_producer_digest( + isolated_database_env: str, + tmp_path: Path, +) -> None: + """Guide producer digest shape stays exact through admission and replay.""" + settings = _settings(tmp_path) + namespace = _namespace(settings) + engine = create_async_engine(isolated_database_env) + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + async with factory() as session: + context = _context() + scratch_limits = replace( + artifact_preparation_limits(), + reservation_ttl_seconds=360, + total_deadline_seconds=300, + ) + async with minted_source( + tmp_path / "guide-digest", + b"retained guide bytes", + media_type="application/pdf", + limits=scratch_limits, + ) as source: + project_id, item_id = await _seed_guide( + session, + context=context, + content_hash="sha256:" + "f" * 64, + media_type=source.commitment.media_type, + ) + lineage = await ArtifactRepository(session).get_guide_lineage(item_id) + assert lineage is not None + await session.rollback() + request = _guide_admission_request(item_id, source, lineage=lineage) + first_authority = _AllowGuidePreparedAuthorization( + context.actor_profile_id + ) + first = await ArtifactAdmissionService( + session, settings, namespace + ).admit( + request, + guide_prepared_authorization=first_authority, # type: ignore[arg-type] + prepared_authorization=first_authority.handle, + ) + assert first.replayed is False + + canonical_guide_digest = canonical_json_hash( + { + "operation_identity": request.operation_identity, + "guide_ingest_request_digest": request.request_digest, + "request_type": "guide", + "producer_type": "actor_profile", + "producer_ref": str(context.actor_profile_id), + "project_id": project_id, + "task_id": None, + "guide_source_item_id": item_id, + "checker_run_id": None, + "logical_role": None, + "pre_submit_evidence_set_id": None, + "sha256": source.commitment.sha256, + "byte_count": source.commitment.byte_count, + "media_type": source.commitment.media_type, + "namespace_fingerprint": namespace.namespace_fingerprint, + "scopes": [ + { + "scope_type": "deployment", + "scope_id": "primary", + "limit_bytes": settings.artifact_admission_deployment_maximum_bytes, + }, + { + "scope_type": "producer", + "scope_id": f"actor_profile:{context.actor_profile_id}", + "limit_bytes": settings.artifact_admission_producer_maximum_bytes, + }, + { + "scope_type": "project", + "scope_id": project_id, + "limit_bytes": settings.artifact_admission_project_maximum_bytes, + }, + ], + } + ) + attempt = await session.get(ArtifactPutAttempt, str(first.attempt_id)) + assert attempt is not None + assert first.request_digest == canonical_guide_digest + await session.rollback() + + replay_authority = _AllowGuidePreparedAuthorization( + context.actor_profile_id + ) + replay = await ArtifactAdmissionService( + session, settings, namespace + ).admit( + request, + guide_prepared_authorization=replay_authority, # type: ignore[arg-type] + prepared_authorization=replay_authority.handle, + ) + assert replay.replayed is True + assert replay.attempt_id == first.attempt_id + assert replay.request_digest == canonical_guide_digest + assert await session.scalar( + select(func.count()).select_from(ArtifactPutAttempt) + ) == 1 + finally: + await engine.dispose() diff --git a/backend/tests/test_checker_output_storage.py b/backend/tests/test_checker_output_storage.py index 77b12a9b2..01f765e7a 100644 --- a/backend/tests/test_checker_output_storage.py +++ b/backend/tests/test_checker_output_storage.py @@ -419,6 +419,71 @@ async def test_exact_verified_checker_output_binding_and_generic_binding_remain_ ) == 2 +@pytest.mark.asyncio +async def test_checker_binding_rejects_null_verification_terminal_result( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + async with case.factory() as session: + control = await session.begin() + await session.execute(_INSERT_BINDING, _binding_values(case)) + assert await _sealed_chain(session, case) == (True, True, True) + await control.rollback() + + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_verification_jobs " + "set terminal_result_code=null where id=:id" + ), + {"id": case.verification_job_id}, + ) + + with pytest.raises( + DBAPIError, + match="checker output binding verified ancestry mismatch", + ): + async with case.factory() as session, session.begin(): + await session.execute(_INSERT_BINDING, _binding_values(case)) + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 0 + + async with case.factory() as session: + mutation = await session.begin() + definition = await session.scalar( + text( + "select pg_get_functiondef(" + "'guard_checker_output_binding_insert()'::regprocedure)" + ) + ) + assert definition is not None + null_safe = "job.terminal_result_code IS DISTINCT FROM 'verified'" + old_comparison = "job.terminal_result_code <> 'verified'" + assert definition.count(null_safe) == 1 + await session.execute(text(definition.replace(null_safe, old_comparison))) + await session.execute(_INSERT_BINDING, _binding_values(case)) + assert await _sealed_chain(session, case) == (True, True, True) + await mutation.rollback() + + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 0 + + @pytest.mark.asyncio async def test_checker_binding_rejects_mixed_or_foreign_ancestry( isolated_database_env: str, From 57a9bdf61268ac18c5b3d7473f686806b95308d6 Mon Sep 17 00:00:00 2001 From: Commitrail Probe Date: Tue, 29 Sep 2026 14:41:03 +0100 Subject: [PATCH 6/6] Reconcile checker custody allowed files with relocated contracts --- .../initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md index 31d80064b..df62ad8b2 100644 --- a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -100,7 +100,8 @@ claim the current structural catalogue supports them. ### Allowed -- `backend/app/interfaces/artifact_operations.py` output requests/results and ports. +- `backend/app/interfaces/artifact_operations.py` only to remove the superseded + output requests/results and ports. - `backend/app/modules/artifacts/checker_outputs.py`, `backend/app/modules/artifacts/checker_output_custody.py` and `backend/app/modules/artifacts/checker_output_bindings.py` (new ART owners). @@ -109,8 +110,13 @@ claim the current structural catalogue supports them. `repository.py` only for affected output custody and traced lookup consumers. The separate Operator resource-to-project CHECKERS lookup remains explicitly outside this mutation boundary; it is not a second output admission path. -- `backend/app/modules/checkers/api/output_custody.py` (new closed owner facts), - `backend/app/modules/checkers/api/__init__.py`; composition in existing +- `backend/app/modules/checkers/api/output_custody.py` for closed owner facts, + output requests/results and consumer ports, and + `backend/app/modules/checkers/api/materialization.py` for the relocated input + materialization contract. + Remove `backend/app/modules/artifacts/api/submission_materialization.py` and its + exports from `backend/app/modules/artifacts/api/__init__.py`; update affected + callers without compatibility aliases. Composition in existing `backend/app/adapters/artifacts/__init__.py`, `backend/app/adapters/checkers/`. - One ART-owned migration `backend/alembic/versions/0007_checker_output_custody.py` after `0006_history_read_authority` for immutable exact output intent/binding custody; no CHECKERS run writer or retained-data deletion.