diff --git a/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json b/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json index f8dd57a45..ff7be2786 100644 --- a/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json +++ b/.ci/auth-boundaries/TEST_STRUCTURE_DEBT.json @@ -110,11 +110,11 @@ }, { "capability": "oversized_authorization_structure", - "content_sha256": "6552c4f594bc0ad05a231f1d9c202b78e5b4e31a0ce05ea32e7eb48b67d440ba", - "end_line": 2864, + "content_sha256": "ef1302419840b5659d01e45efd5806b904507ad8f9c0e457287597ac3ecbfb6f", + "end_line": 2847, "hard_limit": 1200, "kind": "test_file", - "observed_lines": 2864, + "observed_lines": 2847, "path": "backend/tests/test_artifact_admission.py", "qualified_symbol": null, "removal_chunk": "WS-AUTH-003-CLOSE", @@ -171,26 +171,26 @@ { "capability": "oversized_authorization_structure", "content_sha256": "2fac78c747fd82908991bde192ba874d3847b3ad105a827575dfc6afd73614be", - "end_line": 2277, + "end_line": 2281, "hard_limit": 120, "kind": "test_function", "observed_lines": 194, "path": "backend/tests/test_artifact_admission.py", "qualified_symbol": "test_guide_admission_consumes_real_project_manager_prep_atomically", "removal_chunk": "WS-AUTH-003-CLOSE", - "start_line": 2084 + "start_line": 2088 }, { "capability": "oversized_authorization_structure", "content_sha256": "12f1dd1e0632ec3e5fb80ea19eb01bcce2b9a14015a04e120fb304f9b343b828", - "end_line": 2081, + "end_line": 2085, "hard_limit": 120, "kind": "test_function", "observed_lines": 132, "path": "backend/tests/test_artifact_admission.py", "qualified_symbol": "test_guide_admission_derives_three_scopes_without_provider_evidence", "removal_chunk": "WS-AUTH-003-CLOSE", - "start_line": 1950 + "start_line": 1954 }, { "capability": "oversized_authorization_structure", diff --git a/.ci/behavior-ownership/partition.v1.json b/.ci/behavior-ownership/partition.v1.json index 5bdf6758a..5cb0a507d 100644 --- a/.ci/behavior-ownership/partition.v1.json +++ b/.ci/behavior-ownership/partition.v1.json @@ -258,15 +258,23 @@ }, { "group": "artifacts", - "target": "backend/app/modules/artifacts/api/submission_materialization.py" + "target": "backend/app/modules/artifacts/api/submission_preparation.py" }, { "group": "artifacts", - "target": "backend/app/modules/artifacts/api/submission_preparation.py" + "target": "backend/app/modules/artifacts/authorization.py" }, { "group": "artifacts", - "target": "backend/app/modules/artifacts/authorization.py" + "target": "backend/app/modules/artifacts/checker_output_bindings.py" + }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/checker_output_custody.py" + }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/checker_outputs.py" }, { "group": "artifacts", @@ -724,6 +732,14 @@ "group": "artifacts", "target": "backend/app/modules/checkers/api/history.py" }, + { + "group": "artifacts", + "target": "backend/app/modules/checkers/api/materialization.py" + }, + { + "group": "artifacts", + "target": "backend/app/modules/checkers/api/output_custody.py" + }, { "group": "artifacts", "target": "backend/app/modules/checkers/api/policy_compilation.py" @@ -1473,7 +1489,7 @@ "target": "backend/scripts/validate_test_lane_evidence.py" } ], - "authority_digest": "45c8605a1e6b8a29cd5c9fb59771f423f8e87f04ed7da07833139cc1f657e22b", + "authority_digest": "c836827deba926d6b40689de2d7a723c9640fe209b11edb49d78dacb12b5744e", "protected_base_commit": "7676ce4347db0c9694962a9b587a20765e16eac6", "schema": "workstream.behavior-ownership-partition.v1" } diff --git a/.ci/module-boundaries/private-edge-debt.v1.json b/.ci/module-boundaries/private-edge-debt.v1.json index 2b961e53b..9a47e2df4 100644 --- a/.ci/module-boundaries/private-edge-debt.v1.json +++ b/.ci/module-boundaries/private-edge-debt.v1.json @@ -205,24 +205,12 @@ "imported_private_path": "app.modules.tasks.models", "repair_owner": "WS-ARCH-001-03" }, - { - "source_file": "backend/app/modules/artifacts/repository.py", - "target_module": "checkers", - "imported_private_path": "app.modules.checkers.models", - "repair_owner": "WS-ARCH-001-04" - }, { "source_file": "backend/app/modules/artifacts/repository.py", "target_module": "projects", "imported_private_path": "app.modules.projects.models", "repair_owner": "WS-ARCH-001-03" }, - { - "source_file": "backend/app/modules/artifacts/repository.py", - "target_module": "tasks", - "imported_private_path": "app.modules.tasks.models", - "repair_owner": "WS-ARCH-001-03" - }, { "source_file": "backend/app/modules/artifacts/service.py", "target_module": "actors", diff --git a/.commitrail/INDEX.md b/.commitrail/INDEX.md index 051d526d1..5b4555b3b 100644 --- a/.commitrail/INDEX.md +++ b/.commitrail/INDEX.md @@ -8,12 +8,12 @@ for current product capability. |---|---|---| | [WS-DB-002](initiatives/WS-DB-002/OVERVIEW.md) | Complete | Shared UUIDv7 record generation, native-UUID relationships and fresh v0.1 baseline; natural-owner retry custody and aligned CI/local setup | | [WS-MCP-002](initiatives/WS-MCP-002/OVERVIEW.md) | Planned | Three self-service tools delivered through WS-MCP-002-02; 24 tools remain and WS-MCP-002-03 administrative reads are next | -| [WS-ARCH-001](initiatives/WS-ARCH-001/OVERVIEW.md) | Planned | Canonical ARCH-04A delivered; Automatic unified setup delivered; POL-05/06 manager review and separate approvals delivered; POL-07B internal checker phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [WS-ART-001](initiatives/WS-ART-001/OVERVIEW.md) | Planned | Hidden exact post-submit input materialization delivered; ARCH-04B2 output custody next | -| [WS-AUTH-001](initiatives/WS-AUTH-001/OVERVIEW.md) | Planned | Shared dispatcher authority and phase custody after delivered CP05; POL-04B consumes completed finalization authority; AUTH-12F4 supplies proposal authority; POL-05B public composition delivered; AUTH-12G post-policy authority delivered; POL-06B public composition delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [WS-CON-001](initiatives/WS-CON-001/OVERVIEW.md) | Planned | CP06 selected-policy validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [WS-AUTH-003](initiatives/WS-AUTH-003/OVERVIEW.md) | Planned | TASK/checker canonical history authority delivered and alternate gate removed; AUTH-18 public manager activation and ARCH-03D hidden intake delivered; ARCH-04B hidden input delivered; continue boundary recovery with ARCH-04B2 output custody next | -| [WS-POL-003](initiatives/WS-POL-003/OVERVIEW.md) | Planned | Unified setup, separate draft proposals and guide document intake delivered; Hidden proposal review/correction/pre-policy approval custody delivered; AUTH-12F4 proposal authority delivered; POL-05B public review/approval/manual correction dispatch delivered; POL-06A hidden post-policy projection/read/approval/correction delivered; AUTH-12G live authority delivered; POL-06B public access and automatic derivation delivered; POL-07A ART pre-submit attempt recovery delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-ARCH-001](initiatives/WS-ARCH-001/OVERVIEW.md) | Planned | Public guide activation and task reads, hidden approved-guide intake, exact post-submit input materialization and hidden ARCH-04B2 checker-output custody delivered; ARCH-04C durable execution is next | +| [WS-ART-001](initiatives/WS-ART-001/OVERVIEW.md) | Planned | Hidden exact post-submit input materialization and ARCH-04B2 output custody delivered; ARCH-04C durable execution next | +| [WS-AUTH-001](initiatives/WS-AUTH-001/OVERVIEW.md) | Planned | Manager guide activation, public task authority, dispatcher mechanics and hidden intake are delivered; ARCH-04B input and ARCH-04B2 output custody exist with deny-only/unavailable production authority; ARCH-04C durable execution is next | +| [WS-CON-001](initiatives/WS-CON-001/OVERVIEW.md) | Planned | ContributionPolicy administration and guide binding plus hidden intake, post-submit input and ARCH-04B2 output custody delivered; ARCH-04C durable execution precedes shared FinalAcceptance, submitter ContributionRecord and applicable awards | +| [WS-AUTH-003](initiatives/WS-AUTH-003/OVERVIEW.md) | Planned | TASK/checker canonical history authority delivered and alternate gate removed; AUTH-18 public manager activation and ARCH-03D hidden intake delivered; ARCH-04B hidden input and ARCH-04B2 output custody delivered; continue boundary recovery with ARCH-04C durable execution next | +| [WS-POL-003](initiatives/WS-POL-003/OVERVIEW.md) | Planned | Unified setup, public manager policy operations, phase composition, public guide activation, hidden intake, post-submit input and ARCH-04B2 output custody delivered; ARCH-04C durable execution is next | | [WS-REV-001](initiatives/WS-REV-001/OVERVIEW.md) | Planned | Shared acceptance/source and existing fence foundations; human hidden review work remains independently dependency-gated | | [WS-QUAL-002](initiatives/WS-QUAL-002/OVERVIEW.md) | Planned | Populate subsystem ownership before changed-line mutation work | | [WS-QUAL-003](initiatives/WS-QUAL-003/OVERVIEW.md) | Planned | Audit and prune test proof, add missing safety cases, decompose oversized test modules | diff --git a/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md b/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md index eeae50a29..7c4bec26c 100644 --- a/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md @@ -14,7 +14,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), - Current boundary: one CHECKERS catalogue, compiler/parser and implementation per checker ID serve active policy consumers; production post-submit phase execution remains unavailable. -- Next usable boundary: checker output custody (ARCH-04B2), following delivered hidden input materialization (ARCH-04B). Production execution and live materialization authority remain unavailable. +- Delivered storage boundary: hidden [ARCH-04B2 checker-output custody](WS-ARCH-001-04B2.md), following hidden input materialization (ARCH-04B). Typed store, byte-free recovery and flush-only verified binding exist; production CHECKERS reservation and authority remain unavailable. +- Next usable boundary: ARCH-04C durable execution and results, including the current structural catalogue's valid empty output set. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation using the existing CP07 operation. [CP05A](WS-ARCH-001-CP05A.md) supplies public Finance policy administration and recoverable draft selectors. diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md new file mode 100644 index 000000000..df62ad8b2 --- /dev/null +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md @@ -0,0 +1,303 @@ +# ARCH-04B2 — Verified checker output custody + +- Initiative: `WS-ARCH-001` +- Durable disposition: `Complete` +- Intended merge outcome: Hidden ART storage and flush-only verified binding of exact checker-run output, with production access unavailable until CHECKERS reservations and AUTH activation exist. + +## Intent + +Complete the next storage boundary on the path to a usable contributor journey: +claim -> upload ZIP -> pre-submit feedback or immutable Submission -> automatic +post-submit evaluation -> outcome. The first complete no-human-review journey must +include shared FinalAcceptance, the submitter ContributionRecord and applicable +policy awards before live human review/revision becomes a dependency. Human review +packet foundations may proceed independently in another worktree; shared acceptance +is one operation used by both triggers, never a second automated acceptance engine. +Contributor skip and assignment expiry remain deferred. + +## Current behavior + +Main `9c292100` includes merged ARCH-04B exact verified input and bounded scratch. +The output-store and output-binding capabilities begin as unused declarations; +this change gives them CHECKERS-owned consumer contracts and ART implementations. Generic ART preparation, quota admission, durable put, +unknown-outcome recovery and independent verification already exist. Checker +output admission still reads private CheckerRun rows from ART's repository. +Existing CHECKERS rows do not persist 04A request/generation/worker reservation +facts. Current structural checker definitions permit zero generated output bytes +and no required output roles. This change must not invent evaluator outputs or +claim the current structural catalogue supports them. + +## Design and decisions + +1. Replace the unused output/binding declarations with CHECKERS-owned consumer + requests, results and ports in its public API; ART implements them through + composition injection. The input-materialization port uses the same dependency + direction. Delete the replaced declaration paths; no compatibility alias or + second storage engine. A CHECKERS-owned + public reservation-facts port supplies exact run/request/Submission/policy, worker-lease generation and + owner-declared output-slot/budget facts. Each slot has a globally unique bounded + key within its run, media type and byte limit. Its production implementation is explicitly + unavailable until ARCH-04C; hidden proof uses a controlled owner fixture, + clearly distinguished from live reservation/currentness proof. +2. The public store request contains selector plus byte source, never PREP or raw + AuthorizationContext. ART action-specific preflight denies before protected reads, byte-source iteration, scratch or + provider access. Exact reservation facts must match the complete validated + evaluation request and output selector. Role/budget comes from CHECKERS's + reservation, never arbitrary contributor input. No production log/output role is introduced. The current structural plan has + zero slots; 04C must accept that exact empty output set. Controlled nonempty + test slots prove storage mechanics only, not a live evaluator capability. +3. Reuse canonical bounded scratch to commit exact output bytes. Add a per-call + byte ceiling bounded by the existing manager maximum; over-cap input stops + before writing excess bytes and cleans the reservation. Re-resolve owner + facts and authority in a short transaction before generic quota admission and + durable intent. Charge deployment/project/task/fixed producer, not contributor. + Bind exact evaluation/run/output identity into the existing put request digest; + persist Submission identity/version and a narrow checker-request digest on that + attempt, and verified receipt/put + ancestry on the immutable checker binding. Do not copy the evaluation envelope + or add a competing CHECKERS reservation store. This receipt is material lineage, never + independent CHECKERS currentness or authorization. + Operation identity remains (run, globally unique slot). The request digest + binds evaluation request ID/hash/generation, run, immutable owner lineage, + slot key/media/ceiling, commitment, namespace and quota scopes. It excludes + volatile worker lease ID/generation: every store/replay/bind separately resolves + and authorizes the active lease. A replacement worker recovers the same logical + output/put; an old lease cannot publish. Changed immutable facts or bytes conflict. +4. Release transaction/locks/PREP before provider I/O. Reuse existing committed + put, observation and verification operations. Unknown outcomes retain the + original attempt; absent unreproducible bytes cannot be fabricated or + regenerated. Always clean private scratch. A verified reference is returned + only for the exact attempt/replica/independent verification receipt. + `recover(selector)` retrieves the stable operation after a lost response without + a byte source. It reuses observation and verification, never provider put or + regeneration. `store` still commits supplied bytes and rejects changed replay + content. The narrow checker-request digest permits recovery/binding comparisons + without reconstructing historical quota configuration inside the full digest. +5. A separate flush-only binding participant runs inside the caller's transaction. + Caller supplies fresh CHECKERS reservation/fence facts and exact authority + before ART locks. Match persisted intent and full verified chain, then create + or replay one immutable run/role binding. Changed bytes, foreign intent, + incomplete verification, stale owner facts or late revoked authority deny. + Authority custody is prepared before CHECKERS reservation/currentness locks, + followed by ART scope/row locks. ART never locks CHECKERS/TASK privately. + Verify using immutable terminal receipt ancestry or the verifier's compatible + job -> replica -> attempt -> content lock order; do not introduce the inverse. + Generic bindings receive checker-only put_attempt_id and verification_receipt_id. + An INSERT guard verifies exact receipt -> job -> attempt -> replica/content + ancestry, verified state and project/run/slot ownership. Seal terminal + attempt/job facts and replica identity atomically with that binding; preserve + mutable replica health. Reuse the existing binding immutability trigger; do not invent a common receipt FK across distinct + direct-put and observation receipt tables. ARCH-04C later composes this participant + with its final result and outbox; ARCH-04D owns live service activation. +6. Replace the raw-AuthorizationContext checker admission path with a mandatory + action-specific authority participant and owner reservation facts. Default + admission without that participant denies before mutation. Replace the affected + private checker admission/recovery lookup with owner facts. + Trace remaining consumers explicitly; preserve retained records. Do not rewrite + unrelated owner behavior or add compatibility constructors. + +## Bounded change + +### Allowed + +- `backend/app/interfaces/artifact_operations.py` only to remove the superseded + output requests/results and ports. +- `backend/app/modules/artifacts/checker_outputs.py`, + `backend/app/modules/artifacts/checker_output_custody.py` and + `backend/app/modules/artifacts/checker_output_bindings.py` (new ART owners). +- `backend/app/modules/artifacts/preparation.py` for the bounded per-call cap. +- `backend/app/modules/artifacts/models.py`, `schemas.py`, `service.py`, + `repository.py` only for affected output custody and traced lookup consumers. + The separate Operator resource-to-project CHECKERS lookup remains explicitly + outside this mutation boundary; it is not a second output admission path. +- `backend/app/modules/checkers/api/output_custody.py` for closed owner facts, + output requests/results and consumer ports, and + `backend/app/modules/checkers/api/materialization.py` for the relocated input + materialization contract. + Remove `backend/app/modules/artifacts/api/submission_materialization.py` and its + exports from `backend/app/modules/artifacts/api/__init__.py`; update affected + callers without compatibility aliases. Composition in existing + `backend/app/adapters/artifacts/__init__.py`, `backend/app/adapters/checkers/`. +- One ART-owned migration `backend/alembic/versions/0007_checker_output_custody.py` after `0006_history_read_authority` for immutable exact + output intent/binding custody; no CHECKERS run writer or retained-data deletion. +- New `backend/tests/test_checker_output_custody.py`, + `backend/tests/test_checker_output_storage.py`, + `backend/tests/test_artifact_admission_digest.py` for unaffected guide replay, + `backend/tests/checker_output_custody_helpers.py`, + `backend/tests/checker_output_admission_helpers.py`, and + existing `test_artifact_admission.py`, `test_artifact_recovery.py`, + `test_artifact_architecture.py`, `test_artifact_preparation.py`, + `test_artifact_authorization.py`, `backend/tests/conftest.py` (exact migrated + fingerprint and protected-table reset inventory), `backend/alembic/env.py`, + `test_alembic.py`, `test_coverage_contract.py` for exact migration-head registration, module boundary + tests and migration tests, including + `backend/tests/authorization/submission_history/test_migration.py` for exact + predecessor-schema restoration across the new dependent foreign key. +- Existing lane/ownership/module/test-structure inventories and their tests only + to register changed owners and keep existing gates intact. +- This record, linked ARCH output skeleton, current ARCH/ART/AUTH/POL/CON navigation, + `.commitrail/INDEX.md`, `README.md`, `docs/roadmap_status.md`, artifact/checker/data + model specifications, authorization specification, operating manual and + authorization custody page for affected claims. + +### Not allowed + +Public routes, live grants, checker reservation/result/currentness writers, +TASK transitions, policy/catalogue expansion, inference, reviewer workflows, +acceptance implementation, alternate scratch/store/factory, data deletion, +compatibility paths, CI weakening or contributor quota charges for system outputs. + +## Acceptance criteria + +- Default composition fails before any protected or external access. +- Controlled exact reservation plus real PostgreSQL/Local and MinIO storage proves + preparation, exact byte commitment, independent verification and immutable binding. +- Valid foreign stored lineages and mixed reservation/intent/binding selectors deny; + positive controls succeed. Requests cannot select a role or budget outside the + owner-issued reservation. Invalid requests never invoke the byte source/provider. +- Same run/role/bytes replay produces the same intent and binding; changed request + or bytes reject. Concurrent publication and caller rollback preserve one binding + and no partially committed effect. +- Database rejects intent/binding identity mutation, mismatched stored ancestry, + and subsequent changes to sealed verified ancestry, including concurrent writes + under READ COMMITTED and REPEATABLE READ. + Migration refuses retained checker attempts lacking provable evaluation custody; + it never invents the missing checker-request digest or deletes retained data. +- Unknown put outcome resumes observation of the same intent without regenerating + output; missing unreproducible bytes remain unavailable. Existing generic recovery + proof is reused rather than copied. +- Quota/deadline/cancellation cleanup and fresh authority after I/O are demonstrated; + independent sessions prove no row lock spans provider I/O. +- No claim of live CHECKERS request/lease custody: that remains ARCH-04C proof. + +## Risk and review routing + +- Risk class: L1 (untrusted bytes, immutable evidence, storage/schema boundary). +- Required reviewers: architecture/reuse, security, QA/test delta, docs/product; + CI integrity for affected lane/inventory registrations. +- Human review focus: exact run/request/role binding, unavailable production boundary, + shared storage reuse, retained evidence and automated acceptance sequencing. +- Plan review resolved the missing producer with an unavailable owner seam, not + a new runtime fixture; current structural outputs remain empty. Replay binds + immutable attempt facts while fresh authority fences the changing worker lease. + +## Evidence + +The focused proof lives in `test_checker_output_custody.py` and +`test_checker_output_storage.py`: + +- Default composition denies before protected access; valid stored foreign and + mixed selectors reject before source/provider use after two valid controls. +- Local/MinIO store -> independent verify -> binding preserves exact persisted + identities; binding rollback and concurrent replay preserve one result. +- Lost acknowledgement uses typed observation and byte-free recovery; + worker takeover preserves logical output identity and denies the old lease. +- Slot caps stop iteration and clean scratch; revocation during provider I/O + denies the returned reference, with independent-session lock checks. +- Direct SQL rejects mixed binding ancestry and immutable intent changes; + migration refuses retained output lacking provable request custody. + +Affected admission/recovery tests retain quota, service-identity, relationship, +namespace precedence and transaction guarantees under the new closed request. +Existing preparation tests cover deadline, cancellation and aggregate quotas; +new per-slot cases extend that same canonical path. + +Verification uses the existing isolated PostgreSQL/MinIO runner, Ruff, +structure/module-boundary validators, markdown links, stale wording, +Commitrail checks and complete hosted lanes. Exact command results and +review targets belong in the PR. Guard-removal probes must reach the behavioral +assertion after valid controls, not fail during fixture setup. Hidden controlled +reservation/action-authority fixtures prove ART mechanics only; actual CHECKERS +lease custody and AUTH activation remain subsequent work. + +## Plan review disposition + +The focused plan review found the original store PREP lifetime, +per-slot scratch ceiling, missing reservation producer, and binding ancestry +underspecified. The design above incorporates those repairs. The reviewed stable/volatile split keeps worker leases out of logical output +identity while authorizing the caller's exact claimed active lease on every phase. +Existing binding +immutability is reused; no generated-output catalogue expansion or duplicate +custody aggregate is authorized. + +## Review findings + +- Immutable binding custody must cover statement-level deletion as well as row + mutation. Reuse the existing ART fact-mutation rejection function for a binding + no-TRUNCATE trigger; prove direct and cascading truncation rejects and preserves + a real verified binding. The canonical isolated test reset disables/restores + this guard and pins the resulting schema fingerprint. +- Service-level binding proof must substitute valid foreign put/receipt identities + after both valid controls, independently of the database insert guard. The + requested-attempt filter removal must make the negative service test fail. +- Current authorization specifications must show delivered input/output custody, + then ARCH-04C execution/results, then ARCH-04D activation. ARCH-04F gates + remediation/public intake and enabling false, not the earlier true routing branch. +- Retained-data migration probes must remove the exact new dependent ART foreign + key when reconstructing the predecessor checker schema and restore it after + committed concurrency probes; no CASCADE shortcut or weakened assertion. The + existing guide quota-reconciliation fixture supplies explicit null checker + custody fields under the expanded internal admission facts; its rollback and + configured-limit assertions remain intact. + +## Reconciliation + +- Current-source reconciliation: merged ARCH-04B on main; no overlapping open + product PR. AUTH provisioning proof and deferred lease planning are separate. +- Next usable boundary: ARCH-04C durable execution/results, then exact AUTH and + automatic routing; shared acceptance and compensation participants precede + no-human-review activation; remediation and public intake complete the first + contributor milestone before live human review/revision. +- Remaining risks: CHECKERS reservation producer and live authority deliberately + absent; only controlled hidden storage proof is claimed by this child. + +## External review repair scope + +The output capability must have a typed acyclic consumer/implementation seam; +placing its declarations in shared `app.interfaces` does not remove the +ART-to-CHECKERS dependency. CHECKERS owns the consumed output and materialization +ports in its public API; ART implements them and composition injects them. Move +the existing ART materialization contract to that consumer API and delete its +old path. ARCH-04C imports its own consumer ports, never ART API/private owners. +The existing pre-submit archive-execution private ART dependency remains a +separately tracked consumer; it is not a new post-submit dependency or fallback. Replace that affected declaration path, update all +callers and boundary proof, and make ARCH-04C consume the declared seam without +importing ART implementation. No compatibility re-export is allowed. + +Verified binding custody must survive subsequent SQL mutations, not merely pass +an insertion check. Protect the ancestry facts used by the inserted binding, +including put execution state and references, while preserving legitimate +unbound put/observation recovery. Prove post-bind mutations fail and leave the +binding and its exact chain unchanged; include the competing-transaction case. + +Add focused service proofs for identical `store()` replay without another put +and cancellation while provider I/O is paused with scratch cleanup. Share the +canonical run/slot operation identity instead of rebuilding its hash. Reconcile +the current ARCH plan's stale missing-storage sentence. These are repairs within +the existing L1 boundary and require fresh architecture/reuse, security, +QA/test-delta, documentation/product and CI-integrity review. Existing prohibited +changes and human review focus remain unchanged. The affected owner API, +architecture boundary tests and ARCH-04C contract are included in allowed scope. + +The ancestry repair seals the terminal attempt and verification job plus replica +identity in the successful binding transaction. A monotonic row-local seal is +needed because a transaction using an older PostgreSQL snapshot could miss a +newly inserted binding if protection relied only on cross-table existence checks. +Binding takes the existing job -> replica -> attempt -> content order with update +locks on the sealed ancestors, validates the exact chain, and sets seals atomically. +The attempt/job terminal facts and replica identity cannot change afterward; +replica health and availability remain operational facts that may change. Failed +or rolled-back publication must not leave seals behind. Real PostgreSQL proof +covers both READ COMMITTED and REPEATABLE READ interleavings, not only sequential +mutation. No separate custody aggregate or public capability is introduced. + +Follow-up review identified two additional bounded repairs. The binding guard +must reject a null terminal verification result explicitly: SQL's nullable +comparison cannot establish verified ancestry. Direct-SQL proof supplies an +otherwise valid chain and checks rejection without a binding or seals. Checker +lineage fields belong only in the checker-output admission digest; unrelated +guide and submission producers retain their canonical digest inputs. Guide +replay must recover its existing attempt without a conflicting digest. Submission +bundle replay already returns through its owner replay port before this digest; +no submission replay failure is claimed. These changes add no compatibility +path and do not rewrite retained attempts. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md index 0fe4ed520..55cc95b8a 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md @@ -4,7 +4,7 @@ Use the [current dependency contract](PLAN.md#current-dependency-contract). The [preserved map](../pre-cutover/CHUNK_MAP.md) retains the complete original work accounting. Foundations through 02H and CP04B are complete; none restart. AUTH-18 public manager activation/context and ARCH-03D hidden approved-guide intake are delivered. -ARCH-04B hidden exact post-submit input is delivered. ARCH-04B2 output custody is next; +ARCH-04B hidden exact post-submit input and ARCH-04B2 output custody are delivered. ARCH-04C durable execution is next; public intake remains deferred to ARCH-02I. | Boundary | Owner outcome | Risk | Current dependency | @@ -32,15 +32,15 @@ public intake remains deferred to ARCH-02I. | [WS-ARCH-001-03C4](../WS-ARCH-001-03C4.md) | Exact-authorized public task queues | L1 | Complete; contributor, manager and operational projections with signed pagination | | [WS-ARCH-001-03C5](../WS-ARCH-001-03C5.md) | Exact-authorized Contributor and Manager detail and requirements | L1 | Complete; canonical projections, historical policy custody and atomic read evidence | | [WS-ARCH-001-03C6](../WS-ARCH-001-03C6.md) | Distinct exact-authorized locked-context reads | L1 | Complete; bounded Audit Authority history access delivered by 03C7 | -| [WS-ARCH-001-03C7](../WS-ARCH-001-03C7.md) | Exact-authorized bounded task history | L1 | Complete; AUTH-18 public guide activation delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-ARCH-001-03C7](../WS-ARCH-001-03C7.md) | Exact-authorized bounded task history | L1 | Complete; AUTH-18 public guide activation, ARCH-03D hidden approved-guide intake, hidden exact post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next | | [WS-ARCH-001-03C](chunks/WS-ARCH-001-03C-auth-task-readiness.md) | Exact task/assignment public activation and integrated readiness proof | L1 | Complete through 03C7 audit history, 03C4 queues, 03C5 detail/requirements and 03C6 locked-context reads; AUTH-18 public guide activation delivered; ARCH-03D hidden intake delivered; public intake cutover remains separate | | [WS-ARCH-001-04A](../WS-ARCH-001-04A.md) | CHECKER post-submit contract and registered evaluator conformance | L1 | Complete canonical catalogue, phase facts and structural conformance; consumed by delivered POL-04B and POL-07B | | [WS-ARCH-001-04B](../WS-ARCH-001-04B.md) | ART exact verified Submission materialization | L1 | Complete hidden verified input with async scoped reads; production authority remains deny-only until 04D | -| [WS-ARCH-001-04B2](chunks/WS-ARCH-001-04B-art-post-submit-materialization.md#arch-04b2--separate-art-output-custody-child) | ART generated-output/log custody and verified binding | L1 | 04A public request/run facts plus merged ART foundations; no CHECKERS private lookup | -| [WS-ARCH-001-04C](chunks/WS-ARCH-001-04C-checker-current-result.md) | CHECKER hidden durable current output and supersession behavior | L1 | Planned after 04A/04B/04B2; production remains deny-only | +| [WS-ARCH-001-04B2](../WS-ARCH-001-04B2.md) | ART checker-output custody and verified binding | L1 | Complete hidden typed store, byte-free recovery and flush-only binding; production reservation and authority unavailable | +| [WS-ARCH-001-04C](chunks/WS-ARCH-001-04C-checker-current-result.md) | CHECKER hidden durable current output and supersession behavior | L1 | Planned after delivered 04A/04B/04B2; must support the structural catalogue's empty output set; production remains deny-only | | [WS-ARCH-001-04D](chunks/WS-ARCH-001-04D-auth-post-submit-activation.md) | AUTH exact fixed-service post-submit activation (replaces XINT-06B) | L1 | Planned after 04B/04C evidence | | [WS-ARCH-001-04E](chunks/WS-ARCH-001-04E-canonical-allow-review.md) | TASK current routing: true to canonical `allow_review`, false/pass to shared acceptance | L1 | Source 04E1A -> hidden handlers 04E1B -> AUTH 04E2 -> live 04E3, plus 04D/OUTBOX-02; false consumes shared REV/CON/fence proof and activation also requires 04F remediation | -| [WS-ARCH-001-03D](../WS-ARCH-001-03D.md) | Exact activated historical guide through hidden durable intake; obsolete lookup removed | L1 | Complete; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next, public cutover remains deferred | +| [WS-ARCH-001-03D](../WS-ARCH-001-03D.md) | Exact activated historical guide through hidden durable intake; obsolete lookup removed | L1 | Complete; hidden exact post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next, public cutover remains deferred | | [WS-ARCH-001-04F](chunks/WS-ARCH-001-04F-checker-remediation.md) | Contributor-correctable checker failures and same-lineage admission-backed replacement Submission | L1 | Planned after 04E; replaces XINT-05C, required before public 02I, not before REV begins from `allow_review` | CP09, 04E and 04F are coordination parents, not permission for multi-owner PRs. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md index c0aa70b4a..ad412c883 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md @@ -44,13 +44,13 @@ checker-remediation boundary before public Submission cutover. | [ARCH-03C4](../WS-ARCH-001-03C4.md) | ARCH-03C3 | Complete: exact-authorized contributor, manager and operational public queues | | [ARCH-03C5](../WS-ARCH-001-03C5.md) | ARCH-03C4 | Complete: exact-authorized Contributor/Manager detail and requirements | | [ARCH-03C6](../WS-ARCH-001-03C6.md) | ARCH-03C5 | Complete: distinct exact-authorized locked-context reads | -| [ARCH-03C7](../WS-ARCH-001-03C7.md) | ARCH-03C6 and hidden TASK owner contracts | Complete: bounded Audit Authority history access; public guide activation is delivered by AUTH-18; ARCH-03D hidden intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | -| [CP05A](../WS-ARCH-001-CP05A.md) | CP05, existing policy owners | Complete: public Finance policy administration and selector recovery; AUTH-18 public manager activation/context delivered; ARCH-03D hidden intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [ARCH-03C7](../WS-ARCH-001-03C7.md) | ARCH-03C6 and hidden TASK owner contracts | Complete: bounded Audit Authority history access; public guide activation, ARCH-03D hidden intake, hidden post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next | +| [CP05A](../WS-ARCH-001-CP05A.md) | CP05, existing policy owners | Complete: public Finance policy administration and selector recovery; public manager activation, ARCH-03D hidden intake, hidden post-submit materialization and ARCH-04B2 output custody delivered; ARCH-04C next | | CP09 (later cleanup coordination) | All legacy consumers replaced, including CHECKER and public 02I path | Physical economic deletion; not on the allow_review critical path | | [ARCH-03D](../WS-ARCH-001-03D.md) | AUTH-18, ARCH-03A, CP08 and merged ART preparation | Complete: hidden durable intake uses exact historical TASK/PROJECTS ports; public cutover remains ARCH-02I | | ARCH-04B | ARCH-04A, POL-07, ARCH-03C, merged ARCH-02H | Complete: ART hidden verified Submission input; live authority remains deferred | -| ARCH-04B2 | ARCH-04A and merged ART admission/verification/binding foundations | ART bounded checker output/log ingestion and verified binding, no routing | -| ARCH-04C | ARCH-04A, ARCH-04B, ARCH-04B2, POL-07 | CHECKERS durable execution/result/currentness and worker recovery | +| [ARCH-04B2](../WS-ARCH-001-04B2.md) | ARCH-04A and merged ART admission/verification/binding foundations | Complete: hidden bounded checker-output store/recovery and verified binding, no routing or live reservation | +| ARCH-04C | ARCH-04A, ARCH-04B, delivered ARCH-04B2, POL-07 | CHECKERS durable execution/result/currentness and worker recovery, including valid empty output sets | | ARCH-04D | ARCH-04B, ARCH-04C | AUTH post-submit materialization/result activation | | AUTH-OUTBOX-01 | Merged shared outbox persistence and AUTH service/PREP foundations | Complete: planned dispatcher identity/action/matrix and unavailable typed authority contract | | CON-02B | AUTH-OUTBOX-01 | Complete: shared hidden dispatcher/claim fencing, typed handlers and recovery | @@ -69,6 +69,11 @@ the usable ARCH-04F failure route. The true routing foundation may ship first with false still unavailable; neither route invents a new acceptance subsystem. Shared REV acceptance persistence/CON participation can precede live human queues or decisions, so this extension adds no REV-admission dependency cycle. +The first complete contributor milestone includes public claim/upload/pre-check +feedback, immutable Submission, automatic checking, failure remediation and, for +locked false, shared FinalAcceptance plus the submitter ContributionRecord and +applicable awards. Complete it before live human review/revision; independent +review foundations may proceed in another worktree without blocking that path. CP05 and ARCH-04A have independent prerequisites. POL-04B consumes the corrected ARCH-04A catalogue/schema before producing approval-eligible generations. Owners may @@ -82,14 +87,14 @@ ARCH-03C1 completes exact reconciler authority and decision-bound receipts. ARCH-03C2 supplies originating-transaction-only per-assignment producer events and first handler registration with enforced prefork topology; it must never backfill or dispatch retained invalidation rows. Public TASK activation is complete through ARCH-03C7. AUTH-18 delivers public -manager guide activation/context; ARCH-03D hidden intake is delivered; hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. Subsequent +manager guide activation/context; ARCH-03D hidden intake, hidden exact post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C is next. Subsequent PR-sized contracts name exact files, public types, current migration head and runnable proof before implementation; they refine this design, not create a new permission requirement. ### Supporting foundations required by automatic routing -Current supporting contracts are [ARCH-04B2](chunks/WS-ARCH-001-04B-art-post-submit-materialization.md#arch-04b2--separate-art-output-custody-child), +Current supporting contracts are delivered [ARCH-04B2](../WS-ARCH-001-04B2.md), [AUTH-OUTBOX-01/02](../../WS-AUTH-001/planning/PLAN.md#ws-auth-001-outbox-01--unavailable-dispatcher-contract), [CON-02B](../../WS-CON-001/OVERVIEW.md#con-02b-current-dispatcher-contract), and [ARCH-04E1A/04E1B/04E2/04E3](chunks/WS-ARCH-001-04E-canonical-allow-review.md#current-bounded-sequence). @@ -114,15 +119,18 @@ checker finalization or task transitions. The implementation contract must name those feature action/resource manifests, not infer authority from the event type. Lost delivery/redelivery cannot create a new logical evaluation. -ART checker-output storage is also missing, not implicit in CHECKERS result -persistence. An ART-owned child of 04B supplies bounded generated-output/log +Hidden ARCH-04B2 ART checker-output storage and binding are delivered, not +implicit in CHECKERS result persistence. They supply bounded generated-output ingestion, generic quota admission attributed to the fixed service, independent -reread verification and exact checker-output binding. Its controlled hidden -fixtures use 04A public request/run facts, not a future TASK dispatch row or -private CHECKERS import. 04C composes the resulting verified binding references -with final-result persistence; 04D activates the exact ART write/binding and -CHECKERS completion surfaces. External byte I/O occurs before the final caller -transaction; binding publication and final result become visible atomically. +reread verification and exact checker-output binding. Production CHECKERS +reservation/currentness and output authority remain unavailable, and the current +structural catalogue reserves zero output slots; controlled nonempty fixtures +prove ART mechanics only. ARCH-04C is next and must accept that empty output set; +when owner-declared slots exist it composes their verified binding references +with final-result persistence. ARCH-04D then activates the exact ART write/binding +and CHECKERS completion surfaces. In that planned flow, external byte I/O occurs +before the final caller transaction; binding publication and final result become +visible atomically. Failed storage never becomes contributor blame or an `allow_review` result. No second artifact store, quota ledger or historical ART-06B implementation lane is introduced. @@ -175,8 +183,8 @@ owning implementation, not with planning prose or fake database claims. ### Named proof targets for the remaining design ARCH-04B input proof is delivered in `test_post_submit_materialization.py` and -`test_post_submit_selection.py`, including deny-before-I/O composition. Output -custody and durable/live execution proof remain with ARCH-04B2/04C/04D below. +`test_post_submit_selection.py`, including deny-before-I/O composition. ARCH-04B2 +delivers output custody proof; durable/live execution proof remains with 04C/04D below. These are required future implementation tests, not tests claimed present or executed by this planning PR. Each owner's bounded record fixes the final @@ -193,7 +201,7 @@ claims require their real custody, not unit substitutes. | ARCH-04A/POL-07 | `test_registered_evaluator_rejects_invalid_work`, `test_checker_facade_delegates_once` | Actual registered evaluator fixtures and typed composition; presence-only mutant must fail | | CP06/CP07/AUTH-12H | `test_activate_without_legacy_payment_or_task`, `test_activation_requires_exact_selected_policy`, `test_activation_rejects_missing_review_revision_config` | PostgreSQL atomic command plus full response serialization; foreign/retired/incomplete new binding denies | | CP08/ARCH-03A/03B/03C | `test_ready_preserves_screening_policy_lock`, `test_claim_copies_policy_without_current_lookup` | PostgreSQL and actual AUTH/owner composition; later publication leaves existing attempt unchanged | -| ARCH-04B2/04C/04D | `test_late_revocation_cannot_publish_result`, `test_unfinished_checker_recovery_reuses_attempt`, `test_terminal_retry_requires_operator_and_new_attempt` | Local/MinIO, real worker/provider contract, PostgreSQL races; independent sessions and staged/final state | +| ARCH-04C/04D | `test_late_revocation_cannot_publish_result`, `test_unfinished_checker_recovery_reuses_attempt`, `test_terminal_retry_requires_operator_and_new_attempt` | Consume delivered 04B2 custody; Local/MinIO, real worker/provider contract and PostgreSQL races; independent sessions and staged/final state | | ARCH-04E | `test_submission_to_current_allow_review`, `test_superseded_run_cannot_route`, `test_duplicate_dispatch_has_one_manifest`; false tests in the shared acceptance contract | Real DB/worker/storage path, exact authority-event references; true creates no acceptance, false creates the shared atomic acceptance with no human Review | Owner-local schema names and migrations are chosen from the then-current diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md index bb515bf3f..c469217f8 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md @@ -2,25 +2,28 @@ ARCH-04B implementation follows the [current bounded change record](../../WS-ARCH-001-04B.md). It replaces this input skeleton with exact files, predicates and verification. -The output child below remains separate and Planned. +The separate output child is delivered by the +[current ARCH-04B2 record](../../WS-ARCH-001-04B2.md). ## ARCH-04B2 — Separate ART output-custody child -Input materialization does not implement output persistence. Keep a separate -ART-owned bounded change for `CheckerArtifactOutputPort.store` and -`ArtifactBindingPort.bind_checker_output`, using 04A's exact immutable request, -attempt/run and policy facts. Reuse the generic admission, put-intent, -verification, quota and binding infrastructure. Replace the current ART -repository's private CheckerRun lookup with the canonical public-fact boundary; -do not import CHECKERS models in ART behavior. Existing relational run foreign -keys may remain and use controlled foreign-row fixtures for hidden proof. +Input materialization does not implement output persistence. ARCH-04B2 now +supplies typed `CheckerArtifactOutputPort.store`, byte-free +`recover(selector)`, and `CheckerOutputBindingPort.bind_checker_output` using 04A's +exact immutable request and owner reservation facts. It reuses generic +admission, put intent, observation, verification, quota and binding +infrastructure. ART behavior no longer reads CheckerRun through its private +repository; the separate Operator resource-to-project lookup remains explicit. -Runtime CHECKERS reserves its run before output I/O. ART admits bounded output -and logs against project/task/fixed-service/deployment quotas, never contributor -quota, independently rereads/verifies bytes, then supplies a flush-only binding -participant. No row lock or PREP survives storage I/O. 04C later composes final -binding publication with final CHECKER result and completion outbox in one -transaction, after fresh exact action authority supplied by 04D. +Runtime CHECKERS will reserve its run before output I/O. ART admits each +owner-declared slot against project/task/fixed-service/deployment quotas, never +contributor quota, independently rereads/verifies bytes, then supplies a +flush-only binding participant. Authority is fresh for each phase and no row +lock or PREP survives storage I/O. Production reservation and authority remain +unavailable. The current structural catalogue declares zero slots; controlled +nonempty fixtures prove ART mechanics only. 04C must support the empty set and +later composes any reserved bindings with final CHECKER result and completion +outbox in one transaction, after fresh exact action authority supplied by 04D. The output child proves Local/MinIO parity, non-reproducible/unknown outcome handling without fabricated bytes, crash cleanup, deadline/quota races, @@ -28,4 +31,5 @@ foreign-request denial and fixed-service attribution. 04C/04D separately prove the integrated run/binding/result transaction. Neither child owns TASK routing, REV records or contributor-blame decisions. Expand each owner-local child into its exact implementation record rather than combining ART and CHECKERS writes -in one implementation PR. +in one implementation PR. The [ARCH-04B2 record](../../WS-ARCH-001-04B2.md) is +the delivered source; this skeleton no longer assigns the next change. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md index 40113b1b3..89773157e 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04C-checker-current-result.md @@ -1,7 +1,7 @@ # Chunk Contract: WS-ARCH-001-04C CHECKER Current Result Persistence -Disposition: Planned. Dependencies: 04A, 04B, its ART-owned output-custody -child and POL-07. Risk: L1. Outcome: +Disposition: Planned. Dependencies: 04A, 04B, delivered +[ARCH-04B2 output custody](../../WS-ARCH-001-04B2.md) and POL-07. Risk: L1. Outcome: CHECKERS installs hidden, deny-only execution of the exact post-submit plan and persists one immutable final result with explicit supersession/currentness and routing recommendation. @@ -17,6 +17,13 @@ Execution must reuse the POL-003 single checker-service port and or caller-triggered execution path is allowed. Production remains fail-closed until 04D activates the exact fixed-service boundaries. +CHECKERS owns the consumer contracts for post-submit materialization and output +custody in its public `api.materialization` and `api.output_custody` modules. +04C consumes only those CHECKERS-owned ports; application composition injects +the ART implementations. 04C must not import `app.interfaces.artifact_operations`, +the ART public API, or private ART modules. This preserves the one-way ART +implementation-to-CHECKERS-API dependency and prevents a public module cycle. + This is the only owner of the durable post-submit attempt, member-result, final-result, supersession/currentness and worker-recovery writes. POL-07 declares the facade contract, not a competing persistence implementation. @@ -68,12 +75,18 @@ proves that capability; absent it, remain blocked rather than invoking again. Prove slow-worker/takeover completion in both orders and no duplicate provider call or final output from stale leases. -Required logs/generated outputs pass the ART-owned output admission and -independent verification path before final publication. Compose verified -checker-output bindings and CHECKERS final completion in one caller transaction -through owner public ports. Missing, stale, unverified or mismatched required -output prevents final-current result/routing. Do not implement ART writes here, -persist provider locations as evidence, or charge output bytes to a contributor. +The current structural catalogue declares zero output slots, and a completed +evaluation with that exact empty set must finalize without inventing logs or +generated artifacts. When CHECKERS reserves nonempty slots, every produced +output passes the delivered ART-owned admission and independent verification +path before final publication. Compose those verified bindings and CHECKERS +final completion in one caller transaction through owner public ports. Missing, +stale, unverified or mismatched owner-required output prevents final-current +result/routing. Do not implement ART writes here, persist provider locations as +evidence, charge output bytes to a contributor, or turn ARCH-04B2's controlled +nonempty fixtures into catalogue requirements. +This contract relocation does not change the current zero-slot catalogue or +activate output production. The final multi-participant transaction must declare one lock order consistent with AUTH PREP: prepare required service authority custody before feature locks, diff --git a/.commitrail/initiatives/WS-ART-001/OVERVIEW.md b/.commitrail/initiatives/WS-ART-001/OVERVIEW.md index 580993951..6cc29f0b9 100644 --- a/.commitrail/initiatives/WS-ART-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-ART-001/OVERVIEW.md @@ -8,8 +8,10 @@ and the [capability ledger](../../../docs/roadmap_status.md). storage, and bounded private processing scratch. - Current boundary: ready-admission publication and hidden preparation, consumption, and binding dependencies are merged through ARCH-02H. -- Next usable boundary: ARCH-04B2 checker output custody. ARCH-04B hidden input - materialization is delivered; live authority and execution cutover remain later. +- Completed boundary: ARCH-04B hidden input materialization and hidden + [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md). + Production reservation and authority remain unavailable. +- Next usable boundary: ARCH-04C durable execution and results. - Governing sources: artifact specifications, `ArtifactStore`, `ArtifactScratchManager`, code, migrations, and artifact tests. - Preserve: SHA-256/byte-count identity, reread verification, isolation, @@ -32,8 +34,9 @@ ART retains the merged default-plus-project intake compiler/executor; POL-07 is a facade, not a replacement or second precheck run. New unified generations must prove exact approved lineage at preparation, consumption and binding. -1. ARCH-04B hidden exact Submission materialization is delivered. Continue with - ARCH-04B2 output custody, then durable execution and fixed-service authority. +1. ARCH-04B hidden exact Submission materialization and ARCH-04B2 hidden output + custody are delivered. Continue with ARCH-04C durable execution, then + fixed-service authority. 2. ARCH-04F owns checker-remediation resubmission using existing ART ports; later reviewer-requested revision remains a separate REV boundary. Add those dependencies before public Submission cutover. diff --git a/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md b/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md index b68e3f3cf..b5e39aff9 100644 --- a/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md @@ -23,8 +23,8 @@ Historical pre-cutover work records: [`STATUS.md`](pre-cutover/STATUS.md), - Public post-policy composition: POL-06B delivered using existing AUTH-12G. - Completed activation boundary: AUTH-12H exact-project manager authority for CP07 complete-guide activation/binding, with live-authority replay. -- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden - ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04C durable post-submit evaluation after delivered + hidden ARCH-04B input materialization and [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), then live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published @@ -77,5 +77,5 @@ manager proposal review, pre-submit approval and manual correction dispatch. AUTH-OUTBOX-01/02 bracket hidden CON-02B dispatch; ARCH-04E2 activates only the proven TASK routing handler before ARCH-04E3 live composition. TASK queue/read exposure is complete through ARCH-03C7. AUTH-18 public - manager guide activation/context is delivered; ARCH-03D hidden intake is delivered; hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. + manager guide activation/context is delivered; ARCH-03D hidden intake, hidden exact post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C is next. Remaining work must use its exact owner, not the superseded broad designs. diff --git a/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md index 6c86ed494..d9770933d 100644 --- a/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md @@ -13,7 +13,7 @@ work and historical proposals. | [AUTH-12H](../WS-AUTH-001-12H.md) | Complete: exact manager authority for CP07; AUTH-18 exposes manager activation and selections publicly | | CP05 | Complete: exact five policy actions; public Finance exposure delivered by CP05A | | ARCH-03C | Complete through 03C7: task/assignment authority, public queues, projections and audit reads; replaces broad AUTH-13 | -| [AUTH-18](../WS-AUTH-001-18.md) | Complete: public manager activation/context over CP07 and AUTH-12H; ARCH-03D completes hidden approved-guide intake; ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next | +| [AUTH-18](../WS-AUTH-001-18.md) | Complete: public manager activation/context over CP07 and AUTH-12H; ARCH-03D hidden intake, ARCH-04B input and ARCH-04B2 output custody are delivered; ARCH-04C durable execution is next | | ARCH-04D | AUTH materialization/output plus CHECKERS execute/finalize activation after ARCH-04B/04B2/04C; replaces AUTH-14/XINT-06B | | [AUTH-OUTBOX-01](PLAN.md#ws-auth-001-outbox-01--unavailable-dispatcher-contract) | Complete: unavailable exact dispatcher identity/action/phase contract; CON-02B and AUTH-OUTBOX-02 mechanics complete; feature authority/registration remain separate | | [AUTH-OUTBOX-02](PLAN.md#ws-auth-001-outbox-02--exact-dispatcher-activation) | Complete: exact dispatcher mechanics activation, phase audit custody and bounded prefork delivery; ARCH-03C2 subsequently registers assignment invalidation, while future handlers require their own exact authority | diff --git a/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md b/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md index 76edeed5b..aa8bf777e 100644 --- a/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md @@ -25,7 +25,7 @@ AUTH-13/14 cutovers are not additional implementation work. public TASK activation is complete through ARCH-03C7. - [AUTH-18](../WS-AUTH-001-18.md) delivers public manager activation and exact selection discovery over CP07/AUTH-12H. ARCH-03D completes hidden approved-guide - intake; ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. + intake; ARCH-04B hidden exact post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C durable execution is next. - CP05 owns exact ContributionPolicy-action activation after merged CP04B. - CP08 delivered the minimal lineage writers. ARCH-03B8 hidden task audit evidence and 03B9 hidden assignment invalidation are complete. ARCH-03C delivered @@ -53,7 +53,7 @@ and public JSON-packet Submission creation. Admission-backed creation stays hidd ARCH-03B/03C reuse these exact actions and command owners; public TASK access and authority are delivered through 03C7, with invalidation wiring in 03C2. CP08 delivered ContributionPolicyVersion lineage. ARCH-03D completes hidden approved-guide intake. ARCH-04B hidden exact post-submit -input is delivered; ARCH-04B2 output custody is next; public intake remains deferred to ARCH-02I. Do not restore eligibility +input and ARCH-04B2 output custody are delivered; ARCH-04C durable execution is next; public intake remains deferred to ARCH-02I. Do not restore eligibility or register replacement aliases. ## WS-AUTH-001-OUTBOX-01 — unavailable dispatcher contract diff --git a/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md b/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md index e228dabb8..581aba1d8 100644 --- a/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md +++ b/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md @@ -8,8 +8,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), - Completed boundary: recovery foundation and [TASK/checker authorization cleanup](WS-AUTH-003-TASKCHECKER.md). - Intent: route public authorization capability through `authorization.api` and remove cross-module repository/model coupling. -- Next usable boundary: ARCH-04B2 output custody after delivered ARCH-04B hidden input, - ARCH-03D hidden intake and +- Next usable boundary: ARCH-04C durable execution after delivered ARCH-04B + hidden input, [ARCH-04B2 output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), ARCH-03D hidden intake and [AUTH-18 public manager activation](../WS-AUTH-001/WS-AUTH-001-18.md). Submission/checker history uses canonical authority; the alternate gate lifecycle is removed. Continue shrinking the canonical import ledger as implementation diff --git a/.commitrail/initiatives/WS-CON-001/OVERVIEW.md b/.commitrail/initiatives/WS-CON-001/OVERVIEW.md index afb315fff..627008146 100644 --- a/.commitrail/initiatives/WS-CON-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-CON-001/OVERVIEW.md @@ -11,8 +11,8 @@ and the [capability ledger](../../../docs/roadmap_status.md). immutable ContributionRecords and optional project-policy-driven compensation awards without coupling lifecycle truth to an economic provider. -- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden - ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04C durable post-submit evaluation after delivered + hidden ARCH-04B input materialization and [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), then live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published diff --git a/.commitrail/initiatives/WS-POL-003/OVERVIEW.md b/.commitrail/initiatives/WS-POL-003/OVERVIEW.md index a7511f3e2..842cc5be5 100644 --- a/.commitrail/initiatives/WS-POL-003/OVERVIEW.md +++ b/.commitrail/initiatives/WS-POL-003/OVERVIEW.md @@ -27,8 +27,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), Production post-submit phase execution remains unavailable. - Intent: compile one locked guide and its policies into authoritative, versioned project behavior without circular subsystem authority. -- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden - ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04C durable post-submit evaluation after delivered + hidden ARCH-04B input materialization and [ARCH-04B2 checker-output custody](../WS-ARCH-001/WS-ARCH-001-04B2.md), then live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published diff --git a/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md index 69ef63219..85141bfd1 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md @@ -29,7 +29,7 @@ post-task requirement. Historical split parents 05/06 are not extra PRs. | `WS-AUTH-001-12G` | Activate exact fixed-service projection plus PM approval/correction authority for the hidden 06A manifest. | 06A | | `WS-POL-003-06B` | Live deterministic post-submit projection/approval cutover with zero additional inference. | 06A + AUTH-12G | | `WS-POL-003-07` | One typed facade over existing ART pre and CHECKER post contracts; no post-result persistence. | 06B + ARCH-04A registered capability proof + merged ART-04B1-04B3 | -| `WS-AUTH-001-12H` / [AUTH-18](../../WS-AUTH-001/WS-AUTH-001-18.md) | Complete: exact guide activation authority and public manager activation/context over the approved unified chain. CP08 lineage and ARCH-03 task authority/projections are delivered; ARCH-03D completes hidden approved-guide intake. ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next; public intake remains ARCH-02I. CP09 remains later. | POL-07 + corrected AUTH-12B2 + CP05 active ContributionPolicy behavior + CP06 validation + CP07 ProjectGuide binding | +| `WS-AUTH-001-12H` / [AUTH-18](../../WS-AUTH-001/WS-AUTH-001-18.md) | Complete: exact guide activation authority and public manager activation/context over the approved unified chain. CP08 lineage and ARCH-03 task authority/projections are delivered; ARCH-03D hidden intake, ARCH-04B input and ARCH-04B2 output custody are delivered; ARCH-04C is next and public intake remains ARCH-02I. CP09 remains later. | POL-07 + corrected AUTH-12B2 + CP05 active ContributionPolicy behavior + CP06 validation + CP07 ProjectGuide binding | | `WS-POL-003-08` | Supplementary visibility; separate remaining cleanup is parked and handled within each affected module. Essential review/correction belongs to 05A/05B and 06A/06B. | Planned after 07 + AUTH-12H + canonical WS-ARCH-001-04E manifest; any separately authorized retained-data change requires CP09's inventory, mapping and readability/recoverability proof for affected facts; no cleanup prerequisite for 04B | The 04E manifest proves canonical routing, not legacy-history preservation. @@ -54,7 +54,8 @@ mixed-generation chains deny through the existing locked-lineage checks. ARCH-04A contracts/capability conformance precede POL-07 and guide activation. ARCH-04B delivers hidden exact ART post-submit input with deny-only production -authority. ARCH-04B2 output custody is next, followed by ARCH-04C durable CHECKER -execution/results. +authority. ARCH-04B2 hidden output custody is delivered with unavailable +reservation/authority composition, followed by ARCH-04C durable CHECKER +execution/results with valid empty-output support. WS-ARCH-001-04D is the later replacement activation gate. Historical XINT-06B and AUTH-14 contracts are superseded/non-executable. diff --git a/.commitrail/initiatives/WS-POL-003/planning/PLAN.md b/.commitrail/initiatives/WS-POL-003/planning/PLAN.md index 7d41f450c..0b9664c87 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/PLAN.md +++ b/.commitrail/initiatives/WS-POL-003/planning/PLAN.md @@ -56,7 +56,8 @@ aliases or fallbacks. AUTH-12F4 and AUTH-12G supply the delivered pre/post approval authority. AUTH-12H supplies exact guide activation authority; AUTH-18 exposes manager activation and selection discovery publicly. ARCH-03D completes hidden approved-guide intake integration. ARCH-04B hidden exact -post-submit materialization is delivered; ARCH-04B2 output custody is next; public intake remains deferred to ARCH-02I. +post-submit materialization and ARCH-04B2 output custody are delivered; ARCH-04C +durable execution is next; public intake remains deferred to ARCH-02I. The sequence through POL-04B is complete; POL-04B1 supplies automatic request custody for the delivered live cutover: diff --git a/README.md b/README.md index e6dc2f38d..8562e8cf1 100644 --- a/README.md +++ b/README.md @@ -651,9 +651,11 @@ history only to covered Audit Authority and removes the old payload-bearing task-only audit route. AUTH-18 delivers public manager guide activation; ARCH-03D connects hidden intake through final durable intent to canonical TASK/PROJECTS ports, preserving historical guide selection. Public intake remains deferred. ARCH-04B supplies hidden exact verified Submission -input with scoped async file access and cleanup. Production materialization -authority remains deny-only; ARCH-04B2 output custody, ARCH-04C durable execution -and ARCH-04D live authority remain separate steps. +input with scoped async file access and cleanup. ARCH-04B2 adds hidden typed +checker-output storage, byte-free recovery and flush-only verified binding over +the generic ART put/verification path. CHECKERS reservation and output authority +remain unavailable in production; ARCH-04C durable execution and ARCH-04D live +authority are the next separate steps. ## v0.1 Success Standard diff --git a/backend/alembic/env.py b/backend/alembic/env.py index f8258a08a..b6840322b 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -21,7 +21,7 @@ target_metadata = Base.metadata _BASELINE_REVISION = "0001_uuid7_v01" -_CURRENT_HEAD_REVISION = "0006_history_read_authority" +_CURRENT_HEAD_REVISION = "0007_checker_output_custody" _RECREATE_GUIDANCE = ( "Workstream v0.1 requires a fresh database; recreate this database before " "running the 0001_uuid7_v01 migration" @@ -52,7 +52,7 @@ def do_run_migrations(connection: Connection) -> None: .scalars() .all() ) - if revisions not in ((), (_BASELINE_REVISION,), ("0002_task_queue_authority",), ("0003_task_read_authority",), ("0004_task_context_authority",), ("0005_task_evidence_authority",), (_CURRENT_HEAD_REVISION,)): + if revisions not in ((), (_BASELINE_REVISION,), ("0002_task_queue_authority",), ("0003_task_read_authority",), ("0004_task_context_authority",), ("0005_task_evidence_authority",), ("0006_history_read_authority",), (_CURRENT_HEAD_REVISION,)): raise RuntimeError(_RECREATE_GUIDANCE) # The read-only preflight autobegins a SQLAlchemy transaction. End that # transaction before Alembic establishes the migration transaction; diff --git a/backend/alembic/versions/0007_checker_output_custody.py b/backend/alembic/versions/0007_checker_output_custody.py new file mode 100644 index 000000000..a67971218 --- /dev/null +++ b/backend/alembic/versions/0007_checker_output_custody.py @@ -0,0 +1,565 @@ +"""Bind checker output intent and publication to exact verified ART custody.""" + +from alembic import op +import sqlalchemy as sa + + +revision = "0007_checker_output_custody" +down_revision = "0006_history_read_authority" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Install fail-closed checker output intent and binding custody.""" + op.execute( + "lock table artifact_bindings, artifact_contents, artifact_operation_receipts, " + "artifact_put_attempts, artifact_put_observation_receipts, artifact_replicas, " + "artifact_verification_jobs, artifact_verification_receipts, checker_runs, " + "submissions, workstream_tasks in access exclusive mode" + ) + + op.add_column( + "artifact_put_attempts", + sa.Column("submission_id", sa.Uuid(), nullable=True), + ) + op.add_column( + "artifact_put_attempts", + sa.Column("submission_version", sa.Integer(), nullable=True), + ) + op.add_column( + "artifact_put_attempts", + sa.Column("checker_request_digest", sa.String(length=71), nullable=True), + ) + op.add_column( + "artifact_put_attempts", + sa.Column( + "checker_output_custody_sealed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "artifact_verification_jobs", + sa.Column( + "checker_output_custody_sealed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "artifact_replicas", + sa.Column( + "checker_output_custody_sealed", + sa.Boolean(), + nullable=False, + server_default=sa.text("false"), + ), + ) + op.add_column( + "artifact_bindings", + sa.Column("put_attempt_id", sa.Uuid(), nullable=True), + ) + op.add_column( + "artifact_bindings", + sa.Column("verification_receipt_id", sa.Uuid(), nullable=True), + ) + + _refuse_unprovable_retained_checker_output() + + op.execute( + "alter table artifact_put_attempts drop constraint " + "ck_artifact_put_attempts_producer_reference" + ) + op.execute( + "alter table artifact_put_attempts add constraint " + "ck_artifact_put_attempts_producer_reference check (" + "(producer_request_type = 'guide' and guide_source_item_id is not null " + "and checker_run_id is null and task_id is null and submission_id is null " + "and submission_version is null and logical_role is null) or " + "(producer_request_type = 'checker_output' and guide_source_item_id is null " + "and checker_run_id is not null and task_id is not null " + "and submission_id is not null and submission_version is not null " + "and octet_length(logical_role) between 1 and 100) or " + "(producer_request_type = 'submission_bundle' and guide_source_item_id is null " + "and checker_run_id is null and task_id is not null and submission_id is null " + "and submission_version is null and logical_role is null))" + ) + op.execute( + "alter table artifact_put_attempts add constraint " + "ck_artifact_put_attempts_checker_request_digest check (" + "(producer_request_type = 'checker_output' and checker_request_digest is not null " + "and checker_request_digest ~ '^sha256:[0-9a-f]{64}$') or " + "(producer_request_type <> 'checker_output' and checker_request_digest is null))" + ) + op.create_foreign_key( + "fk_artifact_put_attempts_task_project", + "artifact_put_attempts", + "workstream_tasks", + ["task_id", "project_id"], + ["id", "project_id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + "fk_artifact_put_attempts_checker_run_ownership", + "artifact_put_attempts", + "checker_runs", + ["checker_run_id", "task_id", "submission_id"], + ["id", "task_id", "submission_id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + "fk_artifact_put_attempts_submission_version", + "artifact_put_attempts", + "submissions", + ["submission_id", "task_id", "submission_version"], + ["id", "task_id", "version"], + ondelete="RESTRICT", + ) + op.create_index( + "ix_artifact_put_attempts_submission_id", + "artifact_put_attempts", + ["submission_id"], + ) + + op.create_foreign_key( + "fk_artifact_bindings_checker_put_attempt", + "artifact_bindings", + "artifact_put_attempts", + ["put_attempt_id"], + ["id"], + ondelete="RESTRICT", + ) + op.create_foreign_key( + "fk_artifact_bindings_checker_verification_receipt", + "artifact_bindings", + "artifact_verification_receipts", + ["verification_receipt_id"], + ["id"], + ondelete="RESTRICT", + ) + op.execute( + "alter table artifact_bindings add constraint " + "ck_artifact_bindings_checker_output_lineage check (" + "(resource_type = 'checker_run' and scope_version = 1 " + "and put_attempt_id is not null and verification_receipt_id is not null) or " + "(resource_type <> 'checker_run' and put_attempt_id is null " + "and verification_receipt_id is null))" + ) + op.create_index( + "ix_artifact_bindings_put_attempt_id", + "artifact_bindings", + ["put_attempt_id"], + ) + op.create_index( + "ix_artifact_bindings_verification_receipt_id", + "artifact_bindings", + ["verification_receipt_id"], + ) + + _install_checker_output_attempt_custody() + _install_checker_output_ancestor_custody() + _install_checker_output_binding_guard() + + +def downgrade() -> None: + """Reject destructive downgrade of immutable checker output evidence.""" + raise RuntimeError("Workstream v0.1 migrations cannot be downgraded; recreate the database") + + +def _refuse_unprovable_retained_checker_output() -> None: + """Never invent the absent evaluation/slot digest for retained checker output.""" + op.execute( + """ + DO $$ BEGIN + IF EXISTS ( + SELECT 1 + FROM artifact_put_attempts attempt + LEFT JOIN checker_runs run + ON run.id=attempt.checker_run_id AND run.task_id=attempt.task_id + LEFT JOIN submissions submission + ON submission.id=run.submission_id + AND submission.task_id=run.task_id + AND submission.version=run.submission_version + LEFT JOIN workstream_tasks task + ON task.id=run.task_id AND task.project_id=attempt.project_id + WHERE attempt.producer_request_type='checker_output' + AND (run.id IS NULL OR submission.id IS NULL OR task.id IS NULL) + ) THEN + RAISE EXCEPTION 'retained checker output attempt ownership is inconsistent' + USING ERRCODE='23514'; + END IF; + IF EXISTS ( + SELECT 1 FROM artifact_put_attempts + WHERE producer_request_type='checker_output' + ) THEN + RAISE EXCEPTION + 'retained checker output request custody is unprovable; evaluation and slot digest was not persisted' + USING ERRCODE='23514'; + END IF; + IF EXISTS ( + SELECT 1 FROM artifact_bindings WHERE resource_type='checker_run' + ) THEN + RAISE EXCEPTION + 'retained checker output binding ancestry is unprovable' + USING ERRCODE='23514'; + END IF; + END $$ + """ + ) + + +def _install_checker_output_attempt_custody() -> None: + op.execute( + """ + CREATE FUNCTION guard_checker_output_put_attempt_custody() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN + IF OLD.producer_request_type='checker_output' + OR OLD.checker_output_custody_sealed THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN OLD; + END IF; + IF OLD.checker_output_custody_sealed THEN + IF NOT NEW.checker_output_custody_sealed + OR (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF NEW.checker_output_custody_sealed THEN + IF (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') + OR NOT EXISTS ( + SELECT 1 FROM artifact_bindings binding + WHERE binding.resource_type='checker_run' + AND binding.put_attempt_id=OLD.id + ) THEN + RAISE EXCEPTION 'checker output put attempt seal requires binding' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF OLD.producer_request_type='checker_output' + OR NEW.producer_request_type='checker_output' THEN + IF ROW( + NEW.id, NEW.producer_request_type, NEW.producer_type, NEW.producer_ref, + NEW.project_id, NEW.task_id, NEW.guide_source_item_id, NEW.checker_run_id, + NEW.submission_id, NEW.submission_version, NEW.logical_role, + NEW.sha256, NEW.byte_count, NEW.media_type, NEW.storage_namespace_id, + NEW.namespace_fingerprint, NEW.canonical_target, NEW.operation_identity, + NEW.request_digest, NEW.checker_request_digest, NEW.maximum_observations, + NEW.prepared_at, NEW.created_at + ) IS DISTINCT FROM ROW( + OLD.id, OLD.producer_request_type, OLD.producer_type, OLD.producer_ref, + OLD.project_id, OLD.task_id, OLD.guide_source_item_id, OLD.checker_run_id, + OLD.submission_id, OLD.submission_version, OLD.logical_role, + OLD.sha256, OLD.byte_count, OLD.media_type, OLD.storage_namespace_id, + OLD.namespace_fingerprint, OLD.canonical_target, OLD.operation_identity, + OLD.request_digest, OLD.checker_request_digest, OLD.maximum_observations, + OLD.prepared_at, OLD.created_at + ) THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_put_attempt_custody " + "BEFORE DELETE OR UPDATE ON artifact_put_attempts FOR EACH ROW " + "EXECUTE FUNCTION guard_checker_output_put_attempt_custody()" + ) + op.execute( + """ + CREATE FUNCTION guard_checker_output_put_attempt_truncate() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM artifact_put_attempts + WHERE producer_request_type='checker_output' + ) THEN + RAISE EXCEPTION 'checker output put attempt custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NULL; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_put_attempt_no_truncate BEFORE TRUNCATE " + "ON artifact_put_attempts FOR EACH STATEMENT " + "EXECUTE FUNCTION guard_checker_output_put_attempt_truncate()" + ) + + +def _install_checker_output_ancestor_custody() -> None: + op.execute( + """ + CREATE FUNCTION guard_checker_output_verification_job_custody() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN + IF OLD.checker_output_custody_sealed THEN + RAISE EXCEPTION 'checker output verification job custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN OLD; + END IF; + IF OLD.checker_output_custody_sealed THEN + IF NOT NEW.checker_output_custody_sealed + OR (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') THEN + RAISE EXCEPTION 'checker output verification job custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF NEW.checker_output_custody_sealed THEN + IF (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') + OR NOT EXISTS ( + SELECT 1 + FROM artifact_bindings binding + JOIN artifact_verification_receipts receipt + ON receipt.id=binding.verification_receipt_id + WHERE binding.resource_type='checker_run' + AND receipt.verification_job_id=OLD.id + ) THEN + RAISE EXCEPTION 'checker output verification job seal requires binding' + USING ERRCODE='55000'; + END IF; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_verification_job_custody " + "BEFORE DELETE OR UPDATE ON artifact_verification_jobs FOR EACH ROW " + "EXECUTE FUNCTION guard_checker_output_verification_job_custody()" + ) + op.execute( + """ + CREATE FUNCTION guard_checker_output_replica_custody() RETURNS trigger + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP='DELETE' THEN + IF OLD.checker_output_custody_sealed THEN + RAISE EXCEPTION 'checker output replica custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN OLD; + END IF; + IF OLD.checker_output_custody_sealed THEN + IF NOT NEW.checker_output_custody_sealed + OR ROW( + NEW.id, NEW.content_id, NEW.storage_namespace_id, + NEW.namespace_fingerprint, NEW.adapter, + NEW.provider_profile, NEW.provider_object_ref + ) IS DISTINCT FROM ROW( + OLD.id, OLD.content_id, OLD.storage_namespace_id, + OLD.namespace_fingerprint, OLD.adapter, + OLD.provider_profile, OLD.provider_object_ref + ) THEN + RAISE EXCEPTION 'checker output replica custody is immutable' + USING ERRCODE='55000'; + END IF; + RETURN NEW; + END IF; + IF NEW.checker_output_custody_sealed THEN + IF (to_jsonb(NEW) - 'checker_output_custody_sealed') + IS DISTINCT FROM + (to_jsonb(OLD) - 'checker_output_custody_sealed') + OR NOT EXISTS ( + SELECT 1 + FROM artifact_bindings binding + JOIN artifact_verification_receipts receipt + ON receipt.id=binding.verification_receipt_id + JOIN artifact_verification_jobs job + ON job.id=receipt.verification_job_id + WHERE binding.resource_type='checker_run' + AND job.replica_id=OLD.id + ) THEN + RAISE EXCEPTION 'checker output replica seal requires binding' + USING ERRCODE='55000'; + END IF; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_replica_custody " + "BEFORE DELETE OR UPDATE ON artifact_replicas FOR EACH ROW " + "EXECUTE FUNCTION guard_checker_output_replica_custody()" + ) + + +def _install_checker_output_binding_guard() -> None: + op.execute( + """ + CREATE FUNCTION guard_checker_output_binding_insert() RETURNS trigger + LANGUAGE plpgsql AS $$ + DECLARE + verification artifact_verification_receipts%ROWTYPE; + job artifact_verification_jobs%ROWTYPE; + replica artifact_replicas%ROWTYPE; + attempt artifact_put_attempts%ROWTYPE; + content artifact_contents%ROWTYPE; + write_receipt_matches integer; + BEGIN + IF NEW.resource_type <> 'checker_run' THEN + RETURN NEW; + END IF; + + SELECT receipt.* INTO verification + FROM artifact_verification_receipts receipt + WHERE receipt.id=NEW.verification_receipt_id; + IF NOT FOUND THEN + RAISE EXCEPTION 'checker output binding verified ancestry mismatch' + USING ERRCODE='23514'; + END IF; + + SELECT value.* INTO job + FROM artifact_verification_jobs value + WHERE value.id=verification.verification_job_id + FOR UPDATE; + SELECT value.* INTO replica + FROM artifact_replicas value + WHERE value.id=job.replica_id + FOR UPDATE; + SELECT value.* INTO attempt + FROM artifact_put_attempts value + WHERE value.id=job.originating_put_attempt_id + FOR UPDATE; + SELECT value.* INTO content + FROM artifact_contents value + WHERE value.id=replica.content_id + FOR SHARE; + + SELECT count(*) INTO write_receipt_matches + FROM ( + SELECT receipt.id + FROM artifact_operation_receipts receipt + WHERE attempt.receipt_id IS NOT NULL + AND receipt.id=attempt.receipt_id + AND receipt.put_attempt_id=attempt.id + AND receipt.replica_id=replica.id + AND receipt.checker_run_id=attempt.checker_run_id + AND receipt.logical_role=attempt.logical_role + AND receipt.request_digest=attempt.request_digest + AND receipt.provider_object_ref=replica.provider_object_ref + AND receipt.outcome='stored_pending_verification' + UNION ALL + SELECT receipt.id + FROM artifact_put_observation_receipts receipt + WHERE attempt.receipt_id IS NULL + AND receipt.put_attempt_id=attempt.id + AND receipt.execution_generation=attempt.execution_generation + AND receipt.outcome='observed_confirmed' + AND receipt.expected_sha256=attempt.sha256 + AND receipt.observed_sha256=attempt.sha256 + AND receipt.expected_byte_count=attempt.byte_count + AND receipt.observed_byte_count=attempt.byte_count + ) successful_write; + + IF job.id IS NULL OR replica.id IS NULL OR attempt.id IS NULL OR content.id IS NULL + OR NEW.put_attempt_id IS DISTINCT FROM attempt.id + OR NEW.content_id IS DISTINCT FROM content.id + OR NEW.project_id IS DISTINCT FROM attempt.project_id + OR NEW.resource_id IS DISTINCT FROM attempt.checker_run_id::text + OR NEW.logical_role IS DISTINCT FROM attempt.logical_role + OR NEW.actor_id IS DISTINCT FROM attempt.producer_ref + OR NEW.attribution_type IS DISTINCT FROM 'service_identity' + OR NEW.scope_version <> 1 + OR NEW.supersedes_binding_id IS NOT NULL + OR attempt.producer_request_type <> 'checker_output' + OR attempt.producer_type <> 'service_identity' + OR attempt.submission_id IS NULL OR attempt.submission_version IS NULL + OR attempt.checker_request_digest IS NULL + OR attempt.status <> 'object_confirmed' + OR attempt.replica_id IS DISTINCT FROM replica.id + OR attempt.storage_namespace_id IS DISTINCT FROM replica.storage_namespace_id + OR attempt.namespace_fingerprint IS DISTINCT FROM replica.namespace_fingerprint + OR attempt.canonical_target IS DISTINCT FROM replica.provider_object_ref + OR attempt.sha256 IS DISTINCT FROM content.sha256 + OR attempt.byte_count IS DISTINCT FROM content.byte_count + OR attempt.media_type IS DISTINCT FROM content.media_type + OR job.originating_put_attempt_id IS DISTINCT FROM attempt.id + OR job.replica_id IS DISTINCT FROM replica.id + OR job.status <> 'verified' + OR job.terminal_result_code IS DISTINCT FROM 'verified' + OR job.terminal_at IS NULL + OR verification.outcome <> 'verified' + OR verification.execution_generation IS DISTINCT FROM job.execution_generation + OR verification.observed_sha256 IS DISTINCT FROM attempt.sha256 + OR verification.observed_byte_count IS DISTINCT FROM attempt.byte_count + OR replica.verification_state <> 'verified' + OR replica.availability_state <> 'available' + OR replica.integrity_state <> 'valid' + OR write_receipt_matches <> 1 THEN + RAISE EXCEPTION 'checker output binding verified ancestry mismatch' + USING ERRCODE='23514'; + END IF; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_binding_insert BEFORE INSERT ON artifact_bindings " + "FOR EACH ROW EXECUTE FUNCTION guard_checker_output_binding_insert()" + ) + op.execute( + """ + CREATE FUNCTION seal_checker_output_binding_ancestry() RETURNS trigger + LANGUAGE plpgsql AS $$ + DECLARE + verification artifact_verification_receipts%ROWTYPE; + job artifact_verification_jobs%ROWTYPE; + BEGIN + IF NEW.resource_type <> 'checker_run' THEN + RETURN NEW; + END IF; + SELECT receipt.* INTO STRICT verification + FROM artifact_verification_receipts receipt + WHERE receipt.id=NEW.verification_receipt_id; + SELECT value.* INTO STRICT job + FROM artifact_verification_jobs value + WHERE value.id=verification.verification_job_id; + + UPDATE artifact_verification_jobs + SET checker_output_custody_sealed=true + WHERE id=job.id AND NOT checker_output_custody_sealed; + UPDATE artifact_replicas + SET checker_output_custody_sealed=true + WHERE id=job.replica_id AND NOT checker_output_custody_sealed; + UPDATE artifact_put_attempts + SET checker_output_custody_sealed=true + WHERE id=NEW.put_attempt_id AND NOT checker_output_custody_sealed; + RETURN NEW; + END $$ + """ + ) + op.execute( + "CREATE TRIGGER checker_output_binding_seal AFTER INSERT ON artifact_bindings " + "FOR EACH ROW EXECUTE FUNCTION seal_checker_output_binding_ancestry()" + ) + op.execute( + "CREATE TRIGGER trg_artifact_bindings_no_truncate BEFORE TRUNCATE " + "ON artifact_bindings FOR EACH STATEMENT " + "EXECUTE FUNCTION reject_artifact_fact_mutation()" + ) diff --git a/backend/app/adapters/artifacts/__init__.py b/backend/app/adapters/artifacts/__init__.py index e6f97ed1f..59c38b8f9 100644 --- a/backend/app/adapters/artifacts/__init__.py +++ b/backend/app/adapters/artifacts/__init__.py @@ -19,7 +19,12 @@ from app.db.session import get_db_session from app.interfaces.artifact_operations import GuideArtifactIngestCommand from app.modules.projects.api.guide_documents import GuideDocumentUploadTargetPort -from app.modules.artifacts.api import SubmissionBundlePreparationCommand, PostSubmissionMaterializationPort +from app.modules.artifacts.api import SubmissionBundlePreparationCommand +from app.modules.checkers.api.materialization import PostSubmissionMaterializationPort +from app.modules.checkers.api.output_custody import ( + CheckerArtifactOutputPort, + CheckerOutputBindingPort, +) from app.interfaces.artifacts import ( ARTIFACT_STORE_CAPABILITY_KEY, ArtifactConfigurationError, @@ -511,3 +516,26 @@ def post_submission_materialization(*, sessions, store, namespace, preparation, preparation=preparation, inspector=inspector, authority=DenyPostSubmissionMaterializationAuthority(), ) + + +def checker_output_storage( + *, sessions, store, namespace, preparation, settings +) -> CheckerArtifactOutputPort: + """Compose hidden output custody; live producer and write authority remain absent.""" + from app.modules.artifacts.checker_outputs import CheckerArtifactOutputService, DenyCheckerOutputWriteAuthority + from app.modules.artifacts.schemas import DenyArtifactInternalAuthority + from app.modules.checkers.api.output_custody import UnavailableCheckerOutputReservation + return CheckerArtifactOutputService( + sessions=sessions, store=store, namespace=namespace, preparation=preparation, settings=settings, + reservations=lambda session: UnavailableCheckerOutputReservation(), + authority=lambda session: DenyCheckerOutputWriteAuthority(), + internal_authority=lambda session: DenyArtifactInternalAuthority(), + ) + + +def checker_output_binding(session, *, namespace) -> CheckerOutputBindingPort: + """Compose a deny-only caller-transaction binding participant.""" + from app.modules.artifacts.checker_output_bindings import CheckerOutputBindingService, DenyCheckerOutputBindingAuthority + from app.modules.checkers.api.output_custody import UnavailableCheckerOutputReservation + return CheckerOutputBindingService(session, namespace_fingerprint=namespace.namespace_fingerprint, + reservations=UnavailableCheckerOutputReservation(), authority=DenyCheckerOutputBindingAuthority()) diff --git a/backend/app/interfaces/artifact_operations.py b/backend/app/interfaces/artifact_operations.py index 92ab1c2e0..4342692e5 100644 --- a/backend/app/interfaces/artifact_operations.py +++ b/backend/app/interfaces/artifact_operations.py @@ -9,17 +9,12 @@ from app.modules.authorization.prepared import PreparedAuthorizationHandle from app.modules.authorization.runtime import AuthorizationContext - __all__ = ( "ArtifactAuditResourceType", "ArtifactBindingResourceType", - "ArtifactBindingPort", "ArtifactOperatorReadPort", "ArtifactOperatorRecoveryPort", "ArtifactRecoveryRequest", - "CheckerArtifactOutputPort", - "CheckerOutputBindingRequest", - "CheckerOutputArtifactRequest", "GuideArtifactIngestPort", "GuideArtifactIngestCommand", "GuideArtifactIngestRequest", @@ -73,31 +68,6 @@ class GuideArtifactIngestResult: replayed: bool -@dataclass(frozen=True, slots=True) -class CheckerOutputBindingRequest: - """Verified checker output and its exact CheckerRun owner.""" - - prepared_authorization: PreparedAuthorizationHandle - project_id: UUID - task_id: UUID - submission_id: UUID - checker_run_id: UUID - logical_role: str - verified_content_ids: tuple[UUID, ...] - - -@dataclass(frozen=True, slots=True) -class CheckerOutputArtifactRequest: - """Generated checker bytes bound to one fixed service execution.""" - - prepared_authorization: PreparedAuthorizationHandle - task_id: UUID - submission_id: UUID - checker_run_id: UUID - logical_role: str - byte_source: AsyncIterable[bytes] - - @dataclass(frozen=True, slots=True) class ArtifactRecoveryRequest: """Reason-bound Operator retry of one exact verification job.""" @@ -137,20 +107,6 @@ async def ingest( """Prepare authority before delegating to durable byte ingestion.""" -class ArtifactBindingPort(Protocol): - """Create exact action-bound bindings from verified content.""" - - async def bind_checker_output(self, request: CheckerOutputBindingRequest) -> object: - """Bind verified checker output under the checker binding action.""" - - -class CheckerArtifactOutputPort(Protocol): - """Store generated output for one fixed checker execution.""" - - async def store(self, request: CheckerOutputArtifactRequest) -> object: - """Store one generated checker artifact.""" - - class ArtifactOperatorReadPort(Protocol): """Expose bounded Operator reads without provider references.""" diff --git a/backend/app/modules/artifacts/api/__init__.py b/backend/app/modules/artifacts/api/__init__.py index bb1c7602f..47813c349 100644 --- a/backend/app/modules/artifacts/api/__init__.py +++ b/backend/app/modules/artifacts/api/__init__.py @@ -31,15 +31,3 @@ "SubmissionAdmissionConsumptionResult", "SubmissionAdmissionConsumptionStatus", ) - -from app.modules.artifacts.api.submission_materialization import ( - PostSubmissionMaterialConsumer, PostSubmissionMaterializationPort, - PostSubmissionMaterializationResult, PostSubmissionMaterializationUnavailable, - SubmissionMaterialEntry, SubmissionMaterialView, -) - -__all__ += ( - "PostSubmissionMaterialConsumer", "PostSubmissionMaterializationPort", - "PostSubmissionMaterializationResult", "PostSubmissionMaterializationUnavailable", - "SubmissionMaterialEntry", "SubmissionMaterialView", -) diff --git a/backend/app/modules/artifacts/checker_output_bindings.py b/backend/app/modules/artifacts/checker_output_bindings.py new file mode 100644 index 000000000..23285287b --- /dev/null +++ b/backend/app/modules/artifacts/checker_output_bindings.py @@ -0,0 +1,172 @@ +"""Verified checker-output bindings in the caller's final-result transaction.""" + +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.identifiers import new_record_id +from app.modules.checkers.api.output_custody import ( + CheckerOutputBindingRequest, + CheckerOutputBindingResult, + CheckerOutputReservation, + CheckerOutputReservationPort, + CheckerOutputSelector, + CheckerOutputUnavailable, +) +from app.modules.actors.api import ServiceIdentity +from app.modules.artifacts.checker_output_custody import ( + CheckerOutputStoredFacts, + select_checker_output, +) +from app.modules.artifacts.models import ArtifactBinding + + +class CheckerOutputBindingAuthority(Protocol): + """Prepare authority before CHECKERS locks, consume exact ART custody afterward.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Deny before any owner or artifact lookup.""" + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Stabilize exact service authority before feature row locks.""" + + async def consume( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Consume binding action authority in this caller-owned transaction.""" + + def discard(self) -> None: + """Release process-local authority without committing or rolling back.""" + + +class DenyCheckerOutputBindingAuthority: + """No production binding until exact AUTH and CHECKERS custody are implemented.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Reject before protected reads.""" + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Reject binding authority preparation.""" + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + + async def consume( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Reject publication authority.""" + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + + def discard(self) -> None: + """No prepared authority exists.""" + + +class CheckerOutputBindingService: + """Flush one immutable binding; never commit the caller's result transaction.""" + + def __init__( + self, + session: AsyncSession, + *, + reservations: CheckerOutputReservationPort, + authority: CheckerOutputBindingAuthority, + namespace_fingerprint: str, + ) -> None: + self._session, self._reservations, self._authority = session, reservations, authority + self._namespace_fingerprint = namespace_fingerprint + + async def bind_checker_output( + self, request: CheckerOutputBindingRequest + ) -> CheckerOutputBindingResult: + """Fence current worker facts before acquiring ART binding/verification locks.""" + if ( + type(request) is not CheckerOutputBindingRequest + or not self._session.in_transaction() + or self._session.in_nested_transaction() + or type(request.put_attempt_id) is not UUID + or type(request.verification_receipt_id) is not UUID + ): + raise CheckerOutputUnavailable("checker_output_binding_unavailable") + selector = CheckerOutputSelector.model_validate(request.selector) + try: + await self._authority.preflight(selector) + await self._authority.prepare(selector) + reservation = await self._reservations.resolve(selector) + reservation.select(selector) + await self._session.execute( + text("select pg_advisory_xact_lock(hashtextextended(:key, 0))"), + { + "key": f"checker-output:{selector.evaluation.project_id}:{selector.checker_run_id}:{selector.slot_key}", + }, + ) + facts = await select_checker_output( + self._session, + selector=selector, + reservation=reservation, + namespace_fingerprint=self._namespace_fingerprint, + put_attempt_id=request.put_attempt_id, + verification_receipt_id=request.verification_receipt_id, + lock_verified=True, + ) + if ( + facts is None + or facts.status != "verified" + or facts.verification_receipt_id != request.verification_receipt_id + ): + raise CheckerOutputUnavailable("checker_output_verification_unavailable") + await self._authority.consume(reservation, facts) + binding = await self._session.scalar( + select(ArtifactBinding) + .where( + ArtifactBinding.project_id == str(selector.evaluation.project_id), + ArtifactBinding.resource_type == "checker_run", + ArtifactBinding.resource_id == str(selector.checker_run_id), + ArtifactBinding.logical_role == selector.slot_key, + ArtifactBinding.scope_version == 1, + ) + .with_for_update() + .execution_options(populate_existing=True) + ) + replayed = binding is not None + if binding is not None: + if ( + binding.content_id, + binding.put_attempt_id, + binding.verification_receipt_id, + binding.actor_id, + binding.attribution_type, + ) != ( + str(facts.content_id), + str(facts.put_attempt_id), + str(facts.verification_receipt_id), + ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + "service_identity", + ): + raise CheckerOutputUnavailable("checker_output_binding_conflict") + else: + binding = ArtifactBinding( + id=str(new_record_id()), + content_id=str(facts.content_id), + project_id=str(selector.evaluation.project_id), + resource_type="checker_run", + resource_id=str(selector.checker_run_id), + logical_role=selector.slot_key, + scope_version=1, + actor_id=ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + attribution_type="service_identity", + supersedes_binding_id=None, + put_attempt_id=str(facts.put_attempt_id), + verification_receipt_id=str(facts.verification_receipt_id), + ) + self._session.add(binding) + await self._session.flush() + return CheckerOutputBindingResult( + UUID(binding.id), + UUID(binding.content_id), + request.put_attempt_id, + request.verification_receipt_id, + replayed, + ) + finally: + self._authority.discard() diff --git a/backend/app/modules/artifacts/checker_output_custody.py b/backend/app/modules/artifacts/checker_output_custody.py new file mode 100644 index 000000000..beff8fdb2 --- /dev/null +++ b/backend/app/modules/artifacts/checker_output_custody.py @@ -0,0 +1,210 @@ +"""Exact ART-owned output selection shared by storage recovery and binding.""" + +from dataclasses import dataclass +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.hashing import canonical_json_hash +from app.modules.artifacts.models import ( + ArtifactContent, + ArtifactPutAttempt, + ArtifactReplica, + ArtifactStorageNamespace, + ArtifactVerificationJob, + ArtifactVerificationReceipt, +) +from app.modules.artifacts.schemas import ( + checker_output_operation_identity, + checker_output_request_digest_facts, +) +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSelector, + CheckerOutputUnavailable, +) + + +@dataclass(frozen=True, slots=True) +class CheckerOutputStoredFacts: + """Exact storage facts consumed by fresh output action authority.""" + + put_attempt_id: UUID + checker_request_digest: str + sha256: str + byte_count: int + media_type: str + status: str + content_id: UUID | None + replica_id: UUID | None + verification_receipt_id: UUID | None + + + +async def select_checker_output( + session: AsyncSession, + *, + selector: CheckerOutputSelector, + reservation: CheckerOutputReservation, + namespace_fingerprint: str, + put_attempt_id: UUID | None = None, + verification_receipt_id: UUID | None = None, + lock_verified: bool = False, +) -> CheckerOutputStoredFacts | None: + """Match immutable request and verified ancestry; never read foreign owners.""" + slot = reservation.select(selector) + expected_digest = canonical_json_hash( + checker_output_request_digest_facts( + reservation=reservation, + slot=slot, + ) + ) + attempt = await session.scalar( + select(ArtifactPutAttempt) + .where( + ArtifactPutAttempt.operation_identity == checker_output_operation_identity( + checker_run_id=selector.checker_run_id, slot_key=selector.slot_key + ), + ) + .execution_options(populate_existing=True) + ) + if attempt is None: + return None + evaluation = selector.evaluation + if ( + ( + attempt.producer_request_type, + attempt.project_id, + attempt.task_id, + attempt.submission_id, + attempt.submission_version, + attempt.checker_run_id, + attempt.logical_role, + attempt.checker_request_digest, + attempt.media_type, + attempt.namespace_fingerprint, + ) + != ( + "checker_output", + str(evaluation.project_id), + str(evaluation.task_id), + str(evaluation.submission_id), + evaluation.submission_version, + str(selector.checker_run_id), + slot.key, + expected_digest, + slot.media_type, + namespace_fingerprint, + ) + or attempt.byte_count > slot.maximum_bytes + or (put_attempt_id is not None and attempt.id != str(put_attempt_id)) + ): + raise CheckerOutputUnavailable("checker_output_identity_mismatch") + facts = CheckerOutputStoredFacts( + UUID(attempt.id), + expected_digest, + attempt.sha256, + attempt.byte_count, + attempt.media_type, + attempt.status, + None, + None, + None, + ) + if attempt.status != "object_confirmed": + return facts + return await _verified_output_facts( + session, + attempt=attempt, + facts=facts, + namespace_fingerprint=namespace_fingerprint, + verification_receipt_id=verification_receipt_id, + lock_verified=lock_verified, + ) + + +async def _verified_output_facts( + session: AsyncSession, + *, + attempt: ArtifactPutAttempt, + facts: CheckerOutputStoredFacts, + namespace_fingerprint: str, + verification_receipt_id: UUID | None, + lock_verified: bool, +) -> CheckerOutputStoredFacts: + """Resolve and, for publication, lock the exact independent verification chain.""" + query = ( + select( + ArtifactVerificationReceipt, + ArtifactVerificationJob, + ArtifactReplica, + ArtifactContent, + ArtifactStorageNamespace, + ) + .select_from(ArtifactVerificationReceipt) + .join( + ArtifactVerificationJob, + ArtifactVerificationJob.id == ArtifactVerificationReceipt.verification_job_id, + ) + .join(ArtifactReplica, ArtifactReplica.id == ArtifactVerificationJob.replica_id) + .join(ArtifactContent, ArtifactContent.id == ArtifactReplica.content_id) + .join( + ArtifactStorageNamespace, + ArtifactStorageNamespace.id == ArtifactReplica.storage_namespace_id, + ) + .where( + ArtifactVerificationJob.originating_put_attempt_id == attempt.id, + ArtifactVerificationReceipt.outcome == "verified", + ) + .order_by(ArtifactVerificationReceipt.id) + .execution_options(populate_existing=True) + ) + if verification_receipt_id is not None: + query = query.where(ArtifactVerificationReceipt.id == str(verification_receipt_id)) + row = (await session.execute(query.limit(1))).one_or_none() + if row is None: + return facts + receipt, job, replica, content, namespace = row + if lock_verified: + # Match verifier completion's lock order; CHECKERS/authority are already held. + for model, record_id in ( + (ArtifactVerificationJob, job.id), + (ArtifactReplica, replica.id), + (ArtifactPutAttempt, attempt.id), + (ArtifactContent, content.id), + ): + await session.scalar( + select(model) + .where(model.id == record_id) + .with_for_update() + .execution_options(populate_existing=True) + ) + if not ( + attempt.status == "object_confirmed" + and job.status == "verified" + and receipt.execution_generation == job.execution_generation + and job.replica_id == replica.id == attempt.replica_id + and (replica.verification_state, replica.availability_state, replica.integrity_state) + == ("verified", "available", "valid") + and receipt.observed_sha256 == content.sha256 == attempt.sha256 + and receipt.observed_byte_count == content.byte_count == attempt.byte_count + and content.media_type == attempt.media_type + and replica.storage_namespace_id == attempt.storage_namespace_id + and replica.namespace_fingerprint + == namespace.namespace_fingerprint + == namespace_fingerprint + and replica.provider_object_ref == attempt.canonical_target + ): + raise CheckerOutputUnavailable("checker_output_verification_unavailable") + return CheckerOutputStoredFacts( + UUID(attempt.id), + facts.checker_request_digest, + attempt.sha256, + attempt.byte_count, + attempt.media_type, + "verified", + UUID(content.id), + UUID(replica.id), + UUID(receipt.id), + ) diff --git a/backend/app/modules/artifacts/checker_outputs.py b/backend/app/modules/artifacts/checker_outputs.py new file mode 100644 index 000000000..c2a9ce0d7 --- /dev/null +++ b/backend/app/modules/artifacts/checker_outputs.py @@ -0,0 +1,251 @@ +"""Bounded checker bytes through the existing durable ART storage workflow.""" + +from collections.abc import Callable +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.core.config import Settings +from app.modules.checkers.api.output_custody import ( + CheckerOutputArtifactRequest, + CheckerOutputArtifactResult, + CheckerOutputReservation, + CheckerOutputReservationPort, + CheckerOutputSelector, + CheckerOutputUnavailable, +) +from app.interfaces.artifacts import ArtifactStore +from app.modules.artifacts.checker_output_custody import ( + CheckerOutputStoredFacts, + select_checker_output, +) +from app.modules.artifacts.models import ArtifactVerificationJob +from app.modules.artifacts.preparation import ArtifactPreparationService +from app.modules.artifacts.schemas import ( + ArtifactInternalAuthority, + CheckerOutputArtifactAdmissionRequest, +) +from app.modules.artifacts.service import ( + ArtifactAdmissionService, + ArtifactStorageOrchestrator, + ArtifactStorageNamespaceSpec, +) + + +class CheckerOutputWriteAuthority(Protocol): + """Fresh write authority; no prepared handle survives a storage operation.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Deny before owner facts, source iteration or scratch access.""" + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Stabilize service authority before CHECKERS and ART transaction locks.""" + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + """Bind the exact active worker, slot and byte commitment before admission.""" + + async def consume_existing( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Authorize exact stored output recovery with the current worker lease.""" + + def discard(self) -> None: + """Dispose transaction-local authority after each phase.""" + + +class DenyCheckerOutputWriteAuthority: + """Keep production output storage unavailable until ARCH-04D activation.""" + + async def preflight(self, selector: CheckerOutputSelector) -> None: + """Reject before any protected access.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + async def prepare(self, selector: CheckerOutputSelector) -> None: + """Reject transaction authority preparation.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + """Reject durable output admission.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + async def consume_existing( + self, reservation: CheckerOutputReservation, facts: CheckerOutputStoredFacts + ) -> None: + """Reject recovery of protected output facts.""" + raise CheckerOutputUnavailable("checker_output_write_unavailable") + + def discard(self) -> None: + """No authority was retained.""" + + +class CheckerArtifactOutputService: + """Own preparation lifetime; reuse put/observation and independent verification.""" + + def __init__( + self, + *, + sessions: async_sessionmaker[AsyncSession], + reservations: Callable[[AsyncSession], CheckerOutputReservationPort], + authority: Callable[[AsyncSession], CheckerOutputWriteAuthority], + internal_authority: Callable[[AsyncSession], ArtifactInternalAuthority], + preparation: ArtifactPreparationService, + store: ArtifactStore, + namespace: ArtifactStorageNamespaceSpec, + settings: Settings, + ) -> None: + self._sessions, self._reservations, self._authority = sessions, reservations, authority + self._internal_authority, self._preparation = internal_authority, preparation + self._store, self._namespace, self._settings = store, namespace, settings + + async def store(self, request: CheckerOutputArtifactRequest) -> CheckerOutputArtifactResult: + """Commit checked output bytes before I/O, never a CHECKERS completion.""" + if type(request) is not CheckerOutputArtifactRequest: + raise TypeError("invalid checker output request") + selector = CheckerOutputSelector.model_validate(request.selector) + prepared = None + async with self._sessions() as session: + authority = self._authority(session) + try: + await authority.preflight(selector) + async with session.begin(): + await authority.prepare(selector) + reservation = await self._reservations(session).resolve(selector) + slot = reservation.select(selector) + authority.discard() + prepared = await self._preparation.prepare( + request.byte_source, + media_type=slot.media_type, + maximum_bytes=slot.maximum_bytes, + ) + async with session.begin(): + await authority.prepare(selector) + current = await self._reservations(session).resolve(selector) + if current.select(selector) != slot: + raise CheckerOutputUnavailable("checker_output_slot_changed") + admission = await ArtifactAdmissionService( + session, self._settings, self._namespace + ).admit( + CheckerOutputArtifactAdmissionRequest( + reservation=current, + slot_key=slot.key, + source=prepared.committed_source, + ), + checker_output_authority=authority, + existing_transaction=True, + ) + authority.discard() + storage = self._storage(session) + if admission.status == "object_confirmed": + status = admission.status + elif admission.replayed: + status = await storage.resume_committed_put( + attempt_id=admission.attempt_id, + source=prepared.committed_source, + ) + else: + status = await storage.execute_committed_put( + attempt_id=admission.attempt_id, + source=prepared.committed_source, + ) + await self._verify(session, storage, admission.attempt_id, status) + return await self._result(session, authority, selector, replayed=admission.replayed) + finally: + authority.discard() + if prepared is not None: + await prepared.close() + + async def recover(self, selector: CheckerOutputSelector) -> CheckerOutputArtifactResult | None: + """Recover a lost response by logical identity without regenerating bytes.""" + selector = CheckerOutputSelector.model_validate(selector) + async with self._sessions() as session: + authority = self._authority(session) + try: + await authority.preflight(selector) + async with session.begin(): + await authority.prepare(selector) + reservation = await self._reservations(session).resolve(selector) + reservation.select(selector) + facts = await select_checker_output( + session, + selector=selector, + reservation=reservation, + namespace_fingerprint=self._namespace.namespace_fingerprint, + ) + if facts is None: + return None + await authority.consume_existing(reservation, facts) + authority.discard() + storage = self._storage(session) + status = facts.status + if status not in {"object_confirmed", "verified"}: + status = await storage.resolve_put_attempt(facts.put_attempt_id) + await self._verify(session, storage, facts.put_attempt_id, status) + return await self._result(session, authority, selector, replayed=True) + finally: + authority.discard() + + def _storage(self, session: AsyncSession) -> ArtifactStorageOrchestrator: + """Use the shared put, observation and verification implementation.""" + return ArtifactStorageOrchestrator( + session, self._store, self._namespace, self._settings, self._internal_authority(session) + ) + + async def _verify( + self, + session: AsyncSession, + storage: ArtifactStorageOrchestrator, + attempt_id: UUID, + status: str, + ) -> None: + """Independently verify confirmed bytes through the existing ART worker.""" + if status not in { + "object_confirmed", + "verified", + "stored_pending_verification", + "observed_confirmed", + }: + return + async with session.begin(): + job_id = await session.scalar( + select(ArtifactVerificationJob.id) + .where( + ArtifactVerificationJob.originating_put_attempt_id == str(attempt_id), + ArtifactVerificationJob.status == "pending", + ) + .order_by(ArtifactVerificationJob.id) + .limit(1) + ) + if job_id is not None: + await storage.verify_object(UUID(job_id)) + + async def _result( + self, + session: AsyncSession, + authority: CheckerOutputWriteAuthority, + selector: CheckerOutputSelector, + *, + replayed: bool, + ) -> CheckerOutputArtifactResult: + """Recheck current owner custody and authority after provider operations.""" + async with session.begin(): + await authority.prepare(selector) + reservation = await self._reservations(session).resolve(selector) + reservation.select(selector) + facts = await select_checker_output( + session, + selector=selector, + reservation=reservation, + namespace_fingerprint=self._namespace.namespace_fingerprint, + ) + if facts is None: + raise CheckerOutputUnavailable("checker_output_unavailable") + await authority.consume_existing(reservation, facts) + return CheckerOutputArtifactResult( + facts.put_attempt_id, + facts.status, + facts.content_id, + facts.verification_receipt_id, + replayed, + ) diff --git a/backend/app/modules/artifacts/models.py b/backend/app/modules/artifacts/models.py index 47d82e738..9215b89e3 100644 --- a/backend/app/modules/artifacts/models.py +++ b/backend/app/modules/artifacts/models.py @@ -396,6 +396,13 @@ class ArtifactBinding(Base): "(scope_version > 1 and supersedes_binding_id is not null)", name="scope_version_predecessor", ), + CheckConstraint( + "(resource_type = 'checker_run' and scope_version = 1 " + "and put_attempt_id is not null and verification_receipt_id is not null) or " + "(resource_type <> 'checker_run' and put_attempt_id is null " + "and verification_receipt_id is null)", + name="checker_output_lineage", + ), ) id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True) @@ -411,6 +418,22 @@ class ArtifactBinding(Base): scope_version: Mapped[int] = mapped_column(Integer, nullable=False) actor_id: Mapped[str] = mapped_column(String(100), nullable=False) attribution_type: Mapped[str] = mapped_column(String(30), nullable=False) + put_attempt_id: Mapped[str | None] = mapped_column( + ForeignKey( + "artifact_put_attempts.id", + ondelete="RESTRICT", + name="fk_artifact_bindings_checker_put_attempt", + ), + index=True, + ) + verification_receipt_id: Mapped[str | None] = mapped_column( + ForeignKey( + "artifact_verification_receipts.id", + ondelete="RESTRICT", + name="fk_artifact_bindings_checker_verification_receipt", + ), + index=True, + ) supersedes_binding_id: Mapped[str | None] = mapped_column( ForeignKey("artifact_bindings.id", ondelete="RESTRICT"), index=True ) @@ -969,6 +992,24 @@ class ArtifactPutAttempt(Base): ondelete="RESTRICT", name="fk_artifact_put_attempts_namespace_fingerprint", ), + ForeignKeyConstraint( + ["task_id", "project_id"], + ["workstream_tasks.id", "workstream_tasks.project_id"], + ondelete="RESTRICT", + name="fk_artifact_put_attempts_task_project", + ), + ForeignKeyConstraint( + ["checker_run_id", "task_id", "submission_id"], + ["checker_runs.id", "checker_runs.task_id", "checker_runs.submission_id"], + ondelete="RESTRICT", + name="fk_artifact_put_attempts_checker_run_ownership", + ), + ForeignKeyConstraint( + ["submission_id", "task_id", "submission_version"], + ["submissions.id", "submissions.task_id", "submissions.version"], + ondelete="RESTRICT", + name="fk_artifact_put_attempts_submission_version", + ), UniqueConstraint("operation_identity", name="uq_artifact_put_attempt_operation"), CheckConstraint( "producer_request_type in ('guide', 'checker_output', 'submission_bundle')", @@ -1005,6 +1046,12 @@ class ArtifactPutAttempt(Base): SHA256_CHECK.format(column="request_digest"), name="request_digest_shape", ), + CheckConstraint( + "(producer_request_type = 'checker_output' and checker_request_digest is not null " + "and checker_request_digest ~ '^sha256:[0-9a-f]{64}$') or " + "(producer_request_type <> 'checker_output' and checker_request_digest is null)", + name="checker_request_digest", + ), CheckConstraint( "status in ('prepared', 'put_in_flight', 'acknowledgement_unknown', " "'object_confirmed', 'absent_replay_required', 'integrity_mismatch', " @@ -1046,14 +1093,17 @@ class ArtifactPutAttempt(Base): ), CheckConstraint( "(producer_request_type = 'guide' and guide_source_item_id is not null " - "and checker_run_id is null and task_id is null " + "and checker_run_id is null and task_id is null and submission_id is null " + "and submission_version is null " "and logical_role is null) or " "(producer_request_type = 'checker_output' and guide_source_item_id is null " "and checker_run_id is not null and task_id is not null " + "and submission_id is not null and submission_version is not null " "and octet_length(logical_role) between 1 and 100) or " "(producer_request_type = 'submission_bundle' " "and guide_source_item_id is null and checker_run_id is null " - "and task_id is not null and logical_role is null)", + "and task_id is not null and submission_id is null " + "and submission_version is null and logical_role is null)", name="producer_reference", ), ) @@ -1074,6 +1124,8 @@ class ArtifactPutAttempt(Base): checker_run_id: Mapped[str | None] = mapped_column( ForeignKey("checker_runs.id", ondelete="RESTRICT"), index=True ) + submission_id: Mapped[str | None] = mapped_column(Uuid(as_uuid=False), index=True) + submission_version: Mapped[int | None] = mapped_column(Integer) logical_role: Mapped[str | None] = mapped_column(String(100)) sha256: Mapped[str] = mapped_column(String(71), nullable=False) byte_count: Mapped[int] = mapped_column(BigInteger, nullable=False) @@ -1083,6 +1135,10 @@ class ArtifactPutAttempt(Base): canonical_target: Mapped[str] = mapped_column(String(1024), nullable=False) operation_identity: Mapped[str] = mapped_column(String(71), nullable=False) request_digest: Mapped[str] = mapped_column(String(71), nullable=False) + checker_request_digest: Mapped[str | None] = mapped_column(String(71)) + checker_output_custody_sealed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) status: Mapped[str] = mapped_column(String(40), nullable=False, default="prepared", index=True) next_run_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), index=True) executor_id: Mapped[str | None] = mapped_column(Uuid(as_uuid=False)) @@ -1312,6 +1368,9 @@ class ArtifactReplica(Base): adapter: Mapped[str] = mapped_column(String(50), nullable=False) provider_profile: Mapped[str] = mapped_column(String(100), nullable=False) provider_object_ref: Mapped[str] = mapped_column(String(1024), nullable=False) + checker_output_custody_sealed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) verification_state: Mapped[str] = mapped_column(String(30), nullable=False) availability_state: Mapped[str] = mapped_column(String(30), nullable=False) integrity_state: Mapped[str] = mapped_column(String(30), nullable=False) @@ -1453,6 +1512,9 @@ class ArtifactVerificationJob(Base): replica_id: Mapped[str] = mapped_column( ForeignKey("artifact_replicas.id", ondelete="RESTRICT"), nullable=False, index=True ) + checker_output_custody_sealed: Mapped[bool] = mapped_column( + Boolean, nullable=False, server_default=text("false") + ) status: Mapped[str] = mapped_column(String(40), nullable=False, default="pending", index=True) attempt_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) maximum_attempts: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/backend/app/modules/artifacts/post_submit_materialization.py b/backend/app/modules/artifacts/post_submit_materialization.py index 897f291e8..6d84db997 100644 --- a/backend/app/modules/artifacts/post_submit_materialization.py +++ b/backend/app/modules/artifacts/post_submit_materialization.py @@ -8,7 +8,7 @@ from app.core.cancellation import await_cancellation_resistant from app.interfaces.artifacts import ArtifactStore -from app.modules.artifacts.api.submission_materialization import ( +from app.modules.checkers.api.materialization import ( PostSubmissionMaterialConsumer, PostSubmissionMaterializationResult, PostSubmissionMaterializationUnavailable, SubmissionMaterialEntry, ) diff --git a/backend/app/modules/artifacts/post_submit_selection.py b/backend/app/modules/artifacts/post_submit_selection.py index a637d0c76..4e01cada7 100644 --- a/backend/app/modules/artifacts/post_submit_selection.py +++ b/backend/app/modules/artifacts/post_submit_selection.py @@ -7,7 +7,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.interfaces.artifacts import ArtifactStore, artifact_provider_object_ref -from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.checkers.api.materialization import PostSubmissionMaterializationUnavailable from app.modules.artifacts.models import ( ArtifactBinding, ArtifactContent, ArtifactReplica, ArtifactStorageNamespace, ArtifactVerificationJob, ArtifactVerificationReceipt, PreSubmitEvidenceSet, diff --git a/backend/app/modules/artifacts/preparation.py b/backend/app/modules/artifacts/preparation.py index 99b465e71..620380b89 100644 --- a/backend/app/modules/artifacts/preparation.py +++ b/backend/app/modules/artifacts/preparation.py @@ -1481,6 +1481,7 @@ async def prepare( media_type: str, expected_sha256: str | None = None, expected_size: int | None = None, + maximum_bytes: int | None = None, ) -> PreparedArtifact: """Hash and count one complete source before any future provider call.""" self._validate_client_commitment( @@ -1488,6 +1489,16 @@ async def prepare( expected_sha256=expected_sha256, expected_size=expected_size, ) + if maximum_bytes is None: + source_maximum_bytes = self._manager.limits.maximum_source_bytes + else: + if ( + type(maximum_bytes) is not int + or maximum_bytes <= 0 + or maximum_bytes > self._manager.limits.maximum_source_bytes + ): + raise ValueError("artifact preparation per-call byte limit is invalid") + source_maximum_bytes = maximum_bytes loop = asyncio.get_running_loop() deadline = loop.time() + self._manager.limits.total_deadline_seconds reservation: _ScratchReservation | None = None @@ -1506,7 +1517,7 @@ async def prepare( for offset in range(0, len(view), self._manager.limits.stream_buffer_bytes): chunk = view[offset : offset + self._manager.limits.stream_buffer_bytes] byte_count += len(chunk) - if byte_count > self._manager.limits.maximum_source_bytes: + if byte_count > source_maximum_bytes: raise ArtifactLimitExceededError( "artifact source exceeds maximum bytes" ) diff --git a/backend/app/modules/artifacts/repository.py b/backend/app/modules/artifacts/repository.py index 5c369533b..aaafc0b16 100644 --- a/backend/app/modules/artifacts/repository.py +++ b/backend/app/modules/artifacts/repository.py @@ -30,7 +30,6 @@ SubmissionBundleDurableIntent, ) from app.modules.artifacts.schemas import VERIFICATION_PRODUCERS -from app.modules.checkers.models import CheckerRun from app.modules.projects.models import ( GuideSourceArtifactIngest, GuideSourceSnapshot, @@ -38,7 +37,6 @@ Project, ProjectGuide, ) -from app.modules.tasks.models import Submission, WorkstreamTask class GuideSourceIngestConflict(ValueError): @@ -72,15 +70,6 @@ class GuideLineageFacts: media_type: str | None -@dataclass(frozen=True, slots=True) -class CheckerOutputAdmissionFacts: - """Authoritative project/task ownership for one checker run.""" - - checker_run_id: str - project_id: str - task_id: str - - class ArtifactRepository: """Persist artifact state transitions under caller-owned transactions.""" @@ -95,12 +84,6 @@ async def database_now(self) -> datetime: raise RuntimeError("PostgreSQL clock did not return a timestamp") return value - async def lock_checker_run(self, checker_run_id: str) -> CheckerRun | None: - """Lock one checker run for canonical recovery resource derivation.""" - return await self._session.scalar( - select(CheckerRun).where(CheckerRun.id == checker_run_id).with_for_update() - ) - async def get_guide_admission_facts( self, guide_source_item_id: str ) -> GuideAdmissionFacts | None: @@ -237,32 +220,6 @@ async def get_guide_lineage(self, guide_source_item_id: str) -> GuideLineageFact media_type=lineage.media_type, ) - async def get_checker_output_admission_facts( - self, checker_run_id: str - ) -> CheckerOutputAdmissionFacts | None: - """Load canonical project/task ownership for one checker run.""" - row = ( - await self._session.execute( - select(CheckerRun.id, Submission.task_id, WorkstreamTask.project_id) - .join( - Submission, - (Submission.id == CheckerRun.submission_id) - & (Submission.version == CheckerRun.submission_version) - & (Submission.task_id == CheckerRun.task_id), - ) - .join(WorkstreamTask, WorkstreamTask.id == Submission.task_id) - .where(CheckerRun.id == checker_run_id) - .with_for_update(of=(CheckerRun, Submission, WorkstreamTask)) - ) - ).one_or_none() - if row is None: - return None - return CheckerOutputAdmissionFacts( - checker_run_id=row.id, - project_id=row.project_id, - task_id=row.task_id, - ) - async def ensure_and_lock_admission_scopes( self, scopes: Sequence[tuple[str, str, int]], diff --git a/backend/app/modules/artifacts/schemas.py b/backend/app/modules/artifacts/schemas.py index e173dee83..b5a012813 100644 --- a/backend/app/modules/artifacts/schemas.py +++ b/backend/app/modules/artifacts/schemas.py @@ -7,7 +7,12 @@ from typing import Protocol, TypeAlias, final from uuid import UUID +from app.core.hashing import canonical_json_hash from app.modules.artifacts.sources import CommittedArtifactSource +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSlot, +) from app.modules.authorization.runtime import AuthorizationContext from app.modules.authorization.catalogue import ActionId from app.modules.actors.api import ServiceIdentity @@ -32,12 +37,50 @@ class GuideArtifactAdmissionRequest: class CheckerOutputArtifactAdmissionRequest: """One prepared checker output admitted under its exact checker run.""" - authorization_context: AuthorizationContext - checker_run_id: UUID - logical_role: str + reservation: CheckerOutputReservation + slot_key: str source: CommittedArtifactSource +def checker_output_operation_identity(*, checker_run_id: UUID, slot_key: str) -> str: + """Keep one logical output identity across worker-lease replacement.""" + return canonical_json_hash( + { + "request_type": "checker_output", + "checker_run_id": str(checker_run_id), + "logical_role": slot_key, + } + ) + + +def checker_output_request_digest_facts( + *, + reservation: CheckerOutputReservation, + slot: CheckerOutputSlot, +) -> dict[str, object]: + """Return stable owner facts shared by checker admission and binding.""" + evaluation = reservation.evaluation + return { + "evaluation_request_id": str(evaluation.evaluation_request_id), + "evaluation_request_sha256": evaluation.request_sha256, + "evaluation_generation": evaluation.evaluation_generation, + "checker_run_id": str(reservation.checker_run_id), + "project_id": str(evaluation.project_id), + "task_id": str(evaluation.task_id), + "submission_id": str(evaluation.submission_id), + "submission_version": evaluation.submission_version, + "slot_key": slot.key, + "media_type": slot.media_type, + "maximum_bytes": slot.maximum_bytes, + } + + +class CheckerOutputAdmissionAuthority(Protocol): + """Consume caller-prepared authority for one exact output reservation.""" + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: ... + + @final @dataclass(frozen=True, slots=True) class SubmissionBundleArtifactAdmissionRequest: diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index 41af3efaa..69c579354 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -92,23 +92,30 @@ ArtifactRecoveryIneligibleError, ArtifactRecoveryNotFoundError, DenyArtifactInternalAuthority, + CheckerOutputAdmissionAuthority, CheckerOutputArtifactAdmissionRequest, GuideArtifactAdmissionRequest, GuideArtifactIngestAuthorityFacts, SubmissionBundleArtifactAdmissionRequest, SubmissionBundleDurableIntentAuthorityFacts, + checker_output_operation_identity, + checker_output_request_digest_facts, ) from app.modules.artifacts.submission_authorization import ( SubmissionBundlePreparedAuthorization, ) from app.modules.artifacts.sources import CommittedArtifactSource, PreparedArtifact +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSelector, + CheckerOutputUnavailable, +) from app.modules.authorization.runtime import ( ActorKind, ActorStatus, AuthorizationContext, HumanAuthorizationContext, IdentityLinkStatus, - ServiceAuthorizationContext, ) from app.modules.authorization.prepared import PreparedAuthorizationHandle from app.modules.authorization.catalogue import ActionId, PermissionId @@ -181,6 +188,10 @@ class _AdmissionFacts: guide_source_snapshot_id: str | None checker_run_id: str | None logical_role: str | None + submission_id: str | None + submission_version: int | None + checker_request_digest: str | None + checker_request_digest_facts: dict[str, object] | None pre_submit_evidence_set_id: str | None operation_identity: str @@ -1602,12 +1613,7 @@ async def _create_locked( if self._is_exact_replay(existing, request, digest): return self._result(existing, replayed=True) raise ArtifactRecoveryConflictError("artifact recovery source is already owned") - checker_run = ( - await self._repo.lock_checker_run(put_attempt.checker_run_id) - if put_attempt.checker_run_id is not None - else None - ) - canonical_submission_id = checker_run.submission_id if checker_run is not None else None + canonical_submission_id = put_attempt.submission_id canonical_project_id = UUID(put_attempt.project_id) canonical_task_id = UUID(put_attempt.task_id) canonical_submission_uuid = ( @@ -1906,16 +1912,27 @@ async def admit( submission_prepared_authorization: SubmissionBundlePreparedAuthorization | None = None, submission_task_contexts: TaskSubmissionContextPort | None = None, submission_project_contexts: ProjectLockedPolicyContextPort | None = None, + checker_output_authority: CheckerOutputAdmissionAuthority | None = None, prepared_authorization: PreparedAuthorizationHandle | None = None, existing_transaction: bool = False, ) -> ArtifactAdmissionResult: """Reserve every derived scope and persist one prepared attempt atomically.""" + if ( + type(request) is CheckerOutputArtifactAdmissionRequest + and checker_output_authority is None + ): + raise ArtifactAuthorityDeniedError( + "checker output admission authority is unavailable" + ) self._validate_request_boundary(request) commitment = request.source.commitment async with _artifact_admission_transaction( self._session, existing=existing_transaction, ): + if type(request) is CheckerOutputArtifactAdmissionRequest: + assert checker_output_authority is not None + await checker_output_authority.consume(request) if type(request) is GuideArtifactAdmissionRequest: if ( guide_prepared_authorization is None @@ -2040,6 +2057,15 @@ async def admit( "guide_source_item_id": facts.guide_source_item_id, "checker_run_id": facts.checker_run_id, "logical_role": facts.logical_role, + **( + { + "submission_id": facts.submission_id, + "submission_version": facts.submission_version, + "checker_output_request": facts.checker_request_digest_facts, + } + if facts.request_type == "checker_output" + else {} + ), "pre_submit_evidence_set_id": facts.pre_submit_evidence_set_id, "sha256": commitment.sha256, "byte_count": commitment.byte_count, @@ -2103,6 +2129,9 @@ async def admit( guide_source_item_id=facts.guide_source_item_id, checker_run_id=facts.checker_run_id, logical_role=facts.logical_role, + submission_id=facts.submission_id, + submission_version=facts.submission_version, + checker_request_digest=facts.checker_request_digest, sha256=commitment.sha256, byte_count=commitment.byte_count, media_type=commitment.media_type, @@ -2149,7 +2178,22 @@ def _validate_request_boundary(request: ArtifactAdmissionRequest) -> None: if type(request.source) is not CommittedArtifactSource: raise TypeError("invalid artifact admission source") if type(request) is CheckerOutputArtifactAdmissionRequest: - ArtifactAdmissionService._validate_logical_role(request.logical_role) + if type(request.reservation) is not CheckerOutputReservation: + raise TypeError("invalid checker output reservation") + try: + request.reservation.select( + CheckerOutputSelector( + evaluation=request.reservation.evaluation, + checker_run_id=request.reservation.checker_run_id, + worker_lease_id=request.reservation.worker_lease_id, + worker_lease_generation=request.reservation.worker_lease_generation, + slot_key=request.slot_key, + ) + ) + except (CheckerOutputUnavailable, ValueError) as exc: + raise ArtifactAdmissionRelationshipError( + "checker output reservation is unavailable" + ) from exc if type(request) is GuideArtifactAdmissionRequest: lineage_claims = ( request.project_id, @@ -2163,14 +2207,7 @@ def _validate_request_boundary(request: ArtifactAdmissionRequest) -> None: return if type(request) is SubmissionBundleArtifactAdmissionRequest: return - context = request.authorization_context - if type(context) not in {HumanAuthorizationContext, ServiceAuthorizationContext}: - raise TypeError("invalid artifact admission authorization context") - if ( - context.actor_status is not ActorStatus.ACTIVE - or context.identity_link_status is not IdentityLinkStatus.ACTIVE - ): - raise ArtifactAdmissionRelationshipError("artifact admission actor is not active") + return async def _derive_admission_facts(self, request: ArtifactAdmissionRequest) -> _AdmissionFacts: """Load every product and producer relationship from authoritative rows.""" @@ -2208,6 +2245,10 @@ async def _guide_facts(self, request: GuideArtifactAdmissionRequest) -> _Admissi guide_source_snapshot_id=row.guide_source_snapshot_id, checker_run_id=None, logical_role=None, + submission_id=None, + submission_version=None, + checker_request_digest=None, + checker_request_digest_facts=None, pre_submit_evidence_set_id=None, operation_identity=operation_identity, ) @@ -2215,51 +2256,55 @@ async def _guide_facts(self, request: GuideArtifactAdmissionRequest) -> _Admissi async def _checker_output_facts( self, request: CheckerOutputArtifactAdmissionRequest ) -> _AdmissionFacts: - """Bind committed bytes to one run and fixed checker service actor.""" - context = request.authorization_context - if context.actor_kind is not ActorKind.SERVICE: - raise ArtifactAdmissionRelationshipError( - "checker output producer must be a service actor" + """Bind committed bytes to one owner-issued run slot and fixed service.""" + reservation = CheckerOutputReservation.model_validate(request.reservation) + try: + slot = reservation.select( + CheckerOutputSelector( + evaluation=reservation.evaluation, + checker_run_id=reservation.checker_run_id, + worker_lease_id=reservation.worker_lease_id, + worker_lease_generation=reservation.worker_lease_generation, + slot_key=request.slot_key, + ) ) - logical_role = request.logical_role - service_actor = await self._actors.lock_admission_proof( - context.actor_profile_id, - context.identity_link_id, - ) + except (CheckerOutputUnavailable, ValueError) as exc: + raise ArtifactAdmissionRelationshipError( + "checker output reservation is unavailable" + ) from exc + commitment = request.source.commitment if ( - service_actor is None - or service_actor.actor_kind != "service" - or service_actor.actor_status != "active" - or service_actor.identity_link_id != str(context.identity_link_id) - or service_actor.identity_link_subject_kind != "service" - or service_actor.identity_link_status != "active" - or service_actor.service_identity != ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + commitment.media_type != slot.media_type + or commitment.byte_count > slot.maximum_bytes ): raise ArtifactAdmissionRelationshipError( - "checker output service identity is unavailable" + "checker output does not match reserved slot" ) - checker_run_id = str(request.checker_run_id) - row = await self._repo.get_checker_output_admission_facts(checker_run_id) - if row is None: - raise ArtifactAdmissionRelationshipError("checker run relationship is unavailable") - operation_identity = canonical_json_hash( - { - "request_type": "checker_output", - "checker_run_id": checker_run_id, - "logical_role": logical_role, - } + evaluation = reservation.evaluation + checker_run_id = str(reservation.checker_run_id) + logical_role = slot.key + operation_identity = checker_output_operation_identity( + checker_run_id=reservation.checker_run_id, slot_key=logical_role + ) + digest_facts = checker_output_request_digest_facts( + reservation=reservation, + slot=slot, ) return _AdmissionFacts( request_type="checker_output", producer_type="service_identity", producer_ref=ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, - project_id=row.project_id, + project_id=str(evaluation.project_id), guide_id=None, - task_id=row.task_id, + task_id=str(evaluation.task_id), guide_source_item_id=None, guide_source_snapshot_id=None, checker_run_id=checker_run_id, logical_role=logical_role, + submission_id=str(evaluation.submission_id), + submission_version=evaluation.submission_version, + checker_request_digest=canonical_json_hash(digest_facts), + checker_request_digest_facts=digest_facts, pre_submit_evidence_set_id=None, operation_identity=operation_identity, ) @@ -2519,6 +2564,10 @@ async def _submission_bundle_facts( guide_source_snapshot_id=evidence.source_snapshot_id, checker_run_id=None, logical_role=None, + submission_id=None, + submission_version=None, + checker_request_digest=None, + checker_request_digest_facts=None, pre_submit_evidence_set_id=evidence.id, operation_identity=operation_identity, ) @@ -2542,20 +2591,6 @@ async def _require_active_human_actor(self, context: AuthorizationContext) -> No "artifact admission human identity is unavailable" ) - @staticmethod - def _validate_logical_role(value: str) -> str: - """Require one bounded printable checker-output role.""" - if ( - not isinstance(value, str) - or value != value.strip() - or not value - or not value.isascii() - or len(value) > 100 - or any(ord(character) < 32 or ord(character) == 127 for character in value) - ): - raise ArtifactAdmissionRelationshipError("checker output logical role is invalid") - return value - def _derive_scopes(self, facts: _AdmissionFacts) -> tuple[_AdmissionScopeSpec, ...]: """Derive the complete closed scope set without caller participation.""" limits = self._configured_limits() diff --git a/backend/app/modules/artifacts/api/submission_materialization.py b/backend/app/modules/checkers/api/materialization.py similarity index 94% rename from backend/app/modules/artifacts/api/submission_materialization.py rename to backend/app/modules/checkers/api/materialization.py index d6b918f6e..1a109bc75 100644 --- a/backend/app/modules/artifacts/api/submission_materialization.py +++ b/backend/app/modules/checkers/api/materialization.py @@ -4,7 +4,10 @@ from typing import Literal, Protocol from uuid import UUID -from app.modules.checkers.api import PostSubmissionEvaluationRequest, PostSubmissionEvaluationResult +from app.modules.checkers.api.post_submit import ( + PostSubmissionEvaluationRequest, + PostSubmissionEvaluationResult, +) class PostSubmissionMaterializationUnavailable(RuntimeError): diff --git a/backend/app/modules/checkers/api/output_custody.py b/backend/app/modules/checkers/api/output_custody.py new file mode 100644 index 000000000..5e0b6f599 --- /dev/null +++ b/backend/app/modules/checkers/api/output_custody.py @@ -0,0 +1,154 @@ +"""CHECKERS-owned output custody contracts; production reservations are unavailable.""" + +from collections.abc import AsyncIterable +from dataclasses import dataclass +from typing import Annotated, Literal, Protocol, Self +from uuid import UUID + +from pydantic import Field, StrictInt, model_validator + +from app.modules.checkers.api.post_submit import PostSubmissionEvaluationRequest +from app.modules.checkers.api.post_submit_catalogue import ( + Identifier, + PostSubmitValue, + ResourceId, + VersionNumber, +) + + +class CheckerOutputUnavailable(RuntimeError): + """Conceal missing, stale or unauthorized checker output custody.""" + + +class CheckerOutputSelector(PostSubmitValue): + """Select exact immutable evaluation and the caller's claimed execution lease.""" + + evaluation: PostSubmissionEvaluationRequest + checker_run_id: ResourceId + worker_lease_id: ResourceId + worker_lease_generation: VersionNumber + slot_key: Identifier + + +class CheckerOutputSlot(PostSubmitValue): + """One globally unique run slot issued by CHECKERS, not by a contributor.""" + + key: Identifier + media_type: Literal["application/json", "application/octet-stream", "text/plain"] + maximum_bytes: Annotated[StrictInt, Field(ge=1, le=536_870_912)] + + +class CheckerOutputReservation(PostSubmitValue): + """Detached owner facts, never a substitute for fresh action authority.""" + + evaluation: PostSubmissionEvaluationRequest + checker_run_id: ResourceId + worker_lease_id: ResourceId + worker_lease_generation: VersionNumber + slots: tuple[CheckerOutputSlot, ...] = Field(max_length=64) + + @model_validator(mode="after") + def unique_slots(self) -> Self: + """Keep generic ART binding's run/role scope unambiguous.""" + if len({slot.key for slot in self.slots}) != len(self.slots): + raise ValueError("checker output slots repeat") + return self + + def select(self, selector: CheckerOutputSelector) -> CheckerOutputSlot: + """Match the caller's claimed lease as well as immutable evaluation facts.""" + current = CheckerOutputReservation.model_validate(self) + selector = CheckerOutputSelector.model_validate(selector) + if ( + current.evaluation, + current.checker_run_id, + current.worker_lease_id, + current.worker_lease_generation, + ) != ( + selector.evaluation, + selector.checker_run_id, + selector.worker_lease_id, + selector.worker_lease_generation, + ): + raise CheckerOutputUnavailable("checker_output_reservation_unavailable") + for slot in current.slots: + if slot.key == selector.slot_key: + return slot + raise CheckerOutputUnavailable("checker_output_slot_unavailable") + + +class CheckerOutputReservationPort(Protocol): + """Resolve current owner custody before ART locks; CHECKERS owns lease fencing.""" + + async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReservation: + """Return exact current facts under the caller's CHECKERS transaction locks.""" + + +@dataclass(frozen=True, slots=True) +class CheckerOutputBindingRequest: + """Bind one exact independently verified output in the caller transaction.""" + + selector: CheckerOutputSelector + put_attempt_id: UUID + verification_receipt_id: UUID + + +@dataclass(frozen=True, slots=True) +class CheckerOutputArtifactRequest: + """Bounded generated bytes; action authority is freshly prepared by ART.""" + + selector: CheckerOutputSelector + byte_source: AsyncIterable[bytes] + + +@dataclass(frozen=True, slots=True) +class CheckerOutputArtifactResult: + """Durable put identity and verified custody, without provider coordinates.""" + + put_attempt_id: UUID + status: str + content_id: UUID | None + verification_receipt_id: UUID | None + replayed: bool + + +@dataclass(frozen=True, slots=True) +class CheckerOutputBindingResult: + """Immutable binding participating in the caller's final-result transaction.""" + + binding_id: UUID + content_id: UUID + put_attempt_id: UUID + verification_receipt_id: UUID + replayed: bool + + +class CheckerOutputBindingPort(Protocol): + """Create exact action-bound bindings from verified content.""" + + async def bind_checker_output( + self, request: CheckerOutputBindingRequest + ) -> CheckerOutputBindingResult: + """Bind verified checker output under the checker binding action.""" + + +class CheckerArtifactOutputPort(Protocol): + """Store generated output for one fixed checker execution.""" + + async def store( + self, request: CheckerOutputArtifactRequest + ) -> CheckerOutputArtifactResult: + """Store one generated checker artifact.""" + + async def recover( + self, selector: CheckerOutputSelector + ) -> CheckerOutputArtifactResult | None: + """Recover a lost output response without a byte source or regeneration.""" + + +class UnavailableCheckerOutputReservation: + """ARCH-04C must install the real reservation producer before activation.""" + + async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReservation: + """Deny without reading protected facts or fabricating a reservation.""" + del selector + raise CheckerOutputUnavailable("checker_output_reservation_unavailable") diff --git a/backend/scripts/behavior_ownership.py b/backend/scripts/behavior_ownership.py index 5d537f926..d8158a33d 100644 --- a/backend/scripts/behavior_ownership.py +++ b/backend/scripts/behavior_ownership.py @@ -146,13 +146,22 @@ "backend/app/modules/authorization/api/outbox_dispatch.py", } ) +ARCH_04B2_OUTPUT_TARGETS = frozenset({ + "backend/app/modules/artifacts/checker_outputs.py", + "backend/app/modules/artifacts/checker_output_custody.py", + "backend/app/modules/artifacts/checker_output_bindings.py", + "backend/app/modules/checkers/api/output_custody.py", +}) ARCH_04B_MATERIALIZATION_TARGETS = frozenset({ - "backend/app/modules/artifacts/api/submission_materialization.py", + "backend/app/modules/checkers/api/materialization.py", "backend/app/modules/artifacts/post_submit_materialization.py", "backend/app/modules/artifacts/post_submit_selection.py", "backend/app/modules/tasks/api/submitted_bundle.py", "backend/app/modules/tasks/submitted_bundle.py", }) +ARCH_04B_SEAM_REMOVED_TARGETS = frozenset({ + "backend/app/modules/artifacts/api/submission_materialization.py", +}) ARCH_04A_POST_SUBMIT_TARGETS = frozenset( { "backend/app/modules/checkers/api/post_submit.py", @@ -660,7 +669,7 @@ def _validate_additive_partition_transition( ] if ( trusted_targets != sorted(trusted_targets) - or removed - (ARCH_03D_REMOVED_TARGETS | TASK_CHECKER_CLEANUP_REMOVED_TARGETS | ARCH_03C2_REMOVED_TARGETS | OUTBOX_IDENTITY_REMOVED_TARGETS | V01_BASELINE_REMOVED_TARGETS | POL_03B_REMOVED_TARGETS | POL_04B_REMOVED_TARGETS | POL_05A_REMOVED_TARGETS | ARCH_03A_GUIDE_CONTEXT_REMOVED_TARGETS) + or removed - (ARCH_03D_REMOVED_TARGETS | TASK_CHECKER_CLEANUP_REMOVED_TARGETS | ARCH_03C2_REMOVED_TARGETS | OUTBOX_IDENTITY_REMOVED_TARGETS | V01_BASELINE_REMOVED_TARGETS | POL_03B_REMOVED_TARGETS | POL_04B_REMOVED_TARGETS | POL_05A_REMOVED_TARGETS | ARCH_03A_GUIDE_CONTEXT_REMOVED_TARGETS | ARCH_04B_SEAM_REMOVED_TARGETS) or [current_by_target[item["target"]] for item in retained_trusted] != retained_trusted ): @@ -705,6 +714,7 @@ def _validate_additive_partition_transition( | ARCH_CP03B_ADAPTER_BINDING_AUTH_TARGETS | ARCH_CP04A_CONTRIBUTION_POLICY_TARGETS | ARCH_CP04B_CONTRIBUTION_POLICY_TARGETS + | ARCH_04B2_OUTPUT_TARGETS | ARCH_04B_MATERIALIZATION_TARGETS | ARCH_04A_POST_SUBMIT_TARGETS | ARCH_CP05_POLICY_AUTH_TARGETS diff --git a/backend/scripts/test_lane_catalogue.py b/backend/scripts/test_lane_catalogue.py index db9e5c97f..c292d6e14 100644 --- a/backend/scripts/test_lane_catalogue.py +++ b/backend/scripts/test_lane_catalogue.py @@ -217,6 +217,7 @@ class TestLane: "tests/authorization/contribution_policies/test_scope_locks.py", "tests/test_behavior_ownership.py", "tests/test_artifact_admission.py", + "tests/test_artifact_admission_digest.py", "tests/test_submission_bundle_admission.py", "tests/test_submission_bundle_preparation_recovery.py", "tests/checkers/test_phase_service.py", @@ -421,6 +422,8 @@ class TestLane: "tests/test_approved_guide_intake.py", "tests/test_post_submit_materialization.py", "tests/test_post_submit_selection.py", + "tests/test_checker_output_custody.py", + "tests/test_checker_output_storage.py", "tests/test_pre_submit_attempt_recovery.py", "tests/test_pre_submit_attempt_contracts.py", "tests/test_pre_submit_attempt_authority_integration.py", diff --git a/backend/tests/architecture/test_module_boundaries.py b/backend/tests/architecture/test_module_boundaries.py index 649eb20a9..e5615a3d1 100644 --- a/backend/tests/architecture/test_module_boundaries.py +++ b/backend/tests/architecture/test_module_boundaries.py @@ -413,6 +413,53 @@ def test_cyclic_public_dependencies_fail_closed() -> None: boundary._validate_acyclic(graph) # noqa: SLF001 - architecture proof +def test_art_implements_checker_consumer_ports_without_a_public_cycle() -> None: + """The delivered seam is ART implementation to CHECKERS-owned public ports.""" + registry = boundary.load_registry(REGISTRY) + private, graph, _ = boundary.scan(ROOT, registry) + assert "checkers" in graph["artifacts"] + assert "artifacts" not in graph["checkers"] + assert { + (edge.source_file, edge.imported_private_path) + for edge in private + if edge.source_file.startswith("backend/app/modules/checkers/") + and edge.target_module == "artifacts" + } == { + ( + "backend/app/modules/checkers/pre_submit_execution.py", + "app.modules.artifacts.sources", + ), + ( + "backend/app/modules/checkers/pre_submit_execution.py", + "app.modules.artifacts.submission_archive", + ), + ( + "backend/app/modules/checkers/pre_submit_execution.py", + "app.modules.artifacts.submission_manifest", + ), + } + boundary._validate_acyclic(graph) # noqa: SLF001 - architecture proof + + +def test_checker_to_art_public_import_would_close_a_cycle(tmp_path: Path) -> None: + """A concrete future CHECKERS import of ART public API fails closed.""" + registry_path = tmp_path / "registry.json" + _registry(registry_path) + _write( + tmp_path / "backend/app/modules/artifacts/implementation.py", + "from app.modules.checkers.api.output_custody import CheckerArtifactOutputPort\n", + ) + _write( + tmp_path / "backend/app/modules/checkers/future_execution.py", + "from app.modules.artifacts.api import SubmissionBundlePreparationCommand\n", + ) + _, graph, _ = boundary.scan(tmp_path, boundary.load_registry(registry_path)) + assert graph["artifacts"] == {"checkers"} + assert graph["checkers"] == {"artifacts"} + with pytest.raises(boundary.ModuleBoundaryError, match="cyclic_public_dependency"): + boundary._validate_acyclic(graph) # noqa: SLF001 - architecture proof + + def test_general_ledger_rejects_copied_authorization_edges(tmp_path: Path) -> None: """AUTH debt remains exclusively owned by the AUTH-003 ledger.""" document = { diff --git a/backend/tests/authorization/submission_history/test_migration.py b/backend/tests/authorization/submission_history/test_migration.py index 8b197e093..6ac5a7f72 100644 --- a/backend/tests/authorization/submission_history/test_migration.py +++ b/backend/tests/authorization/submission_history/test_migration.py @@ -68,8 +68,20 @@ async def probe(before): async def _before_custody(connection): - """Restore only this unmerged migration's predecessor checker schema in a transaction.""" + """Restore the history migration's predecessor checker schema in a transaction.""" from sqlalchemy import text + dependent_fk = "fk_artifact_put_attempts_checker_run_ownership" + dependent_fk_definition = await connection.scalar( + text( + "select pg_get_constraintdef(oid) from pg_constraint " + "where conname=:name and conrelid='artifact_put_attempts'::regclass" + ), + {"name": dependent_fk}, + ) + assert isinstance(dependent_fk_definition, str) + await connection.execute( + text(f"alter table artifact_put_attempts drop constraint {dependent_fk}") + ) for table, owner in (("checker_runs", "run"), ("checker_results", "result")): for suffix in ("custody", "no_truncate"): await connection.execute(text(f"drop trigger checker_{owner}_{suffix} on {table}")) @@ -85,6 +97,7 @@ async def _before_custody(connection): ("checker_results", "submission_id", "submissions"), ): await connection.execute(text(f"alter table {table} add constraint fk_{table}_{column}_{parent} foreign key ({column}) references {parent}(id)")) + return dependent_fk_definition def _upgrade_custody(connection): @@ -137,7 +150,7 @@ async def test_checker_migration_locks_out_coherent_rebinding(task_client, monke factory = db_session.get_session_factory() async with factory() as prepare: audit_definition = await prepare.scalar(text("select pg_get_constraintdef(oid) from pg_constraint where conname=:name"), {"name": CONSTRAINT}) - await _before_custody(await prepare.connection()) + dependent_fk_definition = await _before_custody(await prepare.connection()) await prepare.commit() preflight = asyncio.Event() original_execute = op.execute @@ -189,4 +202,19 @@ async def rebind(): assert str(row.task_id) == case[1] and str(row.submission_id) == case[2] await session.execute(text(f"alter table audit_events drop constraint {CONSTRAINT}")) await session.execute(text(f"alter table audit_events add constraint {CONSTRAINT} {audit_definition}")) + await session.execute( + text( + "alter table artifact_put_attempts add constraint " + "fk_artifact_put_attempts_checker_run_ownership " + f"{dependent_fk_definition}" + ) + ) + restored_fk_definition = await session.scalar( + text( + "select pg_get_constraintdef(oid) from pg_constraint " + "where conname='fk_artifact_put_attempts_checker_run_ownership' " + "and conrelid='artifact_put_attempts'::regclass" + ) + ) + assert restored_fk_definition == dependent_fk_definition await session.commit() diff --git a/backend/tests/checker_output_admission_helpers.py b/backend/tests/checker_output_admission_helpers.py new file mode 100644 index 000000000..4e1deb985 --- /dev/null +++ b/backend/tests/checker_output_admission_helpers.py @@ -0,0 +1,223 @@ +"""Controlled complete CHECKERS owner facts for ART output-custody tests.""" + +from uuid import UUID + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import async_sessionmaker + +from app.core.identifiers import new_record_id +from app.modules.actors.api import ServiceIdentity +from app.modules.actors.service import ActorService +from app.modules.artifacts.schemas import ( + ArtifactAuthorityDeniedError, + CheckerOutputArtifactAdmissionRequest, +) +from app.modules.artifacts.models import ( + ArtifactAdmissionCharge, + ArtifactAdmissionScope, + ArtifactContent, + ArtifactOperationReceipt, + ArtifactPutAttempt, + ArtifactPutAttemptCharge, + ArtifactReplica, + ArtifactStorageNamespace, +) +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSlot, +) +from app.modules.checkers.models import CheckerRun +from app.modules.checkers.post_submit_contracts import make_post_submit_request +from app.modules.tasks.models import Submission, WorkstreamTask +from tests.checkers.post_submit.support import request as checker_evaluation_request + + +async def make_checker_output_reservation( + session, + checker_run_id: str | UUID, + *, + slot_key: str = "platform-review", + media_type: str = "application/octet-stream", + maximum_bytes: int = 1024, + worker_lease_id: UUID | None = None, + worker_lease_generation: int = 1, +) -> CheckerOutputReservation: + """Derive hash-valid public owner facts from one actual seeded run.""" + checker_run = await session.get(CheckerRun, str(checker_run_id)) + assert checker_run is not None + submission = await session.scalar( + select(Submission).where( + Submission.id == checker_run.submission_id, + Submission.version == checker_run.submission_version, + Submission.task_id == checker_run.task_id, + ) + ) + task = await session.get(WorkstreamTask, checker_run.task_id) + assert submission is not None and task is not None + seed = checker_evaluation_request(project_id=UUID(task.project_id)) + fields = seed.model_dump(exclude={"request_sha256"}) + fields.update( + task_id=UUID(task.id), + assignment_id=UUID(submission.task_assignment_id), + submission_id=UUID(submission.id), + submission_version=submission.version, + ) + evaluation = make_post_submit_request(**fields) + return CheckerOutputReservation( + evaluation=evaluation, + checker_run_id=UUID(checker_run.id), + worker_lease_id=worker_lease_id or new_record_id(), + worker_lease_generation=worker_lease_generation, + slots=( + CheckerOutputSlot( + key=slot_key, + media_type=media_type, + maximum_bytes=maximum_bytes, + ), + ), + ) + + +async def make_checker_output_admission_request( + session, + checker_run_id: str, + source, + *, + slot_key: str = "platform-review", + worker_lease_id: UUID | None = None, + worker_lease_generation: int = 1, +) -> CheckerOutputArtifactAdmissionRequest: + """Build detached owner facts without rolling back or expiring caller state.""" + factory = async_sessionmaker(session.bind, expire_on_commit=False) + async with factory() as reservation_session: + reservation = await make_checker_output_reservation( + reservation_session, + checker_run_id, + slot_key=slot_key, + media_type=source.commitment.media_type, + maximum_bytes=max(1, source.commitment.byte_count), + worker_lease_id=worker_lease_id, + worker_lease_generation=worker_lease_generation, + ) + return CheckerOutputArtifactAdmissionRequest( + reservation=reservation, + slot_key=slot_key, + source=source, + ) + + +_ADMISSION_BASELINE_MODELS = ( + ArtifactStorageNamespace, + ArtifactAdmissionScope, + ArtifactAdmissionCharge, + ArtifactPutAttempt, + ArtifactContent, + ArtifactReplica, + ArtifactOperationReceipt, +) + + +async def checker_admission_baseline(session) -> dict[type, int]: + """Snapshot every ART row family asserted by checker admission proofs.""" + return { + model: int(await session.scalar(select(func.count()).select_from(model)) or 0) + for model in _ADMISSION_BASELINE_MODELS + } + + +async def assert_checker_admission_unchanged(session, baseline: dict[type, int]) -> None: + """Prove a denied checker request created no admission-owned rows.""" + for model in ( + ArtifactStorageNamespace, + ArtifactAdmissionScope, + ArtifactAdmissionCharge, + ArtifactPutAttempt, + ): + assert await session.scalar(select(func.count()).select_from(model)) == baseline[model] + + +async def assert_exact_checker_admission( + session, + *, + result, + replay, + takeover, + request, + authority, + project_id: str, + task_id: str, + checker_run_id: str, + baseline: dict[type, int], +) -> None: + """Retain the complete fixed-service, quota and replay assertion set.""" + attempt = await session.get(ArtifactPutAttempt, str(result.attempt_id)) + scopes = (await session.scalars(select(ArtifactAdmissionScope).order_by( + ArtifactAdmissionScope.scope_type, ArtifactAdmissionScope.scope_id, + ))).all() + links = (await session.scalars(select(ArtifactPutAttemptCharge).where( + ArtifactPutAttemptCharge.attempt_id == str(result.attempt_id), + ))).all() + assert attempt is not None + assert replay.attempt_id == result.attempt_id + assert replay.charge_ids == result.charge_ids + assert takeover.attempt_id == result.attempt_id + assert takeover.replayed is True + assert attempt.status == "prepared" + assert attempt.producer_request_type == "checker_output" + assert attempt.producer_type == "service_identity" + assert attempt.producer_ref == ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + assert attempt.project_id == project_id + assert attempt.task_id == task_id + assert attempt.checker_run_id == checker_run_id + assert attempt.logical_role == "platform-review" + assert attempt.submission_id == str(request.reservation.evaluation.submission_id) + assert attempt.submission_version == request.reservation.evaluation.submission_version + assert attempt.checker_request_digest is not None + assert authority.calls == 3 + assert attempt.executor_id is None + assert attempt.lease_expires_at is None + assert attempt.next_run_at is None + assert attempt.execution_generation == 0 + assert {scope.scope_type for scope in scopes} == { + "deployment", "producer", "project", "task", + } + assert len(result.charge_ids) == 4 + assert len(links) == 4 + assert await session.scalar(select(func.count()).select_from(ArtifactPutAttempt)) == ( + baseline[ArtifactPutAttempt] + 1 + ) + assert await session.scalar(select(func.count()).select_from(ArtifactAdmissionCharge)) == ( + baseline[ArtifactAdmissionCharge] + 4 + ) + for model in (ArtifactContent, ArtifactReplica, ArtifactOperationReceipt): + assert await session.scalar(select(func.count()).select_from(model)) == baseline[model] + + +class ControlledCheckerOutputAdmissionAuthority: + """Hidden active fixed-service proof; never a production activation.""" + + def __init__(self, session, actor_id: UUID, identity_link_id: UUID) -> None: + self._session = session + self._actor_id = actor_id + self._identity_link_id = identity_link_id + self.calls = 0 + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + self.calls += 1 + proof = await ActorService(self._session).lock_admission_proof( + self._actor_id, + self._identity_link_id, + ) + if ( + proof is None + or proof.actor_kind != "service" + or proof.actor_status != "active" + or proof.identity_link_id != str(self._identity_link_id) + or proof.identity_link_subject_kind != "service" + or proof.identity_link_status != "active" + or proof.service_identity + != ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + ): + raise ArtifactAuthorityDeniedError( + "checker output service identity is unavailable" + ) diff --git a/backend/tests/checker_output_custody_helpers.py b/backend/tests/checker_output_custody_helpers.py new file mode 100644 index 000000000..d238eb566 --- /dev/null +++ b/backend/tests/checker_output_custody_helpers.py @@ -0,0 +1,440 @@ +"""Controlled owner facts and real storage harness for checker-output custody.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.adapters.artifacts import create_artifact_store_bootstrap +from app.core.identifiers import new_record_id +from app.modules.checkers.api.output_custody import ( + CheckerOutputArtifactRequest, + CheckerOutputArtifactResult, + CheckerOutputBindingRequest, + CheckerOutputBindingResult, +) +from app.interfaces.artifacts import ( + ArtifactByteRange, + ArtifactStoreNamespaceClaim, + ArtifactStoreUnavailableError, +) +from app.modules.actors.api import ServiceIdentity +from app.modules.actors.models import ActorIdentityLink, ActorProfile +from app.modules.artifacts.checker_output_bindings import ( + CheckerOutputBindingService, +) +from app.modules.artifacts.checker_output_custody import CheckerOutputStoredFacts +from app.modules.artifacts.checker_outputs import CheckerArtifactOutputService +from app.modules.artifacts.models import ArtifactBinding +from app.modules.artifacts.preparation import ( + ArtifactPreparationService, + ArtifactScratchManager, +) +from app.modules.artifacts.schemas import CheckerOutputArtifactAdmissionRequest +from app.modules.artifacts.service import artifact_storage_namespace_spec +from app.modules.checkers.api.output_custody import ( + CheckerOutputReservation, + CheckerOutputSelector, + CheckerOutputUnavailable, +) +from app.modules.checkers.models import CheckerRun +from tests.artifact_store_helpers import ( + artifact_admission_limit_settings, + artifact_preparation_limits, +) +from tests.checker_output_admission_helpers import ( + ControlledCheckerOutputAdmissionAuthority, + make_checker_output_reservation, +) +from tests.projects.unified_policy_fixtures import create_standalone_unified_policy +from tests.test_artifact_admission import ( + _AllowArtifactAuthority, + _add_checker_output_actor, + _seed_checker_output_relationships, + _settings, +) + + +def selector_for(reservation: CheckerOutputReservation) -> CheckerOutputSelector: + """Select the sole controlled output slot under the reservation's current lease.""" + assert len(reservation.slots) == 1 + return CheckerOutputSelector( + evaluation=reservation.evaluation, + checker_run_id=reservation.checker_run_id, + worker_lease_id=reservation.worker_lease_id, + worker_lease_generation=reservation.worker_lease_generation, + slot_key=reservation.slots[0].key, + ) + + +async def byte_stream(*chunks: bytes, observed: list[bytes] | None = None) -> AsyncIterator[bytes]: + """Yield explicit generated bytes and optionally record source iteration.""" + for chunk in chunks: + if observed is not None: + observed.append(chunk) + yield chunk + + +@dataclass(slots=True) +class ControlledReservationState: + """Mutable test control around one immutable owner-issued reservation.""" + + reservation: CheckerOutputReservation + resolved: list[CheckerOutputSelector] = field(default_factory=list) + revoked: bool = False + preflights: list[CheckerOutputSelector] = field(default_factory=list) + prepares: list[CheckerOutputSelector] = field(default_factory=list) + admissions: list[CheckerOutputArtifactAdmissionRequest] = field(default_factory=list) + recoveries: list[tuple[CheckerOutputReservation, CheckerOutputStoredFacts]] = field( + default_factory=list + ) + bindings: list[tuple[CheckerOutputReservation, CheckerOutputStoredFacts]] = field( + default_factory=list + ) + + def require_authority(self) -> None: + """Model one fresh action-authority revocation without product activation.""" + if self.revoked: + raise CheckerOutputUnavailable("checker_output_test_authority_revoked") + + +class ControlledReservationPort: + """Lock a real run, then return controlled detached CHECKERS owner facts.""" + + def __init__(self, session: AsyncSession, state: ControlledReservationState) -> None: + self._session = session + self._state = state + + async def resolve(self, selector: CheckerOutputSelector) -> CheckerOutputReservation: + """Fence the selected real run before ART acquires any artifact row lock.""" + checker_run = await self._session.scalar( + select(CheckerRun) + .where(CheckerRun.id == str(selector.checker_run_id)) + .with_for_update() + .execution_options(populate_existing=True) + ) + if checker_run is None: + raise CheckerOutputUnavailable("checker_output_reservation_unavailable") + self._state.resolved.append(selector) + return self._state.reservation + + +class ControlledWriteAuthority: + """Test-only exact action authority plus real fixed-service admission proof.""" + + def __init__( + self, + session: AsyncSession, + state: ControlledReservationState, + actor_id: UUID, + identity_link_id: UUID, + ) -> None: + self._state = state + self._admission = ControlledCheckerOutputAdmissionAuthority( + session, + actor_id, + identity_link_id, + ) + + async def preflight(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.preflights.append(selector) + + async def prepare(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.prepares.append(selector) + + async def consume(self, request: CheckerOutputArtifactAdmissionRequest) -> None: + self._state.require_authority() + assert request.reservation == self._state.reservation + await self._admission.consume(request) + self._state.admissions.append(request) + + async def consume_existing( + self, + reservation: CheckerOutputReservation, + facts: CheckerOutputStoredFacts, + ) -> None: + self._state.require_authority() + assert reservation == self._state.reservation + self._state.recoveries.append((reservation, facts)) + + def discard(self) -> None: + """The controlled seam retains no prepared capability.""" + + +class ControlledBindingAuthority: + """Test-only binding authority that records exact verified custody.""" + + def __init__(self, state: ControlledReservationState) -> None: + self._state = state + + async def preflight(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.preflights.append(selector) + + async def prepare(self, selector: CheckerOutputSelector) -> None: + self._state.require_authority() + self._state.prepares.append(selector) + + async def consume( + self, + reservation: CheckerOutputReservation, + facts: CheckerOutputStoredFacts, + ) -> None: + self._state.require_authority() + assert reservation == self._state.reservation + self._state.bindings.append((reservation, facts)) + + def discard(self) -> None: + """The controlled seam retains no prepared capability.""" + + +class ObservedStore: + """Count provider calls and optionally pause one put at the provider boundary.""" + + def __init__(self, wrapped: Any) -> None: + self.wrapped = wrapped + self.identity = wrapped.identity + self.puts = 0 + self.opens = 0 + self.put_entered: asyncio.Event | None = None + self.put_release: asyncio.Event | None = None + self.lose_put_acknowledgement = False + + async def put(self, source): + self.puts += 1 + if self.put_entered is not None: + self.put_entered.set() + if self.put_release is not None: + await self.put_release.wait() + result = await self.wrapped.put(source) + if self.lose_put_acknowledgement: + raise ArtifactStoreUnavailableError("controlled lost provider acknowledgement") + return result + + async def observe_put_result(self, commitment): + return await self.wrapped.observe_put_result(commitment) + + def open( + self, + provider_object_ref: str, + byte_range: ArtifactByteRange | None = None, + ): + self.opens += 1 + return self.wrapped.open(provider_object_ref, byte_range) + + async def head(self, provider_object_ref: str): + return await self.wrapped.head(provider_object_ref) + + +@dataclass(slots=True) +class OutputCustodyHarness: + """One real database/provider path with controlled unavailable owner seams.""" + + factory: async_sessionmaker[AsyncSession] + engine: Any + bootstrap: Any + store: ObservedStore + namespace: Any + settings: Any + manager: ArtifactScratchManager + preparation: ArtifactPreparationService + service: CheckerArtifactOutputService + state: ControlledReservationState + actor_id: UUID + identity_link_id: UUID + policy_bundle: Any + + @property + def selector(self) -> CheckerOutputSelector: + return selector_for(self.state.reservation) + + def request( + self, + *chunks: bytes, + selector: CheckerOutputSelector | None = None, + observed: list[bytes] | None = None, + ) -> CheckerOutputArtifactRequest: + return CheckerOutputArtifactRequest( + selector=selector or self.selector, + byte_source=byte_stream(*chunks, observed=observed), + ) + + def binding_service(self, session: AsyncSession) -> CheckerOutputBindingService: + return CheckerOutputBindingService( + session, + reservations=ControlledReservationPort(session, self.state), + authority=ControlledBindingAuthority(self.state), + namespace_fingerprint=self.namespace.namespace_fingerprint, + ) + + async def bind( + self, + selector: CheckerOutputSelector, + stored: CheckerOutputArtifactResult, + ) -> CheckerOutputBindingResult: + """Bind one successful stored-output control in its own transaction.""" + assert stored.verification_receipt_id is not None + async with self.factory() as session: + async with session.begin(): + return await self.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + selector, + stored.put_attempt_id, + stored.verification_receipt_id, + ) + ) + + async def binding_rows(self) -> list[tuple[str, str, str, str, str]]: + """Snapshot exact immutable binding identities for denial-side-effect proof.""" + async with self.factory() as session: + rows = await session.execute( + select( + ArtifactBinding.id, + ArtifactBinding.content_id, + ArtifactBinding.resource_id, + ArtifactBinding.put_attempt_id, + ArtifactBinding.verification_receipt_id, + ).order_by(ArtifactBinding.id) + ) + return [tuple(row) for row in rows] + + async def seed_reservation( + self, + *, + maximum_bytes: int = 1024, + ) -> CheckerOutputReservation: + """Create a second coherent lineage without changing the active test reservation.""" + async with self.factory() as session: + _, _, checker_run_id = await _seed_checker_output_relationships( + session, + self.namespace, + policy_bundle=self.policy_bundle, + ) + reservation = await make_checker_output_reservation( + session, + checker_run_id, + maximum_bytes=maximum_bytes, + ) + await session.rollback() + return reservation + + +@asynccontextmanager +async def output_custody_harness( + tmp_path: Path, + database_url: str, + *, + provider: str = "local", + maximum_bytes: int = 1024, +) -> AsyncIterator[OutputCustodyHarness]: + """Compose real Local/MinIO storage and controlled exact output authority.""" + engine = create_async_engine(database_url) + factory = async_sessionmaker(engine, expire_on_commit=False) + if provider == "minio": + from tests.test_s3_artifact_store import minio_settings + + settings = minio_settings(private_prefix=f"checker-output/{new_record_id()}").model_copy( + update={ + **artifact_admission_limit_settings(1024), + "artifact_scratch_root": tmp_path / "configured-scratch", + "artifact_scratch_minimum_free_bytes": 0, + } + ) + else: + settings = _settings(tmp_path, maximum_bytes=1024) + bootstrap = create_artifact_store_bootstrap(settings) + namespace = artifact_storage_namespace_spec(settings, bootstrap) + raw_store = bootstrap.initialize_after_namespace_claim( + ArtifactStoreNamespaceClaim( + bootstrap.identity, + bootstrap.namespace_identity, + namespace.namespace_fingerprint, + ) + ) + store = ObservedStore(raw_store) + manager = ArtifactScratchManager( + root=tmp_path / "output-scratch", + limits=artifact_preparation_limits(), + ) + preparation = ArtifactPreparationService(manager) + try: + policy_bundle = await create_standalone_unified_policy(factory, namespace) + async with factory() as session: + _, _, checker_run_id = await _seed_checker_output_relationships( + session, + namespace, + policy_bundle=policy_bundle, + ) + existing = ( + await session.execute( + select(ActorProfile.id, ActorIdentityLink.id) + .join(ActorIdentityLink, ActorIdentityLink.actor_profile_id == ActorProfile.id) + .where( + ActorProfile.service_identity + == ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value + ) + ) + ).one_or_none() + if existing is None: + actor_id, identity_link_id = new_record_id(), new_record_id() + _add_checker_output_actor( + session, + actor_id, + identity_link_id, + subject=f"checker-output-{actor_id}", + ) + await session.commit() + else: + actor_id, identity_link_id = (UUID(value) for value in existing) + await session.rollback() + reservation = await make_checker_output_reservation( + session, + checker_run_id, + maximum_bytes=maximum_bytes, + ) + await session.rollback() + state = ControlledReservationState(reservation) + service = CheckerArtifactOutputService( + sessions=factory, + reservations=lambda session: ControlledReservationPort(session, state), + authority=lambda session: ControlledWriteAuthority( + session, + state, + actor_id, + identity_link_id, + ), + internal_authority=lambda session: _AllowArtifactAuthority(), + preparation=preparation, + store=store, + namespace=namespace, + settings=settings, + ) + yield OutputCustodyHarness( + factory=factory, + engine=engine, + bootstrap=bootstrap, + store=store, + namespace=namespace, + settings=settings, + manager=manager, + preparation=preparation, + service=service, + state=state, + actor_id=actor_id, + identity_link_id=identity_link_id, + policy_bundle=policy_bundle, + ) + finally: + manager.close() + bootstrap.close() + await engine.dispose() diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 3174546a3..154eb4d6f 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -23,7 +23,7 @@ DDL_LOCK_DIRECTORY = Path("/tmp") # Match the PostgreSQL 16 engine used by Backend CI. Catalog identity rendering # differs across major versions; regenerate only after comparing actual objects. -EXPECTED_PUBLIC_SCHEMA_SHA256 = "48ad2cc4307b52524c1343311866a7d7e39ff6c9471f37001ea84192ffb86767" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "f7bbe6bcb1c6298f79f612d6627ca47900fa84d5df3c94addc37c94904861504" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", @@ -119,6 +119,8 @@ "workstream_tasks", ) TRUNCATE_GUARDED_TABLES = ( + "artifact_bindings", + "artifact_put_attempts", "checker_runs", "checker_results", "task_command_receipts", diff --git a/backend/tests/test_alembic.py b/backend/tests/test_alembic.py index 039e80129..8319cf4b0 100644 --- a/backend/tests/test_alembic.py +++ b/backend/tests/test_alembic.py @@ -79,7 +79,7 @@ def test_v01_graph_has_one_root_and_head() -> None: script = ScriptDirectory.from_config(config) revisions = list(script.walk_revisions()) - assert [revision.revision for revision in revisions] == [HEAD_REVISION, "0005_task_evidence_authority", "0004_task_context_authority", "0003_task_read_authority", "0002_task_queue_authority", BASELINE_REVISION] + assert [revision.revision for revision in revisions] == [HEAD_REVISION, "0006_history_read_authority", "0005_task_evidence_authority", "0004_task_context_authority", "0003_task_read_authority", "0002_task_queue_authority", BASELINE_REVISION] assert revisions[-1].down_revision is None assert script.get_heads() == [HEAD_REVISION] diff --git a/backend/tests/test_artifact_admission.py b/backend/tests/test_artifact_admission.py index b40687ce1..e8c2bce74 100644 --- a/backend/tests/test_artifact_admission.py +++ b/backend/tests/test_artifact_admission.py @@ -62,7 +62,6 @@ from app.modules.artifacts.schemas import ( ArtifactAuthorityDeniedError, ArtifactInternalResourceType, - CheckerOutputArtifactAdmissionRequest, GuideArtifactAdmissionRequest, ) from app.modules.artifacts.service import ( @@ -101,6 +100,13 @@ artifact_admission_limit_settings, minted_source, ) +from tests.checker_output_admission_helpers import ( + ControlledCheckerOutputAdmissionAuthority, + assert_checker_admission_unchanged, + assert_exact_checker_admission, + checker_admission_baseline, + make_checker_output_admission_request as _checker_output_request, +) class _AllowArtifactAuthority: @@ -521,15 +527,13 @@ async def _admit_checker_output(session, settings, namespace, source, *, policy_ session, actor_id, link_id, subject=f"checker-output-{actor_id}") else: actor_id, link_id = (UUID(value) for value in existing) - context = _context(actor_profile_id=actor_id, identity_link_id=link_id, actor_kind=ActorKind.SERVICE) await session.commit() + request = await _checker_output_request(session, checker_run_id, source) result = await ArtifactAdmissionService(session, settings, namespace).admit( - CheckerOutputArtifactAdmissionRequest( - authorization_context=context, - checker_run_id=UUID(checker_run_id), - logical_role="platform-review", - source=source, - ) + request, + checker_output_authority=ControlledCheckerOutputAdmissionAuthority( + session, actor_id, link_id + ), ) return project_id, task_id, checker_run_id, result @@ -2416,11 +2420,6 @@ async def test_checker_output_requires_exact_active_fixed_service_identity( namespace = _namespace(settings) actor_id = new_record_id() link_id = new_record_id() - context = _context( - actor_profile_id=actor_id, - identity_link_id=link_id, - actor_kind=ActorKind.SERVICE, - ) engine = create_async_engine(admission_database_env) factory = async_sessionmaker(engine, expire_on_commit=False) try: @@ -2439,75 +2438,61 @@ async def test_checker_output_requires_exact_active_fixed_service_identity( await session.commit() await session.rollback() - baseline = {model: await _count(session, model) for model in ( - ArtifactPutAttempt, ArtifactContent, ArtifactReplica, ArtifactOperationReceipt, - )} + baseline = await checker_admission_baseline(session) await session.rollback() async with minted_source(tmp_path / "scratch-source", b"checker") as source: service = ArtifactAdmissionService(session, settings, namespace) - forged = context.model_copy(update={"identity_link_id": new_record_id()}) + request = await _checker_output_request( + session, checker_run_id, source + ) with pytest.raises( - ArtifactAdmissionRelationshipError, + ArtifactAuthorityDeniedError, + match="admission authority is unavailable", + ): + await service.admit(request) + await assert_checker_admission_unchanged(session, baseline) + await session.rollback() + + with pytest.raises( + ArtifactAuthorityDeniedError, match="service identity is unavailable", ): await service.admit( - CheckerOutputArtifactAdmissionRequest( - authorization_context=forged, - checker_run_id=UUID(checker_run_id), - logical_role="platform-review", - source=source, - ) + request, + checker_output_authority=ControlledCheckerOutputAdmissionAuthority( + session, actor_id, new_record_id() + ), ) - assert await _count(session, ArtifactStorageNamespace) == 1 - assert await _count(session, ArtifactAdmissionScope) == 0 - assert await _count(session, ArtifactAdmissionCharge) == 0 - assert await _count(session, ArtifactPutAttempt) == baseline[ArtifactPutAttempt] + await assert_checker_admission_unchanged(session, baseline) await session.rollback() - - request = CheckerOutputArtifactAdmissionRequest( - authorization_context=context, - checker_run_id=UUID(checker_run_id), - logical_role="platform-review", - source=source, + authority = ControlledCheckerOutputAdmissionAuthority( + session, actor_id, link_id + ) + result = await service.admit( + request, checker_output_authority=authority + ) + replay = await service.admit( + request, checker_output_authority=authority + ) + takeover_request = replace( + request, + reservation=request.reservation.model_copy( + update={ + "worker_lease_id": new_record_id(), + "worker_lease_generation": 2, + } + ), + ) + takeover = await service.admit( + takeover_request, + checker_output_authority=authority, ) - result = await service.admit(request) - replay = await service.admit(request) - attempt = await session.get(ArtifactPutAttempt, str(result.attempt_id)) - scopes = (await session.scalars(select(ArtifactAdmissionScope).order_by( - ArtifactAdmissionScope.scope_type, ArtifactAdmissionScope.scope_id, - ))).all() - links = (await session.scalars(select(ArtifactPutAttemptCharge).where( - ArtifactPutAttemptCharge.attempt_id == str(result.attempt_id), - ))).all() - assert attempt is not None - assert replay.attempt_id == result.attempt_id - assert replay.charge_ids == result.charge_ids - assert attempt.status == "prepared" - assert attempt.producer_request_type == "checker_output" - assert attempt.producer_type == "service_identity" - assert attempt.producer_ref == ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value - assert attempt.project_id == project_id - assert attempt.task_id == task_id - assert attempt.checker_run_id == checker_run_id - assert attempt.logical_role == "platform-review" - assert attempt.executor_id is None - assert attempt.lease_expires_at is None - assert attempt.next_run_at is None - assert attempt.execution_generation == 0 - assert {scope.scope_type for scope in scopes} == { - "deployment", - "producer", - "project", - "task", - } - assert len(result.charge_ids) == 4 - assert len(links) == 4 - assert await _count(session, ArtifactPutAttempt) == baseline[ArtifactPutAttempt] + 1 - assert await _count(session, ArtifactAdmissionCharge) == 4 - assert await _count(session, ArtifactContent) == baseline[ArtifactContent] - assert await _count(session, ArtifactReplica) == baseline[ArtifactReplica] - assert await _count(session, ArtifactOperationReceipt) == baseline[ArtifactOperationReceipt] + await assert_exact_checker_admission( + session, result=result, replay=replay, takeover=takeover, + request=request, authority=authority, project_id=project_id, + task_id=task_id, checker_run_id=checker_run_id, baseline=baseline, + ) finally: await engine.dispose() @@ -2703,7 +2688,7 @@ async def test_checker_output_put_observation_terminal_outcomes( await engine.dispose() -async def test_invalid_checker_role_precedes_namespace_drift( +async def test_invalid_checker_slot_precedes_namespace_drift( admission_database_env: str, tmp_path: Path, ) -> None: @@ -2713,34 +2698,32 @@ async def test_invalid_checker_role_precedes_namespace_drift( factory = async_sessionmaker(engine, expire_on_commit=False) try: async with factory() as session: - session.add( - ArtifactStorageNamespace( - id="primary", - backend=namespace.backend, - adapter=namespace.adapter, - provider_profile=namespace.provider_profile, - namespace_descriptor=namespace.namespace_descriptor, - namespace_fingerprint="sha256:" + "f" * 64, - ) + project_id, task_id, checker_run_id = await _seed_checker_output_relationships( + session, namespace + ) + del project_id, task_id + baseline = await checker_admission_baseline(session) + await session.rollback() + drifted_namespace = replace( + namespace, + namespace_fingerprint="sha256:" + "f" * 64, ) - await session.commit() async with minted_source(tmp_path / "scratch-source", b"checker") as source: + valid = await _checker_output_request( + session, checker_run_id, source + ) + invalid = replace(valid, slot_key="missing-slot") with pytest.raises( ArtifactAdmissionRelationshipError, - match="logical role is invalid", + match="reservation is unavailable", ): - await ArtifactAdmissionService(session, settings, namespace).admit( - CheckerOutputArtifactAdmissionRequest( - authorization_context=_context(actor_kind=ActorKind.SERVICE), - checker_run_id=new_record_id(), - logical_role="é" * 100, - source=source, - ) + await ArtifactAdmissionService( + session, settings, drifted_namespace + ).admit( + invalid, + checker_output_authority=AsyncMock(), ) - assert await _count(session, ArtifactStorageNamespace) == 1 - assert await _count(session, ArtifactAdmissionScope) == 0 - assert await _count(session, ArtifactAdmissionCharge) == 0 - assert await _count(session, ArtifactPutAttempt) == 0 + await assert_checker_admission_unchanged(session, baseline) finally: await engine.dispose() diff --git a/backend/tests/test_artifact_admission_digest.py b/backend/tests/test_artifact_admission_digest.py new file mode 100644 index 000000000..5716d580f --- /dev/null +++ b/backend/tests/test_artifact_admission_digest.py @@ -0,0 +1,127 @@ +"""Producer-scoped request-digest replay through the ART admission owner.""" + +from dataclasses import replace +from pathlib import Path + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.hashing import canonical_json_hash +from app.modules.artifacts.models import ArtifactPutAttempt +from app.modules.artifacts.repository import ArtifactRepository +from app.modules.artifacts.service import ArtifactAdmissionService +from tests.artifact_store_helpers import artifact_preparation_limits, minted_source +from tests.test_artifact_admission import ( + _AllowGuidePreparedAuthorization, + _context, + _guide_admission_request, + _namespace, + _seed_guide, + _settings, +) + + +async def test_guide_replay_preserves_canonical_producer_digest( + isolated_database_env: str, + tmp_path: Path, +) -> None: + """Guide producer digest shape stays exact through admission and replay.""" + settings = _settings(tmp_path) + namespace = _namespace(settings) + engine = create_async_engine(isolated_database_env) + factory = async_sessionmaker(engine, expire_on_commit=False) + try: + async with factory() as session: + context = _context() + scratch_limits = replace( + artifact_preparation_limits(), + reservation_ttl_seconds=360, + total_deadline_seconds=300, + ) + async with minted_source( + tmp_path / "guide-digest", + b"retained guide bytes", + media_type="application/pdf", + limits=scratch_limits, + ) as source: + project_id, item_id = await _seed_guide( + session, + context=context, + content_hash="sha256:" + "f" * 64, + media_type=source.commitment.media_type, + ) + lineage = await ArtifactRepository(session).get_guide_lineage(item_id) + assert lineage is not None + await session.rollback() + request = _guide_admission_request(item_id, source, lineage=lineage) + first_authority = _AllowGuidePreparedAuthorization( + context.actor_profile_id + ) + first = await ArtifactAdmissionService( + session, settings, namespace + ).admit( + request, + guide_prepared_authorization=first_authority, # type: ignore[arg-type] + prepared_authorization=first_authority.handle, + ) + assert first.replayed is False + + canonical_guide_digest = canonical_json_hash( + { + "operation_identity": request.operation_identity, + "guide_ingest_request_digest": request.request_digest, + "request_type": "guide", + "producer_type": "actor_profile", + "producer_ref": str(context.actor_profile_id), + "project_id": project_id, + "task_id": None, + "guide_source_item_id": item_id, + "checker_run_id": None, + "logical_role": None, + "pre_submit_evidence_set_id": None, + "sha256": source.commitment.sha256, + "byte_count": source.commitment.byte_count, + "media_type": source.commitment.media_type, + "namespace_fingerprint": namespace.namespace_fingerprint, + "scopes": [ + { + "scope_type": "deployment", + "scope_id": "primary", + "limit_bytes": settings.artifact_admission_deployment_maximum_bytes, + }, + { + "scope_type": "producer", + "scope_id": f"actor_profile:{context.actor_profile_id}", + "limit_bytes": settings.artifact_admission_producer_maximum_bytes, + }, + { + "scope_type": "project", + "scope_id": project_id, + "limit_bytes": settings.artifact_admission_project_maximum_bytes, + }, + ], + } + ) + attempt = await session.get(ArtifactPutAttempt, str(first.attempt_id)) + assert attempt is not None + assert first.request_digest == canonical_guide_digest + await session.rollback() + + replay_authority = _AllowGuidePreparedAuthorization( + context.actor_profile_id + ) + replay = await ArtifactAdmissionService( + session, settings, namespace + ).admit( + request, + guide_prepared_authorization=replay_authority, # type: ignore[arg-type] + prepared_authorization=replay_authority.handle, + ) + assert replay.replayed is True + assert replay.attempt_id == first.attempt_id + assert replay.request_digest == canonical_guide_digest + assert await session.scalar( + select(func.count()).select_from(ArtifactPutAttempt) + ) == 1 + finally: + await engine.dispose() diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index d97128560..cf25148f5 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -20,6 +20,15 @@ APP_ROOT / "modules" / "artifacts" / "api" / "submission_preparation.py" ) SUBMISSION_ADMISSION_API = APP_ROOT / "modules" / "artifacts" / "api" / "submission_admission.py" +CHECKER_OUTPUT_CUSTODY_API = ( + APP_ROOT / "modules" / "checkers" / "api" / "output_custody.py" +) +CHECKER_MATERIALIZATION_API = ( + APP_ROOT / "modules" / "checkers" / "api" / "materialization.py" +) +RETIRED_ARTIFACT_MATERIALIZATION_API = ( + APP_ROOT / "modules" / "artifacts" / "api" / "submission_materialization.py" +) COMPOSITION_ROOT = APP_ROOT / "adapters" / "artifacts" / "__init__.py" AGENT_COMPOSITION_ROOT = APP_ROOT / "adapters/project_agents/__init__.py" AGENT_ADAPTER_MODULE = "app.adapters.project_agents.openai_agent_sdk" @@ -27,15 +36,11 @@ CLOSED_PORTS = { "GuideArtifactIngestCommand", "GuideArtifactIngestPort", - "ArtifactBindingPort", - "CheckerArtifactOutputPort", "ArtifactOperatorReadPort", "ArtifactOperatorRecoveryPort", } CANONICAL_REQUESTS = { "GuideArtifactIngestRequest", - "CheckerOutputBindingRequest", - "CheckerOutputArtifactRequest", "ArtifactRecoveryRequest", } CANONICAL_RESULTS = { @@ -46,9 +51,7 @@ "ArtifactBindingResourceType", } CANONICAL_VALUE_TYPES = set() -PREPARED_MUTATION_REQUESTS = CANONICAL_REQUESTS - { - "ArtifactRecoveryRequest", -} +PREPARED_MUTATION_REQUESTS = {"GuideArtifactIngestRequest"} PREPARED_HANDLE_FORBIDDEN_ROOTS = ( APP_ROOT / "adapters", APP_ROOT / "api", @@ -469,7 +472,7 @@ def test_artifact_operations_exports_only_canonical_closed_contracts() -> None: } -def test_durable_artifact_mutation_ports_require_process_local_prepared_authority() -> None: +def test_artifact_ports_keep_prepared_authority_at_the_transaction_boundary() -> None: tree = _tree(ARTIFACT_OPERATIONS) request_classes = { node.name: node @@ -498,15 +501,9 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit expected_methods = { "GuideArtifactIngestPort": {"ingest"}, - "ArtifactBindingPort": { - "bind_checker_output", - }, - "CheckerArtifactOutputPort": {"store"}, } expected_request_by_method = { "ingest": "GuideArtifactIngestRequest", - "bind_checker_output": "CheckerOutputBindingRequest", - "store": "CheckerOutputArtifactRequest", } protocols = { node.name: node @@ -524,7 +521,7 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): continue assert node.args.posonlyargs == [] - assert [argument.arg for argument in node.args.args] == ["self", "request"] + assert [argument.arg for argument in node.args.args] == ["self", "selector" if node.name == "recover" else "request"] assert node.args.kwonlyargs == [] assert node.args.vararg is None assert node.args.kwarg is None @@ -535,6 +532,116 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit assert "AuthorizationContext" not in _declared_annotation_names(node) +def test_checker_output_contract_has_one_canonical_consumer_owner() -> None: + """CHECKERS owns the ports that ART implements for future durable execution.""" + tree = _tree(CHECKER_OUTPUT_CUSTODY_API) + classes = { + node.name: node for node in tree.body if isinstance(node, ast.ClassDef) + } + assert { + "CheckerOutputArtifactRequest", + "CheckerOutputArtifactResult", + "CheckerOutputBindingRequest", + "CheckerOutputBindingResult", + "CheckerOutputBindingPort", + "CheckerArtifactOutputPort", + } <= set(classes) + assert "ArtifactBindingPort" not in classes + for name in ("CheckerOutputArtifactRequest", "CheckerOutputBindingRequest"): + annotations = _declared_annotation_names(classes[name]) + assert {"PreparedAuthorizationHandle", "AuthorizationContext"}.isdisjoint( + annotations + ) + assert "CheckerOutputSelector" in annotations + + expected_methods = { + "CheckerOutputBindingPort": {"bind_checker_output"}, + "CheckerArtifactOutputPort": {"store", "recover"}, + } + for name, methods in expected_methods.items(): + assert { + item.name + for item in classes[name].body + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)) + } == methods + + artifact_source = ARTIFACT_OPERATIONS.read_text(encoding="utf-8") + assert "app.modules.checkers" not in artifact_source + assert not RETIRED_ARTIFACT_MATERIALIZATION_API.exists() + assert "ArtifactBindingPort" not in "\n".join( + path.read_text(encoding="utf-8") for path in _python_files(APP_ROOT) + ) + + +def test_checker_materialization_contract_has_one_canonical_consumer_owner() -> None: + """The complete material callback contract resides in CHECKERS public API.""" + tree = _tree(CHECKER_MATERIALIZATION_API) + classes = { + node.name: node for node in tree.body if isinstance(node, ast.ClassDef) + } + assert set(classes) == { + "PostSubmissionMaterializationUnavailable", + "SubmissionMaterialEntry", + "SubmissionMaterialView", + "PostSubmissionMaterialConsumer", + "PostSubmissionMaterializationResult", + "PostSubmissionMaterializationPort", + } + assert { + node.name + for node in classes["PostSubmissionMaterializationPort"].body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } == {"materialize"} + artifact_api = APP_ROOT / "modules" / "artifacts" / "api" / "__init__.py" + assert "Materialization" not in artifact_api.read_text(encoding="utf-8") + + +def test_art_implementations_are_the_exact_checker_contract_consumers() -> None: + """Prevent a reverse dependency or an extra consumer of CHECKERS custody ports.""" + expected = { + "app.modules.checkers.api.output_custody": { + "app/adapters/artifacts/__init__.py", + "app/modules/artifacts/checker_output_bindings.py", + "app/modules/artifacts/checker_output_custody.py", + "app/modules/artifacts/checker_outputs.py", + "app/modules/artifacts/schemas.py", + "app/modules/artifacts/service.py", + }, + "app.modules.checkers.api.materialization": { + "app/adapters/artifacts/__init__.py", + "app/modules/artifacts/post_submit_materialization.py", + "app/modules/artifacts/post_submit_selection.py", + }, + } + actual = {module: set() for module in expected} + for path in _python_files(APP_ROOT): + modules = { + node.module + for node in ast.walk(_tree(path)) + if isinstance(node, ast.ImportFrom) and node.module in expected + } + for module in modules: + actual[module].add(path.relative_to(BACKEND_ROOT).as_posix()) + assert actual == expected + + composition = _tree(COMPOSITION_ROOT) + returns = { + node.name: _annotation_names(node.returns) + for node in composition.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + and node.name in { + "post_submission_materialization", + "checker_output_storage", + "checker_output_binding", + } + } + assert returns == { + "post_submission_materialization": {"PostSubmissionMaterializationPort"}, + "checker_output_storage": {"CheckerArtifactOutputPort"}, + "checker_output_binding": {"CheckerOutputBindingPort"}, + } + + def test_submission_preparation_http_request_never_carries_prepared_authority() -> None: tree = _tree(SUBMISSION_PREPARATION_API) request_class = next( diff --git a/backend/tests/test_artifact_authorization.py b/backend/tests/test_artifact_authorization.py index 01b722abd..390248719 100644 --- a/backend/tests/test_artifact_authorization.py +++ b/backend/tests/test_artifact_authorization.py @@ -148,7 +148,9 @@ def test_admission_metrics_are_bounded_and_classified() -> None: def test_readiness_is_static_and_aws_never_active() -> None: - disabled = artifact_provider_readiness(Settings()) + disabled = artifact_provider_readiness( + Settings.model_construct(artifact_store_backend="disabled") + ) assert disabled["status"] == "inactive_disabled" assert disabled["active"] is False @@ -192,6 +194,10 @@ async def test_quota_reconciliation_is_configuration_driven_and_rollback_safe() guide_source_snapshot_id=None, checker_run_id=None, logical_role=None, + submission_id=None, + submission_version=None, + checker_request_digest=None, + checker_request_digest_facts=None, pre_submit_evidence_set_id=None, operation_identity="sha256:" + "a" * 64, ) diff --git a/backend/tests/test_artifact_preparation.py b/backend/tests/test_artifact_preparation.py index d17c73767..2098c7a42 100644 --- a/backend/tests/test_artifact_preparation.py +++ b/backend/tests/test_artifact_preparation.py @@ -365,13 +365,46 @@ async def future_provider_call(_: CommittedArtifactSource) -> None: @pytest.mark.asyncio -async def test_preparation_rejects_oversize_and_non_byte_sources(tmp_path: Path) -> None: - """Enforce the source ceiling and byte-only stream contract.""" - limits = preparation_limits(maximum_source_bytes=4) +async def test_preparation_enforces_source_caps_and_byte_only_streams( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Enforce manager and per-call ceilings before excess bytes reach scratch.""" + limits = preparation_limits(maximum_source_bytes=8) manager = ArtifactScratchManager(root=tmp_path / "scratch", limits=limits) service = ArtifactPreparationService(manager) + prepared = await service.prepare( + byte_stream(b"ab", b"cd"), + media_type="text/plain", + maximum_bytes=4, + ) + assert prepared.commitment.byte_count == 4 + await prepared.close() with pytest.raises(ArtifactLimitExceededError): - await service.prepare(byte_stream(b"abcde"), media_type="text/plain") + await service.prepare(byte_stream(b"123456789"), media_type="text/plain") + + written = bytearray() + yielded: list[bytes] = [] + original_write_chunk = service._write_chunk + + async def track_write(descriptor: int, chunk: memoryview) -> None: + written.extend(chunk) + await original_write_chunk(descriptor, chunk) + + async def over_cap_stream() -> AsyncIterator[bytes]: + for chunk in (b"abcd", b"e", b"must-not-be-read"): + yielded.append(chunk) + yield chunk + + monkeypatch.setattr(service, "_write_chunk", track_write) + with pytest.raises(ArtifactLimitExceededError): + await service.prepare( + over_cap_stream(), + media_type="text/plain", + maximum_bytes=4, + ) + assert written == b"abcd" + assert yielded == [b"abcd", b"e"] async def invalid_stream() -> AsyncIterator[bytes]: yield "not-bytes" # type: ignore[misc] @@ -382,6 +415,36 @@ async def invalid_stream() -> AsyncIterator[bytes]: manager.close() +@pytest.mark.asyncio +@pytest.mark.parametrize("maximum_bytes", [True, 0, -1, 65]) +async def test_preparation_rejects_invalid_per_call_caps_before_iteration( + tmp_path: Path, + maximum_bytes: object, +) -> None: + """Reject malformed or manager-exceeding per-call limits before reserving scratch.""" + manager = ArtifactScratchManager( + root=tmp_path / str(maximum_bytes), + limits=preparation_limits(maximum_source_bytes=64), + ) + iterated = False + + async def tracked_stream() -> AsyncIterator[bytes]: + nonlocal iterated + iterated = True + yield b"data" + + with pytest.raises(ValueError, match="per-call byte limit"): + await ArtifactPreparationService(manager).prepare( + tracked_stream(), + media_type="text/plain", + maximum_bytes=maximum_bytes, # type: ignore[arg-type] + ) + + assert not iterated + assert (await manager.usage()).reservation_count == 0 + manager.close() + + @pytest.mark.asyncio async def test_cross_process_ledger_prevents_concurrent_oversubscription( tmp_path: Path, diff --git a/backend/tests/test_artifact_recovery.py b/backend/tests/test_artifact_recovery.py index 99ff22070..e701b3d2a 100644 --- a/backend/tests/test_artifact_recovery.py +++ b/backend/tests/test_artifact_recovery.py @@ -27,9 +27,11 @@ ArtifactStoreUnavailableError, ) from app.modules.artifacts.models import ( + ArtifactPutAttempt, ArtifactRecoveryAttempt, ArtifactVerificationJob, ) +from app.modules.artifacts.repository import ArtifactRepository from app.modules.artifacts.schemas import ( ArtifactRecoveryAuthorizationEvidence, ArtifactRecoveryConflictError, @@ -42,7 +44,6 @@ ArtifactStorageOrchestrator, artifact_storage_namespace_spec, ) -from app.modules.checkers.models import CheckerRun from app.modules.actors.models import ActorIdentityLink, ActorProfile from app.modules.authorization.runtime import ( ActorKind, @@ -186,7 +187,7 @@ async def _exhausted_job(session, settings, tmp_path, context): async with minted_source( tmp_path / "checker-output", b"recover checker output", limits=limits, ) as source: - project_id, task_id, checker_run_id, admission = await _admit_checker_output( + project_id, task_id, _checker_run_id, admission = await _admit_checker_output( session, settings, namespace, source, policy_bundle=policy_bundle) await _seed_recovery_actor(session, context) await session.commit() @@ -209,14 +210,19 @@ async def _exhausted_job(session, settings, tmp_path, context): await orchestrator.verify_object(UUID(job_id)) job = await session.get(ArtifactVerificationJob, job_id) assert job is not None - checker_run = await session.get(CheckerRun, checker_run_id) - assert checker_run is not None - submission_id = checker_run.submission_id + attempt = await session.get(ArtifactPutAttempt, str(admission.attempt_id)) + assert attempt is not None + submission_id = attempt.submission_id + assert submission_id is not None await session.refresh(job) await session.commit() return project_id, task_id, submission_id, job, orchestrator, bootstrap +def test_recovery_repository_has_no_checker_run_lookup() -> None: + assert not hasattr(ArtifactRepository, "lock_checker_run") + + def _request( context: HumanAuthorizationContext, project_id: str, diff --git a/backend/tests/test_behavior_ownership.py b/backend/tests/test_behavior_ownership.py index df94a0f42..7b39d2611 100644 --- a/backend/tests/test_behavior_ownership.py +++ b/backend/tests/test_behavior_ownership.py @@ -2124,18 +2124,47 @@ def test_approved_guide_intake_removal_is_exact(): def test_post_submit_materialization_partition_additions_are_exact(): targets = { - "backend/app/modules/artifacts/api/submission_materialization.py", + "backend/app/modules/checkers/api/materialization.py", "backend/app/modules/artifacts/post_submit_materialization.py", "backend/app/modules/artifacts/post_submit_selection.py", "backend/app/modules/tasks/api/submitted_bundle.py", "backend/app/modules/tasks/submitted_bundle.py", } assert ownership.ARCH_04B_MATERIALIZATION_TARGETS == targets + assert ownership.ARCH_04B_SEAM_REMOVED_TARGETS == { + "backend/app/modules/artifacts/api/submission_materialization.py" + } retained = "backend/app/core/config.py" + retired = "backend/app/modules/artifacts/api/submission_materialization.py" trusted = _partition([retained]) ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets])), trusted) + prior_targets = (targets - {"backend/app/modules/checkers/api/materialization.py"}) | { + retired + } + ownership._validate_additive_partition_transition( + _partition(sorted([retained, *targets])), + _partition(sorted([retained, *prior_targets])), + ) for forbidden in ("backend/app/modules/artifacts/download.py", "backend/app/modules/tasks/other_material.py"): with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets, forbidden])), trusted) with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): ownership._validate_additive_partition_transition(_partition(sorted(targets)), trusted) + + +def test_checker_output_partition_additions_are_exact(): + targets = { + "backend/app/modules/artifacts/checker_outputs.py", + "backend/app/modules/artifacts/checker_output_custody.py", + "backend/app/modules/artifacts/checker_output_bindings.py", + "backend/app/modules/checkers/api/output_custody.py", + } + assert ownership.ARCH_04B2_OUTPUT_TARGETS == targets + retained = "backend/app/core/config.py" + trusted = _partition([retained]) + ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets])), trusted) + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition( + _partition(sorted([retained, *targets, "backend/app/modules/artifacts/arbitrary_output.py"])), trusted) + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition(_partition(sorted(targets)), trusted) diff --git a/backend/tests/test_checker_output_custody.py b/backend/tests/test_checker_output_custody.py new file mode 100644 index 000000000..c6ec4d3a4 --- /dev/null +++ b/backend/tests/test_checker_output_custody.py @@ -0,0 +1,627 @@ +"""Hidden end-to-end proof for exact verified checker-output custody.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace +from uuid import UUID + +import pytest +from sqlalchemy import func, select + +from app.adapters.artifacts import checker_output_binding, checker_output_storage +from app.core.identifiers import new_record_id +from app.interfaces.artifacts import ArtifactLimitExceededError +from app.modules.actors.api import ServiceIdentity +from app.modules.artifacts.models import ( + ArtifactBinding, + ArtifactOperationReceipt, + ArtifactPutObservationReceipt, + ArtifactPutAttempt, + ArtifactVerificationJob, + ArtifactVerificationReceipt, +) +from app.modules.artifacts.service import ArtifactAdmissionConflictError +from app.modules.checkers.api.output_custody import ( + CheckerOutputArtifactRequest, + CheckerOutputBindingRequest, + CheckerOutputSelector, + CheckerOutputUnavailable, +) +from app.modules.checkers.models import CheckerRun +from tests.checker_output_custody_helpers import ( + byte_stream, + output_custody_harness, + selector_for, +) +from tests.checkers.post_submit.support import request as post_submit_request + + +async def test_default_composition_denies_before_owner_source_or_storage() -> None: + """Keep both production output capabilities unavailable before protected access.""" + + class Forbidden: + def __getattr__(self, name): + pytest.fail(f"protected access: {name}") + + @asynccontextmanager + async def session_scope(): + yield object() + + iterated = False + + async def source(): + nonlocal iterated + iterated = True + yield b"must-not-be-read" + + selector = CheckerOutputSelector( + evaluation=post_submit_request(), + checker_run_id=new_record_id(), + worker_lease_id=new_record_id(), + worker_lease_generation=1, + slot_key="platform-review", + ) + forbidden = Forbidden() + storage = checker_output_storage( + sessions=session_scope, + store=forbidden, + namespace=forbidden, + preparation=forbidden, + settings=forbidden, + ) + with pytest.raises(CheckerOutputUnavailable, match="write_unavailable"): + await storage.store(CheckerOutputArtifactRequest(selector, source())) + assert not iterated + + class Transaction: + def in_transaction(self): + return True + + def in_nested_transaction(self): + return False + + binding = checker_output_binding( + Transaction(), + namespace=SimpleNamespace(namespace_fingerprint="sha256:" + "0" * 64), + ) + with pytest.raises(CheckerOutputUnavailable, match="binding_unavailable"): + await binding.bind_checker_output( + CheckerOutputBindingRequest(selector, new_record_id(), new_record_id()) + ) + + +@pytest.mark.parametrize("provider", ["local", "minio"]) +async def test_real_store_verification_and_binding( + tmp_path, + isolated_database_env, + provider, +) -> None: + """Store, independently verify and bind one exact output through real providers.""" + if provider == "minio": + from tests.test_s3_artifact_store import provision_minio_bucket + + await provision_minio_bucket.__wrapped__() + async with output_custody_harness( + tmp_path, + isolated_database_env, + provider=provider, + ) as harness: + result = await harness.service.store(harness.request(b"verified checker output")) + assert result.status == "verified" + assert result.content_id is not None + assert result.verification_receipt_id is not None + assert not result.replayed + assert harness.store.puts == 1 + assert harness.store.opens == 1 + + async with harness.factory() as session: + async with session.begin(): + bound = await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + harness.selector, + result.put_attempt_id, + result.verification_receipt_id, + ) + ) + row = await session.get(ArtifactBinding, str(bound.binding_id)) + assert row is not None + assert ( + row.resource_type, + row.resource_id, + row.logical_role, + row.actor_id, + row.attribution_type, + row.put_attempt_id, + row.verification_receipt_id, + ) == ( + "checker_run", + str(harness.selector.checker_run_id), + harness.selector.slot_key, + ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + "service_identity", + str(result.put_attempt_id), + str(result.verification_receipt_id), + ) + assert harness.state.admissions + assert harness.state.recoveries + assert harness.state.bindings + + +async def test_identical_store_replay_reuses_verified_custody_without_provider_work( + tmp_path, + isolated_database_env, +) -> None: + """Replay exact bytes through the stable attempt without another put or verify.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + first = await harness.service.store(harness.request(b"stable replay output")) + assert first.status == "verified" + assert first.content_id is not None + assert first.verification_receipt_id is not None + assert first.replayed is False + provider_counts = (harness.store.puts, harness.store.opens) + assert provider_counts == (1, 1) + bound = await harness.bind(harness.selector, first) + assert ( + bound.content_id, + bound.put_attempt_id, + bound.verification_receipt_id, + bound.replayed, + ) == ( + first.content_id, + first.put_attempt_id, + first.verification_receipt_id, + False, + ) + + replay = await harness.service.store(harness.request(b"stable replay output")) + + assert ( + replay.put_attempt_id, + replay.content_id, + replay.verification_receipt_id, + replay.status, + replay.replayed, + ) == ( + first.put_attempt_id, + first.content_id, + first.verification_receipt_id, + "verified", + True, + ) + assert (harness.store.puts, harness.store.opens) == provider_counts + assert (await harness.manager.usage()).reservation_count == 0 + assert list((tmp_path / "output-scratch" / "files").iterdir()) == [] + async with harness.factory() as session: + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactPutAttempt) + .where( + ArtifactPutAttempt.checker_run_id == str(harness.selector.checker_run_id) + ) + ) + == 1 + ) + assert ( + await session.get( + ArtifactVerificationReceipt, + str(first.verification_receipt_id), + ) + is not None + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactVerificationJob) + .where( + ArtifactVerificationJob.originating_put_attempt_id + == str(first.put_attempt_id) + ) + ) + == 1 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactOperationReceipt) + .where(ArtifactOperationReceipt.put_attempt_id == str(first.put_attempt_id)) + ) + == 1 + ) + + +async def test_foreign_lineage_and_changed_bytes_fail_before_provider( + tmp_path, + isolated_database_env, +) -> None: + """Reject coherent foreign ownership before bytes and immutable replay drift before put.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + original_reservation = harness.state.reservation + original_selector = harness.selector + foreign = await harness.seed_reservation() + foreign_selector = selector_for(foreign) + + original = await harness.service.store( + harness.request(b"stable output", selector=original_selector) + ) + assert original.status == "verified" + harness.state.reservation = foreign + foreign_result = await harness.service.store( + harness.request(b"foreign control", selector=foreign_selector) + ) + assert foreign_result.status == "verified" + assert foreign_result.verification_receipt_id is not None + assert harness.store.puts == 2 + + harness.state.reservation = original_reservation + for wrong_selector in ( + foreign_selector, + original_selector.model_copy(update={"evaluation": foreign.evaluation}), + ): + observed: list[bytes] = [] + with pytest.raises(CheckerOutputUnavailable, match="reservation_unavailable"): + await harness.service.store( + harness.request( + b"mismatched bytes", + selector=wrong_selector, + observed=observed, + ) + ) + assert observed == [] + assert harness.store.puts == 2 + assert (await harness.manager.usage()).reservation_count == 0 + + with pytest.raises(ArtifactAdmissionConflictError): + await harness.service.store( + harness.request(b"changed output", selector=original_selector) + ) + assert harness.store.puts == 2 + assert (await harness.manager.usage()).reservation_count == 0 + + +async def test_binding_rejects_mixed_stored_lineage_before_consumption_or_mutation( + tmp_path, + isolated_database_env, +) -> None: + """Bind two valid controls, then reject every original/foreign custody mix.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + original_reservation = harness.state.reservation + original_selector = harness.selector + foreign_reservation = await harness.seed_reservation() + foreign_selector = selector_for(foreign_reservation) + + original = await harness.service.store(harness.request(b"original binding control")) + harness.state.reservation = foreign_reservation + foreign = await harness.service.store( + harness.request(b"foreign binding control", selector=foreign_selector) + ) + assert original.content_id is not None and original.verification_receipt_id is not None + assert foreign.content_id is not None and foreign.verification_receipt_id is not None + + harness.state.reservation = original_reservation + original_bound = await harness.bind(original_selector, original) + harness.state.reservation = foreign_reservation + foreign_bound = await harness.bind(foreign_selector, foreign) + assert ( + original_bound.content_id, + original_bound.put_attempt_id, + original_bound.verification_receipt_id, + original_bound.replayed, + ) == ( + original.content_id, + original.put_attempt_id, + original.verification_receipt_id, + False, + ) + assert ( + foreign_bound.content_id, + foreign_bound.put_attempt_id, + foreign_bound.verification_receipt_id, + foreign_bound.replayed, + ) == ( + foreign.content_id, + foreign.put_attempt_id, + foreign.verification_receipt_id, + False, + ) + + binding_baseline = await harness.binding_rows() + assert {UUID(row[0]) for row in binding_baseline} == { + original_bound.binding_id, + foreign_bound.binding_id, + } + consumed_baseline = len(harness.state.bindings) + assert consumed_baseline == 2 + + harness.state.reservation = original_reservation + for put_attempt_id, receipt_id, failure in ( + (foreign.put_attempt_id, original.verification_receipt_id, "identity_mismatch"), + (original.put_attempt_id, foreign.verification_receipt_id, "verification_unavailable"), + (foreign.put_attempt_id, foreign.verification_receipt_id, "identity_mismatch"), + ): + async with harness.factory() as session: + with pytest.raises(CheckerOutputUnavailable, match=failure): + async with session.begin(): + await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + original_selector, + put_attempt_id, + receipt_id, + ) + ) + assert len(harness.state.bindings) == consumed_baseline + assert await harness.binding_rows() == binding_baseline + + +async def test_binding_rollback_and_concurrent_replay_are_atomic( + tmp_path, + isolated_database_env, +) -> None: + """Keep caller rollback empty and serialize concurrent binding to one row.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + stored = await harness.service.store(harness.request(b"binding output")) + assert stored.verification_receipt_id is not None + request = CheckerOutputBindingRequest( + harness.selector, + stored.put_attempt_id, + stored.verification_receipt_id, + ) + + async with harness.factory() as session: + transaction = await session.begin() + rolled_back = await harness.binding_service(session).bind_checker_output(request) + await transaction.rollback() + async with harness.factory() as session: + assert await session.get(ArtifactBinding, str(rolled_back.binding_id)) is None + + async def bind_once(): + async with harness.factory() as session: + async with session.begin(): + return await harness.binding_service(session).bind_checker_output(request) + + first, second = await asyncio.gather(bind_once(), bind_once()) + assert first.binding_id == second.binding_id + assert sorted((first.replayed, second.replayed)) == [False, True] + async with harness.factory() as session: + count = await session.scalar(select(func.count()).select_from(ArtifactBinding)) + assert count == 1 + + +async def test_unknown_put_is_observed_recovered_and_bound_without_bytes( + tmp_path, + isolated_database_env, +) -> None: + """Recover lost acknowledgement through typed observation evidence and bind it.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + harness.store.lose_put_acknowledgement = True + unknown = await harness.service.store(harness.request(b"observed output")) + assert unknown.status == "acknowledgement_unknown" + assert unknown.content_id is None + assert unknown.verification_receipt_id is None + assert harness.store.puts == 1 + + recovered = await harness.service.recover(harness.selector) + assert recovered is not None + assert recovered.put_attempt_id == unknown.put_attempt_id + assert recovered.status == "verified" + assert recovered.content_id is not None + assert recovered.verification_receipt_id is not None + assert recovered.replayed + assert harness.store.puts == 1 + + async with harness.factory() as session: + attempt = await session.get(ArtifactPutAttempt, str(recovered.put_attempt_id)) + observation = await session.scalar( + select(ArtifactPutObservationReceipt).where( + ArtifactPutObservationReceipt.put_attempt_id == str(recovered.put_attempt_id) + ) + ) + direct_receipt = await session.scalar( + select(ArtifactOperationReceipt).where( + ArtifactOperationReceipt.put_attempt_id == str(recovered.put_attempt_id) + ) + ) + assert attempt is not None and attempt.receipt_id is None + assert observation is not None and observation.outcome == "observed_confirmed" + assert direct_receipt is None + await session.rollback() + async with session.begin(): + bound = await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + harness.selector, + recovered.put_attempt_id, + recovered.verification_receipt_id, + ) + ) + assert bound.content_id == recovered.content_id + + +async def test_worker_takeover_recovers_same_attempt_without_bytes( + tmp_path, + isolated_database_env, +) -> None: + """Fence the old lease and recover exact stored custody under its replacement.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + old_selector = harness.selector + stored = await harness.service.store(harness.request(b"takeover output")) + puts = harness.store.puts + harness.state.reservation = harness.state.reservation.model_copy( + update={ + "worker_lease_id": new_record_id(), + "worker_lease_generation": old_selector.worker_lease_generation + 1, + } + ) + + with pytest.raises(CheckerOutputUnavailable, match="reservation_unavailable"): + await harness.service.recover(old_selector) + recovered = await harness.service.recover(harness.selector) + assert recovered is not None + assert recovered.put_attempt_id == stored.put_attempt_id + assert recovered.verification_receipt_id == stored.verification_receipt_id + assert recovered.replayed + assert harness.store.puts == puts + + +async def test_authority_revocation_during_put_prevents_return_and_binding( + tmp_path, + isolated_database_env, +) -> None: + """Recheck authority after provider I/O and deny publication after revocation.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + harness.store.put_entered = asyncio.Event() + harness.store.put_release = asyncio.Event() + operation = asyncio.create_task(harness.service.store(harness.request(b"revoked output"))) + await asyncio.wait_for(harness.store.put_entered.wait(), timeout=10) + async with harness.factory() as probe, probe.begin(): + locked_run = await probe.scalar( + select(CheckerRun) + .where(CheckerRun.id == str(harness.selector.checker_run_id)) + .with_for_update(nowait=True) + ) + locked_attempt = await probe.scalar( + select(ArtifactPutAttempt) + .where(ArtifactPutAttempt.checker_run_id == str(harness.selector.checker_run_id)) + .with_for_update(nowait=True) + ) + assert locked_run is not None and locked_attempt is not None + harness.state.revoked = True + harness.store.put_release.set() + with pytest.raises(CheckerOutputUnavailable, match="authority_revoked"): + await operation + + async with harness.factory() as session: + attempt = await session.scalar(select(ArtifactPutAttempt)) + receipt = await session.scalar(select(ArtifactVerificationReceipt)) + assert attempt is not None and receipt is not None + attempt_id, receipt_id = UUID(attempt.id), UUID(receipt.id) + await session.rollback() + with pytest.raises(CheckerOutputUnavailable, match="authority_revoked"): + async with session.begin(): + await harness.binding_service(session).bind_checker_output( + CheckerOutputBindingRequest( + harness.selector, + attempt_id, + receipt_id, + ) + ) + binding_count = await session.scalar(select(func.count()).select_from(ArtifactBinding)) + assert binding_count == 0 + + +async def test_cancellation_during_provider_put_cleans_scratch_and_retains_uncertainty( + tmp_path, + isolated_database_env, +) -> None: + """Preserve cancellation while retaining only the durable in-flight attempt.""" + async with output_custody_harness(tmp_path, isolated_database_env) as harness: + harness.store.put_entered = asyncio.Event() + harness.store.put_release = asyncio.Event() + operation = asyncio.create_task(harness.service.store(harness.request(b"cancelled output"))) + try: + await asyncio.wait_for(harness.store.put_entered.wait(), timeout=10) + assert harness.store.puts == 1 + live_usage = await harness.manager.usage() + assert live_usage.reservation_count == 1 + assert live_usage.reserved_bytes > 0 + + operation.cancel("cancelled checker output provider put") + with pytest.raises( + asyncio.CancelledError, + match="cancelled checker output provider put", + ): + await operation + finally: + harness.store.put_release.set() + if not operation.done(): + operation.cancel() + await asyncio.gather(operation, return_exceptions=True) + + assert harness.store.puts == 1 + assert harness.store.opens == 0 + assert (await harness.manager.usage()).reservation_count == 0 + assert list((tmp_path / "output-scratch" / "files").iterdir()) == [] + async with harness.factory() as session: + attempt = await session.scalar( + select(ArtifactPutAttempt).where( + ArtifactPutAttempt.checker_run_id == str(harness.selector.checker_run_id) + ) + ) + assert attempt is not None + assert attempt.status == "put_in_flight" + assert attempt.execution_mode == "caller_put" + assert attempt.executor_id is not None + assert attempt.lease_expires_at is not None + assert attempt.replica_id is None + assert attempt.receipt_id is None + assert attempt.terminal_result_code is None + assert attempt.terminal_at is None + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactOperationReceipt) + .where( + ArtifactOperationReceipt.checker_run_id + == str(harness.selector.checker_run_id) + ) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactPutObservationReceipt) + .where(ArtifactPutObservationReceipt.put_attempt_id == attempt.id) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactVerificationJob) + .where(ArtifactVerificationJob.originating_put_attempt_id == attempt.id) + ) + == 0 + ) + assert ( + await session.scalar( + select(func.count()) + .select_from(ArtifactBinding) + .where( + ArtifactBinding.resource_type == "checker_run", + ArtifactBinding.resource_id == str(harness.selector.checker_run_id), + ) + ) + == 0 + ) + + +async def test_per_slot_cap_stops_source_and_cleans_scratch( + tmp_path, + isolated_database_env, +) -> None: + """Apply the owner-issued cap before excess bytes reach durable intent or provider.""" + async with output_custody_harness( + tmp_path, + isolated_database_env, + maximum_bytes=4, + ) as harness: + observed: list[bytes] = [] + request = CheckerOutputArtifactRequest( + harness.selector, + byte_stream(b"abcd", b"e", b"must-not-be-read", observed=observed), + ) + with pytest.raises(ArtifactLimitExceededError): + await harness.service.store(request) + assert observed == [b"abcd", b"e"] + assert harness.store.puts == 0 + assert (await harness.manager.usage()).reservation_count == 0 + assert list((tmp_path / "output-scratch" / "files").iterdir()) == [] + async with harness.factory() as session: + attempts = await session.scalar( + select(func.count()) + .select_from(ArtifactPutAttempt) + .where(ArtifactPutAttempt.producer_request_type == "checker_output") + ) + assert attempts == 0 diff --git a/backend/tests/test_checker_output_storage.py b/backend/tests/test_checker_output_storage.py new file mode 100644 index 000000000..01f765e7a --- /dev/null +++ b/backend/tests/test_checker_output_storage.py @@ -0,0 +1,989 @@ +"""Direct PostgreSQL proofs for exact checker output intent and binding custody.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from dataclasses import replace +from pathlib import Path +import runpy +from types import SimpleNamespace +from uuid import UUID + +from alembic.migration import MigrationContext +from alembic.operations import Operations +import pytest +from sqlalchemy import select, text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.core.identifiers import new_record_id +from app.modules.actors.api import ServiceIdentity +from app.modules.artifacts.models import ( + ArtifactPutAttempt, + ArtifactVerificationJob, + ArtifactVerificationReceipt, +) +from app.modules.artifacts.service import ArtifactStorageOrchestrator +from projects.unified_policy_fixtures import create_standalone_unified_policy +from tests.artifact_store_helpers import artifact_preparation_limits, minted_source +from tests.test_artifact_admission import ( + _AllowArtifactAuthority, + _admit_checker_output, + _local_store, + _namespace, + _seed_checker_output_relationships, + _settings, +) + + +async def _store_verified_output( + *, + factory, + settings, + namespace, + store, + policy_bundle, + source_path: Path, + payload: bytes, +): + """Store and independently verify one output using shared test infrastructure.""" + async with factory() as session: + limits = replace( + artifact_preparation_limits(), + reservation_ttl_seconds=360, + total_deadline_seconds=300, + ) + async with minted_source( + source_path, + payload, + media_type="text/plain", + limits=limits, + ) as source: + project_id, task_id, checker_run_id, admission = await _admit_checker_output( + session, + settings, + namespace, + source, + policy_bundle=policy_bundle, + ) + orchestrator = ArtifactStorageOrchestrator( + session, + store, + namespace, + settings, + _AllowArtifactAuthority(), + ) + await orchestrator.ensure_storage_namespace() + assert await orchestrator.execute_committed_put( + attempt_id=admission.attempt_id, + source=source, + ) == "stored_pending_verification" + job_id = await session.scalar( + select(ArtifactVerificationJob.id).where( + ArtifactVerificationJob.originating_put_attempt_id + == str(admission.attempt_id) + ) + ) + assert job_id is not None + await session.rollback() + assert await orchestrator.verify_object(UUID(job_id)) == "verified" + receipt_id = await session.scalar( + select(ArtifactVerificationReceipt.id).where( + ArtifactVerificationReceipt.verification_job_id == job_id, + ArtifactVerificationReceipt.outcome == "verified", + ) + ) + attempt = await session.get(ArtifactPutAttempt, str(admission.attempt_id)) + assert attempt is not None and receipt_id is not None + assert attempt.replica_id is not None + logical_role = attempt.logical_role + replica_id = attempt.replica_id + content_id = await session.scalar( + text("select content_id from artifact_replicas where id=:replica_id"), + {"replica_id": replica_id}, + ) + assert content_id is not None + await session.rollback() + return SimpleNamespace( + factory=factory, + project_id=project_id, + task_id=task_id, + checker_run_id=checker_run_id, + put_attempt_id=str(admission.attempt_id), + verification_job_id=str(job_id), + replica_id=str(replica_id), + verification_receipt_id=receipt_id, + content_id=str(content_id), + logical_role=logical_role, + namespace=namespace, + policy_bundle=policy_bundle, + ) + + +@asynccontextmanager +async def _verified_output(database_url: str, tmp_path: Path): + """Create one real local put and independently verified checker output.""" + settings = _settings(tmp_path) + namespace = _namespace(settings) + engine = create_async_engine(database_url) + factory = async_sessionmaker(engine, expire_on_commit=False) + bootstrap, store = _local_store(settings, namespace) + policy_bundle = await create_standalone_unified_policy(factory, namespace) + try: + case = await _store_verified_output( + factory=factory, + settings=settings, + namespace=namespace, + store=store, + policy_bundle=policy_bundle, + source_path=tmp_path / "checker-output", + payload=b"exact checker output", + ) + case.settings = settings + case.store = store + case.engine = engine + yield case + finally: + bootstrap.close() + await engine.dispose() + + +def _binding_values(case, **changes): + values = { + "id": str(new_record_id()), + "content_id": case.content_id, + "project_id": case.project_id, + "resource_type": "checker_run", + "resource_id": case.checker_run_id, + "logical_role": case.logical_role, + "scope_version": 1, + "actor_id": ServiceIdentity.ARTIFACT_CHECKER_OUTPUT.value, + "attribution_type": "service_identity", + "put_attempt_id": case.put_attempt_id, + "verification_receipt_id": case.verification_receipt_id, + "supersedes_binding_id": None, + } + values.update(changes) + return values + + +_INSERT_BINDING = text( + """insert into artifact_bindings( + id,content_id,project_id,resource_type,resource_id,logical_role,scope_version, + actor_id,attribution_type,put_attempt_id,verification_receipt_id,supersedes_binding_id) + values(:id,:content_id,:project_id,:resource_type,:resource_id,:logical_role, + :scope_version,:actor_id,:attribution_type,:put_attempt_id, + :verification_receipt_id,:supersedes_binding_id)""" +) + +_ANCESTOR_MUTATIONS = { + "attempt": ( + "artifact_put_attempts", + "put_attempt_id", + "cas_version=cas_version+1", + "checker_output_put_attempt_custody", + "checker output put attempt custody is immutable", + ), + "job": ( + "artifact_verification_jobs", + "verification_job_id", + "cas_version=cas_version+1", + "checker_output_verification_job_custody", + "checker output verification job custody is immutable", + ), + "replica": ( + "artifact_replicas", + "replica_id", + "provider_profile=provider_profile || '-changed'", + "checker_output_replica_custody", + "checker output replica custody is immutable", + ), +} + + +async def _sealed_chain(session, case) -> tuple[bool, bool, bool]: + values = [] + for table, attribute in ( + ("artifact_verification_jobs", "verification_job_id"), + ("artifact_replicas", "replica_id"), + ("artifact_put_attempts", "put_attempt_id"), + ): + value = await session.scalar( + text( + f"select checker_output_custody_sealed from {table} " + "where id=:id" + ), + {"id": getattr(case, attribute)}, + ) + values.append(bool(value)) + return tuple(values) + + +async def _wait_for_lock(session, backend_pid: int) -> None: + for _ in range(500): + waiting = await session.scalar( + text( + "select exists(select 1 from pg_locks " + "where pid=:pid and not granted)" + ), + {"pid": backend_pid}, + ) + if waiting: + return + await asyncio.sleep(0.01) + raise AssertionError("competing ancestry transaction did not wait") + + +async def _transaction(connection, isolation: str): + transaction = await connection.begin() + await connection.execute(text(f"set transaction isolation level {isolation}")) + return transaction + + +def _ancestor_update(case, ancestor: str): + table, attribute, assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + return text(f"update {table} set {assignment} where id=:id"), { + "id": getattr(case, attribute) + } + + +def _invalid_ancestor_update(case, ancestor: str): + table, attribute, _assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + assignment = { + "attempt": ( + "status='conflict', terminal_result_code='conflict', " + "terminal_at=clock_timestamp(), cas_version=cas_version+1" + ), + "job": ( + "status='conflict', terminal_result_code='conflict', " + "terminal_at=clock_timestamp(), cas_version=cas_version+1" + ), + "replica": "provider_object_ref=provider_object_ref || '-moved'", + }[ancestor] + return text(f"update {table} set {assignment} where id=:id"), { + "id": getattr(case, attribute) + } + + +async def _ancestor_without_seal(session, case, ancestor: str): + table, attribute, _assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + return await session.scalar( + text( + f"select to_jsonb(value) - 'checker_output_custody_sealed' " + f"from {table} value where id=:id" + ), + {"id": getattr(case, attribute)}, + ) + + +def _assert_concurrent_rejection( + error: DBAPIError, + *, + isolation: str, + expected: str, +) -> None: + message = str(error) + if isolation == "REPEATABLE READ": + assert expected in message or "could not serialize access" in message + else: + assert expected in message + + +async def _binding_first_race(case, ancestor: str, isolation: str) -> None: + async with case.factory() as session: + before = await _ancestor_without_seal(session, case, ancestor) + binder = await case.engine.connect() + updater = await case.engine.connect() + binding_transaction = await _transaction(binder, isolation) + update_transaction = await _transaction(updater, isolation) + pending_update = None + try: + updater_pid = await updater.scalar(text("select pg_backend_pid()")) + assert await _sealed_chain(updater, case) == (False, False, False) + assert await _ancestor_without_seal(updater, case, ancestor) == before + await binder.execute(_INSERT_BINDING, _binding_values(case)) + statement, parameters = _ancestor_update(case, ancestor) + pending_update = asyncio.create_task(updater.execute(statement, parameters)) + async with case.engine.connect() as observer: + await _wait_for_lock(observer, updater_pid) + await binding_transaction.commit() + with pytest.raises(DBAPIError) as rejected: + await pending_update + _assert_concurrent_rejection( + rejected.value, + isolation=isolation, + expected=_ANCESTOR_MUTATIONS[ancestor][4], + ) + await update_transaction.rollback() + finally: + if pending_update is not None and not pending_update.done(): + pending_update.cancel() + await asyncio.gather(pending_update, return_exceptions=True) + if binding_transaction.is_active: + await binding_transaction.rollback() + if update_transaction.is_active: + await update_transaction.rollback() + await binder.close() + await updater.close() + async with case.factory() as session: + assert await _sealed_chain(session, case) == (True, True, True) + assert await _ancestor_without_seal(session, case, ancestor) == before + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 1 + + +async def _update_first_race(case, ancestor: str, isolation: str) -> None: + async with case.factory() as session: + before = await _ancestor_without_seal(session, case, ancestor) + updater = await case.engine.connect() + binder = await case.engine.connect() + update_transaction = await _transaction(updater, isolation) + binding_transaction = await _transaction(binder, isolation) + pending_binding = None + try: + binder_pid = await binder.scalar(text("select pg_backend_pid()")) + statement, parameters = _invalid_ancestor_update(case, ancestor) + await updater.execute(statement, parameters) + assert await _sealed_chain(binder, case) == (False, False, False) + assert await _ancestor_without_seal(binder, case, ancestor) == before + pending_binding = asyncio.create_task( + binder.execute(_INSERT_BINDING, _binding_values(case)) + ) + async with case.engine.connect() as observer: + await _wait_for_lock(observer, binder_pid) + await update_transaction.commit() + with pytest.raises(DBAPIError) as rejected: + await pending_binding + _assert_concurrent_rejection( + rejected.value, + isolation=isolation, + expected="checker output binding verified ancestry mismatch", + ) + await binding_transaction.rollback() + finally: + if pending_binding is not None and not pending_binding.done(): + pending_binding.cancel() + await asyncio.gather(pending_binding, return_exceptions=True) + if update_transaction.is_active: + await update_transaction.rollback() + if binding_transaction.is_active: + await binding_transaction.rollback() + await updater.close() + await binder.close() + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await _ancestor_without_seal(session, case, ancestor) != before + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 0 + + +@pytest.mark.asyncio +async def test_exact_verified_checker_output_binding_and_generic_binding_remain_valid( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + async with case.factory() as session, session.begin(): + await session.execute(_INSERT_BINDING, _binding_values(case)) + await session.execute( + _INSERT_BINDING, + _binding_values( + case, + id=str(new_record_id()), + resource_type="task", + resource_id=case.task_id, + logical_role="diagnostic", + actor_id="test-actor", + attribution_type="human", + put_attempt_id=None, + verification_receipt_id=None, + ), + ) + async with case.factory() as session: + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where resource_type in ('checker_run','task')" + ) + ) == 2 + + +@pytest.mark.asyncio +async def test_checker_binding_rejects_null_verification_terminal_result( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + async with case.factory() as session: + control = await session.begin() + await session.execute(_INSERT_BINDING, _binding_values(case)) + assert await _sealed_chain(session, case) == (True, True, True) + await control.rollback() + + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_verification_jobs " + "set terminal_result_code=null where id=:id" + ), + {"id": case.verification_job_id}, + ) + + with pytest.raises( + DBAPIError, + match="checker output binding verified ancestry mismatch", + ): + async with case.factory() as session, session.begin(): + await session.execute(_INSERT_BINDING, _binding_values(case)) + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 0 + + async with case.factory() as session: + mutation = await session.begin() + definition = await session.scalar( + text( + "select pg_get_functiondef(" + "'guard_checker_output_binding_insert()'::regprocedure)" + ) + ) + assert definition is not None + null_safe = "job.terminal_result_code IS DISTINCT FROM 'verified'" + old_comparison = "job.terminal_result_code <> 'verified'" + assert definition.count(null_safe) == 1 + await session.execute(text(definition.replace(null_safe, old_comparison))) + await session.execute(_INSERT_BINDING, _binding_values(case)) + assert await _sealed_chain(session, case) == (True, True, True) + await mutation.rollback() + + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await session.scalar( + text( + "select count(*) from artifact_bindings " + "where put_attempt_id=:put_attempt_id" + ), + {"put_attempt_id": case.put_attempt_id}, + ) == 0 + + +@pytest.mark.asyncio +async def test_checker_binding_rejects_mixed_or_foreign_ancestry( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + other = await _store_verified_output( + factory=case.factory, + settings=case.settings, + namespace=case.namespace, + store=case.store, + policy_bundle=case.policy_bundle, + source_path=tmp_path / "other-checker-output", + payload=b"distinct independently verified checker output", + ) + assert case.put_attempt_id != other.put_attempt_id + assert case.verification_receipt_id != other.verification_receipt_id + assert case.content_id != other.content_id + assert case.namespace.namespace_fingerprint == other.namespace.namespace_fingerprint + + async with case.factory() as session, session.begin(): + control = await session.begin_nested() + await session.execute(_INSERT_BINDING, _binding_values(case)) + await session.execute(_INSERT_BINDING, _binding_values(other)) + assert await session.scalar( + text("select count(*) from artifact_bindings where resource_type='checker_run'") + ) == 2 + await control.rollback() + assert await session.scalar( + text("select count(*) from artifact_bindings where resource_type='checker_run'") + ) == 0 + + mixed_lineages = ( + { + "put_attempt_id": other.put_attempt_id, + "verification_receipt_id": other.verification_receipt_id, + "content_id": other.content_id, + }, + {"put_attempt_id": other.put_attempt_id}, + {"verification_receipt_id": other.verification_receipt_id}, + {"content_id": other.content_id}, + ) + for mixed in mixed_lineages: + with pytest.raises( + DBAPIError, + match="checker output binding verified ancestry mismatch", + ): + async with case.factory() as session, session.begin(): + await session.execute( + _INSERT_BINDING, + _binding_values(case, **mixed), + ) + async with case.factory() as session: + assert await session.scalar( + text("select count(*) from artifact_bindings where resource_type='checker_run'") + ) == 0 + + +@pytest.mark.asyncio +async def test_checker_attempt_static_custody_allows_only_execution_lifecycle( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + for assignment in ( + "checker_request_digest='sha256:' || repeat('b',64)", + "request_digest='sha256:' || repeat('b',64)", + "sha256='sha256:' || repeat('b',64)", + "submission_id='00000000-0000-7000-8000-000000000001'::uuid", + "logical_role='substituted-slot'", + "namespace_fingerprint='sha256:' || repeat('b',64)", + ): + with pytest.raises( + DBAPIError, + match="checker output put attempt custody is immutable", + ): + async with case.factory() as session, session.begin(): + await session.execute( + text( + f"update artifact_put_attempts set {assignment} where id=:id" + ), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="checker output put attempt custody is immutable", + ): + async with case.factory() as session, session.begin(): + await session.execute( + text("delete from artifact_put_attempts where id=:id"), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="checker output put attempt custody is immutable", + ): + async with case.factory() as session, session.begin(): + await session.execute(text("truncate artifact_put_attempts cascade")) + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_put_attempts set cas_version=cas_version+1, " + "updated_at=clock_timestamp() where id=:id" + ), + {"id": case.put_attempt_id}, + ) + + +@pytest.mark.asyncio +async def test_binding_seals_exact_ancestry_but_preserves_replica_health( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + for ancestor in _ANCESTOR_MUTATIONS: + table, attribute, _assignment, _trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + with pytest.raises(DBAPIError, match="seal requires binding"): + async with case.factory() as session, session.begin(): + await session.execute( + text( + f"update {table} set checker_output_custody_sealed=true " + "where id=:id" + ), + {"id": getattr(case, attribute)}, + ) + + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_put_attempts set cas_version=cas_version+1 " + "where id=:id" + ), + {"id": case.put_attempt_id}, + ) + await session.execute( + text( + "update artifact_verification_jobs set cas_version=cas_version+1 " + "where id=:id" + ), + {"id": case.verification_job_id}, + ) + await session.execute( + text( + "update artifact_replicas set last_reconciled_at=clock_timestamp(), " + "updated_at=clock_timestamp() where id=:id" + ), + {"id": case.replica_id}, + ) + await session.execute(_INSERT_BINDING, _binding_values(case)) + assert await _sealed_chain(session, case) == (True, True, True) + + for ancestor in _ANCESTOR_MUTATIONS: + table, attribute, assignment, _trigger, error = _ANCESTOR_MUTATIONS[ancestor] + with pytest.raises(DBAPIError, match=error): + async with case.factory() as session, session.begin(): + await session.execute( + text(f"update {table} set {assignment} where id=:id"), + {"id": getattr(case, attribute)}, + ) + with pytest.raises(DBAPIError, match=error): + async with case.factory() as session, session.begin(): + await session.execute( + text(f"delete from {table} where id=:id"), + {"id": getattr(case, attribute)}, + ) + + async with case.factory() as session, session.begin(): + await session.execute( + text( + "update artifact_replicas set verification_state='missing', " + "availability_state='unavailable', integrity_state='invalid', " + "last_reconciled_at=clock_timestamp(), updated_at=clock_timestamp() " + "where id=:id" + ), + {"id": case.replica_id}, + ) + assert await _sealed_chain(session, case) == (True, True, True) + + for ancestor in _ANCESTOR_MUTATIONS: + table, attribute, assignment, trigger, _error = _ANCESTOR_MUTATIONS[ancestor] + async with case.factory() as session: + before = await session.scalar( + text(f"select to_jsonb(value) from {table} value where id=:id"), + {"id": getattr(case, attribute)}, + ) + transaction = await session.begin_nested() + await session.execute(text(f"alter table {table} disable trigger {trigger}")) + await session.execute( + text(f"update {table} set {assignment} where id=:id"), + {"id": getattr(case, attribute)}, + ) + after = await session.scalar( + text(f"select to_jsonb(value) from {table} value where id=:id"), + {"id": getattr(case, attribute)}, + ) + assert after != before + await transaction.rollback() + assert await session.scalar( + text(f"select to_jsonb(value) from {table} value where id=:id"), + {"id": getattr(case, attribute)}, + ) == before + await session.rollback() + + +@pytest.mark.asyncio +async def test_binding_rollback_removes_all_ancestry_seals( + isolated_database_env: str, + tmp_path: Path, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + binding_id = str(new_record_id()) + async with case.factory() as session: + transaction = await session.begin() + await session.execute( + _INSERT_BINDING, + _binding_values(case, id=binding_id), + ) + assert await _sealed_chain(session, case) == (True, True, True) + await transaction.rollback() + async with case.factory() as session: + assert await _sealed_chain(session, case) == (False, False, False) + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 0 + + +@pytest.mark.parametrize("ancestor", tuple(_ANCESTOR_MUTATIONS)) +@pytest.mark.parametrize("isolation", ("READ COMMITTED", "REPEATABLE READ")) +@pytest.mark.asyncio +async def test_binding_and_ancestor_updates_serialize_fail_closed( + isolated_database_env: str, + tmp_path: Path, + ancestor: str, + isolation: str, +) -> None: + async with _verified_output(isolated_database_env, tmp_path) as case: + await _binding_first_race(case, ancestor, isolation) + + other = await _store_verified_output( + factory=case.factory, + settings=case.settings, + namespace=case.namespace, + store=case.store, + policy_bundle=case.policy_bundle, + source_path=tmp_path / f"update-first-{ancestor}", + payload=f"update-first-{isolation}-{ancestor}".encode(), + ) + other.engine = case.engine + await _update_first_race(other, ancestor, isolation) + + +@pytest.mark.asyncio +async def test_artifact_binding_truncate_custody_blocks_direct_and_cascade_deletion( + isolated_database_env: str, + tmp_path: Path, +) -> None: + """Table-wide deletion cannot bypass immutable artifact binding custody.""" + async with _verified_output(isolated_database_env, tmp_path) as case: + binding_id = str(new_record_id()) + async with case.factory() as session, session.begin(): + await session.execute( + _INSERT_BINDING, + _binding_values(case, id=binding_id), + ) + + for statement in ( + "truncate artifact_bindings", + "truncate artifact_contents cascade", + ): + with pytest.raises(DBAPIError, match="artifact_bindings rows are immutable"): + async with case.factory() as session, session.begin(): + await session.execute(text(statement)) + async with case.factory() as session: + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 1 + + async with case.factory() as session: + transaction = await session.begin() + try: + await session.execute( + text( + "alter table artifact_bindings disable trigger " + "trg_artifact_bindings_no_truncate" + ) + ) + await session.execute(text("truncate artifact_bindings")) + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 0 + finally: + await transaction.rollback() + + async with case.factory() as session: + assert await session.scalar( + text("select count(*) from artifact_bindings where id=:id"), + {"id": binding_id}, + ) == 1 + trigger_definition = await session.scalar( + text( + "select pg_get_triggerdef(oid) from pg_trigger " + "where tgrelid='artifact_bindings'::regclass " + "and tgname='trg_artifact_bindings_no_truncate'" + ) + ) + assert trigger_definition is not None + assert "BEFORE TRUNCATE" in trigger_definition + assert "EXECUTE FUNCTION reject_artifact_fact_mutation()" in trigger_definition + + +@pytest.mark.asyncio +async def test_binding_guard_uses_only_artifact_ancestry_and_exact_owner_fks( + isolated_database_env: str, +) -> None: + engine = create_async_engine(isolated_database_env) + try: + async with engine.connect() as connection: + definition = await connection.scalar( + text( + "select pg_get_functiondef(" + "'guard_checker_output_binding_insert()'::regprocedure)" + ) + ) + assert definition is not None + assert "checker_runs" not in definition + assert "submissions" not in definition + assert "workstream_tasks" not in definition + constraints = dict( + (await connection.execute(text( + "select conname,pg_get_constraintdef(oid) from pg_constraint " + "where conrelid='artifact_put_attempts'::regclass and conname in " + "('fk_artifact_put_attempts_checker_run_ownership'," + "'fk_artifact_put_attempts_submission_version'," + "'fk_artifact_put_attempts_task_project')" + ))).all() + ) + assert set(constraints) == { + "fk_artifact_put_attempts_checker_run_ownership", + "fk_artifact_put_attempts_submission_version", + "fk_artifact_put_attempts_task_project", + } + assert "FOREIGN KEY (checker_run_id, task_id, submission_id)" in constraints[ + "fk_artifact_put_attempts_checker_run_ownership" + ] + assert "FOREIGN KEY (submission_id, task_id, submission_version)" in constraints[ + "fk_artifact_put_attempts_submission_version" + ] + assert "FOREIGN KEY (task_id, project_id)" in constraints[ + "fk_artifact_put_attempts_task_project" + ] + finally: + await engine.dispose() + + +async def _restore_pre_0007_schema(connection) -> None: + """Remove only 0007 custody inside the caller's rollback-only transaction.""" + for table, trigger in ( + ("artifact_bindings", "checker_output_binding_seal"), + ("artifact_bindings", "checker_output_binding_insert"), + ("artifact_bindings", "trg_artifact_bindings_no_truncate"), + ("artifact_verification_jobs", "checker_output_verification_job_custody"), + ("artifact_replicas", "checker_output_replica_custody"), + ("artifact_put_attempts", "checker_output_put_attempt_custody"), + ("artifact_put_attempts", "checker_output_put_attempt_no_truncate"), + ): + await connection.execute(text(f"drop trigger {trigger} on {table}")) + for function in ( + "seal_checker_output_binding_ancestry()", + "guard_checker_output_binding_insert()", + "guard_checker_output_verification_job_custody()", + "guard_checker_output_replica_custody()", + "guard_checker_output_put_attempt_custody()", + "guard_checker_output_put_attempt_truncate()", + ): + await connection.execute(text(f"drop function {function}")) + for constraint in ( + "ck_artifact_bindings_checker_output_lineage", + "fk_artifact_bindings_checker_put_attempt", + "fk_artifact_bindings_checker_verification_receipt", + ): + await connection.execute( + text(f"alter table artifact_bindings drop constraint {constraint}") + ) + for index in ( + "ix_artifact_bindings_put_attempt_id", + "ix_artifact_bindings_verification_receipt_id", + ): + await connection.execute(text(f"drop index {index}")) + await connection.execute( + text( + "alter table artifact_bindings drop column put_attempt_id, " + "drop column verification_receipt_id" + ) + ) + for constraint in ( + "fk_artifact_put_attempts_task_project", + "fk_artifact_put_attempts_checker_run_ownership", + "fk_artifact_put_attempts_submission_version", + "ck_artifact_put_attempts_checker_request_digest", + "ck_artifact_put_attempts_producer_reference", + ): + await connection.execute( + text(f"alter table artifact_put_attempts drop constraint {constraint}") + ) + await connection.execute(text("drop index ix_artifact_put_attempts_submission_id")) + await connection.execute( + text( + "alter table artifact_put_attempts add constraint " + "ck_artifact_put_attempts_producer_reference check (" + "(producer_request_type='guide' and guide_source_item_id is not null " + "and checker_run_id is null and task_id is null and logical_role is null) or " + "(producer_request_type='checker_output' and guide_source_item_id is null " + "and checker_run_id is not null and task_id is not null " + "and octet_length(logical_role) between 1 and 100) or " + "(producer_request_type='submission_bundle' and guide_source_item_id is null " + "and checker_run_id is null and task_id is not null and logical_role is null))" + ) + ) + await connection.execute( + text( + "alter table artifact_put_attempts drop column submission_id, " + "drop column submission_version, drop column checker_request_digest, " + "drop column checker_output_custody_sealed" + ) + ) + await connection.execute( + text( + "alter table artifact_verification_jobs " + "drop column checker_output_custody_sealed" + ) + ) + await connection.execute( + text( + "alter table artifact_replicas " + "drop column checker_output_custody_sealed" + ) + ) + + +def _run_0007_upgrade(connection) -> None: + module = runpy.run_path( + str( + Path(__file__).resolve().parents[1] + / "alembic/versions/0007_checker_output_custody.py" + ) + ) + with Operations.context(MigrationContext.configure(connection)): + module["upgrade"]() + + +@pytest.mark.asyncio +async def test_migration_refuses_unprovable_or_inconsistent_retained_checker_attempts( + isolated_database_env: str, + tmp_path: Path, +) -> None: + """The old schema lacks evaluation/slot custody, so retained attempts fail closed.""" + async with _verified_output(isolated_database_env, tmp_path / "retained") as case: + async with case.factory() as seed_session: + _, other_task_id, _ = await _seed_checker_output_relationships( + seed_session, + case.namespace, + policy_bundle=case.policy_bundle, + ) + async with case.factory() as session: + connection = await session.connection() + await _restore_pre_0007_schema(connection) + before = await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="retained checker output request custody is unprovable", + ): + async with connection.begin_nested(): + await connection.run_sync(_run_0007_upgrade) + assert await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) == before + + await session.execute( + text("update artifact_put_attempts set task_id=:task where id=:id"), + {"id": case.put_attempt_id, "task": other_task_id}, + ) + mismatched = await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) + with pytest.raises( + DBAPIError, + match="retained checker output attempt ownership is inconsistent", + ): + async with connection.begin_nested(): + await connection.run_sync(_run_0007_upgrade) + assert await session.scalar( + text("select to_jsonb(a) from artifact_put_attempts a where id=:id"), + {"id": case.put_attempt_id}, + ) == mismatched + await session.rollback() diff --git a/backend/tests/test_ci_lane_catalogue.py b/backend/tests/test_ci_lane_catalogue.py index 6b374327e..a2a45a389 100644 --- a/backend/tests/test_ci_lane_catalogue.py +++ b/backend/tests/test_ci_lane_catalogue.py @@ -231,6 +231,8 @@ def test_measured_hotspots_have_explicit_semantic_owners() -> None: "tests/test_approved_guide_intake.py", "tests/test_post_submit_materialization.py", "tests/test_post_submit_selection.py", + "tests/test_checker_output_custody.py", + "tests/test_checker_output_storage.py", "tests/test_pre_submit_attempt_recovery.py", "tests/test_pre_submit_attempt_contracts.py", "tests/test_pre_submit_attempt_authority_integration.py", diff --git a/backend/tests/test_coverage_contract.py b/backend/tests/test_coverage_contract.py index f4f0fa9bb..b79a09230 100644 --- a/backend/tests/test_coverage_contract.py +++ b/backend/tests/test_coverage_contract.py @@ -12,7 +12,7 @@ sys.path.insert(0, str(SCRIPTS)) import coverage_policy as policy # noqa: E402 -HEAD = "0006_history_read_authority" +HEAD = "0007_checker_output_custody" SHA = "a" * 40 PEP695_INVALID = sys.version_info < (3, 12) diff --git a/backend/tests/test_post_submit_materialization.py b/backend/tests/test_post_submit_materialization.py index 3869e8e0f..ac10d6d52 100644 --- a/backend/tests/test_post_submit_materialization.py +++ b/backend/tests/test_post_submit_materialization.py @@ -10,7 +10,7 @@ from app.adapters.artifacts import post_submission_materialization from app.core.identifiers import new_record_id from app.interfaces.artifacts import ArtifactInputMismatchError -from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.checkers.api.materialization import PostSubmissionMaterializationUnavailable from tests.checkers.post_submit.support import change_request from tests.checkers.post_submit.test_result_contract import result from tests.post_submit_materialization_helpers import material_fixture diff --git a/backend/tests/test_post_submit_selection.py b/backend/tests/test_post_submit_selection.py index 5001e1405..eecfa4075 100644 --- a/backend/tests/test_post_submit_selection.py +++ b/backend/tests/test_post_submit_selection.py @@ -7,7 +7,7 @@ from app.adapters.tasks import submitted_bundle_port from app.core.identifiers import new_record_id -from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.checkers.api.materialization import PostSubmissionMaterializationUnavailable from app.modules.tasks.api.submitted_bundle import SubmittedBundleRequest, SubmittedBundleUnavailable from tests.checkers.post_submit.support import change_request from tests.post_submit_materialization_helpers import material_fixture diff --git a/docs/architecture_checker_framework.md b/docs/architecture_checker_framework.md index bdb4ed86c..146fa130b 100644 --- a/docs/architecture_checker_framework.md +++ b/docs/architecture_checker_framework.md @@ -331,9 +331,13 @@ Production explicitly uses `UnavailablePostSubmissionExecution`; this does not install durable post-submit execution, authorize material reads, or prove attempt and currentness ownership. ARCH-04B supplies the hidden ART input port: exact consumed Submission bytes, rebuilt manifest, async scoped file access and cleanup, followed -by a fresh material-selection check. Production composition denies materialization; -ARCH-04B2 output custody, ARCH-04C durable execution, ARCH-04D authority and ARCH-04E -routing remain the execution cutover. A returned evaluation value is not a stored +by a fresh material-selection check. Production composition denies materialization. +ARCH-04B2 now supplies hidden typed output storage, byte-free recovery and +flush-only verified binding while its CHECKERS reservation and authority adapters +remain unavailable. The current structural catalogue reserves zero output slots; +controlled nonempty slots prove ART mechanics only. ARCH-04C must support that +empty output set and owns durable execution, followed by ARCH-04D authority and +ARCH-04E routing. A returned evaluation value is not a stored current result or acceptance. Retained run and submission history now use canonical AUTH and separate fixed contributor/manager projections. The alternate execution service and Celery worker are @@ -519,10 +523,13 @@ severities. ## Checker Run Flow — Target Contract The following is the intended end-to-end lifecycle. Pre-submit intake and -retained-history reads are implemented. Canonical durable post-submit execution, -result routing and recovery remain unavailable pending ARCH-04B2/04C/04D/04E/04F; -ARCH-04B hidden input is delivered with deny-only production authority; -the flow below is not a claim that those jobs or transitions are live. +retained-history reads are implemented. Canonical durable post-submit execution +and result routing remain unavailable pending ARCH-04C/04D/04E. ARCH-04F adds +contributor-correctable remediation and gates public intake and enabling the +false-policy acceptance path; the true `allow_review` route may ship before +ARCH-04F. ARCH-04B hidden input and ARCH-04B2 hidden output custody are delivered +with deny-only/unavailable production composition. The flow below is not a claim +that those jobs or transitions are live. ```text Draft packet @@ -602,8 +609,8 @@ The checker run records: - warning count - completion timestamp -With ARCH-04B input delivered, ARCH-04B2 output custody, ARCH-04C result custody, -ARCH-04D activation and +With ARCH-04B input and ARCH-04B2 output custody delivered, ARCH-04C result +custody, ARCH-04D activation and ARCH-04E routing integration, this gives reviewers proof that they are reviewing the same immutable binding and manifest that passed automated checks; legacy caller-owned manifest fields are not authority. diff --git a/docs/architecture_data_model.md b/docs/architecture_data_model.md index 0a0772211..fb6bf39dc 100644 --- a/docs/architecture_data_model.md +++ b/docs/architecture_data_model.md @@ -318,8 +318,15 @@ Submission's three ART references remain all-null during transaction staging or all-present. The production creation command consumes the exact ART admission and fills all three before its root transaction commits. This is separate from the required initial assignment identity. Hidden exact ART-to-CHECKERS input materialization is delivered by ARCH-04B. -Output custody (ARCH-04B2) and durable execution/results (ARCH-04C) remain pending; -production access remains deny-only until ARCH-04D. Retained payment columns on Submission and CheckerRun +ARCH-04B2 adds checker-output attempt custody with exact Submission version and +checker-request digest plus immutable verified put/receipt ancestry on each +run/slot binding. Binding publication atomically seals the terminal put attempt, +verification job and replica identity. Later replica health changes remain +possible without rewriting the historical verified ancestry. Its migration +refuses retained checker attempts or bindings +whose missing evaluation digest or verified ancestry cannot be proven; it does +not invent or delete retained data. Durable execution/results (ARCH-04C) remain +pending, and production access remains deny-only until ARCH-04D. Retained payment columns on Submission and CheckerRun are nullable, so new unified-guide work requires no invented economic configuration. CP09 owns physical economic-schema removal after its remaining consumers change. @@ -949,10 +956,10 @@ cannot reproduce the authoritative result. At the later public cutover, without receiving scratch paths or rerunning the pre-submit plan. Canonical admission-backed creation remains hidden until then. -Before that cutover, hidden ART-04B2 establishes the execution boundary without +Before that cutover, hidden pre-submit materialization establishes the execution boundary without exposing a route. The fixed materializer authorizes before any prepared-byte read or workspace reservation. One callback-scoped sealed tree is checked -against the server commitment and semantic manifest. ART-04B3 extends that same +against the server commitment and semantic manifest. The locked project-policy phase extends that same callback to execute the locked project-policy phase and normalize every platform and project result into one typed envelope. The tree is cleaned before the transaction-bound evidence service reloads the actor, identity link, task, @@ -1647,13 +1654,18 @@ Fields: - `locked_revision_policy_id` - `locked_revision_policy_generation` - `locked_revision_policy_hash` -- `artifact_binding_id` (target after ARCH-04B2/04C custody) -- `submission_bundle_manifest_id` (target after ARCH-04B2/04C custody) -- `package_hash` (legacy; replacement custody in ARCH-04B2/04C) -- `artifact_hash_manifest` (legacy; replacement custody in ARCH-04B2/04C) -- `artifact_manifest_hash` (legacy; replacement custody in ARCH-04B2/04C) +- `artifact_binding_id` (planned canonical input reference; exact 04C shape pending) +- `submission_bundle_manifest_id` (planned canonical input reference; exact 04C shape pending) +- `package_hash` (retained current field; 04C must replace its authority use) +- `artifact_hash_manifest` (retained current field; 04C must replace its authority use) +- `artifact_manifest_hash` (retained current field; 04C must replace its authority use) - `summary` +ARCH-04B2 does not populate or authorize this retained run writer. It stores +checker-output custody in ART attempts and bindings only. ARCH-04C owns the +current CheckerRun/result schema and must select exact canonical input +references without treating these retained caller-manifest fields as authority. + Status: - queued diff --git a/docs/engineering/authorization_activation_custody.md b/docs/engineering/authorization_activation_custody.md index 6e60baed8..88f86b1a8 100644 --- a/docs/engineering/authorization_activation_custody.md +++ b/docs/engineering/authorization_activation_custody.md @@ -58,9 +58,13 @@ The table retains historical planning-custody labels, not a literal mapping of every typed runtime `ActionOwner`. XINT-06B groups runtime `WS-AUTH-001-ART-06A` post-submit materialization and `WS-AUTH-001-ART-06B` output write/binding. ARCH-04D is their current replacement -activation boundary after delivered ARCH-04B hidden input, planned ARCH-04B2 -output custody and ARCH-04C durable execution/results. Production materialization -remains deny-only; the typed catalogue is unchanged by this reconciliation. Do not implement an additional XINT-06B lane. +activation boundary. ARCH-04B hidden input and ARCH-04B2 hidden output custody +are delivered; ARCH-04C durable execution/results is next, followed by ARCH-04D. ARCH-04B2 owns +fresh authority participants internally for each store, recovery and binding +phase; public requests carry selectors and byte sources, never PREP handles. +Default output reservation and authority composition remains unavailable, and +production materialization remains deny-only. The typed catalogue is unchanged +by this reconciliation. Do not implement an additional XINT-06B lane. Runtime owner `WS-XINT-002-07` retains catalogue custody. The only approved v0.1 availability transition is 07A packet materialization. Evidence binding diff --git a/docs/operations_project_operating_manual.md b/docs/operations_project_operating_manual.md index 601216f9f..2b00ae459 100644 --- a/docs/operations_project_operating_manual.md +++ b/docs/operations_project_operating_manual.md @@ -262,7 +262,9 @@ authority and decision-bound receipts. ARCH-03C2 delivers atomic producer publication and first handler registration with enforced prefork topology. Public manager activation and ARCH-03D hidden approved-guide intake are delivered. Hidden exact post-submit input (ARCH-04B) is delivered with deny-only production -authority. ARCH-04B2 output custody is next. Public intake remains deferred +authority. Hidden ARCH-04B2 output storage, recovery and verified binding are +delivered with production reservations and authority unavailable. ARCH-04C durable +execution is next. Public intake remains deferred to ARCH-02I after evaluation and remediation prerequisites. The intended unified flow uses one compilation result for sufficiency and diff --git a/docs/roadmap_status.md b/docs/roadmap_status.md index 75d38136c..8cff89898 100644 --- a/docs/roadmap_status.md +++ b/docs/roadmap_status.md @@ -66,8 +66,15 @@ creation defaults true and omitted replacements inherit the predecessor. Versioned hashing preserves old policy hashes and locks. False remains blocked by the current activation service. See the [implementation record](../.commitrail/changes/pre-review-plan-reconciliation.md#delivered-policy-setting-implementation). Enabling false follows shared final-acceptance, CON and exact AUTH integration proof, not live human-review infrastructure. -This allows an automated end-to-end milestone first; human review/revision -still belongs to the complete v0.1 release. See the +The first complete contributor milestone is claim -> upload ZIP -> pre-submit +feedback or immutable Submission -> automatic post-submit checking -> outcome. +For an eligible project with locked `human_review_required=false`, success must +include shared FinalAcceptance, the submitter ContributionRecord and applicable +policy awards. Failure/remediation and public intake belong to that milestone; +reaching hidden checker success alone does not complete it. Deliver this journey +before live human review/revision. Independent review foundations may proceed in +a separate worktree, but cannot add a human-queue or review-lease dependency to +automated acceptance. Human review/revision still belongs to the complete v0.1 release. See the [product-builder handoff](../.commitrail/changes/pre-review-plan-reconciliation.md#product-builder-handoff-implement-the-setting-next). | Status | Meaning | @@ -95,8 +102,11 @@ adds exact-authorized Contributor/Manager detail and requirements. ARCH-03C6 add separate exact-authorized locked-context reads. ARCH-03C7 adds bounded public Audit Authority history access. ARCH-03D connects hidden intake through durable intent to the activated historical guide using canonical owner ports. ARCH-04B adds hidden verified Submission input -with scoped async access; production materialization authority remains deny-only. -Checker output custody and durable evaluation follow; public intake remains deferred to their cutover prerequisites. Required success +with scoped async access. ARCH-04B2 adds hidden typed checker-output storage, +byte-free recovery and flush-only verified binding over generic ART +put/verification. Production reservation and authority remain unavailable; +durable evaluation follows in ARCH-04C, and public intake remains deferred to +the later cutover prerequisites. Required success then branches on the locked ReviewPolicy: true routes to human `allow_review`; false invokes shared authorized acceptance without a human Review. Both routing integrations remain planned. Human review/revision, contribution and conditional @@ -150,7 +160,7 @@ cannot be reused as post-submission review-gate evidence. See the | Contributor artifact preparation | **Hidden and proven** | One outer ZIP; bounded scratch inspection; canonical manifest; platform and project prechecks; unchanged-work rejection; durable put intent; verification; capacity-charged ready admission; hidden final handoff validates the exact activated historical guide through owner ports | Complete the later public admission-only cutover | | Pre-submission intake checking | **Hidden with approved-guide lineage** | Separate versioned pre-submission catalogue, locked effective-plan compilation, platform/project checks during continuous preparation, blocking feedback before Submission creation, and one internal phase command covering execution/replay with the JSON precheck removed; ARCH-03D connects approved-guide lineage through the final durable handoff | Complete the canonical public cutover after evaluation/remediation prerequisites; passing intake must never substitute for post-submit evaluation | | Immutable Submission creation | **Hidden foundation; public packet creation retired** | Contributor preparation authority; durable pre-submit reservation and exact completed-evidence recovery without rerunning checks; atomic admission consumption; TASK-owned admission-backed creation with exact assignment ContributionPolicyVersion and locked policy lineage; fixed-service artifact binding; replay/concurrency/rollback proof | Finish downstream evaluation and the canonical public integration. The retained submission-list GET is not a usable creation POST | -| Post-submission evaluation and `allow_review` | **Planned; immediate integration milestone** | One canonical CHECKER post-submit catalogue/compiler used by existing consumers, internal phase service with explicitly unavailable post execution, hidden value contracts and structural-handler conformance; ARCH-04B hidden exact verified Submission materialization with deny-only production authority | Add ARCH-04B2 output custody; connect the unavailable phase port to durable execution; evaluate the exact Submission against its locked policy; persist one durable current superseding result; activate fixed services; automatically dispatch it and publish the canonical `allow_review` manifest | +| Post-submission evaluation and `allow_review` | **Planned; immediate integration milestone** | One canonical CHECKER post-submit catalogue/compiler used by existing consumers, internal phase service with explicitly unavailable post execution, hidden value contracts and structural-handler conformance; ARCH-04B hidden exact verified Submission materialization; ARCH-04B2 hidden typed output store/recovery and verified binding, with production reservation and authority unavailable | Add ARCH-04C durable execution with valid empty-output support; evaluate the exact Submission against its locked policy; persist one durable current superseding result; activate fixed services; automatically dispatch it and publish the canonical `allow_review` manifest | | Review queue and lease | **Hidden persistence foundation** | Queue/admission idempotency and ReviewLease/preference persistence; complete unavailable REV action/principal catalogue and typed AUTH contracts | Packet-membership contract and manifest; Review schema; canonical admission from `allow_review`; claim/lease/packet authority; lease copies the Submission-stamped policy version with no CON lookup | | Review decision and revision | **Planned** | Review/revision policy identities and mutation authority; approved same-task revision-rebase semantics | Immutable findings and decisions; `accept`, `needs_revision`, and `reject`; complete-context revision preparation; finding responses; replacement contributor rules; replay and recovery | | Contribution and compensation truth | **Schema foundations plus public policy administration** | ContributionPolicyVersion persistence; lifecycle-audit participant; adapter bindings; public Finance policy administration | Persist ContributionRecord/CompensationAward and one shared FinalAcceptance/submitter operation for human accept or authorized false/pass routing. Only actual Reviews create reviewer records. Evaluate frozen actor rules into zero, one or two awards | @@ -438,8 +448,9 @@ The next dependency-safe product sequence is: data and downstream TASK consumers remain until their scoped cutover; deleting retained data is not authorized by code cleanup. 2. **Produce current post-submit evidence and policy-governed routing.** ARCH-04B's - hidden exact input materialization is delivered. Next add ARCH-04B2 output custody, - execute the locked post-submit plan, persist one current result, + hidden exact input materialization and ARCH-04B2's hidden output custody are + delivered. Next execute the locked post-submit plan through ARCH-04C, support + the current structural catalogue's empty output set, persist one current result, activate only its fixed services, and automatically publish an exact human `allow_review` manifest on true when no blocking failure exists. CHECKERS owns durable execution/currentness; the shared facade does not @@ -523,6 +534,13 @@ read port and ART's consumed admission. Local/MinIO input is independently rerea verified and projected through canonical bounded scratch. Production access remains deny-only; it does not activate durable checker execution or public intake. +ARCH-04B2 adds hidden typed checker-output `store` and byte-free +`recover(selector)` operations, per-slot preparation caps, generic put and +verification reuse, and flush-only immutable verified binding. Controlled +nonempty slots prove ART mechanics only; the current structural catalogue has +zero slots. CHECKERS reservation/currentness and live output authority remain +unavailable, so this does not activate production execution or public intake. + ## Critical Dependency Map ```text @@ -533,6 +551,7 @@ Delivered foundations (not a claim of full public integration) task/assignment/Submission lineage + hidden intake/creation ARCH-03D exact approved historical guide through durable intake handoff ARCH-04B hidden verified Submission input (production authority deny-only) + ARCH-04B2 hidden checker output store/recovery/binding (reservation and authority unavailable) shared dispatcher + exact-authorized hidden assignment invalidation ARCH-03C2 invalidation producer + first handler registration ARCH-03C3 exact manager task create/screen/release + replay @@ -543,7 +562,7 @@ Delivered foundations (not a claim of full public integration) | v Remaining integration - checker output custody + durable evaluation + live authority + remediation + durable evaluation + live authority + remediation -> public intake and immutable admitted Submission cutover -> durable current post-submit result + required checks pass | @@ -653,10 +672,11 @@ remaining trace sequence is: removes the private TASK/PROJECTS lookup at final durable handoff. No public preparation or Submission endpoint is activated. - Post-submit admission: after delivered `POL-07B` and `ARCH-03C`, delivered hidden - [ARCH-04B input materialization](../.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md) - leads to `ARCH-04B2 -> 04C -> 04D -> 04E` for output custody, durable results, - live authority and routing. - An ART-owned output/log custody child precedes `04C` final completion. + [ARCH-04B input materialization](../.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B.md) + and [ARCH-04B2 output custody](../.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B2.md) + lead to `04C -> 04D -> 04E` for durable results, live authority and routing. + ARCH-04C must accept the exact empty output set from the current structural + catalogue; nonempty 04B2 test slots do not claim a live evaluator capability. AUTH-OUTBOX-02 delivers shared live authority, phase audit custody and Celery delivery/recovery scans over CON-02B. Delivery termination is bounded by a 300-second hard limit under prefork. diff --git a/docs/spec_artifact_storage_service.md b/docs/spec_artifact_storage_service.md index 70cfc778e..19de5379c 100644 --- a/docs/spec_artifact_storage_service.md +++ b/docs/spec_artifact_storage_service.md @@ -17,7 +17,7 @@ or Flow Node. FastAPI and Celery composition roots -> ExternalServiceAdapterFactory[ArtifactStoreBootstrap] -> PostgreSQL exact namespace claim --> initialized ArtifactStore v2 +-> initialized ArtifactStore -> LocalStorageAdapter -> S3CompatibleArtifactStore -> MinIO integration @@ -319,7 +319,7 @@ changes. A subsequently exhausted retry job may be the source of a new chained attempt. The earlier source can never be reused, including after the first attempt succeeds or fails and regardless of a different idempotency key. -## ArtifactStore v2 Port +## ArtifactStore Port The asynchronous port is: @@ -355,24 +355,33 @@ SubmissionBundlePreparationCommand.prepare(SubmissionBundlePreparationRequest) SubmissionAdmissionConsumptionPort.consume(SubmissionAdmissionConsumptionRequest) GuideDocumentManifestPort.load(GuideDocumentManifestRequest) GuideDocumentGrant.open(opaque_document_handle) -ArtifactBindingPort.bind_checker_output(CheckerOutputBindingRequest) +CheckerOutputBindingPort.bind_checker_output(CheckerOutputBindingRequest) PostSubmissionMaterializationPort.materialize(PostSubmissionEvaluationRequest, consumer) CheckerArtifactOutputPort.store(CheckerOutputArtifactRequest) +CheckerArtifactOutputPort.recover(CheckerOutputSelector) ArtifactOperatorReadPort.list_bindings/list_replicas/list_receipts/ get_verification_job/get_recovery_attempt/list_audit_events/admission_usage(...) ArtifactOperatorRecoveryPort.retry_verification(ArtifactRecoveryRequest) ``` -Every protected durable mutation in this port family consumes one opaque, -process-local `PreparedAuthorizationHandle` at its transaction boundary. The -hidden HTTP submission-bundle preparation request carries immutable +Every protected durable mutation in this port family obtains fresh, +action-specific authority at its protected transaction boundary. Opaque +`PreparedAuthorizationHandle` values remain process-local implementation +custody; checker-output store/recovery requests do not carry them. The hidden +HTTP submission-bundle preparation request carries immutable `ActorIdentityFacts`, request and correlation IDs, contributor selectors, packet metadata, and an asynchronous byte source rather than a prepared handle; -ART obtains separate prepared handles +ART obtains separate prepared handles internally immediately before materialization and durable put intent. The typed methods fix their expected actions, and handles never enter route schemas, outbox/Celery payloads, provider interfaces, or serialized contracts. +CHECKERS owns the post-submit materialization and output-custody consumer ports +in `app.modules.checkers.api.materialization` and +`app.modules.checkers.api.output_custody`. ART imports those public contracts and +supplies their concrete custody implementations through composition. CHECKERS +execution does not import ART APIs or private ART modules to consume them. + `GuideArtifactIngestRequest` contains prepared authority, exact project, guide, guide-source snapshot and item IDs, logical role, and authorized byte source. The service resolves the server-bound media type from the locked snapshot item. @@ -402,14 +411,23 @@ admission and admitted verified replica through its own records plus the TASK-ow an async consumer receives a scoped read-only file view. The typed result carries material custody plus the existing closed evaluation value; it is not a durable checker result. Production composition denies before protected reads until ARCH-04D. -`CheckerOutputArtifactRequest` contains the fixed -service's prepared authority, task/submission/checker-run IDs, logical role, -and generated byte source. The Submission binding is instead the terminal result of the typed +`CheckerOutputArtifactRequest` contains a CHECKERS-owned reservation selector +and generated byte source. `store` obtains fresh authority for each phase, +resolves the exact evaluation/run/checker-worker-lease/output-slot facts, applies the +owner-declared media type and byte ceiling, and uses generic ART put and +independent verification. `recover(selector)` uses the same current owner and +authority facts to recover the stable operation without a byte source; it never +puts or regenerates bytes. The current production composition supplies an +unavailable reservation port and deny-only authority, so neither method is a +live evaluator capability. The Submission binding is instead the terminal result of the typed `SubmissionAdmissionConsumptionPort`: it consumes one ready admission against the exact TASK-supplied Submission identity and locked lineage. The remaining -checker-output binding request contains its required verified content ID and -exact CheckerRun selector. Each typed method fixes its own authority; no request -selects the action. +checker-output binding request contains the same exact selector plus the put +attempt and independent verification receipt identities. A flush-only ART +participant re-resolves current reservation and authority, verifies complete +receipt/job/attempt/replica/content ancestry, and creates or replays one +immutable run/slot binding in the caller's transaction. Each typed method fixes +its own authority; no request selects the action. `ArtifactRecoveryRequest` contains the authenticated Operator context, exact source verification-job ID, reason, client idempotency key, expected source-job CAS version, and canonical authorization resource facts. Reason-bound Operator @@ -548,7 +566,7 @@ activation evidence. Cloudflare R2 has no v0.1 runtime mode, credential issuer, sidecar, secret contract, configuration profile, or deployment proof. Any future provider -initiative must perform current discovery and pass the same ArtifactStore v2 +initiative must perform current discovery and pass the same `ArtifactStore` conformance suite before any runtime configuration is introduced. Production rejects `local`, plaintext HTTP, loopback/local endpoints, invalid @@ -731,7 +749,7 @@ and the AWS live readiness profile. ## LocalStorage Adapter -LocalStorage implements the same v2 port and conformance vectors. It uses a +LocalStorage implements the same port and conformance vectors. It uses a pre-provisioned private configured root, opaque content-derived paths, exclusive no-overwrite publication, bounded off-event-loop file I/O, no-follow path handling, private permissions, full read verification, and sanitized @@ -753,8 +771,8 @@ must then restart and revalidate the complete layout. Any different name, owner, type, link count, or mode is an integrity incident and must not be repaired by this procedure. -The v1 local provider metadata for retain/release/provider receipts is removed -in the v2 clean cut. No compatibility adapter or dual format remains. +The initial v0.1 provider contract has no retain/release methods or +provider-owned receipt metadata. No compatibility adapter or dual format exists. ## Ingest Choreography @@ -892,7 +910,8 @@ approved operational change. | `WORKSTREAM_ARTIFACT_SUBMISSION_ZIP_MAXIMUM_COMPRESSION_RATIO` | `100` | `10000` | Maximum expanded-to-compressed ratio for one file. | | `WORKSTREAM_ARTIFACT_SUBMISSION_ZIP_MAXIMUM_INSPECTION_SECONDS` | `300` | `1800` | Complete synchronous inspector deadline inside the preparation deadline. | -04B2 uses the same ledger to reserve a submission workspace's complete expanded +The delivered post-submit materializer uses the same ledger to reserve a +submission workspace's complete expanded byte count and entry count before projection. Workspace reservations participate in aggregate byte/free-space checks but do not become artifact storage. Cleanup uses the workspace's own bounded entry reservation rather than the live @@ -904,7 +923,7 @@ the already-inspected manifest into one callback-scoped sealed tree using descriptor-relative creation. Regular files use fixed `0400` or normalized executable `0500`; directories use `0500`. Checker adapters receive no scratch path, shell, subprocess, or execution primitive. The tree is destroyed first; -only then do bounded, path-redacted platform/default results return. 04B2 neither runs +only then do bounded, path-redacted platform/default results return. ARCH-04B neither runs project-policy primitives nor persists final checker evidence. Pre-submission checker catalogue configuration is separate from ZIP inspection @@ -995,8 +1014,9 @@ and the owning durable source can be regenerated, the original caller may prepare it again and compare the complete digest/size. Identical bytes may replay the original operation after admission capacity is reacquired. Changed guide bytes require a new guide version with its own document declaration, -internal snapshot and setup; changed checker output requires a new checker-run -attempt. Neither reuses the old operation, snapshot, +internal snapshot and setup. Changed checker-output bytes conflict with the +existing run/slot operation; a distinct logical run or owner-issued slot is +required. Neither reuses the old operation, snapshot, or binding identity. A generator that cannot reproduce exact bytes fails its old infrastructure attempt instead of fabricating replay. Bytes are never placed in PostgreSQL, Redis, Celery payloads, logs, or audit. @@ -1475,6 +1495,21 @@ implementation identity. Pre-submit evidence and post-submit execution name the same verified admission and exact binding. Checker logs and generated outputs become artifact bindings. +ARCH-04B2 supplies the hidden checker-output custody boundary. The typed +`store` and byte-free `recover(selector)` paths enforce one owner-issued slot +ceiling, reuse generic admission, put observation and verification, and return +only exact verified identities. The flush-only binding participant preserves +the full verified ancestry and never privately reads CHECKERS rows. Store and +recovery resolve owner facts only through the typed CHECKERS port; the separate +Operator resource-to-project lookup remains an explicit read concern and is not +an output-admission path. Production +composition remains unavailable because CHECKERS reservation/currentness and +live output authority are intentionally deferred. The current structural +catalogue declares zero output slots; controlled nonempty test reservations +prove ART mechanics only. ARCH-04C must complete a valid evaluation with an +empty output set and compose bindings only for slots actually reserved by its +owner. + Transient post-submit storage unavailability leaves the task in `evaluation_pending` and uses checker retry infrastructure. A provider object confirmed missing after its content was bound is a terminal artifact incident @@ -1506,12 +1541,10 @@ forbidden in production. ## Migration And Removal -Implementation is a clean cut: +The initial v0.1 implementation is a clean cut: -- committed-source preparation and LocalStorage private refactoring land first - without changing the active v1 port; -- ArtifactStore v1 methods, active callers, LocalStorage's public adapter - surface, and schema then migrate atomically in the v2 clean-cut chunk; +- committed-source preparation and the provider-neutral `ArtifactStore` are the + only writable storage path; - the Flow Node backend is removed from configuration without an alias; - obsolete caller-owned URI/hash and storage-scheme fields are removed in their owning guide/submission cutovers; @@ -1520,9 +1553,9 @@ Implementation is a clean cut: normal submission creation still enters evaluation automatically; - no dual write, nullable shadow field, fake verified backfill, fallback adapter, compatibility constructor, or second factory remains; -- the v0.1 baseline contains no v1 artifact schema or fabricated backfill; - pre-v0.1 development databases/storage namespaces are reprovisioned and - authoritative bytes are reingested through v2; +- the v0.1 baseline contains only the current artifact schema and no fabricated + backfill; pre-v0.1 development databases/storage namespaces are reprovisioned + and authoritative bytes are reingested through the current port; - the v0.1 baseline installs the durable admission ledger and prepared put-attempt tables; - the v0.1 baseline includes polymorphic contract-v2 operation @@ -1543,6 +1576,17 @@ Implementation is a clean cut: The baseline proves fresh installation, fail-closed refusal of old/nonempty databases, and no artifact bytes in PostgreSQL. It is not downgradable. +The checker-output custody migration adds exact Submission version and narrow +checker-request digest fields to checker-output attempts plus verified +put/receipt ancestry on checker bindings. A successful binding atomically seals +the terminal attempt/job and replica identity under the existing +job -> replica -> attempt -> content lock order. The database rejects later +ancestry changes, including writers using an older transaction snapshot; replica +health and availability remain mutable. Rolling back publication also rolls back +the seals. It refuses any retained checker-output +attempt or binding whose missing evaluation/slot digest or verified ancestry +cannot be proven; it neither invents those facts nor deletes retained data. + ## Verification Strategy ### Remaining v0.1 dependency order @@ -1550,8 +1594,8 @@ databases, and no artifact bytes in PostgreSQL. It is not downgradable. The pre-submit checker materializer is a mandatory part of preparation, so its fixed-service AUTH activation must merge after hidden ART-04B1-04B3 and before contributor preparation is activated. ARCH-04B delivers hidden exact post-submit -input with deny-only production authority. ARCH-04B2 checker-output custody is -next; ARCH-04C owns durable CHECKERS execution/results, and ARCH-04D owns their +input with deny-only production authority. ARCH-04B2 hidden checker-output +custody is delivered; ARCH-04C now owns durable CHECKERS execution/results, and ARCH-04D owns their exact live activation. ARCH-04E separately owns TASK routing. Historical ART-06A/06B labels do not open duplicate implementation lanes. This ordering @@ -1597,14 +1641,14 @@ instances or an external deployment barrier. ## Deferred Flow Node Adapter `FN-ART-002` is a separate inactive initiative. It may later implement the same -ArtifactStore v2 port and conformance vectors. It cannot change product records +`ArtifactStore` port and conformance vectors. It cannot change product records or services, cannot introduce a runtime fallback, and requires an explicit maintenance cutover after v0.1 is proven. ## Deferred R2 Adapter Cloudflare R2 is outside the v0.1 dependency graph. Any later provider -initiative must perform current provider discovery, implement the same ArtifactStore v2 +initiative must perform current provider discovery, implement the same `ArtifactStore` contract, pass its own conformance and security proof, and use an explicit no-fallback maintenance cutover. diff --git a/docs/spec_authorization_service.md b/docs/spec_authorization_service.md index d5d274dc7..d5e6c49b1 100644 --- a/docs/spec_authorization_service.md +++ b/docs/spec_authorization_service.md @@ -537,8 +537,9 @@ runtime `ActionOwner` values or the current implementation boundaries. In particular, the XINT-06B grouping corresponds to runtime `WS-AUTH-001-ART-06A` for post-submit materialization and `WS-AUTH-001-ART-06B` for checker-output write/binding. The current replacement -activation contract is ARCH-04D, after delivered ARCH-04B hidden input and -remaining ARCH-04B2 output custody/ARCH-04C durable execution. It does not reopen XINT-06B as a parallel implementation lane. Likewise ARCH-02G/02H +activation contract is ARCH-04D. ARCH-04B hidden input and ARCH-04B2 output +custody are delivered; ARCH-04C durable execution/results is next, followed by +ARCH-04D activation. This does not reopen XINT-06B as a parallel implementation lane. Likewise ARCH-02G/02H are replacement implementation boundaries, not automatic registry renames. Read exact runtime ownership from the typed catalogue. No planning-only change may promote or reassign an action.