diff --git a/.ci/behavior-ownership/partition.v1.json b/.ci/behavior-ownership/partition.v1.json index 85c4a3c7d..5bdf6758a 100644 --- a/.ci/behavior-ownership/partition.v1.json +++ b/.ci/behavior-ownership/partition.v1.json @@ -256,6 +256,10 @@ "group": "artifacts", "target": "backend/app/modules/artifacts/api/submission_admission.py" }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/api/submission_materialization.py" + }, { "group": "artifacts", "target": "backend/app/modules/artifacts/api/submission_preparation.py" @@ -288,6 +292,14 @@ "group": "artifacts", "target": "backend/app/modules/artifacts/operator.py" }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/post_submit_materialization.py" + }, + { + "group": "artifacts", + "target": "backend/app/modules/artifacts/post_submit_selection.py" + }, { "group": "artifacts", "target": "backend/app/modules/artifacts/pre_submit_attempts.py" @@ -1260,6 +1272,10 @@ "group": "lifecycle", "target": "backend/app/modules/tasks/api/submission_history.py" }, + { + "group": "lifecycle", + "target": "backend/app/modules/tasks/api/submitted_bundle.py" + }, { "group": "lifecycle", "target": "backend/app/modules/tasks/api/task_detail.py" @@ -1316,6 +1332,10 @@ "group": "lifecycle", "target": "backend/app/modules/tasks/submission_history.py" }, + { + "group": "lifecycle", + "target": "backend/app/modules/tasks/submitted_bundle.py" + }, { "group": "auth", "target": "backend/app/schemas/auth.py" @@ -1453,7 +1473,7 @@ "target": "backend/scripts/validate_test_lane_evidence.py" } ], - "authority_digest": "a9279c1c9d3150a8ff5d6ba8af2f6305f919f5082e2b885aff2d84baeabdd553", + "authority_digest": "45c8605a1e6b8a29cd5c9fb59771f423f8e87f04ed7da07833139cc1f657e22b", "protected_base_commit": "7676ce4347db0c9694962a9b587a20765e16eac6", "schema": "workstream.behavior-ownership-partition.v1" } diff --git a/.commitrail/INDEX.md b/.commitrail/INDEX.md index f6179b945..051d526d1 100644 --- a/.commitrail/INDEX.md +++ b/.commitrail/INDEX.md @@ -8,12 +8,12 @@ for current product capability. |---|---|---| | [WS-DB-002](initiatives/WS-DB-002/OVERVIEW.md) | Complete | Shared UUIDv7 record generation, native-UUID relationships and fresh v0.1 baseline; natural-owner retry custody and aligned CI/local setup | | [WS-MCP-002](initiatives/WS-MCP-002/OVERVIEW.md) | Planned | Three self-service tools delivered through WS-MCP-002-02; 24 tools remain and WS-MCP-002-03 administrative reads are next | -| [WS-ARCH-001](initiatives/WS-ARCH-001/OVERVIEW.md) | Planned | Canonical ARCH-04A delivered; Automatic unified setup delivered; POL-05/06 manager review and separate approvals delivered; POL-07B internal checker phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; exact post-submit materialization next | -| [WS-ART-001](initiatives/WS-ART-001/OVERVIEW.md) | Planned | Exact post-submit materialization after guide/checker contracts | -| [WS-AUTH-001](initiatives/WS-AUTH-001/OVERVIEW.md) | Planned | Shared dispatcher authority and phase custody after delivered CP05; POL-04B consumes completed finalization authority; AUTH-12F4 supplies proposal authority; POL-05B public composition delivered; AUTH-12G post-policy authority delivered; POL-06B public composition delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; exact post-submit materialization next | -| [WS-CON-001](initiatives/WS-CON-001/OVERVIEW.md) | Planned | CP06 selected-policy validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; exact post-submit materialization next | -| [WS-AUTH-003](initiatives/WS-AUTH-003/OVERVIEW.md) | Planned | TASK/checker canonical history authority delivered and alternate gate removed; continue boundary recovery as manager activation/public guide integration reaches remaining consumers | -| [WS-POL-003](initiatives/WS-POL-003/OVERVIEW.md) | Planned | Unified setup, separate draft proposals and guide document intake delivered; Hidden proposal review/correction/pre-policy approval custody delivered; AUTH-12F4 proposal authority delivered; POL-05B public review/approval/manual correction dispatch delivered; POL-06A hidden post-policy projection/read/approval/correction delivered; AUTH-12G live authority delivered; POL-06B public access and automatic derivation delivered; POL-07A ART pre-submit attempt recovery delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; exact post-submit materialization next | +| [WS-ARCH-001](initiatives/WS-ARCH-001/OVERVIEW.md) | Planned | Canonical ARCH-04A delivered; Automatic unified setup delivered; POL-05/06 manager review and separate approvals delivered; POL-07B internal checker phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-ART-001](initiatives/WS-ART-001/OVERVIEW.md) | Planned | Hidden exact post-submit input materialization delivered; ARCH-04B2 output custody next | +| [WS-AUTH-001](initiatives/WS-AUTH-001/OVERVIEW.md) | Planned | Shared dispatcher authority and phase custody after delivered CP05; POL-04B consumes completed finalization authority; AUTH-12F4 supplies proposal authority; POL-05B public composition delivered; AUTH-12G post-policy authority delivered; POL-06B public composition delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-CON-001](initiatives/WS-CON-001/OVERVIEW.md) | Planned | CP06 selected-policy validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [WS-AUTH-003](initiatives/WS-AUTH-003/OVERVIEW.md) | Planned | TASK/checker canonical history authority delivered and alternate gate removed; AUTH-18 public manager activation and ARCH-03D hidden intake delivered; ARCH-04B hidden input delivered; continue boundary recovery with ARCH-04B2 output custody next | +| [WS-POL-003](initiatives/WS-POL-003/OVERVIEW.md) | Planned | Unified setup, separate draft proposals and guide document intake delivered; Hidden proposal review/correction/pre-policy approval custody delivered; AUTH-12F4 proposal authority delivered; POL-05B public review/approval/manual correction dispatch delivered; POL-06A hidden post-policy projection/read/approval/correction delivered; AUTH-12G live authority delivered; POL-06B public access and automatic derivation delivered; POL-07A ART pre-submit attempt recovery delivered; POL-07B internal phase service delivered; CP06 validation delivered; CP07 hidden activation/binding delivered; AUTH-12H live manager authority delivered; ARCH-03A complete internal guide context delivered; CP08 lineage and minimal writers delivered; ARCH-03B1 detached metadata and ARCH-03B2 ready and ARCH-03B3 management/operational queue facts and ARCH-03B4 hidden task detail and ARCH-03B5 current contributor/manager work context and ARCH-03B6 locked-context and ARCH-03B7 requirements projections delivered; ARCH-03B8 hidden task audit evidence delivered; AUTH-OUTBOX-01 unavailable dispatcher contract delivered; CON-02B hidden delivery/recovery custody delivered; AUTH-OUTBOX-02 live authority/audit binding and worker integration delivered; ARCH-03B9 hidden exact assignment invalidation delivered; ARCH-03C1 exact reconciler authority and decision receipts delivered; ARCH-03C2 atomic publication and registered prefork delivery delivered; ARCH-03C3 exact manager task readiness and replay delivered; ARCH-03C4 exact-authorized public queues delivered; ARCH-03C5 exact-authorized detail and requirements delivered; ARCH-03C6 exact-authorized locked-context reads delivered; ARCH-03C7 bounded audit history delivered; CP05A public Finance policy administration delivered; AUTH-18 public manager activation/context delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | | [WS-REV-001](initiatives/WS-REV-001/OVERVIEW.md) | Planned | Shared acceptance/source and existing fence foundations; human hidden review work remains independently dependency-gated | | [WS-QUAL-002](initiatives/WS-QUAL-002/OVERVIEW.md) | Planned | Populate subsystem ownership before changed-line mutation work | | [WS-QUAL-003](initiatives/WS-QUAL-003/OVERVIEW.md) | Planned | Audit and prune test proof, add missing safety cases, decompose oversized test modules | diff --git a/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md b/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md index 383a6c811..eeae50a29 100644 --- a/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-ARCH-001/OVERVIEW.md @@ -14,7 +14,7 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), - Current boundary: one CHECKERS catalogue, compiler/parser and implementation per checker ID serve active policy consumers; production post-submit phase execution remains unavailable. -- Next usable boundary: exact post-submit materialization (ARCH-04B), following delivered hidden approved-guide intake (ARCH-03D). +- Next usable boundary: checker output custody (ARCH-04B2), following delivered hidden input materialization (ARCH-04B). Production execution and live materialization authority remain unavailable. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation using the existing CP07 operation. [CP05A](WS-ARCH-001-CP05A.md) supplies public Finance policy administration and recoverable draft selectors. diff --git a/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B.md b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B.md new file mode 100644 index 000000000..902c9e6db --- /dev/null +++ b/.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-04B.md @@ -0,0 +1,250 @@ +# ARCH-04B — Exact verified post-submit input + +- Initiative: `WS-ARCH-001` +- Durable disposition: `Complete` +- Intended merge outcome: Hidden verified Submission input with async scoped reads; production authority remains deny-only. + +Dependencies: 04A, POL-07, ARCH-03C, ARCH-03D and merged 02H. +Risk: L1. Migration head: `0006_history_read_authority`; no migration planned. + +## Intent + +Give the fixed post-submit checker service bounded access to the exact verified +ZIP consumed by one immutable Submission. Replace the unused generic +`ArtifactMaterializationPort` / `BindingMaterializationRequest`; no compatibility +alias remains. Public submission intake, durable execution, output storage, +routing, review and acceptance remain deferred to their existing chunks. + +The consumed ART admission alone cannot establish the Submission's TASK-owned +status and frozen policy stamps. Add one small TASK read port rather than reading +TASK tables inside ART or repurposing contributor/history projections. Its query +is project/task/submission-qualified from the start. It returns detached scalar +ownership, artifact and frozen-policy references, not packet content or policy +bodies. No TASK mutation or current-guide rebasing is introduced. + +## Design + +1. Reuse the closed `PostSubmissionEvaluationRequest` from 04A. It selects exact + project/task/assignment/Submission and version, binding/content, evaluation + request/generation/digest and locked context. It cannot choose a provider + object, replica, namespace, path, arbitrary binding list or service identity. +2. Production composition supplies explicit deny-only materialization authority. + Its preflight denies before database/provider/scratch access. Controlled test + authority allows hidden owner proof; this is not live AUTH activation. Exact + server-selected material facts are authorized before provider access. ARCH-04D + supplies live service admission and generation/replay/revocation enforcement. +3. A short read transaction resolves public TASK facts and ART's consumed admission, + exact submission binding/content, admitted replica, verification receipt and + active namespace. Reuse canonical namespace validation. Read fresh scalar rows, + not stale cached ORM objects. Reject missing, foreign, unconsumed, nonverified or + inconsistent material before storage access. The immutable Submission selects + historical locked policies, never today's guide. +4. End the transaction before I/O. Stream through the provider-neutral ArtifactStore + into ArtifactPreparationService. Recompute complete digest/size, inspect the ZIP + with SubmissionArchiveInspector and compare the rebuilt semantic-manifest hash. +5. Reuse `_process_prepared_submission` and `project_and_run` to supply an asynchronous + consumer with a typed read-only entry/bounded-file view. The public call remains + async; blocking projection runs off-loop. The existing synchronous pre-submit + wrapper and the async post-submit consumer share one projection context manager; + no second extraction implementation is introduced. The consumer returns only the existing + closed PostSubmissionEvaluationResult, validated against its request. ART returns + that value together with typed material custody facts after cleanup. No raw path, + provider handle, credential or live tree survives the callback. Cancellation + drains processing before releasing the existing scratch reservations. Abort cancels + and drains the in-flight consumer; abort during projection prevents consumer + entry. Close/revoke the shared projection off-loop in a cancellation-resistant + finally. Prove pre/post parity through this one primitive. +6. Re-read and compare the same persisted selection after I/O before returning. + A changed Submission/replica rejects the result. No transaction or PREP survives + provider I/O. This is materialization freshness, not final execution/result + authority: ARCH-04C/04D still own current attempts and fresh final publication. + Full structural packet construction remains 04C; ART validates only the byte + and locked-context facts owned by its materialization boundary. + +### Exact selection predicates + +TASK returns project/task/assignment/Submission ID and version, contributor ID, +predecessor ID and version, status, contribution-policy version, admission/binding/content IDs, +and guide/source/effective/pre/post/review/revision identities, versions/generations +and hashes. Require Submission status `submitted`, exact project/task/assignment +and Submission/version, and exact binding/content. Compare the full stamped context +with request.expected_context and request.structural_input.observed_context. Other +structural text remains outside this port's authority. + +ART requires the stored admission to be consumed by that exact Submission/version, +with matching project/task/assignment/contributor/content and predecessor. Require +binding project/resource_type= submission/resource_id/logical_role= +submission_bundle_original/scope_version=1/content to match exactly. Require content +ZIP media type, digest and size to match the admission and request. The admitted +replica must match content, active namespace and canonical content-derived provider +reference, with verified/available/valid states. Its receipt must be verified with +matching digest/size and a verification job for that replica. Rebuild the exact +semantic manifest from bytes and compare its hash to admission; require the request's +file manifest paths/hashes/sizes and package hash to match server-owned byte facts. + +Post-I/O selection compares the complete detached TASK facts plus ART admission, +binding/content/replica/receipt IDs, archive and manifest commitments, replica +states and namespace/provider identity. No current-guide lookup substitutes for +Submission stamps; no run-generation state is invented before 04C/04D. + +## Bounded change + +- `backend/app/modules/artifacts/api/submission_materialization.py` (new) and + `backend/app/modules/artifacts/api/__init__.py`. +- `backend/app/modules/artifacts/post_submit_materialization.py` and + `backend/app/modules/artifacts/post_submit_selection.py` (new owner files). +- `backend/app/modules/artifacts/preparation.py` (shared processor wording only) and + `backend/app/modules/artifacts/submission_archive.py` (one shared projection lifetime). +- `backend/app/modules/tasks/api/submitted_bundle.py` and + `backend/app/modules/tasks/submitted_bundle.py` (new read port and owner). +- `backend/app/modules/tasks/api/__init__.py`, + `backend/app/adapters/tasks/__init__.py`, `backend/app/adapters/artifacts/__init__.py`. +- `backend/app/interfaces/artifact_operations.py` (remove superseded empty contract). +- New `backend/tests/test_post_submit_materialization.py`, + `backend/tests/test_post_submit_selection.py`, + `backend/tests/post_submit_materialization_helpers.py`; + `backend/tests/tasks/submission_lineage_support.py` (scope the shared test + admission lookup to its exact put attempt); existing + `backend/tests/test_artifact_architecture.py`, + `backend/tests/test_checker_materialization.py`, `backend/tests/test_submission_archive.py` and + `backend/tests/architecture/test_module_boundaries.py` for affected proof. +- Existing lane catalogue and ownership/structure ledgers and their tests only + to register affected paths and preserve existing proof, never weaken gates. +- This contract, current ARCH/AUTH/POL overviews/plans/maps, ART/CON overviews, `.commitrail/INDEX.md`, + `docs/roadmap_status.md`, `docs/spec_artifact_storage_service.md`, + `docs/architecture_checker_framework.md`, `docs/architecture_data_model.md`, + `docs/engineering/authorization_activation_custody.md`, + `docs/operations_project_operating_manual.md`, `docs/spec_authorization_service.md`, + `planning/chunks/WS-ARCH-001-03B-task-assignment-api.md`, + `../WS-POL-003/planning/chunks/WS-POL-003-07-single-checker-service-port.md`, + `.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md`, `README.md` and applicable ART specs + only for this boundary and navigation identifying 04B2 as next; no 04B2 design + or implementation changes. + +Prohibited: new routes, TASK writes, policy/compiler changes, checker run/result +writes, REV packet access, generic downloads, worker activation, output ingestion, +new persistence states, data deletion, grant activation, serialized process handles, +private cross-owner imports, alternate scratch or provider implementations. + +## Acceptance criteria + +- Valid Local and MinIO originals yield the exact admitted files and executable + flags; output facts identify the stored Submission and ART custody. +- Denial and mismatched selectors fail before provider open or scratch use. + Default composition remains unavailable. Controlled generation/replay denial + proves the authority seam, not production generation custody. +- Full-byte digest/size or semantic-manifest mismatch prevents consumer invocation. + Retained views are revoked after success, consumer failure and cancellation; + quota/deadline failures release scratch. Existing ZIP-safety proofs remain. +- Real migrated PostgreSQL proves owner-qualified selection and independent-session + replica/state change during provider I/O; no row lock or open read transaction + spans that I/O. Valid controls and discriminating guard-removal probes must reach + the claimed behavioral assertion rather than fail during fixture setup. +- A real admitted/consumed fixture proves the complete stored lineage. Pure + contract and scratch tests use focused fixtures; no permissive fake is called + live authorization or durable admission proof. +- Remove the dead interface and update every traced caller/spec/test together. + Preserve pre-submit behavior and separate production execution unavailability. + +The full migrated fixture is the existing +`tests/tasks/test_submission_lineage.py::test_real_zip_admission_and_hidden_creation_copy_exact_assignment` +flow and `tests/tasks/submission_lineage_support.py::_verified_admission`: +real preparation, provider put, independent verifier publication, hidden Submission +creation and admission consumption. Reuse its helpers; do not use the reduced +`test_artifact_bindings_db.py` schema as full-lineage proof. + +Concrete commands (from `backend/`, with the existing local test services/env): + +```sh +.venv/bin/python scripts/run_isolated_tests.py --metadata-json /tmp/arch04b-focused.json --timeout-seconds 900 -- .venv/bin/python -m pytest tests/test_post_submit_materialization.py tests/test_post_submit_selection.py tests/test_checker_materialization.py tests/test_submission_archive.py tests/test_default_pre_submit_execution.py tests/tasks/test_submission_lineage.py -q +.venv/bin/python -m pytest tests/test_artifact_architecture.py tests/architecture/test_module_boundaries.py tests/test_ci_lane_catalogue.py -q +.venv/bin/ruff check app/modules/artifacts app/modules/tasks/api app/modules/tasks/submitted_bundle.py app/adapters/artifacts app/adapters/tasks tests/test_post_submit_materialization.py tests/test_post_submit_selection.py tests/post_submit_materialization_helpers.py +``` + +From repository root run `python3 scripts/check_markdown_links.py`, +`python3 scripts/check_commitrail_records.py --base-ref 5e35f635`, +`python3 scripts/check_stale_artifact_contracts.py`, +`python3 scripts/check_stale_workstream_wording.py`, and `git diff --check 5e35f635 HEAD`. +Use a unique metadata path per isolated rerun. The named test paths above are implemented. + +Run focused tests with `backend/scripts/run_isolated_tests.py` against migrated +PostgreSQL and existing MinIO; the named files above are the implementation proof. Run Ruff, +architecture/ownership/lane tests, `git diff --check`, existing markdown-link and +Commitrail validators, then all required hosted lanes. Coverage is diagnostic. +Do not claim production execution or live AUTH from these tests. + +## Risk and review routing + +Size exception: this L1 input boundary exceeds the preferred 500 changed lines. +The added TASK read, ART selection/lifetime and real Local/MinIO proof must agree +in one reviewable change; splitting them would leave an unused port or an +unverified reader. Current-navigation updates account for many paths. There are +no schema changes, live AUTH activation or additional product workflows. + +Required focused reviews: plan review before implementation; architecture/reuse, +security, QA/test-delta, product-ops/documentation, senior engineering and CI +integrity against a clean candidate. Human review focus: exact source selection, +transaction-free I/O, callback lifetime, fail-closed composition and accurate +separation from future durable execution and live authorization. + +## Proof refinements from review + +The stored semantic-manifest commitment and the caller file list are distinct +checks. A real isolated-database corruption fixture changes only the retained +admission/evidence commitment, with custody triggers disabled solely for that +fixture; the unchanged request must then fail before consumer entry. Normal +writes remain protected. Executable and plain ZIP members prove the callback's +exact executable flags. A paused provider stream plus independently observed +PostgreSQL activity/locks proves the selection transaction ended before byte I/O. +A result for another evaluation generation must fail with scratch/view cleanup. + +Current ARCH/AUTH navigation, checker/storage specs and the operating manual +identify hidden input as delivered and output custody as next. Production remains +deny-only. New-owner docstrings explain these boundaries without changing CI gates. + +### External review correction + +The foreign-record regression provisions two complete valid projects, contributors, +assignments, admitted originals and immutable Submissions in one migrated database, +sharing the configured store and fixed service identities. Both unmixed lineages +must materialize successfully. Mixes in both directions substitute each project, +task, assignment, Submission, binding and content identifier, plus coherent foreign +subsets. Every mix must deny before provider open, scratch preparation or consumer +entry. This closes the gap left by nonexistent-ID negatives; no production change +or compatibility path is introduced. AUTH-003 index navigation and authorization +activation custody now agree on ARCH-04B2 before ARCH-04C/04D. The roadmap already +records that same boundary and needs no capability change for this proof repair. + +## Evidence + +Plan review passed on the expanded contract before product implementation. The +existing full ZIP admission/Submission fixture passed on migrated PostgreSQL. +Focused implementation proof on clean `8fee6012`: 98 tests passed, including full +Local/MinIO admission/consumption/materialization, exact executable metadata, +wrong selectors, stored and supplied manifest mismatches, foreign result rejection, +provider-stream transaction/lock observation, concurrent replica/Submission drift, +async cancellation/deadline/quota cleanup, and retained pre-submit/archive proof. +The ownership, boundary and lane suite passed 227 cases. Module, ownership, +structure and documentation checks passed; no guards were weakened and no required +tests were removed. Production AUTH/currentness and public HTTP remain unavailable +and are not claimed by these owner tests. + +Three isolated guard-removal probes on that clean target made the unchanged +regressions fail at their intended assertions: removing only the stored-manifest +comparison, replacing executable flags with null, or removing result/request +validation. Earlier development probes also caught removed final-selection +freshness, request/Submission identity matching and supplied-manifest validation; +those earlier runs are not claimed as exact-clean-target execution. Tests and +metadata for each reviewable candidate are bound to its committed head; hosted +full-suite evidence remains in the PR checks rather than a durable status claim. + +### Plan review + +Read-only design review identified the missing TASK read boundary and selected +existing closed CHECKERS request/result contracts over generic object returns. +The review required exact selection predicates, a reachable full admission fixture, +async projection lifetime and executable commands. Those corrections are adopted; +implementation follows the reviewed design without live AUTH or execution activation. + + +The separate [ARCH-04B2 output-custody child](planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md#arch-04b2--separate-art-output-custody-child) remains Planned. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md index 535e245c8..0fe4ed520 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/CHUNK_MAP.md @@ -4,7 +4,8 @@ Use the [current dependency contract](PLAN.md#current-dependency-contract). The [preserved map](../pre-cutover/CHUNK_MAP.md) retains the complete original work accounting. Foundations through 02H and CP04B are complete; none restart. AUTH-18 public manager activation/context and ARCH-03D hidden approved-guide intake are delivered. -Exact post-submit materialization is next; public intake remains deferred to ARCH-02I. +ARCH-04B hidden exact post-submit input is delivered. ARCH-04B2 output custody is next; +public intake remains deferred to ARCH-02I. | Boundary | Owner outcome | Risk | Current dependency | |---|---|---|---| @@ -31,15 +32,15 @@ Exact post-submit materialization is next; public intake remains deferred to ARC | [WS-ARCH-001-03C4](../WS-ARCH-001-03C4.md) | Exact-authorized public task queues | L1 | Complete; contributor, manager and operational projections with signed pagination | | [WS-ARCH-001-03C5](../WS-ARCH-001-03C5.md) | Exact-authorized Contributor and Manager detail and requirements | L1 | Complete; canonical projections, historical policy custody and atomic read evidence | | [WS-ARCH-001-03C6](../WS-ARCH-001-03C6.md) | Distinct exact-authorized locked-context reads | L1 | Complete; bounded Audit Authority history access delivered by 03C7 | -| [WS-ARCH-001-03C7](../WS-ARCH-001-03C7.md) | Exact-authorized bounded task history | L1 | Complete; AUTH-18 public guide activation delivered; ARCH-03D hidden approved-guide intake delivered; exact post-submit materialization next | +| [WS-ARCH-001-03C7](../WS-ARCH-001-03C7.md) | Exact-authorized bounded task history | L1 | Complete; AUTH-18 public guide activation delivered; ARCH-03D hidden approved-guide intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | | [WS-ARCH-001-03C](chunks/WS-ARCH-001-03C-auth-task-readiness.md) | Exact task/assignment public activation and integrated readiness proof | L1 | Complete through 03C7 audit history, 03C4 queues, 03C5 detail/requirements and 03C6 locked-context reads; AUTH-18 public guide activation delivered; ARCH-03D hidden intake delivered; public intake cutover remains separate | | [WS-ARCH-001-04A](../WS-ARCH-001-04A.md) | CHECKER post-submit contract and registered evaluator conformance | L1 | Complete canonical catalogue, phase facts and structural conformance; consumed by delivered POL-04B and POL-07B | -| [WS-ARCH-001-04B](chunks/WS-ARCH-001-04B-art-post-submit-materialization.md) | ART exact verified Submission materialization | L1 | Planned after 04A, POL-07, 03C and merged 02H | +| [WS-ARCH-001-04B](../WS-ARCH-001-04B.md) | ART exact verified Submission materialization | L1 | Complete hidden verified input with async scoped reads; production authority remains deny-only until 04D | | [WS-ARCH-001-04B2](chunks/WS-ARCH-001-04B-art-post-submit-materialization.md#arch-04b2--separate-art-output-custody-child) | ART generated-output/log custody and verified binding | L1 | 04A public request/run facts plus merged ART foundations; no CHECKERS private lookup | | [WS-ARCH-001-04C](chunks/WS-ARCH-001-04C-checker-current-result.md) | CHECKER hidden durable current output and supersession behavior | L1 | Planned after 04A/04B/04B2; production remains deny-only | | [WS-ARCH-001-04D](chunks/WS-ARCH-001-04D-auth-post-submit-activation.md) | AUTH exact fixed-service post-submit activation (replaces XINT-06B) | L1 | Planned after 04B/04C evidence | | [WS-ARCH-001-04E](chunks/WS-ARCH-001-04E-canonical-allow-review.md) | TASK current routing: true to canonical `allow_review`, false/pass to shared acceptance | L1 | Source 04E1A -> hidden handlers 04E1B -> AUTH 04E2 -> live 04E3, plus 04D/OUTBOX-02; false consumes shared REV/CON/fence proof and activation also requires 04F remediation | -| [WS-ARCH-001-03D](../WS-ARCH-001-03D.md) | Exact activated historical guide through hidden durable intake; obsolete lookup removed | L1 | Complete; exact post-submit materialization next, public cutover remains deferred | +| [WS-ARCH-001-03D](../WS-ARCH-001-03D.md) | Exact activated historical guide through hidden durable intake; obsolete lookup removed | L1 | Complete; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next, public cutover remains deferred | | [WS-ARCH-001-04F](chunks/WS-ARCH-001-04F-checker-remediation.md) | Contributor-correctable checker failures and same-lineage admission-backed replacement Submission | L1 | Planned after 04E; replaces XINT-05C, required before public 02I, not before REV begins from `allow_review` | CP09, 04E and 04F are coordination parents, not permission for multi-owner PRs. diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md index 7c3356ed6..c0aa70b4a 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/PLAN.md @@ -44,11 +44,11 @@ checker-remediation boundary before public Submission cutover. | [ARCH-03C4](../WS-ARCH-001-03C4.md) | ARCH-03C3 | Complete: exact-authorized contributor, manager and operational public queues | | [ARCH-03C5](../WS-ARCH-001-03C5.md) | ARCH-03C4 | Complete: exact-authorized Contributor/Manager detail and requirements | | [ARCH-03C6](../WS-ARCH-001-03C6.md) | ARCH-03C5 | Complete: distinct exact-authorized locked-context reads | -| [ARCH-03C7](../WS-ARCH-001-03C7.md) | ARCH-03C6 and hidden TASK owner contracts | Complete: bounded Audit Authority history access; public guide activation is delivered by AUTH-18; ARCH-03D hidden intake delivered; exact post-submit materialization next | -| [CP05A](../WS-ARCH-001-CP05A.md) | CP05, existing policy owners | Complete: public Finance policy administration and selector recovery; AUTH-18 public manager activation/context delivered; ARCH-03D hidden intake delivered; exact post-submit materialization next | +| [ARCH-03C7](../WS-ARCH-001-03C7.md) | ARCH-03C6 and hidden TASK owner contracts | Complete: bounded Audit Authority history access; public guide activation is delivered by AUTH-18; ARCH-03D hidden intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | +| [CP05A](../WS-ARCH-001-CP05A.md) | CP05, existing policy owners | Complete: public Finance policy administration and selector recovery; AUTH-18 public manager activation/context delivered; ARCH-03D hidden intake delivered; hidden exact post-submit materialization delivered; ARCH-04B2 output custody next | | CP09 (later cleanup coordination) | All legacy consumers replaced, including CHECKER and public 02I path | Physical economic deletion; not on the allow_review critical path | | [ARCH-03D](../WS-ARCH-001-03D.md) | AUTH-18, ARCH-03A, CP08 and merged ART preparation | Complete: hidden durable intake uses exact historical TASK/PROJECTS ports; public cutover remains ARCH-02I | -| ARCH-04B | ARCH-04A, POL-07, ARCH-03C, merged ARCH-02H | ART exact stored Submission materialization | +| ARCH-04B | ARCH-04A, POL-07, ARCH-03C, merged ARCH-02H | Complete: ART hidden verified Submission input; live authority remains deferred | | ARCH-04B2 | ARCH-04A and merged ART admission/verification/binding foundations | ART bounded checker output/log ingestion and verified binding, no routing | | ARCH-04C | ARCH-04A, ARCH-04B, ARCH-04B2, POL-07 | CHECKERS durable execution/result/currentness and worker recovery | | ARCH-04D | ARCH-04B, ARCH-04C | AUTH post-submit materialization/result activation | @@ -82,7 +82,7 @@ ARCH-03C1 completes exact reconciler authority and decision-bound receipts. ARCH-03C2 supplies originating-transaction-only per-assignment producer events and first handler registration with enforced prefork topology; it must never backfill or dispatch retained invalidation rows. Public TASK activation is complete through ARCH-03C7. AUTH-18 delivers public -manager guide activation/context; ARCH-03D hidden intake is delivered; exact post-submit materialization is next. Subsequent +manager guide activation/context; ARCH-03D hidden intake is delivered; hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. Subsequent PR-sized contracts name exact files, public types, current migration head and runnable proof before implementation; they refine this design, not create a new permission requirement. @@ -174,6 +174,10 @@ owning implementation, not with planning prose or fake database claims. ### Named proof targets for the remaining design +ARCH-04B input proof is delivered in `test_post_submit_materialization.py` and +`test_post_submit_selection.py`, including deny-before-I/O composition. Output +custody and durable/live execution proof remain with ARCH-04B2/04C/04D below. + These are required future implementation tests, not tests claimed present or executed by this planning PR. Each owner's bounded record fixes the final module path alongside implementation; the symbols preserve the behavioral @@ -189,7 +193,7 @@ claims require their real custody, not unit substitutes. | ARCH-04A/POL-07 | `test_registered_evaluator_rejects_invalid_work`, `test_checker_facade_delegates_once` | Actual registered evaluator fixtures and typed composition; presence-only mutant must fail | | CP06/CP07/AUTH-12H | `test_activate_without_legacy_payment_or_task`, `test_activation_requires_exact_selected_policy`, `test_activation_rejects_missing_review_revision_config` | PostgreSQL atomic command plus full response serialization; foreign/retired/incomplete new binding denies | | CP08/ARCH-03A/03B/03C | `test_ready_preserves_screening_policy_lock`, `test_claim_copies_policy_without_current_lookup` | PostgreSQL and actual AUTH/owner composition; later publication leaves existing attempt unchanged | -| ARCH-04B/04C/04D | `test_exact_post_materialization_denies_before_io`, `test_late_revocation_cannot_publish_result`, `test_unfinished_checker_recovery_reuses_attempt`, `test_terminal_retry_requires_operator_and_new_attempt` | Local/MinIO, real worker/provider contract, PostgreSQL races; independent sessions and staged/final state | +| ARCH-04B2/04C/04D | `test_late_revocation_cannot_publish_result`, `test_unfinished_checker_recovery_reuses_attempt`, `test_terminal_retry_requires_operator_and_new_attempt` | Local/MinIO, real worker/provider contract, PostgreSQL races; independent sessions and staged/final state | | ARCH-04E | `test_submission_to_current_allow_review`, `test_superseded_run_cannot_route`, `test_duplicate_dispatch_has_one_manifest`; false tests in the shared acceptance contract | Real DB/worker/storage path, exact authority-event references; true creates no acceptance, false creates the shared atomic acceptance with no human Review | Owner-local schema names and migrations are chosen from the then-current diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-03B-task-assignment-api.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-03B-task-assignment-api.md index 0472ff440..d2b930c50 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-03B-task-assignment-api.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-03B-task-assignment-api.md @@ -7,7 +7,8 @@ projections and assignment recovery through owner ports. Its implementation children are complete; it is not a skeleton or an instruction to restart work. The subsequent ARCH-03C children delivered exact authority and public TASK exposure through 03C7. AUTH-18 delivers public manager guide activation/context; -hidden approved-guide intake is delivered by ARCH-03D; exact post-submit materialization is the next boundary in the +hidden approved-guide intake is delivered by ARCH-03D. ARCH-04B hidden exact +post-submit input is delivered; ARCH-04B2 output custody is next in the [current dependency contract](../PLAN.md#current-dependency-contract). ## Delivered boundaries diff --git a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md index 31bf32602..bb515bf3f 100644 --- a/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md +++ b/.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md @@ -1,27 +1,8 @@ -# Chunk Contract: WS-ARCH-001-04B ART Post-Submit Materialization +# ART post-submit input and output boundaries -Disposition: Planned. Dependencies: 04A, POL-07, ARCH-03C and merged 02H. Risk: L1. -Outcome: ART can internally -materialize the exact verified bytes bound to one immutable Submission for the -fixed post-submit checker service. - -Allowed: ART public API and owner-local materialization/binding code, fixed -adapter composition, focused ART tests, boundary ledgers and evidence/status. -Not allowed: checker policy/result ownership, TASK mutation, REV packet access, -generic download authority, Celery handle serialization or public routes. - -This is the sole replacement for historical ART-06A materialization; do not -run both plans. Preparation may use public fixture facts before activation, -but production remains deny-only until 04D. The capability opens bounded -scratch through ArtifactScratchManager, re-verifies exact stored bytes and -returns typed material facts, never raw credentials or provider handles. - -Acceptance: authority and materialization bind project/task/Submission, -admission, binding/content/replica, digest/size and approved generation; -denial precedes provider/scratch access; stale/replaced/cross-resource/replayed -requests fail closed. Verify Local/MinIO protocol tests, scratch cleanup, -PostgreSQL races, boundary validators, Ruff and hosted coverage. Required -reviews: architecture, security, ART/product ops, QA, senior and CI. +ARCH-04B implementation follows the [current bounded change record](../../WS-ARCH-001-04B.md). +It replaces this input skeleton with exact files, predicates and verification. +The output child below remains separate and Planned. ## ARCH-04B2 — Separate ART output-custody child @@ -48,10 +29,3 @@ the integrated run/binding/result transaction. Neither child owns TASK routing, REV records or contributor-blame decisions. Expand each owner-local child into its exact implementation record rather than combining ART and CHECKERS writes in one implementation PR. - -Before implementation, replace this skeleton with a current-main contract that -enumerates exact files, commands, migration head and reviewers. - -## Merge state - -- Outcome on merge: `planned` diff --git a/.commitrail/initiatives/WS-ART-001/OVERVIEW.md b/.commitrail/initiatives/WS-ART-001/OVERVIEW.md index 781d865f5..580993951 100644 --- a/.commitrail/initiatives/WS-ART-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-ART-001/OVERVIEW.md @@ -8,8 +8,8 @@ and the [capability ledger](../../../docs/roadmap_status.md). storage, and bounded private processing scratch. - Current boundary: ready-admission publication and hidden preparation, consumption, and binding dependencies are merged through ARCH-02H. -- Next usable boundary: exact post-submit materialization after executable - unified guide/checker contracts; live cutover remains later. +- Next usable boundary: ARCH-04B2 checker output custody. ARCH-04B hidden input + materialization is delivered; live authority and execution cutover remain later. - Governing sources: artifact specifications, `ArtifactStore`, `ArtifactScratchManager`, code, migrations, and artifact tests. - Preserve: SHA-256/byte-count identity, reread verification, isolation, @@ -32,8 +32,8 @@ ART retains the merged default-plus-project intake compiler/executor; POL-07 is a facade, not a replacement or second precheck run. New unified generations must prove exact approved lineage at preparation, consumption and binding. -1. Implement the ARCH-04 checker/post-submit materialization chain against the - canonical checker contracts. +1. ARCH-04B hidden exact Submission materialization is delivered. Continue with + ARCH-04B2 output custody, then durable execution and fixed-service authority. 2. ARCH-04F owns checker-remediation resubmission using existing ART ports; later reviewer-requested revision remains a separate REV boundary. Add those dependencies before public Submission cutover. diff --git a/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md b/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md index fb8d37506..b68e3f3cf 100644 --- a/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-AUTH-001/OVERVIEW.md @@ -23,8 +23,8 @@ Historical pre-cutover work records: [`STATUS.md`](pre-cutover/STATUS.md), - Public post-policy composition: POL-06B delivered using existing AUTH-12G. - Completed activation boundary: AUTH-12H exact-project manager authority for CP07 complete-guide activation/binding, with live-authority replay. -- Next usable boundary: exact post-submit materialization after ARCH-03D hidden intake, then durable - post-submit evaluation. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden + ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published @@ -77,5 +77,5 @@ manager proposal review, pre-submit approval and manual correction dispatch. AUTH-OUTBOX-01/02 bracket hidden CON-02B dispatch; ARCH-04E2 activates only the proven TASK routing handler before ARCH-04E3 live composition. TASK queue/read exposure is complete through ARCH-03C7. AUTH-18 public - manager guide activation/context is delivered; ARCH-03D hidden intake is delivered; exact post-submit materialization is next. + manager guide activation/context is delivered; ARCH-03D hidden intake is delivered; hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. Remaining work must use its exact owner, not the superseded broad designs. diff --git a/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md index ef5508f93..6c86ed494 100644 --- a/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-AUTH-001/planning/CHUNK_MAP.md @@ -13,7 +13,7 @@ work and historical proposals. | [AUTH-12H](../WS-AUTH-001-12H.md) | Complete: exact manager authority for CP07; AUTH-18 exposes manager activation and selections publicly | | CP05 | Complete: exact five policy actions; public Finance exposure delivered by CP05A | | ARCH-03C | Complete through 03C7: task/assignment authority, public queues, projections and audit reads; replaces broad AUTH-13 | -| [AUTH-18](../WS-AUTH-001-18.md) | Complete: public manager activation/context over CP07 and AUTH-12H; ARCH-03D completes hidden approved-guide intake; ARCH-04B exact post-submit materialization is next | +| [AUTH-18](../WS-AUTH-001-18.md) | Complete: public manager activation/context over CP07 and AUTH-12H; ARCH-03D completes hidden approved-guide intake; ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next | | ARCH-04D | AUTH materialization/output plus CHECKERS execute/finalize activation after ARCH-04B/04B2/04C; replaces AUTH-14/XINT-06B | | [AUTH-OUTBOX-01](PLAN.md#ws-auth-001-outbox-01--unavailable-dispatcher-contract) | Complete: unavailable exact dispatcher identity/action/phase contract; CON-02B and AUTH-OUTBOX-02 mechanics complete; feature authority/registration remain separate | | [AUTH-OUTBOX-02](PLAN.md#ws-auth-001-outbox-02--exact-dispatcher-activation) | Complete: exact dispatcher mechanics activation, phase audit custody and bounded prefork delivery; ARCH-03C2 subsequently registers assignment invalidation, while future handlers require their own exact authority | diff --git a/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md b/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md index 59bb686a5..76edeed5b 100644 --- a/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md +++ b/.commitrail/initiatives/WS-AUTH-001/planning/PLAN.md @@ -25,7 +25,7 @@ AUTH-13/14 cutovers are not additional implementation work. public TASK activation is complete through ARCH-03C7. - [AUTH-18](../WS-AUTH-001-18.md) delivers public manager activation and exact selection discovery over CP07/AUTH-12H. ARCH-03D completes hidden approved-guide - intake; ARCH-04B exact post-submit materialization is next. + intake; ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next. - CP05 owns exact ContributionPolicy-action activation after merged CP04B. - CP08 delivered the minimal lineage writers. ARCH-03B8 hidden task audit evidence and 03B9 hidden assignment invalidation are complete. ARCH-03C delivered @@ -52,8 +52,8 @@ context and reasoned system-Operator start, and removes self-activated eligibili and public JSON-packet Submission creation. Admission-backed creation stays hidden. ARCH-03B/03C reuse these exact actions and command owners; public TASK access and authority are delivered through 03C7, with invalidation wiring in 03C2. -CP08 delivered ContributionPolicyVersion lineage. ARCH-03D completes hidden approved-guide intake. ARCH-04B exact post-submit -materialization is next; public intake remains deferred to ARCH-02I. Do not restore eligibility +CP08 delivered ContributionPolicyVersion lineage. ARCH-03D completes hidden approved-guide intake. ARCH-04B hidden exact post-submit +input is delivered; ARCH-04B2 output custody is next; public intake remains deferred to ARCH-02I. Do not restore eligibility or register replacement aliases. ## WS-AUTH-001-OUTBOX-01 — unavailable dispatcher contract diff --git a/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md b/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md index fa0602e39..e228dabb8 100644 --- a/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md +++ b/.commitrail/initiatives/WS-AUTH-003/OVERVIEW.md @@ -8,7 +8,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), - Completed boundary: recovery foundation and [TASK/checker authorization cleanup](WS-AUTH-003-TASKCHECKER.md). - Intent: route public authorization capability through `authorization.api` and remove cross-module repository/model coupling. -- Next usable boundary: exact post-submit materialization after ARCH-03D hidden intake and delivered +- Next usable boundary: ARCH-04B2 output custody after delivered ARCH-04B hidden input, + ARCH-03D hidden intake and [AUTH-18 public manager activation](../WS-AUTH-001/WS-AUTH-001-18.md). Submission/checker history uses canonical authority; the alternate gate lifecycle is removed. Continue shrinking the canonical import ledger as implementation diff --git a/.commitrail/initiatives/WS-CON-001/OVERVIEW.md b/.commitrail/initiatives/WS-CON-001/OVERVIEW.md index 17a8d7c2c..afb315fff 100644 --- a/.commitrail/initiatives/WS-CON-001/OVERVIEW.md +++ b/.commitrail/initiatives/WS-CON-001/OVERVIEW.md @@ -11,8 +11,8 @@ and the [capability ledger](../../../docs/roadmap_status.md). immutable ContributionRecords and optional project-policy-driven compensation awards without coupling lifecycle truth to an economic provider. -- Next usable boundary: exact post-submit materialization after ARCH-03D hidden intake, then durable - post-submit evaluation. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden + ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published diff --git a/.commitrail/initiatives/WS-POL-003/OVERVIEW.md b/.commitrail/initiatives/WS-POL-003/OVERVIEW.md index 289dd7957..a7511f3e2 100644 --- a/.commitrail/initiatives/WS-POL-003/OVERVIEW.md +++ b/.commitrail/initiatives/WS-POL-003/OVERVIEW.md @@ -27,8 +27,8 @@ Exact pre-cutover work record: [`STATUS.md`](pre-cutover/STATUS.md), Production post-submit phase execution remains unavailable. - Intent: compile one locked guide and its policies into authoritative, versioned project behavior without circular subsystem authority. -- Next usable boundary: exact post-submit materialization after ARCH-03D hidden intake, then durable - post-submit evaluation. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers +- Next usable boundary: ARCH-04B2 checker output custody after delivered hidden + ARCH-04B input materialization, then durable post-submit evaluation and live authority. [AUTH-18](../WS-AUTH-001/WS-AUTH-001-18.md) delivers public manager activation context and exact guide activation. [CP05A](../WS-ARCH-001/WS-ARCH-001-CP05A.md) delivers public Finance ContributionPolicy administration and recovery of a draft selector, a published diff --git a/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md b/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md index 871116a0a..69ef63219 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md +++ b/.commitrail/initiatives/WS-POL-003/planning/CHUNK_MAP.md @@ -29,7 +29,7 @@ post-task requirement. Historical split parents 05/06 are not extra PRs. | `WS-AUTH-001-12G` | Activate exact fixed-service projection plus PM approval/correction authority for the hidden 06A manifest. | 06A | | `WS-POL-003-06B` | Live deterministic post-submit projection/approval cutover with zero additional inference. | 06A + AUTH-12G | | `WS-POL-003-07` | One typed facade over existing ART pre and CHECKER post contracts; no post-result persistence. | 06B + ARCH-04A registered capability proof + merged ART-04B1-04B3 | -| `WS-AUTH-001-12H` / [AUTH-18](../../WS-AUTH-001/WS-AUTH-001-18.md) | Complete: exact guide activation authority and public manager activation/context over the approved unified chain. CP08 lineage and ARCH-03 task authority/projections are delivered; ARCH-03D completes hidden approved-guide intake. ARCH-04B exact post-submit materialization is next; public intake remains ARCH-02I. CP09 remains later. | POL-07 + corrected AUTH-12B2 + CP05 active ContributionPolicy behavior + CP06 validation + CP07 ProjectGuide binding | +| `WS-AUTH-001-12H` / [AUTH-18](../../WS-AUTH-001/WS-AUTH-001-18.md) | Complete: exact guide activation authority and public manager activation/context over the approved unified chain. CP08 lineage and ARCH-03 task authority/projections are delivered; ARCH-03D completes hidden approved-guide intake. ARCH-04B hidden exact post-submit materialization is delivered; ARCH-04B2 output custody is next; public intake remains ARCH-02I. CP09 remains later. | POL-07 + corrected AUTH-12B2 + CP05 active ContributionPolicy behavior + CP06 validation + CP07 ProjectGuide binding | | `WS-POL-003-08` | Supplementary visibility; separate remaining cleanup is parked and handled within each affected module. Essential review/correction belongs to 05A/05B and 06A/06B. | Planned after 07 + AUTH-12H + canonical WS-ARCH-001-04E manifest; any separately authorized retained-data change requires CP09's inventory, mapping and readability/recoverability proof for affected facts; no cleanup prerequisite for 04B | The 04E manifest proves canonical routing, not legacy-history preservation. @@ -53,7 +53,8 @@ mixed-generation chains deny through the existing locked-lineage checks. ## Post-submit execution gate ARCH-04A contracts/capability conformance precede POL-07 and guide activation. -ART post-submit materialization and durable CHECKER execution follow only through -ARCH-04B/04C after POL-07 and task readiness merge. +ARCH-04B delivers hidden exact ART post-submit input with deny-only production +authority. ARCH-04B2 output custody is next, followed by ARCH-04C durable CHECKER +execution/results. WS-ARCH-001-04D is the later replacement activation gate. Historical XINT-06B and AUTH-14 contracts are superseded/non-executable. diff --git a/.commitrail/initiatives/WS-POL-003/planning/PLAN.md b/.commitrail/initiatives/WS-POL-003/planning/PLAN.md index 266c98ec3..7d41f450c 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/PLAN.md +++ b/.commitrail/initiatives/WS-POL-003/planning/PLAN.md @@ -55,8 +55,8 @@ POL-04B replaced the separate inference paths with unified setup, without aliases or fallbacks. AUTH-12F4 and AUTH-12G supply the delivered pre/post approval authority. AUTH-12H supplies exact guide activation authority; AUTH-18 exposes manager activation and selection discovery publicly. -ARCH-03D completes hidden approved-guide intake integration. ARCH-04B exact -post-submit materialization is next; public intake remains deferred to ARCH-02I. +ARCH-03D completes hidden approved-guide intake integration. ARCH-04B hidden exact +post-submit materialization is delivered; ARCH-04B2 output custody is next; public intake remains deferred to ARCH-02I. The sequence through POL-04B is complete; POL-04B1 supplies automatic request custody for the delivered live cutover: diff --git a/.commitrail/initiatives/WS-POL-003/planning/chunks/WS-POL-003-07-single-checker-service-port.md b/.commitrail/initiatives/WS-POL-003/planning/chunks/WS-POL-003-07-single-checker-service-port.md index ecdf6beb9..407730eab 100644 --- a/.commitrail/initiatives/WS-POL-003/planning/chunks/WS-POL-003-07-single-checker-service-port.md +++ b/.commitrail/initiatives/WS-POL-003/planning/chunks/WS-POL-003-07-single-checker-service-port.md @@ -26,8 +26,9 @@ the earlier overbroad execution claims in this planning chunk. contract. Production explicitly remains unavailable. Exact request, policy, catalogue, generation and member correspondence is value-consistency proof, not authority, durable ownership, attempt recovery or currentness proof. -- ARCH-04B/04C/04D/04E retain post materialization, persistence, authorization, - event invocation and routing. Their eventual event handler invokes the facade; +- ARCH-04B delivers hidden post-submit input with deny-only production authority. + ARCH-04B2 output custody, ARCH-04C persistence, ARCH-04D authority and ARCH-04E + event invocation/routing remain separate. Their eventual event handler invokes the facade; its injected executor must not call the facade recursively. Existing post callers remain until that cutover, not as a second implementation of this port. - Remove the standalone draft JSON precheck, its exclusive code, schemas and diff --git a/README.md b/README.md index 570386423..e6dc2f38d 100644 --- a/README.md +++ b/README.md @@ -650,8 +650,10 @@ locked-context reads with exact current grants. ARCH-03C7 exposes bounded task history only to covered Audit Authority and removes the old payload-bearing task-only audit route. AUTH-18 delivers public manager guide activation; ARCH-03D connects hidden intake through final durable intent to canonical TASK/PROJECTS -ports, preserving historical guide selection. Public intake remains deferred. Canonical checker materialization follows in -ARCH-04B/04C. +ports, preserving historical guide selection. Public intake remains deferred. ARCH-04B supplies hidden exact verified Submission +input with scoped async file access and cleanup. Production materialization +authority remains deny-only; ARCH-04B2 output custody, ARCH-04C durable execution +and ARCH-04D live authority remain separate steps. ## v0.1 Success Standard diff --git a/backend/app/adapters/artifacts/__init__.py b/backend/app/adapters/artifacts/__init__.py index d19b52400..e6f97ed1f 100644 --- a/backend/app/adapters/artifacts/__init__.py +++ b/backend/app/adapters/artifacts/__init__.py @@ -19,7 +19,7 @@ from app.db.session import get_db_session from app.interfaces.artifact_operations import GuideArtifactIngestCommand from app.modules.projects.api.guide_documents import GuideDocumentUploadTargetPort -from app.modules.artifacts.api import SubmissionBundlePreparationCommand +from app.modules.artifacts.api import SubmissionBundlePreparationCommand, PostSubmissionMaterializationPort from app.interfaces.artifacts import ( ARTIFACT_STORE_CAPABILITY_KEY, ArtifactConfigurationError, @@ -498,3 +498,16 @@ def authority(session): await grant.close() finally: manager.close() + + +def post_submission_materialization(*, sessions, store, namespace, preparation, inspector) -> PostSubmissionMaterializationPort: + """Compose hidden verified input; production authority remains explicitly unavailable.""" + from app.adapters.tasks import submitted_bundle_port + from app.modules.artifacts.post_submit_materialization import ( + DenyPostSubmissionMaterializationAuthority, PostSubmissionMaterializer, + ) + return PostSubmissionMaterializer( + sessions=sessions, tasks=submitted_bundle_port, store=store, namespace=namespace, + preparation=preparation, inspector=inspector, + authority=DenyPostSubmissionMaterializationAuthority(), + ) diff --git a/backend/app/adapters/tasks/__init__.py b/backend/app/adapters/tasks/__init__.py index 56a801188..f88282d0e 100644 --- a/backend/app/adapters/tasks/__init__.py +++ b/backend/app/adapters/tasks/__init__.py @@ -1,6 +1,7 @@ """TASK-owned composition adapters and transaction roots.""" from app.modules.tasks.api.submission_history import SubmissionHistoryReadPort +from app.modules.tasks.api.submitted_bundle import SubmittedBundlePort from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import Settings from uuid import UUID @@ -174,3 +175,9 @@ def submission_history_repository(session) -> SubmissionHistoryReadPort: """Compose TASK-owned immutable history selectors and projections.""" from app.modules.tasks.submission_history import SubmissionHistoryRepository return SubmissionHistoryRepository(session) + + +def submitted_bundle_port(session: AsyncSession) -> SubmittedBundlePort: + """Compose the exact immutable Submission read without private owner imports in ART.""" + from app.modules.tasks.submitted_bundle import SubmittedBundleReader + return SubmittedBundleReader(session) diff --git a/backend/app/interfaces/artifact_operations.py b/backend/app/interfaces/artifact_operations.py index 12f0fedff..92ab1c2e0 100644 --- a/backend/app/interfaces/artifact_operations.py +++ b/backend/app/interfaces/artifact_operations.py @@ -14,11 +14,9 @@ "ArtifactAuditResourceType", "ArtifactBindingResourceType", "ArtifactBindingPort", - "ArtifactMaterializationPort", "ArtifactOperatorReadPort", "ArtifactOperatorRecoveryPort", "ArtifactRecoveryRequest", - "BindingMaterializationRequest", "CheckerArtifactOutputPort", "CheckerOutputBindingRequest", "CheckerOutputArtifactRequest", @@ -88,17 +86,6 @@ class CheckerOutputBindingRequest: verified_content_ids: tuple[UUID, ...] -@dataclass(frozen=True, slots=True) -class BindingMaterializationRequest: - """Immutable bindings selected by exact execution context.""" - - prepared_authorization: PreparedAuthorizationHandle - task_id: UUID - submission_id: UUID | None - checker_run_id: UUID - binding_ids: tuple[UUID, ...] - - @dataclass(frozen=True, slots=True) class CheckerOutputArtifactRequest: """Generated checker bytes bound to one fixed service execution.""" @@ -157,16 +144,6 @@ async def bind_checker_output(self, request: CheckerOutputBindingRequest) -> obj """Bind verified checker output under the checker binding action.""" -class ArtifactMaterializationPort(Protocol): - """Materialize only canonical immutable source forms.""" - - async def materialize_bindings( - self, - request: BindingMaterializationRequest, - ) -> object: - """Materialize exact immutable binding IDs.""" - - class CheckerArtifactOutputPort(Protocol): """Store generated output for one fixed checker execution.""" diff --git a/backend/app/modules/artifacts/api/__init__.py b/backend/app/modules/artifacts/api/__init__.py index 47813c349..bb1c7602f 100644 --- a/backend/app/modules/artifacts/api/__init__.py +++ b/backend/app/modules/artifacts/api/__init__.py @@ -31,3 +31,15 @@ "SubmissionAdmissionConsumptionResult", "SubmissionAdmissionConsumptionStatus", ) + +from app.modules.artifacts.api.submission_materialization import ( + PostSubmissionMaterialConsumer, PostSubmissionMaterializationPort, + PostSubmissionMaterializationResult, PostSubmissionMaterializationUnavailable, + SubmissionMaterialEntry, SubmissionMaterialView, +) + +__all__ += ( + "PostSubmissionMaterialConsumer", "PostSubmissionMaterializationPort", + "PostSubmissionMaterializationResult", "PostSubmissionMaterializationUnavailable", + "SubmissionMaterialEntry", "SubmissionMaterialView", +) diff --git a/backend/app/modules/artifacts/api/submission_materialization.py b/backend/app/modules/artifacts/api/submission_materialization.py new file mode 100644 index 000000000..d6b918f6e --- /dev/null +++ b/backend/app/modules/artifacts/api/submission_materialization.py @@ -0,0 +1,63 @@ +"""Exact post-submit byte access; neither durable execution nor live authority.""" + +from dataclasses import dataclass +from typing import Literal, Protocol +from uuid import UUID + +from app.modules.checkers.api import PostSubmissionEvaluationRequest, PostSubmissionEvaluationResult + + +class PostSubmissionMaterializationUnavailable(RuntimeError): + """Reject material without exposing provider or private packet details.""" + + +@dataclass(frozen=True, slots=True) +class SubmissionMaterialEntry: + """Expose verified archive metadata without a filesystem path.""" + normalized_path: str + entry_type: Literal["file", "directory"] + byte_count: int + sha256: str | None + executable: bool | None + + +class SubmissionMaterialView(Protocol): + """Permit bounded reads only during the consumer callback.""" + + @property + def entries(self) -> tuple[SubmissionMaterialEntry, ...]: + """Return verified entries only while the consumer is running.""" + + def read_file(self, normalized_path: str, *, maximum_bytes: int) -> bytes: + """Read one bounded verified file without filesystem authority.""" + + +class PostSubmissionMaterialConsumer(Protocol): + """Evaluate scoped input asynchronously and return detached phase facts.""" + async def evaluate( + self, request: PostSubmissionEvaluationRequest, material: SubmissionMaterialView, + ) -> PostSubmissionEvaluationResult: + """Return detached closed phase facts before ART revokes material access.""" + + +@dataclass(frozen=True, slots=True) +class PostSubmissionMaterializationResult: + """Bind a validated evaluation to its verified immutable input custody.""" + submission_id: UUID + submission_version: int + admission_id: UUID + binding_id: UUID + content_id: UUID + replica_id: UUID + content_sha256: str + byte_count: int + semantic_manifest_sha256: str + evaluation: PostSubmissionEvaluationResult + + +class PostSubmissionMaterializationPort(Protocol): + """Verify and scope exact input without publishing durable checker results.""" + async def materialize( + self, request: PostSubmissionEvaluationRequest, consumer: PostSubmissionMaterialConsumer, + ) -> PostSubmissionMaterializationResult: + """Verify exact stored bytes and finish cleanup before returning phase facts.""" diff --git a/backend/app/modules/artifacts/post_submit_materialization.py b/backend/app/modules/artifacts/post_submit_materialization.py new file mode 100644 index 000000000..897f291e8 --- /dev/null +++ b/backend/app/modules/artifacts/post_submit_materialization.py @@ -0,0 +1,187 @@ +"""Verified post-submit input through the sole bounded artifact scratch owner.""" + +import asyncio +from collections.abc import Callable +from typing import Protocol + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from app.core.cancellation import await_cancellation_resistant +from app.interfaces.artifacts import ArtifactStore +from app.modules.artifacts.api.submission_materialization import ( + PostSubmissionMaterialConsumer, PostSubmissionMaterializationResult, + PostSubmissionMaterializationUnavailable, SubmissionMaterialEntry, +) +from app.modules.artifacts.post_submit_selection import ( + PostSubmissionMaterialSelection, select_post_submission_material, +) +from app.modules.artifacts.preparation import ArtifactPreparationService +from app.modules.artifacts.service import ArtifactStorageNamespaceError, ArtifactStorageNamespaceSpec +from app.modules.artifacts.submission_archive import ( + SealedSubmissionTree, SubmissionArchiveInspector, SubmissionArchiveEntryType, +) +from app.modules.artifacts.submission_manifest import build_submission_manifest +from app.modules.checkers.api import PostSubmissionEvaluationRequest, PostSubmissionEvaluationResult +from app.modules.tasks.api.submitted_bundle import SubmittedBundlePort, SubmittedBundleUnavailable + + +class PostSubmissionMaterializationAuthority(Protocol): + """Separate early service availability from exact resolved-material authority.""" + async def preflight(self, request: PostSubmissionEvaluationRequest) -> None: + """Deny unavailable fixed-service access before protected reads.""" + + async def authorize( + self, request: PostSubmissionEvaluationRequest, selection: PostSubmissionMaterialSelection, + ) -> None: + """Authorize exact resolved facts before provider and scratch access.""" + + +class DenyPostSubmissionMaterializationAuthority: + """Production remains unavailable until ARCH-04D activates exact service authority.""" + + async def preflight(self, request: PostSubmissionEvaluationRequest) -> None: + """Reject production access before any protected selection or byte read.""" + raise PostSubmissionMaterializationUnavailable("post_submit_materialization_unavailable") + + async def authorize( + self, request: PostSubmissionEvaluationRequest, selection: PostSubmissionMaterialSelection, + ) -> None: + """Reject exact material access until live authority is implemented.""" + raise PostSubmissionMaterializationUnavailable("post_submit_materialization_unavailable") + + +class _MaterialView: + """Revoke callback access independently of the private projection lifetime.""" + def __init__(self, tree: SealedSubmissionTree) -> None: + """Wrap the single ART-owned sealed tree without exposing its path.""" + self._tree = tree + self._closed = False + + def close(self) -> None: + """Revoke subsequent metadata and file reads.""" + self._closed = True + + def _require_open(self) -> None: + """Reject retained views after callback completion or cancellation.""" + if self._closed: + raise RuntimeError("submission material view is closed") + + @property + def entries(self) -> tuple[SubmissionMaterialEntry, ...]: + """Project verified metadata while the view remains live.""" + self._require_open() + return tuple(SubmissionMaterialEntry( + item.normalized_path, item.entry_type.value, item.byte_count, item.sha256, item.executable, + ) for item in self._tree.entries) + + def read_file(self, normalized_path: str, *, maximum_bytes: int) -> bytes: + """Delegate bounded reads to the sealed verified tree.""" + self._require_open() + return self._tree.read_file(normalized_path, maximum_bytes=maximum_bytes) + + def __reduce__(self): + """Prevent transfer of a process-local read capability.""" + raise TypeError("submission material view is process-local") + + +class _MaterialProcessor: + """Keep async evaluation inside the shared projection and scratch lifetime.""" + def __init__(self, inspector, inspection, request, consumer) -> None: + """Bind one inspection, request and consumer for preparation-owned execution.""" + self._inspector, self._inspection = inspector, inspection + self._request, self._consumer = request, consumer + self._aborted = False + self._consumer_task: asyncio.Task[PostSubmissionEvaluationResult] | None = None + self._view: _MaterialView | None = None + + def abort(self) -> None: + """Revoke reads and cancel the consumer before preparation drains cleanup.""" + self._aborted = True + if self._view is not None: + self._view.close() + if self._consumer_task is not None: + self._consumer_task.cancel() + + async def process(self, reader, workspace) -> PostSubmissionEvaluationResult: + """Project off-loop, validate callback output, and drain projection cleanup.""" + projection = self._inspector._projected_tree(reader, workspace, expected=self._inspection) + # The preparation owner shields and drains this entire operation, including + # projection entry, before it releases the reader or workspace. + tree = await asyncio.to_thread(projection.__enter__) + try: + if self._aborted: + raise asyncio.CancelledError + self._view = _MaterialView(tree) + self._consumer_task = asyncio.create_task(self._consumer.evaluate(self._request, self._view)) + result = PostSubmissionEvaluationResult.model_validate(await self._consumer_task) + result.validate_request(self._request) + return result + finally: + if self._view is not None: + self._view.close() + await await_cancellation_resistant(asyncio.to_thread(projection.__exit__, None, None, None)) + + +class PostSubmissionMaterializer: + """Resolve, authorize, verify, scope, clean, and revalidate one exact input.""" + + def __init__( + self, *, sessions: async_sessionmaker[AsyncSession], + tasks: Callable[[AsyncSession], SubmittedBundlePort], store: ArtifactStore, + namespace: ArtifactStorageNamespaceSpec, preparation: ArtifactPreparationService, + inspector: SubmissionArchiveInspector, authority: PostSubmissionMaterializationAuthority, + ) -> None: + """Require explicit owner ports, scratch service and material authority.""" + self._sessions, self._tasks, self._store = sessions, tasks, store + self._namespace, self._preparation = namespace, preparation + self._inspector, self._authority = inspector, authority + + async def _select(self, request) -> PostSubmissionMaterialSelection: + """Detach fresh selection facts and close the transaction before I/O.""" + try: + async with self._sessions() as session, session.begin(): + return await select_post_submission_material( + session, tasks=self._tasks(session), request=request, + namespace=self._namespace, store=self._store, + ) + except (SubmittedBundleUnavailable, ArtifactStorageNamespaceError): + raise PostSubmissionMaterializationUnavailable("post_submit_material_unavailable") from None + + async def materialize( + self, request: PostSubmissionEvaluationRequest, consumer: PostSubmissionMaterialConsumer, + ) -> PostSubmissionMaterializationResult: + """Verify bytes, run the scoped consumer, clean up, and reject late drift.""" + request = PostSubmissionEvaluationRequest.model_validate(request) + await self._authority.preflight(request) + selected = await self._select(request) + await self._authority.authorize(request, selected) + prepared = await self._preparation.prepare( + self._store.open(selected.provider_object_ref), media_type=selected.media_type, + expected_sha256=selected.sha256, expected_size=selected.byte_count, + ) + try: + inspection = await prepared.inspect(self._inspector) + manifest = build_submission_manifest(inspection) + expected_files = tuple((entry.normalized_path, entry.sha256, entry.byte_count) + for entry in manifest.entries + if entry.entry_type is SubmissionArchiveEntryType.FILE) + supplied_files = tuple(sorted((entry.artifact, entry.hash, entry.size_bytes) + for entry in request.structural_input.manifest)) + if manifest.sha256 != selected.semantic_manifest_sha256 or supplied_files != expected_files: + raise PostSubmissionMaterializationUnavailable("post_submit_material_manifest_mismatch") + evaluation = await self._preparation._process_prepared_submission( + prepared, _MaterialProcessor(self._inspector, inspection, request, consumer), + reserved_bytes=manifest.total_expanded_bytes, maximum_entries=manifest.entry_count, + ) + finally: + await prepared.close() + if await self._select(request) != selected: + raise PostSubmissionMaterializationUnavailable("post_submit_material_changed") + facts = selected.submission + return PostSubmissionMaterializationResult( + submission_id=facts.submission_id, submission_version=facts.submission_version, + admission_id=facts.admission_id, binding_id=facts.binding_id, content_id=facts.content_id, + replica_id=selected.replica_id, content_sha256=selected.sha256, + byte_count=selected.byte_count, semantic_manifest_sha256=selected.semantic_manifest_sha256, + evaluation=evaluation, + ) diff --git a/backend/app/modules/artifacts/post_submit_selection.py b/backend/app/modules/artifacts/post_submit_selection.py new file mode 100644 index 000000000..a637d0c76 --- /dev/null +++ b/backend/app/modules/artifacts/post_submit_selection.py @@ -0,0 +1,138 @@ +"""ART-owned exact admitted-material selection, without provider I/O or locks.""" + +from dataclasses import asdict, dataclass +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.interfaces.artifacts import ArtifactStore, artifact_provider_object_ref +from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.artifacts.models import ( + ArtifactBinding, ArtifactContent, ArtifactReplica, ArtifactStorageNamespace, + ArtifactVerificationJob, ArtifactVerificationReceipt, PreSubmitEvidenceSet, + SubmissionBundleAdmission, +) +from app.modules.artifacts.service import ( + ArtifactStorageNamespaceSpec, validate_artifact_replica_execution_namespace, +) +from app.modules.artifacts.sources import ArtifactCommitment +from app.modules.artifacts.submission_bindings import SubmissionAdmissionConsumptionService +from app.modules.checkers.api import PostSubmissionEvaluationRequest +from app.modules.tasks.api.submitted_bundle import ( + SubmittedBundleFacts, SubmittedBundlePort, SubmittedBundleRequest, +) + + +@dataclass(frozen=True, slots=True) +class PostSubmissionMaterialSelection: + """Private detached snapshot; storage coordinates never leave ART.""" + + submission: SubmittedBundleFacts + evidence_id: UUID + replica_id: UUID + verification_receipt_id: UUID + verification_job_id: UUID + verification_generation: int + namespace_fingerprint: str + adapter: str + provider_profile: str + provider_object_ref: str + sha256: str + byte_count: int + media_type: str + semantic_manifest_id: UUID + semantic_manifest_sha256: str + + +async def select_post_submission_material( + session: AsyncSession, *, tasks: SubmittedBundlePort, + request: PostSubmissionEvaluationRequest, namespace: ArtifactStorageNamespaceSpec, + store: ArtifactStore, +) -> PostSubmissionMaterialSelection: + """Read fresh exact owner facts in one short caller-owned transaction.""" + facts = await tasks.read(SubmittedBundleRequest( + project_id=request.project_id, task_id=request.task_id, submission_id=request.submission_id, + )) + if ( + facts.status != "submitted" + or (facts.project_id, facts.task_id, facts.assignment_id, facts.submission_id, + facts.submission_version, facts.binding_id, facts.content_id) != ( + request.project_id, request.task_id, request.assignment_id, request.submission_id, + request.submission_version, request.binding_id, request.content_id) + or asdict(facts.context) != request.expected_context.model_dump() + or asdict(facts.context) != request.structural_input.observed_context.model_dump() + ): + raise PostSubmissionMaterializationUnavailable("post_submit_material_identity_mismatch") + row = (await session.execute( + select(SubmissionBundleAdmission, PreSubmitEvidenceSet, ArtifactBinding, + ArtifactContent, ArtifactReplica, ArtifactVerificationReceipt, + ArtifactVerificationJob, ArtifactStorageNamespace) + .select_from(SubmissionBundleAdmission) + .join(PreSubmitEvidenceSet, PreSubmitEvidenceSet.id == SubmissionBundleAdmission.pre_submit_evidence_set_id) + .join(ArtifactBinding, ArtifactBinding.id == str(facts.binding_id)) + .join(ArtifactContent, ArtifactContent.id == SubmissionBundleAdmission.artifact_content_id) + .join(ArtifactReplica, ArtifactReplica.id == SubmissionBundleAdmission.verified_replica_id) + .join(ArtifactVerificationReceipt, ArtifactVerificationReceipt.id == SubmissionBundleAdmission.verification_receipt_id) + .join(ArtifactVerificationJob, ArtifactVerificationJob.id == ArtifactVerificationReceipt.verification_job_id) + .join(ArtifactStorageNamespace, ArtifactStorageNamespace.id == ArtifactReplica.storage_namespace_id) + .where(SubmissionBundleAdmission.id == str(facts.admission_id), + SubmissionBundleAdmission.project_id == str(request.project_id), + SubmissionBundleAdmission.task_id == str(request.task_id)) + .execution_options(populate_existing=True) + )).one_or_none() + if row is None: + raise PostSubmissionMaterializationUnavailable("post_submit_material_unavailable") + admission, evidence, binding, content, replica, receipt, job, persisted = row + context = facts.context + if not ( + SubmissionAdmissionConsumptionService._art_lineage_is_intact(admission, evidence, content) + and admission.status == "consumed" + and admission.consumed_by_submission_id == str(facts.submission_id) + and admission.consumed_by_submission_version == facts.submission_version + and admission.assignment_id == str(facts.assignment_id) + and admission.actor_profile_id == str(facts.contributor_id) + and admission.predecessor_submission_id == (str(facts.predecessor_id) if facts.predecessor_id else None) + and admission.predecessor_submission_version == facts.predecessor_version + and binding.project_id == str(facts.project_id) + and binding.resource_type == "submission" + and binding.resource_id == str(facts.submission_id) + and binding.logical_role == "submission_bundle_original" + and binding.scope_version == 1 + and binding.content_id == content.id == str(facts.content_id) + and content.media_type == "application/zip" + and content.sha256 == request.content_sha256 == request.structural_input.package_hash + and content.byte_count == request.byte_count + and evidence.guide_version == context.guide_version + and evidence.source_snapshot_id == str(context.source_id) + and evidence.source_snapshot_sha256 == context.source_hash + and evidence.effective_policy_id == str(context.effective_policy_id) + and evidence.locked_artifact_policy_sha256 == context.effective_policy_hash + and evidence.pre_submit_policy_id == str(context.pre_policy_id) + and evidence.locked_checker_policy_sha256 == context.pre_policy_hash + and replica.content_id == content.id + and (replica.verification_state, replica.availability_state, replica.integrity_state) + == ("verified", "available", "valid") + and receipt.outcome == "verified" + and receipt.observed_sha256 == content.sha256 + and receipt.observed_byte_count == content.byte_count + and job.replica_id == replica.id + and job.originating_put_attempt_id == admission.put_attempt_id + ): + raise PostSubmissionMaterializationUnavailable("post_submit_material_identity_mismatch") + validate_artifact_replica_execution_namespace( + replica=replica, persisted=persisted, namespace=namespace, store=store, + ) + commitment = ArtifactCommitment(content.sha256, content.byte_count, content.media_type) + if replica.provider_object_ref != artifact_provider_object_ref(commitment): + raise PostSubmissionMaterializationUnavailable("post_submit_material_identity_mismatch") + return PostSubmissionMaterialSelection( + submission=facts, evidence_id=UUID(evidence.id), replica_id=UUID(replica.id), + verification_receipt_id=UUID(receipt.id), verification_job_id=UUID(job.id), + verification_generation=receipt.execution_generation, + namespace_fingerprint=persisted.namespace_fingerprint, adapter=replica.adapter, + provider_profile=replica.provider_profile, provider_object_ref=replica.provider_object_ref, + sha256=content.sha256, byte_count=content.byte_count, media_type=content.media_type, + semantic_manifest_id=UUID(admission.semantic_manifest_id), + semantic_manifest_sha256=admission.semantic_manifest_sha256, + ) diff --git a/backend/app/modules/artifacts/preparation.py b/backend/app/modules/artifacts/preparation.py index 2c00eb601..99b465e71 100644 --- a/backend/app/modules/artifacts/preparation.py +++ b/backend/app/modules/artifacts/preparation.py @@ -1679,7 +1679,7 @@ async def _process_prepared_submission( reserved_bytes: int, maximum_entries: int, ) -> _InspectionResult: - """Serve only the authority-gated hidden submission materializer.""" + """Serve authority-gated pre- and post-submit materializers through one scratch lifetime.""" if type(prepared) is not PreparedArtifact or prepared._owner is not self: raise ArtifactScratchIntegrityError("prepared artifact source is unavailable") binding = prepared._binding diff --git a/backend/app/modules/artifacts/submission_archive.py b/backend/app/modules/artifacts/submission_archive.py index 078bd6d89..a1add340a 100644 --- a/backend/app/modules/artifacts/submission_archive.py +++ b/backend/app/modules/artifacts/submission_archive.py @@ -2,6 +2,8 @@ from __future__ import annotations +from contextlib import contextmanager +from collections.abc import Iterator from dataclasses import dataclass from enum import StrEnum import hashlib @@ -274,6 +276,14 @@ def project_and_run( callback: Callable[[SealedSubmissionTree], _ProjectionResult], ) -> _ProjectionResult: """Project the exact inspected ZIP and keep the tree inside one callback lifetime.""" + with self._projected_tree(reader, workspace, expected=expected) as tree: + return callback(tree) + + @contextmanager + def _projected_tree( + self, reader: BinaryIO, workspace: Path, *, expected: SubmissionArchiveInspectionResult, + ) -> Iterator[SealedSubmissionTree]: + """One shared projection lifetime for synchronous and asynchronous consumers.""" observed = self.inspect(reader) if observed != expected: self._reject(SubmissionArchiveFailureCode.INTEGRITY_FAILURE) @@ -344,7 +354,7 @@ def project_and_run( content=content, ) callback_started = True - return callback(tree) + yield tree except SubmissionArchiveRejectedError: raise except (OSError, ValueError, zipfile.BadZipFile, RuntimeError): diff --git a/backend/app/modules/tasks/api/submitted_bundle.py b/backend/app/modules/tasks/api/submitted_bundle.py new file mode 100644 index 000000000..e189c555f --- /dev/null +++ b/backend/app/modules/tasks/api/submitted_bundle.py @@ -0,0 +1,65 @@ +"""TASK-owned immutable Submission facts for internal materialization.""" + +from dataclasses import dataclass +from typing import Protocol +from uuid import UUID + + +class SubmittedBundleUnavailable(RuntimeError): + """Conceal absent, foreign or ineligible Submission material.""" + + +@dataclass(frozen=True, slots=True) +class SubmittedBundleRequest: + """Select one Submission within its exact project and task.""" + project_id: UUID + task_id: UUID + submission_id: UUID + + +@dataclass(frozen=True, slots=True) +class SubmittedPolicyContext: + """Frozen Submission references, independent of current project selectors.""" + + guide_version: str + source_id: UUID + source_hash: str + effective_policy_id: UUID + effective_policy_hash: str + pre_policy_id: UUID + pre_policy_hash: str + post_policy_id: UUID + post_policy_version: str + post_policy_hash: str + review_policy_id: UUID + review_generation: int + review_hash: str + revision_policy_id: UUID + revision_generation: int + revision_hash: str + + +@dataclass(frozen=True, slots=True) +class SubmittedBundleFacts: + """Detached exact ownership; no packet, policy body or storage coordinates.""" + + project_id: UUID + task_id: UUID + assignment_id: UUID + contributor_id: UUID + submission_id: UUID + submission_version: int + status: str + predecessor_id: UUID | None + predecessor_version: int | None + contribution_policy_version_id: UUID + admission_id: UUID + binding_id: UUID + content_id: UUID + context: SubmittedPolicyContext + + +class SubmittedBundlePort(Protocol): + """Read immutable TASK facts without acquiring mutation authority.""" + async def read(self, request: SubmittedBundleRequest) -> SubmittedBundleFacts: + """Read exact submitted ownership inside the caller's short transaction.""" diff --git a/backend/app/modules/tasks/submitted_bundle.py b/backend/app/modules/tasks/submitted_bundle.py new file mode 100644 index 000000000..826da986f --- /dev/null +++ b/backend/app/modules/tasks/submitted_bundle.py @@ -0,0 +1,101 @@ +"""Owner-qualified, nonlocking immutable Submission material projection.""" + +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import aliased + +from app.modules.tasks.api.submitted_bundle import ( + SubmittedBundleFacts, SubmittedBundleRequest, SubmittedBundleUnavailable, + SubmittedPolicyContext, +) +from app.modules.tasks.models import Submission, WorkstreamTask + + +class SubmittedBundleReader: + """Project detached submitted facts from the owning TASK tables.""" + def __init__(self, session: AsyncSession) -> None: + """Use the caller-owned short read transaction.""" + self._session = session + + async def read(self, request: SubmittedBundleRequest) -> SubmittedBundleFacts: + """Select all scalar facts afresh, scoped before any row is returned.""" + predecessor = aliased(Submission) + columns = ( + Submission.id, + Submission.task_id, + Submission.task_assignment_id, + Submission.contributor_id, + Submission.version, + Submission.status, + Submission.supersedes_submission_id, + Submission.contribution_policy_version_id, + Submission.submission_bundle_admission_id, + Submission.artifact_binding_id, + Submission.artifact_content_id, + Submission.locked_guide_version, + Submission.locked_guide_source_snapshot_id, + Submission.locked_guide_source_snapshot_hash, + Submission.locked_effective_project_submission_artifact_policy_id, + Submission.locked_effective_project_submission_artifact_policy_hash, + Submission.locked_pre_submit_checker_policy_id, + Submission.locked_pre_submit_checker_bundle_hash, + Submission.locked_post_submit_checker_policy_id, + Submission.locked_post_submit_checker_policy_version, + Submission.locked_post_submit_checker_policy_hash, + Submission.locked_review_policy_id, + Submission.locked_review_policy_generation, + Submission.locked_review_policy_hash, + Submission.locked_revision_policy_id, + Submission.locked_revision_policy_generation, + Submission.locked_revision_policy_hash, + ) + row = (await self._session.execute( + select(*columns, WorkstreamTask.project_id, + predecessor.version.label("predecessor_version")) + .join(WorkstreamTask, WorkstreamTask.id == Submission.task_id) + .outerjoin(predecessor, (predecessor.id == Submission.supersedes_submission_id) + & (predecessor.task_id == Submission.task_id)) + .where(WorkstreamTask.project_id == str(request.project_id), + Submission.task_id == str(request.task_id), + Submission.id == str(request.submission_id), Submission.status == "submitted") + )).mappings().one_or_none() + if row is None: + raise SubmittedBundleUnavailable("submitted_bundle_unavailable") + try: + context = SubmittedPolicyContext( + guide_version=row["locked_guide_version"], + source_id=UUID(row["locked_guide_source_snapshot_id"]), + source_hash=row["locked_guide_source_snapshot_hash"], + effective_policy_id=UUID(row["locked_effective_project_submission_artifact_policy_id"]), + effective_policy_hash=row["locked_effective_project_submission_artifact_policy_hash"], + pre_policy_id=UUID(row["locked_pre_submit_checker_policy_id"]), + pre_policy_hash=row["locked_pre_submit_checker_bundle_hash"], + post_policy_id=UUID(row["locked_post_submit_checker_policy_id"]), + post_policy_version=row["locked_post_submit_checker_policy_version"], + post_policy_hash=row["locked_post_submit_checker_policy_hash"], + review_policy_id=UUID(row["locked_review_policy_id"]), + review_generation=row["locked_review_policy_generation"], + review_hash=row["locked_review_policy_hash"], + revision_policy_id=UUID(row["locked_revision_policy_id"]), + revision_generation=row["locked_revision_policy_generation"], + revision_hash=row["locked_revision_policy_hash"], + ) + predecessor_id = row["supersedes_submission_id"] + if (predecessor_id is None) != (row["predecessor_version"] is None): + raise ValueError("predecessor unavailable") + return SubmittedBundleFacts( + project_id=UUID(row["project_id"]), task_id=UUID(row["task_id"]), + assignment_id=UUID(row["task_assignment_id"]), + contributor_id=UUID(row["contributor_id"]), submission_id=UUID(row["id"]), + submission_version=row["version"], status=row["status"], + predecessor_id=UUID(predecessor_id) if predecessor_id else None, + predecessor_version=row["predecessor_version"], + contribution_policy_version_id=row["contribution_policy_version_id"], + admission_id=UUID(row["submission_bundle_admission_id"]), + binding_id=UUID(row["artifact_binding_id"]), + content_id=UUID(row["artifact_content_id"]), context=context, + ) + except (ValueError, TypeError, AttributeError): + raise SubmittedBundleUnavailable("submitted_bundle_unavailable") from None diff --git a/backend/scripts/behavior_ownership.py b/backend/scripts/behavior_ownership.py index ce5de3932..5d537f926 100644 --- a/backend/scripts/behavior_ownership.py +++ b/backend/scripts/behavior_ownership.py @@ -146,6 +146,13 @@ "backend/app/modules/authorization/api/outbox_dispatch.py", } ) +ARCH_04B_MATERIALIZATION_TARGETS = frozenset({ + "backend/app/modules/artifacts/api/submission_materialization.py", + "backend/app/modules/artifacts/post_submit_materialization.py", + "backend/app/modules/artifacts/post_submit_selection.py", + "backend/app/modules/tasks/api/submitted_bundle.py", + "backend/app/modules/tasks/submitted_bundle.py", +}) ARCH_04A_POST_SUBMIT_TARGETS = frozenset( { "backend/app/modules/checkers/api/post_submit.py", @@ -698,6 +705,7 @@ def _validate_additive_partition_transition( | ARCH_CP03B_ADAPTER_BINDING_AUTH_TARGETS | ARCH_CP04A_CONTRIBUTION_POLICY_TARGETS | ARCH_CP04B_CONTRIBUTION_POLICY_TARGETS + | ARCH_04B_MATERIALIZATION_TARGETS | ARCH_04A_POST_SUBMIT_TARGETS | ARCH_CP05_POLICY_AUTH_TARGETS | AUTH_18_PUBLIC_ACTIVATION_TARGETS diff --git a/backend/scripts/test_lane_catalogue.py b/backend/scripts/test_lane_catalogue.py index a27720f83..db9e5c97f 100644 --- a/backend/scripts/test_lane_catalogue.py +++ b/backend/scripts/test_lane_catalogue.py @@ -419,6 +419,8 @@ class TestLane: "tests/checkers/test_effective_intake_rules.py", "tests/test_default_pre_submit_execution.py", "tests/test_approved_guide_intake.py", + "tests/test_post_submit_materialization.py", + "tests/test_post_submit_selection.py", "tests/test_pre_submit_attempt_recovery.py", "tests/test_pre_submit_attempt_contracts.py", "tests/test_pre_submit_attempt_authority_integration.py", diff --git a/backend/tests/post_submit_materialization_helpers.py b/backend/tests/post_submit_materialization_helpers.py new file mode 100644 index 000000000..71539d5cc --- /dev/null +++ b/backend/tests/post_submit_materialization_helpers.py @@ -0,0 +1,181 @@ +"""Real verified admission/Submission fixtures and controlled post-submit authority.""" + +from contextlib import asynccontextmanager +from dataclasses import asdict +from io import BytesIO +from types import SimpleNamespace +import zipfile +import json + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from app.adapters.artifacts import create_artifact_store_bootstrap +from app.adapters.tasks import submitted_bundle_port +from app.api.deps.authorization import compose_hidden_submission_creation_command +from app.core.identifiers import new_record_id +from app.interfaces.artifacts import ArtifactStoreNamespaceClaim +from app.modules.artifacts.api import SubmissionBundlePreparationRequest +from app.modules.artifacts.post_submit_materialization import PostSubmissionMaterializer +from app.modules.artifacts.preparation import ArtifactPreparationService, ArtifactScratchManager +from app.modules.artifacts.service import artifact_storage_namespace_spec +from app.modules.artifacts.submission_archive import SubmissionArchiveInspector, SubmissionArchiveLimits +from app.modules.artifacts.submission_manifest import build_submission_manifest +from app.modules.authorization.api import ActorIdentityFacts, ActorKind +from app.modules.checkers.api import ( + CompiledPostSubmitPolicy, ExpectedPostSubmitContext, ObservedPostSubmitContext, + PostSubmissionStructuralInput, PostSubmitManifestEntry, PostSubmitPolicyInputs, +) +from app.modules.checkers.post_submit_contracts import make_post_submit_request +from app.modules.tasks.api import SubmissionCreationRequest +from app.modules.tasks.api.submitted_bundle import SubmittedBundleRequest +from app.modules.tasks.models import Submission +from tests.artifact_store_helpers import artifact_admission_limit_settings +from tests.checkers.post_submit.support import catalogue +from tests.pre_submit_test_helpers import approved_pre_submit_fixture +from tests.submission_preparation_auth_helpers import install_submitter_grant +from tests.tasks.lineage_fixtures import seed_started_task_for_artifact_test +from tests.tasks.submission_lineage_support import _seed_services, _verified_admission +from tests.test_artifact_admission import _settings, _context, _seed_human_actor +from tests.test_checker_materialization import _limits +from tests.test_default_pre_submit_execution import _archive, _bytes + + +class ControlledAuthority: + """Test-only seam, not a production service admission or generation store.""" + + def __init__(self): + self.requests = [] + self.selections = [] + + async def preflight(self, request): + self.requests.append(request) + + async def authorize(self, request, selection): + self.selections.append((request, selection)) + + +class CountedStore: + def __init__(self, store): + self.wrapped, self.opens = store, [] + self.identity = store.identity + + def open(self, reference): + self.opens.append(reference) + return self.wrapped.open(reference) + + +def archive_with_modes(evidence_path, project_id): + """Build a valid packet with independently known executable/plain members.""" + data = _archive(evidence_path=evidence_path) + # Preserve the valid governed packet and add both Unix file-mode cases. + archive_bytes = BytesIO() + with zipfile.ZipFile(BytesIO(data)) as source, zipfile.ZipFile(archive_bytes, "w") as target: + for item in source.infolist(): + target.writestr(item, source.read(item)) + for name, mode in (("run.sh", 0o100755), ("notes.txt", 0o100644)): + entry = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + entry.create_system = 3 + entry.external_attr = mode << 16 + target.writestr(entry, f"project {project_id}\n".encode()) + return archive_bytes.getvalue() + + +@asynccontextmanager +async def material_fixture(tmp_path, database_url, *, provider="local", scratch_limits=None, + storage_settings=None, provision_services=True): + engine = create_async_engine(database_url) + factory = async_sessionmaker(engine, expire_on_commit=False) + if storage_settings is not None: + settings = storage_settings + elif provider == "minio": + from tests.test_s3_artifact_store import minio_settings + settings = minio_settings(private_prefix=f"material/{new_record_id()}").model_copy(update={ + **artifact_admission_limit_settings(1024 * 1024), + "artifact_scratch_root": tmp_path / "intake-scratch", + "artifact_scratch_minimum_free_bytes": 0, + }) + else: + settings = _settings(tmp_path, maximum_bytes=1024 * 1024) + bootstrap = create_artifact_store_bootstrap(settings) + namespace = artifact_storage_namespace_spec(settings, bootstrap) + store = bootstrap.initialize_after_namespace_claim(ArtifactStoreNamespaceClaim( + bootstrap.identity, bootstrap.namespace_identity, namespace.namespace_fingerprint, + )) + manager = ArtifactScratchManager(root=tmp_path / "post-scratch", limits=scratch_limits if scratch_limits is not None else _limits()) + try: + plan, policy = await approved_pre_submit_fixture(factory, namespace, guide_version="v1") + context = _context() + if provision_services: + await _seed_services(factory) + task_id, assignment_id = new_record_id(), new_record_id() + async with factory.begin() as session: + await _seed_human_actor(session, context) + async with engine.begin() as connection: + params = dict(task=str(task_id), assignment=str(assignment_id), + project=str(plan.lineage.project_id), actor=str(context.actor_profile_id)) + await seed_started_task_for_artifact_test(connection, params) + await install_submitter_grant(connection, params) + data = archive_with_modes(policy["evidence_path"], plan.lineage.project_id) + preparation_request = SubmissionBundlePreparationRequest( + actor=ActorIdentityFacts(context.actor_profile_id, context.identity_link_id, ActorKind.HUMAN), + request_id=context.request_id, correlation_id=context.correlation_id, + task_id=task_id, assignment_id=assignment_id, predecessor_submission_id=None, + idempotency_key=new_record_id(), summary="Completed the required project work and included evidence.", + contributor_attestation="I confirm no confidential client data, credentials, or copied source material is included in this submission; rights_confirmed. " + " ".join(policy["attestation_terms"]), + media_type="application/zip", byte_source=_bytes(data), + ) + admission_id = await _verified_admission(factory, store, namespace, settings, context, preparation_request) + async with factory() as session: + created = await compose_hidden_submission_creation_command( + session, context, request_id=new_record_id(), correlation_id=new_record_id(), + ).create(SubmissionCreationRequest( + task_id=task_id, assignment_id=assignment_id, contributor_id=context.actor_profile_id, + predecessor_submission_id=None, admission_id=admission_id, + summary=preparation_request.summary, + contributor_attestation=preparation_request.contributor_attestation, + )) + async with factory() as session: + facts = await submitted_bundle_port(session).read(SubmittedBundleRequest( + plan.lineage.project_id, task_id, created.submission_id, + )) + submission = await session.scalar(select(Submission).where(Submission.id == str(created.submission_id))) + compiled = CompiledPostSubmitPolicy.model_validate_json(json.dumps(submission.locked_post_submit_checker_policy_body)) + inspector = SubmissionArchiveInspector(SubmissionArchiveLimits()) + manifest = build_submission_manifest(inspector.inspect(BytesIO(data))) + with zipfile.ZipFile(BytesIO(data)) as archive: + files = {name: archive.read(name) for name in archive.namelist() if not name.endswith("/")} + import hashlib + digest = "sha256:" + hashlib.sha256(data).hexdigest() + request = make_post_submit_request( + evaluation_request_id=new_record_id(), evaluation_generation=1, + project_id=facts.project_id, task_id=task_id, assignment_id=assignment_id, + submission_id=created.submission_id, submission_version=created.submission_version, + content_id=created.artifact_content_id, binding_id=created.artifact_binding_id, + content_sha256=digest, byte_count=len(data), expected_context=ExpectedPostSubmitContext(**asdict(facts.context)), + catalogue=catalogue(), policy=compiled, + structural_input=PostSubmissionStructuralInput( + summary=preparation_request.summary, worker_attestation=preparation_request.contributor_attestation, + package_hash=digest, criteria="Deliver the required project work.", + manifest=tuple(PostSubmitManifestEntry(artifact=e.normalized_path, hash=e.sha256, size_bytes=e.byte_count) + for e in manifest.entries if e.sha256 is not None), + evidence=(), policy_inputs=PostSubmitPolicyInputs(), + observed_context=ObservedPostSubmitContext(**asdict(facts.context)), + ), + ) + authority, counted = ControlledAuthority(), CountedStore(store) + preparation = ArtifactPreparationService(manager) + service = PostSubmissionMaterializer( + sessions=factory, tasks=submitted_bundle_port, store=counted, namespace=namespace, + preparation=preparation, inspector=inspector, authority=authority, + ) + yield SimpleNamespace(service=service, factory=factory, engine=engine, request=request, + created=created, facts=facts, files=files, data=data, manifest=manifest, + settings=settings, + store=counted, namespace=namespace, preparation=preparation, + manager=manager, inspector=inspector, authority=authority, + scratch=tmp_path / "post-scratch") + finally: + manager.close() + bootstrap.close() + await engine.dispose() diff --git a/backend/tests/tasks/submission_lineage_support.py b/backend/tests/tasks/submission_lineage_support.py index 76d34e8dc..d6feaf9c9 100644 --- a/backend/tests/tasks/submission_lineage_support.py +++ b/backend/tests/tasks/submission_lineage_support.py @@ -103,7 +103,9 @@ async def _verified_admission(factory, store, namespace, settings, context, requ == "verified" ) async with factory() as session: - admission = (await session.scalars(select(SubmissionBundleAdmission))).one() + admission = (await session.scalars(select(SubmissionBundleAdmission).where( + SubmissionBundleAdmission.put_attempt_id == str(result.put_attempt_id), + ))).one() assert admission.status == "ready" admission_id = UUID(admission.id) return admission_id diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index 2a7cb0dc2..d97128560 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -28,7 +28,6 @@ "GuideArtifactIngestCommand", "GuideArtifactIngestPort", "ArtifactBindingPort", - "ArtifactMaterializationPort", "CheckerArtifactOutputPort", "ArtifactOperatorReadPort", "ArtifactOperatorRecoveryPort", @@ -36,7 +35,6 @@ CANONICAL_REQUESTS = { "GuideArtifactIngestRequest", "CheckerOutputBindingRequest", - "BindingMaterializationRequest", "CheckerOutputArtifactRequest", "ArtifactRecoveryRequest", } @@ -195,7 +193,7 @@ def test_product_api_and_workers_cannot_import_or_inject_raw_artifact_types() -> def test_only_artifact_custody_services_own_provider_execution() -> None: - """Fence store calls to the orchestrator and narrow guide reader.""" + """Fence store calls to the orchestrator and exact guide/Submission readers.""" violations: list[str] = [] for path in _python_files(APP_ROOT / "modules" / "artifacts"): tree = _tree(path) @@ -203,7 +201,7 @@ def test_only_artifact_custody_services_own_provider_execution() -> None: node.name: node for node in tree.body if isinstance(node, ast.ClassDef) - and node.name in {"ArtifactStorageOrchestrator", "ScopedGuideDocumentGrant"} + and node.name in {"ArtifactStorageOrchestrator", "ScopedGuideDocumentGrant", "PostSubmissionMaterializer"} } for node in ast.walk(tree): if ( @@ -224,7 +222,7 @@ def test_only_artifact_custody_services_own_provider_execution() -> None: None, ) if owner is None or ( - owner.name == "ScopedGuideDocumentGrant" and node.func.attr != "open" + owner.name in {"ScopedGuideDocumentGrant", "PostSubmissionMaterializer"} and node.func.attr != "open" ): violations.append(f"{path.relative_to(BACKEND_ROOT)} calls {node.func.attr}") assert violations == [] @@ -503,15 +501,11 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit "ArtifactBindingPort": { "bind_checker_output", }, - "ArtifactMaterializationPort": { - "materialize_bindings", - }, "CheckerArtifactOutputPort": {"store"}, } expected_request_by_method = { "ingest": "GuideArtifactIngestRequest", "bind_checker_output": "CheckerOutputBindingRequest", - "materialize_bindings": "BindingMaterializationRequest", "store": "CheckerOutputArtifactRequest", } protocols = { diff --git a/backend/tests/test_behavior_ownership.py b/backend/tests/test_behavior_ownership.py index bcec7194d..df94a0f42 100644 --- a/backend/tests/test_behavior_ownership.py +++ b/backend/tests/test_behavior_ownership.py @@ -2120,3 +2120,22 @@ def test_approved_guide_intake_removal_is_exact(): ownership._validate_additive_partition_transition(_partition([retained]), trusted) with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): ownership._validate_additive_partition_transition(_partition([]), trusted) + + +def test_post_submit_materialization_partition_additions_are_exact(): + targets = { + "backend/app/modules/artifacts/api/submission_materialization.py", + "backend/app/modules/artifacts/post_submit_materialization.py", + "backend/app/modules/artifacts/post_submit_selection.py", + "backend/app/modules/tasks/api/submitted_bundle.py", + "backend/app/modules/tasks/submitted_bundle.py", + } + assert ownership.ARCH_04B_MATERIALIZATION_TARGETS == targets + retained = "backend/app/core/config.py" + trusted = _partition([retained]) + ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets])), trusted) + for forbidden in ("backend/app/modules/artifacts/download.py", "backend/app/modules/tasks/other_material.py"): + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition(_partition(sorted([retained, *targets, forbidden])), trusted) + with pytest.raises(ownership.BehaviorOwnershipError, match="untrusted_partition_change"): + ownership._validate_additive_partition_transition(_partition(sorted(targets)), trusted) diff --git a/backend/tests/test_ci_lane_catalogue.py b/backend/tests/test_ci_lane_catalogue.py index 98d5b0ef8..6b374327e 100644 --- a/backend/tests/test_ci_lane_catalogue.py +++ b/backend/tests/test_ci_lane_catalogue.py @@ -229,6 +229,8 @@ def test_measured_hotspots_have_explicit_semantic_owners() -> None: "tests/test_checkers.py", "tests/test_default_pre_submit_execution.py", "tests/test_approved_guide_intake.py", + "tests/test_post_submit_materialization.py", + "tests/test_post_submit_selection.py", "tests/test_pre_submit_attempt_recovery.py", "tests/test_pre_submit_attempt_contracts.py", "tests/test_pre_submit_attempt_authority_integration.py", diff --git a/backend/tests/test_post_submit_materialization.py b/backend/tests/test_post_submit_materialization.py new file mode 100644 index 000000000..3869e8e0f --- /dev/null +++ b/backend/tests/test_post_submit_materialization.py @@ -0,0 +1,297 @@ +"""Verified exact Submission input, async lifetime and fail-closed composition.""" + +import asyncio +import pickle +from dataclasses import asdict + +import pytest +from sqlalchemy import text + +from app.adapters.artifacts import post_submission_materialization +from app.core.identifiers import new_record_id +from app.interfaces.artifacts import ArtifactInputMismatchError +from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from tests.checkers.post_submit.support import change_request +from tests.checkers.post_submit.test_result_contract import result +from tests.post_submit_materialization_helpers import material_fixture + + +class Consumer: + def __init__(self, files): + self.files, self.view, self.calls = files, None, 0 + self.entered, self.release = asyncio.Event(), None + self.failure = None + + async def evaluate(self, request, view): + self.calls += 1 + self.view = view + self.entered.set() + await asyncio.sleep(0) # The view must remain usable across real async work. + assert {e.normalized_path for e in view.entries if e.entry_type == "file"} == set(self.files) + flags = {entry.normalized_path: entry.executable for entry in view.entries} + assert flags["run.sh"] is True + assert flags["notes.txt"] is False + for path, data in self.files.items(): + assert view.read_file(path, maximum_bytes=len(data)) == data + if self.release is not None: + await self.release.wait() + if self.failure: + raise self.failure + return result(request) + + +def assert_closed(h, consumer): + with pytest.raises(RuntimeError, match="closed"): + _ = consumer.view.entries + with pytest.raises(RuntimeError, match="closed"): + consumer.view.read_file(next(iter(h.files)), maximum_bytes=1024) + with pytest.raises(TypeError, match="process-local"): + pickle.dumps(consumer.view) + assert list((h.scratch / "workspaces").iterdir()) == [] + assert not h.preparation._active + + +@pytest.mark.parametrize("provider", ["local", "minio"]) +async def test_exact_materialization_reads_verified_original_and_revokes_view(tmp_path, isolated_database_env, provider): + if provider == "minio": + from tests.test_s3_artifact_store import provision_minio_bucket + await provision_minio_bucket.__wrapped__() + async with material_fixture(tmp_path, isolated_database_env, provider=provider) as h: + consumer = Consumer(h.files) + material = await h.service.materialize(h.request, consumer) + assert material.submission_id == h.created.submission_id + assert material.submission_version == h.created.submission_version + assert material.admission_id == h.created.admission_id + assert material.binding_id == h.created.artifact_binding_id + assert material.content_id == h.created.artifact_content_id + assert material.content_sha256 == h.request.content_sha256 + assert material.byte_count == len(h.data) + assert material.semantic_manifest_sha256 == h.manifest.sha256 + assert consumer.calls == len(h.store.opens) == 1 + assert len(h.authority.selections) == 1 + material.evaluation.validate_request(h.request) + assert_closed(h, consumer) + async with h.factory() as session: + replica = (await session.execute(text( + "select verified_replica_id,archive_sha256,archive_byte_count,semantic_manifest_sha256 " + "from submission_bundle_admissions where id=:id" + ), {"id": str(material.admission_id)})).one() + assert str(material.replica_id) == str(replica[0]) + assert (material.content_sha256, material.byte_count, material.semantic_manifest_sha256) == tuple(replica[1:]) + assert set(asdict(material)) == {"submission_id", "submission_version", "admission_id", "binding_id", + "content_id", "replica_id", "content_sha256", "byte_count", + "semantic_manifest_sha256", "evaluation"} + + +async def test_default_composition_denies_before_database_provider_or_scratch(): + from tests.checkers.post_submit.support import request + class Forbidden: + def __getattr__(self, name): + pytest.fail(f"protected access: {name}") + def __call__(self, *args, **kwargs): + pytest.fail("database access") + forbidden = Forbidden() + service = post_submission_materialization(sessions=forbidden, store=forbidden, namespace=forbidden, + preparation=forbidden, inspector=forbidden) + with pytest.raises(PostSubmissionMaterializationUnavailable, match="materialization_unavailable"): + await service.materialize(request(), forbidden) + + +async def test_foreign_or_changed_request_denies_before_provider(tmp_path, isolated_database_env): + async with material_fixture(tmp_path, isolated_database_env) as h: + consumer = Consumer(h.files) + for field in ("task_id", "assignment_id", "submission_id", "binding_id", "content_id", "submission_version", "byte_count", "content_sha256"): + value = (2 if field == "submission_version" else h.request.byte_count + 1 if field == "byte_count" + else "sha256:" + "0" * 64 if field == "content_sha256" else new_record_id()) + with pytest.raises(PostSubmissionMaterializationUnavailable): + await h.service.materialize(change_request(h.request, **{field: value}), consumer) + assert h.store.opens == [] and consumer.calls == 0 + assert not h.preparation._active + # A valid control reaches the same consumer through the real stored selection. + await h.service.materialize(h.request, consumer) + assert consumer.calls == 1 + + +@pytest.mark.parametrize("outcome", ["failure", "cancel", "replica_drift", "status_drift"]) +async def test_async_exit_and_concurrent_drift_never_return_stale_material(tmp_path, isolated_database_env, outcome): + async with material_fixture(tmp_path, isolated_database_env) as h: + consumer = Consumer(h.files) + consumer.release = asyncio.Event() + operation = asyncio.create_task(h.service.materialize(h.request, consumer)) + await asyncio.wait_for(consumer.entered.wait(), 10) + if outcome == "failure": + consumer.failure = ValueError("controlled consumer failure") + expected = ValueError + elif outcome == "cancel": + operation.cancel() + expected = asyncio.CancelledError + else: + # An independent transaction must finish while material is in use: + # no row lock may block a change during the callback. + async with h.factory() as session, session.begin(): + await session.execute(text("set local lock_timeout='300ms'")) + if outcome == "replica_drift": + await session.execute(text("update artifact_replicas set availability_state='unavailable' where id=:id"), + {"id": str(h.authority.selections[0][1].replica_id)}) + else: + await session.execute(text("update submissions set status='checks_failed' where id=:id"), + {"id": str(h.created.submission_id)}) + expected = PostSubmissionMaterializationUnavailable + consumer.release.set() + with pytest.raises(expected): + await asyncio.wait_for(operation, 10) + assert_closed(h, consumer) + + +async def test_wrong_provider_bytes_or_manifest_never_reach_consumer(tmp_path, isolated_database_env): + from tests.test_default_pre_submit_execution import _bytes + async with material_fixture(tmp_path, isolated_database_env) as h: + consumer = Consumer(h.files) + original = h.store.open + h.store.open = lambda _: _bytes(h.data[:-1] + bytes([h.data[-1] ^ 1])) + with pytest.raises(ArtifactInputMismatchError): + await h.service.materialize(h.request, consumer) + h.store.open = original + packet = h.request.structural_input.model_copy(update={"manifest": ()}) + with pytest.raises(PostSubmissionMaterializationUnavailable, match="manifest_mismatch"): + await h.service.materialize(change_request(h.request, structural_input=packet), consumer) + assert consumer.calls == 0 and not h.preparation._active + + +async def test_abort_during_projection_prevents_consumer_entry(tmp_path, isolated_database_env, monkeypatch): + from contextlib import contextmanager + from threading import Event + async with material_fixture(tmp_path, isolated_database_env) as h: + entered, release = Event(), Event() + original = h.inspector._projected_tree + @contextmanager + def paused_projection(*args, **kwargs): + with original(*args, **kwargs) as tree: + entered.set() + assert release.wait(10) + yield tree + monkeypatch.setattr(h.inspector, "_projected_tree", paused_projection) + consumer = Consumer(h.files) + operation = asyncio.create_task(h.service.materialize(h.request, consumer)) + assert await asyncio.to_thread(entered.wait, 10) + operation.cancel() + await asyncio.sleep(0.05) + assert not operation.done() + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(operation, 10) + assert consumer.calls == 0 and not h.preparation._active + assert list((h.scratch / "workspaces").iterdir()) == [] + + +async def test_consumer_deadline_revokes_material_and_releases_scratch(tmp_path, isolated_database_env): + from tests.test_checker_materialization import _limits + from app.modules.artifacts.preparation import ArtifactPreparationDeadlineError + async with material_fixture(tmp_path, isolated_database_env, scratch_limits=_limits(total_deadline_seconds=2)) as h: + consumer = Consumer(h.files) + consumer.release = asyncio.Event() + with pytest.raises(ArtifactPreparationDeadlineError): + await asyncio.wait_for(h.service.materialize(h.request, consumer), 10) + assert consumer.calls == 1 + assert_closed(h, consumer) + + +async def test_post_submit_expansion_cannot_bypass_aggregate_quota(tmp_path, isolated_database_env): + from app.modules.artifacts.preparation import HARD_MAXIMUM_ARTIFACT_BYTES, ArtifactScratchCapacityError + from tests.test_checker_materialization import _limits + async with material_fixture(tmp_path, isolated_database_env, scratch_limits=_limits( + aggregate_reserved_bytes=HARD_MAXIMUM_ARTIFACT_BYTES, + )) as h: + consumer = Consumer(h.files) + with pytest.raises(ArtifactScratchCapacityError): + await h.service.materialize(h.request, consumer) + assert consumer.calls == 0 + assert not h.preparation._active + assert list((h.scratch / "workspaces").iterdir()) == [] + + +async def test_stored_manifest_mismatch_never_reaches_consumer(tmp_path, isolated_database_env): + async with material_fixture(tmp_path, isolated_database_env) as h: + consumer = Consumer(h.files) + # Deliberately corrupt retained custody in this isolated database. Normal + # database guards forbid these writes; exercise ART's independent byte check. + async with h.factory() as session, session.begin(): + for table in ("submission_bundle_admissions", "pre_submit_evidence_sets"): + await session.execute(text(f"alter table {table} disable trigger user")) + await session.execute(text( + "update pre_submit_evidence_sets set semantic_manifest_sha256=:bad where id=" + "(select pre_submit_evidence_set_id from submission_bundle_admissions where id=:id)" + ), {"id": str(h.created.admission_id), "bad": "sha256:" + "0" * 64}) + await session.execute(text( + "update submission_bundle_admissions set semantic_manifest_sha256=:bad where id=:id" + ), {"id": str(h.created.admission_id), "bad": "sha256:" + "0" * 64}) + for table in ("submission_bundle_admissions", "pre_submit_evidence_sets"): + await session.execute(text(f"alter table {table} enable trigger user")) + with pytest.raises(PostSubmissionMaterializationUnavailable, match="manifest_mismatch"): + await h.service.materialize(h.request, consumer) + assert len(h.store.opens) == 1 and consumer.calls == 0 + assert not h.preparation._active + assert list((h.scratch / "workspaces").iterdir()) == [] + + +async def test_foreign_consumer_result_is_rejected_and_cleaned(tmp_path, isolated_database_env): + async with material_fixture(tmp_path, isolated_database_env) as h: + class WrongResult(Consumer): + async def evaluate(self, request, view): + await super().evaluate(request, view) + return result(request, evaluation_generation=request.evaluation_generation + 1) + consumer = WrongResult(h.files) + with pytest.raises(ValueError, match="request mismatch"): + await h.service.materialize(h.request, consumer) + assert consumer.calls == 1 + assert_closed(h, consumer) + + +async def test_provider_stream_has_no_selection_transaction_and_rejects_drift(tmp_path, isolated_database_env): + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + async with material_fixture(tmp_path, isolated_database_env) as h: + # Tag only selection connections so pg_stat_activity can observe the + # actual materializer session, independently of fixture/observer traffic. + tag = "material-selection-" + str(new_record_id()) + selection_engine = create_async_engine(isolated_database_env, connect_args={ + "server_settings": {"application_name": tag}, + }) + h.service._sessions = async_sessionmaker(selection_engine) + entered, release = asyncio.Event(), asyncio.Event() + original = h.store.open + async def paused_stream(reference): + async for chunk in original(reference): + entered.set() + await release.wait() + yield chunk + h.store.open = paused_stream + consumer = Consumer(h.files) + operation = asyncio.create_task(h.service.materialize(h.request, consumer)) + try: + await asyncio.wait_for(entered.wait(), 10) + assert consumer.calls == 0 + async with h.factory() as session, session.begin(): + rows = (await session.execute(text( + "select pid, state, xact_start from pg_stat_activity where application_name=:tag" + ), {"tag": tag})).all() + assert rows and all(row.state == "idle" and row.xact_start is None for row in rows) + locks = await session.scalar(text( + "select count(*) from pg_locks where pid in " + "(select pid from pg_stat_activity where application_name=:tag) " + "and locktype in ('relation', 'tuple', 'transactionid')" + ), {"tag": tag}) + assert locks == 0 + await session.execute(text("set local lock_timeout='300ms'")) + await session.execute(text( + "update artifact_replicas set availability_state='unavailable' where id=:id" + ), {"id": str(h.authority.selections[0][1].replica_id)}) + release.set() + with pytest.raises(PostSubmissionMaterializationUnavailable): + await asyncio.wait_for(operation, 10) + assert_closed(h, consumer) + finally: + release.set() + if not operation.done(): + operation.cancel() + await asyncio.gather(operation, return_exceptions=True) + await selection_engine.dispose() diff --git a/backend/tests/test_post_submit_selection.py b/backend/tests/test_post_submit_selection.py new file mode 100644 index 000000000..5001e1405 --- /dev/null +++ b/backend/tests/test_post_submit_selection.py @@ -0,0 +1,132 @@ +"""Stored ownership, frozen context and independent-session selection proof.""" + +from dataclasses import asdict + +import pytest +from sqlalchemy import event, text + +from app.adapters.tasks import submitted_bundle_port +from app.core.identifiers import new_record_id +from app.modules.artifacts.api.submission_materialization import PostSubmissionMaterializationUnavailable +from app.modules.tasks.api.submitted_bundle import SubmittedBundleRequest, SubmittedBundleUnavailable +from tests.checkers.post_submit.support import change_request +from tests.post_submit_materialization_helpers import material_fixture +from tests.test_post_submit_materialization import Consumer + + +async def test_task_projection_is_owner_qualified_and_exact(tmp_path, isolated_database_env): + async with material_fixture(tmp_path, isolated_database_env) as h: + statements = [] + def observe(connection, cursor, statement, parameters, context, executemany): + statements.append(statement) + event.listen(h.engine.sync_engine, "before_cursor_execute", observe) + try: + async with h.factory() as session: + reader = submitted_bundle_port(session) + with pytest.raises(SubmittedBundleUnavailable): + await reader.read(SubmittedBundleRequest(new_record_id(), h.request.task_id, h.request.submission_id)) + facts = await reader.read(SubmittedBundleRequest(h.request.project_id, h.request.task_id, h.request.submission_id)) + assert len(statements) == 2 + for statement in statements: + assert "workstream_tasks.project_id =" in statement + assert "submissions.task_id =" in statement and "submissions.id =" in statement + assert "FOR UPDATE" not in statement + assert "package_uri" not in statement and "policy_body" not in statement + assert "worker_attestation" not in statement and "summary" not in statement + finally: + event.remove(h.engine.sync_engine, "before_cursor_execute", observe) + assert facts.admission_id == h.created.admission_id + assert facts.binding_id == h.created.artifact_binding_id + assert facts.content_id == h.created.artifact_content_id + async with h.factory() as session: + row = (await session.execute(text("select * from submissions where id=:id"), + {"id": str(h.created.submission_id)})).mappings().one() + assert str(facts.assignment_id) == str(row["task_assignment_id"]) + assert str(facts.contributor_id) == str(row["contributor_id"]) + assert facts.contribution_policy_version_id == row["contribution_policy_version_id"] + assert facts.predecessor_id is None and facts.predecessor_version is None + mapping = { + "guide_version": "locked_guide_version", "source_id": "locked_guide_source_snapshot_id", + "source_hash": "locked_guide_source_snapshot_hash", + "effective_policy_id": "locked_effective_project_submission_artifact_policy_id", + "effective_policy_hash": "locked_effective_project_submission_artifact_policy_hash", + "pre_policy_id": "locked_pre_submit_checker_policy_id", "pre_policy_hash": "locked_pre_submit_checker_bundle_hash", + "post_policy_id": "locked_post_submit_checker_policy_id", "post_policy_version": "locked_post_submit_checker_policy_version", + "post_policy_hash": "locked_post_submit_checker_policy_hash", "review_policy_id": "locked_review_policy_id", + "review_generation": "locked_review_policy_generation", "review_hash": "locked_review_policy_hash", + "revision_policy_id": "locked_revision_policy_id", "revision_generation": "locked_revision_policy_generation", + "revision_hash": "locked_revision_policy_hash", + } + assert {key: str(value) for key, value in asdict(facts.context).items()} == { + key: str(row[column]) for key, column in mapping.items() + } + + +async def test_frozen_context_substitution_and_authority_denial_precede_io(tmp_path, isolated_database_env): + async with material_fixture(tmp_path, isolated_database_env) as h: + consumer = Consumer(h.files) + for name in ("source_id", "effective_policy_id", "pre_policy_id", "post_policy_id", "review_policy_id", "revision_policy_id", "review_generation", "revision_hash"): + value = (2 if name == "review_generation" else "sha256:" + "0" * 64 if name == "revision_hash" else new_record_id()) + expected = h.request.expected_context.model_copy(update={name: value}) + packet = h.request.structural_input.model_copy(update={ + "observed_context": h.request.structural_input.observed_context.model_copy(update={name: value}), + }) + with pytest.raises(PostSubmissionMaterializationUnavailable, match="identity_mismatch"): + await h.service.materialize(change_request(h.request, expected_context=expected, structural_input=packet), consumer) + assert not h.store.opens and consumer.calls == 0 + # Explicit hidden seam denial, not a claim of live AUTH replay storage. + class RejectSelection: + async def preflight(self, request): + pass + async def authorize(self, request, selection): + assert selection.submission.admission_id == h.created.admission_id + raise PostSubmissionMaterializationUnavailable("controlled_generation_denied") + h.service._authority = RejectSelection() + with pytest.raises(PostSubmissionMaterializationUnavailable, match="controlled_generation_denied"): + await h.service.materialize(h.request, consumer) + assert not h.store.opens and not h.preparation._active + + +async def test_valid_foreign_lineage_mixes_deny_before_material_access(tmp_path, isolated_database_env, monkeypatch): + async with material_fixture(tmp_path / "first", isolated_database_env) as first: + # Share the configured store and fixed services, as two real projects do. + async with material_fixture( + tmp_path / "second", isolated_database_env, + storage_settings=first.settings, provision_services=False, + ) as second: + fields = ("project_id", "task_id", "assignment_id", "submission_id", "binding_id", "content_id") + assert all(getattr(first.request, key) != getattr(second.request, key) for key in fields) + assert first.facts.contributor_id != second.facts.contributor_id + assert first.created.admission_id != second.created.admission_id + # Each untouched stored lineage must pass through real materialization. + for own in (first, second): + value = await own.service.materialize(own.request, Consumer(own.files)) + assert value.submission_id == own.created.submission_id + own.store.opens.clear() + own.authority.selections.clear() + def forbidden(*args, **kwargs): + pytest.fail("foreign lineage reached provider, scratch, or consumer") + class ForbiddenConsumer: + evaluate = staticmethod(forbidden) + for own, foreign in ((first, second), (second, first)): + with monkeypatch.context() as patch: + patch.setattr(own.store, "open", forbidden) + patch.setattr(own.preparation, "prepare", forbidden) + mixtures = [{key: getattr(foreign.request, key)} for key in fields] + mixtures.extend(( + {key: getattr(foreign.request, key) for key in fields if key != "project_id"}, + {key: getattr(foreign.request, key) for key in ("binding_id", "content_id")}, + )) + for mix in mixtures: + if "project_id" in mix: + # Keep duplicated project/policy facts schema-valid so + # rejection must come from persisted owner selection. + mix.update(policy=foreign.request.policy, + expected_context=foreign.request.expected_context, + structural_input=foreign.request.structural_input) + mixed = change_request(own.request, **mix) + with pytest.raises(PostSubmissionMaterializationUnavailable): + await own.service.materialize(mixed, ForbiddenConsumer()) + assert not own.authority.selections + assert not own.preparation._active + assert list((own.scratch / "workspaces").iterdir()) == [] diff --git a/docs/architecture_checker_framework.md b/docs/architecture_checker_framework.md index ee7183ae0..bdb4ed86c 100644 --- a/docs/architecture_checker_framework.md +++ b/docs/architecture_checker_framework.md @@ -156,7 +156,8 @@ The canonical `policy_hash` binds ordered entries, configuration and exact catalogue/implementation identities. Domain project policy versions record changes to project rules; they do not select obsolete software implementations. The public phase execution port remains unavailable pending ARCH-04C/04D. -POL-04B connects unified guide setup; separate pre/post approval remains later. +POL-04B connects unified guide setup; POL-05B and POL-06B expose separate +pre-submit and post-submit policy approvals. ## Blocking Policy @@ -328,7 +329,12 @@ obtains fresh authority. Replay returns ART's canonical result unchanged. The post command validates and delegates CHECKER's closed value contract. Production explicitly uses `UnavailablePostSubmissionExecution`; this does not install durable post-submit execution, authorize material reads, or prove attempt and -currentness ownership. ARCH-04B/04C/04D/04E own that execution cutover. +currentness ownership. ARCH-04B supplies the hidden ART input port: exact consumed +Submission bytes, rebuilt manifest, async scoped file access and cleanup, followed +by a fresh material-selection check. Production composition denies materialization; +ARCH-04B2 output custody, ARCH-04C durable execution, ARCH-04D authority and ARCH-04E +routing remain the execution cutover. A returned evaluation value is not a stored +current result or acceptance. Retained run and submission history now use canonical AUTH and separate fixed contributor/manager projections. The alternate execution service and Celery worker are removed; the facade neither wraps them nor adds another policy compiler. @@ -514,7 +520,8 @@ severities. The following is the intended end-to-end lifecycle. Pre-submit intake and retained-history reads are implemented. Canonical durable post-submit execution, -result routing and recovery remain unavailable pending ARCH-04B/04C/04D/04E/04F; +result routing and recovery remain unavailable pending ARCH-04B2/04C/04D/04E/04F; +ARCH-04B hidden input is delivered with deny-only production authority; the flow below is not a claim that those jobs or transitions are live. ```text @@ -595,7 +602,8 @@ The checker run records: - warning count - completion timestamp -After ARCH-04B materialization, ARCH-04C result custody, ARCH-04D activation and +With ARCH-04B input delivered, ARCH-04B2 output custody, ARCH-04C result custody, +ARCH-04D activation and ARCH-04E routing integration, this gives reviewers proof that they are reviewing the same immutable binding and manifest that passed automated checks; legacy caller-owned manifest fields are not authority. diff --git a/docs/architecture_data_model.md b/docs/architecture_data_model.md index 63c072960..0a0772211 100644 --- a/docs/architecture_data_model.md +++ b/docs/architecture_data_model.md @@ -317,8 +317,9 @@ replace the Task and Assignment guards together. Submission's three ART references remain all-null during transaction staging or all-present. The production creation command consumes the exact ART admission and fills all three before its root transaction commits. This is separate from the -required initial assignment identity. Canonical ART-to-CHECKERS materialization -remains ARCH-04B/04C work. Retained payment columns on Submission and CheckerRun +required initial assignment identity. Hidden exact ART-to-CHECKERS input materialization is delivered by ARCH-04B. +Output custody (ARCH-04B2) and durable execution/results (ARCH-04C) remain pending; +production access remains deny-only until ARCH-04D. Retained payment columns on Submission and CheckerRun are nullable, so new unified-guide work requires no invented economic configuration. CP09 owns physical economic-schema removal after its remaining consumers change. @@ -1646,11 +1647,11 @@ Fields: - `locked_revision_policy_id` - `locked_revision_policy_generation` - `locked_revision_policy_hash` -- `artifact_binding_id` (target after ARCH-04B/04C custody) -- `submission_bundle_manifest_id` (target after ARCH-04B/04C custody) -- `package_hash` (legacy; replacement custody in ARCH-04B/04C) -- `artifact_hash_manifest` (legacy; replacement custody in ARCH-04B/04C) -- `artifact_manifest_hash` (legacy; replacement custody in ARCH-04B/04C) +- `artifact_binding_id` (target after ARCH-04B2/04C custody) +- `submission_bundle_manifest_id` (target after ARCH-04B2/04C custody) +- `package_hash` (legacy; replacement custody in ARCH-04B2/04C) +- `artifact_hash_manifest` (legacy; replacement custody in ARCH-04B2/04C) +- `artifact_manifest_hash` (legacy; replacement custody in ARCH-04B2/04C) - `summary` Status: diff --git a/docs/engineering/authorization_activation_custody.md b/docs/engineering/authorization_activation_custody.md index fe8f06c65..6e60baed8 100644 --- a/docs/engineering/authorization_activation_custody.md +++ b/docs/engineering/authorization_activation_custody.md @@ -58,8 +58,9 @@ The table retains historical planning-custody labels, not a literal mapping of every typed runtime `ActionOwner`. XINT-06B groups runtime `WS-AUTH-001-ART-06A` post-submit materialization and `WS-AUTH-001-ART-06B` output write/binding. ARCH-04D is their current replacement -activation boundary after ARCH-04B/04C; the typed catalogue is unchanged by -this planning reconciliation. Do not implement an additional XINT-06B lane. +activation boundary after delivered ARCH-04B hidden input, planned ARCH-04B2 +output custody and ARCH-04C durable execution/results. Production materialization +remains deny-only; the typed catalogue is unchanged by this reconciliation. Do not implement an additional XINT-06B lane. Runtime owner `WS-XINT-002-07` retains catalogue custody. The only approved v0.1 availability transition is 07A packet materialization. Evidence binding diff --git a/docs/operations_project_operating_manual.md b/docs/operations_project_operating_manual.md index e48d4bed8..601216f9f 100644 --- a/docs/operations_project_operating_manual.md +++ b/docs/operations_project_operating_manual.md @@ -261,7 +261,8 @@ AUTH-OUTBOX-02 supplies shared delivery, and ARCH-03C1 supplies exact reconciler authority and decision-bound receipts. ARCH-03C2 delivers atomic producer publication and first handler registration with enforced prefork topology. Public manager activation and ARCH-03D hidden approved-guide intake are delivered. -Exact post-submit materialization (ARCH-04B) is next. Public intake remains deferred +Hidden exact post-submit input (ARCH-04B) is delivered with deny-only production +authority. ARCH-04B2 output custody is next. Public intake remains deferred to ARCH-02I after evaluation and remediation prerequisites. The intended unified flow uses one compilation result for sufficiency and diff --git a/docs/roadmap_status.md b/docs/roadmap_status.md index bbb2a5d2a..75d38136c 100644 --- a/docs/roadmap_status.md +++ b/docs/roadmap_status.md @@ -94,8 +94,9 @@ replay. ARCH-03C4 adds the three exact-authorized public task queues. ARCH-03C5 adds exact-authorized Contributor/Manager detail and requirements. ARCH-03C6 adds separate exact-authorized locked-context reads. ARCH-03C7 adds bounded public Audit Authority history access. ARCH-03D connects hidden intake through durable intent to the activated historical -guide using canonical owner ports. Exact post-submit materialization and durable -evaluation follow; public intake remains deferred to their cutover prerequisites. Required success +guide using canonical owner ports. ARCH-04B adds hidden verified Submission input +with scoped async access; production materialization authority remains deny-only. +Checker output custody and durable evaluation follow; public intake remains deferred to their cutover prerequisites. Required success then branches on the locked ReviewPolicy: true routes to human `allow_review`; false invokes shared authorized acceptance without a human Review. Both routing integrations remain planned. Human review/revision, contribution and conditional @@ -149,7 +150,7 @@ cannot be reused as post-submission review-gate evidence. See the | Contributor artifact preparation | **Hidden and proven** | One outer ZIP; bounded scratch inspection; canonical manifest; platform and project prechecks; unchanged-work rejection; durable put intent; verification; capacity-charged ready admission; hidden final handoff validates the exact activated historical guide through owner ports | Complete the later public admission-only cutover | | Pre-submission intake checking | **Hidden with approved-guide lineage** | Separate versioned pre-submission catalogue, locked effective-plan compilation, platform/project checks during continuous preparation, blocking feedback before Submission creation, and one internal phase command covering execution/replay with the JSON precheck removed; ARCH-03D connects approved-guide lineage through the final durable handoff | Complete the canonical public cutover after evaluation/remediation prerequisites; passing intake must never substitute for post-submit evaluation | | Immutable Submission creation | **Hidden foundation; public packet creation retired** | Contributor preparation authority; durable pre-submit reservation and exact completed-evidence recovery without rerunning checks; atomic admission consumption; TASK-owned admission-backed creation with exact assignment ContributionPolicyVersion and locked policy lineage; fixed-service artifact binding; replay/concurrency/rollback proof | Finish downstream evaluation and the canonical public integration. The retained submission-list GET is not a usable creation POST | -| Post-submission evaluation and `allow_review` | **Planned; immediate integration milestone** | One canonical CHECKER post-submit catalogue/compiler used by existing consumers, internal phase service with explicitly unavailable post execution, hidden value contracts and structural-handler conformance; existing pre-review and materialization foundations | Connect the unavailable phase port to durable execution; evaluate the exact Submission against its locked policy; persist one durable current superseding result; activate fixed services; automatically dispatch it and publish the canonical `allow_review` manifest | +| Post-submission evaluation and `allow_review` | **Planned; immediate integration milestone** | One canonical CHECKER post-submit catalogue/compiler used by existing consumers, internal phase service with explicitly unavailable post execution, hidden value contracts and structural-handler conformance; ARCH-04B hidden exact verified Submission materialization with deny-only production authority | Add ARCH-04B2 output custody; connect the unavailable phase port to durable execution; evaluate the exact Submission against its locked policy; persist one durable current superseding result; activate fixed services; automatically dispatch it and publish the canonical `allow_review` manifest | | Review queue and lease | **Hidden persistence foundation** | Queue/admission idempotency and ReviewLease/preference persistence; complete unavailable REV action/principal catalogue and typed AUTH contracts | Packet-membership contract and manifest; Review schema; canonical admission from `allow_review`; claim/lease/packet authority; lease copies the Submission-stamped policy version with no CON lookup | | Review decision and revision | **Planned** | Review/revision policy identities and mutation authority; approved same-task revision-rebase semantics | Immutable findings and decisions; `accept`, `needs_revision`, and `reject`; complete-context revision preparation; finding responses; replacement contributor rules; replay and recovery | | Contribution and compensation truth | **Schema foundations plus public policy administration** | ContributionPolicyVersion persistence; lifecycle-audit participant; adapter bindings; public Finance policy administration | Persist ContributionRecord/CompensationAward and one shared FinalAcceptance/submitter operation for human accept or authorized false/pass routing. Only actual Reviews create reviewer records. Evaluate frozen actor rules into zero, one or two awards | @@ -436,8 +437,9 @@ The next dependency-safe product sequence is: The operation replaces superseded economic readiness. Retained economic data and downstream TASK consumers remain until their scoped cutover; deleting retained data is not authorized by code cleanup. -2. **Produce current post-submit evidence and policy-governed routing.** Materialize the exact immutable - Submission, execute the locked post-submit plan, persist one current result, +2. **Produce current post-submit evidence and policy-governed routing.** ARCH-04B's + hidden exact input materialization is delivered. Next add ARCH-04B2 output custody, + execute the locked post-submit plan, persist one current result, activate only its fixed services, and automatically publish an exact human `allow_review` manifest on true when no blocking failure exists. CHECKERS owns durable execution/currentness; the shared facade does not @@ -516,6 +518,11 @@ compatibility identity writer, manual checker execution, finalize repair and fabricated-actor Celery gate are removed. Retained records are preserved; canonical durable post-submit execution and recovery remain pending ARCH-04. +ARCH-04B adds hidden exact Submission materialization through TASK's immutable +read port and ART's consumed admission. Local/MinIO input is independently reread, +verified and projected through canonical bounded scratch. Production access remains +deny-only; it does not activate durable checker execution or public intake. + ## Critical Dependency Map ```text @@ -525,6 +532,7 @@ Delivered foundations (not a claim of full public integration) public manager activation context + exact guide activation/binding task/assignment/Submission lineage + hidden intake/creation ARCH-03D exact approved historical guide through durable intake handoff + ARCH-04B hidden verified Submission input (production authority deny-only) shared dispatcher + exact-authorized hidden assignment invalidation ARCH-03C2 invalidation producer + first handler registration ARCH-03C3 exact manager task create/screen/release + replay @@ -535,7 +543,7 @@ Delivered foundations (not a claim of full public integration) | v Remaining integration - exact post-submit materialization + durable evaluation + remediation + checker output custody + durable evaluation + live authority + remediation -> public intake and immutable admitted Submission cutover -> durable current post-submit result + required checks pass | @@ -644,8 +652,10 @@ remaining trace sequence is: - Hidden intake: [ARCH-03D](../.commitrail/initiatives/WS-ARCH-001/WS-ARCH-001-03D.md) removes the private TASK/PROJECTS lookup at final durable handoff. No public preparation or Submission endpoint is activated. -- Post-submit admission: after delivered `POL-07B` and `ARCH-03C`, `ARCH-04B -> 04C -> - 04D -> 04E` supplies materialization, durable results, authority and routing. +- Post-submit admission: after delivered `POL-07B` and `ARCH-03C`, delivered hidden + [ARCH-04B input materialization](../.commitrail/initiatives/WS-ARCH-001/planning/chunks/WS-ARCH-001-04B-art-post-submit-materialization.md) + leads to `ARCH-04B2 -> 04C -> 04D -> 04E` for output custody, durable results, + live authority and routing. An ART-owned output/log custody child precedes `04C` final completion. AUTH-OUTBOX-02 delivers shared live authority, phase audit custody and Celery delivery/recovery scans over CON-02B. Delivery termination is bounded by a diff --git a/docs/spec_artifact_storage_service.md b/docs/spec_artifact_storage_service.md index d05b6b7be..70cfc778e 100644 --- a/docs/spec_artifact_storage_service.md +++ b/docs/spec_artifact_storage_service.md @@ -356,7 +356,7 @@ SubmissionAdmissionConsumptionPort.consume(SubmissionAdmissionConsumptionRequest GuideDocumentManifestPort.load(GuideDocumentManifestRequest) GuideDocumentGrant.open(opaque_document_handle) ArtifactBindingPort.bind_checker_output(CheckerOutputBindingRequest) -ArtifactMaterializationPort.materialize_bindings(BindingMaterializationRequest) +PostSubmissionMaterializationPort.materialize(PostSubmissionEvaluationRequest, consumer) CheckerArtifactOutputPort.store(CheckerOutputArtifactRequest) ArtifactOperatorReadPort.list_bindings/list_replicas/list_receipts/ get_verification_job/get_recovery_attempt/list_audit_events/admission_usage(...) @@ -395,8 +395,14 @@ opaque prepared authority, the exact task/assignment context, the current change gate, packet, and exact policy/checker selectors. It is not a product capability port and may not cross a route, background execution, provider, or module public boundary. -`BindingMaterializationRequest` contains task/submission/checker-run context and -immutable binding IDs. `CheckerOutputArtifactRequest` contains the fixed +`PostSubmissionEvaluationRequest` selects one immutable Submission, its exact +binding/content, locked context and evaluation generation. ART resolves the consumed +admission and admitted verified replica through its own records plus the TASK-owned +`SubmittedBundlePort`. It verifies exact bytes and the canonical manifest before +an async consumer receives a scoped read-only file view. The typed result carries +material custody plus the existing closed evaluation value; it is not a durable +checker result. Production composition denies before protected reads until ARCH-04D. +`CheckerOutputArtifactRequest` contains the fixed service's prepared authority, task/submission/checker-run IDs, logical role, and generated byte source. The Submission binding is instead the terminal result of the typed `SubmissionAdmissionConsumptionPort`: it consumes one ready admission against @@ -965,23 +971,22 @@ set. The root marker binds a canonical fingerprint of those limits and startup fails closed on mismatch; changing limits therefore requires an empty, deliberately reprovisioned scratch root rather than an in-place mixed rollout. -The same canonical `ArtifactScratchManager` also owns checker-workspace -allocations. Authoritative pre-submit introduces one authorized artifact -materializer, and post-submit reuses it without a second workspace manager. -The materializer exposes two separate methods, not one caller-selected source -union: an internal pre-submit method accepts only the current process-local -`PreparedArtifact` submission-bundle workspace, and `materialize_bindings` -accepts immutable `ArtifactBinding` IDs after submission creation. No scratch -path or handle crosses an API/Celery boundary, and staging never creates a -premature product binding. The materializer reserves -the complete workspace against the same aggregate ledger and quotas, streams -exact provider bytes into private no-follow paths, and recomputes SHA-256 and -byte count for every file. Any mismatch becomes an artifact incident before -checker execution. After successful verification, files are sealed read-only -before the checker receives an opaque workspace handle. Success, failure, -cancellation, and crash/stale cleanup use the same API-startup and Celery Beat -ownership; a checker never receives provider references, credentials, or a -writable verified input. +The same canonical `ArtifactScratchManager` owns pre- and post-submit projection +workspaces. Pre-submit uses the current prepared contributor ZIP; post-submit uses +only the original selected by the immutable Submission's consumed admission. +Both share one inspected, sealed-tree projection implementation. Async post-submit +consumers receive only entry facts and bounded file reads, with no paths, provider +handles or credentials. The view is revoked before workspace and preparation +cleanup finishes, including failure and cancellation. + +ARCH-04B implements this hidden input path. It verifies complete original digest +and size, then rebuilds the semantic manifest, reserves expansion against the +existing aggregate scratch quota, and rechecks stored material selection after +I/O. Database transactions and row locks do not span provider/consumer execution. +Production authority remains deny-only; ARCH-04D supplies live authority and +ARCH-04C owns durable attempts/results and their final publication. Byte drift +rejects materialization; this read does not fabricate an incident or mutate +Submission lifecycle state. After ambiguous provider completion, the durable put-attempt resolver first calls read-only `observe_put_result` with the persisted commitment; it does not @@ -1544,9 +1549,10 @@ databases, and no artifact bytes in PostgreSQL. It is not downgradable. The pre-submit checker materializer is a mandatory part of preparation, so its fixed-service AUTH activation must merge after hidden ART-04B1-04B3 and before -contributor preparation is activated. Post-submit materialization and checker -output actions remain planned: ARCH-04B owns exact ART materialization, -ARCH-04C owns CHECKERS result custody, and ARCH-04D owns their exact activation. +contributor preparation is activated. ARCH-04B delivers hidden exact post-submit +input with deny-only production authority. ARCH-04B2 checker-output custody is +next; ARCH-04C owns durable CHECKERS execution/results, and ARCH-04D owns their +exact live activation. ARCH-04E separately owns TASK routing. Historical ART-06A/06B labels do not open duplicate implementation lanes. This ordering prevents a live contributor route whose mandatory checker read is unavailable. diff --git a/docs/spec_authorization_service.md b/docs/spec_authorization_service.md index 71102afef..d5d274dc7 100644 --- a/docs/spec_authorization_service.md +++ b/docs/spec_authorization_service.md @@ -537,8 +537,8 @@ runtime `ActionOwner` values or the current implementation boundaries. In particular, the XINT-06B grouping corresponds to runtime `WS-AUTH-001-ART-06A` for post-submit materialization and `WS-AUTH-001-ART-06B` for checker-output write/binding. The current replacement -activation contract is ARCH-04D, after ARCH-04B/04C hidden behavior; it does -not reopen XINT-06B as a parallel implementation lane. Likewise ARCH-02G/02H +activation contract is ARCH-04D, after delivered ARCH-04B hidden input and +remaining ARCH-04B2 output custody/ARCH-04C durable execution. It does not reopen XINT-06B as a parallel implementation lane. Likewise ARCH-02G/02H are replacement implementation boundaries, not automatic registry renames. Read exact runtime ownership from the typed catalogue. No planning-only change may promote or reassign an action.