diff --git a/apps/presentation/dashboard/smoke/conversation-reply-browser-smoke.mjs b/apps/presentation/dashboard/smoke/conversation-reply-browser-smoke.mjs index 784a95aa7c..67bf327169 100644 --- a/apps/presentation/dashboard/smoke/conversation-reply-browser-smoke.mjs +++ b/apps/presentation/dashboard/smoke/conversation-reply-browser-smoke.mjs @@ -80,11 +80,21 @@ async function checkReturnedFiles(ready, page, errors, writes) { } } await page.screenshot({path: resolve(output, `${baseline ? "before" : "after"}-files-desktop.png`), animations: "disabled"}); + const historyPath = `/api/chat/sessions/${ready.sessionId}`; + const readerRequest = page.context().waitForEvent("request", {predicate: request => + new URL(request.url()).pathname === historyPath + && new URL(request.frame().url()).searchParams.get("reportMessageId") === ready.fileMessageId}); const opened = page.context().waitForEvent("page"); await timeline.getByRole("link", {name: "单独阅读完整答复"}).click(); const report = await opened; report.on("pageerror", error => errors.push(error.message)); await report.getByRole("heading", {name: "结果已核验 / Verified result", exact: true}).waitFor(); + const readerQuery = new URL((await readerRequest).url()).searchParams; + if (!baseline) { + assert.equal(readerQuery.get("turn_id"), answer.turn_id, "The saved answer checks its original Turn"); + assert.equal(readerQuery.get("report_message_id"), answer.message_id, "The reader requests only the chosen answer"); + assert.equal(readerQuery.has("message_limit"), false); + } assert.equal(await report.locator("a[download]").count(), baseline ? 0 : ready.files.length); await report.screenshot({path: resolve(output, `${baseline ? "before" : "after"}-files-reader.png`), animations: "disabled"}); if (!baseline) { @@ -109,6 +119,91 @@ async function checkReturnedFiles(ready, page, errors, writes) { const url = ready.origin + answer.attachments[0].data_url; assert.equal((await page.request.get(url.replace(ready.sessionId, "unrelated-session"))).status(), 404); assert.equal((await page.request.get(url, {headers: {Origin: "https://unrelated.example"}})).status(), 403); + + const legacy = new URL(report.url()); + legacy.searchParams.delete("reportTurnId"); + const savedAnswerRequest = url => url.pathname === historyPath && url.searchParams.get("report_message_id") === ready.fileMessageId; + const legacyRead = report.waitForResponse(response => new URL(response.url()).pathname === historyPath); + await report.goto(legacy.toString()); + await report.getByRole("link", {name: "Download file: source.bin", exact: false}).waitFor(); + const legacyResponse = await legacyRead; + assert.ok(savedAnswerRequest(new URL(legacyResponse.url())), "An existing link selects its saved message without loading the transcript"); + const legacySnapshot = await legacyResponse.json(); + assert.deepEqual(legacySnapshot.messages.filter(row => row.role !== "user").map(row => row.message_id), [ready.fileMessageId]); + const legacyAnswer = legacySnapshot.messages.find(row => row.message_id === ready.fileMessageId); + assert.equal(legacyAnswer.text, answer.text); + assert.deepEqual(legacyAnswer.attachments, answer.attachments); + + await report.route(savedAnswerRequest, route => route.fulfill({status: 400, json: {ok: false, error: "invalid history page arguments"}})); + const fallbackRead = report.waitForRequest(request => { + const url = new URL(request.url()); + return url.pathname === historyPath && url.search === ""; + }); + await report.reload(); + await report.getByRole("link", {name: "Download file: source.bin", exact: false}).waitFor(); + await fallbackRead; + + const fallbackTurnRead = report.waitForRequest(request => { + const url = new URL(request.url()); + return url.pathname === historyPath && url.searchParams.get("turn_id") === answer.turn_id && url.searchParams.has("message_limit"); + }); + const withTurn = new URL(legacy); + withTurn.searchParams.set("reportTurnId", answer.turn_id); + await report.goto(withTurn.toString()); + await report.getByRole("link", {name: "Download file: source.bin", exact: false}).waitFor(); + await fallbackTurnRead; + await report.unroute(savedAnswerRequest); + await report.goto(legacy.toString()); + await report.getByRole("link", {name: "Download file: source.bin", exact: false}).waitFor(); + + // Earlier history readers accept unknown query arguments and return their + // complete v1 snapshot. Both existing and Turn-pinned links still open it. + await report.route(savedAnswerRequest, route => route.fulfill({json: snapshot})); + for (const address of [legacy, withTurn]) { + await report.goto(address.toString()); + await report.getByRole("link", {name: "Download file: source.bin", exact: false}).waitFor(); + assert.equal(await report.locator("a[download]").count(), ready.files.length); + } + const wrongTurn = new URL(withTurn); + wrongTurn.searchParams.set("reportTurnId", "unrelated-turn"); + await report.goto(wrongTurn.toString()); + await report.getByText("This answer was not found. It may have been removed, or the link may be incorrect.").waitFor(); + assert.equal(await report.locator("a[download]").count(), 0); + await report.unroute(savedAnswerRequest); + + for (const invalid of [ + {...snapshot, session: {...snapshot.session, session_id: "another-session"}}, + {...snapshot, report_message_id: "another-message"}, + {...snapshot, message_page: {next_cursor: "another-page"}}, + ]) { + await report.route(savedAnswerRequest, route => route.fulfill({json: invalid})); + await report.goto(legacy.toString()); + await report.getByText("Could not load this answer.").waitFor(); + assert.equal(await report.locator("a[download]").count(), 0); + await report.unroute(savedAnswerRequest); + } + + const refusedReads = []; + const observeRefusal = request => { + if (new URL(request.url()).pathname === historyPath) refusedReads.push(new URL(request.url())); + }; + report.on("request", observeRefusal); + for (const [status, error] of [[403, "Source access was revoked"], [400, "invalid Chat journal record"]]) { + refusedReads.length = 0; + await report.route(savedAnswerRequest, route => route.fulfill({status, json: {ok: false, error}})); + await report.reload(); + await report.getByText("Could not load this answer.").waitFor(); + assert.equal(await report.locator("a[download]").count(), 0); + assert.equal(refusedReads.length, 1); + assert.ok(savedAnswerRequest(refusedReads[0]), "A refused or invalid source must never trigger a wider history read"); + await report.unroute(savedAnswerRequest); + } + report.off("request", observeRefusal); + + legacy.searchParams.set("reportTurnId", "unrelated-turn"); + await report.goto(legacy.toString()); + await report.getByText(/This answer was not found|Could not load this answer/).waitFor(); + assert.equal(await report.locator("a[download]").count(), 0); } child.stdin.write("inspect\n"); const inspection = await next(); diff --git a/apps/presentation/dashboard/src/data/chat.ts b/apps/presentation/dashboard/src/data/chat.ts index ed9ebf8896..c0fdf5234c 100644 --- a/apps/presentation/dashboard/src/data/chat.ts +++ b/apps/presentation/dashboard/src/data/chat.ts @@ -972,6 +972,32 @@ export async function fetchChatSession(sessionId: string, signal?: AbortSignal) return requestJson(`/api/chat/sessions/${encodeURIComponent(sessionId)}`, { signal }); } +export async function fetchSavedChatAnswer(sessionId: string, messageId: string, signal: AbortSignal, turnId?: string) { + const query = new URLSearchParams({report_message_id: messageId}); + if (turnId !== undefined) query.set("turn_id", turnId); + const path = `/api/chat/sessions/${encodeURIComponent(sessionId)}?${query}`; + try { + const snapshot = await requestJson(path, {signal}); + // A legacy service can ignore the selector and return its complete v1 + // snapshot. A conflicting marker or a partial history remains invalid. + if (Object.hasOwn(snapshot, "report_message_id")) { + if (snapshot.report_message_id !== messageId) throw new Error("The saved answer selection changed during read."); + } else if ("message_page" in snapshot || "history_resume" in snapshot) { + throw new Error("The saved answer history is incomplete."); + } + return snapshot; + } catch (error) { + // Older Chat builds reject unknown history selectors before reading the + // Session. Only that response enables the earlier history read paths. + if (!(error instanceof ChatApiError) || error.payload.http_status !== 400 + || error.message !== "invalid history page arguments") throw error; + signal.throwIfAborted(); + return turnId === undefined + ? fetchChatSession(sessionId, signal) + : fetchChatSessionHistory(sessionId, signal, turnId); + } +} + export const chatQueueInputRequestSchema = z.discriminatedUnion("action", [ z.object({action: z.literal("edit"), operation_id: z.string().min(1).max(160), expected_revision: z.number().int().nonnegative(), message: z.string().min(1).max(24000)}).strict(), diff --git a/apps/presentation/dashboard/src/features/personal-workspace/answer-report-page.tsx b/apps/presentation/dashboard/src/features/personal-workspace/answer-report-page.tsx index 5f38dc2034..167c0a129c 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/answer-report-page.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/answer-report-page.tsx @@ -1,6 +1,6 @@ import {useEffect, useMemo, useRef, useState} from "react"; -import {fetchChatSession, type ChatSessionSnapshot, type ChatVisibleMessage} from "../../data/chat"; +import {fetchSavedChatAnswer, type ChatSessionSnapshot, type ChatVisibleMessage} from "../../data/chat"; import {visibleAgentMessage} from "./answer-text"; import {readWorkspaceLocale} from "./i18n"; import {resolveConversationScope} from "../../../../../../loopx/control_plane/collaboration/conversation_scope"; @@ -24,7 +24,8 @@ export function AnswerReportPage({sessionId, messageId, statusUrl}: { messageId: string; statusUrl: string; }) { - const scope = JSON.stringify([statusUrl || "/status.json", sessionId, messageId]); + const turnId = new URLSearchParams(window.location.search).get("reportTurnId"); + const scope = JSON.stringify([statusUrl || "/status.json", sessionId, messageId, turnId]); const [loaded, setLoaded] = useState<{scope: string; value: ReportState} | null>(null); // A late read from the preceding source never appears in the new reader. const state: ReportState = loaded?.scope === scope ? loaded.value : {kind: "loading"}; @@ -44,16 +45,18 @@ export function AnswerReportPage({sessionId, messageId, statusUrl}: { setLoaded({scope, value: {kind: "loading"}}); setCopied(false); setCopyFailed(false); - void fetchChatSession(sessionId, controller.signal).then((snapshot) => { + const read = fetchSavedChatAnswer(sessionId, messageId, controller.signal, turnId ?? undefined); + void read.then((snapshot) => { if (!live) return; if (snapshot.ok !== true || snapshot.schema_version !== "loopx_chat_store_v1" || snapshot.session.session_id !== sessionId) throw new Error("The saved Session changed during read."); const message = snapshot.messages.find((item) => item.message_id === messageId - && ["agent", "assistant"].includes(item.role)); + && ["agent", "assistant"].includes(item.role) + && (turnId === null || item.turn_id === turnId)); setLoaded({scope, value: message ? {kind: "ready", snapshot, message} : {kind: "missing", snapshot}}); }).catch(() => { if (live) setLoaded({scope, value: {kind: "error"}}); }); return () => {live = false; controller.abort(); copyAttempt.current += 1; document.title = previousTitle;}; - }, [scope, sessionId, messageId, zh, readRevision]); + }, [scope, sessionId, messageId, turnId, zh, readRevision]); const navigation = answerReportNavigation( (state.kind === "ready" || state.kind === "missing") ? state.snapshot.session : null, window.location.href, statusUrl, zh, @@ -180,6 +183,7 @@ export function answerReportNavigation( const managed = scope.kind === "owner_goal" && session?.managed_agent_id; url.searchParams.delete("reportSessionId"); url.searchParams.delete("reportMessageId"); + url.searchParams.delete("reportTurnId"); url.searchParams.delete("todo"); url.searchParams.delete("messageSession"); url.searchParams.delete("message"); diff --git a/apps/presentation/dashboard/src/features/personal-workspace/channel-timeline.tsx b/apps/presentation/dashboard/src/features/personal-workspace/channel-timeline.tsx index 43549a1332..2645c89c58 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/channel-timeline.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/channel-timeline.tsx @@ -23,10 +23,12 @@ import { conversationOrder } from "./conversation-order"; import { ConversationFind, conversationMessageKey } from "./conversation-find"; import type { WorkspaceActionPreview, WorkspaceDrawerSelection, WorkspaceGoal, WorkspaceTimelineItem } from "./personal-workspace-model"; -function answerLink(sessionId: string, messageId: string) { +function answerLink(sessionId: string, messageId: string, turnId?: string) { const url = new URL(window.location.href); url.searchParams.set("reportSessionId", sessionId); url.searchParams.set("reportMessageId", messageId); + if (turnId) url.searchParams.set("reportTurnId", turnId); + else url.searchParams.delete("reportTurnId"); url.searchParams.set("view", "conversation"); url.hash = ""; return url.toString(); @@ -288,7 +290,7 @@ export const ChannelTimeline = memo(function ChannelTimeline({ {item.message.role === "assistant" && !item.message.pending && item.message.text.length >= MIN_SEPARATE_ANSWER_LENGTH && item.message.sourceSessionId && item.message.sourceMessageId - ? {locale === "zh-CN" ? "单独阅读完整答复" : "Read full answer separately"} : null} {item.message.role !== "user" && (item.message.pending || item.message.sourceTurnId || item.message.activity?.length || item.message.steps?.length) ? : null} diff --git a/docs/reference/remote-authority-task-service-v0.md b/docs/reference/remote-authority-task-service-v0.md index 3e9ec583c2..aa4063fa51 100644 --- a/docs/reference/remote-authority-task-service-v0.md +++ b/docs/reference/remote-authority-task-service-v0.md @@ -56,6 +56,83 @@ Revocation prevents subsequent admissions. A request already admitted under a policy snapshot may finish and retains that snapshot's revision in its receipt; revocation does not claim to undo an in-flight commit. +### Scoped Todo reads + +For a read-only listener with resource authorization, set `schema_version` to +`loopx_remote_authority_scoped_read_policy_v1` in the existing service policy. +Each scope requires `todo_read_acl`, an array of entries with `goal_ref`, +`todo_id`, `principal_id` and `actor_id`. Each credential grant also requires +`goal_ref` with the exact `goal_id` and `goal_instance_id`. Grant `read` or +`snapshot` for the actions the caller needs. The operator must approve each +Todo and audience for that lifetime. Grant and ACL entries must agree with the +service scope, authenticated principal and actor. An empty ACL grants no access. + +For current Todo or snapshot reads, each entry also needs +`todo_origin_operation_id`: the canonical operation id of that Todo's successful +creation under this exact lifetime, or the trusted source owner's successful +first `bind_existing` transaction that adopted the existing Todo. The service +verifies that original receipt and projection through the existing PostgreSQL +checkpoint owner, bounded by the admitted current head. An ACL is permission, +not proof of ownership: a claim, update, successor activation, no-op rebind, +missing receipt or predecessor's receipt cannot move retained work into a new +lifetime. Entries without a valid origin are excluded; a direct/targeted read +without proof, or a snapshot with no proven resources, is rejected. Shared +first-bind proofs are read once per request; no authorization is cached between +requests. The origin operation id stays in the private policy and is not +returned to callers. Existing bound work without such evidence stays unavailable +until the source owner supplies valid historical proof; do not rebind or rewrite +its history to manufacture one. + +Launch the service with the scoped policy and use the current TLS client with +the existing v0 request JSON. The server supplies the exact Goal reference +from its private policy and verifies it against the active lifecycle on the +same PostgreSQL head used to compose the response. Unbound, retiring and +mismatched lifetimes deny access. A direct read or targeted snapshot outside +the ACL returns rejection without revealing whether the Todo exists. The +scoped listener denies write commands. + +Scoped successes use `loopx_remote_authority_scoped_read_response_v1`. They +return the exact Goal reference, authorized Todo status and assignment, and +matching lease metadata. Snapshots also include role and priority. Free-form +notes and lease keys are withheld. Scoped clients reject unknown nested Todo +and lease fields even when a snapshot digest matches; the trusted v0 decoder +keeps its existing compatibility behavior. The snapshot receipt digest covers only the +authorized projection and its scope. The global provider revision is withheld, +so changing a hidden Todo leaves the returned view and digest intact. Clients +that lack support for this response version report an unavailable read. + +Revoke access by replacing the policy with a new version of the same schema +that removes the matching ACL entry or grant. The service pins the policy +schema at startup and fails closed on a schema change during reload. Stop the +listener to disable it. Keep the v0 policy for an explicitly configured +legacy listener with trusted access to the whole Goal. + +#### Historical receipt proof + +To authorize recovery of a specific remote operation, add its canonical +`operation_id` to `receipt_operation_ids` on the Todo's read ACL entry. The +credential also needs `read` in its grant. The ID comes from the original +operation or its trusted service receipt. An omitted or empty list grants +no receipt access. Use `--recover` with the complete original command. + +The service checks the active Goal and allowed Todo before it opens the +receipt. The original request digest, principal and actor must match the +receipt. The original committed projection must prove the same Goal lifetime. +The service reuses the provider's bounded checkpoint and delta reader for that +exact cursor. Missing, unbound or cross-lifetime receipts are rejected, even +with a matching operation ID. The scoped service role needs read access to the +existing tenant-scoped `authority_checkpoints` table as part of this journal read. + +A scoped success returns the exact Goal, Todo and operation, `historical_only` +and the original receipt digest. It withholds the original result payload, +lease key, cursor and provider revision. The original receipt confirms +recorded history. Execution authority still comes from a current lease and +the existing mutation owner. Remove the operation ID to revoke recovery while +leaving the authorized Todo read available. + +Artifact versions, delegated sources and cross-person writes require separate +qualification before they can be exposed. + ## Commands and recovery POST JSON to `/v0/tasks/execute` with a `Bearer` credential. Every command includes diff --git a/examples/coordination/remote-authority-service.ts b/examples/coordination/remote-authority-service.ts index 45a98cab92..f4b9507dee 100644 --- a/examples/coordination/remote-authority-service.ts +++ b/examples/coordination/remote-authority-service.ts @@ -12,12 +12,12 @@ const port = Number(values.port); if (!values.policy || !values.cert || !values.key || !Number.isSafeInteger(port) || port < 0 || port > 65535 || !process.env.LOOPX_AUTHORITY_DATABASE_URL) throw new Error("Provide policy, TLS certificate/key and service-owned database configuration"); // Validate before listening; updates are read again for each admission. -await readRemoteAuthorityPolicy(values.policy); +const policy = await readRemoteAuthorityPolicy(values.policy); const pool = new Pool({connectionString: process.env.LOOPX_AUTHORITY_DATABASE_URL, max: 8, connectionTimeoutMillis: 2000, query_timeout: 6000, idleTimeoutMillis: 10000, statement_timeout: 5000, lock_timeout: 2000, idle_in_transaction_session_timeout: 10000}); pool.on("error", () => process.stderr.write("Authority database connection unavailable\n")); -const service = new RemoteAuthorityService({admit: fileRemoteAuthorityAdmission(values.policy), +const service = new RemoteAuthorityService({admit: fileRemoteAuthorityAdmission(values.policy, String(policy.schema_version)), database: {connect: async () => { const connection = await pool.connect(); return {query: (sql, args) => connection.query(sql, args ? [...args] : undefined), diff --git a/loopx/capabilities/manager_context/roundtrip.py b/loopx/capabilities/manager_context/roundtrip.py index 3ce56200f9..8359b41340 100644 --- a/loopx/capabilities/manager_context/roundtrip.py +++ b/loopx/capabilities/manager_context/roundtrip.py @@ -427,24 +427,46 @@ def observe_result(result_path, reply): ] +def _saved_answer_rows(messages, message_id, expected_turn_id): + matches = [row for row in messages if row.get("message_id") == message_id] + if len(matches) > 1: + raise ValueError("saved answer has conflicting message identities") + if not matches or matches[0].get("role") not in {"agent", "assistant"}: + return [], [] + answer = matches[0] + turn_id = answer.get("turn_id") + if expected_turn_id is not None and turn_id != expected_turn_id: + return [], [] + if not turn_id: + return [answer], [answer] + original = [row for row in messages if row.get("turn_id") == turn_id] + question = next((row for row in original if row.get("role") == "user"), None) + return ([question, answer] if question else [answer]), original + + def project_chat_session_snapshot(root, store, session_id, *, registry, - message_limit=None, message_cursor=None, turn_id=None, + message_limit=None, message_cursor=None, turn_id=None, report_message_id=None, resume_from=None, project_history_messages=None, include_turn_error=False): """Project return delivery state into one existing Chat snapshot.""" turn = None - if turn_id is not None: + if turn_id is not None and report_message_id is None: if message_limit is None: raise ValueError("a selected original Turn requires paged history") turn = store.load_turn(session_id, turn_id) if not turn or turn.get("session_id") != session_id or turn.get("turn_id") != turn_id: raise ValueError("original Turn is unavailable from this Session") - strict = message_limit is not None or resume_from is not None + strict = message_limit is not None or resume_from is not None or report_message_id is not None # Read the status first: terminal completion persists its answer before # publishing completed. An older message prefix must not appear final. snapshot = (store.session_snapshot(session_id, strict_messages=True) if strict else store.session_snapshot(session_id)) - if turn is not None: + report_context = None + if report_message_id is not None: + snapshot["messages"], report_context = _saved_answer_rows(snapshot["messages"], report_message_id, turn_id) + snapshot["report_message_id"] = report_message_id + snapshot["active_turn"] = None + elif turn is not None: status = turn["status"] snapshot["selected_turn"] = {"turn_id": turn_id, "status": status, "terminal": status in TERMINAL_TURN_STATES} # Failure records persist the reason with the status, before appending the message. @@ -464,7 +486,7 @@ def project_chat_session_snapshot(root, store, session_id, *, registry, from .chat_history import page_chat_messages page = (page_chat_messages(session_id, snapshot["messages"], limit=message_limit, cursor=message_cursor, turn_id=turn_id) if message_limit is not None else None) - context_messages = snapshot["messages"] + context_messages = snapshot["messages"] if report_context is None else report_context if page is not None: context_messages = context_messages[:page["message_page"]["total"]] snapshot["messages"] = page["messages"] @@ -475,8 +497,14 @@ def project_chat_session_snapshot(root, store, session_id, *, registry, # Use the frozen context to choose the carrier, projecting only this page. # A question and answer can straddle pages without duplicating a commitment # or re-reading every off-page Turn for each page. - snapshot["messages"] = project_collaboration(store, root, session_id, snapshot["messages"], + projected = project_collaboration(store, root, session_id, snapshot["messages"], registry=registry, context_messages=context_messages) + if report_message_id is not None: + # Collaboration may include the original receipt carrier. The saved + # answer only exposes its precise answer and the question it displays. + visible_ids = {row.get("message_id") for row in snapshot["messages"]} + projected = [row for row in projected if row.get("message_id") in visible_ids] + snapshot["messages"] = projected # Provider-owned history joins the selected page at the composition root. # Core capabilities never import an extension projection sink. if page is not None: @@ -487,9 +515,13 @@ def project_chat_session_snapshot(root, store, session_id, *, registry, def parse_chat_history_selection(query): """Validate one HTTP history selection before any recovery observation.""" - if (set(query) - {"message_limit", "message_cursor", "turn_id", "known_history", "task_goal_id", "task_todo_id"} + keys = set(query) + report = "report_message_id" in keys + if (keys - {"message_limit", "message_cursor", "turn_id", "known_history", + "task_goal_id", "task_todo_id", "report_message_id"} or any(len(values) != 1 or not values[0] for values in query.values()) - or ("message_limit" not in query and set(query) & {"message_cursor", "turn_id", "known_history"})): + or (report and keys - {"report_message_id", "turn_id"}) + or (not report and "message_limit" not in keys and keys & {"message_cursor", "turn_id", "known_history"})): raise ValueError("invalid history page arguments") limit = int(query["message_limit"][0]) if "message_limit" in query else None if limit is not None and not 1 <= limit <= 200: @@ -512,7 +544,8 @@ def parse_chat_history_selection(query): raise ValueError("invalid known history proof") parsed_basis = (basis, count, proof) return {"message_limit": limit, "message_cursor": query.get("message_cursor", [None])[0], - "turn_id": query.get("turn_id", [None])[0], "known_history": parsed_basis, "task_scope": task_scope} + "turn_id": query.get("turn_id", [None])[0], "known_history": parsed_basis, "task_scope": task_scope, + "report_message_id": query.get("report_message_id", [None])[0]} def prepare_chat_history_session(store, session_id, selection, *, recover_turn): @@ -520,7 +553,7 @@ def prepare_chat_history_session(store, session_id, selection, *, recover_turn): task_scope = selection["task_scope"] turn_id = selection["turn_id"] if task_scope is None: - if turn_id is None: + if turn_id is None and selection["report_message_id"] is None: recover_turn(session_id) return store.load_session(session_id) or {} @@ -553,6 +586,7 @@ def read_chat_history_selection(root, store, session_id, *, registry, selection, snapshot = project_chat_session_snapshot(root, store, session_id, registry=registry, message_limit=selection["message_limit"], message_cursor=selection["message_cursor"], turn_id=selection["turn_id"], project_history_messages=project_history_messages, + report_message_id=selection["report_message_id"], include_turn_error=selection["task_scope"] is not None) snapshot["messages"] = project_provider(snapshot["messages"]) if selection["message_limit"] == 200 and selection["turn_id"] is None: diff --git a/loopx/chat_server.py b/loopx/chat_server.py index ab9a4425ea..65337c4b63 100644 --- a/loopx/chat_server.py +++ b/loopx/chat_server.py @@ -936,7 +936,8 @@ def _session_snapshot(self, session_id: str) -> None: media = self.server.private_media_history contexts = getattr(self.server.runtime_controller, "project_contexts", None) bindings = getattr(contexts, "conversation_bindings", None) - markers = [] if selection["turn_id"] is not None else media.read(store, bindings, session) + scoped = selection["turn_id"] is not None or selection["report_message_id"] is not None + markers = [] if scoped else media.read(store, bindings, session) payload = read_chat_history_selection( self.server.runtime_controller.coordination_runtime_root, store, session_id, registry=self.server.registry_path, selection=selection, diff --git a/loopx/control_plane/coordination/remote_authority_contract.ts b/loopx/control_plane/coordination/remote_authority_contract.ts index d37ef5f7bb..131b35fb73 100644 --- a/loopx/control_plane/coordination/remote_authority_contract.ts +++ b/loopx/control_plane/coordination/remote_authority_contract.ts @@ -1,7 +1,7 @@ /** Candidate wire contract. Clients submit intentions, never authority facts. */ import type {JsonObject} from "../effect_program.ts"; import {canonicalAuthorityObject, canonicalAuthoritySha256, - requireAuthorityStoreId} from "./authority_store_codec.ts"; + hasExactAuthorityKeys, requireAuthorityStoreId} from "./authority_store_codec.ts"; import {POSTGRESQL_STORE_IDENTITY_PATTERN} from "./postgresql_authority_store.ts"; import {taskLeaseOperationIdentity} from "../work_items/task_lease_operation_identity.ts"; import {normalizeTodoId, normalizeGoalId, normalizeOwner} from "../work_items/task_lease_acquire.ts"; @@ -9,9 +9,11 @@ import {BARE_SHA256_PATTERN} from "../content_digest.ts"; import {parseIsoTimestamp} from "../runtime_timestamp.ts"; import {normalizeTodoResumeWhen, TODO_RESUME_NORMALIZE_REQUEST_SCHEMA_VERSION} from "../todos/resume_condition.ts"; import {normalizeTodoPriority} from "../todos/priority.ts"; +import {parseWireExactGoalRef, type WireExactGoalRef} from "../goals/acceptance_lifecycle.ts"; export const REMOTE_AUTHORITY_REQUEST = "loopx_remote_authority_request_v0"; export const REMOTE_AUTHORITY_RESPONSE = "loopx_remote_authority_response_v0"; +export const REMOTE_AUTHORITY_SCOPED_READ_RESPONSE = "loopx_remote_authority_scoped_read_response_v1"; export const REMOTE_AUTHORITY_OPERATIONS = [ "read", "claim", "progress", "renew", "release", "complete", "block", "defer", "todo_add", "snapshot", @@ -173,16 +175,18 @@ export function remoteAuthorityOperationId(request: RemoteAuthorityRequest, prin request_id: requestId ?? ("request_id" in request ? request.request_id : "")})}`; } export function remoteAuthorityResponse(request: unknown, mode: RemoteAuthorityMode, - payload: JsonObject): JsonObject { - return {...payload, schema_version: REMOTE_AUTHORITY_RESPONSE, + payload: JsonObject, schema: typeof REMOTE_AUTHORITY_RESPONSE | typeof REMOTE_AUTHORITY_SCOPED_READ_RESPONSE = REMOTE_AUTHORITY_RESPONSE): JsonObject { + return {...payload, schema_version: schema, request_sha256: canonicalAuthoritySha256({mode, request})}; } export function remoteAuthoritySnapshotDigest(request: RemoteAuthorityScope, revision: unknown, - todos: readonly unknown[], leases: readonly unknown[]): string { - return canonicalAuthoritySha256({schema_version: "loopx_goal_snapshot_receipt_v0", - tenant_id: request.tenant_id, goal_id: request.goal_id, store_identity: request.store_identity, - provider_revision: revision, todos, leases}); + todos: readonly unknown[], leases: readonly unknown[], goalRef?: WireExactGoalRef): string { + const evidence = goalRef + ? {schema_version: REMOTE_AUTHORITY_SCOPED_READ_RESPONSE, goal_ref: goalRef} + : {schema_version: "loopx_goal_snapshot_receipt_v0", provider_revision: revision}; + return canonicalAuthoritySha256({...evidence, tenant_id: request.tenant_id, + goal_id: request.goal_id, store_identity: request.store_identity, todos, leases}); } function responseRequires(condition: unknown): asserts condition { @@ -199,8 +203,14 @@ function responseOperationId(value: unknown, request: RemoteAuthorityRequest): s : id.startsWith("remote-task:") && BARE_SHA256_PATTERN.test(id.slice("remote-task:".length))); return id; } -function responseLease(raw: unknown, request: RemoteAuthorityRequest, mutation: boolean): JsonObject { +function responseFields(value: JsonObject, allowed: readonly string[]): void { + responseRequires(Object.keys(value).every(key => allowed.includes(key))); +} +function responseLease(raw: unknown, request: RemoteAuthorityRequest, mutation: boolean, + scoped: boolean = false, snapshot: boolean = false): JsonObject { const lease = canonicalAuthorityObject(raw, "response lease"); + if (scoped) responseFields(lease, ["owner", "status", "expires_at", "version", "lease_epoch", + ...(snapshot ? ["goal_id", "todo_id"] : [])]); responseRequires(typeof lease.owner === "string" && normalizeOwner(lease.owner) === lease.owner && (lease.status === "active" || lease.status === "released") && Number.isSafeInteger(lease.version) && Number(lease.version) >= 0 && @@ -220,8 +230,9 @@ function responseLease(raw: unknown, request: RemoteAuthorityRequest, mutation: return lease; } -function responseTodo(raw: unknown, todoId?: string): string { +function responseTodo(raw: unknown, todoId?: string, scoped: boolean = false, snapshot: boolean = false): string { const todo = canonicalAuthorityObject(raw, "response Todo"); + if (scoped) responseFields(todo, ["todo_id", "status", "claimed_by", ...(snapshot ? ["role", "priority"] : [])]); const id = normalizeTodoId(todo.todo_id); responseRequires(todo.todo_id === id && (todoId === undefined || id === todoId)); remoteAuthorityId(todo.status, "Todo status"); @@ -231,22 +242,23 @@ function responseTodo(raw: unknown, todoId?: string): string { return id; } -function responseSnapshot(value: JsonObject, request: Extract): void { +function responseSnapshot(value: JsonObject, request: Extract, + goalRef?: WireExactGoalRef): void { responseRequires(value.status === "snapshot" && value.goal_id === request.goal_id && value.store_identity === request.store_identity && Array.isArray(value.todos) && Array.isArray(value.leases)); - responseRevision(value.provider_revision, request); - const todoIds = value.todos.map(todo => responseTodo(todo, request.todo_id)); + if (!goalRef) responseRevision(value.provider_revision, request); + const todoIds = value.todos.map(todo => responseTodo(todo, request.todo_id, goalRef !== undefined, true)); const todoSet = new Set(todoIds); responseRequires(todoSet.size === todoIds.length); const leaseIds = new Set(); for (const raw of value.leases) { - const lease = responseLease(raw, request, false); + const lease = responseLease(raw, request, false, goalRef !== undefined, true); const id = normalizeTodoId(lease.todo_id); responseRequires(lease.todo_id === id && lease.goal_id === request.goal_id && todoSet.has(id) && !leaseIds.has(id) && !Object.hasOwn(lease, "idempotency_key")); leaseIds.add(id); } - responseRequires(value.receipt_sha256 === remoteAuthoritySnapshotDigest(request, value.provider_revision, value.todos, value.leases)); + responseRequires(value.receipt_sha256 === remoteAuthoritySnapshotDigest(request, value.provider_revision, value.todos, value.leases, goalRef)); } function responseCommand(raw: unknown, request: RemoteAuthorityRequest, operationId: string, historical: boolean): void { @@ -301,9 +313,28 @@ function responseCommand(raw: unknown, request: RemoteAuthorityRequest, operatio export function decodeRemoteAuthorityResponse(raw: unknown, request: RemoteAuthorityRequest, mode: RemoteAuthorityMode): JsonObject { const value = canonicalAuthorityObject(raw, "remote authority response"); - responseRequires(value.schema_version === REMOTE_AUTHORITY_RESPONSE && + const scoped = value.schema_version === REMOTE_AUTHORITY_SCOPED_READ_RESPONSE; + responseRequires((value.schema_version === REMOTE_AUTHORITY_RESPONSE || scoped) && value.request_sha256 === canonicalAuthoritySha256({mode, request})); responseRequires(typeof value.status === "string"); + let goalRef: WireExactGoalRef | undefined; + if (scoped) { + goalRef = parseWireExactGoalRef(value.goal_ref, "scoped read Goal"); + responseRequires(goalRef.goal_id === request.goal_id); + if (mode === "recover") { + responseRequires(request.operation !== "read" && request.operation !== "snapshot" && + value.status === "found" && value.historical_only === true && value.todo_id === request.todo_id && + typeof value.receipt_sha256 === "string" && BARE_SHA256_PATTERN.test(value.receipt_sha256)); + responseRequires(hasExactAuthorityKeys(value, ["schema_version", "request_sha256", "status", "goal_ref", + "historical_only", "todo_id", "operation_id", "receipt_sha256"])); + responseOperationId(value.operation_id, request); + return value; + } + responseRequires(mode === "execute" && (request.operation === "read" && value.status === "found" + || request.operation === "snapshot" && value.status === "snapshot")); + const fields = value.status === "found" ? ["todo", "lease"] : ["goal_id", "store_identity", "receipt_sha256", "todos", "leases"]; + responseRequires(hasExactAuthorityKeys(value, ["schema_version", "request_sha256", "status", "goal_ref", ...fields])); + } if (["rejected", "unavailable", "ambiguous"].includes(value.status)) { remoteAuthorityId(value.reason_code, "response reason"); responseRequires(value.status !== "ambiguous" || mode === "execute" && request.operation !== "read" && request.operation !== "snapshot"); @@ -319,12 +350,12 @@ export function decodeRemoteAuthorityResponse(raw: unknown, request: RemoteAutho } else if (mode === "execute" && request.operation === "read") { if (value.status !== "missing") { responseRequires(value.status === "found"); - responseRevision(value.provider_revision, request); - responseTodo(value.todo, request.todo_id); - if (value.lease !== null) responseLease(value.lease, request, false); + if (!goalRef) responseRevision(value.provider_revision, request); + responseTodo(value.todo, request.todo_id, scoped); + if (value.lease !== null) responseLease(value.lease, request, false, scoped); } } else if (mode === "execute" && request.operation === "snapshot") { - responseSnapshot(value, request); + responseSnapshot(value, request, goalRef); } else { responseRequires(mode === "execute" && request.operation !== "read" && request.operation !== "snapshot" && value.status === "result"); responseCommand(value.result, request, responseOperationId(value.operation_id, request), false); diff --git a/loopx/control_plane/coordination/remote_authority_policy.ts b/loopx/control_plane/coordination/remote_authority_policy.ts index 712d22a2e8..79b09459e6 100644 --- a/loopx/control_plane/coordination/remote_authority_policy.ts +++ b/loopx/control_plane/coordination/remote_authority_policy.ts @@ -4,30 +4,63 @@ import {createHash, timingSafeEqual} from "node:crypto"; import {open} from "node:fs/promises"; import type {JsonObject} from "../effect_program.ts"; import {BARE_SHA256_PATTERN} from "../content_digest.ts"; +import {parseWireExactGoalRef, sameExactGoalRef, type WireExactGoalRef} from "../goals/acceptance_lifecycle.ts"; +import {normalizeTodoId} from "../work_items/task_lease_acquire.ts"; import {canonicalAuthorityObject, canonicalAuthoritySha256, hasExactAuthorityKeys} from "./authority_store_codec.ts"; import {POSTGRESQL_STORE_IDENTITY_PATTERN} from "./postgresql_authority_store.ts"; -import {REMOTE_AUTHORITY_OPERATIONS, remoteAuthorityId, type RemoteAuthorityRequest} from "./remote_authority_contract.ts"; +import {REMOTE_AUTHORITY_OPERATIONS, remoteAuthorityId, remoteAuthorityOperationId, + type RemoteAuthorityMode, type RemoteAuthorityRequest} from "./remote_authority_contract.ts"; import type {RemoteAuthorityAdmission} from "./remote_authority_service.ts"; export const REMOTE_AUTHORITY_POLICY = "loopx_remote_authority_policy_v0"; +export const REMOTE_AUTHORITY_SCOPED_READ_POLICY = "loopx_remote_authority_scoped_read_policy_v1"; const sameScope = (a: JsonObject, b: {tenant_id: string; goal_id: string; store_identity: string}) => a.tenant_id === b.tenant_id && a.goal_id === b.goal_id && a.store_identity === b.store_identity; -function object(value: unknown, keys: string[], name: string): JsonObject { +function object(value: unknown, keys: string[], name: string, optionalKeys: string[] = []): JsonObject { const result = canonicalAuthorityObject(value, name); - if (!hasExactAuthorityKeys(result, keys)) throw new TypeError(`Invalid ${name}`); + if (!hasExactAuthorityKeys(result, [...keys, ...optionalKeys.filter(key => Object.hasOwn(result, key))])) { + throw new TypeError(`Invalid ${name}`); + } return result; } export function decodeRemoteAuthorityPolicy(value: unknown): JsonObject { const policy = object(value, ["schema_version", "scopes", "principals"], "service policy"); - if (policy.schema_version !== REMOTE_AUTHORITY_POLICY || !Array.isArray(policy.scopes) || + const scoped = policy.schema_version === REMOTE_AUTHORITY_SCOPED_READ_POLICY; + if (!(scoped || policy.schema_version === REMOTE_AUTHORITY_POLICY) || !Array.isArray(policy.scopes) || !Array.isArray(policy.principals) || policy.scopes.length > 128 || policy.principals.length > 256) throw new TypeError("Invalid service policy"); const scopes = policy.scopes.map(raw => { - const scope = object(raw, ["tenant_id", "goal_id", "store_identity", "registered_agents"], "service scope"); + const scope = object(raw, ["tenant_id", "goal_id", "store_identity", "registered_agents", + ...(scoped ? ["todo_read_acl"] : [])], "service scope"); remoteAuthorityId(scope.tenant_id, "tenant"); remoteAuthorityId(scope.goal_id, "Goal"); if (typeof scope.store_identity !== "string" || !POSTGRESQL_STORE_IDENTITY_PATTERN.test(scope.store_identity) || !Array.isArray(scope.registered_agents) || scope.registered_agents.length === 0 || scope.registered_agents.length > 128) throw new TypeError("Invalid service scope"); scope.registered_agents.forEach(a => remoteAuthorityId(a, "registered actor")); if (new Set(scope.registered_agents).size !== scope.registered_agents.length) throw new TypeError("Duplicate registered actor"); + if (scoped) { + if (!Array.isArray(scope.todo_read_acl)) throw new TypeError("Invalid Todo read ACL"); + const actors = new Set(scope.registered_agents), entries = new Set(); + for (const rawEntry of scope.todo_read_acl) { + const entry = object(rawEntry, ["goal_ref", "todo_id", "principal_id", "actor_id"], + "Todo read ACL entry", ["receipt_operation_ids", "todo_origin_operation_id"]); + const goal = parseWireExactGoalRef(entry.goal_ref, "Todo read ACL Goal"); + remoteAuthorityId(entry.principal_id, "ACL principal"); + remoteAuthorityId(entry.todo_id, "ACL Todo"); + if (goal.goal_id !== scope.goal_id || !actors.has(entry.actor_id) || normalizeTodoId(entry.todo_id) !== entry.todo_id) { + throw new TypeError("Invalid Todo read ACL scope"); + } + const key = JSON.stringify([goal.goal_instance_id, entry.todo_id, entry.principal_id, entry.actor_id]); + if (entries.has(key)) throw new TypeError("Duplicate Todo read ACL entry"); + entries.add(key); + if (Object.hasOwn(entry, "todo_origin_operation_id")) { + remoteAuthorityId(entry.todo_origin_operation_id, "Todo origin operation"); + } + if (Object.hasOwn(entry, "receipt_operation_ids")) { + const ids = entry.receipt_operation_ids; + if (!Array.isArray(ids) || new Set(ids).size !== ids.length) throw new TypeError("Invalid receipt read ACL"); + ids.forEach(id => remoteAuthorityId(id, "receipt ACL operation")); + } + } + } return scope; }); const scopeKeys = scopes.map(s => JSON.stringify([s.tenant_id, s.goal_id])); @@ -39,12 +72,15 @@ export function decodeRemoteAuthorityPolicy(value: unknown): JsonObject { !Array.isArray(principal.grants) || principal.grants.length > 128) throw new TypeError("Invalid principal"); const seen = new Set(); principal.grants.forEach(rawGrant => { - const grant = object(rawGrant, ["tenant_id", "goal_id", "actor_id", "operations"], "grant"); + const grant = object(rawGrant, ["tenant_id", "goal_id", "actor_id", "operations", + ...(scoped ? ["goal_ref"] : [])], "grant"); const scope = scopes.find(s => s.tenant_id === grant.tenant_id && s.goal_id === grant.goal_id); if (!scope || !(scope.registered_agents as string[]).includes(String(grant.actor_id)) || !Array.isArray(grant.operations) || grant.operations.length === 0 || grant.operations.some(op => !REMOTE_AUTHORITY_OPERATIONS.includes(op as typeof REMOTE_AUTHORITY_OPERATIONS[number])) || new Set(grant.operations).size !== grant.operations.length) throw new TypeError("Invalid grant"); + if (scoped && (parseWireExactGoalRef(grant.goal_ref, "read grant Goal").goal_id !== grant.goal_id || + grant.operations.some(op => op !== "read" && op !== "snapshot"))) throw new TypeError("Invalid scoped read grant"); const key = JSON.stringify([grant.tenant_id, grant.goal_id, grant.actor_id]); if (seen.has(key)) throw new TypeError("Duplicate grant"); seen.add(key); @@ -58,24 +94,50 @@ export function decodeRemoteAuthorityPolicy(value: unknown): JsonObject { } export function remoteAuthorityPolicyAdmission(policy: JsonObject, credential: string, - request: RemoteAuthorityRequest): RemoteAuthorityAdmission { + request: RemoteAuthorityRequest, mode: RemoteAuthorityMode = "execute"): RemoteAuthorityAdmission { const digest = createHash("sha256").update(credential).digest(); const principal = (policy.principals as JsonObject[]).find(p => timingSafeEqual(Buffer.from(String(p.credential_sha256), "hex"), digest)); if (!principal) return {status: "rejected", reason_code: "principal_unauthenticated"}; const scope = (policy.scopes as JsonObject[]).find(s => sameScope(s, request)); + const scoped = policy.schema_version === REMOTE_AUTHORITY_SCOPED_READ_POLICY; + const recoverScoped = scoped && mode === "recover"; + const operation = recoverScoped ? "read" : request.operation; const grant = (principal.grants as JsonObject[]).find(g => g.tenant_id === request.tenant_id && - g.goal_id === request.goal_id && g.actor_id === request.actor_id && (g.operations as string[]).includes(request.operation)); + g.goal_id === request.goal_id && g.actor_id === request.actor_id && (g.operations as string[]).includes(operation)); if (!scope || !grant) return {status: "rejected", reason_code: "remote_authority_scope_denied"}; - return {status: "allowed", principal_id: String(principal.principal_id), + const allowed = {status: "allowed" as const, principal_id: String(principal.principal_id), policy_revision: canonicalAuthoritySha256(policy), registered_agents: [...scope.registered_agents as string[]]}; + if (!scoped) return allowed; + + const goalRef = grant.goal_ref as WireExactGoalRef; + const entries = (scope.todo_read_acl as JsonObject[]) + .filter(entry => entry.principal_id === principal.principal_id && entry.actor_id === request.actor_id && + sameExactGoalRef(entry.goal_ref as WireExactGoalRef, goalRef)); + const todoIds = entries.map(entry => String(entry.todo_id)); + if (todoIds.length === 0 || request.todo_id !== undefined && !todoIds.includes(request.todo_id)) { + return {status: "rejected", reason_code: "remote_authority_scope_denied"}; + } + const readScope = {goal_ref: goalRef, todo_ids: todoIds, + todo_origins: Object.fromEntries(entries.filter(entry => entry.todo_origin_operation_id !== undefined) + .map(entry => [String(entry.todo_id), String(entry.todo_origin_operation_id)]))}; + if (!recoverScoped) return {...allowed, read_scope: readScope}; + const id = remoteAuthorityOperationId(request, allowed.principal_id); + if (!entries.some(entry => entry.todo_id === request.todo_id && + (entry.receipt_operation_ids as string[] | undefined)?.includes(id))) { + return {status: "rejected", reason_code: "remote_authority_scope_denied"}; + } + return {...allowed, read_scope: {...readScope, receipt_operation_id: id}}; } -export function fileRemoteAuthorityAdmission(path: string) { - return async (credential: string, request: RemoteAuthorityRequest): Promise => { +export function fileRemoteAuthorityAdmission(path: string, expectedSchema: string = REMOTE_AUTHORITY_POLICY) { + return async (credential: string, request: RemoteAuthorityRequest, + mode: RemoteAuthorityMode = "execute"): Promise => { // A rename publishes one complete policy. Missing, malformed or unreadable // policy fails closed; never retain a stale last-known-good grant. - return remoteAuthorityPolicyAdmission(await readRemoteAuthorityPolicy(path), credential, request); + const policy = await readRemoteAuthorityPolicy(path); + if (policy.schema_version !== expectedSchema) throw new TypeError("Service policy schema changed; restart the listener"); + return remoteAuthorityPolicyAdmission(policy, credential, request, mode); }; } diff --git a/loopx/control_plane/coordination/remote_authority_service.ts b/loopx/control_plane/coordination/remote_authority_service.ts index 75f045e7e0..eff348b4d3 100644 --- a/loopx/control_plane/coordination/remote_authority_service.ts +++ b/loopx/control_plane/coordination/remote_authority_service.ts @@ -1,32 +1,36 @@ /** Authenticated task commands over the existing semantic and persistence owners. */ import type {JsonObject} from "../effect_program.ts"; import type {AuthorityStore, AuthorityStoreReadFailure} from "./authority_store.ts"; -import {canonicalAuthoritySha256} from "./authority_store_codec.ts"; +import {canonicalAuthoritySha256, parseAuthorityCursor} from "./authority_store_codec.ts"; import {PostgreSqlAuthorityService} from "./postgresql_authority_service.ts"; import type {PostgreSqlAuthorityDatabase} from "./postgresql_authority_store.ts"; import {executeCoordinationTodoClaim} from "./todo_claim.ts"; import {executeCoordinationTodoUpdate} from "./todo_update.ts"; -import {executeCoordinationTodoCreate} from "./todo_create.ts"; +import {executeCoordinationTodoCreate, COORDINATION_TODO_CREATE_RECEIPT_SCHEMA} from "./todo_create.ts"; import {executeCoordinationTodoTerminalLifecycle, type CoordinationTodoTerminalLifecycleInput} from "./todo_terminal_lifecycle.ts"; import {executeCanonicalTaskLeaseLifecycle} from "./task_lease_lifecycle.ts"; import {leaseVersion, leaseEpoch} from "../work_items/task_lease_acquire.ts"; import {TODO_DOMAIN_ITEM_SCHEMA} from "./coordination_state_contract.ts"; -import {indexCoordinationProjection, validateCoordinationTodoReadModel} from "./coordination_projection.ts"; -import {decodeRemoteAuthorityRequest, remoteAuthorityOperationId, remoteAuthorityResponse, remoteAuthoritySnapshotDigest, +import {CoordinationProjectionRead} from "./coordination_projection.ts"; +import {GOAL_ACCEPTANCE_LIFECYCLE_OPERATION_SCHEMA, readGoalAcceptanceLifecycle, sameExactGoalRef, type WireExactGoalRef} from "../goals/acceptance_lifecycle.ts"; +import {decodeRemoteAuthorityRequest, remoteAuthorityOperationId, remoteAuthorityResponse, remoteAuthoritySnapshotDigest, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE, type RemoteAuthorityMode, type RemoteAuthorityRequest} from "./remote_authority_contract.ts"; export type RemoteAuthorityAdmission = - | {status: "allowed"; principal_id: string; policy_revision: string; registered_agents: readonly string[]} + | {status: "allowed"; principal_id: string; policy_revision: string; registered_agents: readonly string[]; + read_scope?: {goal_ref: WireExactGoalRef; todo_ids: readonly string[]; + todo_origins: Readonly>; receipt_operation_id?: string}} | {status: "rejected"; reason_code: string}; export interface RemoteAuthorityDependencies { database: PostgreSqlAuthorityDatabase; /** One immutable, server-owned authorization decision per request. Revocation * affects later admissions; it never lies about an already admitted commit. */ - admit(credential: string, request: RemoteAuthorityRequest): Promise; + admit(credential: string, request: RemoteAuthorityRequest, mode: RemoteAuthorityMode): Promise; now?: () => Date; } const incarnationFailure: AuthorityStoreReadFailure = {status: "unavailable", reason_code: "remote_authority_incarnation_changed", reason: "Rebind the authorized service scope before continuing"}; +const scopeDenied = {status: "rejected", reason_code: "remote_authority_scope_denied"} as const; const MAX_READ_NOTE_BYTES = 4096; /** Pin every observation and write to the admitted incarnation. The provider's @@ -82,8 +86,9 @@ function pinnedStore(store: AuthorityStore, identity: string, admission: JsonObj }; } -function projectReadTodo(todo: JsonObject, includePlanning: boolean): JsonObject { - const fields = ["todo_id", "status", "claimed_by", "note", ...(includePlanning ? ["role", "priority"] : [])]; +function projectReadTodo(todo: JsonObject, includePlanning: boolean, includeNote: boolean): JsonObject { + const fields = ["todo_id", "status", "claimed_by", ...(includeNote ? ["note"] : []), + ...(includePlanning ? ["role", "priority"] : [])]; const output = Object.fromEntries(fields.filter(field => Object.hasOwn(todo, field)).map(field => [field, todo[field]])); if (Object.hasOwn(output, "note") && Buffer.byteLength(JSON.stringify(output.note)) > MAX_READ_NOTE_BYTES) { delete output.note; @@ -92,6 +97,69 @@ function projectReadTodo(todo: JsonObject, includePlanning: boolean): JsonObject return output; } +function isActiveGoal(head: JsonObject, goalRef: WireExactGoalRef): boolean { + const lifecycle = readGoalAcceptanceLifecycle(head, goalRef.goal_id); + return lifecycle?.state === "active" && sameExactGoalRef(lifecycle.goal_ref, goalRef); +} + +/** Reconstruct one exact committed operation through the existing checkpoint + * owner. Proof is bounded by the admitted head; neither current state nor a + * later receipt can substitute for missing historical evidence. */ +async function readScopedTransaction(store: AuthorityStore, headCursor: string, + goalRef: WireExactGoalRef, operationId: string) { + const unavailable = {status: "unavailable", reason_code: "remote_authority_read_unavailable"} as const; + const receipt = await store.readReceipt(operationId); + if (receipt.status !== "found") return receipt.status === "missing" || + receipt.reason_code === "remote_authority_operation_mismatch" ? scopeDenied : unavailable; + const cursor = parseAuthorityCursor(receipt.cursor); + if (cursor > parseAuthorityCursor(headCursor) || receipt.receipts.length !== 1) return scopeDenied; + const offset = cursor - 1n; + const page = await store.scanCommitted(offset === 0n ? null : offset.toString(), 1); + if (page.status !== "page" || page.transactions.length !== 1) return unavailable; + const transaction = page.transactions[0]!; + const original = receipt.receipts[0]!; + const digest = canonicalAuthoritySha256(original); + if (transaction.cursor !== receipt.cursor || transaction.provider_revision !== receipt.provider_revision || + transaction.operation_id !== operationId || transaction.receipts.length !== 1 || + canonicalAuthoritySha256(transaction.receipts[0]) !== digest) return unavailable; + if (!isActiveGoal(transaction.projection, goalRef)) return scopeDenied; + return {status: "found" as const, original, projection: transaction.projection, digest}; +} + +/** The policy selects evidence, not resource ownership. Only a successful Todo + * creation or the trusted first bind of existing work can prove its lifetime. + * A claim/update, successor activation or idempotent rebind cannot relabel it. */ +function provesTodoOrigin(proof: Extract>, {status: "found"}>, + goalRef: WireExactGoalRef, todoId: string): boolean { + const read = new CoordinationProjectionRead(proof.projection, goalRef.goal_id); + read.validateTodoReadModel(); + const todo = read.todoIndex.todos.get(todoId); + if (!todo || proof.original.goal_id !== goalRef.goal_id) return false; + if (proof.original.schema_version === COORDINATION_TODO_CREATE_RECEIPT_SCHEMA) { + return proof.original.todo_id === todoId && canonicalAuthoritySha256(proof.original.todo) === canonicalAuthoritySha256(todo); + } + const result = proof.original.result; + return proof.original.schema_version === GOAL_ACCEPTANCE_LIFECYCLE_OPERATION_SCHEMA && + proof.original.request_sha256 === canonicalAuthoritySha256({transition: {kind: "bind_existing", goal_ref: goalRef}}) && + result !== null && typeof result === "object" && !Array.isArray(result) && (result as JsonObject).changed === true; +} + +/** Authorize from the current pinned head, then prove the original transaction + * belonged to the same Goal lifetime. Operation access stays separate from + * current Todo access and never returns the original execution payload. */ +async function readScopedReceipt(store: AuthorityStore, goalRef: WireExactGoalRef, + todoId: string, operationId: string): Promise { + const head = await store.loadAuthority(); + if (head.status !== "loaded") return {status: "unavailable", reason_code: "remote_authority_read_unavailable"}; + if (!isActiveGoal(head.head, goalRef)) return scopeDenied; + const current = new CoordinationProjectionRead(head.head, goalRef.goal_id); + current.validateTodoReadModel(); + if (!current.todoIndex.todos.has(todoId)) return scopeDenied; + const proof = await readScopedTransaction(store, head.cursor, goalRef, operationId); + if (proof.status !== "found") return proof; + return {status: "found", historical_only: true, todo_id: todoId, operation_id: operationId, receipt_sha256: proof.digest}; +} + /** Deliberately bounded response projection: no SQL errors, paths, arbitrary * canonical fields, workspace observations or provider credentials cross here. */ function commandResult(result: JsonObject): JsonObject { @@ -135,9 +203,16 @@ export class RemoteAuthorityService { return reply({status: "rejected", reason_code: "invalid_remote_authority_mode"}); } let admission: RemoteAuthorityAdmission; - try { admission = structuredClone(await this.#dependencies.admit(credential, request)); } + try { admission = structuredClone(await this.#dependencies.admit(credential, request, mode)); } catch { return reply({status: "unavailable", reason_code: "remote_authority_policy_unavailable"}); } if (admission.status !== "allowed") return reply(admission); + const isRead = request.operation === "read" || request.operation === "snapshot"; + const readScope = admission.read_scope; + const allowedTodos = readScope && new Set(readScope.todo_ids); + const operationId = remoteAuthorityOperationId(request, admission.principal_id); + if (allowedTodos && (allowedTodos.size === 0 || request.todo_id && !allowedTodos.has(request.todo_id))) return reply(scopeDenied); + if (readScope && !isRead && (mode !== "recover" || request.todo_id === undefined || + readScope.receipt_operation_id !== operationId)) return reply(scopeDenied); // Open through the established service seam with the already authenticated // snapshot. Never re-evaluate a different policy halfway through a request. const service = new PostgreSqlAuthorityService({database: this.#dependencies.database, @@ -145,7 +220,6 @@ export class RemoteAuthorityService { authorizeTenant: () => ({status: "allowed"})}); const opened = await service.openStore({...request, credential: null}); if (opened.status !== "opened") return reply({status: "unavailable", reason_code: opened.reason_code}); - const operationId = remoteAuthorityOperationId(request, admission.principal_id); // Explicitly reauthorizing a restored incarnation may recover old history. // The incarnation gates current access; it is not part of business intent. const {store_identity: admittedIdentity, ...intent} = request; @@ -155,7 +229,13 @@ export class RemoteAuthorityService { const store = pinnedStore(opened.store, request.store_identity, binding, operationId); let dispatched = false; try { - const previous = (request.operation === "read" || request.operation === "snapshot") ? null : await store.readReceipt(operationId); + if (readScope && mode === "recover" && request.todo_id !== undefined) { + const result = await readScopedReceipt(store, readScope.goal_ref, request.todo_id, operationId); + return result.status === "found" + ? remoteAuthorityResponse(request, mode, {...result, goal_ref: readScope.goal_ref}, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE) + : reply(result); + } + const previous = isRead ? null : await store.readReceipt(operationId); if (previous?.status === "found") { const original = previous.receipts[0]; const recorded = original?.remote_admission as JsonObject | undefined; @@ -175,22 +255,44 @@ export class RemoteAuthorityService { if (mode === "recover") return reply({status: "missing", operation_id: operationId}); const head = await store.loadAuthority(); if (head.status !== "loaded") return reply({status: "unavailable", reason_code: "remote_authority_head_unavailable"}); - validateCoordinationTodoReadModel(head.head, request.goal_id); - const index = indexCoordinationProjection(head.head, request.goal_id); + if (readScope && !isActiveGoal(head.head, readScope.goal_ref)) return reply(scopeDenied); + const projection = new CoordinationProjectionRead(head.head, request.goal_id); + projection.validateTodoReadModel(); + const index = projection.coordinationIndex; + const provenTodos = readScope ? new Set() : undefined; + if (readScope && provenTodos) { + // One request-local proof per origin, shared by Todos first bound in + // the same transaction. Never cache authorization across admissions. + const proofs = new Map>>(); + for (const id of readScope.todo_ids) { + if (request.todo_id && request.todo_id !== id || !index.todos.has(id)) continue; + const origin = readScope.todo_origins[id]; + if (!origin) continue; + let proof = proofs.get(origin); + if (!proof) { proof = await readScopedTransaction(store, head.cursor, readScope.goal_ref, origin); proofs.set(origin, proof); } + if (proof.status === "unavailable") return reply(proof); + if (proof.status === "found" && provesTodoOrigin(proof, readScope.goal_ref, id)) provenTodos.add(id); + } + if (request.todo_id ? !provenTodos.has(request.todo_id) : provenTodos.size === 0) return reply(scopeDenied); + } + const replyRead = (payload: JsonObject) => readScope + ? remoteAuthorityResponse(request, mode, {...payload, goal_ref: readScope.goal_ref}, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE) + : reply({...payload, provider_revision: head.provider_revision}); if (request.operation === "read") { const todo = index.todos.get(request.todo_id), lease = index.leases.get(request.todo_id); if (!todo) return reply({status: "missing"}); - return reply({status: "found", provider_revision: head.provider_revision, - todo: projectReadTodo(todo, false), + return replyRead({status: "found", + todo: projectReadTodo(todo, false, readScope === undefined), lease: lease ? {...Object.fromEntries(["owner", "status", "expires_at"].filter(k => k in lease).map(k => [k, lease[k]])), version: leaseVersion(lease), lease_epoch: leaseEpoch(lease)} : null}); } if (request.operation === "snapshot") { + const visible = (id: string) => (!request.todo_id || id === request.todo_id) && (!allowedTodos || allowedTodos.has(id)) && (!provenTodos || provenTodos.has(id)); const todos = index.todo_ids - .filter(id => !request.todo_id || id === request.todo_id) - .map(id => projectReadTodo(index.todos.get(id)!, true)); + .filter(visible) + .map(id => projectReadTodo(index.todos.get(id)!, true, readScope === undefined)); const leases = index.lease_todo_ids - .filter(id => !request.todo_id || id === request.todo_id) + .filter(visible) .map(id => { const l = index.leases.get(id)!; return { @@ -199,12 +301,11 @@ export class RemoteAuthorityService { lease_epoch: leaseEpoch(l), }; }); - return reply({ + return replyRead({ status: "snapshot", - provider_revision: head.provider_revision, store_identity: request.store_identity, goal_id: request.goal_id, - receipt_sha256: remoteAuthoritySnapshotDigest(request, head.provider_revision, todos, leases), + receipt_sha256: remoteAuthoritySnapshotDigest(request, head.provider_revision, todos, leases, readScope?.goal_ref), todos, leases, }); @@ -290,6 +391,7 @@ export class RemoteAuthorityService { expected_version: request.expected_version, ttl_seconds: request.operation === "renew" ? request.ttl_seconds : null}); return reply({status: "result", operation_id: operationId, result: commandResult(result)}); } catch { + if (readScope) return reply({status: "unavailable", reason_code: "remote_authority_read_unavailable"}); return reply({status: dispatched ? "ambiguous" : "unavailable", operation_id: operationId, reason_code: dispatched ? "remote_authority_recovery_required" : "remote_authority_read_unavailable"}); } diff --git a/loopx/control_plane/goals/acceptance_authority.ts b/loopx/control_plane/goals/acceptance_authority.ts index d68cf6e30a..554d59b158 100644 --- a/loopx/control_plane/goals/acceptance_authority.ts +++ b/loopx/control_plane/goals/acceptance_authority.ts @@ -17,7 +17,7 @@ import {GOAL_ACCEPTANCE_SCHEMA, clearedGoalAcceptance, acceptanceKeys, acceptanc resolveGoalAcceptanceDocument, projectGoalAcceptance, readGoalAcceptance, readGoalAcceptanceAuthority, type AcceptanceDocument, type AcceptanceState, type AcceptanceVerification} from "./acceptance_contract.ts"; import { BARE_SHA256_PATTERN } from "../content_digest.ts"; -import {GOAL_ACCEPTANCE_LIFECYCLE_SCHEMA, parseGoalAcceptanceLifecycleTransition, parseWireExactGoalRef, +import {GOAL_ACCEPTANCE_LIFECYCLE_SCHEMA, GOAL_ACCEPTANCE_LIFECYCLE_OPERATION_SCHEMA, parseGoalAcceptanceLifecycleTransition, parseWireExactGoalRef, readGoalAcceptanceLifecycle, sameExactGoalRef, type GoalAcceptanceLifecycle, type GoalAcceptanceLifecycleTransition, type WireExactGoalRef} from "./acceptance_lifecycle.ts"; @@ -27,7 +27,7 @@ import {projectCoreGoal} from "../work_items/replan_context.ts"; const RESULT_SCHEMA = "loopx_goal_acceptance_result_v0"; const RECEIPT_SCHEMA = "loopx_goal_acceptance_operation_v0"; -const LIFECYCLE_RECEIPT_SCHEMA = "loopx_goal_acceptance_lifecycle_operation_v0"; +const LIFECYCLE_RECEIPT_SCHEMA = GOAL_ACCEPTANCE_LIFECYCLE_OPERATION_SCHEMA; const REQUEST_FIELDS = ["goal_id", "operation_id", "actor_agent_id", "expected_provider_revision"]; const LOCAL_FIELDS = ["runtime_root", "dry_run"]; diff --git a/loopx/control_plane/goals/acceptance_lifecycle.ts b/loopx/control_plane/goals/acceptance_lifecycle.ts index ab4a35294d..102de76144 100644 --- a/loopx/control_plane/goals/acceptance_lifecycle.ts +++ b/loopx/control_plane/goals/acceptance_lifecycle.ts @@ -3,6 +3,7 @@ import {AuthorityStoreProtocolError, canonicalAuthorityObject} from "../coordina import {parseExactGoalRef} from "./goal_instance_identity.ts"; export const GOAL_ACCEPTANCE_LIFECYCLE_SCHEMA = "loopx_goal_acceptance_lifecycle_v0"; +export const GOAL_ACCEPTANCE_LIFECYCLE_OPERATION_SCHEMA = "loopx_goal_acceptance_lifecycle_operation_v0"; export interface WireExactGoalRef extends JsonObject { goal_id: string; diff --git a/loopx/control_plane/quota/accounting_artifact_transaction.ts b/loopx/control_plane/quota/accounting_artifact_transaction.ts index 54867f699d..e70a5c7bb9 100644 --- a/loopx/control_plane/quota/accounting_artifact_transaction.ts +++ b/loopx/control_plane/quota/accounting_artifact_transaction.ts @@ -508,13 +508,35 @@ export function resolveQuotaAccountingEffect( return { kind: "absent" }; } +function spendIdentityMismatch(index: JsonObject, retained: JsonObject): boolean { + // The spend writers retain these bindings even before commit metadata was + // introduced. Missing historical fields are unknown; explicit conflicts are + // never authenticated by a matching timestamp or billing terms. + return ["agent_id", "effect_ref", "turn_instance_id", "todo_id", + "replan_obligation_id", "settlement_identity", "goal_ref"].some(field => { + const expected = index[field], actual = retained[field]; + return expected !== undefined && expected !== null && expected !== "" && + actual !== undefined && actual !== null && actual !== "" && + canonicalJson(expected) !== canonicalJson(actual); + }); +} + export async function readQuotaSpendEventFromRun( runsDir: string, run: JsonObject, goalId: string, ): Promise { + const contract = contractFor("spend"); + const indexCommit = run[contract.metadataField]; const inline = jsonObject(run.quota_event); - if (inline) return inline; + if (inline && indexCommit === undefined) { + if (spendIdentityMismatch(run, inline)) { + throw new EffectRuntimeRequestError( + "quota spend JSON artifact identity does not match its index row", + ); + } + return inline; + } if (typeof run.json_path !== "string" || !run.json_path.trim()) return null; let targetPath: string; try { @@ -550,15 +572,30 @@ export async function readQuotaSpendEventFromRun( throw new EffectRuntimeRequestError("quota spend JSON artifact is malformed"); } const record = requiredObject(value, "quota spend JSON artifact"); + const artifactCommit = jsonObject(record[contract.metadataField]); + // The writer repeats its commit identity in the index and artifact. An + // in-directory file can still describe a different debit for the same Goal. + const commitMismatch = indexCommit !== undefined && ( + jsonObject(indexCommit) === null || artifactCommit === null || + canonicalJson(indexCommit) !== canonicalJson(artifactCommit) + ); + // A historical row without a commit reference can still have an explicit + // Run timestamp. Preserve omitted historical fields, but reject conflicts. + const timestampMismatch = typeof run.generated_at === "string" && run.generated_at !== "" && + typeof record.generated_at === "string" && record.generated_at !== "" && + run.generated_at !== record.generated_at; + const event = jsonObject(record.quota_event); if ( - record.classification !== contractFor("spend").classification || - (record.goal_id !== undefined && record.goal_id !== goalId) + record.classification !== contract.classification || + (record.goal_id !== undefined && record.goal_id !== goalId) || + commitMismatch || timestampMismatch || spendIdentityMismatch(run, record) || + (event !== null && spendIdentityMismatch(run, event)) ) { throw new EffectRuntimeRequestError( "quota spend JSON artifact identity does not match its index row", ); } - return jsonObject(record.quota_event); + return event; } export async function lookupQuotaAccountingReplay( diff --git a/loopx/control_plane/quota/settlement_readback.ts b/loopx/control_plane/quota/settlement_readback.ts index e0b3f977b5..e8b00ac642 100644 --- a/loopx/control_plane/quota/settlement_readback.ts +++ b/loopx/control_plane/quota/settlement_readback.ts @@ -134,6 +134,19 @@ interface ResultBundle extends JsonObject { payload: JsonObject; } +interface MatchedSpendRecords { + run: JsonObject | null; + receipt: JsonObject | null; + admissionSource?: unknown; +} + +interface SpendTerms { + slots: unknown; + source: unknown; +} + +const ORIGINAL_TURN_SPEND_SLOTS = 1; + type SettlementProgressState = "identity_required" | "writeback_required" | "writeback_receipt_required" | "spend_required" | "spend_receipt_required" | "settled"; @@ -1206,6 +1219,7 @@ function readQuotaSettlementFromRequest( request: ReadbackRequest, snapshot: QuotaSettlementReadbackSnapshot, nameOwedPriorTurn = true, + matchedSpend?: MatchedSpendRecords, ): JsonObject { if ( snapshot.runtimeRoot !== request.runtime_root || @@ -1292,6 +1306,11 @@ function readQuotaSettlementFromRequest( identity, ); const spendEvent = findStepEvent(events, identity, "quota_spend"); + if (matchedSpend) { + matchedSpend.run = spendRun; + matchedSpend.receipt = spendEvent; + matchedSpend.admissionSource = receiptDetails.quota_spend_source; + } const completionEvent = findStepEvent(events, identity, "todo_complete"); const supersedeEvent = findStepEvent(events, identity, "todo_supersede"); const closeoutStarted = [writebackRun, writebackEvent, spendRun, spendEvent, completionEvent, supersedeEvent] @@ -1511,9 +1530,33 @@ export async function readQuotaSettlement(value: unknown): Promise { return await withOwner( request.owner, async () => { + const matchedSpend: MatchedSpendRecords = {run: null, receipt: null}; const result = readQuotaSettlementFromRequest(request, - await readQuotaSettlementSnapshot(request.runtime_root, request.goal_id)); + await readQuotaSettlementSnapshot(request.runtime_root, request.goal_id), true, matchedSpend); const identity = jsonObject(jsonObject(jsonObject(result.identity)?.result)?.value); + const receipt = spendReceiptTerms(matchedSpend.receipt); + // Only exact readback opens its matched debit artifact. Check its + // identity even when the receipt is missing or omits billing terms. + // Snapshot readers for history and guard discovery use the bounded index. + const retained = matchedSpend.run !== null + ? await readQuotaSpendEventFromRun(join(request.runtime_root, "goals", request.goal_id, "runs"), + matchedSpend.run, request.goal_id) + : null; + const ledger = {slots: retained?.slots ?? undefined, source: retained?.source ?? undefined}; + let conflict = spendTermsConflict(ledger, receipt); + if (retained === null && matchedSpend.run !== null && matchedSpend.receipt !== null) { + // An index row identifies the original debit, but does not provide its + // terms. Check explicit receipt terms against the matching admission. + const source = matchedSpend.admissionSource ?? undefined; + conflict = spendTermsConflict( + {slots: ORIGINAL_TURN_SPEND_SLOTS, source}, receipt, "the original Turn admission", + ) ?? (source === undefined && receipt.source !== undefined + ? "the original Turn admission has no verifiable billing source for the quota spend receipt" + : null); + } + if (conflict !== null && spendRequest === null) { + throw new EffectRuntimeRequestError(conflict, "malformed_settlement_state"); + } if (identity && request.refresh_retry && jsonObject(result.refresh_recovery)?.decision !== "reject" && isTurnScopedSettlementOutcome(request.refresh_retry.delivery_outcome, @@ -1529,49 +1572,68 @@ export async function readQuotaSettlement(value: unknown): Promise { // only its receipts and whether a spend Run already needs receipt repair. const settlement = requireJsonObject(result.settlement, "quota settlement bundle"); const outcome = requireJsonObject(settlement.result, "quota settlement result"); - const spendRun = jsonObject(result.spend_run); - const spendRejection = spendRequest && identity - ? await originalSpendRejection(request, result, spendRequest, spendRun) - : null; + const rejection = conflict === null + ? spendRequest && identity ? originalSpendRejection(result, spendRequest, ledger, receipt) : null + : rejectedSpend(conflict); return { schema_version: result.schema_version, found: true, identity: result.identity, settlement: {...settlement, result: {...outcome, value: null}}, progress: result.progress, - spend_run_present: spendRun !== null, - ...(spendRejection === null ? {} : {spend_request_rejection: bundle(spendRejection)}), + spend_run_present: matchedSpend.run !== null, + ...(rejection === null ? {} : {spend_request_rejection: bundle(rejection)}), }; }, ); } -async function originalSpendRejection( - request: ReadbackRequest, readback: JsonObject, proposed: JsonObject, - spendRun: JsonObject | null, -): Promise | null> { +function spendReceiptTerms(event: JsonObject | null): SpendTerms { + const {slots, source} = event === null ? {} : details(event); + // Older rollout envelopes and repaired receipts can omit billing terms. + return { + slots: slots == null || slots === "" ? undefined : slots, + source: source == null || source === "" ? undefined : source, + }; +} + +function spendTermsConflict( + recorded: SpendTerms, + receipt: SpendTerms, + description = "the recorded debit", +): string | null { + for (const field of ["slots", "source"] as const) { + if (recorded[field] !== undefined && receipt[field] !== undefined && recorded[field] !== receipt[field]) { + return `${description} and quota spend receipt disagree on ${field}`; + } + } + return null; +} + +function rejectedSpend(reason: string): SettlementResult { + return settlementFailed({kind: "quota_spend_rejected", step_kind: "quota_spend", reason}); +} + +function originalSpendRejection( + readback: JsonObject, proposed: JsonObject, + ledger: SpendTerms, receipt: SpendTerms, +): SettlementResult | null { const progress = requireJsonObject(readback.progress, "quota settlement progress"); const guard = jsonObject(readback.heartbeat_receipt); const authorizedSource = guard ? details(guard).quota_spend_source : null; // Older ledgers can have a rollout receipt but no readable artifact. Keep // their receipt facts available and never repair an already mismatched debit. const spendEvent = jsonObject(readback.spend_event); - const receipt = spendEvent ? details(spendEvent) : {}; - const retained = spendRun === null ? null : await readQuotaSpendEventFromRun( - join(request.runtime_root, "goals", request.goal_id, "runs"), spendRun, request.goal_id, - ); - const recordedSlots = retained?.slots ?? (receipt.slots === "" ? undefined : receipt.slots); - const recordedSource = retained?.source ?? (receipt.source === "" ? undefined : receipt.source); + const spendRun = jsonObject(readback.spend_run); + const recordedSlots = ledger.slots ?? receipt.slots; + const recordedSource = ledger.source ?? receipt.source; const fallbackSource = recordedSource === "heartbeat" || recordedSource === "visible-goal" ? recordedSource : progress.quota_spend_source; const source = authorizedSource ?? fallbackSource; // Every admitted CLI settlement plan authorizes one slot and the original // source. The same check gates initial spend, receipt repair, and replay. - if (proposed.slots !== 1 || proposed.source !== source) { - return settlementFailed({ - kind: "quota_spend_rejected", step_kind: "quota_spend", - reason: `original Turn settlement requires --slots 1 --source ${String(source)}`, - }); + if (proposed.slots !== ORIGINAL_TURN_SPEND_SLOTS || proposed.source !== source) { + return rejectedSpend(`original Turn settlement requires --slots ${ORIGINAL_TURN_SPEND_SLOTS} --source ${String(source)}`); } const slotsConflict = recordedSlots !== undefined && recordedSlots !== proposed.slots; const sourceConflict = recordedSource !== undefined && recordedSource !== source; @@ -1579,10 +1641,7 @@ async function originalSpendRejection( // debit needs the original charge facts before we can certify its receipt. const unverifiableRepair = spendRun !== null && spendEvent === null && (recordedSlots === undefined || recordedSource === undefined); - return slotsConflict || sourceConflict || unverifiableRepair ? settlementFailed({ - kind: "quota_spend_rejected", step_kind: "quota_spend", - reason: unverifiableRepair + return slotsConflict || sourceConflict || unverifiableRepair ? rejectedSpend(unverifiableRepair ? "the recorded debit has no verifiable billing terms for receipt repair" - : "the recorded debit differs from the original Turn settlement plan", - }) : null; + : "the recorded debit differs from the original Turn settlement plan") : null; } diff --git a/loopx/extensions/lark/private_conversations.py b/loopx/extensions/lark/private_conversations.py index 8ec48634ef..0adb10bf31 100644 --- a/loopx/extensions/lark/private_conversations.py +++ b/loopx/extensions/lark/private_conversations.py @@ -33,7 +33,7 @@ from ...control_plane.collaboration import conversation_reply_context from ...control_plane.content_digest import BARE_SHA256_PATTERN from ...control_plane.effect_runtime import effect_runtime_request_scope -from ...file_lock import exclusive_file_lock +from ...file_lock import exclusive_file_lock, try_exclusive_file_lock from ...presentation.markdown import markdown_scalar from ...presentation.renderers.conversation_status_markdown import render_conversation_status from .conversation_identity import identity_ref, lark_private_source @@ -41,7 +41,9 @@ from .event_collector_runtime import enrich_lark_event_reply_context from .goal_channel_transport import call, json_payload, lark_args, lark_provider_mention_identities, record_event_timing from .identity_shapes import LARK_APP_ID_PATTERN as APP_ID_PATTERN, LARK_MESSAGE_ID_PATTERN -from .inbox_reply import _message, reply_lark_event_inbox, update_lark_inbox_reply, verify_lark_inbox_reply +from .inbox_reply import ( + SourceAuthorityVerifier, _message, reply_lark_event_inbox, update_lark_inbox_reply, verify_lark_inbox_reply, +) from .manager_context import manager_failure_reply from .inbox_reactions import (_reconcile_outbound_input_reactions, mark_lark_event_inbox_processing, mark_lark_event_inbox_received) @@ -1262,6 +1264,7 @@ def _feedback(self, path: Path, record: dict[str, Any], *, inbox: Callable[[], P def _deliver(self, path: Path, record: dict[str, Any], phase: str, text: str, *, inbox: Callable[[], Path], edit_only: bool = False, authorization_check: Callable[[], bool] | None = None, + source_authority_verifier: SourceAuthorityVerifier | None = None, reply_placement: str | None = None, delivery_id: str | None = None, finalize_source: bool = True, image_attachments: list[dict[str, Any]] | None = None, historical_source_check: Callable[[], bool] | None = None) -> bool: @@ -1276,9 +1279,8 @@ def _deliver(self, path: Path, record: dict[str, Any], phase: str, text: str, *, if historical_source_check is not None and phase != "answer_images.historical": raise ValueError("historical source verification requires a media repair") phase = presentation_phase(record, phase) - source_authority = partial(self._source_authority, record) - if kind == "answer_images": - source_authority = partial(source_authority, expected_text=record["event"]["content"]) + expected_text = record["event"]["content"] if kind == "answer_images" else None + source_authority = source_authority_verifier or partial(self._source_authority, record, expected_text=expected_text) phase_state = record["deliveries"].get(phase, {}) if phase_state.get("verified"): return True @@ -1289,10 +1291,12 @@ def _deliver(self, path: Path, record: dict[str, Any], phase: str, text: str, *, text = _presentation_text(text) if phase_state.get("text") not in (None, text): raise ValueError("delivery content changed after an attempt") - if image_attachments and (authorization_check is None or edit_only - or not ((kind == "web" and reply_placement == "chat_root") + if image_attachments and (edit_only + or not ((kind == "web" and reply_placement == "chat_root" + and (authorization_check is not None or source_authority_verifier is not None)) or (kind == "answer_images" and record["event"].get("chat_type") == "p2p" - and record["source"].get("private_human_message") is True))): + and record["source"].get("private_human_message") is True + and authorization_check is not None))): raise ValueError("Image projection requires its original private outbox") record_event_timing(record["profile"], record["event"]["message_id"], "delivery_started") config = inbox() @@ -1318,8 +1322,9 @@ def verified_source() -> bool: def image_authorized() -> bool: # A provider read can overlap Stop, Close, or an edit to the old reply. - return authorization_check is not None and (provider_images_current() and authorization_check() - if kind == "answer_images" else authorization_check() and verified_source()) + if kind == "answer_images": + return provider_images_current() and authorization_check is not None and authorization_check() + return (authorization_check is None or authorization_check()) and verified_source() image_keys = () if image_attachments: @@ -2114,11 +2119,17 @@ def _reconcile_web_projection(self, path: Path) -> int: for message in messages) else "delivered" _atomic_write_json(path, row) return 0 - def authorized() -> bool: + def authorized_source(profile: str, chat_id: str, message_id: str) -> tuple[str | None, bool]: + # Check the original provider source first. The fresh App + # grant and canonical body checks then cover any changes + # during that read, both before preview and before dispatch. + name, verified = self._source_authority(row, profile, chat_id, message_id) + if not name or not verified: + return None, False try: - return pending in self._web_projection_messages(row) + return name, pending in self._web_projection_messages(row) except (OSError, ValueError, KeyError): - return False + return None, False who = "你" if pending["role"] == "user" else "LoopX" text = f"**{who}(App)**\n\n{pending['text']}" if self.reaction_feedback and pending["role"] == "user": @@ -2127,7 +2138,7 @@ def authorized() -> bool: row["deliveries"].setdefault("web." + pending["message_id"], {}).setdefault("feedback", {}) _atomic_write_json(path, row) delivered = self._deliver(path, row, "web." + pending["message_id"], text, - inbox=lambda: self._inbox(row), authorization_check=authorized, + inbox=lambda: self._inbox(row), source_authority_verifier=authorized_source, reply_placement="chat_root", delivery_id=self._web_projection_delivery_id(row, pending), finalize_source=False, image_attachments=pending.get("attachments")) if delivered: @@ -2140,18 +2151,17 @@ def authorized() -> bool: return 0 def reconcile_app_feedback(self) -> int: - """Reconcile optional reactions independently of the body outbox. - - The live service reserves at most one existing reply worker for this - pass. A slow reaction must never own the body-delivery fence or prevent - a later canonical answer from reaching the same audience. - """ + """Reconcile optional reactions independently of the body outbox.""" if not self.reaction_feedback: return 0 for path in sorted((self.root / "web-projections").glob("*.json")): with exclusive_file_lock(path.with_suffix(".feedback"), operation="reconcile_web_input_feedback"): try: - with exclusive_file_lock(path, operation="read_web_input_feedback"): + # Leave a busy body outbox for the next feedback pass so + # independent Sessions can still acknowledge their inputs. + with try_exclusive_file_lock(path, operation="read_web_input_feedback") as lock: + if lock is None: + continue row = self._read_web_projection_journal(path) except (OSError, ValueError): continue @@ -2176,7 +2186,11 @@ def reconcile_app_feedback(self) -> int: self._web_projection_feedback(row, message) if "feedback" not in phase or phase["feedback"] == prior_feedback: continue - with exclusive_file_lock(path, operation="record_web_input_feedback"): + # The reaction has its own receipt. Defer writeback when + # a body worker acquired the journal during provider I/O. + with try_exclusive_file_lock(path, operation="record_web_input_feedback") as lock: + if lock is None: + continue try: current = self._read_web_projection_journal(path) except (OSError, ValueError): diff --git a/loopx/global_todos.py b/loopx/global_todos.py index 71103af9b1..11aa3d97f7 100644 --- a/loopx/global_todos.py +++ b/loopx/global_todos.py @@ -4,6 +4,10 @@ from pathlib import Path from typing import Any +from .control_plane.goals.contract_health import ( + CONTRACT_ERROR_DIAGNOSTIC_SCHEMA_VERSION, + contract_error_views, +) from .control_plane.projection_envelope_facts import ( render_projection_envelope_markdown, seal_projection_envelope, @@ -392,10 +396,50 @@ def _deduplicate_todos( return list(deduplicated.values()) +def _scoped_status_failure( + status_payload: dict[str, Any], +) -> tuple[set[str], set[str]] | None: + registry = as_dict(status_payload.get("global_registry")) + contract = as_dict(status_payload.get("contract")) + diagnostics = contract.get("error_diagnostics") + if ( + registry.get("available") is not True + or registry.get("ok") is not True + or contract.get("ok") is not False + or contract.get("global_errors") + or not isinstance(diagnostics, list) + or not diagnostics + or not all( + isinstance(row, dict) + and row.get("schema_version") == CONTRACT_ERROR_DIAGNOSTIC_SCHEMA_VERSION + for row in diagnostics + ) + ): + return None + views = contract_error_views(diagnostics) + if ( + views["global_errors"] + or views["errors"] != contract.get("errors") + or len(views["error_diagnostics"]) != len(diagnostics) + ): + return None + failed_goals = set(views["goal_errors"]) + registered_goals = { + row["id"] + for row in as_list(as_dict(status_payload.get("run_history")).get("goals")) + if isinstance(row, dict) and row.get("registry_member") is True + and isinstance(row.get("id"), str) and row["id"] + } + if not failed_goals or not failed_goals <= registered_goals: + return None + return registered_goals, failed_goals + + def _queue_goal_ids( status_payload: dict[str, Any], *, scan_limit: int, + eligible_goals: set[str] | None = None, ) -> tuple[list[str], int]: queue = as_dict(status_payload.get("attention_queue")) unique_goal_ids: list[str] = [] @@ -412,6 +456,12 @@ def _queue_goal_ids( reported_count = max(0, int(queue.get("item_count") or 0)) except (TypeError, ValueError): reported_count = 0 + # A degraded status queue can include a synthetic contract warning in + # place of the failed Goal. Only the observed registry can name read scopes. + if eligible_goals is not None: + eligible = [goal_id for goal_id in unique_goal_ids if goal_id in eligible_goals] + reported_count -= len(unique_goal_ids) - len(eligible) + unique_goal_ids = eligible available_count = max(len(unique_goal_ids), reported_count) return unique_goal_ids[:scan_limit], available_count @@ -426,6 +476,7 @@ def _todos_envelope( goal_scan_omitted: int, shown_count: int, available_count: int, + failed_status_goals: set[str] | None = None, ) -> dict[str, Any]: global_registry = as_dict(status_payload.get("global_registry")) status_envelope = as_dict(status_payload.get("projection_envelope")) @@ -467,7 +518,13 @@ def _todos_envelope( omitted.append( {"reason": "attention_goals_not_scanned", "count": goal_scan_omitted} ) - if status_payload and status_payload.get("ok") is not True: + if failed_status_goals: + omitted.append({ + "reason": "status_unavailable", + "count": len(failed_status_goals), + "refs": [_redact_text(goal_id, limit=120) for goal_id in sorted(failed_status_goals)[:8]], + }) + elif status_payload and status_payload.get("ok") is not True: omitted.append({"reason": "status_unavailable", "count": 1}) queue = status_payload.get("attention_queue") queue_read = isinstance(queue, dict) and isinstance(queue.get("items"), list) @@ -516,7 +573,7 @@ def _todos_envelope( "scope": "global", "expected_count": expected, "included_count": ( - max(0, expected - excluded - goal_scan_omitted) + max(0, expected - excluded - goal_scan_omitted - len(failed_status_goals or ())) if expected is not None else 0 ), @@ -554,21 +611,27 @@ def build_global_todos( limit=scan_limit, ) status_read_at = now_utc_iso() - if status_payload.get("ok") is not True: + status_ok = status_payload.get("ok") is True + scoped_failure = None if status_ok else _scoped_status_failure(status_payload) + registered_goals, failed_status_goals = scoped_failure or (set(), set()) + goal_ids, available_goal_count = _queue_goal_ids( + status_payload, + scan_limit=scan_limit, + eligible_goals=registered_goals - failed_status_goals if scoped_failure else None, + ) + if not status_ok and (scoped_failure is None or not goal_ids): return build_global_todos_error( "Global status source unavailable.", status_payload=status_payload, status_read_at=status_read_at, ) - - goal_ids, available_goal_count = _queue_goal_ids( - status_payload, - scan_limit=scan_limit, - ) - warnings: list[dict[str, Any]] = [] + warnings = [ + _warning("quota_unavailable", goal_id=goal_id, detail="The Goal's status contract check failed.") + for goal_id in sorted(failed_status_goals) + ] classified: list[dict[str, Any]] = [] quota_read_at = now_utc_iso() - quota_unavailable = 0 + quota_unavailable = len(failed_status_goals) for goal_id in goal_ids: quota_payloads, was_unavailable = _quotas_for_goal( status_payload, @@ -655,6 +718,7 @@ def build_global_todos( goal_scan_omitted=max(0, available_goal_count - len(goal_ids)), shown_count=returned_count, available_count=matched_count, + failed_status_goals=failed_status_goals, ), "omissions": [ ( diff --git a/loopx/quota.py b/loopx/quota.py index 4d774c4980..0c8bbfbfb6 100644 --- a/loopx/quota.py +++ b/loopx/quota.py @@ -1837,12 +1837,15 @@ def spend_quota_slot( ), } if settlement_readback.spend_run_present: + # The precheck verified these terms against the retained debit. + # Carry them to the existing receipt writer without charging again. return { "ok": True, "mode": "spend-slot", "dry_run": not execute, "appended": False, "receipt_repair_required": bool(execute), + **spend_request, "goal_id": safe_goal_id, "agent_id": identity.agent_id, "todo_id": identity.todo_id, diff --git a/tests/control_plane/test_refresh_settle_turn.py b/tests/control_plane/test_refresh_settle_turn.py index 5c3d86da21..4941264756 100644 --- a/tests/control_plane/test_refresh_settle_turn.py +++ b/tests/control_plane/test_refresh_settle_turn.py @@ -169,6 +169,229 @@ def fail_child(argv, **kwargs): assert replay["idempotent_replay"] and _spend_run_count(runtime) == 1 +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +def test_exact_spend_and_combined_replay_reject_disagreeing_billing_terms(tmp_path, monkeypatch, provider): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path) + _initialize_fixture_authority(project, runtime, provider=provider) + binding = ["--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", TURN_ID] + rc, guard = _run_cli(registry, runtime, "quota", "should-run", "--codex-app", *binding, cwd=project) + assert rc == 0, guard + original = guard["heartbeat_receipt"]["settlement_identity"] + refresh = ["refresh-state", *binding, "--classification", "verified_billing_replay", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--delivery-boundary", "in_flight_continuation", "--no-global-sync", "--suppress-external-sinks"] + combined = [*refresh, "--settle-turn", "--settlement-receipt"] + spend = ["quota", "spend-slot", *binding, "--execute"] + debit = [*spend, "--slots", "1", "--source", "heartbeat"] + rc, delivery = _run_cli(registry, runtime, *refresh, cwd=project) + assert rc == 0 and delivery["settlement_progress"]["state"] == "spend_required", delivery + for slots, source in (("2", "heartbeat"), ("1", "visible-goal")): + rc, wrong = _run_cli(registry, runtime, *spend, "--slots", slots, "--source", source, cwd=project) + assert rc != 0 and not wrong["ok"] and not wrong["appended"], wrong + assert _spend_run_count(runtime) == 0 + + rc, settled = _run_cli(registry, runtime, *combined, cwd=project) + assert rc == 0 and settled["settlement_progress"]["state"] == "settled", settled + assert settled["settlement_identity"]["effect_id"] == original["effect_id"] + log = runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl" + index = log.parent / "runs" / "index.jsonl" + receipt_log = log.read_text(encoding="utf-8") + initial_index = index.read_bytes() + debit_record, = (record for line in initial_index.splitlines() + if (record := json.loads(line))["classification"] == "quota_slot_spent") + artifact = Path(debit_record["json_path"]) + assert artifact.resolve().is_relative_to(runtime.resolve()) + for field, bad_value in ((None, None), ("slots", 2), ("source", "visible-goal"), + ("missing", None), ("legacy_empty", ""), ("legacy_empty", None), + ("missing_artifact", None), ("missing", None), ("legacy_empty", None), + ("slots", 2), ("source", "visible-goal"), + ("unknown_admission_source", None), ("unknown_legacy_terms", None)): + rows = [json.loads(line) for line in receipt_log.splitlines()] + receipt, = (row for row in rows if row["event_kind"] == "quota_spend" and row["agent_id"] == AGENT_ID) + details = receipt["details"] + assert details["settlement_effect_id"] == original["effect_id"] + assert details["slots"] == 1 and details["source"] == "heartbeat" + if field == "missing": + details.pop("slots") + details.pop("source") + elif field == "legacy_empty": + details.update(slots=bad_value, source=bad_value) + elif field == "missing_artifact": + artifact.rename(tmp_path / "legacy-unavailable-spend.json") + elif field in ("unknown_admission_source", "unknown_legacy_terms"): + admission, = (row for row in rows if row["event_kind"] == "quota_should_run") + assert admission["details"].pop("quota_spend_source") == "heartbeat" + if field == "unknown_legacy_terms": + details.pop("slots") + details.pop("source") + elif field is not None: + details[field] = bad_value + log.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + before = log.read_bytes() + missing_artifact = not artifact.exists() + # A caller cannot make a conflicting source valid by supplying that + # same source on replay once only the index and receipt remain. + commands = (combined, debit) if missing_artifact else (debit, combined) + if field == "source" and missing_artifact: + commands += ((*spend, "--slots", "1", "--source", bad_value),) + for command in commands: + rc, replay = _run_cli(registry, runtime, *command, cwd=project) + if field in ("slots", "source", "unknown_admission_source"): + assert rc != 0 and not replay["ok"], replay + basis = "original Turn admission" if missing_artifact else "recorded debit" + diagnostic = replay.get("reason") or replay.get("error") or "" + expected = ("original Turn admission has no verifiable billing source" + if field == "unknown_admission_source" + else f"{basis} and quota spend receipt disagree on {field}") + assert expected in diagnostic, replay + assert replay.get("settlement_progress", {}).get("state") != "settled", replay + assert not replay.get("settlement_owed"), replay + else: + assert rc == 0 and replay["ok"] and replay["idempotent_replay"], replay + assert replay["settlement_identity"]["effect_id"] == original["effect_id"] + assert not replay.get("appended"), replay + assert log.read_bytes() == before and index.read_bytes() == initial_index + assert _heartbeat_receipt_count(runtime, TURN_ID) == 1 and _spend_run_count(runtime) == 1 + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("repair_entry", ["spend", "refresh"]) +def test_missing_spend_receipt_preserves_the_original_debit(tmp_path, monkeypatch, provider, repair_entry): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path) + _initialize_fixture_authority(project, runtime, provider=provider) + binding = ["--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", TURN_ID] + rc, guard = _run_cli(registry, runtime, "quota", "should-run", "--codex-app", *binding, cwd=project) + assert rc == 0, guard + original = guard["heartbeat_receipt"]["settlement_identity"] + combined = ["refresh-state", *binding, "--classification", "verified_billing_repair", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--delivery-boundary", "in_flight_continuation", "--no-global-sync", "--suppress-external-sinks", + "--settle-turn", "--settlement-receipt"] + debit = ["quota", "spend-slot", *binding, "--slots", "1", "--source", "heartbeat", "--execute"] + rc, settled = _run_cli(registry, runtime, *combined, cwd=project) + assert rc == 0 and settled["settlement_progress"]["state"] == "settled", settled + + log = runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl" + index = log.parent / "runs" / "index.jsonl" + original_index = index.read_bytes() + debit_record, = (row for line in original_index.splitlines() + if (row := json.loads(line))["classification"] == "quota_slot_spent") + artifact = Path(debit_record["json_path"]) + original_artifact = artifact.read_bytes() + records = [json.loads(line) for line in log.read_text(encoding="utf-8").splitlines()] + receipt, = (row for row in records if row["event_kind"] == "quota_spend") + retained = [row for row in records if row is not receipt] + log.write_text("".join(json.dumps(row) + "\n" for row in retained), encoding="utf-8") + + rc, repaired = _run_cli(registry, runtime, *(debit if repair_entry == "spend" else combined), cwd=project) + assert rc == 0 and repaired["ok"], repaired + assert repaired["settlement_progress"]["state"] == "settled", repaired + assert repaired["settlement_identity"]["effect_id"] == original["effect_id"] + assert not repaired.get("settlement_owed"), repaired + recovered_log = log.read_bytes() + after = [json.loads(line) for line in recovered_log.splitlines()] + assert [row for row in after if row["event_kind"] != "quota_spend"] == retained + recovered, = (row for row in after if row["event_kind"] == "quota_spend") + assert recovered["details"]["settlement_effect_id"] == original["effect_id"] + assert recovered["details"]["slots"] == receipt["details"]["slots"] == 1 + assert recovered["details"]["source"] == receipt["details"]["source"] == "heartbeat" + for command in (debit, combined): + rc, replay = _run_cli(registry, runtime, *command, cwd=project) + assert rc == 0 and replay["ok"] and replay["idempotent_replay"], replay + assert replay["settlement_identity"]["effect_id"] == original["effect_id"] + assert not replay.get("appended") and not replay.get("settlement_owed"), replay + assert log.read_bytes() == recovered_log + assert index.read_bytes() == original_index and artifact.read_bytes() == original_artifact + assert _heartbeat_receipt_count(runtime, TURN_ID) == 1 and _spend_run_count(runtime) == 1 + + +@pytest.mark.parametrize("provider", ["file", "sqlite"]) +@pytest.mark.parametrize("artifact_format", ["current", "legacy"]) +def test_exact_spend_and_refresh_reject_another_turns_debit_file( + tmp_path, monkeypatch, provider, artifact_format, +): + isolate_sqlite_runtime(tmp_path, monkeypatch) + project, runtime, registry = _write_fixture(tmp_path) + _initialize_fixture_authority(project, runtime, provider=provider) + commands, identities = {}, {} + for turn in (TURN_ID, f"{TURN_ID}-second"): + binding = ["--goal-id", GOAL_ID, "--agent-id", AGENT_ID, + "--todo-id", TODO_ID, "--turn-instance-id", turn] + rc, guard = _run_cli(registry, runtime, "quota", "should-run", "--codex-app", *binding, cwd=project) + assert rc == 0 and guard["should_run"], guard + identities[turn] = guard["heartbeat_receipt"]["settlement_identity"] + combined = ["refresh-state", *binding, "--classification", "verified_debit_identity", + "--delivery-batch-scale", "implementation", "--delivery-outcome", "outcome_progress", + "--delivery-boundary", "in_flight_continuation", "--no-global-sync", "--suppress-external-sinks", + "--settle-turn", "--settlement-receipt"] + spend = ["quota", "spend-slot", *binding, "--slots", "1", "--source", "heartbeat", "--execute", "--verbose"] + rc, settled = _run_cli(registry, runtime, *combined, cwd=project) + assert rc == 0 and settled["settlement_progress"]["state"] == "settled", settled + assert settled["settlement_identity"] == identities[turn] + commands[turn] = (combined, spend) + + log = runtime / "goals" / GOAL_ID / "rollout-event-log.jsonl" + index = log.parent / "runs" / "index.jsonl" + rows = [json.loads(line) for line in index.read_bytes().splitlines()] + target, sibling = (row for row in rows if row["classification"] == "quota_slot_spent") + assert target["settlement_identity"] == identities[TURN_ID] + assert sibling["settlement_identity"] == identities[f"{TURN_ID}-second"] + artifacts = [Path(row["json_path"]) for row in (target, sibling)] + if artifact_format == "legacy": + # The historical Python writer retained Turn/effect bindings, but no + # commit metadata. Equal timestamps must not authenticate another Turn. + for row, path in zip((target, sibling), artifacts): + record = json.loads(path.read_bytes()) + row.pop("quota_spend_commit") + record.pop("quota_spend_commit") + row["generated_at"] = record["generated_at"] = "2026-11-01T12:00:00Z" + path.write_text(json.dumps(record) + "\n", encoding="utf-8") + index.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + original_artifacts = [path.read_bytes() for path in artifacts] + assert [(json.loads(data)["quota_event"]["slots"], json.loads(data)["quota_event"]["source"]) + for data in original_artifacts] == [(1, "heartbeat"), (1, "heartbeat")] + original_index = index.read_bytes() + original_log = log.read_bytes() + target["json_path"] = sibling["json_path"] + index.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8") + redirected_index = index.read_bytes() + + for receipt_kind in ("current", "legacy", "missing"): + if receipt_kind != "current": + retained = [] + for line in original_log.splitlines(): + row = json.loads(line) + if row["event_kind"] == "quota_spend" and row["run_id"] == TURN_ID: + if receipt_kind == "missing": + continue + row["details"].pop("slots") + row["details"].pop("source") + retained.append(row) + log.write_text("".join(json.dumps(row) + "\n" for row in retained), encoding="utf-8") + before = log.read_bytes() + for command in commands[TURN_ID]: + rc, rejected = _run_cli(registry, runtime, *command, cwd=project) + assert rc != 0 and not rejected["ok"], rejected + reason = (rejected.get("verbose_debug", {}).get("error") + or rejected.get("error") or rejected.get("reason") or "") + assert "quota spend JSON artifact identity does not match its index row" in reason, rejected + assert rejected.get("settlement_progress", {}).get("state") != "settled", rejected + assert log.read_bytes() == before and index.read_bytes() == redirected_index + assert _spend_run_count(runtime) == 2 + + assert [path.read_bytes() for path in artifacts] == original_artifacts + index.write_bytes(original_index) + rc, recovered = _run_cli(registry, runtime, *commands[TURN_ID][0], cwd=project) + assert rc == 0 and recovered["settlement_progress"]["state"] == "settled", recovered + assert recovered["settlement_identity"] == identities[TURN_ID] + assert _spend_run_count(runtime) == 2 and index.read_bytes() == original_index + + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_receipt_capture_failure_keeps_committed_receipts_and_exact_retry(tmp_path, monkeypatch, provider): isolate_sqlite_runtime(tmp_path, monkeypatch) diff --git a/tests/control_plane/test_unmigrated_goal_reads.py b/tests/control_plane/test_unmigrated_goal_reads.py index f7720c3b5f..6d4d5a52b0 100644 --- a/tests/control_plane/test_unmigrated_goal_reads.py +++ b/tests/control_plane/test_unmigrated_goal_reads.py @@ -11,6 +11,8 @@ from loopx.control_plane.coordination.legacy_writer_fence import ( legacy_coordination_writer_fence_path, ) +from loopx.control_plane.testing.canary_harness import run_json_cli_result +from canonical_authority_fixture import initialize_canonical_source_fixture, isolate_sqlite_runtime from test_cold_source_import_cli import canonical_todos, workspace @@ -66,6 +68,66 @@ def test_ordinary_cli_refuses_unmigrated_goal_without_importing_it( assert not (runtime / "authority").exists() +def test_global_todos_retains_healthy_lanes_with_a_stopped_unmigrated_goal(tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + runtime = tmp_path / "runtime" + runtime.mkdir() + registry = runtime / "registry.global.json" + goals = [] + state_by_goal = {} + for goal_id in ("ready", "stopped"): + project = tmp_path / goal_id + project.mkdir() + state = project / "ACTIVE_GOAL_STATE.md" + state.write_text( + "---\nstatus: active\nowner_mode: goal\n---\n\n" + "## Agent Todo\n\n- [ ] Verify the registered work.\n" + f" \n" + ) + goal = { + "id": goal_id, "repo": str(project), "state_file": state.name, "status": "active", + "domain": "software", "adapter": {"kind": "read_only_project_map_v0", "status": "connected-read-only"}, + "coordination": {"registered_agents": ["agent-a"]}, + } + if goal_id == "stopped": + goal["activation"] = { + "schema_version": "loopx_goal_activation_v1", "state": "stopped", + "updated_at": "2026-01-01T00:00:00Z", "actor_kind": "owner", "reason": "Stop this fixture.", + } + else: + initialize_canonical_source_fixture(runtime, goal_id, state_path=state, provider="sqlite") + goals.append(goal) + state_by_goal[goal_id] = state + registry.write_text(json.dumps({ + "schema_version": "0.1", "registry_role": "global-local", + "common_runtime_root": str(runtime), "goals": goals, + })) + original = registry.read_bytes(), state_by_goal["stopped"].read_bytes() + + code, status = run_json_cli_result("status", registry_path=registry) + assert code == 1 and status["ok"] is False + assert [(e["goal_id"], e["code"]) for e in status["contract"]["error_diagnostics"]] == [ + ("stopped", "canonical_authority_required"), + ] + assert any(row["goal_id"] == "loopx-contract" for row in status["attention_queue"]["items"]) + + code, observed = run_json_cli_result("global-todos", registry_path=registry) + assert code == 0 and observed["ok"] is True, observed + assert {(todo["goal_id"], todo["todo_id"]) for todo in observed["todos"]} == {("ready", "todo_ready")} + assert [(row["goal_id"], row["reason_code"]) for row in observed["source_warnings"]] == [ + ("stopped", "quota_unavailable"), + ] + envelope = observed["projection_envelope"] + assert envelope["complete"] is False + assert envelope["coverage"]["expected_count"] == 2 + assert envelope["coverage"]["included_count"] == 1 + quota = next(row for row in envelope["sources"] if row["source_id"] == "goal_quota") + assert (quota["read_status"], quota["item_count"], quota["unreadable_count"]) == ("read", 1, 1) + assert (registry.read_bytes(), state_by_goal["stopped"].read_bytes()) == original + assert not legacy_coordination_writer_fence_path(runtime_root=runtime, goal_id="stopped").exists() + + @pytest.mark.parametrize("provider", ["file", "sqlite"]) def test_explicit_import_retains_full_source_and_enables_ordinary_reads( tmp_path, monkeypatch, provider, diff --git a/tests/control_plane_ts/quota_void_commit.test.ts b/tests/control_plane_ts/quota_void_commit.test.ts index 9f1e9592ba..b70c4abc99 100644 --- a/tests/control_plane_ts/quota_void_commit.test.ts +++ b/tests/control_plane_ts/quota_void_commit.test.ts @@ -304,6 +304,81 @@ test("target lookup falls back to the bounded legacy JSON artifact", async (t) = assert.equal((event.after as Record).spent_slots, 1); }); +test("void does not use another debit's artifact even at the same timestamp", async (t) => { + const fixture = await targetFixture(t, { inline: false, slots: 4 }); + const record = JSON.parse(await readFile(fixture.targetJsonPath, "utf8")); + const commit = { schema_version: "quota_spend_commit_receipt_v0", effect_id: "original-debit", + request_digest: `sha256:${"a".repeat(64)}` }; + const index = JSON.parse(fixture.indexContent); + const otherDebit = `${JSON.stringify({ ...record, + quota_spend_commit: { ...commit, effect_id: "another-debit" } })}\n`; + assert.equal(index.generated_at, record.generated_at); + await writeFile(fixture.targetJsonPath, otherDebit, "utf8"); + + for (const inline of [false, true]) { + const retainedIndex = `${JSON.stringify({ ...index, quota_spend_commit: commit, + ...(inline ? { quota_event: record.quota_event } : {}) })}\n`; + await writeFile(fixture.indexPath, retainedIndex, "utf8"); + await assert.rejects( + async () => evaluateQuotaVoidCommit(await request(fixture, { execute: false })), + /quota spend JSON artifact identity does not match its index row/, + ); + assert.equal(await readFile(fixture.indexPath, "utf8"), retainedIndex); + } + assert.equal(await readFile(fixture.targetJsonPath, "utf8"), otherDebit); +}); + +test("legacy debit bindings must agree in the index, artifact and event", async (t) => { + const fixture = await targetFixture(t, { inline: false }); + const record = JSON.parse(await readFile(fixture.targetJsonPath, "utf8")); + const index = JSON.parse(fixture.indexContent); + // Each identity was persisted by the legacy writer independently of its + // timestamp. Exercise both artifact locations through the actual void path. + const bindings = { + agent_id: ["agent-a", "agent-b"], + effect_ref: ["debit-a", "debit-b"], + turn_instance_id: ["turn-a", "turn-b"], + todo_id: ["todo-a", "todo-b"], + replan_obligation_id: ["replan-a", "replan-b"], + settlement_identity: [{ effect_id: "turn-a" }, { effect_id: "turn-b" }], + }; + for (const [field, [original, other]] of Object.entries(bindings)) { + for (const nested of [false, true]) { + const retainedIndex = `${JSON.stringify({ ...index, [field]: original })}\n`; + await writeFile(fixture.indexPath, retainedIndex); + const conflict = nested + ? { ...record, quota_event: { ...record.quota_event, [field]: other } } + : { ...record, [field]: other }; + const artifact = `${JSON.stringify(conflict)}\n`; + await writeFile(fixture.targetJsonPath, artifact); + await assert.rejects( + async () => evaluateQuotaVoidCommit(await request(fixture, { execute: false })), + /quota spend JSON artifact identity does not match its index row/, + ); + assert.equal(await readFile(fixture.indexPath, "utf8"), retainedIndex); + assert.equal(await readFile(fixture.targetJsonPath, "utf8"), artifact); + if (nested) { + const inlineIndex = JSON.stringify({ ...index, [field]: original, + quota_event: conflict.quota_event }); + await writeFile(fixture.indexPath, inlineIndex); + await assert.rejects( + async () => evaluateQuotaVoidCommit(await request(fixture, { execute: false })), + /quota spend JSON artifact identity does not match its index row/, + ); + assert.equal(await readFile(fixture.indexPath, "utf8"), inlineIndex); + await writeFile(fixture.indexPath, retainedIndex); + } + } + // Omitted or empty historical identity fields remain compatible. An + // explicit conflict is stronger evidence than a matching timestamp. + for (const absent of [undefined, null, ""]) { + await writeFile(fixture.targetJsonPath, JSON.stringify({ ...record, [field]: absent })); + const result = await evaluateQuotaVoidCommit(await request(fixture, { execute: false })); + assert.equal(result.status, "preview"); + } + } +}); + test("legacy index rows without goal identity use the requested goal", async (t) => { const fixture = await targetFixture(t, { inline: false }); const indexRecord = JSON.parse(fixture.indexContent) as Record; diff --git a/tests/control_plane_ts/remote_authority.integration.test.ts b/tests/control_plane_ts/remote_authority.integration.test.ts index 23c1cf5205..96436bc47c 100644 --- a/tests/control_plane_ts/remote_authority.integration.test.ts +++ b/tests/control_plane_ts/remote_authority.integration.test.ts @@ -21,10 +21,12 @@ import {authorityProjectionFixture} from "./authority_projection_fixture.ts"; import {resolveTestPython} from "../../scripts/test-python.mjs"; import {registerRemoteAuthorityModeConformance} from "./remote_authority_mode_conformance.ts"; import {registerRemoteAuthorityLifecycleConformance} from "./remote_authority_lifecycle_conformance.ts"; +import {registerRemoteAuthorityReadAclConformance} from "./remote_authority_read_acl_conformance.ts"; const connectionString = process.env.LOOPX_TEST_POSTGRES_URL; registerRemoteAuthorityModeConformance(connectionString); registerRemoteAuthorityLifecycleConformance(connectionString); +registerRemoteAuthorityReadAclConformance(connectionString); test("quota CLI verifies the exact remote binding and never grants local execution", {skip: !connectionString}, async t => { const fixture = await postgresRemoteFixture(t, connectionString!); await fixture.seed(["todo_gateway_probe"], {todo_gateway_probe: "private-synthetic-note"}); diff --git a/tests/control_plane_ts/remote_authority.test.ts b/tests/control_plane_ts/remote_authority.test.ts index e65dae3eb3..0c4d8012d7 100644 --- a/tests/control_plane_ts/remote_authority.test.ts +++ b/tests/control_plane_ts/remote_authority.test.ts @@ -9,7 +9,7 @@ import {tmpdir} from "node:os"; import {join} from "node:path"; import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; import {decodeRemoteAuthorityRequest, decodeRemoteAuthorityResponse, remoteAuthorityOperationId, - remoteAuthorityResponse} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; + remoteAuthorityResponse, remoteAuthoritySnapshotDigest, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; import {canonicalAuthoritySha256} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; import {decodeRemoteAuthorityPolicy, remoteAuthorityPolicyAdmission, readRemoteAuthorityPolicy} from "../../loopx/control_plane/coordination/remote_authority_policy.ts"; import {RemoteAuthorityService} from "../../loopx/control_plane/coordination/remote_authority_service.ts"; @@ -76,6 +76,66 @@ test("missing-lease release responses require explicit no-op evidence", () => { } }); +test("scoped read responses require their Goal reference and exclude global authority metadata", () => { + const {request_id: _, ...scope} = claimRequest("a"); + const request = decodeRemoteAuthorityRequest({...scope, operation: "read"}); + const goal_ref = {goal_id: request.goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}; + const payload = {status: "found", goal_ref, todo: {todo_id: request.todo_id, status: "open"}, lease: null}; + const wrap = (value: JsonObject) => remoteAuthorityResponse(request, "execute", value, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE); + assert.deepEqual(decodeRemoteAuthorityResponse(wrap(payload), request, "execute"), wrap(payload)); + for (const invalid of [ + {...payload, goal_ref: {goal_id: request.goal_id}}, + {...payload, goal_ref: {...goal_ref, goal_id: "other-goal"}}, + {...payload, provider_revision: `${request.store_identity}:2`}, + {...payload, cursor: "2"}, + ]) assert.throws(() => decodeRemoteAuthorityResponse(wrap(invalid), request, "execute")); +}); + +test("scoped response nested records reject fields outside their read projection", () => { + const {request_id: _, ...scope} = claimRequest("a"); + const goal_ref = {goal_id: scope.goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}; + const todo = {todo_id: scope.todo_id, status: "open", claimed_by: "actor-a"}; + const lease = {owner: "actor-a", status: "active", expires_at: "2099-01-01T00:00:00Z", version: 1, lease_epoch: 1}; + const read = decodeRemoteAuthorityRequest({...scope, operation: "read"}); + const snapshot = decodeRemoteAuthorityRequest({...scope, operation: "snapshot"}); + const wrap = (request: typeof read, payload: JsonObject) => remoteAuthorityResponse(request, "execute", + {...payload, goal_ref}, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE); + const found = (row: JsonObject, held: JsonObject) => wrap(read, {status: "found", todo: row, lease: held}); + const snap = (row: JsonObject, held: JsonObject) => wrap(snapshot, {status: "snapshot", goal_id: scope.goal_id, + store_identity: scope.store_identity, todos: [row], leases: [held], + receipt_sha256: remoteAuthoritySnapshotDigest(snapshot, undefined, [row], [held], goal_ref)}); + const snapshotTodo = {...todo, role: "agent", priority: "P1"}; + const snapshotLease = {...lease, goal_id: scope.goal_id, todo_id: scope.todo_id}; + assert.deepEqual(decodeRemoteAuthorityResponse(found(todo, lease), read, "execute"), found(todo, lease)); + assert.deepEqual(decodeRemoteAuthorityResponse(snap(snapshotTodo, snapshotLease), snapshot, "execute"), snap(snapshotTodo, snapshotLease)); + for (const field of ["note", "note_omitted", "text", "idempotency_key", "provider_revision", "cursor", "unknown"]) { + const value = field === "note_omitted" ? true : "must-not-cross-the-read-boundary"; + assert.throws(() => decodeRemoteAuthorityResponse(found({...todo, [field]: value}, lease), read, "execute"), `read Todo ${field}`); + assert.throws(() => decodeRemoteAuthorityResponse(found(todo, {...lease, [field]: value}), read, "execute"), `read lease ${field}`); + // A matching digest cannot authorize extra fields in a scoped snapshot. + assert.throws(() => decodeRemoteAuthorityResponse(snap({...snapshotTodo, [field]: value}, snapshotLease), snapshot, "execute"), `snapshot Todo ${field}`); + assert.throws(() => decodeRemoteAuthorityResponse(snap(snapshotTodo, {...snapshotLease, [field]: value}), snapshot, "execute"), `snapshot lease ${field}`); + } + const trusted = remoteAuthorityResponse(read, "execute", {status: "found", provider_revision: `${scope.store_identity}:3`, + todo: {...todo, note: "trusted v0 note"}, lease: {...lease, idempotency_key: "trusted-v0-key"}}); + assert.deepEqual(decodeRemoteAuthorityResponse(trusted, read, "execute"), trusted); +}); + +test("scoped historical receipt responses expose proof for the original Todo without its execution payload", () => { + const request = claimRequest("a"); + const goal_ref = {goal_id: request.goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}; + const payload = {status: "found", historical_only: true, goal_ref, todo_id: request.todo_id, + operation_id: remoteAuthorityOperationId(request, "principal-a"), receipt_sha256: "d".repeat(64)}; + const wrap = (value: JsonObject) => remoteAuthorityResponse(request, "recover", value, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE); + assert.deepEqual(decodeRemoteAuthorityResponse(wrap(payload), request, "recover"), wrap(payload)); + for (const invalid of [ + {...payload, todo_id: "todo_other"}, + {...payload, historical_only: false}, + {...payload, result: {lease: {idempotency_key: "out-of-scope"}}}, + {...payload, cursor: "2"}, + ]) assert.throws(() => decodeRemoteAuthorityResponse(wrap(invalid), request, "recover")); +}); + test("invalid policy cannot silently broaden or retain grants", async t => { for (const mutate of [(p: ReturnType) => p.principals.push(p.principals[0]), (p: ReturnType) => p.scopes.push(p.scopes[0]), diff --git a/tests/control_plane_ts/remote_authority_fixture.ts b/tests/control_plane_ts/remote_authority_fixture.ts index 93125f5136..892ebbf116 100644 --- a/tests/control_plane_ts/remote_authority_fixture.ts +++ b/tests/control_plane_ts/remote_authority_fixture.ts @@ -88,6 +88,7 @@ export async function postgresRemoteFixture(t: TestContext, connectionString: st await setup.query(`GRANT CONNECT ON DATABASE ${name} TO ${role}`); await setup.query(`GRANT USAGE ON SCHEMA loopx_control_plane TO ${role}`); await setup.query(`GRANT SELECT ON loopx_control_plane.authority_store_metadata TO ${role}`); + await setup.query(`GRANT SELECT ON loopx_control_plane.authority_checkpoints TO ${role}`); await setup.query(`GRANT SELECT, INSERT, UPDATE ON loopx_control_plane.authority_heads TO ${role}`); await setup.query(`GRANT SELECT, INSERT ON loopx_control_plane.authority_commits, loopx_control_plane.authority_events, loopx_control_plane.authority_receipts TO ${role}`); @@ -102,7 +103,7 @@ export async function postgresRemoteFixture(t: TestContext, connectionString: st }); } const policyPath = join(tls.root, "policy.json"); - async function writePolicy(policy = servicePolicy()) { + async function writePolicy(policy: unknown = servicePolicy()) { await writeFile(`${policyPath}.next`, JSON.stringify(policy), {mode: 0o600}); await rename(`${policyPath}.next`, policyPath); } diff --git a/tests/control_plane_ts/remote_authority_read_acl_conformance.ts b/tests/control_plane_ts/remote_authority_read_acl_conformance.ts new file mode 100644 index 0000000000..fe4260876e --- /dev/null +++ b/tests/control_plane_ts/remote_authority_read_acl_conformance.ts @@ -0,0 +1,277 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import {createServer} from "node:https"; +import type {AddressInfo} from "node:net"; +import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; +import {transitionGoalAcceptanceLifecycle} from "../../loopx/control_plane/goals/acceptance_authority.ts"; +import type {GoalAcceptanceLifecycleTransition} from "../../loopx/control_plane/goals/acceptance_lifecycle.ts"; +import {prepareCoordinationProjectionCommit} from "../../loopx/control_plane/coordination/coordination_projection.ts"; +import {TODO_DOMAIN_ITEM_SCHEMA} from "../../loopx/control_plane/coordination/coordination_state_contract.ts"; +import {executeCoordinationTodoCreate} from "../../loopx/control_plane/coordination/todo_create.ts"; +import {REMOTE_AUTHORITY_REQUEST, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE, + remoteAuthorityOperationId, remoteAuthoritySnapshotDigest} from "../../loopx/control_plane/coordination/remote_authority_contract.ts"; +import {REMOTE_AUTHORITY_SCOPED_READ_POLICY} from "../../loopx/control_plane/coordination/remote_authority_policy.ts"; +import {claimRequest, postgresRemoteFixture, serviceIdentity, servicePolicy} from "./remote_authority_fixture.ts"; + +export function registerRemoteAuthorityReadAclConformance(connectionString: string | undefined) { + test("remote read ACL pins the Goal lifetime and projects only the caller's resources", {skip: !connectionString}, async t => { + const fixture = await postgresRemoteFixture(t, connectionString!); + const goal_id = "goal-one"; + const a = {goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}; + const b = {goal_id, goal_instance_id: "ginst_" + "b".repeat(32)}; + const {scopes, principals} = servicePolicy(); + const policy = {schema_version: REMOTE_AUTHORITY_SCOPED_READ_POLICY, + scopes: scopes.map(scope => ({...scope, todo_read_acl: [ + {goal_ref: a, todo_id: "todo_visible", principal_id: "principal-a", actor_id: "actor-a", + todo_origin_operation_id: "bind-first-goal"}, + {goal_ref: b, todo_id: "todo_next", principal_id: "principal-b", actor_id: "actor-b", + todo_origin_operation_id: "create-successor-todo"}, + // A new grant must not relabel retained predecessor resources. + {goal_ref: b, todo_id: "todo_visible", principal_id: "principal-b", actor_id: "actor-b", + todo_origin_operation_id: "bind-first-goal"}, + ]})), + principals: principals.map((principal, index) => ({...principal, grants: principal.grants.map(grant => ({ + ...grant, goal_ref: index === 0 ? a : b, operations: ["read", "snapshot"], + }))})), + }; + await fixture.seed(["todo_visible", "todo_hidden"], {todo_visible: "unqualified source text", todo_hidden: "private hidden note"}); + const legacy = await fixture.start(); + const originalClaim = claimRequest("a", "todo_visible"); + assert.equal((await fixture.client(legacy.endpoint, "a", originalClaim)).code, 0); + assert.equal((await fixture.client(legacy.endpoint, "b", claimRequest("b", "todo_hidden"))).code, 0); + await legacy.stop(); + await fixture.writePolicy(policy); + const service = await fixture.start(); + const scope = (actor: "a" | "b") => ({schema_version: REMOTE_AUTHORITY_REQUEST, goal_id, + tenant_id: "tenant-one", actor_id: `actor-${actor}`, store_identity: serviceIdentity} as const); + const snapshot = (actor: "a" | "b", todo_id?: string) => + fixture.client(service.endpoint, actor, {...scope(actor), operation: "snapshot", ...(todo_id ? {todo_id} : {})}); + const read = (actor: "a" | "b", todo_id: string) => + fixture.client(service.endpoint, actor, {...scope(actor), operation: "read", todo_id}); + function denied(response: Awaited>) { + assert.equal(response.code, 1); + assert.equal(response.value.status, "rejected"); + assert.equal(response.value.reason_code, "remote_authority_scope_denied"); + assert.deepEqual(Object.keys(response.value).sort(), ["reason_code", "request_sha256", "schema_version", "status"]); + } + async function transition(operation_id: string, step: GoalAcceptanceLifecycleTransition, expected = "applied") { + const result = await transitionGoalAcceptanceLifecycle(fixture.store, { + goal_id, actor_agent_id: null, operation_id, transition: step, + }); + assert.equal(result.status, expected); + } + + const unbound = await fixture.store.loadAuthority(); + denied(await read("a", "todo_visible")); + denied(await snapshot("a")); + assert.deepEqual(await fixture.store.loadAuthority(), unbound); + await transition("bind-first-goal", {kind: "bind_existing", goal_ref: a}); + const head = await fixture.store.loadAuthority(); + assert.equal(head.status, "loaded"); + if (head.status !== "loaded") throw new Error("Missing fixture head"); + + const visible = await read("a", "todo_visible"); + assert.equal(visible.code, 0); + assert.equal(visible.value.schema_version, REMOTE_AUTHORITY_SCOPED_READ_RESPONSE); + assert.deepEqual(visible.value.goal_ref, a); + assert.deepEqual(visible.value.todo, {todo_id: "todo_visible", status: "open", claimed_by: "actor-a"}); + assert.equal((visible.value.lease as JsonObject).owner, "actor-a"); + assert(!("idempotency_key" in (visible.value.lease as JsonObject))); + const first = await snapshot("a"); + assert.equal(first.code, 0); + assert.deepEqual(first.value.goal_ref, a); + for (const field of ["todos", "leases"]) { + assert.deepEqual((first.value[field] as JsonObject[]).map(row => row.todo_id), ["todo_visible"]); + } + assert(!Object.hasOwn(first.value, "provider_revision")); + assert(!JSON.stringify(first.value).includes("unqualified source text")); + await t.test("TLS CLI rejects nested disclosure even with correlation and a recomputed snapshot digest", async () => { + let reply: JsonObject = visible.value; + const proxy = createServer({key: fixture.key, cert: fixture.cert}, (incoming, outgoing) => { + incoming.resume(); outgoing.setHeader("Content-Type", "application/json"); outgoing.end(JSON.stringify(reply)); + }); + await new Promise(resolve => proxy.listen(0, "127.0.0.1", resolve)); + try { + const endpoint = `https://127.0.0.1:${(proxy.address() as AddressInfo).port}`; + const request = {...scope("a"), operation: "read" as const, todo_id: "todo_visible"}; + assert.equal((await fixture.client(endpoint, "a", request)).code, 0); + for (const [record, field] of [["todo", "note"], ["lease", "idempotency_key"]]) { + reply = {...visible.value, [record!]: {...visible.value[record!] as JsonObject, [field!]: "private-response-marker"}}; + const rejected = await fixture.client(endpoint, "a", request); + assert.equal(rejected.code, 1); assert.equal(rejected.value.status, "unavailable"); + assert(!JSON.stringify(rejected.value).includes("private-response-marker")); + } + const snapshotRequest = {...scope("a"), operation: "snapshot" as const}; + for (const [collection, field] of [["todos", "note"], ["leases", "idempotency_key"]]) { + reply = structuredClone(first.value); + (reply[collection!] as JsonObject[])[0]![field!] = "private-response-marker"; + reply.receipt_sha256 = remoteAuthoritySnapshotDigest(snapshotRequest, undefined, + reply.todos as JsonObject[], reply.leases as JsonObject[], a); + const rejected = await fixture.client(endpoint, "a", snapshotRequest); + assert.equal(rejected.code, 1); assert.equal(rejected.value.status, "unavailable"); + assert(!JSON.stringify(rejected.value).includes("private-response-marker")); + } + reply = first.value; + assert.equal((await fixture.client(endpoint, "a", snapshotRequest)).code, 0); + } finally { await new Promise(resolve => proxy.close(() => resolve())); } + }); + for (const origin of [undefined, "missing-origin-operation", remoteAuthorityOperationId(originalClaim, "principal-a")]) { + await fixture.writePolicy({...policy, scopes: policy.scopes.map(s => ({...s, + todo_read_acl: s.todo_read_acl.map(entry => ({...entry, todo_origin_operation_id: origin}))}))}); + denied(await read("a", "todo_visible")); + denied(await snapshot("a")); + } + await fixture.writePolicy(policy); + denied(await read("a", "todo_hidden")); + denied(await read("a", "todo_missing")); + denied(await snapshot("a", "todo_hidden")); + denied(await snapshot("b")); + denied(await fixture.client(service.endpoint, "a", {...scope("b"), operation: "snapshot"})); + denied(await fixture.client(service.endpoint, "a", originalClaim)); + denied(await fixture.client(service.endpoint, "a", originalClaim, true)); + assert.deepEqual(await fixture.store.loadAuthority(), head); + + const hidden = (head.head.todos as JsonObject[]).find(todo => todo.todo_id === "todo_hidden")!; + assert.equal((await fixture.store.commitAuthority(prepareCoordinationProjectionCommit({ + goal_id, projection: head.head, expected_provider_revision: head.provider_revision, operation_id: "change-hidden-todo", + mutations: [{kind: "todo_upsert", todo: {...hidden, note: "different private hidden note"}}], + }))).status, "applied"); + assert.deepEqual((await snapshot("a")).value, first.value); + + await fixture.writePolicy({...policy, scopes: policy.scopes.map(s => ({...s, + todo_read_acl: s.todo_read_acl.filter(entry => entry.principal_id !== "principal-a")}))}); + denied(await snapshot("a")); + await fixture.writePolicy(servicePolicy()); + const downgrade = await snapshot("a"); + assert.equal(downgrade.value.status, "unavailable"); + assert.equal(downgrade.value.reason_code, "remote_authority_policy_unavailable"); + await fixture.writePolicy(policy); + assert.deepEqual((await snapshot("a")).value, first.value); + + await transition("retire-first-goal", {kind: "retire", goal_ref: a}); + denied(await snapshot("a")); + await transition("activate-second-goal", {kind: "activate_successor", retired_goal_ref: a, goal_ref: b}); + const added = await executeCoordinationTodoCreate(fixture.store, { + goal_id, actor_agent_id: "actor-b", registered_agents: ["actor-a", "actor-b"], + operation_id: "create-successor-todo", dry_run: false, now: new Date(), + todo: {schema_version: TODO_DOMAIN_ITEM_SCHEMA, todo_id: "todo_next", role: "agent", + status: "open", done: false, archive_state: "active", text: "Second lifetime task"}, + }); + assert.equal(added.status, "applied"); + const successor = await fixture.store.loadAuthority(); + denied(await snapshot("a")); + denied(await read("a", "todo_visible")); + denied(await read("a", "todo_next")); + denied(await read("b", "todo_visible")); + const second = await snapshot("b"); + assert.equal(second.code, 0); + assert.deepEqual(second.value.goal_ref, b); + assert.deepEqual((second.value.todos as JsonObject[]).map(todo => todo.todo_id), ["todo_next"]); + assert.deepEqual(second.value.leases, []); + assert.equal((await read("b", "todo_next")).code, 0); + assert.deepEqual(await fixture.store.loadAuthority(), successor); + await transition("bind-current-again", {kind: "bind_existing", goal_ref: b}, "no_change"); + for (const origin of ["activate-second-goal", "bind-current-again", "create-successor-todo"]) { + await fixture.writePolicy({...policy, scopes: policy.scopes.map(s => ({...s, + todo_read_acl: s.todo_read_acl.map(entry => entry.goal_ref === b && entry.todo_id === "todo_visible" + ? {...entry, todo_origin_operation_id: origin} : entry)}))}); + denied(await read("b", "todo_visible")); + assert.deepEqual(((await snapshot("b")).value.todos as JsonObject[]).map(todo => todo.todo_id), ["todo_next"]); + } + }); + + test("scoped receipt recovery proves the original Goal lifetime and current exact operation access", {skip: !connectionString}, async t => { + const fixture = await postgresRemoteFixture(t, connectionString!); + const goal_id = "goal-one"; + const a = {goal_id, goal_instance_id: "ginst_" + "a".repeat(32)}; + const b = {goal_id, goal_instance_id: "ginst_" + "b".repeat(32)}; + const unboundClaim = claimRequest("a", "todo_legacy"); + const boundClaim = claimRequest("a", "todo_bound"); + const missingClaim = claimRequest("a", "todo_bound", "never-committed"); + const id = (request: typeof boundClaim) => remoteAuthorityOperationId(request, "principal-a"); + async function transition(operation_id: string, step: GoalAcceptanceLifecycleTransition) { + const result = await transitionGoalAcceptanceLifecycle(fixture.store, {goal_id, actor_agent_id: null, + operation_id, transition: step}); + assert.equal(result.status, "applied"); + } + + await fixture.seed(["todo_legacy", "todo_bound", "todo_hidden"]); + const legacy = await fixture.start(); + assert.equal((await fixture.client(legacy.endpoint, "a", unboundClaim)).code, 0); + await transition("bind-receipt-goal", {kind: "bind_existing", goal_ref: a}); + assert.equal((await fixture.client(legacy.endpoint, "a", boundClaim)).code, 0); + const original = await fixture.client(legacy.endpoint, "a", boundClaim, true); + assert.equal(original.code, 0); + assert("cursor" in original.value); + assert("lease" in (original.value.result as JsonObject)); + await legacy.stop(); + + // Read an earlier receipt after the authority has crossed a checkpoint. + // This uses the normal journal and keeps the original receipt intact. + for (let sequence = 0; sequence < 64; sequence++) { + const head = await fixture.store.loadAuthority(); + if (head.status !== "loaded") throw new Error("Missing fixture head"); + const hidden = (head.head.todos as JsonObject[]).find(todo => todo.todo_id === "todo_hidden")!; + const result = await fixture.store.commitAuthority(prepareCoordinationProjectionCommit({ + goal_id, projection: head.head, expected_provider_revision: head.provider_revision, + operation_id: `append-receipt-fixture-${sequence}`, + mutations: [{kind: "todo_upsert", todo: {...hidden, note: `Hidden fixture update ${sequence}`}}], + })); + assert.equal(result.status, "applied"); + } + function policy(goalRef: typeof a, allowed: readonly string[], operations = ["read", "snapshot"]) { + const {scopes, principals} = servicePolicy(); + return {schema_version: REMOTE_AUTHORITY_SCOPED_READ_POLICY, + scopes: scopes.map(scope => ({...scope, todo_read_acl: [ + {goal_ref: goalRef, todo_id: "todo_bound", principal_id: "principal-a", actor_id: "actor-a", + todo_origin_operation_id: "bind-receipt-goal", receipt_operation_ids: allowed}, + {goal_ref: goalRef, todo_id: "todo_legacy", principal_id: "principal-a", actor_id: "actor-a", + receipt_operation_ids: [id(unboundClaim), id(boundClaim)]}, + ]})), + principals: principals.slice(0, 1).map(principal => ({...principal, grants: principal.grants.map(grant => + ({...grant, goal_ref: goalRef, operations}))})), + }; + } + await fixture.writePolicy(policy(a, [id(boundClaim), id(missingClaim)])); + const service = await fixture.start(); + const recover = (request: typeof boundClaim) => fixture.client(service.endpoint, "a", request, true); + function denied(result: Awaited>) { + assert.equal(result.code, 1); + assert.equal(result.value.status, "rejected"); + assert.equal(result.value.reason_code, "remote_authority_scope_denied"); + assert.deepEqual(Object.keys(result.value).sort(), ["reason_code", "request_sha256", "schema_version", "status"]); + } + const current = await fixture.store.loadAuthority(); + const allowed = await recover(boundClaim); + assert.equal(allowed.code, 0); + assert.deepEqual(allowed.value, { + schema_version: REMOTE_AUTHORITY_SCOPED_READ_RESPONSE, status: "found", historical_only: true, + todo_id: boundClaim.todo_id, goal_ref: a, operation_id: original.value.operation_id, + receipt_sha256: original.value.receipt_sha256, request_sha256: original.value.request_sha256, + }); + denied(await recover(unboundClaim)); + denied(await recover(missingClaim)); + denied(await recover({...boundClaim, todo_id: unboundClaim.todo_id})); + denied(await fixture.client(service.endpoint, "a", boundClaim)); // Execute stays outside the read grant. + await fixture.writePolicy(policy(a, [])); + denied(await recover(boundClaim)); + const currentTodo = await fixture.client(service.endpoint, "a", { + operation: "read", schema_version: REMOTE_AUTHORITY_REQUEST, tenant_id: boundClaim.tenant_id, + goal_id, actor_id: boundClaim.actor_id, store_identity: serviceIdentity, todo_id: boundClaim.todo_id, + }); + assert.equal(currentTodo.code, 0); + await fixture.writePolicy(policy(a, [id(boundClaim)], ["snapshot"])); + denied(await recover(boundClaim)); + assert.deepEqual(await fixture.store.loadAuthority(), current); + + await fixture.writePolicy(policy(a, [id(boundClaim)])); + await transition("retire-receipt-goal", {kind: "retire", goal_ref: a}); + denied(await recover(boundClaim)); + await transition("activate-next-receipt-goal", {kind: "activate_successor", retired_goal_ref: a, goal_ref: b}); + const successor = await fixture.store.loadAuthority(); + denied(await recover(boundClaim)); // Stale credential. + await fixture.writePolicy(policy(b, [id(boundClaim)])); + denied(await recover(boundClaim)); // Current ACL cannot reassign a receipt from A to B. + assert.deepEqual(await fixture.store.loadAuthority(), successor); + }); +} diff --git a/tests/test_global_todos.py b/tests/test_global_todos.py index 1f2b578a63..a6d3c6bb6a 100644 --- a/tests/test_global_todos.py +++ b/tests/test_global_todos.py @@ -1,7 +1,13 @@ from pathlib import Path from typing import Any +import pytest + import loopx.global_todos as global_todos +from loopx.control_plane.goals.contract_health import ( + contract_error_diagnostic, + contract_error_views, +) def build_payload( @@ -137,6 +143,32 @@ def test_global_todos_error_redacts_local_paths(tmp_path) -> None: assert "" in payload["error"] +@pytest.mark.parametrize("failure", ["global_contract", "unknown_goal", "registry"]) +def test_global_todos_keeps_unverified_scope_unavailable(monkeypatch, tmp_path, failure) -> None: + diagnostic = contract_error_diagnostic(code="canonical_authority_required", + message="Unavailable canonical source.", goal_id="unknown" if failure == "unknown_goal" else "cold") + diagnostics = [diagnostic] + if failure == "global_contract": + diagnostics.append(contract_error_diagnostic(code="registry_contract", message="Global contract failed.")) + status = { + **status_for("ready"), + "ok": False, + "global_registry": {"available": True, "ok": failure != "registry"}, + "contract": {"ok": False, **contract_error_views(diagnostics)}, + "run_history": {"goals": [ + {"id": "ready", "registry_member": True}, + {"id": "cold", "registry_member": True}, + ]}, + } + calls = patch_manager_reads(monkeypatch, status=status, quotas={}) + + payload = build_payload(tmp_path) + + assert payload["ok"] is False + assert "todos" not in payload + assert calls == {"status": 1, "quota": []} + + def test_global_todos_error_survives_projection_runtime_failure(monkeypatch) -> None: monkeypatch.setattr( global_todos, diff --git a/tests/test_lark_private_conversations.py b/tests/test_lark_private_conversations.py index 393ee609af..bef4c20103 100644 --- a/tests/test_lark_private_conversations.py +++ b/tests/test_lark_private_conversations.py @@ -4040,6 +4040,108 @@ def test_app_projection_refuses_a_replaced_provider_identity(ordinary): # noqa: runtime.close() +@pytest.mark.parametrize("fault", ["revoked", "provider_changed", "body_changed"]) +def test_app_projection_checks_current_authority_after_reading_original_source(ordinary, monkeypatch, fault): # noqa: F811 + store, runtime, provider, transport, original, turns, before = _app_projection_journey(ordinary) + sid = original["session_id"] + transcript = store.sessions_root / sid / "messages.jsonl" + request_path = transport.root / f"{original['request_ref']}.json" + source_id = json.loads(request_path.read_text())["event"]["message_id"] + call = Provider.__call__ + previewed = changed = False + + def change_after_source_read(self, args, *pos, **kwargs): + nonlocal previewed, changed + result = call(self, args, *pos, **kwargs) + if "+messages-send" in args and "--dry-run" in args: + previewed = True + elif (previewed and not changed and "+messages-mget" in args + and args[args.index("--message-ids") + 1] == source_id): + if fault == "revoked": + transport.bindings.disconnect(original["binding_id"], expected_revision=transport.bindings.read()["revision"]) + elif fault == "provider_changed": + provider.profile_apps["notes-app"] = "cli_replaced_app" + else: + messages = store.messages(sid) + next(message for message in messages if message.get("turn_id") == turns[0])["text"] = "Changed during source read" + transcript.write_text("".join(json.dumps(message) + "\n" for message in messages)) + changed = True + return result + + try: + starts = ordinary[4].read_bytes() + monkeypatch.setattr(Provider, "__call__", change_after_source_read) + transport._web_projection_scan_at = 0 + path, = transport._web_projection_paths() + assert transport.reconcile_request(path) == 0 + assert changed and len(provider.writes) == before + assert ordinary[4].read_bytes() == starts + assert not any(phase.get("started") for phase in transport._read_web_projection_journal(path)["deliveries"].values()) + finally: + runtime.close() + + +@pytest.mark.parametrize("lock_at", ["read", "writeback"]) +def test_busy_app_body_does_not_delay_feedback_for_another_session(ordinary, monkeypatch, lock_at): # noqa: F811 + from contextlib import ExitStack + + from loopx.file_lock import exclusive_file_lock, lock_incident_path + + _store, runtime, provider, transport = connect(ordinary) + try: + for profile in ("notes-app", "steward-app"): + event = provider.event(profile, f"feedback-source-{profile}", "Admit the original private conversation") + transport.admit(profile, event) + request = transport.core.read_request(identity_ref(transport._binding(profile)["provider_ref"], event["message_id"])) + sid = request["session_id"] + assert runtime.wait_for_turn(session_id=sid, turn_id=request["turn_id"], timeout_sec=10)["status"] == "completed" + transport.reconcile_request(transport.root / f"{request['request_ref']}.json") + status, turn = ordinary[3](f"/api/chat/sessions/{sid}/turns", {"message": "New App input", "client_turn_id": "feedback"}) + assert status == 202 + assert runtime.wait_for_turn(session_id=sid, turn_id=turn["turn_id"], timeout_sec=10)["status"] == "completed" + + transport._web_projection_scan_at = 0 + busy, ready = sorted(transport._web_projection_paths()) + targets = {} + for path in (busy, ready): + assert transport.reconcile_request(path) == 1 + row = transport._read_web_projection_journal(path) + phase, = row["deliveries"].values() + assert phase["verified"] and phase["feedback"] == {} + targets[path] = (row["profile"], phase["attempt"]["message_ref"], "Get") + + with ExitStack() as body_worker, monkeypatch.context() as patch: + body_lock = exclusive_file_lock(busy, operation="deliver_web_conversation_projection") + if lock_at == "read": + body_worker.enter_context(body_lock) + else: + feedback = transport._web_projection_feedback + busy_session = transport._read_web_projection_journal(busy)["session_id"] + + def hold_body_after_feedback(row, message): + feedback(row, message) + if row["session_id"] == busy_session: + assert targets[busy] in provider.reaction_creates + body_worker.enter_context(body_lock) + + patch.setattr(transport, "_web_projection_feedback", hold_body_after_feedback) + + transport.reconcile_app_feedback() + assert not lock_incident_path(busy).exists() + busy_phase, = transport._read_web_projection_journal(busy)["deliveries"].values() + assert busy_phase["feedback"] == {} + assert (targets[busy] in provider.reaction_creates) == (lock_at == "writeback") + assert targets[ready] in provider.reaction_creates + ready_phase, = transport._read_web_projection_journal(ready)["deliveries"].values() + assert ready_phase["feedback"]["ok"] + transport.reconcile_app_feedback() + assert all(provider.reaction_creates.count(target) == 1 for target in targets.values()) + busy_phase, = transport._read_web_projection_journal(busy)["deliveries"].values() + assert busy_phase["feedback"]["ok"] + finally: + runtime.close() + + @pytest.mark.parametrize("fault", ["revoked", "app", "source", "session", "turn", "context_recovery"]) def test_app_feedback_rechecks_exact_source_and_turn_after_mirror_delivery(ordinary, monkeypatch, fault): # noqa: F811 store, runtime, provider, transport, original, turns, before = _app_projection_journey(ordinary) diff --git a/tests/test_manager_context_roundtrip.py b/tests/test_manager_context_roundtrip.py index 2a396f4c1c..60ab6e7d72 100644 --- a/tests/test_manager_context_roundtrip.py +++ b/tests/test_manager_context_roundtrip.py @@ -136,6 +136,34 @@ def test_frozen_private_reply_projects_original_text_without_mutating_model_inpu assert {str(p): p.read_bytes() for p in store.root.rglob("*") if p.is_file()} == before +def test_saved_answer_reads_only_its_original_question_and_exact_answer(frozen_private_reply, private_snapshot): + store, session, turn, _, _, _, source_text, _ = frozen_private_reply + sid, tid = session["session_id"], turn["turn_id"] + answer = store.append_message(sid, role="agent", text="The saved answer", turn_id=tid) + store.append_message(sid, role="agent", text="Later answer from the same Turn", turn_id=tid) + no_turn = store.append_message(sid, role="agent", text="An older saved answer without a Turn") + other_session = store.create_session(goal_id="notes", agent_id="codex", adapter_kind="codex_app_server", + upstream_thread_id="another-conversation") + foreign = store.append_message(other_session["session_id"], role="agent", text="An answer from another Session") + journal = store.sessions_root / sid / "messages.jsonl" + before = journal.read_bytes() + + response = private_snapshot(report_message_id=answer["message_id"]) + assert response["session"]["session_id"] == sid + assert response["report_message_id"] == answer["message_id"] + assert response["active_turn"] is None + question, saved = response["messages"] + assert question["reply_context"]["source_text"] == source_text + assert saved == answer + assert private_snapshot(report_message_id=answer["message_id"], turn_id=tid) == response + assert private_snapshot(report_message_id=answer["message_id"], turn_id="unrelated-turn")["messages"] == [] + assert private_snapshot(report_message_id=no_turn["message_id"])["messages"] == [no_turn] + for unavailable in (foreign["message_id"], question["message_id"]): + assert private_snapshot(report_message_id=unavailable)["messages"] == [] + private_snapshot(report_message_id=answer["message_id"], message_limit=200, expected_status=400) + assert journal.read_bytes() == before + + def test_history_pages_reuse_unchanged_journal_but_reread_provider_receipts( frozen_private_reply, private_snapshot, monkeypatch): store, session, turn, request, *_ = frozen_private_reply