Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions loopx/cli_commands/quota_scheduler_followup.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@ def build_scheduler_followup_payload(
failed_rrule=args.failed_rrule,
observed_host_rrule=observed_rrule,
failure_kind=args.failure_kind,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
)
return record_quota_scheduler_ack(
status_payload,
Expand All @@ -248,4 +250,6 @@ def build_scheduler_followup_payload(
scheduler_execution_context=scheduler_context,
operator_inbox_urgency_projector=operator_inbox_urgency_projector,
before_decision=before_decision,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
)
26 changes: 23 additions & 3 deletions loopx/control_plane/quota/scheduler_ack.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import json
import subprocess
from collections.abc import Mapping
from pathlib import Path

from ..effect_runtime import _node_executable
Expand Down Expand Up @@ -184,14 +185,25 @@ def _run_native_followup(
host_facts: dict[str, Any],
use_current_hint: bool,
reason_summary: str | None,
turn_instance_id: str | None = None,
goal_ref: Mapping[str, object] | None = None,
) -> dict[str, Any]:
"""Decision-free compatibility transport for in-process Python callers."""
"""Decision-free compatibility transport for Python callers.

A caller that already bound the follow-up to a Turn passes it here, so the
native transaction applies the same heartbeat-receipt fence (missing or
superseded) as the generated route. The exact GoalRef this caller resolved
travels with it, so the native successor scan compares receipts inside the
same instance instead of across every goal that shares the log. Callers
without a Turn keep the receipt-less compatibility request.
"""

request = {
"schema_version": _FOLLOWUP_REQUEST_SCHEMA,
"runtime_root": str(runtime_root),
"turn_instance_id": None,
"require_heartbeat_receipt": False,
"turn_instance_id": turn_instance_id,
"goal_ref": dict(goal_ref) if goal_ref is not None else None,
"require_heartbeat_receipt": turn_instance_id is not None,
"before": before,
"host_facts": host_facts,
"use_current_hint": use_current_hint,
Expand Down Expand Up @@ -298,6 +310,8 @@ def record_quota_scheduler_ack_for_decision(
generated_at: str | None = None,
use_current_hint: bool = False,
host_match_observed: bool = False,
turn_instance_id: str | None = None,
goal_ref: Mapping[str, object] | None = None,
) -> dict[str, Any]:
state_key = _followup_state_key(before, surface=surface, state_key=state_key)
safe_agent_id = normalize_todo_claimed_by(agent_id)
Expand Down Expand Up @@ -367,6 +381,8 @@ def record_quota_scheduler_ack_for_decision(
host_facts=facts,
use_current_hint=use_current_hint,
reason_summary=reason_summary,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
)
except ValueError as exc:
return _failure_payload(
Expand Down Expand Up @@ -396,6 +412,8 @@ def record_quota_scheduler_failure_for_decision(
observed_host_rrule: str | None = None,
failure_kind: str = "host_tool_failure",
generated_at: str | None = None,
turn_instance_id: str | None = None,
goal_ref: Mapping[str, object] | None = None,
) -> dict[str, Any]:
state_key = _followup_state_key(before, surface=surface, state_key=state_key)
safe_agent_id = normalize_todo_claimed_by(agent_id)
Expand Down Expand Up @@ -443,6 +461,8 @@ def record_quota_scheduler_failure_for_decision(
host_facts=facts,
use_current_hint=False,
reason_summary=None,
turn_instance_id=turn_instance_id,
goal_ref=goal_ref,
)
except ValueError as exc:
return _failure_payload(
Expand Down
50 changes: 46 additions & 4 deletions loopx/control_plane/quota/source_admission.ts
Original file line number Diff line number Diff line change
Expand Up @@ -113,14 +113,56 @@ export function requireQuotaOwnerProjection(
}
}

export function quotaOwnerOwnsProjection(
owner: QuotaAccountingOwner,
/**
* The Goal-instance scope one persisted fact was written under.
*
* `alias` is the legacy projection: the fact carries no `goal_ref` at all.
* Anything else is the one Goal instance the writer presented, so two goals
* that share a display name stay separate even inside one shared log.
*/
export type QuotaProjectionScope =
| Readonly<{ kind: "alias" }>
| Readonly<{ kind: "exact_source"; goalRef: ExactGoalRef }>;

/**
* Read one fact's scope, or null when its `goal_ref` is not a usable scope.
*
* Absent and unparseable refs are different facts: absent is the legacy
* projection every pre-instance writer produced, while a present-but-unusable
* ref belongs to no scope at all and must not be compared loosely.
*/
export function quotaProjectionScope(
value: unknown,
): QuotaProjectionScope | null {
if (value === undefined) return { kind: "alias" };
const projected = parseExactGoalRef(value);
return projected.kind === "parsed"
? { kind: "exact_source", goalRef: projected.value }
: null;
}

/**
* One judge for "did this fact belong to that scope".
*
* Legacy facts only compare with legacy facts and an exact GoalRef only
* compares with that same instance, so a caller cannot widen a scope by
* asking a looser question than the writer answered.
*/
export function quotaScopeOwnsProjection(
scope: QuotaProjectionScope,
value: unknown,
): boolean {
if (owner.kind === "alias") return value === undefined;
if (scope.kind === "alias") return value === undefined;
const projected = parseExactGoalRef(value);
return projected.kind === "parsed"
&& sameGoalRef(projected.value, owner.goalRef);
&& sameGoalRef(projected.value, scope.goalRef);
}

export function quotaOwnerOwnsProjection(
owner: QuotaAccountingOwner,
value: unknown,
): boolean {
return quotaScopeOwnsProjection(owner, value);
}

function quotaSourceLock(
Expand Down
126 changes: 100 additions & 26 deletions loopx/control_plane/rollout_receipt_log.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ import { relative, resolve, sep } from "node:path";

import type { JsonObject } from "./effect_program.ts";
import { EffectRuntimeRequestError } from "./effect_runtime_errors.ts";
import {
quotaProjectionScope,
quotaScopeOwnsProjection,
type QuotaProjectionScope,
} from "./quota/source_admission.ts";
import { requireNonEmptyString } from "./runtime_decode.ts";
import { readReceiptLogSnapshot } from "./runtime/receipt_log_snapshot.ts";

Expand Down Expand Up @@ -118,19 +123,32 @@ export function goalHeartbeatReceiptsFromSnapshot(
return heartbeatReceiptsFromEvents(snapshot.events, goalId, agentId);
}

/**
* One receipt predicate, shared by discovery and by successor scans.
*
* A failed invocation without a real Turn identity is an audit: the host
* protocol keeps it visible in the log, but lets it neither grant host
* writeback nor expire the receipt of the Turn it follows.
*/
export function isHeartbeatReceiptEvent(
event: JsonObject,
goalId: string,
agentId?: string | null,
): boolean {
return event.event_kind === HEARTBEAT_RECEIPT_EVENT_KIND &&
event.goal_id === goalId &&
typeof event.run_id === "string" && event.run_id.trim().length > 0 &&
(agentId === undefined || event.agent_id === agentId);
}

/** The heartbeat receipt filter over events a caller already read. */
export function heartbeatReceiptsFromEvents(
events: readonly JsonObject[],
goalId: string,
agentId?: string | null,
): JsonObject[] {
return events.filter((event) =>
event.event_kind === HEARTBEAT_RECEIPT_EVENT_KIND &&
event.goal_id === goalId &&
// A failed invocation without a real Turn identity is an audit, not a
// receipt that can grant host writeback or supersede an admitted Turn.
typeof event.run_id === "string" && event.run_id.trim().length > 0 &&
(agentId === undefined || event.agent_id === agentId)
isHeartbeatReceiptEvent(event, goalId, agentId)
);
}

Expand Down Expand Up @@ -159,7 +177,9 @@ export async function heartbeatReceiptStatus(params: {
goalId: string;
agentId: string;
turnInstanceId: string;
goalRef?: unknown;
}): Promise<HeartbeatReceiptStatus> {
const requestedScope = requestedReceiptScope(params.goalId, params.goalRef);
const snapshot = await readGoalRolloutEventSnapshot(
params.runtimeRoot,
params.goalId,
Expand All @@ -170,28 +190,82 @@ export async function heartbeatReceiptStatus(params: {
params.agentId,
);
if (snapshot === null || receipts === null) return "missing";
const firstReceipt = receipts.find(
(event) => event.run_id === params.turnInstanceId,
const firstReceiptMatches = receipts.filter((event) =>
event.run_id === params.turnInstanceId &&
(requestedScope === null ||
quotaScopeOwnsProjection(requestedScope, event.goal_ref))
);
const firstMatch = firstReceipt === undefined
? -1
: snapshot.events.indexOf(firstReceipt);
if (firstReceiptMatches.length === 0) return "missing";
// A command generated before the origin scope travelled with its facts cannot
// say which instance its Turn belonged to. Two Goal instances may share one
// Turn id inside a log they both write, and taking whichever row landed first
// would scan the wrong instance's successors and let a stale writeback
// through, so the ambiguity is refused instead of guessed. The host re-reads
// a scoped hint and retries with the origin receipt's own scope.
if (
requestedScope === null &&
new Set(firstReceiptMatches.map((event) => receiptScopeKey(event.goal_ref)))
.size > 1
) {
throw new EffectRuntimeRequestError(
"heartbeat receipt Turn identity matches more than one Goal instance",
"heartbeat_receipt_ambiguous",
);
}
const firstReceipt = firstReceiptMatches[0];
const firstMatch = snapshot.events.indexOf(firstReceipt);
if (firstMatch < 0) return "missing";
return snapshot.events.slice(firstMatch + 1).some((event) => {
if (
event.event_kind !== HEARTBEAT_RECEIPT_EVENT_KIND ||
event.goal_id !== params.goalId ||
event.agent_id !== params.agentId
) return false;
if (typeof event.run_id === "string" && event.run_id.trim().length > 0) {
return event.run_id !== params.turnInstanceId;
}
const details = event.details;
return event.status === "normal_run" &&
typeof details === "object" && details !== null && !Array.isArray(details) &&
(details as JsonObject).ok === true &&
(details as JsonObject).should_run === true;
})
// The receipt itself states the scope its successor scan may use: a legacy
// receipt is only superseded by another legacy receipt, and an instance
// receipt only by that same Goal instance. A differently named Goal that
// happens to share this log therefore cannot expire an admitted Turn, and an
// unscopable `goal_ref` fails closed instead of authorizing a writeback.
const receiptScope = quotaProjectionScope(firstReceipt.goal_ref);
if (receiptScope === null) return "missing";
// Only a real Turn receipt expires an admitted one. Identity-less quota rows
// are diagnostics: the protocol neither grants them host writeback nor lets
// them supersede the receipt they follow, so the successor scan reuses the
// discovery predicate rather than re-deciding what a receipt is.
return snapshot.events.slice(firstMatch + 1).some((event) =>
isHeartbeatReceiptEvent(event, params.goalId, params.agentId) &&

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 另一 Goal 实例的回执会使有效调度回报过期。 先用真实 should-run 准入旧格式 Turn,再在同名 Goal 和同 Agent 的共享日志里加入另一 Goal 实例的有效回执。重试原 ACK 或失败回报。 仅增加一条带其他 GoalRef 的回执,File 与 SQLite 的 ACK 和失败回报均从正常写入变为 heartbeat_receipt_stale。原回执仍在,重试也无法写回。相同条件下基线写入;同范围真实后继应当继续拒绝。 让 Python 把原回执作用域带到原生传输,或由原生从已匹配的原回执推导其作用域。用同一个类型化作用域判断过滤查找与替代,并在提交锁内复验。旧格式只和同类回执比较,精确 GoalRef 只和同一实例比较。

event.run_id !== params.turnInstanceId &&
quotaScopeOwnsProjection(receiptScope, event.goal_ref)
)
? "stale"
: "fresh";
}

/** One comparable key per scope, so ambiguity is decided by set size. */
function receiptScopeKey(value: unknown): string {
const scope = quotaProjectionScope(value);
return scope === null ? "unscopable" : JSON.stringify(scope);
}

/**
* Resolve the scope a caller asked the receipt fence to run in.
*
* Absent means "use the scope the matching receipt states", so the persisted
* receipt stays the only authority. A caller that does know its Goal instance
* passes it, and a malformed or foreign scope is refused rather than silently
* widening the comparison.
*/
function requestedReceiptScope(
goalId: string,
value: unknown,
): QuotaProjectionScope | null {
if (value === undefined || value === null) return null;
const scope = quotaProjectionScope(value);
if (scope === null || scope.kind === "alias") {
throw new EffectRuntimeRequestError(
"heartbeat receipt scope requires goal_id and goal_instance_id",
"heartbeat_receipt_scope_invalid",
);
}
if (scope.goalRef.goalId.value !== goalId) {
throw new EffectRuntimeRequestError(
"heartbeat receipt scope does not belong to the requested goal",
"heartbeat_receipt_scope_mismatch",
);
}
return scope;
}
3 changes: 3 additions & 0 deletions loopx/control_plane/scheduler/heartbeat_commit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ export interface SchedulerHeartbeatCommitResult extends JsonObject {

export interface SchedulerHeartbeatCommitOptions {
readonly heartbeatReceiptTurnInstanceId?: string | null;
/** The Goal instance the caller bound the follow-up to, when it knows it. */
readonly heartbeatReceiptGoalRef?: JsonObject | null;
}

function stableValue(value: unknown): unknown {
Expand Down Expand Up @@ -853,6 +855,7 @@ export async function evaluateSchedulerHeartbeatCommit(
goalId: request.scope.goalId,
agentId: request.scope.agentId,
turnInstanceId: receiptTurnInstanceId,
goalRef: options.heartbeatReceiptGoalRef,
});
if (status !== "fresh") {
return result(
Expand Down
9 changes: 9 additions & 0 deletions loopx/control_plane/scheduler/heartbeat_followup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ export interface SchedulerHeartbeatFollowupRequest extends JsonObject {
schema_version: typeof SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA;
runtime_root: string;
turn_instance_id: string | null;
/** The Goal instance the Python transport resolved for this follow-up. */
goal_ref: JsonObject | null;
require_heartbeat_receipt: boolean;
before: JsonObject;
host_facts: JsonObject;
Expand Down Expand Up @@ -95,6 +97,9 @@ function requestObject(value: unknown): SchedulerHeartbeatFollowupRequest {
schema_version: SCHEDULER_HEARTBEAT_FOLLOWUP_REQUEST_SCHEMA,
runtime_root: runtimeRoot,
turn_instance_id: turnInstanceId,
goal_ref: input.goal_ref === undefined || input.goal_ref === null
? null
: requireJsonObject(input.goal_ref, "goal_ref"),
require_heartbeat_receipt: requireReceipt,
before: requireJsonObject(input.before ?? {}, "before"),
host_facts: hostFacts,
Expand Down Expand Up @@ -341,6 +346,7 @@ export async function evaluateSchedulerHeartbeatFollowup(
goalId: String(facts.goal_id),
agentId: String(facts.agent_id),
turnInstanceId: String(request.turn_instance_id),
goalRef: request.goal_ref,
});
if (status !== "fresh") return receiptFailure(request, status);
}
Expand All @@ -351,6 +357,9 @@ export async function evaluateSchedulerHeartbeatFollowup(
heartbeatReceiptTurnInstanceId: request.require_heartbeat_receipt
? request.turn_instance_id
: null,
heartbeatReceiptGoalRef: request.require_heartbeat_receipt
? request.goal_ref
: null,
});
const receiptStatus = commitReceiptFailureStatus(commit);
if (receiptStatus !== null) return receiptFailure(request, receiptStatus);
Expand Down
9 changes: 8 additions & 1 deletion loopx/control_plane/scheduler/heartbeat_followup_cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import {
effectRuntimeErrorPayload,
EffectRuntimeRequestError,
} from "../effect_runtime_errors.ts";
import { requireJsonObject } from "../runtime_decode.ts";
import { jsonObject, requireJsonObject } from "../runtime_decode.ts";
import { isSupportedNodeVersion } from "../runtime/node_version.ts";
import {
evaluateSchedulerHeartbeatFollowup,
Expand Down Expand Up @@ -214,6 +214,12 @@ function requestFromArgs(argv: string[]): { request: Record<string, unknown>; fo
const facts = requireJsonObject(hint.host_facts, "host_facts");
matchingText(facts, "goal_id", parsed.goal_id, "--goal-id");
matchingText(facts, "agent_id", parsed.agent_id, "--agent-id");
// The origin receipt's scope travels with the facts it was generated for.
// Passing it into the transaction is what keeps the pre-flight check and the
// in-lock recheck on that receipt instead of resolving the bare Turn id again.
const goalRef = jsonObject(facts.goal_ref);

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[P2] 拒绝格式无效的显式 GoalRef。 File 和 SQLite 上,省略字段的兼容回报返回成功。显式格式无效的回报也返回成功,并创建调度状态。结构正确但没有匹配回执时,两种后端均拒绝写入。 按字段是否存在区分旧包和显式引用。让原生入口把显式值交给现有 GoalRef 解码与匹配规则,任何形状错误都在写入前返回结构化错误。

if (goalRef !== null)
matchingText(goalRef, "goal_id", parsed.goal_id, "--goal-id");
const expectedOperation = parsed.command === "scheduler-ack-current"
? "ack"
: "host_failure";
Expand Down Expand Up @@ -291,6 +297,7 @@ function requestFromArgs(argv: string[]): { request: Record<string, unknown>; fo
runtime_root: parsed.runtime_root,
turn_instance_id: parsed.turn_instance_id,
require_heartbeat_receipt: true,
...(goalRef === null ? {} : { goal_ref: goalRef }),
before: hint.before ?? {},
use_current_hint: hint.use_current_hint ?? expectedOperation === "ack",
reason_summary: values["--reason-summary"] ?? null,
Expand Down
Loading
Loading