diff --git a/.github/workflows/package-smoke.yml b/.github/workflows/package-smoke.yml
index d130915e13..72495bf7b7 100644
--- a/.github/workflows/package-smoke.yml
+++ b/.github/workflows/package-smoke.yml
@@ -5,8 +5,10 @@ on:
paths:
- ".github/workflows/package-smoke.yml"
- "packages/loopx-core/**"
+ - "packages/loopx-capability-benchmark-toolkit/**"
- "loopx/**"
- "tests/test_core_distribution.py"
+ - "tests/test_core_benchmark_distribution.py"
- "tests/control_plane/test_installed_core_context.py"
- "packages/loopx-repo-health/**"
- "packages/loopx-obelisk/**"
@@ -19,8 +21,10 @@ on:
paths:
- ".github/workflows/package-smoke.yml"
- "packages/loopx-core/**"
+ - "packages/loopx-capability-benchmark-toolkit/**"
- "loopx/**"
- "tests/test_core_distribution.py"
+ - "tests/test_core_benchmark_distribution.py"
- "tests/control_plane/test_installed_core_context.py"
- "packages/loopx-repo-health/**"
- "packages/loopx-obelisk/**"
@@ -65,6 +69,16 @@ jobs:
uv run --extra test python -m pytest -q
tests/test_core_distribution.py
tests/control_plane/test_installed_core_context.py
+ - name: Qualify matching Benchmark Toolkit installation and recovery
+ env:
+ LOOPX_USAGE_PING: "0"
+ run: |
+ uv run --extra test python packages/loopx-core/prepare.py \
+ --distribution loopx-capability-benchmark-toolkit --output "$RUNNER_TEMP/benchmark-source"
+ uv build --wheel "$RUNNER_TEMP/benchmark-source" --out-dir "$RUNNER_TEMP/benchmark-dist"
+ export LOOPX_CORE_TEST_WHEEL=$(printf '%s' "$RUNNER_TEMP"/core-dist/loopx_core-*.whl)
+ export LOOPX_BENCHMARK_TEST_WHEEL=$(printf '%s' "$RUNNER_TEMP"/benchmark-dist/loopx_capability_benchmark_toolkit-*.whl)
+ uv run --extra test python -m pytest -q tests/test_core_benchmark_distribution.py
codex-provider-routing:
name: loopx-codex-provider-routing
diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md
index 6e5521ac0a..29cb4f79a8 100644
--- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md
+++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md
@@ -4,7 +4,7 @@
- **Delivery maturity:** Proposal
- **Authors / owners:** LoopX maintainers
- **Created:** 2026-09-26
-- **Last normative revision:** 2026-11-17
+- **Last normative revision:** 2026-11-19
- **Implementation baseline:** packaging inventory `2f3d13ae9`; runtime composition audit `9cae3b28939f72fa4aadab5cfe49e19bcbdab372`
- **Related contracts:** [TypeScript Control-Plane Migration v0](typescript-control-plane-migration-v0.md), [Extensions reference](../../reference/extensions.md), [Capability catalog](../../../loopx/capabilities/README.md), [Overall Roadmap v0](loopx-overall-roadmap-v0.md) (S2, S8, S12), [import-boundary tests](../../../tests/architecture/test_control_plane_import_boundaries.py)
- **Tracking issue:** [#5072](https://github.com/loopx-project/loopx/issues/5072)
@@ -35,7 +35,7 @@ retain their original baselines.
wheel is replaced over time by `loopx-core`, `loopx-workspace`, and
per-capability packages under `packages/`, with `loopx` retained as a
meta-package that depends on all of them.
-3. **Top-level regrouping happens before any packaging change.** `loopx/chat_*`
+3. **Regroup the selected dependency closure as needed for independent packaging.** `loopx/chat_*`
moves to `loopx/chat/`, `loopx/*_goal_mode/` moves to `loopx/hosts/`, with
compatibility re-exports for established public imports for at least one minor
release. Internal-only callers move together and the old internal entry is deleted.
@@ -63,6 +63,20 @@ can be installed, not what every command must load. Installing standard grants
no service activation or permission. Existing defaults and machine-enforced
work obligations remain until a qualified, explicitly disclosed migration.
+The immediate outcome is an independently installed core that executes a task,
+recovers its original result and returns evidence. Unrelated directory moves,
+workspace packaging or another model session do not block that core-artifact
+slice; full M3 retains its complete distribution acceptance. Ordinary callers
+supply a task and consequential constraints, without first learning Vision,
+Quota, Lease or method names. The host carries bookkeeping; relevant permission,
+budget, uncertainty and recovery decisions remain visible when needed.
+
+The **10% complexity / 90% current-function coverage** target remains an unproven
+hypothesis. Fix a versioned journey denominator and compare necessary concepts,
+mandatory steps, loaded/activated code, model reads, complete costs and accepted
+outcomes. Report kernel-only and assembled coverage; do not drop specialized
+journeys from the denominator or treat code counts as model benefit.
+
## 2. Problem and motivation
A developer who wants a CLI-only or kernel-only LoopX installs the same wheel as
@@ -269,6 +283,19 @@ contract into prose guidance. The kernel is a cohesive set of domains, not one
large file. The adjacent refactor should localize the next change and preserve
characterized behavior.
+### Required mechanisms and optional task methods
+
+Core owns legal scoped work, durable evidence, stop, budget and original-attempt
+recovery. Bench, workbench and manager compose specialized workflows around the
+same owners. Their absence must not block unrelated authorized core work.
+**Yardstick is an independently optional decision method, not a provider or a
+mandatory kernel stage.** Methods choose how to reason or measure; providers
+supply execution, storage or concrete external operations. Reuse existing
+capability owners and `yardstick_enabled`; add no method registry, provider id
+or enable switch. Ordinary result validation remains required when a method is
+off. Existing records, configured obligations, authority and defaults survive;
+method/dependency removal needs its own absence, parity and recovery evidence.
+
### Presets over the distribution tiers
| Composition | Required content | Explicit optional content |
@@ -283,6 +310,16 @@ flag applied after standard fully initializes. A host-free read-only kernel
library does not qualify that executable preset. Exact contents and the first
supported host/authority combination are D2/D5 decisions.
+The initial proposed **Standard** selection is `agent-turn-recall` for relevant
+prior guidance, `issue-fix` for validated repository changes, and
+`pull-request-review` for delivery review. These retain their existing capability
+owners and default-off settings; installation grants no source access, starts
+no watcher and authorizes no publication or merge. Load only the selected task's
+implementation. Basic recovery stays in Minimal without advisory recall.
+Yardstick, Stride, research/benchmark machinery and manager routing remain
+separate optional additions. This selects the R2 target; the current monolithic
+and core-candidate wheels do not yet implement that isolated package selection.
+
Keep composition, managed/attached mode, host provider, one/two model sessions,
storage provider and benchmark arm independent. Minimal does not imply attached,
standard does not require two model sessions, and research does not mean
@@ -343,6 +380,17 @@ objective. Moving a file while retaining its shim does not reduce that count;
combining unrelated responsibilities to meet it is not a successful refactor.
Measure representative change locality and installation cost separately.
+Standalone Core holds the benchmark catalog and its referenced four-arm
+contract. The matching [Benchmark Toolkit wheel](../../../packages/loopx-capability-benchmark-toolkit/README.md)
+owns the existing execution modules and CLI adapters. The capability catalog
+reports availability from matching installed files, and ordinary discovery
+leaves execution unloaded. An explicit benchmark command or scoped Workspace
+analysis request uses the original toolkit. Installing or removing the wheel
+preserves the existing task and analysis records. The ordinary distribution
+continues to bundle the same implementation. This qualifies a D2 package
+boundary for the tested toolkit journey. Benchmark-family execution, public
+release and model-visible overhead have separate acceptance.
+
### State model and schema
No canonical record changes. The only new durable artefact is the architecture
@@ -611,6 +659,14 @@ moves; it does not qualify the kernel. R3/R4 are optional and do not block a
qualified CLI or managed task. No unused provider, field or framework is added
ahead of its real caller. Report the applied/deferred adjacent refactor in each PR.
+### Core discovery checkpoint — 2026-11-19
+
+| Boundary | Delivered delta / evidence | Still open |
+| --- | --- | --- |
+| V1/V3, capability discovery | Benchmark package exports resolve their selected existing owner on demand, retaining all 199 public symbols and their order; explicit star imports still request all. Help/list/detail succeed with benchmark execution implementations unavailable, no Node, process launch, network or state writes. | Does not remove benchmark files from the wheel or qualify absent-capability installation. |
+| Metadata and active consumers | The same catalog and all 25 details are byte-identical to the frozen `main@e235cde9c` workload; 53 CLI help/list/detail/unknown-id observations match. Discovery loads 2 benchmark children instead of 28, and 298 LoopX modules instead of 326. Existing benchmark semantics and canonical export identities remain unchanged. | Import counts are not latency, total cost, model benefit or full D2/D5 closure. |
+| Acceptance and continuation | 52 installed-core discovery/execution/context/recovery cases pass on the rebuilt artifact; 383 selected consumer cases and 43 architecture/documentation cases pass separately. No new state owner, provider id, activation toggle or top-level module. | Keep R1/M3 and V1–V5/V10 open; continue capability package isolation and selected Standard delivery. Public-channel C1 qualification remains with its release owner. |
+
### Subtraction Gate & Anti-Context-Gravity Rules
diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md
index 2303929af4..f4c64c300e 100644
--- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md
+++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md
@@ -4,7 +4,7 @@
- **交付成熟度:** Proposal
- **作者 / 负责人:** LoopX maintainers
- **创建:** 2026-09-26
-- **最近规范性修订:** 2026-11-17
+- **最近规范性修订:** 2026-11-19
- **实现基线:** 发行盘点 `2f3d13ae9`;运行时装配审计 `9cae3b28939f72fa4aadab5cfe49e19bcbdab372`
- **相关契约:** [TypeScript 控制面迁移 v0](typescript-control-plane-migration-v0.zh-CN.md)、[Extensions 参考](../../reference/extensions.md)、[Capability 目录](../../../loopx/capabilities/README.md)、[总体路线图 v0](loopx-overall-roadmap-v0.zh-CN.md)(S2、S8、S12)、[import 边界测试](../../../tests/architecture/test_control_plane_import_boundaries.py)
- **跟踪 issue:** [#5072](https://github.com/loopx-project/loopx/issues/5072)
@@ -25,7 +25,7 @@
1. **仓库保持单一 monorepo。** 一条 PR 流程、一套 CI、一个发布列车。拆成多个 Git 仓库是本 RFC 的非目标。
2. **安装物拆成多个发行物。** 单一 `loopx` wheel 逐步替换为 `loopx-core`、`loopx-workspace` 以及 `packages/` 下的按 capability 打包;`loopx` 保留为依赖全部子包的 meta-package。
-3. **先做顶层目录重组,再做打包变更。** `loopx/chat_*` 迁入 `loopx/chat/`,`loopx/*_goal_mode/` 迁入 `loopx/hosts/`,已建立的公开 import 保留至少一个 minor 版本的兼容 re-export;内部调用方一起迁移并删除旧内部入口。
+3. **按独立打包所需重组所选依赖闭包。** `loopx/chat_*` 迁入 `loopx/chat/`,`loopx/*_goal_mode/` 迁入 `loopx/hosts/`,已建立的公开 import 保留至少一个 minor 版本的兼容 re-export;内部调用方一起迁移并删除旧内部入口。
4. **新增一条架构测试钉住顶层增长。** `loopx/*.py` 的文件数只能减少,与现有零例外 import 边界测试并列。
5. **仅内核安装不得在 import 阶段要求 Node.js。** TS effect runtime 对有副作用的命令仍是必需的;缺失时必须以现有的类型化 `node_unavailable` 诊断呈现,而不是安装或 import 失败。
6. **不变:** 内核权威、CLI 兼容基线、extension 生命周期规则、公开/私有边界、全部稳定协议。
@@ -39,6 +39,15 @@ Minimal 必须执行并恢复真实的受治理任务;standard 围绕同一内
发行表定义可安装内容,不代表每条命令都必须加载。安装 standard 不授予服务激活或权限。
现有默认值与机器强制工作义务持续生效,直到合格且明确披露的迁移改变它们。
+当前优先结果是可独立安装、执行任务、恢复原结果并返回证据的 core。无关目录移动、
+workspace 打包或额外模型会话不阻塞这一 core 发行物切片;完整 M3 仍保留全部发行
+验收。普通调用者提供任务与关键约束,不必先学 Vision、Quota、Lease 或方法名称。
+Host 承担记账;必要的权限、预算、不确定性与恢复决策仍在相关时可见。
+
+**10% 复杂度/90% 当前功能覆盖率**仍是未验证假设。固定版本化用户旅程分母,比较
+必要概念、必经步骤、加载/启用代码、模型读取、完整成本和已验收成果。分别报告
+仅内核与组合后的覆盖率;不能从分母删除特化旅程,也不能以代码计数代替模型收益。
+
## 2. 问题与动机
只想要 CLI 或内核的开发者,安装的和运行桌面工作台、Lark 管家、全部内置 capability 的人是同一个 wheel。`2f3d13ae9` 上的历史发行盘点:
@@ -181,6 +190,16 @@ Planner 可以建议工作或 stride,所属迁移规则决定准入与义务
预设不能把机器强制的 `must_attempt_work` 契约变成文字 guidance。
内核是内聚领域的集合,不是一个大文件。相邻重构应让下一次修改更局部,并保持已刻画的行为。
+### 必要机制与可选任务方法
+
+Core 负责合法限定工作、持久证据、停止、预算及原 attempt 恢复。Bench、workbench、
+管家围绕相同 owner 组合特化工作流;它们缺席不能阻塞无关的已授权 core 工作。
+**Yardstick 是独立可选的决策方法,不是 provider 或强制内核阶段。** 方法选择如何
+推理或衡量;provider 提供执行、存储或具体外部操作。复用既有 capability owner 和
+`yardstick_enabled`,不增加方法 registry、provider id 或开关。关闭方法不免除普通
+结果验收。保留既有记录、已配置义务、权限与默认值;移除方法/依赖须独立证明组件
+缺席、一致性与恢复。
+
### 发行层之上的预设
| 装配 | 必需内容 | 显式可选内容 |
@@ -194,6 +213,13 @@ Minimal 是可执行装配,不是在 standard 完成全部初始化后加开
不带宿主的只读 kernel library 不构成可执行预设资格。
精确内容与首个受支持宿主/authority 组合由 D2/D5 决定。
+首批 **Standard** 选型提案为:`agent-turn-recall` 取回相关既有指导,`issue-fix`
+交付已验证仓库变更,`pull-request-review` 执行交付审查。保留各自既有 owner 和默认
+关闭设置;安装不授予来源访问、不启动 watcher,也不授权发布或合并。仅加载所选
+任务实现;可选指导召回缺席时,Minimal 仍保有基本恢复。Yardstick、Stride、研究/
+benchmark 机制与管家路由继续独立可选。这选定 R2 的目标,现有单体和 core 候选 wheel
+尚未实现该隔离分包组合。
+
装配、managed/attached 模式、宿主 provider、一个/两个模型 Session、存储 provider、benchmark arm
保持独立。Minimal 不意味着 attached,standard 不要求双模型 Session,research 不等于不受治理。
只发布已测组合,保留现有模式默认值;改变装配不能静默改变 binding 的执行 owner。
@@ -239,6 +265,14 @@ import 方向、剩余兼容接缝和从失败定位 owner 的路径。每个发
把无关职责塞进一个文件来满足预算不构成成功重构。代表性修改局部性与安装成本
分别测量。
+独立 Core 保留 benchmark 目录及其引用的四臂实验契约。同版本的
+[Benchmark Toolkit wheel](../../../packages/loopx-capability-benchmark-toolkit/README.md)
+包含既有执行模块和 CLI 适配器。能力目录根据同版本安装文件报告可用性,常规发现
+不会加载执行模块。显式 benchmark 命令或经范围校验的 Workspace 分析请求使用原有
+工具包。安装和卸载保留既有任务及分析记录。普通发行物继续包含同一实现。已测试的
+工具包流程验证了 D2 的这项打包边界。具体 benchmark 家族的执行、公开发布和模型可见
+开销仍分别验收。
+
### 状态模型与 schema
不改任何规范记录。唯一新增的持久产物是架构 fixture:
@@ -443,6 +477,14 @@ M0–M5 保留原发行里程碑;R0–R4 是同一计划中的运行时和旅
R0 可以先于目录移动交付,但不代表内核合格。R3/R4 可选,不阻塞已合格的 CLI 或 managed 任务。
真实调用者出现前不增加闲置 provider、字段或框架。每个 PR 记录应用/推迟的相邻重构。
+### Core 发现入口检查点 — 2026-11-19
+
+| 边界 | 交付增量/证据 | 仍未完成 |
+| --- | --- | --- |
+| V1/V3,能力发现 | Benchmark 包公开符号按需解析到所选既有 owner,保留全部 199 个符号及顺序;显式星号导入仍请求全部。执行实现不可用且无 Node 时,帮助/列表/详情仍可用,无进程启动、网络或状态写入。 | 未从 wheel 删除 benchmark 文件,也不验收 capability 分包缺席安装。 |
+| 元数据与活跃调用方 | 同冻结 `main@e235cde9c` workload 的目录和全部 25 项详情逐字节一致;53 项 CLI 帮助/列表/详情/未知 id 观察一致。发现入口的 benchmark 子模块从 28 降到 2,LoopX 模块从 326 降到 298。保留 benchmark 语义及公开符号的 canonical 对象身份。 | Import 数量不代表时延、完整成本、模型收益或完整 D2/D5 闭包。 |
+| 验收与继续方向 | 重建发行物通过 52 项安装 core 发现/执行/上下文/恢复用例;383 项所选消费方与 43 项架构/文档用例分别通过。没有新增状态 owner、provider id、启用开关或顶层模块。 | R1/M3 与 V1–V5/V10 继续开放;下一步仍是 capability 分包隔离与所选 Standard 交付。公开渠道 C1 资格仍归既有 release owner。 |
+
### 减法门禁与反上下文重力规则
diff --git a/loopx/capabilities/benchmark_toolkit/__init__.py b/loopx/capabilities/benchmark_toolkit/__init__.py
index f024c7c7e9..993fdb7b31 100644
--- a/loopx/capabilities/benchmark_toolkit/__init__.py
+++ b/loopx/capabilities/benchmark_toolkit/__init__.py
@@ -1,455 +1,484 @@
-"""Built-in provider-neutral benchmark experiment toolkit."""
+"""Benchmark exports load only their selected implementation.
-from .adaptive_concurrency import (
- BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION,
- BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION,
- BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION,
- build_benchmark_adaptive_concurrency_decision,
- build_benchmark_adaptive_concurrency_policy,
- normalize_benchmark_adaptive_concurrency_policy,
- normalize_benchmark_concurrency_feedback,
- tune_benchmark_concurrency_target,
-)
-from .artifacts import (
- BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION,
- build_benchmark_candidate_source_boundary,
- classify_benchmark_artifact_path,
- classify_benchmark_candidate_source_path,
- filter_public_benchmark_artifact_paths,
- materialize_public_benchmark_artifacts,
-)
-from .behavior_finding import (
- BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION,
- build_benchmark_behavior_report,
- normalize_benchmark_behavior_finding,
-)
-from .concurrency_envelope import (
- BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME,
- BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION,
- BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION,
- admit_benchmark_case,
- build_benchmark_concurrency_config,
- build_benchmark_concurrency_status,
- configure_benchmark_concurrency_envelope,
- default_benchmark_concurrency_envelope_path,
- normalize_benchmark_concurrency_config,
- normalize_benchmark_concurrency_envelope,
- normalize_benchmark_resource_headroom_receipt,
- read_benchmark_concurrency_envelope,
- release_benchmark_case,
- render_benchmark_concurrency_markdown,
-)
-from .container_binding import (
- BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION,
- DockerContainerBinding,
- DockerContainerBindingError,
- compact_docker_container_binding_receipt,
- select_exact_docker_container,
-)
-from .experiment_board import (
- BenchmarkExperimentBoardValidationError,
- BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME,
- BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION,
- BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION,
- benchmark_experiment_board_row_key,
- build_benchmark_experiment_board,
- default_benchmark_experiment_board_path,
- normalize_benchmark_experiment_board_row,
- preview_benchmark_experiment_board_reconcile,
- preview_benchmark_experiment_board_upsert,
- read_benchmark_experiment_board_rows,
- render_benchmark_experiment_board_markdown,
- upsert_benchmark_experiment_board_row,
-)
-from .environment_access import (
- BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION,
- BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION,
- BENCHMARK_TASK_ENVIRONMENT_READ_TOOL,
-)
-from .continuation import (
- BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION,
- BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION,
- BenchmarkContinuationDecision,
- build_benchmark_continuation_decision,
- normalize_benchmark_public_progress,
-)
-from .factorial_contrast import (
- BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION,
- build_benchmark_factorial_contrasts,
-)
-from .four_arm_contract import (
- BENCHMARK_FOUR_ARM_ATTESTATIONS,
- BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION,
- BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE,
- BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS,
- BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION,
- build_benchmark_four_arm_contract,
- build_benchmark_four_arm_contract_from_spec,
- compact_benchmark_four_arm_contract,
-)
-from .integrity import (
- BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION,
- BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION,
- BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION,
- BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION,
- BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION,
- BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION,
- BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION,
- BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION,
- INTEGRITY_EVIDENCE_CATEGORIES,
- REQUIRED_RUNTIME_ATTESTATIONS,
- RestrictedAccessAdjudicationDecision,
- build_benchmark_integrity_qualification,
-)
-from .native_codex_goal import (
- NativeGoalConfig,
- NativeGoalDeadlineExceeded,
- NativeGoalEventTransport,
- NativeGoalProtocolError,
- NativeGoalTransport,
- NativeGoalTurn,
- StdioNativeGoalTransport,
- attach_native_goal,
- compact_native_goal_receipt,
- observe_native_goal_event,
- probe_native_goal_process,
- refresh_native_goal_status,
- run_native_goal_process,
- run_native_goal_process_until_terminal,
- run_native_goal_turn,
- run_native_goal_until_terminal,
- start_native_goal_turn,
- wait_native_goal_turn,
-)
-from .native_codex_isolation import (
- NativeCodexIsolationEnvelope,
- NativeCodexIsolationError,
- NativeCodexLoopXStateRebase,
- build_native_codex_isolation_envelope,
- rebase_native_codex_loopx_workspace_state,
-)
-from .native_codex_profile import (
- NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION,
- NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS,
- NATIVE_CODEX_PROFILE_SCHEMA_VERSION,
- NativeCodexGoalPrompt,
- NativeCodexProfile,
- NativeCodexProfileError,
- compact_native_codex_goal_prompt_receipt,
- compact_native_codex_profile_receipt,
- inspect_native_codex_profile,
- install_native_codex_profile,
- native_codex_app_server_shell_policy_args,
- native_codex_profile_environment,
- render_native_codex_goal_prompt,
-)
-from .provider_gateway import (
- RunnerOwnedProviderGateway,
- serve_runner_owned_provider_gateway,
-)
-from .public_trajectory import (
- NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION,
- PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION,
- NativeGoalLifecycleFacts,
- PublicTrajectoryLifecycleState,
- PublicTrajectorySummaryError,
- build_native_goal_public_trajectory_summary,
-)
-from .reward_contract import (
- REWARD_CONTRACT_SCHEMA_VERSION,
- verify_verifier_reward_file,
- verify_verifier_reward_json,
-)
-from .rebalance_plan import (
- BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION,
- BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION,
- BenchmarkRebalanceAction,
- BenchmarkRebalanceCandidateState,
- build_benchmark_rebalance_plan,
- build_benchmark_rebalance_plan_from_inventory,
- normalize_benchmark_rebalance_inventory,
- render_benchmark_rebalance_plan_markdown,
-)
-from .run_permissions import (
- DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS,
- DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS,
- RUN_PERMISSION_POLICY_SCHEMA_VERSION,
- RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION,
- RunPermissionAction,
- build_run_permission_policy,
- compact_run_permission_policy_for_quota,
- validate_run_permission_policy,
-)
-from .runtime_continuity import (
- BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION,
- BenchmarkEventWindowState,
- BenchmarkRuntimeContinuityClassification,
- BenchmarkRuntimeContinuityTransition,
- build_benchmark_runtime_continuity,
-)
-from .runtime_observation import (
- BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION,
- BenchmarkJobReceiptState,
- BenchmarkRunnerOwnerState,
- BenchmarkRuntimeClassification,
- BenchmarkRuntimeTransition,
- build_benchmark_runtime_observation,
-)
-from .source_revision_fence import (
- BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION,
- BenchmarkSourceRevisionFence,
- BenchmarkSourceRevisionFenceError,
- compact_benchmark_source_revision_fence_receipt,
- inspect_benchmark_source_revision_fence,
-)
-from .study_projection import (
- BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION,
- BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION,
- BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION,
- BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION,
- BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION,
- BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION,
- build_benchmark_study_dashboard,
- build_benchmark_upload_envelope,
- normalize_benchmark_case_insight_projection,
- normalize_benchmark_study_manifest,
- normalize_benchmark_upload_envelope,
- read_benchmark_local_upload_records,
- read_benchmark_upload_receipt,
- simulate_benchmark_upload,
-)
-from .traex_evidence import (
- ATIF_SCHEMA_VERSION,
- BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION,
- TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION,
- build_traex_model_route_receipt,
- capture_traex_benchmark_evidence,
- convert_traex_events_to_atif,
-)
-from .treatment_continuation import (
- BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION,
- BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION,
- build_benchmark_treatment_continuation_receipt,
-)
-from .token_capacity import (
- BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION,
- BENCHMARK_TOKEN_CAPACITY_FILENAME,
- BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION,
- BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION,
- BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION,
- admit_benchmark_token_capacity,
- build_benchmark_token_capacity_policy,
- build_benchmark_token_capacity_status,
- configure_benchmark_token_capacity,
- default_benchmark_token_capacity_path,
- normalize_benchmark_token_capacity_policy,
- normalize_benchmark_token_capacity_state,
- normalize_benchmark_token_usage_sample,
- observe_benchmark_token_usage,
- read_benchmark_token_capacity_state,
- record_benchmark_token_capacity_rejection,
- release_benchmark_token_capacity,
- render_benchmark_token_capacity_markdown,
-)
+Capability discovery reads catalog metadata without loading execution, capacity,
+installation or experiment IO. Public names retain their canonical owners.
+"""
+from __future__ import annotations
-__all__ = [
- "BenchmarkExperimentBoardValidationError",
- "ATIF_SCHEMA_VERSION",
- "BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION",
- "BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION",
- "BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION",
- "BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION",
- "BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION",
- "BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME",
- "BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION",
- "BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION",
- "BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION",
- "BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION",
- "BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME",
- "BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION",
- "BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION",
- "BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION",
- "BENCHMARK_FOUR_ARM_ATTESTATIONS",
- "BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION",
- "BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE",
- "BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS",
- "BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION",
- "BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION",
- "BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION",
- "BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION",
- "BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION",
- "BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION",
- "BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION",
- "BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION",
- "BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION",
- "BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION",
- "BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION",
- "BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION",
- "BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION",
- "BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION",
- "BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION",
- "BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION",
- "BENCHMARK_TASK_ENVIRONMENT_READ_TOOL",
- "BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION",
- "BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION",
- "BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION",
- "BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION",
- "BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION",
- "BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION",
- "BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION",
- "BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION",
- "BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION",
- "BENCHMARK_TOKEN_CAPACITY_FILENAME",
- "BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION",
- "BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION",
- "BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION",
- "BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION",
- "BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION",
- "DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS",
- "DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS",
- "INTEGRITY_EVIDENCE_CATEGORIES",
- "NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION",
- "NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS",
- "NATIVE_CODEX_PROFILE_SCHEMA_VERSION",
- "NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION",
- "PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION",
- "REQUIRED_RUNTIME_ATTESTATIONS",
- "REWARD_CONTRACT_SCHEMA_VERSION",
- "RUN_PERMISSION_POLICY_SCHEMA_VERSION",
- "RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION",
- "TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION",
- "BenchmarkContinuationDecision",
- "BenchmarkRebalanceAction",
- "BenchmarkRebalanceCandidateState",
- "BenchmarkEventWindowState",
- "BenchmarkJobReceiptState",
- "BenchmarkRunnerOwnerState",
- "BenchmarkRuntimeClassification",
- "BenchmarkRuntimeContinuityClassification",
- "BenchmarkRuntimeContinuityTransition",
- "BenchmarkRuntimeTransition",
- "BenchmarkSourceRevisionFence",
- "BenchmarkSourceRevisionFenceError",
- "DockerContainerBinding",
- "DockerContainerBindingError",
- "NativeCodexGoalPrompt",
- "NativeCodexIsolationEnvelope",
- "NativeCodexIsolationError",
- "NativeCodexLoopXStateRebase",
- "NativeCodexProfile",
- "NativeCodexProfileError",
- "NativeGoalConfig",
- "NativeGoalDeadlineExceeded",
- "NativeGoalEventTransport",
- "NativeGoalLifecycleFacts",
- "NativeGoalProtocolError",
- "NativeGoalTransport",
- "NativeGoalTurn",
- "PublicTrajectoryLifecycleState",
- "PublicTrajectorySummaryError",
- "RestrictedAccessAdjudicationDecision",
- "RunPermissionAction",
- "RunnerOwnedProviderGateway",
- "StdioNativeGoalTransport",
- "admit_benchmark_case",
- "admit_benchmark_token_capacity",
- "attach_native_goal",
- "benchmark_experiment_board_row_key",
- "build_benchmark_adaptive_concurrency_decision",
- "build_benchmark_adaptive_concurrency_policy",
- "build_benchmark_behavior_report",
- "build_benchmark_candidate_source_boundary",
- "build_benchmark_concurrency_config",
- "build_benchmark_concurrency_status",
- "build_benchmark_continuation_decision",
- "build_benchmark_experiment_board",
- "build_benchmark_factorial_contrasts",
- "build_benchmark_rebalance_plan",
- "build_benchmark_rebalance_plan_from_inventory",
- "build_benchmark_four_arm_contract",
- "build_benchmark_four_arm_contract_from_spec",
- "build_benchmark_integrity_qualification",
- "build_benchmark_runtime_continuity",
- "build_benchmark_runtime_observation",
- "build_benchmark_study_dashboard",
- "build_benchmark_treatment_continuation_receipt",
- "build_benchmark_token_capacity_policy",
- "build_benchmark_token_capacity_status",
- "build_benchmark_upload_envelope",
- "build_native_codex_isolation_envelope",
- "build_native_goal_public_trajectory_summary",
- "build_run_permission_policy",
- "build_traex_model_route_receipt",
- "capture_traex_benchmark_evidence",
- "classify_benchmark_artifact_path",
- "classify_benchmark_candidate_source_path",
- "compact_benchmark_four_arm_contract",
- "compact_benchmark_source_revision_fence_receipt",
- "compact_docker_container_binding_receipt",
- "compact_native_codex_goal_prompt_receipt",
- "compact_native_codex_profile_receipt",
- "compact_native_goal_receipt",
- "compact_run_permission_policy_for_quota",
- "configure_benchmark_concurrency_envelope",
- "configure_benchmark_token_capacity",
- "convert_traex_events_to_atif",
- "default_benchmark_concurrency_envelope_path",
- "default_benchmark_experiment_board_path",
- "default_benchmark_token_capacity_path",
- "filter_public_benchmark_artifact_paths",
- "inspect_benchmark_source_revision_fence",
- "inspect_native_codex_profile",
- "install_native_codex_profile",
- "materialize_public_benchmark_artifacts",
- "native_codex_app_server_shell_policy_args",
- "native_codex_profile_environment",
- "normalize_benchmark_adaptive_concurrency_policy",
- "normalize_benchmark_behavior_finding",
- "normalize_benchmark_case_insight_projection",
- "normalize_benchmark_concurrency_config",
- "normalize_benchmark_concurrency_envelope",
- "normalize_benchmark_concurrency_feedback",
- "normalize_benchmark_experiment_board_row",
- "normalize_benchmark_public_progress",
- "normalize_benchmark_resource_headroom_receipt",
- "normalize_benchmark_rebalance_inventory",
- "normalize_benchmark_study_manifest",
- "normalize_benchmark_token_capacity_policy",
- "normalize_benchmark_token_capacity_state",
- "normalize_benchmark_token_usage_sample",
- "normalize_benchmark_upload_envelope",
- "observe_native_goal_event",
- "observe_benchmark_token_usage",
- "preview_benchmark_experiment_board_reconcile",
- "preview_benchmark_experiment_board_upsert",
- "probe_native_goal_process",
- "read_benchmark_concurrency_envelope",
- "read_benchmark_experiment_board_rows",
- "read_benchmark_local_upload_records",
- "read_benchmark_upload_receipt",
- "read_benchmark_token_capacity_state",
- "rebase_native_codex_loopx_workspace_state",
- "refresh_native_goal_status",
- "release_benchmark_case",
- "record_benchmark_token_capacity_rejection",
- "release_benchmark_token_capacity",
- "render_benchmark_concurrency_markdown",
- "render_benchmark_rebalance_plan_markdown",
- "render_benchmark_token_capacity_markdown",
- "render_benchmark_experiment_board_markdown",
- "render_native_codex_goal_prompt",
- "run_native_goal_process",
- "run_native_goal_process_until_terminal",
- "run_native_goal_turn",
- "run_native_goal_until_terminal",
- "select_exact_docker_container",
- "serve_runner_owned_provider_gateway",
- "simulate_benchmark_upload",
- "start_native_goal_turn",
- "tune_benchmark_concurrency_target",
- "upsert_benchmark_experiment_board_row",
- "validate_run_permission_policy",
- "verify_verifier_reward_file",
- "verify_verifier_reward_json",
- "wait_native_goal_turn",
-]
+from importlib import import_module
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+ from .adaptive_concurrency import (
+ BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION as BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION,
+ BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION as BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION,
+ BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION as BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION,
+ build_benchmark_adaptive_concurrency_decision as build_benchmark_adaptive_concurrency_decision,
+ build_benchmark_adaptive_concurrency_policy as build_benchmark_adaptive_concurrency_policy,
+ normalize_benchmark_adaptive_concurrency_policy as normalize_benchmark_adaptive_concurrency_policy,
+ normalize_benchmark_concurrency_feedback as normalize_benchmark_concurrency_feedback,
+ tune_benchmark_concurrency_target as tune_benchmark_concurrency_target,
+ )
+ from .artifacts import (
+ BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION as BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION,
+ build_benchmark_candidate_source_boundary as build_benchmark_candidate_source_boundary,
+ classify_benchmark_artifact_path as classify_benchmark_artifact_path,
+ classify_benchmark_candidate_source_path as classify_benchmark_candidate_source_path,
+ filter_public_benchmark_artifact_paths as filter_public_benchmark_artifact_paths,
+ materialize_public_benchmark_artifacts as materialize_public_benchmark_artifacts,
+ )
+ from .behavior_finding import (
+ BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION as BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION,
+ build_benchmark_behavior_report as build_benchmark_behavior_report,
+ normalize_benchmark_behavior_finding as normalize_benchmark_behavior_finding,
+ )
+ from .concurrency_envelope import (
+ BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME as BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME,
+ BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION as BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION,
+ BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION as BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION,
+ admit_benchmark_case as admit_benchmark_case,
+ build_benchmark_concurrency_config as build_benchmark_concurrency_config,
+ build_benchmark_concurrency_status as build_benchmark_concurrency_status,
+ configure_benchmark_concurrency_envelope as configure_benchmark_concurrency_envelope,
+ default_benchmark_concurrency_envelope_path as default_benchmark_concurrency_envelope_path,
+ normalize_benchmark_concurrency_config as normalize_benchmark_concurrency_config,
+ normalize_benchmark_concurrency_envelope as normalize_benchmark_concurrency_envelope,
+ normalize_benchmark_resource_headroom_receipt as normalize_benchmark_resource_headroom_receipt,
+ read_benchmark_concurrency_envelope as read_benchmark_concurrency_envelope,
+ release_benchmark_case as release_benchmark_case,
+ render_benchmark_concurrency_markdown as render_benchmark_concurrency_markdown,
+ )
+ from .container_binding import (
+ BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION as BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION,
+ DockerContainerBinding as DockerContainerBinding,
+ DockerContainerBindingError as DockerContainerBindingError,
+ compact_docker_container_binding_receipt as compact_docker_container_binding_receipt,
+ select_exact_docker_container as select_exact_docker_container,
+ )
+ from .experiment_board import (
+ BenchmarkExperimentBoardValidationError as BenchmarkExperimentBoardValidationError,
+ BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME as BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME,
+ BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION as BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION,
+ BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION as BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION,
+ benchmark_experiment_board_row_key as benchmark_experiment_board_row_key,
+ build_benchmark_experiment_board as build_benchmark_experiment_board,
+ default_benchmark_experiment_board_path as default_benchmark_experiment_board_path,
+ normalize_benchmark_experiment_board_row as normalize_benchmark_experiment_board_row,
+ preview_benchmark_experiment_board_reconcile as preview_benchmark_experiment_board_reconcile,
+ preview_benchmark_experiment_board_upsert as preview_benchmark_experiment_board_upsert,
+ read_benchmark_experiment_board_rows as read_benchmark_experiment_board_rows,
+ render_benchmark_experiment_board_markdown as render_benchmark_experiment_board_markdown,
+ upsert_benchmark_experiment_board_row as upsert_benchmark_experiment_board_row,
+ )
+ from .environment_access import (
+ BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION,
+ BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION,
+ BENCHMARK_TASK_ENVIRONMENT_READ_TOOL as BENCHMARK_TASK_ENVIRONMENT_READ_TOOL,
+ )
+ from .continuation import (
+ BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION as BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION,
+ BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION as BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION,
+ BenchmarkContinuationDecision as BenchmarkContinuationDecision,
+ build_benchmark_continuation_decision as build_benchmark_continuation_decision,
+ normalize_benchmark_public_progress as normalize_benchmark_public_progress,
+ )
+ from .factorial_contrast import (
+ BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION as BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION,
+ build_benchmark_factorial_contrasts as build_benchmark_factorial_contrasts,
+ )
+ from .four_arm_contract import (
+ BENCHMARK_FOUR_ARM_ATTESTATIONS as BENCHMARK_FOUR_ARM_ATTESTATIONS,
+ BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION as BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION,
+ BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE as BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE,
+ BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS as BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS,
+ BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION as BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION,
+ build_benchmark_four_arm_contract as build_benchmark_four_arm_contract,
+ build_benchmark_four_arm_contract_from_spec as build_benchmark_four_arm_contract_from_spec,
+ compact_benchmark_four_arm_contract as compact_benchmark_four_arm_contract,
+ )
+ from .integrity import (
+ BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION as BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION,
+ BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION as BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION,
+ BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION as BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION,
+ BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION as BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION,
+ BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION,
+ BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION,
+ BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION as BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION,
+ BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION as BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION,
+ INTEGRITY_EVIDENCE_CATEGORIES as INTEGRITY_EVIDENCE_CATEGORIES,
+ REQUIRED_RUNTIME_ATTESTATIONS as REQUIRED_RUNTIME_ATTESTATIONS,
+ RestrictedAccessAdjudicationDecision as RestrictedAccessAdjudicationDecision,
+ build_benchmark_integrity_qualification as build_benchmark_integrity_qualification,
+ )
+ from .native_codex_goal import (
+ NativeGoalConfig as NativeGoalConfig,
+ NativeGoalDeadlineExceeded as NativeGoalDeadlineExceeded,
+ NativeGoalEventTransport as NativeGoalEventTransport,
+ NativeGoalProtocolError as NativeGoalProtocolError,
+ NativeGoalTransport as NativeGoalTransport,
+ NativeGoalTurn as NativeGoalTurn,
+ StdioNativeGoalTransport as StdioNativeGoalTransport,
+ attach_native_goal as attach_native_goal,
+ compact_native_goal_receipt as compact_native_goal_receipt,
+ observe_native_goal_event as observe_native_goal_event,
+ probe_native_goal_process as probe_native_goal_process,
+ refresh_native_goal_status as refresh_native_goal_status,
+ run_native_goal_process as run_native_goal_process,
+ run_native_goal_process_until_terminal as run_native_goal_process_until_terminal,
+ run_native_goal_turn as run_native_goal_turn,
+ run_native_goal_until_terminal as run_native_goal_until_terminal,
+ start_native_goal_turn as start_native_goal_turn,
+ wait_native_goal_turn as wait_native_goal_turn,
+ )
+ from .native_codex_isolation import (
+ NativeCodexIsolationEnvelope as NativeCodexIsolationEnvelope,
+ NativeCodexIsolationError as NativeCodexIsolationError,
+ NativeCodexLoopXStateRebase as NativeCodexLoopXStateRebase,
+ build_native_codex_isolation_envelope as build_native_codex_isolation_envelope,
+ rebase_native_codex_loopx_workspace_state as rebase_native_codex_loopx_workspace_state,
+ )
+ from .native_codex_profile import (
+ NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION as NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION,
+ NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS as NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS,
+ NATIVE_CODEX_PROFILE_SCHEMA_VERSION as NATIVE_CODEX_PROFILE_SCHEMA_VERSION,
+ NativeCodexGoalPrompt as NativeCodexGoalPrompt,
+ NativeCodexProfile as NativeCodexProfile,
+ NativeCodexProfileError as NativeCodexProfileError,
+ compact_native_codex_goal_prompt_receipt as compact_native_codex_goal_prompt_receipt,
+ compact_native_codex_profile_receipt as compact_native_codex_profile_receipt,
+ inspect_native_codex_profile as inspect_native_codex_profile,
+ install_native_codex_profile as install_native_codex_profile,
+ native_codex_app_server_shell_policy_args as native_codex_app_server_shell_policy_args,
+ native_codex_profile_environment as native_codex_profile_environment,
+ render_native_codex_goal_prompt as render_native_codex_goal_prompt,
+ )
+ from .provider_gateway import (
+ RunnerOwnedProviderGateway as RunnerOwnedProviderGateway,
+ serve_runner_owned_provider_gateway as serve_runner_owned_provider_gateway,
+ )
+ from .public_trajectory import (
+ NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION as NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION,
+ PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION as PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION,
+ NativeGoalLifecycleFacts as NativeGoalLifecycleFacts,
+ PublicTrajectoryLifecycleState as PublicTrajectoryLifecycleState,
+ PublicTrajectorySummaryError as PublicTrajectorySummaryError,
+ build_native_goal_public_trajectory_summary as build_native_goal_public_trajectory_summary,
+ )
+ from .reward_contract import (
+ REWARD_CONTRACT_SCHEMA_VERSION as REWARD_CONTRACT_SCHEMA_VERSION,
+ verify_verifier_reward_file as verify_verifier_reward_file,
+ verify_verifier_reward_json as verify_verifier_reward_json,
+ )
+ from .rebalance_plan import (
+ BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION as BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION,
+ BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION as BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION,
+ BenchmarkRebalanceAction as BenchmarkRebalanceAction,
+ BenchmarkRebalanceCandidateState as BenchmarkRebalanceCandidateState,
+ build_benchmark_rebalance_plan as build_benchmark_rebalance_plan,
+ build_benchmark_rebalance_plan_from_inventory as build_benchmark_rebalance_plan_from_inventory,
+ normalize_benchmark_rebalance_inventory as normalize_benchmark_rebalance_inventory,
+ render_benchmark_rebalance_plan_markdown as render_benchmark_rebalance_plan_markdown,
+ )
+ from .run_permissions import (
+ DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS as DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS,
+ DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS as DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS,
+ RUN_PERMISSION_POLICY_SCHEMA_VERSION as RUN_PERMISSION_POLICY_SCHEMA_VERSION,
+ RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION as RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION,
+ RunPermissionAction as RunPermissionAction,
+ build_run_permission_policy as build_run_permission_policy,
+ compact_run_permission_policy_for_quota as compact_run_permission_policy_for_quota,
+ validate_run_permission_policy as validate_run_permission_policy,
+ )
+ from .runtime_continuity import (
+ BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION as BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION,
+ BenchmarkEventWindowState as BenchmarkEventWindowState,
+ BenchmarkRuntimeContinuityClassification as BenchmarkRuntimeContinuityClassification,
+ BenchmarkRuntimeContinuityTransition as BenchmarkRuntimeContinuityTransition,
+ build_benchmark_runtime_continuity as build_benchmark_runtime_continuity,
+ )
+ from .runtime_observation import (
+ BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION as BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION,
+ BenchmarkJobReceiptState as BenchmarkJobReceiptState,
+ BenchmarkRunnerOwnerState as BenchmarkRunnerOwnerState,
+ BenchmarkRuntimeClassification as BenchmarkRuntimeClassification,
+ BenchmarkRuntimeTransition as BenchmarkRuntimeTransition,
+ build_benchmark_runtime_observation as build_benchmark_runtime_observation,
+ )
+ from .source_revision_fence import (
+ BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION as BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION,
+ BenchmarkSourceRevisionFence as BenchmarkSourceRevisionFence,
+ BenchmarkSourceRevisionFenceError as BenchmarkSourceRevisionFenceError,
+ compact_benchmark_source_revision_fence_receipt as compact_benchmark_source_revision_fence_receipt,
+ inspect_benchmark_source_revision_fence as inspect_benchmark_source_revision_fence,
+ )
+ from .study_projection import (
+ BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION as BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION,
+ BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION as BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION,
+ BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION as BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION,
+ BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION as BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION,
+ BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION as BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION,
+ BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION as BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION,
+ build_benchmark_study_dashboard as build_benchmark_study_dashboard,
+ build_benchmark_upload_envelope as build_benchmark_upload_envelope,
+ normalize_benchmark_case_insight_projection as normalize_benchmark_case_insight_projection,
+ normalize_benchmark_study_manifest as normalize_benchmark_study_manifest,
+ normalize_benchmark_upload_envelope as normalize_benchmark_upload_envelope,
+ read_benchmark_local_upload_records as read_benchmark_local_upload_records,
+ read_benchmark_upload_receipt as read_benchmark_upload_receipt,
+ simulate_benchmark_upload as simulate_benchmark_upload,
+ )
+ from .traex_evidence import (
+ ATIF_SCHEMA_VERSION as ATIF_SCHEMA_VERSION,
+ BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION as BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION,
+ TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION as TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION,
+ build_traex_model_route_receipt as build_traex_model_route_receipt,
+ capture_traex_benchmark_evidence as capture_traex_benchmark_evidence,
+ convert_traex_events_to_atif as convert_traex_events_to_atif,
+ )
+ from .treatment_continuation import (
+ BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION as BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION,
+ BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION as BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION,
+ build_benchmark_treatment_continuation_receipt as build_benchmark_treatment_continuation_receipt,
+ )
+ from .token_capacity import (
+ BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION as BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION,
+ BENCHMARK_TOKEN_CAPACITY_FILENAME as BENCHMARK_TOKEN_CAPACITY_FILENAME,
+ BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION as BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION,
+ BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION as BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION,
+ BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION as BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION,
+ admit_benchmark_token_capacity as admit_benchmark_token_capacity,
+ build_benchmark_token_capacity_policy as build_benchmark_token_capacity_policy,
+ build_benchmark_token_capacity_status as build_benchmark_token_capacity_status,
+ configure_benchmark_token_capacity as configure_benchmark_token_capacity,
+ default_benchmark_token_capacity_path as default_benchmark_token_capacity_path,
+ normalize_benchmark_token_capacity_policy as normalize_benchmark_token_capacity_policy,
+ normalize_benchmark_token_capacity_state as normalize_benchmark_token_capacity_state,
+ normalize_benchmark_token_usage_sample as normalize_benchmark_token_usage_sample,
+ observe_benchmark_token_usage as observe_benchmark_token_usage,
+ read_benchmark_token_capacity_state as read_benchmark_token_capacity_state,
+ record_benchmark_token_capacity_rejection as record_benchmark_token_capacity_rejection,
+ release_benchmark_token_capacity as release_benchmark_token_capacity,
+ render_benchmark_token_capacity_markdown as render_benchmark_token_capacity_markdown,
+ )
+
+_EXPORTS = {
+ "BenchmarkExperimentBoardValidationError": "experiment_board",
+ "ATIF_SCHEMA_VERSION": "traex_evidence",
+ "BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION": "adaptive_concurrency",
+ "BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION": "adaptive_concurrency",
+ "BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION": "behavior_finding",
+ "BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION": "artifacts",
+ "BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION": "study_projection",
+ "BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME": "concurrency_envelope",
+ "BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION": "concurrency_envelope",
+ "BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION": "adaptive_concurrency",
+ "BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION": "continuation",
+ "BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION": "container_binding",
+ "BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME": "experiment_board",
+ "BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION": "experiment_board",
+ "BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION": "experiment_board",
+ "BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION": "factorial_contrast",
+ "BENCHMARK_FOUR_ARM_ATTESTATIONS": "four_arm_contract",
+ "BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION": "four_arm_contract",
+ "BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE": "four_arm_contract",
+ "BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS": "four_arm_contract",
+ "BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION": "four_arm_contract",
+ "BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION": "study_projection",
+ "BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION": "traex_evidence",
+ "BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION": "continuation",
+ "BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION": "concurrency_envelope",
+ "BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION": "rebalance_plan",
+ "BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION": "rebalance_plan",
+ "BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION": "runtime_continuity",
+ "BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION": "environment_access",
+ "BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION": "environment_access",
+ "BENCHMARK_TASK_ENVIRONMENT_READ_TOOL": "environment_access",
+ "BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION": "integrity",
+ "BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION": "runtime_observation",
+ "BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION": "source_revision_fence",
+ "BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION": "study_projection",
+ "BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION": "study_projection",
+ "BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION": "treatment_continuation",
+ "BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION": "treatment_continuation",
+ "BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION": "token_capacity",
+ "BENCHMARK_TOKEN_CAPACITY_FILENAME": "token_capacity",
+ "BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION": "token_capacity",
+ "BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION": "token_capacity",
+ "BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION": "token_capacity",
+ "BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION": "study_projection",
+ "BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION": "study_projection",
+ "DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS": "run_permissions",
+ "DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS": "run_permissions",
+ "INTEGRITY_EVIDENCE_CATEGORIES": "integrity",
+ "NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION": "native_codex_profile",
+ "NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS": "native_codex_profile",
+ "NATIVE_CODEX_PROFILE_SCHEMA_VERSION": "native_codex_profile",
+ "NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION": "public_trajectory",
+ "PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION": "public_trajectory",
+ "REQUIRED_RUNTIME_ATTESTATIONS": "integrity",
+ "REWARD_CONTRACT_SCHEMA_VERSION": "reward_contract",
+ "RUN_PERMISSION_POLICY_SCHEMA_VERSION": "run_permissions",
+ "RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION": "run_permissions",
+ "TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION": "traex_evidence",
+ "BenchmarkContinuationDecision": "continuation",
+ "BenchmarkRebalanceAction": "rebalance_plan",
+ "BenchmarkRebalanceCandidateState": "rebalance_plan",
+ "BenchmarkEventWindowState": "runtime_continuity",
+ "BenchmarkJobReceiptState": "runtime_observation",
+ "BenchmarkRunnerOwnerState": "runtime_observation",
+ "BenchmarkRuntimeClassification": "runtime_observation",
+ "BenchmarkRuntimeContinuityClassification": "runtime_continuity",
+ "BenchmarkRuntimeContinuityTransition": "runtime_continuity",
+ "BenchmarkRuntimeTransition": "runtime_observation",
+ "BenchmarkSourceRevisionFence": "source_revision_fence",
+ "BenchmarkSourceRevisionFenceError": "source_revision_fence",
+ "DockerContainerBinding": "container_binding",
+ "DockerContainerBindingError": "container_binding",
+ "NativeCodexGoalPrompt": "native_codex_profile",
+ "NativeCodexIsolationEnvelope": "native_codex_isolation",
+ "NativeCodexIsolationError": "native_codex_isolation",
+ "NativeCodexLoopXStateRebase": "native_codex_isolation",
+ "NativeCodexProfile": "native_codex_profile",
+ "NativeCodexProfileError": "native_codex_profile",
+ "NativeGoalConfig": "native_codex_goal",
+ "NativeGoalDeadlineExceeded": "native_codex_goal",
+ "NativeGoalEventTransport": "native_codex_goal",
+ "NativeGoalLifecycleFacts": "public_trajectory",
+ "NativeGoalProtocolError": "native_codex_goal",
+ "NativeGoalTransport": "native_codex_goal",
+ "NativeGoalTurn": "native_codex_goal",
+ "PublicTrajectoryLifecycleState": "public_trajectory",
+ "PublicTrajectorySummaryError": "public_trajectory",
+ "RestrictedAccessAdjudicationDecision": "integrity",
+ "RunPermissionAction": "run_permissions",
+ "RunnerOwnedProviderGateway": "provider_gateway",
+ "StdioNativeGoalTransport": "native_codex_goal",
+ "admit_benchmark_case": "concurrency_envelope",
+ "admit_benchmark_token_capacity": "token_capacity",
+ "attach_native_goal": "native_codex_goal",
+ "benchmark_experiment_board_row_key": "experiment_board",
+ "build_benchmark_adaptive_concurrency_decision": "adaptive_concurrency",
+ "build_benchmark_adaptive_concurrency_policy": "adaptive_concurrency",
+ "build_benchmark_behavior_report": "behavior_finding",
+ "build_benchmark_candidate_source_boundary": "artifacts",
+ "build_benchmark_concurrency_config": "concurrency_envelope",
+ "build_benchmark_concurrency_status": "concurrency_envelope",
+ "build_benchmark_continuation_decision": "continuation",
+ "build_benchmark_experiment_board": "experiment_board",
+ "build_benchmark_factorial_contrasts": "factorial_contrast",
+ "build_benchmark_rebalance_plan": "rebalance_plan",
+ "build_benchmark_rebalance_plan_from_inventory": "rebalance_plan",
+ "build_benchmark_four_arm_contract": "four_arm_contract",
+ "build_benchmark_four_arm_contract_from_spec": "four_arm_contract",
+ "build_benchmark_integrity_qualification": "integrity",
+ "build_benchmark_runtime_continuity": "runtime_continuity",
+ "build_benchmark_runtime_observation": "runtime_observation",
+ "build_benchmark_study_dashboard": "study_projection",
+ "build_benchmark_treatment_continuation_receipt": "treatment_continuation",
+ "build_benchmark_token_capacity_policy": "token_capacity",
+ "build_benchmark_token_capacity_status": "token_capacity",
+ "build_benchmark_upload_envelope": "study_projection",
+ "build_native_codex_isolation_envelope": "native_codex_isolation",
+ "build_native_goal_public_trajectory_summary": "public_trajectory",
+ "build_run_permission_policy": "run_permissions",
+ "build_traex_model_route_receipt": "traex_evidence",
+ "capture_traex_benchmark_evidence": "traex_evidence",
+ "classify_benchmark_artifact_path": "artifacts",
+ "classify_benchmark_candidate_source_path": "artifacts",
+ "compact_benchmark_four_arm_contract": "four_arm_contract",
+ "compact_benchmark_source_revision_fence_receipt": "source_revision_fence",
+ "compact_docker_container_binding_receipt": "container_binding",
+ "compact_native_codex_goal_prompt_receipt": "native_codex_profile",
+ "compact_native_codex_profile_receipt": "native_codex_profile",
+ "compact_native_goal_receipt": "native_codex_goal",
+ "compact_run_permission_policy_for_quota": "run_permissions",
+ "configure_benchmark_concurrency_envelope": "concurrency_envelope",
+ "configure_benchmark_token_capacity": "token_capacity",
+ "convert_traex_events_to_atif": "traex_evidence",
+ "default_benchmark_concurrency_envelope_path": "concurrency_envelope",
+ "default_benchmark_experiment_board_path": "experiment_board",
+ "default_benchmark_token_capacity_path": "token_capacity",
+ "filter_public_benchmark_artifact_paths": "artifacts",
+ "inspect_benchmark_source_revision_fence": "source_revision_fence",
+ "inspect_native_codex_profile": "native_codex_profile",
+ "install_native_codex_profile": "native_codex_profile",
+ "materialize_public_benchmark_artifacts": "artifacts",
+ "native_codex_app_server_shell_policy_args": "native_codex_profile",
+ "native_codex_profile_environment": "native_codex_profile",
+ "normalize_benchmark_adaptive_concurrency_policy": "adaptive_concurrency",
+ "normalize_benchmark_behavior_finding": "behavior_finding",
+ "normalize_benchmark_case_insight_projection": "study_projection",
+ "normalize_benchmark_concurrency_config": "concurrency_envelope",
+ "normalize_benchmark_concurrency_envelope": "concurrency_envelope",
+ "normalize_benchmark_concurrency_feedback": "adaptive_concurrency",
+ "normalize_benchmark_experiment_board_row": "experiment_board",
+ "normalize_benchmark_public_progress": "continuation",
+ "normalize_benchmark_resource_headroom_receipt": "concurrency_envelope",
+ "normalize_benchmark_rebalance_inventory": "rebalance_plan",
+ "normalize_benchmark_study_manifest": "study_projection",
+ "normalize_benchmark_token_capacity_policy": "token_capacity",
+ "normalize_benchmark_token_capacity_state": "token_capacity",
+ "normalize_benchmark_token_usage_sample": "token_capacity",
+ "normalize_benchmark_upload_envelope": "study_projection",
+ "observe_native_goal_event": "native_codex_goal",
+ "observe_benchmark_token_usage": "token_capacity",
+ "preview_benchmark_experiment_board_reconcile": "experiment_board",
+ "preview_benchmark_experiment_board_upsert": "experiment_board",
+ "probe_native_goal_process": "native_codex_goal",
+ "read_benchmark_concurrency_envelope": "concurrency_envelope",
+ "read_benchmark_experiment_board_rows": "experiment_board",
+ "read_benchmark_local_upload_records": "study_projection",
+ "read_benchmark_upload_receipt": "study_projection",
+ "read_benchmark_token_capacity_state": "token_capacity",
+ "rebase_native_codex_loopx_workspace_state": "native_codex_isolation",
+ "refresh_native_goal_status": "native_codex_goal",
+ "release_benchmark_case": "concurrency_envelope",
+ "record_benchmark_token_capacity_rejection": "token_capacity",
+ "release_benchmark_token_capacity": "token_capacity",
+ "render_benchmark_concurrency_markdown": "concurrency_envelope",
+ "render_benchmark_rebalance_plan_markdown": "rebalance_plan",
+ "render_benchmark_token_capacity_markdown": "token_capacity",
+ "render_benchmark_experiment_board_markdown": "experiment_board",
+ "render_native_codex_goal_prompt": "native_codex_profile",
+ "run_native_goal_process": "native_codex_goal",
+ "run_native_goal_process_until_terminal": "native_codex_goal",
+ "run_native_goal_turn": "native_codex_goal",
+ "run_native_goal_until_terminal": "native_codex_goal",
+ "select_exact_docker_container": "container_binding",
+ "serve_runner_owned_provider_gateway": "provider_gateway",
+ "simulate_benchmark_upload": "study_projection",
+ "start_native_goal_turn": "native_codex_goal",
+ "tune_benchmark_concurrency_target": "adaptive_concurrency",
+ "upsert_benchmark_experiment_board_row": "experiment_board",
+ "validate_run_permission_policy": "run_permissions",
+ "verify_verifier_reward_file": "reward_contract",
+ "verify_verifier_reward_json": "reward_contract",
+ "wait_native_goal_turn": "native_codex_goal",
+}
+
+__all__ = list(_EXPORTS)
+
+
+def __getattr__(name: str) -> Any:
+ owner = _EXPORTS.get(name)
+ if owner is not None:
+ value = getattr(import_module(f".{owner}", __name__), name)
+ elif name in _EXPORTS.values():
+ # Preserve explicitly accessed child-module attributes from the former
+ # eager package facade without loading the other owners.
+ value = import_module(f".{name}", __name__)
+ else:
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+ globals()[name] = value
+ return value
+
+
+def __dir__() -> list[str]:
+ return sorted(set(globals()) | set(__all__) | set(_EXPORTS.values()))
diff --git a/loopx/capabilities/benchmark_toolkit/catalog_entry.py b/loopx/capabilities/benchmark_toolkit/catalog_entry.py
index 05418399ee..c4f9fde463 100644
--- a/loopx/capabilities/benchmark_toolkit/catalog_entry.py
+++ b/loopx/capabilities/benchmark_toolkit/catalog_entry.py
@@ -2,11 +2,29 @@
from typing import Any
+from ...python_install_owner import BENCHMARK_TOOLKIT_DISTRIBUTION, builtin_capability_installed
from .four_arm_contract import (
BENCHMARK_FOUR_ARM_ATTESTATIONS,
BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS,
)
+BENCHMARK_TOOLKIT_INSTALL_HINT = (
+ "Install the matching loopx-capability-benchmark-toolkit wheel "
+ "or use the ordinary LoopX installation."
+)
+
+
+def benchmark_toolkit_installed(*, builtin_cli_present: bool | None = None) -> bool:
+ return builtin_capability_installed(
+ BENCHMARK_TOOLKIT_DISTRIBUTION,
+ (
+ "loopx.cli_commands.benchmark_dispatch",
+ "loopx.capabilities.benchmark_toolkit.trajectory_assessment_store",
+ ),
+ builtin_cli_present=builtin_cli_present,
+ )
+
+
BENCHMARK_TOOLKIT_CATALOG_ENTRY: dict[str, Any] = {
"id": "benchmark-toolkit",
"origin": "builtin",
diff --git a/loopx/capabilities/catalog.py b/loopx/capabilities/catalog.py
index 13b7571ac1..2cfe2a4a7a 100644
--- a/loopx/capabilities/catalog.py
+++ b/loopx/capabilities/catalog.py
@@ -1,11 +1,17 @@
from __future__ import annotations
from collections.abc import Iterable, Mapping
+from importlib.util import find_spec
from pathlib import Path
from typing import Any
from ..extensions.runtime import extension_catalog_entries
-from .benchmark_toolkit.catalog_entry import BENCHMARK_TOOLKIT_CATALOG_ENTRY
+from ..python_install_owner import BENCHMARK_TOOLKIT_DISTRIBUTION
+from .benchmark_toolkit.catalog_entry import (
+ BENCHMARK_TOOLKIT_CATALOG_ENTRY,
+ BENCHMARK_TOOLKIT_INSTALL_HINT,
+ benchmark_toolkit_installed,
+)
from .integration_branch.catalog_entry import INTEGRATION_BRANCH_CATALOG_ENTRY
from .repository_change_window.catalog_entry import (
REPOSITORY_CHANGE_WINDOW_CATALOG_ENTRY,
@@ -155,7 +161,19 @@ def build_capability_registry(
"ready": True,
}
)
+ # Split installs publish the matching toolkit owner's availability without
+ # importing the command group or its execution modules.
+ has_full_cli = find_spec("loopx.cli") is not None
+ has_benchmark = not has_full_cli and benchmark_toolkit_installed(builtin_cli_present=False)
+ if not has_full_cli:
+ registry.register_provider(
+ {"id": BENCHMARK_TOOLKIT_DISTRIBUTION, "origin": "builtin", "installed": has_benchmark}
+ )
for record in BUILTIN_CAPABILITIES:
+ if record is BENCHMARK_TOOLKIT_CATALOG_ENTRY and not has_full_cli:
+ record = record | {"provider_id": BENCHMARK_TOOLKIT_DISTRIBUTION}
+ if not has_benchmark:
+ record["next_real_step"] = BENCHMARK_TOOLKIT_INSTALL_HINT
registry.register_capability(record)
for record in BUILTIN_CAPABILITIES:
_register_declared_implementation_providers(registry, record)
diff --git a/loopx/cli_runtime.py b/loopx/cli_runtime.py
index bc674449aa..acb5bbebf7 100644
--- a/loopx/cli_runtime.py
+++ b/loopx/cli_runtime.py
@@ -722,9 +722,16 @@ def _dispatch_common_command(
build_command_reference_payload, render_command_reference_markdown,
)
+ available_commands = None
+ if _standalone_core():
+ from .capabilities.benchmark_toolkit.catalog_entry import benchmark_toolkit_installed
+
+ available_commands = _SELECTED_COMMANDS
+ if benchmark_toolkit_installed(builtin_cli_present=False):
+ available_commands |= {"benchmark"}
print_payload(
build_command_reference_payload(
- available_commands=_SELECTED_COMMANDS if _standalone_core() else None,
+ available_commands=available_commands,
), output_format(args),
render_command_reference_markdown,
)
@@ -932,6 +939,12 @@ def _dispatch_selected(args: argparse.Namespace, raw_argv: list[str]) -> int:
args, registry_path=registry_path, runtime_root_arg=args.runtime_root,
print_payload=print_payload,
)
+ elif args.command == "benchmark":
+ from .cli_commands.benchmark_dispatch import handle_benchmark_command
+
+ result = handle_benchmark_command(
+ args, registry_path=registry_path, print_payload=print_payload, output_format=output_format,
+ )
else:
result = dispatch_common_command(
args,
@@ -1011,6 +1024,18 @@ def main(argv: list[str] | None = None) -> int:
def _run_command(command: str | None, raw_argv: list[str]) -> int:
+ if command == "benchmark" and _standalone_core():
+ from .capabilities.benchmark_toolkit.catalog_entry import (
+ BENCHMARK_TOOLKIT_INSTALL_HINT, benchmark_toolkit_installed,
+ )
+
+ parser, subparsers = build_cli_parser()
+ if not benchmark_toolkit_installed(builtin_cli_present=False):
+ parser.error(f"benchmark is unavailable in the standalone core. {BENCHMARK_TOOLKIT_INSTALL_HINT}")
+ from .cli_commands.benchmark_dispatch import register_benchmark_command_group
+
+ register_benchmark_command_group(subparsers, add_subcommand_format)
+ return _dispatch_selected(parse_cli_args(parser, raw_argv), raw_argv)
if command == "version":
version_exports = sys.modules.get("loopx.cli_commands")
# Inspect bound exports without triggering the lazy facade. Explicit
diff --git a/loopx/presentation/research_analysis_api.py b/loopx/presentation/research_analysis_api.py
index fd63c36ce6..a5bfa93e89 100644
--- a/loopx/presentation/research_analysis_api.py
+++ b/loopx/presentation/research_analysis_api.py
@@ -6,7 +6,10 @@
"""
from urllib.parse import parse_qs, urlparse
-from ..capabilities.benchmark_toolkit import trajectory_assessment_store as store
+from ..capabilities.benchmark_toolkit.catalog_entry import (
+ BENCHMARK_TOOLKIT_INSTALL_HINT,
+ benchmark_toolkit_installed,
+)
from ..feedback import validate_goal_id
from ..control_plane.effect_runtime import EffectRuntimeRejected
from .goal_source_routing import goal_source_catalog, linked_goal_source
@@ -15,6 +18,18 @@
class ResearchAnalysisRequestMixin:
+ def _research_analysis_store(self):
+ if not benchmark_toolkit_installed():
+ self._send_error(
+ BENCHMARK_TOOLKIT_INSTALL_HINT,
+ status=503,
+ error_code="research_analysis_unavailable",
+ )
+ return None
+ from ..capabilities.benchmark_toolkit import trajectory_assessment_store
+
+ return trajectory_assessment_store
+
def _research_analysis_owner_goal(self, goal_id):
linked = linked_goal_source(self.server, goal_id)
return linked[1].source.goal_id if linked is not None else goal_id
@@ -31,7 +46,7 @@ def _research_analysis_query(self, goal_id=None):
raise ValueError("Analysis source identity changed")
return query
- def _research_analysis_verify_source_goal(self, runtime, goal_id, pack_id, pack=None):
+ def _research_analysis_verify_source_goal(self, runtime, goal_id, pack_id, pack=None, *, store):
linked = linked_goal_source(self.server, goal_id)
if getattr(self.server, "analysis_source_namespace", None) is not None or linked is not None:
owner_goal_id = self._research_analysis_owner_goal(goal_id)
@@ -66,6 +81,7 @@ def _research_analysis_scope(self, goal_id, *, cancel=False):
return self._goal_result_scope(goal_id)
def _research_analysis_read(self):
+ store = None
try:
query = self._research_analysis_query()
usage_turn = urlparse(self.path).path.endswith("/usage-turn")
@@ -84,6 +100,9 @@ def _research_analysis_read(self):
runtime = self._research_analysis_scope(goal_id)
if runtime is None:
return
+ store = self._research_analysis_store()
+ if store is None:
+ return
owner_goal_id = self._research_analysis_owner_goal(goal_id)
if "compare_pack_id" in values:
if "pack_id" not in values:
@@ -92,15 +111,15 @@ def _research_analysis_read(self):
pack_id=values["pack_id"], output_language=language)
right = store.inspect_pack(runtime_root=runtime, owner_goal_id=owner_goal_id,
pack_id=values["compare_pack_id"], output_language=language)
- self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"], left)
- self._research_analysis_verify_source_goal(runtime, goal_id, values["compare_pack_id"], right)
+ self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"], left, store=store)
+ self._research_analysis_verify_source_goal(runtime, goal_id, values["compare_pack_id"], right, store=store)
result = {"ok": True, "comparison": store.compare_captured_usage(left, right)}
elif usage_turn:
- self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"])
+ self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"], store=store)
result = store.read_usage_turn(runtime_root=runtime, owner_goal_id=owner_goal_id,
output_language=language, **values)
elif citation:
- self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"])
+ self._research_analysis_verify_source_goal(runtime, goal_id, values["pack_id"], store=store)
first = int(values.pop("first_line"))
last = int(values.pop("last_line"))
if first < 1 or last < first:
@@ -112,21 +131,23 @@ def _research_analysis_read(self):
pack_id = values.get("pack_id") or catalog["default_pack_id"]
pack = store.inspect_pack(runtime_root=runtime, owner_goal_id=owner_goal_id, pack_id=pack_id, output_language=language) if pack_id else None
if pack_id:
- self._research_analysis_verify_source_goal(runtime, goal_id, pack_id, pack)
+ self._research_analysis_verify_source_goal(runtime, goal_id, pack_id, pack, store=store)
result = {"ok": True, "catalog": catalog, "pack": pack}
if hasattr(self.server, "analysis_execution_enabled"):
result["execution_enabled"] = self.server.analysis_execution_enabled
elif linked_goal_source(self.server, goal_id) is not None:
result["execution_enabled"] = True
self._send_json(result)
- except store.PackUnavailable as exc:
- self._send_error("The registered analysis source is unavailable.", status=409, error_code=exc.reason_code)
- except (ValueError, TypeError, KeyError, EffectRuntimeRejected):
- self._send_error("Analysis needs a registered source in this Goal.", status=400, error_code="research_analysis_scope_invalid")
+ except (ValueError, TypeError, KeyError, EffectRuntimeRejected) as exc:
+ if store is not None and isinstance(exc, store.PackUnavailable):
+ self._send_error("The registered analysis source is unavailable.", status=409, error_code=exc.reason_code)
+ else:
+ self._send_error("Analysis needs a registered source in this Goal.", status=400, error_code="research_analysis_scope_invalid")
except (OSError, RuntimeError):
self._send_error("Analysis could not be read. Research history remains available.", status=503, error_code="research_analysis_unavailable")
def _research_analysis_write(self):
+ store = None
try:
if getattr(self.server, "analysis_source_namespace", None) is not None and self._research_analysis_query():
raise ValueError("Only the bound analysis source selector is accepted")
@@ -148,17 +169,21 @@ def _research_analysis_write(self):
runtime = self._research_analysis_scope(body["goal_id"], cancel=cancel)
if runtime is None:
return
+ store = self._research_analysis_store()
+ if store is None:
+ return
args = dict(runtime_root=runtime, owner_goal_id=self._research_analysis_owner_goal(body["goal_id"]),
pack_id=body["pack_id"], analysis_key=body["analysis_key"], output_language=body.get("output_language"))
- self._research_analysis_verify_source_goal(runtime, body["goal_id"], body["pack_id"])
+ self._research_analysis_verify_source_goal(runtime, body["goal_id"], body["pack_id"], store=store)
# The server's existing threaded request is the asynchronous host.
# Provider key/claim, source verification and cancellation
# remain with benchmark-toolkit; nothing executes on GET.
job = store.cancel_pack(**args) if cancel else store.execute_pack(**args, retry=body.get("retry", False))
self._send_json({"ok": True, "job": job})
- except store.PackUnavailable as exc:
- self._send_error("Analysis source changed or is unavailable; refresh before retrying.", status=409, error_code=exc.reason_code)
- except (ValueError, TypeError, KeyError, EffectRuntimeRejected):
- self._send_error("Analysis request was not accepted. Refresh its source and retry.", status=400, error_code="research_analysis_request_invalid")
+ except (ValueError, TypeError, KeyError, EffectRuntimeRejected) as exc:
+ if store is not None and isinstance(exc, store.PackUnavailable):
+ self._send_error("Analysis source changed or is unavailable; refresh before retrying.", status=409, error_code=exc.reason_code)
+ else:
+ self._send_error("Analysis request was not accepted. Refresh its source and retry.", status=400, error_code="research_analysis_request_invalid")
except (OSError, RuntimeError):
self._send_error("Analysis did not complete. Read its status before retrying.", status=503, error_code="research_analysis_unavailable")
diff --git a/loopx/python_install_owner.py b/loopx/python_install_owner.py
index 8270d105ce..89b3c161d3 100644
--- a/loopx/python_install_owner.py
+++ b/loopx/python_install_owner.py
@@ -9,7 +9,8 @@
from dataclasses import dataclass
from collections.abc import Callable
-from importlib.metadata import Distribution, PackageNotFoundError
+from importlib.metadata import Distribution, PackageNotFoundError, distribution
+from importlib.util import find_spec
import json
import os
from pathlib import Path
@@ -29,6 +30,42 @@ def installed_loopx_distribution(lookup: Callable[[str], Distribution]) -> Distr
return lookup("loopx-core")
+def builtin_capability_installed(
+ distribution_name: str,
+ modules: tuple[str, ...],
+ *,
+ builtin_cli_present: bool | None = None,
+) -> bool:
+ """Check import locations and matching installed wheel ownership without loading code."""
+ if builtin_cli_present is None:
+ builtin_cli_present = find_spec("loopx.cli") is not None
+ if not builtin_cli_present:
+ try:
+ implementation = distribution(distribution_name)
+ if implementation.version != distribution("loopx-core").version:
+ return False
+ except PackageNotFoundError:
+ return False
+ origins = [
+ Path(spec.origin).resolve()
+ for module in modules
+ if (spec := find_spec(module)) is not None and spec.origin is not None
+ ]
+ if len(origins) != len(modules):
+ return False
+ if builtin_cli_present:
+ return True
+ recorded = {
+ Path(str(implementation.locate_file(path))).resolve()
+ for path in implementation.files or ()
+ }
+ return all(origin in recorded for origin in origins)
+
+
+WORKSPACE_DISTRIBUTION = "loopx-workspace"
+BENCHMARK_TOOLKIT_DISTRIBUTION = "loopx-capability-benchmark-toolkit"
+
+
@dataclass(frozen=True)
class PythonInstallOwner:
manager: str
diff --git a/loopx/release_manifest.py b/loopx/release_manifest.py
index d0eb679d27..1961f35bf2 100644
--- a/loopx/release_manifest.py
+++ b/loopx/release_manifest.py
@@ -1,7 +1,7 @@
from __future__ import annotations
from datetime import datetime, timezone
-from importlib.metadata import PackageNotFoundError, distribution
+from importlib.metadata import Distribution, PackageNotFoundError, distribution
import hashlib
import json
import os
@@ -10,7 +10,11 @@
from . import __version__
from .doctor_git import GitSourceState, git_source_metadata
-from .python_install_owner import installed_loopx_distribution
+from .python_install_owner import (
+ BENCHMARK_TOOLKIT_DISTRIBUTION,
+ WORKSPACE_DISTRIBUTION,
+ installed_loopx_distribution,
+)
RELEASE_MANIFEST_SCHEMA_VERSION = "loopx_release_manifest_v0"
@@ -310,27 +314,48 @@ def _distribution_artifact_file(path: Path) -> bool:
return "__pycache__" not in path.parts and path.suffix not in {".pyc", ".pyo"}
+def _owned_runtime_files(installed: Distribution) -> dict[str, Path]:
+ direct_url = json.loads(installed.read_text("direct_url.json") or "{}")
+ if direct_url.get("dir_info", {}).get("editable") or installed.version != __version__:
+ return {}
+ return {
+ item.as_posix(): Path(item.locate())
+ for item in installed.files or ()
+ if item.as_posix().startswith("loopx/")
+ and _distribution_artifact_file(Path(item.as_posix()))
+ }
+
+
def _distribution_runtime_fingerprint() -> str | None:
"""Hash the actual imported, RECORD-owned wheel contents, not RECORD hashes.
+ Include the exact disjoint contents of the installed Core companion wheels.
Refuse editable, redirected, incomplete or ambiguous ownership. Bytecode
caches are generated by Python and are not part of the installed artifact.
"""
try:
installed = installed_loopx_distribution(distribution)
- direct_url = json.loads(installed.read_text("direct_url.json") or "{}")
- if direct_url.get("dir_info", {}).get("editable") or installed.version != __version__:
- return None
package_root = Path(__file__).resolve().parent
- files = {
- item.as_posix(): Path(item.locate())
- for item in installed.files or ()
- if item.as_posix().startswith("loopx/")
- and _distribution_artifact_file(Path(item.as_posix()))
- }
+ files = _owned_runtime_files(installed)
module = files.get("loopx/release_manifest.py")
if module is None or module.resolve() != Path(__file__).resolve():
return None
+ if installed.name == "loopx-core":
+ for name in (
+ WORKSPACE_DISTRIBUTION,
+ BENCHMARK_TOOLKIT_DISTRIBUTION,
+ "loopx-capability-agent-turn-recall",
+ "loopx-capability-pull-request-review",
+ "loopx-capability-issue-fix",
+ ):
+ try:
+ companion = distribution(name)
+ except PackageNotFoundError:
+ continue
+ owned = _owned_runtime_files(companion)
+ if not owned or files.keys() & owned.keys():
+ return None
+ files.update(owned)
entries = list(package_root.rglob("*"))
if any(path.is_symlink() for path in entries):
return None
diff --git a/packages/README.md b/packages/README.md
index ba612e0714..f21ce86cd9 100644
--- a/packages/README.md
+++ b/packages/README.md
@@ -26,6 +26,8 @@ Current co-located distributions, in directory order:
- [`loopx-ark-turn`](loopx-ark-turn/README.md): an optional host for one
LoopX-governed work unit on Ark Managed Agents (Python distribution, no
`extension.toml`; selected with `turn run-once --host generic-cli`);
+- [`loopx-capability-benchmark-toolkit`](loopx-capability-benchmark-toolkit/README.md)
+ adds benchmark commands and registered analysis packs to the matching Core.
- [`loopx-codex-provider-routing`](loopx-codex-provider-routing/README.md):
native Codex account slots, allowance observation and automatic handoff at
safe boundaries, with legacy CPA qualification and rollback compatibility;
diff --git a/packages/loopx-capability-benchmark-toolkit/README.md b/packages/loopx-capability-benchmark-toolkit/README.md
new file mode 100644
index 0000000000..506a654953
--- /dev/null
+++ b/packages/loopx-capability-benchmark-toolkit/README.md
@@ -0,0 +1,36 @@
+# Benchmark Toolkit for LoopX Core
+
+Install this source-built wheel when you need benchmark commands or registered
+analysis packs. It uses the same version of [LoopX Core](../loopx-core/README.md)
+and the same Goal registry and runtime state. Capability discovery reads its
+catalog without loading the toolkit's execution modules.
+
+Build both wheels from the same checkout and add the toolkit to your Core environment.
+
+```sh
+uv run --extra test python packages/loopx-core/prepare.py --distribution loopx-capability-benchmark-toolkit --output /tmp/loopx-bench-source
+uv build --wheel /tmp/loopx-bench-source --out-dir /tmp/loopx-bench-dist
+uv pip install --python /tmp/loopx-core-env/bin/python /tmp/loopx-bench-dist/loopx_capability_benchmark_toolkit-*.whl
+```
+
+Check readiness and discover a registered Goal's analysis packs using the same
+registry and runtime you used to run it.
+
+```sh
+/tmp/loopx-core-env/bin/loopx capability show benchmark-toolkit
+/tmp/loopx-core-env/bin/loopx --registry .loopx/registry.json benchmark trajectory-packs --goal-id ""
+```
+
+Pass the original `--runtime-root` if you supplied one. The toolkit exposes the
+existing authority and evidence checks. See the
+[capability guide](../../loopx/capabilities/benchmark_toolkit/README.md) for
+benchmark families and execution prerequisites.
+
+When using a matching Workspace wheel, restart Workspace after adding or
+removing the toolkit. Research Analysis reads the same registered packs.
+Remove the toolkit when you no longer need those commands. The existing Goal
+registry, task history and analysis records remain in your runtime directory.
+
+```sh
+uv pip uninstall --python /tmp/loopx-core-env/bin/python loopx-capability-benchmark-toolkit
+```
diff --git a/packages/loopx-capability-benchmark-toolkit/pyproject.toml b/packages/loopx-capability-benchmark-toolkit/pyproject.toml
new file mode 100644
index 0000000000..9a74fb9d8e
--- /dev/null
+++ b/packages/loopx-capability-benchmark-toolkit/pyproject.toml
@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools==83.0.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "loopx-capability-benchmark-toolkit"
+dynamic = ["version", "dependencies"]
+description = "Optional Benchmark Toolkit for the matching LoopX Core release."
+readme = "README.md"
+requires-python = ">=3.11"
+license = "Apache-2.0"
+license-files = ["LICENSE", "NOTICE", "LICENSE-MIT"]
+
+[tool.setuptools.dynamic]
+version = {file = "VERSION"}
+dependencies = {file = "requires-core.txt"}
+
+[tool.setuptools.packages.find]
+include = ["loopx.capabilities.benchmark_toolkit", "loopx.capabilities.benchmark_toolkit.*", "loopx.cli_commands"]
+
+[tool.setuptools.package-data]
+"*" = ["*.ts", "*.json", "*.mjs", "*.js", "*.md", "*.toml"]
diff --git a/packages/loopx-core/README.md b/packages/loopx-core/README.md
index ae5ef9df5d..c4b797f9e3 100644
--- a/packages/loopx-core/README.md
+++ b/packages/loopx-core/README.md
@@ -48,6 +48,12 @@ The installed-only qualification in
exercises setup, independent result validation, process loss, replay and refusal.
No model service, credentials or paid task execution is supplied by this wheel.
+Core keeps the benchmark catalog and the four-arm contract it references.
+Install the matching [Benchmark Toolkit wheel](../loopx-capability-benchmark-toolkit/README.md)
+to run `loopx benchmark` or use registered analysis packs in Workspace.
+`loopx capability show benchmark-toolkit` reports availability and installation
+guidance. Normal discovery leaves benchmark execution unloaded.
+
Goal-start help and preview use issue-fix command metadata without importing
its execution providers. An explicit `--capability-route issue-fix` retains
the existing candidate-preflight contract. The installed context qualification
@@ -66,7 +72,7 @@ new default, migration, automatic upgrade, PyPI publication or platform release.
This is the first independently installable core-artifact slice of R1/M3 in the
[small-kernel RFC](../../docs/architecture/rfcs/monorepo-distribution-split-v0.md).
-It still retains built-in capability implementations, policy helpers and several
+It still retains other built-in capability implementations, policy helpers and several
host/CLI dependencies because the current imports are coupled. This makes it a
workspace-free installation prerequisite, not a qualified minimal mechanism
closure. Those dependencies are migration debt, not evidence that every
diff --git a/packages/loopx-core/prepare.py b/packages/loopx-core/prepare.py
index d53291b160..0322cd9ad4 100644
--- a/packages/loopx-core/prepare.py
+++ b/packages/loopx-core/prepare.py
@@ -1,4 +1,4 @@
-"""Prepare a standalone core source distribution from the single source tree.
+"""Prepare standalone Core and its matching benchmark wheel from one source tree.
Build the returned directory with a PEP 517 frontend. No runtime code is copied
back into the repository and the ordinary distribution remains unchanged.
@@ -9,31 +9,59 @@
import shutil
import subprocess
from pathlib import Path
+from runpy import run_path
REPOSITORY = Path(__file__).resolve().parents[2]
# These are distribution exclusions, not an authority or activation policy.
-EXCLUDED_TREES = ("loopx/chat/", "loopx/web/", "loopx/canary/")
-EXCLUDED_MODULES = {"loopx/cli.py", "loopx/chat.py"}
+EXCLUDED_PREFIXES = (
+ "loopx/chat/",
+ "loopx/web/",
+ "loopx/canary/",
+)
+BENCHMARK_DISTRIBUTION = "loopx-capability-benchmark-toolkit"
+BENCHMARK_EXECUTION_PREFIXES = (
+ "loopx/capabilities/benchmark_toolkit/",
+ "loopx/cli_commands/benchmark_",
+)
+EXCLUDED_MODULES = {
+ "loopx/cli.py",
+ "loopx/chat.py",
+ "loopx/presentation/research_analysis_api.py",
+ "loopx/presentation/runtime_workspace.py",
+}
+# Keep the catalog in a namespace package so discovery skips the execution facade.
+BENCHMARK_CATALOG_FILES = {
+ "loopx/capabilities/benchmark_toolkit/README.md",
+ "loopx/capabilities/benchmark_toolkit/catalog_entry.py",
+ "loopx/capabilities/benchmark_toolkit/four_arm_contract.py",
+}
SOURCE_SUFFIXES = {".py", ".ts", ".json", ".mjs", ".js", ".md", ".toml"}
+def _benchmark_execution_source(path: Path) -> bool:
+ name = path.as_posix()
+ return name.startswith(BENCHMARK_EXECUTION_PREFIXES) and name not in BENCHMARK_CATALOG_FILES
+
+
def prepare_core_source(output: Path, *, repository: Path = REPOSITORY) -> Path:
repository = repository.resolve()
- output = output.resolve()
- if output == repository or output.is_relative_to(repository):
- raise ValueError("prepare the core distribution outside the source checkout")
- if output.exists() and any(output.iterdir()):
- raise ValueError("the output directory must be empty")
- names = subprocess.check_output(
- ["git", "ls-files", "--cached", "-z", "--", "loopx"], cwd=repository,
- ).split(b"\0")
- sources = [Path(name.decode("utf-8")) for name in names if name]
- sources = [path for path in sources if path.suffix in SOURCE_SUFFIXES
+ sources = [path for path in _tracked_loopx_sources(repository)
+ if path.suffix in SOURCE_SUFFIXES
and path.as_posix() not in EXCLUDED_MODULES
and not (path.parent == Path("loopx") and path.name.startswith("chat_"))
- and not path.as_posix().startswith(EXCLUDED_TREES)]
+ and not path.as_posix().startswith(EXCLUDED_PREFIXES)
+ and not _benchmark_execution_source(path)]
if not sources or not (repository / "loopx/entrypoint.py").is_file():
raise ValueError("a complete LoopX source checkout is required")
+ return _stage_source(output, repository, "loopx-core", sources)
+
+
+def _stage_source(output: Path, repository: Path, distribution: str, sources: list[Path]) -> Path:
+ output = output.resolve()
+ if output == repository or output.is_relative_to(repository):
+ raise ValueError("prepare the distribution outside the source checkout")
+ if output.exists() and any(output.iterdir()):
+ raise ValueError("the output directory must be empty")
output.mkdir(parents=True, exist_ok=True)
for relative in sources:
source = repository / relative
@@ -42,7 +70,7 @@ def prepare_core_source(output: Path, *, repository: Path = REPOSITORY) -> Path:
target = output / relative
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(source, target)
- package = repository / "packages/loopx-core"
+ package = repository / "packages" / distribution
for name in ("pyproject.toml", "README.md"):
shutil.copyfile(package / name, output / name)
for name in ("LICENSE", "LICENSE-MIT", "NOTICE"):
@@ -53,11 +81,33 @@ def prepare_core_source(output: Path, *, repository: Path = REPOSITORY) -> Path:
return output
+def _tracked_loopx_sources(repository: Path) -> list[Path]:
+ names = subprocess.check_output(
+ ["git", "ls-files", "--cached", "-z", "--", "loopx"], cwd=repository,
+ ).split(b"\0")
+ return [Path(name.decode("utf-8")) for name in names if name]
+
+
+def prepare_benchmark_source(output: Path, *, repository: Path = REPOSITORY) -> Path:
+ repository = repository.resolve()
+ sources = [path for path in _tracked_loopx_sources(repository)
+ if path.suffix in SOURCE_SUFFIXES and _benchmark_execution_source(path)]
+ if Path("loopx/cli_commands/benchmark_dispatch.py") not in sources:
+ raise ValueError("the complete benchmark toolkit source is required")
+ output = _stage_source(output, repository, BENCHMARK_DISTRIBUTION, sources)
+ version = run_path(str(repository / "loopx/__init__.py"))["__version__"]
+ (output / "VERSION").write_text(f"{version}\n", encoding="utf-8")
+ (output / "requires-core.txt").write_text(f"loopx-core=={version}\n", encoding="utf-8")
+ return output
+
+
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--distribution", choices=["loopx-core", BENCHMARK_DISTRIBUTION], default="loopx-core")
args = parser.parse_args()
- print(prepare_core_source(args.output))
+ prepare = prepare_benchmark_source if args.distribution == BENCHMARK_DISTRIBUTION else prepare_core_source
+ print(prepare(args.output))
return 0
diff --git a/tests/capabilities/test_benchmark_toolkit_loading.py b/tests/capabilities/test_benchmark_toolkit_loading.py
new file mode 100644
index 0000000000..02c4f42eb4
--- /dev/null
+++ b/tests/capabilities/test_benchmark_toolkit_loading.py
@@ -0,0 +1,81 @@
+"""Discovery must not depend on optional benchmark execution implementations."""
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+
+import pytest
+
+
+DISCOVERY_SCRIPT = '''import importlib.abc,json,sys
+class MetadataOnly(importlib.abc.MetaPathFinder):
+ def find_spec(self,fullname,path=None,target=None):
+ prefix='loopx.capabilities.benchmark_toolkit.'
+ if fullname.startswith(prefix) and fullname not in (prefix+'catalog_entry',prefix+'four_arm_contract'):
+ raise AssertionError('unselected benchmark implementation was imported: '+fullname)
+sys.meta_path.insert(0,MetadataOnly())
+def no_execution(event,args):
+ if event in ('subprocess.Popen','os.system','os.posix_spawn','socket.connect'):
+ raise AssertionError('discovery executed an external operation: '+event)
+sys.addaudithook(no_execution)
+from loopx.entrypoint import main
+raise SystemExit(main(json.loads(sys.argv[1])))
+'''
+
+
+@pytest.mark.parametrize("arguments", [
+ ["capability", "--help"],
+ ["capability", "list", "--format", "json"],
+ ["capability", "show", "benchmark-toolkit", "--format", "json"],
+ ["capability", "show", "issue-fix", "--format", "json"],
+])
+def test_capability_discovery_survives_unavailable_benchmark_execution(tmp_path, arguments):
+ result = subprocess.run([sys.executable, "-I", "-c", DISCOVERY_SCRIPT,
+ json.dumps(["--registry", str(tmp_path / "registry.json"),
+ "--runtime-root", str(tmp_path / "runtime"), *arguments])],
+ cwd=tmp_path, env={**os.environ, "PATH": "", "LOOPX_USAGE_PING": "0"},
+ text=True, capture_output=True, timeout=30)
+ assert result.returncode == 0, (result.stdout, result.stderr)
+ if arguments[-1] == "json":
+ assert json.loads(result.stdout)["ok"] is True
+ assert not list(tmp_path.iterdir())
+
+
+def test_toolkit_import_and_selected_export_leave_other_implementations_unloaded(tmp_path):
+ source = '''import importlib.abc,sys
+class SelectedOnly(importlib.abc.MetaPathFinder):
+ def find_spec(self,fullname,path=None,target=None):
+ if fullname.startswith('loopx.capabilities.benchmark_toolkit.') and not fullname.endswith('.four_arm_contract'):
+ raise ModuleNotFoundError('optional implementation unavailable',name=fullname)
+sys.meta_path.insert(0,SelectedOnly())
+import loopx.capabilities.benchmark_toolkit as toolkit
+assert not any(name.startswith(toolkit.__name__+'.') for name in sys.modules)
+assert 'build_benchmark_four_arm_contract' in dir(toolkit)
+from loopx.capabilities.benchmark_toolkit import build_benchmark_four_arm_contract
+from loopx.capabilities.benchmark_toolkit.four_arm_contract import build_benchmark_four_arm_contract as canonical
+assert build_benchmark_four_arm_contract is canonical
+assert toolkit.build_benchmark_four_arm_contract is canonical
+assert toolkit.four_arm_contract.build_benchmark_four_arm_contract is canonical
+try:toolkit.run_native_goal_turn
+except ModuleNotFoundError as exc:assert exc.name==toolkit.__name__+'.native_codex_goal'
+else:raise AssertionError('a selected missing implementation must remain an error')
+try:toolkit.unknown_export
+except AttributeError:pass
+else:raise AssertionError('unknown exports must remain unavailable')
+'''
+ result = subprocess.run([sys.executable, "-I", "-c", source], cwd=tmp_path,
+ env={**os.environ, "LOOPX_USAGE_PING": "0"}, text=True, capture_output=True, timeout=30)
+ assert result.returncode == 0, (result.stdout, result.stderr)
+
+
+def test_explicit_star_import_retains_public_exports():
+ import loopx.capabilities.benchmark_toolkit as toolkit
+ from loopx.capabilities.benchmark_toolkit import native_codex_goal
+
+ exports = {}
+ exec("from loopx.capabilities.benchmark_toolkit import *", exports)
+ assert {name for name in exports if not name.startswith("_")} == set(toolkit.__all__)
+ assert exports["NativeGoalConfig"] is native_codex_goal.NativeGoalConfig
+ assert exports["run_native_goal_turn"] is native_codex_goal.run_native_goal_turn
diff --git a/tests/test_core_benchmark_distribution.py b/tests/test_core_benchmark_distribution.py
new file mode 100644
index 0000000000..4c6763fa0d
--- /dev/null
+++ b/tests/test_core_benchmark_distribution.py
@@ -0,0 +1,146 @@
+"""Installed Benchmark Toolkit over a checked Core task and an offline case pack."""
+
+from hashlib import sha256
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+from zipfile import ZipFile
+
+import pytest
+
+from tests.test_core_distribution import GOAL, _assert_committed, _environment, _run
+from tests.test_core_distribution import task as task
+
+
+@pytest.fixture
+def wheels():
+ core = os.environ.get("LOOPX_CORE_TEST_WHEEL")
+ toolkit = os.environ.get("LOOPX_BENCHMARK_TEST_WHEEL")
+ if not core or not toolkit:
+ pytest.skip("set LOOPX_CORE_TEST_WHEEL and LOOPX_BENCHMARK_TEST_WHEEL to the built artifacts")
+ return Path(core).resolve(strict=True), Path(toolkit).resolve(strict=True)
+
+
+@pytest.fixture
+def core_python(tmp_path, wheels):
+ uv = shutil.which("uv")
+ assert uv, "uv is required to qualify a clean split installation"
+ environment = tmp_path / "environment"
+ subprocess.run([uv, "venv", "--python", sys.executable, str(environment)], check=True, timeout=45)
+ python = str(environment / ("Scripts/python.exe" if os.name == "nt" else "bin/python"))
+ subprocess.run([uv, "pip", "install", "--offline", "--python", python, str(wheels[0])],
+ check=True, timeout=45)
+ return python
+
+
+def test_benchmark_wheel_owns_execution_and_keeps_the_catalog_in_core(wheels):
+ with ZipFile(wheels[0]) as core, ZipFile(wheels[1]) as toolkit:
+ assert not set(core.namelist()) & set(toolkit.namelist())
+ assert "loopx/capabilities/benchmark_toolkit/catalog_entry.py" in core.namelist()
+ assert "loopx/capabilities/benchmark_toolkit/four_arm_contract.py" in core.namelist()
+ assert "loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py" in toolkit.namelist()
+ assert "loopx/cli_commands/benchmark_dispatch.py" in toolkit.namelist()
+ assert "loopx/cli.py" not in toolkit.namelist()
+
+
+def _python_json(python, cwd, source):
+ result = subprocess.run([python, "-I", "-c", source], cwd=cwd, env=_environment(python),
+ capture_output=True, text=True, check=True, timeout=30)
+ return json.loads(result.stdout)
+
+
+_IDENTITY = """import json
+from loopx.release_manifest import release_runtime_identity
+print(json.dumps(release_runtime_identity()))
+"""
+
+
+def test_installed_benchmark_reopens_the_same_analysis_after_removal(core_python, task, wheels, tmp_path):
+ workspace, runtime, registry, arguments = task
+ root = ["--registry", str(registry), "--runtime-root", str(runtime), "--format", "json"]
+ bench = [*root, "benchmark"]
+ discover = [*bench, "trajectory-packs", "--goal-id", GOAL]
+ before = _python_json(core_python, workspace, _IDENTITY)
+ assert before["package_fingerprint"]
+ assert "install the matching" in _run(core_python, discover, workspace, expected=2).stderr.lower()
+ turn = json.loads(_run(core_python, arguments, workspace).stdout)
+ turn_key = _assert_committed(core_python, workspace, runtime, turn)
+ registry_bytes = registry.read_bytes()
+
+ install = ["uv", "pip", "install", "--offline", "--python", core_python, str(wheels[1])]
+ subprocess.run(install, check=True, timeout=45)
+ probe = _python_json(core_python, workspace, """import contextlib, io, json, sys
+from importlib import metadata
+from pathlib import Path
+from loopx.entrypoint import main
+owner = metadata.distribution("loopx-capability-benchmark-toolkit")
+assert owner.requires == [f"loopx-core=={metadata.version('loopx-core')}"]
+with contextlib.redirect_stdout(io.StringIO()):
+ assert main(["capability", "list", "--format", "json"]) == 0
+ assert main(["commands", "--format", "json"]) == 0
+assert "loopx.cli_commands.benchmark_dispatch" not in sys.modules
+assert "loopx.capabilities.benchmark_toolkit.trajectory_assessment_store" not in sys.modules
+assert "loopx.capabilities.benchmark_toolkit.native_codex_goal" not in sys.modules
+record = next(record for record in owner.files if record.as_posix().endswith(".dist-info/METADATA"))
+print(json.dumps({"metadata": str(Path(owner.locate_file(record))), "version": owner.version}))
+""")
+ detail = [*root, "capability", "show", "benchmark-toolkit"]
+ record = json.loads(_run(core_python, detail, workspace).stdout)["capability"]
+ assert record["provider_id"] == "loopx-capability-benchmark-toolkit"
+ assert record["provider_state"] == {"declared": True, "installed": True, "enabled": True, "ready": True}
+ current = _python_json(core_python, workspace, _IDENTITY)
+ assert current["package_fingerprint"] != before["package_fingerprint"]
+ assert json.loads(_run(core_python, discover, workspace).stdout)["packs"] == []
+
+ path = Path(probe["metadata"])
+ contents = path.read_bytes()
+ try:
+ path.write_bytes(contents.replace(f"Version: {probe['version']}".encode(), b"Version: 0.0.0", 1))
+ rejected = json.loads(_run(core_python, detail, workspace).stdout)["capability"]["provider_state"]
+ assert rejected["installed"] is rejected["ready"] is False
+ _run(core_python, discover, workspace, expected=2)
+ finally:
+ path.write_bytes(contents)
+
+ evidence = b"Independent validation found eight expected values.\n"
+ source = tmp_path / "source"
+ source.mkdir()
+ (source / "check.txt").write_bytes(evidence)
+ input_file = tmp_path / "input.json"
+ payload = {
+ "schema_version": "trajectory_assessment_input_v1",
+ "scope": dict(goal_id=GOAL, case_id="checked-task", run_id="local-check", revision="fixed",
+ window="first accepted result"),
+ "objective": "Read the original independently accepted task evidence.",
+ "selection": dict(rule="One synthetic checked task", population_episodes=1, exhaustive=True),
+ "sources": [dict(id="check", artifact="check.txt", artifact_sha256=sha256(evidence).hexdigest(),
+ first_line=1, text=evidence.decode().strip())],
+ "episodes": [dict(id="episode", question="Did the independent check run?", source_ids=["check"],
+ eligible_metrics=["benefit_test"])],
+ }
+ register = [*bench, "trajectory-pack-register", "--goal-id", GOAL,
+ "--input-json", str(input_file), "--source-root", str(source), "--title", "Original checked result"]
+ payload["sources"][0]["artifact_sha256"] = "0" * 64
+ input_file.write_text(json.dumps(payload), encoding="utf-8")
+ _run(core_python, register, workspace, expected=1)
+ assert json.loads(_run(core_python, discover, workspace).stdout)["packs"] == []
+ payload["sources"][0]["artifact_sha256"] = sha256(evidence).hexdigest()
+ input_file.write_text(json.dumps(payload), encoding="utf-8")
+ pack = json.loads(_run(core_python, register, workspace).stdout)
+ observed = json.loads(_run(core_python, discover, workspace).stdout)
+ assert observed["default_pack_id"] == pack["pack_id"]
+ assert [item["title"] for item in observed["packs"]] == ["Original checked result"]
+
+ subprocess.run(["uv", "pip", "uninstall", "--python", core_python, "loopx-capability-benchmark-toolkit"],
+ check=True, timeout=45)
+ assert _python_json(core_python, workspace, _IDENTITY) == before
+ _run(core_python, discover, workspace, expected=2)
+ replay = json.loads(_run(core_python, [*arguments, "--resume-turn-key", turn_key], workspace).stdout)
+ assert not any(replay["effects"][name] for name in ("host_invoked", "state_written", "quota_spent"))
+ assert _assert_committed(core_python, workspace, runtime, replay) == turn_key
+ assert registry.read_bytes() == registry_bytes
+ subprocess.run(install, check=True, timeout=45)
+ assert json.loads(_run(core_python, discover, workspace).stdout) == observed
diff --git a/tests/test_core_distribution.py b/tests/test_core_distribution.py
index f39fae8197..9c0bec797e 100644
--- a/tests/test_core_distribution.py
+++ b/tests/test_core_distribution.py
@@ -59,6 +59,13 @@ def test_core_staging_excludes_workspace_and_reuses_kernel_bytes(tmp_path):
for name in ("chat", "web", "canary", "cli.py", "chat.py"):
assert not (staged / "loopx" / name).exists()
assert not list((staged / "loopx").glob("chat_*.py"))
+ package = staged / "loopx"
+ assert {file.name for file in (package / "capabilities/benchmark_toolkit").iterdir()} == {
+ "README.md", "catalog_entry.py", "four_arm_contract.py",
+ }
+ assert not list((package / "cli_commands").glob("benchmark_*.py"))
+ assert not (package / "presentation/research_analysis_api.py").exists()
+ assert not (package / "presentation/runtime_workspace.py").exists()
for name in ("entrypoint.py", "control_plane/turn_driver/executor.py",
"control_plane/coordination/sqlite_authority_store.ts",
"extensions/lark/extension.toml"):
@@ -132,6 +139,12 @@ def test_installed_core_identity_help_and_unavailable_workspace(core_python, tmp
script = '''import importlib.util as u,json,loopx
from pathlib import Path
for name in ('loopx.cli','loopx.chat','loopx.web','loopx.canary'):assert u.find_spec(name) is None,name
+for name in ('loopx.capabilities.benchmark_toolkit.native_codex_goal',
+ 'loopx.capabilities.benchmark_toolkit.experiment_board',
+ 'loopx.capabilities.benchmark_toolkit.trajectory_assessment_store',
+ 'loopx.cli_commands.benchmark_dispatch','loopx.presentation.research_analysis_api',
+ 'loopx.presentation.runtime_workspace'):
+ assert u.find_spec(name) is None,name
from loopx.release_manifest import _distribution_runtime_fingerprint
assert _distribution_runtime_fingerprint() is not None
from loopx import doctor
@@ -158,6 +171,26 @@ def test_installed_core_identity_help_and_unavailable_workspace(core_python, tmp
assert not list(tmp_path.iterdir())
+def test_installed_core_lists_benchmark_as_unavailable(core_python, tmp_path):
+ catalog = json.loads(_run(core_python, ["capability", "list", "--format", "json"], tmp_path,
+ environment={"PATH": str(tmp_path)}).stdout)
+ records = {record["id"]: record for record in catalog["capabilities"]}
+ detail = json.loads(_run(core_python, ["capability", "show", "benchmark-toolkit", "--format", "json"],
+ tmp_path).stdout)["capability"]
+ unavailable = {"declared": True, "installed": False, "enabled": False, "ready": False}
+ assert records["decision-context"]["provider_state"]["ready"] is True
+ for record in (records["benchmark-toolkit"], detail):
+ assert record["provider_id"] == "loopx-capability-benchmark-toolkit"
+ assert record["provider_state"] == unavailable
+ assert "ordinary LoopX installation" in record["next_real_step"]
+ provider = next(provider for provider in catalog["providers"]
+ if provider["id"] == "loopx-capability-benchmark-toolkit")
+ assert {key: provider[key] for key in unavailable} == unavailable
+ refusal = _run(core_python, ["benchmark", "--help"], tmp_path, expected=2)
+ assert "unavailable in the standalone core" in refusal.stderr
+ assert not list(tmp_path.iterdir())
+
+
@pytest.mark.parametrize("host_surface", ("shell", "traex-cli"))
def test_installed_core_guided_start_identity_and_readback(core_python, tmp_path, monkeypatch, host_surface):
from tests.control_plane.canonical_authority_fixture import isolate_sqlite_runtime
@@ -298,6 +331,26 @@ def test_installed_core_start_metadata_defers_issue_fix_execution(core_python, t
assert not list(tmp_path.iterdir())
+@pytest.mark.parametrize("arguments", [
+ ["capability", "--help"],
+ ["capability", "list", "--format", "json"],
+ ["capability", "show", "benchmark-toolkit", "--format", "json"],
+ ["capability", "show", "issue-fix", "--format", "json"],
+])
+def test_installed_core_discovers_capabilities_without_benchmark_execution(core_python, tmp_path, arguments):
+ from tests.capabilities.test_benchmark_toolkit_loading import DISCOVERY_SCRIPT
+
+ result = subprocess.run([core_python, "-I", "-c", DISCOVERY_SCRIPT,
+ json.dumps(["--registry", str(tmp_path / "registry.json"),
+ "--runtime-root", str(tmp_path / "runtime"), *arguments])],
+ cwd=tmp_path, env=_environment(core_python, {"PATH": ""}),
+ capture_output=True, text=True, timeout=30)
+ assert result.returncode == 0, (result.stdout, result.stderr)
+ if arguments[-1] == "json":
+ assert json.loads(result.stdout)["ok"] is True
+ assert not list(tmp_path.iterdir())
+
+
def test_installed_core_catalog_entries_have_executable_parsers(core_python, tmp_path):
reference = json.loads(_run(core_python, ["commands", "--format", "json"], tmp_path).stdout)
for group in reference["groups"]:
diff --git a/tests/test_distribution_runtime_identity.py b/tests/test_distribution_runtime_identity.py
index dbaee0d644..4540451a54 100644
--- a/tests/test_distribution_runtime_identity.py
+++ b/tests/test_distribution_runtime_identity.py
@@ -27,7 +27,7 @@ def owned_package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
path.write_text(content)
items = [SimpleNamespace(as_posix=lambda name=name: "loopx/" + name,
locate=lambda name=name: package / name) for name in contents]
- installed = SimpleNamespace(version=manifest.__version__, files=items,
+ installed = SimpleNamespace(name="loopx", version=manifest.__version__, files=items,
read_text=lambda _name: None)
monkeypatch.setattr(manifest, "__file__", str(package / "release_manifest.py"))
monkeypatch.setattr(manifest, "distribution", lambda _name: installed)
@@ -134,3 +134,31 @@ def test_record_root_alias_keeps_exact_physical_ownership(owned_package):
name = item.as_posix()
item.locate = lambda name=name: alias / name
assert manifest.release_runtime_identity() == before
+
+
+@pytest.mark.parametrize("change", ["owned", "wrong_version", "overlap", "absent"])
+def test_matching_workspace_owns_only_its_records(owned_package, monkeypatch, change):
+ _package, core = owned_package
+ original = manifest.release_runtime_identity()
+ core.name = "loopx-core"
+ asset = core.files.pop()
+ workspace = SimpleNamespace(name="loopx-workspace", version=manifest.__version__,
+ files=[asset], read_text=lambda _name: None)
+ if change == "wrong_version":
+ workspace.version = "0.0.0"
+ elif change == "overlap":
+ workspace.files.append(core.files[0])
+
+ def lookup(name):
+ if name == "loopx-core":
+ return core
+ if name == "loopx-workspace" and change != "absent":
+ return workspace
+ raise manifest.PackageNotFoundError(name)
+
+ monkeypatch.setattr(manifest, "distribution", lookup)
+ result = manifest.release_runtime_identity()
+ if change == "owned":
+ assert result == original
+ else:
+ assert "package_fingerprint" not in result