From 6c481cfee739e9e22237e282735d5440efd1a5f5 Mon Sep 17 00:00:00 2001 From: cicolord Date: Thu, 19 Nov 2026 00:38:23 +0800 Subject: [PATCH] refactor(core): defer benchmark implementations during capability discovery Preserve the benchmark toolkit public exports and static types while keeping execution and experiment implementations out of metadata discovery. Qualify source and installed Core paths, and reconcile the paired minimal/Standard RFC boundary without changing activation or authority defaults. Signed-off-by: cicolord --- .../rfcs/monorepo-distribution-split-v0.md | 49 +- .../monorepo-distribution-split-v0.zh-CN.md | 38 +- .../benchmark_toolkit/__init__.py | 935 +++++++++--------- .../test_benchmark_toolkit_loading.py | 81 ++ tests/test_core_distribution.py | 20 + 5 files changed, 666 insertions(+), 457 deletions(-) create mode 100644 tests/capabilities/test_benchmark_toolkit_loading.py diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.md index 6e5521ac0a..8a3723da1d 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.md @@ -4,7 +4,7 @@ - **Delivery maturity:** Proposal - **Authors / owners:** LoopX maintainers - **Created:** 2026-09-26 -- **Last normative revision:** 2026-11-17 +- **Last normative revision:** 2026-11-19 - **Implementation baseline:** packaging inventory `2f3d13ae9`; runtime composition audit `9cae3b28939f72fa4aadab5cfe49e19bcbdab372` - **Related contracts:** [TypeScript Control-Plane Migration v0](typescript-control-plane-migration-v0.md), [Extensions reference](../../reference/extensions.md), [Capability catalog](../../../loopx/capabilities/README.md), [Overall Roadmap v0](loopx-overall-roadmap-v0.md) (S2, S8, S12), [import-boundary tests](../../../tests/architecture/test_control_plane_import_boundaries.py) - **Tracking issue:** [#5072](https://github.com/loopx-project/loopx/issues/5072) @@ -35,7 +35,7 @@ retain their original baselines. wheel is replaced over time by `loopx-core`, `loopx-workspace`, and per-capability packages under `packages/`, with `loopx` retained as a meta-package that depends on all of them. -3. **Top-level regrouping happens before any packaging change.** `loopx/chat_*` +3. **Regroup the selected dependency closure as needed for independent packaging.** `loopx/chat_*` moves to `loopx/chat/`, `loopx/*_goal_mode/` moves to `loopx/hosts/`, with compatibility re-exports for established public imports for at least one minor release. Internal-only callers move together and the old internal entry is deleted. @@ -63,6 +63,20 @@ can be installed, not what every command must load. Installing standard grants no service activation or permission. Existing defaults and machine-enforced work obligations remain until a qualified, explicitly disclosed migration. +The immediate outcome is an independently installed core that executes a task, +recovers its original result and returns evidence. Unrelated directory moves, +workspace packaging or another model session do not block that core-artifact +slice; full M3 retains its complete distribution acceptance. Ordinary callers +supply a task and consequential constraints, without first learning Vision, +Quota, Lease or method names. The host carries bookkeeping; relevant permission, +budget, uncertainty and recovery decisions remain visible when needed. + +The **10% complexity / 90% current-function coverage** target remains an unproven +hypothesis. Fix a versioned journey denominator and compare necessary concepts, +mandatory steps, loaded/activated code, model reads, complete costs and accepted +outcomes. Report kernel-only and assembled coverage; do not drop specialized +journeys from the denominator or treat code counts as model benefit. + ## 2. Problem and motivation A developer who wants a CLI-only or kernel-only LoopX installs the same wheel as @@ -269,6 +283,19 @@ contract into prose guidance. The kernel is a cohesive set of domains, not one large file. The adjacent refactor should localize the next change and preserve characterized behavior. +### Required mechanisms and optional task methods + +Core owns legal scoped work, durable evidence, stop, budget and original-attempt +recovery. Bench, workbench and manager compose specialized workflows around the +same owners. Their absence must not block unrelated authorized core work. +**Yardstick is an independently optional decision method, not a provider or a +mandatory kernel stage.** Methods choose how to reason or measure; providers +supply execution, storage or concrete external operations. Reuse existing +capability owners and `yardstick_enabled`; add no method registry, provider id +or enable switch. Ordinary result validation remains required when a method is +off. Existing records, configured obligations, authority and defaults survive; +method/dependency removal needs its own absence, parity and recovery evidence. + ### Presets over the distribution tiers | Composition | Required content | Explicit optional content | @@ -283,6 +310,16 @@ flag applied after standard fully initializes. A host-free read-only kernel library does not qualify that executable preset. Exact contents and the first supported host/authority combination are D2/D5 decisions. +The initial proposed **Standard** selection is `agent-turn-recall` for relevant +prior guidance, `issue-fix` for validated repository changes, and +`pull-request-review` for delivery review. These retain their existing capability +owners and default-off settings; installation grants no source access, starts +no watcher and authorizes no publication or merge. Load only the selected task's +implementation. Basic recovery stays in Minimal without advisory recall. +Yardstick, Stride, research/benchmark machinery and manager routing remain +separate optional additions. This selects the R2 target; the current monolithic +and core-candidate wheels do not yet implement that isolated package selection. + Keep composition, managed/attached mode, host provider, one/two model sessions, storage provider and benchmark arm independent. Minimal does not imply attached, standard does not require two model sessions, and research does not mean @@ -611,6 +648,14 @@ moves; it does not qualify the kernel. R3/R4 are optional and do not block a qualified CLI or managed task. No unused provider, field or framework is added ahead of its real caller. Report the applied/deferred adjacent refactor in each PR. +### Core discovery checkpoint — 2026-11-19 + +| Boundary | Delivered delta / evidence | Still open | +| --- | --- | --- | +| V1/V3, capability discovery | Benchmark package exports resolve their selected existing owner on demand, retaining all 199 public symbols and their order; explicit star imports still request all. Help/list/detail succeed with benchmark execution implementations unavailable, no Node, process launch, network or state writes. | Does not remove benchmark files from the wheel or qualify absent-capability installation. | +| Metadata and active consumers | The same catalog and all 25 details are byte-identical to the frozen `main@e235cde9c` workload; 53 CLI help/list/detail/unknown-id observations match. Discovery loads 2 benchmark children instead of 28, and 298 LoopX modules instead of 326. Existing benchmark semantics and canonical export identities remain unchanged. | Import counts are not latency, total cost, model benefit or full D2/D5 closure. | +| Acceptance and continuation | 52 installed-core discovery/execution/context/recovery cases pass on the rebuilt artifact; 383 selected consumer cases and 43 architecture/documentation cases pass separately. No new state owner, provider id, activation toggle or top-level module. | Keep R1/M3 and V1–V5/V10 open; continue capability package isolation and selected Standard delivery. Public-channel C1 qualification remains with its release owner. | + ### Subtraction Gate & Anti-Context-Gravity Rules diff --git a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md index 2303929af4..82df0bbd40 100644 --- a/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md +++ b/docs/architecture/rfcs/monorepo-distribution-split-v0.zh-CN.md @@ -4,7 +4,7 @@ - **交付成熟度:** Proposal - **作者 / 负责人:** LoopX maintainers - **创建:** 2026-09-26 -- **最近规范性修订:** 2026-11-17 +- **最近规范性修订:** 2026-11-19 - **实现基线:** 发行盘点 `2f3d13ae9`;运行时装配审计 `9cae3b28939f72fa4aadab5cfe49e19bcbdab372` - **相关契约:** [TypeScript 控制面迁移 v0](typescript-control-plane-migration-v0.zh-CN.md)、[Extensions 参考](../../reference/extensions.md)、[Capability 目录](../../../loopx/capabilities/README.md)、[总体路线图 v0](loopx-overall-roadmap-v0.zh-CN.md)(S2、S8、S12)、[import 边界测试](../../../tests/architecture/test_control_plane_import_boundaries.py) - **跟踪 issue:** [#5072](https://github.com/loopx-project/loopx/issues/5072) @@ -25,7 +25,7 @@ 1. **仓库保持单一 monorepo。** 一条 PR 流程、一套 CI、一个发布列车。拆成多个 Git 仓库是本 RFC 的非目标。 2. **安装物拆成多个发行物。** 单一 `loopx` wheel 逐步替换为 `loopx-core`、`loopx-workspace` 以及 `packages/` 下的按 capability 打包;`loopx` 保留为依赖全部子包的 meta-package。 -3. **先做顶层目录重组,再做打包变更。** `loopx/chat_*` 迁入 `loopx/chat/`,`loopx/*_goal_mode/` 迁入 `loopx/hosts/`,已建立的公开 import 保留至少一个 minor 版本的兼容 re-export;内部调用方一起迁移并删除旧内部入口。 +3. **按独立打包所需重组所选依赖闭包。** `loopx/chat_*` 迁入 `loopx/chat/`,`loopx/*_goal_mode/` 迁入 `loopx/hosts/`,已建立的公开 import 保留至少一个 minor 版本的兼容 re-export;内部调用方一起迁移并删除旧内部入口。 4. **新增一条架构测试钉住顶层增长。** `loopx/*.py` 的文件数只能减少,与现有零例外 import 边界测试并列。 5. **仅内核安装不得在 import 阶段要求 Node.js。** TS effect runtime 对有副作用的命令仍是必需的;缺失时必须以现有的类型化 `node_unavailable` 诊断呈现,而不是安装或 import 失败。 6. **不变:** 内核权威、CLI 兼容基线、extension 生命周期规则、公开/私有边界、全部稳定协议。 @@ -39,6 +39,15 @@ Minimal 必须执行并恢复真实的受治理任务;standard 围绕同一内 发行表定义可安装内容,不代表每条命令都必须加载。安装 standard 不授予服务激活或权限。 现有默认值与机器强制工作义务持续生效,直到合格且明确披露的迁移改变它们。 +当前优先结果是可独立安装、执行任务、恢复原结果并返回证据的 core。无关目录移动、 +workspace 打包或额外模型会话不阻塞这一 core 发行物切片;完整 M3 仍保留全部发行 +验收。普通调用者提供任务与关键约束,不必先学 Vision、Quota、Lease 或方法名称。 +Host 承担记账;必要的权限、预算、不确定性与恢复决策仍在相关时可见。 + +**10% 复杂度/90% 当前功能覆盖率**仍是未验证假设。固定版本化用户旅程分母,比较 +必要概念、必经步骤、加载/启用代码、模型读取、完整成本和已验收成果。分别报告 +仅内核与组合后的覆盖率;不能从分母删除特化旅程,也不能以代码计数代替模型收益。 + ## 2. 问题与动机 只想要 CLI 或内核的开发者,安装的和运行桌面工作台、Lark 管家、全部内置 capability 的人是同一个 wheel。`2f3d13ae9` 上的历史发行盘点: @@ -181,6 +190,16 @@ Planner 可以建议工作或 stride,所属迁移规则决定准入与义务 预设不能把机器强制的 `must_attempt_work` 契约变成文字 guidance。 内核是内聚领域的集合,不是一个大文件。相邻重构应让下一次修改更局部,并保持已刻画的行为。 +### 必要机制与可选任务方法 + +Core 负责合法限定工作、持久证据、停止、预算及原 attempt 恢复。Bench、workbench、 +管家围绕相同 owner 组合特化工作流;它们缺席不能阻塞无关的已授权 core 工作。 +**Yardstick 是独立可选的决策方法,不是 provider 或强制内核阶段。** 方法选择如何 +推理或衡量;provider 提供执行、存储或具体外部操作。复用既有 capability owner 和 +`yardstick_enabled`,不增加方法 registry、provider id 或开关。关闭方法不免除普通 +结果验收。保留既有记录、已配置义务、权限与默认值;移除方法/依赖须独立证明组件 +缺席、一致性与恢复。 + ### 发行层之上的预设 | 装配 | 必需内容 | 显式可选内容 | @@ -194,6 +213,13 @@ Minimal 是可执行装配,不是在 standard 完成全部初始化后加开 不带宿主的只读 kernel library 不构成可执行预设资格。 精确内容与首个受支持宿主/authority 组合由 D2/D5 决定。 +首批 **Standard** 选型提案为:`agent-turn-recall` 取回相关既有指导,`issue-fix` +交付已验证仓库变更,`pull-request-review` 执行交付审查。保留各自既有 owner 和默认 +关闭设置;安装不授予来源访问、不启动 watcher,也不授权发布或合并。仅加载所选 +任务实现;可选指导召回缺席时,Minimal 仍保有基本恢复。Yardstick、Stride、研究/ +benchmark 机制与管家路由继续独立可选。这选定 R2 的目标,现有单体和 core 候选 wheel +尚未实现该隔离分包组合。 + 装配、managed/attached 模式、宿主 provider、一个/两个模型 Session、存储 provider、benchmark arm 保持独立。Minimal 不意味着 attached,standard 不要求双模型 Session,research 不等于不受治理。 只发布已测组合,保留现有模式默认值;改变装配不能静默改变 binding 的执行 owner。 @@ -443,6 +469,14 @@ M0–M5 保留原发行里程碑;R0–R4 是同一计划中的运行时和旅 R0 可以先于目录移动交付,但不代表内核合格。R3/R4 可选,不阻塞已合格的 CLI 或 managed 任务。 真实调用者出现前不增加闲置 provider、字段或框架。每个 PR 记录应用/推迟的相邻重构。 +### Core 发现入口检查点 — 2026-11-19 + +| 边界 | 交付增量/证据 | 仍未完成 | +| --- | --- | --- | +| V1/V3,能力发现 | Benchmark 包公开符号按需解析到所选既有 owner,保留全部 199 个符号及顺序;显式星号导入仍请求全部。执行实现不可用且无 Node 时,帮助/列表/详情仍可用,无进程启动、网络或状态写入。 | 未从 wheel 删除 benchmark 文件,也不验收 capability 分包缺席安装。 | +| 元数据与活跃调用方 | 同冻结 `main@e235cde9c` workload 的目录和全部 25 项详情逐字节一致;53 项 CLI 帮助/列表/详情/未知 id 观察一致。发现入口的 benchmark 子模块从 28 降到 2,LoopX 模块从 326 降到 298。保留 benchmark 语义及公开符号的 canonical 对象身份。 | Import 数量不代表时延、完整成本、模型收益或完整 D2/D5 闭包。 | +| 验收与继续方向 | 重建发行物通过 52 项安装 core 发现/执行/上下文/恢复用例;383 项所选消费方与 43 项架构/文档用例分别通过。没有新增状态 owner、provider id、启用开关或顶层模块。 | R1/M3 与 V1–V5/V10 继续开放;下一步仍是 capability 分包隔离与所选 Standard 交付。公开渠道 C1 资格仍归既有 release owner。 | + ### 减法门禁与反上下文重力规则 diff --git a/loopx/capabilities/benchmark_toolkit/__init__.py b/loopx/capabilities/benchmark_toolkit/__init__.py index f024c7c7e9..993fdb7b31 100644 --- a/loopx/capabilities/benchmark_toolkit/__init__.py +++ b/loopx/capabilities/benchmark_toolkit/__init__.py @@ -1,455 +1,484 @@ -"""Built-in provider-neutral benchmark experiment toolkit.""" +"""Benchmark exports load only their selected implementation. -from .adaptive_concurrency import ( - BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION, - BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION, - BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION, - build_benchmark_adaptive_concurrency_decision, - build_benchmark_adaptive_concurrency_policy, - normalize_benchmark_adaptive_concurrency_policy, - normalize_benchmark_concurrency_feedback, - tune_benchmark_concurrency_target, -) -from .artifacts import ( - BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION, - build_benchmark_candidate_source_boundary, - classify_benchmark_artifact_path, - classify_benchmark_candidate_source_path, - filter_public_benchmark_artifact_paths, - materialize_public_benchmark_artifacts, -) -from .behavior_finding import ( - BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION, - build_benchmark_behavior_report, - normalize_benchmark_behavior_finding, -) -from .concurrency_envelope import ( - BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME, - BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION, - BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION, - admit_benchmark_case, - build_benchmark_concurrency_config, - build_benchmark_concurrency_status, - configure_benchmark_concurrency_envelope, - default_benchmark_concurrency_envelope_path, - normalize_benchmark_concurrency_config, - normalize_benchmark_concurrency_envelope, - normalize_benchmark_resource_headroom_receipt, - read_benchmark_concurrency_envelope, - release_benchmark_case, - render_benchmark_concurrency_markdown, -) -from .container_binding import ( - BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION, - DockerContainerBinding, - DockerContainerBindingError, - compact_docker_container_binding_receipt, - select_exact_docker_container, -) -from .experiment_board import ( - BenchmarkExperimentBoardValidationError, - BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME, - BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION, - BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION, - benchmark_experiment_board_row_key, - build_benchmark_experiment_board, - default_benchmark_experiment_board_path, - normalize_benchmark_experiment_board_row, - preview_benchmark_experiment_board_reconcile, - preview_benchmark_experiment_board_upsert, - read_benchmark_experiment_board_rows, - render_benchmark_experiment_board_markdown, - upsert_benchmark_experiment_board_row, -) -from .environment_access import ( - BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION, - BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION, - BENCHMARK_TASK_ENVIRONMENT_READ_TOOL, -) -from .continuation import ( - BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION, - BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION, - BenchmarkContinuationDecision, - build_benchmark_continuation_decision, - normalize_benchmark_public_progress, -) -from .factorial_contrast import ( - BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION, - build_benchmark_factorial_contrasts, -) -from .four_arm_contract import ( - BENCHMARK_FOUR_ARM_ATTESTATIONS, - BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION, - BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE, - BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS, - BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION, - build_benchmark_four_arm_contract, - build_benchmark_four_arm_contract_from_spec, - compact_benchmark_four_arm_contract, -) -from .integrity import ( - BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION, - BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION, - BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION, - BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION, - BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION, - BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION, - BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION, - BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, - INTEGRITY_EVIDENCE_CATEGORIES, - REQUIRED_RUNTIME_ATTESTATIONS, - RestrictedAccessAdjudicationDecision, - build_benchmark_integrity_qualification, -) -from .native_codex_goal import ( - NativeGoalConfig, - NativeGoalDeadlineExceeded, - NativeGoalEventTransport, - NativeGoalProtocolError, - NativeGoalTransport, - NativeGoalTurn, - StdioNativeGoalTransport, - attach_native_goal, - compact_native_goal_receipt, - observe_native_goal_event, - probe_native_goal_process, - refresh_native_goal_status, - run_native_goal_process, - run_native_goal_process_until_terminal, - run_native_goal_turn, - run_native_goal_until_terminal, - start_native_goal_turn, - wait_native_goal_turn, -) -from .native_codex_isolation import ( - NativeCodexIsolationEnvelope, - NativeCodexIsolationError, - NativeCodexLoopXStateRebase, - build_native_codex_isolation_envelope, - rebase_native_codex_loopx_workspace_state, -) -from .native_codex_profile import ( - NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION, - NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS, - NATIVE_CODEX_PROFILE_SCHEMA_VERSION, - NativeCodexGoalPrompt, - NativeCodexProfile, - NativeCodexProfileError, - compact_native_codex_goal_prompt_receipt, - compact_native_codex_profile_receipt, - inspect_native_codex_profile, - install_native_codex_profile, - native_codex_app_server_shell_policy_args, - native_codex_profile_environment, - render_native_codex_goal_prompt, -) -from .provider_gateway import ( - RunnerOwnedProviderGateway, - serve_runner_owned_provider_gateway, -) -from .public_trajectory import ( - NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION, - PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION, - NativeGoalLifecycleFacts, - PublicTrajectoryLifecycleState, - PublicTrajectorySummaryError, - build_native_goal_public_trajectory_summary, -) -from .reward_contract import ( - REWARD_CONTRACT_SCHEMA_VERSION, - verify_verifier_reward_file, - verify_verifier_reward_json, -) -from .rebalance_plan import ( - BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION, - BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION, - BenchmarkRebalanceAction, - BenchmarkRebalanceCandidateState, - build_benchmark_rebalance_plan, - build_benchmark_rebalance_plan_from_inventory, - normalize_benchmark_rebalance_inventory, - render_benchmark_rebalance_plan_markdown, -) -from .run_permissions import ( - DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS, - DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS, - RUN_PERMISSION_POLICY_SCHEMA_VERSION, - RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION, - RunPermissionAction, - build_run_permission_policy, - compact_run_permission_policy_for_quota, - validate_run_permission_policy, -) -from .runtime_continuity import ( - BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION, - BenchmarkEventWindowState, - BenchmarkRuntimeContinuityClassification, - BenchmarkRuntimeContinuityTransition, - build_benchmark_runtime_continuity, -) -from .runtime_observation import ( - BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION, - BenchmarkJobReceiptState, - BenchmarkRunnerOwnerState, - BenchmarkRuntimeClassification, - BenchmarkRuntimeTransition, - build_benchmark_runtime_observation, -) -from .source_revision_fence import ( - BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION, - BenchmarkSourceRevisionFence, - BenchmarkSourceRevisionFenceError, - compact_benchmark_source_revision_fence_receipt, - inspect_benchmark_source_revision_fence, -) -from .study_projection import ( - BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION, - BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION, - BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION, - BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION, - BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION, - BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION, - build_benchmark_study_dashboard, - build_benchmark_upload_envelope, - normalize_benchmark_case_insight_projection, - normalize_benchmark_study_manifest, - normalize_benchmark_upload_envelope, - read_benchmark_local_upload_records, - read_benchmark_upload_receipt, - simulate_benchmark_upload, -) -from .traex_evidence import ( - ATIF_SCHEMA_VERSION, - BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION, - TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION, - build_traex_model_route_receipt, - capture_traex_benchmark_evidence, - convert_traex_events_to_atif, -) -from .treatment_continuation import ( - BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION, - BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION, - build_benchmark_treatment_continuation_receipt, -) -from .token_capacity import ( - BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION, - BENCHMARK_TOKEN_CAPACITY_FILENAME, - BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION, - BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION, - BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION, - admit_benchmark_token_capacity, - build_benchmark_token_capacity_policy, - build_benchmark_token_capacity_status, - configure_benchmark_token_capacity, - default_benchmark_token_capacity_path, - normalize_benchmark_token_capacity_policy, - normalize_benchmark_token_capacity_state, - normalize_benchmark_token_usage_sample, - observe_benchmark_token_usage, - read_benchmark_token_capacity_state, - record_benchmark_token_capacity_rejection, - release_benchmark_token_capacity, - render_benchmark_token_capacity_markdown, -) +Capability discovery reads catalog metadata without loading execution, capacity, +installation or experiment IO. Public names retain their canonical owners. +""" +from __future__ import annotations -__all__ = [ - "BenchmarkExperimentBoardValidationError", - "ATIF_SCHEMA_VERSION", - "BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION", - "BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION", - "BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION", - "BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION", - "BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION", - "BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME", - "BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION", - "BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION", - "BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION", - "BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION", - "BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME", - "BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION", - "BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION", - "BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION", - "BENCHMARK_FOUR_ARM_ATTESTATIONS", - "BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION", - "BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE", - "BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS", - "BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION", - "BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION", - "BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION", - "BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION", - "BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION", - "BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION", - "BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION", - "BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION", - "BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION", - "BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION", - "BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION", - "BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION", - "BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION", - "BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION", - "BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION", - "BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION", - "BENCHMARK_TASK_ENVIRONMENT_READ_TOOL", - "BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION", - "BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION", - "BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION", - "BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION", - "BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION", - "BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION", - "BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION", - "BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION", - "BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION", - "BENCHMARK_TOKEN_CAPACITY_FILENAME", - "BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION", - "BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION", - "BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION", - "BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION", - "BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION", - "DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS", - "DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS", - "INTEGRITY_EVIDENCE_CATEGORIES", - "NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION", - "NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS", - "NATIVE_CODEX_PROFILE_SCHEMA_VERSION", - "NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION", - "PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION", - "REQUIRED_RUNTIME_ATTESTATIONS", - "REWARD_CONTRACT_SCHEMA_VERSION", - "RUN_PERMISSION_POLICY_SCHEMA_VERSION", - "RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION", - "TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION", - "BenchmarkContinuationDecision", - "BenchmarkRebalanceAction", - "BenchmarkRebalanceCandidateState", - "BenchmarkEventWindowState", - "BenchmarkJobReceiptState", - "BenchmarkRunnerOwnerState", - "BenchmarkRuntimeClassification", - "BenchmarkRuntimeContinuityClassification", - "BenchmarkRuntimeContinuityTransition", - "BenchmarkRuntimeTransition", - "BenchmarkSourceRevisionFence", - "BenchmarkSourceRevisionFenceError", - "DockerContainerBinding", - "DockerContainerBindingError", - "NativeCodexGoalPrompt", - "NativeCodexIsolationEnvelope", - "NativeCodexIsolationError", - "NativeCodexLoopXStateRebase", - "NativeCodexProfile", - "NativeCodexProfileError", - "NativeGoalConfig", - "NativeGoalDeadlineExceeded", - "NativeGoalEventTransport", - "NativeGoalLifecycleFacts", - "NativeGoalProtocolError", - "NativeGoalTransport", - "NativeGoalTurn", - "PublicTrajectoryLifecycleState", - "PublicTrajectorySummaryError", - "RestrictedAccessAdjudicationDecision", - "RunPermissionAction", - "RunnerOwnedProviderGateway", - "StdioNativeGoalTransport", - "admit_benchmark_case", - "admit_benchmark_token_capacity", - "attach_native_goal", - "benchmark_experiment_board_row_key", - "build_benchmark_adaptive_concurrency_decision", - "build_benchmark_adaptive_concurrency_policy", - "build_benchmark_behavior_report", - "build_benchmark_candidate_source_boundary", - "build_benchmark_concurrency_config", - "build_benchmark_concurrency_status", - "build_benchmark_continuation_decision", - "build_benchmark_experiment_board", - "build_benchmark_factorial_contrasts", - "build_benchmark_rebalance_plan", - "build_benchmark_rebalance_plan_from_inventory", - "build_benchmark_four_arm_contract", - "build_benchmark_four_arm_contract_from_spec", - "build_benchmark_integrity_qualification", - "build_benchmark_runtime_continuity", - "build_benchmark_runtime_observation", - "build_benchmark_study_dashboard", - "build_benchmark_treatment_continuation_receipt", - "build_benchmark_token_capacity_policy", - "build_benchmark_token_capacity_status", - "build_benchmark_upload_envelope", - "build_native_codex_isolation_envelope", - "build_native_goal_public_trajectory_summary", - "build_run_permission_policy", - "build_traex_model_route_receipt", - "capture_traex_benchmark_evidence", - "classify_benchmark_artifact_path", - "classify_benchmark_candidate_source_path", - "compact_benchmark_four_arm_contract", - "compact_benchmark_source_revision_fence_receipt", - "compact_docker_container_binding_receipt", - "compact_native_codex_goal_prompt_receipt", - "compact_native_codex_profile_receipt", - "compact_native_goal_receipt", - "compact_run_permission_policy_for_quota", - "configure_benchmark_concurrency_envelope", - "configure_benchmark_token_capacity", - "convert_traex_events_to_atif", - "default_benchmark_concurrency_envelope_path", - "default_benchmark_experiment_board_path", - "default_benchmark_token_capacity_path", - "filter_public_benchmark_artifact_paths", - "inspect_benchmark_source_revision_fence", - "inspect_native_codex_profile", - "install_native_codex_profile", - "materialize_public_benchmark_artifacts", - "native_codex_app_server_shell_policy_args", - "native_codex_profile_environment", - "normalize_benchmark_adaptive_concurrency_policy", - "normalize_benchmark_behavior_finding", - "normalize_benchmark_case_insight_projection", - "normalize_benchmark_concurrency_config", - "normalize_benchmark_concurrency_envelope", - "normalize_benchmark_concurrency_feedback", - "normalize_benchmark_experiment_board_row", - "normalize_benchmark_public_progress", - "normalize_benchmark_resource_headroom_receipt", - "normalize_benchmark_rebalance_inventory", - "normalize_benchmark_study_manifest", - "normalize_benchmark_token_capacity_policy", - "normalize_benchmark_token_capacity_state", - "normalize_benchmark_token_usage_sample", - "normalize_benchmark_upload_envelope", - "observe_native_goal_event", - "observe_benchmark_token_usage", - "preview_benchmark_experiment_board_reconcile", - "preview_benchmark_experiment_board_upsert", - "probe_native_goal_process", - "read_benchmark_concurrency_envelope", - "read_benchmark_experiment_board_rows", - "read_benchmark_local_upload_records", - "read_benchmark_upload_receipt", - "read_benchmark_token_capacity_state", - "rebase_native_codex_loopx_workspace_state", - "refresh_native_goal_status", - "release_benchmark_case", - "record_benchmark_token_capacity_rejection", - "release_benchmark_token_capacity", - "render_benchmark_concurrency_markdown", - "render_benchmark_rebalance_plan_markdown", - "render_benchmark_token_capacity_markdown", - "render_benchmark_experiment_board_markdown", - "render_native_codex_goal_prompt", - "run_native_goal_process", - "run_native_goal_process_until_terminal", - "run_native_goal_turn", - "run_native_goal_until_terminal", - "select_exact_docker_container", - "serve_runner_owned_provider_gateway", - "simulate_benchmark_upload", - "start_native_goal_turn", - "tune_benchmark_concurrency_target", - "upsert_benchmark_experiment_board_row", - "validate_run_permission_policy", - "verify_verifier_reward_file", - "verify_verifier_reward_json", - "wait_native_goal_turn", -] +from importlib import import_module +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .adaptive_concurrency import ( + BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION as BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION, + BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION as BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION, + BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION as BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION, + build_benchmark_adaptive_concurrency_decision as build_benchmark_adaptive_concurrency_decision, + build_benchmark_adaptive_concurrency_policy as build_benchmark_adaptive_concurrency_policy, + normalize_benchmark_adaptive_concurrency_policy as normalize_benchmark_adaptive_concurrency_policy, + normalize_benchmark_concurrency_feedback as normalize_benchmark_concurrency_feedback, + tune_benchmark_concurrency_target as tune_benchmark_concurrency_target, + ) + from .artifacts import ( + BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION as BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION, + build_benchmark_candidate_source_boundary as build_benchmark_candidate_source_boundary, + classify_benchmark_artifact_path as classify_benchmark_artifact_path, + classify_benchmark_candidate_source_path as classify_benchmark_candidate_source_path, + filter_public_benchmark_artifact_paths as filter_public_benchmark_artifact_paths, + materialize_public_benchmark_artifacts as materialize_public_benchmark_artifacts, + ) + from .behavior_finding import ( + BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION as BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION, + build_benchmark_behavior_report as build_benchmark_behavior_report, + normalize_benchmark_behavior_finding as normalize_benchmark_behavior_finding, + ) + from .concurrency_envelope import ( + BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME as BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME, + BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION as BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION, + BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION as BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION, + admit_benchmark_case as admit_benchmark_case, + build_benchmark_concurrency_config as build_benchmark_concurrency_config, + build_benchmark_concurrency_status as build_benchmark_concurrency_status, + configure_benchmark_concurrency_envelope as configure_benchmark_concurrency_envelope, + default_benchmark_concurrency_envelope_path as default_benchmark_concurrency_envelope_path, + normalize_benchmark_concurrency_config as normalize_benchmark_concurrency_config, + normalize_benchmark_concurrency_envelope as normalize_benchmark_concurrency_envelope, + normalize_benchmark_resource_headroom_receipt as normalize_benchmark_resource_headroom_receipt, + read_benchmark_concurrency_envelope as read_benchmark_concurrency_envelope, + release_benchmark_case as release_benchmark_case, + render_benchmark_concurrency_markdown as render_benchmark_concurrency_markdown, + ) + from .container_binding import ( + BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION as BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION, + DockerContainerBinding as DockerContainerBinding, + DockerContainerBindingError as DockerContainerBindingError, + compact_docker_container_binding_receipt as compact_docker_container_binding_receipt, + select_exact_docker_container as select_exact_docker_container, + ) + from .experiment_board import ( + BenchmarkExperimentBoardValidationError as BenchmarkExperimentBoardValidationError, + BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME as BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME, + BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION as BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION, + BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION as BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION, + benchmark_experiment_board_row_key as benchmark_experiment_board_row_key, + build_benchmark_experiment_board as build_benchmark_experiment_board, + default_benchmark_experiment_board_path as default_benchmark_experiment_board_path, + normalize_benchmark_experiment_board_row as normalize_benchmark_experiment_board_row, + preview_benchmark_experiment_board_reconcile as preview_benchmark_experiment_board_reconcile, + preview_benchmark_experiment_board_upsert as preview_benchmark_experiment_board_upsert, + read_benchmark_experiment_board_rows as read_benchmark_experiment_board_rows, + render_benchmark_experiment_board_markdown as render_benchmark_experiment_board_markdown, + upsert_benchmark_experiment_board_row as upsert_benchmark_experiment_board_row, + ) + from .environment_access import ( + BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_READ_TOOL as BENCHMARK_TASK_ENVIRONMENT_READ_TOOL, + ) + from .continuation import ( + BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION as BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION, + BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION as BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION, + BenchmarkContinuationDecision as BenchmarkContinuationDecision, + build_benchmark_continuation_decision as build_benchmark_continuation_decision, + normalize_benchmark_public_progress as normalize_benchmark_public_progress, + ) + from .factorial_contrast import ( + BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION as BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION, + build_benchmark_factorial_contrasts as build_benchmark_factorial_contrasts, + ) + from .four_arm_contract import ( + BENCHMARK_FOUR_ARM_ATTESTATIONS as BENCHMARK_FOUR_ARM_ATTESTATIONS, + BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION as BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION, + BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE as BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE, + BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS as BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS, + BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION as BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION, + build_benchmark_four_arm_contract as build_benchmark_four_arm_contract, + build_benchmark_four_arm_contract_from_spec as build_benchmark_four_arm_contract_from_spec, + compact_benchmark_four_arm_contract as compact_benchmark_four_arm_contract, + ) + from .integrity import ( + BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION as BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION, + BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION as BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION, + BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION as BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION, + BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION as BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION, + BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION as BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION, + BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION as BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION, + BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION as BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION, + INTEGRITY_EVIDENCE_CATEGORIES as INTEGRITY_EVIDENCE_CATEGORIES, + REQUIRED_RUNTIME_ATTESTATIONS as REQUIRED_RUNTIME_ATTESTATIONS, + RestrictedAccessAdjudicationDecision as RestrictedAccessAdjudicationDecision, + build_benchmark_integrity_qualification as build_benchmark_integrity_qualification, + ) + from .native_codex_goal import ( + NativeGoalConfig as NativeGoalConfig, + NativeGoalDeadlineExceeded as NativeGoalDeadlineExceeded, + NativeGoalEventTransport as NativeGoalEventTransport, + NativeGoalProtocolError as NativeGoalProtocolError, + NativeGoalTransport as NativeGoalTransport, + NativeGoalTurn as NativeGoalTurn, + StdioNativeGoalTransport as StdioNativeGoalTransport, + attach_native_goal as attach_native_goal, + compact_native_goal_receipt as compact_native_goal_receipt, + observe_native_goal_event as observe_native_goal_event, + probe_native_goal_process as probe_native_goal_process, + refresh_native_goal_status as refresh_native_goal_status, + run_native_goal_process as run_native_goal_process, + run_native_goal_process_until_terminal as run_native_goal_process_until_terminal, + run_native_goal_turn as run_native_goal_turn, + run_native_goal_until_terminal as run_native_goal_until_terminal, + start_native_goal_turn as start_native_goal_turn, + wait_native_goal_turn as wait_native_goal_turn, + ) + from .native_codex_isolation import ( + NativeCodexIsolationEnvelope as NativeCodexIsolationEnvelope, + NativeCodexIsolationError as NativeCodexIsolationError, + NativeCodexLoopXStateRebase as NativeCodexLoopXStateRebase, + build_native_codex_isolation_envelope as build_native_codex_isolation_envelope, + rebase_native_codex_loopx_workspace_state as rebase_native_codex_loopx_workspace_state, + ) + from .native_codex_profile import ( + NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION as NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION, + NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS as NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS, + NATIVE_CODEX_PROFILE_SCHEMA_VERSION as NATIVE_CODEX_PROFILE_SCHEMA_VERSION, + NativeCodexGoalPrompt as NativeCodexGoalPrompt, + NativeCodexProfile as NativeCodexProfile, + NativeCodexProfileError as NativeCodexProfileError, + compact_native_codex_goal_prompt_receipt as compact_native_codex_goal_prompt_receipt, + compact_native_codex_profile_receipt as compact_native_codex_profile_receipt, + inspect_native_codex_profile as inspect_native_codex_profile, + install_native_codex_profile as install_native_codex_profile, + native_codex_app_server_shell_policy_args as native_codex_app_server_shell_policy_args, + native_codex_profile_environment as native_codex_profile_environment, + render_native_codex_goal_prompt as render_native_codex_goal_prompt, + ) + from .provider_gateway import ( + RunnerOwnedProviderGateway as RunnerOwnedProviderGateway, + serve_runner_owned_provider_gateway as serve_runner_owned_provider_gateway, + ) + from .public_trajectory import ( + NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION as NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION, + PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION as PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION, + NativeGoalLifecycleFacts as NativeGoalLifecycleFacts, + PublicTrajectoryLifecycleState as PublicTrajectoryLifecycleState, + PublicTrajectorySummaryError as PublicTrajectorySummaryError, + build_native_goal_public_trajectory_summary as build_native_goal_public_trajectory_summary, + ) + from .reward_contract import ( + REWARD_CONTRACT_SCHEMA_VERSION as REWARD_CONTRACT_SCHEMA_VERSION, + verify_verifier_reward_file as verify_verifier_reward_file, + verify_verifier_reward_json as verify_verifier_reward_json, + ) + from .rebalance_plan import ( + BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION as BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION, + BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION as BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION, + BenchmarkRebalanceAction as BenchmarkRebalanceAction, + BenchmarkRebalanceCandidateState as BenchmarkRebalanceCandidateState, + build_benchmark_rebalance_plan as build_benchmark_rebalance_plan, + build_benchmark_rebalance_plan_from_inventory as build_benchmark_rebalance_plan_from_inventory, + normalize_benchmark_rebalance_inventory as normalize_benchmark_rebalance_inventory, + render_benchmark_rebalance_plan_markdown as render_benchmark_rebalance_plan_markdown, + ) + from .run_permissions import ( + DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS as DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS, + DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS as DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS, + RUN_PERMISSION_POLICY_SCHEMA_VERSION as RUN_PERMISSION_POLICY_SCHEMA_VERSION, + RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION as RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION, + RunPermissionAction as RunPermissionAction, + build_run_permission_policy as build_run_permission_policy, + compact_run_permission_policy_for_quota as compact_run_permission_policy_for_quota, + validate_run_permission_policy as validate_run_permission_policy, + ) + from .runtime_continuity import ( + BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION as BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION, + BenchmarkEventWindowState as BenchmarkEventWindowState, + BenchmarkRuntimeContinuityClassification as BenchmarkRuntimeContinuityClassification, + BenchmarkRuntimeContinuityTransition as BenchmarkRuntimeContinuityTransition, + build_benchmark_runtime_continuity as build_benchmark_runtime_continuity, + ) + from .runtime_observation import ( + BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION as BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION, + BenchmarkJobReceiptState as BenchmarkJobReceiptState, + BenchmarkRunnerOwnerState as BenchmarkRunnerOwnerState, + BenchmarkRuntimeClassification as BenchmarkRuntimeClassification, + BenchmarkRuntimeTransition as BenchmarkRuntimeTransition, + build_benchmark_runtime_observation as build_benchmark_runtime_observation, + ) + from .source_revision_fence import ( + BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION as BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION, + BenchmarkSourceRevisionFence as BenchmarkSourceRevisionFence, + BenchmarkSourceRevisionFenceError as BenchmarkSourceRevisionFenceError, + compact_benchmark_source_revision_fence_receipt as compact_benchmark_source_revision_fence_receipt, + inspect_benchmark_source_revision_fence as inspect_benchmark_source_revision_fence, + ) + from .study_projection import ( + BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION as BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION, + BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION as BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION, + BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION as BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION, + BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION as BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION, + BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION as BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION, + BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION as BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION, + build_benchmark_study_dashboard as build_benchmark_study_dashboard, + build_benchmark_upload_envelope as build_benchmark_upload_envelope, + normalize_benchmark_case_insight_projection as normalize_benchmark_case_insight_projection, + normalize_benchmark_study_manifest as normalize_benchmark_study_manifest, + normalize_benchmark_upload_envelope as normalize_benchmark_upload_envelope, + read_benchmark_local_upload_records as read_benchmark_local_upload_records, + read_benchmark_upload_receipt as read_benchmark_upload_receipt, + simulate_benchmark_upload as simulate_benchmark_upload, + ) + from .traex_evidence import ( + ATIF_SCHEMA_VERSION as ATIF_SCHEMA_VERSION, + BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION as BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION, + TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION as TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION, + build_traex_model_route_receipt as build_traex_model_route_receipt, + capture_traex_benchmark_evidence as capture_traex_benchmark_evidence, + convert_traex_events_to_atif as convert_traex_events_to_atif, + ) + from .treatment_continuation import ( + BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION as BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION, + BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION as BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION, + build_benchmark_treatment_continuation_receipt as build_benchmark_treatment_continuation_receipt, + ) + from .token_capacity import ( + BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION as BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION, + BENCHMARK_TOKEN_CAPACITY_FILENAME as BENCHMARK_TOKEN_CAPACITY_FILENAME, + BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION as BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION, + BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION as BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION, + BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION as BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION, + admit_benchmark_token_capacity as admit_benchmark_token_capacity, + build_benchmark_token_capacity_policy as build_benchmark_token_capacity_policy, + build_benchmark_token_capacity_status as build_benchmark_token_capacity_status, + configure_benchmark_token_capacity as configure_benchmark_token_capacity, + default_benchmark_token_capacity_path as default_benchmark_token_capacity_path, + normalize_benchmark_token_capacity_policy as normalize_benchmark_token_capacity_policy, + normalize_benchmark_token_capacity_state as normalize_benchmark_token_capacity_state, + normalize_benchmark_token_usage_sample as normalize_benchmark_token_usage_sample, + observe_benchmark_token_usage as observe_benchmark_token_usage, + read_benchmark_token_capacity_state as read_benchmark_token_capacity_state, + record_benchmark_token_capacity_rejection as record_benchmark_token_capacity_rejection, + release_benchmark_token_capacity as release_benchmark_token_capacity, + render_benchmark_token_capacity_markdown as render_benchmark_token_capacity_markdown, + ) + +_EXPORTS = { + "BenchmarkExperimentBoardValidationError": "experiment_board", + "ATIF_SCHEMA_VERSION": "traex_evidence", + "BENCHMARK_ADAPTIVE_CONCURRENCY_DECISION_SCHEMA_VERSION": "adaptive_concurrency", + "BENCHMARK_ADAPTIVE_CONCURRENCY_POLICY_SCHEMA_VERSION": "adaptive_concurrency", + "BENCHMARK_BEHAVIOR_FINDING_SCHEMA_VERSION": "behavior_finding", + "BENCHMARK_CANDIDATE_SOURCE_BOUNDARY_SCHEMA_VERSION": "artifacts", + "BENCHMARK_CASE_INSIGHT_PROJECTION_SCHEMA_VERSION": "study_projection", + "BENCHMARK_CONCURRENCY_ENVELOPE_FILENAME": "concurrency_envelope", + "BENCHMARK_CONCURRENCY_ENVELOPE_SCHEMA_VERSION": "concurrency_envelope", + "BENCHMARK_CONCURRENCY_FEEDBACK_SCHEMA_VERSION": "adaptive_concurrency", + "BENCHMARK_CONTINUATION_DECISION_SCHEMA_VERSION": "continuation", + "BENCHMARK_EXACT_CONTAINER_BINDING_SCHEMA_VERSION": "container_binding", + "BENCHMARK_EXPERIMENT_BOARD_LEDGER_FILENAME": "experiment_board", + "BENCHMARK_EXPERIMENT_BOARD_ROW_SCHEMA_VERSION": "experiment_board", + "BENCHMARK_EXPERIMENT_BOARD_SCHEMA_VERSION": "experiment_board", + "BENCHMARK_FACTORIAL_CONTRAST_SCHEMA_VERSION": "factorial_contrast", + "BENCHMARK_FOUR_ARM_ATTESTATIONS": "four_arm_contract", + "BENCHMARK_FOUR_ARM_CONTRACT_SCHEMA_VERSION": "four_arm_contract", + "BENCHMARK_FOUR_ARM_QUALIFICATION_SCOPE": "four_arm_contract", + "BENCHMARK_FOUR_ARM_RUNNER_OBLIGATIONS": "four_arm_contract", + "BENCHMARK_FOUR_ARM_SPEC_SCHEMA_VERSION": "four_arm_contract", + "BENCHMARK_INTEGRITY_POLICY_SCHEMA_VERSION": "integrity", + "BENCHMARK_INTEGRITY_QUALIFICATION_SCHEMA_VERSION": "integrity", + "BENCHMARK_LOCAL_UPLOAD_RECORD_SCHEMA_VERSION": "study_projection", + "BENCHMARK_MODEL_ROUTE_RECEIPT_SCHEMA_VERSION": "traex_evidence", + "BENCHMARK_PUBLIC_PROGRESS_SCHEMA_VERSION": "continuation", + "BENCHMARK_RESOURCE_HEADROOM_RECEIPT_SCHEMA_VERSION": "concurrency_envelope", + "BENCHMARK_REBALANCE_INVENTORY_SCHEMA_VERSION": "rebalance_plan", + "BENCHMARK_REBALANCE_PLAN_SCHEMA_VERSION": "rebalance_plan", + "BENCHMARK_RESTRICTED_ACCESS_ADJUDICATION_SCHEMA_VERSION": "integrity", + "BENCHMARK_RUNTIME_CONTINUITY_SCHEMA_VERSION": "runtime_continuity", + "BENCHMARK_RUNTIME_INTEGRITY_ATTESTATION_SCHEMA_VERSION": "integrity", + "BENCHMARK_TASK_ENVIRONMENT_ACCESS_RECEIPT_SCHEMA_VERSION": "integrity", + "BENCHMARK_TASK_ENVIRONMENT_ACCESS_SCHEMA_VERSION": "integrity", + "BENCHMARK_TASK_ENVIRONMENT_READ_RESULT_SCHEMA_VERSION": "environment_access", + "BENCHMARK_TASK_ENVIRONMENT_READ_SCHEMA_VERSION": "environment_access", + "BENCHMARK_TASK_ENVIRONMENT_READ_TOOL": "environment_access", + "BENCHMARK_TASK_SERVICE_ACCESS_RECEIPT_SCHEMA_VERSION": "integrity", + "BENCHMARK_TASK_SERVICE_ACCESS_SCHEMA_VERSION": "integrity", + "BENCHMARK_RUNTIME_OBSERVATION_SCHEMA_VERSION": "runtime_observation", + "BENCHMARK_SOURCE_REVISION_FENCE_SCHEMA_VERSION": "source_revision_fence", + "BENCHMARK_STUDY_DASHBOARD_SCHEMA_VERSION": "study_projection", + "BENCHMARK_STUDY_MANIFEST_SCHEMA_VERSION": "study_projection", + "BENCHMARK_TREATMENT_CONTINUATION_OBSERVATION_SCHEMA_VERSION": "treatment_continuation", + "BENCHMARK_TREATMENT_CONTINUATION_RECEIPT_SCHEMA_VERSION": "treatment_continuation", + "BENCHMARK_TOKEN_CAPACITY_DECISION_SCHEMA_VERSION": "token_capacity", + "BENCHMARK_TOKEN_CAPACITY_FILENAME": "token_capacity", + "BENCHMARK_TOKEN_CAPACITY_POLICY_SCHEMA_VERSION": "token_capacity", + "BENCHMARK_TOKEN_CAPACITY_STATE_SCHEMA_VERSION": "token_capacity", + "BENCHMARK_TOKEN_USAGE_SAMPLE_SCHEMA_VERSION": "token_capacity", + "BENCHMARK_UPLOAD_ENVELOPE_SCHEMA_VERSION": "study_projection", + "BENCHMARK_UPLOAD_READBACK_RECEIPT_SCHEMA_VERSION": "study_projection", + "DEFAULT_RUN_PERMISSION_ALLOWED_ACTIONS": "run_permissions", + "DEFAULT_RUN_PERMISSION_FORBIDDEN_ACTIONS": "run_permissions", + "INTEGRITY_EVIDENCE_CATEGORIES": "integrity", + "NATIVE_CODEX_GOAL_PROMPT_SCHEMA_VERSION": "native_codex_profile", + "NATIVE_CODEX_PROFILE_REQUIRED_SKILL_IDS": "native_codex_profile", + "NATIVE_CODEX_PROFILE_SCHEMA_VERSION": "native_codex_profile", + "NATIVE_GOAL_LIFECYCLE_SCHEMA_VERSION": "public_trajectory", + "PUBLIC_TRAJECTORY_SUMMARY_SCHEMA_VERSION": "public_trajectory", + "REQUIRED_RUNTIME_ATTESTATIONS": "integrity", + "REWARD_CONTRACT_SCHEMA_VERSION": "reward_contract", + "RUN_PERMISSION_POLICY_SCHEMA_VERSION": "run_permissions", + "RUN_PERMISSION_QUOTA_PROJECTION_SCHEMA_VERSION": "run_permissions", + "TRAE_BENCHMARK_EVIDENCE_SCHEMA_VERSION": "traex_evidence", + "BenchmarkContinuationDecision": "continuation", + "BenchmarkRebalanceAction": "rebalance_plan", + "BenchmarkRebalanceCandidateState": "rebalance_plan", + "BenchmarkEventWindowState": "runtime_continuity", + "BenchmarkJobReceiptState": "runtime_observation", + "BenchmarkRunnerOwnerState": "runtime_observation", + "BenchmarkRuntimeClassification": "runtime_observation", + "BenchmarkRuntimeContinuityClassification": "runtime_continuity", + "BenchmarkRuntimeContinuityTransition": "runtime_continuity", + "BenchmarkRuntimeTransition": "runtime_observation", + "BenchmarkSourceRevisionFence": "source_revision_fence", + "BenchmarkSourceRevisionFenceError": "source_revision_fence", + "DockerContainerBinding": "container_binding", + "DockerContainerBindingError": "container_binding", + "NativeCodexGoalPrompt": "native_codex_profile", + "NativeCodexIsolationEnvelope": "native_codex_isolation", + "NativeCodexIsolationError": "native_codex_isolation", + "NativeCodexLoopXStateRebase": "native_codex_isolation", + "NativeCodexProfile": "native_codex_profile", + "NativeCodexProfileError": "native_codex_profile", + "NativeGoalConfig": "native_codex_goal", + "NativeGoalDeadlineExceeded": "native_codex_goal", + "NativeGoalEventTransport": "native_codex_goal", + "NativeGoalLifecycleFacts": "public_trajectory", + "NativeGoalProtocolError": "native_codex_goal", + "NativeGoalTransport": "native_codex_goal", + "NativeGoalTurn": "native_codex_goal", + "PublicTrajectoryLifecycleState": "public_trajectory", + "PublicTrajectorySummaryError": "public_trajectory", + "RestrictedAccessAdjudicationDecision": "integrity", + "RunPermissionAction": "run_permissions", + "RunnerOwnedProviderGateway": "provider_gateway", + "StdioNativeGoalTransport": "native_codex_goal", + "admit_benchmark_case": "concurrency_envelope", + "admit_benchmark_token_capacity": "token_capacity", + "attach_native_goal": "native_codex_goal", + "benchmark_experiment_board_row_key": "experiment_board", + "build_benchmark_adaptive_concurrency_decision": "adaptive_concurrency", + "build_benchmark_adaptive_concurrency_policy": "adaptive_concurrency", + "build_benchmark_behavior_report": "behavior_finding", + "build_benchmark_candidate_source_boundary": "artifacts", + "build_benchmark_concurrency_config": "concurrency_envelope", + "build_benchmark_concurrency_status": "concurrency_envelope", + "build_benchmark_continuation_decision": "continuation", + "build_benchmark_experiment_board": "experiment_board", + "build_benchmark_factorial_contrasts": "factorial_contrast", + "build_benchmark_rebalance_plan": "rebalance_plan", + "build_benchmark_rebalance_plan_from_inventory": "rebalance_plan", + "build_benchmark_four_arm_contract": "four_arm_contract", + "build_benchmark_four_arm_contract_from_spec": "four_arm_contract", + "build_benchmark_integrity_qualification": "integrity", + "build_benchmark_runtime_continuity": "runtime_continuity", + "build_benchmark_runtime_observation": "runtime_observation", + "build_benchmark_study_dashboard": "study_projection", + "build_benchmark_treatment_continuation_receipt": "treatment_continuation", + "build_benchmark_token_capacity_policy": "token_capacity", + "build_benchmark_token_capacity_status": "token_capacity", + "build_benchmark_upload_envelope": "study_projection", + "build_native_codex_isolation_envelope": "native_codex_isolation", + "build_native_goal_public_trajectory_summary": "public_trajectory", + "build_run_permission_policy": "run_permissions", + "build_traex_model_route_receipt": "traex_evidence", + "capture_traex_benchmark_evidence": "traex_evidence", + "classify_benchmark_artifact_path": "artifacts", + "classify_benchmark_candidate_source_path": "artifacts", + "compact_benchmark_four_arm_contract": "four_arm_contract", + "compact_benchmark_source_revision_fence_receipt": "source_revision_fence", + "compact_docker_container_binding_receipt": "container_binding", + "compact_native_codex_goal_prompt_receipt": "native_codex_profile", + "compact_native_codex_profile_receipt": "native_codex_profile", + "compact_native_goal_receipt": "native_codex_goal", + "compact_run_permission_policy_for_quota": "run_permissions", + "configure_benchmark_concurrency_envelope": "concurrency_envelope", + "configure_benchmark_token_capacity": "token_capacity", + "convert_traex_events_to_atif": "traex_evidence", + "default_benchmark_concurrency_envelope_path": "concurrency_envelope", + "default_benchmark_experiment_board_path": "experiment_board", + "default_benchmark_token_capacity_path": "token_capacity", + "filter_public_benchmark_artifact_paths": "artifacts", + "inspect_benchmark_source_revision_fence": "source_revision_fence", + "inspect_native_codex_profile": "native_codex_profile", + "install_native_codex_profile": "native_codex_profile", + "materialize_public_benchmark_artifacts": "artifacts", + "native_codex_app_server_shell_policy_args": "native_codex_profile", + "native_codex_profile_environment": "native_codex_profile", + "normalize_benchmark_adaptive_concurrency_policy": "adaptive_concurrency", + "normalize_benchmark_behavior_finding": "behavior_finding", + "normalize_benchmark_case_insight_projection": "study_projection", + "normalize_benchmark_concurrency_config": "concurrency_envelope", + "normalize_benchmark_concurrency_envelope": "concurrency_envelope", + "normalize_benchmark_concurrency_feedback": "adaptive_concurrency", + "normalize_benchmark_experiment_board_row": "experiment_board", + "normalize_benchmark_public_progress": "continuation", + "normalize_benchmark_resource_headroom_receipt": "concurrency_envelope", + "normalize_benchmark_rebalance_inventory": "rebalance_plan", + "normalize_benchmark_study_manifest": "study_projection", + "normalize_benchmark_token_capacity_policy": "token_capacity", + "normalize_benchmark_token_capacity_state": "token_capacity", + "normalize_benchmark_token_usage_sample": "token_capacity", + "normalize_benchmark_upload_envelope": "study_projection", + "observe_native_goal_event": "native_codex_goal", + "observe_benchmark_token_usage": "token_capacity", + "preview_benchmark_experiment_board_reconcile": "experiment_board", + "preview_benchmark_experiment_board_upsert": "experiment_board", + "probe_native_goal_process": "native_codex_goal", + "read_benchmark_concurrency_envelope": "concurrency_envelope", + "read_benchmark_experiment_board_rows": "experiment_board", + "read_benchmark_local_upload_records": "study_projection", + "read_benchmark_upload_receipt": "study_projection", + "read_benchmark_token_capacity_state": "token_capacity", + "rebase_native_codex_loopx_workspace_state": "native_codex_isolation", + "refresh_native_goal_status": "native_codex_goal", + "release_benchmark_case": "concurrency_envelope", + "record_benchmark_token_capacity_rejection": "token_capacity", + "release_benchmark_token_capacity": "token_capacity", + "render_benchmark_concurrency_markdown": "concurrency_envelope", + "render_benchmark_rebalance_plan_markdown": "rebalance_plan", + "render_benchmark_token_capacity_markdown": "token_capacity", + "render_benchmark_experiment_board_markdown": "experiment_board", + "render_native_codex_goal_prompt": "native_codex_profile", + "run_native_goal_process": "native_codex_goal", + "run_native_goal_process_until_terminal": "native_codex_goal", + "run_native_goal_turn": "native_codex_goal", + "run_native_goal_until_terminal": "native_codex_goal", + "select_exact_docker_container": "container_binding", + "serve_runner_owned_provider_gateway": "provider_gateway", + "simulate_benchmark_upload": "study_projection", + "start_native_goal_turn": "native_codex_goal", + "tune_benchmark_concurrency_target": "adaptive_concurrency", + "upsert_benchmark_experiment_board_row": "experiment_board", + "validate_run_permission_policy": "run_permissions", + "verify_verifier_reward_file": "reward_contract", + "verify_verifier_reward_json": "reward_contract", + "wait_native_goal_turn": "native_codex_goal", +} + +__all__ = list(_EXPORTS) + + +def __getattr__(name: str) -> Any: + owner = _EXPORTS.get(name) + if owner is not None: + value = getattr(import_module(f".{owner}", __name__), name) + elif name in _EXPORTS.values(): + # Preserve explicitly accessed child-module attributes from the former + # eager package facade without loading the other owners. + value = import_module(f".{name}", __name__) + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + globals()[name] = value + return value + + +def __dir__() -> list[str]: + return sorted(set(globals()) | set(__all__) | set(_EXPORTS.values())) diff --git a/tests/capabilities/test_benchmark_toolkit_loading.py b/tests/capabilities/test_benchmark_toolkit_loading.py new file mode 100644 index 0000000000..02c4f42eb4 --- /dev/null +++ b/tests/capabilities/test_benchmark_toolkit_loading.py @@ -0,0 +1,81 @@ +"""Discovery must not depend on optional benchmark execution implementations.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys + +import pytest + + +DISCOVERY_SCRIPT = '''import importlib.abc,json,sys +class MetadataOnly(importlib.abc.MetaPathFinder): + def find_spec(self,fullname,path=None,target=None): + prefix='loopx.capabilities.benchmark_toolkit.' + if fullname.startswith(prefix) and fullname not in (prefix+'catalog_entry',prefix+'four_arm_contract'): + raise AssertionError('unselected benchmark implementation was imported: '+fullname) +sys.meta_path.insert(0,MetadataOnly()) +def no_execution(event,args): + if event in ('subprocess.Popen','os.system','os.posix_spawn','socket.connect'): + raise AssertionError('discovery executed an external operation: '+event) +sys.addaudithook(no_execution) +from loopx.entrypoint import main +raise SystemExit(main(json.loads(sys.argv[1]))) +''' + + +@pytest.mark.parametrize("arguments", [ + ["capability", "--help"], + ["capability", "list", "--format", "json"], + ["capability", "show", "benchmark-toolkit", "--format", "json"], + ["capability", "show", "issue-fix", "--format", "json"], +]) +def test_capability_discovery_survives_unavailable_benchmark_execution(tmp_path, arguments): + result = subprocess.run([sys.executable, "-I", "-c", DISCOVERY_SCRIPT, + json.dumps(["--registry", str(tmp_path / "registry.json"), + "--runtime-root", str(tmp_path / "runtime"), *arguments])], + cwd=tmp_path, env={**os.environ, "PATH": "", "LOOPX_USAGE_PING": "0"}, + text=True, capture_output=True, timeout=30) + assert result.returncode == 0, (result.stdout, result.stderr) + if arguments[-1] == "json": + assert json.loads(result.stdout)["ok"] is True + assert not list(tmp_path.iterdir()) + + +def test_toolkit_import_and_selected_export_leave_other_implementations_unloaded(tmp_path): + source = '''import importlib.abc,sys +class SelectedOnly(importlib.abc.MetaPathFinder): + def find_spec(self,fullname,path=None,target=None): + if fullname.startswith('loopx.capabilities.benchmark_toolkit.') and not fullname.endswith('.four_arm_contract'): + raise ModuleNotFoundError('optional implementation unavailable',name=fullname) +sys.meta_path.insert(0,SelectedOnly()) +import loopx.capabilities.benchmark_toolkit as toolkit +assert not any(name.startswith(toolkit.__name__+'.') for name in sys.modules) +assert 'build_benchmark_four_arm_contract' in dir(toolkit) +from loopx.capabilities.benchmark_toolkit import build_benchmark_four_arm_contract +from loopx.capabilities.benchmark_toolkit.four_arm_contract import build_benchmark_four_arm_contract as canonical +assert build_benchmark_four_arm_contract is canonical +assert toolkit.build_benchmark_four_arm_contract is canonical +assert toolkit.four_arm_contract.build_benchmark_four_arm_contract is canonical +try:toolkit.run_native_goal_turn +except ModuleNotFoundError as exc:assert exc.name==toolkit.__name__+'.native_codex_goal' +else:raise AssertionError('a selected missing implementation must remain an error') +try:toolkit.unknown_export +except AttributeError:pass +else:raise AssertionError('unknown exports must remain unavailable') +''' + result = subprocess.run([sys.executable, "-I", "-c", source], cwd=tmp_path, + env={**os.environ, "LOOPX_USAGE_PING": "0"}, text=True, capture_output=True, timeout=30) + assert result.returncode == 0, (result.stdout, result.stderr) + + +def test_explicit_star_import_retains_public_exports(): + import loopx.capabilities.benchmark_toolkit as toolkit + from loopx.capabilities.benchmark_toolkit import native_codex_goal + + exports = {} + exec("from loopx.capabilities.benchmark_toolkit import *", exports) + assert {name for name in exports if not name.startswith("_")} == set(toolkit.__all__) + assert exports["NativeGoalConfig"] is native_codex_goal.NativeGoalConfig + assert exports["run_native_goal_turn"] is native_codex_goal.run_native_goal_turn diff --git a/tests/test_core_distribution.py b/tests/test_core_distribution.py index f39fae8197..e2ec4c4272 100644 --- a/tests/test_core_distribution.py +++ b/tests/test_core_distribution.py @@ -298,6 +298,26 @@ def test_installed_core_start_metadata_defers_issue_fix_execution(core_python, t assert not list(tmp_path.iterdir()) +@pytest.mark.parametrize("arguments", [ + ["capability", "--help"], + ["capability", "list", "--format", "json"], + ["capability", "show", "benchmark-toolkit", "--format", "json"], + ["capability", "show", "issue-fix", "--format", "json"], +]) +def test_installed_core_discovers_capabilities_without_benchmark_execution(core_python, tmp_path, arguments): + from tests.capabilities.test_benchmark_toolkit_loading import DISCOVERY_SCRIPT + + result = subprocess.run([core_python, "-I", "-c", DISCOVERY_SCRIPT, + json.dumps(["--registry", str(tmp_path / "registry.json"), + "--runtime-root", str(tmp_path / "runtime"), *arguments])], + cwd=tmp_path, env=_environment(core_python, {"PATH": ""}), + capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.stdout, result.stderr) + if arguments[-1] == "json": + assert json.loads(result.stdout)["ok"] is True + assert not list(tmp_path.iterdir()) + + def test_installed_core_catalog_entries_have_executable_parsers(core_python, tmp_path): reference = json.loads(_run(core_python, ["commands", "--format", "json"], tmp_path).stdout) for group in reference["groups"]: