diff --git a/adapters/cli/commands.py b/adapters/cli/commands.py index f92d256..fccd2d2 100644 --- a/adapters/cli/commands.py +++ b/adapters/cli/commands.py @@ -2,6 +2,7 @@ import argparse import asyncio +import json import os import shutil from pathlib import Path @@ -33,6 +34,7 @@ def _notify_missing_github_token(printer: Printer) -> None: def build_parser( *, all_runtime_agents: list[str] | tuple[str, ...], + internal_team_agents: list[str] | tuple[str, ...], team_roles: list[str] | tuple[str, ...], relay_transport_internal: str, relay_transport_discord: str, @@ -94,6 +96,18 @@ def build_parser( ) list_parser.add_argument("--request-id", help="Optional request identifier to print.") + metrics_parser = subparsers.add_parser("metrics", help="Summarize local model cost and latency telemetry.") + metrics_parser.add_argument( + "--workspace-root", + default=None, + help=workspace_root_help_text, + ) + metrics_parser.add_argument("--hours", type=float, default=24.0, help="Positive lookback window in hours.") + metrics_parser.add_argument("--request-id", default="", help="Optional request identifier filter.") + metrics_parser.add_argument("--sprint-id", default="", help="Optional sprint identifier filter.") + metrics_parser.add_argument("--agent", choices=all_runtime_agents, help="Optional role filter.") + metrics_parser.add_argument("--json", action="store_true", help="Render stable aggregate JSON output.") + config_parser = subparsers.add_parser("config") config_subparsers = config_parser.add_subparsers(dest="config_command", required=True) role_parser = config_subparsers.add_parser("role") @@ -108,6 +122,32 @@ def build_parser( role_set_parser.add_argument("--model", help="Optional model override to save in team_runtime.yaml.") role_set_parser.add_argument("--reasoning", help="Optional reasoning level to save in team_runtime.yaml.") + internal_parser = config_subparsers.add_parser("internal") + internal_subparsers = internal_parser.add_subparsers( + dest="internal_command", + required=True, + ) + internal_set_parser = internal_subparsers.add_parser("set") + internal_set_parser.add_argument( + "--workspace-root", + default=None, + help=workspace_root_help_text, + ) + internal_set_parser.add_argument( + "--agent", + choices=internal_team_agents, + required=True, + help="Target internal helper agent.", + ) + internal_set_parser.add_argument( + "--model", + help="Optional model override to save in team_runtime.yaml.", + ) + internal_set_parser.add_argument( + "--reasoning", + help="Optional reasoning level to save in team_runtime.yaml.", + ) + research_parser = config_subparsers.add_parser("research") research_subparsers = research_parser.add_subparsers(dest="research_command", required=True) research_set_parser = research_subparsers.add_parser("set") @@ -196,6 +236,84 @@ def build_parser( help=workspace_root_help_text, ) + benchmark_parser = subparsers.add_parser( + "benchmark", + help="Run an explicitly opted-in performance benchmark.", + ) + benchmark_subparsers = benchmark_parser.add_subparsers( + dest="benchmark_command", + required=True, + ) + sprint_ab_parser = benchmark_subparsers.add_parser( + "sprint-ab", + help="Compare full sprints with prompt event compaction disabled and enabled.", + ) + sprint_ab_parser.add_argument( + "--live", + action="store_true", + help="Permit live provider calls; also requires TEAMS_RUNTIME_LIVE_BENCHMARK=1.", + ) + sprint_ab_parser.add_argument( + "--runtime-config", + required=True, + help="Path to the deployed team_runtime.yaml or its workspace directory.", + ) + sprint_ab_parser.add_argument( + "--repetitions", + type=int, + default=1, + help="Number of paired runs; execution order alternates AB then BA.", + ) + sprint_ab_parser.add_argument( + "--max-invocations", + type=int, + default=20, + help="Hard physical model-invocation cap per arm.", + ) + sprint_ab_parser.add_argument( + "--call-timeout-seconds", + type=float, + default=300.0, + help="Hard provider-call timeout.", + ) + sprint_ab_parser.add_argument( + "--run-timeout-seconds", + type=float, + default=1800.0, + help="Hard full-arm timeout.", + ) + sprint_ab_parser.add_argument( + "--keep-workspaces", + choices=("none", "failures", "all"), + default="failures", + help="Retain no workspaces, failed workspaces, or every workspace.", + ) + sprint_ab_parser.add_argument( + "--rate-card-file", + default="", + help="Optional YAML rate card used only for estimated-cost reporting.", + ) + sprint_ab_parser.add_argument( + "--output-dir", + default="", + help="Optional parent directory for benchmark artifacts.", + ) + sprint_ab_parser.add_argument( + "--benchmark-id", + default="", + help="Optional stable artifact directory name.", + ) + sprint_ab_parser.add_argument( + "--allow-dirty-source", + action="store_true", + help="Allow a dirty source checkout and record its state hash in provenance.", + ) + sprint_ab_parser.add_argument( + "--json", + action="store_true", + help="Print a machine-readable result summary.", + ) + return parser @@ -212,6 +330,7 @@ def dispatch_main( cmd_restart: DispatchSyncCallback, cmd_list: DispatchSyncCallback, cmd_config_role_set: DispatchSyncCallback, + cmd_config_internal_set: DispatchSyncCallback, cmd_config_research_set: DispatchSyncCallback, cmd_sprint_start: DispatchSyncCallback, cmd_sprint_stop: DispatchSyncCallback, @@ -223,6 +342,8 @@ def dispatch_main( cmd_goal_resume: DispatchSyncCallback, cmd_goal_cancel: DispatchSyncCallback, default_relay_transport: str, + cmd_metrics: DispatchSyncCallback | None = None, + cmd_benchmark_sprint_ab: DispatchSyncCallback | None = None, ) -> int: if args.command == "init": return cmd_init( @@ -263,6 +384,18 @@ def dispatch_main( ) if args.command == "list": return cmd_list(workspace_root, args.request_id) + if args.command == "metrics": + if cmd_metrics is None: + parser.error("Metrics command is unavailable.") + return 2 + return cmd_metrics( + workspace_root, + hours=float(getattr(args, "hours", 24.0)), + request_id=str(getattr(args, "request_id", "") or ""), + sprint_id=str(getattr(args, "sprint_id", "") or ""), + role=str(getattr(args, "agent", "") or ""), + as_json=bool(getattr(args, "json", False)), + ) if args.command == "config": if args.config_command == "role" and args.role_command == "set": return cmd_config_role_set( @@ -271,6 +404,13 @@ def dispatch_main( model=getattr(args, "model", None), reasoning=getattr(args, "reasoning", None), ) + if args.config_command == "internal" and args.internal_command == "set": + return cmd_config_internal_set( + workspace_root, + args.agent, + model=getattr(args, "model", None), + reasoning=getattr(args, "reasoning", None), + ) if args.config_command == "research" and args.research_command == "set": return cmd_config_research_set( workspace_root, @@ -315,6 +455,24 @@ def dispatch_main( return cmd_goal_resume(workspace_root) if args.goal_command in {"cancel", "terminate"}: return cmd_goal_cancel(workspace_root) + if args.command == "benchmark" and args.benchmark_command == "sprint-ab": + if cmd_benchmark_sprint_ab is None: + parser.error("Sprint benchmark command is unavailable.") + return 2 + return cmd_benchmark_sprint_ab( + live=bool(getattr(args, "live", False)), + runtime_config=str(getattr(args, "runtime_config", "") or ""), + repetitions=int(getattr(args, "repetitions", 1)), + max_invocations=int(getattr(args, "max_invocations", 20)), + call_timeout_seconds=float(getattr(args, "call_timeout_seconds", 300.0)), + run_timeout_seconds=float(getattr(args, "run_timeout_seconds", 1800.0)), + keep_workspaces=str(getattr(args, "keep_workspaces", "failures") or "failures"), + rate_card_file=str(getattr(args, "rate_card_file", "") or ""), + output_dir=str(getattr(args, "output_dir", "") or ""), + benchmark_id=str(getattr(args, "benchmark_id", "") or ""), + allow_dirty_source=bool(getattr(args, "allow_dirty_source", False)), + as_json=bool(getattr(args, "json", False)), + ) parser.error(f"Unsupported command: {args.command}") return 2 @@ -477,6 +635,8 @@ def _format_role_runtime_summary(runtime_config: Any, role: str) -> str: f"callback_timeout={int(research_runtime.callback_timeout)}" ) role_runtime = runtime_config.role_defaults.get(role) + if role_runtime is None: + role_runtime = runtime_config.internal_agent_defaults.get(role) if role_runtime is None: return "model=N/A reasoning=N/A" model = str(role_runtime.model or "").strip() or "N/A" @@ -548,6 +708,41 @@ def cmd_status_impl( return 0 +def cmd_metrics_impl( + workspace_root: Path, + *, + hours: float, + request_id: str, + sprint_id: str, + role: str, + as_json: bool, + runtime_paths_cls: Any, + aggregate_model_invocations: Callable[..., dict[str, Any]], + render_model_metrics_summary: Callable[[dict[str, Any]], str], + printer: Printer = print, +) -> int: + if hours <= 0: + printer("metrics --hours must be a positive number.") + return 2 + paths = runtime_paths_cls.from_root(workspace_root) + try: + summary = aggregate_model_invocations( + paths, + hours=hours, + request_id=request_id, + sprint_id=sprint_id, + role=role, + ) + except ValueError as exc: + printer(str(exc)) + return 2 + if as_json: + printer(json.dumps(summary, ensure_ascii=False, indent=2)) + else: + printer(render_model_metrics_summary(summary)) + return 0 + + def cmd_stop_impl( workspace_root: Path, role: str | None, @@ -662,6 +857,41 @@ def cmd_config_role_set_impl( return 0 +def cmd_config_internal_set_impl( + workspace_root: Path, + agent: str, + *, + model: str | None = None, + reasoning: str | None = None, + update_team_runtime_internal_agent_defaults: Callable[..., Any], + runtime_paths_cls: Any, + printer: Printer = print, +) -> int: + updated = update_team_runtime_internal_agent_defaults( + workspace_root, + agent, + model=model, + reasoning=reasoning, + ) + effective_reasoning = ( + "None" if "gemini" in updated.model.lower() else updated.reasoning + ) + config_path = ( + runtime_paths_cls.from_root(workspace_root).workspace_root + / "team_runtime.yaml" + ) + printer(f"Updated {config_path}") + printer( + f"internal_agent={agent} model={updated.model} " + f"reasoning={effective_reasoning}" + ) + printer( + "Restart the orchestrator to apply helper changes: " + "python -m teams_runtime restart --agent orchestrator" + ) + return 0 + + def cmd_config_research_set_impl( workspace_root: Path, *, @@ -848,6 +1078,7 @@ def cmd_goal_cancel_impl( __all__ = [ "build_parser", + "cmd_config_internal_set_impl", "cmd_config_research_set_impl", "cmd_config_role_set_impl", "cmd_goal_cancel_impl", @@ -857,6 +1088,7 @@ def cmd_goal_cancel_impl( "cmd_goal_stop_impl", "cmd_init_impl", "cmd_list_impl", + "cmd_metrics_impl", "cmd_restart_impl", "cmd_sprint_restart_impl", "cmd_sprint_start_impl", diff --git a/benchmarking/__init__.py b/benchmarking/__init__.py new file mode 100644 index 0000000..4f4944a --- /dev/null +++ b/benchmarking/__init__.py @@ -0,0 +1,25 @@ +"""Repeatable before/after benchmarks for teams_runtime.""" + +from teams_runtime.benchmarking.models import ( + ArmPlan, + BenchmarkOptions, + BenchmarkResult, + BenchmarkWorker, + QualityEvidence, + SprintEvidence, + WorkerContext, + WorkerOutcome, +) +from teams_runtime.benchmarking.runner import run_sprint_ab_benchmark + +__all__ = [ + "ArmPlan", + "BenchmarkOptions", + "BenchmarkResult", + "BenchmarkWorker", + "QualityEvidence", + "SprintEvidence", + "WorkerContext", + "WorkerOutcome", + "run_sprint_ab_benchmark", +] diff --git a/benchmarking/metrics.py b/benchmarking/metrics.py new file mode 100644 index 0000000..2aab01d --- /dev/null +++ b/benchmarking/metrics.py @@ -0,0 +1,925 @@ +from __future__ import annotations + +import hashlib +import json +import math +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable, Mapping + +from teams_runtime.benchmarking.scenario import ( + BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + BENCHMARK_TARGET_INCLUDED_EVENTS, + BENCHMARK_TARGET_OMITTED_EVENTS, + BENCHMARK_TARGET_PURPOSE, + BENCHMARK_TARGET_ROLE, + BENCHMARK_TARGET_TOTAL_EVENTS, + BENCHMARK_TARGET_WORKFLOW_STEP, +) +from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY + + +SAFE_INVOCATION_FIELDS = frozenset( + { + "schema_version", + "invocation_id", + "operation_id", + "logical_call_id", + "attempt_index", + "attempt_kind", + "started_at", + "ended_at", + "duration_ms", + "pid", + "runtime_identity", + "role", + "purpose", + "workflow_step", + "request_id", + "sprint_id", + "todo_id", + "backlog_id", + "goal_id", + "provider", + "model", + "reasoning", + "cli_version", + "session_mode", + "session_id_hash", + "status", + "exit_code", + "error_category", + "prompt_chars", + "output_chars", + "tool_calls", + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + "usage_source", + "estimated_cost_usd", + "rate_card", + "prompt_context_enabled", + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + "prompt_context_recent_events", + "prompt_context_max_events", + "prompt_context_selection_policy", + "prompt_context_representation_conflict", + "prompt_context", + } +) +_RATE_CARD_FIELDS = frozenset( + { + "input_per_million_usd", + "cached_input_per_million_usd", + "output_per_million_usd", + "per_invocation_usd", + } +) +_PROMPT_CONTEXT_COUNT_FIELDS = ( + "total_events", + "included_events", + "omitted_events", + "recent_events", + "max_events", +) + + +def _non_negative_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + normalized = int(value) + except (OverflowError, TypeError, ValueError): + return None + return normalized if normalized >= 0 else None + + +def _strict_non_negative_int(value: Any) -> int | None: + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + return None + return value + + +def _finite_number(value: Any) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + normalized = float(value) + return normalized if math.isfinite(normalized) else None + + +def _sanitize_rate_card(value: Any) -> dict[str, float | None] | None: + if not isinstance(value, dict): + return None + sanitized: dict[str, float | None] = {} + for field_name in _RATE_CARD_FIELDS: + raw_value = value.get(field_name) + if raw_value is None: + sanitized[field_name] = None + continue + normalized = _finite_number(raw_value) + if normalized is not None and normalized >= 0: + sanitized[field_name] = normalized + return sanitized or None + + +def _sanitize_prompt_context(value: Any) -> dict[str, Any] | None: + if not isinstance(value, dict): + return None + sanitized: dict[str, Any] = {} + if isinstance(value.get("enabled"), bool): + sanitized["enabled"] = value["enabled"] + if isinstance(value.get("compacted"), bool): + sanitized["compacted"] = value["compacted"] + for field_name in _PROMPT_CONTEXT_COUNT_FIELDS: + normalized = _strict_non_negative_int(value.get(field_name)) + if normalized is not None: + sanitized[field_name] = normalized + selection_policy = str(value.get("selection_policy") or "").strip() + if selection_policy == PROMPT_EVENT_SELECTION_POLICY: + sanitized["selection_policy"] = selection_policy + return sanitized or None + + +def _flat_prompt_context(record: Mapping[str, Any]) -> dict[str, Any]: + return { + "enabled": record.get("prompt_context_enabled"), + "total_events": record.get("prompt_context_total_events"), + "included_events": record.get("prompt_context_included_events"), + "omitted_events": record.get("prompt_context_omitted_events"), + "recent_events": record.get("prompt_context_recent_events"), + "max_events": record.get("prompt_context_max_events"), + "selection_policy": record.get("prompt_context_selection_policy"), + } + + +def _prompt_context_present(context: Mapping[str, Any]) -> bool: + return isinstance(context.get("enabled"), bool) or any( + context.get(field_name) is not None + for field_name in _PROMPT_CONTEXT_COUNT_FIELDS + ) or bool(str(context.get("selection_policy") or "").strip()) + + +def _prompt_context_identity(context: Mapping[str, Any]) -> tuple[Any, ...]: + return ( + context.get("enabled") + if isinstance(context.get("enabled"), bool) + else None, + *( + _non_negative_int(context.get(field_name)) + for field_name in _PROMPT_CONTEXT_COUNT_FIELDS + ), + str(context.get("selection_policy") or "").strip(), + ) + + +def sanitize_invocation_record(record: Mapping[str, Any]) -> dict[str, Any]: + sanitized = { + key: value + for key, value in record.items() + if key in SAFE_INVOCATION_FIELDS + } + rate_card = _sanitize_rate_card(sanitized.get("rate_card")) + if rate_card is None: + sanitized.pop("rate_card", None) + else: + sanitized["rate_card"] = rate_card + context = _sanitize_prompt_context(sanitized.get("prompt_context")) + if context is None: + sanitized.pop("prompt_context", None) + else: + sanitized["prompt_context"] = context + if not isinstance(sanitized.get("prompt_context_enabled"), bool): + sanitized["prompt_context_enabled"] = None + for field_name in ( + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + "prompt_context_recent_events", + "prompt_context_max_events", + ): + sanitized[field_name] = _strict_non_negative_int( + sanitized.get(field_name) + ) + if ( + str(sanitized.get("prompt_context_selection_policy") or "").strip() + != PROMPT_EVENT_SELECTION_POLICY + ): + sanitized["prompt_context_selection_policy"] = "" + flat_context = _flat_prompt_context(sanitized) + sanitized["prompt_context_representation_conflict"] = bool( + sanitized.get("prompt_context_representation_conflict") is True + or ( + context is not None + and _prompt_context_present(context) + and _prompt_context_present(flat_context) + and _prompt_context_identity(context) + != _prompt_context_identity(flat_context) + ) + ) + return sanitized + + +def load_telemetry_directory(metrics_root: Path) -> tuple[dict[str, Any], ...]: + """Load and sanitize recorder shards from one trusted telemetry directory.""" + + records: list[dict[str, Any]] = [] + metrics_root = Path(metrics_root).expanduser().resolve() + if not metrics_root.is_dir(): + return () + for shard in sorted(metrics_root.rglob("*.jsonl")): + try: + lines = shard.read_text(encoding="utf-8").splitlines() + except OSError: + continue + for line in lines: + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(record, dict): + records.append(sanitize_invocation_record(record)) + records.sort( + key=lambda item: ( + str(item.get("started_at") or ""), + str(item.get("invocation_id") or ""), + ) + ) + return tuple(records) + + +def load_workspace_telemetry(workspace_root: Path) -> tuple[dict[str, Any], ...]: + """Load the normal runtime telemetry store outside benchmark evidence paths.""" + + return load_telemetry_directory( + workspace_root / ".teams_runtime" / "metrics" / "model_invocations" + ) + + +def _nearest_rank(values: list[int], percentile: float) -> int: + if not values: + return 0 + ordered = sorted(values) + index = max(math.ceil(percentile * len(ordered)) - 1, 0) + return ordered[index] + + +def _prompt_context(record: Mapping[str, Any]) -> dict[str, Any]: + flat = _flat_prompt_context(record) + if _prompt_context_present(flat): + return flat + nested = record.get("prompt_context") + if isinstance(nested, dict): + return dict(nested) + return flat + + +def _identity_digest(values: Iterable[Any]) -> str: + normalized = sorted(str(value or "").strip() for value in values) + canonical = json.dumps( + normalized, + ensure_ascii=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def is_v2_target_projection( + value: Mapping[str, Any], + *, + state_field: str, +) -> bool: + """Match the exact target using only canonical, journaled flat fields.""" + + if value.get("prompt_context_representation_conflict") is True: + return False + enabled = value.get("prompt_context_enabled") + invocation_id = str(value.get("invocation_id") or "").strip() + if not isinstance(enabled, bool) or not invocation_id: + return False + expected_included = ( + BENCHMARK_TARGET_INCLUDED_EVENTS + if enabled + else BENCHMARK_TARGET_TOTAL_EVENTS + ) + expected_omitted = BENCHMARK_TARGET_OMITTED_EVENTS if enabled else 0 + return ( + str(value.get(state_field) or "").strip() == "completed" + and str(value.get("attempt_kind") or "").strip() == "primary" + and str(value.get("role") or "").strip() == BENCHMARK_TARGET_ROLE + and str(value.get("purpose") or "").strip() == BENCHMARK_TARGET_PURPOSE + and str(value.get("workflow_step") or "").strip() + == BENCHMARK_TARGET_WORKFLOW_STEP + and _strict_non_negative_int(value.get("prompt_context_total_events")) + == BENCHMARK_TARGET_TOTAL_EVENTS + and _strict_non_negative_int(value.get("prompt_context_included_events")) + == expected_included + and _strict_non_negative_int(value.get("prompt_context_omitted_events")) + == expected_omitted + and _strict_non_negative_int(value.get("prompt_context_recent_events")) + == BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS + and _strict_non_negative_int(value.get("prompt_context_max_events")) + == BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS + and str(value.get("prompt_context_selection_policy") or "").strip() + == PROMPT_EVENT_SELECTION_POLICY + ) + + +def reduce_telemetry( + records: Iterable[Mapping[str, Any]], + *, + expected_invocation_count: int | None = None, + coverage_available: bool = True, + verified_target_projection_count: int = 0, + verified_target_invocation_ids_sha256: str = "", +) -> dict[str, Any]: + normalized = [sanitize_invocation_record(record) for record in records] + if not isinstance(coverage_available, bool): + raise ValueError("coverage_available must be a boolean") + if expected_invocation_count is not None and ( + isinstance(expected_invocation_count, bool) + or not isinstance(expected_invocation_count, int) + or expected_invocation_count < 0 + ): + raise ValueError("expected_invocation_count must be a non-negative integer") + if not coverage_available and expected_invocation_count is not None: + raise ValueError( + "expected_invocation_count must be omitted when coverage is unavailable" + ) + if ( + isinstance(verified_target_projection_count, bool) + or not isinstance(verified_target_projection_count, int) + or verified_target_projection_count < 0 + ): + raise ValueError( + "verified_target_projection_count must be a non-negative integer" + ) + normalized_verified_target_digest = str( + verified_target_invocation_ids_sha256 or "" + ).strip().lower() + if normalized_verified_target_digest and ( + len(normalized_verified_target_digest) != 64 + or any( + character not in "0123456789abcdef" + for character in normalized_verified_target_digest + ) + ): + raise ValueError( + "verified_target_invocation_ids_sha256 must be empty or a SHA-256 digest" + ) + observed_invocation_count = len(normalized) + coverage_denominator = max( + observed_invocation_count, + ( + expected_invocation_count + if expected_invocation_count is not None + else observed_invocation_count + ), + ) + logical_calls = { + str(record.get("logical_call_id") or "") + for record in normalized + if str(record.get("logical_call_id") or "") + } + durations: list[int] = [] + totals = { + "invocation_count": observed_invocation_count, + "coverage_basis": ( + "call_journal" + if coverage_available and expected_invocation_count is not None + else ( + "observed_telemetry" + if coverage_available + else "unavailable_untrusted_call_journal" + ) + ), + "expected_invocation_count": ( + coverage_denominator if coverage_available else None + ), + "unobserved_invocation_count": ( + max(coverage_denominator - observed_invocation_count, 0) + if coverage_available + else None + ), + "logical_call_count": len(logical_calls), + "primary_count": 0, + "contract_repair_count": 0, + "sandbox_retry_count": 0, + "completed_count": 0, + "failed_count": 0, + "tool_call_count": 0, + "prompt_chars": 0, + "output_chars": 0, + } + tokens = { + "input": 0, + "cached_input": 0, + "uncached_input": 0, + "output": 0, + "reasoning_output": 0, + "total": 0, + } + native_usage_count = 0 + tool_call_usage_count = 0 + priced_count = 0 + total_cost = 0.0 + compaction = { + "observed_invocation_count": 0, + "unobserved_invocation_count": 0, + "enabled_invocation_count": 0, + "eligible_invocation_count": 0, + "disabled_eligible_invocation_count": 0, + "compacted_invocation_count": 0, + "invalid_projection_count": 0, + "total_events": 0, + "included_events": 0, + "omitted_events": 0, + "max_observed_events": 0, + "max_included_events": 0, + "expected_recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "expected_max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "expected_selection_policy": PROMPT_EVENT_SELECTION_POLICY, + "target_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_role": BENCHMARK_TARGET_ROLE, + "target_purpose": BENCHMARK_TARGET_PURPOSE, + "target_workflow_step": BENCHMARK_TARGET_WORKFLOW_STEP, + "target_included_events_when_enabled": BENCHMARK_TARGET_INCLUDED_EVENTS, + "target_omitted_events_when_enabled": BENCHMARK_TARGET_OMITTED_EVENTS, + "target_projection_count": 0, + "target_projection_candidate_count": 0, + "target_projection_verification_mismatch_count": 0, + "target_projection_invocation_ids_sha256": "", + "target_projection_identity_reconciled": False, + "selection_policies": [], + } + selection_policies: set[str] = set() + target_candidate_invocation_ids: list[str] = [] + groups: dict[tuple[str, str, str, str], dict[str, Any]] = {} + for record in normalized: + attempt_kind = str(record.get("attempt_kind") or "") + if attempt_kind == "primary": + totals["primary_count"] += 1 + elif attempt_kind == "contract_repair": + totals["contract_repair_count"] += 1 + elif attempt_kind == "sandbox_retry": + totals["sandbox_retry_count"] += 1 + if str(record.get("status") or "") == "completed": + totals["completed_count"] += 1 + else: + totals["failed_count"] += 1 + duration = _non_negative_int(record.get("duration_ms")) or 0 + durations.append(duration) + tool_calls = _non_negative_int(record.get("tool_calls")) + if tool_calls is not None: + tool_call_usage_count += 1 + totals["tool_call_count"] += tool_calls + totals["prompt_chars"] += _non_negative_int(record.get("prompt_chars")) or 0 + totals["output_chars"] += _non_negative_int(record.get("output_chars")) or 0 + raw_input_tokens = _non_negative_int(record.get("input_tokens")) + raw_cached_tokens = _non_negative_int(record.get("cached_input_tokens")) + raw_output_tokens = _non_negative_int(record.get("output_tokens")) + raw_total_tokens = _non_negative_int(record.get("total_tokens")) + input_tokens = raw_input_tokens or 0 + cached_tokens = min(raw_cached_tokens or 0, input_tokens) + output_tokens = raw_output_tokens or 0 + tokens["input"] += input_tokens + tokens["cached_input"] += cached_tokens + tokens["uncached_input"] += max(input_tokens - cached_tokens, 0) + tokens["output"] += output_tokens + tokens["reasoning_output"] += _non_negative_int(record.get("reasoning_output_tokens")) or 0 + effective_total_tokens = ( + raw_total_tokens + if raw_total_tokens is not None + else input_tokens + output_tokens + ) + tokens["total"] += effective_total_tokens + complete_native_usage = ( + str(record.get("usage_source") or "") == "native" + and raw_input_tokens is not None + and raw_output_tokens is not None + and effective_total_tokens >= raw_input_tokens + raw_output_tokens + ) + if complete_native_usage: + native_usage_count += 1 + cost = _finite_number(record.get("estimated_cost_usd")) + if cost is not None: + priced_count += 1 + total_cost += cost + context = _prompt_context(record) + total_events = _non_negative_int(context.get("total_events")) + included_events = _non_negative_int(context.get("included_events")) + omitted_events = _non_negative_int(context.get("omitted_events")) + recent_events = _non_negative_int(context.get("recent_events")) + max_events = _non_negative_int(context.get("max_events")) + enabled = context.get("enabled") + selection_policy = str(context.get("selection_policy") or "").strip() + representation_conflict = ( + record.get("prompt_context_representation_conflict") is True + ) + projection_candidate = isinstance(enabled, bool) or any( + value is not None + for value in ( + total_events, + included_events, + omitted_events, + recent_events, + max_events, + ) + ) or bool(selection_policy) + projection_observed = isinstance(enabled, bool) and all( + value is not None + for value in ( + total_events, + included_events, + omitted_events, + recent_events, + max_events, + ) + ) + projection_valid = False + if projection_observed: + compaction["observed_invocation_count"] += 1 + compaction["enabled_invocation_count"] += int(enabled) + if selection_policy: + selection_policies.add(selection_policy) + projection_valid = ( + not representation_conflict + and max_events == BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS + and recent_events == BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS + and total_events == included_events + omitted_events + and selection_policy == PROMPT_EVENT_SELECTION_POLICY + and ( + ( + enabled + and ( + (total_events <= max_events and included_events == total_events and omitted_events == 0) + or ( + total_events > max_events + and recent_events <= included_events <= max_events + and omitted_events > 0 + ) + ) + ) + or ( + not enabled + and included_events == total_events + and omitted_events == 0 + ) + ) + ) + if not projection_valid: + compaction["invalid_projection_count"] += 1 + elif projection_candidate: + compaction["invalid_projection_count"] += 1 + if projection_valid and total_events is not None: + compaction["max_observed_events"] = max(compaction["max_observed_events"], total_events) + compaction["total_events"] += total_events + if max_events is not None and total_events > max_events: + compaction["eligible_invocation_count"] += 1 + if not enabled: + compaction["disabled_eligible_invocation_count"] += 1 + if projection_valid and included_events is not None: + compaction["included_events"] += included_events + compaction["max_included_events"] = max( + compaction["max_included_events"], + included_events, + ) + if projection_valid and omitted_events is not None: + compaction["omitted_events"] += omitted_events + if enabled and omitted_events > 0: + compaction["compacted_invocation_count"] += 1 + if projection_valid and is_v2_target_projection( + record, + state_field="status", + ): + compaction["target_projection_candidate_count"] += 1 + target_candidate_invocation_ids.append( + str(record.get("invocation_id") or "").strip() + ) + + key = tuple( + str(record.get(field_name) or "") + for field_name in ("role", "purpose", "provider", "model") + ) + group = groups.setdefault( + key, + { + "role": key[0], + "purpose": key[1], + "provider": key[2], + "model": key[3], + "invocation_count": 0, + "failed_count": 0, + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "duration_ms": 0, + "estimated_cost_usd": None, + "_priced_count": 0, + }, + ) + group["invocation_count"] += 1 + group["failed_count"] += int(str(record.get("status") or "") != "completed") + group["input_tokens"] += input_tokens + group["cached_input_tokens"] += cached_tokens + group["output_tokens"] += output_tokens + group["total_tokens"] += effective_total_tokens + group["duration_ms"] += duration + if cost is not None: + group["_priced_count"] += 1 + group["estimated_cost_usd"] = round( + (group["estimated_cost_usd"] or 0.0) + cost, + 12, + ) + + count = observed_invocation_count + for group in groups.values(): + if ( + group.pop("_priced_count") != group["invocation_count"] + or not coverage_available + or count != coverage_denominator + ): + group["estimated_cost_usd"] = None + token_coverage = ( + round(native_usage_count * 100 / coverage_denominator, 2) + if coverage_denominator + else 0.0 + ) + tool_call_coverage = ( + round(tool_call_usage_count * 100 / coverage_denominator, 2) + if coverage_denominator + else 0.0 + ) + pricing_coverage = ( + round(priced_count * 100 / coverage_denominator, 2) + if coverage_denominator + else 0.0 + ) + totals["token_coverage_percent"] = ( + token_coverage if coverage_available else None + ) + totals["tool_call_coverage_percent"] = ( + tool_call_coverage if coverage_available else None + ) + totals["pricing_coverage_percent"] = ( + pricing_coverage if coverage_available else None + ) + totals["estimated_cost_usd"] = ( + round(total_cost, 12) + if coverage_available + and coverage_denominator + and count == coverage_denominator + and priced_count == coverage_denominator + else None + ) + compaction["unobserved_invocation_count"] = ( + coverage_denominator - compaction["observed_invocation_count"] + if coverage_available + else None + ) + target_candidate_count = int( + compaction["target_projection_candidate_count"] + ) + compaction["target_projection_verification_mismatch_count"] = abs( + target_candidate_count - verified_target_projection_count + ) + target_candidate_digest = _identity_digest(target_candidate_invocation_ids) + compaction["target_projection_invocation_ids_sha256"] = ( + target_candidate_digest + ) + target_identity_reconciled = bool( + normalized_verified_target_digest + and target_candidate_count == verified_target_projection_count + and target_candidate_digest == normalized_verified_target_digest + ) + compaction["target_projection_identity_reconciled"] = ( + target_identity_reconciled + ) + if target_identity_reconciled: + compaction["target_projection_count"] = verified_target_projection_count + compaction["selection_policies"] = sorted(selection_policies) + return { + "totals": totals, + "tokens": tokens, + "latency_ms": { + "provider_total": sum(durations), + "p50": _nearest_rank(durations, 0.50), + "p95": _nearest_rank(durations, 0.95), + "max": max(durations, default=0), + }, + "compaction": compaction, + "groups": sorted( + groups.values(), + key=lambda item: ( + -int(item["total_tokens"]), + -int(item["duration_ms"]), + item["role"], + item["purpose"], + ), + ), + } + + +def _primary_groups( + records: Iterable[Mapping[str, Any]], +) -> dict[tuple[str, str, str], list[Mapping[str, Any]]]: + grouped: defaultdict[ + tuple[str, str, str], + list[Mapping[str, Any]], + ] = defaultdict(list) + for record in records: + if str(record.get("attempt_kind") or "") != "primary": + continue + base = tuple( + str(record.get(field_name) or "") + for field_name in ("role", "purpose", "workflow_step") + ) + grouped[base].append(record) + return dict(grouped) + + +def _primary_identity(record: Mapping[str, Any]) -> tuple[Any, ...]: + context = _prompt_context(record) + return ( + str(record.get("provider") or ""), + str(record.get("model") or ""), + str(record.get("reasoning") or ""), + str(record.get("status") or ""), + _non_negative_int(context.get("total_events")), + ) + + +def _delta(before: float | int | None, after: float | int | None) -> dict[str, Any]: + if before is None or after is None: + return { + "before": before, + "after": after, + "delta": None, + "change_percent": None, + "reduction": None, + "reduction_percent": None, + } + delta = after - before + reduction = before - after + return { + "before": before, + "after": after, + "delta": delta, + "change_percent": round(delta * 100 / before, 4) if before else None, + "reduction": reduction, + "reduction_percent": round(reduction * 100 / before, 4) if before else None, + } + + +def compare_metrics( + before_metrics: Mapping[str, Any], + after_metrics: Mapping[str, Any], + *, + before_wall_duration_ms: int, + after_wall_duration_ms: int, + before_records: Iterable[Mapping[str, Any]], + after_records: Iterable[Mapping[str, Any]], +) -> dict[str, Any]: + before_totals = dict(before_metrics.get("totals") or {}) + after_totals = dict(after_metrics.get("totals") or {}) + before_tokens = dict(before_metrics.get("tokens") or {}) + after_tokens = dict(after_metrics.get("tokens") or {}) + before_latency = dict(before_metrics.get("latency_ms") or {}) + after_latency = dict(after_metrics.get("latency_ms") or {}) + end_to_end = { + "invocation_count": _delta( + before_totals.get("invocation_count"), + after_totals.get("invocation_count"), + ), + "logical_call_count": _delta( + before_totals.get("logical_call_count"), + after_totals.get("logical_call_count"), + ), + "contract_repair_count": _delta( + before_totals.get("contract_repair_count"), + after_totals.get("contract_repair_count"), + ), + "sandbox_retry_count": _delta( + before_totals.get("sandbox_retry_count"), + after_totals.get("sandbox_retry_count"), + ), + "failed_count": _delta( + before_totals.get("failed_count"), + after_totals.get("failed_count"), + ), + "tool_call_count": _delta( + before_totals.get("tool_call_count"), + after_totals.get("tool_call_count"), + ), + "prompt_chars": _delta( + before_totals.get("prompt_chars"), + after_totals.get("prompt_chars"), + ), + "input_tokens": _delta( + before_tokens.get("input"), + after_tokens.get("input"), + ), + "cached_input_tokens": _delta( + before_tokens.get("cached_input"), + after_tokens.get("cached_input"), + ), + "uncached_input_tokens": _delta( + before_tokens.get("uncached_input"), + after_tokens.get("uncached_input"), + ), + "output_tokens": _delta( + before_tokens.get("output"), + after_tokens.get("output"), + ), + "reasoning_output_tokens": _delta( + before_tokens.get("reasoning_output"), + after_tokens.get("reasoning_output"), + ), + "total_tokens": _delta( + before_tokens.get("total"), + after_tokens.get("total"), + ), + "provider_duration_ms": _delta( + before_latency.get("provider_total"), + after_latency.get("provider_total"), + ), + "wall_duration_ms": _delta(before_wall_duration_ms, after_wall_duration_ms), + "estimated_cost_usd": _delta( + before_totals.get("estimated_cost_usd"), + after_totals.get("estimated_cost_usd"), + ), + } + + before_primary = _primary_groups(before_records) + after_primary = _primary_groups(after_records) + matched: list[dict[str, Any]] = [] + unmatched_before = 0 + unmatched_after = 0 + ambiguous_groups = 0 + for key in sorted(before_primary.keys() | after_primary.keys()): + before_group = before_primary.get(key, []) + after_group = after_primary.get(key, []) + unambiguous = ( + len(before_group) == 1 + and len(after_group) == 1 + and _primary_identity(before_group[0]) == _primary_identity(after_group[0]) + ) + if not unambiguous: + unmatched_before += len(before_group) + unmatched_after += len(after_group) + ambiguous_groups += int(bool(before_group) and bool(after_group)) + continue + before = before_group[0] + after = after_group[0] + matched.append( + { + "role": key[0], + "purpose": key[1], + "workflow_step": key[2], + "occurrence": 1, + "prompt_chars": _delta( + _non_negative_int(before.get("prompt_chars")), + _non_negative_int(after.get("prompt_chars")), + ), + "input_tokens": _delta( + _non_negative_int(before.get("input_tokens")), + _non_negative_int(after.get("input_tokens")), + ), + "cached_input_tokens": _delta( + _non_negative_int(before.get("cached_input_tokens")), + _non_negative_int(after.get("cached_input_tokens")), + ), + "total_tokens": _delta( + _non_negative_int(before.get("total_tokens")), + _non_negative_int(after.get("total_tokens")), + ), + "duration_ms": _delta( + _non_negative_int(before.get("duration_ms")), + _non_negative_int(after.get("duration_ms")), + ), + } + ) + return { + "end_to_end": end_to_end, + "matched_primary_invocations": matched, + "matched_primary_count": len(matched), + "unmatched_before_primary_count": unmatched_before, + "unmatched_after_primary_count": unmatched_after, + "ambiguous_primary_group_count": ambiguous_groups, + } + + +__all__ = [ + "SAFE_INVOCATION_FIELDS", + "compare_metrics", + "is_v2_target_projection", + "load_telemetry_directory", + "load_workspace_telemetry", + "reduce_telemetry", + "sanitize_invocation_record", +] diff --git a/benchmarking/models.py b/benchmarking/models.py new file mode 100644 index 0000000..a0d5000 --- /dev/null +++ b/benchmarking/models.py @@ -0,0 +1,390 @@ +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Iterable, Literal, Mapping, Protocol, Sequence + + +BenchmarkVariant = Literal["before", "after"] +KeepWorkspaces = Literal["none", "failures", "all"] +RunStatus = Literal[ + "completed", + "failed", + "timeout", + "call_budget_exhausted", + "preflight_failed", +] +_INVOCATION_ATTEMPT_BOOLEAN_FIELDS = frozenset( + { + "journal_available", + "context_reconciled", + "identity_reconciled", + "reconciled", + } +) +_INVOCATION_ATTEMPT_INTEGER_FIELDS = frozenset( + { + "schema_version", + "journal_schema_version", + "max_invocations", + "reserved_count", + "entry_count", + "telemetry_record_count", + "unobserved_attempt_count", + "telemetry_overage_count", + "completed_count", + "failed_count", + "timeout_count", + "launch_failed_count", + "terminated_count", + "active_count", + "unknown_state_count", + "malformed_entry_count", + "unaccounted_count", + "overaccounted_count", + "rejected_count", + "remaining_budget", + "journal_invocation_id_missing_count", + "journal_invocation_id_duplicate_count", + "telemetry_invocation_id_missing_count", + "telemetry_invocation_id_duplicate_count", + "telemetry_invocation_id_unmatched_count", + "journal_invocation_id_unobserved_count", + "journal_telemetry_context_mismatch_count", + "verified_target_projection_count", + } +) +_INVOCATION_ATTEMPT_FLOAT_FIELDS = frozenset( + { + "telemetry_coverage_percent", + } +) +_INVOCATION_ATTEMPT_HASH_FIELDS = frozenset( + { + "journal_invocation_ids_sha256", + "verified_target_invocation_ids_sha256", + } +) + + +def invocation_identity_digest(values: Iterable[Any]) -> str: + normalized = sorted( + str(value or "").strip() + for value in values + ) + canonical = json.dumps( + normalized, + ensure_ascii=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def sanitize_invocation_attempts( + value: Any, +) -> dict[str, bool | float | int | str]: + """Keep only bounded count fields at the worker/report privacy boundary.""" + + if not isinstance(value, Mapping): + return {} + sanitized: dict[str, bool | float | int | str] = {} + for field_name in _INVOCATION_ATTEMPT_BOOLEAN_FIELDS: + raw_value = value.get(field_name) + if isinstance(raw_value, bool): + sanitized[field_name] = raw_value + for field_name in _INVOCATION_ATTEMPT_INTEGER_FIELDS: + raw_value = value.get(field_name) + if raw_value is None or isinstance(raw_value, bool): + continue + try: + normalized = int(raw_value) + except (OverflowError, TypeError, ValueError): + continue + if normalized >= 0: + sanitized[field_name] = normalized + for field_name in _INVOCATION_ATTEMPT_FLOAT_FIELDS: + raw_value = value.get(field_name) + if isinstance(raw_value, bool): + continue + try: + normalized = float(raw_value) + except (OverflowError, TypeError, ValueError): + continue + if math.isfinite(normalized) and 0.0 <= normalized <= 100.0: + sanitized[field_name] = normalized + for field_name in _INVOCATION_ATTEMPT_HASH_FIELDS: + normalized = str(value.get(field_name) or "").strip().lower() + if ( + len(normalized) == 64 + and all(character in "0123456789abcdef" for character in normalized) + ): + sanitized[field_name] = normalized + return dict(sorted(sanitized.items())) + + +class BenchmarkWorkerSafetyError(RuntimeError): + """Raised when benchmark worker isolation cannot be proven safe.""" + + +@dataclass(slots=True, frozen=True) +class BenchmarkOptions: + """Operator-selected controls for a sprint A/B benchmark.""" + + source_root: Path + runtime_config_path: Path + output_dir: Path | None = None + rate_card_path: Path | None = None + repetitions: int = 1 + max_invocations: int = 20 + call_timeout_seconds: float = 300.0 + run_timeout_seconds: float = 1800.0 + keep_workspaces: KeepWorkspaces = "failures" + allow_dirty_source: bool = False + live: bool = False + benchmark_id: str = "" + + def validate(self) -> None: + if self.repetitions <= 0: + raise ValueError("repetitions must be a positive integer") + if self.max_invocations <= 0: + raise ValueError("max_invocations must be a positive integer") + if ( + not math.isfinite(self.call_timeout_seconds) + or self.call_timeout_seconds <= 0 + ): + raise ValueError("call_timeout_seconds must be positive and finite") + if ( + not math.isfinite(self.run_timeout_seconds) + or self.run_timeout_seconds <= 0 + ): + raise ValueError("run_timeout_seconds must be positive and finite") + if self.keep_workspaces not in {"none", "failures", "all"}: + raise ValueError("keep_workspaces must be none, failures, or all") + if not self.source_root.expanduser().is_dir(): + raise FileNotFoundError(f"Source root does not exist: {self.source_root}") + config_path = self.runtime_config_path.expanduser() + if not config_path.exists(): + raise FileNotFoundError(f"Runtime config does not exist: {config_path}") + if self.rate_card_path is not None and not self.rate_card_path.expanduser().is_file(): + raise FileNotFoundError(f"Rate card does not exist: {self.rate_card_path}") + + +@dataclass(slots=True, frozen=True) +class ArmPlan: + pair_index: int + order_index: int + variant: BenchmarkVariant + run_id: str + prompt_context_enabled: bool + + +@dataclass(slots=True, frozen=True) +class SprintEvidence: + sprint_id: str = "" + status: str = "" + closeout_status: str = "" + todo_count: int = 0 + completed_todo_count: int = 0 + blocked_todo_count: int = 0 + failed_todo_count: int = 0 + commit_sha: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "sprint_id": self.sprint_id, + "status": self.status, + "closeout_status": self.closeout_status, + "todo_count": self.todo_count, + "completed_todo_count": self.completed_todo_count, + "blocked_todo_count": self.blocked_todo_count, + "failed_todo_count": self.failed_todo_count, + "commit_sha": self.commit_sha, + } + + +@dataclass(slots=True, frozen=True) +class QualityEvidence: + behavior_oracle_passed: bool = False + sprint_terminal: bool = False + closeout_verified: bool = False + protected_files_unchanged: bool = False + git_clean: bool = False + commit_created: bool = False + no_git_remotes: bool = False + blocked_todo_count: int = 0 + failed_todo_count: int = 0 + notes: tuple[str, ...] = () + + @property + def passed(self) -> bool: + return ( + self.behavior_oracle_passed + and self.sprint_terminal + and self.closeout_verified + and self.protected_files_unchanged + and self.git_clean + and self.commit_created + and self.no_git_remotes + and self.blocked_todo_count == 0 + and self.failed_todo_count == 0 + ) + + def to_dict(self) -> dict[str, Any]: + return { + "passed": self.passed, + "behavior_oracle_passed": self.behavior_oracle_passed, + "sprint_terminal": self.sprint_terminal, + "closeout_verified": self.closeout_verified, + "protected_files_unchanged": self.protected_files_unchanged, + "git_clean": self.git_clean, + "commit_created": self.commit_created, + "no_git_remotes": self.no_git_remotes, + "blocked_todo_count": self.blocked_todo_count, + "failed_todo_count": self.failed_todo_count, + "notes": list(self.notes), + } + + +@dataclass(slots=True, frozen=True) +class WorkerContext: + """Everything an orchestration worker needs for one isolated arm.""" + + benchmark_id: str + arm: ArmPlan + workspace_root: Path + run_output_dir: Path + milestone: str + history_seed: tuple[Mapping[str, Any], ...] + max_invocations: int + call_timeout_seconds: float + run_timeout_seconds: float + live: bool + + +@dataclass(slots=True, frozen=True) +class WorkerOutcome: + """Privacy-safe result returned by an injected sprint worker.""" + + status: RunStatus + sprint: SprintEvidence = field(default_factory=SprintEvidence) + quality: QualityEvidence = field(default_factory=QualityEvidence) + telemetry_records: tuple[Mapping[str, Any], ...] = () + invocation_attempts: Mapping[str, Any] = field(default_factory=dict) + started_at: str = "" + ended_at: str = "" + wall_duration_ms: int = 0 + worker_duration_ms: int = 0 + stop_reason: str = "" + error_category: str = "" + + +class BenchmarkWorker(Protocol): + def __call__(self, context: WorkerContext) -> WorkerOutcome: + """Run one complete sprint arm and return privacy-safe evidence.""" + + +@dataclass(slots=True, frozen=True) +class ArmResult: + arm: ArmPlan + status: RunStatus + started_at: str + ended_at: str + wall_duration_ms: int + worker_duration_ms: int + stop_reason: str + error_category: str + config_hash: str + comparable_config_hash: str + metrics: Mapping[str, Any] + quality: QualityEvidence + sprint: SprintEvidence + invocation_attempts: Mapping[str, Any] = field(default_factory=dict) + invocation_records: tuple[Mapping[str, Any], ...] = () + retained_workspace: str = "" + + def to_dict(self, *, include_records: bool = False) -> dict[str, Any]: + result = { + "run_id": self.arm.run_id, + "pair_index": self.arm.pair_index, + "order_index": self.arm.order_index, + "variant": self.arm.variant, + "prompt_context_enabled": self.arm.prompt_context_enabled, + "status": self.status, + "started_at": self.started_at, + "ended_at": self.ended_at, + "wall_duration_ms": self.wall_duration_ms, + "worker_duration_ms": self.worker_duration_ms, + "stop_reason": self.stop_reason, + "error_category": self.error_category, + "config_hash": self.config_hash, + "comparable_config_hash": self.comparable_config_hash, + "metrics": dict(self.metrics), + "quality": self.quality.to_dict(), + "sprint": self.sprint.to_dict(), + "invocation_attempts": sanitize_invocation_attempts( + self.invocation_attempts + ), + "retained_workspace": self.retained_workspace, + } + if include_records: + result["invocations"] = [dict(record) for record in self.invocation_records] + return result + + +@dataclass(slots=True, frozen=True) +class BenchmarkResult: + benchmark_id: str + status: Literal["comparable", "inconclusive"] + classification: Literal["preliminary_smoke", "repeated_experiment"] + output_dir: Path + report_json: Path + report_markdown: Path + runs: tuple[ArmResult, ...] + report: Mapping[str, Any] + + @property + def exit_code(self) -> int: + return 0 if self.status == "comparable" else 1 + + +def make_arm_schedule(repetitions: int) -> tuple[ArmPlan, ...]: + if repetitions <= 0: + raise ValueError("repetitions must be a positive integer") + schedule: list[ArmPlan] = [] + order_index = 0 + for pair_index in range(1, repetitions + 1): + variants: Sequence[BenchmarkVariant] = ( + ("before", "after") if pair_index % 2 else ("after", "before") + ) + for variant in variants: + order_index += 1 + schedule.append( + ArmPlan( + pair_index=pair_index, + order_index=order_index, + variant=variant, + run_id=f"pair-{pair_index:03d}-{variant}", + prompt_context_enabled=variant == "after", + ) + ) + return tuple(schedule) + + +__all__ = [ + "ArmPlan", + "ArmResult", + "BenchmarkOptions", + "BenchmarkResult", + "BenchmarkWorker", + "BenchmarkWorkerSafetyError", + "QualityEvidence", + "SprintEvidence", + "WorkerContext", + "WorkerOutcome", + "invocation_identity_digest", + "make_arm_schedule", + "sanitize_invocation_attempts", +] diff --git a/benchmarking/reporting.py b/benchmarking/reporting.py new file mode 100644 index 0000000..cc5f020 --- /dev/null +++ b/benchmarking/reporting.py @@ -0,0 +1,443 @@ +from __future__ import annotations + +import json +import os +import statistics +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping + +from teams_runtime.benchmarking.metrics import compare_metrics +from teams_runtime.benchmarking.models import ArmResult, BenchmarkOptions +from teams_runtime.benchmarking.scenario import ( + BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + BENCHMARK_TARGET_INCLUDED_EVENTS, + BENCHMARK_TARGET_OMITTED_EVENTS, + BENCHMARK_TARGET_PURPOSE, + BENCHMARK_TARGET_ROLE, + BENCHMARK_TARGET_TOTAL_EVENTS, + BENCHMARK_TARGET_WORKFLOW_STEP, + DEFAULT_HISTORY_SEED_COUNT, + SCENARIO_ID, +) +from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY + + +REPORT_SCHEMA_VERSION = 3 + + +def utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def write_text_atomic(path: Path, content: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{path.name}.", + suffix=".tmp", + dir=path.parent, + text=True, + ) + temporary_path = Path(temporary_name) + try: + with os.fdopen(file_descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + finally: + temporary_path.unlink(missing_ok=True) + + +def write_json_atomic(path: Path, payload: Mapping[str, Any]) -> None: + write_text_atomic( + path, + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + ) + + +def write_jsonl_atomic(path: Path, records: Iterable[Mapping[str, Any]]) -> None: + content = "".join( + json.dumps(record, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + "\n" + for record in records + ) + write_text_atomic(path, content) + + +def write_run_artifacts(run_dir: Path, result: ArmResult) -> None: + run_dir.mkdir(parents=True, exist_ok=True) + write_json_atomic(run_dir / "run.json", result.to_dict()) + write_json_atomic(run_dir / "metrics.json", dict(result.metrics)) + write_json_atomic(run_dir / "sprint.json", result.sprint.to_dict()) + write_json_atomic(run_dir / "quality.json", result.quality.to_dict()) + write_jsonl_atomic(run_dir / "model_invocations.jsonl", result.invocation_records) + + +def _pair_comparability(before: ArmResult, after: ArmResult) -> tuple[bool, list[str]]: + reasons: list[str] = [] + if before.comparable_config_hash != after.comparable_config_hash: + reasons.append("non_feature_configuration_differs") + for label, arm in (("before", before), ("after", after)): + if arm.status != "completed": + reasons.append(f"{label}_status_{arm.status}") + if not arm.quality.passed: + reasons.append(f"{label}_quality_failed") + attempts = dict(arm.invocation_attempts) + if attempts.get("journal_available") is not True: + reasons.append(f"{label}_call_journal_missing") + elif int(attempts.get("journal_schema_version") or 0) not in {1, 2, 3}: + reasons.append(f"{label}_call_journal_schema_unsupported") + else: + if attempts.get("reconciled") is not True: + reasons.append(f"{label}_call_journal_not_reconciled") + if attempts.get("identity_reconciled") is not True: + reasons.append( + f"{label}_invocation_identity_not_reconciled" + ) + if int(attempts.get("journal_schema_version") or 0) != 3: + reasons.append(f"{label}_call_journal_context_unavailable") + elif attempts.get("context_reconciled") is not True: + reasons.append(f"{label}_invocation_context_not_reconciled") + for field_name, reason_suffix in ( + ("active_count", "active_attempts_present"), + ("unknown_state_count", "unknown_attempt_states"), + ("malformed_entry_count", "malformed_attempt_entries"), + ("unaccounted_count", "unaccounted_attempts"), + ("overaccounted_count", "overaccounted_attempts"), + ("telemetry_overage_count", "telemetry_attempt_overage"), + ("unobserved_attempt_count", "unobserved_attempts"), + ("terminated_count", "terminated_attempts"), + ("rejected_count", "rejected_attempts"), + ( + "journal_invocation_id_missing_count", + "journal_invocation_ids_missing", + ), + ( + "journal_invocation_id_duplicate_count", + "journal_invocation_ids_duplicated", + ), + ( + "telemetry_invocation_id_missing_count", + "telemetry_invocation_ids_missing", + ), + ( + "telemetry_invocation_id_duplicate_count", + "telemetry_invocation_ids_duplicated", + ), + ( + "telemetry_invocation_id_unmatched_count", + "telemetry_invocation_ids_unmatched", + ), + ): + if int(attempts.get(field_name) or 0) > 0: + reasons.append(f"{label}_{reason_suffix}") + totals = dict(arm.metrics.get("totals") or {}) + if float(totals.get("token_coverage_percent") or 0.0) != 100.0: + reasons.append(f"{label}_native_token_coverage_incomplete") + compaction = dict(arm.metrics.get("compaction") or {}) + if int(compaction.get("invalid_projection_count") or 0): + reasons.append(f"{label}_prompt_projection_invalid") + if int( + compaction.get("target_projection_verification_mismatch_count") + or 0 + ): + reasons.append(f"{label}_v2_target_projection_not_reconciled") + if compaction.get("target_projection_identity_reconciled") is not True: + reasons.append( + f"{label}_v2_target_projection_identity_not_reconciled" + ) + if int(compaction.get("max_observed_events") or 0) < DEFAULT_HISTORY_SEED_COUNT: + reasons.append(f"{label}_backfill_not_observed") + if compaction.get("selection_policies") != [PROMPT_EVENT_SELECTION_POLICY]: + reasons.append(f"{label}_selection_policy_unverified") + before_compaction = dict(before.metrics.get("compaction") or {}) + after_compaction = dict(after.metrics.get("compaction") or {}) + if int(before_compaction.get("enabled_invocation_count") or 0): + reasons.append("before_compaction_unexpectedly_enabled") + if int(before_compaction.get("eligible_invocation_count") or 0) <= 0: + reasons.append("before_compaction_eligibility_not_observed") + if int(before_compaction.get("disabled_eligible_invocation_count") or 0) <= 0: + reasons.append("before_disabled_projection_not_observed") + if int(before_compaction.get("compacted_invocation_count") or 0): + reasons.append("before_compaction_unexpectedly_observed") + if int(before_compaction.get("target_projection_count") or 0) <= 0: + reasons.append("before_v2_target_projection_not_observed") + if int(after_compaction.get("enabled_invocation_count") or 0) <= 0: + reasons.append("after_compaction_not_enabled") + if int(after_compaction.get("enabled_invocation_count") or 0) != int( + after_compaction.get("observed_invocation_count") or 0 + ): + reasons.append("after_prompt_projection_not_uniformly_enabled") + if int(after_compaction.get("disabled_eligible_invocation_count") or 0): + reasons.append("after_disabled_projection_observed") + if int(after_compaction.get("compacted_invocation_count") or 0) <= 0: + reasons.append("after_compaction_not_observed") + if int(after_compaction.get("target_projection_count") or 0) <= 0: + reasons.append("after_v2_target_projection_not_observed") + return not reasons, reasons + + +def _build_pairs(runs: tuple[ArmResult, ...]) -> list[dict[str, Any]]: + pair_indexes = sorted({run.arm.pair_index for run in runs}) + pairs: list[dict[str, Any]] = [] + for pair_index in pair_indexes: + pair_runs = { + run.arm.variant: run + for run in runs + if run.arm.pair_index == pair_index + } + before = pair_runs.get("before") + after = pair_runs.get("after") + if before is None or after is None: + pairs.append( + { + "pair_index": pair_index, + "comparable": False, + "inconclusive_reasons": ["missing_arm"], + } + ) + continue + comparable, reasons = _pair_comparability(before, after) + comparison = compare_metrics( + before.metrics, + after.metrics, + before_wall_duration_ms=before.wall_duration_ms, + after_wall_duration_ms=after.wall_duration_ms, + before_records=before.invocation_records, + after_records=after.invocation_records, + ) + pairs.append( + { + "pair_index": pair_index, + "execution_order": [ + run.arm.variant + for run in sorted((before, after), key=lambda item: item.arm.order_index) + ], + "before_run_id": before.arm.run_id, + "after_run_id": after.arm.run_id, + "comparable": comparable, + "inconclusive_reasons": reasons, + "comparison": comparison, + } + ) + return pairs + + +def _aggregate_pair_metrics(pairs: list[dict[str, Any]]) -> dict[str, Any]: + comparable_pairs = [pair for pair in pairs if pair.get("comparable")] + metric_values: dict[str, list[float]] = {} + for pair in comparable_pairs: + end_to_end = dict((pair.get("comparison") or {}).get("end_to_end") or {}) + for metric_name, delta_payload in end_to_end.items(): + reduction = (delta_payload or {}).get("reduction") + if isinstance(reduction, (int, float)) and not isinstance(reduction, bool): + metric_values.setdefault(metric_name, []).append(float(reduction)) + result: dict[str, Any] = {} + for metric_name, values in sorted(metric_values.items()): + result[metric_name] = { + "pair_count": len(values), + "mean_reduction": statistics.fmean(values), + "median_reduction": statistics.median(values), + "sample_standard_deviation": ( + statistics.stdev(values) if len(values) > 1 else None + ), + } + return result + + +def build_report( + *, + benchmark_id: str, + options: BenchmarkOptions, + source_revision: Mapping[str, Any], + source_config_hash: str, + runtime_model_map: Mapping[str, Mapping[str, str]], + rate_cards: Mapping[str, Mapping[str, float | None]], + history_hash: str, + runs: tuple[ArmResult, ...], + started_at: str, + ended_at: str, +) -> dict[str, Any]: + pairs = _build_pairs(runs) + comparable = bool(pairs) and all(pair.get("comparable") for pair in pairs) + status = "comparable" if comparable else "inconclusive" + return { + "schema_version": REPORT_SCHEMA_VERSION, + "benchmark_id": benchmark_id, + "benchmark": "sprint_ab", + "classification": ( + "preliminary_smoke" if options.repetitions == 1 else "repeated_experiment" + ), + "status": status, + "started_at": started_at, + "ended_at": ended_at, + "provenance": { + "scenario_id": SCENARIO_ID, + "source": dict(source_revision), + "source_config_hash": source_config_hash, + "history_hash": history_hash, + "runtime_model_map": { + role: dict(values) + for role, values in runtime_model_map.items() + }, + "rate_cards": { + key: dict(values) + for key, values in rate_cards.items() + }, + }, + "controls": { + "repetitions": options.repetitions, + "max_invocations_per_arm": options.max_invocations, + "call_timeout_seconds": options.call_timeout_seconds, + "run_timeout_seconds": options.run_timeout_seconds, + "keep_workspaces": options.keep_workspaces, + "live": options.live, + "target_invocation": { + "attempt_kind": "primary", + "role": BENCHMARK_TARGET_ROLE, + "purpose": BENCHMARK_TARGET_PURPOSE, + "workflow_step": BENCHMARK_TARGET_WORKFLOW_STEP, + }, + "a_b_definition": { + "before": { + "prompt_context_enabled": False, + "target_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_included_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_omitted_events": 0, + }, + "after": { + "prompt_context_enabled": True, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "target_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_included_events": BENCHMARK_TARGET_INCLUDED_EVENTS, + "target_omitted_events": BENCHMARK_TARGET_OMITTED_EVENTS, + }, + }, + }, + "runs": [run.to_dict() for run in runs], + "pairs": pairs, + "aggregate_reductions": _aggregate_pair_metrics(pairs), + "interpretation": { + "statistical_significance_claimed": False, + "note": ( + "A one-pair run is preliminary. Full-sprint model routing is " + "nondeterministic, so end-to-end deltas are not solely attributable " + "to prompt compaction." + ), + }, + } + + +def _display(value: Any, *, missing: str = "N/A") -> str: + if value is None: + return missing + if isinstance(value, float): + return f"{value:.4f}" + return str(value) + + +def render_markdown(report: Mapping[str, Any]) -> str: + lines = [ + "# Sprint Performance Benchmark", + "", + f"- Benchmark: `{report.get('benchmark_id', '')}`", + f"- Status: **{report.get('status', 'inconclusive')}**", + f"- Classification: `{report.get('classification', '')}`", + f"- Scenario: `{(report.get('provenance') or {}).get('scenario_id', '')}`", + f"- Started: `{report.get('started_at', '')}`", + f"- Ended: `{report.get('ended_at', '')}`", + "", + "## Runs", + "", + "| Run | Variant | Status | Reserved | Telemetry | Completed | Failed | Timed out | Launch failed | Terminated | Active | Rejected | Repairs | Input tokens | Total tokens | V2 verified | Wall ms | Quality |", + "| --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + ] + for run in report.get("runs") or []: + metrics = dict(run.get("metrics") or {}) + totals = dict(metrics.get("totals") or {}) + tokens = dict(metrics.get("tokens") or {}) + compaction = dict(metrics.get("compaction") or {}) + attempts = dict(run.get("invocation_attempts") or {}) + quality = dict(run.get("quality") or {}) + lines.append( + "| {run_id} | {variant} | {status} | {reserved} | {telemetry} | " + "{completed} | {failed} | {timed_out} | {launch_failed} | {terminated} | " + "{active} | {rejected} | {repairs} | {input_tokens} | {total_tokens} | " + "{v2_targets} | {wall_ms} | {quality} |".format( + run_id=run.get("run_id", ""), + variant=run.get("variant", ""), + status=run.get("status", ""), + reserved=_display(attempts.get("reserved_count")), + telemetry=totals.get("invocation_count", 0), + completed=_display(attempts.get("completed_count")), + failed=_display(attempts.get("failed_count")), + timed_out=_display(attempts.get("timeout_count")), + launch_failed=_display(attempts.get("launch_failed_count")), + terminated=_display(attempts.get("terminated_count")), + active=_display(attempts.get("active_count")), + rejected=_display(attempts.get("rejected_count")), + repairs=totals.get("contract_repair_count", 0), + input_tokens=tokens.get("input", 0), + total_tokens=tokens.get("total", 0), + v2_targets=int(compaction.get("target_projection_count") or 0), + wall_ms=run.get("wall_duration_ms", 0), + quality="pass" if quality.get("passed") else "fail", + ) + ) + for pair in report.get("pairs") or []: + lines.extend( + [ + "", + f"## Pair {int(pair.get('pair_index') or 0):03d}", + "", + f"- Comparable: `{str(bool(pair.get('comparable'))).lower()}`", + ] + ) + reasons = pair.get("inconclusive_reasons") or [] + if reasons: + lines.append(f"- Inconclusive reasons: `{', '.join(str(item) for item in reasons)}`") + comparison = dict(pair.get("comparison") or {}) + end_to_end = dict(comparison.get("end_to_end") or {}) + if end_to_end: + lines.extend( + [ + "", + "| Metric | Before | After | Delta | Reduction | Reduction % |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for metric_name, values in end_to_end.items(): + values = dict(values or {}) + missing = "unpriced" if metric_name == "estimated_cost_usd" else "N/A" + lines.append( + f"| {metric_name} | {_display(values.get('before'), missing=missing)} | " + f"{_display(values.get('after'), missing=missing)} | " + f"{_display(values.get('delta'), missing=missing)} | " + f"{_display(values.get('reduction'), missing=missing)} | " + f"{_display(values.get('reduction_percent'), missing=missing)} |" + ) + lines.extend( + [ + "", + "## Interpretation", + "", + str((report.get("interpretation") or {}).get("note") or ""), + "", + ] + ) + return "\n".join(lines) + + +__all__ = [ + "REPORT_SCHEMA_VERSION", + "build_report", + "render_markdown", + "utc_now_iso", + "write_json_atomic", + "write_jsonl_atomic", + "write_run_artifacts", + "write_text_atomic", +] diff --git a/benchmarking/runner.py b/benchmarking/runner.py new file mode 100644 index 0000000..ceaacd4 --- /dev/null +++ b/benchmarking/runner.py @@ -0,0 +1,734 @@ +from __future__ import annotations + +import hashlib +import os +import platform +import re +import shutil +import subprocess +import tempfile +import time +import uuid +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Mapping + +from teams_runtime.benchmarking.metrics import ( + reduce_telemetry, + sanitize_invocation_record, +) +from teams_runtime.benchmarking.models import ( + ArmResult, + BenchmarkOptions, + BenchmarkResult, + BenchmarkWorker, + BenchmarkWorkerSafetyError, + QualityEvidence, + WorkerContext, + WorkerOutcome, + invocation_identity_digest, + make_arm_schedule, + sanitize_invocation_attempts, +) +from teams_runtime.benchmarking.reporting import ( + build_report, + render_markdown, + utc_now_iso, + write_json_atomic, + write_run_artifacts, + write_text_atomic, +) +from teams_runtime.benchmarking.scenario import ( + SCENARIO_MILESTONE, + ScenarioWorkspace, + create_scenario_workspace, + inspect_scenario_workspace, + load_runtime_settings, +) + + +_BENCHMARK_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$") +_RETAINED_BASELINE_FILES = ( + (".benchmark/scenario.json", ".benchmark/scenario.json"), + (".benchmark/history_seed.json", ".benchmark/history_seed.json"), + ("benchmark_app.py", "benchmark_app.baseline.py"), + ("tests/__init__.py", "tests/__init__.py"), + ("tests/test_benchmark_app.py", "tests/test_benchmark_app.py"), + ("BENCHMARK_TASK.md", "BENCHMARK_TASK.md"), + ("team_runtime.yaml", "team_runtime.yaml"), +) +_RETENTION_NOTICE = """# Sanitized Benchmark Snapshot + +This directory is an allowlisted diagnostic snapshot, not the execution workspace. +Baseline files are captured before any model call. The mutable implementation is +represented only by a content hash and byte count. Runtime state, model sessions, +provider output, logs, Git metadata, and unrecognized files are intentionally excluded. +""" + + +class BenchmarkPreflightError(RuntimeError): + pass + + +def _git( + root: Path, + *args: str, +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ("git", *args), + cwd=root, + text=True, + capture_output=True, + check=False, + ) + + +def _dirty_hash(porcelain: str) -> str: + return hashlib.sha256(porcelain.encode("utf-8")).hexdigest() if porcelain else "" + + +def _source_revision(source_root: Path, *, allow_dirty: bool) -> dict[str, Any]: + root = source_root.expanduser().resolve() + head = _git(root, "rev-parse", "HEAD") + if head.returncode: + raise BenchmarkPreflightError(f"Source root is not a Git repository: {root}") + status = _git(root, "status", "--porcelain=v1", "--untracked-files=all") + if status.returncode: + raise BenchmarkPreflightError("Unable to inspect source Git status") + dirty = bool(status.stdout.strip()) + if dirty and not allow_dirty: + raise BenchmarkPreflightError( + "Source worktree is dirty; commit changes or use allow_dirty_source explicitly" + ) + describe = _git(root, "describe", "--always", "--dirty", "--tags") + return { + "commit_sha": head.stdout.strip(), + "describe": describe.stdout.strip() if describe.returncode == 0 else head.stdout.strip()[:12], + "dirty": dirty, + "dirty_state_hash": _dirty_hash(status.stdout), + "python": platform.python_version(), + "platform": platform.platform(), + } + + +def _benchmark_id(options: BenchmarkOptions) -> str: + if options.benchmark_id: + if not _BENCHMARK_ID_PATTERN.fullmatch(options.benchmark_id): + raise ValueError( + "benchmark_id must be 1-96 ASCII letters, numbers, dots, underscores, or hyphens" + ) + return options.benchmark_id + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + return f"sprint-ab-{timestamp}-{uuid.uuid4().hex[:8]}" + + +def _output_root(options: BenchmarkOptions, benchmark_id: str) -> Path: + base = ( + options.output_dir.expanduser().resolve() + if options.output_dir is not None + else options.source_root.expanduser().resolve() / ".teams_runtime" / "benchmarks" + ) + root = base / benchmark_id + root.mkdir(parents=True, mode=0o700, exist_ok=False) + root.chmod(0o700) + return root + + +def _merge_quality( + outcome: WorkerOutcome, + scenario: ScenarioWorkspace, +) -> QualityEvidence: + inspection = inspect_scenario_workspace(scenario) + worker = outcome.quality + sprint = outcome.sprint + notes = tuple(dict.fromkeys((*worker.notes, *inspection.notes))) + return QualityEvidence( + behavior_oracle_passed=inspection.behavior_oracle_passed, + sprint_terminal=worker.sprint_terminal, + closeout_verified=worker.closeout_verified, + protected_files_unchanged=inspection.protected_files_unchanged, + git_clean=inspection.git_clean, + commit_created=inspection.commit_created, + no_git_remotes=inspection.no_git_remotes, + blocked_todo_count=max(worker.blocked_todo_count, sprint.blocked_todo_count), + failed_todo_count=max(worker.failed_todo_count, sprint.failed_todo_count), + notes=notes, + ) + + +_REQUIRED_INVOCATION_ATTEMPT_FIELDS = frozenset( + { + "schema_version", + "journal_schema_version", + "max_invocations", + "reserved_count", + "entry_count", + "telemetry_record_count", + "unobserved_attempt_count", + "telemetry_overage_count", + "completed_count", + "failed_count", + "timeout_count", + "launch_failed_count", + "terminated_count", + "active_count", + "unknown_state_count", + "malformed_entry_count", + "unaccounted_count", + "overaccounted_count", + "rejected_count", + "remaining_budget", + "journal_invocation_ids_sha256", + "journal_invocation_id_missing_count", + "journal_invocation_id_duplicate_count", + "telemetry_invocation_id_missing_count", + "telemetry_invocation_id_duplicate_count", + "telemetry_invocation_id_unmatched_count", + "journal_invocation_id_unobserved_count", + } +) +_INVOCATION_ATTEMPT_STATE_COUNTS = ( + "completed_count", + "failed_count", + "timeout_count", + "launch_failed_count", + "terminated_count", + "active_count", + "unknown_state_count", + "malformed_entry_count", +) + + +def _journal_coverage_available( + invocation_attempts: Mapping[str, Any], + *, + expected_max_invocations: int, +) -> bool: + if ( + invocation_attempts.get("journal_available") is not True + or invocation_attempts.get("reconciled") is not True + or invocation_attempts.get("identity_reconciled") is not True + or not _REQUIRED_INVOCATION_ATTEMPT_FIELDS.issubset( + invocation_attempts + ) + ): + return False + summary_schema = int(invocation_attempts["schema_version"]) + journal_schema = int(invocation_attempts["journal_schema_version"]) + maximum = int(invocation_attempts["max_invocations"]) + reserved = int(invocation_attempts["reserved_count"]) + entries = int(invocation_attempts["entry_count"]) + observed = int(invocation_attempts["telemetry_record_count"]) + accounted = sum( + int(invocation_attempts[field_name]) + for field_name in _INVOCATION_ATTEMPT_STATE_COUNTS + ) + context_reconciled = ( + journal_schema < 3 + or ( + { + "journal_telemetry_context_mismatch_count", + "verified_target_projection_count", + "verified_target_invocation_ids_sha256", + }.issubset(invocation_attempts) + and invocation_attempts.get("context_reconciled") is True + and int( + invocation_attempts.get( + "journal_telemetry_context_mismatch_count" + ) + or 0 + ) + == 0 + ) + ) + return ( + summary_schema == 1 + and journal_schema in {1, 2, 3} + and context_reconciled + and maximum == expected_max_invocations + and reserved <= maximum + and int(invocation_attempts["remaining_budget"]) + == maximum - reserved + and reserved == entries == accounted + and int(invocation_attempts["unknown_state_count"]) == 0 + and int(invocation_attempts["malformed_entry_count"]) == 0 + and int(invocation_attempts["unaccounted_count"]) == 0 + and int(invocation_attempts["overaccounted_count"]) == 0 + and int( + invocation_attempts["journal_invocation_id_missing_count"] + ) + == 0 + and int( + invocation_attempts["journal_invocation_id_duplicate_count"] + ) + == 0 + and int( + invocation_attempts["telemetry_invocation_id_missing_count"] + ) + == 0 + and int( + invocation_attempts["telemetry_invocation_id_duplicate_count"] + ) + == 0 + and int( + invocation_attempts["telemetry_invocation_id_unmatched_count"] + ) + == 0 + and int( + invocation_attempts["journal_invocation_id_unobserved_count"] + ) + == max(reserved - observed, 0) + and int(invocation_attempts["unobserved_attempt_count"]) + == max(reserved - observed, 0) + and int(invocation_attempts["telemetry_overage_count"]) == 0 + ) + + +def _safe_worker_failure( + started_at: str, + started_monotonic: float, + exc: BaseException, +) -> WorkerOutcome: + return WorkerOutcome( + status="failed", + started_at=started_at, + ended_at=utc_now_iso(), + wall_duration_ms=max(int((time.monotonic() - started_monotonic) * 1000), 0), + worker_duration_ms=max(int((time.monotonic() - started_monotonic) * 1000), 0), + stop_reason="worker_exception", + error_category=type(exc).__name__, + ) + + +def _is_safe_regular_file(source_root: Path, relative_name: str) -> bool: + candidate = source_root + for component in Path(relative_name).parts: + candidate = candidate / component + if candidate.is_symlink(): + return False + if not candidate.is_file(): + return False + try: + candidate.resolve().relative_to(source_root.resolve()) + except (OSError, RuntimeError, ValueError): + return False + return True + + +def _capture_retention_baseline(source_root: Path) -> dict[str, bytes]: + baseline: dict[str, bytes] = {} + for source_name, retained_name in _RETAINED_BASELINE_FILES: + if not _is_safe_regular_file(source_root, source_name): + raise BenchmarkPreflightError( + f"Benchmark baseline file is missing or unsafe: {source_name}" + ) + baseline[retained_name] = (source_root / source_name).read_bytes() + return baseline + + +def _implementation_result_summary( + source_root: Path, + baseline: Mapping[str, bytes], +) -> dict[str, Any]: + source_name = "benchmark_app.py" + baseline_content = baseline["benchmark_app.baseline.py"] + baseline_hash = hashlib.sha256(baseline_content).hexdigest() + if not _is_safe_regular_file(source_root, source_name): + return { + "schema_version": 1, + "path": source_name, + "status": "missing_or_unsafe", + "baseline_sha256": baseline_hash, + "sha256": None, + "size_bytes": None, + "changed_from_baseline": None, + } + source = source_root / source_name + digest = hashlib.sha256() + size_bytes = 0 + with source.open("rb") as handle: + for chunk in iter(lambda: handle.read(64 * 1024), b""): + digest.update(chunk) + size_bytes += len(chunk) + result_hash = digest.hexdigest() + return { + "schema_version": 1, + "path": source_name, + "status": "hashed", + "baseline_sha256": baseline_hash, + "sha256": result_hash, + "size_bytes": size_bytes, + "changed_from_baseline": result_hash != baseline_hash, + } + + +def _retain_workspace_snapshot( + source_root: Path, + retained_root: Path, + *, + baseline: Mapping[str, bytes], +) -> None: + retained_root.mkdir(parents=True, mode=0o700, exist_ok=False) + retained_root.chmod(0o700) + for relative_name, content in baseline.items(): + destination = retained_root / relative_name + destination.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + destination.write_bytes(content) + destination.chmod(0o600) + write_json_atomic( + retained_root / "benchmark_app.result.json", + _implementation_result_summary(source_root, baseline), + ) + notice_path = retained_root / "RETENTION_NOTICE.md" + notice_path.write_text( + _RETENTION_NOTICE, + encoding="utf-8", + ) + notice_path.chmod(0o600) + + +def _run_arm( + *, + benchmark_id: str, + output_root: Path, + temporary_root: Path, + options: BenchmarkOptions, + worker: BenchmarkWorker, + settings: Any, + arm: Any, +) -> ArmResult: + workspace_parent = temporary_root / arm.run_id + scenario = create_scenario_workspace( + workspace_parent, + benchmark_id=benchmark_id, + run_id=arm.run_id, + prompt_context_enabled=arm.prompt_context_enabled, + settings=settings, + ) + retention_baseline = _capture_retention_baseline(scenario.root) + run_dir = output_root / "runs" / arm.run_id + started_at = utc_now_iso() + started_monotonic = time.monotonic() + context = WorkerContext( + benchmark_id=benchmark_id, + arm=arm, + workspace_root=scenario.root, + run_output_dir=run_dir, + milestone=SCENARIO_MILESTONE, + history_seed=scenario.history_seed, + max_invocations=options.max_invocations, + call_timeout_seconds=options.call_timeout_seconds, + run_timeout_seconds=options.run_timeout_seconds, + live=options.live, + ) + try: + outcome = worker(context) + if not isinstance(outcome, WorkerOutcome): + raise TypeError("Benchmark worker must return WorkerOutcome") + except BenchmarkWorkerSafetyError: + raise + except Exception as exc: + outcome = _safe_worker_failure(started_at, started_monotonic, exc) + measured_wall_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + if measured_wall_ms > int(options.run_timeout_seconds * 1000) and outcome.status == "completed": + outcome = replace( + outcome, + status="timeout", + stop_reason="run_timeout_exceeded", + ) + raw_records = tuple( + sanitize_invocation_record(record) + for record in outcome.telemetry_records + ) + invocation_attempts = sanitize_invocation_attempts( + outcome.invocation_attempts + ) + if invocation_attempts.get("journal_available") is True: + reserved_count = int(invocation_attempts.get("reserved_count") or 0) + telemetry_record_count = len(raw_records) + telemetry_invocation_ids = [ + str(record.get("invocation_id") or "").strip() + for record in raw_records + ] + telemetry_nonempty_ids = [ + invocation_id + for invocation_id in telemetry_invocation_ids + if invocation_id + ] + telemetry_missing_id_count = ( + len(telemetry_invocation_ids) - len(telemetry_nonempty_ids) + ) + telemetry_duplicate_id_count = ( + len(telemetry_nonempty_ids) + - len(set(telemetry_nonempty_ids)) + ) + telemetry_identity_digest = invocation_identity_digest( + telemetry_invocation_ids + ) + invocation_attempts.update( + { + "telemetry_record_count": telemetry_record_count, + "unobserved_attempt_count": max( + reserved_count - telemetry_record_count, + 0, + ), + "telemetry_overage_count": max( + telemetry_record_count - reserved_count, + 0, + ), + "telemetry_coverage_percent": ( + round( + min( + telemetry_record_count * 100 / reserved_count, + 100.0, + ), + 2, + ) + if reserved_count + else 0.0 + ), + "telemetry_invocation_id_missing_count": ( + telemetry_missing_id_count + ), + "telemetry_invocation_id_duplicate_count": ( + telemetry_duplicate_id_count + ), + "identity_reconciled": bool( + invocation_attempts.get("identity_reconciled") + and telemetry_missing_id_count == 0 + and telemetry_duplicate_id_count == 0 + and int( + invocation_attempts.get( + "telemetry_invocation_id_unmatched_count" + ) + or 0 + ) + == 0 + and int( + invocation_attempts.get( + "journal_invocation_id_unobserved_count" + ) + or 0 + ) + == max( + reserved_count - telemetry_record_count, + 0, + ) + and ( + telemetry_record_count < reserved_count + or invocation_attempts.get( + "journal_invocation_ids_sha256" + ) + == telemetry_identity_digest + ) + ), + } + ) + accounted_count = sum( + int(invocation_attempts.get(field_name) or 0) + for field_name in _INVOCATION_ATTEMPT_STATE_COUNTS + ) + invocation_attempts["reconciled"] = bool( + invocation_attempts.get("reconciled") + and int(invocation_attempts.get("max_invocations") or 0) + == options.max_invocations + and reserved_count + == int(invocation_attempts.get("entry_count") or 0) + == accounted_count + and int(invocation_attempts.get("unaccounted_count") or 0) == 0 + and int(invocation_attempts.get("overaccounted_count") or 0) == 0 + ) + coverage_available = _journal_coverage_available( + invocation_attempts, + expected_max_invocations=options.max_invocations, + ) + if not coverage_available: + invocation_attempts.pop("telemetry_coverage_percent", None) + expected_invocation_count = ( + int(invocation_attempts.get("reserved_count") or 0) + if coverage_available + else None + ) + metrics = reduce_telemetry( + raw_records, + expected_invocation_count=expected_invocation_count, + coverage_available=coverage_available, + verified_target_projection_count=( + int( + invocation_attempts.get("verified_target_projection_count") + or 0 + ) + if int(invocation_attempts.get("journal_schema_version") or 0) == 3 + and invocation_attempts.get("context_reconciled") is True + and int( + invocation_attempts.get( + "journal_telemetry_context_mismatch_count" + ) + or 0 + ) + == 0 + else 0 + ), + verified_target_invocation_ids_sha256=( + str( + invocation_attempts.get( + "verified_target_invocation_ids_sha256" + ) + or "" + ) + if int(invocation_attempts.get("journal_schema_version") or 0) == 3 + and invocation_attempts.get("context_reconciled") is True + and int( + invocation_attempts.get( + "journal_telemetry_context_mismatch_count" + ) + or 0 + ) + == 0 + else "" + ), + ) + quality = _merge_quality(outcome, scenario) + result = ArmResult( + arm=arm, + status=outcome.status, + started_at=outcome.started_at or started_at, + ended_at=outcome.ended_at or utc_now_iso(), + wall_duration_ms=outcome.wall_duration_ms or measured_wall_ms, + worker_duration_ms=outcome.worker_duration_ms or measured_wall_ms, + stop_reason=outcome.stop_reason, + error_category=outcome.error_category, + config_hash=scenario.config_hash, + comparable_config_hash=scenario.comparable_config_hash, + metrics=metrics, + quality=quality, + sprint=outcome.sprint, + invocation_attempts=invocation_attempts, + invocation_records=raw_records, + ) + keep = options.keep_workspaces == "all" or ( + options.keep_workspaces == "failures" + and (result.status != "completed" or not result.quality.passed) + ) + if keep: + retained_root = output_root / "workspaces" / arm.run_id + retained_root.parent.mkdir(parents=True, exist_ok=True) + _retain_workspace_snapshot( + scenario.root, + retained_root, + baseline=retention_baseline, + ) + shutil.rmtree(scenario.root, ignore_errors=True) + result = replace( + result, + retained_workspace=f"workspaces/{arm.run_id}", + ) + else: + shutil.rmtree(scenario.root, ignore_errors=True) + write_run_artifacts(run_dir, result) + return result + + +def run_sprint_ab_benchmark( + options: BenchmarkOptions, + *, + worker: BenchmarkWorker, +) -> BenchmarkResult: + """Run isolated sprint arms and write a privacy-safe A/B report. + + The injected worker owns TeamService orchestration and provider-process + enforcement. This core owns fixtures, arm isolation, telemetry reduction, + quality checks, comparison semantics, retention, and report persistence. + """ + + options.validate() + source_revision = _source_revision( + options.source_root, + allow_dirty=options.allow_dirty_source, + ) + settings = load_runtime_settings( + options.runtime_config_path, + rate_card_path=options.rate_card_path, + ) + benchmark_id = _benchmark_id(options) + output_root = _output_root(options, benchmark_id) + started_at = utc_now_iso() + schedule = make_arm_schedule(options.repetitions) + runs: list[ArmResult] = [] + history_hash = "" + with tempfile.TemporaryDirectory(prefix=f"{benchmark_id}-") as temp_directory: + temporary_root = Path(temp_directory) + temporary_root.chmod(0o700) + for arm in schedule: + result = _run_arm( + benchmark_id=benchmark_id, + output_root=output_root, + temporary_root=temporary_root, + options=options, + worker=worker, + settings=settings, + arm=arm, + ) + runs.append(result) + if result.status == "preflight_failed": + # The paired arm uses the same source, model, and safety controls. + # Repeating a failed safety/configuration preflight cannot produce + # a valid comparison and may obscure the original failure. + break + if not history_hash: + scenario_file = ( + output_root / result.retained_workspace / ".benchmark" / "scenario.json" + if result.retained_workspace + else None + ) + if scenario_file is not None and scenario_file.is_file(): + import json + + history_hash = str( + (json.loads(scenario_file.read_text(encoding="utf-8")) or {}).get( + "history_hash" + ) + or "" + ) + if not history_hash: + from teams_runtime.benchmarking.scenario import build_history_seed, canonical_hash + + history_hash = canonical_hash(build_history_seed()) + + ended_at = utc_now_iso() + report = build_report( + benchmark_id=benchmark_id, + options=options, + source_revision=source_revision, + source_config_hash=settings.source_config_hash, + runtime_model_map={ + **settings.role_defaults, + **settings.internal_agent_defaults, + }, + rate_cards=settings.rate_cards, + history_hash=history_hash, + runs=tuple(runs), + started_at=started_at, + ended_at=ended_at, + ) + report_json = output_root / "report.json" + report_markdown = output_root / "report.md" + write_json_atomic(report_json, report) + write_text_atomic(report_markdown, render_markdown(report)) + return BenchmarkResult( + benchmark_id=benchmark_id, + status=str(report["status"]), # type: ignore[arg-type] + classification=str(report["classification"]), # type: ignore[arg-type] + output_dir=output_root, + report_json=report_json, + report_markdown=report_markdown, + runs=tuple(runs), + report=report, + ) + + +__all__ = [ + "BenchmarkPreflightError", + "run_sprint_ab_benchmark", +] diff --git a/benchmarking/scenario.py b/benchmarking/scenario.py new file mode 100644 index 0000000..93ae795 --- /dev/null +++ b/benchmarking/scenario.py @@ -0,0 +1,906 @@ +from __future__ import annotations + +import ast +import hashlib +import json +import os +import shutil +import stat +import subprocess +import sys +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Mapping + +import yaml + +from teams_runtime.core.template import scaffold_workspace +from teams_runtime.shared.models import INTERNAL_TEAM_AGENTS, TEAM_ROLES + + +SCENARIO_ID = "sum-positive-full-sprint-v2" +DEFAULT_HISTORY_SEED_COUNT = 48 +BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS = 8 +BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS = 16 +BENCHMARK_TARGET_TOTAL_EVENTS = DEFAULT_HISTORY_SEED_COUNT + 2 +BENCHMARK_TARGET_INCLUDED_EVENTS = BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS +BENCHMARK_TARGET_OMITTED_EVENTS = ( + BENCHMARK_TARGET_TOTAL_EVENTS - BENCHMARK_TARGET_INCLUDED_EVENTS +) +BENCHMARK_TARGET_ROLE = "research" +BENCHMARK_TARGET_PURPOSE = "research_decision" +BENCHMARK_TARGET_WORKFLOW_STEP = "research_initial" +SCENARIO_MILESTONE = ( + "Fix sum_positive(values) so it returns the sum of positive values only. " + "Use exactly `return sum(value for value in values if value > 0)` as its only " + "non-docstring statement; do not add imports, decorators, annotations, defaults, " + "or other definitions. Preserve the public function, do not alter benchmark tests " + "or scenario metadata, run the unittest suite, and commit the completed change." +) +PROTECTED_PATHS = ( + ".benchmark/scenario.json", + ".benchmark/history_seed.json", + "tests/__init__.py", + "tests/test_benchmark_app.py", +) +_RATE_FIELDS = ( + "input_per_million_usd", + "cached_input_per_million_usd", + "output_per_million_usd", + "per_invocation_usd", +) +_MAX_ORACLE_SOURCE_BYTES = 32 * 1024 +_MAX_PROTECTED_FILE_BYTES = 1024 * 1024 +_GIT_COMMAND_TIMEOUT_SECONDS = 10.0 +_EMPTY_FILE_SHA256 = hashlib.sha256(b"").hexdigest() +_GIT_CONFIG_OVERRIDES = ( + "-c", + f"core.hooksPath={os.devnull}", + "-c", + "core.fsmonitor=false", + "-c", + f"core.attributesFile={os.devnull}", + "-c", + "diff.external=", + "-c", + "core.pager=", + "-c", + "pager.status=false", + "-c", + "pager.remote=false", + "-c", + "interactive.diffFilter=", + "-c", + "maintenance.auto=false", + "-c", + "gc.auto=0", +) + + +class ScenarioError(RuntimeError): + pass + + +@dataclass(slots=True, frozen=True) +class RuntimeSettings: + role_defaults: Mapping[str, Mapping[str, str]] + internal_agent_defaults: Mapping[str, Mapping[str, str]] + rate_cards: Mapping[str, Mapping[str, float | None]] + source_config_hash: str + + +@dataclass(slots=True, frozen=True) +class ScenarioWorkspace: + root: Path + initial_commit: str + initial_commit_count: int + protected_hashes: Mapping[str, str] + config_hash: str + comparable_config_hash: str + history_hash: str + history_seed: tuple[Mapping[str, Any], ...] + git_executable: Path | None = None + + +@dataclass(slots=True, frozen=True) +class WorkspaceInspection: + behavior_oracle_passed: bool + protected_files_unchanged: bool + git_clean: bool + commit_created: bool + no_git_remotes: bool + head_sha: str + notes: tuple[str, ...] + + +def _canonical_json(value: Any) -> str: + return json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")) + + +def canonical_hash(value: Any) -> str: + return hashlib.sha256(_canonical_json(value).encode("utf-8")).hexdigest() + + +def _read_regular_file_at( + root: Path, + relative_path: str, + *, + max_bytes: int, +) -> bytes: + relative = Path(relative_path) + if ( + relative.is_absolute() + or not relative.parts + or any(part in {"", ".", ".."} for part in relative.parts) + ): + raise ValueError(f"Unsafe relative path: {relative_path!r}") + + directory_flags = os.O_RDONLY | os.O_DIRECTORY + file_flags = os.O_RDONLY | os.O_NONBLOCK + if hasattr(os, "O_CLOEXEC"): + directory_flags |= os.O_CLOEXEC + file_flags |= os.O_CLOEXEC + if hasattr(os, "O_NOFOLLOW"): + directory_flags |= os.O_NOFOLLOW + file_flags |= os.O_NOFOLLOW + + directory_fd = os.open(root, directory_flags) + try: + for component in relative.parts[:-1]: + next_fd = os.open(component, directory_flags, dir_fd=directory_fd) + os.close(directory_fd) + directory_fd = next_fd + file_fd = os.open(relative.parts[-1], file_flags, dir_fd=directory_fd) + try: + metadata = os.fstat(file_fd) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > max_bytes: + raise ValueError(f"Unsafe file type or size: {relative_path}") + chunks: list[bytes] = [] + remaining = max_bytes + 1 + while remaining: + chunk = os.read(file_fd, min(64 * 1024, remaining)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + payload = b"".join(chunks) + if len(payload) > max_bytes: + raise ValueError(f"File exceeds size limit: {relative_path}") + return payload + finally: + os.close(file_fd) + finally: + os.close(directory_fd) + + +def _protected_file_hash(root: Path, relative_path: str) -> str: + payload = _read_regular_file_at( + root, + relative_path, + max_bytes=_MAX_PROTECTED_FILE_BYTES, + ) + return hashlib.sha256(payload).hexdigest() + + +def _is_string_literal(statement: ast.stmt) -> bool: + return ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and isinstance(statement.value.value, str) + ) + + +def _without_optional_docstring(statements: list[ast.stmt]) -> list[ast.stmt]: + if statements and _is_string_literal(statements[0]): + return statements[1:] + return statements + + +def _is_name(node: ast.AST, identifier: str, context: type[ast.expr_context]) -> bool: + return ( + isinstance(node, ast.Name) + and node.id == identifier + and isinstance(node.ctx, context) + ) + + +def _is_constrained_sum_positive(module: ast.Module) -> bool: + module_body = _without_optional_docstring(module.body) + if len(module_body) != 1 or not isinstance(module_body[0], ast.FunctionDef): + return False + function = module_body[0] + arguments = function.args + if ( + function.name != "sum_positive" + or function.decorator_list + or function.returns is not None + or getattr(function, "type_params", ()) + or arguments.posonlyargs + or len(arguments.args) != 1 + or arguments.args[0].arg != "values" + or arguments.args[0].annotation is not None + or arguments.vararg is not None + or arguments.kwonlyargs + or arguments.kw_defaults + or arguments.kwarg is not None + or arguments.defaults + ): + return False + function_body = _without_optional_docstring(function.body) + if len(function_body) != 1 or not isinstance(function_body[0], ast.Return): + return False + call = function_body[0].value + if ( + not isinstance(call, ast.Call) + or not _is_name(call.func, "sum", ast.Load) + or len(call.args) != 1 + or call.keywords + or not isinstance(call.args[0], ast.GeneratorExp) + ): + return False + generator = call.args[0] + if ( + not _is_name(generator.elt, "value", ast.Load) + or len(generator.generators) != 1 + ): + return False + comprehension = generator.generators[0] + if ( + comprehension.is_async + or not _is_name(comprehension.target, "value", ast.Store) + or not _is_name(comprehension.iter, "values", ast.Load) + or len(comprehension.ifs) != 1 + ): + return False + predicate = comprehension.ifs[0] + return ( + isinstance(predicate, ast.Compare) + and _is_name(predicate.left, "value", ast.Load) + and len(predicate.ops) == 1 + and isinstance(predicate.ops[0], ast.Gt) + and len(predicate.comparators) == 1 + and isinstance(predicate.comparators[0], ast.Constant) + and type(predicate.comparators[0].value) is int + and predicate.comparators[0].value == 0 + ) + + +def _sum_positive_ast_oracle(root: Path) -> bool: + try: + payload = _read_regular_file_at( + root, + "benchmark_app.py", + max_bytes=_MAX_ORACLE_SOURCE_BYTES, + ) + source = payload.decode("utf-8") + module = ast.parse(source, filename="benchmark_app.py", mode="exec") + except (MemoryError, OSError, RecursionError, SyntaxError, UnicodeError, ValueError): + return False + return _is_constrained_sum_positive(module) + + +def _read_yaml(path: Path) -> dict[str, Any]: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"Expected a YAML mapping: {path}") + return payload + + +def _runtime_config_file(path: Path) -> Path: + resolved = path.expanduser().resolve() + return resolved / "team_runtime.yaml" if resolved.is_dir() else resolved + + +def _normalize_role_defaults(payload: Mapping[str, Any]) -> dict[str, dict[str, str]]: + raw_defaults = payload.get("role_defaults") + if not isinstance(raw_defaults, dict): + raise ValueError("Runtime config must define role_defaults") + normalized: dict[str, dict[str, str]] = {} + for role in TEAM_ROLES: + raw = raw_defaults.get(role) + if not isinstance(raw, dict): + raise ValueError(f"Runtime config must define role_defaults.{role}") + model = str(raw.get("model") or "").strip() + reasoning = str(raw.get("reasoning") or "").strip() + if not model or not reasoning: + raise ValueError(f"role_defaults.{role} must define model and reasoning") + normalized[role] = {"model": model, "reasoning": reasoning} + return normalized + + +def _normalize_internal_agent_defaults( + payload: Mapping[str, Any], + *, + inherited_runtime: Mapping[str, str], +) -> dict[str, dict[str, str]]: + raw_defaults = payload.get("internal_agent_defaults") + if raw_defaults is None: + return { + agent: dict(inherited_runtime) + for agent in INTERNAL_TEAM_AGENTS + } + if not isinstance(raw_defaults, dict): + raise ValueError("internal_agent_defaults must be a mapping") + + normalized: dict[str, dict[str, str]] = {} + for agent in INTERNAL_TEAM_AGENTS: + raw = raw_defaults.get(agent) + if raw is None: + normalized[agent] = dict(inherited_runtime) + continue + if not isinstance(raw, dict): + raise ValueError( + f"internal_agent_defaults.{agent} must be a mapping" + ) + model = ( + str(raw.get("model") or "").strip() + or str(inherited_runtime["model"]) + ) + reasoning = ( + str(raw.get("reasoning") or "").strip() + or str(inherited_runtime["reasoning"]) + ) + normalized[agent] = {"model": model, "reasoning": reasoning} + return normalized + + +def _normalize_rate(value: Any, *, field_name: str) -> float | None: + if value is None: + return None + if isinstance(value, bool): + raise ValueError(f"{field_name} must be a finite non-negative number") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a finite non-negative number") from exc + if normalized < 0 or normalized in {float("inf"), float("-inf")} or normalized != normalized: + raise ValueError(f"{field_name} must be a finite non-negative number") + return normalized + + +def _normalize_rate_cards(payload: Mapping[str, Any]) -> dict[str, dict[str, float | None]]: + raw_cards: Any = payload.get("rate_cards") + if raw_cards is None and isinstance(payload.get("telemetry"), dict): + raw_cards = payload["telemetry"].get("rate_cards") + if raw_cards in (None, {}): + return {} + if not isinstance(raw_cards, dict): + raise ValueError("rate_cards must be a mapping") + cards: dict[str, dict[str, float | None]] = {} + for raw_key, raw_card in raw_cards.items(): + key = str(raw_key or "").strip() + if "/" not in key or not isinstance(raw_card, dict): + raise ValueError(f"Invalid rate card entry: {raw_key!r}") + normalized = { + field_name: _normalize_rate( + raw_card.get(field_name), + field_name=f"rate_cards.{key}.{field_name}", + ) + for field_name in _RATE_FIELDS + } + if normalized["per_invocation_usd"] is None and ( + normalized["input_per_million_usd"] is None + or normalized["output_per_million_usd"] is None + ): + raise ValueError( + f"rate_cards.{key} requires per_invocation_usd or both input and output rates" + ) + cards[key] = normalized + return cards + + +def load_runtime_settings( + runtime_config_path: Path, + *, + rate_card_path: Path | None = None, +) -> RuntimeSettings: + config_file = _runtime_config_file(runtime_config_path) + payload = _read_yaml(config_file) + role_defaults = _normalize_role_defaults(payload) + internal_agent_defaults = _normalize_internal_agent_defaults( + payload, + inherited_runtime=role_defaults["orchestrator"], + ) + rate_cards: dict[str, dict[str, float | None]] = {} + if rate_card_path is not None: + rate_cards = _normalize_rate_cards(_read_yaml(rate_card_path.expanduser().resolve())) + source_snapshot = { + "role_defaults": role_defaults, + "internal_agent_defaults": internal_agent_defaults, + "rate_cards": rate_cards, + } + return RuntimeSettings( + role_defaults=role_defaults, + internal_agent_defaults=internal_agent_defaults, + rate_cards=rate_cards, + source_config_hash=canonical_hash(source_snapshot), + ) + + +def build_history_seed( + count: int = DEFAULT_HISTORY_SEED_COUNT, +) -> tuple[Mapping[str, Any], ...]: + if count < 24: + raise ValueError("History seed must contain at least 24 events") + roles = ( + "research", + "planner", + "designer", + "architect", + "developer", + "qa", + "version_controller", + "orchestrator", + ) + started = datetime(2026, 1, 1, tzinfo=timezone.utc) + events: list[Mapping[str, Any]] = [] + for index in range(count): + timestamp = (started + timedelta(minutes=index)).isoformat() + if index < len(roles) * 2 and index % 2 == 1: + role = roles[index // 2] + event: Mapping[str, Any] = { + "created_at": timestamp, + "type": "role_report", + "actor": role, + "summary": f"Historical {role} checkpoint {index + 1:02d}.", + "payload": { + "role": role, + "status": "completed", + "summary": f"Stable benchmark evidence {index + 1:02d}.", + }, + } + else: + event = { + "created_at": timestamp, + "type": "benchmark_checkpoint", + "actor": "orchestrator", + "summary": f"Neutral historical checkpoint {index + 1:02d}.", + "payload": {"sequence": index + 1}, + } + events.append(event) + return tuple(events) + + +def _benchmark_config( + workspace_root: Path, + *, + benchmark_id: str, + run_id: str, + prompt_context_enabled: bool, + settings: RuntimeSettings, +) -> tuple[str, str]: + path = workspace_root / "team_runtime.yaml" + payload = _read_yaml(path) + sprint = dict(payload.get("sprint") or {}) + sprint.update( + { + "id": f"{benchmark_id}-{run_id.rsplit('-', 1)[0]}", + "mode": "hybrid", + "start_mode": "manual_daily", + "ingress_mode": "backlog_first", + "discovery_scope": "workspace_only", + "discovery_actions": [], + } + ) + payload["sprint"] = sprint + payload["role_defaults"] = { + role: dict(settings.role_defaults[role]) + for role in TEAM_ROLES + } + payload["internal_agent_defaults"] = { + agent: dict(settings.internal_agent_defaults[agent]) + for agent in INTERNAL_TEAM_AGENTS + } + payload["research_defaults"] = { + "app": "", + "notebook": "", + "files": [], + "mode": "", + "profile_path": "", + "completion_timeout": 600, + "callback_timeout": 1200, + "cleanup": False, + "reasoning_level": "Standard", + } + payload["prompt_context"] = { + "enabled": prompt_context_enabled, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + } + payload["telemetry"] = { + "enabled": True, + "rate_cards": { + key: { + field_name: value + for field_name, value in card.items() + if value is not None + } + for key, card in settings.rate_cards.items() + }, + } + payload["actions"] = {} + path.write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=False), + encoding="utf-8", + ) + comparable_payload = json.loads(json.dumps(payload)) + comparable_payload["prompt_context"].pop("enabled", None) + return canonical_hash(payload), canonical_hash(comparable_payload) + + +def _resolve_git_executable(root: Path) -> Path: + candidate = shutil.which("git", path=os.defpath) + if not candidate: + raise ScenarioError("Git executable is unavailable") + try: + resolved = Path(candidate).expanduser().resolve(strict=True) + metadata = os.stat(resolved, follow_symlinks=False) + except OSError as exc: + raise ScenarioError("Git executable cannot be resolved safely") from exc + if ( + not resolved.is_absolute() + or not stat.S_ISREG(metadata.st_mode) + or not os.access(resolved, os.X_OK) + or resolved.is_relative_to(root) + ): + raise ScenarioError("Git executable is not a safe external executable") + return resolved + + +def _run_git( + root: Path, + *args: str, + git_executable: Path, + attributes_source: str | None = None, + check: bool = True, +) -> subprocess.CompletedProcess[str]: + try: + metadata = os.stat(git_executable, follow_symlinks=False) + except OSError as exc: + raise ScenarioError("Pinned Git executable is unavailable") from exc + if ( + not git_executable.is_absolute() + or not stat.S_ISREG(metadata.st_mode) + or not os.access(git_executable, os.X_OK) + or git_executable.is_relative_to(root) + ): + raise ScenarioError("Pinned Git executable is unsafe") + environment = { + "HOME": os.devnull, + "PATH": os.defpath, + "LC_ALL": "C", + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_ATTR_NOSYSTEM": "1", + "GIT_EXTERNAL_DIFF": "", + "GIT_PAGER": "", + "PAGER": "", + "GIT_TERMINAL_PROMPT": "0", + "GIT_PROTOCOL_FROM_USER": "0", + "GIT_OPTIONAL_LOCKS": "0", + } + if attributes_source is not None: + environment["GIT_ATTR_SOURCE"] = attributes_source + try: + completed = subprocess.run( + (str(git_executable), "--no-pager", *_GIT_CONFIG_OVERRIDES, *args), + cwd=root, + text=True, + capture_output=True, + check=False, + timeout=_GIT_COMMAND_TIMEOUT_SECONDS, + env=environment, + ) + except subprocess.TimeoutExpired as exc: + raise ScenarioError("Pinned Git command timed out") from exc + if check and completed.returncode: + raise ScenarioError(f"Git command failed: git {' '.join(args)}") + return completed + + +def _initialize_git(root: Path, *, git_executable: Path) -> tuple[str, int]: + _run_git(root, "init", "-b", "benchmark", git_executable=git_executable) + (root / ".git" / "info" / "attributes").write_bytes(b"") + _run_git( + root, + "config", + "--local", + "user.name", + "teams-runtime-benchmark", + git_executable=git_executable, + ) + _run_git( + root, + "config", + "--local", + "user.email", + "benchmark@invalid.local", + git_executable=git_executable, + ) + _run_git( + root, + "config", + "--local", + "commit.gpgsign", + "false", + git_executable=git_executable, + ) + _run_git( + root, + "config", + "--local", + "tag.gpgsign", + "false", + git_executable=git_executable, + ) + _run_git( + root, + "config", + "--local", + "core.hooksPath", + ".git/benchmark-disabled-hooks", + git_executable=git_executable, + ) + (root / ".git" / "benchmark-disabled-hooks").mkdir(mode=0o700, exist_ok=True) + _run_git(root, "add", "--all", git_executable=git_executable) + _run_git( + root, + "commit", + "-m", + "[benchmark] seed defective sum_positive scenario", + git_executable=git_executable, + ) + head = _run_git( + root, + "rev-parse", + "HEAD", + git_executable=git_executable, + ).stdout.strip() + count = int( + _run_git( + root, + "rev-list", + "--count", + "HEAD", + git_executable=git_executable, + ).stdout.strip() + ) + if _run_git(root, "remote", git_executable=git_executable).stdout.strip(): + raise ScenarioError("Benchmark repository unexpectedly has a Git remote") + return head, count + + +def _assert_defect_reproduces(root: Path) -> None: + result = subprocess.run( + (sys.executable, "-m", "unittest", "discover", "-s", "tests"), + cwd=root, + text=True, + capture_output=True, + check=False, + timeout=30, + env={"PATH": os.environ.get("PATH", ""), "PYTHONPATH": str(root), "LC_ALL": "C"}, + ) + if result.returncode == 0: + raise ScenarioError("Benchmark fixture must fail its baseline behavior oracle") + + +def create_scenario_workspace( + workspace_root: Path, + *, + benchmark_id: str, + run_id: str, + prompt_context_enabled: bool, + settings: RuntimeSettings, +) -> ScenarioWorkspace: + root = workspace_root.expanduser().resolve() + root.mkdir(parents=True, mode=0o700, exist_ok=False) + root.chmod(0o700) + git_executable = _resolve_git_executable(root) + scaffold_workspace(root) + history_seed = build_history_seed() + scenario_payload = { + "schema_version": 1, + "scenario_id": SCENARIO_ID, + "milestone": SCENARIO_MILESTONE, + "protected_paths": list(PROTECTED_PATHS), + "quality_command": ["python", "-m", "unittest", "discover", "-s", "tests"], + "history_event_count": len(history_seed), + "history_hash": canonical_hash(history_seed), + } + files = { + ".benchmark/scenario.json": json.dumps(scenario_payload, indent=2, sort_keys=True) + "\n", + ".benchmark/history_seed.json": json.dumps(history_seed, indent=2, sort_keys=True) + "\n", + "benchmark_app.py": ( + '"""Small benchmark target with an intentional defect."""\n\n' + "\n" + "def sum_positive(values):\n" + ' """Return the sum of positive numeric values."""\n' + " return sum(values)\n" + ), + "tests/__init__.py": "", + "tests/test_benchmark_app.py": ( + "import unittest\n\n" + "from benchmark_app import sum_positive\n\n\n" + "class SumPositiveTests(unittest.TestCase):\n" + " def test_mixed_values(self):\n" + " self.assertEqual(sum_positive([5, -8, 2]), 7)\n\n" + " def test_non_positive_values(self):\n" + " self.assertEqual(sum_positive([-5, 0, -3]), 0)\n\n" + " def test_empty_values(self):\n" + " self.assertEqual(sum_positive([]), 0)\n\n\n" + 'if __name__ == "__main__":\n' + " unittest.main()\n" + ), + "BENCHMARK_TASK.md": ( + "# Benchmark Task\n\n" + f"{SCENARIO_MILESTONE}\n\n" + "Accepted implementation shape (comments, whitespace, and the existing " + "module/function docstrings are optional):\n\n" + "```python\n" + "def sum_positive(values):\n" + " return sum(value for value in values if value > 0)\n" + "```\n\n" + "Acceptance command: `python -m unittest discover -s tests`\n" + ), + ".gitignore": ( + ".teams_runtime/\n" + "logs/\n" + "__pycache__/\n" + "*.py[cod]\n" + ".teams_runtime_codex_output.txt\n" + ), + } + for relative_path, content in files.items(): + target = root / relative_path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + config_hash, comparable_hash = _benchmark_config( + root, + benchmark_id=benchmark_id, + run_id=run_id, + prompt_context_enabled=prompt_context_enabled, + settings=settings, + ) + protected_hashes = { + relative_path: _protected_file_hash(root, relative_path) + for relative_path in PROTECTED_PATHS + } + _assert_defect_reproduces(root) + initial_commit, commit_count = _initialize_git( + root, + git_executable=git_executable, + ) + return ScenarioWorkspace( + root=root, + initial_commit=initial_commit, + initial_commit_count=commit_count, + protected_hashes=protected_hashes, + config_hash=config_hash, + comparable_config_hash=comparable_hash, + history_hash=canonical_hash(history_seed), + history_seed=history_seed, + git_executable=git_executable, + ) + + +def inspect_scenario_workspace( + scenario: ScenarioWorkspace, + *, + timeout_seconds: float = 30.0, +) -> WorkspaceInspection: + # Retained for API compatibility; final verification is deliberately non-executing. + del timeout_seconds + root = scenario.root + notes: list[str] = [] + oracle_passed = _sum_positive_ast_oracle(root) + if not oracle_passed: + notes.append("behavior_oracle_failed") + + protected_unchanged = True + for relative_path, expected_hash in scenario.protected_hashes.items(): + try: + observed_hash = _protected_file_hash(root, relative_path) + except (OSError, ValueError): + observed_hash = "" + if observed_hash != expected_hash: + protected_unchanged = False + notes.append(f"protected_file_changed:{relative_path}") + + git_executable = scenario.git_executable + if git_executable is None: + git_clean = False + head_sha = "" + commit_created = False + no_git_remotes = False + notes.append("git_inspection_unavailable") + else: + try: + try: + attributes_unchanged = ( + _protected_file_hash(root, ".git/info/attributes") + == _EMPTY_FILE_SHA256 + ) + except (OSError, ValueError): + attributes_unchanged = False + if attributes_unchanged: + status = _run_git( + root, + "status", + "--porcelain", + "--untracked-files=all", + "--ignore-submodules=all", + "--no-ahead-behind", + git_executable=git_executable, + attributes_source=scenario.initial_commit, + check=False, + ) + git_clean = status.returncode == 0 and not status.stdout.strip() + else: + git_clean = False + notes.append("git_attributes_changed") + head_result = _run_git( + root, + "rev-parse", + "HEAD", + git_executable=git_executable, + check=False, + ) + head_sha = ( + head_result.stdout.strip() if head_result.returncode == 0 else "" + ) + commit_created = bool(head_sha and head_sha != scenario.initial_commit) + remotes = _run_git( + root, + "remote", + git_executable=git_executable, + check=False, + ) + no_git_remotes = remotes.returncode == 0 and not remotes.stdout.strip() + except ScenarioError: + git_clean = False + head_sha = "" + commit_created = False + no_git_remotes = False + notes.append("git_inspection_failed") + if not git_clean: + notes.append("git_worktree_not_clean") + if not commit_created: + notes.append("task_commit_missing") + if not no_git_remotes: + notes.append("git_remote_detected") + return WorkspaceInspection( + behavior_oracle_passed=oracle_passed, + protected_files_unchanged=protected_unchanged, + git_clean=git_clean, + commit_created=commit_created, + no_git_remotes=no_git_remotes, + head_sha=head_sha, + notes=tuple(notes), + ) + + +__all__ = [ + "BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS", + "BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS", + "BENCHMARK_TARGET_INCLUDED_EVENTS", + "BENCHMARK_TARGET_OMITTED_EVENTS", + "BENCHMARK_TARGET_PURPOSE", + "BENCHMARK_TARGET_ROLE", + "BENCHMARK_TARGET_TOTAL_EVENTS", + "BENCHMARK_TARGET_WORKFLOW_STEP", + "DEFAULT_HISTORY_SEED_COUNT", + "PROTECTED_PATHS", + "RuntimeSettings", + "SCENARIO_ID", + "SCENARIO_MILESTONE", + "ScenarioError", + "ScenarioWorkspace", + "WorkspaceInspection", + "build_history_seed", + "canonical_hash", + "create_scenario_workspace", + "inspect_scenario_workspace", + "load_runtime_settings", +] diff --git a/benchmarking/worker.py b/benchmarking/worker.py new file mode 100644 index 0000000..371b987 --- /dev/null +++ b/benchmarking/worker.py @@ -0,0 +1,2242 @@ +from __future__ import annotations + +import argparse +import asyncio +import contextlib +import hashlib +import json +import math +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +import uuid +from dataclasses import replace +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable, Mapping + +from teams_runtime.benchmarking.metrics import ( + is_v2_target_projection, + load_telemetry_directory, + sanitize_invocation_record, +) +from teams_runtime.benchmarking.models import ( + ArmPlan, + BenchmarkWorkerSafetyError, + QualityEvidence, + SprintEvidence, + WorkerContext, + WorkerOutcome, + invocation_identity_digest, + sanitize_invocation_attempts, +) +from teams_runtime.benchmarking.scenario import ( + DEFAULT_HISTORY_SEED_COUNT, + canonical_hash, +) +from teams_runtime.runtime.execution_policy import ( + InvocationBudget, + InvocationBudgetExceeded, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, + ModelInvocationTimeout, + quarantine_unsafe_workspace_entries, +) +from teams_runtime.shared.models import TEAM_ROLES +from teams_runtime.shared.paths import RuntimePaths +from teams_runtime.workflows.orchestration.team_service import TeamService +from teams_runtime.workflows.sprints.lifecycle import ( + INITIAL_PHASE_STEP_MILESTONE_REFINEMENT, + apply_initial_plan_confirmation, +) +from teams_runtime.workflows.state.sprint_store import iter_sprint_states + + +LIVE_BENCHMARK_ENV = "TEAMS_RUNTIME_LIVE_BENCHMARK" +_TERMINAL_SPRINT_STATUSES = frozenset({"completed", "failed", "blocked"}) +_COMPLETED_TODO_STATUSES = frozenset({"completed", "committed"}) +_MAX_RESUME_PASSES = 16 +_RELAY_POLL_SECONDS = 0.02 +_CHILD_TERMINATION_GRACE_SECONDS = 5.0 +_CALL_JOURNAL_STATES = ( + "reserved", + "running", + "completed", + "failed", + "timeout", + "launch_failed", + "terminated", +) +_JOURNAL_CONTEXT_STRING_FIELDS = ( + "provider", + "operation_id", + "logical_call_id", + "attempt_kind", + "runtime_identity", + "role", + "purpose", + "workflow_step", + "request_id", + "sprint_id", + "todo_id", + "backlog_id", + "goal_id", + "prompt_context_selection_policy", +) +_JOURNAL_CONTEXT_INTEGER_FIELDS = ( + "attempt_index", + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + "prompt_context_recent_events", + "prompt_context_max_events", +) +_JOURNAL_CONTEXT_FIELDS = ( + *_JOURNAL_CONTEXT_STRING_FIELDS, + *_JOURNAL_CONTEXT_INTEGER_FIELDS, + "prompt_context_enabled", +) +_PROVIDER_AUTH_ENVIRONMENT_KEYS = ( + "CODEX_API_KEY", + "OPENAI_API_KEY", +) +_CHILD_ENVIRONMENT_KEYS = ( + "CODEX_API_KEY", + "CODEX_HOME", + "CURL_CA_BUNDLE", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TEMP", + "TMP", + "TMPDIR", +) + + +class _BenchmarkRunTimeout(TimeoutError): + pass + + +class _SprintDidNotTerminate(RuntimeError): + pass + + +class _InitialPlanNotReady(RuntimeError): + pass + + +class _WorkerCleanupFailure(BenchmarkWorkerSafetyError): + pass + + +class _BenchmarkHistorySeedState: + def __init__(self) -> None: + self.lock = threading.RLock() + self.request_id = "" + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +class _WorkerLog: + """Append-only, content-free worker diagnostics.""" + + def __init__(self, path: Path, *, reset: bool = False): + self.path = path + self.path.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + try: + self.path.parent.chmod(0o700) + except OSError: + pass + if reset or not self.path.exists(): + self.path.write_text("", encoding="utf-8") + try: + self.path.chmod(0o600) + except OSError: + pass + + def append(self, event: str, **fields: Any) -> None: + safe_fields = " ".join( + f"{key}={str(value).replace(chr(10), ' ').replace(chr(13), ' ')}" + for key, value in sorted(fields.items()) + ) + line = f"{_utc_now_iso()} event={event}" + if safe_fields: + line += f" {safe_fields}" + with self.path.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") + + +class _BenchmarkTeamService(TeamService): + """Production TeamService with benchmark-only outbound and history seams.""" + + def __init__( + self, + *args: Any, + benchmark_context: WorkerContext, + benchmark_history_state: _BenchmarkHistorySeedState | None = None, + **kwargs: Any, + ): + self._benchmark_context = benchmark_context + self._benchmark_history_state = ( + benchmark_history_state or _BenchmarkHistorySeedState() + ) + self._benchmark_history_seeded = False + super().__init__(*args, **kwargs) + + def _prepare_benchmark_history(self, request_record: dict[str, Any]) -> bool: + params = ( + dict(request_record.get("params") or {}) + if isinstance(request_record.get("params"), dict) + else {} + ) + if ( + str(params.get("_teams_kind") or "").strip() != "sprint_internal" + or str(params.get("sprint_phase") or "").strip() != "initial" + or str(params.get("initial_phase_step") or "").strip() + != INITIAL_PHASE_STEP_MILESTONE_REFINEMENT + ): + return False + + request_id = str(request_record.get("request_id") or "").strip() + if not request_id: + raise ValueError("Benchmark history target request must have an id") + seed = _copy_history_seed(self._benchmark_context.history_seed) + expected_marker = { + "event_count": len(seed), + "sha256": _history_seed_hash(seed), + } + existing_marker = params.get("_benchmark_history_seed") + if existing_marker is not None: + events = [ + dict(event) + for event in (request_record.get("events") or []) + if isinstance(event, dict) + ] + if ( + existing_marker != expected_marker + or _history_seed_hash(events[: len(seed)]) + != expected_marker["sha256"] + ): + raise ValueError("Benchmark request contains an invalid history seed marker") + seeded_request_id = self._benchmark_history_state.request_id + if seeded_request_id and seeded_request_id != request_id: + raise ValueError( + "Benchmark history seed marker appears on multiple requests" + ) + return True + if self._benchmark_history_state.request_id: + return False + + request_record["events"] = [ + *seed, + *[ + dict(event) + for event in (request_record.get("events") or []) + if isinstance(event, dict) + ], + ] + params["_benchmark_history_seed"] = expected_marker + request_record["params"] = params + return True + + def _save_request(self, request_record: dict[str, Any]) -> None: + with self._benchmark_history_state.lock: + history_prepared = self._prepare_benchmark_history(request_record) + super()._save_request(request_record) + if history_prepared: + self._benchmark_history_state.request_id = str( + request_record.get("request_id") or "" + ).strip() + self._benchmark_history_seeded = bool( + self._benchmark_history_state.request_id + ) + + def _mark_github_publish_skipped(self, sprint_state: dict[str, Any]) -> None: + sprint_state["github_issue_number"] = "" + sprint_state["github_issue_url"] = "" + sprint_state["github_issue_publish_status"] = "skipped_benchmark" + sprint_state["github_issue_publish_updated_at"] = _utc_now_iso() + sprint_state.pop("github_issue_publish_error", None) + self._save_sprint_state(sprint_state) + + def _schedule_sprint_issue_publish(self, sprint_state: dict[str, Any]) -> None: + self._mark_github_publish_skipped(sprint_state) + + async def _publish_sprint_issue_best_effort( + self, + sprint_state: dict[str, Any], + ) -> None: + self._mark_github_publish_skipped(sprint_state) + return None + + async def _publish_sprint_issue_before_terminal_reports( + self, + sprint_state: dict[str, Any], + ) -> None: + self._mark_github_publish_skipped(sprint_state) + + +def _copy_history_seed( + history_seed: tuple[Mapping[str, Any], ...], +) -> list[dict[str, Any]]: + payload = json.loads( + json.dumps( + list(history_seed), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + ) + if not isinstance(payload, list) or not all(isinstance(item, dict) for item in payload): + raise ValueError("Benchmark history seed must contain JSON object events") + return [dict(item) for item in payload] + + +def _history_seed_hash(history_seed: list[dict[str, Any]]) -> str: + normalized: list[dict[str, Any]] = [] + for raw_event in history_seed: + event = dict(raw_event) + created_at = str(event.get("created_at") or "").strip() + if created_at: + parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError("Benchmark history timestamps must include a timezone") + event["created_at"] = parsed.astimezone(timezone.utc).isoformat() + normalized.append(event) + return canonical_hash(normalized) + + +def _validate_context(context: WorkerContext) -> None: + if not context.live: + raise ModelExecutionPolicyViolation( + "Live benchmark worker requires WorkerContext.live=True" + ) + if os.environ.get(LIVE_BENCHMARK_ENV) != "1": + raise ModelExecutionPolicyViolation( + f"Live benchmark worker requires {LIVE_BENCHMARK_ENV}=1" + ) + workspace_root = context.workspace_root.expanduser().resolve() + run_output_dir = context.run_output_dir.expanduser().resolve() + if run_output_dir.is_relative_to(workspace_root): + raise ModelExecutionPolicyViolation( + "Benchmark run output must be outside the provider-writable workspace" + ) + if not workspace_root.is_dir(): + raise FileNotFoundError(f"Benchmark workspace does not exist: {workspace_root}") + for relative_path in ("team_runtime.yaml", ".git", ".benchmark/scenario.json"): + if not (workspace_root / relative_path).exists(): + raise FileNotFoundError( + f"Benchmark workspace is missing required path: {relative_path}" + ) + if len(context.history_seed) != DEFAULT_HISTORY_SEED_COUNT: + raise ValueError( + "Live sprint benchmark requires the fixed " + f"{DEFAULT_HISTORY_SEED_COUNT}-event history seed" + ) + if not str(context.milestone or "").strip(): + raise ValueError("Benchmark milestone must not be empty") + if context.max_invocations <= 0: + raise ValueError("Benchmark invocation budget must be positive") + if ( + not math.isfinite(context.call_timeout_seconds) + or context.call_timeout_seconds <= 0 + ): + raise ValueError("Benchmark call timeout must be positive and finite") + if ( + not math.isfinite(context.run_timeout_seconds) + or context.run_timeout_seconds <= 0 + ): + raise ValueError("Benchmark run timeout must be positive and finite") + + +def _source_import_root() -> Path: + # teams_runtime/benchmarking/worker.py -> import parent containing teams_runtime. + return Path(__file__).resolve().parents[2] + + +def _private_telemetry_dir(context: WorkerContext) -> Path: + return ( + context.run_output_dir.expanduser().resolve() + / ".private_model_invocations" + ) + + +def _benchmark_unsafe_path_roots(context: WorkerContext) -> tuple[Path, ...]: + roots = { + context.workspace_root.expanduser().resolve(), + context.run_output_dir.expanduser().resolve(), + Path("/tmp").resolve(), + Path(tempfile.gettempdir()).expanduser().resolve(), + } + for name in ("TEMP", "TMP", "TMPDIR"): + raw_value = str(os.environ.get(name) or "").strip() + if raw_value: + roots.add(Path(raw_value).expanduser().resolve()) + return tuple(sorted(roots, key=str)) + + +def _sanitized_benchmark_path(context: WorkerContext) -> str: + safe_directories: list[str] = [] + seen: set[str] = set() + unsafe_roots = _benchmark_unsafe_path_roots(context) + for raw_entry in (os.environ.get("PATH") or os.defpath).split(os.pathsep): + if not raw_entry: + continue + entry = Path(raw_entry).expanduser() + if not entry.is_absolute(): + continue + try: + resolved = entry.resolve(strict=True) + except (OSError, RuntimeError): + continue + if not resolved.is_dir() or any( + resolved.is_relative_to(root) for root in unsafe_roots + ): + continue + resolved_text = str(resolved) + if resolved_text not in seen: + seen.add(resolved_text) + safe_directories.append(resolved_text) + if not safe_directories: + raise ModelExecutionPolicyViolation( + "Benchmark PATH contains no safe external executable directories" + ) + return os.pathsep.join(safe_directories) + + +def _resolve_benchmark_codex_executable( + context: WorkerContext, + *, + search_path: str, +) -> Path: + located = shutil.which("codex", path=search_path) + if not located: + raise ModelExecutionPolicyViolation( + "Live sprint benchmark requires the Codex CLI on a safe PATH" + ) + try: + executable = Path(located).resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ModelExecutionPolicyViolation( + "Benchmark Codex executable could not be resolved safely" + ) from exc + if ( + not executable.is_file() + or not os.access(executable, os.X_OK) + or any( + executable.is_relative_to(root) + for root in _benchmark_unsafe_path_roots(context) + ) + ): + raise ModelExecutionPolicyViolation( + "Benchmark Codex executable is not a safe external executable" + ) + return executable + + +def _initialize_private_telemetry(context: WorkerContext) -> Path: + telemetry_dir = _private_telemetry_dir(context) + if telemetry_dir.exists() or telemetry_dir.is_symlink(): + raise _WorkerCleanupFailure( + "Benchmark private telemetry directory already exists" + ) + try: + telemetry_dir.mkdir(parents=True, mode=0o700) + telemetry_dir.chmod(0o700) + except OSError as exc: + raise _WorkerCleanupFailure( + "Failed to initialize benchmark private telemetry directory" + ) from exc + return telemetry_dir + + +def _load_private_telemetry( + context: WorkerContext, +) -> tuple[dict[str, Any], ...]: + return load_telemetry_directory(_private_telemetry_dir(context)) + + +def _consume_private_telemetry( + context: WorkerContext, +) -> tuple[dict[str, Any], ...]: + telemetry_dir = _private_telemetry_dir(context) + records = load_telemetry_directory(telemetry_dir) + if not telemetry_dir.exists() and not telemetry_dir.is_symlink(): + return records + try: + shutil.rmtree(telemetry_dir) + except OSError as exc: + raise _WorkerCleanupFailure( + "Failed to remove benchmark private telemetry shards" + ) from exc + if telemetry_dir.exists() or telemetry_dir.is_symlink(): + raise _WorkerCleanupFailure( + "Benchmark private telemetry shards remain after cleanup" + ) + return records + + +def _build_execution_policy( + context: WorkerContext, + *, + budget: InvocationBudget, +) -> ModelExecutionPolicy: + if not any( + str(os.environ.get(name) or "").strip() + for name in _PROVIDER_AUTH_ENVIRONMENT_KEYS + ): + raise ModelExecutionPolicyViolation( + "Live sprint benchmark requires provider-only authentication through " + "CODEX_API_KEY or OPENAI_API_KEY; operator Codex home credentials are isolated." + ) + safe_path = _sanitized_benchmark_path(context) + codex_executable = _resolve_benchmark_codex_executable( + context, + search_path=safe_path, + ) + return ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=context.workspace_root, + invocation_budget=budget, + call_timeout_seconds=context.call_timeout_seconds, + codex_executable=codex_executable, + telemetry_output_dir=_private_telemetry_dir(context), + shell_environment={ + "LANG": "C", + "LC_ALL": "C", + "PATH": safe_path, + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": str(_source_import_root()), + "PYTHONUNBUFFERED": "1", + }, + ) + + +def _build_services( + context: WorkerContext, + *, + policy: ModelExecutionPolicy, +) -> dict[str, _BenchmarkTeamService]: + history_state = _BenchmarkHistorySeedState() + services = { + role: _BenchmarkTeamService( + context.workspace_root, + role, + enable_discord_client=False, + relay_transport="internal", + model_execution_policy=policy, + allow_external_research=False, + benchmark_context=context, + benchmark_history_state=history_state, + ) + for role in TEAM_ROLES + } + configured_models = { + str(config.model or "").strip() + for service in services.values() + for config in ( + *service.runtime_config.role_defaults.values(), + *service.runtime_config.internal_agent_defaults.values(), + ) + } + unsupported_models = sorted( + model for model in configured_models if not model or "gemini" in model.lower() + ) + if unsupported_models: + raise ModelExecutionPolicyViolation( + "Live sprint benchmark requires Codex-compatible role models" + ) + return services + + +async def _relay_pump( + services: Mapping[str, _BenchmarkTeamService], + *, + worker_log: _WorkerLog, +) -> None: + worker_log.append("relay_pump_started", role_count=len(services)) + while True: + for role in TEAM_ROLES: + await services[role]._consume_internal_relay_once() + await asyncio.sleep(_RELAY_POLL_SECONDS) + + +def _active_process_entries( + budget_or_snapshot: InvocationBudget | Mapping[str, Any], +) -> list[dict[str, Any]]: + snapshot = ( + budget_or_snapshot.snapshot() + if isinstance(budget_or_snapshot, InvocationBudget) + else budget_or_snapshot + ) + return [ + dict(entry) + for entry in (snapshot.get("entries") or []) + if isinstance(entry, dict) + and str(entry.get("state") or "") == "running" + and isinstance(entry.get("pid"), int) + ] + + +def _provider_entry_key(entry: Mapping[str, Any]) -> tuple[int, int | None]: + raw_pid = entry.get("pid") + raw_process_group_id = entry.get("process_group_id") + pid = int(raw_pid) if isinstance(raw_pid, int) and not isinstance(raw_pid, bool) else 0 + process_group_id = ( + int(raw_process_group_id) + if isinstance(raw_process_group_id, int) + and not isinstance(raw_process_group_id, bool) + else None + ) + return pid, process_group_id + + +def _merge_active_process_entries( + *snapshots: Mapping[str, Any], +) -> list[dict[str, Any]]: + merged: dict[tuple[int, int | None], dict[str, Any]] = {} + for snapshot in snapshots: + for entry in _active_process_entries(snapshot): + key = _provider_entry_key(entry) + if key[0] > 1: + merged[key] = entry + return list(merged.values()) + + +def _merge_launched_process_entries( + *snapshots: Mapping[str, Any], +) -> list[dict[str, Any]]: + """Retain only unresolved provider groups for parent cleanup.""" + + merged: dict[tuple[int, int | None], dict[str, Any]] = {} + for snapshot in snapshots: + for raw_entry in (snapshot.get("entries") or []): + if not isinstance(raw_entry, dict): + continue + entry = dict(raw_entry) + if str(entry.get("state") or "").strip() not in {"reserved", "running"}: + continue + key = _provider_entry_key(entry) + if key[0] > 1: + merged[key] = entry + return list(merged.values()) + + +def _journal_non_negative_int(value: Any, *, default: int = 0) -> int: + if not isinstance(value, int) or isinstance(value, bool): + return default + return value if value >= 0 else default + + +def _journal_optional_non_negative_int(value: Any) -> int | None: + if not isinstance(value, int) or isinstance(value, bool): + return None + return value if value >= 0 else None + + +def _journal_context_matches( + entry: Mapping[str, Any], + record: Mapping[str, Any], +) -> bool: + if record.get("prompt_context_representation_conflict") is True: + return False + for field_name in _JOURNAL_CONTEXT_STRING_FIELDS: + if str(entry.get(field_name) or "").strip() != str( + record.get(field_name) or "" + ).strip(): + return False + for field_name in _JOURNAL_CONTEXT_INTEGER_FIELDS: + if _journal_optional_non_negative_int( + entry.get(field_name) + ) != _journal_optional_non_negative_int(record.get(field_name)): + return False + journal_enabled = entry.get("prompt_context_enabled") + telemetry_enabled = record.get("prompt_context_enabled") + if journal_enabled is not telemetry_enabled: + return False + expected_status = ( + "completed" + if str(entry.get("state") or "").strip() == "completed" + else "failed" + ) + return str(record.get("status") or "").strip() == expected_status + + +def _summarize_call_journal( + snapshot: Mapping[str, Any], + *, + telemetry_records: Iterable[Mapping[str, Any]], +) -> dict[str, Any]: + raw_entries = snapshot.get("entries") + raw_entry_list = list(raw_entries) if isinstance(raw_entries, list) else [] + entries = [dict(entry) for entry in raw_entry_list if isinstance(entry, dict)] + malformed_entry_count = len(raw_entry_list) - len(entries) + state_counts = {state: 0 for state in _CALL_JOURNAL_STATES} + unknown_count = 0 + for entry in entries: + state = str(entry.get("state") or "").strip() + if state in state_counts: + state_counts[state] += 1 + else: + unknown_count += 1 + + reserved_count = _journal_non_negative_int(snapshot.get("reserved_count")) + entry_count = len(raw_entry_list) + telemetry_record_list = tuple( + sanitize_invocation_record(record) + for record in telemetry_records + ) + observed_count = len(telemetry_record_list) + journal_invocation_ids = [ + str(entry.get("invocation_id") or "").strip() + for entry in entries + ] + telemetry_invocation_ids = [ + str(record.get("invocation_id") or "").strip() + for record in telemetry_record_list + ] + journal_nonempty_ids = [ + invocation_id + for invocation_id in journal_invocation_ids + if invocation_id + ] + telemetry_nonempty_ids = [ + invocation_id + for invocation_id in telemetry_invocation_ids + if invocation_id + ] + journal_missing_id_count = ( + len(journal_invocation_ids) - len(journal_nonempty_ids) + ) + telemetry_missing_id_count = ( + len(telemetry_invocation_ids) - len(telemetry_nonempty_ids) + ) + journal_duplicate_id_count = ( + len(journal_nonempty_ids) - len(set(journal_nonempty_ids)) + ) + telemetry_duplicate_id_count = ( + len(telemetry_nonempty_ids) - len(set(telemetry_nonempty_ids)) + ) + journal_id_set = set(journal_nonempty_ids) + telemetry_id_set = set(telemetry_nonempty_ids) + telemetry_unmatched_id_count = len( + telemetry_id_set - journal_id_set + ) + journal_unobserved_id_count = len( + journal_id_set - telemetry_id_set + ) + identity_reconciled = ( + journal_missing_id_count == 0 + and telemetry_missing_id_count == 0 + and journal_duplicate_id_count == 0 + and telemetry_duplicate_id_count == 0 + and telemetry_unmatched_id_count == 0 + ) + journal_entries_by_invocation_id = { + str(entry.get("invocation_id") or "").strip(): entry + for entry in entries + if str(entry.get("invocation_id") or "").strip() + } + context_mismatch_count = 0 + verified_target_invocation_ids: list[str] = [] + for record in telemetry_record_list: + invocation_id = str(record.get("invocation_id") or "").strip() + entry = journal_entries_by_invocation_id.get(invocation_id) + if entry is None: + continue + if not _journal_context_matches(entry, record): + context_mismatch_count += 1 + continue + if is_v2_target_projection( + entry, + state_field="state", + ) and is_v2_target_projection( + record, + state_field="status", + ): + verified_target_invocation_ids.append(invocation_id) + journal_schema_version = _journal_non_negative_int( + snapshot.get("schema_version") + ) + context_reconciled = ( + journal_schema_version == 3 + and identity_reconciled + and context_mismatch_count == 0 + ) + unaccounted_count = max(reserved_count - entry_count, 0) + overaccounted_count = max(entry_count - reserved_count, 0) + return { + "schema_version": 1, + "journal_available": bool(snapshot), + "journal_schema_version": _journal_non_negative_int( + snapshot.get("schema_version") + ), + "max_invocations": _journal_non_negative_int( + snapshot.get("max_invocations") + ), + "reserved_count": reserved_count, + "entry_count": entry_count, + "telemetry_record_count": observed_count, + "unobserved_attempt_count": max(reserved_count - observed_count, 0), + "telemetry_overage_count": max(observed_count - reserved_count, 0), + "telemetry_coverage_percent": ( + round(min(observed_count * 100 / reserved_count, 100.0), 2) + if reserved_count + else 0.0 + ), + "identity_reconciled": identity_reconciled, + "context_reconciled": context_reconciled, + "journal_telemetry_context_mismatch_count": context_mismatch_count, + "verified_target_projection_count": len(verified_target_invocation_ids), + "verified_target_invocation_ids_sha256": invocation_identity_digest( + verified_target_invocation_ids + ), + "journal_invocation_ids_sha256": invocation_identity_digest( + journal_invocation_ids + ), + "journal_invocation_id_missing_count": journal_missing_id_count, + "journal_invocation_id_duplicate_count": journal_duplicate_id_count, + "telemetry_invocation_id_missing_count": telemetry_missing_id_count, + "telemetry_invocation_id_duplicate_count": telemetry_duplicate_id_count, + "telemetry_invocation_id_unmatched_count": ( + telemetry_unmatched_id_count + ), + "journal_invocation_id_unobserved_count": ( + journal_unobserved_id_count + ), + "completed_count": state_counts["completed"], + "failed_count": state_counts["failed"], + "timeout_count": state_counts["timeout"], + "launch_failed_count": state_counts["launch_failed"], + "terminated_count": state_counts["terminated"], + "active_count": state_counts["reserved"] + state_counts["running"], + "unknown_state_count": unknown_count, + "malformed_entry_count": malformed_entry_count, + "unaccounted_count": unaccounted_count, + "overaccounted_count": overaccounted_count, + "reconciled": ( + reserved_count == entry_count + and malformed_entry_count == 0 + and unknown_count == 0 + ), + "rejected_count": _journal_non_negative_int( + snapshot.get("rejected_count") + ), + "remaining_budget": _journal_non_negative_int(snapshot.get("remaining")), + } + + +def _finalize_call_journal_after_cleanup( + journal_path: Path, + snapshot: Mapping[str, Any], + *, + stop_reason: str, +) -> dict[str, Any]: + if not snapshot: + return {} + normalized = dict(snapshot) + entries: list[dict[str, Any]] = [] + changed = False + completed_at = _utc_now_iso() + for raw_entry in (snapshot.get("entries") or []): + if not isinstance(raw_entry, dict): + continue + entry = dict(raw_entry) + if str(entry.get("state") or "").strip() in {"reserved", "running"}: + entry.update( + { + "state": "terminated", + "completed_at": completed_at, + "exit_code": None, + "stop_reason": str(stop_reason or "worker_cleanup").strip(), + } + ) + changed = True + entries.append(entry) + normalized["entries"] = entries + normalized["reserved_count"] = max( + _journal_non_negative_int(snapshot.get("reserved_count")), + len(entries), + ) + if changed: + normalized["schema_version"] = _journal_non_negative_int( + snapshot.get("schema_version"), + default=1, + ) + try: + _write_private_json(journal_path, normalized) + except (OSError, TypeError, ValueError) as exc: + raise _WorkerCleanupFailure( + "Failed to finalize benchmark call journal after cleanup" + ) from exc + return normalized + + +def _process_exists(pid: int) -> bool: + if pid <= 1: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return True + return True + + +def _process_group_exists(process_group_id: int) -> bool: + if process_group_id <= 1 or not hasattr(os, "killpg"): + return False + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return True + return True + + +def _provider_entry_alive(entry: Mapping[str, Any]) -> bool: + pid, process_group_id = _provider_entry_key(entry) + if process_group_id is not None and hasattr(os, "killpg"): + return _process_group_exists(process_group_id) + return _process_exists(pid) + + +def _signal_provider_entries( + entries: list[dict[str, Any]], + process_signal: signal.Signals, +) -> int: + signaled = 0 + own_process_group = os.getpgrp() if hasattr(os, "getpgrp") else None + for entry in entries: + pid, process_group_id = _provider_entry_key(entry) + if pid <= 1: + continue + if process_group_id is not None and hasattr(os, "killpg"): + if process_group_id == own_process_group: + continue + try: + os.killpg(process_group_id, process_signal) + except ProcessLookupError: + continue + except OSError: + # Cleanup confirmation will fail closed if the group survives. + continue + signaled += 1 + continue + try: + os.kill(pid, process_signal) + except ProcessLookupError: + continue + except OSError: + # Continue so one inaccessible process cannot hide later entries. + continue + signaled += 1 + return signaled + + +def _signal_active_provider_processes( + budget_or_snapshot: InvocationBudget | Mapping[str, Any], + process_signal: signal.Signals, +) -> int: + return _signal_provider_entries( + _active_process_entries(budget_or_snapshot), + process_signal, + ) + + +async def _terminate_active_provider_processes( + budget: InvocationBudget, + *, + grace_seconds: float, + worker_log: _WorkerLog, +) -> None: + terminated = _signal_active_provider_processes(budget, signal.SIGTERM) + worker_log.append("provider_termination_requested", process_count=terminated) + if not terminated: + return + await asyncio.sleep(max(min(grace_seconds, 5.0), 0.0)) + killed = _signal_active_provider_processes(budget, signal.SIGKILL) + if killed: + worker_log.append("provider_kill_requested", process_count=killed) + + +def _latest_sprint_state(workspace_root: Path) -> dict[str, Any]: + paths = RuntimePaths.from_root(workspace_root) + states = [ + dict(state) + for state in iter_sprint_states(paths) + if isinstance(state, dict) and str(state.get("sprint_id") or "").strip() + ] + if not states: + return {} + return max( + states, + key=lambda state: ( + str(state.get("started_at") or ""), + str(state.get("sprint_id") or ""), + ), + ) + + +def _sprint_evidence(sprint_state: Mapping[str, Any]) -> SprintEvidence: + todos = [ + dict(todo) + for todo in (sprint_state.get("todos") or []) + if isinstance(todo, dict) + ] + statuses = [ + str(todo.get("status") or "").strip().lower() + for todo in todos + ] + return SprintEvidence( + sprint_id=str(sprint_state.get("sprint_id") or "").strip(), + status=str(sprint_state.get("status") or "").strip(), + closeout_status=str(sprint_state.get("closeout_status") or "").strip(), + todo_count=len(todos), + completed_todo_count=sum( + status in _COMPLETED_TODO_STATUSES for status in statuses + ), + blocked_todo_count=statuses.count("blocked"), + failed_todo_count=statuses.count("failed"), + commit_sha=str( + sprint_state.get("commit_sha") + or sprint_state.get("version_control_sha") + or sprint_state.get("auto_commit_sha") + or "" + ).strip(), + ) + + +def _quality_evidence( + sprint_state: Mapping[str, Any], + sprint: SprintEvidence, +) -> QualityEvidence: + notes: list[str] = [] + if not sprint.sprint_id: + notes.append("sprint_state_missing") + if sprint.status != "completed": + notes.append("sprint_not_completed") + if sprint.closeout_status != "verified": + notes.append("closeout_not_verified") + if any( + str(todo.get("status") or "").strip().lower() == "uncommitted" + for todo in (sprint_state.get("todos") or []) + if isinstance(todo, dict) + ): + notes.append("uncommitted_todo_present") + return QualityEvidence( + sprint_terminal=sprint.status == "completed", + closeout_verified=sprint.closeout_status == "verified", + blocked_todo_count=sprint.blocked_todo_count, + failed_todo_count=sprint.failed_todo_count, + notes=tuple(notes), + ) + + +def _confirmation_is_pending(sprint_state: Mapping[str, Any]) -> bool: + confirmation = sprint_state.get("initial_plan_confirmation") + return ( + isinstance(confirmation, dict) + and str(confirmation.get("status") or "").strip().lower() == "pending" + ) + + +def _confirm_initial_plan( + orchestrator: _BenchmarkTeamService, + sprint_state: dict[str, Any], + *, + worker_log: _WorkerLog, +) -> None: + confirmation = apply_initial_plan_confirmation( + sprint_state, + confirmed_by={ + "type": "benchmark_harness", + "author_id": "benchmark-harness", + "author_name": "benchmark-harness", + }, + message_id="benchmark-auto-confirm", + parser_reason="isolated benchmark auto-confirm policy", + parser_confidence="high", + confirmed_at=_utc_now_iso(), + ) + sprint_id = str(sprint_state.get("sprint_id") or "").strip() + orchestrator._save_sprint_state(sprint_state) + orchestrator._append_sprint_event( + sprint_id, + event_type="initial_plan_confirmed", + summary="Benchmark harness auto-confirmed the initial implementation plan.", + payload={ + "revision": int(confirmation.get("revision") or 0), + "confirmation_source": "benchmark_harness", + }, + ) + worker_log.append( + "initial_plan_auto_confirmed", + revision=int(confirmation.get("revision") or 0), + ) + + +async def _drive_sprint( + context: WorkerContext, + orchestrator: _BenchmarkTeamService, + *, + worker_log: _WorkerLog, +) -> None: + await orchestrator.start_sprint_lifecycle( + context.milestone, + trigger="benchmark", + resume_mode="await", + kickoff_brief=( + "Execute the deterministic local benchmark task. Use only files in the " + "workspace, preserve protected benchmark inputs, run the stated unittest " + "command, and commit the implementation." + ), + kickoff_requirements=[context.milestone], + kickoff_request_text=context.milestone, + kickoff_reference_artifacts=[ + "./BENCHMARK_TASK.md", + "./.benchmark/scenario.json", + "./tests/test_benchmark_app.py", + ], + kickoff_requester_route={ + "type": "benchmark_harness", + "author_id": "benchmark-harness", + "author_name": "benchmark-harness", + }, + ) + sprint_state = orchestrator._load_active_sprint_state() + if not sprint_state: + raise _InitialPlanNotReady("Sprint state was not created") + + sprint_id = str(sprint_state.get("sprint_id") or "").strip() + worker_log.append("sprint_created") + confirmation_performed = False + for _resume_pass in range(_MAX_RESUME_PASSES): + sprint_state = orchestrator._load_sprint_state(sprint_id) + status = str(sprint_state.get("status") or "").strip().lower() + if status in _TERMINAL_SPRINT_STATUSES: + worker_log.append( + "sprint_terminal", + closeout_status=str(sprint_state.get("closeout_status") or ""), + status=status, + ) + return + if _confirmation_is_pending(sprint_state): + if confirmation_performed: + raise _InitialPlanNotReady( + "Initial implementation plan returned to pending state" + ) + _confirm_initial_plan( + orchestrator, + sprint_state, + worker_log=worker_log, + ) + confirmation_performed = True + elif not confirmation_performed: + raise _InitialPlanNotReady( + "Initial implementation plan did not reach pending confirmation" + ) + await orchestrator._resume_active_sprint(sprint_id) + + raise _SprintDidNotTerminate( + f"Sprint did not terminate after {_MAX_RESUME_PASSES} resume passes" + ) + + +async def _execute_live_arm( + context: WorkerContext, + *, + budget: InvocationBudget, + policy: ModelExecutionPolicy, + worker_log: _WorkerLog, +) -> None: + services = _build_services(context, policy=policy) + worker_log.append( + "services_ready", + discord="disabled", + external_research="disabled", + relay="internal", + role_count=len(services), + ) + pump_task = asyncio.create_task( + _relay_pump(services, worker_log=worker_log), + name=f"benchmark-relay-{context.arm.run_id}", + ) + sprint_task = asyncio.create_task( + _drive_sprint( + context, + services["orchestrator"], + worker_log=worker_log, + ), + name=f"benchmark-sprint-{context.arm.run_id}", + ) + try: + await asyncio.wait_for( + sprint_task, + timeout=context.run_timeout_seconds, + ) + except ModelInvocationTimeout: + raise + except TimeoutError as exc: + await _terminate_active_provider_processes( + budget, + grace_seconds=policy.kill_grace_seconds, + worker_log=worker_log, + ) + raise _BenchmarkRunTimeout( + f"Sprint arm exceeded {context.run_timeout_seconds:g} seconds" + ) from exc + finally: + if not sprint_task.done(): + sprint_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await sprint_task + pump_task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await pump_task + for service in services.values(): + with contextlib.suppress(Exception): + await service.discord_client.close() + + +def _classify_status( + *, + forced_status: str, + sprint: SprintEvidence, + budget_snapshot: Mapping[str, Any], +) -> tuple[str, str, str]: + if int(budget_snapshot.get("rejected_count") or 0) > 0: + return ( + "call_budget_exhausted", + "invocation_budget_exhausted", + "invocation_budget_exceeded", + ) + entries = [ + entry + for entry in (budget_snapshot.get("entries") or []) + if isinstance(entry, dict) + ] + if forced_status == "timeout" or any( + str(entry.get("state") or "") == "timeout" for entry in entries + ): + return ("timeout", "timeout", "model_or_run_timeout") + if forced_status: + return ( + forced_status, + "worker_exception", + "worker_exception", + ) + if sprint.status == "completed": + return ("completed", "sprint_completed", "") + return ( + "failed", + "sprint_not_completed", + "sprint_not_completed", + ) + + +def _preflight_failure_outcome( + *, + started_at: str, + started_monotonic: float, + worker_log: _WorkerLog, + exc: BaseException, +) -> WorkerOutcome: + worker_log.append("preflight_failed", error_category=type(exc).__name__) + duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + return WorkerOutcome( + status="preflight_failed", + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=duration_ms, + worker_duration_ms=duration_ms, + stop_reason="preflight_failed", + error_category=type(exc).__name__, + ) + + +def _run_live_sprint_arm_in_child(context: WorkerContext) -> WorkerOutcome: + started_at = _utc_now_iso() + started_monotonic = time.monotonic() + run_output_dir = context.run_output_dir.expanduser().resolve() + worker_log = _WorkerLog(run_output_dir / "worker.log") + worker_log.append("child_worker_started", run_id=context.arm.run_id) + try: + _validate_context(context) + except Exception as exc: + return _preflight_failure_outcome( + started_at=started_at, + started_monotonic=started_monotonic, + worker_log=worker_log, + exc=exc, + ) + + budget = InvocationBudget( + context.max_invocations, + journal_path=run_output_dir / "call_journal.json", + ) + try: + policy = _build_execution_policy(context, budget=budget) + except Exception as exc: + return _preflight_failure_outcome( + started_at=started_at, + started_monotonic=started_monotonic, + worker_log=worker_log, + exc=exc, + ) + + forced_status = "" + forced_error_category = "" + try: + asyncio.run( + _execute_live_arm( + context, + budget=budget, + policy=policy, + worker_log=worker_log, + ) + ) + except _BenchmarkRunTimeout: + forced_status = "timeout" + forced_error_category = "run_timeout" + except ModelInvocationTimeout: + forced_status = "timeout" + forced_error_category = "model_invocation_timeout" + except InvocationBudgetExceeded: + forced_status = "call_budget_exhausted" + forced_error_category = "invocation_budget_exceeded" + except ModelExecutionPolicyViolation: + forced_status = "preflight_failed" + forced_error_category = "execution_policy_violation" + except Exception as exc: + forced_status = "failed" + forced_error_category = type(exc).__name__ + + sprint_state = _latest_sprint_state(context.workspace_root) + sprint = _sprint_evidence(sprint_state) + quality = _quality_evidence(sprint_state, sprint) + budget_snapshot = budget.snapshot() + status, stop_reason, classified_error = _classify_status( + forced_status=forced_status, + sprint=sprint, + budget_snapshot=budget_snapshot, + ) + error_category = forced_error_category or classified_error + telemetry_records = _load_private_telemetry(context) + duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + worker_log.append( + "worker_finished", + error_category=error_category or "none", + invocation_count=len(telemetry_records), + status=status, + ) + return WorkerOutcome( + status=status, # type: ignore[arg-type] + sprint=sprint, + quality=quality, + telemetry_records=telemetry_records, + invocation_attempts=_summarize_call_journal( + budget_snapshot, + telemetry_records=telemetry_records, + ), + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=duration_ms, + worker_duration_ms=duration_ms, + stop_reason=stop_reason, + error_category=error_category, + ) + + +def _sha256_text(value: str) -> str: + return hashlib.sha256(value.encode("utf-8")).hexdigest() + + +def _write_private_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + temporary_path = path.parent / f".{path.name}.{uuid.uuid4().hex}.tmp" + try: + descriptor = os.open( + temporary_path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, + ) + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump( + payload, + handle, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, path) + try: + path.chmod(0o600) + except OSError: + pass + finally: + with contextlib.suppress(FileNotFoundError): + temporary_path.unlink() + + +def _read_json_mapping(path: Path) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError, OSError): + return {} + return dict(payload) if isinstance(payload, dict) else {} + + +def _read_call_journal_strict( + path: Path, + *, + required: bool = False, +) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + if required: + raise _WorkerCleanupFailure( + "Required benchmark call journal is missing" + ) + return {} + except (json.JSONDecodeError, OSError) as exc: + raise _WorkerCleanupFailure( + "Existing benchmark call journal is unreadable or malformed" + ) from exc + if not isinstance(payload, dict): + raise _WorkerCleanupFailure("Benchmark call journal must be a JSON object") + snapshot = dict(payload) + journal_schema_version = _journal_non_negative_int( + snapshot.get("schema_version") + ) + if journal_schema_version not in {1, 2, 3}: + raise _WorkerCleanupFailure("Benchmark call journal schema is invalid") + entries = snapshot.get("entries") + if not isinstance(entries, list) or not all( + isinstance(entry, dict) for entry in entries + ): + raise _WorkerCleanupFailure("Benchmark call journal entries are invalid") + for field_name in ( + "max_invocations", + "reserved_count", + "remaining", + "rejected_count", + ): + raw_value = snapshot.get(field_name) + if ( + raw_value is None + or isinstance(raw_value, bool) + or not isinstance(raw_value, int) + or raw_value < 0 + ): + raise _WorkerCleanupFailure( + f"Benchmark call journal {field_name} is invalid" + ) + max_invocations = int(snapshot["max_invocations"]) + reserved_count = int(snapshot["reserved_count"]) + if ( + max_invocations <= 0 + or reserved_count != len(entries) + or reserved_count > max_invocations + or int(snapshot["remaining"]) != max(max_invocations - reserved_count, 0) + ): + raise _WorkerCleanupFailure("Benchmark call journal counts do not reconcile") + reservation_ids: set[str] = set() + for entry in entries: + reservation_id = str(entry.get("reservation_id") or "").strip() + if not reservation_id or reservation_id in reservation_ids: + raise _WorkerCleanupFailure( + "Benchmark call journal reservation ids are missing or duplicated" + ) + reservation_ids.add(reservation_id) + state = str(entry.get("state") or "").strip() + if state not in _CALL_JOURNAL_STATES: + raise _WorkerCleanupFailure( + "Benchmark call journal contains an invalid invocation state" + ) + if state == "running": + pid = entry.get("pid") + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 1: + raise _WorkerCleanupFailure( + "Running benchmark journal entry has an invalid process id" + ) + process_group_id = entry.get("process_group_id") + if ( + not isinstance(process_group_id, int) + or isinstance(process_group_id, bool) + or process_group_id <= 1 + or process_group_id != pid + ): + raise _WorkerCleanupFailure( + "Running benchmark journal entry has an invalid process group id" + ) + elif entry.get("process_group_id") is not None: + process_group_id = entry.get("process_group_id") + if ( + not isinstance(process_group_id, int) + or isinstance(process_group_id, bool) + or process_group_id <= 1 + ): + raise _WorkerCleanupFailure( + "Benchmark journal entry has an invalid process group id" + ) + if journal_schema_version == 3: + if not all(field_name in entry for field_name in _JOURNAL_CONTEXT_FIELDS): + raise _WorkerCleanupFailure( + "Benchmark call journal context fields are incomplete" + ) + prompt_context_enabled = entry.get("prompt_context_enabled") + if prompt_context_enabled is not None and not isinstance( + prompt_context_enabled, + bool, + ): + raise _WorkerCleanupFailure( + "Benchmark call journal prompt context flag is invalid" + ) + for field_name in _JOURNAL_CONTEXT_INTEGER_FIELDS: + raw_value = entry.get(field_name) + if raw_value is not None and ( + isinstance(raw_value, bool) + or not isinstance(raw_value, int) + or raw_value < 0 + ): + raise _WorkerCleanupFailure( + "Benchmark call journal context count is invalid" + ) + for field_name in _JOURNAL_CONTEXT_STRING_FIELDS: + if not isinstance(entry.get(field_name), str): + raise _WorkerCleanupFailure( + "Benchmark call journal context identity is invalid" + ) + return snapshot + + +def _manifest_payload( + context: WorkerContext, + *, + result_path: Path, +) -> dict[str, Any]: + return { + "schema_version": 1, + "benchmark_id": context.benchmark_id, + "arm": { + "pair_index": context.arm.pair_index, + "order_index": context.arm.order_index, + "variant": context.arm.variant, + "run_id": context.arm.run_id, + "prompt_context_enabled": context.arm.prompt_context_enabled, + }, + "workspace_root": str(context.workspace_root.expanduser().resolve()), + "run_output_dir": str(context.run_output_dir.expanduser().resolve()), + "result_path": str(result_path), + "controls": { + "max_invocations": context.max_invocations, + "call_timeout_seconds": context.call_timeout_seconds, + "run_timeout_seconds": context.run_timeout_seconds, + "live": context.live, + }, + "fixture_evidence": { + "milestone_sha256": _sha256_text(context.milestone), + "history_sha256": canonical_hash(context.history_seed), + "history_event_count": len(context.history_seed), + }, + } + + +def _child_context_from_manifest(payload: Mapping[str, Any]) -> tuple[WorkerContext, Path]: + if int(payload.get("schema_version") or 0) != 1: + raise ValueError("Unsupported benchmark worker manifest schema") + workspace_root = Path(str(payload.get("workspace_root") or "")).expanduser().resolve() + run_output_dir = Path(str(payload.get("run_output_dir") or "")).expanduser().resolve() + result_path = Path(str(payload.get("result_path") or "")).expanduser().resolve() + if result_path.parent != run_output_dir: + raise ValueError("Child result path must be inside the run output directory") + + scenario_payload = _read_json_mapping(workspace_root / ".benchmark" / "scenario.json") + milestone = str(scenario_payload.get("milestone") or "").strip() + try: + raw_history = json.loads( + (workspace_root / ".benchmark" / "history_seed.json").read_text( + encoding="utf-8" + ) + ) + except (FileNotFoundError, json.JSONDecodeError, OSError) as exc: + raise ValueError("Unable to load benchmark history fixture") from exc + if not isinstance(raw_history, list) or not all( + isinstance(item, dict) for item in raw_history + ): + raise ValueError("Benchmark history fixture must be a JSON object array") + history_seed = tuple(dict(item) for item in raw_history) + + expected_fixture = ( + dict(payload.get("fixture_evidence") or {}) + if isinstance(payload.get("fixture_evidence"), dict) + else {} + ) + if _sha256_text(milestone) != str( + expected_fixture.get("milestone_sha256") or "" + ): + raise ValueError("Benchmark milestone fixture hash differs from the parent context") + if canonical_hash(history_seed) != str( + expected_fixture.get("history_sha256") or "" + ): + raise ValueError("Benchmark history fixture hash differs from the parent context") + if len(history_seed) != int( + expected_fixture.get("history_event_count") or 0 + ): + raise ValueError("Benchmark history fixture count differs from the parent context") + + raw_arm = ( + dict(payload.get("arm") or {}) + if isinstance(payload.get("arm"), dict) + else {} + ) + variant = str(raw_arm.get("variant") or "") + if variant not in {"before", "after"}: + raise ValueError("Benchmark worker manifest has an invalid arm variant") + arm = ArmPlan( + pair_index=int(raw_arm.get("pair_index") or 0), + order_index=int(raw_arm.get("order_index") or 0), + variant=variant, # type: ignore[arg-type] + run_id=str(raw_arm.get("run_id") or "").strip(), + prompt_context_enabled=bool(raw_arm.get("prompt_context_enabled")), + ) + controls = ( + dict(payload.get("controls") or {}) + if isinstance(payload.get("controls"), dict) + else {} + ) + return ( + WorkerContext( + benchmark_id=str(payload.get("benchmark_id") or "").strip(), + arm=arm, + workspace_root=workspace_root, + run_output_dir=run_output_dir, + milestone=milestone, + history_seed=history_seed, + max_invocations=int(controls.get("max_invocations") or 0), + call_timeout_seconds=float(controls.get("call_timeout_seconds") or 0), + run_timeout_seconds=float(controls.get("run_timeout_seconds") or 0), + live=bool(controls.get("live")), + ), + result_path, + ) + + +def _worker_outcome_payload(outcome: WorkerOutcome) -> dict[str, Any]: + return { + "schema_version": 1, + "status": outcome.status, + "sprint": outcome.sprint.to_dict(), + "quality": outcome.quality.to_dict(), + # The parent reloads telemetry from its private directory after cleanup. + "telemetry_records": [], + "invocation_attempts": sanitize_invocation_attempts( + outcome.invocation_attempts + ), + "started_at": outcome.started_at, + "ended_at": outcome.ended_at, + "wall_duration_ms": outcome.wall_duration_ms, + "worker_duration_ms": outcome.worker_duration_ms, + "stop_reason": outcome.stop_reason, + "error_category": outcome.error_category, + } + + +def _worker_outcome_from_payload(payload: Mapping[str, Any]) -> WorkerOutcome: + if int(payload.get("schema_version") or 0) != 1: + raise ValueError("Unsupported benchmark worker result schema") + status = str(payload.get("status") or "") + if status not in { + "completed", + "failed", + "timeout", + "call_budget_exhausted", + "preflight_failed", + }: + raise ValueError("Benchmark worker result has an invalid status") + raw_sprint = ( + dict(payload.get("sprint") or {}) + if isinstance(payload.get("sprint"), dict) + else {} + ) + sprint = SprintEvidence( + sprint_id=str(raw_sprint.get("sprint_id") or ""), + status=str(raw_sprint.get("status") or ""), + closeout_status=str(raw_sprint.get("closeout_status") or ""), + todo_count=int(raw_sprint.get("todo_count") or 0), + completed_todo_count=int(raw_sprint.get("completed_todo_count") or 0), + blocked_todo_count=int(raw_sprint.get("blocked_todo_count") or 0), + failed_todo_count=int(raw_sprint.get("failed_todo_count") or 0), + commit_sha=str(raw_sprint.get("commit_sha") or ""), + ) + raw_quality = ( + dict(payload.get("quality") or {}) + if isinstance(payload.get("quality"), dict) + else {} + ) + quality = QualityEvidence( + behavior_oracle_passed=bool(raw_quality.get("behavior_oracle_passed")), + sprint_terminal=bool(raw_quality.get("sprint_terminal")), + closeout_verified=bool(raw_quality.get("closeout_verified")), + protected_files_unchanged=bool( + raw_quality.get("protected_files_unchanged") + ), + git_clean=bool(raw_quality.get("git_clean")), + commit_created=bool(raw_quality.get("commit_created")), + no_git_remotes=bool(raw_quality.get("no_git_remotes")), + blocked_todo_count=int(raw_quality.get("blocked_todo_count") or 0), + failed_todo_count=int(raw_quality.get("failed_todo_count") or 0), + notes=tuple( + str(note) + for note in (raw_quality.get("notes") or []) + if str(note).strip() + ), + ) + invocation_attempts = sanitize_invocation_attempts( + payload.get("invocation_attempts") + ) + return WorkerOutcome( + status=status, # type: ignore[arg-type] + sprint=sprint, + quality=quality, + # Child result files are not an evidence channel. The parent replaces + # this with records read from its private recorder directory. + telemetry_records=(), + invocation_attempts=invocation_attempts, + started_at=str(payload.get("started_at") or ""), + ended_at=str(payload.get("ended_at") or ""), + wall_duration_ms=max(int(payload.get("wall_duration_ms") or 0), 0), + worker_duration_ms=max(int(payload.get("worker_duration_ms") or 0), 0), + stop_reason=str(payload.get("stop_reason") or ""), + error_category=str(payload.get("error_category") or ""), + ) + + +def _child_environment(context: WorkerContext) -> dict[str, str]: + environment = { + key: value + for key in _CHILD_ENVIRONMENT_KEYS + if (value := os.environ.get(key)) is not None + } + environment["PATH"] = _sanitized_benchmark_path(context) + environment["PYTHONPATH"] = str(_source_import_root()) + environment["PYTHONDONTWRITEBYTECODE"] = "1" + environment["PYTHONUNBUFFERED"] = "1" + environment["NO_COLOR"] = "1" + environment[LIVE_BENCHMARK_ENV] = "1" + return environment + + +def _signal_child_group( + process: subprocess.Popen[Any], + process_signal: signal.Signals, +) -> None: + try: + if hasattr(os, "killpg"): + os.killpg(process.pid, process_signal) + elif process.poll() is None: + process.send_signal(process_signal) + except OSError: + pass + + +def _worker_group_alive(process: subprocess.Popen[Any]) -> bool: + if hasattr(os, "killpg"): + try: + process.poll() + except OSError: + return True + return _process_group_exists(process.pid) + return process.poll() is None + + +def _append_cleanup_log( + worker_log: _WorkerLog, + event: str, + **fields: Any, +) -> None: + try: + worker_log.append(event, **fields) + except OSError: + # Diagnostics must not prevent process termination. + pass + + +def _wait_for_cleanup_confirmation( + process: subprocess.Popen[Any], + *, + provider_entries: list[dict[str, Any]], + worker_log: _WorkerLog, + timeout_seconds: float, +) -> None: + deadline = time.monotonic() + max(timeout_seconds, 0.0) + while True: + worker_alive = _worker_group_alive(process) + surviving_providers = [ + entry for entry in provider_entries if _provider_entry_alive(entry) + ] + if not worker_alive and not surviving_providers: + _append_cleanup_log( + worker_log, + "worker_cleanup_confirmed", + provider_group_count=len(provider_entries), + ) + return + _signal_provider_entries(surviving_providers, signal.SIGKILL) + if worker_alive: + _signal_child_group(process, signal.SIGKILL) + if time.monotonic() >= deadline: + _append_cleanup_log( + worker_log, + "worker_cleanup_failed", + provider_group_count=len(surviving_providers), + worker_group_alive=worker_alive, + ) + raise _WorkerCleanupFailure( + "Timed out confirming benchmark worker and provider process termination" + ) + time.sleep(0.05) + + +def _terminate_worker_child( + process: subprocess.Popen[Any], + *, + journal_path: Path, + worker_log: _WorkerLog, + stop_reason: str = "worker_cleanup", +) -> dict[str, Any]: + cleanup_failures: list[tuple[str, _WorkerCleanupFailure]] = [] + + def defer_failure(stage: str, failure: _WorkerCleanupFailure) -> None: + cleanup_failures.append((stage, failure)) + _append_cleanup_log( + worker_log, + "worker_cleanup_failure_deferred", + stage=stage, + error_category=type(failure).__name__, + ) + + def read_journal(stage: str) -> dict[str, Any]: + try: + snapshot = _read_call_journal_strict( + journal_path, + required=True, + ) + except _WorkerCleanupFailure as exc: + defer_failure(stage, exc) + return {} + if any( + str(entry.get("state") or "").strip() == "reserved" + for entry in (snapshot.get("entries") or []) + if isinstance(entry, dict) + ): + defer_failure( + f"{stage}_reserved_attempt", + _WorkerCleanupFailure( + "Benchmark provider launch registration was incomplete" + ), + ) + return snapshot + + initial_snapshot = read_journal("initial_journal") + provider_entries = _merge_launched_process_entries(initial_snapshot) + provider_count = _signal_provider_entries( + provider_entries, + signal.SIGTERM, + ) + _append_cleanup_log( + worker_log, + "parent_provider_termination_requested", + process_count=provider_count, + ) + _signal_child_group(process, signal.SIGTERM) + term_timed_out = False + try: + process.wait(timeout=_CHILD_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + term_timed_out = True + except OSError: + term_timed_out = True + defer_failure( + "worker_term_wait", + _WorkerCleanupFailure( + "Unable to reap benchmark worker after SIGTERM" + ), + ) + + # The worker may reserve and launch a provider between the first journal + # read and delivery of SIGTERM. This read is required regardless of whether + # the worker exited during its grace window. + pre_kill_snapshot = read_journal("pre_kill_journal") + provider_entries = _merge_launched_process_entries( + initial_snapshot, + pre_kill_snapshot, + ) + _signal_provider_entries(provider_entries, signal.SIGKILL) + _signal_child_group(process, signal.SIGKILL) + if term_timed_out: + try: + process.wait(timeout=_CHILD_TERMINATION_GRACE_SECONDS) + except subprocess.TimeoutExpired: + _append_cleanup_log( + worker_log, + "worker_process_reap_failed", + provider_group_count=len(provider_entries), + ) + defer_failure( + "worker_kill_wait", + _WorkerCleanupFailure( + "Benchmark worker did not exit after SIGKILL" + ), + ) + except OSError: + defer_failure( + "worker_kill_wait", + _WorkerCleanupFailure( + "Unable to reap benchmark worker after SIGKILL" + ), + ) + + # Once the worker has received SIGKILL it cannot intentionally launch + # another call. A final read closes the remaining write/read race before + # liveness verification. + final_snapshot = read_journal("final_journal") + provider_entries = _merge_launched_process_entries( + initial_snapshot, + pre_kill_snapshot, + final_snapshot, + ) + _signal_provider_entries(provider_entries, signal.SIGKILL) + _signal_child_group(process, signal.SIGKILL) + _wait_for_cleanup_confirmation( + process, + provider_entries=provider_entries, + worker_log=worker_log, + timeout_seconds=_CHILD_TERMINATION_GRACE_SECONDS, + ) + + finalized_snapshot: dict[str, Any] = {} + if final_snapshot: + try: + finalized_snapshot = _finalize_call_journal_after_cleanup( + journal_path, + final_snapshot, + stop_reason=stop_reason, + ) + except _WorkerCleanupFailure as exc: + defer_failure("journal_finalization", exc) + + if cleanup_failures: + stages = ",".join(stage for stage, _failure in cleanup_failures) + _append_cleanup_log( + worker_log, + "worker_cleanup_failed_closed", + failure_count=len(cleanup_failures), + stages=stages, + ) + raise _WorkerCleanupFailure( + f"Benchmark worker cleanup could not be verified: {stages}" + ) from cleanup_failures[0][1] + return finalized_snapshot + + +def _partial_outcome( + context: WorkerContext, + *, + status: str, + started_at: str, + started_monotonic: float, + stop_reason: str, + error_category: str, + telemetry_records: tuple[Mapping[str, Any], ...], +) -> WorkerOutcome: + sprint_state = _latest_sprint_state(context.workspace_root) + sprint = _sprint_evidence(sprint_state) + quality = _quality_evidence(sprint_state, sprint) + duration_ms = max(int((time.monotonic() - started_monotonic) * 1000), 0) + journal_snapshot = _read_call_journal_strict( + context.run_output_dir.expanduser().resolve() / "call_journal.json", + required=True, + ) + return WorkerOutcome( + status=status, # type: ignore[arg-type] + sprint=sprint, + quality=quality, + telemetry_records=telemetry_records, + invocation_attempts=_summarize_call_journal( + journal_snapshot, + telemetry_records=telemetry_records, + ), + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=duration_ms, + worker_duration_ms=duration_ms, + stop_reason=stop_reason, + error_category=error_category, + ) + + +def run_live_sprint_arm(context: WorkerContext) -> WorkerOutcome: + """Run one benchmark arm behind a hard child-process timeout boundary. + + The child executes the production sprint. The parent stores only bounded + controls in its temporary manifest and recovers privacy-safe partial evidence + if the child or one of its provider process groups must be terminated. + """ + + started_at = _utc_now_iso() + started_monotonic = time.monotonic() + run_output_dir = context.run_output_dir.expanduser().resolve() + worker_log = _WorkerLog(run_output_dir / "worker.log", reset=True) + worker_log.append("worker_parent_started", run_id=context.arm.run_id) + try: + _validate_context(context) + except Exception as exc: + return _preflight_failure_outcome( + started_at=started_at, + started_monotonic=started_monotonic, + worker_log=worker_log, + exc=exc, + ) + + nonce = uuid.uuid4().hex + manifest_path = run_output_dir / f".worker-manifest-{nonce}.json" + result_path = run_output_dir / f".worker-result-{nonce}.json" + journal_path = run_output_dir / "call_journal.json" + with contextlib.suppress(FileNotFoundError): + journal_path.unlink() + _initialize_private_telemetry(context) + try: + _write_private_json( + journal_path, + { + "schema_version": 3, + "max_invocations": context.max_invocations, + "reserved_count": 0, + "remaining": context.max_invocations, + "rejected_count": 0, + "entries": [], + }, + ) + except (OSError, TypeError, ValueError) as exc: + _consume_private_telemetry(context) + raise _WorkerCleanupFailure( + "Failed to initialize benchmark call journal" + ) from exc + try: + _write_private_json( + manifest_path, + _manifest_payload(context, result_path=result_path), + ) + except (OSError, TypeError, ValueError) as exc: + _consume_private_telemetry(context) + raise _WorkerCleanupFailure( + "Failed to initialize benchmark worker manifest" + ) from exc + with contextlib.suppress(FileNotFoundError): + result_path.unlink() + command = ( + sys.executable, + "-m", + "teams_runtime.benchmarking.worker", + "--child-manifest", + str(manifest_path), + ) + try: + process = subprocess.Popen( + command, + cwd=str(context.workspace_root.expanduser().resolve()), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=_child_environment(context), + start_new_session=True, + ) + except Exception as exc: + with contextlib.suppress(FileNotFoundError): + manifest_path.unlink() + worker_log.append("worker_child_launch_failed", error_category=type(exc).__name__) + return _partial_outcome( + context, + status="preflight_failed", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="worker_child_launch_failed", + error_category=type(exc).__name__, + telemetry_records=_consume_private_telemetry(context), + ) + + timed_out = False + final_journal_snapshot: dict[str, Any] = {} + try: + process.wait(timeout=context.run_timeout_seconds) + except subprocess.TimeoutExpired: + timed_out = True + worker_log.append("worker_child_timeout") + final_journal_snapshot = _terminate_worker_child( + process, + journal_path=journal_path, + worker_log=worker_log, + stop_reason="run_timeout_exceeded", + ) + finally: + with contextlib.suppress(FileNotFoundError): + manifest_path.unlink() + + if not timed_out: + # The child's own asyncio deadline can fire just before the parent's. + # Verify cleanup even when process.wait() observed a normal child exit. + final_journal_snapshot = _terminate_worker_child( + process, + journal_path=journal_path, + worker_log=worker_log, + stop_reason="worker_exit_cleanup", + ) + + try: + quarantined_entries = quarantine_unsafe_workspace_entries( + context.workspace_root, + ) + except (ModelExecutionPolicyViolation, OSError, ValueError) as exc: + raise _WorkerCleanupFailure( + "Benchmark workspace integrity could not be verified after child cleanup" + ) from exc + if quarantined_entries: + worker_log.append( + "unsafe_workspace_entries_quarantined_by_parent", + entry_count=len(quarantined_entries), + ) + raise _WorkerCleanupFailure( + "Benchmark child left unsafe filesystem entries after provider cleanup" + ) + + if timed_out: + with contextlib.suppress(FileNotFoundError): + result_path.unlink() + outcome = _partial_outcome( + context, + status="timeout", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="run_timeout_exceeded", + error_category="run_timeout", + telemetry_records=_consume_private_telemetry(context), + ) + worker_log.append( + "worker_parent_finished", + invocation_count=len(outcome.telemetry_records), + status=outcome.status, + ) + return outcome + + result_payload = _read_json_mapping(result_path) + with contextlib.suppress(FileNotFoundError): + result_path.unlink() + telemetry_records = _consume_private_telemetry(context) + if process.returncode != 0 or not result_payload: + outcome = _partial_outcome( + context, + status="failed", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="worker_child_failed", + error_category="worker_child_failed", + telemetry_records=telemetry_records, + ) + else: + try: + outcome = replace( + _worker_outcome_from_payload(result_payload), + telemetry_records=telemetry_records, + ) + except (TypeError, ValueError): + outcome = _partial_outcome( + context, + status="failed", + started_at=started_at, + started_monotonic=started_monotonic, + stop_reason="worker_result_invalid", + error_category="worker_result_invalid", + telemetry_records=telemetry_records, + ) + + parent_duration_ms = max( + int((time.monotonic() - started_monotonic) * 1000), + 0, + ) + outcome = replace( + outcome, + started_at=started_at, + ended_at=_utc_now_iso(), + wall_duration_ms=parent_duration_ms, + invocation_attempts=( + _summarize_call_journal( + final_journal_snapshot, + telemetry_records=outcome.telemetry_records, + ) + if final_journal_snapshot + else outcome.invocation_attempts + ), + ) + worker_log.append( + "worker_parent_finished", + invocation_count=len(outcome.telemetry_records), + status=outcome.status, + ) + return outcome + + +def _child_main(manifest_path: Path) -> int: + payload = _read_json_mapping(manifest_path.expanduser().resolve()) + if not payload: + return 2 + try: + context, result_path = _child_context_from_manifest(payload) + except (TypeError, ValueError): + return 2 + outcome = _run_live_sprint_arm_in_child(context) + _write_private_json(result_path, _worker_outcome_payload(outcome)) + return 0 + + +def _main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument("--child-manifest", type=Path, required=True) + args = parser.parse_args(argv) + return _child_main(args.child_manifest) + + +__all__ = [ + "LIVE_BENCHMARK_ENV", + "run_live_sprint_arm", +] + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/cli.py b/cli.py index 4f5b9a8..50e9e77 100644 --- a/cli.py +++ b/cli.py @@ -2,12 +2,14 @@ import argparse import asyncio +import json import logging import os from collections.abc import Awaitable from pathlib import Path from teams_runtime.adapters.cli.commands import build_parser as build_cli_parser +from teams_runtime.adapters.cli.commands import cmd_config_internal_set_impl from teams_runtime.adapters.cli.commands import cmd_config_research_set_impl from teams_runtime.adapters.cli.commands import cmd_config_role_set_impl from teams_runtime.adapters.cli.commands import cmd_goal_cancel_impl @@ -17,6 +19,7 @@ from teams_runtime.adapters.cli.commands import cmd_goal_stop_impl from teams_runtime.adapters.cli.commands import cmd_init_impl from teams_runtime.adapters.cli.commands import cmd_list_impl +from teams_runtime.adapters.cli.commands import cmd_metrics_impl from teams_runtime.adapters.cli.commands import cmd_restart_impl from teams_runtime.adapters.cli.commands import cmd_sprint_restart_impl from teams_runtime.adapters.cli.commands import cmd_sprint_start_impl @@ -36,6 +39,7 @@ ) from teams_runtime.shared.config import ( load_team_runtime_config, + update_team_runtime_internal_agent_defaults, update_team_runtime_research_defaults, update_team_runtime_role_defaults, validate_runtime_discord_agents_config, @@ -52,6 +56,7 @@ from teams_runtime.core.template import refresh_workspace_prompt_assets, scaffold_workspace from teams_runtime.shared.models import ALL_RUNTIME_AGENTS, INTERNAL_TEAM_AGENTS, TEAM_ROLES from teams_runtime.runtime.session_manager import RoleSessionManager +from teams_runtime.runtime.model_telemetry import aggregate_model_invocations, render_model_metrics_summary logging.basicConfig( @@ -266,6 +271,7 @@ def resolve_workspace_root(raw: str | None) -> Path: def build_parser() -> argparse.ArgumentParser: return build_cli_parser( all_runtime_agents=ALL_RUNTIME_AGENTS, + internal_team_agents=INTERNAL_TEAM_AGENTS, team_roles=TEAM_ROLES, relay_transport_internal=RELAY_TRANSPORT_INTERNAL, relay_transport_discord=RELAY_TRANSPORT_DISCORD, @@ -391,6 +397,119 @@ def cmd_list(workspace_root: Path, request_id: str | None) -> int: ) +def cmd_metrics( + workspace_root: Path, + *, + hours: float = 24.0, + request_id: str = "", + sprint_id: str = "", + role: str = "", + as_json: bool = False, +) -> int: + return cmd_metrics_impl( + workspace_root, + hours=hours, + request_id=request_id, + sprint_id=sprint_id, + role=role, + as_json=as_json, + runtime_paths_cls=RuntimePaths, + aggregate_model_invocations=aggregate_model_invocations, + render_model_metrics_summary=render_model_metrics_summary, + ) + + +def cmd_benchmark_sprint_ab( + *, + live: bool, + runtime_config: str, + repetitions: int, + max_invocations: int, + call_timeout_seconds: float, + run_timeout_seconds: float, + keep_workspaces: str, + rate_card_file: str = "", + output_dir: str = "", + benchmark_id: str = "", + allow_dirty_source: bool = False, + as_json: bool = False, +) -> int: + from teams_runtime.benchmarking.models import ( + BenchmarkOptions, + BenchmarkWorkerSafetyError, + ) + from teams_runtime.benchmarking.runner import ( + BenchmarkPreflightError, + run_sprint_ab_benchmark, + ) + from teams_runtime.benchmarking.worker import ( + LIVE_BENCHMARK_ENV, + run_live_sprint_arm, + ) + + if not live or os.environ.get(LIVE_BENCHMARK_ENV) != "1": + print( + "Live benchmark calls require both --live and " + f"{LIVE_BENCHMARK_ENV}=1." + ) + return 2 + + normalized_runtime_config = str(runtime_config or "").strip() + if not normalized_runtime_config: + print("--runtime-config is required.") + return 2 + + options = BenchmarkOptions( + source_root=Path(__file__).resolve().parent, + runtime_config_path=Path(normalized_runtime_config).expanduser(), + output_dir=Path(output_dir).expanduser() if str(output_dir or "").strip() else None, + rate_card_path=( + Path(rate_card_file).expanduser() + if str(rate_card_file or "").strip() + else None + ), + repetitions=repetitions, + max_invocations=max_invocations, + call_timeout_seconds=call_timeout_seconds, + run_timeout_seconds=run_timeout_seconds, + keep_workspaces=keep_workspaces, # type: ignore[arg-type] + allow_dirty_source=allow_dirty_source, + live=True, + benchmark_id=str(benchmark_id or "").strip(), + ) + try: + result = run_sprint_ab_benchmark( + options, + worker=run_live_sprint_arm, + ) + except BenchmarkWorkerSafetyError as exc: + print(f"Benchmark safety abort: {exc}") + return 2 + except (BenchmarkPreflightError, FileNotFoundError, OSError, ValueError) as exc: + print(f"Benchmark preflight failed: {exc}") + return 2 + + summary = { + "benchmark_id": result.benchmark_id, + "status": result.status, + "classification": result.classification, + "output_dir": str(result.output_dir), + "report_json": str(result.report_json), + "report_markdown": str(result.report_markdown), + "exit_code": result.exit_code, + } + if as_json: + print(json.dumps(summary, ensure_ascii=True, indent=2, sort_keys=True)) + else: + print( + f"benchmark_id={result.benchmark_id} status={result.status} " + f"classification={result.classification}" + ) + print(f"report_json={result.report_json}") + print(f"report_markdown={result.report_markdown}") + return result.exit_code + + def cmd_config_role_set( workspace_root: Path, role: str, @@ -408,6 +527,25 @@ def cmd_config_role_set( ) +def cmd_config_internal_set( + workspace_root: Path, + agent: str, + *, + model: str | None = None, + reasoning: str | None = None, +) -> int: + return cmd_config_internal_set_impl( + workspace_root, + agent, + model=model, + reasoning=reasoning, + update_team_runtime_internal_agent_defaults=( + update_team_runtime_internal_agent_defaults + ), + runtime_paths_cls=RuntimePaths, + ) + + def cmd_config_research_set( workspace_root: Path, *, @@ -530,6 +668,8 @@ def main(argv: list[str] | None = None) -> int: cmd_stop=cmd_stop, cmd_restart=cmd_restart, cmd_list=cmd_list, + cmd_metrics=cmd_metrics, + cmd_config_internal_set=cmd_config_internal_set, cmd_config_role_set=cmd_config_role_set, cmd_config_research_set=cmd_config_research_set, cmd_sprint_start=cmd_sprint_start, @@ -541,6 +681,7 @@ def main(argv: list[str] | None = None) -> int: cmd_goal_stop=cmd_goal_stop, cmd_goal_resume=cmd_goal_resume, cmd_goal_cancel=cmd_goal_cancel, + cmd_benchmark_sprint_ab=cmd_benchmark_sprint_ab, default_relay_transport=DEFAULT_RELAY_TRANSPORT, ) diff --git a/docs/README.md b/docs/README.md index eb588a5..510be30 100644 --- a/docs/README.md +++ b/docs/README.md @@ -10,6 +10,8 @@ Package-local documentation for the standalone `teams_runtime` module lives here - explains `discord_agents_config.yaml`, `team_runtime.yaml`, bot IDs, sprint IDs, and actions - `operations_guide.md` - day-to-day commands, request flow, sprint rollover, and troubleshooting +- `telemetry.md` + - model cost, token, latency, retry, privacy, storage, CLI, and analysis guidance ## Maintainer Reference diff --git a/docs/call_amplification_controls.md b/docs/call_amplification_controls.md new file mode 100644 index 0000000..95001ca --- /dev/null +++ b/docs/call_amplification_controls.md @@ -0,0 +1,185 @@ +# Call Amplification Controls + +This document describes the runtime controls introduced for optimization issue #2: + +- independent model and reasoning tiers for internal helper agents +- a bounded architect review-cycle budget +- a separate bounded workflow-reopen budget +- benchmark provenance for every effective model tier + +The controls reduce avoidable model cost while preserving the public-role workflow and the evidence needed to evaluate quality. + +## Why These Calls Matter + +The internal `parser`, `sourcer`, and `version_controller` agents perform narrow, high-frequency tasks. Before independent configuration, all three inherited the orchestrator model and reasoning level. That made a routine classification, milestone sourcing pass, or commit check use the same tier as broad orchestration reasoning. + +Review and reopen paths can amplify this baseline further. A single implementation todo may traverse: + +```text +architect -> developer -> architect -> developer -> qa +``` + +If review or QA repeatedly requests revisions, each new handoff adds another model call and carries more accumulated context. The two workflow budgets provide deterministic upper bounds on that amplification. + +## Internal Helper Tiers + +Newly scaffolded workspaces use these defaults: + +| Agent | Model | Reasoning | Workload rationale | +|---|---|---|---| +| `parser` | `gpt-5.4-mini` | `low` | Narrow intent normalization and status classification | +| `sourcer` | `gpt-5.4-mini` | `medium` | Goal and milestone framing needs more synthesis than classification | +| `version_controller` | `gpt-5.4-mini` | `low` | Constrained Git inspection and policy-guided commit execution | + +Public role defaults are unchanged. In particular, planner, architect, developer, QA, and orchestrator quality settings do not change as part of helper tiering. + +The helper default uses [`gpt-5.4-mini`](https://developers.openai.com/api/docs/models/gpt-5.4-mini) because OpenAI positions it as a faster, efficient model for high-volume coding and subagent workloads. This is a rollout hypothesis, not a guarantee of equivalent quality, so the benchmark and rollback gates below remain mandatory. + +The configured tiers live in `team_runtime.yaml`: + +```yaml +internal_agent_defaults: + parser: + model: "gpt-5.4-mini" + reasoning: "low" + sourcer: + model: "gpt-5.4-mini" + reasoning: "medium" + version_controller: + model: "gpt-5.4-mini" + reasoning: "low" +``` + +Change one helper with the CLI: + +```bash +python -m teams_runtime config internal set \ + --workspace-root teams_generated \ + --agent sourcer \ + --model gpt-5.4-mini \ + --reasoning medium +python -m teams_runtime restart \ + --workspace-root teams_generated \ + --agent orchestrator +``` + +The helpers are orchestrator-local runtimes, so an orchestrator restart is required after a change. + +## Backward Compatibility + +An existing workspace does not need to add `internal_agent_defaults` immediately. If the section or an individual helper entry is absent, that helper inherits the effective orchestrator model and reasoning level. + +For example, this legacy configuration remains valid: + +```yaml +role_defaults: + orchestrator: + model: "gpt-5.5" + reasoning: "medium" +``` + +Its effective helper configuration is: + +```yaml +internal_agent_defaults: + parser: {model: "gpt-5.5", reasoning: "medium"} + sourcer: {model: "gpt-5.5", reasoning: "medium"} + version_controller: {model: "gpt-5.5", reasoning: "medium"} +``` + +This fallback preserves legacy behavior. Running `config internal set` creates only the requested override; omitted fields continue to inherit during loading. + +## Review-Cycle Budget + +`implementation_review_cycle_limit` limits architect implementation reviews. The default is `3`. + +This lowers the default architect-review entry ceiling from `20` to `3`: 17 fewer possible review entries, or an 85% reduction in that structural bound. Actual call savings depend on how often work would otherwise revise, and must be measured rather than inferred from the ceiling alone. + +A review cycle is counted when the workflow routes developer output to `architect_review`. Initial architecture guidance is not a review cycle. When the architect is already processing the review at the configured limit, an explicit continuation to another developer revision is blocked instead of opening another implementation loop. A successful review may still advance to QA at the limit. + +Configure it in the generated orchestrator policy: + +```yaml +workflow_contract: + implementation_review_cycle_limit: 3 +``` + +## Reopen Budget + +`implementation_reopen_limit` is an independent cap for orchestrator-governed `reopen` transitions. The default is `3`. + +The counter increments once when a valid reopen request is accepted and routed. Categories include `scope`, `ux`, `architecture`, `implementation`, and `verification`. Provider retries and contract-repair attempts are not reopens and do not affect this counter. + +The limit is checked before the next model handoff. With a limit of `3`, the first three reopen transitions are routed; the fourth is terminally blocked and requires operator intervention. + +Configure it beside the review limit: + +```yaml +workflow_contract: + implementation_review_cycle_limit: 3 + implementation_reopen_limit: 3 +``` + +The orchestrator copies both limits into each new internal request's persisted workflow state. Restart the orchestrator after changing the policy. Existing in-flight requests retain their persisted limits so a deployment does not silently change an active workflow's contract. + +### Reopen Example + +Input state and QA transition: + +```json +{ + "workflow": { + "phase": "validation", + "step": "qa_validation", + "reopen_count": 3, + "reopen_limit": 3 + }, + "transition": { + "outcome": "reopen", + "target_phase": "implementation", + "target_step": "developer_revision", + "reopen_category": "verification" + } +} +``` + +Result before any fourth developer call: + +```json +{ + "next_role": "", + "terminal_status": "blocked", + "workflow_state": { + "phase": "validation", + "step": "qa_validation", + "phase_status": "blocked", + "reopen_count": 3, + "reopen_limit": 3, + "reopen_category": "verification" + } +} +``` + +The persisted terminal summary includes the observed count and configured limit. + +## Benchmark And Telemetry Evidence + +The sprint benchmark copies both `role_defaults` and `internal_agent_defaults` into every isolated arm. Its source configuration hash includes the effective helper values, including inherited values from a legacy config. Reports record a single `runtime_model_map` containing public roles and internal helpers. + +This prevents two runs with different helper tiers from being treated as the same configuration. It also makes per-agent telemetry groups directly reconcilable with benchmark provenance. + +For a controlled before/after experiment: + +1. Keep the scenario, source revision, repetition count, prompt-context policy, and public role tiers fixed. +2. Run the baseline with helpers set to the orchestrator tier and higher workflow budgets. +3. Run the candidate with the scaffold helper tiers and three-call budgets. +4. Compare invocations per completed todo, input/output tokens, estimated cost, p95 latency, repair rate, reopen count, completion status, and QA evidence. +5. Treat a lower-cost run as acceptable only when completion and QA evidence remain equivalent. + +## Rollout And Rollback + +Roll out one workspace at a time and inspect helper-group telemetry after representative sprints. If helper quality regresses, raise only the affected helper's reasoning level or model before changing public roles. + +To restore the legacy helper behavior, set each helper to the current orchestrator model and reasoning level. To relax workflow limits during an incident, set explicit higher integers in the workspace policy and restart the orchestrator. Keep a finite bound so a malformed workflow cannot reopen indefinitely. + +Do not compare runs whose model maps or source configuration hashes differ in unplanned ways. The benchmark report now exposes those differences for this reason. diff --git a/docs/configuration_guide.md b/docs/configuration_guide.md index 8d8218d..e9cd387 100644 --- a/docs/configuration_guide.md +++ b/docs/configuration_guide.md @@ -182,6 +182,45 @@ role_defaults: After changing a running role's config, restart that role to apply the new settings. +### `internal_agent_defaults` + +Configure the orchestrator-local `parser`, `sourcer`, and `version_controller` independently from public roles: + +```yaml +internal_agent_defaults: + parser: + model: "gpt-5.4-mini" + reasoning: "low" + sourcer: + model: "gpt-5.4-mini" + reasoning: "medium" + version_controller: + model: "gpt-5.4-mini" + reasoning: "low" +``` + +Use the CLI for one helper override: + +```bash +python -m teams_runtime config internal set --agent parser --model gpt-5.4-mini --reasoning low +``` + +If this section or one helper entry is absent, that helper inherits the effective `role_defaults.orchestrator` model and reasoning level. This keeps existing workspaces backward compatible. Restart the orchestrator after a helper change because all three helpers run inside that service. + +See [`docs/call_amplification_controls.md`](call_amplification_controls.md) for tier rationale, measurement, migration, and rollback details. + +### Workflow Budgets + +The generated orchestrator skill policy at `orchestrator/.agents/skills/agent_utilization/policy.yaml` controls implementation amplification: + +```yaml +workflow_contract: + implementation_review_cycle_limit: 3 + implementation_reopen_limit: 3 +``` + +The review limit counts routes into architect implementation review. The reopen limit independently counts accepted orchestrator-governed reopen transitions. On reaching either applicable limit, the workflow blocks before another revision handoff. Both values must be positive integers, and workspace policy values override the bundled defaults. Restart orchestrator after a policy change; existing in-flight requests retain their persisted limits, while new internal requests receive the updated values. + ### `actions` Defines which `execute` actions are allowed. @@ -238,6 +277,82 @@ Public service runtimes still use role-name identities such as `planner`, so the Many `backlog_id` and `request_id` values may exist while one configured sprint session scope remains active. +### `prompt_context` + +Controls how much persisted request event history is copied into model prompts: + +```yaml +prompt_context: + enabled: true + recent_events: 8 + max_events: 16 +``` + +Defaults: + +- `enabled: true` +- `recent_events: 8` +- `max_events: 16` + +Both event limits must be positive integers, and `max_events` must be greater than or equal to `recent_events`. Role services load these values at startup, so restart them after changing this section. There is no CLI mutation command for this policy. + +Compaction changes only the prompt projection. The canonical request JSON under `.teams_runtime/requests/` retains every event, and the current request metadata, result, artifacts, and selected event payloads are not truncated or summarized. + +#### Compaction And Backfill + +**Compaction** is executed only when `enabled` is `true` and the request contains more than `max_events` events. The runtime: + +1. Includes the last `recent_events` entries by list position, regardless of event shape. +2. Treats roles represented in that recent tail as already covered. +3. Scans older events from newest to oldest. +4. Backfills the newest evidence for each role not already represented, stopping at `max_events`. +5. Restores the selected events to their original chronological order. + +**Backfill** means using remaining capacity to retain older role evidence that would otherwise disappear behind the recent-event boundary. An older event qualifies as role evidence when either `type` or legacy `event_type` is `role_report`, or when its `payload` contains non-empty `role` and `status` fields. The role identity comes from `payload.role`, falling back to the event `actor`. + +Backfill is not a summary, database repair, or write to history. It copies complete existing event objects into the prompt. Repeated reports for one role do not consume multiple backfill slots; the newest qualifying older report wins. If `max_events` equals `recent_events`, no backfill capacity exists and the projection is recent-only. + +#### Worked Example + +With `recent_events: 4` and `max_events: 7`, suppose the persisted request has this abbreviated event list: + +```json +[ + {"timestamp": "T01", "type": "created", "actor": "orchestrator"}, + {"timestamp": "T02", "type": "role_report", "payload": {"role": "research", "status": "completed"}}, + {"timestamp": "T03", "type": "delegated", "actor": "orchestrator"}, + {"timestamp": "T04", "type": "role_report", "payload": {"role": "planner", "status": "completed", "summary": "initial plan"}}, + {"timestamp": "T05", "type": "role_report", "payload": {"role": "designer", "status": "completed"}}, + {"timestamp": "T06", "type": "role_report", "payload": {"role": "planner", "status": "completed", "summary": "final plan"}}, + {"timestamp": "T07", "type": "retried", "actor": "orchestrator"}, + {"timestamp": "T08", "type": "role_report", "payload": {"role": "developer", "status": "completed"}}, + {"timestamp": "T09", "type": "role_report", "payload": {"role": "architect", "status": "completed"}}, + {"timestamp": "T10", "type": "delegated", "actor": "orchestrator"}, + {"timestamp": "T11", "type": "role_report", "payload": {"role": "qa", "status": "blocked"}}, + {"timestamp": "T12", "type": "resumed", "actor": "orchestrator"} +] +``` + +The recent tail is `T09` through `T12`, representing `architect` and `qa`. Three slots remain. Scanning backward selects `T08` for `developer`, `T06` for `planner`, and `T05` for `designer`. `T04` is skipped because the newer planner report already represents that role. Capacity is then full, so older `research` evidence at `T02` is not selected. + +The prompt receives: + +```json +[ + {"timestamp": "T05", "type": "role_report", "payload": {"role": "designer", "status": "completed"}}, + {"timestamp": "T06", "type": "role_report", "payload": {"role": "planner", "status": "completed", "summary": "final plan"}}, + {"timestamp": "T08", "type": "role_report", "payload": {"role": "developer", "status": "completed"}}, + {"timestamp": "T09", "type": "role_report", "payload": {"role": "architect", "status": "completed"}}, + {"timestamp": "T10", "type": "delegated", "actor": "orchestrator"}, + {"timestamp": "T11", "type": "role_report", "payload": {"role": "qa", "status": "blocked"}}, + {"timestamp": "T12", "type": "resumed", "actor": "orchestrator"} +] +``` + +The adjacent prompt notice reports `total_events: 12`, `included_events: 7`, `omitted_events: 5`, the selection policy, and the canonical request path. A role may open that canonical file when its current decision needs omitted evidence. + +For immediate rollback, set `enabled: false` and restart role services. This restores full event-history inclusion in prompts without changing persisted request data. + ## Changing `sprint.id` To rotate the configured sprint session scope: diff --git a/docs/implementation.md b/docs/implementation.md index 3ae5860..bf4fc7f 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -136,6 +136,8 @@ Current ownership notes: - canonical shared contracts: role/session config dataclasses plus typed request/backlog/sprint/workflow/result shapes - `teams_runtime/shared/config.py` - canonical runtime and Discord config loading, validation, placeholder-ID guardrails, and runtime config mutation helpers +- `teams_runtime/shared/prompt_context.py` + - canonical pure request-event projection, role-evidence backfill selection, and compacted-history prompt notice rendering - `teams_runtime/shared/paths.py` - canonical `RuntimePaths` workspace, runtime state, log, role, shared workspace, sprint artifact, and archive path contract - `teams_runtime/shared/persistence.py` @@ -275,6 +277,7 @@ Current ownership notes: - Shared contracts now belong in `shared/models.py`. - Canonical report/progress formatting helpers, backlog item construction, backlog markdown rendering, and current-sprint markdown rendering belong in `shared/formatting.py`; `core/reports.py` remains compatibility-only and `core/sprints.py` re-exports moved formatting helpers. - Canonical runtime and Discord config loading/updating belongs in `shared/config.py`; `core/config.py` remains compatibility-only. +- Canonical model-facing request event-history projection and backfill selection belongs in `shared/prompt_context.py`; request persistence remains unchanged. - Canonical workspace/runtime path helpers belong in `shared/paths.py`; `core/paths.py` remains compatibility-only. - Canonical shared JSON/JSONL persistence, ID/fingerprint generation, and KST timestamp helpers belong in `shared/persistence.py`; `core/persistence.py` remains compatibility-only. - Canonical request/backlog/sprint/goal file IO, event helpers, planner-review request predicates/lookups/record assembly, internal sprint request predicates/iteration, blocked-backlog review candidate normalization/rendering, non-actionable backlog classification/drop/repair, backlog status/blocker/todo-state helpers, sprint selected-backlog view derivation, backlog kind/acceptance normalization, goal lifecycle reports, and backlog status-report context helpers belong in `workflows/state/request_store.py`, `workflows/state/backlog_store.py`, `workflows/state/sprint_store.py`, and `workflows/state/goal_store.py`; `core/*_store.py` remains compatibility-only. diff --git a/docs/operations_guide.md b/docs/operations_guide.md index ece5a32..94b4813 100644 --- a/docs/operations_guide.md +++ b/docs/operations_guide.md @@ -102,6 +102,7 @@ python -m teams_runtime goal cancel - Sprint backlog definition items should carry concrete `acceptance_criteria` plus planner trace in `origin.milestone_ref`, `origin.requirement_refs`, `origin.spec_refs`, `origin.plan_action_refs`, and `origin.research_refs` when a source-backed or local-evidence research report is available. - During an active sprint, clear new user requirements are stored as sprint-local `REQ-CAND-*` entries in `pending_requirement_candidates`. They are not accepted scope, not acknowledged as registered requirements, and not included in role context until planner reaches the next completed/committed TODO checkpoint. - At that checkpoint, planner receives pending candidates in an `ongoing_review` request and may return `proposals.sprint_requirement_reconciliation` with `registered_requirements`, `merged_candidates`, `deferred_candidates`, and `rejected_candidates`. Only registered candidates become `REQ-*`; unresolved candidates expire into `requirement_candidate_archive` at sprint closeout. +- Manual sprints batch all currently pending candidates into that single checkpoint review. A completed or committed TODO does not force an `ongoing_review` when no valid pending candidates exist; interval-based planner reviews continue to use `sprint.interval_minutes`. - Legacy planner aliases such as `planned_backlog_updates` are compatibility inputs only inside role-runtime normalization. They are not accepted by the canonical backlog helper interface. ## Sprint Requirement Feedback @@ -278,17 +279,19 @@ params: {"_teams_kind":"delegate"} ### 2. Persisted request record -The target role also receives the full persisted request record through its prompt context. +The target role receives a prompt projection of the persisted request record. Request metadata, current status, reply route, artifacts, and the most recent role result remain available. Event history is included in full while it is within the configured limit; longer histories use the bounded `prompt_context` compaction policy. That record contains: - request metadata - current status - reply route -- event history +- full or compacted event history - most recent role result -So later roles can see earlier role output even though the relay message itself stays small. +Compaction always keeps the configured recent tail, then uses remaining capacity to backfill the newest older evidence for roles missing from that tail. Here, backfill means selecting complete historical event objects; it does not summarize or modify them. An adjacent notice gives the total, included, and omitted counts plus the canonical `.teams_runtime/requests/.json` path. The persisted file remains complete and can be inspected when omitted evidence is required. + +This keeps normal later-role prompts bounded while preserving a path to the complete audit history. See [`configuration_guide.md`](./configuration_guide.md#prompt_context) for the exact algorithm and worked input/output example. ## Request Examples diff --git a/docs/performance_benchmarking.md b/docs/performance_benchmarking.md new file mode 100644 index 0000000..1390152 --- /dev/null +++ b/docs/performance_benchmarking.md @@ -0,0 +1,503 @@ +# Full-Sprint Performance Benchmarking + +This guide defines the repeatable integration benchmark used to quantify model-call +cost and performance changes in `teams_runtime`. It is intentionally separate from +the README because it describes an operator-only live experiment, not normal runtime +startup. + +## Objective + +The benchmark answers a narrow question: + +> For the same complete sprint, how do call count, duration, prompt size, native token +> usage, and optional estimated cost change when request-event prompt compaction is +> enabled? + +The first experiment targets the optimization merged in PR #5. Both arms run the same +merged source revision and deployed role/model configuration: + +| Arm | `prompt_context.enabled` | Event projection | +| --- | --- | --- | +| Before | `false` | The complete request `events` array is embedded in each applicable prompt. | +| After | `true` | At most 16 events are embedded: the latest 8 events plus the latest older evidence for roles not represented in that tail. | + +This is a feature-toggle comparison, not a comparison between two Git commits. Keeping +the source revision fixed removes unrelated code changes from the experiment. + +The default run is one paired smoke test. A single pair can reveal large regressions +and validate measurement coverage, but it is not statistically significant. Provider +routing, tool use, cache state, and model behavior are nondeterministic, so end-to-end +deltas are not attributable to compaction alone. Use repeated pairs before making a +capacity or budget commitment. + +## Full-Sprint Scenario + +Each arm receives a fresh, isolated, no-remote Git repository. The benchmark scaffolds +the normal team workspace and adds a deliberately defective Python function: + +```python +def sum_positive(values): + return sum(values) +``` + +The protected functional examples are: + +```python +assert sum_positive([5, -8, 2]) == 7 +assert sum_positive([-5, 0, -3]) == 0 +assert sum_positive([]) == 0 +``` + +The trusted fixture's initial test run must fail. The sprint milestone asks the team +to preserve the public function, fix the behavior, run the `unittest` suite, leave +protected benchmark files unchanged, and commit the result. To keep parent-side final +inspection from importing or executing model-modified Python, the accepted repair has +one deliberately narrow AST shape: + +```python +def sum_positive(values): + return sum(value for value in values if value > 0) +``` + +Comments, whitespace, and an optional string docstring at module and function scope +do not affect acceptance. The module must otherwise contain only that synchronous +function. The function must have exactly the unannotated `values` parameter, no +decorators or defaults, and exactly the shown generator-expression return as its only +non-docstring statement. Imports, additional definitions or statements, annotations, +list comprehensions, renamed operands, and alternate predicates are rejected even if +they could produce the same outputs. This intentionally trades general semantic +equivalence for a deterministic, non-executing oracle. The complete production sprint +workflow is still used: planning, explicit benchmark auto-confirmation, backlog/TODO +execution, governed role handoffs, QA, version control, and closeout. + +An arm passes its behavior and workflow gates only when all of the following are true: + +- the initial defective fixture was reproduced before the arm started +- the final constrained AST behavior oracle passes without importing or executing + workspace code +- protected scenario and test files are regular files reached without following + symlinks and have the same SHA-256 hashes +- the sprint reaches a terminal completed state +- closeout is verified +- no TODO is blocked or failed +- at least one task commit exists +- the final Git worktree is clean +- the isolated repository still has no Git remotes +- every persisted invocation has native token usage +- the call journal is present, uses a supported schema, and reconciles every + reservation to exactly one terminal attempt +- a completed primary call in the Before arm records exactly `50 total / 50 + included / 0 omitted` events +- a completed primary call in the After arm records exactly `50 total / 16 + included / 34 omitted` events +- the Before and After non-feature configuration fingerprints match + +## Backfill + +In this benchmark, **Backfill** means adding a deterministic history prefix to the +canonical initial sprint-planning request before it is relayed to research or +planner. The same canonical request, including the prefix, is used by later routing +steps. It does not mean importing production telemetry, replaying customer data, or +modifying an existing workspace. + +The benchmark generates 48 content-safe historical events. They contain neutral +checkpoints and one evidence checkpoint for each workflow role: + +```json +[ + { + "created_at": "2026-01-01T00:01:00+00:00", + "type": "role_report", + "actor": "research", + "summary": "Historical research checkpoint 02.", + "payload": { + "role": "research", + "status": "completed", + "summary": "Stable benchmark evidence 02." + } + }, + { + "created_at": "2026-01-01T00:02:00+00:00", + "type": "benchmark_checkpoint", + "actor": "orchestrator", + "summary": "Neutral historical checkpoint 03.", + "payload": {"sequence": 3} + } +] +``` + +The full 48-event sequence and its canonical SHA-256 hash are identical in both arms. +It is prepended exactly once, before the initial planning request's normal `created` +and `delegated` events. The benchmark persists and hash-verifies the prefix before +the first provider call. This produces a realistic long-history prompt from the +beginning of the sprint without introducing facts that could change the desired +implementation. + +Backfill serves three purposes: + +1. It guarantees that the request exceeds the 16-event compaction threshold. +2. It gives the selector older evidence from every role to preserve. +3. It makes prompt-size and token deltas reproducible across paired runs. + +The generated history is saved as `.benchmark/history_seed.json` inside a retained, +sanitized arm snapshot. Reports store only its hash and counts. The benchmark never +backfills the normal telemetry store with fabricated model invocations. + +## Compaction + +Compaction is executed immediately before an applicable role prompt is built. The +canonical request JSON on disk remains complete. Only the request projection embedded +in the model prompt changes. + +The selection policy is +`recent_tail_plus_latest_role_evidence`: + +1. If compaction is disabled, or the request has at most `max_events`, include every + event. +2. Select the final `recent_events` events. The benchmark uses 8. +3. Record which roles already have evidence in that recent tail. +4. Scan older events from newest to oldest. +5. For each role not yet represented, include its most recent `role_report` evidence. +6. Stop when every available role is represented or `max_events` is reached. The + benchmark uses 16. +7. Restore selected events to chronological order and embed their complete objects, + not summaries. +8. Add a projection notice with total, included, and omitted counts plus the path to + the complete canonical request. + +For example, the first relayed planning request normally has the 48-event Backfill +followed by its `created` and `delegated` events. Before compaction, the prompt input +contains: + +```json +{ + "request_id": "req-example", + "events": [ + {"type": "role_report", "payload": {"role": "research", "status": "completed"}}, + "... 47 additional deterministic historical events ...", + {"type": "created", "actor": "sprint_runner"}, + {"type": "delegated", "actor": "orchestrator"} + ] +} +``` + +The Before arm embeds all 50 events. The After arm's projection contains the latest +8 events and the most recent older evidence for up to 8 missing roles: + +```json +{ + "compacted": true, + "total_events": 50, + "included_events": 16, + "omitted_events": 34, + "recent_events": 8, + "max_events": 16, + "selection": "recent_tail_plus_latest_role_evidence", + "canonical_request": "./.teams_runtime/requests/req-example.json" +} +``` + +The projected `events` array contains 16 complete event objects in chronological +order. The other 34 events still exist in the canonical request. The prompt tells the +role to open that file only when a decision requires evidence missing from the +projection. + +Telemetry records the projection policy and counts on the physical provider attempt. +This proves that compaction was eligible and executed; token reduction by itself is +not accepted as proof. Pair comparability requires at least one completed primary +provider attempt for `research / research_decision / research_initial` with the exact +v2 projection in each arm. Before provider launch, the runtime writes that invocation +identity and projection tuple to the private call journal outside the model-writable +workspace. During execution, every runtime recorder writes raw usage shards to a +second parent-owned directory under the arm report path, also outside that workspace. +After child and provider cleanup, the parent loads and removes those raw shards; it +does not read the normal workspace telemetry store or trust telemetry carried in the +child result. Parent-side reconciliation requires the journal and telemetry values to +match by invocation ID. The reducer also requires the SHA-256 digest of its exact +target invocation-ID set to match the journal-verified set, so an equal count from a +different call cannot substitute for the target. Only reconciled evidence becomes +`metrics.compaction.target_projection_count`; an exact-shaped telemetry record alone +is only an untrusted candidate and cannot satisfy the gate. `metrics.json`, +`run.json`, and `report.json` retain the gate result without relying on manual prompt +or response inspection. The exact paths are `compaction.target_projection_count` in +`metrics.json`, `metrics.compaction.target_projection_count` in `run.json`, and +`runs[].metrics.compaction.target_projection_count` in `report.json`. + +## Running The Benchmark + +Run from the `teams_runtime` source checkout: + +```bash +TEAMS_RUNTIME_LIVE_BENCHMARK=1 PYTHONPATH=.. \ +python -m teams_runtime benchmark sprint-ab \ + --live \ + --runtime-config ../teams_generated/team_runtime.yaml \ + --repetitions 1 \ + --max-invocations 20 \ + --call-timeout-seconds 300 \ + --run-timeout-seconds 1800 \ + --keep-workspaces failures +``` + +Live calls require both the environment variable and `--live`. Omitting either is a +preflight failure and makes no model call. + +Both timeout values must be positive, finite numbers. Zero, negative values, `NaN`, +and positive or negative infinity are rejected before a worker or provider process is +launched; a non-finite run timeout is never allowed to disable the hard deadline. + +The provider does not inherit the operator's `HOME`, `CODEX_HOME`, or temporary +directories. It uses private paths under the arm's ignored `.teams_runtime` state. +Provide authentication through a supported provider-only environment variable such as +`CODEX_API_KEY` or `OPENAI_API_KEY`; stored credentials from the operator's normal +Codex home are intentionally unavailable. Missing provider-only authentication fails +preflight before a call is reserved. These authentication variables are not injected +into model tool shells. Use a rate card matching the selected credential and backend. + +The worker also constructs a benchmark-only `PATH`. It keeps only existing absolute +directories outside the provider-writable workspace, protected run output, and system +temporary roots. It resolves `codex` once through that path, canonicalizes symlinks, +verifies that the target is a regular executable, and pins that absolute path for CLI +version discovery and every provider launch. A workspace-local `codex` shim therefore +cannot replace the measured provider executable between arms. + +Options: + +| Option | Default | Meaning | +| --- | ---: | --- | +| `--runtime-config PATH` | required | Deployed `team_runtime.yaml` or the workspace directory containing it. | +| `--repetitions N` | `1` | Number of paired experiments. Pair 1 runs Before/After; pair 2 runs After/Before, then order alternates. | +| `--max-invocations N` | `20` | Hard physical model-call cap for each arm, including retries and contract repairs. | +| `--call-timeout-seconds N` | `300` | Hard timeout for one provider process group. | +| `--run-timeout-seconds N` | `1800` | Hard wall-clock timeout for one complete arm. | +| `--keep-workspaces MODE` | `failures` | `none`, `failures`, or `all`. | +| `--rate-card-file PATH` | unset | Optional YAML pricing snapshot. | +| `--output-dir PATH` | `.teams_runtime/benchmarks` | Parent directory for benchmark artifacts. | +| `--benchmark-id ID` | generated | Stable 1-96 character artifact directory name. | +| `--allow-dirty-source` | false | Permit a dirty checkout and record its content-free state hash. | +| `--json` | false | Print a machine-readable completion summary. | + +The runner copies effective public `role_defaults` and internal `internal_agent_defaults` into every arm. For a legacy config without helper overrides, parser, sourcer, and version-controller inherit the orchestrator tier. The effective values participate in `source_config_hash` and appear in `provenance.runtime_model_map`, so runs with different helper tiers are not silently treated as the same deployed model map. + +Exit codes: + +| Code | Meaning | +| ---: | --- | +| `0` | Every pair passed quality and comparability gates. | +| `1` | Artifacts were preserved, but at least one pair is partial, inconclusive, or stopped at an arm-level preflight. | +| `2` | CLI usage, top-level preflight failure, or fatal cleanup-safety abort; no complete benchmark report is available. | + +## Execution Safety + +The live worker is deliberately narrower than normal runtime execution: + +- every role uses the internal file relay; no Discord listener or message is used +- sprint GitHub issue publication is replaced with a local `skipped_benchmark` record +- external deep research is disabled and remains an unresolved risk if requested +- the fixture repository has no remote; final inspection uses the external Git + executable resolved before model access, disables prompts and paging, ignores + system/global configuration, and overrides repository-controlled hooks, fsmonitor, + external diff, diff filters, and automatic maintenance; status reads attributes + from the pre-model seed commit and requires the repository-local attributes override + to remain empty, preventing later clean-filter commands from running +- final source verification reads size-bounded regular files through directory-anchored, + no-follow descriptors and parses `benchmark_app.py` as AST only; it never imports or + executes model-controlled workspace code +- benchmark model execution supports Codex only; Gemini CLI requests are rejected +- approval policy is `never` and sandbox mode is `workspace-write` +- the workspace-write sandbox explicitly excludes `/tmp` and the `TMPDIR` path, + so report output beneath a system temporary directory remains model-read-only +- relative, missing, provider-writable, report-owned, and system-temporary entries are + removed from the benchmark `PATH`; the resolved Codex executable is pinned outside + those roots before any call is reserved +- dangerous sandbox-bypass requests and automatic bypass retries are rejected +- MCP servers, web search, plugins, hooks, computer use, and multi-agent features are + disabled for the provider process +- the provider's `HOME`, `CODEX_HOME`, and temporary directories resolve inside private + ignored benchmark state rather than the operator's home or system temporary directory +- the provider receives only the authentication and transport variables needed by the + outer Codex CLI +- tool shells inherit no outer environment and receive an explicit non-secret + allowlist +- writable paths must resolve inside the isolated arm root +- a shared atomic journal reserves a call before launch, so concurrent roles cannot + exceed the arm's physical-call budget +- journal schema v3 stores content-free invocation identity and prompt-projection + counts before launch outside the model-writable workspace; parent reconciliation + rejects changed, unrelated, unmatched, or internally conflicting telemetry evidence +- raw benchmark telemetry is written to a parent-owned arm directory outside the + provider-writable workspace, consumed only after process cleanup, and then removed; + workspace telemetry and child-result telemetry are never accepted as evidence +- a benchmark-only launcher waits on a private pipe and executes the provider only + after its PID and process group are durably journaled; parent death closes the + pipe and exits the launcher without starting the provider +- provider processes run in dedicated process groups and are terminated on call + timeout +- an arm timeout terminates active provider groups before the worker is stopped +- benchmark Codex commands do not use `-o`/`--output-last-message`; the final message + is parsed from the CLI's JSONL stdout, so the unsandboxed outer CLI is never asked to + follow a model-created output-file symlink + +When a call cap or timeout is reached, the benchmark does not silently raise the +limit. It preserves partial telemetry, marks the arm inconclusive, and proceeds to the +other arm when doing so remains safe. + +If provider cleanup cannot be proven, the runner aborts instead of producing a +normal inconclusive result. The CLI returns `2`; a partially created private output +directory may remain for diagnosis, but automation must not assume `report.json` or +`report.md` exists. + +Live benchmarks are never run automatically in CI. Deterministic fake-worker +integration tests exercise scheduling, fixtures, aggregation, reporting, and failure +paths without credentials or provider calls. + +## Reports + +`report.json` uses report schema v3. This version identifies the v2 scenario and +changes comparability semantics to require journal-v3 target-context reconciliation. + +Artifacts are written under: + +```text +.teams_runtime/benchmarks// +├── report.json +├── report.md +├── runs/ +│ ├── pair-001-before/ +│ │ ├── run.json +│ │ ├── metrics.json +│ │ ├── model_invocations.jsonl +│ │ ├── sprint.json +│ │ ├── quality.json +│ │ ├── call_journal.json +│ │ └── worker.log +│ └── pair-001-after/ +└── workspaces/ + └── ... sanitized snapshots retained according to --keep-workspaces +``` + +The execution report includes: + +- scenario identity and the exact v2 target projection for both arms +- journal-reserved, telemetry-observed, completed, failed, timed-out, + launch-failed, parent-terminated, active, and rejected attempt counts +- physical telemetry invocation and logical-call counts +- primary, contract-repair, sandbox-retry, failed, and completed counts +- tool-call count and coverage +- provider and end-to-end wall duration, including p50 and p95 provider latency +- prompt and output character counts +- input, cached input, uncached input, output, reasoning-output, and total tokens +- native-token coverage +- optional estimated cost and pricing coverage +- compaction eligibility, executions, total/included/omitted events, maximum + included events, untrusted target candidates, journal/telemetry mismatches, and + exact-target journal-verified completed-primary evidence count +- per-role/provider/model groups +- matched primary calls keyed by role, purpose, workflow step, and occurrence +- full-sprint Before/After deltas and reduction percentages + +Provider usage is reported only for telemetry-observed calls. Native-token, +tool-call, and pricing coverage use journal-reserved attempts as their denominator. +If the hard arm deadline terminates a provider before telemetry can be finalized, +the call journal records a terminal `terminated` attempt while its token usage +remains unmeasured. Reports show both counts, leave aggregate and per-group costs +unpriced, and never infer tokens for that attempt. If the journal is absent, +unsupported, unreconciled, incomplete, or has more telemetry records than +reservations, all coverage percentages and cost totals remain unknown. Coverage +also requires unique, nonempty telemetry invocation IDs that match a subset of the +journaled invocation IDs. Reports persist only bounded mismatch counts and a +SHA-256 identity digest, not the journal's raw identity list. + +Reports are content-safe: they do not contain prompts, model responses, tool output, +raw errors, raw session IDs, credentials, or environment values. +The persisted `model_invocations.jsonl` file is a sanitized report artifact produced +from the consumed private shards; the raw private shard directory is not retained. + +### Optional Rate Card + +Pricing is operator-supplied and never fetched automatically. A rate-card file can +use a top-level `rate_cards` mapping: + +```yaml +rate_cards: + "codex_cli/gpt-5.5": + input_per_million_usd: 0.00 + cached_input_per_million_usd: 0.00 + output_per_million_usd: 0.00 + "codex_cli/gpt-5.3-codex-spark": + input_per_million_usd: 0.00 + cached_input_per_million_usd: 0.00 + output_per_million_usd: 0.00 +``` + +Replace zero placeholders with the pricing agreement applicable to the deployment. +Cost is reported only when every invocation in both arms has a matching rate and the +native usage required by that rate. Otherwise cost remains `null`/`unpriced`; partial +coverage is never presented as a complete total. + +## Interpreting Results + +Prioritize evidence in this order: + +1. Verify both arms passed all quality and comparability gates. +2. Verify the same history hash and non-feature configuration hash were used. +3. Verify 100% native token coverage. +4. Verify both arms have a positive `target_projection_count`: Before proves + journal-reconciled `50/50/0`, and After proves journal-reconciled `50/16/34`. +5. Compare matched primary calls to isolate prompts that exercised the same role, + purpose, and workflow step. +6. Compare full-sprint totals to capture routing, retries, and downstream effects. +7. Treat cost as an estimate only when pricing coverage is 100%. + +A one-pair report is labeled `preliminary_smoke`; its sample standard deviation is +`null`. For a stronger estimate, rerun with at least three pairs. Execution order +alternates automatically to reduce a simple warm-cache or time-order bias. Do not +combine reports from different source revisions, deployed model maps, history hashes, +or rate-card snapshots. + +## Troubleshooting + +`preflight_failed`: + +- confirm both live opt-ins are present +- provide `CODEX_API_KEY` or `OPENAI_API_KEY`; an arm-level authentication preflight + produces an inconclusive report with exit code `1` and zero reserved calls +- confirm `codex` resolves to a regular executable through an absolute external + `PATH` directory, not the arm workspace, report output, or a temporary directory +- confirm the runtime config exists and defines every role model/reasoning pair +- commit source changes, or deliberately use `--allow-dirty-source` +- choose a new benchmark ID or remove only an explicitly disposable old output + +`call_budget_exhausted`: + +- inspect `call_journal.json` and the role/purpose aggregates +- do not increase the cap to make an individual result appear comparable +- simplify the fixture only by creating a new scenario version, or deliberately + schedule a separately documented higher-budget experiment + +`timeout`: + +- inspect `worker.log`, terminal journal entries, and partial invocation telemetry +- distinguish a single provider timeout from the full-arm timeout +- rerun the unchanged pair after a transient provider incident; do not merge a + replacement arm into the old pair + +`native_token_coverage_incomplete`: + +- confirm the installed Codex CLI emits terminal usage in JSON mode +- retain the result as evidence of a measurement gap +- do not substitute character-count estimates for native tokens in a comparable pair + +`after_compaction_not_observed`: + +- verify the deterministic history hash +- inspect the sanitized `model_invocations.jsonl` projection metadata in the arm report +- treat the pair as inconclusive even if input tokens decreased + +Retained paths are sanitized, allowlisted snapshots rather than execution workspaces. +They include only pre-execution benchmark metadata, deterministic fixture/test files, +the baseline `benchmark_app.py`, the benchmark task, and the arm configuration. The +post-execution implementation is represented only by a SHA-256 digest, byte count, and +changed/not-changed flag. Git metadata, `.teams_runtime`, logs, model sessions, provider +output, role workspaces, model-mutated file content, and all unrecognized files are +excluded. Remove snapshots according to the project's normal retention policy. diff --git a/docs/specification.md b/docs/specification.md index f610af6..5101a7c 100644 --- a/docs/specification.md +++ b/docs/specification.md @@ -151,8 +151,27 @@ Supported sections: - `mentions` - `allowed_guild_ids` - `role_defaults` +- `prompt_context` + - `enabled` + - `recent_events` + - `max_events` +- `telemetry` + - `enabled` + - `rate_cards` - `actions` +### Prompt event-history projection + +- persisted request records remain the canonical, complete audit history +- model-facing normal role, role-result repair, research-decision, and version-controller prompts share one event projection policy +- when enabled and event count exceeds `max_events`, the projection always includes the final `recent_events` entries +- remaining capacity is backfilled by scanning older events newest-first and selecting the newest evidence for each role not represented in the recent tail +- qualifying evidence is a `role_report` in `type` or `event_type`, or an event whose payload contains non-empty `role` and `status`; identity uses `payload.role` then `actor` +- selected events retain their complete payload and original chronological order +- prompt compaction never mutates the persisted request record +- compacted prompts include counts, policy name, and canonical request path so omitted history remains discoverable +- disabling `prompt_context` restores full event-history prompt inclusion after role services restart + ## Backlog And Sprint Model Normal change and enhancement requests are backlog-first. diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..4becde1 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,557 @@ +# `teams_runtime` Telemetry Guide + +This guide describes the local model telemetry system used to measure model-call cost, latency, retries, and context growth in `teams_runtime`. + +## Purpose + +`teams_runtime` coordinates several model-backed roles. A single user request, goal, or sprint todo may invoke the orchestrator, research, planner, architect, developer, QA, version controller, parser, and goal sourcer. Contract repair and workflow reopening can add more calls. + +Before telemetry, runtime logs showed that a role started and completed, but they did not provide a reliable answer to questions such as: + +- which role consumed the most input tokens +- which workflow step dominated latency +- whether session reuse produced meaningful cached-input usage +- how many provider calls were contract repairs or sandbox retries +- which request or sprint caused model-call amplification +- what portion of activity could be assigned a monetary estimate + +Telemetry supplies that baseline without changing workflow policy. It is intentionally the first optimization step: later model-tier, prompt-compaction, retry, and batching changes can be evaluated against measured behavior instead of static assumptions. + +## Goals And Non-Goals + +Goals: + +- record one event for every real provider attempt +- correlate calls with runtime, role, request, sprint, todo, backlog item, and workflow step +- capture native token usage when a provider exposes it +- measure prompt size, output size, latency, failures, retries, and repairs +- calculate optional cost estimates from operator-supplied rate cards +- remain local, append-only, inspectable, and safe when partially corrupted +- avoid changing the result or availability of model execution + +Non-goals: + +- sending telemetry to a hosted collector +- reconciling estimates with a provider invoice or subscription allowance +- storing prompts, responses, tool output, or raw errors +- changing models, reasoning levels, prompts, workflow transitions, or retry budgets +- adding a dashboard, database, automatic retention, or historical backfill + +## Data Flow + +```mermaid +flowchart LR + R[Role runtime] --> C[Invocation context] + C --> P[Codex or Gemini CLI] + C --> D[Deep research] + P --> O[Normal role result handling] + D --> O + P --> N[Usage normalization] + D --> N + N --> J[Daily per-process JSONL shard] + J --> A[Streaming aggregator] + A --> H[Human metrics report] + A --> M[Versioned JSON report] +``` + +Normal result handling and telemetry are separate. A telemetry write failure logs a warning but does not fail, retry, or modify the role result. + +## Configuration + +Telemetry is configured in `team_runtime.yaml`: + +```yaml +telemetry: + enabled: true + rate_cards: {} +``` + +The block is optional. Existing workspaces that do not contain it behave as follows: + +- telemetry is enabled +- token and latency metrics are recorded +- monetary estimates are unavailable until a rate card is configured + +Role services load this configuration at startup. Restart affected services after changing it. + +### Disable Telemetry + +```yaml +telemetry: + enabled: false + rate_cards: {} +``` + +Disabled telemetry does not create invocation records or perform provider-version discovery. It does not delete records that already exist. + +### Token Rate Cards + +Rate-card keys use the exact `provider/model` identifier recorded by telemetry: + +```yaml +telemetry: + enabled: true + rate_cards: + "codex_cli/gpt-5.5": + input_per_million_usd: 0.00 + cached_input_per_million_usd: 0.00 + output_per_million_usd: 0.00 +``` + +Required token fields: + +- `input_per_million_usd` +- `output_per_million_usd` + +Optional token field: + +- `cached_input_per_million_usd`, which defaults to the normal input rate + +The example deliberately uses placeholder zero values. Supply the rate that matches the deployment's billing arrangement. `teams_runtime` does not fetch or hardcode current provider prices. + +### Flat Rate Cards + +External operations without native token usage may use a per-invocation estimate: + +```yaml +telemetry: + enabled: true + rate_cards: + "gemini_deep_research/default": + per_invocation_usd: 0.00 +``` + +A flat rate is applied to every attempted invocation, including a failed attempt. This is an explicit accounting assumption made by the operator. + +### Validation + +Configuration loading rejects: + +- a non-Boolean `enabled` value +- a non-mapping `rate_cards` value +- keys without the `provider/model` form +- negative or non-finite rates +- token cards missing input or output rates +- cards that mix token and flat pricing +- empty cards without a pricing method + +Invalid telemetry configuration prevents service startup in the same way as other invalid runtime policy. + +## Provider Coverage + +### Codex CLI + +Codex execution uses `--json` together with `--output-last-message`. + +The final-message file remains the authoritative role response. JSONL standard output is used only for: + +- thread or session identity +- terminal token usage +- final-message recovery when the output file is unavailable + +Telemetry normalizes: + +- input tokens +- cached-input tokens +- output tokens +- reasoning-output tokens +- total tokens + +Unknown events and malformed JSONL lines are ignored. If a terminal usage event is absent, the role result still completes and telemetry records `usage_source=unavailable`. + +### Gemini CLI + +Gemini continues to use `--output-format json`. Its response and session ID are processed as before. The `stats.models` payload is normalized across every reported model: + +| Gemini statistic | Telemetry field | +|---|---| +| `tokens.prompt` | `input_tokens` | +| `tokens.cached` | `cached_input_tokens` | +| `tokens.candidates` | `output_tokens` | +| `tokens.thoughts` | `reasoning_output_tokens` | +| `tokens.total` | `total_tokens` | +| `tools.totalCalls` | `tool_calls` | + +The raw statistics object is not persisted. + +### Deep Research + +The external deep-research library does not expose reliable token usage. Telemetry records: + +- attempt status +- elapsed time +- prompt characters +- response characters +- configured app and reasoning mode + +Token fields remain null. A flat rate can provide an optional accounting estimate. + +A provider result that completed successfully remains a completed invocation even if later artifact writing or research-report validation fails. Those are downstream workflow failures rather than provider failures. + +## Invocation Lifecycle + +Telemetry uses three correlation levels. + +### Operation + +An operation is one high-level role action, such as a planner task, intent classification, or research prepass. All provider activity generated by that action shares `operation_id`. + +### Logical Call + +A logical call groups the primary attempt with retries whose purpose is to produce the same logical result. A research decision and external deep research use different logical calls under one research operation. + +### Invocation + +An invocation is one real provider attempt. Each invocation has a unique `invocation_id` and one of these attempt kinds: + +| Attempt kind | Meaning | +|---|---| +| `primary` | Initial provider attempt for a logical call | +| `sandbox_retry` | Retry after a detected write or sandbox denial | +| `contract_repair` | Follow-up attempt to repair invalid role-result JSON | + +Purposes distinguish runtime responsibilities: + +| Purpose | Runtime activity | +|---|---| +| `role_task` | Normal public-role execution | +| `research_decision` | Research need and subject classification | +| `deep_research` | External source-backed research | +| `intent_classification` | Internal semantic intake parser | +| `goal_sourcing` | Internal goal milestone sourcing | +| `version_control` | Version-controller commit preparation | +| `sprint_closeout_report` | Planner-authored terminal sprint report | + +## Record Schema + +Each JSONL line is one schema-versioned object. + +| Field | Description | +|---|---| +| `schema_version` | Record contract version, currently `1` | +| `invocation_id` | Unique provider attempt | +| `operation_id` | Enclosing role operation | +| `logical_call_id` | Primary call and related retry group | +| `attempt_index` | One-based order within the logical call | +| `attempt_kind` | Primary, sandbox retry, or contract repair | +| `started_at`, `ended_at` | Runtime-timezone ISO-8601 timestamps | +| `duration_ms` | Monotonic elapsed time | +| `pid` | Writer process identifier | +| `runtime_identity` | Service or local helper identity | +| `role` | Public or internal agent role | +| `purpose` | Normalized invocation purpose | +| `workflow_step` | Governed workflow step when available | +| `request_id` | Associated request or empty string | +| `sprint_id` | Associated sprint or empty string | +| `todo_id` | Associated todo or empty string | +| `backlog_id` | Associated backlog item or empty string | +| `goal_id` | Associated operator goal or empty string | +| `provider` | Provider adapter identifier | +| `model` | Configured model or research app | +| `reasoning` | Configured reasoning mode | +| `cli_version` | Lazily detected provider CLI version | +| `session_mode` | `new`, `resume`, or `not_applicable` | +| `session_id_hash` | Truncated SHA-256 session correlation value | +| `status` | Provider invocation completion status | +| `exit_code` | Subprocess exit code or null | +| `error_category` | Content-free normalized failure class | +| `prompt_chars`, `output_chars` | Text sizes without text content | +| `tool_calls` | Native tool-call count or null | +| `input_tokens` | Native input usage or null | +| `cached_input_tokens` | Native cached-input usage or null | +| `output_tokens` | Native output usage or null | +| `reasoning_output_tokens` | Native reasoning usage or null | +| `total_tokens` | Native or consistently derived total | +| `usage_source` | `native` or `unavailable` | +| `estimated_cost_usd` | Rate-card estimate or null | +| `rate_card` | Applied pricing snapshot or null | + +Sanitized example: + +```json +{ + "schema_version": 1, + "invocation_id": "a91c...", + "operation_id": "b72d...", + "logical_call_id": "c13e...", + "attempt_index": 1, + "attempt_kind": "primary", + "started_at": "2026-07-18T12:00:00+09:00", + "ended_at": "2026-07-18T12:00:08+09:00", + "duration_ms": 8000, + "pid": 42424, + "runtime_identity": "planner", + "role": "planner", + "purpose": "role_task", + "workflow_step": "planner_draft", + "request_id": "request-20260718-001", + "sprint_id": "2026-Sprint-03", + "provider": "codex_cli", + "model": "gpt-5.5", + "session_mode": "resume", + "session_id_hash": "4f1e8c35a20e9d22", + "status": "completed", + "prompt_chars": 18250, + "output_chars": 2400, + "input_tokens": 8200, + "cached_input_tokens": 6100, + "output_tokens": 900, + "total_tokens": 9100, + "usage_source": "native", + "estimated_cost_usd": null, + "rate_card": null +} +``` + +## Storage + +Telemetry is stored under the generated workspace runtime root: + +```text +.teams_runtime/ + metrics/ + model_invocations/ + 2026-07-18/ + planner.42424.jsonl + orchestrator.local.parser.42425.jsonl +``` + +Design properties: + +- dates use the runtime timezone +- PID-specific shards avoid normal cross-process write contention +- records are compact, append-only JSON lines +- files are flushed after each provider attempt +- a crash may leave one partial final line +- readers skip malformed lines and report their count +- queries visit only date directories in the requested interval +- no automatic deletion or retention policy is applied +- `init --reset` keeps its existing behavior and does not specially preserve metrics + +## Privacy And Security + +Telemetry is local to the runtime workspace. It does not transmit records to a collector. + +The recorder never stores: + +- prompt or response text +- raw stdout or stderr +- raw exception messages +- full session IDs +- workspace paths +- commands or command arguments +- environment variables +- credentials +- Discord message content + +Session IDs are hashed only to measure reuse. Error details are reduced to categories such as `cli_not_found`, `nonzero_exit`, `provider_output_invalid`, `provider_incomplete`, and `provider_exception`. + +Operational role logs remain separate and may contain more diagnostic context according to their existing behavior. + +## Cost Estimation + +For token-priced models: + +```text +uncached_input = max(input_tokens - cached_input_tokens, 0) + +estimated_cost = + uncached_input * input_rate / 1,000,000 + + cached_input_tokens * cached_input_rate / 1,000,000 + + output_tokens * output_rate / 1,000,000 +``` + +Reasoning-output tokens are reported separately but are not added again because provider output totals may already include them. + +For flat-priced operations: + +```text +estimated_cost = per_invocation_usd +``` + +The complete applied rate card is stored with a priced record. Historical estimates therefore remain stable if configuration later changes. + +An unpriced report is shown as `unpriced`, not `$0.00`. Pricing coverage is the percentage of matched invocations with an estimate. + +These values are accounting estimates. They do not incorporate subscription allowances, credits, rate limits, refunds, or provider invoice adjustments. + +## CLI Reference + +Basic report: + +```bash +python -m teams_runtime metrics --hours 24 +``` + +Request report: + +```bash +python -m teams_runtime metrics --request-id request-20260718-001 --hours 72 +``` + +Sprint and role report: + +```bash +python -m teams_runtime metrics --sprint-id 2026-Sprint-03 --agent planner --hours 168 +``` + +Machine-readable report: + +```bash +python -m teams_runtime metrics --hours 24 --json +``` + +Options: + +| Option | Behavior | +|---|---| +| `--workspace-root PATH` | Select a generated runtime workspace | +| `--hours NUMBER` | Positive lookback duration, default `24` | +| `--request-id ID` | Exact request filter | +| `--sprint-id ID` | Exact sprint filter | +| `--agent ROLE` | Exact public or internal role filter | +| `--json` | Emit stable aggregate JSON | + +Filters use AND semantics. A request, sprint, and role supplied together must all match one record. + +Human output reports totals followed by grouped rows. The JSON output contains: + +- `schema_version` +- `generated_at` +- `filters` +- `totals` +- `tokens` +- `latency_ms` +- `groups` + +No matching records returns exit code `0` and an explicit no-data message. Invalid hours return exit code `2`. + +Latency percentiles use nearest-rank calculation. Native token coverage and pricing coverage are reported independently. + +## Analysis Playbook + +Use one representative sprint or at least 24 hours of normal traffic before changing policy. + +| Observed concentration | Likely next investigation | +|---|---| +| High input tokens in later workflow stages | Compact request events and prompt context | +| Low cached-input ratio on resumed sessions | Inspect session identity and rollover behavior | +| High contract-repair count | Stabilize role-result prompts and contract validation | +| High invocations per logical call | Inspect sandbox and repair retry causes | +| High invocations per request or todo | Reduce workflow fan-out or review cycles | +| High parser or goal-sourcer cost | Configure independent lower-cost helper models | +| High closeout planner cost | Evaluate deterministic report drafting | +| High deep-research latency | Tighten research gating and timeout policy | +| High latency with low token usage | Inspect tools, Git operations, or provider waiting | +| Low pricing coverage | Add exact rate cards or treat tokens as the cost proxy | + +Do not compare only total role cost. Normalize by request count, todo count, and logical-call count so a frequently used inexpensive role is not confused with an inefficient role. + +### Measuring Helper Tiers And Workflow Budgets + +Group telemetry by `role`, `model`, `reasoning`, and `purpose` before changing helper tiers. Compare equivalent workloads with the public role tiers held constant. The sprint benchmark report's `provenance.runtime_model_map` includes `parser`, `sourcer`, and `version_controller`, and its source configuration hash includes their effective settings even when a legacy workspace inherits them from orchestrator. + +For review and reopen controls, compare invocations per completed todo and inspect terminal outcomes alongside cost. A reduced call count is not a successful optimization if blocked todos, repair calls, or QA failures increase. See [`docs/call_amplification_controls.md`](call_amplification_controls.md) for exact counter semantics and the controlled experiment procedure. + +### Measuring Prompt Compaction + +Use comparable requests with long event histories before and after enabling `prompt_context`. Keep the provider, model, reasoning level, request shape, and workflow path stable. Compare: + +- `prompt_chars` and native input tokens per logical call +- cached-input tokens and cached-input ratio +- total input tokens per completed request or todo +- contract-repair and retry counts +- failures, QA outcomes, and p95 latency + +The expected result is lower prompt size and input-token usage in later workflow stages without a higher repair, retry, failure, or reopen rate. Session reuse can change cached-input behavior independently, so do not attribute every cached-token change to compaction. Telemetry stores prompt size and usage totals, not prompt content or the selected event list; use the content-free `prompt_context_compacted` runtime log for total/included/omitted counts. + +For a controlled rollback comparison, set `prompt_context.enabled: false`, restart the same role services, and repeat the same request shape. The canonical persisted event history is identical in either mode. + +## Operational Validation + +After restarting role services and completing a model-backed request: + +```bash +find teams_generated/.teams_runtime/metrics/model_invocations -type f -name '*.jsonl' +python -m teams_runtime metrics --workspace-root teams_generated --hours 1 +``` + +Confirm: + +- the expected role and purpose appear +- invocation count matches actual attempts +- repair or retry counts match logs +- token coverage is nonzero for supported native provider events +- no prompt or response content appears in the JSONL record +- estimated cost remains unpriced until a rate card is configured + +To validate disabled mode, set `telemetry.enabled` to `false`, restart the role service, run a new request, and confirm no new JSONL line appears. + +## Troubleshooting + +### The Report Is Empty + +Check the workspace root, time window, service restart, and `telemetry.enabled`. Metrics are stored in the generated workspace, not necessarily the project source directory. + +### Token Coverage Is Zero + +The provider call completed without a recognized native usage event. Confirm the recorded `provider` and `cli_version`. The runtime preserves the role result and marks usage unavailable rather than inventing zeros. + +### Cost Is Unpriced + +No exact `provider/model` rate-card key matched, or the provider did not expose the token fields required by the token rate. Inspect the grouped provider and model names in the report. + +### Invalid Record Count Is Nonzero + +A shard contains a partial line, unsupported schema, invalid timestamp, or malformed JSON. Other records remain usable. A single partial final line can result from abrupt process termination. + +### Telemetry Write Warning Appears + +Check permissions and available disk space under `.teams_runtime/metrics`. The model result is not failed when recording fails. Warnings are throttled to avoid log flooding. + +### Provider Schema Changes + +Unknown provider fields are ignored. If native coverage drops after a CLI upgrade, add a parser fixture for the new terminal usage shape while preserving schema version `1` when the normalized record contract remains unchanged. + +## Compatibility And Versioning + +Telemetry records use an explicit `schema_version`. + +Compatibility rules: + +- additive fields may be introduced without changing version `1` +- readers ignore unknown fields +- missing nullable fields behave as unavailable +- malformed and unsupported records are skipped and counted +- a breaking field meaning or type requires a new schema version +- aggregate JSON is separately versioned through its `schema_version` +- no records are synthesized for calls made before telemetry deployment + +The `CodexRunner.run` tuple return contract remains unchanged. Telemetry context is an optional internal keyword, so existing callers can continue to invoke the runner without telemetry metadata. + +## Known Limitations + +- no hosted dashboard or exporter +- no automatic retention or compression +- no billing reconciliation +- no distributed trace across separate machines +- no historical backfill +- no native deep-research token usage +- no automatic provider-price updates +- percentile calculation retains matched durations in memory + +Daily sharding bounds normal query work, but very large installations may eventually require a database or metrics backend. + +## Choosing The Next Optimization + +Collect a representative sprint and rank groups by total input tokens, total duration, repair count, and estimated cost. Select the next change from the most concentrated measured driver. + +Recommended decision order: + +1. Fix unexpectedly high repair or retry rates because they spend tokens without advancing workflow. +2. Compact prompts when later stages repeatedly carry large request histories. +3. Move helper or classification purposes to lower-cost models when they dominate call volume. +4. Reduce workflow fan-out when calls per request are high despite stable contracts. +5. Introduce safe parallelism only when latency is dominated by independent work and Git isolation is available. + +Measure the same request or sprint shape after the optimization. Compare cost per completed todo, tokens per logical call, repair rate, p95 latency, and QA outcome rather than comparing raw totals from different workloads. diff --git a/models.py b/models.py index 27b5bed..45f4cab 100644 --- a/models.py +++ b/models.py @@ -14,6 +14,8 @@ DiscordAgentsConfig, INTERNAL_TEAM_AGENTS, MessageEnvelope, + ModelRateCard, + PromptContextRuntimeConfig, ReplyRoute, RequestEvent, RequestRecord, @@ -27,6 +29,7 @@ TEAM_ROLES, TERMINAL_REQUEST_STATUSES, TeamRuntimeConfig, + TelemetryRuntimeConfig, WorkflowState, ) @@ -37,6 +40,8 @@ "DiscordAgentsConfig", "INTERNAL_TEAM_AGENTS", "MessageEnvelope", + "ModelRateCard", + "PromptContextRuntimeConfig", "ReplyRoute", "RequestEvent", "RequestRecord", @@ -50,5 +55,6 @@ "TEAM_ROLES", "TERMINAL_REQUEST_STATUSES", "TeamRuntimeConfig", + "TelemetryRuntimeConfig", "WorkflowState", ] diff --git a/runtime/base_runtime.py b/runtime/base_runtime.py index ffb30e3..664dffe 100644 --- a/runtime/base_runtime.py +++ b/runtime/base_runtime.py @@ -8,7 +8,13 @@ from teams_runtime.shared.paths import RuntimePaths from teams_runtime.runtime.codex_runner import CodexRunner, extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.identities import service_runtime_identity +from teams_runtime.runtime.model_telemetry import ( + InvocationSequence, + ModelTelemetryRecorder, + run_with_optional_telemetry, +) from teams_runtime.runtime.role_result_contract import ( ALLOWED_ROLE_STATUSES, CONTRACT_STATUS_INVALID, @@ -25,7 +31,20 @@ validate_role_result_contract, ) from teams_runtime.runtime.session_manager import RoleSessionManager -from teams_runtime.shared.models import MessageEnvelope, RequestRecord, RoleResult, RoleRuntimeConfig +from teams_runtime.shared.models import ( + MessageEnvelope, + PromptContextRuntimeConfig, + RequestRecord, + RoleResult, + RoleRuntimeConfig, + TelemetryRuntimeConfig, +) +from teams_runtime.shared.prompt_context import ( + PROMPT_EVENT_SELECTION_POLICY, + PromptRequestProjection, + project_request_record_for_prompt, + render_prompt_event_history_notice, +) from teams_runtime.workflows.roles import render_role_prompt_spec from teams_runtime.workflows.roles.planner import normalize_planner_proposals @@ -254,6 +273,9 @@ def __init__( runtime_config: RoleRuntimeConfig, agent_root: Path | None = None, session_identity: str | None = None, + telemetry_config: TelemetryRuntimeConfig | None = None, + prompt_context_config: PromptContextRuntimeConfig | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.paths = paths self.role = role @@ -270,9 +292,55 @@ def __init__( self._session_managers: dict[str, RoleSessionManager] = { self.sprint_id: self.session_manager, } - self.codex_runner = CodexRunner(runtime_config, role=role) + self.telemetry_recorder = ModelTelemetryRecorder( + paths, + self.runtime_identity, + telemetry_config, + output_dir=( + execution_policy.telemetry_output_dir + if execution_policy is not None + else None + ), + ) + self.prompt_context_config = prompt_context_config or PromptContextRuntimeConfig() + self.codex_runner = CodexRunner( + runtime_config, + role=role, + telemetry_recorder=self.telemetry_recorder, + execution_policy=execution_policy, + ) self.runtime_config = runtime_config - self._run_lock = threading.Lock() + self._run_lock = ( + execution_policy.execution_lock + if execution_policy is not None + and execution_policy.execution_lock is not None + else threading.Lock() + ) + + def _project_request_for_prompt( + self, + request_record: RequestRecord, + *, + purpose: str, + ) -> PromptRequestProjection: + projection = project_request_record_for_prompt( + request_record, + self.prompt_context_config, + ) + if projection.compacted: + LOGGER.info( + "[%s] prompt_context_compacted request_id=%s purpose=%s total_events=%s included_events=%s " + "omitted_events=%s recent_events=%s max_events=%s", + self.role, + str(request_record.get("request_id") or "unknown"), + purpose, + projection.total_events, + projection.included_events, + projection.omitted_events, + projection.recent_events, + projection.max_events, + ) + return projection def _resolve_request_sprint_id( self, @@ -312,17 +380,45 @@ def _request_requires_default_bypass( envelope: MessageEnvelope, request_record: RequestRecord, ) -> bool: - return True + return not self._benchmark_execution_enabled() + + def _benchmark_execution_enabled(self) -> bool: + execution_policy = getattr(self.codex_runner, "execution_policy", None) + return bool(getattr(execution_policy, "benchmark_mode", False)) - def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> RoleResult: + def run_task( + self, + envelope: MessageEnvelope, + request_record: RequestRecord, + *, + telemetry_purpose: str = "role_task", + ) -> RoleResult: with self._run_lock: current_sprint_id = self._resolve_request_sprint_id(envelope, request_record) + invocation_sequence = InvocationSequence.from_request( + runtime_identity=self.runtime_identity, + role=self.role, + purpose=telemetry_purpose, + request_record=request_record, + envelope=envelope, + sprint_id=current_sprint_id, + ) session_manager = self._session_manager_for_sprint(current_sprint_id) state = session_manager.ensure_session() + request_projection = self._project_request_for_prompt( + request_record, + purpose=telemetry_purpose, + ) + invocation_sequence.set_prompt_context_projection( + request_projection, + enabled=self.prompt_context_config.enabled, + selection_policy=PROMPT_EVENT_SELECTION_POLICY, + ) prompt = self._build_prompt( envelope, request_record, current_sprint_id=current_sprint_id, + request_projection=request_projection, ) force_fresh_role_session = ( bool((envelope.params or {}).get("_repair_invalid_role_payload_on_resume")) @@ -363,15 +459,21 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> ) default_bypass = self._request_requires_default_bypass(envelope, request_record) try: - output, resolved_session_id = self.codex_runner.run( + output, resolved_session_id = run_with_optional_telemetry( + self.codex_runner, Path(state.workspace_path), prompt, active_session_id, bypass_sandbox=default_bypass, + invocation_context=invocation_sequence.next("primary"), ) active_session_id = resolved_session_id or active_session_id payload = self._parse_role_output(output, request_record) - if not default_bypass and self._should_retry_with_bypass(payload): + if ( + not self._benchmark_execution_enabled() + and not default_bypass + and self._should_retry_with_bypass(payload) + ): retry_session_id = None if active_session_id else active_session_id LOGGER.warning( "[%s] sandbox_denial_detected retrying_with_bypass request_id=%s sprint_id=%s todo_id=%s backlog_id=%s workspace=%s session_id=%s retry_session_mode=%s", @@ -384,11 +486,13 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> active_session_id or "new", "fresh" if retry_session_id is None else "resume", ) - output, resolved_session_id = self.codex_runner.run( + output, resolved_session_id = run_with_optional_telemetry( + self.codex_runner, Path(state.workspace_path), prompt, retry_session_id, bypass_sandbox=True, + invocation_context=invocation_sequence.next("sandbox_retry"), ) active_session_id = resolved_session_id or active_session_id payload = self._parse_role_output(output, request_record) @@ -399,6 +503,7 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> workspace_path=Path(state.workspace_path), active_session_id=active_session_id, bypass_sandbox=default_bypass, + invocation_sequence=invocation_sequence, ) except RuntimeError as exc: LOGGER.warning( @@ -494,6 +599,7 @@ def _repair_invalid_role_payload_once( workspace_path: Path, active_session_id: str | None, bypass_sandbox: bool, + invocation_sequence: InvocationSequence, ) -> tuple[RoleResult, str | None, bool]: if not is_invalid_contract_payload(payload): return payload, active_session_id, False @@ -533,11 +639,13 @@ def _repair_invalid_role_payload_once( attempts, latest_session_id, ) - output, resolved_session_id = self.codex_runner.run( + output, resolved_session_id = run_with_optional_telemetry( + self.codex_runner, workspace_path, repair_prompt, repair_session_id, bypass_sandbox=bypass_sandbox, + invocation_context=invocation_sequence.next("contract_repair"), ) latest_session_id = resolved_session_id if repair_session_id is None else (resolved_session_id or latest_session_id) latest_payload = self._parse_role_output(output, request_record) @@ -604,6 +712,11 @@ def _build_role_result_repair_prompt( *, current_sprint_id: str, ) -> str: + request_projection = self._project_request_for_prompt( + request_record, + purpose="contract_repair", + ) + event_history_notice = render_prompt_event_history_notice(request_projection) team_workspace_hint = "./workspace/teams_generated" if self.paths.workspace_root.name == "teams_generated" else "./workspace" role_specific_rules, extra_fields = render_role_prompt_spec(self.role, team_workspace_hint) contract_block = render_role_result_contract( @@ -635,8 +748,9 @@ def _build_role_result_repair_prompt( If validation errors mention copied placeholder or scaffold text, do not reuse any wording from the shape block. Write concrete Korean summary and workflow reason text from the actual request state, or return `failed` with a concrete Korean reason. {role_specific_rules} +{event_history_notice} Current request: -{json.dumps(request_record, ensure_ascii=False, indent=2)} +{json.dumps(request_projection.request_record, ensure_ascii=False, indent=2)} """ def _should_retry_with_bypass(self, payload: dict[str, Any]) -> bool: @@ -694,7 +808,13 @@ def _build_prompt( request_record: RequestRecord, *, current_sprint_id: str | None = None, + request_projection: PromptRequestProjection | None = None, ) -> str: + request_projection = request_projection or self._project_request_for_prompt( + request_record, + purpose="role_task", + ) + event_history_notice = render_prompt_event_history_notice(request_projection) resolved_sprint_id = str(current_sprint_id or "").strip() or self._resolve_request_sprint_id( envelope, request_record, @@ -735,8 +855,9 @@ def _build_prompt( When you claim a file change or validation result, leave enough evidence in `summary`, `insights`, or `proposals` for orchestrator to verify what you actually checked. {role_specific_rules} +{event_history_notice} Current request: -{json.dumps(request_record, ensure_ascii=False, indent=2)} +{json.dumps(request_projection.request_record, ensure_ascii=False, indent=2)} Incoming envelope: {json.dumps(envelope.to_dict(), ensure_ascii=False, indent=2)} diff --git a/runtime/benchmark_launcher.py b/runtime/benchmark_launcher.py new file mode 100644 index 0000000..1070bed --- /dev/null +++ b/runtime/benchmark_launcher.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import os +import sys + + +_READY_BYTE = b"\x01" + + +def main(argv: list[str] | None = None) -> int: + arguments = list(sys.argv[1:] if argv is None else argv) + if len(arguments) < 4 or arguments[0] != "--ready-fd": + return 64 + try: + ready_fd = int(arguments[1]) + except ValueError: + return 64 + if arguments[2] != "--": + return 64 + command = arguments[3:] + if not command: + return 64 + + try: + with os.fdopen(ready_fd, "rb", closefd=True) as ready_pipe: + ready = ready_pipe.read(1) + except OSError: + return 70 + if ready != _READY_BYTE: + return 70 + + try: + os.execvpe(command[0], command, os.environ) + except OSError: + return 71 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/runtime/codex_runner.py b/runtime/codex_runner.py index 1f9c661..62d1a94 100644 --- a/runtime/codex_runner.py +++ b/runtime/codex_runner.py @@ -1,18 +1,246 @@ from __future__ import annotations import json +import math import logging import os import re +import signal import subprocess +import sys +import time +from datetime import datetime from pathlib import Path from typing import Any +from teams_runtime.runtime.execution_policy import ( + DEFAULT_MODEL_EXECUTION_POLICY, + InvocationReservation, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, + ModelInvocationTimeout, + quarantine_unsafe_workspace_entries, +) +from teams_runtime.runtime.model_telemetry import ( + ModelInvocationContext, + ModelTelemetryRecorder, + ModelUsage, + normalized_error_category, +) from teams_runtime.shared.models import RoleRuntimeConfig +from teams_runtime.shared.persistence import runtime_now SESSION_ID_PATTERN = re.compile(r"session id:\s*([0-9a-fA-F-]+)", re.IGNORECASE) LOGGER = logging.getLogger(__name__) +CODEX_TOOL_ITEM_TYPES = frozenset( + { + "command_execution", + "file_change", + "mcp_tool_call", + "web_search", + } +) +BENCHMARK_DISABLED_CODEX_FEATURES = ( + "apps", + "browser_use", + "browser_use_external", + "computer_use", + "enable_fanout", + "enable_mcp_apps", + "hooks", + "image_generation", + "in_app_browser", + "multi_agent", + "multi_agent_v2", + "plugin_sharing", + "plugins", + "standalone_web_search", + "web_search_request", + "workspace_dependencies", +) +BENCHMARK_PROVIDER_ENVIRONMENT_KEYS = ( + "CODEX_API_KEY", + "CODEX_HOME", + "CURL_CA_BUNDLE", + "LANG", + "LC_ALL", + "LC_CTYPE", + "OPENAI_API_KEY", + "OPENAI_BASE_URL", + "OPENAI_ORG_ID", + "OPENAI_PROJECT_ID", + "PATH", + "REQUESTS_CA_BUNDLE", + "SSL_CERT_DIR", + "SSL_CERT_FILE", + "TEMP", + "TMP", + "TMPDIR", +) +_BENCHMARK_LAUNCHER_PATH = Path(__file__).with_name( + "benchmark_launcher.py" +) +_BENCHMARK_LAUNCH_READY_BYTE = b"\x01" + + +class _BenchmarkProcessSafetyAbort(BaseException): + """Abort the dedicated worker when provider cleanup cannot be proven.""" + + +def _nested_mapping(payload: Any, *keys: str) -> dict[str, Any]: + current = payload + for key in keys: + if not isinstance(current, dict): + return {} + current = current.get(key) + return current if isinstance(current, dict) else {} + + +def _first_value(payload: dict[str, Any], *names: str) -> Any: + for name in names: + if name in payload: + return payload.get(name) + return None + + +def _usage_from_mapping(payload: dict[str, Any]) -> ModelUsage: + return ModelUsage.from_values( + input_tokens=_first_value(payload, "input_tokens", "inputTokens", "prompt_tokens", "promptTokens"), + cached_input_tokens=_first_value( + payload, + "cached_input_tokens", + "cachedInputTokens", + "cached_tokens", + "cachedTokens", + "cached", + ), + output_tokens=_first_value(payload, "output_tokens", "outputTokens", "candidate_tokens", "candidateTokens"), + reasoning_output_tokens=_first_value( + payload, + "reasoning_output_tokens", + "reasoningOutputTokens", + "reasoning_tokens", + "thoughts", + ), + total_tokens=_first_value(payload, "total_tokens", "totalTokens"), + tool_calls=_first_value(payload, "tool_calls", "toolCalls"), + ) + + +def parse_codex_jsonl(stdout: str) -> tuple[str | None, ModelUsage, str]: + session_id: str | None = None + usage = ModelUsage() + final_message = "" + completed_tool_calls = 0 + completed_tool_item_ids: set[tuple[str, str]] = set() + for raw_line in str(stdout or "").splitlines(): + try: + event = json.loads(raw_line) + except json.JSONDecodeError: + continue + if not isinstance(event, dict): + continue + event_type = str(event.get("type") or event.get("method") or "").strip() + params = event.get("params") if isinstance(event.get("params"), dict) else {} + candidate_session = ( + event.get("thread_id") + or event.get("session_id") + or params.get("thread_id") + or params.get("session_id") + ) + if candidate_session: + session_id = str(candidate_session).strip() or session_id + if event_type in {"turn.completed", "turn/completed", "task_complete"}: + usage_payload = event.get("usage") + if not isinstance(usage_payload, dict): + usage_payload = params.get("usage") if isinstance(params.get("usage"), dict) else {} + if not usage_payload: + usage_payload = _nested_mapping(event, "turn", "usage") + candidate_usage = _usage_from_mapping(usage_payload) + if candidate_usage.source == "native": + usage = candidate_usage + if event_type in {"item.completed", "item/completed", "agent_message"}: + item = event.get("item") if isinstance(event.get("item"), dict) else params.get("item") + item = item if isinstance(item, dict) else event + item_type = str(item.get("type") or item.get("kind") or "").strip() + normalized_item_type = item_type.lower().replace("-", "_").replace(".", "_") + if ( + event_type in {"item.completed", "item/completed"} + and ( + normalized_item_type in CODEX_TOOL_ITEM_TYPES + or normalized_item_type.endswith("_tool_call") + ) + ): + item_id = str(item.get("id") or item.get("item_id") or "").strip() + item_identity = (normalized_item_type, item_id) + if not item_id or item_identity not in completed_tool_item_ids: + completed_tool_calls += 1 + if item_id: + completed_tool_item_ids.add(item_identity) + if item_type in {"agent_message", "message"} or event_type == "agent_message": + candidate_text = item.get("text") or item.get("message") or item.get("content") + if isinstance(candidate_text, list): + candidate_text = "".join( + str(part.get("text") or "") if isinstance(part, dict) else str(part) + for part in candidate_text + ) + if candidate_text: + final_message = str(candidate_text).strip() + if usage.source == "native" or completed_tool_calls: + usage = ModelUsage.from_values( + input_tokens=usage.input_tokens, + cached_input_tokens=usage.cached_input_tokens, + output_tokens=usage.output_tokens, + reasoning_output_tokens=usage.reasoning_output_tokens, + total_tokens=usage.total_tokens, + tool_calls=max(usage.tool_calls or 0, completed_tool_calls), + ) + return session_id, usage, final_message + + +def parse_gemini_usage(stats: Any) -> ModelUsage: + if not isinstance(stats, dict): + return ModelUsage() + if any(key in stats for key in ("input_tokens", "inputTokens", "total_tokens", "totalTokens")): + return _usage_from_mapping(stats) + models = stats.get("models") + if not isinstance(models, dict): + return ModelUsage() + totals = { + "input_tokens": 0, + "cached_input_tokens": 0, + "output_tokens": 0, + "reasoning_output_tokens": 0, + "total_tokens": 0, + } + observed = {name: False for name in totals} + for model_metrics in models.values(): + if not isinstance(model_metrics, dict): + continue + tokens = model_metrics.get("tokens") if isinstance(model_metrics.get("tokens"), dict) else model_metrics + aliases = { + "input_tokens": ("prompt", "input_tokens", "inputTokens", "input"), + "cached_input_tokens": ("cached", "cached_input_tokens", "cachedInputTokens"), + "output_tokens": ("candidates", "output_tokens", "outputTokens"), + "reasoning_output_tokens": ("thoughts", "reasoning_output_tokens", "reasoningOutputTokens"), + "total_tokens": ("total", "total_tokens", "totalTokens"), + } + for target, names in aliases.items(): + value = _first_value(tokens, *names) + if ( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + and value >= 0 + ): + totals[target] += int(value) + observed[target] = True + tools = stats.get("tools") if isinstance(stats.get("tools"), dict) else {} + return ModelUsage.from_values( + **{name: totals[name] if observed[name] else None for name in totals}, + tool_calls=_first_value(tools, "totalCalls", "total_calls", "tool_calls"), + ) def extract_json_object(text: str) -> dict[str, Any]: @@ -69,9 +297,62 @@ def extract_json_object(text: str) -> dict[str, Any]: class CodexRunner: - def __init__(self, runtime_config: RoleRuntimeConfig, *, role: str = ""): + _version_cache: dict[str, str] = {} + + def __init__( + self, + runtime_config: RoleRuntimeConfig, + *, + role: str = "", + telemetry_recorder: ModelTelemetryRecorder | None = None, + execution_policy: ModelExecutionPolicy | None = None, + ): self.runtime_config = runtime_config self.role = str(role or "").strip() + self.telemetry_recorder = telemetry_recorder + self.execution_policy = execution_policy or DEFAULT_MODEL_EXECUTION_POLICY + + def _cli_version(self, cli_name: str) -> str: + if self.telemetry_recorder is None or not self.telemetry_recorder.enabled: + return "" + executable = ( + self._codex_executable() + if cli_name == "codex" + else cli_name + ) + cached = self._version_cache.get(executable) + if cached is not None: + return cached + try: + run_options: dict[str, Any] = {} + if self.execution_policy.benchmark_mode: + run_options["env"] = self._provider_environment() + run_options["timeout"] = min( + float(self.execution_policy.call_timeout_seconds or 10.0), + 10.0, + ) + process = subprocess.run( + [executable, "--version"], + capture_output=True, + text=True, + check=False, + **run_options, + ) + version = str(process.stdout or process.stderr or "").strip().splitlines()[0] + except Exception: + version = "" + self._version_cache[executable] = version + return version + + def _codex_executable(self) -> str: + if not self.execution_policy.benchmark_mode: + return "codex" + executable = self.execution_policy.codex_executable + if executable is None: + raise ModelExecutionPolicyViolation( + "Benchmark execution has no pinned Codex executable." + ) + return str(executable) def _discover_extra_writable_dirs(self, workspace: Path) -> list[str]: extra_dirs: list[str] = [] @@ -86,22 +367,308 @@ def _discover_extra_writable_dirs(self, workspace: Path) -> list[str]: continue resolved_text = str(resolved) if resolved != workspace and resolved_text not in seen: + self.execution_policy.assert_workspace_allowed(resolved) seen.add(resolved_text) extra_dirs.append(resolved_text) return extra_dirs + def _provider_environment(self) -> dict[str, str]: + if not self.execution_policy.benchmark_mode: + return {**os.environ, "HOME": str(Path.home())} + environment = { + key: value + for key in BENCHMARK_PROVIDER_ENVIRONMENT_KEYS + if (value := os.environ.get(key)) is not None + } + # The explicit policy must override outer HOME, CODEX_HOME, temp, Git, + # and locale values while provider-only auth variables remain available. + environment.update(self.execution_policy.shell_environment) + environment.setdefault("PATH", os.defpath) + environment["NO_COLOR"] = "1" + return environment + + def _append_benchmark_codex_controls( + self, + command: list[str], + *, + supports_sandbox_option: bool, + ) -> None: + if supports_sandbox_option: + command.extend(["--sandbox", "workspace-write"]) + command.extend( + [ + "--ignore-user-config", + "--ignore-rules", + "-c", + 'approval_policy="never"', + "-c", + 'sandbox_mode="workspace-write"', + "-c", + "sandbox_workspace_write.exclude_slash_tmp=true", + "-c", + "sandbox_workspace_write.exclude_tmpdir_env_var=true", + "-c", + "mcp_servers={}", + "-c", + 'shell_environment_policy.inherit="none"', + ] + ) + for name, value in self.execution_policy.shell_environment.items(): + command.extend( + [ + "-c", + f"shell_environment_policy.set.{name}={json.dumps(value, ensure_ascii=True)}", + ] + ) + for feature_name in BENCHMARK_DISABLED_CODEX_FEATURES: + command.extend(["--disable", feature_name]) + + @staticmethod + def _process_group_exists(process_group_id: int | None) -> bool: + if ( + process_group_id is None + or process_group_id <= 1 + or not hasattr(os, "killpg") + ): + return False + try: + os.killpg(process_group_id, 0) + except ProcessLookupError: + return False + except OSError: + return True + return True + + @classmethod + def _stop_remaining_process_group( + cls, + process_group_id: int | None, + *, + grace_seconds: float, + ) -> None: + if not cls._process_group_exists(process_group_id): + return + assert process_group_id is not None + try: + os.killpg(process_group_id, signal.SIGTERM) + except ProcessLookupError: + return + except OSError: + pass + deadline = time.monotonic() + max(grace_seconds, 0.0) + while cls._process_group_exists(process_group_id) and time.monotonic() < deadline: + time.sleep(0.05) + if cls._process_group_exists(process_group_id): + try: + os.killpg(process_group_id, signal.SIGKILL) + except ProcessLookupError: + return + except OSError: + pass + kill_deadline = time.monotonic() + max(grace_seconds, 0.1) + while cls._process_group_exists(process_group_id) and time.monotonic() < kill_deadline: + time.sleep(0.05) + if cls._process_group_exists(process_group_id): + raise _BenchmarkProcessSafetyAbort( + "Benchmark provider process-group cleanup could not be proven." + ) + + def _quarantine_workspace_after_provider(self, workspace: Path) -> None: + integrity_root = self.execution_policy.allowed_workspace_root or workspace + try: + removed_kinds = quarantine_unsafe_workspace_entries(integrity_root) + except (ModelExecutionPolicyViolation, OSError, ValueError) as exc: + raise _BenchmarkProcessSafetyAbort( + "Benchmark workspace integrity could not be proven after provider exit." + ) from exc + if removed_kinds: + raise ModelExecutionPolicyViolation( + "Benchmark provider created unsafe filesystem entries; they were quarantined." + ) + + @classmethod + def _terminate_process_group( + cls, + process: subprocess.Popen[str], + *, + process_group_id: int | None, + grace_seconds: float, + ) -> tuple[str, str]: + def send_group_signal(group_signal: signal.Signals) -> None: + try: + if process_group_id is not None and hasattr(os, "killpg"): + os.killpg(process_group_id, group_signal) + elif process.poll() is None and group_signal == signal.SIGTERM: + process.terminate() + elif process.poll() is None: + process.kill() + except ProcessLookupError: + pass + + send_group_signal(signal.SIGTERM) + try: + stdout, stderr = process.communicate(timeout=grace_seconds) + except subprocess.TimeoutExpired: + send_group_signal(signal.SIGKILL) + stdout, stderr = process.communicate() + cls._stop_remaining_process_group( + process_group_id, + grace_seconds=grace_seconds, + ) + return str(stdout or ""), str(stderr or "") + + def _run_benchmark_process( + self, + command: list[str], + *, + cwd: Path, + stdin_input: str | None, + env: dict[str, str], + reservation: InvocationReservation, + ) -> subprocess.CompletedProcess[str]: + if os.name != "posix": + raise ModelExecutionPolicyViolation( + "Benchmark provider launch requires POSIX file-descriptor handoff." + ) + ready_read_fd, ready_write_fd = os.pipe() + try: + try: + process = subprocess.Popen( + [ + sys.executable, + str(_BENCHMARK_LAUNCHER_PATH), + "--ready-fd", + str(ready_read_fd), + "--", + *command, + ], + cwd=str(cwd), + stdin=( + subprocess.PIPE + if stdin_input is not None + else None + ), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + start_new_session=True, + pass_fds=(ready_read_fd,), + ) + finally: + os.close(ready_read_fd) + except BaseException: + try: + os.close(ready_write_fd) + except OSError: + pass + raise + + # The launcher is the future provider process: exec preserves both its + # PID and its session/process-group identity. + process_group_id = process.pid + try: + reservation.mark_started( + pid=process.pid, + process_group_id=process_group_id, + ) + if ( + os.write( + ready_write_fd, + _BENCHMARK_LAUNCH_READY_BYTE, + ) + != len(_BENCHMARK_LAUNCH_READY_BYTE) + ): + raise OSError("Benchmark provider launch handoff was incomplete.") + except BaseException: + try: + os.close(ready_write_fd) + except OSError: + pass + self._terminate_process_group( + process, + process_group_id=process_group_id, + grace_seconds=float(self.execution_policy.kill_grace_seconds), + ) + raise + else: + try: + os.close(ready_write_fd) + except OSError: + pass + + timeout_seconds = float(self.execution_policy.call_timeout_seconds or 0) + try: + stdout, stderr = process.communicate( + input=stdin_input, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired: + stdout, stderr = self._terminate_process_group( + process, + process_group_id=process_group_id, + grace_seconds=float(self.execution_policy.kill_grace_seconds), + ) + completed_process = subprocess.CompletedProcess( + command, + process.returncode, + stdout, + stderr, + ) + raise ModelInvocationTimeout( + timeout_seconds, + completed_process=completed_process, + ) + self._stop_remaining_process_group( + process_group_id, + grace_seconds=float(self.execution_policy.kill_grace_seconds), + ) + return subprocess.CompletedProcess( + command, + process.returncode, + str(stdout or ""), + str(stderr or ""), + ) + + @staticmethod + def _reservation_result( + *, + completed: bool, + process: Any, + captured_error: BaseException | None, + ) -> tuple[str, str]: + if isinstance(captured_error, ModelInvocationTimeout): + return "timeout", "timeout" + if process is None: + return "launch_failed", ( + normalized_error_category(captured_error) or "launch_failed" + ) + if process.returncode not in (None, 0): + return "failed", "nonzero_exit" + if captured_error is not None or not completed: + return "failed", ( + normalized_error_category(captured_error, exit_code=process.returncode) + or "runner_error" + ) + return "completed", "completed" + def _build_command( self, *, workspace: Path, prompt: str, session_id: str | None, - output_file: Path, + output_file: Path | None, bypass_sandbox: bool, ) -> tuple[list[str], str | None]: is_gemini = "gemini" in self.runtime_config.model.lower() if is_gemini: + if self.execution_policy.benchmark_mode: + raise ModelExecutionPolicyViolation( + "Benchmark execution currently supports only the Codex CLI because " + "Gemini cannot provide the same non-interactive workspace-write policy." + ) command = ["gemini"] if session_id: command.extend(["--resume", session_id]) @@ -117,44 +684,46 @@ def _build_command( command.extend(["--prompt", prompt]) return command, None - command = ["codex", "exec"] + command = [self._codex_executable(), "exec"] if session_id: - command.extend( - [ - "resume", - "--model", - self.runtime_config.model, - "-o", - str(output_file), - "--skip-git-repo-check", - ] - ) - if bypass_sandbox: + command.extend(["resume", "--model", self.runtime_config.model]) + if not self.execution_policy.benchmark_mode: + if output_file is None: + raise ValueError("Codex output path is required") + command.extend(["-o", str(output_file)]) + command.append("--skip-git-repo-check") + if self.execution_policy.benchmark_mode: + self._append_benchmark_codex_controls( + command, + supports_sandbox_option=False, + ) + elif bypass_sandbox: command.append("--dangerously-bypass-approvals-and-sandbox") else: command.append("--full-auto") + command.append("--json") command.extend(["-c", f'model_reasoning_effort="{self.runtime_config.reasoning}"']) command.extend(["-c", 'personality="friendly"']) command.extend([session_id, "-"]) return command, prompt - command.extend( - [ - "-", - "--model", - self.runtime_config.model, - "-o", - str(output_file), - "--skip-git-repo-check", - "-C", - str(workspace), - ] - ) + command.extend(["-", "--model", self.runtime_config.model]) + if not self.execution_policy.benchmark_mode: + if output_file is None: + raise ValueError("Codex output path is required") + command.extend(["-o", str(output_file)]) + command.extend(["--skip-git-repo-check", "-C", str(workspace)]) for extra_dir in self._discover_extra_writable_dirs(workspace): command.extend(["--add-dir", extra_dir]) - if bypass_sandbox: + if self.execution_policy.benchmark_mode: + self._append_benchmark_codex_controls( + command, + supports_sandbox_option=True, + ) + elif bypass_sandbox: command.append("--dangerously-bypass-approvals-and-sandbox") else: command.append("--full-auto") + command.append("--json") command.extend(["-c", f'model_reasoning_effort="{self.runtime_config.reasoning}"']) command.extend(["-c", 'personality="friendly"']) return command, prompt @@ -166,13 +735,32 @@ def run( session_id: str | None, *, bypass_sandbox: bool = False, + invocation_context: ModelInvocationContext | None = None, ) -> tuple[str, str | None]: abs_workspace = workspace.expanduser().resolve() - output_file = abs_workspace / ".teams_runtime_codex_output.txt" - try: - output_file.unlink() - except FileNotFoundError: - pass + self.execution_policy.assert_workspace_allowed(abs_workspace) + if self.execution_policy.benchmark_mode and bypass_sandbox: + raise ModelExecutionPolicyViolation( + "Benchmark execution forbids sandbox bypass requests." + ) + if self.execution_policy.benchmark_mode: + for directory_key in ("HOME", "CODEX_HOME", "TMPDIR", "TMP", "TEMP"): + directory_value = self.execution_policy.shell_environment.get(directory_key) + if not directory_value: + continue + directory_path = Path(directory_value).expanduser().resolve() + self.execution_policy.assert_workspace_allowed(directory_path) + directory_path.mkdir(mode=0o700, parents=True, exist_ok=True) + output_file = ( + None + if self.execution_policy.benchmark_mode + else abs_workspace / ".teams_runtime_codex_output.txt" + ) + if output_file is not None: + try: + output_file.unlink() + except FileNotFoundError: + pass command, stdin_input = self._build_command( workspace=abs_workspace, prompt=prompt, @@ -182,7 +770,7 @@ def run( ) is_gemini = "gemini" in self.runtime_config.model.lower() - env = {**os.environ, "HOME": str(Path.home())} + env = self._provider_environment() if is_gemini: env["GEMINI_SYSTEM_MD"] = str(abs_workspace / "GEMINI.md") gemini_dir = abs_workspace / ".gemini" @@ -195,54 +783,134 @@ def run( except OSError: pass - process = subprocess.run( - command, - cwd=str(abs_workspace), - capture_output=True, - input=stdin_input, - text=True, - env=env, - check=False, - ) - + started_at = runtime_now() + started_monotonic = time.monotonic() + process = None output = "" resolved_session_id = session_id - - if is_gemini: - try: - res_json = json.loads(process.stdout) - output = res_json.get("response", "").strip() - resolved_session_id = res_json.get("session_id") or res_json.get("sessionId") or session_id - if not output and res_json.get("error"): - error_info = res_json.get("error") - output = error_info.get("message") if isinstance(error_info, dict) else str(error_info) - except json.JSONDecodeError: - output = process.stdout.strip() or process.stderr.strip() - else: - combined = "\n".join(part for part in [process.stdout.strip(), process.stderr.strip()] if part).strip() - session_match = SESSION_ID_PATTERN.search(combined) - resolved_session_id = session_match.group(1).strip() if session_match else session_id - if output_file.exists(): - output = output_file.read_text(encoding="utf-8").strip() - if not output: - output = process.stdout.strip() or combined - - if process.returncode != 0: - cli_name = "Gemini" if is_gemini else "Codex" - if output: + usage = ModelUsage() + captured_error: BaseException | None = None + completed = False + reservation: InvocationReservation | None = None + try: + if self.execution_policy.benchmark_mode: + budget = self.execution_policy.invocation_budget + if budget is None: + raise ModelExecutionPolicyViolation( + "Benchmark execution has no invocation budget." + ) + reservation = budget.reserve( + invocation_context, + provider="gemini_cli" if is_gemini else "codex_cli", + role=self.role, + ) try: - extract_json_object(output) - except ValueError: - raise RuntimeError(output or f"{cli_name} command failed.") - LOGGER.warning( - "[%s] %s command exited with code %s but produced a valid JSON payload; preserving role result", - self.role, - cli_name, - process.returncode, + process = self._run_benchmark_process( + command, + cwd=abs_workspace, + stdin_input=stdin_input, + env=env, + reservation=reservation, + ) + except ModelInvocationTimeout as exc: + process = exc.completed_process + self._quarantine_workspace_after_provider(abs_workspace) + raise + self._quarantine_workspace_after_provider(abs_workspace) + else: + process = subprocess.run( + command, + cwd=str(abs_workspace), + capture_output=True, + input=stdin_input, + text=True, + env=env, + check=False, ) + if is_gemini: + try: + res_json = json.loads(process.stdout) + output = str(res_json.get("response") or "").strip() + resolved_session_id = res_json.get("session_id") or res_json.get("sessionId") or session_id + usage = parse_gemini_usage(res_json.get("stats")) + if not output and res_json.get("error"): + error_info = res_json.get("error") + output = error_info.get("message") if isinstance(error_info, dict) else str(error_info) + except json.JSONDecodeError: + output = process.stdout.strip() or process.stderr.strip() else: - raise RuntimeError(f"{cli_name} command failed.") - return output, resolved_session_id + combined = "\n".join(part for part in [process.stdout.strip(), process.stderr.strip()] if part).strip() + event_session_id, usage, final_message = parse_codex_jsonl(process.stdout) + session_match = SESSION_ID_PATTERN.search(combined) + resolved_session_id = event_session_id or (session_match.group(1).strip() if session_match else session_id) + if output_file is not None and output_file.exists(): + output = output_file.read_text(encoding="utf-8").strip() + if not output: + output = final_message or process.stderr.strip() or process.stdout.strip() + + if process.returncode != 0: + cli_label = "Gemini" if is_gemini else "Codex" + if output: + try: + extract_json_object(output) + except ValueError: + raise RuntimeError(output or f"{cli_label} command failed.") + LOGGER.warning( + "[%s] %s command exited with code %s but produced a valid JSON payload; preserving role result", + self.role, + cli_label, + process.returncode, + ) + else: + raise RuntimeError(f"{cli_label} command failed.") + completed = True + return output, resolved_session_id + except BaseException as exc: + captured_error = exc + raise + finally: + if reservation is not None: + reservation_state, stop_reason = self._reservation_result( + completed=completed, + process=process, + captured_error=captured_error, + ) + reservation.complete( + state=reservation_state, + exit_code=process.returncode if process is not None else None, + stop_reason=stop_reason, + ) + should_record_telemetry = ( + not self.execution_policy.benchmark_mode or reservation is not None + ) + if ( + should_record_telemetry + and self.telemetry_recorder is not None + and invocation_context is not None + ): + cli_name = "gemini" if is_gemini else "codex" + exit_code = process.returncode if process is not None else None + ended_at = runtime_now() + duration_ms = int((time.monotonic() - started_monotonic) * 1000) + cli_version = self._cli_version(cli_name) + self.telemetry_recorder.record( + invocation_context, + provider="gemini_cli" if is_gemini else "codex_cli", + model=self.runtime_config.model, + reasoning="" if is_gemini else self.runtime_config.reasoning, + cli_version=cli_version, + started_at=started_at, + ended_at=ended_at, + duration_ms=duration_ms, + session_id_before=session_id, + session_id_after=resolved_session_id, + status="completed" if completed else "failed", + exit_code=exit_code, + error_category=normalized_error_category(captured_error, exit_code=exit_code), + prompt_chars=len(prompt), + output_chars=len(output), + usage=usage, + ) -__all__ = ["CodexRunner", "extract_json_object"] +__all__ = ["CodexRunner", "extract_json_object", "parse_codex_jsonl", "parse_gemini_usage"] diff --git a/runtime/execution_policy.py b/runtime/execution_policy.py new file mode 100644 index 0000000..0352596 --- /dev/null +++ b/runtime/execution_policy.py @@ -0,0 +1,638 @@ +from __future__ import annotations + +import json +import math +import os +import re +import stat +import tempfile +import threading +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from types import MappingProxyType +from typing import Any, Mapping + + +_ENVIRONMENT_NAME_PATTERN = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") +_SENSITIVE_ENVIRONMENT_NAME_PATTERN = re.compile( + r"(?:^|_)(?:API_?KEY|AUTH|CREDENTIALS?|PASSWORD|SECRET|TOKEN)(?:$|_)", + re.IGNORECASE, +) +_TERMINAL_STATES = { + "completed", + "failed", + "timeout", + "launch_failed", + "terminated", +} + + +def _utc_timestamp() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _positive_finite_number(value: Any, *, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a positive finite number.") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a positive finite number.") from exc + if not math.isfinite(normalized) or normalized <= 0: + raise ValueError(f"{name} must be a positive finite number.") + return normalized + + +def _non_negative_finite_number(value: Any, *, name: str) -> float: + if isinstance(value, bool): + raise ValueError(f"{name} must be a non-negative finite number.") + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{name} must be a non-negative finite number.") from exc + if not math.isfinite(normalized) or normalized < 0: + raise ValueError(f"{name} must be a non-negative finite number.") + return normalized + + +def _optional_non_negative_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + normalized = int(value) + except (OverflowError, TypeError, ValueError): + return None + return normalized if normalized >= 0 else None + + +class InvocationBudgetExceeded(RuntimeError): + def __init__(self, max_invocations: int, reserved_count: int): + self.max_invocations = max_invocations + self.reserved_count = reserved_count + super().__init__( + f"Model invocation budget exhausted: {reserved_count}/{max_invocations} calls are already reserved." + ) + + +class ModelExecutionPolicyViolation(RuntimeError): + """Raised before launch when a benchmark request violates its safety policy.""" + + +class ModelInvocationTimeout(RuntimeError, TimeoutError): + def __init__( + self, + timeout_seconds: float, + *, + completed_process: Any = None, + ): + self.timeout_seconds = timeout_seconds + self.completed_process = completed_process + super().__init__(f"Model invocation exceeded the {timeout_seconds:g}-second timeout.") + + +def quarantine_unsafe_workspace_entries( + workspace_root: str | os.PathLike[str], + *, + max_entries: int = 100_000, +) -> tuple[str, ...]: + """Remove filesystem entries that could redirect trusted writes outside a benchmark. + + The provider is stopped before this sweep runs, so validation and removal do not + race model-owned processes. Internal symlinks are retained because session + workspaces intentionally use them; outward/broken-loop symlinks, hard-linked + regular files, and special files are quarantined. + """ + + if isinstance(max_entries, bool) or not isinstance(max_entries, int) or max_entries <= 0: + raise ValueError("max_entries must be a positive integer.") + try: + root = Path(workspace_root).expanduser().resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ModelExecutionPolicyViolation( + "Benchmark workspace could not be resolved for its integrity sweep." + ) from exc + if not root.is_dir(): + raise ModelExecutionPolicyViolation( + "Benchmark workspace integrity sweep requires a directory." + ) + + unsafe: list[tuple[Path, str]] = [] + pending = [root] + entry_count = 0 + while pending: + directory = pending.pop() + try: + entries = os.scandir(directory) + except OSError as exc: + raise ModelExecutionPolicyViolation( + "Benchmark workspace integrity sweep could not inspect a directory." + ) from exc + with entries: + for entry in entries: + entry_count += 1 + if entry_count > max_entries: + raise ModelExecutionPolicyViolation( + "Benchmark workspace exceeded the integrity sweep entry limit." + ) + path = Path(entry.path) + try: + metadata = entry.stat(follow_symlinks=False) + except OSError as exc: + raise ModelExecutionPolicyViolation( + "Benchmark workspace integrity sweep could not inspect an entry." + ) from exc + mode = metadata.st_mode + if stat.S_ISLNK(mode): + try: + target = path.resolve(strict=False) + except (OSError, RuntimeError): + unsafe.append((path, "unresolvable_symlink")) + continue + if not target.is_relative_to(root): + unsafe.append((path, "outward_symlink")) + continue + if stat.S_ISDIR(mode): + pending.append(path) + continue + if stat.S_ISREG(mode): + if metadata.st_nlink != 1: + unsafe.append((path, "hard_link")) + continue + unsafe.append((path, "special_file")) + + removed_kinds: list[str] = [] + for path, kind in sorted(unsafe, key=lambda item: len(item[0].parts), reverse=True): + try: + path.unlink() + except FileNotFoundError: + continue + except OSError as exc: + raise ModelExecutionPolicyViolation( + "Benchmark workspace contained an unsafe entry that could not be quarantined." + ) from exc + removed_kinds.append(kind) + return tuple(sorted(removed_kinds)) + + +class InvocationReservation: + __slots__ = ("_budget", "reservation_id") + + def __init__(self, budget: "InvocationBudget", reservation_id: str): + self._budget = budget + self.reservation_id = reservation_id + + def mark_started(self, *, pid: int, process_group_id: int | None) -> None: + self._budget._mark_started( # noqa: SLF001 - reservation is the budget's public mutation handle + self.reservation_id, + pid=pid, + process_group_id=process_group_id, + ) + + def complete( + self, + *, + state: str, + exit_code: int | None, + stop_reason: str, + ) -> None: + self._budget._complete( # noqa: SLF001 - reservation is the budget's public mutation handle + self.reservation_id, + state=state, + exit_code=exit_code, + stop_reason=stop_reason, + ) + + +class InvocationBudget: + """Thread-safe physical-call budget with an atomic, privacy-safe journal.""" + + def __init__( + self, + max_invocations: int, + *, + journal_path: str | os.PathLike[str] | None = None, + ): + if isinstance(max_invocations, bool) or not isinstance(max_invocations, int) or max_invocations <= 0: + raise ValueError("max_invocations must be a positive integer.") + self.max_invocations = max_invocations + self.journal_path = ( + Path(journal_path).expanduser().resolve() + if journal_path is not None + else None + ) + self._lock = threading.RLock() + self._entries: list[dict[str, Any]] = [] + self._entries_by_id: dict[str, dict[str, Any]] = {} + self._rejected_count = 0 + + @property + def reserved_count(self) -> int: + with self._lock: + return len(self._entries) + + @property + def remaining(self) -> int: + with self._lock: + return max(self.max_invocations - len(self._entries), 0) + + @property + def rejected_count(self) -> int: + with self._lock: + return self._rejected_count + + def reserve( + self, + invocation_context: Any = None, + *, + provider: str, + role: str = "", + ) -> InvocationReservation: + with self._lock: + if len(self._entries) >= self.max_invocations: + self._rejected_count += 1 + self._persist_locked() + raise InvocationBudgetExceeded(self.max_invocations, len(self._entries)) + + reservation_id = uuid.uuid4().hex + entry = { + "reservation_id": reservation_id, + "provider": str(provider or "").strip(), + "invocation_id": str(getattr(invocation_context, "invocation_id", "") or "").strip(), + "operation_id": str(getattr(invocation_context, "operation_id", "") or "").strip(), + "logical_call_id": str(getattr(invocation_context, "logical_call_id", "") or "").strip(), + "attempt_index": getattr(invocation_context, "attempt_index", None), + "attempt_kind": str(getattr(invocation_context, "attempt_kind", "") or "").strip(), + "runtime_identity": str( + getattr(invocation_context, "runtime_identity", "") or "" + ).strip(), + "role": str(getattr(invocation_context, "role", "") or role or "").strip(), + "purpose": str(getattr(invocation_context, "purpose", "") or "").strip(), + "workflow_step": str( + getattr(invocation_context, "workflow_step", "") or "" + ).strip(), + "request_id": str(getattr(invocation_context, "request_id", "") or "").strip(), + "sprint_id": str(getattr(invocation_context, "sprint_id", "") or "").strip(), + "todo_id": str(getattr(invocation_context, "todo_id", "") or "").strip(), + "backlog_id": str(getattr(invocation_context, "backlog_id", "") or "").strip(), + "goal_id": str(getattr(invocation_context, "goal_id", "") or "").strip(), + "prompt_context_enabled": ( + getattr(invocation_context, "prompt_context_enabled", None) + if isinstance( + getattr(invocation_context, "prompt_context_enabled", None), + bool, + ) + else None + ), + "prompt_context_total_events": _optional_non_negative_int( + getattr(invocation_context, "prompt_context_total_events", None) + ), + "prompt_context_included_events": _optional_non_negative_int( + getattr(invocation_context, "prompt_context_included_events", None) + ), + "prompt_context_omitted_events": _optional_non_negative_int( + getattr(invocation_context, "prompt_context_omitted_events", None) + ), + "prompt_context_recent_events": _optional_non_negative_int( + getattr(invocation_context, "prompt_context_recent_events", None) + ), + "prompt_context_max_events": _optional_non_negative_int( + getattr(invocation_context, "prompt_context_max_events", None) + ), + "prompt_context_selection_policy": str( + getattr( + invocation_context, + "prompt_context_selection_policy", + "", + ) + or "" + ).strip(), + "state": "reserved", + "reserved_at": _utc_timestamp(), + "started_at": "", + "completed_at": "", + "pid": None, + "process_group_id": None, + "exit_code": None, + "stop_reason": "", + } + self._entries.append(entry) + self._entries_by_id[reservation_id] = entry + self._persist_locked() + return InvocationReservation(self, reservation_id) + + def snapshot(self) -> dict[str, Any]: + with self._lock: + return self._snapshot_locked() + + def _snapshot_locked(self) -> dict[str, Any]: + return { + "schema_version": 3, + "max_invocations": self.max_invocations, + "reserved_count": len(self._entries), + "remaining": max(self.max_invocations - len(self._entries), 0), + "rejected_count": self._rejected_count, + "entries": [dict(entry) for entry in self._entries], + } + + def _mark_started( + self, + reservation_id: str, + *, + pid: int, + process_group_id: int | None, + ) -> None: + with self._lock: + entry = self._entries_by_id[reservation_id] + if entry["state"] != "reserved": + raise RuntimeError(f"Invocation reservation {reservation_id} is already started.") + entry.update( + { + "state": "running", + "started_at": _utc_timestamp(), + "pid": int(pid), + "process_group_id": ( + int(process_group_id) + if process_group_id is not None + else None + ), + } + ) + self._persist_locked() + + def _complete( + self, + reservation_id: str, + *, + state: str, + exit_code: int | None, + stop_reason: str, + ) -> None: + normalized_state = str(state or "").strip() + if normalized_state not in _TERMINAL_STATES: + raise ValueError(f"Unsupported invocation terminal state: {state}") + with self._lock: + entry = self._entries_by_id[reservation_id] + if entry["state"] in _TERMINAL_STATES: + return + entry.update( + { + "state": normalized_state, + "completed_at": _utc_timestamp(), + "exit_code": int(exit_code) if exit_code is not None else None, + "stop_reason": str(stop_reason or "").strip(), + } + ) + self._persist_locked() + + def _persist_locked(self) -> None: + if self.journal_path is None: + return + journal_path = self.journal_path + journal_path.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + try: + journal_path.parent.chmod(0o700) + except OSError: + pass + descriptor, temporary_name = tempfile.mkstemp( + dir=str(journal_path.parent), + prefix=f".{journal_path.name}.", + suffix=".tmp", + ) + temporary_path = Path(temporary_name) + try: + try: + os.fchmod(descriptor, 0o600) + except OSError: + pass + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + json.dump( + self._snapshot_locked(), + handle, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary_path, journal_path) + try: + journal_path.chmod(0o600) + except OSError: + pass + except BaseException: + try: + os.close(descriptor) + except OSError: + pass + try: + temporary_path.unlink() + except FileNotFoundError: + pass + raise + + +def _normalize_shell_environment(values: Mapping[str, str]) -> Mapping[str, str]: + normalized: dict[str, str] = {} + for raw_name, raw_value in values.items(): + name = str(raw_name or "").strip() + if not _ENVIRONMENT_NAME_PATTERN.fullmatch(name): + raise ValueError(f"Invalid shell environment variable name: {raw_name!r}") + if _SENSITIVE_ENVIRONMENT_NAME_PATTERN.search(name): + raise ValueError( + f"Benchmark shell environment must not expose secret-bearing variable {name}." + ) + if not isinstance(raw_value, str): + raise ValueError(f"Benchmark shell environment value for {name} must be a string.") + if "\x00" in raw_value or "\n" in raw_value or "\r" in raw_value: + raise ValueError(f"Benchmark shell environment value for {name} contains control characters.") + normalized[name] = raw_value + return MappingProxyType(dict(sorted(normalized.items()))) + + +@dataclass(slots=True, frozen=True) +class ModelExecutionPolicy: + """Immutable opt-in controls for benchmark model execution.""" + + benchmark_mode: bool = False + call_timeout_seconds: float | None = None + kill_grace_seconds: float = 5.0 + invocation_budget: InvocationBudget | None = field(default=None, compare=False) + allowed_workspace_root: Path | None = None + telemetry_output_dir: Path | None = field(default=None, compare=False) + codex_executable: Path | None = field(default=None, compare=False) + execution_lock: threading.RLock | None = field( + default=None, + compare=False, + hash=False, + repr=False, + ) + shell_environment: Mapping[str, str] = field( + default_factory=lambda: MappingProxyType({}), + hash=False, + ) + + def __post_init__(self) -> None: + normalized_environment = _normalize_shell_environment(self.shell_environment) + object.__setattr__(self, "shell_environment", normalized_environment) + object.__setattr__( + self, + "kill_grace_seconds", + _non_negative_finite_number( + self.kill_grace_seconds, + name="kill_grace_seconds", + ), + ) + if not self.benchmark_mode: + if ( + self.call_timeout_seconds is not None + or self.invocation_budget is not None + or self.allowed_workspace_root is not None + or self.telemetry_output_dir is not None + or self.codex_executable is not None + or self.execution_lock is not None + or normalized_environment + ): + raise ValueError( + "Bounded execution controls require benchmark_mode=True; use " + "ModelExecutionPolicy.for_benchmark()." + ) + return + + object.__setattr__( + self, + "call_timeout_seconds", + _positive_finite_number( + self.call_timeout_seconds, + name="call_timeout_seconds", + ), + ) + if self.invocation_budget is None: + raise ValueError("Benchmark execution requires an invocation budget.") + if self.execution_lock is None: + raise ValueError("Benchmark execution requires a shared execution lock.") + if self.allowed_workspace_root is None: + raise ValueError("Benchmark execution requires an allowed workspace root.") + allowed_root = Path(self.allowed_workspace_root).expanduser().resolve() + object.__setattr__(self, "allowed_workspace_root", allowed_root) + if self.telemetry_output_dir is not None: + telemetry_output_dir = ( + Path(self.telemetry_output_dir).expanduser().resolve() + ) + if telemetry_output_dir.is_relative_to(allowed_root): + raise ValueError( + "Benchmark telemetry output must be outside the provider-writable workspace." + ) + object.__setattr__( + self, + "telemetry_output_dir", + telemetry_output_dir, + ) + if self.codex_executable is None: + raise ValueError( + "Benchmark execution requires a pinned Codex executable." + ) + try: + codex_executable = ( + Path(self.codex_executable).expanduser().resolve(strict=True) + ) + except (OSError, RuntimeError) as exc: + raise ValueError( + "Benchmark Codex executable could not be resolved safely." + ) from exc + if not codex_executable.is_file() or not os.access( + codex_executable, + os.X_OK, + ): + raise ValueError( + "Benchmark Codex executable must be a regular executable file." + ) + if codex_executable.is_relative_to(allowed_root): + raise ValueError( + "Benchmark Codex executable must be outside the provider-writable workspace." + ) + if ( + self.telemetry_output_dir is not None + and codex_executable.is_relative_to( + self.telemetry_output_dir.parent + ) + ): + raise ValueError( + "Benchmark Codex executable must be outside the protected run output." + ) + object.__setattr__(self, "codex_executable", codex_executable) + + @classmethod + def for_benchmark( + cls, + *, + allowed_workspace_root: str | os.PathLike[str], + invocation_budget: InvocationBudget, + call_timeout_seconds: float, + codex_executable: str | os.PathLike[str], + kill_grace_seconds: float = 5.0, + shell_environment: Mapping[str, str] | None = None, + telemetry_output_dir: str | os.PathLike[str] | None = None, + ) -> "ModelExecutionPolicy": + allowed_root = Path(allowed_workspace_root).expanduser().resolve() + provider_state_root = allowed_root / ".teams_runtime" / "benchmark_provider" + provider_tmp = provider_state_root / "tmp" + environment = { + "CODEX_HOME": str(provider_state_root / "codex_home"), + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_TERMINAL_PROMPT": "0", + "HOME": str(provider_state_root / "home"), + "PATH": os.environ.get("PATH") or os.defpath, + "TEMP": str(provider_tmp), + "TMP": str(provider_tmp), + "TMPDIR": str(provider_tmp), + } + environment.update(shell_environment or {}) + return cls( + benchmark_mode=True, + call_timeout_seconds=call_timeout_seconds, + kill_grace_seconds=kill_grace_seconds, + invocation_budget=invocation_budget, + allowed_workspace_root=allowed_root, + codex_executable=Path(codex_executable).expanduser(), + telemetry_output_dir=( + Path(telemetry_output_dir).expanduser().resolve() + if telemetry_output_dir is not None + else None + ), + execution_lock=threading.RLock(), + shell_environment=environment, + ) + + def assert_workspace_allowed(self, workspace: Path) -> None: + if not self.benchmark_mode: + return + allowed_root = self.allowed_workspace_root + try: + resolved_workspace = Path(workspace).expanduser().resolve() + except (OSError, RuntimeError) as exc: + raise ModelExecutionPolicyViolation( + f"Benchmark workspace {workspace} could not be resolved safely." + ) from exc + if allowed_root is None or not resolved_workspace.is_relative_to(allowed_root): + raise ModelExecutionPolicyViolation( + f"Benchmark workspace {resolved_workspace} is outside the allowed root {allowed_root}." + ) + + +DEFAULT_MODEL_EXECUTION_POLICY = ModelExecutionPolicy() + + +__all__ = [ + "DEFAULT_MODEL_EXECUTION_POLICY", + "InvocationBudget", + "InvocationBudgetExceeded", + "InvocationReservation", + "ModelExecutionPolicy", + "ModelExecutionPolicyViolation", + "ModelInvocationTimeout", + "quarantine_unsafe_workspace_entries", +] diff --git a/runtime/internal/goal_sourcing.py b/runtime/internal/goal_sourcing.py index cfee225..c79c4ff 100644 --- a/runtime/internal/goal_sourcing.py +++ b/runtime/internal/goal_sourcing.py @@ -8,9 +8,15 @@ from typing import Any from teams_runtime.runtime.codex_runner import CodexRunner, extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.identities import service_runtime_identity +from teams_runtime.runtime.model_telemetry import ( + InvocationSequence, + ModelTelemetryRecorder, + run_with_optional_telemetry, +) from teams_runtime.runtime.session_manager import RoleSessionManager -from teams_runtime.shared.models import RoleRuntimeConfig +from teams_runtime.shared.models import RoleRuntimeConfig, TelemetryRuntimeConfig from teams_runtime.shared.paths import RuntimePaths from teams_runtime.shared.persistence import utc_now_iso @@ -221,6 +227,8 @@ def __init__( sprint_id: str, runtime_config: RoleRuntimeConfig, session_identity: str | None = None, + telemetry_config: TelemetryRuntimeConfig | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.paths = paths self.role = "sourcer" @@ -233,8 +241,28 @@ def __init__( agent_root=paths.internal_agent_root("sourcer"), runtime_identity=self.runtime_identity, ) - self.codex_runner = CodexRunner(runtime_config, role=self.role) - self._run_lock = threading.Lock() + self.telemetry_recorder = ModelTelemetryRecorder( + paths, + self.runtime_identity, + telemetry_config, + output_dir=( + execution_policy.telemetry_output_dir + if execution_policy is not None + else None + ), + ) + self.codex_runner = CodexRunner( + runtime_config, + role=self.role, + telemetry_recorder=self.telemetry_recorder, + execution_policy=execution_policy, + ) + self._run_lock = ( + execution_policy.execution_lock + if execution_policy is not None + and execution_policy.execution_lock is not None + else threading.Lock() + ) def source( self, @@ -283,10 +311,19 @@ def source( state.workspace_path, state.session_id or "new", ) - output, session_id = self.codex_runner.run( + invocation_sequence = InvocationSequence( + runtime_identity=self.runtime_identity, + role=self.role, + purpose="goal_sourcing", + sprint_id=str(current_sprint.get("sprint_id") or self.sprint_id), + goal_id=str(goal_state.get("goal_id") or ""), + ) + output, session_id = run_with_optional_telemetry( + self.codex_runner, Path(state.workspace_path), prompt, state.session_id or None, + invocation_context=invocation_sequence.next("primary"), ) except Exception: monitoring["codex_run_status"] = "failed" diff --git a/runtime/internal/intent_parser.py b/runtime/internal/intent_parser.py index e3760de..d693000 100644 --- a/runtime/internal/intent_parser.py +++ b/runtime/internal/intent_parser.py @@ -9,9 +9,15 @@ from teams_runtime.workflows.orchestration.ingress import is_manual_sprint_finalize_text, is_manual_sprint_start_text from teams_runtime.shared.paths import RuntimePaths from teams_runtime.runtime.codex_runner import CodexRunner, extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.identities import service_runtime_identity +from teams_runtime.runtime.model_telemetry import ( + InvocationSequence, + ModelTelemetryRecorder, + run_with_optional_telemetry, +) from teams_runtime.runtime.session_manager import RoleSessionManager -from teams_runtime.shared.models import MessageEnvelope, RoleRuntimeConfig +from teams_runtime.shared.models import MessageEnvelope, RoleRuntimeConfig, TelemetryRuntimeConfig REQUEST_ID_TEXT_PATTERN = re.compile(r"\brequest[_\s-]*id\s*[:=]?\s*([A-Za-z0-9._-]+)", re.IGNORECASE) @@ -196,6 +202,8 @@ def __init__( sprint_id: str, runtime_config: RoleRuntimeConfig, session_identity: str | None = None, + telemetry_config: TelemetryRuntimeConfig | None = None, + execution_policy: ModelExecutionPolicy | None = None, ): self.paths = paths self.role = "parser" @@ -208,8 +216,28 @@ def __init__( agent_root=paths.internal_agent_root("parser"), runtime_identity=self.runtime_identity, ) - self.codex_runner = CodexRunner(runtime_config, role=self.role) - self._run_lock = threading.Lock() + self.telemetry_recorder = ModelTelemetryRecorder( + paths, + self.runtime_identity, + telemetry_config, + output_dir=( + execution_policy.telemetry_output_dir + if execution_policy is not None + else None + ), + ) + self.codex_runner = CodexRunner( + runtime_config, + role=self.role, + telemetry_recorder=self.telemetry_recorder, + execution_policy=execution_policy, + ) + self._run_lock = ( + execution_policy.execution_lock + if execution_policy is not None + and execution_policy.execution_lock is not None + else threading.Lock() + ) def classify( self, @@ -231,7 +259,20 @@ def classify( backlog_counts=backlog_counts, forwarded=forwarded, ) - output, session_id = self.codex_runner.run(Path(state.workspace_path), prompt, state.session_id or None) + invocation_sequence = InvocationSequence( + runtime_identity=self.runtime_identity, + role=self.role, + purpose="intent_classification", + request_id=str(envelope.request_id or ""), + sprint_id=str(active_sprint.get("sprint_id") or self.sprint_id), + ) + output, session_id = run_with_optional_telemetry( + self.codex_runner, + Path(state.workspace_path), + prompt, + state.session_id or None, + invocation_context=invocation_sequence.next("primary"), + ) state = self.session_manager.finalize_session_id(state, session_id) try: payload = extract_json_object(output) diff --git a/runtime/model_telemetry.py b/runtime/model_telemetry.py new file mode 100644 index 0000000..95996f2 --- /dev/null +++ b/runtime/model_telemetry.py @@ -0,0 +1,865 @@ +from __future__ import annotations + +import hashlib +import inspect +import json +import logging +import math +import os +import time +import uuid +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Iterable + +from teams_runtime.runtime.identities import sanitize_runtime_identity +from teams_runtime.shared.models import ModelRateCard, TelemetryRuntimeConfig +from teams_runtime.shared.paths import RuntimePaths +from teams_runtime.shared.persistence import RUNTIME_TIMEZONE, normalize_runtime_datetime, runtime_now, runtime_now_iso + + +LOGGER = logging.getLogger(__name__) +TELEMETRY_SCHEMA_VERSION = 1 +TELEMETRY_WARNING_INTERVAL_SECONDS = 60.0 +VALID_ATTEMPT_KINDS = {"primary", "sandbox_retry", "contract_repair"} + + +def _text(value: Any) -> str: + return str(value or "").strip() + + +def _optional_non_negative_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + normalized = int(value) + except (OverflowError, TypeError, ValueError): + return None + return normalized if normalized >= 0 else None + + +def _timestamp(value: datetime | None = None) -> str: + return normalize_runtime_datetime(value).isoformat() if value is not None else runtime_now_iso() + + +def hash_session_id(value: str | None) -> str: + normalized = _text(value) + if not normalized: + return "" + return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16] + + +def normalized_error_category(exc: BaseException | None, *, exit_code: int | None = None) -> str: + if exc is None and exit_code in (None, 0): + return "" + if isinstance(exc, FileNotFoundError): + return "cli_not_found" + if isinstance(exc, TimeoutError): + return "timeout" + if exit_code not in (None, 0): + return "nonzero_exit" + if isinstance(exc, (json.JSONDecodeError, ValueError)): + return "provider_output_invalid" + return "provider_exception" if exc is not None else "unknown" + + +@dataclass(slots=True, frozen=True) +class ModelUsage: + input_tokens: int | None = None + cached_input_tokens: int | None = None + output_tokens: int | None = None + reasoning_output_tokens: int | None = None + total_tokens: int | None = None + tool_calls: int | None = None + source: str = "unavailable" + + @classmethod + def from_values( + cls, + *, + input_tokens: Any = None, + cached_input_tokens: Any = None, + output_tokens: Any = None, + reasoning_output_tokens: Any = None, + total_tokens: Any = None, + tool_calls: Any = None, + ) -> "ModelUsage": + values = { + "input_tokens": _optional_non_negative_int(input_tokens), + "cached_input_tokens": _optional_non_negative_int(cached_input_tokens), + "output_tokens": _optional_non_negative_int(output_tokens), + "reasoning_output_tokens": _optional_non_negative_int(reasoning_output_tokens), + "total_tokens": _optional_non_negative_int(total_tokens), + "tool_calls": _optional_non_negative_int(tool_calls), + } + if values["total_tokens"] is None and values["input_tokens"] is not None and values["output_tokens"] is not None: + values["total_tokens"] = values["input_tokens"] + values["output_tokens"] + token_values = tuple(values[name] for name in values if name != "tool_calls") + return cls(**values, source="native" if any(value is not None for value in token_values) else "unavailable") + + +@dataclass(slots=True, frozen=True) +class ModelInvocationContext: + invocation_id: str + operation_id: str + logical_call_id: str + attempt_index: int + attempt_kind: str + runtime_identity: str + role: str + purpose: str + workflow_step: str = "" + request_id: str = "" + sprint_id: str = "" + todo_id: str = "" + backlog_id: str = "" + goal_id: str = "" + prompt_context_enabled: bool | None = None + prompt_context_total_events: int | None = None + prompt_context_included_events: int | None = None + prompt_context_omitted_events: int | None = None + prompt_context_recent_events: int | None = None + prompt_context_max_events: int | None = None + prompt_context_selection_policy: str = "" + + +class InvocationSequence: + def __init__( + self, + *, + runtime_identity: str, + role: str, + purpose: str, + workflow_step: str = "", + request_id: str = "", + sprint_id: str = "", + todo_id: str = "", + backlog_id: str = "", + goal_id: str = "", + operation_id: str | None = None, + ): + self.runtime_identity = _text(runtime_identity) + self.role = _text(role) + self.purpose = _text(purpose) or "role_task" + self.workflow_step = _text(workflow_step) + self.request_id = _text(request_id) + self.sprint_id = _text(sprint_id) + self.todo_id = _text(todo_id) + self.backlog_id = _text(backlog_id) + self.goal_id = _text(goal_id) + self.operation_id = _text(operation_id) or uuid.uuid4().hex + self.logical_call_id = uuid.uuid4().hex + self._attempt_index = 0 + self.prompt_context_enabled: bool | None = None + self.prompt_context_total_events: int | None = None + self.prompt_context_included_events: int | None = None + self.prompt_context_omitted_events: int | None = None + self.prompt_context_recent_events: int | None = None + self.prompt_context_max_events: int | None = None + self.prompt_context_selection_policy = "" + + @classmethod + def from_request( + cls, + *, + runtime_identity: str, + role: str, + purpose: str, + request_record: dict[str, Any], + envelope: Any = None, + sprint_id: str = "", + ) -> "InvocationSequence": + params = request_record.get("params") if isinstance(request_record.get("params"), dict) else {} + workflow = params.get("workflow") if isinstance(params.get("workflow"), dict) else request_record.get("workflow") + workflow = workflow if isinstance(workflow, dict) else {} + return cls( + runtime_identity=runtime_identity, + role=role, + purpose=purpose, + workflow_step=_text(workflow.get("step")), + request_id=_text(request_record.get("request_id") or getattr(envelope, "request_id", "")), + sprint_id=_text(request_record.get("sprint_id") or sprint_id), + todo_id=_text(request_record.get("todo_id")), + backlog_id=_text(request_record.get("backlog_id")), + goal_id=_text(request_record.get("goal_id") or params.get("goal_id")), + ) + + def start_logical_call(self, *, purpose: str | None = None) -> None: + self.logical_call_id = uuid.uuid4().hex + self._attempt_index = 0 + if purpose is not None: + self.purpose = _text(purpose) or self.purpose + + def set_prompt_context_projection( + self, + projection: Any, + *, + enabled: bool, + selection_policy: str = "", + ) -> None: + """Attach content-free prompt projection evidence to subsequent attempts.""" + self.prompt_context_enabled = bool(enabled) + self.prompt_context_total_events = _optional_non_negative_int( + getattr(projection, "total_events", None) + ) + self.prompt_context_included_events = _optional_non_negative_int( + getattr(projection, "included_events", None) + ) + self.prompt_context_omitted_events = _optional_non_negative_int( + getattr(projection, "omitted_events", None) + ) + self.prompt_context_recent_events = _optional_non_negative_int( + getattr(projection, "recent_events", None) + ) + self.prompt_context_max_events = _optional_non_negative_int( + getattr(projection, "max_events", None) + ) + self.prompt_context_selection_policy = _text(selection_policy) + + def clear_prompt_context_projection(self) -> None: + self.prompt_context_enabled = None + self.prompt_context_total_events = None + self.prompt_context_included_events = None + self.prompt_context_omitted_events = None + self.prompt_context_recent_events = None + self.prompt_context_max_events = None + self.prompt_context_selection_policy = "" + + def next(self, attempt_kind: str = "primary") -> ModelInvocationContext: + normalized_kind = _text(attempt_kind) + if normalized_kind not in VALID_ATTEMPT_KINDS: + raise ValueError(f"Unsupported telemetry attempt kind: {attempt_kind}") + self._attempt_index += 1 + return ModelInvocationContext( + invocation_id=uuid.uuid4().hex, + operation_id=self.operation_id, + logical_call_id=self.logical_call_id, + attempt_index=self._attempt_index, + attempt_kind=normalized_kind, + runtime_identity=self.runtime_identity, + role=self.role, + purpose=self.purpose, + workflow_step=self.workflow_step, + request_id=self.request_id, + sprint_id=self.sprint_id, + todo_id=self.todo_id, + backlog_id=self.backlog_id, + goal_id=self.goal_id, + prompt_context_enabled=self.prompt_context_enabled, + prompt_context_total_events=self.prompt_context_total_events, + prompt_context_included_events=self.prompt_context_included_events, + prompt_context_omitted_events=self.prompt_context_omitted_events, + prompt_context_recent_events=self.prompt_context_recent_events, + prompt_context_max_events=self.prompt_context_max_events, + prompt_context_selection_policy=self.prompt_context_selection_policy, + ) + + +def calculate_estimated_cost( + usage: ModelUsage, + rate_card: ModelRateCard | None, +) -> float | None: + if rate_card is None: + return None + if rate_card.per_invocation_usd is not None: + return rate_card.per_invocation_usd + if usage.input_tokens is None or usage.output_tokens is None: + return None + input_rate = rate_card.input_per_million_usd + output_rate = rate_card.output_per_million_usd + if input_rate is None or output_rate is None: + return None + cached_tokens = min(usage.cached_input_tokens or 0, usage.input_tokens) + uncached_tokens = max(usage.input_tokens - cached_tokens, 0) + cached_rate = rate_card.cached_input_per_million_usd + if cached_rate is None: + cached_rate = input_rate + cost = ( + uncached_tokens * input_rate + + cached_tokens * cached_rate + + usage.output_tokens * output_rate + ) / 1_000_000 + return round(cost, 12) + + +class ModelTelemetryRecorder: + def __init__( + self, + paths: RuntimePaths, + runtime_identity: str, + config: TelemetryRuntimeConfig | None = None, + *, + output_dir: Path | None = None, + ): + self.paths = paths + self.runtime_identity = _text(runtime_identity) or "unknown" + self.config = config or TelemetryRuntimeConfig() + self.output_dir = ( + Path(output_dir).expanduser().resolve() + if output_dir is not None + else paths.model_invocations_dir + ) + self._last_warning_at = 0.0 + + @property + def enabled(self) -> bool: + return bool(self.config.enabled) + + def _warning(self, exc: BaseException) -> None: + now = time.monotonic() + if now - self._last_warning_at < TELEMETRY_WARNING_INTERVAL_SECONDS: + return + self._last_warning_at = now + LOGGER.warning("Model telemetry write failed for %s: %s", self.runtime_identity, type(exc).__name__) + + def record( + self, + context: ModelInvocationContext, + *, + provider: str, + model: str, + reasoning: str, + cli_version: str, + started_at: datetime, + ended_at: datetime, + duration_ms: int, + session_id_before: str | None, + session_id_after: str | None, + status: str, + exit_code: int | None, + error_category: str, + prompt_chars: int, + output_chars: int, + usage: ModelUsage | None = None, + ) -> None: + if not self.enabled: + return + try: + normalized_usage = usage or ModelUsage() + provider_key = f"{_text(provider)}/{_text(model)}" + rate_card = self.config.rate_cards.get(provider_key) + estimated_cost = calculate_estimated_cost(normalized_usage, rate_card) + session_id = _text(session_id_after or session_id_before) + record = { + "schema_version": TELEMETRY_SCHEMA_VERSION, + "invocation_id": context.invocation_id, + "operation_id": context.operation_id, + "logical_call_id": context.logical_call_id, + "attempt_index": context.attempt_index, + "attempt_kind": context.attempt_kind, + "started_at": _timestamp(started_at), + "ended_at": _timestamp(ended_at), + "duration_ms": max(int(duration_ms), 0), + "pid": os.getpid(), + "runtime_identity": context.runtime_identity or self.runtime_identity, + "role": context.role, + "purpose": context.purpose, + "workflow_step": context.workflow_step, + "request_id": context.request_id, + "sprint_id": context.sprint_id, + "todo_id": context.todo_id, + "backlog_id": context.backlog_id, + "goal_id": context.goal_id, + "prompt_context_enabled": context.prompt_context_enabled, + "prompt_context_total_events": context.prompt_context_total_events, + "prompt_context_included_events": context.prompt_context_included_events, + "prompt_context_omitted_events": context.prompt_context_omitted_events, + "prompt_context_recent_events": context.prompt_context_recent_events, + "prompt_context_max_events": context.prompt_context_max_events, + "prompt_context_selection_policy": context.prompt_context_selection_policy, + "provider": _text(provider), + "model": _text(model), + "reasoning": _text(reasoning), + "cli_version": _text(cli_version), + "session_mode": ( + "not_applicable" + if _text(provider) == "gemini_deep_research" + else ("resume" if _text(session_id_before) else "new") + ), + "session_id_hash": hash_session_id(session_id), + "status": "completed" if _text(status) == "completed" else "failed", + "exit_code": exit_code, + "error_category": _text(error_category), + "prompt_chars": max(int(prompt_chars), 0), + "output_chars": max(int(output_chars), 0), + "tool_calls": normalized_usage.tool_calls, + "input_tokens": normalized_usage.input_tokens, + "cached_input_tokens": normalized_usage.cached_input_tokens, + "output_tokens": normalized_usage.output_tokens, + "reasoning_output_tokens": normalized_usage.reasoning_output_tokens, + "total_tokens": normalized_usage.total_tokens, + "usage_source": normalized_usage.source, + "estimated_cost_usd": estimated_cost, + "rate_card": asdict(rate_card) if rate_card is not None else None, + } + day = normalize_runtime_datetime(started_at).date().isoformat() + identity = sanitize_runtime_identity(context.runtime_identity or self.runtime_identity) + path = self.output_dir / day / f"{identity}.{os.getpid()}.jsonl" + path.parent.mkdir(parents=True, mode=0o700, exist_ok=True) + try: + path.parent.chmod(0o700) + except OSError: + pass + line = json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n" + with path.open("a", encoding="utf-8") as handle: + handle.write(line) + handle.flush() + try: + path.chmod(0o600) + except OSError: + pass + except Exception as exc: # Telemetry is deliberately fail-open. + self._warning(exc) + + +def record_external_invocation( + recorder: ModelTelemetryRecorder, + context: ModelInvocationContext, + *, + provider: str, + model: str, + reasoning: str, + started_at: datetime, + started_monotonic: float, + status: str, + prompt_chars: int, + output_chars: int, + error_category: str = "", +) -> None: + ended_at = runtime_now() + recorder.record( + context, + provider=provider, + model=model, + reasoning=reasoning, + cli_version="", + started_at=started_at, + ended_at=ended_at, + duration_ms=int((time.monotonic() - started_monotonic) * 1000), + session_id_before=None, + session_id_after=None, + status=status, + exit_code=None, + error_category=error_category, + prompt_chars=prompt_chars, + output_chars=output_chars, + usage=ModelUsage(), + ) + + +def run_with_optional_telemetry( + runner: Any, + workspace: Path, + prompt: str, + session_id: str | None, + *, + invocation_context: ModelInvocationContext, + bypass_sandbox: bool = False, +) -> tuple[str, str | None]: + run_method = runner.run + try: + parameters = inspect.signature(run_method).parameters + accepts_kwargs = any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters.values() + ) + accepts_context = "invocation_context" in parameters or accepts_kwargs + accepts_bypass = "bypass_sandbox" in parameters or accepts_kwargs + except (TypeError, ValueError): + accepts_context = True + accepts_bypass = True + kwargs: dict[str, Any] = {} + if accepts_bypass: + kwargs["bypass_sandbox"] = bypass_sandbox + if accepts_context: + kwargs["invocation_context"] = invocation_context + return run_method(workspace, prompt, session_id, **kwargs) + + +def run_task_with_optional_telemetry_purpose( + runtime: Any, + envelope: Any, + request_record: dict[str, Any], + *, + telemetry_purpose: str, +) -> dict[str, Any]: + run_method = runtime.run_task + try: + parameters = inspect.signature(run_method).parameters + accepts_purpose = "telemetry_purpose" in parameters or any( + parameter.kind == inspect.Parameter.VAR_KEYWORD + for parameter in parameters.values() + ) + except (TypeError, ValueError): + accepts_purpose = True + if accepts_purpose: + return run_method( + envelope, + request_record, + telemetry_purpose=telemetry_purpose, + ) + return run_method(envelope, request_record) + + +def _parse_record_timestamp(value: Any) -> datetime | None: + try: + parsed = datetime.fromisoformat(_text(value)) + except ValueError: + return None + return normalize_runtime_datetime(parsed) + + +def _date_range(started_at: datetime, ended_at: datetime) -> Iterable[str]: + current = normalize_runtime_datetime(started_at).date() + last = normalize_runtime_datetime(ended_at).date() + while current <= last: + yield current.isoformat() + current += timedelta(days=1) + + +def _nearest_rank(values: list[int], percentile: float) -> int: + if not values: + return 0 + ordered = sorted(values) + index = max(math.ceil(percentile * len(ordered)) - 1, 0) + return ordered[index] + + +def _empty_group(role: str, purpose: str, provider: str, model: str) -> dict[str, Any]: + return { + "role": role, + "purpose": purpose, + "provider": provider, + "model": model, + "invocation_count": 0, + "primary_count": 0, + "failed_count": 0, + "contract_repair_count": 0, + "sandbox_retry_count": 0, + "tool_call_count": 0, + "input_tokens": 0, + "cached_input_tokens": 0, + "uncached_input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "prompt_context_observed_count": 0, + "prompt_context_compacted_count": 0, + "duration_ms": 0, + "estimated_cost_usd": None, + } + + +def aggregate_model_invocations( + paths: RuntimePaths, + *, + hours: float = 24.0, + request_id: str = "", + sprint_id: str = "", + role: str = "", + now: datetime | None = None, +) -> dict[str, Any]: + if not math.isfinite(hours) or hours <= 0: + raise ValueError("hours must be a positive finite number") + ended_at = normalize_runtime_datetime(now or runtime_now()) + started_at = ended_at - timedelta(hours=hours) + filters = { + "started_at": started_at.isoformat(), + "ended_at": ended_at.isoformat(), + "request_id": _text(request_id), + "sprint_id": _text(sprint_id), + "role": _text(role), + } + invocations: list[dict[str, Any]] = [] + invalid_count = 0 + for day in _date_range(started_at, ended_at): + day_dir = paths.model_invocations_dir / day + if not day_dir.is_dir(): + continue + for shard in sorted(day_dir.glob("*.jsonl")): + try: + handle = shard.open("r", encoding="utf-8") + except OSError: + invalid_count += 1 + continue + with handle: + for line in handle: + try: + record = json.loads(line) + except json.JSONDecodeError: + invalid_count += 1 + continue + if not isinstance(record, dict) or record.get("schema_version") != TELEMETRY_SCHEMA_VERSION: + invalid_count += 1 + continue + record_started = _parse_record_timestamp(record.get("started_at")) + if record_started is None: + invalid_count += 1 + continue + if record_started < started_at or record_started > ended_at: + continue + if filters["request_id"] and _text(record.get("request_id")) != filters["request_id"]: + continue + if filters["sprint_id"] and _text(record.get("sprint_id")) != filters["sprint_id"]: + continue + if filters["role"] and _text(record.get("role")) != filters["role"]: + continue + invocations.append(record) + + durations: list[int] = [] + logical_calls: set[str] = set() + groups: dict[tuple[str, str, str, str], dict[str, Any]] = {} + group_priced_counts: dict[tuple[str, str, str, str], int] = {} + token_fields = ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + ) + tokens = {name: 0 for name in token_fields} + completed_count = failed_count = primary_count = repair_count = sandbox_count = 0 + prompt_chars = output_chars = 0 + native_usage_count = priced_count = tool_call_coverage_count = 0 + tool_call_count = 0 + prompt_context_observed_count = 0 + prompt_context_enabled_count = 0 + prompt_context_eligible_count = 0 + prompt_context_compacted_count = 0 + prompt_context_total_events = 0 + prompt_context_included_events = 0 + prompt_context_omitted_events = 0 + prompt_context_selection_policies: set[str] = set() + total_cost = 0.0 + for record in invocations: + logical_id = _text(record.get("logical_call_id")) + if logical_id: + logical_calls.add(logical_id) + duration = _optional_non_negative_int(record.get("duration_ms")) or 0 + durations.append(duration) + prompt_chars += _optional_non_negative_int(record.get("prompt_chars")) or 0 + output_chars += _optional_non_negative_int(record.get("output_chars")) or 0 + if _text(record.get("status")) == "completed": + completed_count += 1 + else: + failed_count += 1 + attempt_kind = _text(record.get("attempt_kind")) + if attempt_kind == "primary": + primary_count += 1 + if attempt_kind == "contract_repair": + repair_count += 1 + if attempt_kind == "sandbox_retry": + sandbox_count += 1 + if _text(record.get("usage_source")) == "native": + native_usage_count += 1 + for name in token_fields: + tokens[name] += _optional_non_negative_int(record.get(name)) or 0 + input_tokens = _optional_non_negative_int(record.get("input_tokens")) + cached_input_tokens = _optional_non_negative_int(record.get("cached_input_tokens")) or 0 + uncached_input_tokens = ( + max(input_tokens - min(cached_input_tokens, input_tokens), 0) + if input_tokens is not None + else 0 + ) + tool_calls = _optional_non_negative_int(record.get("tool_calls")) + if tool_calls is not None: + tool_call_coverage_count += 1 + tool_call_count += tool_calls + + prompt_context_enabled = record.get("prompt_context_enabled") + context_counts = tuple( + _optional_non_negative_int(record.get(name)) + for name in ( + "prompt_context_total_events", + "prompt_context_included_events", + "prompt_context_omitted_events", + ) + ) + prompt_context_observed = isinstance(prompt_context_enabled, bool) and all( + value is not None for value in context_counts + ) + prompt_context_compacted = False + if prompt_context_observed: + total_events, included_events, omitted_events = context_counts + prompt_context_observed_count += 1 + prompt_context_enabled_count += int(prompt_context_enabled) + prompt_context_total_events += int(total_events or 0) + prompt_context_included_events += int(included_events or 0) + prompt_context_omitted_events += int(omitted_events or 0) + max_events = _optional_non_negative_int(record.get("prompt_context_max_events")) + if max_events is not None and int(total_events or 0) > max_events: + prompt_context_eligible_count += 1 + prompt_context_compacted = bool(prompt_context_enabled) and int(omitted_events or 0) > 0 + prompt_context_compacted_count += int(prompt_context_compacted) + selection_policy = _text(record.get("prompt_context_selection_policy")) + if selection_policy: + prompt_context_selection_policies.add(selection_policy) + cost = record.get("estimated_cost_usd") + if isinstance(cost, (int, float)) and math.isfinite(float(cost)): + priced_count += 1 + total_cost += float(cost) + + key = tuple(_text(record.get(name)) for name in ("role", "purpose", "provider", "model")) + group = groups.setdefault(key, _empty_group(*key)) + group["invocation_count"] += 1 + group["duration_ms"] += duration + if attempt_kind == "primary": + group["primary_count"] += 1 + if _text(record.get("status")) != "completed": + group["failed_count"] += 1 + if attempt_kind == "contract_repair": + group["contract_repair_count"] += 1 + if attempt_kind == "sandbox_retry": + group["sandbox_retry_count"] += 1 + group["tool_call_count"] += tool_calls or 0 + group["uncached_input_tokens"] += uncached_input_tokens + if prompt_context_observed: + group["prompt_context_observed_count"] += 1 + if prompt_context_compacted: + group["prompt_context_compacted_count"] += 1 + for source, target in ( + ("input_tokens", "input_tokens"), + ("cached_input_tokens", "cached_input_tokens"), + ("output_tokens", "output_tokens"), + ("total_tokens", "total_tokens"), + ): + group[target] += _optional_non_negative_int(record.get(source)) or 0 + if isinstance(cost, (int, float)) and math.isfinite(float(cost)): + group["estimated_cost_usd"] = round((group["estimated_cost_usd"] or 0.0) + float(cost), 12) + group_priced_counts[key] = group_priced_counts.get(key, 0) + 1 + + count = len(invocations) + for key, group in groups.items(): + if group_priced_counts.get(key, 0) != group["invocation_count"]: + group["estimated_cost_usd"] = None + return { + "schema_version": TELEMETRY_SCHEMA_VERSION, + "generated_at": runtime_now_iso(), + "filters": filters, + "totals": { + "invocation_count": count, + "physical_attempt_count": count, + "logical_call_count": len(logical_calls), + "primary_count": primary_count, + "completed_count": completed_count, + "failed_count": failed_count, + "contract_repair_count": repair_count, + "sandbox_retry_count": sandbox_count, + "tool_call_count": tool_call_count, + "prompt_chars": prompt_chars, + "output_chars": output_chars, + "estimated_cost_usd": ( + round(total_cost, 12) if count and priced_count == count else None + ), + "token_coverage_percent": round(native_usage_count * 100 / count, 2) if count else 0.0, + "tool_call_coverage_percent": ( + round(tool_call_coverage_count * 100 / count, 2) if count else 0.0 + ), + "pricing_coverage_percent": round(priced_count * 100 / count, 2) if count else 0.0, + "invalid_record_count": invalid_count, + }, + "tokens": { + "input": tokens["input_tokens"], + "cached_input": tokens["cached_input_tokens"], + "uncached_input": sum( + int(group["uncached_input_tokens"]) for group in groups.values() + ), + "output": tokens["output_tokens"], + "reasoning_output": tokens["reasoning_output_tokens"], + "total": tokens["total_tokens"], + }, + "prompt_context": { + "observed_invocation_count": prompt_context_observed_count, + "enabled_invocation_count": prompt_context_enabled_count, + "eligible_invocation_count": prompt_context_eligible_count, + "compacted_invocation_count": prompt_context_compacted_count, + "total_events": prompt_context_total_events, + "included_events": prompt_context_included_events, + "omitted_events": prompt_context_omitted_events, + "coverage_percent": ( + round(prompt_context_observed_count * 100 / count, 2) if count else 0.0 + ), + "selection_policies": sorted(prompt_context_selection_policies), + }, + "latency_ms": { + "total": sum(durations), + "p50": _nearest_rank(durations, 0.50), + "p95": _nearest_rank(durations, 0.95), + "max": max(durations, default=0), + }, + "groups": sorted( + groups.values(), + key=lambda item: (-int(item["total_tokens"]), -int(item["duration_ms"]), item["role"], item["purpose"]), + ), + } + + +def render_model_metrics_summary(summary: dict[str, Any]) -> str: + totals = dict(summary.get("totals") or {}) + tokens = dict(summary.get("tokens") or {}) + latency = dict(summary.get("latency_ms") or {}) + filters = dict(summary.get("filters") or {}) + lines = [ + "Model telemetry", + f"window={filters.get('started_at')}..{filters.get('ended_at')}", + ( + f"invocations={totals.get('invocation_count', 0)} logical_calls={totals.get('logical_call_count', 0)} " + f"completed={totals.get('completed_count', 0)} failed={totals.get('failed_count', 0)} " + f"repairs={totals.get('contract_repair_count', 0)} sandbox_retries={totals.get('sandbox_retry_count', 0)}" + ), + ( + f"tokens input={tokens.get('input', 0)} cached={tokens.get('cached_input', 0)} " + f"output={tokens.get('output', 0)} reasoning={tokens.get('reasoning_output', 0)} " + f"total={tokens.get('total', 0)} coverage={totals.get('token_coverage_percent', 0.0):.2f}%" + ), + ( + f"latency_ms total={latency.get('total', 0)} p50={latency.get('p50', 0)} " + f"p95={latency.get('p95', 0)} max={latency.get('max', 0)}" + ), + ] + cost = totals.get("estimated_cost_usd") + cost_text = "unpriced" if cost is None else f"${float(cost):.6f}" + lines.append( + f"estimated_cost={cost_text} pricing_coverage={totals.get('pricing_coverage_percent', 0.0):.2f}% " + f"invalid_records={totals.get('invalid_record_count', 0)}" + ) + if not totals.get("invocation_count"): + lines.append("No model telemetry matched the requested filters.") + return "\n".join(lines) + lines.append("role\tpurpose\tprovider/model\tcalls\tfailures\trepairs\ttokens\tduration_ms\tcost") + for group in summary.get("groups") or []: + group_cost = group.get("estimated_cost_usd") + group_cost_text = "unpriced" if group_cost is None else f"${float(group_cost):.6f}" + lines.append( + "\t".join( + ( + _text(group.get("role")) or "N/A", + _text(group.get("purpose")) or "N/A", + f"{_text(group.get('provider'))}/{_text(group.get('model'))}", + str(group.get("invocation_count", 0)), + str(group.get("failed_count", 0)), + str(group.get("contract_repair_count", 0)), + str(group.get("total_tokens", 0)), + str(group.get("duration_ms", 0)), + group_cost_text, + ) + ) + ) + return "\n".join(lines) + + +__all__ = [ + "InvocationSequence", + "ModelInvocationContext", + "ModelTelemetryRecorder", + "ModelUsage", + "TELEMETRY_SCHEMA_VERSION", + "aggregate_model_invocations", + "calculate_estimated_cost", + "hash_session_id", + "normalized_error_category", + "record_external_invocation", + "render_model_metrics_summary", + "run_task_with_optional_telemetry_purpose", + "run_with_optional_telemetry", +] diff --git a/runtime/research_runtime.py b/runtime/research_runtime.py index 166ec4d..2842d28 100644 --- a/runtime/research_runtime.py +++ b/runtime/research_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import time from dataclasses import asdict from pathlib import Path from typing import Any @@ -9,11 +10,13 @@ from teams_runtime.shared.paths import RuntimePaths from teams_runtime.shared.models import ( MessageEnvelope, + PromptContextRuntimeConfig, RequestRecord, ResearchRuntimeConfig, RoleResult, RoleRuntimeConfig, RoleSessionState, + TelemetryRuntimeConfig, ) from teams_runtime.runtime.base_runtime import ( RoleAgentRuntime, @@ -21,6 +24,18 @@ normalize_role_payload, ) from teams_runtime.runtime.codex_runner import extract_json_object +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy +from teams_runtime.runtime.model_telemetry import ( + InvocationSequence, + normalized_error_category, + record_external_invocation, + run_with_optional_telemetry, +) +from teams_runtime.shared.persistence import runtime_now +from teams_runtime.shared.prompt_context import ( + PROMPT_EVENT_SELECTION_POLICY, + project_request_record_for_prompt, +) from teams_runtime.workflows.roles.research import ( RESEARCH_REPORT_LIST_FIELDS, RESEARCH_REASON_CODE_BLOCKED_DECISION_FAILED, @@ -83,6 +98,10 @@ def __init__( research_defaults: ResearchRuntimeConfig, agent_root: Path | None = None, session_identity: str | None = None, + telemetry_config: TelemetryRuntimeConfig | None = None, + prompt_context_config: PromptContextRuntimeConfig | None = None, + allow_external_research: bool = True, + execution_policy: ModelExecutionPolicy | None = None, ): super().__init__( paths=paths, @@ -91,12 +110,30 @@ def __init__( runtime_config=runtime_config, agent_root=agent_root, session_identity=session_identity, + telemetry_config=telemetry_config, + prompt_context_config=prompt_context_config, + execution_policy=execution_policy, ) self.research_defaults = research_defaults + self.allow_external_research = bool(allow_external_research) - def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> RoleResult: + def run_task( + self, + envelope: MessageEnvelope, + request_record: RequestRecord, + *, + telemetry_purpose: str = "research_decision", + ) -> RoleResult: with self._run_lock: current_sprint_id = self._resolve_request_sprint_id(envelope, request_record) + invocation_sequence = InvocationSequence.from_request( + runtime_identity=self.runtime_identity, + role=self.role, + purpose=telemetry_purpose, + request_record=request_record, + envelope=envelope, + sprint_id=current_sprint_id, + ) session_manager = self._session_manager_for_sprint(current_sprint_id) state = session_manager.ensure_session() request_id = str(request_record.get("request_id") or envelope.request_id or "").strip() or "unknown" @@ -111,6 +148,7 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> request_record, state=state, local_sources_checked=local_sources_checked, + invocation_sequence=invocation_sequence, ) active_session_id = resolved_session_id or active_session_id except Exception as exc: @@ -182,6 +220,46 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> "session_id": active_session_id or "", "session_workspace": state.workspace_path, } + if signal["needed"] and not self.allow_external_research: + disabled_details = { + "failure_stage": "external_research_policy", + "reason": "external_research_disabled", + } + requirement_traceability_matrix = mark_requirement_traceability_research_status( + requirement_traceability_matrix, + research_status="failed", + failure_details=disabled_details, + ) + sprint_prepass = _is_sprint_research_prepass_request(envelope, request_record) + payload["status"] = "completed" if sprint_prepass else "blocked" + payload["summary"] = ( + "외부 research 실행은 비활성화되어 unresolved research risk를 planner에 전달합니다." + if sprint_prepass + else "외부 research가 필요하지만 현재 실행 정책에서 비활성화되어 있습니다." + ) + payload["error"] = "" if sprint_prepass else "external_research_disabled" + payload["proposals"]["requirement_traceability_matrix"] = requirement_traceability_matrix + payload["proposals"]["research_report"] = { + "report_artifact": "", + "research_url": "", + "headline": "External research disabled by execution policy", + "planner_guidance": ( + f"{planner_guidance} 외부 research는 실행되지 않았으므로 관련 가정을 unresolved risk로 유지하세요." + ).strip(), + "research_subject_definition": subject_definition, + "requirement_traceability_matrix": requirement_traceability_matrix, + "research_execution_status": "disabled_by_policy", + "backing_sources": [], + **{field: [] for field in RESEARCH_REPORT_LIST_FIELDS}, + "open_questions": [ + str(signal.get("research_query") or signal.get("subject") or "외부 근거 확인 필요").strip() + ], + "effective_config": asdict(effective_config), + } + state = session_manager.finalize_session_id(state, active_session_id) + payload["session_id"] = state.session_id + payload["session_workspace"] = state.workspace_path + return normalize_role_payload(payload) if signal["needed"]: prompt = build_research_prompt( envelope, @@ -196,18 +274,25 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> response_text = "" parsed_report: dict[str, Any] | None = None failure_stage = "run_deep_research" + effective_reasoning = effective_config.reasoning_level or ( + self.runtime_config.reasoning if self.runtime_config else None + ) + reasoning_level = None + if effective_reasoning: + raw_reasoning = str(effective_reasoning).strip() + if raw_reasoning.lower() in ("extended", "xhigh", "high"): + reasoning_level = "Extended" + elif raw_reasoning.lower() in ("standard", "medium", "low"): + reasoning_level = "Standard" + else: + reasoning_level = raw_reasoning + invocation_sequence.start_logical_call(purpose="deep_research") + invocation_sequence.clear_prompt_context_projection() + external_context = invocation_sequence.next("primary") + external_started_at = runtime_now() + external_started_monotonic = time.monotonic() + external_recorded = False try: - effective_reasoning = effective_config.reasoning_level or (self.runtime_config.reasoning if self.runtime_config else None) - reasoning_level = None - if effective_reasoning: - raw_reasoning = str(effective_reasoning).strip() - if raw_reasoning.lower() in ("extended", "xhigh", "high"): - reasoning_level = "Extended" - elif raw_reasoning.lower() in ("standard", "medium", "low"): - reasoning_level = "Standard" - else: - reasoning_level = raw_reasoning - deep_research_result = run_deep_research_sync( prompt, app_name=effective_config.app, @@ -232,6 +317,19 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> failure_stage = "response_validation" if not response_text: raise RuntimeError("Deep research returned an empty report.") + record_external_invocation( + self.telemetry_recorder, + external_context, + provider="gemini_deep_research", + model=str(effective_config.app or "default"), + reasoning=str(reasoning_level or ""), + started_at=external_started_at, + started_monotonic=external_started_monotonic, + status="completed", + prompt_chars=len(prompt), + output_chars=len(response_text), + ) + external_recorded = True if deep_research_result.url: response_text += f"\n\n---\n**Deep Research URL:** {deep_research_result.url}\n" failure_stage = "write_artifact" @@ -273,6 +371,24 @@ def run_task(self, envelope: MessageEnvelope, request_record: RequestRecord) -> } payload["artifacts"] = [artifact_hint] except Exception as exc: + if not external_recorded: + record_external_invocation( + self.telemetry_recorder, + external_context, + provider="gemini_deep_research", + model=str(effective_config.app or "default"), + reasoning=str(reasoning_level or ""), + started_at=external_started_at, + started_monotonic=external_started_monotonic, + status="failed", + prompt_chars=len(prompt), + output_chars=len(response_text), + error_category=( + "provider_incomplete" + if failure_stage == "await_final_report" + else normalized_error_category(exc) + ), + ) artifact_written = artifact_path.exists() parsed_backing_sources = ( parsed_report.get("backing_sources") @@ -467,17 +583,30 @@ def _run_research_decision( *, state: RoleSessionState, local_sources_checked: list[str], + invocation_sequence: InvocationSequence, ) -> tuple[dict[str, Any], str | None]: + request_projection = project_request_record_for_prompt( + request_record, + self.prompt_context_config, + ) + invocation_sequence.set_prompt_context_projection( + request_projection, + enabled=self.prompt_context_config.enabled, + selection_policy=PROMPT_EVENT_SELECTION_POLICY, + ) prompt = build_research_decision_prompt( envelope, request_record, local_sources_checked=local_sources_checked, + prompt_context_config=self.prompt_context_config, ) - output, resolved_session_id = self.codex_runner.run( + output, resolved_session_id = run_with_optional_telemetry( + self.codex_runner, Path(state.workspace_path), prompt, state.session_id or None, bypass_sandbox=self._request_requires_default_bypass(envelope, request_record), + invocation_context=invocation_sequence.next("primary"), ) raw_payload = extract_json_object(output) return normalize_research_decision(raw_payload, request_record=request_record), resolved_session_id diff --git a/shared/__init__.py b/shared/__init__.py index 010cc02..173d7b0 100644 --- a/shared/__init__.py +++ b/shared/__init__.py @@ -5,6 +5,7 @@ DiscordAgentsConfig, INTERNAL_TEAM_AGENTS, MessageEnvelope, + ModelRateCard, ReplyRoute, RequestEvent, RequestRecord, @@ -18,6 +19,7 @@ TEAM_ROLES, TERMINAL_REQUEST_STATUSES, TeamRuntimeConfig, + TelemetryRuntimeConfig, WorkflowState, ) @@ -28,6 +30,7 @@ "DiscordAgentsConfig", "INTERNAL_TEAM_AGENTS", "MessageEnvelope", + "ModelRateCard", "ReplyRoute", "RequestEvent", "RequestRecord", @@ -41,5 +44,6 @@ "TEAM_ROLES", "TERMINAL_REQUEST_STATUSES", "TeamRuntimeConfig", + "TelemetryRuntimeConfig", "WorkflowState", ] diff --git a/shared/config.py b/shared/config.py index ce49a3d..3121ed2 100644 --- a/shared/config.py +++ b/shared/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import os import re from pathlib import Path @@ -12,11 +13,15 @@ from teams_runtime.shared.models import ( ActionConfig, DiscordAgentsConfig, + INTERNAL_TEAM_AGENTS, + ModelRateCard, + PromptContextRuntimeConfig, ResearchRuntimeConfig, RoleAgentConfig, RoleRuntimeConfig, TEAM_ROLES, TeamRuntimeConfig, + TelemetryRuntimeConfig, ) @@ -65,6 +70,39 @@ def _normalize_snowflake(value: Any, *, field_name: str) -> str: } +def _normalize_internal_agent_defaults( + value: Any, + *, + inherited_runtime: RoleRuntimeConfig, +) -> dict[str, RoleRuntimeConfig]: + if value is None: + payload: dict[str, Any] = {} + elif isinstance(value, dict): + payload = value + else: + raise ValueError("team_runtime.yaml internal_agent_defaults must be a mapping.") + + normalized: dict[str, RoleRuntimeConfig] = {} + for agent in INTERNAL_TEAM_AGENTS: + raw_defaults = payload.get(agent) + if raw_defaults is None: + defaults: dict[str, Any] = {} + elif not isinstance(raw_defaults, dict): + raise ValueError( + f"team_runtime.yaml internal_agent_defaults.{agent} must be a mapping." + ) + else: + defaults = raw_defaults + normalized[agent] = RoleRuntimeConfig( + model=str(defaults.get("model") or "").strip() or inherited_runtime.model, + reasoning=( + str(defaults.get("reasoning") or "").strip() + or inherited_runtime.reasoning + ), + ) + return normalized + + def _normalize_cutoff_time(value: Any) -> str: normalized = str(value or "22:00").strip() match = _TIME_PATTERN.fullmatch(normalized) @@ -131,6 +169,117 @@ def _normalize_research_defaults(value: Any) -> ResearchRuntimeConfig: ) +def _normalize_positive_integer(value: Any, *, field_name: str, default: int) -> int: + if value is None: + return default + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{field_name} must be a positive integer.") + return value + + +def _normalize_prompt_context_config(value: Any) -> PromptContextRuntimeConfig: + if value in (None, {}): + return PromptContextRuntimeConfig() + if not isinstance(value, dict): + raise ValueError("team_runtime.yaml prompt_context must be a mapping.") + enabled = value.get("enabled", True) + if not isinstance(enabled, bool): + raise ValueError("team_runtime.yaml prompt_context.enabled must be a boolean.") + recent_events = _normalize_positive_integer( + value.get("recent_events"), + field_name="team_runtime.yaml prompt_context.recent_events", + default=8, + ) + max_events = _normalize_positive_integer( + value.get("max_events"), + field_name="team_runtime.yaml prompt_context.max_events", + default=16, + ) + if max_events < recent_events: + raise ValueError( + "team_runtime.yaml prompt_context.max_events must be greater than or equal to recent_events." + ) + return PromptContextRuntimeConfig( + enabled=enabled, + recent_events=recent_events, + max_events=max_events, + ) + + +def _normalize_non_negative_rate(value: Any, *, field_name: str) -> float: + try: + normalized = float(value) + except (TypeError, ValueError) as exc: + raise ValueError(f"{field_name} must be a non-negative finite number.") from exc + if not math.isfinite(normalized) or normalized < 0: + raise ValueError(f"{field_name} must be a non-negative finite number.") + return normalized + + +def _normalize_telemetry_config(value: Any) -> TelemetryRuntimeConfig: + if value in (None, {}): + return TelemetryRuntimeConfig() + if not isinstance(value, dict): + raise ValueError("team_runtime.yaml telemetry must be a mapping.") + enabled = value.get("enabled", True) + if not isinstance(enabled, bool): + raise ValueError("team_runtime.yaml telemetry.enabled must be a boolean.") + raw_rate_cards = value.get("rate_cards") or {} + if not isinstance(raw_rate_cards, dict): + raise ValueError("team_runtime.yaml telemetry.rate_cards must be a mapping.") + + rate_cards: dict[str, ModelRateCard] = {} + for raw_key, raw_card in raw_rate_cards.items(): + key = str(raw_key or "").strip() + field_prefix = f"team_runtime.yaml telemetry.rate_cards.{key or ''}" + provider, separator, model = key.partition("/") + if not separator or not provider.strip() or not model.strip(): + raise ValueError(f"{field_prefix} must use an exact provider/model key.") + if not isinstance(raw_card, dict): + raise ValueError(f"{field_prefix} must be a mapping.") + has_flat_rate = "per_invocation_usd" in raw_card + has_token_rate = any( + name in raw_card + for name in ( + "input_per_million_usd", + "cached_input_per_million_usd", + "output_per_million_usd", + ) + ) + if has_flat_rate and has_token_rate: + raise ValueError(f"{field_prefix} cannot mix token and per-invocation pricing.") + if has_flat_rate: + rate_cards[key] = ModelRateCard( + per_invocation_usd=_normalize_non_negative_rate( + raw_card.get("per_invocation_usd"), + field_name=f"{field_prefix}.per_invocation_usd", + ) + ) + continue + if not has_token_rate: + raise ValueError(f"{field_prefix} must define token rates or per_invocation_usd.") + if "input_per_million_usd" not in raw_card or "output_per_million_usd" not in raw_card: + raise ValueError(f"{field_prefix} token pricing requires input and output rates.") + input_rate = _normalize_non_negative_rate( + raw_card.get("input_per_million_usd"), + field_name=f"{field_prefix}.input_per_million_usd", + ) + cached_rate = _normalize_non_negative_rate( + raw_card.get("cached_input_per_million_usd", input_rate), + field_name=f"{field_prefix}.cached_input_per_million_usd", + ) + output_rate = _normalize_non_negative_rate( + raw_card.get("output_per_million_usd"), + field_name=f"{field_prefix}.output_per_million_usd", + ) + rate_cards[key] = ModelRateCard( + input_per_million_usd=input_rate, + cached_input_per_million_usd=cached_rate, + output_per_million_usd=output_rate, + ) + return TelemetryRuntimeConfig(enabled=enabled, rate_cards=rate_cards) + + def _ensure_non_placeholder_snowflake( value: str, *, @@ -375,10 +524,16 @@ def load_team_runtime_config(workspace_root: str | Path) -> TeamRuntimeConfig: reasoning = str(defaults.get("reasoning") or "").strip() or default_runtime.reasoning model = str(defaults.get("model") or "").strip() or default_runtime.model role_defaults[role] = RoleRuntimeConfig(model=model, reasoning=reasoning) + internal_agent_defaults = _normalize_internal_agent_defaults( + payload.get("internal_agent_defaults"), + inherited_runtime=role_defaults["orchestrator"], + ) raw_research_defaults = payload.get("research_defaults") if raw_research_defaults not in (None, {}) and not isinstance(raw_research_defaults, dict): raise ValueError("team_runtime.yaml research_defaults must be a mapping.") research_defaults = _normalize_research_defaults(raw_research_defaults) + prompt_context = _normalize_prompt_context_config(payload.get("prompt_context")) + telemetry = _normalize_telemetry_config(payload.get("telemetry")) actions: dict[str, ActionConfig] = {} raw_actions = payload.get("actions") or {} @@ -429,7 +584,10 @@ def load_team_runtime_config(workspace_root: str | Path) -> TeamRuntimeConfig: ingress_mentions=bool(ingress.get("mentions", True)), allowed_guild_ids=tuple(str(item).strip() for item in allowed_guild_ids if str(item).strip()), role_defaults=role_defaults, + internal_agent_defaults=internal_agent_defaults, research_defaults=research_defaults, + prompt_context=prompt_context, + telemetry=telemetry, actions=actions, ) @@ -484,6 +642,59 @@ def update_team_runtime_role_defaults( return runtime_config.role_defaults[normalized_role] +def update_team_runtime_internal_agent_defaults( + workspace_root: str | Path, + agent: str, + *, + model: str | None = None, + reasoning: str | None = None, +) -> RoleRuntimeConfig: + normalized_agent = str(agent or "").strip() + if normalized_agent not in INTERNAL_TEAM_AGENTS: + raise ValueError(f"Unsupported internal agent: {normalized_agent or agent}") + + normalized_model = None if model is None else str(model).strip() + normalized_reasoning = None if reasoning is None else str(reasoning).strip() + if normalized_model == "": + raise ValueError("model must be a non-empty string when provided.") + if normalized_reasoning == "": + raise ValueError("reasoning must be a non-empty string when provided.") + if normalized_model is None and normalized_reasoning is None: + raise ValueError("At least one of model or reasoning must be provided.") + + workspace_path = Path(workspace_root).expanduser().resolve() + config_path = workspace_path / "team_runtime.yaml" + payload = _load_yaml(config_path) + raw_defaults = payload.get("internal_agent_defaults") + if raw_defaults is None: + raw_defaults = {} + payload["internal_agent_defaults"] = raw_defaults + if not isinstance(raw_defaults, dict): + raise ValueError("team_runtime.yaml internal_agent_defaults must be a mapping.") + + current_defaults = raw_defaults.get(normalized_agent) + if current_defaults is None: + current_defaults = {} + if not isinstance(current_defaults, dict): + raise ValueError( + "team_runtime.yaml " + f"internal_agent_defaults.{normalized_agent} must be a mapping." + ) + updated_defaults = dict(current_defaults) + if normalized_model is not None: + updated_defaults["model"] = normalized_model + if normalized_reasoning is not None: + updated_defaults["reasoning"] = normalized_reasoning + raw_defaults[normalized_agent] = updated_defaults + + config_path.write_text( + yaml.safe_dump(payload, sort_keys=False, allow_unicode=True), + encoding="utf-8", + ) + runtime_config = load_team_runtime_config(workspace_path) + return runtime_config.internal_agent_defaults[normalized_agent] + + def update_team_runtime_research_defaults( workspace_root: str | Path, *, diff --git a/shared/models.py b/shared/models.py index af615ae..b7a1be0 100644 --- a/shared/models.py +++ b/shared/models.py @@ -62,7 +62,10 @@ class WorkflowState(TypedDict, total=False): policy_source: str contract_version: int advisory_pass_count: int + reopen_count: int + reopen_limit: int review_cycle_count: int + review_cycle_limit: int reopen_category: str last_transition_at: str last_completed_role: str @@ -237,6 +240,27 @@ class ResearchRuntimeConfig: reasoning_level: str | None = "Standard" +@dataclass(slots=True, frozen=True) +class ModelRateCard: + input_per_million_usd: float | None = None + cached_input_per_million_usd: float | None = None + output_per_million_usd: float | None = None + per_invocation_usd: float | None = None + + +@dataclass(slots=True, frozen=True) +class TelemetryRuntimeConfig: + enabled: bool = True + rate_cards: dict[str, ModelRateCard] = field(default_factory=dict) + + +@dataclass(slots=True, frozen=True) +class PromptContextRuntimeConfig: + enabled: bool = True + recent_events: int = 8 + max_events: int = 16 + + @dataclass(slots=True, frozen=True) class ActionConfig: name: str @@ -262,7 +286,10 @@ class TeamRuntimeConfig: ingress_mentions: bool = True allowed_guild_ids: tuple[str, ...] = () role_defaults: dict[str, RoleRuntimeConfig] = field(default_factory=dict) + internal_agent_defaults: dict[str, RoleRuntimeConfig] = field(default_factory=dict) research_defaults: ResearchRuntimeConfig = field(default_factory=ResearchRuntimeConfig) + prompt_context: PromptContextRuntimeConfig = field(default_factory=PromptContextRuntimeConfig) + telemetry: TelemetryRuntimeConfig = field(default_factory=TelemetryRuntimeConfig) actions: dict[str, ActionConfig] = field(default_factory=dict) @@ -329,6 +356,7 @@ def from_dict(cls, payload: dict[str, Any]) -> "RoleSessionState": "GoalState", "INTERNAL_TEAM_AGENTS", "MessageEnvelope", + "PromptContextRuntimeConfig", "ReplyRoute", "RequestEvent", "RequestRecord", diff --git a/shared/paths.py b/shared/paths.py index 1eba826..b684b5c 100644 --- a/shared/paths.py +++ b/shared/paths.py @@ -73,6 +73,14 @@ def role_sessions_dir(self) -> Path: def archive_dir(self) -> Path: return self.runtime_root / "archive" + @property + def metrics_dir(self) -> Path: + return self.runtime_root / "metrics" + + @property + def model_invocations_dir(self) -> Path: + return self.metrics_dir / "model_invocations" + def agent_runtime_dir(self, role: str) -> Path: return self.runtime_root / "agents" / role @@ -266,6 +274,8 @@ def ensure_runtime_dirs(self) -> None: self.operations_dir, self.role_sessions_dir, self.archive_dir, + self.metrics_dir, + self.model_invocations_dir, self.shared_workspace_root, self.shared_attachments_root, self.sprint_artifacts_root, diff --git a/shared/prompt_context.py b/shared/prompt_context.py new file mode 100644 index 0000000..9e0a4be --- /dev/null +++ b/shared/prompt_context.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + +from teams_runtime.shared.models import PromptContextRuntimeConfig, RequestRecord + + +PROMPT_EVENT_SELECTION_POLICY = "recent_tail_plus_latest_role_evidence" + + +@dataclass(slots=True, frozen=True) +class PromptRequestProjection: + request_record: RequestRecord + total_events: int + included_events: int + omitted_events: int + recent_events: int + max_events: int + canonical_request: str + + @property + def compacted(self) -> bool: + return self.omitted_events > 0 + + def notice(self) -> dict[str, Any]: + if not self.compacted: + return {} + return { + "compacted": True, + "total_events": self.total_events, + "included_events": self.included_events, + "omitted_events": self.omitted_events, + "recent_events": self.recent_events, + "max_events": self.max_events, + "selection": PROMPT_EVENT_SELECTION_POLICY, + "canonical_request": self.canonical_request, + } + + +def _role_evidence_identity(event: Any) -> str: + if not isinstance(event, dict): + return "" + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + payload_role = str(payload.get("role") or "").strip().lower() + payload_status = str(payload.get("status") or "").strip().lower() + event_types = { + str(event.get(field_name) or "").strip().lower() + for field_name in ("type", "event_type") + } + if "role_report" not in event_types and not (payload_role and payload_status): + return "" + return payload_role or str(event.get("actor") or "").strip().lower() + + +def _canonical_request_path(request_record: RequestRecord) -> str: + request_id = str(request_record.get("request_id") or "").strip() + if not request_id: + return "" + return f"./.teams_runtime/requests/{request_id}.json" + + +def project_request_record_for_prompt( + request_record: RequestRecord, + config: PromptContextRuntimeConfig | None = None, +) -> PromptRequestProjection: + resolved_config = config or PromptContextRuntimeConfig() + projected_record: RequestRecord = dict(request_record) + raw_events = request_record.get("events") + if not isinstance(raw_events, list): + return PromptRequestProjection( + request_record=projected_record, + total_events=0, + included_events=0, + omitted_events=0, + recent_events=resolved_config.recent_events, + max_events=resolved_config.max_events, + canonical_request=_canonical_request_path(request_record), + ) + + total_events = len(raw_events) + if not resolved_config.enabled or total_events <= resolved_config.max_events: + return PromptRequestProjection( + request_record=projected_record, + total_events=total_events, + included_events=total_events, + omitted_events=0, + recent_events=resolved_config.recent_events, + max_events=resolved_config.max_events, + canonical_request=_canonical_request_path(request_record), + ) + + tail_start = total_events - resolved_config.recent_events + selected_indices = set(range(tail_start, total_events)) + represented_roles = { + role + for role in (_role_evidence_identity(raw_events[index]) for index in selected_indices) + if role + } + + for index in range(tail_start - 1, -1, -1): + if len(selected_indices) >= resolved_config.max_events: + break + role = _role_evidence_identity(raw_events[index]) + if not role or role in represented_roles: + continue + selected_indices.add(index) + represented_roles.add(role) + + selected_events = [raw_events[index] for index in sorted(selected_indices)] + projected_record["events"] = selected_events # type: ignore[typeddict-item] + included_events = len(selected_events) + return PromptRequestProjection( + request_record=projected_record, + total_events=total_events, + included_events=included_events, + omitted_events=total_events - included_events, + recent_events=resolved_config.recent_events, + max_events=resolved_config.max_events, + canonical_request=_canonical_request_path(request_record), + ) + + +def render_prompt_event_history_notice(projection: PromptRequestProjection) -> str: + if not projection.compacted: + return "" + return f"""Current request event-history projection: +{json.dumps(projection.notice(), ensure_ascii=False, indent=2)} +The `events` array below contains complete selected events, not summaries. +Omitted events still exist in the canonical request and must not be treated as events that never happened. +Open `canonical_request` only when the current decision requires evidence missing from the selected events. +""" + + +__all__ = [ + "PROMPT_EVENT_SELECTION_POLICY", + "PromptRequestProjection", + "project_request_record_for_prompt", + "render_prompt_event_history_notice", +] diff --git a/templates/prompts/orchestrator.md b/templates/prompts/orchestrator.md index 2dddffb..426637f 100644 --- a/templates/prompts/orchestrator.md +++ b/templates/prompts/orchestrator.md @@ -5,7 +5,8 @@ ## 핵심 책임 - orchestrator 작업에서는 로컬 workspace의 `./.agents/skills/` 아래에 사용 가능한 skill이 있는지 먼저 확인하고 활용 -- role runtime model/reasoning 변경은 prompt 파일이 아니라 `team_runtime.yaml` `role_defaults` 또는 `python -m teams_runtime config role set ...`로 관리한다 +- public role runtime model/reasoning 변경은 prompt 파일이 아니라 `team_runtime.yaml` `role_defaults` 또는 `python -m teams_runtime config role set ...`로 관리한다 +- internal parser/sourcer/version_controller model/reasoning 변경은 `team_runtime.yaml` `internal_agent_defaults` 또는 `python -m teams_runtime config internal set ...`로 관리하고 변경 후 orchestrator를 재시작한다 - `./.agents/skills/agent_utilization/policy.yaml`을 orchestrator routing/scoring의 machine-readable source of truth로 사용한다 - 각 agent의 역할, skill, 강점, 행동 특성을 이해하고 현재 작업에 가장 잘 맞는 agent를 선택한다 - role이 후속 역할을 선택한다고 가정하지 말고, orchestrator가 결과/문맥/정책을 읽어 `next_role`을 중앙에서 결정하고 handoff에 남긴다 @@ -28,6 +29,7 @@ - sprint internal request에 `Current request.params.workflow`가 있으면 그 workflow contract가 일반 capability scoring보다 우선한다 - planning phase는 planner owner + 최대 2회의 shared advisory pass(designer/architect)로 제한하고, pass 소진 뒤에는 planner finalization 또는 blocked로 종료한다 - implementation phase는 `architect guidance -> developer build -> architect review -> developer revision -> qa validation` 순서를 표준값으로 사용한다 +- architect review cycle과 orchestrator-governed reopen은 `policy.yaml`의 독립된 limit를 적용하며 limit 도달 후 추가 revision model handoff를 열지 않는다 - execution/qa 단계에서 reopen이 필요하면 역할이 직접 다음 역할을 고르지 말고 `workflow_transition`에 category를 남기고 orchestrator가 다음 역할을 결정한다 - 역할 보고는 `summary`, `proposals`, `artifacts`로 다음 단계에 필요한 근거를 남기고, `next_role` 선택 책임은 전적으로 orchestrator가 가진다 - 실제로 확인하지 않은 파일 수정, 테스트 통과, 문서 반영, 검증 결과를 완료로 보고하지 않는다 diff --git a/templates/scaffold/orchestrator/.agents/skills/agent_utilization/SKILL.md b/templates/scaffold/orchestrator/.agents/skills/agent_utilization/SKILL.md index 6d497b8..019556b 100644 --- a/templates/scaffold/orchestrator/.agents/skills/agent_utilization/SKILL.md +++ b/templates/scaffold/orchestrator/.agents/skills/agent_utilization/SKILL.md @@ -48,6 +48,8 @@ Do not use this skill to do the role-specific work itself. Reinforce planner-owned backlog persistence and version_controller-owned commit execution while still keeping orchestrator in charge of workflow. 8. Enforce the standard sprint collaboration path. Use planner-owned planning, bounded advisory passes, mandatory architect guidance before developer work, mandatory architect review before QA, and orchestrator-chosen reopen routing. +9. Enforce independent implementation budgets. + Count architect review cycles and accepted reopen transitions separately. Block before another revision handoff when the applicable `workflow_contract` limit is reached. ## Guardrails @@ -56,5 +58,6 @@ Do not use this skill to do the role-specific work itself. - Do not let implementation go to developer when the real need is still UX or architecture shaping. - Do not bypass planner for backlog-management ownership. - Do not reopen routing from non-planner backlog proposals or terminal verification results. +- Do not bypass `implementation_review_cycle_limit` or `implementation_reopen_limit`; a blocked limit requires operator intervention or an explicit policy change. - Do not route commit work away from version_controller. - Do not leave the reason for role selection implicit. diff --git a/templates/scaffold/team_runtime.yaml b/templates/scaffold/team_runtime.yaml index 11c952f..da85ccf 100644 --- a/templates/scaffold/team_runtime.yaml +++ b/templates/scaffold/team_runtime.yaml @@ -39,6 +39,17 @@ role_defaults: model: "gpt-5.5" reasoning: "medium" +internal_agent_defaults: + parser: + model: "gpt-5.4-mini" + reasoning: "low" + sourcer: + model: "gpt-5.4-mini" + reasoning: "medium" + version_controller: + model: "gpt-5.4-mini" + reasoning: "low" + research_defaults: app: "" notebook: "" @@ -50,4 +61,13 @@ research_defaults: cleanup: false reasoning_level: "Standard" +prompt_context: + enabled: true + recent_events: 8 + max_events: 16 + +telemetry: + enabled: true + rate_cards: {} + actions: {} diff --git a/tests/orchestration_test_utils.py b/tests/orchestration_test_utils.py index 8ca4ef8..6220647 100644 --- a/tests/orchestration_test_utils.py +++ b/tests/orchestration_test_utils.py @@ -198,7 +198,9 @@ def _make_workflow_request_record( planning_pass_count=0, planning_pass_limit=2, review_cycle_count=0, - review_cycle_limit=20, + review_cycle_limit=3, + reopen_count=0, + reopen_limit=3, reopen_source_role="", reopen_category="", ): @@ -225,6 +227,8 @@ def _make_workflow_request_record( "planning_final_owner": "planner", "reopen_source_role": reopen_source_role, "reopen_category": reopen_category, + "reopen_count": reopen_count, + "reopen_limit": reopen_limit, "review_cycle_count": review_cycle_count, "review_cycle_limit": review_cycle_limit, }, diff --git a/tests/test_benchmark_worker_cleanup.py b/tests/test_benchmark_worker_cleanup.py new file mode 100644 index 0000000..e218bee --- /dev/null +++ b/tests/test_benchmark_worker_cleanup.py @@ -0,0 +1,1124 @@ +from __future__ import annotations + +import json +import math +import os +import signal +import subprocess +import sys +import tempfile +import unittest +from dataclasses import replace +from pathlib import Path +from unittest import mock + +from teams_runtime.benchmarking import runner, worker +from teams_runtime.benchmarking.models import ( + ArmPlan, + BenchmarkOptions, + BenchmarkWorkerSafetyError, + WorkerContext, + WorkerOutcome, + invocation_identity_digest, +) +from teams_runtime.benchmarking.scenario import ( + BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + BENCHMARK_TARGET_INCLUDED_EVENTS, + BENCHMARK_TARGET_OMITTED_EVENTS, + BENCHMARK_TARGET_PURPOSE, + BENCHMARK_TARGET_ROLE, + BENCHMARK_TARGET_TOTAL_EVENTS, + BENCHMARK_TARGET_WORKFLOW_STEP, + DEFAULT_HISTORY_SEED_COUNT, + ScenarioWorkspace, +) +from teams_runtime.shared.prompt_context import PROMPT_EVENT_SELECTION_POLICY + + +def _snapshot(*entries: dict[str, object]) -> dict[str, object]: + return { + "schema_version": 1, + "entries": list(entries), + } + + +def _running_entry(pid: int, process_group_id: int) -> dict[str, object]: + return { + "pid": pid, + "process_group_id": process_group_id, + "state": "running", + } + + +def _target_context(*, enabled: bool) -> dict[str, object]: + return { + "provider": "codex_cli", + "invocation_id": "target-invocation", + "operation_id": "target-operation", + "logical_call_id": "target-logical-call", + "attempt_index": 1, + "attempt_kind": "primary", + "runtime_identity": "role", + "role": BENCHMARK_TARGET_ROLE, + "purpose": BENCHMARK_TARGET_PURPOSE, + "workflow_step": BENCHMARK_TARGET_WORKFLOW_STEP, + "request_id": "target-request", + "sprint_id": "target-sprint", + "todo_id": "", + "backlog_id": "", + "goal_id": "", + "prompt_context_enabled": enabled, + "prompt_context_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "prompt_context_included_events": ( + BENCHMARK_TARGET_INCLUDED_EVENTS + if enabled + else BENCHMARK_TARGET_TOTAL_EVENTS + ), + "prompt_context_omitted_events": ( + BENCHMARK_TARGET_OMITTED_EVENTS if enabled else 0 + ), + "prompt_context_recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "prompt_context_max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "prompt_context_selection_policy": PROMPT_EVENT_SELECTION_POLICY, + } + + +def _worker_context( + root: Path, + *, + run_output_dir: Path | None = None, +) -> WorkerContext: + workspace_root = root / "workspace" + workspace_root.mkdir(parents=True, exist_ok=True) + output_dir = run_output_dir or root / "run-output" + output_dir.mkdir(parents=True, exist_ok=True) + return WorkerContext( + benchmark_id="private-telemetry-test", + arm=ArmPlan( + pair_index=1, + order_index=1, + variant="before", + run_id="pair-001-before", + prompt_context_enabled=False, + ), + workspace_root=workspace_root, + run_output_dir=output_dir, + milestone="Repair the benchmark fixture.", + history_seed=(), + max_invocations=2, + call_timeout_seconds=30, + run_timeout_seconds=60, + live=True, + ) + + +class _FakeProcess: + def __init__(self, *wait_results: object): + self.pid = 41001 + self._wait_results = list(wait_results) + self.wait_calls: list[float] = [] + + def wait(self, *, timeout: float) -> int: + self.wait_calls.append(timeout) + result = self._wait_results.pop(0) + if isinstance(result, BaseException): + raise result + return int(result) + + def poll(self) -> int | None: + return None + + def send_signal(self, _process_signal: signal.Signals) -> None: + return None + + +class BenchmarkWorkerCleanupTests(unittest.TestCase): + def test_worker_context_rejects_non_finite_timeouts(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + context = _worker_context(Path(temporary_directory)) + (context.workspace_root / "team_runtime.yaml").write_text( + "roles: {}\n", + encoding="utf-8", + ) + (context.workspace_root / ".git").mkdir() + scenario_dir = context.workspace_root / ".benchmark" + scenario_dir.mkdir() + (scenario_dir / "scenario.json").write_text("{}\n", encoding="utf-8") + context = replace( + context, + history_seed=tuple( + {"event": index} + for index in range(DEFAULT_HISTORY_SEED_COUNT) + ), + ) + + with mock.patch.dict( + os.environ, + {worker.LIVE_BENCHMARK_ENV: "1"}, + ): + for field_name in ( + "call_timeout_seconds", + "run_timeout_seconds", + ): + for value in (math.nan, math.inf, -math.inf): + with self.subTest(field_name=field_name, value=value): + invalid_context = replace( + context, + **{field_name: value}, + ) + with self.assertRaises(ValueError): + worker._validate_context(invalid_context) + + def test_execution_policy_routes_telemetry_outside_provider_workspace( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + context = _worker_context(Path(temporary_directory)) + budget = worker.InvocationBudget(context.max_invocations) + + with mock.patch.dict( + os.environ, + {"OPENAI_API_KEY": "benchmark-test-key"}, + ), mock.patch.object( + worker, + "_resolve_benchmark_codex_executable", + return_value=Path(sys.executable).resolve(), + ): + policy = worker._build_execution_policy( + context, + budget=budget, + ) + + expected = ( + context.run_output_dir.resolve() + / ".private_model_invocations" + ) + self.assertEqual(policy.telemetry_output_dir, expected) + self.assertFalse( + expected.is_relative_to(context.workspace_root.resolve()) + ) + with self.assertRaises(worker.ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(expected) + + def test_benchmark_path_excludes_untrusted_and_relative_directories(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + context = _worker_context(root) + workspace_bin = context.workspace_root / "bin" + run_output_bin = context.run_output_dir / "bin" + temporary_bin = root / "system-temp" / "bin" + safe_bin = root / "safe-bin" + for directory in ( + workspace_bin, + run_output_bin, + temporary_bin, + safe_bin, + ): + directory.mkdir(parents=True) + workspace_link = root / "workspace-bin-link" + workspace_link.symlink_to(workspace_bin, target_is_directory=True) + supplied_path = os.pathsep.join( + ( + "", + ".", + str(root / "missing-bin"), + str(workspace_bin), + str(workspace_link), + str(run_output_bin), + str(temporary_bin), + str(safe_bin), + str(safe_bin), + ) + ) + + with ( + mock.patch.dict(os.environ, {"PATH": supplied_path}, clear=False), + mock.patch.object( + worker, + "_benchmark_unsafe_path_roots", + return_value=( + context.workspace_root.resolve(), + context.run_output_dir.resolve(), + (root / "system-temp").resolve(), + ), + ), + ): + sanitized = worker._sanitized_benchmark_path(context) + + self.assertEqual(sanitized, str(safe_bin.resolve())) + + with ( + mock.patch.dict( + os.environ, + {"PATH": os.pathsep.join((".", str(workspace_link)))}, + clear=False, + ), + mock.patch.object( + worker, + "_benchmark_unsafe_path_roots", + return_value=(context.workspace_root.resolve(),), + ), + ): + with self.assertRaisesRegex( + worker.ModelExecutionPolicyViolation, + "no safe external executable directories", + ): + worker._sanitized_benchmark_path(context) + + def test_codex_resolution_rejects_provider_writable_target(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + context = _worker_context(Path(temporary_directory)) + for label, executable in ( + ("workspace", context.workspace_root / "codex"), + ("run_output", context.run_output_dir / "codex"), + ): + executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + executable.chmod(0o700) + with self.subTest(label=label), mock.patch.object( + worker.shutil, + "which", + return_value=str(executable), + ): + with self.assertRaisesRegex( + worker.ModelExecutionPolicyViolation, + "not a safe external executable", + ): + worker._resolve_benchmark_codex_executable( + context, + search_path=os.defpath, + ) + + def test_private_telemetry_consumption_ignores_workspace_forgery_and_removes_raw_shards( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + context = _worker_context(Path(temporary_directory)) + private_dir = worker._initialize_private_telemetry(context) + private_shard = private_dir / "2026-08-12" / "role.100.jsonl" + private_shard.parent.mkdir(parents=True) + private_shard.write_text( + json.dumps( + { + "invocation_id": "trusted-private-invocation", + "input_tokens": 25, + "usage_source": "native", + } + ) + + "\n", + encoding="utf-8", + ) + forged_shard = ( + context.workspace_root + / ".teams_runtime" + / "metrics" + / "model_invocations" + / "2026-08-12" + / "role.200.jsonl" + ) + forged_shard.parent.mkdir(parents=True) + forged_shard.write_text( + json.dumps( + { + "invocation_id": "forged-workspace-invocation", + "input_tokens": 9_999_999, + "usage_source": "native", + } + ) + + "\n", + encoding="utf-8", + ) + + records = worker._consume_private_telemetry(context) + + self.assertEqual( + [record["invocation_id"] for record in records], + ["trusted-private-invocation"], + ) + self.assertEqual(records[0]["input_tokens"], 25) + self.assertFalse(private_dir.exists()) + self.assertTrue(forged_shard.is_file()) + + def test_private_telemetry_deletion_failure_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + context = _worker_context(Path(temporary_directory)) + private_dir = worker._initialize_private_telemetry(context) + shard = private_dir / "2026-08-12" / "role.100.jsonl" + shard.parent.mkdir(parents=True) + shard.write_text( + json.dumps({"invocation_id": "trusted-invocation"}) + "\n", + encoding="utf-8", + ) + + with mock.patch.object( + worker.shutil, + "rmtree", + side_effect=OSError("permission denied"), + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "Failed to remove benchmark private telemetry shards", + ): + worker._consume_private_telemetry(context) + + self.assertTrue(private_dir.is_dir()) + + def test_worker_outcome_payload_cannot_transport_telemetry(self) -> None: + outcome = WorkerOutcome( + status="completed", + telemetry_records=( + { + "invocation_id": "child-supplied-invocation", + "input_tokens": 9_999_999, + }, + ), + ) + + payload = worker._worker_outcome_payload(outcome) + + self.assertEqual(payload["telemetry_records"], []) + payload["telemetry_records"] = [ + { + "invocation_id": "forged-result-invocation", + "input_tokens": 9_999_999, + } + ] + restored = worker._worker_outcome_from_payload(payload) + self.assertEqual(restored.telemetry_records, ()) + + def test_worker_context_rejects_run_output_inside_provider_workspace( + self, + ) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + context = _worker_context( + root, + run_output_dir=root / "workspace" / "run-output", + ) + + with mock.patch.dict( + os.environ, + {worker.LIVE_BENCHMARK_ENV: "1"}, + ): + with self.assertRaisesRegex( + worker.ModelExecutionPolicyViolation, + "run output must be outside", + ): + worker._validate_context(context) + + def test_timeout_cleanup_rereads_journal_before_kill(self) -> None: + first_timeout = subprocess.TimeoutExpired( + cmd=("worker",), + timeout=worker._CHILD_TERMINATION_GRACE_SECONDS, + ) + process = _FakeProcess(first_timeout, -signal.SIGKILL) + initial_entry = _running_entry(42001, 42001) + late_entry = _running_entry(42002, 42002) + snapshots = ( + _snapshot(initial_entry), + _snapshot(initial_entry, late_entry), + _snapshot(initial_entry, late_entry), + ) + log = mock.Mock() + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=snapshots, + ) as read_journal, + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ) as signal_providers, + mock.patch.object(worker, "_signal_child_group") as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + final_snapshot = worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path("/content-safe/call_journal.json"), + worker_log=log, + stop_reason="run_timeout_exceeded", + ) + + self.assertEqual(read_journal.call_count, 3) + self.assertEqual(len(process.wait_calls), 2) + self.assertEqual(signal_providers.call_args_list[0].args[1], signal.SIGTERM) + pre_kill_entries = signal_providers.call_args_list[1].args[0] + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in pre_kill_entries + }, + {(42001, 42001), (42002, 42002)}, + ) + self.assertEqual( + [entry["state"] for entry in final_snapshot["entries"]], + ["terminated", "terminated"], + ) + self.assertTrue( + all( + entry["stop_reason"] == "run_timeout_exceeded" + for entry in final_snapshot["entries"] + ) + ) + write_journal.assert_called_once_with( + Path("/content-safe/call_journal.json"), + final_snapshot, + ) + self.assertEqual(signal_providers.call_args_list[1].args[1], signal.SIGKILL) + self.assertEqual(signal_child.call_args_list[0].args[1], signal.SIGTERM) + self.assertEqual(signal_child.call_args_list[1].args[1], signal.SIGKILL) + confirmed_entries = confirm_cleanup.call_args.kwargs["provider_entries"] + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in confirmed_entries + }, + {(42001, 42001), (42002, 42002)}, + ) + + def test_second_worker_wait_timeout_fails_closed(self) -> None: + first_timeout = subprocess.TimeoutExpired( + cmd=("worker",), + timeout=worker._CHILD_TERMINATION_GRACE_SECONDS, + ) + second_timeout = subprocess.TimeoutExpired( + cmd=("worker",), + timeout=worker._CHILD_TERMINATION_GRACE_SECONDS, + ) + process = _FakeProcess(first_timeout, second_timeout) + snapshots = ( + _snapshot(_running_entry(42001, 42001)), + _snapshot(_running_entry(42001, 42001)), + _snapshot(_running_entry(42001, 42001)), + ) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=snapshots, + ), + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ), + mock.patch.object(worker, "_signal_child_group"), + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaises(worker._WorkerCleanupFailure): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path("/content-safe/call_journal.json"), + worker_log=mock.Mock(), + ) + + self.assertEqual(len(process.wait_calls), 2) + confirm_cleanup.assert_called_once() + write_journal.assert_called_once() + + def test_journal_read_failures_are_deferred_until_cleanup_is_confirmed( + self, + ) -> None: + entries = ( + _running_entry(42001, 42001), + _running_entry(42002, 42002), + _running_entry(42003, 42003), + ) + + for failed_read_index in range(3): + with self.subTest(failed_read_index=failed_read_index): + process = _FakeProcess(0) + side_effects: list[object] = [ + _snapshot(entries[0]), + _snapshot(entries[1]), + _snapshot(entries[2]), + ] + side_effects[failed_read_index] = ( + worker._WorkerCleanupFailure("journal unavailable") + ) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=side_effects, + ) as read_journal, + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ) as signal_providers, + mock.patch.object( + worker, + "_signal_child_group", + ) as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "cleanup could not be verified", + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path( + "/content-safe/call_journal.json" + ), + worker_log=mock.Mock(), + ) + + self.assertEqual(read_journal.call_count, 3) + self.assertTrue( + all( + call.kwargs["required"] is True + for call in read_journal.call_args_list + ) + ) + self.assertEqual(len(process.wait_calls), 1) + child_signals = [ + call.args[1] + for call in signal_child.call_args_list + ] + self.assertEqual(child_signals[0], signal.SIGTERM) + self.assertIn(signal.SIGKILL, child_signals) + confirm_cleanup.assert_called_once() + confirmed_entries = confirm_cleanup.call_args.kwargs[ + "provider_entries" + ] + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in confirmed_entries + }, + { + (entry["pid"], entry["process_group_id"]) + for index, entry in enumerate(entries) + if index != failed_read_index + }, + ) + final_provider_sweep = ( + signal_providers.call_args_list[-1] + ) + self.assertEqual( + final_provider_sweep.args[1], + signal.SIGKILL, + ) + self.assertEqual( + { + (entry["pid"], entry["process_group_id"]) + for entry in final_provider_sweep.args[0] + }, + { + (entry["pid"], entry["process_group_id"]) + for index, entry in enumerate(entries) + if index != failed_read_index + }, + ) + if failed_read_index == 2: + write_journal.assert_not_called() + else: + write_journal.assert_called_once() + + def test_all_journal_reads_failing_still_cleans_up_worker_group(self) -> None: + process = _FakeProcess(0) + journal_failure = worker._WorkerCleanupFailure( + "journal unavailable" + ) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=( + journal_failure, + journal_failure, + journal_failure, + ), + ) as read_journal, + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ), + mock.patch.object( + worker, + "_signal_child_group", + ) as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "initial_journal,pre_kill_journal,final_journal", + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path( + "/content-safe/call_journal.json" + ), + worker_log=mock.Mock(), + ) + + self.assertEqual(read_journal.call_count, 3) + self.assertEqual(len(process.wait_calls), 1) + self.assertEqual( + [call.args[1] for call in signal_child.call_args_list], + [signal.SIGTERM, signal.SIGKILL, signal.SIGKILL], + ) + confirm_cleanup.assert_called_once() + self.assertEqual( + confirm_cleanup.call_args.kwargs["provider_entries"], + [], + ) + write_journal.assert_not_called() + + def test_reserved_launch_observed_during_cleanup_fails_closed(self) -> None: + process = _FakeProcess(0) + reserved_entry = { + "reservation_id": "reservation-1", + "state": "reserved", + } + snapshot = _snapshot(reserved_entry) + + with ( + mock.patch.object( + worker, + "_read_call_journal_strict", + side_effect=(snapshot, snapshot, snapshot), + ), + mock.patch.object( + worker, + "_signal_provider_entries", + return_value=0, + ), + mock.patch.object( + worker, + "_signal_child_group", + ) as signal_child, + mock.patch.object( + worker, + "_wait_for_cleanup_confirmation", + ) as confirm_cleanup, + mock.patch.object( + worker, + "_write_private_json", + ) as write_journal, + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "reserved_attempt", + ): + worker._terminate_worker_child( + process, # type: ignore[arg-type] + journal_path=Path( + "/content-safe/call_journal.json" + ), + worker_log=mock.Mock(), + ) + + self.assertEqual( + [call.args[1] for call in signal_child.call_args_list], + [signal.SIGTERM, signal.SIGKILL, signal.SIGKILL], + ) + confirm_cleanup.assert_called_once() + write_journal.assert_called_once() + + def test_existing_malformed_journal_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + journal_path = Path(temporary_directory) / "call_journal.json" + journal_path.write_text("{not-json", encoding="utf-8") + + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "unreadable or malformed", + ): + worker._read_call_journal_strict(journal_path) + + journal_path.unlink() + self.assertEqual( + worker._read_call_journal_strict(journal_path), + {}, + ) + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "Required benchmark call journal is missing", + ): + worker._read_call_journal_strict( + journal_path, + required=True, + ) + + def test_journal_reservation_ids_must_be_unique_and_present(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + journal_path = Path(temporary_directory) / "call_journal.json" + cases = { + "duplicate": [ + {"reservation_id": "same", "state": "completed"}, + {"reservation_id": "same", "state": "completed"}, + ], + "missing": [ + {"reservation_id": "first", "state": "completed"}, + {"state": "completed"}, + ], + } + for label, entries in cases.items(): + with self.subTest(label=label): + journal_path.write_text( + json.dumps( + { + "schema_version": 2, + "max_invocations": 2, + "reserved_count": 2, + "remaining": 0, + "rejected_count": 0, + "entries": entries, + } + ), + encoding="utf-8", + ) + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "reservation ids", + ): + worker._read_call_journal_strict(journal_path) + + def test_call_journal_summary_keeps_attempt_and_telemetry_counts_distinct(self) -> None: + states = [ + *(["completed"] * 5), + *(["timeout"] * 2), + "terminated", + ] + summary = worker._summarize_call_journal( + { + "schema_version": 2, + "max_invocations": 20, + "reserved_count": 8, + "remaining": 12, + "rejected_count": 1, + "entries": [ + { + "state": state, + "invocation_id": f"invocation-{index}", + } + for index, state in enumerate(states) + ], + }, + telemetry_records=tuple( + {"invocation_id": f"invocation-{index}"} + for index in range(7) + ), + ) + + self.assertEqual(summary["reserved_count"], 8) + self.assertEqual(summary["telemetry_record_count"], 7) + self.assertEqual(summary["unobserved_attempt_count"], 1) + self.assertEqual(summary["telemetry_coverage_percent"], 87.5) + self.assertEqual(summary["completed_count"], 5) + self.assertEqual(summary["timeout_count"], 2) + self.assertEqual(summary["terminated_count"], 1) + self.assertEqual(summary["active_count"], 0) + self.assertEqual(summary["rejected_count"], 1) + self.assertTrue(summary["identity_reconciled"]) + self.assertEqual( + summary["journal_invocation_id_unobserved_count"], + 1, + ) + + def test_parent_cleanup_ignores_terminal_provider_groups(self) -> None: + entries = worker._merge_launched_process_entries( + { + "entries": [ + { + "pid": 42001, + "process_group_id": 42001, + "state": "completed", + }, + { + "pid": 42002, + "process_group_id": 42002, + "state": "failed", + }, + ] + } + ) + + self.assertEqual(entries, []) + + def test_v3_journal_verifies_exact_target_projection_outside_workspace(self) -> None: + for enabled in (False, True): + with self.subTest(enabled=enabled): + context = _target_context(enabled=enabled) + entry = { + **context, + "reservation_id": "target-reservation", + "state": "completed", + } + telemetry = {**context, "status": "completed"} + summary = worker._summarize_call_journal( + { + "schema_version": 3, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "rejected_count": 0, + "entries": [entry], + }, + telemetry_records=(telemetry,), + ) + + self.assertTrue(summary["identity_reconciled"]) + self.assertTrue(summary["context_reconciled"]) + self.assertEqual( + summary["journal_telemetry_context_mismatch_count"], + 0, + ) + self.assertEqual( + summary["verified_target_projection_count"], + 1, + ) + self.assertEqual( + summary["verified_target_invocation_ids_sha256"], + invocation_identity_digest(["target-invocation"]), + ) + + def test_v3_journal_rejects_tampered_or_unrelated_target_telemetry(self) -> None: + context = _target_context(enabled=True) + entry = { + **context, + "reservation_id": "target-reservation", + "state": "completed", + } + tampered = { + **context, + "status": "completed", + "prompt_context_included_events": BENCHMARK_TARGET_INCLUDED_EVENTS - 1, + "prompt_context_omitted_events": BENCHMARK_TARGET_OMITTED_EVENTS + 1, + } + tampered_summary = worker._summarize_call_journal( + { + "schema_version": 3, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "rejected_count": 0, + "entries": [entry], + }, + telemetry_records=(tampered,), + ) + + self.assertFalse(tampered_summary["context_reconciled"]) + self.assertEqual( + tampered_summary["journal_telemetry_context_mismatch_count"], + 1, + ) + self.assertEqual( + tampered_summary["verified_target_projection_count"], + 0, + ) + + conflicting_representation = { + **context, + "status": "completed", + "prompt_context": { + "enabled": False, + "total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "included_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "omitted_events": 0, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "selection_policy": PROMPT_EVENT_SELECTION_POLICY, + }, + } + conflicting_summary = worker._summarize_call_journal( + { + "schema_version": 3, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "rejected_count": 0, + "entries": [entry], + }, + telemetry_records=(conflicting_representation,), + ) + + self.assertFalse(conflicting_summary["context_reconciled"]) + self.assertEqual( + conflicting_summary["journal_telemetry_context_mismatch_count"], + 1, + ) + self.assertEqual( + conflicting_summary["verified_target_projection_count"], + 0, + ) + + unrelated_context = { + **context, + "role": "developer", + "purpose": "implement", + "workflow_step": "todo_execution", + } + unrelated_summary = worker._summarize_call_journal( + { + "schema_version": 3, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "rejected_count": 0, + "entries": [ + { + **unrelated_context, + "reservation_id": "unrelated-reservation", + "state": "completed", + } + ], + }, + telemetry_records=( + {**unrelated_context, "status": "completed"}, + ), + ) + + self.assertTrue(unrelated_summary["context_reconciled"]) + self.assertEqual( + unrelated_summary["verified_target_projection_count"], + 0, + ) + + def test_journal_finalization_failure_fails_closed(self) -> None: + snapshot = { + "schema_version": 2, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "rejected_count": 0, + "entries": [ + { + "reservation_id": "reservation-1", + "state": "running", + } + ], + } + + with mock.patch.object( + worker, + "_write_private_json", + side_effect=OSError("disk full"), + ): + with self.assertRaisesRegex( + worker._WorkerCleanupFailure, + "Failed to finalize", + ): + worker._finalize_call_journal_after_cleanup( + Path("/content-safe/call_journal.json"), + snapshot, + stop_reason="run_timeout_exceeded", + ) + + def test_cleanup_confirmation_raises_while_any_group_survives(self) -> None: + process = _FakeProcess(0) + provider_entry = _running_entry(42001, 42001) + + with ( + mock.patch.object(worker, "_worker_group_alive", return_value=False), + mock.patch.object(worker, "_provider_entry_alive", return_value=True), + mock.patch.object(worker, "_signal_provider_entries", return_value=1), + mock.patch.object(worker.time, "monotonic", side_effect=(10.0, 10.0)), + ): + with self.assertRaises(worker._WorkerCleanupFailure): + worker._wait_for_cleanup_confirmation( + process, # type: ignore[arg-type] + provider_entries=[provider_entry], + worker_log=mock.Mock(), + timeout_seconds=0.0, + ) + + def test_runner_does_not_convert_worker_safety_failure_to_arm_result(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scenario_root = root / "scenario" + scenario_root.mkdir() + scenario = ScenarioWorkspace( + root=scenario_root, + initial_commit="initial", + initial_commit_count=1, + protected_hashes={}, + config_hash="config", + comparable_config_hash="comparable", + history_hash="history", + history_seed=(), + ) + options = BenchmarkOptions( + source_root=root, + runtime_config_path=root / "runtime.yaml", + output_dir=root / "output", + live=True, + ) + arm = ArmPlan( + pair_index=1, + order_index=1, + variant="before", + run_id="pair-001-before", + prompt_context_enabled=False, + ) + + def unsafe_worker(_context: object) -> object: + raise BenchmarkWorkerSafetyError("cleanup could not be confirmed") + + with ( + mock.patch.object( + runner, + "create_scenario_workspace", + return_value=scenario, + ), + mock.patch.object( + runner, + "_capture_retention_baseline", + return_value={}, + ), + mock.patch.object(runner, "_safe_worker_failure") as safe_failure, + ): + with self.assertRaises(BenchmarkWorkerSafetyError): + runner._run_arm( + benchmark_id="safety-test", + output_root=root / "output", + temporary_root=root / "temporary", + options=options, + worker=unsafe_worker, # type: ignore[arg-type] + settings=None, + arm=arm, + ) + + safe_failure.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_config.py b/tests/test_config.py index 4b0dbcb..7c3d909 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -15,6 +15,7 @@ from teams_runtime.cli import ( DEFAULT_WORKSPACE_DIRNAME, InternalAgentService, + cmd_config_internal_set, cmd_config_research_set, cmd_config_role_set, cmd_start, @@ -304,6 +305,22 @@ def test_scaffold_workspace_and_load_configs(self): self.assertEqual(runtime_config.role_defaults["developer"].model, "gpt-5.5") self.assertEqual(runtime_config.role_defaults["developer"].reasoning, "high") self.assertEqual(runtime_config.role_defaults["qa"].reasoning, "medium") + self.assertEqual( + runtime_config.internal_agent_defaults["parser"].model, + "gpt-5.4-mini", + ) + self.assertEqual( + runtime_config.internal_agent_defaults["parser"].reasoning, + "low", + ) + self.assertEqual( + runtime_config.internal_agent_defaults["sourcer"].reasoning, + "medium", + ) + self.assertEqual( + runtime_config.internal_agent_defaults["version_controller"].reasoning, + "low", + ) self.assertIsNone(runtime_config.research_defaults.profile_path) self.assertEqual(runtime_config.research_defaults.completion_timeout, 600.0) self.assertEqual(runtime_config.research_defaults.callback_timeout, 1200.0) @@ -626,6 +643,45 @@ def test_load_team_runtime_config_rejects_legacy_approval_block(self): with self.assertRaisesRegex(ValueError, "approval is no longer supported"): load_team_runtime_config(tmpdir) + def test_internal_agent_defaults_inherit_orchestrator_for_legacy_config(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + config_path = Path(tmpdir) / "team_runtime.yaml" + config_path.write_text( + """sprint:\n id: legacy-runtime\nrole_defaults:\n orchestrator:\n model: gpt-legacy\n reasoning: high\n""", + encoding="utf-8", + ) + + runtime_config = load_team_runtime_config(tmpdir) + + for internal_agent in ("parser", "sourcer", "version_controller"): + self.assertEqual( + runtime_config.internal_agent_defaults[internal_agent].model, + "gpt-legacy", + ) + self.assertEqual( + runtime_config.internal_agent_defaults[internal_agent].reasoning, + "high", + ) + + def test_internal_agent_defaults_reject_non_mapping_entry(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + config_path = Path(tmpdir) / "team_runtime.yaml" + content = config_path.read_text(encoding="utf-8") + content = content.replace( + ' parser:\n model: "gpt-5.4-mini"\n reasoning: "low"\n', + " parser: invalid\n", + 1, + ) + config_path.write_text(content, encoding="utf-8") + + with self.assertRaisesRegex( + ValueError, + "internal_agent_defaults.parser must be a mapping", + ): + load_team_runtime_config(tmpdir) + def test_load_team_runtime_config_rejects_action_approval_required(self): with tempfile.TemporaryDirectory() as tmpdir: scaffold_workspace(tmpdir) @@ -667,10 +723,42 @@ def test_load_agent_utilization_policy_falls_back_when_skill_policy_missing(self self.assertTrue(policy.planner_reentry_requires_explicit_signal) self.assertTrue(policy.verification_result_terminal) self.assertTrue(policy.ignore_non_planner_backlog_proposals_for_routing) + self.assertEqual(policy.implementation_review_cycle_limit, 3) + self.assertEqual(policy.implementation_reopen_limit, 3) self.assertTrue((Path(tmpdir) / "shared_workspace" / "current_sprint.md").exists()) self.assertTrue((Path(tmpdir) / "shared_workspace" / "sprints" / "README.md").exists()) self.assertTrue((Path(tmpdir) / "shared_workspace" / "sprint_history" / "index.md").exists()) + def test_load_agent_utilization_policy_accepts_custom_workflow_budgets(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + policy_path = ( + Path(tmpdir) + / "orchestrator" + / ".agents" + / "skills" + / "agent_utilization" + / "policy.yaml" + ) + content = policy_path.read_text(encoding="utf-8") + content = content.replace( + "implementation_review_cycle_limit: 3", + "implementation_review_cycle_limit: 5", + 1, + ) + content = content.replace( + "implementation_reopen_limit: 3", + "implementation_reopen_limit: 2", + 1, + ) + policy_path.write_text(content, encoding="utf-8") + + policy = load_agent_utilization_policy(tmpdir) + + self.assertEqual(policy.policy_source, "workspace_skill_policy") + self.assertEqual(policy.implementation_review_cycle_limit, 5) + self.assertEqual(policy.implementation_reopen_limit, 2) + def test_load_team_runtime_config_supports_manual_daily_sprint_mode(self): with tempfile.TemporaryDirectory() as tmpdir: scaffold_workspace(tmpdir) @@ -1383,6 +1471,37 @@ def test_cmd_config_role_set_updates_runtime_yaml_without_restart(self): self.assertIn("role=developer model=gpt-5.5 reasoning=low", rendered) self.assertIn("python -m teams_runtime restart --agent developer", rendered) + def test_cmd_config_internal_set_updates_helper_tier_without_restart(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + output = io.StringIO() + with patch("teams_runtime.cli.cmd_restart") as restart_mock: + with redirect_stdout(output): + exit_code = cmd_config_internal_set( + Path(tmpdir), + "sourcer", + model="gpt-5.4-mini", + reasoning="low", + ) + + self.assertEqual(exit_code, 0) + restart_mock.assert_not_called() + runtime_config = load_team_runtime_config(tmpdir) + self.assertEqual( + runtime_config.internal_agent_defaults["sourcer"].model, + "gpt-5.4-mini", + ) + self.assertEqual( + runtime_config.internal_agent_defaults["sourcer"].reasoning, + "low", + ) + rendered = output.getvalue() + self.assertIn( + "internal_agent=sourcer model=gpt-5.4-mini reasoning=low", + rendered, + ) + self.assertIn("restart --agent orchestrator", rendered) + def test_cmd_config_research_set_updates_runtime_yaml_without_restart(self): with tempfile.TemporaryDirectory() as tmpdir: scaffold_workspace(tmpdir) @@ -1612,7 +1731,7 @@ def test_main_status_accepts_internal_sourcer_agent(self): self.assertEqual(exit_code, 0) self.assertIn("sourcer: status=stopped", output.getvalue()) - self.assertIn("model=N/A reasoning=N/A", output.getvalue()) + self.assertIn("model=gpt-5.4-mini reasoning=medium", output.getvalue()) self.assertIn("listener=n/a", output.getvalue()) def test_main_status_accepts_internal_version_controller_agent(self): @@ -1624,7 +1743,7 @@ def test_main_status_accepts_internal_version_controller_agent(self): self.assertEqual(exit_code, 0) self.assertIn("version_controller: status=stopped", output.getvalue()) - self.assertIn("model=N/A reasoning=N/A", output.getvalue()) + self.assertIn("model=gpt-5.4-mini reasoning=low", output.getvalue()) self.assertIn("listener=n/a", output.getvalue()) def test_main_status_surfaces_internal_agent_listener_health(self): @@ -1813,6 +1932,37 @@ def test_main_config_role_set_requires_model_or_reasoning(self): ] ) + def test_main_config_internal_set_updates_runtime_yaml(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + output = io.StringIO() + with redirect_stdout(output): + exit_code = main( + [ + "config", + "internal", + "set", + "--workspace-root", + tmpdir, + "--agent", + "parser", + "--reasoning", + "medium", + ] + ) + + self.assertEqual(exit_code, 0) + runtime_config = load_team_runtime_config(tmpdir) + self.assertEqual( + runtime_config.internal_agent_defaults["parser"].model, + "gpt-5.4-mini", + ) + self.assertEqual( + runtime_config.internal_agent_defaults["parser"].reasoning, + "medium", + ) + self.assertIn("internal_agent=parser", output.getvalue()) + def test_run_foreground_role_service_does_not_persist_reload_snapshot_state(self): with tempfile.TemporaryDirectory() as tmpdir: scaffold_workspace(tmpdir) diff --git a/tests/test_execution_policy.py b/tests/test_execution_policy.py new file mode 100644 index 0000000..f786cd8 --- /dev/null +++ b/tests/test_execution_policy.py @@ -0,0 +1,695 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from teams_runtime.runtime import benchmark_launcher +from teams_runtime.runtime.codex_runner import CodexRunner +from teams_runtime.runtime.execution_policy import ( + InvocationBudget, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, + ModelInvocationTimeout, + quarantine_unsafe_workspace_entries, +) +from teams_runtime.shared.models import RoleRuntimeConfig + + +class BenchmarkExecutionPolicyTests(unittest.TestCase): + def _policy( + self, + root: Path, + *, + max_invocations: int = 2, + timeout_seconds: float = 1.0, + ) -> tuple[ModelExecutionPolicy, InvocationBudget]: + budget = InvocationBudget( + max_invocations, + journal_path=root / "call_journal.json", + ) + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=root, + invocation_budget=budget, + call_timeout_seconds=timeout_seconds, + codex_executable=sys.executable, + kill_grace_seconds=0.1, + shell_environment={"PYTHONPATH": str(root)}, + ) + return policy, budget + + def test_benchmark_codex_command_is_sandboxed_without_bypass(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark", reasoning="high"), + role="developer", + execution_policy=policy, + ) + + command, stdin_input = runner._build_command( + workspace=workspace, + prompt="content-safe test prompt", + session_id=None, + output_file=workspace / "output.txt", + bypass_sandbox=False, + ) + + self.assertEqual(stdin_input, "content-safe test prompt") + self.assertIn("--sandbox", command) + self.assertIn("workspace-write", command) + self.assertIn("--ignore-user-config", command) + self.assertIn("--ignore-rules", command) + self.assertIn( + "sandbox_workspace_write.exclude_slash_tmp=true", + command, + ) + self.assertIn( + "sandbox_workspace_write.exclude_tmpdir_env_var=true", + command, + ) + self.assertIn("mcp_servers={}", command) + self.assertNotIn( + "--dangerously-bypass-approvals-and-sandbox", + command, + ) + self.assertNotIn("--full-auto", command) + + def test_benchmark_codex_commands_use_pinned_executable_without_output_path(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark", reasoning="high"), + role="developer", + execution_policy=policy, + ) + workspace_output = workspace / ".teams_runtime_codex_output.txt" + + for label, session_id in ( + ("fresh", None), + ("resume", "session-123"), + ): + with self.subTest(label=label): + command, stdin_input = runner._build_command( + workspace=workspace, + prompt="content-safe test prompt", + session_id=session_id, + output_file=workspace_output, + bypass_sandbox=False, + ) + + self.assertEqual(command[0], str(policy.codex_executable)) + self.assertTrue(Path(command[0]).is_absolute()) + self.assertEqual( + Path(command[0]), + Path(sys.executable).resolve(), + ) + self.assertNotIn("-o", command) + self.assertNotIn("--output-last-message", command) + self.assertNotIn(str(workspace_output), command) + self.assertEqual(stdin_input, "content-safe test prompt") + + def test_benchmark_run_returns_jsonl_message_without_output_file(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, budget = self._policy(root, max_invocations=1) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark", reasoning="high"), + role="developer", + execution_policy=policy, + ) + completed = subprocess.CompletedProcess( + [str(policy.codex_executable)], + 0, + ( + '{"type":"item.completed","item":{"type":"agent_message",' + '"text":"result-from-jsonl"}}\n' + ), + "", + ) + + with mock.patch.object( + runner, + "_run_benchmark_process", + return_value=completed, + ): + output, _session_id = runner.run( + workspace, + "content-safe test prompt", + None, + ) + + self.assertEqual(output, "result-from-jsonl") + self.assertFalse( + (workspace / ".teams_runtime_codex_output.txt").exists() + ) + self.assertEqual(budget.snapshot()["entries"][0]["state"], "completed") + + def test_cli_version_uses_pinned_benchmark_executable(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + telemetry = mock.Mock(enabled=True) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + telemetry_recorder=telemetry, + ) + executable = str(policy.codex_executable) + CodexRunner._version_cache.pop(executable, None) + + with mock.patch( + "teams_runtime.runtime.codex_runner.subprocess.run", + return_value=subprocess.CompletedProcess( + [executable, "--version"], + 0, + "codex-cli benchmark-test\n", + "", + ), + ) as run_version: + version = runner._cli_version("codex") + + self.assertEqual(version, "codex-cli benchmark-test") + self.assertEqual(run_version.call_args.args[0], [executable, "--version"]) + self.assertIn("env", run_version.call_args.kwargs) + + def test_benchmark_codex_executable_is_canonicalized_and_validated(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "workspace" + workspace.mkdir() + trusted_executable = root / "trusted-bin" / "codex" + trusted_executable.parent.mkdir() + trusted_executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + trusted_executable.chmod(0o700) + executable_symlink = root / "codex-link" + executable_symlink.symlink_to(trusted_executable) + + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=workspace, + invocation_budget=InvocationBudget(1), + call_timeout_seconds=1, + codex_executable=executable_symlink, + ) + + self.assertEqual( + policy.codex_executable, + trusted_executable.resolve(), + ) + self.assertTrue(policy.codex_executable.is_absolute()) + + missing = root / "missing-codex" + non_executable = root / "non-executable-codex" + non_executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + workspace_executable = workspace / "codex" + workspace_executable.write_text( + "#!/bin/sh\nexit 0\n", + encoding="utf-8", + ) + workspace_executable.chmod(0o700) + cases = ( + ( + "missing", + missing, + "could not be resolved safely", + ), + ( + "not_executable", + non_executable, + "regular executable file", + ), + ( + "workspace_contained", + workspace_executable, + "outside the provider-writable workspace", + ), + ) + for label, executable, expected_message in cases: + with self.subTest(label=label): + with self.assertRaisesRegex(ValueError, expected_message): + ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=workspace, + invocation_budget=InvocationBudget(1), + call_timeout_seconds=1, + codex_executable=executable, + ) + + def test_workspace_quarantine_removes_cross_boundary_entries(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_root = Path(temporary_directory).resolve() + workspace = temporary_root / "workspace" + outside = temporary_root / "outside" + workspace.mkdir() + outside.mkdir() + internal_target = workspace / "internal.txt" + internal_target.write_text("internal\n", encoding="utf-8") + internal_link = workspace / "internal-link" + internal_link.symlink_to(internal_target) + outside_target = outside / "host-state.txt" + outside_target.write_text("unchanged\n", encoding="utf-8") + outward_link = workspace / "outward-link" + outward_link.symlink_to(outside_target) + hard_link = workspace / "hard-link" + os.link(outside_target, hard_link) + special_file = workspace / "provider-pipe" + if hasattr(os, "mkfifo"): + os.mkfifo(special_file) + + removed = quarantine_unsafe_workspace_entries(workspace) + + self.assertTrue(internal_link.is_symlink()) + self.assertEqual(internal_link.resolve(), internal_target) + self.assertFalse(outward_link.exists()) + self.assertFalse(outward_link.is_symlink()) + self.assertFalse(hard_link.exists()) + if hasattr(os, "mkfifo"): + self.assertFalse(special_file.exists()) + self.assertIn("special_file", removed) + self.assertIn("outward_symlink", removed) + self.assertIn("hard_link", removed) + self.assertEqual( + outside_target.read_text(encoding="utf-8"), + "unchanged\n", + ) + + def test_runner_quarantines_outward_symlink_before_returning(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_root = Path(temporary_directory).resolve() + allowed_root = temporary_root / "allowed" + outside_root = temporary_root / "outside" + workspace = allowed_root / "role" + workspace.mkdir(parents=True) + outside_root.mkdir() + outside_target = outside_root / "host-state.txt" + outside_target.write_text("unchanged\n", encoding="utf-8") + policy, budget = self._policy(allowed_root, max_invocations=1) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark", reasoning="high"), + role="developer", + execution_policy=policy, + ) + + def completed_provider(*_args: object, **_kwargs: object) -> object: + (workspace / "history.md").symlink_to(outside_target) + return subprocess.CompletedProcess( + [str(policy.codex_executable)], + 0, + '{"type":"item.completed","item":{"type":"agent_message","text":"{}"}}\n', + "", + ) + + with mock.patch.object( + runner, + "_run_benchmark_process", + side_effect=completed_provider, + ): + with self.assertRaisesRegex( + ModelExecutionPolicyViolation, + "unsafe filesystem entries", + ): + runner.run(workspace, "content-safe prompt", None) + + self.assertFalse((workspace / "history.md").is_symlink()) + self.assertEqual( + outside_target.read_text(encoding="utf-8"), + "unchanged\n", + ) + entry = budget.snapshot()["entries"][0] + self.assertEqual(entry["state"], "failed") + + def test_provider_environment_excludes_github_and_discord_secrets(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + source_environment = { + "CODEX_HOME": "/operator/codex-home", + "OPENAI_API_KEY": "provider-secret", + "GH_TOKEN": "github-secret", + "DISCORD_TOKEN": "discord-secret", + "HOME": "/operator/home", + "PATH": os.defpath, + "TEMP": "/operator/temp", + "TMP": "/operator/tmp", + "TMPDIR": "/operator/tmpdir", + } + + with mock.patch.dict(os.environ, source_environment, clear=True): + environment = runner._provider_environment() + + self.assertEqual(environment["OPENAI_API_KEY"], "provider-secret") + self.assertNotIn("GH_TOKEN", environment) + self.assertNotIn("DISCORD_TOKEN", environment) + provider_state = root / ".teams_runtime" / "benchmark_provider" + provider_tmp = str(provider_state / "tmp") + self.assertEqual(environment["HOME"], str(provider_state / "home")) + self.assertEqual( + environment["CODEX_HOME"], + str(provider_state / "codex_home"), + ) + self.assertEqual(environment["TMPDIR"], provider_tmp) + self.assertEqual(environment["TMP"], provider_tmp) + self.assertEqual(environment["TEMP"], provider_tmp) + self.assertEqual(environment["GIT_CONFIG_GLOBAL"], os.devnull) + self.assertEqual(environment["GIT_CONFIG_NOSYSTEM"], "1") + self.assertEqual(environment["PYTHONPATH"], str(root)) + + def test_benchmark_rejects_provider_state_directory_escape(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + outside = root.parent / "outside-codex-home" + budget = InvocationBudget(1) + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=root, + invocation_budget=budget, + call_timeout_seconds=1, + codex_executable=sys.executable, + shell_environment={"CODEX_HOME": str(outside)}, + ) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + + with self.assertRaises(ModelExecutionPolicyViolation): + runner.run(workspace, "prompt", None) + + self.assertEqual(budget.reserved_count, 0) + + def test_benchmark_telemetry_output_must_be_outside_workspace(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "workspace" + workspace.mkdir() + budget = InvocationBudget(1) + + for output_dir in (workspace, workspace / ".teams_runtime" / "metrics"): + with self.subTest(output_dir=output_dir): + with self.assertRaisesRegex( + ValueError, + "outside the provider-writable workspace", + ): + ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=workspace, + invocation_budget=budget, + call_timeout_seconds=1, + codex_executable=sys.executable, + telemetry_output_dir=output_dir, + ) + + external_output = root / "private-telemetry" + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=workspace, + invocation_budget=budget, + call_timeout_seconds=1, + codex_executable=sys.executable, + telemetry_output_dir=external_output, + ) + + self.assertEqual( + policy.telemetry_output_dir, + external_output.resolve(), + ) + + def test_benchmark_rejects_bypass_and_gemini_before_launch(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, budget = self._policy(root) + codex_runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + with self.assertRaises(ModelExecutionPolicyViolation): + codex_runner.run( + workspace, + "prompt", + None, + bypass_sandbox=True, + ) + self.assertEqual(budget.reserved_count, 0) + + gemini_runner = CodexRunner( + RoleRuntimeConfig(model="gemini-benchmark"), + execution_policy=policy, + ) + with self.assertRaises(ModelExecutionPolicyViolation): + gemini_runner.run(workspace, "prompt", None) + self.assertEqual(budget.reserved_count, 0) + + def test_real_timeout_marks_journal_and_terminates_process_group(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + workspace = root / "role" + workspace.mkdir() + policy, budget = self._policy( + root, + max_invocations=1, + timeout_seconds=0.1, + ) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + role="developer", + execution_policy=policy, + ) + sleeping_command = [ + sys.executable, + "-c", + "import time; time.sleep(30)", + ] + + with mock.patch.object( + runner, + "_build_command", + return_value=(sleeping_command, None), + ): + with self.assertRaises(ModelInvocationTimeout): + runner.run(workspace, "prompt", None) + + snapshot = budget.snapshot() + self.assertEqual(snapshot["reserved_count"], 1) + self.assertEqual(snapshot["entries"][0]["state"], "timeout") + self.assertEqual(snapshot["entries"][0]["stop_reason"], "timeout") + persisted = json.loads( + (root / "call_journal.json").read_text(encoding="utf-8") + ) + self.assertEqual(persisted["schema_version"], 3) + self.assertEqual(persisted["entries"][0]["state"], "timeout") + self.assertIn( + "prompt_context_enabled", + persisted["entries"][0], + ) + + def test_reservation_journals_content_free_prompt_projection(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + budget = InvocationBudget( + 1, + journal_path=root / "call_journal.json", + ) + context = SimpleNamespace( + invocation_id="invocation-1", + operation_id="operation-1", + logical_call_id="logical-1", + attempt_index=1, + attempt_kind="primary", + runtime_identity="role", + role="research", + purpose="research_decision", + workflow_step="research_initial", + request_id="request-1", + sprint_id="sprint-1", + todo_id="", + backlog_id="", + goal_id="", + prompt_context_enabled=True, + prompt_context_total_events=50, + prompt_context_included_events=16, + prompt_context_omitted_events=34, + prompt_context_recent_events=8, + prompt_context_max_events=16, + prompt_context_selection_policy=( + "recent_tail_plus_latest_role_evidence" + ), + ) + + budget.reserve( + context, + provider="codex_cli", + role="research", + ) + entry = budget.snapshot()["entries"][0] + + self.assertTrue(entry["prompt_context_enabled"]) + self.assertEqual(entry["prompt_context_total_events"], 50) + self.assertEqual(entry["prompt_context_included_events"], 16) + self.assertEqual(entry["prompt_context_omitted_events"], 34) + self.assertNotIn("prompt", entry) + self.assertNotIn("response", entry) + + def test_provider_launch_waits_for_durable_pid_registration(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + process = mock.Mock(pid=43210, returncode=0) + process.communicate.return_value = ("provider output", "") + events: list[str] = [] + reservation = mock.Mock() + reservation.mark_started.side_effect = ( + lambda **_kwargs: events.append("registered") + ) + + def release_provider( + file_descriptor: int, + payload: bytes, + ) -> int: + self.assertEqual(file_descriptor, 12) + self.assertEqual(payload, b"\x01") + events.append("released") + return len(payload) + + command = [sys.executable, "-c", "print('provider')"] + with ( + mock.patch( + "teams_runtime.runtime.codex_runner.os.pipe", + return_value=(11, 12), + ), + mock.patch( + "teams_runtime.runtime.codex_runner.os.close", + ) as close_fd, + mock.patch( + "teams_runtime.runtime.codex_runner.os.write", + side_effect=release_provider, + ), + mock.patch( + "teams_runtime.runtime.codex_runner.subprocess.Popen", + return_value=process, + ) as popen, + ): + completed = runner._run_benchmark_process( + command, + cwd=root, + stdin_input=None, + env={"PATH": os.defpath}, + reservation=reservation, + ) + + self.assertEqual(events, ["registered", "released"]) + reservation.mark_started.assert_called_once_with( + pid=43210, + process_group_id=43210, + ) + launch_command = popen.call_args.args[0] + self.assertEqual( + launch_command[-len(command):], + command, + ) + self.assertEqual( + popen.call_args.kwargs["pass_fds"], + (11,), + ) + self.assertEqual(close_fd.call_args_list[0].args, (11,)) + self.assertEqual(close_fd.call_args_list[-1].args, (12,)) + self.assertEqual(completed.returncode, 0) + + def test_registration_failure_never_releases_provider(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + policy, _budget = self._policy(root) + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-benchmark"), + execution_policy=policy, + ) + process = mock.Mock(pid=43210, returncode=None) + reservation = mock.Mock() + reservation.mark_started.side_effect = OSError( + "journal unavailable" + ) + + with ( + mock.patch( + "teams_runtime.runtime.codex_runner.os.pipe", + return_value=(11, 12), + ), + mock.patch( + "teams_runtime.runtime.codex_runner.os.close", + ) as close_fd, + mock.patch( + "teams_runtime.runtime.codex_runner.os.write", + ) as release_provider, + mock.patch( + "teams_runtime.runtime.codex_runner.subprocess.Popen", + return_value=process, + ), + mock.patch.object( + runner, + "_terminate_process_group", + return_value=("", ""), + ) as terminate_process, + ): + with self.assertRaisesRegex( + OSError, + "journal unavailable", + ): + runner._run_benchmark_process( + [sys.executable, "-c", "print('provider')"], + cwd=root, + stdin_input=None, + env={"PATH": os.defpath}, + reservation=reservation, + ) + + release_provider.assert_not_called() + self.assertIn(mock.call(11), close_fd.call_args_list) + self.assertIn(mock.call(12), close_fd.call_args_list) + terminate_process.assert_called_once() + + def test_launcher_exits_without_exec_when_parent_closes_gate(self) -> None: + ready_read_fd, ready_write_fd = os.pipe() + os.close(ready_write_fd) + + with mock.patch.object( + benchmark_launcher.os, + "execvpe", + ) as execute_provider: + exit_code = benchmark_launcher.main( + [ + "--ready-fd", + str(ready_read_fd), + "--", + sys.executable, + "-c", + "print('must not run')", + ] + ) + + self.assertEqual(exit_code, 70) + execute_provider.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_goal_store.py b/tests/test_goal_store.py index 88c55c7..0ccf66d 100644 --- a/tests/test_goal_store.py +++ b/tests/test_goal_store.py @@ -155,6 +155,7 @@ def test_goal_terminate_dispatches_to_cancel_alias(self): with tempfile.TemporaryDirectory() as tmpdir: parser = build_parser( all_runtime_agents=["orchestrator"], + internal_team_agents=["parser"], team_roles=["orchestrator"], relay_transport_internal="internal", relay_transport_discord="discord", @@ -179,6 +180,7 @@ def _noop(*_args, **_kwargs) -> int: cmd_restart=_noop, cmd_list=_noop, cmd_config_role_set=_noop, + cmd_config_internal_set=_noop, cmd_config_research_set=_noop, cmd_sprint_start=_noop, cmd_sprint_stop=_noop, diff --git a/tests/test_model_telemetry.py b/tests/test_model_telemetry.py new file mode 100644 index 0000000..a121baf --- /dev/null +++ b/tests/test_model_telemetry.py @@ -0,0 +1,910 @@ +from __future__ import annotations + +import io +import json +import math +import os +import stat +import tempfile +import unittest +from contextlib import redirect_stdout +from datetime import timedelta +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import yaml + +from teams_runtime.cli import build_parser, cmd_metrics +from teams_runtime.core.template import scaffold_workspace +from teams_runtime.runtime.base_runtime import RoleAgentRuntime +from teams_runtime.runtime.codex_runner import CodexRunner, parse_codex_jsonl, parse_gemini_usage +from teams_runtime.runtime.model_telemetry import ( + InvocationSequence, + ModelTelemetryRecorder, + ModelUsage, + aggregate_model_invocations, + calculate_estimated_cost, + hash_session_id, + render_model_metrics_summary, +) +from teams_runtime.shared.config import load_team_runtime_config +from teams_runtime.shared.models import ( + MessageEnvelope, + ModelRateCard, + RoleRuntimeConfig, + TelemetryRuntimeConfig, +) +from teams_runtime.shared.paths import RuntimePaths +from teams_runtime.shared.persistence import runtime_now + + +class ModelTelemetryTests(unittest.TestCase): + def test_codex_jsonl_parser_recovers_session_usage_and_final_message(self): + stdout = "\n".join( + ( + json.dumps({"type": "thread.started", "thread_id": "thread-123"}), + "not-json", + json.dumps( + { + "type": "item.completed", + "item": {"type": "agent_message", "text": '{"status":"completed"}'}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"id": "command-1", "type": "command_execution"}, + } + ), + json.dumps( + { + "type": "item/completed", + "item": {"id": "mcp-1", "type": "mcp_tool_call"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"id": "mcp-1", "type": "mcp_tool_call"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"type": "file_change"}, + } + ), + json.dumps( + { + "type": "item.completed", + "item": {"type": "web_search"}, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 100, + "cached_input_tokens": 60, + "output_tokens": 25, + "reasoning_output_tokens": 5, + "total_tokens": 125, + }, + } + ), + ) + ) + + session_id, usage, final_message = parse_codex_jsonl(stdout) + + self.assertEqual(session_id, "thread-123") + self.assertEqual(usage.input_tokens, 100) + self.assertEqual(usage.cached_input_tokens, 60) + self.assertEqual(usage.output_tokens, 25) + self.assertEqual(usage.reasoning_output_tokens, 5) + self.assertEqual(usage.total_tokens, 125) + self.assertEqual(usage.tool_calls, 4) + self.assertEqual(usage.source, "native") + self.assertEqual(final_message, '{"status":"completed"}') + + def test_codex_jsonl_parser_does_not_double_count_terminal_tool_usage(self): + stdout = "\n".join( + ( + json.dumps( + { + "type": "item.completed", + "item": {"id": "command-1", "type": "command_execution"}, + } + ), + json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": 10, + "output_tokens": 2, + "total_tokens": 12, + "tool_calls": 3, + }, + } + ), + ) + ) + + _session_id, usage, _final_message = parse_codex_jsonl(stdout) + + self.assertEqual(usage.input_tokens, 10) + self.assertEqual(usage.output_tokens, 2) + self.assertEqual(usage.total_tokens, 12) + self.assertEqual(usage.tool_calls, 3) + self.assertEqual(usage.source, "native") + + def test_gemini_usage_parser_sums_models_and_tool_calls(self): + usage = parse_gemini_usage( + { + "models": { + "model-a": { + "tokens": { + "prompt": 40, + "cached": 20, + "candidates": 10, + "thoughts": 3, + "total": 50, + } + }, + "model-b": { + "tokens": { + "prompt": 60, + "cached": 15, + "candidates": 25, + "thoughts": 7, + "total": 85, + } + }, + }, + "tools": {"totalCalls": 4}, + } + ) + + self.assertEqual(usage.input_tokens, 100) + self.assertEqual(usage.cached_input_tokens, 35) + self.assertEqual(usage.output_tokens, 35) + self.assertEqual(usage.reasoning_output_tokens, 10) + self.assertEqual(usage.total_tokens, 135) + self.assertEqual(usage.tool_calls, 4) + + def test_tool_only_usage_does_not_claim_native_token_coverage(self): + usage = ModelUsage.from_values(tool_calls=3) + + self.assertEqual(usage.tool_calls, 3) + self.assertEqual(usage.source, "unavailable") + + def test_usage_parsers_ignore_non_finite_counts(self): + for value in (math.nan, math.inf, -math.inf): + with self.subTest(source="model_usage", value=value): + usage = ModelUsage.from_values( + input_tokens=value, + cached_input_tokens=value, + output_tokens=value, + reasoning_output_tokens=value, + total_tokens=value, + tool_calls=value, + ) + self.assertEqual(usage, ModelUsage()) + + codex_event = json.dumps( + { + "type": "turn.completed", + "usage": { + "input_tokens": math.inf, + "output_tokens": math.nan, + }, + } + ) + _session_id, codex_usage, _final_message = parse_codex_jsonl(codex_event) + self.assertEqual(codex_usage, ModelUsage()) + + gemini_usage = parse_gemini_usage( + { + "models": { + "model-a": { + "tokens": { + "prompt": math.inf, + "candidates": math.nan, + } + } + }, + "tools": {"totalCalls": -math.inf}, + } + ) + self.assertEqual(gemini_usage, ModelUsage()) + + def test_cost_calculation_separates_cached_input(self): + usage = ModelUsage.from_values(input_tokens=1_000_000, cached_input_tokens=400_000, output_tokens=200_000) + rate = ModelRateCard( + input_per_million_usd=2.0, + cached_input_per_million_usd=0.5, + output_per_million_usd=8.0, + ) + + self.assertEqual(calculate_estimated_cost(usage, rate), 3.0) + self.assertEqual( + calculate_estimated_cost(ModelUsage(), ModelRateCard(per_invocation_usd=1.25)), + 1.25, + ) + + def test_invocation_sequence_carries_prompt_context_projection_across_attempts(self): + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=16, + omitted_events=84, + recent_events=8, + max_events=16, + ), + enabled=True, + selection_policy="recent_tail_plus_latest_role_evidence", + ) + + primary = sequence.next("primary") + repair = sequence.next("contract_repair") + + for context in (primary, repair): + self.assertTrue(context.prompt_context_enabled) + self.assertEqual(context.prompt_context_total_events, 100) + self.assertEqual(context.prompt_context_included_events, 16) + self.assertEqual(context.prompt_context_omitted_events, 84) + self.assertEqual(context.prompt_context_recent_events, 8) + self.assertEqual(context.prompt_context_max_events, 16) + self.assertEqual( + context.prompt_context_selection_policy, + "recent_tail_plus_latest_role_evidence", + ) + + def test_recorder_writes_privacy_safe_daily_shard_and_rate_snapshot(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + rate = ModelRateCard( + input_per_million_usd=2.0, + cached_input_per_million_usd=1.0, + output_per_million_usd=4.0, + ) + recorder = ModelTelemetryRecorder( + paths, + "local:planner", + TelemetryRuntimeConfig(rate_cards={"codex_cli/gpt-5.5": rate}), + ) + sequence = InvocationSequence( + runtime_identity="local:planner", + role="planner", + purpose="role_task", + workflow_step="planner_draft", + request_id="request-1", + sprint_id="sprint-a", + goal_id="goal-1", + ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=16, + omitted_events=84, + recent_events=8, + max_events=16, + ), + enabled=True, + selection_policy="recent_tail_plus_latest_role_evidence", + ) + now = runtime_now() + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="codex-cli test", + started_at=now, + ended_at=now + timedelta(seconds=1), + duration_ms=1000, + session_id_before="secret-session-id", + session_id_after="secret-session-id", + status="completed", + exit_code=0, + error_category="", + prompt_chars=500, + output_chars=100, + usage=ModelUsage.from_values(input_tokens=100, cached_input_tokens=40, output_tokens=20), + ) + + shards = list(paths.model_invocations_dir.rglob("*.jsonl")) + self.assertEqual(len(shards), 1) + raw_text = shards[0].read_text(encoding="utf-8") + record = json.loads(raw_text) + self.assertNotIn("secret-session-id", raw_text) + self.assertNotIn("prompt", record) + self.assertNotIn("response", record) + self.assertNotIn("workspace", record) + self.assertEqual(record["session_id_hash"], hash_session_id("secret-session-id")) + self.assertEqual(record["session_mode"], "resume") + self.assertEqual(record["goal_id"], "goal-1") + self.assertTrue(record["prompt_context_enabled"]) + self.assertEqual(record["prompt_context_total_events"], 100) + self.assertEqual(record["prompt_context_included_events"], 16) + self.assertEqual(record["prompt_context_omitted_events"], 84) + self.assertEqual(record["prompt_context_recent_events"], 8) + self.assertEqual(record["prompt_context_max_events"], 16) + self.assertEqual( + record["prompt_context_selection_policy"], + "recent_tail_plus_latest_role_evidence", + ) + self.assertIsNotNone(record["estimated_cost_usd"]) + self.assertEqual(record["rate_card"]["input_per_million_usd"], 2.0) + self.assertEqual(shards[0].parent.name, now.date().isoformat()) + self.assertTrue(shards[0].name.endswith(f".{os.getpid()}.jsonl")) + + def test_recorder_custom_output_is_external_private_daily_shard(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir).resolve() + workspace = root / "workspace" + workspace.mkdir() + paths = RuntimePaths.from_root(workspace) + output_dir = root / "private-telemetry" + recorder = ModelTelemetryRecorder( + paths, + "benchmark:planner", + output_dir=output_dir, + ) + now = runtime_now() + context = InvocationSequence( + runtime_identity="benchmark:planner", + role="planner", + purpose="role_task", + ).next() + + recorder.record( + context, + provider="codex_cli", + model="gpt-5.5", + reasoning="high", + cli_version="codex-cli test", + started_at=now, + ended_at=now, + duration_ms=1, + session_id_before=None, + session_id_after=None, + status="completed", + exit_code=0, + error_category="", + prompt_chars=1, + output_chars=1, + ) + + expected_day_dir = output_dir / now.date().isoformat() + expected_shard = expected_day_dir / f"benchmark_planner.{os.getpid()}.jsonl" + self.assertEqual(recorder.output_dir, output_dir.resolve()) + self.assertTrue(expected_shard.is_file()) + self.assertFalse(paths.model_invocations_dir.exists()) + self.assertEqual( + json.loads(expected_shard.read_text(encoding="utf-8"))["invocation_id"], + context.invocation_id, + ) + if os.name == "posix": + self.assertEqual(stat.S_IMODE(expected_day_dir.stat().st_mode), 0o700) + self.assertEqual(stat.S_IMODE(expected_shard.stat().st_mode), 0o600) + + def test_disabled_recorder_writes_nothing(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder( + paths, + "service-planner", + TelemetryRuntimeConfig(enabled=False), + ) + now = runtime_now() + sequence = InvocationSequence(runtime_identity="service-planner", role="planner", purpose="role_task") + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="medium", + cli_version="", + started_at=now, + ended_at=now, + duration_ms=0, + session_id_before=None, + session_id_after=None, + status="completed", + exit_code=0, + error_category="", + prompt_chars=1, + output_chars=1, + ) + + self.assertFalse(paths.model_invocations_dir.exists()) + + def test_recorder_is_fail_open_during_record_preparation(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder(paths, "service-planner") + sequence = InvocationSequence(runtime_identity="service-planner", role="planner", purpose="role_task") + now = runtime_now() + + with patch( + "teams_runtime.runtime.model_telemetry.calculate_estimated_cost", + side_effect=RuntimeError("telemetry preparation failed"), + ): + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="medium", + cli_version="", + started_at=now, + ended_at=now, + duration_ms=0, + session_id_before=None, + session_id_after=None, + status="completed", + exit_code=0, + error_category="", + prompt_chars=1, + output_chars=1, + ) + + self.assertFalse(paths.model_invocations_dir.exists()) + + def test_aggregation_filters_counts_percentiles_cost_and_invalid_lines(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder( + paths, + "service-planner", + TelemetryRuntimeConfig( + rate_cards={ + "codex_cli/gpt-5.5": ModelRateCard( + input_per_million_usd=1.0, + cached_input_per_million_usd=1.0, + output_per_million_usd=1.0, + ) + } + ), + ) + now = runtime_now() + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + request_id="request-1", + sprint_id="sprint-a", + ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=16, + omitted_events=84, + recent_events=8, + max_events=16, + ), + enabled=True, + selection_policy="recent_tail_plus_latest_role_evidence", + ) + for index, duration in enumerate((100, 200, 300), start=1): + recorder.record( + sequence.next("primary" if index == 1 else "contract_repair"), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now - timedelta(minutes=5), + ended_at=now - timedelta(minutes=4), + duration_ms=duration, + session_id_before=None, + session_id_after=f"session-{index}", + status="completed" if index < 3 else "failed", + exit_code=0 if index < 3 else 1, + error_category="" if index < 3 else "nonzero_exit", + prompt_chars=10, + output_chars=5, + usage=ModelUsage.from_values( + input_tokens=100, + cached_input_tokens=25, + output_tokens=20, + tool_calls=index, + ), + ) + shard = next(paths.model_invocations_dir.rglob("*.jsonl")) + with shard.open("a", encoding="utf-8") as handle: + handle.write("partial-json\n") + + summary = aggregate_model_invocations( + paths, + hours=1, + request_id="request-1", + sprint_id="sprint-a", + role="planner", + now=now, + ) + + self.assertEqual(summary["totals"]["invocation_count"], 3) + self.assertEqual(summary["totals"]["physical_attempt_count"], 3) + self.assertEqual(summary["totals"]["logical_call_count"], 1) + self.assertEqual(summary["totals"]["primary_count"], 1) + self.assertEqual(summary["totals"]["contract_repair_count"], 2) + self.assertEqual(summary["totals"]["failed_count"], 1) + self.assertEqual(summary["totals"]["tool_call_count"], 6) + self.assertEqual(summary["totals"]["tool_call_coverage_percent"], 100.0) + self.assertEqual(summary["totals"]["invalid_record_count"], 1) + self.assertEqual(summary["tokens"]["input"], 300) + self.assertEqual(summary["tokens"]["uncached_input"], 225) + self.assertEqual( + summary["prompt_context"], + { + "observed_invocation_count": 3, + "enabled_invocation_count": 3, + "eligible_invocation_count": 3, + "compacted_invocation_count": 3, + "total_events": 300, + "included_events": 48, + "omitted_events": 252, + "coverage_percent": 100.0, + "selection_policies": ["recent_tail_plus_latest_role_evidence"], + }, + ) + self.assertEqual(summary["latency_ms"]["p50"], 200) + self.assertEqual(summary["latency_ms"]["p95"], 300) + self.assertEqual(summary["totals"]["token_coverage_percent"], 100.0) + self.assertEqual(summary["totals"]["pricing_coverage_percent"], 100.0) + self.assertEqual(len(summary["groups"]), 1) + self.assertEqual(summary["groups"][0]["primary_count"], 1) + self.assertEqual(summary["groups"][0]["tool_call_count"], 6) + self.assertEqual(summary["groups"][0]["uncached_input_tokens"], 225) + self.assertEqual(summary["groups"][0]["prompt_context_observed_count"], 3) + self.assertEqual(summary["groups"][0]["prompt_context_compacted_count"], 3) + self.assertIn("role\tpurpose", render_model_metrics_summary(summary)) + + def test_aggregation_accepts_records_without_optional_projection_or_tool_metadata(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder(paths, "service-planner") + now = runtime_now() + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + ) + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now, + ended_at=now, + duration_ms=10, + session_id_before=None, + session_id_after="session-1", + status="completed", + exit_code=0, + error_category="", + prompt_chars=10, + output_chars=5, + usage=ModelUsage.from_values( + input_tokens=100, + cached_input_tokens=150, + output_tokens=20, + ), + ) + shard = next(paths.model_invocations_dir.rglob("*.jsonl")) + legacy_record = json.loads(shard.read_text(encoding="utf-8")) + legacy_record.pop("tool_calls") + for key in tuple(legacy_record): + if key.startswith("prompt_context_"): + legacy_record.pop(key) + shard.write_text(json.dumps(legacy_record) + "\n", encoding="utf-8") + + summary = aggregate_model_invocations(paths, hours=1, now=now) + + self.assertEqual(summary["tokens"]["uncached_input"], 0) + self.assertEqual(summary["totals"]["tool_call_count"], 0) + self.assertEqual(summary["totals"]["tool_call_coverage_percent"], 0.0) + self.assertEqual(summary["prompt_context"]["observed_invocation_count"], 0) + self.assertEqual(summary["prompt_context"]["coverage_percent"], 0.0) + + def test_aggregation_hides_partial_cost_totals_and_group_subtotals(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder( + paths, + "service-planner", + TelemetryRuntimeConfig( + rate_cards={ + "codex_cli/gpt-5.5": ModelRateCard( + input_per_million_usd=1.0, + output_per_million_usd=1.0, + ) + } + ), + ) + now = runtime_now() + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + ) + usages = ( + ModelUsage.from_values(input_tokens=100, output_tokens=20), + ModelUsage(), + ) + for index, usage in enumerate(usages): + recorder.record( + sequence.next("primary" if index == 0 else "contract_repair"), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now, + ended_at=now, + duration_ms=10, + session_id_before=None, + session_id_after=f"session-{index}", + status="completed", + exit_code=0, + error_category="", + prompt_chars=10, + output_chars=5, + usage=usage, + ) + + summary = aggregate_model_invocations(paths, hours=1, now=now) + + self.assertEqual(summary["totals"]["pricing_coverage_percent"], 50.0) + self.assertIsNone(summary["totals"]["estimated_cost_usd"]) + self.assertEqual(len(summary["groups"]), 1) + self.assertIsNone(summary["groups"][0]["estimated_cost_usd"]) + + def test_aggregation_distinguishes_disabled_eligible_history_from_compaction(self): + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + recorder = ModelTelemetryRecorder(paths, "service-planner") + now = runtime_now() + variants = ( + (False, 100, 0), + (True, 16, 84), + ) + for index, (enabled, included_events, omitted_events) in enumerate(variants): + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose=f"variant-{index}", + ) + sequence.set_prompt_context_projection( + SimpleNamespace( + total_events=100, + included_events=included_events, + omitted_events=omitted_events, + recent_events=8, + max_events=16, + ), + enabled=enabled, + selection_policy="recent_tail_plus_latest_role_evidence", + ) + recorder.record( + sequence.next(), + provider="codex_cli", + model="gpt-5.5", + reasoning="xhigh", + cli_version="test", + started_at=now, + ended_at=now, + duration_ms=10, + session_id_before=None, + session_id_after=f"session-{index}", + status="completed", + exit_code=0, + error_category="", + prompt_chars=10, + output_chars=5, + ) + + prompt_context = aggregate_model_invocations( + paths, + hours=1, + now=now, + )["prompt_context"] + + self.assertEqual(prompt_context["observed_invocation_count"], 2) + self.assertEqual(prompt_context["enabled_invocation_count"], 1) + self.assertEqual(prompt_context["eligible_invocation_count"], 2) + self.assertEqual(prompt_context["compacted_invocation_count"], 1) + self.assertEqual(prompt_context["total_events"], 200) + self.assertEqual(prompt_context["included_events"], 116) + self.assertEqual(prompt_context["omitted_events"], 84) + + def test_codex_runner_records_native_usage_without_changing_tuple_result(self): + with tempfile.TemporaryDirectory() as tmpdir: + workspace = Path(tmpdir) + paths = RuntimePaths.from_root(workspace) + recorder = ModelTelemetryRecorder(paths, "service-planner") + runner = CodexRunner( + RoleRuntimeConfig(model="gpt-5.5", reasoning="xhigh"), + role="planner", + telemetry_recorder=recorder, + ) + sequence = InvocationSequence( + runtime_identity="service-planner", + role="planner", + purpose="role_task", + request_id="request-1", + ) + output_file = workspace / ".teams_runtime_codex_output.txt" + role_output = '{"request_id":"request-1","role":"planner","status":"completed","summary":"ok"}' + stdout = "\n".join( + ( + json.dumps({"type": "thread.started", "thread_id": "thread-1"}), + json.dumps( + { + "type": "turn.completed", + "usage": {"input_tokens": 10, "cached_input_tokens": 3, "output_tokens": 4}, + } + ), + ) + ) + + def fake_run(*_args, **_kwargs): + output_file.write_text(role_output, encoding="utf-8") + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + CodexRunner._version_cache["codex"] = "codex-cli test" + with patch("teams_runtime.runtime.codex_runner.subprocess.run", side_effect=fake_run): + result = runner.run( + workspace, + "private prompt text", + None, + invocation_context=sequence.next(), + ) + + self.assertEqual(result, (role_output, "thread-1")) + record = json.loads(next(paths.model_invocations_dir.rglob("*.jsonl")).read_text(encoding="utf-8")) + self.assertEqual(record["input_tokens"], 10) + self.assertEqual(record["cached_input_tokens"], 3) + self.assertEqual(record["output_tokens"], 4) + self.assertEqual(record["total_tokens"], 14) + self.assertEqual(record["tool_calls"], 0) + self.assertNotIn("private prompt text", json.dumps(record)) + + def test_role_contract_repair_records_correlated_attempts(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + paths = RuntimePaths.from_root(tmpdir) + runtime = RoleAgentRuntime( + paths=paths, + role="planner", + sprint_id="sprint-a", + runtime_config=RoleRuntimeConfig(), + ) + responses = [ + "not-json", + '{"request_id":"request-1","role":"planner","status":"completed","summary":"repaired","artifacts":[]}', + ] + + def fake_run(_command, **kwargs): + workspace = Path(kwargs["cwd"]) + output_file = workspace / ".teams_runtime_codex_output.txt" + output_file.write_text(responses.pop(0), encoding="utf-8") + stdout = "\n".join( + ( + json.dumps({"type": "thread.started", "thread_id": "thread-1"}), + json.dumps( + { + "type": "turn.completed", + "usage": {"input_tokens": 10, "output_tokens": 2, "total_tokens": 12}, + } + ), + ) + ) + return SimpleNamespace(returncode=0, stdout=stdout, stderr="") + + envelope = MessageEnvelope( + request_id="request-1", + sender="orchestrator", + target="planner", + intent="plan", + urgency="normal", + scope="plan", + ) + request = {"request_id": "request-1", "scope": "plan", "body": "", "artifacts": []} + CodexRunner._version_cache["codex"] = "codex-cli test" + with patch("teams_runtime.runtime.codex_runner.subprocess.run", side_effect=fake_run): + result = runtime.run_task(envelope, request) + + self.assertEqual(result["status"], "completed") + records = [] + for shard in paths.model_invocations_dir.rglob("*.jsonl"): + records.extend(json.loads(line) for line in shard.read_text(encoding="utf-8").splitlines()) + self.assertEqual(len(records), 2) + self.assertEqual([record["attempt_kind"] for record in records], ["primary", "contract_repair"]) + self.assertEqual({record["operation_id"] for record in records}, {records[0]["operation_id"]}) + self.assertEqual({record["logical_call_id"] for record in records}, {records[0]["logical_call_id"]}) + self.assertEqual([record["attempt_index"] for record in records], [1, 2]) + + def test_config_defaults_and_rate_card_validation(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + config_path = Path(tmpdir) / "team_runtime.yaml" + payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) + payload.pop("telemetry", None) + config_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + + config = load_team_runtime_config(tmpdir) + + self.assertTrue(config.telemetry.enabled) + self.assertEqual(config.telemetry.rate_cards, {}) + + payload["telemetry"] = { + "enabled": True, + "rate_cards": { + "codex_cli/gpt-5.5": { + "input_per_million_usd": 2, + "output_per_million_usd": 8, + } + }, + } + config_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + config = load_team_runtime_config(tmpdir) + rate = config.telemetry.rate_cards["codex_cli/gpt-5.5"] + self.assertEqual(rate.cached_input_per_million_usd, 2.0) + + payload["telemetry"]["rate_cards"]["codex_cli/gpt-5.5"]["input_per_million_usd"] = math.inf + config_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "non-negative finite"): + load_team_runtime_config(tmpdir) + + for invalid_key in ("codex_cli", "/gpt-5.5", "codex_cli/"): + payload["telemetry"]["rate_cards"] = { + invalid_key: {"per_invocation_usd": 1}, + } + config_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "provider/model"): + load_team_runtime_config(tmpdir) + + def test_metrics_cli_parser_and_json_output(self): + args = build_parser().parse_args( + ["metrics", "--hours", "12", "--request-id", "request-1", "--agent", "planner", "--json"] + ) + self.assertEqual(args.command, "metrics") + self.assertEqual(args.hours, 12.0) + self.assertTrue(args.json) + + with tempfile.TemporaryDirectory() as tmpdir: + output = io.StringIO() + with redirect_stdout(output): + exit_code = cmd_metrics(Path(tmpdir), hours=1, as_json=True) + payload = json.loads(output.getvalue()) + self.assertEqual(exit_code, 0) + self.assertEqual(payload["schema_version"], 1) + self.assertEqual(payload["totals"]["invocation_count"], 0) + + output = io.StringIO() + with redirect_stdout(output): + exit_code = cmd_metrics(Path(tmpdir), hours=0) + self.assertEqual(exit_code, 2) + + def test_dedicated_telemetry_document_is_indexed(self): + package_root = Path(__file__).resolve().parents[1] + document = package_root / "docs" / "telemetry.md" + docs_index = package_root / "docs" / "README.md" + + self.assertTrue(document.is_file()) + content = document.read_text(encoding="utf-8") + self.assertIn("## Record Schema", content) + self.assertIn("## Privacy And Security", content) + self.assertIn("## CLI Reference", content) + self.assertIn("## Troubleshooting", content) + self.assertIn("telemetry.md", docs_index.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_orchestration_delegation.py b/tests/test_orchestration_delegation.py index a05e530..bbd64f6 100644 --- a/tests/test_orchestration_delegation.py +++ b/tests/test_orchestration_delegation.py @@ -1379,7 +1379,63 @@ def test_internal_sprint_request_record_initializes_workflow_contract(self): self.assertEqual(record["next_role"], "planner") self.assertEqual(workflow["planning_pass_limit"], 2) self.assertEqual(workflow["planning_pass_count"], 0) - self.assertEqual(workflow["review_cycle_limit"], 20) + self.assertEqual(workflow["review_cycle_limit"], 3) + self.assertEqual(workflow["reopen_limit"], 3) + self.assertEqual(workflow["reopen_count"], 0) + + def test_orchestrator_builds_internal_runtimes_from_helper_tiers(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + with patch("teams_runtime.core.orchestration.DiscordClient", FakeDiscordClient): + service = TeamService(tmpdir, "orchestrator") + + self.assertEqual( + service.intent_parser.codex_runner.runtime_config.model, + "gpt-5.4-mini", + ) + self.assertEqual( + service.intent_parser.codex_runner.runtime_config.reasoning, + "low", + ) + self.assertEqual( + service.goal_sourcer.codex_runner.runtime_config.reasoning, + "medium", + ) + self.assertEqual( + service.version_controller_runtime.codex_runner.runtime_config.reasoning, + "low", + ) + + def test_orchestrator_copies_workspace_workflow_budgets_into_new_state(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + policy_path = ( + Path(tmpdir) + / "orchestrator" + / ".agents" + / "skills" + / "agent_utilization" + / "policy.yaml" + ) + content = policy_path.read_text(encoding="utf-8") + content = content.replace( + "implementation_review_cycle_limit: 3", + "implementation_review_cycle_limit: 5", + 1, + ) + content = content.replace( + "implementation_reopen_limit: 3", + "implementation_reopen_limit: 2", + 1, + ) + policy_path.write_text(content, encoding="utf-8") + with patch("teams_runtime.core.orchestration.DiscordClient", FakeDiscordClient): + service = TeamService(tmpdir, "orchestrator") + + state = service._initial_workflow_state_for_internal_request() + + self.assertEqual(state["review_cycle_limit"], 5) + self.assertEqual(state["reopen_limit"], 2) def test_non_orchestrator_ready_resumes_pending_delegated_request(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/tests/test_orchestration_sprint_execution.py b/tests/test_orchestration_sprint_execution.py index 33fb857..9e3aba5 100644 --- a/tests/test_orchestration_sprint_execution.py +++ b/tests/test_orchestration_sprint_execution.py @@ -1,4 +1,5 @@ from teams_runtime.tests.orchestration_test_utils import * +from teams_runtime.workflows.sprints.lifecycle import pending_requirement_candidates_for_planner def _no_subject_definition(rationale="The sprint handoff is repo-local and does not need external research."): @@ -1659,6 +1660,123 @@ async def fake_execute(_sprint_state, todo): self.assertEqual(sprint_state["last_resume_checkpoint_status"], "running") self.assertEqual(str(sprint_state.get("resume_from_checkpoint_requested_at") or ""), "") + def test_manual_daily_sprint_does_not_force_planner_review_without_pending_candidates(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + with patch("teams_runtime.core.orchestration.DiscordClient", FakeDiscordClient): + service = TeamService(tmpdir, "orchestrator") + sprint_state = service._build_manual_sprint_state( + milestone_title="batch no-op planner reviews", + trigger="manual_start", + ) + sprint_state["phase"] = "ongoing" + sprint_state["status"] = "running" + sprint_state["last_planner_review_at"] = datetime.now(timezone.utc).isoformat() + sprint_state["todos"] = [ + build_todo_item( + build_backlog_item( + title=f"todo {index}", + summary=f"todo {index}", + kind="enhancement", + source="user", + scope=f"todo {index}", + ), + owner_role="developer", + ) + for index in (1, 2) + ] + + executed_todos: list[str] = [] + + async def fake_execute(_sprint_state, todo): + executed_todos.append(str(todo.get("todo_id") or "")) + todo["status"] = "completed" + + with ( + patch.object(service, "_save_sprint_state", return_value=None), + patch.object(service, "_sync_manual_sprint_queue", return_value=None), + patch.object(service, "_is_manual_sprint_cutoff_reached", return_value=False), + patch.object( + service, + "_run_internal_request_chain", + new=AsyncMock(side_effect=AssertionError("no-op review must not invoke planner")), + ) as planner_chain_mock, + patch.object(service, "_execute_sprint_todo", side_effect=fake_execute), + patch.object(service, "_finalize_sprint", new=AsyncMock(return_value=None)) as finalize_mock, + ): + asyncio.run(service._continue_manual_daily_sprint(sprint_state, announce=False)) + + self.assertEqual(len(executed_todos), 2) + planner_chain_mock.assert_not_awaited() + self.assertEqual(list(service.paths.requests_dir.glob("*.json")), []) + finalize_mock.assert_awaited_once_with(sprint_state) + + def test_manual_daily_sprint_batches_pending_candidates_into_one_checkpoint_review(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + with patch("teams_runtime.core.orchestration.DiscordClient", FakeDiscordClient): + service = TeamService(tmpdir, "orchestrator") + sprint_state = service._build_manual_sprint_state( + milestone_title="batch pending requirements", + trigger="manual_start", + ) + sprint_state["phase"] = "ongoing" + sprint_state["status"] = "running" + sprint_state["last_planner_review_at"] = datetime.now(timezone.utc).isoformat() + sprint_state["todos"] = [ + build_todo_item( + build_backlog_item( + title="todo with requirement feedback", + summary="todo with requirement feedback", + kind="enhancement", + source="user", + scope="todo with requirement feedback", + ), + owner_role="developer", + ) + ] + + review_calls: list[tuple[bool, bool, int]] = [] + + async def fake_review(review_state, *, force=False, requirement_checkpoint=False): + review_calls.append( + ( + force, + requirement_checkpoint, + len(pending_requirement_candidates_for_planner(review_state)), + ) + ) + if requirement_checkpoint: + review_state["pending_requirement_candidates"] = [] + + async def fake_execute(execution_state, todo): + todo["status"] = "committed" + execution_state["pending_requirement_candidates"] = [ + { + "candidate_id": "REQ-CAND-001", + "status": "pending", + "candidate_text": "Add keyboard-only acceptance.", + }, + { + "candidate_id": "REQ-CAND-002", + "status": "pending", + "candidate_text": "Preserve mobile approval flow.", + }, + ] + + with ( + patch.object(service, "_save_sprint_state", return_value=None), + patch.object(service, "_sync_manual_sprint_queue", return_value=None), + patch.object(service, "_is_manual_sprint_cutoff_reached", return_value=False), + patch.object(service, "_run_ongoing_sprint_review", side_effect=fake_review), + patch.object(service, "_execute_sprint_todo", side_effect=fake_execute), + patch.object(service, "_finalize_sprint", new=AsyncMock(return_value=None)) as finalize_mock, + ): + asyncio.run(service._continue_manual_daily_sprint(sprint_state, announce=False)) + + self.assertEqual(review_calls, [(False, False, 0), (True, True, 2)]) + finalize_mock.assert_awaited_once_with(sprint_state) + def test_manual_daily_sprint_wraps_up_when_only_terminal_todos_remain(self): with tempfile.TemporaryDirectory() as tmpdir: scaffold_workspace(tmpdir) diff --git a/tests/test_prompt_context.py b/tests/test_prompt_context.py new file mode 100644 index 0000000..227d69e --- /dev/null +++ b/tests/test_prompt_context.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import copy +import tempfile +import unittest +from pathlib import Path + +import yaml + +from teams_runtime.core.template import scaffold_workspace +from teams_runtime.runtime.base_runtime import RoleAgentRuntime +from teams_runtime.shared.config import load_team_runtime_config +from teams_runtime.shared.models import ( + MessageEnvelope, + PromptContextRuntimeConfig, + RoleRuntimeConfig, +) +from teams_runtime.shared.paths import RuntimePaths +from teams_runtime.shared.prompt_context import ( + project_request_record_for_prompt, + render_prompt_event_history_notice, +) +from teams_runtime.workflows.orchestration.team_service import TeamService +from teams_runtime.workflows.roles.research import build_research_decision_prompt + + +def _role_report(timestamp: str, role: str, summary: str) -> dict[str, object]: + return { + "timestamp": timestamp, + "type": "role_report", + "actor": role, + "payload": { + "role": role, + "status": "completed", + "summary": summary, + }, + } + + +def _envelope(request_id: str = "request-123") -> MessageEnvelope: + return MessageEnvelope( + request_id=request_id, + sender="orchestrator", + target="planner", + intent="plan", + urgency="normal", + scope="Compact the request prompt.", + body="ENVELOPE-CONTEXT-MARKER", + ) + + +class PromptContextProjectionTests(unittest.TestCase): + def test_projection_backfills_latest_missing_role_evidence(self): + events = [ + {"timestamp": "T01", "type": "created", "actor": "orchestrator", "summary": "Request created."}, + _role_report("T02", "research", "Research completed."), + {"timestamp": "T03", "type": "delegated", "actor": "orchestrator", "summary": "To planner."}, + _role_report("T04", "planner", "Initial plan drafted."), + _role_report("T05", "designer", "Design constraints recorded."), + _role_report("T06", "planner", "Final plan completed."), + {"timestamp": "T07", "type": "retried", "actor": "orchestrator", "summary": "Retried."}, + _role_report("T08", "developer", "Implementation completed."), + _role_report("T09", "architect", "Implementation reviewed."), + {"timestamp": "T10", "type": "delegated", "actor": "orchestrator", "summary": "To QA."}, + _role_report("T11", "qa", "A regression remains."), + {"timestamp": "T12", "type": "resumed", "actor": "orchestrator", "summary": "Resumed."}, + ] + request_record = { + "request_id": "request-123", + "status": "delegated", + "artifacts": ["shared_workspace/spec.md"], + "result": { + "role": "qa", + "status": "blocked", + "summary": "CURRENT-RESULT-MARKER", + }, + "events": events, + } + original = copy.deepcopy(request_record) + + projection = project_request_record_for_prompt( + request_record, + PromptContextRuntimeConfig(recent_events=4, max_events=7), + ) + + self.assertEqual( + [event["timestamp"] for event in projection.request_record["events"]], + ["T05", "T06", "T08", "T09", "T10", "T11", "T12"], + ) + self.assertEqual(projection.request_record["events"][0], events[4]) + self.assertEqual(projection.total_events, 12) + self.assertEqual(projection.included_events, 7) + self.assertEqual(projection.omitted_events, 5) + self.assertEqual( + projection.notice(), + { + "compacted": True, + "total_events": 12, + "included_events": 7, + "omitted_events": 5, + "recent_events": 4, + "max_events": 7, + "selection": "recent_tail_plus_latest_role_evidence", + "canonical_request": "./.teams_runtime/requests/request-123.json", + }, + ) + self.assertEqual(request_record, original) + + def test_disabled_and_under_limit_histories_remain_complete(self): + request_record = { + "request_id": "request-disabled", + "events": [ + _role_report(f"T{index:02d}", "planner", f"report-{index}") + for index in range(20) + ], + } + + disabled = project_request_record_for_prompt( + request_record, + PromptContextRuntimeConfig(enabled=False, recent_events=2, max_events=3), + ) + under_limit = project_request_record_for_prompt( + {"request_id": "request-short", "events": request_record["events"][:3]}, + PromptContextRuntimeConfig(recent_events=2, max_events=3), + ) + + self.assertEqual(disabled.request_record["events"], request_record["events"]) + self.assertFalse(disabled.compacted) + self.assertEqual(render_prompt_event_history_notice(disabled), "") + self.assertEqual(under_limit.request_record["events"], request_record["events"][:3]) + self.assertFalse(under_limit.compacted) + + def test_recent_only_and_malformed_histories_are_fail_safe(self): + events: list[object] = [ + _role_report("T01", "research", "old research"), + _role_report("T02", "planner", "old planner"), + {"timestamp": "T03", "type": "created"}, + "MALFORMED-RECENT-EVENT", + {"timestamp": "T05", "type": "resumed", "summary": "LATEST-EVENT"}, + ] + recent_only = project_request_record_for_prompt( + {"request_id": "request-recent", "events": events}, # type: ignore[typeddict-item] + PromptContextRuntimeConfig(recent_events=2, max_events=2), + ) + malformed_history = project_request_record_for_prompt( + {"request_id": "request-malformed", "events": "not-a-list"}, # type: ignore[typeddict-item] + PromptContextRuntimeConfig(recent_events=2, max_events=2), + ) + + self.assertEqual( + recent_only.request_record["events"], + ["MALFORMED-RECENT-EVENT", {"timestamp": "T05", "type": "resumed", "summary": "LATEST-EVENT"}], + ) + self.assertEqual(malformed_history.request_record["events"], "not-a-list") + self.assertFalse(malformed_history.compacted) + + def test_backfill_skips_roles_in_tail_and_accepts_legacy_evidence_shapes(self): + events = [ + {"timestamp": "T01", "type": "created"}, + _role_report("T02", "research", "Research evidence."), + _role_report("T03", "planner", "Stale planner evidence."), + { + "timestamp": "T04", + "type": "legacy", + "event_type": "role_report", + "actor": "architect", + "payload": {"summary": "Legacy architect evidence."}, + }, + { + "timestamp": "T05", + "type": "commit_inspected", + "actor": "orchestrator", + "payload": { + "role": "version_controller", + "status": "completed", + "summary": "Version-control evidence.", + }, + }, + _role_report("T06", "planner", "Planner is already represented in the tail."), + {"timestamp": "T07", "type": "resumed"}, + ] + + projection = project_request_record_for_prompt( + {"request_id": "request-evidence-shapes", "events": events}, + PromptContextRuntimeConfig(recent_events=2, max_events=5), + ) + + self.assertEqual( + [event["timestamp"] for event in projection.request_record["events"]], + ["T02", "T04", "T05", "T06", "T07"], + ) + self.assertNotIn(events[2], projection.request_record["events"]) + + def test_normal_repair_and_research_prompts_share_projection(self): + request_record = { + "request_id": "request-prompt", + "scope": "Compact the request prompt.", + "body": "", + "artifacts": ["ARTIFACT-MARKER"], + "params": {"workflow": {"phase": "implementation", "step": "developer_build"}}, + "result": { + "role": "planner", + "status": "completed", + "summary": "CURRENT-RESULT-MARKER", + }, + "events": [ + {"timestamp": "T01", "type": "created", "summary": "OMITTED-EVENT-MARKER"}, + _role_report("T02", "research", "RESEARCH-EVIDENCE-MARKER"), + _role_report("T03", "planner", "OLD-PLANNER-MARKER"), + {"timestamp": "T04", "type": "delegated", "summary": "RECENT-EVENT-ONE"}, + {"timestamp": "T05", "type": "resumed", "summary": "RECENT-EVENT-TWO"}, + ], + } + config = PromptContextRuntimeConfig(recent_events=2, max_events=3) + envelope = _envelope("request-prompt") + + with tempfile.TemporaryDirectory() as tmpdir: + runtime = RoleAgentRuntime( + paths=RuntimePaths.from_root(tmpdir), + role="planner", + sprint_id="sprint-a", + runtime_config=RoleRuntimeConfig(), + prompt_context_config=config, + ) + normal_prompt = runtime._build_prompt(envelope, request_record) + repair_prompt = runtime._build_role_result_repair_prompt( + request_record, + { + "request_id": "request-prompt", + "role": "planner", + "status": "failed", + "summary": "Invalid result.", + }, + ["missing_summary"], + current_sprint_id="sprint-a", + ) + + research_prompt = build_research_decision_prompt( + envelope, + request_record, + local_sources_checked=["request.scope"], + prompt_context_config=config, + ) + + for prompt in (normal_prompt, repair_prompt, research_prompt): + self.assertIn('"compacted": true', prompt) + self.assertIn("OLD-PLANNER-MARKER", prompt) + self.assertIn("RECENT-EVENT-ONE", prompt) + self.assertIn("RECENT-EVENT-TWO", prompt) + self.assertNotIn("OMITTED-EVENT-MARKER", prompt) + self.assertNotIn("RESEARCH-EVIDENCE-MARKER", prompt) + self.assertIn("CURRENT-RESULT-MARKER", prompt) + self.assertIn("ARTIFACT-MARKER", prompt) + + self.assertIn("ENVELOPE-CONTEXT-MARKER", normal_prompt) + self.assertIn("ENVELOPE-CONTEXT-MARKER", research_prompt) + + def test_large_history_prompt_is_bounded_and_materially_smaller(self): + large_text = "payload-context-" * 80 + roles = ("orchestrator", "research", "planner", "designer", "architect", "developer", "qa") + events = [ + { + **_role_report(f"T{index:03d}", roles[index % len(roles)], f"{index}-{large_text}"), + "sequence": index, + } + for index in range(100) + ] + request_record = { + "request_id": "request-large", + "scope": "Large history", + "body": "", + "artifacts": [], + "result": { + "role": "qa", + "status": "completed", + "summary": "CURRENT-LARGE-RESULT", + }, + "events": events, + } + envelope = _envelope("request-large") + + with tempfile.TemporaryDirectory() as tmpdir: + paths = RuntimePaths.from_root(tmpdir) + compact_runtime = RoleAgentRuntime( + paths=paths, + role="qa", + sprint_id="sprint-a", + runtime_config=RoleRuntimeConfig(), + prompt_context_config=PromptContextRuntimeConfig(), + ) + full_runtime = RoleAgentRuntime( + paths=paths, + role="qa", + sprint_id="sprint-a", + runtime_config=RoleRuntimeConfig(), + prompt_context_config=PromptContextRuntimeConfig(enabled=False), + ) + + compact_prompt = compact_runtime._build_prompt(envelope, request_record) + full_prompt = full_runtime._build_prompt(envelope, request_record) + + projection = project_request_record_for_prompt( + request_record, + PromptContextRuntimeConfig(), + ) + self.assertLessEqual(projection.included_events, 16) + self.assertLessEqual(len(compact_prompt), len(full_prompt) * 0.30) + self.assertIn("CURRENT-LARGE-RESULT", compact_prompt) + + +class PromptContextConfigTests(unittest.TestCase): + def test_config_defaults_custom_values_and_scaffold(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + config_path = Path(tmpdir) / "team_runtime.yaml" + scaffold_payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) + + default_config = load_team_runtime_config(tmpdir) + self.assertEqual(default_config.prompt_context, PromptContextRuntimeConfig()) + self.assertEqual( + scaffold_payload["prompt_context"], + {"enabled": True, "recent_events": 8, "max_events": 16}, + ) + + scaffold_payload.pop("prompt_context") + config_path.write_text(yaml.safe_dump(scaffold_payload, sort_keys=False), encoding="utf-8") + missing_config = load_team_runtime_config(tmpdir) + self.assertEqual(missing_config.prompt_context, PromptContextRuntimeConfig()) + + scaffold_payload["prompt_context"] = { + "enabled": False, + "recent_events": 4, + "max_events": 7, + } + config_path.write_text(yaml.safe_dump(scaffold_payload, sort_keys=False), encoding="utf-8") + custom_config = load_team_runtime_config(tmpdir) + self.assertEqual( + custom_config.prompt_context, + PromptContextRuntimeConfig(enabled=False, recent_events=4, max_events=7), + ) + + def test_config_rejects_invalid_prompt_context_values(self): + invalid_values = [ + ("not-a-mapping", "must be a mapping"), + ({"enabled": "yes"}, "enabled must be a boolean"), + ({"recent_events": True}, "recent_events must be a positive integer"), + ({"recent_events": 0}, "recent_events must be a positive integer"), + ({"max_events": 0}, "max_events must be a positive integer"), + ({"recent_events": 8, "max_events": 7}, "greater than or equal"), + ] + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + config_path = Path(tmpdir) / "team_runtime.yaml" + base_payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) + + for prompt_context, error_pattern in invalid_values: + with self.subTest(prompt_context=prompt_context): + payload = dict(base_payload) + payload["prompt_context"] = prompt_context + config_path.write_text( + yaml.safe_dump(payload, sort_keys=False), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, error_pattern): + load_team_runtime_config(tmpdir) + + def test_team_service_propagates_config_to_all_model_facing_role_runtimes(self): + with tempfile.TemporaryDirectory() as tmpdir: + scaffold_workspace(tmpdir) + config_path = Path(tmpdir) / "team_runtime.yaml" + payload = yaml.safe_load(config_path.read_text(encoding="utf-8")) + payload["prompt_context"] = { + "enabled": True, + "recent_events": 3, + "max_events": 6, + } + config_path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + expected = PromptContextRuntimeConfig(recent_events=3, max_events=6) + + service = TeamService(tmpdir, "planner", enable_discord_client=False) + research_runtime = service._runtime_for_role("research", service.runtime_config.sprint_id) + + self.assertEqual(service.role_runtime.prompt_context_config, expected) + self.assertEqual(research_runtime.prompt_context_config, expected) + self.assertEqual(service.version_controller_runtime.prompt_context_config, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sprint_benchmark.py b/tests/test_sprint_benchmark.py new file mode 100644 index 0000000..c52f02c --- /dev/null +++ b/tests/test_sprint_benchmark.py @@ -0,0 +1,2915 @@ +from __future__ import annotations + +import asyncio +import json +import io +import math +import os +import shutil +import stat +import subprocess +import sys +import tempfile +import unittest +from contextlib import redirect_stdout +from dataclasses import replace +from pathlib import Path +from typing import Any +from unittest import mock + +import yaml + +from teams_runtime.benchmarking import scenario as benchmark_scenario +from teams_runtime.benchmarking.metrics import ( + SAFE_INVOCATION_FIELDS, + compare_metrics, + reduce_telemetry, + sanitize_invocation_record, +) +from teams_runtime.benchmarking.models import ( + ArmPlan, + ArmResult, + BenchmarkOptions, + BenchmarkWorkerSafetyError, + QualityEvidence, + SprintEvidence, + WorkerContext, + WorkerOutcome, + invocation_identity_digest, + make_arm_schedule, +) +from teams_runtime.benchmarking.reporting import build_report +from teams_runtime.benchmarking.runner import ( + _journal_coverage_available, + run_sprint_ab_benchmark, +) +from teams_runtime.benchmarking.scenario import ( + BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + BENCHMARK_TARGET_INCLUDED_EVENTS, + BENCHMARK_TARGET_OMITTED_EVENTS, + BENCHMARK_TARGET_PURPOSE, + BENCHMARK_TARGET_ROLE, + BENCHMARK_TARGET_TOTAL_EVENTS, + BENCHMARK_TARGET_WORKFLOW_STEP, + PROTECTED_PATHS, + RuntimeSettings, + SCENARIO_ID, + SCENARIO_MILESTONE, + build_history_seed, + canonical_hash, + create_scenario_workspace, + load_runtime_settings, +) +from teams_runtime.benchmarking.worker import ( + LIVE_BENCHMARK_ENV, + _BenchmarkHistorySeedState, + _BenchmarkTeamService, + _build_execution_policy, + _history_seed_hash, + _summarize_call_journal, + run_live_sprint_arm, +) +from teams_runtime.cli import build_parser, cmd_benchmark_sprint_ab +from teams_runtime.runtime.execution_policy import ( + InvocationBudget, + InvocationBudgetExceeded, + ModelExecutionPolicy, + ModelExecutionPolicyViolation, +) +from teams_runtime.shared.models import ( + INTERNAL_TEAM_AGENTS, + PromptContextRuntimeConfig, + TEAM_ROLES, +) +from teams_runtime.shared.prompt_context import ( + PROMPT_EVENT_SELECTION_POLICY, + project_request_record_for_prompt, +) +from teams_runtime.workflows.orchestration.team_service import TeamService +from teams_runtime.workflows.sprints.lifecycle import apply_initial_plan_confirmation + + +_MISSING = object() +_HISTORY_SEED_HASH = "13024f26fb93918509533bfc5797e4fb3512944ae885234fd6da1393af72a365" + + +def _role_defaults() -> dict[str, dict[str, str]]: + return { + role: { + "model": "gpt-benchmark-test", + "reasoning": "medium", + } + for role in TEAM_ROLES + } + + +def _internal_agent_defaults() -> dict[str, dict[str, str]]: + return { + agent: { + "model": "gpt-benchmark-helper", + "reasoning": "low", + } + for agent in INTERNAL_TEAM_AGENTS + } + + +def _settings() -> RuntimeSettings: + role_defaults = _role_defaults() + internal_agent_defaults = _internal_agent_defaults() + return RuntimeSettings( + role_defaults=role_defaults, + internal_agent_defaults=internal_agent_defaults, + rate_cards={}, + source_config_hash=canonical_hash( + { + "role_defaults": role_defaults, + "internal_agent_defaults": internal_agent_defaults, + "rate_cards": {}, + } + ), + ) + + +def _git(root: Path, *args: str) -> subprocess.CompletedProcess[str]: + environment = { + "HOME": os.environ.get("HOME", ""), + "PATH": os.environ.get("PATH", ""), + "LC_ALL": "C", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + } + return subprocess.run( + ("git", *args), + cwd=root, + text=True, + capture_output=True, + check=True, + env=environment, + ) + + +def _initialize_source_repository(root: Path) -> None: + root.mkdir(parents=True) + _git(root, "init") + _git(root, "config", "--local", "user.name", "benchmark-test") + _git(root, "config", "--local", "user.email", "benchmark-test@invalid.local") + _git(root, "config", "--local", "commit.gpgsign", "false") + (root / "source-marker.txt").write_text("benchmark source\n", encoding="utf-8") + _git(root, "add", "source-marker.txt") + _git(root, "commit", "-m", "seed benchmark source") + + +def _write_runtime_config(path: Path) -> None: + path.write_text( + yaml.safe_dump({"role_defaults": _role_defaults()}, sort_keys=False), + encoding="utf-8", + ) + + +def _telemetry_record( + *, + variant: str, + occurrence: int = 1, + native_usage: bool = True, + estimated_cost: float | object = 0.001, + compacted: bool | None = None, +) -> dict[str, Any]: + is_after = variant == "after" + if compacted is None: + compacted = is_after + input_tokens = 240 if is_after else 480 + cached_tokens = 40 if is_after else 80 + output_tokens = 30 + included_events = ( + BENCHMARK_TARGET_INCLUDED_EVENTS + if compacted + else BENCHMARK_TARGET_TOTAL_EVENTS + ) + omitted_events = BENCHMARK_TARGET_OMITTED_EVENTS if compacted else 0 + record: dict[str, Any] = { + "schema_version": 1, + "invocation_id": f"{variant}-invocation-{occurrence}", + "operation_id": f"{variant}-operation-{occurrence}", + "logical_call_id": f"{variant}-logical-{occurrence}", + "attempt_index": 1, + "attempt_kind": "primary", + "started_at": f"2026-07-27T00:00:0{occurrence}+00:00", + "ended_at": f"2026-07-27T00:00:1{occurrence}+00:00", + "duration_ms": 400 if is_after else 700, + "runtime_identity": "role", + "role": BENCHMARK_TARGET_ROLE if occurrence == 1 else "developer", + "purpose": BENCHMARK_TARGET_PURPOSE if occurrence == 1 else "implement", + "workflow_step": ( + BENCHMARK_TARGET_WORKFLOW_STEP + if occurrence == 1 + else "todo_execution" + ), + "request_id": f"request-{occurrence}", + "sprint_id": "benchmark-sprint", + "todo_id": "todo-1", + "provider": "codex_cli", + "model": "gpt-benchmark-test", + "reasoning": "medium", + "status": "completed", + "exit_code": 0, + "prompt_chars": 1800 if is_after else 4200, + "output_chars": 300, + "tool_calls": 2, + "input_tokens": input_tokens, + "cached_input_tokens": cached_tokens, + "output_tokens": output_tokens, + "reasoning_output_tokens": 10, + "total_tokens": input_tokens + output_tokens, + "usage_source": "native" if native_usage else "", + "prompt_context_enabled": is_after, + "prompt_context_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "prompt_context_included_events": included_events, + "prompt_context_omitted_events": omitted_events, + "prompt_context_recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "prompt_context_max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "prompt_context_selection_policy": PROMPT_EVENT_SELECTION_POLICY, + "prompt_context": { + "enabled": is_after, + "compacted": bool(compacted), + "total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "included_events": included_events, + "omitted_events": omitted_events, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "selection_policy": PROMPT_EVENT_SELECTION_POLICY, + "raw_history": "SENSITIVE_HISTORY_SHOULD_NOT_PERSIST", + }, + "prompt": "SENSITIVE_PROMPT_SHOULD_NOT_PERSIST", + "response": "SENSITIVE_RESPONSE_SHOULD_NOT_PERSIST", + "api_key": "SENSITIVE_API_KEY_SHOULD_NOT_PERSIST", + "session_id": "SENSITIVE_SESSION_ID_SHOULD_NOT_PERSIST", + } + if estimated_cost is not _MISSING: + record["estimated_cost_usd"] = estimated_cost + return record + + +def _passing_quality() -> QualityEvidence: + return QualityEvidence( + behavior_oracle_passed=True, + sprint_terminal=True, + closeout_verified=True, + protected_files_unchanged=True, + git_clean=True, + commit_created=True, + no_git_remotes=True, + ) + + +def _arm_result( + variant: str, + records: tuple[dict[str, Any], ...], + *, + pair_index: int = 1, + order_index: int | None = None, + comparable_config_hash: str = "same-non-feature-config", +) -> ArmResult: + invocation_count = len(records) + completed_count = sum( + str(record.get("status") or "") == "completed" + for record in records + ) + invocation_ids = [ + str(record.get("invocation_id") or "") + for record in records + ] + candidate_metrics = reduce_telemetry(records) + verified_target_projection_count = int( + candidate_metrics["compaction"]["target_projection_candidate_count"] + ) + verified_target_invocation_ids_sha256 = str( + candidate_metrics["compaction"][ + "target_projection_invocation_ids_sha256" + ] + ) + return ArmResult( + arm=ArmPlan( + pair_index=pair_index, + order_index=order_index or (1 if variant == "before" else 2), + variant=variant, # type: ignore[arg-type] + run_id=f"pair-{pair_index:03d}-{variant}", + prompt_context_enabled=variant == "after", + ), + status="completed", + started_at="2026-07-27T00:00:00+00:00", + ended_at="2026-07-27T00:01:00+00:00", + wall_duration_ms=1_000 if variant == "before" else 700, + worker_duration_ms=900 if variant == "before" else 600, + stop_reason="", + error_category="", + config_hash=f"{variant}-config", + comparable_config_hash=comparable_config_hash, + metrics=reduce_telemetry( + records, + verified_target_projection_count=verified_target_projection_count, + verified_target_invocation_ids_sha256=( + verified_target_invocation_ids_sha256 + ), + ), + quality=_passing_quality(), + sprint=SprintEvidence( + sprint_id="benchmark-sprint", + status="completed", + closeout_status="verified", + todo_count=1, + completed_todo_count=1, + commit_sha=f"{variant}-commit", + ), + invocation_attempts={ + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 3, + "reconciled": True, + "identity_reconciled": True, + "context_reconciled": True, + "max_invocations": 20, + "reserved_count": invocation_count, + "entry_count": invocation_count, + "telemetry_record_count": invocation_count, + "unobserved_attempt_count": 0, + "telemetry_overage_count": 0, + "completed_count": completed_count, + "failed_count": invocation_count - completed_count, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 0, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 20 - invocation_count, + "journal_invocation_ids_sha256": ( + invocation_identity_digest(invocation_ids) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 0, + "journal_telemetry_context_mismatch_count": 0, + "verified_target_projection_count": verified_target_projection_count, + "verified_target_invocation_ids_sha256": ( + verified_target_invocation_ids_sha256 + ), + }, + invocation_records=records, + ) + + +def _report_for_runs( + runs: tuple[ArmResult, ...], + *, + repetitions: int = 1, +) -> dict[str, Any]: + options = BenchmarkOptions( + source_root=Path("/unused/source"), + runtime_config_path=Path("/unused/team_runtime.yaml"), + repetitions=repetitions, + ) + return build_report( + benchmark_id="deterministic-report", + options=options, + source_revision={"commit_sha": "source-sha", "dirty": False}, + source_config_hash="source-config-hash", + runtime_model_map={**_role_defaults(), **_internal_agent_defaults()}, + rate_cards={}, + history_hash=_HISTORY_SEED_HASH, + runs=runs, + started_at="2026-07-27T00:00:00+00:00", + ended_at="2026-07-27T00:02:00+00:00", + ) + + +class SprintBenchmarkScenarioTests(unittest.TestCase): + def test_runtime_settings_hash_and_record_effective_internal_agent_tiers(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + config_path = root / "team_runtime.yaml" + config_path.write_text( + yaml.safe_dump({"role_defaults": _role_defaults()}, sort_keys=False), + encoding="utf-8", + ) + + inherited = load_runtime_settings(config_path) + self.assertEqual( + inherited.internal_agent_defaults["parser"], + inherited.role_defaults["orchestrator"], + ) + + config_path.write_text( + yaml.safe_dump( + { + "role_defaults": _role_defaults(), + "internal_agent_defaults": { + "parser": {"model": "gpt-benchmark-helper"}, + }, + }, + sort_keys=False, + ), + encoding="utf-8", + ) + partial = load_runtime_settings(config_path) + self.assertEqual( + partial.internal_agent_defaults["parser"], + { + "model": "gpt-benchmark-helper", + "reasoning": _role_defaults()["orchestrator"]["reasoning"], + }, + ) + self.assertEqual( + partial.internal_agent_defaults["sourcer"], + partial.role_defaults["orchestrator"], + ) + + explicit_payload = { + "role_defaults": _role_defaults(), + "internal_agent_defaults": _internal_agent_defaults(), + } + config_path.write_text( + yaml.safe_dump(explicit_payload, sort_keys=False), + encoding="utf-8", + ) + explicit = load_runtime_settings(config_path) + + self.assertEqual( + explicit.internal_agent_defaults["sourcer"]["model"], + "gpt-benchmark-helper", + ) + self.assertNotEqual( + inherited.source_config_hash, + explicit.source_config_hash, + ) + + def test_schedule_alternates_pair_order_and_only_after_enables_compaction(self) -> None: + schedule = make_arm_schedule(3) + + self.assertEqual( + [ + ( + arm.pair_index, + arm.order_index, + arm.variant, + arm.run_id, + arm.prompt_context_enabled, + ) + for arm in schedule + ], + [ + (1, 1, "before", "pair-001-before", False), + (1, 2, "after", "pair-001-after", True), + (2, 3, "after", "pair-002-after", True), + (2, 4, "before", "pair-002-before", False), + (3, 5, "before", "pair-003-before", False), + (3, 6, "after", "pair-003-after", True), + ], + ) + with self.assertRaisesRegex(ValueError, "positive integer"): + make_arm_schedule(0) + + def test_history_seed_is_deterministic_balanced_and_has_a_golden_hash(self) -> None: + first = build_history_seed() + second = build_history_seed() + role_reports = [event for event in first if event["type"] == "role_report"] + + self.assertEqual(first, second) + self.assertEqual(len(first), 48) + self.assertEqual(canonical_hash(first), _HISTORY_SEED_HASH) + self.assertEqual( + [event["actor"] for event in role_reports], + [ + "research", + "planner", + "designer", + "architect", + "developer", + "qa", + "version_controller", + "orchestrator", + ], + ) + self.assertEqual(first[0]["created_at"], "2026-01-01T00:00:00+00:00") + self.assertEqual(first[-1]["created_at"], "2026-01-01T00:47:00+00:00") + with self.assertRaisesRegex(ValueError, "at least 24"): + build_history_seed(23) + + def test_fixture_reproduces_defect_and_config_fingerprints_only_feature_toggle(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + before = create_scenario_workspace( + root / "before", + benchmark_id="fixture-fingerprint", + run_id="pair-001-before", + prompt_context_enabled=False, + settings=_settings(), + ) + after = create_scenario_workspace( + root / "after", + benchmark_id="fixture-fingerprint", + run_id="pair-001-after", + prompt_context_enabled=True, + settings=_settings(), + ) + + before_config = yaml.safe_load( + (before.root / "team_runtime.yaml").read_text(encoding="utf-8") + ) + after_config = yaml.safe_load( + (after.root / "team_runtime.yaml").read_text(encoding="utf-8") + ) + self.assertNotEqual(before.config_hash, after.config_hash) + self.assertEqual(before.comparable_config_hash, after.comparable_config_hash) + self.assertEqual(before.history_hash, after.history_hash) + self.assertEqual(before.history_hash, _HISTORY_SEED_HASH) + scenario = json.loads( + (before.root / ".benchmark" / "scenario.json").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(scenario["scenario_id"], "sum-positive-full-sprint-v2") + accepted_return = ( + "return sum(value for value in values if value > 0)" + ) + self.assertIn(accepted_return, scenario["milestone"]) + self.assertIn( + accepted_return, + (before.root / "BENCHMARK_TASK.md").read_text(encoding="utf-8"), + ) + self.assertFalse(before_config["prompt_context"]["enabled"]) + self.assertTrue(after_config["prompt_context"]["enabled"]) + self.assertEqual( + before_config["internal_agent_defaults"], + _internal_agent_defaults(), + ) + before_config["prompt_context"].pop("enabled") + after_config["prompt_context"].pop("enabled") + self.assertEqual(before_config, after_config) + self.assertEqual(before.initial_commit_count, 1) + self.assertEqual(set(before.protected_hashes), set(PROTECTED_PATHS)) + self.assertIn( + "return sum(values)", + (before.root / "benchmark_app.py").read_text(encoding="utf-8"), + ) + baseline = subprocess.run( + ("python", "-m", "unittest", "discover", "-s", "tests"), + cwd=before.root, + text=True, + capture_output=True, + check=False, + env={ + "PATH": os.environ.get("PATH", ""), + "PYTHONPATH": str(before.root), + "LC_ALL": "C", + }, + ) + self.assertNotEqual(baseline.returncode, 0) + self.assertEqual(_git(before.root, "remote").stdout.strip(), "") + + def test_constrained_ast_oracle_accepts_only_the_documented_repair(self) -> None: + accepted = ( + '"""Optional module docstring."""\n\n' + "def sum_positive(values):\n" + ' """Optional function docstring."""\n' + " return sum(value for value in values if value > 0)\n" + ) + rejected = { + "top_level_statement": ( + "sentinel = 'would execute under an importing oracle'\n" + accepted + ), + "annotation": ( + "def sum_positive(values: list[int]):\n" + " return sum(value for value in values if value > 0)\n" + ), + "default": ( + "def sum_positive(values=()):\n" + " return sum(value for value in values if value > 0)\n" + ), + "list_comprehension": ( + "def sum_positive(values):\n" + " return sum([value for value in values if value > 0])\n" + ), + "renamed_operand": ( + "def sum_positive(values):\n" + " return sum(item for item in values if item > 0)\n" + ), + "extra_statement": ( + "def sum_positive(values):\n" + " positives = (value for value in values if value > 0)\n" + " return sum(positives)\n" + ), + } + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + target = root / "benchmark_app.py" + target.write_text(accepted, encoding="utf-8") + self.assertTrue(benchmark_scenario._sum_positive_ast_oracle(root)) + + for label, source in rejected.items(): + with self.subTest(label=label): + target.write_text(source, encoding="utf-8") + self.assertFalse( + benchmark_scenario._sum_positive_ast_oracle(root) + ) + + def test_final_oracle_never_executes_model_modified_python(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scenario = create_scenario_workspace( + root / "workspace", + benchmark_id="non-executing-oracle", + run_id="pair-001-before", + prompt_context_enabled=False, + settings=_settings(), + ) + marker = root / "model-code-executed" + (scenario.root / "benchmark_app.py").write_text( + "from pathlib import Path\n" + f"Path({str(marker)!r}).write_text('unsafe')\n\n" + "def sum_positive(values):\n" + " return sum(value for value in values if value > 0)\n", + encoding="utf-8", + ) + + def fake_git( + _root: Path, + *args: str, + **_kwargs: object, + ) -> subprocess.CompletedProcess[str]: + stdout = "" + if args[:2] == ("rev-parse", "HEAD"): + stdout = "new-commit\n" + return subprocess.CompletedProcess(args, 0, stdout, "") + + with ( + mock.patch.object( + benchmark_scenario, + "_run_git", + side_effect=fake_git, + ), + mock.patch.object( + benchmark_scenario.subprocess, + "run", + side_effect=AssertionError("workspace code must not be executed"), + ), + ): + inspection = benchmark_scenario.inspect_scenario_workspace(scenario) + + self.assertFalse(marker.exists()) + self.assertFalse(inspection.behavior_oracle_passed) + self.assertIn("behavior_oracle_failed", inspection.notes) + + def test_protected_hashing_rejects_symlinked_components(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + workspace = root / "workspace" + external = root / "external" + workspace.mkdir() + external.mkdir() + (external / "scenario.json").write_text("same bytes\n", encoding="utf-8") + (workspace / ".benchmark").symlink_to(external, target_is_directory=True) + + with self.assertRaises(OSError): + benchmark_scenario._protected_file_hash( + workspace, + ".benchmark/scenario.json", + ) + + def test_git_inspection_pins_binary_and_overrides_executable_config(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scenario = create_scenario_workspace( + root / "workspace", + benchmark_id="safe-git-inspection", + run_id="pair-001-after", + prompt_context_enabled=True, + settings=_settings(), + ) + (scenario.root / "benchmark_app.py").write_text( + "def sum_positive(values):\n" + " return sum(value for value in values if value > 0)\n", + encoding="utf-8", + ) + (scenario.root / ".gitattributes").write_text( + "benchmark_app.py filter=model-filter\n", + encoding="utf-8", + ) + _git(scenario.root, "add", "benchmark_app.py", ".gitattributes") + _git(scenario.root, "commit", "-m", "repair fixture") + + config_marker = root / "config-command-executed" + config_command = root / "model-config-command" + config_command.write_text( + "#!/bin/sh\n" + f": > {str(config_marker)!r}\n" + "cat\n", + encoding="utf-8", + ) + config_command.chmod(0o700) + for key in ( + "core.fsmonitor", + "core.hooksPath", + "diff.external", + "core.pager", + "filter.model-filter.clean", + ): + _git(scenario.root, "config", "--local", key, str(config_command)) + # Force status to inspect content rather than accepting the cached stat. + target = scenario.root / "benchmark_app.py" + target.write_bytes(target.read_bytes()) + + fake_bin = root / "fake-bin" + fake_bin.mkdir() + fake_git_marker = root / "fake-git-executed" + fake_git = fake_bin / "git" + fake_git.write_text( + "#!/bin/sh\n" + f": > {str(fake_git_marker)!r}\n" + "exit 0\n", + encoding="utf-8", + ) + fake_git.chmod(0o700) + + with mock.patch.dict(os.environ, {"PATH": str(fake_bin)}): + inspection = benchmark_scenario.inspect_scenario_workspace(scenario) + + self.assertTrue(inspection.behavior_oracle_passed) + self.assertTrue(inspection.protected_files_unchanged) + self.assertTrue(inspection.git_clean) + self.assertTrue(inspection.commit_created) + self.assertTrue(inspection.no_git_remotes) + self.assertFalse(config_marker.exists()) + self.assertFalse(fake_git_marker.exists()) + + (scenario.root / ".git" / "info" / "attributes").write_text( + "benchmark_app.py filter=model-filter\n", + encoding="utf-8", + ) + tampered_attributes = benchmark_scenario.inspect_scenario_workspace( + scenario + ) + self.assertFalse(tampered_attributes.git_clean) + self.assertIn("git_attributes_changed", tampered_attributes.notes) + self.assertFalse(config_marker.exists()) + + def test_git_command_uses_non_executing_inspection_overrides(self) -> None: + completed = subprocess.CompletedProcess(("git", "status"), 0, "", "") + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + with mock.patch.object( + benchmark_scenario.subprocess, + "run", + return_value=completed, + ) as run: + benchmark_scenario._run_git( + root, + "status", + git_executable=Path(sys.executable).resolve(), + ) + + command = run.call_args.args[0] + environment = run.call_args.kwargs["env"] + self.assertEqual(command[0], str(Path(sys.executable).resolve())) + self.assertIn("--no-pager", command) + self.assertIn(f"core.hooksPath={os.devnull}", command) + self.assertIn("core.fsmonitor=false", command) + self.assertIn(f"core.attributesFile={os.devnull}", command) + self.assertIn("diff.external=", command) + self.assertIn("core.pager=", command) + self.assertEqual(environment["GIT_EXTERNAL_DIFF"], "") + self.assertEqual(environment["GIT_PAGER"], "") + self.assertEqual(environment["PATH"], os.defpath) + self.assertEqual(run.call_args.kwargs["timeout"], 10.0) + + +class SprintBenchmarkBackfillTests(unittest.TestCase): + @staticmethod + def _service( + state: _BenchmarkHistorySeedState | None = None, + ) -> _BenchmarkTeamService: + service = object.__new__(_BenchmarkTeamService) + service._benchmark_context = mock.Mock(history_seed=build_history_seed()) + service._benchmark_history_state = state or _BenchmarkHistorySeedState() + service._benchmark_history_seeded = False + return service + + def test_first_sprint_request_is_seeded_before_persistence(self) -> None: + service = self._service() + persisted: list[dict[str, Any]] = [] + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + second_request = { + "request_id": "todo-request", + "params": {"_teams_kind": "sprint_internal"}, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + def capture(_service: TeamService, record: dict[str, Any]) -> None: + persisted.append(json.loads(json.dumps(record))) + + with mock.patch.object( + TeamService, + "_save_request", + autospec=True, + side_effect=capture, + ): + service._save_request(request_record) + service._save_request(second_request) + + seed = build_history_seed() + self.assertEqual(len(persisted), 2) + self.assertEqual(persisted[0]["events"][: len(seed)], list(seed)) + self.assertEqual( + canonical_hash(persisted[0]["events"][: len(seed)]), + _HISTORY_SEED_HASH, + ) + self.assertEqual( + persisted[0]["params"]["_benchmark_history_seed"], + { + "event_count": len(seed), + "sha256": _HISTORY_SEED_HASH, + }, + ) + self.assertEqual( + persisted[0]["events"][-1], + {"type": "created", "actor": "sprint_runner"}, + ) + self.assertNotIn("_benchmark_history_seed", persisted[1]["params"]) + self.assertEqual(len(persisted[1]["events"]), 1) + + def test_persisted_seed_is_idempotent_across_role_services(self) -> None: + shared_state = _BenchmarkHistorySeedState() + first_service = self._service(shared_state) + relay_service = self._service(shared_state) + later_service = self._service(shared_state) + seed = build_history_seed() + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + with mock.patch.object(TeamService, "_save_request", autospec=True): + first_service._save_request(request_record) + first_event_count = len(request_record["events"]) + relay_service._save_request(request_record) + later_request = { + "request_id": "later-planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + later_service._save_request(later_request) + + self.assertEqual(first_event_count, len(seed) + 1) + self.assertEqual(len(request_record["events"]), first_event_count) + self.assertTrue(relay_service._benchmark_history_seeded) + self.assertEqual(len(later_request["events"]), 1) + self.assertNotIn( + "_benchmark_history_seed", + later_request["params"], + ) + + def test_non_initial_sprint_request_cannot_consume_backfill(self) -> None: + service = self._service() + request_record = { + "request_id": "todo-request", + "params": {"_teams_kind": "sprint_internal"}, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + with mock.patch.object(TeamService, "_save_request", autospec=True): + service._save_request(request_record) + + self.assertEqual(len(request_record["events"]), 1) + self.assertNotIn( + "_benchmark_history_seed", + request_record["params"], + ) + self.assertFalse(service._benchmark_history_seeded) + + def test_seed_marker_conflicts_fail_before_persistence(self) -> None: + service = self._service() + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + "_benchmark_history_seed": { + "event_count": 48, + "sha256": "wrong-hash", + }, + }, + "events": list(build_history_seed()), + } + + with mock.patch.object( + TeamService, + "_save_request", + autospec=True, + ) as save_request: + with self.assertRaisesRegex(ValueError, "invalid history seed marker"): + service._save_request(request_record) + + save_request.assert_not_called() + self.assertFalse(service._benchmark_history_seeded) + + def test_failed_persistence_does_not_mark_seed_as_complete(self) -> None: + service = self._service() + request_record = { + "request_id": "planning-request", + "params": { + "_teams_kind": "sprint_internal", + "sprint_phase": "initial", + "initial_phase_step": "milestone_refinement", + }, + "events": [{"type": "created", "actor": "sprint_runner"}], + } + + with mock.patch.object( + TeamService, + "_save_request", + autospec=True, + side_effect=OSError("write failed"), + ): + with self.assertRaisesRegex(OSError, "write failed"): + service._save_request(request_record) + + self.assertFalse(service._benchmark_history_seeded) + + def test_real_planning_lifecycle_produces_exact_v2_projection(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scenario = create_scenario_workspace( + root / "arm", + benchmark_id="planning-seed-boundary", + run_id="pair-001-before", + prompt_context_enabled=False, + settings=_settings(), + ) + context = WorkerContext( + benchmark_id="planning-seed-boundary", + arm=ArmPlan( + pair_index=1, + order_index=1, + variant="before", + run_id="pair-001-before", + prompt_context_enabled=False, + ), + workspace_root=scenario.root, + run_output_dir=root / "run", + milestone="Verify deterministic Backfill persistence.", + history_seed=scenario.history_seed, + max_invocations=1, + call_timeout_seconds=1, + run_timeout_seconds=1, + live=True, + ) + service = _BenchmarkTeamService( + scenario.root, + "orchestrator", + enable_discord_client=False, + relay_transport="internal", + allow_external_research=False, + benchmark_context=context, + ) + sprint_state = service._build_manual_sprint_state( + milestone_title="Verify deterministic Backfill persistence.", + trigger="benchmark", + ) + + request_record = service._build_sprint_planning_request_record( + sprint_state, + phase="initial", + iteration=1, + step="milestone_refinement", + ) + persisted = service._load_request(request_record["request_id"]) + relay_service = _BenchmarkTeamService( + scenario.root, + "research", + enable_discord_client=False, + relay_transport="internal", + allow_external_research=False, + benchmark_context=context, + benchmark_history_state=service._benchmark_history_state, + ) + relay_service._save_request(persisted) + persisted = relay_service._load_request(request_record["request_id"]) + + with ( + mock.patch.object( + service, + "_delegate_request", + new=mock.AsyncMock(return_value=True), + ), + mock.patch.object( + service, + "_wait_for_internal_request_result", + new=mock.AsyncMock(return_value={"status": "completed"}), + ), + mock.patch.object(service, "_append_role_history"), + mock.patch.object(service, "_record_internal_sprint_activity"), + ): + asyncio.run( + service._run_internal_request_chain( + sprint_id=str(sprint_state["sprint_id"]), + request_record=persisted, + initial_role="research", + ) + ) + delegated = service._load_request(request_record["request_id"]) + + seed_count = len(build_history_seed()) + persisted_prefix = list(delegated["events"][:seed_count]) + self.assertEqual(len(delegated["events"]), BENCHMARK_TARGET_TOTAL_EVENTS) + self.assertEqual(_history_seed_hash(persisted_prefix), _HISTORY_SEED_HASH) + self.assertEqual( + delegated["params"]["_benchmark_history_seed"], + { + "event_count": seed_count, + "sha256": _HISTORY_SEED_HASH, + }, + ) + self.assertEqual( + [event["type"] for event in delegated["events"][-2:]], + ["created", "delegated"], + ) + before_projection = project_request_record_for_prompt( + delegated, + PromptContextRuntimeConfig( + enabled=False, + recent_events=BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + max_events=BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + ), + ) + after_projection = project_request_record_for_prompt( + delegated, + PromptContextRuntimeConfig( + enabled=True, + recent_events=BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + max_events=BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + ), + ) + self.assertEqual( + ( + before_projection.total_events, + before_projection.included_events, + before_projection.omitted_events, + ), + (BENCHMARK_TARGET_TOTAL_EVENTS, BENCHMARK_TARGET_TOTAL_EVENTS, 0), + ) + self.assertEqual( + ( + after_projection.total_events, + after_projection.included_events, + after_projection.omitted_events, + ), + ( + BENCHMARK_TARGET_TOTAL_EVENTS, + BENCHMARK_TARGET_INCLUDED_EVENTS, + BENCHMARK_TARGET_OMITTED_EVENTS, + ), + ) + + +class SprintBenchmarkCliTests(unittest.TestCase): + def test_options_reject_non_finite_timeouts(self) -> None: + options = BenchmarkOptions( + source_root=Path.cwd(), + runtime_config_path=Path(__file__), + ) + + for field_name in ("call_timeout_seconds", "run_timeout_seconds"): + for value in (math.nan, math.inf, -math.inf): + with self.subTest(field_name=field_name, value=value): + with self.assertRaises(ValueError): + replace(options, **{field_name: value}).validate() + + def test_parser_exposes_bounded_sprint_ab_defaults(self) -> None: + args = build_parser().parse_args( + [ + "benchmark", + "sprint-ab", + "--runtime-config", + "deployed/team_runtime.yaml", + ] + ) + + self.assertEqual(args.command, "benchmark") + self.assertEqual(args.benchmark_command, "sprint-ab") + self.assertFalse(args.live) + self.assertEqual(args.repetitions, 1) + self.assertEqual(args.max_invocations, 20) + self.assertEqual(args.call_timeout_seconds, 300.0) + self.assertEqual(args.run_timeout_seconds, 1800.0) + self.assertEqual(args.keep_workspaces, "failures") + + def test_cli_requires_both_live_opt_ins_before_calling_runner(self) -> None: + cases = ( + (False, {"TEAMS_RUNTIME_LIVE_BENCHMARK": "1"}), + (True, {}), + ) + for live, environment in cases: + with self.subTest(live=live, environment=environment): + with ( + mock.patch.dict(os.environ, environment, clear=True), + mock.patch( + "teams_runtime.benchmarking.runner.run_sprint_ab_benchmark" + ) as runner, + redirect_stdout(io.StringIO()), + ): + exit_code = cmd_benchmark_sprint_ab( + live=live, + runtime_config="unused.yaml", + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1800, + keep_workspaces="failures", + ) + self.assertEqual(exit_code, 2) + runner.assert_not_called() + + def test_cli_renders_successful_result_as_json(self) -> None: + result = mock.Mock( + benchmark_id="json-success", + status="comparable", + classification="preliminary_smoke", + output_dir=Path("/tmp/json-success"), + report_json=Path("/tmp/json-success/report.json"), + report_markdown=Path("/tmp/json-success/report.md"), + exit_code=0, + ) + output = io.StringIO() + + with ( + mock.patch.dict( + os.environ, + {"TEAMS_RUNTIME_LIVE_BENCHMARK": "1"}, + clear=True, + ), + mock.patch( + "teams_runtime.benchmarking.runner.run_sprint_ab_benchmark", + return_value=result, + ) as runner, + redirect_stdout(output), + ): + exit_code = cmd_benchmark_sprint_ab( + live=True, + runtime_config="unused.yaml", + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1800, + keep_workspaces="failures", + as_json=True, + ) + + self.assertEqual(exit_code, 0) + self.assertEqual( + json.loads(output.getvalue()), + { + "benchmark_id": "json-success", + "status": "comparable", + "classification": "preliminary_smoke", + "output_dir": "/tmp/json-success", + "report_json": "/tmp/json-success/report.json", + "report_markdown": "/tmp/json-success/report.md", + "exit_code": 0, + }, + ) + runner.assert_called_once() + + def test_cli_returns_two_for_fatal_worker_safety_abort(self) -> None: + output = io.StringIO() + + with ( + mock.patch.dict( + os.environ, + {"TEAMS_RUNTIME_LIVE_BENCHMARK": "1"}, + clear=True, + ), + mock.patch( + "teams_runtime.benchmarking.runner.run_sprint_ab_benchmark", + side_effect=BenchmarkWorkerSafetyError( + "provider cleanup could not be confirmed" + ), + ), + redirect_stdout(output), + ): + exit_code = cmd_benchmark_sprint_ab( + live=True, + runtime_config="unused.yaml", + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1800, + keep_workspaces="failures", + ) + + self.assertEqual(exit_code, 2) + self.assertIn("Benchmark safety abort", output.getvalue()) + + +class SprintBenchmarkReportTests(unittest.TestCase): + def test_fake_worker_generates_comparable_private_full_report(self) -> None: + seen_contexts: list[dict[str, Any]] = [] + + def fake_worker(context: WorkerContext) -> WorkerOutcome: + config = yaml.safe_load( + (context.workspace_root / "team_runtime.yaml").read_text(encoding="utf-8") + ) + seen_contexts.append( + { + "run_id": context.arm.run_id, + "variant": context.arm.variant, + "prompt_context_enabled": config["prompt_context"]["enabled"], + "history_hash": canonical_hash(context.history_seed), + "max_invocations": context.max_invocations, + "call_timeout_seconds": context.call_timeout_seconds, + "run_timeout_seconds": context.run_timeout_seconds, + "live": context.live, + } + ) + target = context.workspace_root / "benchmark_app.py" + self.assertIn("return sum(values)", target.read_text(encoding="utf-8")) + target.write_text( + '"""Small benchmark target with a repaired implementation."""\n\n' + "\n" + "def sum_positive(values):\n" + ' """Return the sum of positive numeric values."""\n' + " return sum(value for value in values if value > 0)\n", + encoding="utf-8", + ) + _git(context.workspace_root, "add", "benchmark_app.py") + _git( + context.workspace_root, + "commit", + "-m", + f"repair fixture for {context.arm.variant}", + ) + commit_sha = _git(context.workspace_root, "rev-parse", "HEAD").stdout.strip() + records = tuple( + _telemetry_record( + variant=context.arm.variant, + occurrence=occurrence, + estimated_cost=0.0005 if context.arm.variant == "after" else 0.001, + ) + for occurrence in (1, 2) + ) + return WorkerOutcome( + status="completed", + sprint=SprintEvidence( + sprint_id="benchmark-sprint", + status="completed", + closeout_status="verified", + todo_count=1, + completed_todo_count=1, + commit_sha=commit_sha, + ), + quality=QualityEvidence( + sprint_terminal=True, + closeout_verified=True, + ), + telemetry_records=records, + invocation_attempts={ + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 3, + "reconciled": True, + "identity_reconciled": True, + "context_reconciled": True, + "max_invocations": 20, + "reserved_count": 2, + "entry_count": 2, + "telemetry_record_count": 2, + "completed_count": 2, + "failed_count": 0, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 0, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 18, + "journal_invocation_ids_sha256": ( + invocation_identity_digest( + record["invocation_id"] + for record in records + ) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 0, + "journal_telemetry_context_mismatch_count": 0, + "verified_target_projection_count": 1, + "verified_target_invocation_ids_sha256": ( + invocation_identity_digest( + (records[0]["invocation_id"],) + ) + ), + "prompt": "SENSITIVE_ATTEMPT_SUMMARY_SHOULD_NOT_PERSIST", + }, + started_at="2026-07-27T00:00:00+00:00", + ended_at="2026-07-27T00:00:02+00:00", + wall_duration_ms=1_200 if context.arm.variant == "before" else 800, + worker_duration_ms=1_100 if context.arm.variant == "before" else 700, + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + output_root = root / "reports" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + options = BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=output_root, + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1_800, + keep_workspaces="none", + live=False, + benchmark_id="fake-worker-full-report", + ) + + result = run_sprint_ab_benchmark(options, worker=fake_worker) + + self.assertEqual(result.status, "comparable") + self.assertEqual(result.exit_code, 0) + self.assertEqual(result.classification, "preliminary_smoke") + self.assertTrue(result.report_json.is_file()) + self.assertTrue(result.report_markdown.is_file()) + self.assertEqual( + [(item["variant"], item["prompt_context_enabled"]) for item in seen_contexts], + [("before", False), ("after", True)], + ) + self.assertEqual( + {item["history_hash"] for item in seen_contexts}, + {_HISTORY_SEED_HASH}, + ) + self.assertTrue( + all( + item["max_invocations"] == 20 + and item["call_timeout_seconds"] == 300 + and item["run_timeout_seconds"] == 1_800 + and item["live"] is False + for item in seen_contexts + ) + ) + self.assertNotEqual(result.runs[0].config_hash, result.runs[1].config_hash) + self.assertEqual( + result.runs[0].comparable_config_hash, + result.runs[1].comparable_config_hash, + ) + self.assertTrue(all(run.quality.passed for run in result.runs)) + self.assertTrue(all(not run.retained_workspace for run in result.runs)) + + report = json.loads(result.report_json.read_text(encoding="utf-8")) + self.assertEqual(report["schema_version"], 3) + self.assertEqual(report["provenance"]["scenario_id"], SCENARIO_ID) + self.assertEqual( + report["controls"]["target_invocation"], + { + "attempt_kind": "primary", + "role": BENCHMARK_TARGET_ROLE, + "purpose": BENCHMARK_TARGET_PURPOSE, + "workflow_step": BENCHMARK_TARGET_WORKFLOW_STEP, + }, + ) + self.assertEqual( + report["controls"]["a_b_definition"], + { + "before": { + "prompt_context_enabled": False, + "target_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_included_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_omitted_events": 0, + }, + "after": { + "prompt_context_enabled": True, + "recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "target_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "target_included_events": BENCHMARK_TARGET_INCLUDED_EVENTS, + "target_omitted_events": BENCHMARK_TARGET_OMITTED_EVENTS, + }, + }, + ) + self.assertEqual( + report["runs"][0]["invocation_attempts"]["reserved_count"], + 2, + ) + self.assertTrue( + report["runs"][0]["invocation_attempts"][ + "context_reconciled" + ] + ) + self.assertEqual( + report["runs"][0]["invocation_attempts"][ + "verified_target_projection_count" + ], + 1, + ) + self.assertNotIn( + "prompt", + report["runs"][0]["invocation_attempts"], + ) + pair = report["pairs"][0] + self.assertTrue(pair["comparable"]) + self.assertEqual(pair["execution_order"], ["before", "after"]) + self.assertEqual(pair["inconclusive_reasons"], []) + self.assertEqual( + pair["comparison"]["end_to_end"]["input_tokens"]["reduction"], + 480, + ) + self.assertEqual(pair["comparison"]["matched_primary_count"], 2) + self.assertEqual( + report["aggregate_reductions"]["input_tokens"]["pair_count"], + 1, + ) + self.assertIsNone( + report["aggregate_reductions"]["input_tokens"][ + "sample_standard_deviation" + ] + ) + self.assertFalse( + report["interpretation"]["statistical_significance_claimed"] + ) + self.assertIn("preliminary", report["interpretation"]["note"].lower()) + markdown = result.report_markdown.read_text(encoding="utf-8") + self.assertIn("| Reserved | Telemetry | Completed |", markdown) + self.assertIn(f"- Scenario: `{SCENARIO_ID}`", markdown) + self.assertEqual(markdown.count("| 1 | 1200 | pass |"), 1) + self.assertEqual(markdown.count("| 1 | 800 | pass |"), 1) + + for run in result.runs: + run_dir = result.output_dir / "runs" / run.arm.run_id + self.assertTrue((run_dir / "run.json").is_file()) + self.assertTrue((run_dir / "metrics.json").is_file()) + self.assertTrue((run_dir / "model_invocations.jsonl").is_file()) + persisted_metrics = json.loads( + (run_dir / "metrics.json").read_text(encoding="utf-8") + ) + self.assertEqual( + persisted_metrics["compaction"]["target_projection_count"], + 1, + ) + invocation_lines = ( + run_dir / "model_invocations.jsonl" + ).read_text(encoding="utf-8").splitlines() + self.assertEqual(len(invocation_lines), 2) + persisted = json.loads(invocation_lines[0]) + self.assertLessEqual(set(persisted), SAFE_INVOCATION_FIELDS) + self.assertNotIn("raw_history", persisted["prompt_context"]) + self.assertNotIn( + "invocations", + json.loads((run_dir / "run.json").read_text(encoding="utf-8")), + ) + + for artifact in result.output_dir.rglob("*"): + if artifact.is_file(): + content = artifact.read_text(encoding="utf-8") + self.assertNotIn("SENSITIVE_", content, artifact) + + def test_runner_does_not_fall_back_to_workspace_telemetry(self) -> None: + def fake_worker(context: WorkerContext) -> WorkerOutcome: + metrics_root = ( + context.workspace_root + / ".teams_runtime" + / "metrics" + / "model_invocations" + ) + metrics_root.mkdir(parents=True, exist_ok=True) + forged_record = _telemetry_record(variant=context.arm.variant) + (metrics_root / "forged.jsonl").write_text( + json.dumps(forged_record, sort_keys=True) + "\n", + encoding="utf-8", + ) + return WorkerOutcome(status="completed", telemetry_records=()) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=root / "reports", + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1_800, + keep_workspaces="none", + live=False, + benchmark_id="workspace-telemetry-isolation", + ), + worker=fake_worker, + ) + + self.assertEqual(result.status, "inconclusive") + self.assertEqual(len(result.runs), 2) + for run in result.runs: + self.assertEqual(run.invocation_records, ()) + self.assertEqual(run.metrics["totals"]["invocation_count"], 0) + self.assertEqual( + run.metrics["compaction"]["observed_invocation_count"], + 0, + ) + + def test_untrusted_journal_never_persists_coverage_or_cost(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + output_root = root / "reports" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + + for case_name in ( + "missing", + "unsupported_schema", + "unreconciled", + "telemetry_overage", + "duplicate_telemetry", + "mismatched_telemetry", + ): + with self.subTest(case_name=case_name): + + def fake_worker(context: WorkerContext) -> WorkerOutcome: + occurrences = ( + (1, 2) + if case_name + in { + "telemetry_overage", + "duplicate_telemetry", + } + else (1,) + ) + records = tuple( + _telemetry_record( + variant=context.arm.variant, + occurrence=occurrence, + estimated_cost=0.002, + ) + for occurrence in occurrences + ) + journal_invocation_ids = [ + str(record["invocation_id"]) + for record in records + ] + if case_name == "duplicate_telemetry": + records[1]["invocation_id"] = records[0][ + "invocation_id" + ] + elif case_name == "mismatched_telemetry": + records[0]["invocation_id"] = ( + "unmatched-telemetry-invocation" + ) + reserved_count = ( + 2 + if case_name == "duplicate_telemetry" + else 1 + ) + attempts: dict[str, Any] = { + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "identity_reconciled": True, + "max_invocations": 20, + "reserved_count": reserved_count, + "entry_count": reserved_count, + "telemetry_record_count": len(records), + "unobserved_attempt_count": 0, + "telemetry_overage_count": 0, + "completed_count": reserved_count, + "failed_count": 0, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 0, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 20 - reserved_count, + "journal_invocation_ids_sha256": ( + invocation_identity_digest( + journal_invocation_ids[ + :reserved_count + ] + ) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 0, + } + if case_name == "missing": + attempts = {} + elif case_name == "unsupported_schema": + attempts["journal_schema_version"] = 99 + elif case_name == "unreconciled": + attempts["reconciled"] = False + return WorkerOutcome( + status="completed", + telemetry_records=records, + invocation_attempts=attempts, + ) + + benchmark_id = f"untrusted-journal-{case_name}" + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=output_root, + repetitions=1, + max_invocations=20, + call_timeout_seconds=300, + run_timeout_seconds=1_800, + keep_workspaces="none", + live=False, + benchmark_id=benchmark_id, + ), + worker=fake_worker, + ) + report = json.loads( + result.report_json.read_text(encoding="utf-8") + ) + report_runs = { + str(run["run_id"]): run + for run in report["runs"] + } + + for run in result.runs: + persisted_metrics = json.loads( + ( + result.output_dir + / "runs" + / run.arm.run_id + / "metrics.json" + ).read_text(encoding="utf-8") + ) + for candidate in ( + run.metrics, + persisted_metrics, + report_runs[run.arm.run_id]["metrics"], + ): + totals = candidate["totals"] + self.assertEqual( + totals["coverage_basis"], + "unavailable_untrusted_call_journal", + ) + for field_name in ( + "expected_invocation_count", + "token_coverage_percent", + "tool_call_coverage_percent", + "pricing_coverage_percent", + "estimated_cost_usd", + ): + self.assertIsNone(totals[field_name]) + self.assertTrue( + all( + group["estimated_cost_usd"] is None + for group in candidate["groups"] + ) + ) + self.assertNotIn( + "telemetry_coverage_percent", + run.invocation_attempts, + ) + self.assertNotIn( + "telemetry_coverage_percent", + report_runs[run.arm.run_id][ + "invocation_attempts" + ], + ) + + def test_missing_usage_is_inconclusive_but_missing_price_is_explicitly_unpriced(self) -> None: + before_records = ( + _telemetry_record(variant="before", estimated_cost=0.001), + ) + after_unpriced_records = ( + _telemetry_record(variant="after", estimated_cost=_MISSING), + ) + unpriced_report = _report_for_runs( + ( + _arm_result("before", before_records), + _arm_result("after", after_unpriced_records), + ) + ) + + self.assertEqual(unpriced_report["status"], "comparable") + self.assertEqual( + unpriced_report["runs"][1]["metrics"]["totals"]["pricing_coverage_percent"], + 0.0, + ) + self.assertIsNone( + unpriced_report["runs"][1]["metrics"]["totals"]["estimated_cost_usd"] + ) + cost_delta = unpriced_report["pairs"][0]["comparison"]["end_to_end"][ + "estimated_cost_usd" + ] + self.assertIsNone(cost_delta["delta"]) + self.assertIsNone(cost_delta["reduction_percent"]) + + after_missing_usage = ( + _telemetry_record( + variant="after", + native_usage=False, + estimated_cost=_MISSING, + ), + ) + missing_usage_report = _report_for_runs( + ( + _arm_result("before", before_records), + _arm_result("after", after_missing_usage), + ) + ) + self.assertEqual(missing_usage_report["status"], "inconclusive") + self.assertFalse(missing_usage_report["pairs"][0]["comparable"]) + self.assertIn( + "after_native_token_coverage_incomplete", + missing_usage_report["pairs"][0]["inconclusive_reasons"], + ) + + def test_reducer_reports_partial_coverage_without_inventing_cost_or_usage(self) -> None: + native_priced = _telemetry_record( + variant="before", + occurrence=1, + estimated_cost=0.002, + ) + missing = _telemetry_record( + variant="before", + occurrence=2, + native_usage=False, + estimated_cost=_MISSING, + ) + for field in ( + "input_tokens", + "cached_input_tokens", + "output_tokens", + "reasoning_output_tokens", + "total_tokens", + ): + missing.pop(field) + + metrics = reduce_telemetry((native_priced, missing)) + + self.assertEqual(metrics["totals"]["invocation_count"], 2) + self.assertEqual(metrics["totals"]["token_coverage_percent"], 50.0) + self.assertEqual(metrics["totals"]["pricing_coverage_percent"], 50.0) + self.assertIsNone(metrics["totals"]["estimated_cost_usd"]) + self.assertEqual(metrics["tokens"]["input"], native_priced["input_tokens"]) + self.assertEqual( + {group["estimated_cost_usd"] for group in metrics["groups"]}, + {None, 0.002}, + ) + + def test_reserved_attempt_without_telemetry_keeps_cost_and_usage_incomplete( + self, + ) -> None: + observed = _telemetry_record( + variant="after", + estimated_cost=0.002, + ) + + metrics = reduce_telemetry( + (observed,), + expected_invocation_count=2, + ) + + self.assertEqual(metrics["totals"]["invocation_count"], 1) + self.assertEqual(metrics["totals"]["expected_invocation_count"], 2) + self.assertEqual(metrics["totals"]["unobserved_invocation_count"], 1) + self.assertEqual(metrics["totals"]["token_coverage_percent"], 50.0) + self.assertEqual(metrics["totals"]["pricing_coverage_percent"], 50.0) + self.assertIsNone(metrics["totals"]["estimated_cost_usd"]) + self.assertTrue( + all( + group["estimated_cost_usd"] is None + for group in metrics["groups"] + ) + ) + self.assertEqual( + metrics["compaction"]["unobserved_invocation_count"], + 1, + ) + + def test_missing_call_journal_keeps_coverage_and_cost_unknown(self) -> None: + observed = _telemetry_record( + variant="after", + estimated_cost=0.002, + ) + + metrics = reduce_telemetry( + (observed,), + coverage_available=False, + ) + + totals = metrics["totals"] + self.assertEqual( + totals["coverage_basis"], + "unavailable_untrusted_call_journal", + ) + self.assertIsNone(totals["expected_invocation_count"]) + self.assertIsNone(totals["unobserved_invocation_count"]) + self.assertIsNone(totals["token_coverage_percent"]) + self.assertIsNone(totals["tool_call_coverage_percent"]) + self.assertIsNone(totals["pricing_coverage_percent"]) + self.assertIsNone(totals["estimated_cost_usd"]) + self.assertIsNone( + metrics["compaction"]["unobserved_invocation_count"] + ) + self.assertTrue( + all( + group["estimated_cost_usd"] is None + for group in metrics["groups"] + ) + ) + + def test_coverage_requires_a_supported_reconciled_call_journal(self) -> None: + trusted = { + "schema_version": 1, + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "identity_reconciled": True, + "max_invocations": 20, + "reserved_count": 2, + "entry_count": 2, + "telemetry_record_count": 1, + "unobserved_attempt_count": 1, + "telemetry_overage_count": 0, + "completed_count": 1, + "failed_count": 0, + "timeout_count": 0, + "launch_failed_count": 0, + "terminated_count": 1, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + "rejected_count": 0, + "remaining_budget": 18, + "journal_invocation_ids_sha256": ( + invocation_identity_digest(("invocation-1", "invocation-2")) + ), + "journal_invocation_id_missing_count": 0, + "journal_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_missing_count": 0, + "telemetry_invocation_id_duplicate_count": 0, + "telemetry_invocation_id_unmatched_count": 0, + "journal_invocation_id_unobserved_count": 1, + } + + self.assertTrue( + _journal_coverage_available( + trusted, + expected_max_invocations=20, + ) + ) + invalid_cases = { + "missing": {}, + "unsupported_summary_schema": { + **trusted, + "schema_version": 2, + }, + "unsupported_journal_schema": { + **trusted, + "journal_schema_version": 99, + }, + "unreconciled": { + **trusted, + "reconciled": False, + }, + "identity_unreconciled": { + **trusted, + "identity_reconciled": False, + }, + "missing_count": { + key: value + for key, value in trusted.items() + if key != "remaining_budget" + }, + "maximum_mismatch": { + **trusted, + "max_invocations": 19, + "remaining_budget": 17, + }, + "state_count_mismatch": { + **trusted, + "completed_count": 0, + }, + "telemetry_overage": { + **trusted, + "telemetry_record_count": 3, + "unobserved_attempt_count": 0, + "telemetry_overage_count": 1, + }, + "duplicate_telemetry_identity": { + **trusted, + "telemetry_invocation_id_duplicate_count": 1, + }, + "unmatched_telemetry_identity": { + **trusted, + "telemetry_invocation_id_unmatched_count": 1, + }, + "unobserved_identity_mismatch": { + **trusted, + "journal_invocation_id_unobserved_count": 0, + }, + } + for label, attempts in invalid_cases.items(): + with self.subTest(label=label): + self.assertFalse( + _journal_coverage_available( + attempts, + expected_max_invocations=20, + ) + ) + + trusted_v3 = { + **trusted, + "journal_schema_version": 3, + "context_reconciled": True, + "journal_telemetry_context_mismatch_count": 0, + "verified_target_projection_count": 1, + "verified_target_invocation_ids_sha256": ( + invocation_identity_digest(("invocation-1",)) + ), + } + self.assertTrue( + _journal_coverage_available( + trusted_v3, + expected_max_invocations=20, + ) + ) + self.assertFalse( + _journal_coverage_available( + {**trusted_v3, "context_reconciled": False}, + expected_max_invocations=20, + ) + ) + + def test_unobserved_terminated_attempt_makes_pair_inconclusive(self) -> None: + before = _arm_result( + "before", + (_telemetry_record(variant="before"),), + ) + after = replace( + _arm_result( + "after", + (_telemetry_record(variant="after"),), + ), + invocation_attempts={ + "journal_available": True, + "journal_schema_version": 2, + "reconciled": True, + "reserved_count": 2, + "entry_count": 2, + "telemetry_record_count": 1, + "unobserved_attempt_count": 1, + "completed_count": 1, + "terminated_count": 1, + "active_count": 0, + "unknown_state_count": 0, + "malformed_entry_count": 0, + "unaccounted_count": 0, + "overaccounted_count": 0, + }, + ) + + report = _report_for_runs((before, after)) + + self.assertEqual(report["status"], "inconclusive") + reasons = report["pairs"][0]["inconclusive_reasons"] + self.assertIn("after_unobserved_attempts", reasons) + self.assertIn("after_terminated_attempts", reasons) + + def test_missing_call_journal_makes_pair_inconclusive(self) -> None: + before = _arm_result( + "before", + (_telemetry_record(variant="before"),), + ) + after = replace( + _arm_result( + "after", + (_telemetry_record(variant="after"),), + ), + invocation_attempts={}, + ) + + report = _report_for_runs((before, after)) + + self.assertEqual(report["status"], "inconclusive") + self.assertIn( + "after_call_journal_missing", + report["pairs"][0]["inconclusive_reasons"], + ) + + def test_native_usage_requires_complete_consistent_provider_counts(self) -> None: + complete = _telemetry_record(variant="before") + missing_output = _telemetry_record(variant="before") + missing_output.pop("output_tokens") + inconsistent_total = _telemetry_record(variant="before") + inconsistent_total["total_tokens"] = ( + inconsistent_total["input_tokens"] + + inconsistent_total["output_tokens"] + - 1 + ) + derived_total = _telemetry_record(variant="before") + derived_total.pop("total_tokens") + + cases = ( + ("complete", complete, 100.0), + ("missing_output", missing_output, 0.0), + ("inconsistent_total", inconsistent_total, 0.0), + ("derived_total", derived_total, 100.0), + ) + for label, record, expected_coverage in cases: + with self.subTest(label=label): + metrics = reduce_telemetry((record,)) + self.assertEqual( + metrics["totals"]["token_coverage_percent"], + expected_coverage, + ) + self.assertEqual( + reduce_telemetry((derived_total,))["tokens"]["total"], + derived_total["input_tokens"] + derived_total["output_tokens"], + ) + + def test_v2_target_projection_requires_a_completed_primary_attempt(self) -> None: + repair = _telemetry_record(variant="after", occurrence=1) + repair["attempt_kind"] = "contract_repair" + failed_primary = _telemetry_record(variant="after", occurrence=1) + failed_primary["invocation_id"] = "after-failed-target" + failed_primary["status"] = "failed" + unrelated_primary = _telemetry_record(variant="after", occurrence=2) + + metrics = reduce_telemetry( + (repair, failed_primary, unrelated_primary), + verified_target_projection_count=1, + ) + + self.assertEqual(metrics["compaction"]["observed_invocation_count"], 3) + self.assertEqual(metrics["compaction"]["compacted_invocation_count"], 3) + self.assertEqual( + metrics["compaction"]["target_projection_candidate_count"], + 0, + ) + self.assertEqual(metrics["compaction"]["target_projection_count"], 0) + self.assertEqual( + metrics["compaction"][ + "target_projection_verification_mismatch_count" + ], + 1, + ) + + def test_v2_target_projection_requires_verified_identity_digest(self) -> None: + record = _telemetry_record(variant="after") + cases = { + "missing": "", + "wrong": invocation_identity_digest(("different-invocation",)), + } + + for label, digest in cases.items(): + with self.subTest(label=label): + metrics = reduce_telemetry( + (record,), + verified_target_projection_count=1, + verified_target_invocation_ids_sha256=digest, + ) + compaction = metrics["compaction"] + self.assertEqual( + compaction["target_projection_candidate_count"], + 1, + ) + self.assertEqual( + compaction[ + "target_projection_verification_mismatch_count" + ], + 0, + ) + self.assertFalse( + compaction["target_projection_identity_reconciled"] + ) + self.assertEqual(compaction["target_projection_count"], 0) + + def test_v2_target_projection_identity_digest_is_order_independent(self) -> None: + first = _telemetry_record(variant="after", occurrence=1) + second = _telemetry_record(variant="after", occurrence=2) + second.update( + { + "role": BENCHMARK_TARGET_ROLE, + "purpose": BENCHMARK_TARGET_PURPOSE, + "workflow_step": BENCHMARK_TARGET_WORKFLOW_STEP, + } + ) + reversed_ids = ( + second["invocation_id"], + first["invocation_id"], + ) + + metrics = reduce_telemetry( + (first, second), + verified_target_projection_count=2, + verified_target_invocation_ids_sha256=( + invocation_identity_digest(reversed_ids) + ), + ) + + compaction = metrics["compaction"] + self.assertTrue(compaction["target_projection_identity_reconciled"]) + self.assertEqual(compaction["target_projection_count"], 2) + + def test_v2_target_projection_rejects_cross_invocation_substitution(self) -> None: + telemetry_target = _telemetry_record(variant="after") + telemetry_target["invocation_id"] = "telemetry-target" + + metrics = reduce_telemetry( + (telemetry_target,), + verified_target_projection_count=1, + verified_target_invocation_ids_sha256=( + invocation_identity_digest(("journal-verified-target",)) + ), + ) + + compaction = metrics["compaction"] + self.assertEqual(compaction["target_projection_candidate_count"], 1) + self.assertEqual( + compaction["target_projection_verification_mismatch_count"], + 0, + ) + self.assertFalse(compaction["target_projection_identity_reconciled"]) + self.assertEqual(compaction["target_projection_count"], 0) + + def test_exact_prompt_context_counts_require_json_integers(self) -> None: + for label, invalid_value in ( + ("fractional", BENCHMARK_TARGET_TOTAL_EVENTS + 0.9), + ("numeric_string", str(BENCHMARK_TARGET_TOTAL_EVENTS)), + ): + with self.subTest(label=label): + journal_entry = _telemetry_record(variant="after") + journal_entry["state"] = "completed" + telemetry_record = _telemetry_record(variant="after") + telemetry_record["prompt_context_total_events"] = invalid_value + telemetry_record["prompt_context"][ + "total_events" + ] = invalid_value + snapshot = { + "schema_version": 3, + "max_invocations": 1, + "reserved_count": 1, + "remaining": 0, + "entries": [journal_entry], + } + + sanitized = sanitize_invocation_record(telemetry_record) + summary = _summarize_call_journal( + snapshot, + telemetry_records=(telemetry_record,), + ) + metrics = reduce_telemetry( + (telemetry_record,), + verified_target_projection_count=1, + verified_target_invocation_ids_sha256=( + invocation_identity_digest( + (telemetry_record["invocation_id"],) + ) + ), + ) + + self.assertIsNone( + sanitized["prompt_context_total_events"] + ) + self.assertNotIn( + "total_events", + sanitized["prompt_context"], + ) + self.assertFalse(summary["context_reconciled"]) + self.assertEqual( + summary["journal_telemetry_context_mismatch_count"], + 1, + ) + self.assertEqual( + summary["verified_target_projection_count"], + 0, + ) + self.assertEqual( + metrics["compaction"]["invalid_projection_count"], + 1, + ) + self.assertEqual( + metrics["compaction"][ + "target_projection_candidate_count" + ], + 0, + ) + self.assertEqual( + metrics["compaction"]["target_projection_count"], + 0, + ) + + def test_pair_requires_exact_v2_target_projection_in_each_arm(self) -> None: + wrong_before = _telemetry_record(variant="before") + wrong_before["prompt_context"].update( + { + "total_events": BENCHMARK_TARGET_TOTAL_EVENTS - 1, + "included_events": BENCHMARK_TARGET_TOTAL_EVENTS - 1, + "omitted_events": 0, + } + ) + wrong_before.update( + { + "prompt_context_total_events": BENCHMARK_TARGET_TOTAL_EVENTS - 1, + "prompt_context_included_events": ( + BENCHMARK_TARGET_TOTAL_EVENTS - 1 + ), + "prompt_context_omitted_events": 0, + } + ) + wrong_after = _telemetry_record(variant="after") + wrong_after["prompt_context"].update( + { + "total_events": BENCHMARK_TARGET_TOTAL_EVENTS + 1, + "included_events": BENCHMARK_TARGET_INCLUDED_EVENTS, + "omitted_events": BENCHMARK_TARGET_OMITTED_EVENTS + 1, + } + ) + wrong_after.update( + { + "prompt_context_total_events": BENCHMARK_TARGET_TOTAL_EVENTS + 1, + "prompt_context_included_events": BENCHMARK_TARGET_INCLUDED_EVENTS, + "prompt_context_omitted_events": BENCHMARK_TARGET_OMITTED_EVENTS + 1, + } + ) + cases = ( + ( + _arm_result("before", (wrong_before,)), + _arm_result("after", (_telemetry_record(variant="after"),)), + "before_v2_target_projection_not_observed", + ), + ( + _arm_result("before", (_telemetry_record(variant="before"),)), + _arm_result("after", (wrong_after,)), + "after_v2_target_projection_not_observed", + ), + ) + + for before, after, expected_reason in cases: + with self.subTest(expected_reason=expected_reason): + report = _report_for_runs((before, after)) + + self.assertEqual(report["status"], "inconclusive") + reasons = report["pairs"][0]["inconclusive_reasons"] + self.assertIn(expected_reason, reasons) + self.assertNotIn("after_compaction_not_observed", reasons) + + def test_invalid_compaction_projection_cannot_make_a_pair_comparable(self) -> None: + invalid_cases: dict[str, dict[str, Any]] = {} + inconsistent_total = _telemetry_record(variant="after") + inconsistent_total["prompt_context"]["total_events"] = 49 + invalid_cases["inconsistent_total"] = inconsistent_total + below_recent_tail = _telemetry_record(variant="after") + below_recent_tail["prompt_context"].update( + {"included_events": 7, "omitted_events": 43} + ) + invalid_cases["below_recent_tail"] = below_recent_tail + wrong_recent_limit = _telemetry_record(variant="after") + wrong_recent_limit["prompt_context"]["recent_events"] = 7 + invalid_cases["wrong_recent_limit"] = wrong_recent_limit + wrong_max_limit = _telemetry_record(variant="after") + wrong_max_limit["prompt_context"]["max_events"] = 17 + invalid_cases["wrong_max_limit"] = wrong_max_limit + + for label, invalid_after in invalid_cases.items(): + with self.subTest(label=label): + compaction = reduce_telemetry((invalid_after,))["compaction"] + self.assertEqual(compaction["observed_invocation_count"], 1) + self.assertEqual(compaction["invalid_projection_count"], 1) + self.assertEqual(compaction["compacted_invocation_count"], 0) + self.assertEqual(compaction["omitted_events"], 0) + + report = _report_for_runs( + ( + _arm_result( + "before", + (_telemetry_record(variant="before"),), + ), + _arm_result("after", (inconsistent_total,)), + ) + ) + self.assertEqual(report["status"], "inconclusive") + self.assertFalse(report["pairs"][0]["comparable"]) + self.assertIn( + "after_compaction_not_observed", + report["pairs"][0]["inconclusive_reasons"], + ) + + def test_after_arm_rejects_mixed_enabled_and_disabled_projections(self) -> None: + enabled = _telemetry_record(variant="after", occurrence=1) + disabled_short_history = _telemetry_record( + variant="after", + occurrence=2, + compacted=False, + ) + disabled_short_history["prompt_context"].update( + { + "enabled": False, + "total_events": 4, + "included_events": 4, + "omitted_events": 0, + } + ) + disabled_short_history.update( + { + "prompt_context_enabled": False, + "prompt_context_total_events": 4, + "prompt_context_included_events": 4, + "prompt_context_omitted_events": 0, + } + ) + + report = _report_for_runs( + ( + _arm_result( + "before", + (_telemetry_record(variant="before"),), + ), + _arm_result( + "after", + (enabled, disabled_short_history), + ), + ) + ) + + self.assertEqual(report["status"], "inconclusive") + reasons = report["pairs"][0]["inconclusive_reasons"] + self.assertIn( + "after_prompt_projection_not_uniformly_enabled", + reasons, + ) + self.assertNotIn("after_disabled_projection_observed", reasons) + + def test_repeated_primary_groups_are_left_unmatched_as_ambiguous(self) -> None: + before_records = [ + _telemetry_record(variant="before", occurrence=index) + for index in (1, 2) + ] + after_records = [ + _telemetry_record(variant="after", occurrence=index) + for index in (1, 2) + ] + for record in (*before_records, *after_records): + record.update( + { + "role": "developer", + "purpose": "implement", + "workflow_step": "todo_execution", + } + ) + + comparison = compare_metrics( + reduce_telemetry(before_records), + reduce_telemetry(after_records), + before_wall_duration_ms=1_000, + after_wall_duration_ms=900, + before_records=before_records, + after_records=after_records, + ) + + self.assertEqual(comparison["matched_primary_count"], 0) + self.assertEqual(comparison["unmatched_before_primary_count"], 2) + self.assertEqual(comparison["unmatched_after_primary_count"], 2) + self.assertEqual(comparison["ambiguous_primary_group_count"], 1) + + def test_sanitizer_drops_prompt_response_secrets_and_nested_history(self) -> None: + sanitized = sanitize_invocation_record(_telemetry_record(variant="after")) + + self.assertLessEqual(set(sanitized), SAFE_INVOCATION_FIELDS) + self.assertNotIn("prompt", sanitized) + self.assertNotIn("response", sanitized) + self.assertNotIn("api_key", sanitized) + self.assertNotIn("session_id", sanitized) + self.assertNotIn("raw_history", sanitized["prompt_context"]) + self.assertEqual( + sanitized["prompt_context"]["omitted_events"], + BENCHMARK_TARGET_OMITTED_EVENTS, + ) + + def test_divergent_flat_and_nested_prompt_context_fails_closed(self) -> None: + record = _telemetry_record(variant="before") + record.update( + { + "prompt_context_enabled": True, + "prompt_context_total_events": BENCHMARK_TARGET_TOTAL_EVENTS, + "prompt_context_included_events": BENCHMARK_TARGET_INCLUDED_EVENTS, + "prompt_context_omitted_events": BENCHMARK_TARGET_OMITTED_EVENTS, + "prompt_context_recent_events": BENCHMARK_PROMPT_CONTEXT_RECENT_EVENTS, + "prompt_context_max_events": BENCHMARK_PROMPT_CONTEXT_MAX_EVENTS, + "prompt_context_selection_policy": PROMPT_EVENT_SELECTION_POLICY, + } + ) + + sanitized = sanitize_invocation_record(record) + metrics = reduce_telemetry( + (record,), + verified_target_projection_count=1, + ) + + self.assertTrue( + sanitized["prompt_context_representation_conflict"] + ) + self.assertEqual(metrics["compaction"]["invalid_projection_count"], 1) + self.assertEqual( + metrics["compaction"]["target_projection_candidate_count"], + 0, + ) + self.assertEqual(metrics["compaction"]["target_projection_count"], 0) + self.assertEqual( + metrics["compaction"][ + "target_projection_verification_mismatch_count" + ], + 1, + ) + report = _report_for_runs( + ( + replace( + _arm_result("before", (record,)), + metrics=metrics, + ), + _arm_result( + "after", + (_telemetry_record(variant="after"),), + ), + ) + ) + + self.assertEqual(report["status"], "inconclusive") + self.assertIn( + "before_v2_target_projection_not_reconciled", + report["pairs"][0]["inconclusive_reasons"], + ) + + def test_sanitizer_allowlists_nested_rate_card_fields(self) -> None: + record = _telemetry_record(variant="after") + record["rate_card"] = { + "input_per_million_usd": 1.25, + "cached_input_per_million_usd": 0.25, + "output_per_million_usd": 2.5, + "per_invocation_usd": None, + "api_key": "SENSITIVE_RATE_CARD_SECRET", + "metadata": {"authorization": "SENSITIVE_NESTED_SECRET"}, + } + + sanitized = sanitize_invocation_record(record) + + self.assertEqual( + sanitized["rate_card"], + { + "input_per_million_usd": 1.25, + "cached_input_per_million_usd": 0.25, + "output_per_million_usd": 2.5, + "per_invocation_usd": None, + }, + ) + self.assertNotIn( + "SENSITIVE_", + json.dumps(sanitized["rate_card"], sort_keys=True), + ) + + +class SprintBenchmarkExecutionSafetyTests(unittest.TestCase): + def test_missing_auth_live_worker_preflight_reserves_no_provider_call(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + scenario = create_scenario_workspace( + root / "workspace", + benchmark_id="missing-auth-boundary", + run_id="pair-001-before", + prompt_context_enabled=False, + settings=_settings(), + ) + run_output = root / "run-output" + run_output.mkdir() + context = WorkerContext( + benchmark_id="missing-auth-boundary", + arm=ArmPlan( + pair_index=1, + order_index=1, + variant="before", + run_id="pair-001-before", + prompt_context_enabled=False, + ), + workspace_root=scenario.root, + run_output_dir=run_output, + milestone=SCENARIO_MILESTONE, + history_seed=scenario.history_seed, + max_invocations=2, + call_timeout_seconds=5, + run_timeout_seconds=10, + live=True, + ) + + with mock.patch.dict( + os.environ, + {LIVE_BENCHMARK_ENV: "1", "PATH": os.defpath}, + clear=True, + ): + outcome = run_live_sprint_arm(context) + + journal = json.loads( + (run_output / "call_journal.json").read_text(encoding="utf-8") + ) + self.assertEqual(outcome.status, "preflight_failed") + self.assertEqual(journal["schema_version"], 3) + self.assertEqual(journal["reserved_count"], 0) + self.assertEqual(journal["entries"], []) + self.assertEqual(outcome.invocation_attempts["reserved_count"], 0) + self.assertEqual(outcome.telemetry_records, ()) + self.assertFalse( + (run_output / ".private_model_invocations").exists() + ) + + def test_live_policy_requires_provider_only_auth_before_reserving_call(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory).resolve() + context = mock.Mock( + workspace_root=root / "workspace", + run_output_dir=root / "run-output", + call_timeout_seconds=30, + ) + budget = InvocationBudget(1) + + with mock.patch.dict(os.environ, {"PATH": os.defpath}, clear=True): + with self.assertRaisesRegex( + ModelExecutionPolicyViolation, + "CODEX_API_KEY or OPENAI_API_KEY", + ): + _build_execution_policy(context, budget=budget) + + self.assertEqual(budget.reserved_count, 0) + + with mock.patch.dict( + os.environ, + { + "CODEX_API_KEY": "provider-only-secret", + "PATH": str(Path(sys.executable).resolve().parent), + }, + clear=True, + ), mock.patch.object( + shutil, + "which", + return_value=sys.executable, + ): + policy = _build_execution_policy(context, budget=budget) + + self.assertNotIn("CODEX_API_KEY", policy.shell_environment) + self.assertEqual( + policy.codex_executable, + Path(sys.executable).resolve(), + ) + + def test_invocation_budget_rejects_the_twenty_first_call_and_journals_no_content(self) -> None: + class InvocationContext: + invocation_id = "safe-invocation-id" + operation_id = "safe-operation-id" + logical_call_id = "safe-logical-id" + attempt_index = 1 + attempt_kind = "primary" + role = "developer" + purpose = "implementation" + workflow_step = "todo_execution" + prompt = "SENSITIVE_BUDGET_PROMPT" + + with tempfile.TemporaryDirectory() as temporary_directory: + journal = Path(temporary_directory) / "budget" / "journal.json" + budget = InvocationBudget(20, journal_path=journal) + reservations = [ + budget.reserve(InvocationContext(), provider="codex_cli") + for _ in range(20) + ] + + self.assertEqual(len({item.reservation_id for item in reservations}), 20) + self.assertEqual(budget.reserved_count, 20) + self.assertEqual(budget.remaining, 0) + with self.assertRaises(InvocationBudgetExceeded) as raised: + budget.reserve(InvocationContext(), provider="codex_cli") + self.assertEqual(raised.exception.max_invocations, 20) + self.assertEqual(raised.exception.reserved_count, 20) + self.assertEqual(budget.rejected_count, 1) + + persisted_text = journal.read_text(encoding="utf-8") + persisted = json.loads(persisted_text) + self.assertEqual(persisted["reserved_count"], 20) + self.assertEqual(persisted["rejected_count"], 1) + self.assertEqual(len(persisted["entries"]), 20) + self.assertNotIn("SENSITIVE_BUDGET_PROMPT", persisted_text) + if os.name == "posix": + self.assertEqual( + stat.S_IMODE(journal.stat().st_mode), + 0o600, + ) + + def test_execution_policy_rejects_secret_environment_and_workspace_escape(self) -> None: + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + allowed = root / "allowed" + outside = root / "outside" + allowed.mkdir() + outside.mkdir() + budget = InvocationBudget(20) + + for name in ("OPENAI_API_KEY", "GH_TOKEN", "DATABASE_PASSWORD"): + with self.subTest(name=name): + with self.assertRaisesRegex(ValueError, "secret-bearing"): + ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=allowed, + invocation_budget=budget, + call_timeout_seconds=30, + codex_executable=sys.executable, + shell_environment={name: "must-not-leak"}, + ) + + policy = ModelExecutionPolicy.for_benchmark( + allowed_workspace_root=allowed, + invocation_budget=budget, + call_timeout_seconds=30, + codex_executable=sys.executable, + shell_environment={"PYTHONPATH": str(allowed)}, + ) + nested = allowed / "nested" + nested.mkdir() + policy.assert_workspace_allowed(nested) + with self.assertRaises(ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(outside) + with self.assertRaises(ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(allowed / ".." / "outside") + if hasattr(os, "symlink"): + escape_link = allowed / "escape-link" + escape_link.symlink_to(outside, target_is_directory=True) + with self.assertRaises(ModelExecutionPolicyViolation): + policy.assert_workspace_allowed(escape_link) + + def test_retained_workspaces_are_allowlisted_sanitized_snapshots(self) -> None: + sensitive_marker = "SENSITIVE_RAW_PROVIDER_AND_SESSION_STATE" + expected_files = { + ".benchmark/history_seed.json", + ".benchmark/scenario.json", + "BENCHMARK_TASK.md", + "RETENTION_NOTICE.md", + "benchmark_app.baseline.py", + "benchmark_app.result.json", + "team_runtime.yaml", + "tests/__init__.py", + "tests/test_benchmark_app.py", + } + + def failing_worker(context: WorkerContext) -> WorkerOutcome: + (context.workspace_root / ".teams_runtime_codex_output.txt").write_text( + sensitive_marker, + encoding="utf-8", + ) + session_file = ( + context.workspace_root + / ".teams_runtime" + / "role_sessions" + / "developer.json" + ) + session_file.parent.mkdir(parents=True, exist_ok=True) + session_file.write_text( + json.dumps({"session_id": sensitive_marker}), + encoding="utf-8", + ) + log_file = context.workspace_root / "logs" / "provider.log" + log_file.parent.mkdir(parents=True, exist_ok=True) + log_file.write_text(sensitive_marker, encoding="utf-8") + (context.workspace_root / "unknown-model-note.txt").write_text( + sensitive_marker, + encoding="utf-8", + ) + for relative_name in ( + ".benchmark/history_seed.json", + ".benchmark/scenario.json", + "BENCHMARK_TASK.md", + "team_runtime.yaml", + ): + (context.workspace_root / relative_name).write_text( + sensitive_marker, + encoding="utf-8", + ) + shutil.rmtree(context.workspace_root / "tests") + redirected_tests = ( + context.workspace_root / ".teams_runtime" / "redirected-tests" + ) + redirected_tests.mkdir(parents=True) + for filename in ("__init__.py", "test_benchmark_app.py"): + (redirected_tests / filename).write_text( + f"# {sensitive_marker}\n", + encoding="utf-8", + ) + (context.workspace_root / "tests").symlink_to( + redirected_tests, + target_is_directory=True, + ) + (context.workspace_root / "benchmark_app.py").unlink() + (context.workspace_root / "benchmark_app.py").symlink_to( + session_file, + ) + return WorkerOutcome( + status="failed", + stop_reason="fixture_failure", + error_category="FixtureFailure", + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=root / "reports", + repetitions=1, + keep_workspaces="failures", + benchmark_id="sanitized-retention", + ), + worker=failing_worker, + ) + + self.assertEqual(len(result.runs), 2) + for run in result.runs: + retained_root = result.output_dir / run.retained_workspace + retained_files = { + path.relative_to(retained_root).as_posix() + for path in retained_root.rglob("*") + if path.is_file() + } + self.assertEqual(retained_files, expected_files) + self.assertFalse((retained_root / ".git").exists()) + self.assertFalse((retained_root / ".teams_runtime").exists()) + self.assertFalse((retained_root / "logs").exists()) + self.assertIn( + "return sum(values)", + (retained_root / "benchmark_app.baseline.py").read_text( + encoding="utf-8" + ), + ) + self.assertEqual( + json.loads( + (retained_root / "benchmark_app.result.json").read_text( + encoding="utf-8" + ) + )["status"], + "missing_or_unsafe", + ) + self.assertIn( + "allowlisted diagnostic snapshot", + (retained_root / "RETENTION_NOTICE.md").read_text( + encoding="utf-8" + ), + ) + + for artifact in result.output_dir.rglob("*"): + if artifact.is_file(): + self.assertNotIn( + sensitive_marker, + artifact.read_text(encoding="utf-8"), + artifact, + ) + + def test_preflight_failure_stops_all_subsequent_arms(self) -> None: + seen_variants: list[str] = [] + + def preflight_failure(context: WorkerContext) -> WorkerOutcome: + seen_variants.append(context.arm.variant) + return WorkerOutcome( + status="preflight_failed", + stop_reason="provider_preflight_failed", + error_category="BenchmarkPreflightError", + ) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + source_root = root / "source" + runtime_config = root / "runtime.yaml" + _initialize_source_repository(source_root) + _write_runtime_config(runtime_config) + result = run_sprint_ab_benchmark( + BenchmarkOptions( + source_root=source_root, + runtime_config_path=runtime_config, + output_dir=root / "reports", + repetitions=3, + keep_workspaces="none", + benchmark_id="preflight-short-circuit", + ), + worker=preflight_failure, + ) + report_json_exists = result.report_json.is_file() + report_markdown_exists = result.report_markdown.is_file() + + self.assertEqual(seen_variants, ["before"]) + self.assertEqual(len(result.runs), 1) + self.assertEqual(result.runs[0].status, "preflight_failed") + self.assertEqual(result.status, "inconclusive") + self.assertEqual(result.exit_code, 1) + self.assertTrue(report_json_exists) + self.assertTrue(report_markdown_exists) + self.assertEqual( + result.report["pairs"][0]["inconclusive_reasons"], + ["missing_arm"], + ) + + +class SprintBenchmarkLifecycleTests(unittest.TestCase): + def test_initial_plan_auto_confirmation_preserves_draft_and_records_actor(self) -> None: + draft = { + "revision": 3, + "milestone_title": "Fix deterministic fixture", + "plan_actions": [{"plan_action_id": "PLAN-001", "title": "Repair defect"}], + } + state = { + "initial_plan_confirmation": { + "status": "pending", + "revision": 3, + "draft_proposal": draft, + "plan_artifact": "shared_workspace/sprints/test/implementation_plan.md", + "created_at": "2026-07-27T00:00:00+00:00", + } + } + actor = { + "type": "benchmark_auto_approval", + "id": "sprint-ab-harness", + "name": "Sprint A/B harness", + } + + confirmation = apply_initial_plan_confirmation( + state, + confirmed_by=actor, + message_id="benchmark-auto-confirm", + parser_reason="isolated benchmark policy", + parser_confidence="high", + confirmed_at="2026-07-27T00:01:00+00:00", + ) + + self.assertIs(confirmation, state["initial_plan_confirmation"]) + self.assertEqual(confirmation["status"], "confirmed") + self.assertEqual(confirmation["confirmed_at"], "2026-07-27T00:01:00+00:00") + self.assertEqual(confirmation["updated_at"], "2026-07-27T00:01:00+00:00") + self.assertEqual(confirmation["confirmed_by"], actor) + self.assertEqual(confirmation["confirmed_message_id"], "benchmark-auto-confirm") + self.assertEqual(confirmation["parser_reason"], "isolated benchmark policy") + self.assertEqual(confirmation["parser_confidence"], "high") + self.assertEqual(confirmation["draft_proposal"], draft) + with self.assertRaisesRegex(ValueError, "not awaiting confirmation"): + apply_initial_plan_confirmation(state, confirmed_by=actor) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sprint_lifecycle.py b/tests/test_sprint_lifecycle.py index 9601495..1e0dbd1 100644 --- a/tests/test_sprint_lifecycle.py +++ b/tests/test_sprint_lifecycle.py @@ -43,6 +43,7 @@ next_initial_phase_step, prepare_requested_restart_checkpoint, record_sprint_planning_iteration, + requirement_checkpoint_review_due, requirement_traceability_matrix_for_sprint, recover_sprint_todos_from_recovered, render_initial_implementation_plan_markdown, @@ -206,6 +207,12 @@ def test_ongoing_planning_request_exposes_only_pending_requirement_candidates(se "status": "rejected", "candidate_text": "Rejected text.", }, + { + "candidate_id": "REQ-CAND-003", + "status": "pending", + "candidate_text": "Preserve mobile approval flow.", + "created_at": "2026-04-21T19:40:02+09:00", + }, ], } @@ -223,6 +230,7 @@ def test_ongoing_planning_request_exposes_only_pending_requirement_candidates(se self.assertIn("pending_requirement_candidates:", record["body"]) self.assertIn("REQ-CAND-001: Add keyboard-only acceptance.", record["body"]) + self.assertIn("REQ-CAND-003: Preserve mobile approval flow.", record["body"]) self.assertNotIn("Rejected text.", record["body"]) self.assertEqual( record["params"]["pending_requirement_candidates"], @@ -233,7 +241,14 @@ def test_ongoing_planning_request_exposes_only_pending_requirement_candidates(se "raw_body": "", "artifacts": ["docs/a.md"], "created_at": "2026-04-21T19:40:00+09:00", - } + }, + { + "candidate_id": "REQ-CAND-003", + "candidate_text": "Preserve mobile approval flow.", + "raw_body": "", + "artifacts": [], + "created_at": "2026-04-21T19:40:02+09:00", + }, ], ) self.assertTrue(record["params"]["requirement_reconciliation_checkpoint"]) @@ -252,6 +267,36 @@ def test_ongoing_planning_request_exposes_only_pending_requirement_candidates(se self.assertEqual(non_checkpoint_record["params"]["pending_requirement_candidates"], []) self.assertFalse(non_checkpoint_record["params"]["requirement_reconciliation_checkpoint"]) + def test_requirement_checkpoint_review_requires_success_and_valid_pending_candidates(self) -> None: + sprint_state = { + "pending_requirement_candidates": [ + { + "candidate_id": "REQ-CAND-001", + "status": "pending", + "candidate_text": "Add keyboard-only acceptance.", + }, + { + "candidate_id": "REQ-CAND-002", + "status": "rejected", + "candidate_text": "Rejected candidate.", + }, + { + "candidate_id": "REQ-CAND-003", + "status": "pending", + "candidate_text": "", + }, + ] + } + + self.assertTrue(requirement_checkpoint_review_due(sprint_state, todo_status="completed")) + self.assertTrue(requirement_checkpoint_review_due(sprint_state, todo_status=" COMMITTED ")) + for status in ("queued", "running", "blocked", "failed", "uncommitted", ""): + with self.subTest(status=status): + self.assertFalse(requirement_checkpoint_review_due(sprint_state, todo_status=status)) + + sprint_state["pending_requirement_candidates"][0]["status"] = "registered" + self.assertFalse(requirement_checkpoint_review_due(sprint_state, todo_status="completed")) + def test_sprint_research_prepass_body_lines_include_planning_hints(self) -> None: lines = sprint_research_prepass_body_lines( { diff --git a/tests/test_workflow_engine.py b/tests/test_workflow_engine.py index 0a4afd9..9472a9f 100644 --- a/tests/test_workflow_engine.py +++ b/tests/test_workflow_engine.py @@ -203,6 +203,40 @@ def test_architect_review_explicit_continuation_blocks_at_limit(self): self.assertEqual(decision["workflow_state"]["phase_status"], "blocked") self.assertEqual(decision["workflow_state"]["reopen_category"], "implementation") + def test_architect_review_reopen_counts_before_developer_revision(self): + workflow_state = default_workflow_state() + workflow_state["phase"] = "implementation" + workflow_state["step"] = WORKFLOW_STEP_ARCHITECT_REVIEW + workflow_state["phase_owner"] = "architect" + workflow_state["review_cycle_count"] = 1 + transition = workflow_transition( + { + "proposals": { + "workflow_transition": { + "outcome": "reopen", + "target_phase": "implementation", + "target_step": "developer_revision", + "reopen_category": "implementation", + } + } + } + ) + + decision = derive_workflow_routing_decision( + workflow_state, + transition, + current_role="architect", + reason="The implementation requires one revision.", + ) + + self.assertIsNotNone(decision) + self.assertEqual(decision["next_role"], "developer") + self.assertEqual( + decision["workflow_state"]["step"], + "developer_revision", + ) + self.assertEqual(decision["workflow_state"]["reopen_count"], 1) + def test_qa_verification_reopen_returns_to_developer_revision(self): workflow_state = default_workflow_state() workflow_state["phase"] = "validation" @@ -234,6 +268,41 @@ def test_qa_verification_reopen_returns_to_developer_revision(self): self.assertEqual(decision["workflow_state"]["step"], "developer_revision") self.assertEqual(decision["workflow_state"]["phase_owner"], "developer") self.assertEqual(decision["workflow_state"]["reopen_category"], "verification") + self.assertEqual(decision["workflow_state"]["reopen_count"], 1) + + def test_qa_reopen_blocks_before_fourth_model_handoff(self): + workflow_state = default_workflow_state() + workflow_state["phase"] = "validation" + workflow_state["step"] = WORKFLOW_STEP_QA_VALIDATION + workflow_state["phase_owner"] = "qa" + workflow_state["reopen_count"] = 3 + workflow_state["reopen_limit"] = 3 + transition = workflow_transition( + { + "proposals": { + "workflow_transition": { + "outcome": "reopen", + "target_phase": "implementation", + "target_step": "developer_revision", + "reopen_category": "verification", + } + } + } + ) + + decision = derive_workflow_routing_decision( + workflow_state, + transition, + current_role="qa", + reason="A fourth implementation handoff would exceed the budget.", + ) + + self.assertIsNotNone(decision) + self.assertEqual(decision["next_role"], "") + self.assertEqual(decision["terminal_status"], "blocked") + self.assertEqual(decision["workflow_state"]["phase_status"], "blocked") + self.assertEqual(decision["workflow_state"]["reopen_count"], 3) + self.assertIn("reopen limit 3", decision["terminal_summary"]) def test_qa_ux_reopen_routes_to_designer_advisory(self): workflow_state = default_workflow_state() diff --git a/tests/test_workflow_state.py b/tests/test_workflow_state.py index 25603f0..10ab9ff 100644 --- a/tests/test_workflow_state.py +++ b/tests/test_workflow_state.py @@ -8,6 +8,7 @@ WORKFLOW_STEP_RESEARCH_INITIAL, default_workflow_state, infer_legacy_internal_workflow_state, + initial_workflow_state, normalize_workflow_state, workflow_complete_state, workflow_route_to_architect_review_state, @@ -37,9 +38,18 @@ def test_normalize_workflow_state_filters_invalid_values(self): self.assertEqual(normalized["planning_pass_count"], 0) self.assertEqual(normalized["planning_pass_limit"], 2) self.assertEqual(normalized["review_cycle_count"], 0) - self.assertEqual(normalized["review_cycle_limit"], 20) + self.assertEqual(normalized["review_cycle_limit"], 3) + self.assertEqual(normalized["reopen_count"], 0) + self.assertEqual(normalized["reopen_limit"], 3) self.assertEqual(normalized["reopen_category"], "") + def test_initial_workflow_state_accepts_policy_review_and_reopen_limits(self): + state = initial_workflow_state(review_cycle_limit=5, reopen_limit=2) + + self.assertEqual(state["review_cycle_limit"], 5) + self.assertEqual(state["reopen_limit"], 2) + self.assertEqual(state["reopen_count"], 0) + def test_infer_legacy_internal_workflow_state_for_planner_after_advisory(self): request_record = { "current_role": "planner", diff --git a/workflows/orchestration/engine.py b/workflows/orchestration/engine.py index 10a00ea..e1ce8f5 100644 --- a/workflows/orchestration/engine.py +++ b/workflows/orchestration/engine.py @@ -36,7 +36,8 @@ WORKFLOW_STEP_DEVELOPER_REVISION = "developer_revision" WORKFLOW_STEP_QA_VALIDATION = "qa_validation" WORKFLOW_STEP_CLOSEOUT = "closeout" -DEFAULT_WORKFLOW_REVIEW_CYCLE_LIMIT = 20 +DEFAULT_WORKFLOW_REVIEW_CYCLE_LIMIT = 3 +DEFAULT_WORKFLOW_REOPEN_LIMIT = 3 WORKFLOW_STEPS = { WORKFLOW_STEP_RESEARCH_INITIAL, WORKFLOW_STEP_PLANNER_DRAFT, @@ -76,6 +77,8 @@ def default_workflow_state() -> WorkflowState: "planning_final_owner": "planner", "reopen_source_role": "", "reopen_category": "", + "reopen_count": 0, + "reopen_limit": DEFAULT_WORKFLOW_REOPEN_LIMIT, "review_cycle_count": 0, "review_cycle_limit": DEFAULT_WORKFLOW_REVIEW_CYCLE_LIMIT, } @@ -88,10 +91,15 @@ def research_first_workflow_state() -> WorkflowState: return state -def initial_workflow_state(review_cycle_limit: int | None = None) -> WorkflowState: +def initial_workflow_state( + review_cycle_limit: int | None = None, + reopen_limit: int | None = None, +) -> WorkflowState: state = dict(default_workflow_state()) if review_cycle_limit is not None: state["review_cycle_limit"] = max(1, int(review_cycle_limit)) + if reopen_limit is not None: + state["reopen_limit"] = max(1, int(reopen_limit)) return state @@ -140,6 +148,14 @@ def normalize_workflow_state(raw: Any) -> WorkflowState: state["planning_pass_limit"] = max(1, int(raw.get("planning_pass_limit") or state["planning_pass_limit"])) state["review_cycle_count"] = max(0, int(raw.get("review_cycle_count") or state["review_cycle_count"])) state["review_cycle_limit"] = max(1, int(raw.get("review_cycle_limit") or state["review_cycle_limit"])) + state["reopen_count"] = max( + 0, + int(raw.get("reopen_count") or state["reopen_count"]), + ) + state["reopen_limit"] = max( + 1, + int(raw.get("reopen_limit") or state["reopen_limit"]), + ) return state @@ -311,6 +327,18 @@ def workflow_review_cycle_limit_reached(workflow_state: dict[str, Any]) -> bool: return review_cycle_count >= review_cycle_limit +def workflow_reopen_limit_reached(workflow_state: dict[str, Any]) -> bool: + reopen_count = max(0, int((workflow_state or {}).get("reopen_count") or 0)) + reopen_limit = max( + 1, + int( + (workflow_state or {}).get("reopen_limit") + or DEFAULT_WORKFLOW_REOPEN_LIMIT + ), + ) + return reopen_count >= reopen_limit + + def workflow_reason(result: dict[str, Any], transition: dict[str, Any], default: str) -> str: return ( str(transition.get("reason") or "").strip() @@ -441,10 +469,12 @@ def workflow_mark_reopen_state( updated_state = dict(workflow_state or default_workflow_state()) updated_state["reopen_source_role"] = current_role updated_state["reopen_category"] = category if category in WORKFLOW_REOPEN_CATEGORIES else "" + updated_state["reopen_count"] = int(updated_state.get("reopen_count") or 0) + 1 return updated_state _WORKFLOW_STATE_EXPORTS = [ + "DEFAULT_WORKFLOW_REOPEN_LIMIT", "DEFAULT_WORKFLOW_REVIEW_CYCLE_LIMIT", "PLANNING_ADVISORY_ROLES", "PLANNING_ADVISORY_ROLE_TO_STEP", @@ -481,6 +511,7 @@ def workflow_mark_reopen_state( "workflow_complete_state", "workflow_mark_reopen_state", "workflow_reason", + "workflow_reopen_limit_reached", "workflow_review_cycle_limit_reached", "workflow_route_to_architect_guidance_state", "workflow_route_to_architect_review_state", @@ -1235,6 +1266,12 @@ def workflow_reopen_decision( category: str, reason: str, ) -> dict[str, Any]: + if workflow_reopen_limit_reached(workflow_state): + return workflow_reopen_limit_block_decision( + workflow_state, + reason=reason, + category=category, + ) updated_state = workflow_mark_reopen_state( workflow_state, current_role=current_role, @@ -1269,6 +1306,36 @@ def workflow_reopen_decision( return workflow_route_to_planner_finalize_decision(updated_state, reason=reason, category=category) +def workflow_reopen_limit_block_decision( + workflow_state: dict[str, Any], + *, + reason: str, + category: str = "", +) -> dict[str, Any]: + reopen_count = max(0, int((workflow_state or {}).get("reopen_count") or 0)) + reopen_limit = max( + 1, + int( + (workflow_state or {}).get("reopen_limit") + or DEFAULT_WORKFLOW_REOPEN_LIMIT + ), + ) + limit_summary = ( + f"workflow reopen이 {reopen_count}회 실행되어 reopen limit " + f"{reopen_limit}에 도달했습니다." + ) + combined_summary = " ".join( + part + for part in (limit_summary, str(reason or "").strip()) + if str(part).strip() + ).strip() + return workflow_terminal_block_decision( + workflow_state, + summary=combined_summary or limit_summary, + category=category, + ) + + def workflow_review_cycle_limit_block_decision( workflow_state: dict[str, Any], *, @@ -1420,17 +1487,10 @@ def derive_workflow_routing_decision( category=reopen_category or "implementation", ) if outcome == "reopen": - if reopen_category in {"", "implementation"}: - return workflow_route_to_developer_build_decision( - workflow_state, - reason=reason or "architect review 결과를 developer가 반영합니다.", - step=WORKFLOW_STEP_DEVELOPER_REVISION, - category=reopen_category, - ) return workflow_reopen_decision( workflow_state, current_role=current_role or "architect", - category=reopen_category, + category=reopen_category or "implementation", reason=reason, ) return workflow_route_to_developer_build_decision( @@ -2133,6 +2193,7 @@ def build_governed_routing_selection( "strongest_domain_matches", "workflow_complete_decision", "workflow_reopen_decision", + "workflow_reopen_limit_block_decision", "workflow_review_cycle_limit_block_decision", "workflow_route_decision", "workflow_route_to_architect_guidance_decision", diff --git a/workflows/orchestration/team_service.py b/workflows/orchestration/team_service.py index 70c8263..f98ce71 100644 --- a/workflows/orchestration/team_service.py +++ b/workflows/orchestration/team_service.py @@ -60,6 +60,7 @@ update_goal_stop_condition, ) from teams_runtime.shared.config import load_discord_agents_config, load_team_runtime_config +from teams_runtime.runtime.execution_policy import ModelExecutionPolicy from teams_runtime.runtime.role_result_contract import is_restart_repairable_invalid_contract_payload from teams_runtime.workflows.orchestration.relay import ( archive_internal_relay_file, @@ -115,6 +116,7 @@ select_backlog_items_for_sprint as select_backlog_items_for_sprint_helper, ) from teams_runtime.workflows.orchestration.engine import ( + DEFAULT_WORKFLOW_REOPEN_LIMIT, DEFAULT_WORKFLOW_REVIEW_CYCLE_LIMIT, WORKFLOW_CONTRACT_VERSION, WORKFLOW_PHASE_CLOSEOUT, @@ -383,6 +385,7 @@ INITIAL_PHASE_STEP_TODO_FINALIZATION, INITIAL_PHASE_STEPS, SPRINT_ACTIVE_BACKLOG_STATUSES, + apply_initial_plan_confirmation as apply_initial_plan_confirmation_helper, apply_sprint_planning_result as apply_sprint_planning_result_helper, archive_pending_requirement_candidates as archive_pending_requirement_candidates_helper, build_idle_current_sprint_markdown as build_idle_current_sprint_markdown_helper, @@ -484,6 +487,7 @@ from teams_runtime.runtime.base_runtime import RoleAgentRuntime, normalize_role_payload from teams_runtime.runtime.internal.goal_sourcing import GoalSourcingRuntime, normalize_goal_sourcing_payload from teams_runtime.runtime.internal.intent_parser import IntentParserRuntime, normalize_intent_payload +from teams_runtime.runtime.model_telemetry import run_task_with_optional_telemetry_purpose from teams_runtime.runtime.research_runtime import ResearchAgentRuntime from teams_runtime.runtime.identities import local_runtime_identity, service_runtime_identity @@ -946,6 +950,8 @@ def __init__( *, enable_discord_client: bool = True, relay_transport: str = RELAY_TRANSPORT_DISCORD, + model_execution_policy: ModelExecutionPolicy | None = None, + allow_external_research: bool = True, ): if role not in TEAM_ROLES: raise ValueError(f"Unsupported role: {role}") @@ -959,6 +965,8 @@ def __init__( + ", ".join(sorted(VALID_RELAY_TRANSPORTS)) ) self.relay_transport = normalized_relay_transport + self.model_execution_policy = model_execution_policy + self.allow_external_research = bool(allow_external_research) self.discord_config = load_discord_agents_config(self.paths.workspace_root) self.runtime_config = load_team_runtime_config(self.paths.workspace_root) self.agent_utilization_policy = load_agent_utilization_policy(self.paths.workspace_root) @@ -972,22 +980,31 @@ def __init__( self.intent_parser = IntentParserRuntime( paths=self.paths, sprint_id=self.runtime_config.sprint_id, - runtime_config=self.runtime_config.role_defaults["orchestrator"], + runtime_config=self.runtime_config.internal_agent_defaults["parser"], session_identity=self._local_runtime_session_identity("parser"), + telemetry_config=self.runtime_config.telemetry, + execution_policy=self.model_execution_policy, ) self.goal_sourcer = GoalSourcingRuntime( paths=self.paths, sprint_id=self.runtime_config.sprint_id, - runtime_config=self.runtime_config.role_defaults["orchestrator"], + runtime_config=self.runtime_config.internal_agent_defaults["sourcer"], session_identity=self._local_runtime_session_identity("sourcer"), + telemetry_config=self.runtime_config.telemetry, + execution_policy=self.model_execution_policy, ) self.version_controller_runtime = RoleAgentRuntime( paths=self.paths, role="version_controller", sprint_id=self.runtime_config.sprint_id, - runtime_config=self.runtime_config.role_defaults["orchestrator"], + runtime_config=( + self.runtime_config.internal_agent_defaults["version_controller"] + ), agent_root=self.paths.internal_agent_root("version_controller"), session_identity=self._local_runtime_session_identity("version_controller"), + telemetry_config=self.runtime_config.telemetry, + prompt_context_config=self.runtime_config.prompt_context, + execution_policy=self.model_execution_policy, ) self._purge_request_scoped_role_output_files() self._role_runtime_cache: dict[tuple[str, str, str], RoleAgentRuntime] = { @@ -1251,13 +1268,20 @@ def _research_first_workflow_state(self) -> dict[str, Any]: def _initial_workflow_state_for_internal_request(self) -> dict[str, Any]: return initial_workflow_state( - max( + review_cycle_limit=max( 1, int( self.agent_utilization_policy.implementation_review_cycle_limit or DEFAULT_WORKFLOW_REVIEW_CYCLE_LIMIT ), - ) + ), + reopen_limit=max( + 1, + int( + self.agent_utilization_policy.implementation_reopen_limit + or DEFAULT_WORKFLOW_REOPEN_LIMIT + ), + ), ) def _request_workflow_state(self, request_record: RequestRecord) -> WorkflowState: @@ -2760,7 +2784,12 @@ async def _invoke_version_controller( f"summary={summary}" ), ) - return await asyncio.to_thread(self.version_controller_runtime.run_task, envelope, request_context) + return await asyncio.to_thread( + self.version_controller_runtime.run_task, + envelope, + request_context, + telemetry_purpose="version_control", + ) async def _run_task_version_controller( self, @@ -2934,6 +2963,10 @@ def _build_role_runtime( runtime_config=self.runtime_config.role_defaults[role], research_defaults=self.runtime_config.research_defaults, session_identity=session_identity, + telemetry_config=self.runtime_config.telemetry, + prompt_context_config=self.runtime_config.prompt_context, + allow_external_research=self.allow_external_research, + execution_policy=self.model_execution_policy, ) return RoleAgentRuntime( paths=self.paths, @@ -2941,6 +2974,9 @@ def _build_role_runtime( sprint_id=sprint_id, runtime_config=self.runtime_config.role_defaults[role], session_identity=session_identity, + telemetry_config=self.runtime_config.telemetry, + prompt_context_config=self.runtime_config.prompt_context, + execution_policy=self.model_execution_policy, ) def _runtime_for_role(self, role: str, sprint_id: str) -> RoleAgentRuntime: @@ -3717,18 +3753,14 @@ async def _maybe_handle_initial_plan_feedback( now = utc_now_iso() sprint_id = str(sprint_state.get("sprint_id") or "") if interpreted_intent == "plan_confirm": - confirmation.update( - { - "status": "confirmed", - "confirmed_at": now, - "updated_at": now, - "confirmed_by": build_requester_route(message, envelope, forwarded=forwarded), - "confirmed_message_id": str(message.message_id or "").strip(), - "parser_reason": str(interpreted.params.get("parser_reason") or "").strip(), - "parser_confidence": interpreted_confidence, - } + confirmation = apply_initial_plan_confirmation_helper( + sprint_state, + confirmed_by=build_requester_route(message, envelope, forwarded=forwarded), + message_id=str(message.message_id or "").strip(), + parser_reason=str(interpreted.params.get("parser_reason") or "").strip(), + parser_confidence=interpreted_confidence, + confirmed_at=now, ) - sprint_state["initial_plan_confirmation"] = confirmation self._save_sprint_state(sprint_state) self._append_sprint_event( sprint_id, @@ -5852,7 +5884,13 @@ async def _draft_sprint_report_via_planner( ) try: runtime = self._runtime_for_role("planner", self.runtime_config.sprint_id) - result = await asyncio.to_thread(runtime.run_task, envelope, request_context) + result = await asyncio.to_thread( + run_task_with_optional_telemetry_purpose, + runtime, + envelope, + request_context, + telemetry_purpose="sprint_closeout_report", + ) except Exception: LOGGER.exception( "Planner closeout report drafting failed for sprint %s", diff --git a/workflows/repository_ops.py b/workflows/repository_ops.py index a8902fa..0ba0ffc 100644 --- a/workflows/repository_ops.py +++ b/workflows/repository_ops.py @@ -2,6 +2,7 @@ import argparse import json +import os import secrets import subprocess import sys @@ -285,12 +286,57 @@ def _select_commit_target_path(changed_paths: list[str]) -> str: def _run_git(repo_root: Path, args: list[str]) -> subprocess.CompletedProcess[str]: + benchmark_mode = os.environ.get("TEAMS_RUNTIME_LIVE_BENCHMARK") == "1" + if benchmark_mode and (repo_root / ".gitattributes").exists(): + # Repository-controlled clean/process filters execute during `git add`. + # The benchmark fails closed instead of allowing model-authored metadata + # to run in the trusted worker process. + return subprocess.CompletedProcess( + ["git", *args], 78, "", "benchmark repository attributes are unsupported" + ) + command = ["git"] + environment = None + if benchmark_mode: + command.extend( + [ + "--no-pager", + "--no-replace-objects", + "-c", + "core.hooksPath=/dev/null", + "-c", + "core.fsmonitor=false", + "-c", + "core.attributesFile=/dev/null", + "-c", + "diff.external=", + "-c", + "maintenance.auto=false", + "-c", + "gc.auto=0", + ] + ) + environment = { + "HOME": os.devnull, + "PATH": os.defpath, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_ATTR_NOSYSTEM": "1", + "GIT_EXTERNAL_DIFF": "", + "GIT_PAGER": "", + "PAGER": "", + "GIT_TERMINAL_PROMPT": "0", + "GIT_NO_REPLACE_OBJECTS": "1", + } + command.extend(args) return subprocess.run( - ["git", *args], + command, cwd=str(repo_root), capture_output=True, text=True, check=False, + env=environment, + timeout=10 if benchmark_mode else None, ) diff --git a/workflows/roles/__init__.py b/workflows/roles/__init__.py index 0947781..746d89e 100644 --- a/workflows/roles/__init__.py +++ b/workflows/roles/__init__.py @@ -76,6 +76,7 @@ class AgentUtilizationPolicy: verification_result_terminal: bool ignore_non_planner_backlog_proposals_for_routing: bool implementation_review_cycle_limit: int + implementation_reopen_limit: int load_error: str = "" @@ -181,7 +182,8 @@ def _weights_from_payload(payload: dict[str, Any]) -> RoutingWeights: "planning_advisory_roles": ["designer", "architect"], "planning_shared_pass_limit": 2, "planning_pass_limit_behavior": "planner_finalize_then_block", - "implementation_review_cycle_limit": 20, + "implementation_review_cycle_limit": 3, + "implementation_reopen_limit": 3, "implementation_sequence": [ "architect_guidance", "developer_build", @@ -657,7 +659,11 @@ def build_agent_utilization_policy( ), implementation_review_cycle_limit=max( 1, - _coerce_int(workflow_contract.get("implementation_review_cycle_limit"), 20), + _coerce_int(workflow_contract.get("implementation_review_cycle_limit"), 3), + ), + implementation_reopen_limit=max( + 1, + _coerce_int(workflow_contract.get("implementation_reopen_limit"), 3), ), load_error=load_error, ) diff --git a/workflows/roles/research.py b/workflows/roles/research.py index 8217c04..498ae48 100644 --- a/workflows/roles/research.py +++ b/workflows/roles/research.py @@ -1,11 +1,22 @@ from __future__ import annotations import json +import logging import re from typing import Any -from teams_runtime.shared.models import MessageEnvelope, RequestRecord +from teams_runtime.shared.models import ( + MessageEnvelope, + PromptContextRuntimeConfig, + RequestRecord, +) +from teams_runtime.shared.prompt_context import ( + project_request_record_for_prompt, + render_prompt_event_history_notice, +) + +LOGGER = logging.getLogger(__name__) RESEARCH_REASON_CODE_NEEDED_EXTERNAL_GROUNDING = "needed_external_grounding" RESEARCH_REASON_CODE_NOT_NEEDED_LOCAL_EVIDENCE = "not_needed_local_evidence" @@ -607,7 +618,24 @@ def build_research_decision_prompt( request_record: RequestRecord, *, local_sources_checked: list[str], + prompt_context_config: PromptContextRuntimeConfig | None = None, ) -> str: + request_projection = project_request_record_for_prompt( + request_record, + prompt_context_config, + ) + if request_projection.compacted: + LOGGER.info( + "[research] prompt_context_compacted request_id=%s purpose=research_decision total_events=%s " + "included_events=%s omitted_events=%s recent_events=%s max_events=%s", + str(request_record.get("request_id") or "unknown"), + request_projection.total_events, + request_projection.included_events, + request_projection.omitted_events, + request_projection.recent_events, + request_projection.max_events, + ) + event_history_notice = render_prompt_event_history_notice(request_projection) params = dict(request_record.get("params") or {}) if isinstance(request_record.get("params"), dict) else {} public_targeted = str(params.get("user_requested_role") or "").strip().lower() == "research" closeout_requirements = closeout_original_requirements_from_request(request_record) @@ -689,8 +717,9 @@ def build_research_decision_prompt( "Local sources already checked:", *[f"- {item}" for item in local_sources_checked], "", + event_history_notice, "Current request:", - json.dumps(request_record, ensure_ascii=False, indent=2), + json.dumps(request_projection.request_record, ensure_ascii=False, indent=2), "", "Incoming envelope:", json.dumps(envelope.to_dict(), ensure_ascii=False, indent=2), diff --git a/workflows/sprints/lifecycle.py b/workflows/sprints/lifecycle.py index b8b9a44..8096e8c 100644 --- a/workflows/sprints/lifecycle.py +++ b/workflows/sprints/lifecycle.py @@ -1078,6 +1078,38 @@ def confirmed_initial_plan(sprint_state: dict[str, Any]) -> dict[str, Any]: ) +def apply_initial_plan_confirmation( + sprint_state: dict[str, Any], + *, + confirmed_by: dict[str, Any], + message_id: str = "", + parser_reason: str = "", + parser_confidence: str = "high", + confirmed_at: str = "", +) -> dict[str, Any]: + confirmation = ( + dict(sprint_state.get("initial_plan_confirmation") or {}) + if isinstance(sprint_state.get("initial_plan_confirmation"), dict) + else {} + ) + if str(confirmation.get("status") or "").strip().lower() != "pending": + raise ValueError("Initial implementation plan is not awaiting confirmation.") + now = str(confirmed_at or "").strip() or utc_now_iso() + confirmation.update( + { + "status": "confirmed", + "confirmed_at": now, + "updated_at": now, + "confirmed_by": dict(confirmed_by or {}), + "confirmed_message_id": str(message_id or "").strip(), + "parser_reason": str(parser_reason or "").strip(), + "parser_confidence": str(parser_confidence or "").strip() or "high", + } + ) + sprint_state["initial_plan_confirmation"] = confirmation + return confirmation + + def initial_plan_action_records(sprint_state: dict[str, Any]) -> list[dict[str, Any]]: plan = confirmed_initial_plan(sprint_state) return [dict(item) for item in (plan.get("plan_actions") or []) if isinstance(item, dict)] @@ -1150,6 +1182,17 @@ def pending_requirement_candidates_for_planner(sprint_state: dict[str, Any]) -> return candidates +def requirement_checkpoint_review_due( + sprint_state: dict[str, Any], + *, + todo_status: str, +) -> bool: + return ( + str(todo_status or "").strip().lower() in {"completed", "committed"} + and bool(pending_requirement_candidates_for_planner(sprint_state)) + ) + + def format_requirement_candidate_ref(candidate: dict[str, Any]) -> str: candidate_id = str(candidate.get("candidate_id") or "").strip().upper() text = _normalize_requirement_text(candidate.get("candidate_text") or candidate.get("raw_body") or "") @@ -3946,7 +3989,10 @@ async def continue_manual_daily_sprint( return await service._execute_sprint_todo(sprint_state, next_todo) service._save_sprint_state(sprint_state) - requirement_checkpoint_review = str(next_todo.get("status") or "").strip().lower() in {"completed", "committed"} + requirement_checkpoint_review = requirement_checkpoint_review_due( + sprint_state, + todo_status=str(next_todo.get("status") or ""), + ) force_review = requirement_checkpoint_review @@ -4011,7 +4057,7 @@ async def continue_sprint( todo_status = str(todo.get("status") or "").strip().lower() if todo_status == "uncommitted": return - if todo_status in {"completed", "committed"} and pending_requirement_candidates_for_planner(sprint_state): + if requirement_checkpoint_review_due(sprint_state, todo_status=todo_status): await service._run_ongoing_sprint_review( sprint_state, force=True,