Reduce model-call amplification with helper tiers and workflow budgets

adapters/cli/commands.py

@@ -2,6 +2,7 @@
 
 import argparse
 import asyncio
+import json
 import os
 import shutil
 from pathlib import Path
@@ -33,6 +34,7 @@ def _notify_missing_github_token(printer: Printer) -> None:
 def build_parser(
     *,
     all_runtime_agents: list[str] | tuple[str, ...],
+    internal_team_agents: list[str] | tuple[str, ...],
     team_roles: list[str] | tuple[str, ...],
     relay_transport_internal: str,
     relay_transport_discord: str,
@@ -94,6 +96,18 @@ def build_parser(
     )
     list_parser.add_argument("--request-id", help="Optional request identifier to print.")
 
+    metrics_parser = subparsers.add_parser("metrics", help="Summarize local model cost and latency telemetry.")
+    metrics_parser.add_argument(
+        "--workspace-root",
+        default=None,
+        help=workspace_root_help_text,
+    )
+    metrics_parser.add_argument("--hours", type=float, default=24.0, help="Positive lookback window in hours.")
+    metrics_parser.add_argument("--request-id", default="", help="Optional request identifier filter.")
+    metrics_parser.add_argument("--sprint-id", default="", help="Optional sprint identifier filter.")
+    metrics_parser.add_argument("--agent", choices=all_runtime_agents, help="Optional role filter.")
+    metrics_parser.add_argument("--json", action="store_true", help="Render stable aggregate JSON output.")
+
     config_parser = subparsers.add_parser("config")
     config_subparsers = config_parser.add_subparsers(dest="config_command", required=True)
     role_parser = config_subparsers.add_parser("role")
@@ -108,6 +122,32 @@ def build_parser(
     role_set_parser.add_argument("--model", help="Optional model override to save in team_runtime.yaml.")
     role_set_parser.add_argument("--reasoning", help="Optional reasoning level to save in team_runtime.yaml.")
 
+    internal_parser = config_subparsers.add_parser("internal")
+    internal_subparsers = internal_parser.add_subparsers(
+        dest="internal_command",
+        required=True,
+    )
+    internal_set_parser = internal_subparsers.add_parser("set")
+    internal_set_parser.add_argument(
+        "--workspace-root",
+        default=None,
+        help=workspace_root_help_text,
+    )
+    internal_set_parser.add_argument(
+        "--agent",
+        choices=internal_team_agents,
+        required=True,
+        help="Target internal helper agent.",
+    )
+    internal_set_parser.add_argument(
+        "--model",
+        help="Optional model override to save in team_runtime.yaml.",
+    )
+    internal_set_parser.add_argument(
+        "--reasoning",
+        help="Optional reasoning level to save in team_runtime.yaml.",
+    )
+
     research_parser = config_subparsers.add_parser("research")
     research_subparsers = research_parser.add_subparsers(dest="research_command", required=True)
     research_set_parser = research_subparsers.add_parser("set")
@@ -196,6 +236,84 @@ def build_parser(
             help=workspace_root_help_text,
         )
 
+    benchmark_parser = subparsers.add_parser(
+        "benchmark",
+        help="Run an explicitly opted-in performance benchmark.",
+    )
+    benchmark_subparsers = benchmark_parser.add_subparsers(
+        dest="benchmark_command",
+        required=True,
+    )
+    sprint_ab_parser = benchmark_subparsers.add_parser(
+        "sprint-ab",
+        help="Compare full sprints with prompt event compaction disabled and enabled.",
+    )
+    sprint_ab_parser.add_argument(
+        "--live",
+        action="store_true",
+        help="Permit live provider calls; also requires TEAMS_RUNTIME_LIVE_BENCHMARK=1.",
+    )
+    sprint_ab_parser.add_argument(
+        "--runtime-config",
+        required=True,
+        help="Path to the deployed team_runtime.yaml or its workspace directory.",
+    )
+    sprint_ab_parser.add_argument(
+        "--repetitions",
+        type=int,
+        default=1,
+        help="Number of paired runs; execution order alternates AB then BA.",
+    )
+    sprint_ab_parser.add_argument(
+        "--max-invocations",
+        type=int,
+        default=20,
+        help="Hard physical model-invocation cap per arm.",
+    )
+    sprint_ab_parser.add_argument(
+        "--call-timeout-seconds",
+        type=float,
+        default=300.0,
+        help="Hard provider-call timeout.",
+    )
+    sprint_ab_parser.add_argument(
+        "--run-timeout-seconds",
+        type=float,
+        default=1800.0,
+        help="Hard full-arm timeout.",
+    )
+    sprint_ab_parser.add_argument(
+        "--keep-workspaces",
+        choices=("none", "failures", "all"),
+        default="failures",
+        help="Retain no workspaces, failed workspaces, or every workspace.",
+    )
+    sprint_ab_parser.add_argument(
+        "--rate-card-file",
+        default="",
+        help="Optional YAML rate card used only for estimated-cost reporting.",
+    )
+    sprint_ab_parser.add_argument(
+        "--output-dir",
+        default="",
+        help="Optional parent directory for benchmark artifacts.",
+    )
+    sprint_ab_parser.add_argument(
+        "--benchmark-id",
+        default="",
+        help="Optional stable artifact directory name.",
+    )
+    sprint_ab_parser.add_argument(
+        "--allow-dirty-source",
+        action="store_true",
+        help="Allow a dirty source checkout and record its state hash in provenance.",
+    )
+    sprint_ab_parser.add_argument(
+        "--json",
+        action="store_true",
+        help="Print a machine-readable result summary.",
+    )
+
     return parser
 
 
@@ -212,6 +330,7 @@ def dispatch_main(
     cmd_restart: DispatchSyncCallback,
     cmd_list: DispatchSyncCallback,
     cmd_config_role_set: DispatchSyncCallback,
+    cmd_config_internal_set: DispatchSyncCallback,
     cmd_config_research_set: DispatchSyncCallback,
     cmd_sprint_start: DispatchSyncCallback,
     cmd_sprint_stop: DispatchSyncCallback,
@@ -223,6 +342,8 @@ def dispatch_main(
     cmd_goal_resume: DispatchSyncCallback,
     cmd_goal_cancel: DispatchSyncCallback,
     default_relay_transport: str,
+    cmd_metrics: DispatchSyncCallback | None = None,
+    cmd_benchmark_sprint_ab: DispatchSyncCallback | None = None,
 ) -> int:
     if args.command == "init":
         return cmd_init(
@@ -263,6 +384,18 @@ def dispatch_main(
         )
     if args.command == "list":
         return cmd_list(workspace_root, args.request_id)
+    if args.command == "metrics":
+        if cmd_metrics is None:
+            parser.error("Metrics command is unavailable.")
+            return 2
+        return cmd_metrics(
+            workspace_root,
+            hours=float(getattr(args, "hours", 24.0)),
+            request_id=str(getattr(args, "request_id", "") or ""),
+            sprint_id=str(getattr(args, "sprint_id", "") or ""),
+            role=str(getattr(args, "agent", "") or ""),
+            as_json=bool(getattr(args, "json", False)),
+        )
     if args.command == "config":
         if args.config_command == "role" and args.role_command == "set":
             return cmd_config_role_set(
@@ -271,6 +404,13 @@ def dispatch_main(
                 model=getattr(args, "model", None),
                 reasoning=getattr(args, "reasoning", None),
             )
+        if args.config_command == "internal" and args.internal_command == "set":
+            return cmd_config_internal_set(
+                workspace_root,
+                args.agent,
+                model=getattr(args, "model", None),
+                reasoning=getattr(args, "reasoning", None),
+            )
         if args.config_command == "research" and args.research_command == "set":
             return cmd_config_research_set(
                 workspace_root,
@@ -315,6 +455,24 @@ def dispatch_main(
             return cmd_goal_resume(workspace_root)
         if args.goal_command in {"cancel", "terminate"}:
             return cmd_goal_cancel(workspace_root)
+    if args.command == "benchmark" and args.benchmark_command == "sprint-ab":
+        if cmd_benchmark_sprint_ab is None:
+            parser.error("Sprint benchmark command is unavailable.")
+            return 2
+        return cmd_benchmark_sprint_ab(
+            live=bool(getattr(args, "live", False)),
+            runtime_config=str(getattr(args, "runtime_config", "") or ""),
+            repetitions=int(getattr(args, "repetitions", 1)),
+            max_invocations=int(getattr(args, "max_invocations", 20)),
+            call_timeout_seconds=float(getattr(args, "call_timeout_seconds", 300.0)),
+            run_timeout_seconds=float(getattr(args, "run_timeout_seconds", 1800.0)),
+            keep_workspaces=str(getattr(args, "keep_workspaces", "failures") or "failures"),
+            rate_card_file=str(getattr(args, "rate_card_file", "") or ""),
+            output_dir=str(getattr(args, "output_dir", "") or ""),
+            benchmark_id=str(getattr(args, "benchmark_id", "") or ""),
+            allow_dirty_source=bool(getattr(args, "allow_dirty_source", False)),
+            as_json=bool(getattr(args, "json", False)),
+        )
     parser.error(f"Unsupported command: {args.command}")
     return 2
 
@@ -477,6 +635,8 @@ def _format_role_runtime_summary(runtime_config: Any, role: str) -> str:
             f"callback_timeout={int(research_runtime.callback_timeout)}"
         )
     role_runtime = runtime_config.role_defaults.get(role)
+    if role_runtime is None:
+        role_runtime = runtime_config.internal_agent_defaults.get(role)
     if role_runtime is None:
         return "model=N/A reasoning=N/A"
     model = str(role_runtime.model or "").strip() or "N/A"
@@ -548,6 +708,41 @@ def cmd_status_impl(
     return 0
 
 
+def cmd_metrics_impl(
+    workspace_root: Path,
+    *,
+    hours: float,
+    request_id: str,
+    sprint_id: str,
+    role: str,
+    as_json: bool,
+    runtime_paths_cls: Any,
+    aggregate_model_invocations: Callable[..., dict[str, Any]],
+    render_model_metrics_summary: Callable[[dict[str, Any]], str],
+    printer: Printer = print,
+) -> int:
+    if hours <= 0:
+        printer("metrics --hours must be a positive number.")
+        return 2
+    paths = runtime_paths_cls.from_root(workspace_root)
+    try:
+        summary = aggregate_model_invocations(
+            paths,
+            hours=hours,
+            request_id=request_id,
+            sprint_id=sprint_id,
+            role=role,
+        )
+    except ValueError as exc:
+        printer(str(exc))
+        return 2
+    if as_json:
+        printer(json.dumps(summary, ensure_ascii=False, indent=2))
+    else:
+        printer(render_model_metrics_summary(summary))
+    return 0
+
+
 def cmd_stop_impl(
     workspace_root: Path,
     role: str | None,
@@ -662,6 +857,41 @@ def cmd_config_role_set_impl(
     return 0
 
 
+def cmd_config_internal_set_impl(
+    workspace_root: Path,
+    agent: str,
+    *,
+    model: str | None = None,
+    reasoning: str | None = None,
+    update_team_runtime_internal_agent_defaults: Callable[..., Any],
+    runtime_paths_cls: Any,
+    printer: Printer = print,
+) -> int:
+    updated = update_team_runtime_internal_agent_defaults(
+        workspace_root,
+        agent,
+        model=model,
+        reasoning=reasoning,
+    )
+    effective_reasoning = (
+        "None" if "gemini" in updated.model.lower() else updated.reasoning
+    )
+    config_path = (
+        runtime_paths_cls.from_root(workspace_root).workspace_root
+        / "team_runtime.yaml"
+    )
+    printer(f"Updated {config_path}")
+    printer(
+        f"internal_agent={agent} model={updated.model} "
+        f"reasoning={effective_reasoning}"
+    )
+    printer(
+        "Restart the orchestrator to apply helper changes: "
+        "python -m teams_runtime restart --agent orchestrator"
+    )
+    return 0
+
+
 def cmd_config_research_set_impl(
     workspace_root: Path,
     *,
@@ -848,6 +1078,7 @@ def cmd_goal_cancel_impl(
 
 __all__ = [
     "build_parser",
+    "cmd_config_internal_set_impl",
     "cmd_config_research_set_impl",
     "cmd_config_role_set_impl",
     "cmd_goal_cancel_impl",
@@ -857,6 +1088,7 @@ def cmd_goal_cancel_impl(
     "cmd_goal_stop_impl",
     "cmd_init_impl",
     "cmd_list_impl",
+    "cmd_metrics_impl",
     "cmd_restart_impl",
     "cmd_sprint_restart_impl",
     "cmd_sprint_start_impl",

benchmarking/__init__.py

@@ -0,0 +1,25 @@
+"""Repeatable before/after benchmarks for teams_runtime."""
+
+from teams_runtime.benchmarking.models import (
+    ArmPlan,
+    BenchmarkOptions,
+    BenchmarkResult,
+    BenchmarkWorker,
+    QualityEvidence,
+    SprintEvidence,
+    WorkerContext,
+    WorkerOutcome,
+)
+from teams_runtime.benchmarking.runner import run_sprint_ab_benchmark
+
+__all__ = [
+    "ArmPlan",
+    "BenchmarkOptions",
+    "BenchmarkResult",
+    "BenchmarkWorker",
+    "QualityEvidence",
+    "SprintEvidence",
+    "WorkerContext",
+    "WorkerOutcome",
+    "run_sprint_ab_benchmark",
+]