From 8ba848010d368bf988c6b61034bf26fee752341d Mon Sep 17 00:00:00 2001 From: rotatedcoded Date: Tue, 14 Jul 2026 08:14:28 +0700 Subject: [PATCH] Add runtime readiness checker --- check_runtime_readiness.py | 7 + src/runtime/__init__.py | 1 + src/runtime/readiness.py | 364 ++++++++++++++++++++++++++++++++ tests/test_runtime_readiness.py | 183 ++++++++++++++++ 4 files changed, 555 insertions(+) create mode 100644 check_runtime_readiness.py create mode 100644 src/runtime/__init__.py create mode 100644 src/runtime/readiness.py create mode 100644 tests/test_runtime_readiness.py diff --git a/check_runtime_readiness.py b/check_runtime_readiness.py new file mode 100644 index 0000000..71ce62c --- /dev/null +++ b/check_runtime_readiness.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from src.runtime.readiness import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/runtime/__init__.py b/src/runtime/__init__.py new file mode 100644 index 0000000..596ea65 --- /dev/null +++ b/src/runtime/__init__.py @@ -0,0 +1 @@ +"""Runtime readiness helpers.""" diff --git a/src/runtime/readiness.py b/src/runtime/readiness.py new file mode 100644 index 0000000..6b6b4c7 --- /dev/null +++ b/src/runtime/readiness.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path +from typing import Sequence + +from src.engine.stockfish_service import DEFAULT_ENGINE_PATH +from src.ui.main_ui_elite_recommendation import ( + MainUIEliteRecommendation, + build_main_ui_elite_recommendation, +) +from src.ui.move_policy import BOT_MOVE_POLICY_STATISTICAL + +DEFAULT_PERSONA_ROOT = Path("data/personas") +DEFAULT_DIRECT_MODELS_ROOT = Path("data/ml/direct_candidate_models") +DEFAULT_ELITE_EVIDENCE_ROOT = Path("data/ml/direct_ranker_elite_guarded_evidence") + + +@dataclass(frozen=True) +class RuntimeReadinessCheck: + name: str + ok: bool + status: str + path: Path | None = None + required: bool = True + detail: str = "" + + +@dataclass(frozen=True) +class RuntimeReadinessReport: + username: str + time_class: str + ready: bool + checks: tuple[RuntimeReadinessCheck, ...] + recommendation: MainUIEliteRecommendation + + @property + def exit_code(self) -> int: + return 0 if self.ready else 2 + + +def normalize_runtime_name(value: object) -> str: + return str(value or "").strip().lower() + + +def persona_candidate_paths( + *, + username: object, + time_class: object, + persona_root: Path | str = DEFAULT_PERSONA_ROOT, +) -> tuple[Path, ...]: + user = normalize_runtime_name(username) + tc = normalize_runtime_name(time_class) + root = Path(persona_root) + return ( + root / user / f"{tc}_complete_persona.json", + root / user / f"{tc}_persona.json", + root / user / f"{tc}_description.json", + root / f"{user}_{tc}_complete_persona.json", + root / f"{user}_{tc}_persona.json", + ) + + +def first_existing_path(paths: Sequence[Path]) -> Path | None: + for path in paths: + if path.exists(): + return path + return None + + +def direct_ranker_model_path( + *, + username: object, + time_class: object, + models_root: Path | str = DEFAULT_DIRECT_MODELS_ROOT, +) -> Path: + user = normalize_runtime_name(username) + tc = normalize_runtime_name(time_class) + return Path(models_root) / user / f"{tc}_direct_ranker.joblib" + + +def direct_ranker_metrics_candidate_paths( + *, + username: object, + time_class: object, + models_root: Path | str = DEFAULT_DIRECT_MODELS_ROOT, +) -> tuple[Path, ...]: + user = normalize_runtime_name(username) + tc = normalize_runtime_name(time_class) + root = Path(models_root) + + # Important: keep this scoped to models_root. + # Do not read global data/ml release-gate files here because isolated tests + # can accidentally pass when local artifacts exist for the same username. + return ( + root / user / f"{tc}_direct_ranker_metrics.json", + root / user / f"{tc}_metrics.json", + root / user / f"{tc}_direct_ranker_model_metrics.json", + root / user / f"{tc}_training_metrics.json", + ) + +def _path_check( + *, + name: str, + path: Path, + required: bool = True, + ok_status: str = "OK", + missing_status: str = "MISSING", +) -> RuntimeReadinessCheck: + exists = path.exists() + return RuntimeReadinessCheck( + name=name, + ok=exists, + status=ok_status if exists else missing_status, + path=path, + required=required, + ) + + +def _stockfish_check(engine_path: Path | str) -> RuntimeReadinessCheck: + return _path_check( + name="Stockfish", + path=Path(engine_path), + required=True, + ) + + +def _persona_check( + *, + username: object, + time_class: object, + persona_root: Path | str, +) -> RuntimeReadinessCheck: + candidates = persona_candidate_paths( + username=username, + time_class=time_class, + persona_root=persona_root, + ) + found = first_existing_path(candidates) + if found is not None: + return RuntimeReadinessCheck( + name="Persona", + ok=True, + status="OK", + path=found, + required=True, + ) + return RuntimeReadinessCheck( + name="Persona", + ok=False, + status="MISSING", + path=candidates[0], + required=True, + detail="Tidak menemukan complete/persona/description JSON untuk username dan time class ini.", + ) + + +def _model_check( + *, + username: object, + time_class: object, + models_root: Path | str, +) -> RuntimeReadinessCheck: + return _path_check( + name="Direct Ranker model", + path=direct_ranker_model_path( + username=username, + time_class=time_class, + models_root=models_root, + ), + required=True, + ) + + +def _metrics_check( + *, + username: object, + time_class: object, + models_root: Path | str, + strict_metrics: bool, +) -> RuntimeReadinessCheck: + candidates = direct_ranker_metrics_candidate_paths( + username=username, + time_class=time_class, + models_root=models_root, + ) + found = first_existing_path(candidates) + if found is not None: + return RuntimeReadinessCheck( + name="Direct Ranker metrics", + ok=True, + status="OK", + path=found, + required=strict_metrics, + ) + return RuntimeReadinessCheck( + name="Direct Ranker metrics", + ok=False, + status="MISSING", + path=candidates[0], + required=strict_metrics, + detail=( + "Metrics JSON tidak ditemukan. Secara default ini warning; " + "pakai --strict-metrics untuk menjadikannya blocking." + ), + ) + + +def _elite_evidence_check( + recommendation: MainUIEliteRecommendation, +) -> RuntimeReadinessCheck: + return RuntimeReadinessCheck( + name="Elite evidence", + ok=bool(recommendation.should_select_policy), + status=recommendation.decision, + path=recommendation.evidence_path, + required=True, + detail=recommendation.reason, + ) + + +def _ui_runtime_check( + recommendation: MainUIEliteRecommendation, +) -> RuntimeReadinessCheck: + if recommendation.should_select_policy: + return RuntimeReadinessCheck( + name="UI runtime", + ok=True, + status="ready", + path=None, + required=True, + detail=recommendation.move_policy, + ) + return RuntimeReadinessCheck( + name="UI runtime", + ok=False, + status="statistical_only", + path=None, + required=True, + detail="UI tidak boleh auto-select elite policy sampai evidence ready.", + ) + + +def build_runtime_readiness_report( + *, + username: object, + time_class: object, + engine_path: Path | str = DEFAULT_ENGINE_PATH, + persona_root: Path | str = DEFAULT_PERSONA_ROOT, + models_root: Path | str = DEFAULT_DIRECT_MODELS_ROOT, + evidence_root: Path | str = DEFAULT_ELITE_EVIDENCE_ROOT, + strict_metrics: bool = False, +) -> RuntimeReadinessReport: + user = normalize_runtime_name(username) + tc = normalize_runtime_name(time_class) + recommendation = build_main_ui_elite_recommendation( + username=user, + time_class=tc, + evidence_root=evidence_root, + ) + checks = ( + _persona_check( + username=user, + time_class=tc, + persona_root=persona_root, + ), + _stockfish_check(engine_path), + _model_check( + username=user, + time_class=tc, + models_root=models_root, + ), + _metrics_check( + username=user, + time_class=tc, + models_root=models_root, + strict_metrics=strict_metrics, + ), + _elite_evidence_check(recommendation), + _ui_runtime_check(recommendation), + ) + ready = all(check.ok for check in checks if check.required) + return RuntimeReadinessReport( + username=user, + time_class=tc, + ready=ready, + checks=checks, + recommendation=recommendation, + ) + + +def render_runtime_readiness_report( + report: RuntimeReadinessReport, +) -> str: + recommendation = report.recommendation + title = "READY" if report.ready else "NOT READY" + lines = [ + f"Runtime readiness: {title}", + "", + f"Persona: {report.username} / {report.time_class}", + ] + for check in report.checks: + suffix = "" + if check.path is not None: + suffix = f" ({check.path})" + if not check.required and not check.ok: + suffix += " [warning]" + lines.append(f"{check.name}: {check.status}{suffix}") + if check.detail: + lines.append(f" - {check.detail}") + lines.extend( + [ + f"Recommended UI policy: {recommendation.move_policy_label}", + f"Move policy key: {recommendation.move_policy}", + ] + ) + return "\n".join(lines) + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Check whether a ChessPersona runtime is ready for UI play." + ) + parser.add_argument("--username", required=True) + parser.add_argument( + "--time-class", + choices=("rapid", "blitz"), + required=True, + ) + parser.add_argument("--engine-path", default=str(DEFAULT_ENGINE_PATH)) + parser.add_argument("--persona-root", default=str(DEFAULT_PERSONA_ROOT)) + parser.add_argument("--models-root", default=str(DEFAULT_DIRECT_MODELS_ROOT)) + parser.add_argument("--evidence-root", default=str(DEFAULT_ELITE_EVIDENCE_ROOT)) + parser.add_argument( + "--strict-metrics", + action="store_true", + help="Treat missing metrics/release-gate JSON as blocking.", + ) + return parser + + +def run_from_args(args: argparse.Namespace) -> int: + report = build_runtime_readiness_report( + username=args.username, + time_class=args.time_class, + engine_path=args.engine_path, + persona_root=args.persona_root, + models_root=args.models_root, + evidence_root=args.evidence_root, + strict_metrics=bool(args.strict_metrics), + ) + print(render_runtime_readiness_report(report)) + return report.exit_code + + +def main(argv: Sequence[str] | None = None) -> int: + parser = build_arg_parser() + args = parser.parse_args(argv) + return run_from_args(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_runtime_readiness.py b/tests/test_runtime_readiness.py new file mode 100644 index 0000000..09cdad6 --- /dev/null +++ b/tests/test_runtime_readiness.py @@ -0,0 +1,183 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from src.runtime.readiness import ( + build_runtime_readiness_report, + direct_ranker_model_path, + render_runtime_readiness_report, + run_from_args, +) +from src.ui.move_policy import ( + BOT_MOVE_POLICY_DIRECT_RANKER_ELITE_GUARDED_STRICT, + BOT_MOVE_POLICY_STATISTICAL, +) + + +def write_ready_fixture( + root: Path, + *, + username: str = "hikaru", + time_class: str = "rapid", + decision: str = "elite_guarded_strict_ready", +) -> dict[str, Path]: + persona_root = root / "personas" + models_root = root / "models" + evidence_root = root / "evidence" + engine_path = root / "engines" / "stockfish.exe" + + persona_path = persona_root / username / f"{time_class}_complete_persona.json" + persona_path.parent.mkdir(parents=True, exist_ok=True) + persona_path.write_text("{}", encoding="utf-8") + + model_path = direct_ranker_model_path( + username=username, + time_class=time_class, + models_root=models_root, + ) + model_path.parent.mkdir(parents=True, exist_ok=True) + model_path.write_bytes(b"dummy model marker") + + metrics_path = models_root / username / f"{time_class}_direct_ranker_metrics.json" + metrics_path.write_text("{}", encoding="utf-8") + + evidence_path = evidence_root / username / f"{time_class}_elite_guarded_evidence.json" + evidence_path.parent.mkdir(parents=True, exist_ok=True) + evidence_path.write_text( + json.dumps( + { + "decision": decision, + "reason": "elite_guarded_evidence_ready", + "recommended_policy": "Direct Ranker Elite Guarded Strict", + "failed_checks": [], + } + ), + encoding="utf-8", + ) + + engine_path.parent.mkdir(parents=True, exist_ok=True) + engine_path.write_text("dummy engine marker", encoding="utf-8") + + return { + "persona_root": persona_root, + "models_root": models_root, + "evidence_root": evidence_root, + "engine_path": engine_path, + "persona_path": persona_path, + "model_path": model_path, + "metrics_path": metrics_path, + "evidence_path": evidence_path, + } + + +def test_runtime_readiness_ready_for_elite_strict(tmp_path: Path) -> None: + paths = write_ready_fixture(tmp_path) + report = build_runtime_readiness_report( + username="Hikaru", + time_class="rapid", + engine_path=paths["engine_path"], + persona_root=paths["persona_root"], + models_root=paths["models_root"], + evidence_root=paths["evidence_root"], + ) + assert report.ready is True + assert report.exit_code == 0 + assert report.recommendation.move_policy == BOT_MOVE_POLICY_DIRECT_RANKER_ELITE_GUARDED_STRICT + rendered = render_runtime_readiness_report(report) + assert "Runtime readiness: READY" in rendered + assert "Elite evidence: elite_guarded_strict_ready" in rendered + assert "Move policy key: direct_ranker_elite_guarded_strict" in rendered + + +def test_runtime_readiness_missing_evidence_blocks_ui_runtime(tmp_path: Path) -> None: + paths = write_ready_fixture(tmp_path) + paths["evidence_path"].unlink() + report = build_runtime_readiness_report( + username="hikaru", + time_class="rapid", + engine_path=paths["engine_path"], + persona_root=paths["persona_root"], + models_root=paths["models_root"], + evidence_root=paths["evidence_root"], + ) + assert report.ready is False + assert report.exit_code == 2 + assert report.recommendation.move_policy == BOT_MOVE_POLICY_STATISTICAL + rendered = render_runtime_readiness_report(report) + assert "Runtime readiness: NOT READY" in rendered + assert "Elite evidence: elite_evidence_missing" in rendered + + +def test_runtime_readiness_needs_more_evidence_blocks(tmp_path: Path) -> None: + paths = write_ready_fixture(tmp_path, decision="needs_more_evidence") + report = build_runtime_readiness_report( + username="hikaru", + time_class="rapid", + engine_path=paths["engine_path"], + persona_root=paths["persona_root"], + models_root=paths["models_root"], + evidence_root=paths["evidence_root"], + ) + assert report.ready is False + assert report.recommendation.move_policy == BOT_MOVE_POLICY_STATISTICAL + assert any( + check.name == "Elite evidence" and check.status == "needs_more_evidence" + for check in report.checks + ) + + +def test_runtime_readiness_metrics_are_warning_by_default(tmp_path: Path) -> None: + paths = write_ready_fixture(tmp_path) + paths["metrics_path"].unlink() + report = build_runtime_readiness_report( + username="hikaru", + time_class="rapid", + engine_path=paths["engine_path"], + persona_root=paths["persona_root"], + models_root=paths["models_root"], + evidence_root=paths["evidence_root"], + ) + assert report.ready is True + metrics_check = next(check for check in report.checks if check.name == "Direct Ranker metrics") + assert metrics_check.ok is False + assert metrics_check.required is False + + +def test_runtime_readiness_strict_metrics_blocks(tmp_path: Path) -> None: + paths = write_ready_fixture(tmp_path) + paths["metrics_path"].unlink() + report = build_runtime_readiness_report( + username="hikaru", + time_class="rapid", + engine_path=paths["engine_path"], + persona_root=paths["persona_root"], + models_root=paths["models_root"], + evidence_root=paths["evidence_root"], + strict_metrics=True, + ) + assert report.ready is False + metrics_check = next(check for check in report.checks if check.name == "Direct Ranker metrics") + assert metrics_check.required is True + + +def test_runtime_readiness_cli_exit_code(tmp_path: Path, capsys) -> None: + paths = write_ready_fixture(tmp_path) + exit_code = run_from_args( + type( + "Args", + (), + { + "username": "hikaru", + "time_class": "rapid", + "engine_path": str(paths["engine_path"]), + "persona_root": str(paths["persona_root"]), + "models_root": str(paths["models_root"]), + "evidence_root": str(paths["evidence_root"]), + "strict_metrics": False, + }, + )() + ) + assert exit_code == 0 + output = capsys.readouterr().out + assert "Runtime readiness: READY" in output