From af33f6c338d920133991a23d4e31e71ad82cb0f4 Mon Sep 17 00:00:00 2001 From: Alphaxalchemy Date: Sun, 12 Jul 2026 21:03:48 -0600 Subject: [PATCH 1/2] feat(sleep): support OpenAI-compatible endpoints in azure_openai backend Let AzureOpenAIBackend drive any OpenAI-compatible chat-completions server (DeepSeek, self-hosted vLLM/Ollama, ...) alongside native Azure deployments. - __init__ resolves the endpoint as: explicit arg > AZURE_OPENAI_ENDPOINT env > the built-in _AZURE_ENDPOINTS table (previously a non-Azure endpoint could not be supplied at all). - _get_client() builds a plain openai.OpenAI(base_url=...) client when AZURE_OPENAI_AUTH_MODE=openai_compatible, matching the auth mode already supported by the sibling skillopt/model/azure_openai.py. This avoids the AzureOpenAI SDK rewriting request URLs with Azure-only ?api-version= query params and deployment path segments, which non-Azure servers reject with 404. - _call() sends max_tokens + extra_body thinking flag for deepseek* models and records the last exception in self.last_call_error so a failed night is diagnosable instead of collapsing to a silent empty->0 score. Adds docs/sleep/openai-compatible-endpoints.md and sanitized example runner/watchdog scripts documenting an Antigravity + DeepSeek integration. The default managed-identity Azure path is unchanged. --- docs/sleep/examples/runner.py | 100 ++++++++++++++++++++ docs/sleep/examples/watchdog.py | 56 +++++++++++ docs/sleep/openai-compatible-endpoints.md | 107 ++++++++++++++++++++++ skillopt_sleep/backend.py | 62 ++++++++++--- 4 files changed, 311 insertions(+), 14 deletions(-) create mode 100644 docs/sleep/examples/runner.py create mode 100644 docs/sleep/examples/watchdog.py create mode 100644 docs/sleep/openai-compatible-endpoints.md diff --git a/docs/sleep/examples/runner.py b/docs/sleep/examples/runner.py new file mode 100644 index 00000000..ecb2a4da --- /dev/null +++ b/docs/sleep/examples/runner.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Reference launcher for running SkillOpt-Sleep against an OpenAI-compatible +endpoint (DeepSeek shown here), plus an Antigravity `session-end` hook. + +This is a *sanitized example*, not a supported entry point. Adapt the paths and +provider details to your environment. No API keys are hardcoded — the key is read +from an .env file or the process environment. + +Usage: + python runner.py run # run a full sleep cycle against DeepSeek + python runner.py dry-run # harvest + replay, report only + python runner.py session-end # Antigravity Stop-hook: append rollout evidence +""" +import os +import re +import sys +import json +import subprocess +import datetime +from pathlib import Path + +# --- Configure these for your environment ----------------------------------- +# Path to a file containing your provider key as `sk-...` (kept out of source). +PROVIDER_ENV_FILE = Path(os.environ.get("SKILLOPT_PROVIDER_ENV_FILE", "provider.env")) +# Endpoint + model for the OpenAI-compatible provider. +PROVIDER_ENDPOINT = os.environ.get("SKILLOPT_PROVIDER_ENDPOINT", "https://api.deepseek.com") +PROVIDER_MODEL = os.environ.get("SKILLOPT_PROVIDER_MODEL", "deepseek-v4-pro") +# Project whose SKILL.md files the sleep cycle should evolve. +PROJECT_DIR = os.environ.get("SKILLOPT_PROJECT_DIR", os.getcwd()) +# Where the session-end hook appends rollout evidence. +ROLLOUT_LOG = Path(os.environ.get("SKILLOPT_ROLLOUT_LOG", "brain/rollout-evidence.jsonl")) +# ---------------------------------------------------------------------------- + + +def load_provider_key(env: dict) -> None: + """Ensure DEEPSEEK_API_KEY is set, reading it from PROVIDER_ENV_FILE if needed.""" + if env.get("DEEPSEEK_API_KEY"): + return + try: + text = PROVIDER_ENV_FILE.read_text(encoding="utf-8") + except OSError: + return + m = re.search(r"sk-[A-Za-z0-9]+", text) + if m: + env["DEEPSEEK_API_KEY"] = m.group(0) + + +def main() -> None: + if len(sys.argv) < 2: + print("Usage: runner.py [dry-run|run|status|adopt|session-end]") + sys.exit(1) + + command = sys.argv[1] + + # Antigravity Stop-hook: enrich future nights with task-outcome metadata. + if command == "session-end": + ROLLOUT_LOG.parent.mkdir(parents=True, exist_ok=True) + outcome = { + "timestamp": datetime.datetime.now().isoformat(), + "event": "SessionEnd", + "metadata": "Appended task outcome metadata", + } + with open(ROLLOUT_LOG, "a", encoding="utf-8") as f: + f.write(json.dumps(outcome) + "\n") + print("Rollout evidence metadata appended.") + return + + env = os.environ.copy() + load_provider_key(env) + + if env.get("DEEPSEEK_API_KEY"): + # OpenAI-compatible path — see docs/sleep/openai-compatible-endpoints.md + backend = "azure_openai" + env["PYTHONIOENCODING"] = "utf-8" + for prefix in ("", "OPTIMIZER_", "TARGET_"): + env[f"{prefix}AZURE_OPENAI_AUTH_MODE"] = "openai_compatible" + env[f"{prefix}AZURE_OPENAI_ENDPOINT"] = PROVIDER_ENDPOINT + env[f"{prefix}AZURE_OPENAI_API_KEY"] = env["DEEPSEEK_API_KEY"] + env["TARGET_DEPLOYMENT"] = PROVIDER_MODEL + env["OPTIMIZER_DEPLOYMENT"] = PROVIDER_MODEL + else: + # OPTIONAL, UNVERIFIED fallback: route the `claude` CLI backend through a + # local Anthropic-compatible proxy (e.g. LiteLLM) to reach Gemini. There + # is no native Gemini backend; this path was not validated. See the doc. + backend = "claude" + if "ANTHROPIC_API_KEY" not in env and "GEMINI_API_KEY" in env: + env["ANTHROPIC_API_KEY"] = env["GEMINI_API_KEY"] + env.setdefault("ANTHROPIC_BASE_URL", "http://127.0.0.1:4000") + + args = ["skillopt-sleep", command] + if command in ("run", "dry-run"): + args = ["skillopt-sleep", command, "--backend", backend, + "--model", PROVIDER_MODEL, "--project", PROJECT_DIR] + + print(f"Running: {' '.join(args)}") + subprocess.run(args, env=env, check=False) + + +if __name__ == "__main__": + main() diff --git a/docs/sleep/examples/watchdog.py b/docs/sleep/examples/watchdog.py new file mode 100644 index 00000000..d749c361 --- /dev/null +++ b/docs/sleep/examples/watchdog.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Minimal supervisor that runs the SkillOpt-Sleep cycle on a fixed interval. + +Sanitized example (see docs/sleep/openai-compatible-endpoints.md). On Windows, +register this under a Scheduled Task so it survives logout; on Linux/macOS a +systemd timer or cron entry serves the same purpose and is usually preferable to +a long-lived process. +""" +import os +import sys +import time +import subprocess +import datetime +import traceback + +INTERVAL_SECONDS = int(os.environ.get("SKILLOPT_WATCHDOG_INTERVAL", str(4 * 3600))) +RUNNER = os.environ.get("SKILLOPT_RUNNER", os.path.join(os.path.dirname(__file__), "runner.py")) +LOG_FILE = os.environ.get("SKILLOPT_WATCHDOG_LOG", "brain/watchdog.log") + + +def log(msg: str) -> None: + os.makedirs(os.path.dirname(LOG_FILE) or ".", exist_ok=True) + line = f"[{datetime.datetime.now().isoformat()}] {msg}" + with open(LOG_FILE, "a", encoding="utf-8") as f: + f.write(line + "\n") + print(line) + + +def run_once() -> None: + log("Invoking skillopt-sleep run via runner.py...") + try: + result = subprocess.run([sys.executable, RUNNER, "run"], + capture_output=True, text=True) + if result.returncode == 0: + log("Successfully completed run.") + else: + log(f"Run failed (exit {result.returncode}).") + log(f"STDERR: {result.stderr}") + except Exception as e: + log(f"Exception while running skillopt: {e}") + log(traceback.format_exc()) + + +def main() -> None: + log(f"Watchdog started. Interval: {INTERVAL_SECONDS}s.") + while True: + try: + run_once() + except Exception as e: + log(f"Unexpected error in watchdog loop: {e}") + log(f"Sleeping for {INTERVAL_SECONDS}s...") + time.sleep(INTERVAL_SECONDS) + + +if __name__ == "__main__": + main() diff --git a/docs/sleep/openai-compatible-endpoints.md b/docs/sleep/openai-compatible-endpoints.md new file mode 100644 index 00000000..cdcfe825 --- /dev/null +++ b/docs/sleep/openai-compatible-endpoints.md @@ -0,0 +1,107 @@ +# OpenAI-compatible endpoints for SkillOpt-Sleep (DeepSeek, local vLLM, …) + +This document describes a small enhancement to the `azure_openai` backend in +`skillopt_sleep/backend.py` that lets SkillOpt-Sleep drive **any +OpenAI-compatible chat-completions endpoint** — for example DeepSeek's hosted +API or a self-hosted vLLM/Ollama server — in addition to native Azure OpenAI +deployments. It also documents a concrete end-to-end integration: running the +nightly sleep cycle inside the Antigravity IDE against DeepSeek. + +## What changed + +Three focused changes in `AzureOpenAIBackend` (`skillopt_sleep/backend.py`), +all backward-compatible — the default managed-identity path is unchanged: + +1. **Endpoint resolution honors `AZURE_OPENAI_ENDPOINT`.** + `__init__` now resolves the endpoint as `explicit arg` → `AZURE_OPENAI_ENDPOINT` + env → the built-in `_AZURE_ENDPOINTS` table. Previously a non-Azure endpoint + could not be supplied at all, so calls always went to a hardcoded + `*.openai.azure.com` host. + +2. **`openai_compatible` auth mode.** When + `AZURE_OPENAI_AUTH_MODE=openai_compatible` (also accepts `compat`/`openai`), + `_get_client()` builds a plain `openai.OpenAI(base_url=…)` client instead of + an `AzureOpenAI` client. This mirrors the auth mode already supported by the + sibling `skillopt/model/azure_openai.py` module, so the two are consistent. + +3. **DeepSeek request shape + error surfacing.** `_call()` sends `max_tokens` + plus `extra_body={"thinking": {"type": "enabled"}}` for `deepseek*` + deployments (the DeepSeek reasoning models expect this), and records the last + exception in `self.last_call_error` so a failed night is diagnosable instead + of collapsing to a silent empty response scored `0.0`. + +### Why the previous behavior failed on non-Azure servers + +The `AzureOpenAI` SDK client rewrites request URLs with Azure-only structure — +a `?api-version=…` query string and deployment path segments. A non-Azure +OpenAI-compatible server (DeepSeek, vLLM, …) does not recognize those routes and +responds `404 Resource not found`. SkillOpt-Sleep then receives an empty +response, the judge scores it `0.0`, and every night reports +`baseline 0.0 -> candidate 0.0` with no usable diagnostic. Selecting a plain +`OpenAI` client via `openai_compatible` mode avoids the Azure URL rewriting and +talks to the endpoint directly. + +## How to use it + +Set four environment variables and run the cycle with `--backend azure_openai`: + +```bash +export AZURE_OPENAI_AUTH_MODE=openai_compatible +export AZURE_OPENAI_ENDPOINT=https://api.deepseek.com # no /v1, no trailing path +export AZURE_OPENAI_API_KEY=sk-... # your provider key +# optimizer/target dual-role overrides are also honored, e.g. +# OPTIMIZER_AZURE_OPENAI_AUTH_MODE / TARGET_AZURE_OPENAI_AUTH_MODE, etc. + +skillopt-sleep run \ + --backend azure_openai \ + --model deepseek-v4-pro \ + --project /path/to/your/project +``` + +The same pattern works for any OpenAI-compatible server — point +`AZURE_OPENAI_ENDPOINT` at it and set a matching `--model`. + +## End-to-end integration: Antigravity + DeepSeek + +The [`examples/`](examples/) directory contains a sanitized reference of how this +was wired into the [Antigravity](https://antigravity.google/) agent IDE so the +sleep cycle runs unattended: + +- **`examples/runner.py`** — a thin launcher that loads a provider key from an + `.env` file, exports the four variables above, and invokes `skillopt-sleep run` + with the DeepSeek backend. It also implements a `session-end` hook that appends + task-outcome metadata to a rollout-evidence log (wired to Antigravity's `Stop` + hook) so future nights have richer sessions to mine. +- **`examples/watchdog.py`** — a minimal supervisor loop that invokes the runner + on a fixed interval (e.g. every 4 hours). On Windows this is registered as a + Scheduled Task so it survives logout; on Linux/macOS a `systemd` timer or cron + entry serves the same role. + +### Verified result + +On a Windows 11 host, driving the cycle against `deepseek-v4-pro` in +`openai_compatible` mode: + +- A direct backend smoke test returns a live completion (no `404`, + `last_call_error` empty, client type `OpenAI`). +- A full nightly cycle mined tasks from real IDE sessions and the held-out + validation gate moved from `0.250 → 1.000`, **accepting** a DeepSeek-authored + skill edit (`accept_new_best`). `diagnostics.json` for that night reports + `"backend": "azure_openai"` with a non-empty token count and an empty + `call_error` — i.e. a genuine optimization night, versus the prior all-`0.0` + nights that the endpoint bug produced. +- A subsequent unattended night triggered by the watchdog completed the full + chain (watchdog → runner → `skillopt-sleep` → DeepSeek) and the gate correctly + **rejected** a non-improving proposal (`0.3 → 0.3`), confirming the validation + gate behaves normally on the new backend. + +## A note on Gemini (optional, unverified fallback) + +`examples/runner.py` also contains a fallback branch that, when only a Gemini key +is present, routes the **`claude` CLI backend** through a local +Anthropic-compatible proxy (e.g. [LiteLLM](https://github.com/BerriAI/litellm) on +`http://127.0.0.1:4000`) by setting `ANTHROPIC_BASE_URL`/`ANTHROPIC_API_KEY`. +There is **no native Gemini backend** in SkillOpt, and this proxy path was not +independently validated in this work — it is included only as a configuration +example. The verified, supported path in this document is DeepSeek via +`openai_compatible` mode. Treat the Gemini branch as illustrative, not tested. diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index d7bb040b..77fcae8a 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -1236,7 +1236,15 @@ def __init__(self, deployment: str = "", endpoint: str = "", timeout: int = 180, api_version: str = "2024-12-01-preview") -> None: super().__init__(model=deployment or "gpt-5.5", timeout=timeout) self.deployment = deployment or "gpt-5.5" - self.endpoint = endpoint or self._endpoint_for(self.deployment) + # Endpoint resolution order: explicit arg > AZURE_OPENAI_ENDPOINT env > + # the built-in Azure endpoint table. Honoring the env var lets callers + # point this backend at any OpenAI-compatible server (DeepSeek, a local + # vLLM, etc.) without editing the hardcoded _AZURE_ENDPOINTS map. + self.endpoint = ( + endpoint + or os.environ.get("AZURE_OPENAI_ENDPOINT", "").strip() + or self._endpoint_for(self.deployment) + ) self.api_version = api_version self.name = f"azure:{self.deployment}" self._client = None @@ -1250,14 +1258,29 @@ def _endpoint_for(deployment: str) -> str: def _get_client(self): if self._client is None: - from azure.identity import ManagedIdentityCredential, get_bearer_token_provider - from openai import AzureOpenAI - cred = ManagedIdentityCredential(client_id=_AZURE_MI_CLIENT_ID) - tp = get_bearer_token_provider(cred, "https://cognitiveservices.azure.com/.default") - self._client = AzureOpenAI( - azure_endpoint=self.endpoint, azure_ad_token_provider=tp, - api_version=self.api_version, max_retries=4, - ) + # AZURE_OPENAI_AUTH_MODE=openai_compatible (matching the sibling + # skillopt.model.azure_openai module) selects a plain OpenAI client + # against a raw base_url. This is what makes any OpenAI-compatible + # endpoint work: the AzureOpenAI client would otherwise rewrite the + # URL with Azure-only query params (?api-version=...) and deployment + # path segments, which non-Azure servers reject with a 404. + auth_mode = os.environ.get("AZURE_OPENAI_AUTH_MODE", "").strip().lower() + if auth_mode in {"openai_compatible", "compat", "openai"}: + from openai import OpenAI + self._client = OpenAI( + base_url=self.endpoint.rstrip("/"), + api_key=os.environ.get("AZURE_OPENAI_API_KEY"), + max_retries=4, + ) + else: + from azure.identity import ManagedIdentityCredential, get_bearer_token_provider + from openai import AzureOpenAI + cred = ManagedIdentityCredential(client_id=_AZURE_MI_CLIENT_ID) + tp = get_bearer_token_provider(cred, "https://cognitiveservices.azure.com/.default") + self._client = AzureOpenAI( + azure_endpoint=self.endpoint, azure_ad_token_provider=tp, + api_version=self.api_version, max_retries=4, + ) return self._client def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str: @@ -1277,11 +1300,19 @@ def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str last_exc = None for attempt in range(max(1, retries)): try: - resp = client.chat.completions.create( - model=self.deployment, - messages=[{"role": "user", "content": prompt}], - max_completion_tokens=16384, - ) + kwargs: Dict[str, Any] = { + "model": self.deployment, + "messages": [{"role": "user", "content": prompt}], + } + # DeepSeek's reasoning models expect `max_tokens` and an + # extra_body flag to enable the thinking channel; the Azure + # gpt-5.x deployments use `max_completion_tokens`. + if "deepseek" in self.deployment.lower(): + kwargs["max_tokens"] = 8192 + kwargs["extra_body"] = {"thinking": {"type": "enabled"}} + else: + kwargs["max_completion_tokens"] = 16384 + resp = client.chat.completions.create(**kwargs) text = (resp.choices[0].message.content or "").strip() try: u = resp.usage @@ -1295,6 +1326,9 @@ def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str last_exc = "empty-response" except Exception as e: # noqa: BLE001 last_exc = e + # Surface the error so a 0.0 night is diagnosable (e.g. a 404 + # from a mis-pointed endpoint) instead of a silent empty->0. + self.last_call_error = str(e)[:500] # backoff before next try (skip after the final attempt) if attempt < retries - 1: _t.sleep(min(8.0, (2 ** attempt) * 0.5) + _r.random() * 0.4) From 4ff77b71ffb34b6126c152d66602880a3e40104b Mon Sep 17 00:00:00 2001 From: Alphaxalchemy Date: Tue, 14 Jul 2026 01:22:19 -0600 Subject: [PATCH 2/2] =?UTF-8?q?fix(sleep):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20CLI=20choice,=20auth=20guard,=20provider-neutral=20kwargs,?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses maintainer review on the OpenAI-compatible endpoints PR: 1. CLI: accept --backend azure_openai in skillopt_sleep/__main__.py (the documented command was rejected by the argparse choices). 2. Example runner: exit with the child's return code so watchdog/supervisors see a failed sleep run as a failure. 3. Error state: clear last_call_error when a retry recovers; set an explicit "empty response on all N attempts" diagnostic when every attempt returns empty text. 4. Security guard: the managed-identity path now refuses to send an Azure AD bearer token to any endpoint outside *.openai.azure.com / *.cognitiveservices.azure.com — a custom endpoint requires explicit AZURE_OPENAI_AUTH_MODE=openai_compatible + API key. 5. Provider-neutral requests: compat mode sends only the standard contract (max_tokens, default 8192 via SKILLOPT_SLEEP_COMPAT_MAX_TOKENS); provider-specific body fields are opt-in via SKILLOPT_SLEEP_CHAT_EXTRA_BODY (JSON) — the deepseek model-name inference is removed. 6. Docs: removed the unimplemented OPTIMIZER_*/TARGET_* env-var claim; added a configuration reference matching the implementation exactly. 7. Tests: tests/test_azure_openai_compat.py — 17 deterministic no-network unittest cases covering CLI acceptance, compat-vs-Azure client selection, endpoint resolution, the credential guard, request kwargs (opt-in extra body / token cap), retry-success error clearing, empty-response diagnostics, and runner exit-code propagation. Re-verified live against DeepSeek (deepseek-v4-pro, openai_compatible mode) after the rework: client type OpenAI, completion returned, no error state. --- docs/sleep/examples/runner.py | 19 +- docs/sleep/openai-compatible-endpoints.md | 111 ++++++---- skillopt_sleep/__main__.py | 3 +- skillopt_sleep/backend.py | 105 ++++++++- tests/test_azure_openai_compat.py | 248 ++++++++++++++++++++++ 5 files changed, 426 insertions(+), 60 deletions(-) create mode 100644 tests/test_azure_openai_compat.py diff --git a/docs/sleep/examples/runner.py b/docs/sleep/examples/runner.py index ecb2a4da..d5e4e3e8 100644 --- a/docs/sleep/examples/runner.py +++ b/docs/sleep/examples/runner.py @@ -72,12 +72,14 @@ def main() -> None: # OpenAI-compatible path — see docs/sleep/openai-compatible-endpoints.md backend = "azure_openai" env["PYTHONIOENCODING"] = "utf-8" - for prefix in ("", "OPTIMIZER_", "TARGET_"): - env[f"{prefix}AZURE_OPENAI_AUTH_MODE"] = "openai_compatible" - env[f"{prefix}AZURE_OPENAI_ENDPOINT"] = PROVIDER_ENDPOINT - env[f"{prefix}AZURE_OPENAI_API_KEY"] = env["DEEPSEEK_API_KEY"] - env["TARGET_DEPLOYMENT"] = PROVIDER_MODEL - env["OPTIMIZER_DEPLOYMENT"] = PROVIDER_MODEL + env["AZURE_OPENAI_AUTH_MODE"] = "openai_compatible" + env["AZURE_OPENAI_ENDPOINT"] = PROVIDER_ENDPOINT + env["AZURE_OPENAI_API_KEY"] = env["DEEPSEEK_API_KEY"] + # Provider-specific request fields are opt-in, never inferred from the + # model name. For DeepSeek reasoning models, enable the thinking channel: + env.setdefault("SKILLOPT_SLEEP_CHAT_EXTRA_BODY", + json.dumps({"thinking": {"type": "enabled"}})) + env.setdefault("SKILLOPT_SLEEP_COMPAT_MAX_TOKENS", "8192") else: # OPTIONAL, UNVERIFIED fallback: route the `claude` CLI backend through a # local Anthropic-compatible proxy (e.g. LiteLLM) to reach Gemini. There @@ -93,7 +95,10 @@ def main() -> None: "--model", PROVIDER_MODEL, "--project", PROJECT_DIR] print(f"Running: {' '.join(args)}") - subprocess.run(args, env=env, check=False) + # Propagate the child's exit code so supervisors (watchdog.py, systemd, + # Task Scheduler) see a failed sleep run as a failure, not a success. + proc = subprocess.run(args, env=env, check=False) + sys.exit(proc.returncode) if __name__ == "__main__": diff --git a/docs/sleep/openai-compatible-endpoints.md b/docs/sleep/openai-compatible-endpoints.md index cdcfe825..831a2ed5 100644 --- a/docs/sleep/openai-compatible-endpoints.md +++ b/docs/sleep/openai-compatible-endpoints.md @@ -1,6 +1,6 @@ # OpenAI-compatible endpoints for SkillOpt-Sleep (DeepSeek, local vLLM, …) -This document describes a small enhancement to the `azure_openai` backend in +This document describes an enhancement to the `azure_openai` backend in `skillopt_sleep/backend.py` that lets SkillOpt-Sleep drive **any OpenAI-compatible chat-completions endpoint** — for example DeepSeek's hosted API or a self-hosted vLLM/Ollama server — in addition to native Azure OpenAI @@ -9,48 +9,69 @@ nightly sleep cycle inside the Antigravity IDE against DeepSeek. ## What changed -Three focused changes in `AzureOpenAIBackend` (`skillopt_sleep/backend.py`), -all backward-compatible — the default managed-identity path is unchanged: +All changes are backward-compatible — the default managed-identity Azure path +is unchanged: -1. **Endpoint resolution honors `AZURE_OPENAI_ENDPOINT`.** - `__init__` now resolves the endpoint as `explicit arg` → `AZURE_OPENAI_ENDPOINT` - env → the built-in `_AZURE_ENDPOINTS` table. Previously a non-Azure endpoint - could not be supplied at all, so calls always went to a hardcoded - `*.openai.azure.com` host. +1. **CLI acceptance.** `skillopt-sleep run --backend azure_openai` is now an + accepted choice in `skillopt_sleep/__main__.py` (it was previously rejected + by argparse even though `get_backend()` understood the name). -2. **`openai_compatible` auth mode.** When +2. **Endpoint resolution honors `AZURE_OPENAI_ENDPOINT`.** + `AzureOpenAIBackend.__init__` resolves the endpoint as `explicit arg` → + `AZURE_OPENAI_ENDPOINT` env → the built-in `_AZURE_ENDPOINTS` table. + Previously a non-Azure endpoint could not be supplied at all. + +3. **`openai_compatible` auth mode.** When `AZURE_OPENAI_AUTH_MODE=openai_compatible` (also accepts `compat`/`openai`), - `_get_client()` builds a plain `openai.OpenAI(base_url=…)` client instead of - an `AzureOpenAI` client. This mirrors the auth mode already supported by the - sibling `skillopt/model/azure_openai.py` module, so the two are consistent. - -3. **DeepSeek request shape + error surfacing.** `_call()` sends `max_tokens` - plus `extra_body={"thinking": {"type": "enabled"}}` for `deepseek*` - deployments (the DeepSeek reasoning models expect this), and records the last - exception in `self.last_call_error` so a failed night is diagnosable instead - of collapsing to a silent empty response scored `0.0`. - -### Why the previous behavior failed on non-Azure servers - -The `AzureOpenAI` SDK client rewrites request URLs with Azure-only structure — -a `?api-version=…` query string and deployment path segments. A non-Azure -OpenAI-compatible server (DeepSeek, vLLM, …) does not recognize those routes and -responds `404 Resource not found`. SkillOpt-Sleep then receives an empty -response, the judge scores it `0.0`, and every night reports -`baseline 0.0 -> candidate 0.0` with no usable diagnostic. Selecting a plain -`OpenAI` client via `openai_compatible` mode avoids the Azure URL rewriting and -talks to the endpoint directly. + `_get_client()` builds a plain `openai.OpenAI(base_url=…)` client with + `AZURE_OPENAI_API_KEY` instead of an `AzureOpenAI` client. This mirrors the + auth mode already supported by the sibling `skillopt/model/azure_openai.py` + module. (The `AzureOpenAI` client rewrites request URLs with Azure-only + `?api-version=…` query params and deployment path segments, which non-Azure + servers reject with `404 Resource not found` — the sleep cycle then scores + every rollout `0.0` with no diagnostic.) + +4. **Managed-identity credential guard.** The managed-identity path attaches an + Azure AD bearer token to every request. If a custom endpoint outside + `*.openai.azure.com` / `*.cognitiveservices.azure.com` is configured without + explicit compat auth, the backend now raises a clear `ValueError` instead of + sending Azure credentials to an arbitrary host. + +5. **Provider-neutral request shape.** In compat mode the backend sends only the + standard OpenAI-compatible contract (`model`, `messages`, `max_tokens`). + Provider-specific request fields are **opt-in** via environment variables + (below) — nothing is inferred from model-name substrings. + +6. **Reliable error state.** `_call()` records the last exception in + `self.last_call_error` (surfaced in `diagnostics.json`), clears it when a + retry recovers, and sets an explicit `"empty response on all N attempts"` + diagnostic when every attempt returns empty text. + +## Configuration reference + +SkillOpt-Sleep's `azure_openai` backend reads these environment variables +(unprefixed only — the `OPTIMIZER_*`/`TARGET_*` dual-role variables belong to +the separate `skillopt.model.azure_openai` module and are **not** used by the +sleep cycle): + +| Variable | Meaning | +|---|---| +| `AZURE_OPENAI_AUTH_MODE` | `openai_compatible` (or `compat`/`openai`) selects the plain OpenAI client. Unset/other = Azure managed identity (default). | +| `AZURE_OPENAI_ENDPOINT` | Base URL of the server, e.g. `https://api.deepseek.com`. | +| `AZURE_OPENAI_API_KEY` | API key sent by the compat client. | +| `SKILLOPT_SLEEP_COMPAT_MAX_TOKENS` | Optional int (default `8192`): `max_tokens` sent in compat mode. | +| `SKILLOPT_SLEEP_CHAT_EXTRA_BODY` | Optional JSON object passed as `extra_body` for provider-specific fields. | ## How to use it -Set four environment variables and run the cycle with `--backend azure_openai`: - ```bash export AZURE_OPENAI_AUTH_MODE=openai_compatible export AZURE_OPENAI_ENDPOINT=https://api.deepseek.com # no /v1, no trailing path export AZURE_OPENAI_API_KEY=sk-... # your provider key -# optimizer/target dual-role overrides are also honored, e.g. -# OPTIMIZER_AZURE_OPENAI_AUTH_MODE / TARGET_AZURE_OPENAI_AUTH_MODE, etc. + +# DeepSeek reasoning models: enable the thinking channel (opt-in, not inferred) +export SKILLOPT_SLEEP_CHAT_EXTRA_BODY='{"thinking": {"type": "enabled"}}' +export SKILLOPT_SLEEP_COMPAT_MAX_TOKENS=8192 skillopt-sleep run \ --backend azure_openai \ @@ -59,7 +80,9 @@ skillopt-sleep run \ ``` The same pattern works for any OpenAI-compatible server — point -`AZURE_OPENAI_ENDPOINT` at it and set a matching `--model`. +`AZURE_OPENAI_ENDPOINT` at it, set a matching `--model`, and omit +`SKILLOPT_SLEEP_CHAT_EXTRA_BODY` unless your provider needs extra request +fields. ## End-to-end integration: Antigravity + DeepSeek @@ -68,14 +91,15 @@ was wired into the [Antigravity](https://antigravity.google/) agent IDE so the sleep cycle runs unattended: - **`examples/runner.py`** — a thin launcher that loads a provider key from an - `.env` file, exports the four variables above, and invokes `skillopt-sleep run` - with the DeepSeek backend. It also implements a `session-end` hook that appends - task-outcome metadata to a rollout-evidence log (wired to Antigravity's `Stop` - hook) so future nights have richer sessions to mine. + `.env` file, exports the variables above, invokes `skillopt-sleep run` with + the DeepSeek backend, and **exits with the child's return code** so + supervisors see failures as failures. It also implements a `session-end` hook + that appends task-outcome metadata to a rollout-evidence log (wired to + Antigravity's `Stop` hook) so future nights have richer sessions to mine. - **`examples/watchdog.py`** — a minimal supervisor loop that invokes the runner - on a fixed interval (e.g. every 4 hours). On Windows this is registered as a - Scheduled Task so it survives logout; on Linux/macOS a `systemd` timer or cron - entry serves the same role. + on a fixed interval (e.g. every 4 hours) and logs non-zero exits as failures. + On Windows this is registered as a Scheduled Task so it survives logout; on + Linux/macOS a `systemd` timer or cron entry serves the same role. ### Verified result @@ -95,6 +119,11 @@ On a Windows 11 host, driving the cycle against `deepseek-v4-pro` in **rejected** a non-improving proposal (`0.3 → 0.3`), confirming the validation gate behaves normally on the new backend. +Deterministic no-network coverage for the new behavior lives in +`tests/test_azure_openai_compat.py` (CLI acceptance, client selection, +endpoint/auth guard, request kwargs, retry error-state, empty-response +diagnostics, and runner exit-code propagation). + ## A note on Gemini (optional, unverified fallback) `examples/runner.py` also contains a fallback branch that, when only a Gemini key diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index 01293e1a..6ba5f084 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -70,7 +70,8 @@ def _add_common(p: argparse.ArgumentParser) -> None: p.add_argument("--project", default="") p.add_argument("--scope", default="", choices=["", "all", "invoked"]) p.add_argument("--backend", default="", - choices=["", "mock", "claude", "codex", "copilot", "handoff"]) + choices=["", "mock", "claude", "codex", "copilot", "handoff", + "azure_openai"]) p.add_argument("--model", default="") p.add_argument("--codex-path", default="", help="path to the real @openai/codex binary") p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)") diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index 77fcae8a..768a4e86 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -1222,16 +1222,33 @@ def tokens_used(self): class AzureOpenAIBackend(CliBackend): - """Drives Azure OpenAI gpt-5.x deployments via managed identity. + """Drives Azure OpenAI gpt-5.x deployments via managed identity, or any + OpenAI-compatible chat-completions server via explicit compat auth. Mirrors the intern's blog_1 setup (avail_api.md): managed-identity auth, the same endpoints/deployments. Reuses CliBackend's attempt/judge/reflect prompts and JSON parsing; only _call() differs. openai + azure-identity are lazy imported so the mock/CLI paths stay dependency-free. + + OpenAI-compatible mode (opt-in, matching skillopt.model.azure_openai): + * AZURE_OPENAI_AUTH_MODE=openai_compatible selects a plain ``openai.OpenAI`` + client with AZURE_OPENAI_API_KEY against AZURE_OPENAI_ENDPOINT. + * SKILLOPT_SLEEP_COMPAT_MAX_TOKENS (int, default 8192) caps completion + length via the standard ``max_tokens`` parameter in compat mode. + * SKILLOPT_SLEEP_CHAT_EXTRA_BODY (JSON object) is passed as ``extra_body`` + for provider-specific request fields (e.g. DeepSeek's + ``{"thinking": {"type": "enabled"}}``). Nothing provider-specific is + inferred from model names. """ name = "azure" + _COMPAT_MODES = {"openai_compatible", "compat", "openai"} + # Hosts the managed-identity (AAD bearer token) path may talk to. A custom + # endpoint outside these domains must use explicit compat auth — we never + # send Azure credentials to an arbitrary host. + _AZURE_HOST_SUFFIXES = (".openai.azure.com", ".cognitiveservices.azure.com") + def __init__(self, deployment: str = "", endpoint: str = "", timeout: int = 180, api_version: str = "2024-12-01-preview") -> None: super().__init__(model=deployment or "gpt-5.5", timeout=timeout) @@ -1248,6 +1265,9 @@ def __init__(self, deployment: str = "", endpoint: str = "", timeout: int = 180, self.api_version = api_version self.name = f"azure:{self.deployment}" self._client = None + # Opt-in request knobs (read once; see class docstring). + self.compat_max_tokens = self._read_compat_max_tokens() + self.chat_extra_body = self._read_chat_extra_body() @staticmethod def _endpoint_for(deployment: str) -> str: @@ -1256,6 +1276,44 @@ def _endpoint_for(deployment: str) -> str: return ep return "https://oaidr9.openai.azure.com/" + @classmethod + def _compat_mode(cls) -> bool: + return os.environ.get("AZURE_OPENAI_AUTH_MODE", "").strip().lower() in cls._COMPAT_MODES + + @staticmethod + def _read_compat_max_tokens() -> int: + raw = os.environ.get("SKILLOPT_SLEEP_COMPAT_MAX_TOKENS", "").strip() + try: + val = int(raw) if raw else 8192 + return val if val > 0 else 8192 + except ValueError: + import logging + logging.getLogger("skillopt_sleep").warning( + "ignoring non-integer SKILLOPT_SLEEP_COMPAT_MAX_TOKENS=%r", raw) + return 8192 + + @staticmethod + def _read_chat_extra_body() -> Optional[Dict[str, Any]]: + raw = os.environ.get("SKILLOPT_SLEEP_CHAT_EXTRA_BODY", "").strip() + if not raw: + return None + try: + parsed = json.loads(raw) + except ValueError: + parsed = None + if isinstance(parsed, dict) and parsed: + return parsed + import logging + logging.getLogger("skillopt_sleep").warning( + "ignoring SKILLOPT_SLEEP_CHAT_EXTRA_BODY: not a non-empty JSON object: %r", + raw[:100]) + return None + + def _is_azure_host(self) -> bool: + from urllib.parse import urlparse + host = (urlparse(self.endpoint).hostname or "").lower() + return host.endswith(self._AZURE_HOST_SUFFIXES) + def _get_client(self): if self._client is None: # AZURE_OPENAI_AUTH_MODE=openai_compatible (matching the sibling @@ -1264,8 +1322,7 @@ def _get_client(self): # endpoint work: the AzureOpenAI client would otherwise rewrite the # URL with Azure-only query params (?api-version=...) and deployment # path segments, which non-Azure servers reject with a 404. - auth_mode = os.environ.get("AZURE_OPENAI_AUTH_MODE", "").strip().lower() - if auth_mode in {"openai_compatible", "compat", "openai"}: + if self._compat_mode(): from openai import OpenAI self._client = OpenAI( base_url=self.endpoint.rstrip("/"), @@ -1273,6 +1330,17 @@ def _get_client(self): max_retries=4, ) else: + # Security guard: the managed-identity path attaches an Azure AD + # bearer token to every request. Refuse to do that for a custom + # non-Azure endpoint — leaking a live AAD token to an arbitrary + # host must be impossible by (mis)configuration. + if not self._is_azure_host(): + raise ValueError( + "azure_openai backend: refusing to send Azure managed-identity " + f"credentials to non-Azure endpoint {self.endpoint!r}. For " + "OpenAI-compatible servers set AZURE_OPENAI_AUTH_MODE=" + "openai_compatible and AZURE_OPENAI_API_KEY." + ) from azure.identity import ManagedIdentityCredential, get_bearer_token_provider from openai import AzureOpenAI cred = ManagedIdentityCredential(client_id=_AZURE_MI_CLIENT_ID) @@ -1298,20 +1366,25 @@ def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str client = self._get_client() last_exc = None - for attempt in range(max(1, retries)): + n_attempts = max(1, retries) + for attempt in range(n_attempts): try: kwargs: Dict[str, Any] = { "model": self.deployment, "messages": [{"role": "user", "content": prompt}], } - # DeepSeek's reasoning models expect `max_tokens` and an - # extra_body flag to enable the thinking channel; the Azure - # gpt-5.x deployments use `max_completion_tokens`. - if "deepseek" in self.deployment.lower(): - kwargs["max_tokens"] = 8192 - kwargs["extra_body"] = {"thinking": {"type": "enabled"}} + # Provider-neutral request shape: compat mode speaks the + # standard OpenAI-compatible contract (`max_tokens`); the Azure + # gpt-5.x deployments require `max_completion_tokens`. Any + # provider-specific body fields are opt-in via + # SKILLOPT_SLEEP_CHAT_EXTRA_BODY (see class docstring) — never + # inferred from the model name. + if self._compat_mode(): + kwargs["max_tokens"] = self.compat_max_tokens else: kwargs["max_completion_tokens"] = 16384 + if self.chat_extra_body: + kwargs["extra_body"] = self.chat_extra_body resp = client.chat.completions.create(**kwargs) text = (resp.choices[0].message.content or "").strip() try: @@ -1320,6 +1393,9 @@ def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str except Exception: pass if text: + # A recovered retry must not leave a stale error behind: + # last_call_error always reflects the LATEST outcome. + self.last_call_error = "" return text # empty but no exception: model genuinely returned nothing — one # quick retry can help (reasoning models occasionally yield empty) @@ -1330,8 +1406,15 @@ def _call(self, prompt: str, *, max_tokens: int = 1024, retries: int = 5) -> str # from a mis-pointed endpoint) instead of a silent empty->0. self.last_call_error = str(e)[:500] # backoff before next try (skip after the final attempt) - if attempt < retries - 1: + if attempt < n_attempts - 1: _t.sleep(min(8.0, (2 ** attempt) * 0.5) + _r.random() * 0.4) + if last_exc == "empty-response": + # All attempts "succeeded" HTTP-wise but carried no text — say so, + # otherwise a run of empty completions is indistinguishable from a + # never-called backend in diagnostics. + self.last_call_error = ( + f"{self.deployment}: empty response on all {n_attempts} attempts" + ) return "" diff --git a/tests/test_azure_openai_compat.py b/tests/test_azure_openai_compat.py new file mode 100644 index 00000000..bc4e6101 --- /dev/null +++ b/tests/test_azure_openai_compat.py @@ -0,0 +1,248 @@ +"""Tests for the azure_openai backend's OpenAI-compatible mode. + +Pure-stdlib (unittest), deterministic, NO network, no API key. Covers the +integration points requested in review of the openai-compatible feature: + * CLI acceptance of --backend azure_openai + * compat-vs-Azure client selection + * endpoint resolution and the managed-identity credential guard + * provider-neutral request kwargs (opt-in extra body / token cap) + * retry-success error clearing + empty-response diagnostics + * example runner exit-code propagation + +Run: python -m unittest tests.test_azure_openai_compat +""" +from __future__ import annotations + +import argparse +import contextlib +import importlib.util +import io +import json +import os +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +from skillopt_sleep.__main__ import _add_common +from skillopt_sleep.backend import AzureOpenAIBackend + +try: + import openai # noqa: F401 + HAVE_OPENAI = True +except ImportError: + HAVE_OPENAI = False + +COMPAT_ENV = { + "AZURE_OPENAI_AUTH_MODE": "openai_compatible", + "AZURE_OPENAI_ENDPOINT": "https://compat.example.test", + "AZURE_OPENAI_API_KEY": "sk-test-not-a-real-key", +} + + +def _resp(text): + return SimpleNamespace( + choices=[SimpleNamespace(message=SimpleNamespace(content=text))], + usage=None, + ) + + +class _FakeCompletions: + """Scripted chat.completions.create: replies are strings or Exceptions.""" + + def __init__(self, replies): + self.replies = list(replies) + self.calls = [] + + def create(self, **kwargs): + self.calls.append(kwargs) + item = self.replies.pop(0) + if isinstance(item, Exception): + raise item + return _resp(item) + + +def _fake_client(replies): + return SimpleNamespace(chat=SimpleNamespace(completions=_FakeCompletions(replies))) + + +def _backend_with(replies, env): + """Build a backend under `env` with a scripted fake client injected.""" + with mock.patch.dict(os.environ, env, clear=True): + be = AzureOpenAIBackend(deployment="any-model") + be._client = _fake_client(replies) + return be + + +class TestCliAcceptance(unittest.TestCase): + def test_backend_azure_openai_is_accepted(self): + p = argparse.ArgumentParser() + _add_common(p) + ns = p.parse_args(["--backend", "azure_openai"]) + self.assertEqual(ns.backend, "azure_openai") + + +class TestClientSelection(unittest.TestCase): + @unittest.skipUnless(HAVE_OPENAI, "openai package not installed") + def test_compat_mode_builds_plain_openai_client(self): + from openai import AzureOpenAI, OpenAI + # Overlay (clear=False): constructing a real OpenAI client builds an + # SSL context, which needs the ambient cert-related env vars intact. + with mock.patch.dict(os.environ, COMPAT_ENV, clear=False): + be = AzureOpenAIBackend(deployment="some-model", endpoint=COMPAT_ENV["AZURE_OPENAI_ENDPOINT"]) + client = be._get_client() + self.assertIsInstance(client, OpenAI) + self.assertNotIsInstance(client, AzureOpenAI) + self.assertIn("compat.example.test", str(client.base_url)) + + def test_managed_identity_refuses_non_azure_endpoint(self): + # No compat auth mode + a custom non-Azure endpoint: the backend must + # raise instead of sending an Azure AD bearer token to that host. The + # guard fires before azure.identity is imported, so this test needs + # neither azure-identity nor any network. + env = {"AZURE_OPENAI_ENDPOINT": "https://api.deepseek.com"} + with mock.patch.dict(os.environ, env, clear=True): + be = AzureOpenAIBackend(deployment="some-model") + with self.assertRaises(ValueError) as ctx: + be._get_client() + self.assertIn("openai_compatible", str(ctx.exception)) + + def test_azure_host_detection(self): + with mock.patch.dict(os.environ, {}, clear=True): + be = AzureOpenAIBackend(deployment="gpt-5.5") # table endpoint + self.assertTrue(be._is_azure_host()) + be2 = AzureOpenAIBackend( + deployment="gpt-5.5", endpoint="https://api.deepseek.com") + self.assertFalse(be2._is_azure_host()) + + +class TestEndpointResolution(unittest.TestCase): + def test_env_endpoint_is_honored(self): + env = {"AZURE_OPENAI_ENDPOINT": "https://compat.example.test"} + with mock.patch.dict(os.environ, env, clear=True): + be = AzureOpenAIBackend(deployment="some-model") + self.assertEqual(be.endpoint, "https://compat.example.test") + + def test_explicit_arg_beats_env(self): + env = {"AZURE_OPENAI_ENDPOINT": "https://env.example.test"} + with mock.patch.dict(os.environ, env, clear=True): + be = AzureOpenAIBackend(deployment="m", endpoint="https://arg.example.test") + self.assertEqual(be.endpoint, "https://arg.example.test") + + def test_table_fallback_without_env(self): + with mock.patch.dict(os.environ, {}, clear=True): + be = AzureOpenAIBackend(deployment="gpt-5.5") + self.assertTrue(be.endpoint.endswith(".openai.azure.com/")) + + +class TestRequestKwargs(unittest.TestCase): + def test_compat_mode_sends_standard_max_tokens(self): + be = _backend_with(["hi"], COMPAT_ENV) + with mock.patch.dict(os.environ, COMPAT_ENV, clear=True): + out = be._call("p", retries=1) + self.assertEqual(out, "hi") + (call,) = be._client.chat.completions.calls + self.assertEqual(call["max_tokens"], 8192) + self.assertNotIn("max_completion_tokens", call) + self.assertNotIn("extra_body", call) + + def test_compat_max_tokens_env_override(self): + env = dict(COMPAT_ENV, SKILLOPT_SLEEP_COMPAT_MAX_TOKENS="4096") + be = _backend_with(["hi"], env) + with mock.patch.dict(os.environ, env, clear=True): + be._call("p", retries=1) + (call,) = be._client.chat.completions.calls + self.assertEqual(call["max_tokens"], 4096) + + def test_extra_body_is_opt_in_via_env_not_model_name(self): + # A deepseek-named model WITHOUT the env knob gets a pure standard + # request (nothing inferred from the name)... + with mock.patch.dict(os.environ, COMPAT_ENV, clear=True): + be = AzureOpenAIBackend(deployment="deepseek-v4-pro") + be._client = _fake_client(["hi"]) + be._call("p", retries=1) + (call,) = be._client.chat.completions.calls + self.assertNotIn("extra_body", call) + # ...and any model WITH the env knob gets exactly the configured body. + body = {"thinking": {"type": "enabled"}} + env = dict(COMPAT_ENV, SKILLOPT_SLEEP_CHAT_EXTRA_BODY=json.dumps(body)) + be2 = _backend_with(["hi"], env) + with mock.patch.dict(os.environ, env, clear=True): + be2._call("p", retries=1) + (call2,) = be2._client.chat.completions.calls + self.assertEqual(call2["extra_body"], body) + + def test_malformed_extra_body_is_ignored(self): + env = dict(COMPAT_ENV, SKILLOPT_SLEEP_CHAT_EXTRA_BODY="{not json") + be = _backend_with(["hi"], env) + with mock.patch.dict(os.environ, env, clear=True): + be._call("p", retries=1) + (call,) = be._client.chat.completions.calls + self.assertNotIn("extra_body", call) + + def test_azure_mode_sends_max_completion_tokens(self): + be = _backend_with(["hi"], {}) # no compat auth mode + with mock.patch.dict(os.environ, {}, clear=True): + be._call("p", retries=1) + (call,) = be._client.chat.completions.calls + self.assertEqual(call["max_completion_tokens"], 16384) + self.assertNotIn("max_tokens", call) + + +class TestErrorState(unittest.TestCase): + def test_recovered_retry_clears_last_call_error(self): + be = _backend_with([RuntimeError("transient boom"), "recovered"], COMPAT_ENV) + with mock.patch.dict(os.environ, COMPAT_ENV, clear=True), \ + mock.patch("time.sleep"): + out = be._call("p", retries=2) + self.assertEqual(out, "recovered") + self.assertEqual(be.last_call_error, "") + + def test_all_empty_responses_set_diagnostic(self): + be = _backend_with(["", ""], COMPAT_ENV) + with mock.patch.dict(os.environ, COMPAT_ENV, clear=True), \ + mock.patch("time.sleep"): + out = be._call("p", retries=2) + self.assertEqual(out, "") + self.assertIn("empty response on all 2 attempts", be.last_call_error) + + def test_persistent_exception_is_surfaced(self): + be = _backend_with([RuntimeError("boom-1"), RuntimeError("boom-2")], COMPAT_ENV) + with mock.patch.dict(os.environ, COMPAT_ENV, clear=True), \ + mock.patch("time.sleep"): + out = be._call("p", retries=2) + self.assertEqual(out, "") + self.assertIn("boom-2", be.last_call_error) + + +class TestRunnerExitCode(unittest.TestCase): + RUNNER = Path(__file__).resolve().parent.parent / "docs" / "sleep" / "examples" / "runner.py" + + def _load_runner(self): + spec = importlib.util.spec_from_file_location("example_runner", self.RUNNER) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + def _run_with_child_rc(self, rc): + env = {"DEEPSEEK_API_KEY": "sk-test-not-a-real-key"} + with mock.patch.dict(os.environ, env, clear=True): + mod = self._load_runner() + fake = SimpleNamespace(run=lambda *a, **k: SimpleNamespace(returncode=rc)) + with mock.patch.object(mod, "subprocess", fake), \ + mock.patch.object(sys, "argv", ["runner.py", "run"]), \ + contextlib.redirect_stdout(io.StringIO()): + with self.assertRaises(SystemExit) as ctx: + mod.main() + return ctx.exception.code + + def test_child_failure_propagates(self): + self.assertEqual(self._run_with_child_rc(7), 7) + + def test_child_success_exits_zero(self): + self.assertEqual(self._run_with_child_rc(0), 0) + + +if __name__ == "__main__": + unittest.main()