From df89e7047f25b4475a6367b38ec54fbe642bc064 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:31:36 +0900 Subject: [PATCH 1/9] test(opencode): require bounded gateway failure telemetry --- tests/test_opencode_model_pool_runner.py | 130 +++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 2965d4c55c..dfcfd226b7 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -408,6 +408,136 @@ def test_failed_provider_without_reason_logs_explicit_absence(tmp_path: Path) -> ) in result.stdout +def gateway_failure_event(detail: dict[str, object]) -> str: + """Return one production-shaped OpenCode event containing a gateway error.""" + response_body = json.dumps({"error": {"detail": detail}}) + return json.dumps( + { + "type": "error", + "error": { + "name": "ProviderError", + "data": { + "message": "provider-controlled message must stay suppressed", + "responseBody": response_body, + }, + }, + } + ) + + +@pytest.mark.parametrize( + ("detail", "expected_telemetry"), + [ + ( + { + "model": "meta-llama/llama-3.3-70b-instruct", + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [ + { + "provider_name": "openrouter", + "phase": "response_error", + "provider_status": 429, + "error_code": "rate_limited", + } + ], + }, + "phase=response_error reason=rate_limited provider=openrouter " + "status=429 duration=0s " + "served_model=meta-llama/llama-3.3-70b-instruct", + ), + ( + { + "model": "deepseek-ai/deepseek-v4-pro-0813", + "attempts": [ + { + "provider_name": "nvidia_nim", + "phase": "connecting", + "provider_status": 502, + "error_code": "provider_transport", + } + ], + }, + "phase=connecting reason=provider_transport provider=nvidia_nim " + "status=502 duration=0s " + "served_model=deepseek-ai/deepseek-v4-pro-0813", + ), + ( + { + "attempts": [ + { + "provider_name": "openrouter", + "phase": "response_error", + "provider_status": 413, + "error_code": "request_too_large", + } + ], + }, + "phase=response_error reason=request_too_large provider=openrouter " + "status=413 duration=0s served_model=unknown", + ), + ( + { + "attempts": [ + { + "provider_name": "contextual-orchestrator", + "phase": "queue_admission", + "provider_status": 503, + "error_code": "queue_admission_failed", + } + ], + }, + "phase=queue_admission reason=queue_admission_failed " + "provider=contextual-orchestrator status=503 duration=0s " + "served_model=unknown", + ), + ( + { + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [ + { + "provider_name": "bytez", + "phase": "validating", + "provider_status": 500, + "error_code": "malformed_model_output", + } + ], + }, + "phase=validating reason=malformed_model_output provider=bytez " + "status=500 duration=0s served_model=unknown", + ), + ], +) +def test_gateway_failure_logs_allowlisted_root_cause_fields( + tmp_path: Path, + detail: dict[str, object], + expected_telemetry: str, +) -> None: + """Gateway errors retain bounded routing evidence without raw response text.""" + result = run_failed_model(tmp_path, json_line=gateway_failure_event(detail)) + + assert result.returncode == 1 + assert expected_telemetry in result.stdout + assert "provider-controlled message must stay suppressed" not in result.stdout + + +def test_malformed_gateway_failure_logs_only_bounded_decode_state( + tmp_path: Path, +) -> None: + """Malformed provider JSON reports a stable decode state without echoing bytes.""" + secret = "sk" + "-malformed-never-print" + result = run_failed_model( + tmp_path, + json_line=f'{{"type":"error","secret":"{secret}"', + ) + + assert result.returncode == 1 + assert ( + "phase=decode_error reason=malformed_gateway_envelope provider=unknown " + "status=unknown duration=0s served_model=unknown" + ) in result.stdout + assert secret not in result.stdout + + def test_backoff_environment_rejects_recursive_arithmetic_injection( tmp_path: Path, ) -> None: From 7abbb6a4fd5403aeb6c55fb97f649488bcd9678b Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:31:48 +0900 Subject: [PATCH 2/9] test(opencode): allow measured failure duration --- tests/test_opencode_model_pool_runner.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index dfcfd226b7..b1f2cfacd6 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -516,7 +516,10 @@ def test_gateway_failure_logs_allowlisted_root_cause_fields( result = run_failed_model(tmp_path, json_line=gateway_failure_event(detail)) assert result.returncode == 1 - assert expected_telemetry in result.stdout + telemetry_pattern = re.escape(expected_telemetry).replace( + "duration=0s", r"duration=\d+s" + ) + assert re.search(telemetry_pattern, result.stdout) assert "provider-controlled message must stay suppressed" not in result.stdout @@ -531,10 +534,11 @@ def test_malformed_gateway_failure_logs_only_bounded_decode_state( ) assert result.returncode == 1 - assert ( - "phase=decode_error reason=malformed_gateway_envelope provider=unknown " - "status=unknown duration=0s served_model=unknown" - ) in result.stdout + assert re.search( + r"phase=decode_error reason=malformed_gateway_envelope provider=unknown " + r"status=unknown duration=\d+s served_model=unknown", + result.stdout, + ) assert secret not in result.stdout From f8554c8bca3e61d2c44e2c0fb110de32c3a450a1 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:32:53 +0900 Subject: [PATCH 3/9] fix(opencode): retain safe provider failure evidence --- scripts/ci/run_opencode_review_model_pool.sh | 62 +++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 80f57d1d43..b24614c96a 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -267,7 +267,8 @@ is_credit_exhausted_failure() { emit_sanitized_opencode_failure_detail() { local opencode_json_file="$1" local opencode_stderr_file="$2" - local json_bytes stderr_bytes failure_class + local attempt_duration_seconds="${3:-0}" + local json_bytes stderr_bytes failure_class gateway_telemetry json_bytes=0 stderr_bytes=0 @@ -300,6 +301,60 @@ emit_sanitized_opencode_failure_detail() { fi printf 'OpenCode provider failure metadata: class=%s json-bytes=%s stderr-bytes=%s; provider-controlled content suppressed.\n' \ "$failure_class" "$json_bytes" "$stderr_bytes" + + gateway_telemetry="$( + jq -Rrs --arg duration "${attempt_duration_seconds}s" ' + def safe_value($fallback): + if type == "string" and length > 0 and length <= 128 and + test("^[A-Za-z0-9._:/+-]+$") + then . else $fallback end; + def safe_phase: + if . == "connecting" or . == "requesting" or . == "reading" or + . == "decoding" or . == "validating" or + . == "response_error" or . == "queue_admission" + then . else "unknown" end; + def safe_reason: + if . == "rate_limited" or . == "provider_transport" or + . == "request_too_large" or . == "queue_admission_failed" or + . == "malformed_model_output" or + . == "eligible_candidates_exhausted" or + . == "discovery_failure" or . == "model_unavailable" or + . == "quota_exhausted" or . == "authentication_failed" + then . else "unknown" end; + [ + splits("\\n") | fromjson? | + select(.type == "error") | + (.error.data.responseBody? // .error.data.response_body? // empty) | + if type == "string" then fromjson? else . end | + select(type == "object") | + .error.detail? | + select(type == "object") + ] | last // empty | + . as $detail | + (if ($detail.attempts | type) == "array" and + ($detail.attempts | length) > 0 and + ($detail.attempts | length) <= 64 and + ($detail.attempts[-1] | type) == "object" + then $detail.attempts[-1] else {} end) as $attempt | + [ + "phase=" + (($attempt.phase // "unknown") | safe_value("unknown") | safe_phase), + "reason=" + (($attempt.error_code // $detail.terminal_reason // "unknown") | safe_value("unknown") | safe_reason), + "provider=" + (($attempt.provider_name // "unknown") | safe_value("unknown")), + "status=" + (if ($attempt.provider_status | type) == "number" and + $attempt.provider_status >= 100 and $attempt.provider_status <= 599 and + ($attempt.provider_status | floor) == $attempt.provider_status + then ($attempt.provider_status | tostring) else "unknown" end), + "duration=" + $duration, + "served_model=" + (($detail.model // "unknown") | safe_value("unknown")) + ] | join(" ") + ' "$opencode_json_file" 2>/dev/null || true + )" + if [ -n "$gateway_telemetry" ]; then + printf 'OpenCode gateway failure telemetry: %s; provider-controlled content suppressed.\n' "$gateway_telemetry" + elif [ "$json_bytes" -gt 0 ]; then + printf 'OpenCode gateway failure telemetry: phase=decode_error reason=malformed_gateway_envelope provider=unknown status=unknown duration=%ss served_model=unknown; provider-controlled content suppressed.\n' \ + "$attempt_duration_seconds" + fi } emit_rejected_opencode_artifact_metadata() { @@ -398,6 +453,7 @@ run_one_model_attempt() { local opencode_export_file="$8" local export_timeout_seconds opencode_status session_id opencode_stderr_file local opencode_pid fatal_kill_grace_seconds fatal_poll_seconds + local attempt_started_seconds attempt_duration_seconds export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" fatal_poll_seconds="${OPENCODE_FATAL_ERROR_POLL_SECONDS:-5}" @@ -405,6 +461,7 @@ run_one_model_attempt() { opencode_stderr_file="${opencode_json_file}.stderr" rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" + attempt_started_seconds="$SECONDS" set +e env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ @@ -437,10 +494,11 @@ run_one_model_attempt() { done wait "$opencode_pid" opencode_status=$? + attempt_duration_seconds=$((SECONDS - attempt_started_seconds)) set -e if [ "$opencode_status" -ne 0 ]; then printf 'OpenCode %s attempt %s/%s failed with exit %s.\n' "$model_candidate" "$attempt" "$attempts" "$opencode_status" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" "$attempt_duration_seconds" if is_fatal_provider_failure "$opencode_json_file"; then printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 From 5cec2195571eaeee075a919df9a18a5d66593d56 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:32:54 +0900 Subject: [PATCH 4/9] docs(opencode): record failure provenance boundary --- ...912-opencode-provider-failure-telemetry.md | 1 + ...ntextual-orchestrator-vendored-free-zdr.md | 12 ++++++++++ docs/product-technical-gap-baseline.md | 22 +++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md diff --git a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md new file mode 100644 index 0000000000..3051ba0a51 --- /dev/null +++ b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md @@ -0,0 +1 @@ +Preserve bounded phase, reason, provider, HTTP status, duration, and served-model evidence for OpenCode gateway failures while suppressing raw provider content and credentials. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 9b0749f258..f65a380655 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -282,3 +282,15 @@ all five, and auto-optimize routing by cost. per-agent attempt; it changes only *which* agent gets tried next, never any per-attempt timeout, consistent with the 2026-08-31 amendment above. No other contextual-orchestrator behavior changes with this pin advance. + +- **2026-09-12 proposed amendment: preserve redaction-safe OpenCode failure + provenance.** The OpenCode model-pool adapter must keep the gateway-owned + canonical `error.detail` receipt useful after suppressing raw provider + content. For a bounded structured error it emits only allowlisted phase, + normalized reason, provider identifier, HTTP status, caller-measured + duration, and served-model identifier. Unknown, malformed, and absent fields + become fixed `unknown`/`malformed_gateway_envelope` values; arbitrary + messages, response bodies, headers, credentials, and unbounded identifiers + never reach public Actions logs. This does not add a retry, timeout, provider + choice, or model policy to `.github`; contextual-orchestrator remains the + owner of discovery, routing, and failover. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..f81f4f60b1 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3353,3 +3353,25 @@ queries the check-runs API at its own time, order-independently. The implementin their change was safe because they had scoped it narrowly, not because they had checked for the name collision — which is the more useful lesson: **a job name is unique only within one workflow file, and the same name in another file can carry the opposite safety property.** + +## 2026-09-12 OpenCode gateway failure provenance — Proposed + +**Observed exact evidence.** `ContextualWisdomLab/.github#2106` OpenCode run +[`34693400612`](https://github.com/ContextualWisdomLab/.github/actions/runs/34693400612) +reached `contextual-orchestrator/orchestrator/free`, failed after five seconds, +and logged only `class=provider-error json-bytes=836 stderr-bytes=0`. Raw-body +suppression worked, but it also discarded the gateway's bounded phase, reason, +provider, status, and served-model receipt, so the control plane could not name +the causal owner or distinguish 429, provider 5xx, request-too-large, queue +admission, and malformed-output failures. + +**Owner boundary and repair.** contextual-orchestrator continues to own +provider discovery, routing, failover, and the canonical `error.detail` +envelope. `.github` owns the OpenCode adapter and public-log sanitizer. The +proposed adapter change parses only the canonical bounded envelope and emits +allowlisted scalar fields plus caller-measured duration; provider-controlled +messages and raw bodies remain suppressed. Production-shaped regression +fixtures cover 429, provider 502, HTTP 413 request-too-large, queue admission, +malformed model output, missing served-model, malformed JSON, and credential +non-disclosure. This is diagnostic evidence only: it cannot turn provider +failure into approval, retry a model, or relax an exact-head merge gate. From bdd93f3861cd9d08a1436690cb37c45ce669d8cb Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:49:13 +0900 Subject: [PATCH 5/9] test(opencode): bound failure telemetry input --- tests/test_opencode_model_pool_runner.py | 73 ++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index b1f2cfacd6..2f4849fe24 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -542,6 +542,79 @@ def test_malformed_gateway_failure_logs_only_bounded_decode_state( assert secret not in result.stdout +def test_gateway_failure_identifier_fields_reject_credential_shapes( + tmp_path: Path, +) -> None: + """Credential-shaped provider and model identifiers never reach public logs.""" + credential = "github" + "_pat_THISMUSTNEVERLEAK123456789" + result = run_failed_model( + tmp_path, + json_line=gateway_failure_event( + { + "model": credential, + "attempts": [ + { + "provider_name": credential, + "phase": "response_error", + "provider_status": 502, + "error_code": "provider_transport", + } + ], + } + ), + ) + + assert result.returncode == 1 + assert re.search( + r"phase=response_error reason=provider_transport provider=unknown " + r"status=502 duration=\d+s served_model=unknown", + result.stdout, + ) + assert credential not in result.stdout + + +@pytest.mark.parametrize( + "response_body", + [ + json.dumps( + { + "error": { + "detail": { + "model": "safe/model", + "padding": "x" * (16 * 1024), + } + } + } + ), + "[" * 600 + "]" * 600, + ], +) +def test_gateway_failure_parser_fails_closed_on_oversized_or_deep_body( + tmp_path: Path, + response_body: str, +) -> None: + """Provider-controlled envelopes cannot force unbounded parsing or logging.""" + outer_event = json.dumps( + { + "type": "error", + "error": { + "data": { + "responseBody": response_body, + } + }, + } + ) + result = run_failed_model(tmp_path, json_line=outer_event) + + assert result.returncode == 1 + assert re.search( + r"phase=decode_error reason=malformed_gateway_envelope provider=unknown " + r"status=unknown duration=\d+s served_model=unknown", + result.stdout, + ) + assert "safe/model" not in result.stdout + + def test_backoff_environment_rejects_recursive_arithmetic_injection( tmp_path: Path, ) -> None: From 17a5efb2e231ec7e929775e233ff1c7284816e95 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:49:25 +0900 Subject: [PATCH 6/9] fix(opencode): bound and scrub failure telemetry --- scripts/ci/run_opencode_review_model_pool.sh | 26 +++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index b24614c96a..7876467aed 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -3,6 +3,8 @@ set -euo pipefail : "${GITHUB_OUTPUT:=/dev/null}" +MAX_OPENCODE_FAILURE_TELEMETRY_BYTES=16384 + record_review_status() { printf 'review_status=%s\n' "$1" >>"$GITHUB_OUTPUT" } @@ -303,11 +305,17 @@ emit_sanitized_opencode_failure_detail() { "$failure_class" "$json_bytes" "$stderr_bytes" gateway_telemetry="$( - jq -Rrs --arg duration "${attempt_duration_seconds}s" ' - def safe_value($fallback): - if type == "string" and length > 0 and length <= 128 and - test("^[A-Za-z0-9._:/+-]+$") - then . else $fallback end; + tail -c "$MAX_OPENCODE_FAILURE_TELEMETRY_BYTES" "$opencode_json_file" 2>/dev/null | + jq -Rrs --arg duration "${attempt_duration_seconds}s" ' + def safe_value($fallback): + if type == "string" and length > 0 and length <= 128 and + test("^[A-Za-z0-9._:/+-]+$") + then . else $fallback end; + def safe_identifier($fallback): + if type == "string" and length > 0 and length <= 128 and + test("^[A-Za-z0-9._:/+-]+$") and + (test("github_pat_|gh[pousr]_|sk-[A-Za-z0-9]|xox[baprs]-|nvapi-|AIza"; "i") | not) + then . else $fallback end; def safe_phase: if . == "connecting" or . == "requesting" or . == "reading" or . == "decoding" or . == "validating" or @@ -339,15 +347,15 @@ emit_sanitized_opencode_failure_detail() { [ "phase=" + (($attempt.phase // "unknown") | safe_value("unknown") | safe_phase), "reason=" + (($attempt.error_code // $detail.terminal_reason // "unknown") | safe_value("unknown") | safe_reason), - "provider=" + (($attempt.provider_name // "unknown") | safe_value("unknown")), + "provider=" + (($attempt.provider_name // "unknown") | safe_identifier("unknown")), "status=" + (if ($attempt.provider_status | type) == "number" and $attempt.provider_status >= 100 and $attempt.provider_status <= 599 and ($attempt.provider_status | floor) == $attempt.provider_status then ($attempt.provider_status | tostring) else "unknown" end), "duration=" + $duration, - "served_model=" + (($detail.model // "unknown") | safe_value("unknown")) - ] | join(" ") - ' "$opencode_json_file" 2>/dev/null || true + "served_model=" + (($detail.model // "unknown") | safe_identifier("unknown")) + ] | join(" ") + ' 2>/dev/null || true )" if [ -n "$gateway_telemetry" ]; then printf 'OpenCode gateway failure telemetry: %s; provider-controlled content suppressed.\n' "$gateway_telemetry" From a6d70b879c8f511e76babd3d7dadfb0df790ab59 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 22:49:50 +0900 Subject: [PATCH 7/9] docs(opencode): specify telemetry safety bounds --- .../20260912-opencode-provider-failure-telemetry.md | 2 +- .../0003-contextual-orchestrator-vendored-free-zdr.md | 9 ++++++--- docs/product-technical-gap-baseline.md | 5 ++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md index 3051ba0a51..8e2c771198 100644 --- a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md +++ b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md @@ -1 +1 @@ -Preserve bounded phase, reason, provider, HTTP status, duration, and served-model evidence for OpenCode gateway failures while suppressing raw provider content and credentials. +Preserve bounded phase, reason, provider, HTTP status, duration, and served-model evidence for OpenCode gateway failures while suppressing raw provider content and credentials, capping failure input, and rejecting credential-shaped identifiers. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index f65a380655..1152314efc 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -291,6 +291,9 @@ all five, and auto-optimize routing by cost. duration, and served-model identifier. Unknown, malformed, and absent fields become fixed `unknown`/`malformed_gateway_envelope` values; arbitrary messages, response bodies, headers, credentials, and unbounded identifiers - never reach public Actions logs. This does not add a retry, timeout, provider - choice, or model policy to `.github`; contextual-orchestrator remains the - owner of discovery, routing, and failover. + never reach public Actions logs. The adapter reads at most the final 16 KiB + of the JSONL failure stream, rejects credential-shaped identifier values, + and fails oversized or deeply nested envelopes closed to the fixed malformed + state. This does not add a retry, timeout, provider choice, or model policy + to `.github`; contextual-orchestrator remains the owner of discovery, + routing, and failover. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index f81f4f60b1..e083ee9911 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3370,7 +3370,10 @@ provider discovery, routing, failover, and the canonical `error.detail` envelope. `.github` owns the OpenCode adapter and public-log sanitizer. The proposed adapter change parses only the canonical bounded envelope and emits allowlisted scalar fields plus caller-measured duration; provider-controlled -messages and raw bodies remain suppressed. Production-shaped regression +messages and raw bodies remain suppressed. Parsing is capped to the final +16 KiB of the failure stream, credential-shaped provider/model identifiers are +replaced with `unknown`, and oversized or deeply nested envelopes fail closed. +Production-shaped regression fixtures cover 429, provider 502, HTTP 413 request-too-large, queue admission, malformed model output, missing served-model, malformed JSON, and credential non-disclosure. This is diagnostic evidence only: it cannot turn provider From 89de8288772b2cc09bdba7f70fc85ccbd2ad3792 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 12 Sep 2026 23:15:06 +0900 Subject: [PATCH 8/9] fix(opencode): suppress unverified telemetry identifiers --- ...912-opencode-provider-failure-telemetry.md | 2 +- ...ntextual-orchestrator-vendored-free-zdr.md | 10 +++-- docs/product-technical-gap-baseline.md | 13 +++--- scripts/ci/run_opencode_review_model_pool.sh | 37 +++++++-------- tests/test_opencode_model_pool_runner.py | 45 ++++++++++++++++--- 5 files changed, 68 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md index 8e2c771198..5258f31a1b 100644 --- a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md +++ b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md @@ -1 +1 @@ -Preserve bounded phase, reason, provider, HTTP status, duration, and served-model evidence for OpenCode gateway failures while suppressing raw provider content and credentials, capping failure input, and rejecting credential-shaped identifiers. +Preserve bounded phase, reason, HTTP status, and duration evidence for OpenCode gateway failures while suppressing raw provider content and unverified provider/model identifiers and capping failure input. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 1152314efc..027036db95 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -287,12 +287,14 @@ all five, and auto-optimize routing by cost. provenance.** The OpenCode model-pool adapter must keep the gateway-owned canonical `error.detail` receipt useful after suppressing raw provider content. For a bounded structured error it emits only allowlisted phase, - normalized reason, provider identifier, HTTP status, caller-measured - duration, and served-model identifier. Unknown, malformed, and absent fields - become fixed `unknown`/`malformed_gateway_envelope` values; arbitrary + normalized reason, HTTP status, and caller-measured duration. Provider and + served-model identifiers remain `unknown` until a versioned CO-issued + non-secret identifier contract can be validated locally. Unknown, malformed, + and absent fields become fixed `unknown`/`malformed_gateway_envelope` + values; arbitrary messages, response bodies, headers, credentials, and unbounded identifiers never reach public Actions logs. The adapter reads at most the final 16 KiB - of the JSONL failure stream, rejects credential-shaped identifier values, + of the JSONL failure stream, suppresses unverified identifier values, and fails oversized or deeply nested envelopes closed to the fixed malformed state. This does not add a retry, timeout, provider choice, or model policy to `.github`; contextual-orchestrator remains the owner of discovery, diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index e083ee9911..bd9d799719 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3369,12 +3369,13 @@ admission, and malformed-output failures. provider discovery, routing, failover, and the canonical `error.detail` envelope. `.github` owns the OpenCode adapter and public-log sanitizer. The proposed adapter change parses only the canonical bounded envelope and emits -allowlisted scalar fields plus caller-measured duration; provider-controlled -messages and raw bodies remain suppressed. Parsing is capped to the final -16 KiB of the failure stream, credential-shaped provider/model identifiers are -replaced with `unknown`, and oversized or deeply nested envelopes fail closed. -Production-shaped regression -fixtures cover 429, provider 502, HTTP 413 request-too-large, queue admission, +allowlisted phase, reason, HTTP status, and caller-measured duration; +provider-controlled messages and raw bodies remain suppressed. Provider and +served-model identifiers stay `unknown` until a versioned CO-issued non-secret +identifier contract can be validated locally. Parsing is capped to the final +16 KiB of the failure stream, and oversized or deeply nested envelopes fail +closed. Production-shaped regression fixtures cover 429, provider 502, HTTP +413 request-too-large, queue admission, malformed model output, missing served-model, malformed JSON, and credential non-disclosure. This is diagnostic evidence only: it cannot turn provider failure into approval, retry a model, or relax an exact-head merge gate. diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 7876467aed..c3f2f1532f 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -311,24 +311,19 @@ emit_sanitized_opencode_failure_detail() { if type == "string" and length > 0 and length <= 128 and test("^[A-Za-z0-9._:/+-]+$") then . else $fallback end; - def safe_identifier($fallback): - if type == "string" and length > 0 and length <= 128 and - test("^[A-Za-z0-9._:/+-]+$") and - (test("github_pat_|gh[pousr]_|sk-[A-Za-z0-9]|xox[baprs]-|nvapi-|AIza"; "i") | not) - then . else $fallback end; - def safe_phase: - if . == "connecting" or . == "requesting" or . == "reading" or - . == "decoding" or . == "validating" or - . == "response_error" or . == "queue_admission" - then . else "unknown" end; - def safe_reason: - if . == "rate_limited" or . == "provider_transport" or - . == "request_too_large" or . == "queue_admission_failed" or - . == "malformed_model_output" or - . == "eligible_candidates_exhausted" or - . == "discovery_failure" or . == "model_unavailable" or - . == "quota_exhausted" or . == "authentication_failed" - then . else "unknown" end; + def safe_phase: + if . == "connecting" or . == "requesting" or . == "reading" or + . == "decoding" or . == "validating" or + . == "response_error" or . == "queue_admission" + then . else "unknown" end; + def safe_reason: + if . == "rate_limited" or . == "provider_transport" or + . == "request_too_large" or . == "queue_admission_failed" or + . == "malformed_model_output" or + . == "eligible_candidates_exhausted" or + . == "discovery_failure" or . == "model_unavailable" or + . == "quota_exhausted" or . == "authentication_failed" + then . else "unknown" end; [ splits("\\n") | fromjson? | select(.type == "error") | @@ -347,14 +342,14 @@ emit_sanitized_opencode_failure_detail() { [ "phase=" + (($attempt.phase // "unknown") | safe_value("unknown") | safe_phase), "reason=" + (($attempt.error_code // $detail.terminal_reason // "unknown") | safe_value("unknown") | safe_reason), - "provider=" + (($attempt.provider_name // "unknown") | safe_identifier("unknown")), + "provider=unknown", "status=" + (if ($attempt.provider_status | type) == "number" and $attempt.provider_status >= 100 and $attempt.provider_status <= 599 and ($attempt.provider_status | floor) == $attempt.provider_status then ($attempt.provider_status | tostring) else "unknown" end), "duration=" + $duration, - "served_model=" + (($detail.model // "unknown") | safe_identifier("unknown")) - ] | join(" ") + "served_model=unknown" + ] | join(" ") ' 2>/dev/null || true )" if [ -n "$gateway_telemetry" ]; then diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 2f4849fe24..733780230b 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -441,9 +441,9 @@ def gateway_failure_event(detail: dict[str, object]) -> str: } ], }, - "phase=response_error reason=rate_limited provider=openrouter " + "phase=response_error reason=rate_limited provider=unknown " "status=429 duration=0s " - "served_model=meta-llama/llama-3.3-70b-instruct", + "served_model=unknown", ), ( { @@ -457,9 +457,9 @@ def gateway_failure_event(detail: dict[str, object]) -> str: } ], }, - "phase=connecting reason=provider_transport provider=nvidia_nim " + "phase=connecting reason=provider_transport provider=unknown " "status=502 duration=0s " - "served_model=deepseek-ai/deepseek-v4-pro-0813", + "served_model=unknown", ), ( { @@ -472,7 +472,7 @@ def gateway_failure_event(detail: dict[str, object]) -> str: } ], }, - "phase=response_error reason=request_too_large provider=openrouter " + "phase=response_error reason=request_too_large provider=unknown " "status=413 duration=0s served_model=unknown", ), ( @@ -487,7 +487,7 @@ def gateway_failure_event(detail: dict[str, object]) -> str: ], }, "phase=queue_admission reason=queue_admission_failed " - "provider=contextual-orchestrator status=503 duration=0s " + "provider=unknown status=503 duration=0s " "served_model=unknown", ), ( @@ -502,7 +502,7 @@ def gateway_failure_event(detail: dict[str, object]) -> str: } ], }, - "phase=validating reason=malformed_model_output provider=bytez " + "phase=validating reason=malformed_model_output provider=unknown " "status=500 duration=0s served_model=unknown", ), ], @@ -573,6 +573,37 @@ def test_gateway_failure_identifier_fields_reject_credential_shapes( assert credential not in result.stdout +def test_gateway_failure_identifier_fields_fail_closed_without_catalog_proof( + tmp_path: Path, +) -> None: + """Unverified identifier-shaped secrets never reach public logs.""" + credential = "BYTEZ_TEST_SECRET_1234567890" + result = run_failed_model( + tmp_path, + json_line=gateway_failure_event( + { + "model": credential, + "attempts": [ + { + "provider_name": credential, + "phase": "response_error", + "provider_status": 502, + "error_code": "provider_transport", + } + ], + } + ), + ) + + assert result.returncode == 1 + assert re.search( + r"phase=response_error reason=provider_transport provider=unknown " + r"status=502 duration=\d+s served_model=unknown", + result.stdout, + ) + assert credential not in result.stdout + + @pytest.mark.parametrize( "response_body", [ From 76ca9f83f4538d33f7219b35e46646b459b37c63 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 13 Sep 2026 00:03:30 +0900 Subject: [PATCH 9/9] test(opencode): keep synthetic credential out of scanner source --- tests/test_opencode_model_pool_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 733780230b..f8eb71a083 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -577,7 +577,7 @@ def test_gateway_failure_identifier_fields_fail_closed_without_catalog_proof( tmp_path: Path, ) -> None: """Unverified identifier-shaped secrets never reach public logs.""" - credential = "BYTEZ_TEST_SECRET_1234567890" + credential = "BYTEZ" + "_TEST_SECRET_1234567890" result = run_failed_model( tmp_path, json_line=gateway_failure_event(