diff --git a/CHANGELOG.md b/CHANGELOG.md index 34281625cb..5a722bead3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +### Strix resolves its evidence binder from the trusted gate, not the scanned repository + +- On every consumer repository, Strix exited 2 after reporting zero vulnerabilities (fast-mlsirm#2005 and #2018) with "Strix evidence binder is missing: …/trusted-workspace/scripts/ci/strix_evidence_binding.py". `strix.yml` runs the trusted `.github` gate with `STRIX_REPO_ROOT` pointing at the consumer's base checkout, but `sanitize_remediation_evidence_claims` looked for the central binder under `$REPO_ROOT` instead of next to the gate script. Only `.github` itself, where both locations are the same repository, passed. The binder is now `$SCRIPT_DIR/strix_evidence_binding.py`, the same way the gate already loads `strix_model_utils.sh`. It still fails closed with exit 2 when the trusted copy is absent. +- `tests/test_strix_evidence_binder_trusted_path.py` runs the real function with a consumer `REPO_ROOT` that has no `scripts/ci`. Both cases fail on the previous gate and pass now. +- `scripts/ci/test_strix_quick_gate.sh` fixture repositories now also copy the binder. Fifteen pull-request-target cases had been failing there since 1e150464 (2026-09-17) with the same "binder is missing" error. After this change none of those cases fail locally. Three unrelated pre-existing contract strings (OpenCode coverage-source wording) still fail and are out of scope. + ### Noema transport capacity schedules a bounded continuation re-dispatch - After gateway failover, HTTP 429/5xx no longer end only as a permanent required-check failure with `caller attempts=1`. ADR-0031 classifies that class as `provider_capacity_unavailable`, keeps the single gateway request per job, surfaces `provider_attempt_count` from the orchestrator error envelope, and authorizes at most two same-head `repository_dispatch` retries after a capped `Retry-After` or deterministic 60–180 s jitter. Review is never skipped. Refs #2165. diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index c7d3667465..a90a670fb3 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -243,7 +243,9 @@ PY sanitize_remediation_evidence_claims() { local log_file="$1" local report_root="$2" - local binder="$REPO_ROOT/scripts/ci/strix_evidence_binding.py" + # The binder is central trusted code: resolve it next to this gate script, + # not under REPO_ROOT, which is the scanned consumer repository checkout. + local binder="$SCRIPT_DIR/strix_evidence_binding.py" local report_file if [ ! -f "$binder" ] || [ -L "$binder" ]; then diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 150b9102b3..acd6dbd3e5 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -3296,6 +3296,7 @@ run_gate_case() { local gate_under_test="$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$GATE_SCRIPT" "$gate_under_test" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$gate_under_test" local fake_strix="$bin_dir/strix" local path_hijack_log="$tmp_dir/path-hijack.log" @@ -7026,6 +7027,7 @@ run_pull_request_target_head_scope_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -7174,6 +7176,7 @@ run_pull_request_target_plaintext_runner_token_fails_closed_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -7296,6 +7299,7 @@ run_pull_request_target_bounded_head_context_scope_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -7401,6 +7405,7 @@ run_pull_request_target_changed_context_scope_uses_pr_head_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -7580,6 +7585,7 @@ run_pull_request_target_changed_backend_context_scope_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -7839,6 +7845,7 @@ run_pull_request_target_frontend_email_context_scope_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8029,6 +8036,7 @@ run_pull_request_target_shallow_head_merge_base_fallback_case() { cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8144,6 +8152,7 @@ run_pull_request_target_aborts_on_pr_head_blob_failure_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local real_git @@ -8268,6 +8277,7 @@ run_pull_request_target_rejects_invalid_sha_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8361,6 +8371,7 @@ run_pull_request_target_irregular_head_entry_fails_closed_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8444,6 +8455,7 @@ run_pull_request_target_gitlink_is_explicitly_skipped_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8526,6 +8538,7 @@ run_full_head_scope_skips_gitlink_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8640,6 +8653,7 @@ run_pull_request_target_rejects_unsafe_changed_path_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" @@ -8732,6 +8746,7 @@ run_timeout_cleanup_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" local child_pid_file="$tmp_dir/child.pid" @@ -8814,6 +8829,7 @@ run_vertex_model_ignores_untrusted_llm_api_base_file_case() { mkdir -p "$repo_root_dir/scripts/ci" "$allowed_input_dir" "$outside_dir" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cat >"$fake_strix" <<'EOF' @@ -8866,6 +8882,7 @@ run_total_timeout_case() { mkdir -p "$bin_dir" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" local output_log="$tmp_dir/output.log" @@ -9193,6 +9210,7 @@ run_llm_api_base_file_outside_input_root_fails_closed_case() { mkdir -p "$repo_root_dir/scripts/ci" "$allowed_input_dir" "$outside_dir" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cat >"$fake_strix" <<'EOF' @@ -9248,6 +9266,7 @@ run_pr_scoped_llm_api_base_file_config_failure_exits_2_case() { mkdir -p "$repo_root_dir/scripts/ci" "$repo_root_dir/src" "$allowed_input_dir" "$outside_dir" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" printf '%s\n' 'print("one")' >"$repo_root_dir/src/one.py" printf '%s\n' 'print("two")' >"$repo_root_dir/src/two.py" @@ -9309,6 +9328,7 @@ run_required_input_file_outside_input_root_fails_closed_case() { mkdir -p "$repo_root_dir/scripts/ci" "$allowed_input_dir" "$outside_dir" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cat >"$fake_strix" <<'EOF' @@ -9379,6 +9399,7 @@ run_input_file_root_override_takes_precedence_over_runner_temp_case() { mkdir -p "$repo_root_dir/scripts/ci" "$explicit_input_root" "$inherited_runner_temp" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cat >"$fake_strix" <<'EOF' @@ -9433,6 +9454,7 @@ run_stale_report_case() { mkdir -p "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" mkdir -p "$stale_report_dir" @@ -9488,6 +9510,7 @@ run_symlink_report_case() { mkdir -p "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" mkdir -p "$external_report_dir" "$repo_root_dir/strix_runs" @@ -9544,6 +9567,7 @@ run_unsafe_target_path_case() { mkdir -p "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cat >"$fake_strix" <<'EOF' @@ -9592,6 +9616,7 @@ run_absolute_outside_target_path_case() { mkdir -p "$bin_dir" "$repo_root_dir/src" "$repo_root_dir/scripts/ci" cp "$GATE_SCRIPT" "$repo_root_dir/scripts/ci/strix_quick_gate.sh" cp "$REPO_ROOT/scripts/ci/strix_model_utils.sh" "$repo_root_dir/scripts/ci/strix_model_utils.sh" + cp "$REPO_ROOT/scripts/ci/strix_evidence_binding.py" "$repo_root_dir/scripts/ci/strix_evidence_binding.py" chmod +x "$repo_root_dir/scripts/ci/strix_quick_gate.sh" local fake_strix="$bin_dir/strix" local call_log="$tmp_dir/calls.log" diff --git a/tests/test_strix_evidence_binder_trusted_path.py b/tests/test_strix_evidence_binder_trusted_path.py new file mode 100644 index 0000000000..c488e00a09 --- /dev/null +++ b/tests/test_strix_evidence_binder_trusted_path.py @@ -0,0 +1,62 @@ +"""The Strix evidence binder resolves from the trusted gate, not the scanned repo.""" + +from __future__ import annotations + +import re +import subprocess +from pathlib import Path + +GATE = Path("scripts/ci/strix_quick_gate.sh") + + +def _function_source(name: str) -> str: + """Return one top-level bash function from the gate script.""" + text = GATE.read_text(encoding="utf-8") + match = re.search(rf"(?ms)^{name}\(\) \{{\n.*?^\}}\n", text) + assert match is not None, name + return match.group(0) + + +def test_binder_runs_when_the_scanned_repository_has_no_central_scripts(tmp_path: Path) -> None: + """A consumer repository (for example fast-mlsirm) has no scripts/ci binder. + + ``strix.yml`` runs the trusted ``.github`` gate with ``STRIX_REPO_ROOT`` set + to the consumer's base checkout, so ``REPO_ROOT`` has no + ``scripts/ci/strix_evidence_binding.py``. fast-mlsirm#2005/#2018 then + exited 2 after zero vulnerabilities with "Strix evidence binder is + missing: …/trusted-workspace/scripts/ci/strix_evidence_binding.py". The + binder must resolve next to the gate script instead. + """ + consumer = tmp_path / "trusted-workspace" + consumer.mkdir() + reports = tmp_path / "strix_runs" + reports.mkdir() + report = reports / "penetration_test_report.md" + report.write_text("# Report\n\nNo vulnerabilities were identified.\n", encoding="utf-8") + log = tmp_path / "strix.log" + log.write_text("scan complete\n", encoding="utf-8") + script = ( + "set -u\n" + f'SCRIPT_DIR="{GATE.parent.resolve()}"\n' + f'REPO_ROOT="{consumer}"\n' + + _function_source("sanitize_remediation_evidence_claims") + + f'sanitize_remediation_evidence_claims "{log}" "{reports}"\n' + ) + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, check=False) + assert result.returncode == 0, result.stderr + assert "binder is missing" not in result.stderr + assert report.read_text(encoding="utf-8").startswith("# Report") + + +def test_binder_still_fails_closed_when_the_trusted_copy_is_absent(tmp_path: Path) -> None: + """Without the trusted binder the gate still refuses to continue.""" + script = ( + "set -u\n" + f'SCRIPT_DIR="{tmp_path}"\n' + f'REPO_ROOT="{GATE.parent.parent.resolve()}"\n' + + _function_source("sanitize_remediation_evidence_claims") + + 'sanitize_remediation_evidence_claims "" ""\n' + ) + result = subprocess.run(["bash", "-c", script], capture_output=True, text=True, check=False) + assert result.returncode == 2 + assert f"Strix evidence binder is missing: {tmp_path}/strix_evidence_binding.py" in result.stderr