diff --git a/gateway/failure_lab/claims.py b/gateway/failure_lab/claims.py index 7b1d520a..7c4f0c6e 100644 --- a/gateway/failure_lab/claims.py +++ b/gateway/failure_lab/claims.py @@ -165,7 +165,8 @@ def from_dict(cls, document: Mapping[str, Any]) -> ClaimRecord: if reported is None: matches = bool(expected) and not rows_diverge else: - matches = bool(reported) and not rows_diverge + # Only a real boolean true counts. The string "false" is truthy. + matches = reported is True and not rows_diverge status, status_lines = _observed_status( str(document.get("status", "")), observed ) diff --git a/gateway/failure_lab/effect_ledger.py b/gateway/failure_lab/effect_ledger.py index add62e68..dac5d7f5 100644 --- a/gateway/failure_lab/effect_ledger.py +++ b/gateway/failure_lab/effect_ledger.py @@ -281,9 +281,15 @@ def execution_count(self, operation_id: str | None = None) -> int: return int(connection.execute(sql, params).fetchone()[0]) def snapshot(self) -> list[dict[str, Any]]: - """The PRD's ledger shape: one entry per effect with its execution count.""" + """The PRD's ledger shape: one entry per effect with its execution count. + + Counts and rows come from one read. A second read can contain an + operation the first read never counted, and the snapshot would then + raise or mix two moments. + """ + records = self.effects() counts: dict[str, int] = {} - for record in self.effects(): + for record in records: counts[record.operation_id] = counts.get(record.operation_id, 0) + 1 return [ { @@ -299,7 +305,7 @@ def snapshot(self) -> list[dict[str, Any]]: "worker_pid": record.worker_pid, "configuration": record.configuration, } - for record in self.effects() + for record in records ] def reset(self) -> None: diff --git a/gateway/failure_lab/evidence.py b/gateway/failure_lab/evidence.py index 9101a83a..bc8af683 100644 --- a/gateway/failure_lab/evidence.py +++ b/gateway/failure_lab/evidence.py @@ -62,7 +62,7 @@ from dataclasses import dataclass from dataclasses import fields as dataclass_fields from datetime import datetime, timezone -from pathlib import Path +from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any from failure_lab import TEST_DEFINITION_VERSION @@ -1266,6 +1266,36 @@ def as_dict(self) -> dict[str, Any]: } +def _unsafe_bundle_path(root: Path, relative: str) -> str | None: + """Reject a manifest path that is not a regular file inside the bundle. + + Joining an absolute path, or a path with ``..``, drops the bundle + directory. A symlink can point at the same outside file. Either one + would let a matching hash pass for bytes that are not in the bundle. + """ + if ( + not isinstance(relative, str) + or not relative + or relative.startswith(("/", "\\")) + or re.match(r"^[A-Za-z]:", relative) + or "\\" in relative + or "\x00" in relative + ): + return "path escapes the bundle and was not read" + pure = PurePosixPath(relative) + if pure.is_absolute() or ".." in pure.parts: + return "path escapes the bundle and was not read" + try: + root_real = root.resolve() + candidate = root_real.joinpath(*pure.parts) + if candidate.is_symlink(): + return "path is a symlink and was not read" + candidate.resolve().relative_to(root_real) + except (RuntimeError, OSError, ValueError): + return "path escapes the bundle and was not read" + return None + + def verify_bundle_integrity(directory: Path | str) -> IntegrityReport: """Re-hash every file in a bundle against its manifest. @@ -1279,7 +1309,15 @@ def verify_bundle_integrity(directory: Path | str) -> IntegrityReport: """ root = Path(directory) manifest_path = root / MANIFEST_NAME - if not manifest_path.is_file(): + try: + manifest_is_regular = ( + not root.is_symlink() + and not manifest_path.is_symlink() + and manifest_path.is_file() + ) + except (RuntimeError, OSError, ValueError): + manifest_is_regular = False + if not manifest_is_regular: return IntegrityReport( directory=root, ok=False, @@ -1291,7 +1329,7 @@ def verify_bundle_integrity(directory: Path | str) -> IntegrityReport: ) try: manifest = json.loads(manifest_path.read_text(encoding="utf-8")) - except ValueError as exc: + except (RuntimeError, OSError, ValueError) as exc: return IntegrityReport( directory=root, ok=False, @@ -1302,7 +1340,17 @@ def verify_bundle_integrity(directory: Path | str) -> IntegrityReport: problems=[f"{MANIFEST_NAME} is not valid JSON: {exc}"], ) - entries = manifest.get("files") or [] + if not isinstance(manifest, dict) or not isinstance(manifest.get("files"), list): + return IntegrityReport( + directory=root, + ok=False, + checked=0, + changed=[], + missing=[], + unexpected=[], + problems=[f"{MANIFEST_NAME} must contain a files list"], + ) + entries = manifest["files"] changed: list[str] = [] missing: list[str] = [] problems: list[str] = [] @@ -1310,31 +1358,65 @@ def verify_bundle_integrity(directory: Path | str) -> IntegrityReport: listed: set[str] = set() for entry in entries: - relative = str(entry.get("path", "")) + if not isinstance(entry, dict): + problems.append("manifest file entry must be an object") + continue + relative = entry.get("path", "") + if not isinstance(relative, str): + problems.append("path escapes the bundle and was not read") + continue listed.add(relative) + unsafe = _unsafe_bundle_path(root, relative) + if unsafe is not None: + problems.append(f"{relative}: {unsafe}") + continue path = root / relative - if not path.is_file(): + try: + regular = path.is_file() and not path.is_symlink() + except (RuntimeError, OSError, ValueError): + regular = False + if not regular: missing.append(relative) continue + try: + actual = _sha256_file(path) + size = path.stat().st_size + except (RuntimeError, OSError, ValueError) as exc: + problems.append(f"{relative}: could not read bundle file: {exc}") + continue checked += 1 - actual = _sha256_file(path) expected = str(entry.get("sha256", "")) if actual != expected: changed.append(relative) problems.append( f"{relative}: sha256 is {actual[:16]}..., manifest says {expected[:16]}..." ) - size = path.stat().st_size - if entry.get("bytes") is not None and int(entry["bytes"]) != size: + expected_size = entry.get("bytes") + if expected_size is not None and ( + not isinstance(expected_size, int) + or isinstance(expected_size, bool) + or expected_size != size + ): problems.append( f"{relative}: {size} bytes on disk, manifest says {entry['bytes']}" ) - on_disk = { - str(path.relative_to(root)).replace(os.sep, "/") - for path in root.rglob("*") - if path.is_file() - } + on_disk: set[str] = set() + pending = [root] + while pending: + directory_to_scan = pending.pop() + try: + with os.scandir(directory_to_scan) as entries: + for entry in entries: + path = Path(entry.path) + if entry.is_dir(follow_symlinks=False): + pending.append(path) + else: + on_disk.add(str(path.relative_to(root)).replace(os.sep, "/")) + except (OSError, RuntimeError) as exc: + problems.append( + f"{directory_to_scan}: could not scan bundle directory: {exc}" + ) unexpected = sorted(on_disk - listed - {MANIFEST_NAME}) problems.extend( f"{name}: present on disk but not listed in the manifest" for name in unexpected diff --git a/gateway/failure_lab/refund_tool.py b/gateway/failure_lab/refund_tool.py index 79d9b989..191780be 100644 --- a/gateway/failure_lab/refund_tool.py +++ b/gateway/failure_lab/refund_tool.py @@ -312,7 +312,16 @@ async def refund_create( def _control_ok(request: Request) -> bool: provided = request.headers.get(CONTROL_HEADER) - return provided is not None and hmac.compare_digest(provided, control_token) + if not provided: + return False + try: + # compare_digest on str raises TypeError for non-ASCII. Compare + # UTF-8 bytes so a latin-1 header is a denial, not a 500. + return hmac.compare_digest( + provided.encode("utf-8"), control_token.encode("utf-8") + ) + except (TypeError, ValueError, UnicodeError): + return False def _denied() -> JSONResponse: return JSONResponse({"detail": "failure_lab_control_denied"}, status_code=403) diff --git a/gateway/failure_lab/report.py b/gateway/failure_lab/report.py index 3618659d..4d9d7056 100644 --- a/gateway/failure_lab/report.py +++ b/gateway/failure_lab/report.py @@ -81,7 +81,7 @@ class ConclusionKind(str, Enum): "and no comparison was made either way" ), ConclusionKind.INCONCLUSIVE.value: ( - "too few configurations ran for this test to say anything" + "the measurements are not enough to say a baseline handled this test" ), } @@ -293,6 +293,28 @@ def _differences( return added, lost +def _baseline_is_credible(column: ConfigurationView | None) -> bool: + """A baseline column the report may treat as having run the operation. + + Zero duplicate effects is not evidence on its own. A column that never + reached the one intended execution suppressed duplicates by doing + nothing. OBSERVED records measurements without establishing that the + baseline handled the test correctly; only PASS supports that claim. + """ + return ( + column is not None + and column.ran + and column.verdict == Verdict.PASS.value + and column.downstream_effects >= INTENDED_EXECUTIONS + ) + + +def _baseline_supports_sufficiency(column: ConfigurationView | None) -> bool: + if column is None or not _baseline_is_credible(column): + return False + return column.duplicate_effects == 0 + + def _conclude( result: ScenarioResult, existing: ConfigurationView | None, @@ -371,7 +393,7 @@ def _conclude( disadvantages, ) - if existing is not None and existing.ran and existing.duplicate_effects == 0: + if _baseline_supports_sufficiency(existing): return Conclusion( ConclusionKind.EXISTING_INTEGRATION_SUFFICIENT, "Your existing integration handled this test correctly. We did not " @@ -384,28 +406,42 @@ def _conclude( disadvantages, ) - if native is not None and native.ran and native.duplicate_effects == 0: - if prevented_vs_native == 0: - cost = ( - " On this test the baseline also did something the gateway did " - "not: see what it cost, below." - if disadvantages - else "" - ) + if _baseline_supports_sufficiency(native): + cost = ( + " On this test the baseline also did something the gateway did " + "not: see what it cost, below." + if disadvantages + else "" + ) + return Conclusion( + ConclusionKind.NATIVE_CONTROLS_SUFFICIENT, + "Correct native controls handled this test on their own. We did " + "not observe an additional duplicate effect prevented by Agent " + "Middleware in this scenario. If your downstream already honors " + "a durable operation id the way this baseline does, this " + "failure mode is already covered for you." + cost, + prevented_vs_native, + prevented_vs_existing, + differences, + disadvantages, + ) + + if prevented_vs_native == 0 and prevented_vs_existing == 0: + if (existing_ran or native_ran) and not ( + _baseline_is_credible(existing) or _baseline_is_credible(native) + ): return Conclusion( - ConclusionKind.NATIVE_CONTROLS_SUFFICIENT, - "Correct native controls handled this test on their own. We did " - "not observe an additional duplicate effect prevented by Agent " - "Middleware in this scenario. If your downstream already honors " - "a durable operation id the way this baseline does, this " - "failure mode is already covered for you." + cost, - prevented_vs_native, - prevented_vs_existing, + ConclusionKind.INCONCLUSIVE, + "The available baseline measurements do not establish success " + "in this test: none has both a PASS verdict and " + "the required downstream execution. A zero duplicate count " + "alone cannot establish that the intended business action " + "succeeded.", + 0, + 0, differences, disadvantages, ) - - if prevented_vs_native == 0 and prevented_vs_existing == 0: if differences: return Conclusion( ConclusionKind.GATEWAY_ADDED_EVIDENCE_ONLY, @@ -418,9 +454,9 @@ def _conclude( disadvantages, ) return Conclusion( - ConclusionKind.NATIVE_CONTROLS_SUFFICIENT, - "No difference was observed between the baseline and the gateway " - "in this scenario.", + ConclusionKind.INCONCLUSIVE, + "No difference was observed, but no baseline established that it " + "handled the intended business action without duplicates.", prevented_vs_native, prevented_vs_existing, differences, diff --git a/gateway/tests/test_failure_lab_claims.py b/gateway/tests/test_failure_lab_claims.py index 64b35f9c..b1273d56 100644 --- a/gateway/tests/test_failure_lab_claims.py +++ b/gateway/tests/test_failure_lab_claims.py @@ -182,6 +182,25 @@ def test_loaded_manifest_cannot_promote_negative_rows_to_pass(tmp_path, row_verd assert supported_claims(loaded) == [] +@pytest.mark.parametrize("reported", ["false", "true", "yes", 1]) +def test_a_non_boolean_match_flag_does_not_grant_support(tmp_path, reported): + """The gate accepts JSON true, not every truthy value. + + The string "false" is truthy in Python. A hand-written manifest must not + become supportable because of that. + """ + manifest = build_claims_manifest([_scenario("T01")], environment="local") + document = manifest.as_dict() + document["claims"][0]["matches_documented_expectation"] = reported + path = tmp_path / "claims.json" + path.write_text(json.dumps(document)) + loaded = load_claims_manifest(path) + assert loaded.records[0].matches_documented_expectation is False + with pytest.raises(UnsupportedClaimError): + assert_claim_is_supported(loaded, CLAIM) + assert supported_claims(loaded) == [] + + @pytest.mark.parametrize("documented", [Verdict.PASS, Verdict.FAIL]) def test_loaded_manifest_keeps_passing_rows_and_expectation_gate(tmp_path, documented): manifest = build_claims_manifest( diff --git a/gateway/tests/test_failure_lab_evidence.py b/gateway/tests/test_failure_lab_evidence.py index 28956602..e0e27259 100644 --- a/gateway/tests/test_failure_lab_evidence.py +++ b/gateway/tests/test_failure_lab_evidence.py @@ -8,10 +8,14 @@ from __future__ import annotations +import hashlib +import errno import json +from pathlib import Path import pytest +import failure_lab.evidence as evidence from failure_lab.evidence import ( MIN_SCANNABLE_SECRET_LENGTH, build_evidence_bundle, @@ -20,6 +24,7 @@ assert_no_secret_leak, find_leaked_secrets, redact, + verify_bundle_integrity, ) from failure_lab.telemetry import ( CONVERSION_SOURCES, @@ -238,3 +243,311 @@ def test_a_bundle_that_leaked_is_never_left_on_disk(self, tmp_path): if path.is_file() and secret in path.read_text(errors="ignore") ] assert not survivors, survivors + + +def _manifest(bundle, files): + (bundle / "manifest.json").write_text(json.dumps({"files": files})) + + +def test_integrity_check_accepts_a_regular_file_inside_the_bundle(tmp_path): + bundle = tmp_path / "bundle" + bundle.mkdir() + body = b"inside" + (bundle / "report.txt").write_bytes(body) + _manifest( + bundle, + [ + { + "path": "report.txt", + "sha256": hashlib.sha256(body).hexdigest(), + "bytes": len(body), + } + ], + ) + report = verify_bundle_integrity(bundle) + assert report.ok, report.problems + assert report.checked == 1 + + +def test_integrity_check_does_not_hash_a_path_that_leaves_the_bundle(tmp_path): + """A manifest path of ../ or an absolute path is not a bundle file. + + pathlib joins an absolute path by discarding the bundle directory, and + a matching hash would then make the check pass for bytes that are not + in the bundle. + """ + outside = tmp_path / "outside.txt" + payload = b"not-in-the-bundle" + outside.write_bytes(payload) + digest = hashlib.sha256(payload).hexdigest() + entry = {"sha256": digest, "bytes": len(payload)} + + parent_ref = tmp_path / "parent-ref" + parent_ref.mkdir() + _manifest(parent_ref, [{**entry, "path": "../outside.txt"}]) + parent_report = verify_bundle_integrity(parent_ref) + assert not parent_report.ok + assert parent_report.checked == 0 + assert any("not read" in problem for problem in parent_report.problems) + + absolute = tmp_path / "absolute" + absolute.mkdir() + _manifest(absolute, [{**entry, "path": str(outside)}]) + absolute_report = verify_bundle_integrity(absolute) + assert not absolute_report.ok + assert absolute_report.checked == 0 + + linked = tmp_path / "linked" + linked.mkdir() + (linked / "report.txt").symlink_to(outside) + _manifest(linked, [{**entry, "path": "report.txt"}]) + linked_report = verify_bundle_integrity(linked) + assert not linked_report.ok + assert linked_report.checked == 0 + assert outside.read_bytes() == payload + + +def test_integrity_check_accepts_nested_files_and_reports_missing_files(tmp_path): + bundle = tmp_path / "bundle" + (bundle / "nested").mkdir(parents=True) + body = b"nested" + (bundle / "nested" / "report.txt").write_bytes(body) + _manifest( + bundle, + [ + { + "path": "nested/report.txt", + "sha256": hashlib.sha256(body).hexdigest(), + "bytes": len(body), + }, + {"path": "missing.txt", "sha256": "0" * 64, "bytes": 1}, + ], + ) + report = verify_bundle_integrity(bundle) + assert report.checked == 1 + assert report.missing == ["missing.txt"] + assert not report.ok + + +@pytest.mark.parametrize( + "unsafe", + [ + "../outside.txt", + "/outside.txt", + "bad\x00name", + 4, + "C:/x", + "C:\\x", + "c:x", + "//server/share", + "\\\\server\\share", + ], +) +def test_integrity_check_never_hashes_unsafe_manifest_paths( + tmp_path, monkeypatch, unsafe +): + bundle = tmp_path / "bundle" + bundle.mkdir() + _manifest(bundle, [{"path": unsafe, "sha256": "0" * 64, "bytes": 1}]) + + def forbidden_hash(_path): + pytest.fail("unsafe target was hashed") + + monkeypatch.setattr(evidence, "_sha256_file", forbidden_hash) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert report.checked == 0 + assert any("not read" in problem for problem in report.problems) + + +def test_integrity_check_rejects_escaping_parent_link_and_linked_manifest( + tmp_path, monkeypatch +): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "payload.txt").write_bytes(b"outside") + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "linked").symlink_to(outside, target_is_directory=True) + _manifest(bundle, [{"path": "linked/payload.txt", "sha256": "0" * 64}]) + monkeypatch.setattr( + evidence, "_sha256_file", lambda _path: pytest.fail("outside hashed") + ) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert report.checked == 0 + + linked_bundle = tmp_path / "linked-bundle" + linked_bundle.mkdir() + (linked_bundle / "manifest.json").symlink_to(bundle / "manifest.json") + monkeypatch.setattr( + Path, "read_text", lambda _path, **_kw: pytest.fail("linked manifest read") + ) + linked_report = verify_bundle_integrity(linked_bundle) + assert not linked_report.ok + assert linked_report.checked == 0 + + +def test_integrity_check_reports_unlisted_symlink_without_reading_target( + tmp_path, monkeypatch +): + outside = tmp_path / "outside.txt" + outside.write_bytes(b"outside") + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "unlisted.txt").symlink_to(outside) + _manifest(bundle, []) + monkeypatch.setattr( + evidence, "_sha256_file", lambda _path: pytest.fail("unlisted link was hashed") + ) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert report.checked == 0 + assert report.unexpected == ["unlisted.txt"] + assert outside.read_bytes() == b"outside" + + +def test_integrity_check_accepts_empty_and_nested_regular_bundles(tmp_path): + empty = tmp_path / "empty" + empty.mkdir() + _manifest(empty, []) + assert verify_bundle_integrity(empty).ok + + nested = tmp_path / "nested" + (nested / "subdir").mkdir(parents=True) + payload = b"regular" + (nested / "subdir" / "payload.txt").write_bytes(payload) + _manifest( + nested, + [ + { + "path": "subdir/payload.txt", + "sha256": hashlib.sha256(payload).hexdigest(), + "bytes": len(payload), + } + ], + ) + report = verify_bundle_integrity(nested) + assert report.ok, report.problems + assert report.checked == 1 + + +@pytest.mark.parametrize("nested", [False, True], ids=["root", "nested"]) +@pytest.mark.parametrize("error", [errno.EIO, errno.EACCES], ids=["eio", "permission"]) +def test_integrity_check_reports_directory_scan_failure( + tmp_path, monkeypatch, nested, error +): + bundle = tmp_path / "bundle" + bundle.mkdir() + blocked = bundle / "subdir" if nested else bundle + if nested: + blocked.mkdir() + (blocked / "unlisted.txt").write_bytes(b"not indexed") + _manifest(bundle, []) + original_scandir = evidence.os.scandir + + def failing_scandir(path): + if Path(path) == blocked: + raise OSError(error, "synthetic directory scan failure") + return original_scandir(path) + + with monkeypatch.context() as patch: + patch.setattr(evidence.os, "scandir", failing_scandir) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert report.checked == 0 + assert any("could not scan" in problem for problem in report.problems) + + +def test_integrity_check_does_not_descend_into_unlisted_directory_link( + tmp_path, monkeypatch +): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "payload.txt").write_bytes(b"outside") + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "linked").symlink_to(outside, target_is_directory=True) + _manifest(bundle, []) + original_scandir = evidence.os.scandir + + def guarded_scandir(path): + assert Path(path) != outside, "outside directory was scanned" + return original_scandir(path) + + with monkeypatch.context() as patch: + patch.setattr(evidence.os, "scandir", guarded_scandir) + patch.setattr( + evidence, + "_sha256_file", + lambda _path: pytest.fail("outside target was hashed"), + ) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert report.unexpected == ["linked"] + + +@pytest.mark.parametrize("failure", [OSError, RuntimeError]) +def test_integrity_check_reports_entry_classification_failure( + tmp_path, monkeypatch, failure +): + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "unlisted.txt").write_bytes(b"not indexed") + _manifest(bundle, []) + + class BrokenEntry: + path = str(bundle / "unlisted.txt") + + def is_dir(self, *, follow_symlinks): + assert follow_symlinks is False + raise failure("synthetic entry classification failure") + + class BrokenListing: + def __enter__(self): + return iter([BrokenEntry()]) + + def __exit__(self, *_args): + return False + + with monkeypatch.context() as patch: + patch.setattr(evidence.os, "scandir", lambda _path: BrokenListing()) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert any("could not scan" in problem for problem in report.problems) + + +@pytest.mark.parametrize( + "files", [None, {}, [1], [None], [{"path": "file", "bytes": "invalid"}]] +) +def test_integrity_check_handles_malformed_manifest_shapes(tmp_path, files): + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "file").write_bytes(b"ok") + _manifest(bundle, files) + report = verify_bundle_integrity(bundle) + assert not report.ok + + +@pytest.mark.parametrize("failure", [RuntimeError, OSError, ValueError]) +def test_integrity_check_handles_resolution_failure_without_hashing( + tmp_path, monkeypatch, failure +): + bundle = tmp_path / "bundle" + bundle.mkdir() + (bundle / "file").write_bytes(b"ok") + _manifest(bundle, [{"path": "file", "sha256": "0" * 64}]) + original_resolve = Path.resolve + + def broken_resolve(path, *args, **kwargs): + if path.name == "file": + raise failure("cannot resolve") + return original_resolve(path, *args, **kwargs) + + monkeypatch.setattr(Path, "resolve", broken_resolve) + monkeypatch.setattr( + evidence, "_sha256_file", lambda _path: pytest.fail("unresolved target hashed") + ) + report = verify_bundle_integrity(bundle) + assert not report.ok + assert report.checked == 0 diff --git a/gateway/tests/test_failure_lab_harness.py b/gateway/tests/test_failure_lab_harness.py index 2b2e3ab0..3dfc4c6d 100644 --- a/gateway/tests/test_failure_lab_harness.py +++ b/gateway/tests/test_failure_lab_harness.py @@ -22,6 +22,57 @@ from failure_lab.refund_tool import RefundRequest +def test_snapshot_does_not_mix_two_reads(tmp_path, monkeypatch): + """snapshot() must describe one read of the ledger. + + A second read can include an operation the first read never counted. + Indexing the second read with the first read's counts raises KeyError + and, short of that, reports two different moments as one snapshot. + """ + from failure_lab.effect_ledger import EffectRecord + + ledger = EffectLedger(tmp_path / "effects.db") + common = dict( + request_id="req_1", + idempotency_key="k", + amount=5000, + currency="USD", + customer_id="cus", + payment_id="pay", + configuration="test", + ) + ledger.execute(operation_id="refund:pay", **common) + original = ledger.effects + calls = {"n": 0} + + def two_moments(operation_id=None): + calls["n"] += 1 + rows = original(operation_id) + if calls["n"] == 1 or operation_id is not None: + return rows + late = EffectRecord( + effect_id="effect_late", + operation_id="refund:other", + request_id="req_2", + idempotency_key="k2", + amount=100, + currency="USD", + customer_id="cus", + payment_id="other", + executed_at=rows[0].executed_at, + worker_pid=rows[0].worker_pid, + configuration="test", + execution_ordinal=1, + ) + return [*rows, late] + + monkeypatch.setattr(ledger, "effects", two_moments) + rows = ledger.snapshot() + assert calls["n"] == 1 + assert [row["operation_id"] for row in rows] == ["refund:pay"] + assert rows[0]["execution_count"] == 1 + + def test_effect_ledger_is_duplicate_visible(tmp_path): ledger = EffectLedger(tmp_path / "effects.db") common = dict( @@ -84,6 +135,116 @@ def test_identity_distinguishes_restart_policies(): assert restarted.request_id != attempt.request_id +@pytest.mark.anyio +async def test_non_ascii_control_header_is_denied(tmp_path): + """A control header compare must deny, not raise. + + hmac.compare_digest rejects non-ASCII strings with TypeError. HTTP header + values are latin-1, so a client can send that byte. httpx refuses it + before the app runs. This calls the ASGI app with the raw header bytes. + """ + import json + + from failure_lab.faults import FaultInjector + from failure_lab.refund_tool import SimulatedRefundService, build_downstream_app + + service = SimulatedRefundService( + EffectLedger(tmp_path / "effects.db"), + native_idempotency=False, + configuration="test", + ) + injector = FaultInjector() + downstream = build_downstream_app( + service, + injector, + bearer_token="bearer-token-value", + control_token="control-token-value", + ) + + async def call( + header: bytes | None, + *, + method: str = "GET", + path: str = "/__lab/health", + body: bytes = b"", + ) -> tuple[int, bytes]: + messages: list[dict] = [] + + async def receive(): + return {"type": "http.request", "body": body, "more_body": False} + + async def send(message): + messages.append(message) + + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "headers": [] if header is None else [(b"x-failure-lab-control", header)], + "client": ("127.0.0.1", 1234), + "server": ("127.0.0.1", 80), + } + await downstream.asgi(scope, receive, send) + status = next( + message["status"] + for message in messages + if message["type"] == "http.response.start" + ) + body = b"".join( + message.get("body", b"") + for message in messages + if message["type"] == "http.response.body" + ) + return status, body + + non_ascii = "caf\u00e9-not-the-token".encode("latin-1") + refused_status, refused_body = await call(non_ascii) + wrong_status, wrong_body = await call(b"nope") + ok_status, _ok_body = await call(b"control-token-value") + assert refused_status == 403 + assert wrong_status == 403 + assert json.loads(refused_body)["detail"] == "failure_lab_control_denied" + assert json.loads(wrong_body)["detail"] == "failure_lab_control_denied" + assert ok_status == 200 + + plan = json.dumps({"mode": FaultMode.HTTP_ERROR_BEFORE_EXECUTION.value}).encode() + for denied_header in (None, b"", b"wrong", non_ascii): + for method, path, request_body in ( + ("POST", "/__lab/faults", plan), + ("DELETE", "/__lab/faults", b""), + ("POST", "/__lab/restart", b""), + ): + status, _ = await call( + denied_header, method=method, path=path, body=request_body + ) + assert status == 403 + assert injector.armed() == [] + + status, _ = await call( + b"control-token-value", method="POST", path="/__lab/faults", body=plan + ) + assert status == 200 + assert len(injector.armed()) == 1 + status, _ = await call( + b"control-token-value", method="DELETE", path="/__lab/faults" + ) + assert status == 200 + assert injector.armed() == [] + + injector.crashed = True + status, _ = await call(b"wrong", method="POST", path="/__lab/restart") + assert status == 403 + assert injector.crashed + status, _ = await call(b"control-token-value", method="POST", path="/__lab/restart") + assert status == 200 + assert not injector.crashed + + def test_refund_request_is_strict_about_representation(): RefundRequest.from_mapping( {"operation_id": "o", "customer_id": "c", "payment_id": "p", "amount": 5000} diff --git a/gateway/tests/test_failure_lab_report.py b/gateway/tests/test_failure_lab_report.py index 3cbc91f5..c78c9949 100644 --- a/gateway/tests/test_failure_lab_report.py +++ b/gateway/tests/test_failure_lab_report.py @@ -140,6 +140,193 @@ def test_the_gateway_adding_nothing_is_a_reachable_conclusion(): ) +def test_a_baseline_that_executed_nothing_is_not_called_sufficient(): + """Zero duplicates from a tool that never ran is not a handled test. + + A baseline that records no downstream effect suppresses duplicates by + doing nothing. The telemetry path already refuses to call that a pass. + The visitor-facing conclusion has to refuse it too, including when the + other baseline really did duplicate and the gateway prevented that. + """ + never_ran = _result( + [ + _entry(Configuration.DIRECT_NAIVE, verdict=Verdict.OBSERVED, executions=0), + _entry(Configuration.DIRECT_NATIVE, verdict=Verdict.FAIL, executions=0), + _entry( + Configuration.GATEWAY_NATIVE, + verdict=Verdict.PASS, + executions=1, + dispatches=1, + ), + ] + ) + conclusion = build_comparison(never_ran).conclusion + assert conclusion.kind is ConclusionKind.INCONCLUSIVE + assert "handled" not in conclusion.text + assert conclusion.duplicates_prevented_vs_existing == 0 + assert conclusion.duplicates_prevented_vs_native == 0 + + hidden_duplicate = _result( + [ + _entry(Configuration.DIRECT_NAIVE, verdict=Verdict.OBSERVED, executions=2), + _entry(Configuration.DIRECT_NATIVE, verdict=Verdict.FAIL, executions=0), + _entry( + Configuration.GATEWAY_NATIVE, + verdict=Verdict.PASS, + executions=1, + dispatches=1, + ), + ] + ) + prevented = build_comparison(hidden_duplicate).conclusion + assert prevented.kind is ConclusionKind.GATEWAY_PREVENTED_DUPLICATES + assert prevented.duplicates_prevented_vs_existing == 1 + + +def test_a_failing_baseline_is_not_reported_as_sufficient(): + """Verdict FAIL means the baseline did not hold. Do not invert that.""" + result = _result( + [ + _entry(Configuration.DIRECT_NAIVE, verdict=Verdict.FAIL, executions=1), + _entry(Configuration.DIRECT_NATIVE, verdict=Verdict.FAIL, executions=1), + _entry( + Configuration.GATEWAY_NATIVE, + verdict=Verdict.PASS, + executions=1, + dispatches=1, + ), + ] + ) + conclusion = build_comparison(result).conclusion + assert conclusion.kind is ConclusionKind.INCONCLUSIVE + assert conclusion.kind is not ConclusionKind.EXISTING_INTEGRATION_SUFFICIENT + assert conclusion.kind is not ConclusionKind.NATIVE_CONTROLS_SUFFICIENT + assert ( + "none has both a PASS verdict and the required downstream execution" + in conclusion.text + ) + assert ( + "cannot establish that the intended business action succeeded" + in conclusion.text + ) + + +@pytest.mark.parametrize( + "baseline", [Configuration.DIRECT_NAIVE, Configuration.DIRECT_NATIVE] +) +def test_pass_verdict_without_downstream_execution_is_inconclusive(baseline): + result = _result( + [ + _entry(baseline, verdict=Verdict.PASS, executions=0), + _entry(Configuration.GATEWAY_NATIVE, verdict=Verdict.PASS, executions=1), + ] + ) + + conclusion = build_comparison(result).conclusion + assert conclusion.kind is ConclusionKind.INCONCLUSIVE + assert ( + "none has both a PASS verdict and the required downstream execution" + in conclusion.text + ) + + +@pytest.mark.parametrize( + ("existing_verdict", "existing_count", "native_verdict", "native_count"), + [ + (Verdict.OBSERVED, 0, Verdict.FAIL, 0), + (Verdict.FAIL, 1, Verdict.FAIL, 1), + (Verdict.ERROR, 0, Verdict.NOT_RUN, 0), + (Verdict.NOT_APPLICABLE, 0, Verdict.NOT_RUN, 0), + ], +) +def test_unqualified_baselines_never_get_a_sufficient_conclusion( + existing_verdict, existing_count, native_verdict, native_count +): + result = _result( + [ + _entry( + Configuration.DIRECT_NAIVE, + verdict=existing_verdict, + executions=existing_count, + ), + _entry( + Configuration.DIRECT_NATIVE, + verdict=native_verdict, + executions=native_count, + ), + _entry(Configuration.GATEWAY_NATIVE, verdict=Verdict.PASS, executions=1), + ] + ) + comparison = build_comparison(result) + assert comparison.conclusion.kind not in { + ConclusionKind.EXISTING_INTEGRATION_SUFFICIENT, + ConclusionKind.NATIVE_CONTROLS_SUFFICIENT, + } + rendered = render_run_json([comparison], environment=ENVIRONMENT) + assert comparison.conclusion.kind.value in rendered + + +def test_equal_duplicate_counts_do_not_claim_native_sufficiency(): + comparison = build_comparison( + _result( + [ + _entry( + Configuration.DIRECT_NAIVE, verdict=Verdict.OBSERVED, executions=2 + ), + _entry( + Configuration.DIRECT_NATIVE, verdict=Verdict.OBSERVED, executions=2 + ), + _entry( + Configuration.GATEWAY_NATIVE, verdict=Verdict.PASS, executions=2 + ), + ] + ) + ) + assert comparison.conclusion.kind is ConclusionKind.INCONCLUSIVE + assert comparison.conclusion.duplicates_prevented_vs_native == 0 + assert comparison.conclusion.duplicates_prevented_vs_existing == 0 + assert all(column.duplicate_effects == 1 for column in comparison.columns) + + +def test_observed_single_effect_does_not_prove_existing_integration_sufficient(): + comparison = build_comparison( + _result( + [ + _entry( + Configuration.DIRECT_NAIVE, verdict=Verdict.OBSERVED, executions=1 + ), + _entry(Configuration.DIRECT_NATIVE, verdict=Verdict.FAIL, executions=1), + _entry( + Configuration.GATEWAY_NATIVE, verdict=Verdict.PASS, executions=1 + ), + ] + ) + ) + assert comparison.conclusion.kind is ConclusionKind.INCONCLUSIVE + assert comparison.conclusion.duplicates_prevented_vs_existing == 0 + assert comparison.conclusion.duplicates_prevented_vs_native == 0 + + +def test_a_short_existing_column_does_not_hide_a_native_baseline_that_ran(): + result = _result( + [ + _entry(Configuration.DIRECT_NAIVE, verdict=Verdict.OBSERVED, executions=0), + _entry( + Configuration.DIRECT_NATIVE, verdict=Verdict.PASS, executions=1, known=1 + ), + _entry( + Configuration.GATEWAY_NATIVE, + verdict=Verdict.PASS, + executions=1, + dispatches=1, + known=1, + ), + ] + ) + conclusion = build_comparison(result).conclusion + assert conclusion.kind is ConclusionKind.NATIVE_CONTROLS_SUFFICIENT + + def test_an_already_correct_existing_integration_is_told_so(): result = _result( [