diff --git a/application/reconciliation_service.py b/application/reconciliation_service.py index 1392c35..3fdeaf8 100644 --- a/application/reconciliation_service.py +++ b/application/reconciliation_service.py @@ -2,26 +2,206 @@ from __future__ import annotations +import hashlib import json +import re import tempfile +from collections.abc import Mapping from pathlib import Path from typing import Any import pandas as pd +_NON_LIVE_ENVELOPE_VERSION = "v1" +_NON_LIVE_ENVELOPE_STRATEGY_PROFILE = "soxl_soxx_trend_income" +_NON_LIVE_ENVELOPE_EVIDENCE_SCOPE = "NON_LIVE_STATIC" +_FORBIDDEN_METADATA_KEY_SEQUENCES = frozenset( + { + ("access", "key"), + ("access", "key", "id"), + ("accesskey",), + ("accesskeyid",), + ("account",), + ("api", "key"), + ("apikey",), + ("authorization",), + ("balance",), + ("capital",), + ("cookie",), + ("credential",), + ("fill",), + ("fills",), + ("header",), + ("headers",), + ("jwt",), + ("notional",), + ("order",), + ("passphrase",), + ("password",), + ("position",), + ("private", "key"), + ("privatekey",), + ("provider",), + ("quantity",), + ("raw",), + ("secret",), + ("token",), + ("verified", "active"), + ("verifiedactive",), + } +) +_FORBIDDEN_METADATA_VALUES = {"matched", "mismatched", "verifiedactive"} +_NON_LIVE_ENVELOPE_KEYS = { + "envelope_version", + "strategy_profile", + "evidence_scope", + "reconciliation", + "learning_only", + "promotion_eligible", + "live_ready", + "size_zero_required", + "no_order", + "learning_disposition", + "source_revision", + "source_digests", +} + + def _json_safe(value: Any): if isinstance(value, pd.Timestamp): return value.isoformat() if isinstance(value, Path): return str(value) - if isinstance(value, dict): + if isinstance(value, Mapping): return {str(key): _json_safe(item) for key, item in value.items()} if isinstance(value, (list, tuple)): return [_json_safe(item) for item in value] return value +def _normalized_metadata_key(value: Any) -> str: + return "".join(character for character in str(value).lower() if character.isalnum()) + + +def _metadata_key_tokens(value: Any) -> tuple[str, ...]: + tokens = [] + for segment in re.findall(r"[A-Za-z0-9]+", str(value)): + tokens.extend( + token.lower() + for token in re.findall(r"[A-Z]+(?=[A-Z][a-z]|\d|$)|[A-Z]?[a-z]+|\d+", segment) + ) + return tuple(tokens) + + +def _contains_forbidden_metadata_key(value: Any) -> bool: + tokens = _metadata_key_tokens(value) + return any( + tokens[index : index + len(sequence)] == sequence + for sequence in _FORBIDDEN_METADATA_KEY_SEQUENCES + for index in range(len(tokens) - len(sequence) + 1) + ) + + +def _reject_non_live_metadata(value: Any) -> None: + if isinstance(value, Mapping): + for key, item in value.items(): + if _contains_forbidden_metadata_key(key): + raise ValueError("non-live reconciliation metadata contains a value that is not allowed") + _reject_non_live_metadata(item) + return + if isinstance(value, (list, tuple)): + for item in value: + _reject_non_live_metadata(item) + return + if isinstance(value, str) and _normalized_metadata_key(value) in _FORBIDDEN_METADATA_VALUES: + raise ValueError("non-live reconciliation metadata contains a value that is not allowed") + + +def _validate_optional_provenance_fields(envelope: Mapping[str, Any]) -> None: + if "source_revision" in envelope: + source_revision = envelope["source_revision"] + if not isinstance(source_revision, str) or not source_revision.strip(): + raise ValueError("non-live reconciliation envelope contains a value that is not allowed") + _reject_non_live_metadata({"source_revision": source_revision}) + + if "source_digests" in envelope: + source_digests = envelope["source_digests"] + if not isinstance(source_digests, Mapping) or any( + not isinstance(key, str) or not key.strip() or not isinstance(value, str) or not value.strip() + for key, value in source_digests.items() + ): + raise ValueError("non-live reconciliation envelope contains a value that is not allowed") + _reject_non_live_metadata(source_digests) + + +def build_non_live_reconciliation_envelope( + *, + learning_disposition: str, + source_revision: str | None = None, + source_digests: Mapping[str, str] | None = None, + metadata: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Build the fixed, fail-closed envelope for the SOXL static evidence slice.""" + if learning_disposition != "negative": + raise ValueError("non-live reconciliation envelopes require learning_disposition='negative'") + if metadata is not None: + _reject_non_live_metadata(metadata) + + envelope = { + "envelope_version": _NON_LIVE_ENVELOPE_VERSION, + "strategy_profile": _NON_LIVE_ENVELOPE_STRATEGY_PROFILE, + "evidence_scope": _NON_LIVE_ENVELOPE_EVIDENCE_SCOPE, + "reconciliation": {"status": "MISSING"}, + "learning_only": True, + "promotion_eligible": False, + "live_ready": False, + "size_zero_required": True, + "no_order": True, + "learning_disposition": "negative", + } + if source_revision is not None: + _validate_optional_provenance_fields({"source_revision": source_revision}) + envelope["source_revision"] = source_revision + if source_digests is not None: + _validate_optional_provenance_fields({"source_digests": source_digests}) + envelope["source_digests"] = dict(source_digests) + return envelope + + +def canonical_reconciliation_envelope_json(envelope: Mapping[str, Any]) -> str: + """Serialize a static non-live envelope deterministically without unsafe metadata.""" + if set(envelope).difference(_NON_LIVE_ENVELOPE_KEYS): + raise ValueError("non-live reconciliation envelope contains a value that is not allowed") + required_values = { + "envelope_version": _NON_LIVE_ENVELOPE_VERSION, + "strategy_profile": _NON_LIVE_ENVELOPE_STRATEGY_PROFILE, + "evidence_scope": _NON_LIVE_ENVELOPE_EVIDENCE_SCOPE, + "reconciliation": {"status": "MISSING"}, + "learning_only": True, + "promotion_eligible": False, + "live_ready": False, + "size_zero_required": True, + "no_order": True, + "learning_disposition": "negative", + } + for key, expected_value in required_values.items(): + actual_value = envelope.get(key) + if expected_value is True or expected_value is False: + if type(actual_value) is not bool or actual_value is not expected_value: + raise ValueError("non-live reconciliation envelope contains a value that is not allowed") + elif actual_value != expected_value: + raise ValueError("non-live reconciliation envelope contains a value that is not allowed") + _validate_optional_provenance_fields(envelope) + return json.dumps(_json_safe(envelope), ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def reconciliation_envelope_digest(envelope: Mapping[str, Any]) -> str: + """Return the stable SHA-256 digest for a static non-live envelope.""" + canonical_json = canonical_reconciliation_envelope_json(envelope) + return hashlib.sha256(canonical_json.encode("utf-8")).hexdigest() + + def default_reconciliation_output_path(strategy_profile: str | None) -> Path: profile = str(strategy_profile or "unknown").strip() or "unknown" safe_profile = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in profile) diff --git a/tests/test_reconciliation_service.py b/tests/test_reconciliation_service.py new file mode 100644 index 0000000..4ab1d12 --- /dev/null +++ b/tests/test_reconciliation_service.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +from types import MappingProxyType + +import pytest + +from application.reconciliation_service import ( + build_non_live_reconciliation_envelope, + canonical_reconciliation_envelope_json, + reconciliation_envelope_digest, +) + + +def test_static_envelope_is_fail_closed_despite_runtime_descriptive_metadata(): + envelope = build_non_live_reconciliation_envelope( + learning_disposition="negative", + source_revision="0dbca440", + source_digests={"selection": "a" * 64}, + metadata={ + "strategy_profile": "soxl_soxx_trend_income", + "mode": "live", + "execution_report": {"result": "executed"}, + }, + ) + + assert envelope == { + "envelope_version": "v1", + "strategy_profile": "soxl_soxx_trend_income", + "evidence_scope": "NON_LIVE_STATIC", + "reconciliation": {"status": "MISSING"}, + "learning_only": True, + "promotion_eligible": False, + "live_ready": False, + "size_zero_required": True, + "no_order": True, + "learning_disposition": "negative", + "source_revision": "0dbca440", + "source_digests": {"selection": "a" * 64}, + } + + +@pytest.mark.parametrize( + "metadata", + [ + {"reconciliation": {"status": "MATCHED"}}, + {"reconciliation": {"status": "MISMATCHED"}}, + {"verified_active": True}, + {"fills": [{"symbol": "SOXL"}]}, + {"capital": 1}, + {"order": {"id": "order-1"}}, + {"balance": 1}, + {"position": {"symbol": "SOXL"}}, + {"account_identifier": "redacted"}, + {"provider_row": {"close": 1}}, + {"raw_market_data": {"close": 1}}, + ], +) +def test_static_envelope_rejects_material_runtime_assertions(metadata): + with pytest.raises(ValueError, match="not allowed"): + build_non_live_reconciliation_envelope( + learning_disposition="negative", + metadata=metadata, + ) + + +@pytest.mark.parametrize( + "metadata", + [ + {"outer": {"secret": "value"}}, + {"outer": {"token": "value"}}, + {"outer": {"headers": {"Authorization": "Bearer value"}}}, + {"outer": {"jwt": "value"}}, + {"outer": {"cookie": "value"}}, + {"outer": {"api_key": "value"}}, + ], +) +def test_static_envelope_rejects_nested_sensitive_metadata(metadata): + with pytest.raises(ValueError, match="not allowed"): + build_non_live_reconciliation_envelope( + learning_disposition="negative", + metadata=metadata, + ) + + +def test_static_envelope_canonical_serialization_and_digest_are_deterministic(): + first = build_non_live_reconciliation_envelope( + learning_disposition="negative", + source_digests={"z": "2", "a": "1"}, + ) + second = build_non_live_reconciliation_envelope( + learning_disposition="negative", + source_digests={"a": "1", "z": "2"}, + ) + + assert canonical_reconciliation_envelope_json(first) == canonical_reconciliation_envelope_json(second) + assert reconciliation_envelope_digest(first) == reconciliation_envelope_digest(second) + + +def test_canonical_serialization_rejects_attempts_to_override_fail_closed_flags(): + envelope = build_non_live_reconciliation_envelope(learning_disposition="negative") + envelope["reconciliation"] = {"status": "MATCHED"} + + with pytest.raises(ValueError, match="not allowed"): + canonical_reconciliation_envelope_json(envelope) + + +@pytest.mark.parametrize( + ("key", "integer_substitute"), + [ + ("learning_only", 1), + ("promotion_eligible", 0), + ("live_ready", 0), + ("size_zero_required", 1), + ("no_order", 1), + ], +) +def test_canonical_serialization_rejects_integer_substitutes_for_fixed_boolean_flags(key, integer_substitute): + envelope = build_non_live_reconciliation_envelope(learning_disposition="negative") + envelope[key] = integer_substitute + + with pytest.raises(ValueError, match="not allowed"): + canonical_reconciliation_envelope_json(envelope) + + +@pytest.mark.parametrize("sensitive_key", ["password", "passphrase"]) +def test_static_envelope_rejects_password_bearing_metadata(sensitive_key): + with pytest.raises(ValueError, match="not allowed"): + build_non_live_reconciliation_envelope( + learning_disposition="negative", + metadata={sensitive_key: "redacted"}, + ) + + +@pytest.mark.parametrize("sensitive_key", ["private_key", "access_key", "access_key_id"]) +def test_static_envelope_rejects_conventional_access_credentials(sensitive_key): + with pytest.raises(ValueError, match="not allowed"): + build_non_live_reconciliation_envelope( + learning_disposition="negative", + metadata={"nested": {sensitive_key: "redacted"}}, + ) + + +@pytest.mark.parametrize("metadata_key", ["max_drawdown", "drawdown_report"]) +def test_static_envelope_accepts_descriptive_drawdown_metadata(metadata_key): + envelope = build_non_live_reconciliation_envelope( + learning_disposition="negative", + metadata={metadata_key: "research-only"}, + ) + + assert envelope["reconciliation"] == {"status": "MISSING"} + + +@pytest.mark.parametrize( + ("key", "value"), + [ + ("source_revision", 123), + ("source_revision", ""), + ("source_digests", None), + ("source_digests", []), + ("source_digests", {"selection": float("nan")}), + ("source_digests", {1: "a" * 64}), + ], +) +def test_canonical_serialization_rejects_invalid_optional_provenance_types(key, value): + envelope = build_non_live_reconciliation_envelope(learning_disposition="negative") + envelope[key] = value + + with pytest.raises(ValueError, match="not allowed"): + canonical_reconciliation_envelope_json(envelope) + + +@pytest.mark.parametrize("assertion", ["VERIFIED-ACTIVE", "verified active"]) +def test_static_envelope_rejects_normalized_forbidden_assertions(assertion): + with pytest.raises(ValueError, match="not allowed"): + build_non_live_reconciliation_envelope( + learning_disposition="negative", + metadata={"reconciliation_assertion": assertion}, + ) + + +def test_canonical_serialization_accepts_arbitrary_mapping_envelopes(): + envelope = build_non_live_reconciliation_envelope(learning_disposition="negative") + + assert canonical_reconciliation_envelope_json(MappingProxyType(envelope)) == canonical_reconciliation_envelope_json( + envelope + )