feat(reconciliation): rebuild SOXL non-live static envelope from fresh main

application/reconciliation_service.py

@@ -2,26 +2,206 @@
 
 from __future__ import annotations
 
+import hashlib
 import json
+import re
 import tempfile
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Any
 
 import pandas as pd
 
 
+_NON_LIVE_ENVELOPE_VERSION = "v1"
+_NON_LIVE_ENVELOPE_STRATEGY_PROFILE = "soxl_soxx_trend_income"
+_NON_LIVE_ENVELOPE_EVIDENCE_SCOPE = "NON_LIVE_STATIC"
+_FORBIDDEN_METADATA_KEY_SEQUENCES = frozenset(
+    {
+        ("access", "key"),
+        ("access", "key", "id"),
+        ("accesskey",),
+        ("accesskeyid",),
+        ("account",),
+        ("api", "key"),
+        ("apikey",),
+        ("authorization",),
+        ("balance",),
+        ("capital",),
+        ("cookie",),
+        ("credential",),
+        ("fill",),
+        ("fills",),
+        ("header",),
+        ("headers",),
+        ("jwt",),
+        ("notional",),
+        ("order",),
+        ("passphrase",),
+        ("password",),
+        ("position",),
+        ("private", "key"),
+        ("privatekey",),
+        ("provider",),
+        ("quantity",),
+        ("raw",),
+        ("secret",),
+        ("token",),
+        ("verified", "active"),
+        ("verifiedactive",),
+    }
+)
+_FORBIDDEN_METADATA_VALUES = {"matched", "mismatched", "verifiedactive"}
+_NON_LIVE_ENVELOPE_KEYS = {
+    "envelope_version",
+    "strategy_profile",
+    "evidence_scope",
+    "reconciliation",
+    "learning_only",
+    "promotion_eligible",
+    "live_ready",
+    "size_zero_required",
+    "no_order",
+    "learning_disposition",
+    "source_revision",
+    "source_digests",
+}
+
+
 def _json_safe(value: Any):
     if isinstance(value, pd.Timestamp):
         return value.isoformat()
     if isinstance(value, Path):
         return str(value)
-    if isinstance(value, dict):
+    if isinstance(value, Mapping):
         return {str(key): _json_safe(item) for key, item in value.items()}
     if isinstance(value, (list, tuple)):
         return [_json_safe(item) for item in value]
     return value
 
 
+def _normalized_metadata_key(value: Any) -> str:
+    return "".join(character for character in str(value).lower() if character.isalnum())
+
+
+def _metadata_key_tokens(value: Any) -> tuple[str, ...]:
+    tokens = []
+    for segment in re.findall(r"[A-Za-z0-9]+", str(value)):
+        tokens.extend(
+            token.lower()
+            for token in re.findall(r"[A-Z]+(?=[A-Z][a-z]|\d|$)|[A-Z]?[a-z]+|\d+", segment)
+        )
+    return tuple(tokens)
+
+
+def _contains_forbidden_metadata_key(value: Any) -> bool:
+    tokens = _metadata_key_tokens(value)
+    return any(
+        tokens[index : index + len(sequence)] == sequence
+        for sequence in _FORBIDDEN_METADATA_KEY_SEQUENCES
+        for index in range(len(tokens) - len(sequence) + 1)
+    )
+
+
+def _reject_non_live_metadata(value: Any) -> None:
+    if isinstance(value, Mapping):
+        for key, item in value.items():
+            if _contains_forbidden_metadata_key(key):
+                raise ValueError("non-live reconciliation metadata contains a value that is not allowed")
+            _reject_non_live_metadata(item)
+        return
+    if isinstance(value, (list, tuple)):
+        for item in value:
+            _reject_non_live_metadata(item)
+        return
+    if isinstance(value, str) and _normalized_metadata_key(value) in _FORBIDDEN_METADATA_VALUES:
+        raise ValueError("non-live reconciliation metadata contains a value that is not allowed")
+
+
+def _validate_optional_provenance_fields(envelope: Mapping[str, Any]) -> None:
+    if "source_revision" in envelope:
+        source_revision = envelope["source_revision"]
+        if not isinstance(source_revision, str) or not source_revision.strip():
+            raise ValueError("non-live reconciliation envelope contains a value that is not allowed")
+        _reject_non_live_metadata({"source_revision": source_revision})
+
+    if "source_digests" in envelope:
+        source_digests = envelope["source_digests"]
+        if not isinstance(source_digests, Mapping) or any(
+            not isinstance(key, str) or not key.strip() or not isinstance(value, str) or not value.strip()
+            for key, value in source_digests.items()
+        ):
+            raise ValueError("non-live reconciliation envelope contains a value that is not allowed")
+        _reject_non_live_metadata(source_digests)
+
+
+def build_non_live_reconciliation_envelope(
+    *,
+    learning_disposition: str,
+    source_revision: str | None = None,
+    source_digests: Mapping[str, str] | None = None,
+    metadata: Mapping[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Build the fixed, fail-closed envelope for the SOXL static evidence slice."""
+    if learning_disposition != "negative":
+        raise ValueError("non-live reconciliation envelopes require learning_disposition='negative'")
+    if metadata is not None:
+        _reject_non_live_metadata(metadata)
+
+    envelope = {
+        "envelope_version": _NON_LIVE_ENVELOPE_VERSION,
+        "strategy_profile": _NON_LIVE_ENVELOPE_STRATEGY_PROFILE,
+        "evidence_scope": _NON_LIVE_ENVELOPE_EVIDENCE_SCOPE,
+        "reconciliation": {"status": "MISSING"},
+        "learning_only": True,
+        "promotion_eligible": False,
+        "live_ready": False,
+        "size_zero_required": True,
+        "no_order": True,
+        "learning_disposition": "negative",
+    }
+    if source_revision is not None:
+        _validate_optional_provenance_fields({"source_revision": source_revision})
+        envelope["source_revision"] = source_revision
+    if source_digests is not None:
+        _validate_optional_provenance_fields({"source_digests": source_digests})
+        envelope["source_digests"] = dict(source_digests)
+    return envelope
+
+
+def canonical_reconciliation_envelope_json(envelope: Mapping[str, Any]) -> str:
+    """Serialize a static non-live envelope deterministically without unsafe metadata."""
+    if set(envelope).difference(_NON_LIVE_ENVELOPE_KEYS):
+        raise ValueError("non-live reconciliation envelope contains a value that is not allowed")
+    required_values = {
+        "envelope_version": _NON_LIVE_ENVELOPE_VERSION,
+        "strategy_profile": _NON_LIVE_ENVELOPE_STRATEGY_PROFILE,
+        "evidence_scope": _NON_LIVE_ENVELOPE_EVIDENCE_SCOPE,
+        "reconciliation": {"status": "MISSING"},
+        "learning_only": True,
+        "promotion_eligible": False,
+        "live_ready": False,
+        "size_zero_required": True,
+        "no_order": True,
+        "learning_disposition": "negative",
+    }
+    for key, expected_value in required_values.items():
+        actual_value = envelope.get(key)
+        if expected_value is True or expected_value is False:
+            if type(actual_value) is not bool or actual_value is not expected_value:
+                raise ValueError("non-live reconciliation envelope contains a value that is not allowed")
+        elif actual_value != expected_value:
+            raise ValueError("non-live reconciliation envelope contains a value that is not allowed")
+    _validate_optional_provenance_fields(envelope)
+    return json.dumps(_json_safe(envelope), ensure_ascii=False, separators=(",", ":"), sort_keys=True)
+
+
+def reconciliation_envelope_digest(envelope: Mapping[str, Any]) -> str:
+    """Return the stable SHA-256 digest for a static non-live envelope."""
+    canonical_json = canonical_reconciliation_envelope_json(envelope)
+    return hashlib.sha256(canonical_json.encode("utf-8")).hexdigest()
+
+
 def default_reconciliation_output_path(strategy_profile: str | None) -> Path:
     profile = str(strategy_profile or "unknown").strip() or "unknown"
     safe_profile = "".join(ch if ch.isalnum() or ch in {"-", "_", "."} else "_" for ch in profile)

tests/test_reconciliation_service.py

@@ -0,0 +1,186 @@
+from __future__ import annotations
+
+from types import MappingProxyType
+
+import pytest
+
+from application.reconciliation_service import (
+    build_non_live_reconciliation_envelope,
+    canonical_reconciliation_envelope_json,
+    reconciliation_envelope_digest,
+)
+
+
+def test_static_envelope_is_fail_closed_despite_runtime_descriptive_metadata():
+    envelope = build_non_live_reconciliation_envelope(
+        learning_disposition="negative",
+        source_revision="0dbca440",
+        source_digests={"selection": "a" * 64},
+        metadata={
+            "strategy_profile": "soxl_soxx_trend_income",
+            "mode": "live",
+            "execution_report": {"result": "executed"},
+        },
+    )
+
+    assert envelope == {
+        "envelope_version": "v1",
+        "strategy_profile": "soxl_soxx_trend_income",
+        "evidence_scope": "NON_LIVE_STATIC",
+        "reconciliation": {"status": "MISSING"},
+        "learning_only": True,
+        "promotion_eligible": False,
+        "live_ready": False,
+        "size_zero_required": True,
+        "no_order": True,
+        "learning_disposition": "negative",
+        "source_revision": "0dbca440",
+        "source_digests": {"selection": "a" * 64},
+    }
+
+
+@pytest.mark.parametrize(
+    "metadata",
+    [
+        {"reconciliation": {"status": "MATCHED"}},
+        {"reconciliation": {"status": "MISMATCHED"}},
+        {"verified_active": True},
+        {"fills": [{"symbol": "SOXL"}]},
+        {"capital": 1},
+        {"order": {"id": "order-1"}},
+        {"balance": 1},
+        {"position": {"symbol": "SOXL"}},
+        {"account_identifier": "redacted"},
+        {"provider_row": {"close": 1}},
+        {"raw_market_data": {"close": 1}},
+    ],
+)
+def test_static_envelope_rejects_material_runtime_assertions(metadata):
+    with pytest.raises(ValueError, match="not allowed"):
+        build_non_live_reconciliation_envelope(
+            learning_disposition="negative",
+            metadata=metadata,
+        )
+
+
+@pytest.mark.parametrize(
+    "metadata",
+    [
+        {"outer": {"secret": "value"}},
+        {"outer": {"token": "value"}},
+        {"outer": {"headers": {"Authorization": "Bearer value"}}},
+        {"outer": {"jwt": "value"}},
+        {"outer": {"cookie": "value"}},
+        {"outer": {"api_key": "value"}},
+    ],
+)
+def test_static_envelope_rejects_nested_sensitive_metadata(metadata):
+    with pytest.raises(ValueError, match="not allowed"):
+        build_non_live_reconciliation_envelope(
+            learning_disposition="negative",
+            metadata=metadata,
+        )
+
+
+def test_static_envelope_canonical_serialization_and_digest_are_deterministic():
+    first = build_non_live_reconciliation_envelope(
+        learning_disposition="negative",
+        source_digests={"z": "2", "a": "1"},
+    )
+    second = build_non_live_reconciliation_envelope(
+        learning_disposition="negative",
+        source_digests={"a": "1", "z": "2"},
+    )
+
+    assert canonical_reconciliation_envelope_json(first) == canonical_reconciliation_envelope_json(second)
+    assert reconciliation_envelope_digest(first) == reconciliation_envelope_digest(second)
+
+
+def test_canonical_serialization_rejects_attempts_to_override_fail_closed_flags():
+    envelope = build_non_live_reconciliation_envelope(learning_disposition="negative")
+    envelope["reconciliation"] = {"status": "MATCHED"}
+
+    with pytest.raises(ValueError, match="not allowed"):
+        canonical_reconciliation_envelope_json(envelope)
+
+
+@pytest.mark.parametrize(
+    ("key", "integer_substitute"),
+    [
+        ("learning_only", 1),
+        ("promotion_eligible", 0),
+        ("live_ready", 0),
+        ("size_zero_required", 1),
+        ("no_order", 1),
+    ],
+)
+def test_canonical_serialization_rejects_integer_substitutes_for_fixed_boolean_flags(key, integer_substitute):
+    envelope = build_non_live_reconciliation_envelope(learning_disposition="negative")
+    envelope[key] = integer_substitute
+
+    with pytest.raises(ValueError, match="not allowed"):
+        canonical_reconciliation_envelope_json(envelope)
+
+
+@pytest.mark.parametrize("sensitive_key", ["password", "passphrase"])
+def test_static_envelope_rejects_password_bearing_metadata(sensitive_key):
+    with pytest.raises(ValueError, match="not allowed"):
+        build_non_live_reconciliation_envelope(
+            learning_disposition="negative",
+            metadata={sensitive_key: "redacted"},
+        )
+
+
+@pytest.mark.parametrize("sensitive_key", ["private_key", "access_key", "access_key_id"])
+def test_static_envelope_rejects_conventional_access_credentials(sensitive_key):
+    with pytest.raises(ValueError, match="not allowed"):
+        build_non_live_reconciliation_envelope(
+            learning_disposition="negative",
+            metadata={"nested": {sensitive_key: "redacted"}},
+        )
+
+
+@pytest.mark.parametrize("metadata_key", ["max_drawdown", "drawdown_report"])
+def test_static_envelope_accepts_descriptive_drawdown_metadata(metadata_key):
+    envelope = build_non_live_reconciliation_envelope(
+        learning_disposition="negative",
+        metadata={metadata_key: "research-only"},
+    )
+
+    assert envelope["reconciliation"] == {"status": "MISSING"}
+
+
+@pytest.mark.parametrize(
+    ("key", "value"),
+    [
+        ("source_revision", 123),
+        ("source_revision", ""),
+        ("source_digests", None),
+        ("source_digests", []),
+        ("source_digests", {"selection": float("nan")}),
+        ("source_digests", {1: "a" * 64}),
+    ],
+)
+def test_canonical_serialization_rejects_invalid_optional_provenance_types(key, value):
+    envelope = build_non_live_reconciliation_envelope(learning_disposition="negative")
+    envelope[key] = value
+
+    with pytest.raises(ValueError, match="not allowed"):
+        canonical_reconciliation_envelope_json(envelope)
+
+
+@pytest.mark.parametrize("assertion", ["VERIFIED-ACTIVE", "verified active"])
+def test_static_envelope_rejects_normalized_forbidden_assertions(assertion):
+    with pytest.raises(ValueError, match="not allowed"):
+        build_non_live_reconciliation_envelope(
+            learning_disposition="negative",
+            metadata={"reconciliation_assertion": assertion},
+        )
+
+
+def test_canonical_serialization_accepts_arbitrary_mapping_envelopes():
+    envelope = build_non_live_reconciliation_envelope(learning_disposition="negative")
+
+    assert canonical_reconciliation_envelope_json(MappingProxyType(envelope)) == canonical_reconciliation_envelope_json(
+        envelope
+    )