From d79e19bd644b8c3d03f307fc0d4679e66bc5d55f Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:25:42 +0800 Subject: [PATCH 1/2] feat: add research input manifest contract Co-Authored-By: Codex --- src/quant_platform_kit/data/research_input.py | 362 ++++++++++ .../research-input-manifest.v1.schema.json | 226 +++++++ .../test_research_input_manifest_contract.py | 631 ++++++++++++++++++ 3 files changed, 1219 insertions(+) create mode 100644 src/quant_platform_kit/data/research_input.py create mode 100644 src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json create mode 100644 tests/test_research_input_manifest_contract.py diff --git a/src/quant_platform_kit/data/research_input.py b/src/quant_platform_kit/data/research_input.py new file mode 100644 index 0000000..1dd3c20 --- /dev/null +++ b/src/quant_platform_kit/data/research_input.py @@ -0,0 +1,362 @@ +"""Pure validation and canonicalization for research-input manifest v1.""" + +from __future__ import annotations + +from collections.abc import Mapping as _Mapping +from datetime import date as _date +from datetime import datetime as _datetime +import hashlib as _hashlib +import json as _json +import re as _re +from typing import TYPE_CHECKING as _TYPE_CHECKING +from zoneinfo import ZoneInfo as _ZoneInfo +from zoneinfo import available_timezones as _available_timezones + +del annotations + +if _TYPE_CHECKING: + from typing import Mapping + + +__all__ = [ + "InvalidResearchInputEvidence", + "validate_research_input_manifest", + "canonical_research_input_manifest_bytes", + "research_input_manifest_sha256", + "read_research_input_manifest_json", +] + + +_TOP_LEVEL_REQUIRED = frozenset( + { + "schema_version", + "manifest_id", + "research_input_contract_id", + "domain", + "profile", + "artifact_type", + "observed_at", + "effective_at", + "as_of", + "producer", + "calendar", + "adjustment", + "sources", + "members", + } +) +_PRODUCER_REQUIRED = frozenset( + {"repository", "commit_sha", "tree_sha", "tool", "tool_version"} +) +_CALENDAR_REQUIRED = frozenset( + {"calendar_id", "timezone", "session_date", "source", "source_revision"} +) +_ADJUSTMENT_REQUIRED = frozenset({"policy", "source", "source_revision"}) +_SOURCE_REQUIRED = frozenset( + {"source_id", "revision", "observed_at", "content_sha256"} +) +_MEMBER_REQUIRED = frozenset({"path", "media_type", "size_bytes", "sha256"}) +_ADJUSTMENT_POLICIES = frozenset( + {"raw", "split_adjusted", "total_return_adjusted"} +) +_COMMIT_PATTERN = _re.compile(r"[0-9a-f]{40}") +_SHA256_PATTERN = _re.compile(r"[0-9a-f]{64}") +_DATE_PATTERN = _re.compile(r"\d{4}-\d{2}-\d{2}") +_TIMESTAMP_PATTERN = _re.compile( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?" + r"(?:Z|[+-]\d{2}:\d{2})" +) + + +class InvalidResearchInputEvidence(ValueError): + """The manifest is not strict, canonical research-input evidence.""" + + +def _fail(message: str) -> None: + raise InvalidResearchInputEvidence(message) + + +def _object( + value: object, + required: frozenset[str], + optional: frozenset[str] = frozenset(), +) -> _Mapping[str, object]: + if not isinstance(value, _Mapping): + _fail("expected an object") + if any(type(key) is not str for key in value): + _fail("object keys must be strings") + keys = frozenset(value) + if not required <= keys or not keys <= required | optional: + _fail("object fields do not match the frozen contract") + return value + + +def _nonempty(value: object) -> str: + if type(value) is not str or not value.strip(): + _fail("expected a non-empty string") + return value + + +def _matches(value: object, pattern: _re.Pattern[str]) -> str: + text = _nonempty(value) + if pattern.fullmatch(text) is None: + _fail("string does not match the frozen format") + return text + + +def _timestamp(value: object) -> tuple[str, _datetime]: + text = _nonempty(value) + if _TIMESTAMP_PATTERN.fullmatch(text) is None: + _fail("timestamp must use strict ISO-8601 date-time syntax") + try: + parsed = _datetime.fromisoformat( + text[:-1] + "+00:00" if text.endswith("Z") else text + ) + if parsed.tzinfo is None or parsed.utcoffset() is None: + _fail("timestamp must be timezone-aware") + except InvalidResearchInputEvidence: + raise + except Exception: + _fail("timestamp must be valid ISO-8601") + return text, parsed + + +def _calendar_date(value: object) -> str: + text = _nonempty(value) + if _DATE_PATTERN.fullmatch(text) is None: + _fail("session_date must use YYYY-MM-DD") + try: + _date.fromisoformat(text) + except Exception: + _fail("session_date must be a valid ISO date") + return text + + +def _timezone(value: object) -> str: + text = _nonempty(value) + try: + if text not in _available_timezones(): + _fail("timezone must be a valid IANA timezone") + _ZoneInfo(text) + except Exception: + _fail("timezone must be a valid IANA timezone") + return text + + +def _member_path(value: object) -> str: + path = _nonempty(value) + if path.startswith("/") or "\\" in path or "\x00" in path: + _fail("member path must be relative POSIX") + if any(part in {".", ".."} for part in path.split("/")): + _fail("member path contains an unsafe segment") + return path + + +def _canonical_order(value: object) -> object: + if isinstance(value, dict): + return {key: _canonical_order(value[key]) for key in sorted(value)} + if isinstance(value, list): + return [_canonical_order(child) for child in value] + return value + + +def _validate(manifest: Mapping[str, object]) -> dict[str, object]: + root = _object( + manifest, + _TOP_LEVEL_REQUIRED, + frozenset({"parent_manifest_sha256"}), + ) + if root["schema_version"] != "research_input_manifest.v1": + _fail("schema_version is invalid") + + result: dict[str, object] = { + "schema_version": "research_input_manifest.v1", + "manifest_id": _nonempty(root["manifest_id"]), + "research_input_contract_id": _nonempty( + root["research_input_contract_id"] + ), + "domain": _nonempty(root["domain"]), + "profile": _nonempty(root["profile"]), + "artifact_type": _nonempty(root["artifact_type"]), + } + observed_text, observed_at = _timestamp(root["observed_at"]) + effective_text, effective_at = _timestamp(root["effective_at"]) + as_of_text, as_of = _timestamp(root["as_of"]) + if observed_at > as_of or effective_at > as_of: + _fail("manifest timestamps exceed as_of") + result.update( + { + "observed_at": observed_text, + "effective_at": effective_text, + "as_of": as_of_text, + } + ) + + producer = _object(root["producer"], _PRODUCER_REQUIRED) + result["producer"] = { + "repository": _nonempty(producer["repository"]), + "commit_sha": _matches(producer["commit_sha"], _COMMIT_PATTERN), + "tree_sha": _matches(producer["tree_sha"], _COMMIT_PATTERN), + "tool": _nonempty(producer["tool"]), + "tool_version": _nonempty(producer["tool_version"]), + } + + calendar = _object(root["calendar"], _CALENDAR_REQUIRED) + result["calendar"] = { + "calendar_id": _nonempty(calendar["calendar_id"]), + "timezone": _timezone(calendar["timezone"]), + "session_date": _calendar_date(calendar["session_date"]), + "source": _nonempty(calendar["source"]), + "source_revision": _nonempty(calendar["source_revision"]), + } + + adjustment = _object(root["adjustment"], _ADJUSTMENT_REQUIRED) + policy = _nonempty(adjustment["policy"]) + if policy not in _ADJUSTMENT_POLICIES: + _fail("adjustment policy is invalid") + result["adjustment"] = { + "policy": policy, + "source": _nonempty(adjustment["source"]), + "source_revision": _nonempty(adjustment["source_revision"]), + } + + sources_value = root["sources"] + if type(sources_value) is not list or not sources_value: + _fail("sources must be a non-empty array") + sources: list[dict[str, object]] = [] + source_ids: list[str] = [] + for source_value in sources_value: + source = _object(source_value, _SOURCE_REQUIRED) + source_id = _nonempty(source["source_id"]) + source_observed_text, source_observed_at = _timestamp( + source["observed_at"] + ) + if source_observed_at > as_of: + _fail("source observed_at exceeds as_of") + source_ids.append(source_id) + sources.append( + { + "source_id": source_id, + "revision": _nonempty(source["revision"]), + "observed_at": source_observed_text, + "content_sha256": _matches( + source["content_sha256"], _SHA256_PATTERN + ), + } + ) + if source_ids != sorted(source_ids) or len(source_ids) != len(set(source_ids)): + _fail("sources must be strictly ordered and unique by source_id") + result["sources"] = sources + + members_value = root["members"] + if type(members_value) is not list or not members_value: + _fail("members must be a non-empty array") + members: list[dict[str, object]] = [] + member_paths: list[str] = [] + for member_value in members_value: + member = _object(member_value, _MEMBER_REQUIRED) + path = _member_path(member["path"]) + size_bytes = member["size_bytes"] + if type(size_bytes) is not int or size_bytes < 0: + _fail("size_bytes must be a non-negative integer") + member_paths.append(path) + members.append( + { + "path": path, + "media_type": _nonempty(member["media_type"]), + "size_bytes": size_bytes, + "sha256": _matches(member["sha256"], _SHA256_PATTERN), + } + ) + if member_paths != sorted(member_paths) or len(member_paths) != len( + set(member_paths) + ): + _fail("members must be strictly ordered and unique by path") + result["members"] = members + + if "parent_manifest_sha256" in root: + result["parent_manifest_sha256"] = _matches( + root["parent_manifest_sha256"], _SHA256_PATTERN + ) + return _canonical_order(result) + + +def validate_research_input_manifest( + manifest: Mapping[str, object], +) -> dict[str, object]: + """Validate and return an independent canonical-order manifest.""" + try: + return _validate(manifest) + except InvalidResearchInputEvidence: + raise + except Exception: + raise InvalidResearchInputEvidence("invalid research-input manifest") from None + + +def canonical_research_input_manifest_bytes( + manifest: Mapping[str, object], +) -> bytes: + """Return strict canonical UTF-8 JSON bytes without a trailing newline.""" + try: + validated = validate_research_input_manifest(manifest) + return _json.dumps( + validated, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + except InvalidResearchInputEvidence: + raise + except Exception: + raise InvalidResearchInputEvidence("cannot canonicalize manifest") from None + + +def research_input_manifest_sha256(manifest: Mapping[str, object]) -> str: + """Return the SHA-256 digest of strict canonical manifest bytes.""" + try: + return _hashlib.sha256( + canonical_research_input_manifest_bytes(manifest) + ).hexdigest() + except InvalidResearchInputEvidence: + raise + except Exception: + raise InvalidResearchInputEvidence("cannot digest manifest") from None + + +def _reject_duplicate_keys( + pairs: list[tuple[str, object]], +) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + _fail("duplicate JSON object key") + result[key] = value + return result + + +def _reject_nonfinite(_value: str) -> object: + _fail("non-finite JSON constants are forbidden") + + +def read_research_input_manifest_json(payload: bytes | str) -> dict[str, object]: + """Strictly parse and validate one JSON object from bytes or text.""" + try: + if type(payload) is bytes: + text = payload.decode("utf-8") + elif type(payload) is str: + text = payload + else: + _fail("payload must be bytes or str") + parsed = _json.loads( + text, + object_pairs_hook=_reject_duplicate_keys, + parse_constant=_reject_nonfinite, + ) + if not isinstance(parsed, dict): + _fail("manifest JSON root must be an object") + return validate_research_input_manifest(parsed) + except InvalidResearchInputEvidence: + raise + except Exception: + raise InvalidResearchInputEvidence("invalid research-input JSON") from None diff --git a/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json b/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json new file mode 100644 index 0000000..d6def63 --- /dev/null +++ b/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json @@ -0,0 +1,226 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Research Input Manifest V1", + "type": "object", + "required": [ + "schema_version", + "manifest_id", + "research_input_contract_id", + "domain", + "profile", + "artifact_type", + "observed_at", + "effective_at", + "as_of", + "producer", + "calendar", + "adjustment", + "sources", + "members" + ], + "additionalProperties": false, + "properties": { + "schema_version": { + "const": "research_input_manifest.v1" + }, + "manifest_id": { + "$ref": "#/$defs/nonemptyString" + }, + "research_input_contract_id": { + "$ref": "#/$defs/nonemptyString" + }, + "domain": { + "$ref": "#/$defs/nonemptyString" + }, + "profile": { + "$ref": "#/$defs/nonemptyString" + }, + "artifact_type": { + "$ref": "#/$defs/nonemptyString" + }, + "observed_at": { + "type": "string", + "format": "date-time" + }, + "effective_at": { + "type": "string", + "format": "date-time" + }, + "as_of": { + "type": "string", + "format": "date-time" + }, + "producer": { + "$ref": "#/$defs/producer" + }, + "calendar": { + "$ref": "#/$defs/calendar" + }, + "adjustment": { + "$ref": "#/$defs/adjustment" + }, + "sources": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/source" + } + }, + "members": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/member" + } + }, + "parent_manifest_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "$defs": { + "nonemptyString": { + "type": "string", + "minLength": 1, + "pattern": "\\S" + }, + "producer": { + "type": "object", + "required": [ + "repository", + "commit_sha", + "tree_sha", + "tool", + "tool_version" + ], + "additionalProperties": false, + "properties": { + "repository": { + "$ref": "#/$defs/nonemptyString" + }, + "commit_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "tree_sha": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "tool": { + "$ref": "#/$defs/nonemptyString" + }, + "tool_version": { + "$ref": "#/$defs/nonemptyString" + } + } + }, + "calendar": { + "type": "object", + "required": [ + "calendar_id", + "timezone", + "session_date", + "source", + "source_revision" + ], + "additionalProperties": false, + "properties": { + "calendar_id": { + "$ref": "#/$defs/nonemptyString" + }, + "timezone": { + "$ref": "#/$defs/nonemptyString" + }, + "session_date": { + "type": "string", + "format": "date" + }, + "source": { + "$ref": "#/$defs/nonemptyString" + }, + "source_revision": { + "$ref": "#/$defs/nonemptyString" + } + } + }, + "adjustment": { + "type": "object", + "required": [ + "policy", + "source", + "source_revision" + ], + "additionalProperties": false, + "properties": { + "policy": { + "type": "string", + "enum": [ + "raw", + "split_adjusted", + "total_return_adjusted" + ] + }, + "source": { + "$ref": "#/$defs/nonemptyString" + }, + "source_revision": { + "$ref": "#/$defs/nonemptyString" + } + } + }, + "source": { + "type": "object", + "required": [ + "source_id", + "revision", + "observed_at", + "content_sha256" + ], + "additionalProperties": false, + "properties": { + "source_id": { + "$ref": "#/$defs/nonemptyString" + }, + "revision": { + "$ref": "#/$defs/nonemptyString" + }, + "observed_at": { + "type": "string", + "format": "date-time" + }, + "content_sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + "member": { + "type": "object", + "required": [ + "path", + "media_type", + "size_bytes", + "sha256" + ], + "additionalProperties": false, + "properties": { + "path": { + "type": "string", + "minLength": 1, + "pattern": "^(?!/)(?!.*\\\\)(?!.*\\u0000)(?!.*(?:^|/)\\.{1,2}(?:/|$)).+$" + }, + "media_type": { + "$ref": "#/$defs/nonemptyString" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + } + } +} diff --git a/tests/test_research_input_manifest_contract.py b/tests/test_research_input_manifest_contract.py new file mode 100644 index 0000000..0f8fbce --- /dev/null +++ b/tests/test_research_input_manifest_contract.py @@ -0,0 +1,631 @@ +from __future__ import annotations + +import hashlib +import importlib +import inspect +import json +from pathlib import Path +import re +from types import MappingProxyType + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA_PATH = ( + ROOT + / "src" + / "quant_platform_kit" + / "schemas" + / "research-input-manifest.v1.schema.json" +) +PUBLIC_SURFACE = { + "InvalidResearchInputEvidence", + "validate_research_input_manifest", + "canonical_research_input_manifest_bytes", + "research_input_manifest_sha256", + "read_research_input_manifest_json", +} + + +# Frozen contract-to-test traceability matrix (contract sections 4-6): +# - exact Python surface and normalized failures: test_exact_python_surface, +# test_all_public_functions_normalize_failures +# - complete fields / exact objects / no defaults: test_valid_manifest_and_deep_copy, +# test_rejects_missing_or_unknown_fields +# - required non-blank strings and exact SHA shapes: +# test_rejects_blank_strings, test_rejects_bad_hashes +# - timestamps, ordering, calendar timezone/date, adjustment policy: +# test_rejects_bad_timestamps_and_time_order, +# test_rejects_bad_calendar_and_adjustment +# - ordered unique source/member identities and safe POSIX paths: +# test_rejects_bad_source_identity_order, +# test_rejects_bad_member_identity_or_path +# - JSON-compatible scalar discipline: +# test_rejects_bad_size_or_non_json_values +# - canonical bytes and digest: test_canonical_bytes_and_digest_are_deterministic +# - canonical-order independent deep return: +# test_valid_manifest_and_deep_copy, +# test_mapping_key_order_does_not_change_canonical_output +# - strict JSON readback: test_strict_json_readback, +# test_readback_rejects_duplicate_keys_at_every_depth +# - schema structural isomorphism / no fake semantic extensions: +# test_schema_matches_frozen_structural_contract + + +def _module(): + return importlib.import_module("quant_platform_kit.data.research_input") + + +def _valid_manifest() -> dict[str, object]: + return { + "schema_version": "research_input_manifest.v1", + "manifest_id": "pit-us-equity-20260730", + "research_input_contract_id": "shared-us-equity-pit.v1", + "domain": "us_equity", + "profile": "shared_daily", + "artifact_type": "immutable_pit_bundle", + "observed_at": "2026-07-30T08:00:00+00:00", + "effective_at": "2026-07-29T13:30:00-04:00", + "as_of": "2026-07-30T09:00:00Z", + "producer": { + "repository": "QuantStrategyLab/UsEquitySnapshotPipelines", + "commit_sha": "a" * 40, + "tree_sha": "b" * 40, + "tool": "build_shared_pit", + "tool_version": "1.0.0", + }, + "calendar": { + "calendar_id": "XNYS", + "timezone": "America/New_York", + "session_date": "2026-07-29", + "source": "exchange_calendars", + "source_revision": "4.11.1", + }, + "adjustment": { + "policy": "split_adjusted", + "source": "vendor-adjustments", + "source_revision": "2026-07-30", + }, + "sources": [ + { + "source_id": "daily-bars", + "revision": "2026-07-30T07:00:00Z", + "observed_at": "2026-07-30T07:30:00Z", + "content_sha256": "c" * 64, + }, + { + "source_id": "security-master", + "revision": "2026-07-30", + "observed_at": "2026-07-30T07:45:00+00:00", + "content_sha256": "d" * 64, + }, + ], + "members": [ + { + "path": "data/bars.parquet", + "media_type": "application/vnd.apache.parquet", + "size_bytes": 1024, + "sha256": "e" * 64, + }, + { + "path": "metadata/universe.json", + "media_type": "application/json", + "size_bytes": 128, + "sha256": "f" * 64, + }, + ], + "parent_manifest_sha256": "0" * 64, + } + + +def _assert_invalid(manifest: object) -> None: + module = _module() + with pytest.raises(module.InvalidResearchInputEvidence): + module.validate_research_input_manifest(manifest) + + +def _load_schema() -> dict[str, object]: + def reject_duplicate_keys(pairs): + result = {} + for key, value in pairs: + assert key not in result, f"duplicate schema key: {key}" + result[key] = value + return result + + return json.loads( + SCHEMA_PATH.read_text(encoding="utf-8"), + object_pairs_hook=reject_duplicate_keys, + ) + + +def _reverse_mapping_keys(value): + if isinstance(value, dict): + return { + key: _reverse_mapping_keys(child) + for key, child in reversed(tuple(value.items())) + } + if isinstance(value, list): + return [_reverse_mapping_keys(child) for child in value] + return value + + +def _assert_canonical_key_order(value: object) -> None: + if isinstance(value, dict): + assert list(value) == sorted(value) + for child in value.values(): + _assert_canonical_key_order(child) + elif isinstance(value, list): + for child in value: + _assert_canonical_key_order(child) + + +def test_exact_python_surface() -> None: + module = _module() + + assert set(module.__all__) == PUBLIC_SURFACE + assert { + name for name in vars(module) if not name.startswith("_") + } == PUBLIC_SURFACE + assert issubclass(module.InvalidResearchInputEvidence, ValueError) + assert list(inspect.signature(module.validate_research_input_manifest).parameters) == [ + "manifest" + ] + assert list( + inspect.signature(module.canonical_research_input_manifest_bytes).parameters + ) == ["manifest"] + assert list(inspect.signature(module.research_input_manifest_sha256).parameters) == [ + "manifest" + ] + assert list(inspect.signature(module.read_research_input_manifest_json).parameters) == [ + "payload" + ] + + +def test_valid_manifest_and_deep_copy() -> None: + module = _module() + original = _valid_manifest() + validated = module.validate_research_input_manifest(MappingProxyType(original)) + + assert validated == original + assert type(validated) is dict + _assert_canonical_key_order(validated) + + original["producer"]["tool"] = "mutated" + original["sources"][0]["revision"] = "mutated" + original["members"].append({"path": "mutated"}) + assert validated["producer"]["tool"] == "build_shared_pit" + assert validated["sources"][0]["revision"] == "2026-07-30T07:00:00Z" + assert len(validated["members"]) == 2 + + validated["producer"]["tool"] = "returned-mutation" + fresh = module.validate_research_input_manifest(_valid_manifest()) + assert fresh["producer"]["tool"] == "build_shared_pit" + + +@pytest.mark.parametrize( + ("path", "key"), + [ + *(((), key) for key in ( + "schema_version", + "manifest_id", + "research_input_contract_id", + "domain", + "profile", + "artifact_type", + "observed_at", + "effective_at", + "as_of", + "producer", + "calendar", + "adjustment", + "sources", + "members", + )), + *((("producer",), key) for key in ( + "repository", + "commit_sha", + "tree_sha", + "tool", + "tool_version", + )), + *((("calendar",), key) for key in ( + "calendar_id", + "timezone", + "session_date", + "source", + "source_revision", + )), + *((("adjustment",), key) for key in ( + "policy", + "source", + "source_revision", + )), + *((("sources", 0), key) for key in ( + "source_id", + "revision", + "observed_at", + "content_sha256", + )), + *((("members", 0), key) for key in ( + "path", + "media_type", + "size_bytes", + "sha256", + )), + ], +) +def test_rejects_every_missing_required_field(path, key) -> None: + missing = _valid_manifest() + target = missing + for part in path: + target = target[part] + target.pop(key) + _assert_invalid(missing) + + +@pytest.mark.parametrize( + "path", + [(), ("producer",), ("calendar",), ("adjustment",), ("sources", 0), ("members", 0)], +) +def test_rejects_unknown_fields_in_every_object(path) -> None: + unknown = _valid_manifest() + target = unknown + for part in path: + target = target[part] + target["unknown"] = "forbidden" + _assert_invalid(unknown) + + +@pytest.mark.parametrize( + ("path", "key"), + [ + ((), "manifest_id"), + ((), "research_input_contract_id"), + ((), "domain"), + ((), "profile"), + ((), "artifact_type"), + (("producer",), "repository"), + (("producer",), "tool"), + (("producer",), "tool_version"), + (("calendar",), "calendar_id"), + (("calendar",), "source"), + (("calendar",), "source_revision"), + (("adjustment",), "source"), + (("adjustment",), "source_revision"), + (("sources", 0), "source_id"), + (("sources", 0), "revision"), + (("members", 0), "media_type"), + ], +) +def test_rejects_blank_strings(path, key) -> None: + manifest = _valid_manifest() + target = manifest + for part in path: + target = target[part] + target[key] = " \t\n" + _assert_invalid(manifest) + + +@pytest.mark.parametrize( + ("path", "key", "value"), + [ + ((), "schema_version", "research_input_manifest.v2"), + (("producer",), "commit_sha", "A" * 40), + (("producer",), "commit_sha", "a" * 39), + (("producer",), "tree_sha", "g" * 40), + (("sources", 0), "content_sha256", "C" * 64), + (("sources", 0), "content_sha256", "c" * 63), + (("members", 0), "sha256", "z" * 64), + ((), "parent_manifest_sha256", "0" * 65), + ], +) +def test_rejects_bad_hashes(path, key, value) -> None: + manifest = _valid_manifest() + target = manifest + for part in path: + target = target[part] + target[key] = value + _assert_invalid(manifest) + + +@pytest.mark.parametrize( + ("path", "key", "value"), + [ + ((), "observed_at", "2026-07-30T08:00:00"), + ((), "observed_at", "2026-07-30Q08:00:00Z"), + ((), "effective_at", "not-a-timestamp"), + ((), "as_of", "2026-07-30"), + (("sources", 0), "observed_at", "2026-07-30T07:30:00"), + ((), "observed_at", "2026-07-30T10:00:00Z"), + ((), "effective_at", "2026-07-30T10:00:00Z"), + (("sources", 0), "observed_at", "2026-07-30T10:00:00Z"), + ], +) +def test_rejects_bad_timestamps_and_time_order(path, key, value) -> None: + manifest = _valid_manifest() + target = manifest + for part in path: + target = target[part] + target[key] = value + _assert_invalid(manifest) + + +@pytest.mark.parametrize( + ("path", "key", "value"), + [ + (("calendar",), "timezone", "UTC+08:00"), + (("calendar",), "timezone", "Not/A_Zone"), + (("calendar",), "timezone", "posixrules"), + (("calendar",), "session_date", "2026-02-30"), + (("calendar",), "session_date", "20260730"), + (("adjustment",), "policy", "back_adjusted"), + ], +) +def test_rejects_bad_calendar_and_adjustment(path, key, value) -> None: + manifest = _valid_manifest() + target = manifest + for part in path: + target = target[part] + target[key] = value + _assert_invalid(manifest) + + +def test_rejects_bad_source_identity_order() -> None: + empty = _valid_manifest() + empty["sources"] = [] + _assert_invalid(empty) + + unsorted = _valid_manifest() + unsorted["sources"].reverse() + _assert_invalid(unsorted) + + duplicate = _valid_manifest() + duplicate["sources"][1]["source_id"] = duplicate["sources"][0]["source_id"] + _assert_invalid(duplicate) + + +@pytest.mark.parametrize( + "path", + [ + "", + " \t", + "/absolute/data.json", + ".", + "./data.json", + "../data.json", + "data/./bars.parquet", + "data/../bars.parquet", + r"data\bars.parquet", + "data/\x00bars.parquet", + ], +) +def test_rejects_bad_member_identity_or_path(path) -> None: + manifest = _valid_manifest() + manifest["members"][0]["path"] = path + _assert_invalid(manifest) + + +def test_rejects_empty_unsorted_or_duplicate_members() -> None: + empty = _valid_manifest() + empty["members"] = [] + _assert_invalid(empty) + + unsorted = _valid_manifest() + unsorted["members"].reverse() + _assert_invalid(unsorted) + + duplicate = _valid_manifest() + duplicate["members"][1]["path"] = duplicate["members"][0]["path"] + _assert_invalid(duplicate) + + +@pytest.mark.parametrize("value", [-1, True, False, 1.0, float("nan"), float("inf"), -float("inf"), set()]) +def test_rejects_bad_size_or_non_json_values(value) -> None: + manifest = _valid_manifest() + manifest["members"][0]["size_bytes"] = value + _assert_invalid(manifest) + + +def test_canonical_bytes_and_digest_are_deterministic() -> None: + module = _module() + manifest = _valid_manifest() + manifest["profile"] = "共享输入" + expected = json.dumps( + manifest, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + actual = module.canonical_research_input_manifest_bytes(manifest) + assert actual == expected + assert "共享输入".encode() in actual + assert b"\\u" not in actual + assert not actual.endswith(b"\n") + assert module.research_input_manifest_sha256(manifest) == hashlib.sha256( + expected + ).hexdigest() + assert "sha256" not in module.validate_research_input_manifest(manifest) + + +def test_mapping_key_order_does_not_change_canonical_output() -> None: + module = _module() + manifest = _valid_manifest() + reordered = _reverse_mapping_keys(manifest) + + assert module.validate_research_input_manifest(reordered) == manifest + assert module.canonical_research_input_manifest_bytes( + reordered + ) == module.canonical_research_input_manifest_bytes(manifest) + + +def test_optional_parent_and_untrimmed_values_are_preserved() -> None: + module = _module() + manifest = _valid_manifest() + manifest.pop("parent_manifest_sha256") + manifest["manifest_id"] = " stable identity " + + validated = module.validate_research_input_manifest(manifest) + assert "parent_manifest_sha256" not in validated + assert validated["manifest_id"] == " stable identity " + + +@pytest.mark.parametrize("path", [(), ("producer",), ("sources", 0), ("members", 0)]) +def test_rejects_non_string_object_keys(path) -> None: + manifest = _valid_manifest() + target = manifest + for part in path: + target = target[part] + target[1] = "not-json" + _assert_invalid(manifest) + + +@pytest.mark.parametrize( + "payload", + [ + b"\xff", + "[]", + "null", + '{"schema_version": NaN}', + '{"schema_version": Infinity}', + '{"schema_version": -Infinity}', + '{"schema_version":"research_input_manifest.v1"} trailing', + "{}{}", + ], +) +def test_strict_json_readback_rejects_invalid_payload(payload) -> None: + module = _module() + with pytest.raises(module.InvalidResearchInputEvidence): + module.read_research_input_manifest_json(payload) + + +def test_strict_json_readback_accepts_bytes_and_text() -> None: + module = _module() + manifest = _valid_manifest() + payload = module.canonical_research_input_manifest_bytes(manifest) + + assert module.read_research_input_manifest_json(payload) == manifest + assert module.read_research_input_manifest_json(payload.decode("utf-8")) == manifest + + +@pytest.mark.parametrize( + "payload", + [ + '{"manifest_id":"one","manifest_id":"two"}', + '{"producer":{"tool":"one","tool":"two"}}', + '{"sources":[{"source_id":"one","source_id":"two"}]}', + ], +) +def test_readback_rejects_duplicate_keys_at_every_depth(payload) -> None: + module = _module() + with pytest.raises(module.InvalidResearchInputEvidence): + module.read_research_input_manifest_json(payload) + + +def test_all_public_functions_normalize_failures() -> None: + module = _module() + invalid = {"schema_version": "research_input_manifest.v1"} + + for function in ( + module.validate_research_input_manifest, + module.canonical_research_input_manifest_bytes, + module.research_input_manifest_sha256, + ): + with pytest.raises(module.InvalidResearchInputEvidence) as exc_info: + function(invalid) + assert type(exc_info.value) is module.InvalidResearchInputEvidence + + for payload in (object(), b"\xff", "{"): + with pytest.raises(module.InvalidResearchInputEvidence) as exc_info: + module.read_research_input_manifest_json(payload) + assert type(exc_info.value) is module.InvalidResearchInputEvidence + + +def test_schema_matches_frozen_structural_contract() -> None: + schema = _load_schema() + required = { + "schema_version", + "manifest_id", + "research_input_contract_id", + "domain", + "profile", + "artifact_type", + "observed_at", + "effective_at", + "as_of", + "producer", + "calendar", + "adjustment", + "sources", + "members", + } + + assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + assert schema["type"] == "object" + assert set(schema["required"]) == required + assert schema["additionalProperties"] is False + assert schema["properties"]["schema_version"] == { + "const": "research_input_manifest.v1" + } + assert schema["properties"]["parent_manifest_sha256"]["pattern"] == "^[0-9a-f]{64}$" + assert schema["$defs"]["nonemptyString"]["pattern"] == "\\S" + + producer = schema["$defs"]["producer"] + assert set(producer["required"]) == { + "repository", + "commit_sha", + "tree_sha", + "tool", + "tool_version", + } + assert producer["additionalProperties"] is False + assert producer["properties"]["commit_sha"]["pattern"] == "^[0-9a-f]{40}$" + assert producer["properties"]["tree_sha"]["pattern"] == "^[0-9a-f]{40}$" + + calendar = schema["$defs"]["calendar"] + assert set(calendar["required"]) == { + "calendar_id", + "timezone", + "session_date", + "source", + "source_revision", + } + assert calendar["additionalProperties"] is False + assert calendar["properties"]["session_date"]["format"] == "date" + + adjustment = schema["$defs"]["adjustment"] + assert set(adjustment["required"]) == {"policy", "source", "source_revision"} + assert adjustment["additionalProperties"] is False + assert adjustment["properties"]["policy"]["enum"] == [ + "raw", + "split_adjusted", + "total_return_adjusted", + ] + + source = schema["$defs"]["source"] + assert set(source["required"]) == { + "source_id", + "revision", + "observed_at", + "content_sha256", + } + assert source["additionalProperties"] is False + assert source["properties"]["observed_at"]["format"] == "date-time" + assert source["properties"]["content_sha256"]["pattern"] == "^[0-9a-f]{64}$" + + member = schema["$defs"]["member"] + assert set(member["required"]) == {"path", "media_type", "size_bytes", "sha256"} + assert member["additionalProperties"] is False + assert member["properties"]["size_bytes"] == {"type": "integer", "minimum": 0} + assert member["properties"]["sha256"]["pattern"] == "^[0-9a-f]{64}$" + assert re.fullmatch(member["properties"]["path"]["pattern"], "data/\x00x") is None + + assert schema["properties"]["sources"]["minItems"] == 1 + assert schema["properties"]["members"]["minItems"] == 1 + assert all( + schema["properties"][name]["format"] == "date-time" + for name in ("observed_at", "effective_at", "as_of") + ) + assert "x-" not in json.dumps(schema) From d1e82553b68913a9cde71483cad436b59b5a4f8d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:47:16 +0800 Subject: [PATCH 2/2] fix: harden research input manifest validation Co-Authored-By: Codex --- src/quant_platform_kit/data/research_input.py | 71 ++++++++++----- .../research-input-manifest.v1.schema.json | 2 +- .../test_research_input_manifest_contract.py | 86 ++++++++++++++++++- 3 files changed, 137 insertions(+), 22 deletions(-) diff --git a/src/quant_platform_kit/data/research_input.py b/src/quant_platform_kit/data/research_input.py index 1dd3c20..02ce28e 100644 --- a/src/quant_platform_kit/data/research_input.py +++ b/src/quant_platform_kit/data/research_input.py @@ -1,23 +1,24 @@ -"""Pure validation and canonicalization for research-input manifest v1.""" +"""Pure validation and canonicalization for research-input manifest v1. + +The runtime must provide an IANA timezone database through the operating +system or the ``tzdata`` package; validation fails closed when it is absent. +""" from __future__ import annotations from collections.abc import Mapping as _Mapping from datetime import date as _date from datetime import datetime as _datetime +from datetime import timezone as _datetime_timezone +from decimal import Decimal as _Decimal import hashlib as _hashlib import json as _json import re as _re -from typing import TYPE_CHECKING as _TYPE_CHECKING from zoneinfo import ZoneInfo as _ZoneInfo from zoneinfo import available_timezones as _available_timezones del annotations -if _TYPE_CHECKING: - from typing import Mapping - - __all__ = [ "InvalidResearchInputEvidence", "validate_research_input_manifest", @@ -63,9 +64,13 @@ _SHA256_PATTERN = _re.compile(r"[0-9a-f]{64}") _DATE_PATTERN = _re.compile(r"\d{4}-\d{2}-\d{2}") _TIMESTAMP_PATTERN = _re.compile( - r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?" - r"(?:Z|[+-]\d{2}:\d{2})" + r"(?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})" + r"(?:\.(?P\d+))?" + r"(?PZ|[+-]\d{2}:\d{2})" ) +_EPOCH = _datetime(1970, 1, 1, tzinfo=_datetime_timezone.utc) +_HOST_LOCAL_TIMEZONE_KEYS = frozenset({"Factory", "localtime", "posixrules"}) +_HOST_LOCAL_TIMEZONE_PREFIXES = ("posix/", "right/") class InvalidResearchInputEvidence(ValueError): @@ -94,6 +99,8 @@ def _object( def _nonempty(value: object) -> str: if type(value) is not str or not value.strip(): _fail("expected a non-empty string") + if any("\ud800" <= character <= "\udfff" for character in value): + _fail("strings must contain only Unicode scalar values") return value @@ -104,21 +111,32 @@ def _matches(value: object, pattern: _re.Pattern[str]) -> str: return text -def _timestamp(value: object) -> tuple[str, _datetime]: +def _timestamp(value: object) -> tuple[str, _Decimal]: text = _nonempty(value) - if _TIMESTAMP_PATTERN.fullmatch(text) is None: + match = _TIMESTAMP_PATTERN.fullmatch(text) + if match is None: _fail("timestamp must use strict ISO-8601 date-time syntax") + if match.group("offset") == "-00:00": + _fail("timestamp must not use an unknown UTC offset") try: parsed = _datetime.fromisoformat( text[:-1] + "+00:00" if text.endswith("Z") else text ) if parsed.tzinfo is None or parsed.utcoffset() is None: _fail("timestamp must be timezone-aware") + whole_utc = parsed.replace(microsecond=0).astimezone( + _datetime_timezone.utc + ) + delta = whole_utc - _EPOCH + exact = _Decimal(delta.days * 86_400 + delta.seconds) + fraction = match.group("fraction") + if fraction is not None: + exact += _Decimal(f"0.{fraction}") except InvalidResearchInputEvidence: raise except Exception: _fail("timestamp must be valid ISO-8601") - return text, parsed + return text, exact def _calendar_date(value: object) -> str: @@ -135,11 +153,21 @@ def _calendar_date(value: object) -> str: def _timezone(value: object) -> str: text = _nonempty(value) try: - if text not in _available_timezones(): - _fail("timezone must be a valid IANA timezone") - _ZoneInfo(text) + available = _available_timezones() except Exception: + _fail("IANA timezone database is unavailable") + if not available: + _fail("IANA timezone database is unavailable") + if ( + text in _HOST_LOCAL_TIMEZONE_KEYS + or text.startswith(_HOST_LOCAL_TIMEZONE_PREFIXES) + or text not in available + ): _fail("timezone must be a valid IANA timezone") + try: + _ZoneInfo(text) + except Exception: + _fail("IANA timezone database cannot load the requested timezone") return text @@ -147,7 +175,7 @@ def _member_path(value: object) -> str: path = _nonempty(value) if path.startswith("/") or "\\" in path or "\x00" in path: _fail("member path must be relative POSIX") - if any(part in {".", ".."} for part in path.split("/")): + if any(part in {"", ".", ".."} for part in path.split("/")): _fail("member path contains an unsafe segment") return path @@ -160,7 +188,7 @@ def _canonical_order(value: object) -> object: return value -def _validate(manifest: Mapping[str, object]) -> dict[str, object]: +def _validate(manifest: _Mapping[str, object]) -> dict[str, object]: root = _object( manifest, _TOP_LEVEL_REQUIRED, @@ -282,9 +310,12 @@ def _validate(manifest: Mapping[str, object]) -> dict[str, object]: def validate_research_input_manifest( - manifest: Mapping[str, object], + manifest: _Mapping[str, object], ) -> dict[str, object]: - """Validate and return an independent canonical-order manifest.""" + """Validate and return an independent canonical-order manifest. + + The runtime must provide a system tzdb or the ``tzdata`` package. + """ try: return _validate(manifest) except InvalidResearchInputEvidence: @@ -294,7 +325,7 @@ def validate_research_input_manifest( def canonical_research_input_manifest_bytes( - manifest: Mapping[str, object], + manifest: _Mapping[str, object], ) -> bytes: """Return strict canonical UTF-8 JSON bytes without a trailing newline.""" try: @@ -312,7 +343,7 @@ def canonical_research_input_manifest_bytes( raise InvalidResearchInputEvidence("cannot canonicalize manifest") from None -def research_input_manifest_sha256(manifest: Mapping[str, object]) -> str: +def research_input_manifest_sha256(manifest: _Mapping[str, object]) -> str: """Return the SHA-256 digest of strict canonical manifest bytes.""" try: return _hashlib.sha256( diff --git a/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json b/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json index d6def63..d9f7f3c 100644 --- a/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json +++ b/src/quant_platform_kit/schemas/research-input-manifest.v1.schema.json @@ -207,7 +207,7 @@ "path": { "type": "string", "minLength": 1, - "pattern": "^(?!/)(?!.*\\\\)(?!.*\\u0000)(?!.*(?:^|/)\\.{1,2}(?:/|$)).+$" + "pattern": "^(?!/)(?!.*//)(?!.*/$)(?!.*\\\\)(?!.*\\u0000)(?!.*(?:^|/)\\.{1,2}(?:/|$)).+$" }, "media_type": { "$ref": "#/$defs/nonemptyString" diff --git a/tests/test_research_input_manifest_contract.py b/tests/test_research_input_manifest_contract.py index 0f8fbce..9bf4900 100644 --- a/tests/test_research_input_manifest_contract.py +++ b/tests/test_research_input_manifest_contract.py @@ -7,6 +7,7 @@ from pathlib import Path import re from types import MappingProxyType +import typing import pytest @@ -37,10 +38,14 @@ # test_rejects_blank_strings, test_rejects_bad_hashes # - timestamps, ordering, calendar timezone/date, adjustment policy: # test_rejects_bad_timestamps_and_time_order, -# test_rejects_bad_calendar_and_adjustment +# test_preserves_fractional_timestamp_precision, +# test_rejects_bad_calendar_and_adjustment, +# test_timezone_validation_fails_closed_without_tzdb # - ordered unique source/member identities and safe POSIX paths: # test_rejects_bad_source_identity_order, # test_rejects_bad_member_identity_or_path +# - Unicode scalar strings and runtime-resolvable annotations: +# test_rejects_lone_surrogates, test_public_annotations_resolve_at_runtime # - JSON-compatible scalar discipline: # test_rejects_bad_size_or_non_json_values # - canonical bytes and digest: test_canonical_bytes_and_digest_are_deterministic @@ -182,6 +187,18 @@ def test_exact_python_surface() -> None: ] +def test_public_annotations_resolve_at_runtime() -> None: + module = _module() + + for function in ( + module.validate_research_input_manifest, + module.canonical_research_input_manifest_bytes, + module.research_input_manifest_sha256, + module.read_research_input_manifest_json, + ): + assert typing.get_type_hints(function) + + def test_valid_manifest_and_deep_copy() -> None: module = _module() original = _valid_manifest() @@ -336,7 +353,9 @@ def test_rejects_bad_hashes(path, key, value) -> None: ((), "observed_at", "2026-07-30Q08:00:00Z"), ((), "effective_at", "not-a-timestamp"), ((), "as_of", "2026-07-30"), + ((), "as_of", "2026-07-30T09:00:00-00:00"), (("sources", 0), "observed_at", "2026-07-30T07:30:00"), + (("sources", 0), "observed_at", "2026-07-30T07:30:00-00:00"), ((), "observed_at", "2026-07-30T10:00:00Z"), ((), "effective_at", "2026-07-30T10:00:00Z"), (("sources", 0), "observed_at", "2026-07-30T10:00:00Z"), @@ -351,12 +370,32 @@ def test_rejects_bad_timestamps_and_time_order(path, key, value) -> None: _assert_invalid(manifest) +@pytest.mark.parametrize( + "path", + [ + ("observed_at",), + ("effective_at",), + ("sources", 0, "observed_at"), + ], +) +def test_preserves_fractional_timestamp_precision(path) -> None: + manifest = _valid_manifest() + manifest["as_of"] = "2026-07-30T09:00:00.00000001Z" + target = manifest + for part in path[:-1]: + target = target[part] + target[path[-1]] = "2026-07-30T09:00:00.00000009Z" + + _assert_invalid(manifest) + + @pytest.mark.parametrize( ("path", "key", "value"), [ (("calendar",), "timezone", "UTC+08:00"), (("calendar",), "timezone", "Not/A_Zone"), (("calendar",), "timezone", "posixrules"), + (("calendar",), "timezone", "localtime"), (("calendar",), "session_date", "2026-02-30"), (("calendar",), "session_date", "20260730"), (("adjustment",), "policy", "back_adjusted"), @@ -371,6 +410,27 @@ def test_rejects_bad_calendar_and_adjustment(path, key, value) -> None: _assert_invalid(manifest) +def test_timezone_validation_fails_closed_without_tzdb(monkeypatch) -> None: + module = _module() + monkeypatch.setattr(module, "_available_timezones", lambda: set()) + + with pytest.raises( + module.InvalidResearchInputEvidence, + match="timezone database", + ): + module.validate_research_input_manifest(_valid_manifest()) + + +def test_rejects_host_local_timezone_alias_even_when_exposed(monkeypatch) -> None: + module = _module() + monkeypatch.setattr(module, "_available_timezones", lambda: {"localtime"}) + monkeypatch.setattr(module, "_ZoneInfo", lambda _key: object()) + + manifest = _valid_manifest() + manifest["calendar"]["timezone"] = "localtime" + _assert_invalid(manifest) + + def test_rejects_bad_source_identity_order() -> None: empty = _valid_manifest() empty["sources"] = [] @@ -396,6 +456,8 @@ def test_rejects_bad_source_identity_order() -> None: "../data.json", "data/./bars.parquet", "data/../bars.parquet", + "data//bars.parquet", + "data/bars.parquet/", r"data\bars.parquet", "data/\x00bars.parquet", ], @@ -427,6 +489,26 @@ def test_rejects_bad_size_or_non_json_values(value) -> None: _assert_invalid(manifest) +@pytest.mark.parametrize( + ("path", "key"), + [ + ((), "manifest_id"), + (("producer",), "tool"), + (("calendar",), "source"), + (("sources", 0), "revision"), + (("members", 0), "media_type"), + ], +) +def test_rejects_lone_surrogates(path, key) -> None: + manifest = _valid_manifest() + target = manifest + for part in path: + target = target[part] + target[key] = "invalid\ud800text" + + _assert_invalid(manifest) + + def test_canonical_bytes_and_digest_are_deterministic() -> None: module = _module() manifest = _valid_manifest() @@ -621,6 +703,8 @@ def test_schema_matches_frozen_structural_contract() -> None: assert member["properties"]["size_bytes"] == {"type": "integer", "minimum": 0} assert member["properties"]["sha256"]["pattern"] == "^[0-9a-f]{64}$" assert re.fullmatch(member["properties"]["path"]["pattern"], "data/\x00x") is None + assert re.fullmatch(member["properties"]["path"]["pattern"], "data//x") is None + assert re.fullmatch(member["properties"]["path"]["pattern"], "data/x/") is None assert schema["properties"]["sources"]["minItems"] == 1 assert schema["properties"]["members"]["minItems"] == 1