diff --git a/CMakeLists.txt b/CMakeLists.txt index 72550f6e8..d930ba056 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -148,6 +148,8 @@ set(PINEFORGE_KERNEL_SOURCES src/native_calendar.cpp src/native_run_spec.cpp src/native_fx_curve.cpp + src/selected_report_view.cpp + src/selected_window_plan.cpp ) # The kernel-only archive. Same objects, same flags as the kernel half of @@ -260,6 +262,46 @@ foreach(_pf_library ${PINEFORGE_LIBRARY_TARGETS}) target_link_libraries(${_pf_library} PUBLIC Eigen3::Eigen) endforeach() +# === Selected primary planner C bridge ================================= +# A standalone shared helper for callers that cannot link C++ (Python ctypes): +# the C bridge over pineforge::plan_selected_primary and nothing else. Its one +# source is deliberately NOT in PINEFORGE_KERNEL_SOURCES, so neither +# libpineforge.a nor PineForge::kernel gains a source or a symbol. It links the +# kernel archive PRIVATE (the planner and the native calendar it stands on), so +# no source layer, strategy or compiled-strategy export is part of it. PF_API +# symbols are exported through the same hidden-visibility rule the runtime +# uses. C++ callers keep linking PineForge::kernel and call the C++ planner +# directly. Test registration is not here (tests/CMakeLists.txt). +add_library(pineforge_window_plan SHARED src/selected_window_plan_c.cpp) +add_library(PineForge::window_plan ALIAS pineforge_window_plan) +set_target_properties(pineforge_window_plan PROPERTIES + EXPORT_NAME window_plan + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CXX_EXTENSIONS OFF +) +# PF_API is dllexport on Windows only under this definition. +target_compile_definitions(pineforge_window_plan PRIVATE PINEFORGE_BUILD_SHARED) +target_compile_options(pineforge_window_plan PRIVATE + $<$>:-O2> + -fPIC + -fvisibility=hidden + -fvisibility-inlines-hidden + -Wall + -Wextra + -Wpedantic + -Wno-unused-parameter + # The kernel's rule, see the library loop above: no FMA contraction. + -ffp-contract=off +) +target_include_directories(pineforge_window_plan + PUBLIC + $ + $ + $ +) +target_link_libraries(pineforge_window_plan PRIVATE pineforge_kernel) + # === Coverage instrumentation (opt-in) ================================= # Build with: # cmake -B build -DCMAKE_BUILD_TYPE=Debug -DPINEFORGE_ENABLE_COVERAGE=ON @@ -408,7 +450,7 @@ endif() include(GNUInstallDirs) include(CMakePackageConfigHelpers) -install(TARGETS pineforge pineforge_kernel +install(TARGETS pineforge pineforge_kernel pineforge_window_plan EXPORT PineForgeTargets LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/README.md b/README.md index bab3614fb..62b141cf9 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ The separate PineForge compiler, [`pineforge-codegen`](https://github.com/pinefo - **Open runtime.** The engine and native live runner are Apache-2.0. The separately distributed [PineForge compiler](https://github.com/pineforge-4pass/pineforge-codegen-oss/blob/main/LICENSE) is source-available under the PineForge Source License 1.2 from its release 1.3.0 (release 1.2.0 keeps the PineForge Source License 1.1, and releases up to 1.1.0 keep the PolyForm Noncommercial terms they shipped with); personal trading is free, and commercial use requires a separate license; the capital in an account a proprietary-trading firm or funded-trader program provides or allocates, including a challenge, evaluation or simulated account, is investment capital. Public reference strategies, benchmarks and validation tooling are available in their respective repositories; the community-script test set is not redistributed. - **Fast.** In-process, no interpreter: median **36× faster than PyneCore** on the 200 strategies both engines time, measured at engine `35db01c8` on 2026-09-29 on an AWS c7a.8xlarge with PyneCore timed as a subprocess, its interpreter start-up included (PineForge runs a median 807k bars/s per strategy over its 201 slots with the bar magnifier on; [method](benchmarks/results/speed.md)). Parameter sweeps re-run a loaded `.so` with new inputs — no recompile, no fork. - **Deterministic to the bit.** Two runs with the same inputs produce identical trade lists. Same on Linux and macOS. -- **Yours to embed.** 136 `PF_API` declarations across two headers — `pineforge.h`'s 93 (73 runtime exports and up to 20 per-strategy functions a generated module defines) and `native_c_api.h`'s 43 native-host declarations — an ABI that is append-only within a major version from 1.0 on. Call it from C, Python, Rust, Go, Node, Julia — or let an AI agent drive it over MCP. +- **Yours to embed.** 145 `PF_API` declarations across five installed headers — `pineforge.h`'s 94 (74 of the 80 runtime exports, the other six being declared in `execution_observer.h` and `selected_window.h`, three each, and up to 20 per-strategy functions a generated module defines), `native_c_api.h`'s 43 native-host declarations and `selected_window_plan.h`'s two planner declarations — an ABI that is append-only within a major version from 1.0 on. Call it from C, Python, Rust, Go, Node, Julia — or let an AI agent drive it over MCP. Recorded outputs and their additional API declarations described here are unreleased work targeted for **1.5.0**; they are not part of 1.4.0. @@ -506,11 +506,11 @@ is in [CHANGELOG.md](CHANGELOG.md). The counts below include recorded outputs and describe the unreleased API targeted for 1.5.0; recorded outputs are not part of 1.4.0. -A newly generated strategy `.so` exposes 89 compiled-strategy `PF_API` declarations -(73 runtime implementations plus sixteen generated exports) plus 43 native-host -declarations: 132 `PF_API` exports in total; `use_bar_magnifier = true` adds +A newly generated strategy `.so` exposes 96 compiled-strategy `PF_API` declarations +(80 runtime implementations plus sixteen generated exports) plus 43 native-host +declarations: 139 `PF_API` exports in total; `use_bar_magnifier = true` adds `strategy_declares_bar_magnifier` and a module that records [outputs](docs/outputs.md) its -three outputs exports, bringing the inventories to at most 93 and 136. Older modules lack the six opt-in +three outputs exports, bringing the inventories to at most 100 and 143. Older modules lack the six opt-in [checked-settings exports](docs/checked-settings.md). Older modules can also lack the two opt-in [execution-capability exports](docs/strategy-capabilities.md). In an optimized build `nm -gU` also shows libc++'s diff --git a/docker/bind_compiled_inventory.py b/docker/bind_compiled_inventory.py new file mode 100644 index 000000000..d66836c0e --- /dev/null +++ b/docker/bind_compiled_inventory.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""Bind a source-bound request-feed inventory to the strategy library just linked. + +docker/entrypoint.sh calls this once, after g++ linked the strategy and before it +starts run_json.py, on the selected-window path only: + + bind_compiled_inventory.py --inventory + --cpp + --so + --out + +The inventory is the one the pinned transpiler returned beside the generated C++ +(transpile_with_request_inventory), or the trusted caller's for a precompiled C++. +This helper does file and JSON work and reports its outcome by exit status, and +nothing else. The binding itself, the exact-shape check and the source check +included, is pineforge_codegen.request_feed_inventory.bind_request_feed_inventory +of the pinned codegen the image bundles: nothing is canonicalized, hashed or +compared here, so there is no second implementation to drift from the pinned one. +The library is read as bytes and never loaded, and no source or library is +evaluated to infer a dependency. + +The bound inventory is written and synced as a temporary file beside --out, then +published with link(2), which creates --out only if nothing has that name. A file +(or a dangling symlink) that exists at that instant, one that appeared after the +early check included, makes the link fail with EEXIST and the run is refused: --out +is never replaced, the caller's inventory included, and a reader of --out sees all +of the synced bytes or no file. The temporary name stays beside --out, a second +name of the same inode, and after a refusal it stays as well. Nothing is deleted +here: the pipeline's per-run work dir, which the entrypoint removes on exit, takes +the temporary name with the rest. + +Fails closed, one diagnostic line on stderr and no output file, with exit status: + + 2 usage error (argparse) + 10 the inventory cannot be read, or is over 8 MiB + 11 the inventory is not strict JSON (bad UTF-8, syntax, a duplicate key or a + non-finite number) + 12 the compiled C++ or the linked library cannot be read + 13 the bundled pineforge-codegen does not provide the pinned binder + 14 the binder refused the inventory (shape, already bound, or a source_sha256 + that is not that of the C++ bytes given) + 15 the output cannot be written, or exists already (a file that appears while + the link is made included) + 0 bound; --out holds the bound inventory +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +from typing import Any, List, Optional, Tuple + +PROG = "bind_compiled_inventory" + +EXIT_INVENTORY_UNREADABLE = 10 +EXIT_INVENTORY_NOT_JSON = 11 +EXIT_ARTIFACT_UNREADABLE = 12 +EXIT_BINDER_MISSING = 13 +EXIT_BINDER_REFUSED = 14 +EXIT_OUTPUT_REFUSED = 15 + +# The consumer (docker/request_feed_inventory.py) reads at most this much; a +# larger inventory is incomplete there, so it is refused here, before binding. +INVENTORY_MAX_BYTES = 8 * 1024 * 1024 + + +class Refusal(Exception): + """An outcome that ends the run with exit_code and one line on stderr.""" + + def __init__(self, exit_code: int, message: str) -> None: + super().__init__(message) + self.exit_code = exit_code + + +def _no_duplicate_keys(pairs: List[Tuple[str, Any]]) -> dict: + out: dict = {} + for key, value in pairs: + if key in out: + raise ValueError(f"duplicate key {key!r:.80}") + out[key] = value + return out + + +def _no_constant(name: str) -> Any: + raise ValueError(f"non-finite JSON number {name}") + + +def _read_inventory(path: str) -> Any: + try: + with open(path, "rb") as handle: + data = handle.read(INVENTORY_MAX_BYTES + 1) + except (OSError, ValueError) as error: + raise Refusal(EXIT_INVENTORY_UNREADABLE, + f"the inventory cannot be read: {error}") from None + if len(data) > INVENTORY_MAX_BYTES: + raise Refusal(EXIT_INVENTORY_UNREADABLE, + f"the inventory is over {INVENTORY_MAX_BYTES} bytes") + try: + return json.loads(data.decode("utf-8"), object_pairs_hook=_no_duplicate_keys, + parse_constant=_no_constant) + except (ValueError, RecursionError) as error: + raise Refusal(EXIT_INVENTORY_NOT_JSON, + f"the inventory is not strict JSON: {error}") from None + + +def _read_bytes(path: str, what: str) -> bytes: + try: + with open(path, "rb") as handle: + return handle.read() + except (OSError, ValueError) as error: + raise Refusal(EXIT_ARTIFACT_UNREADABLE, f"the {what} cannot be read: {error}") from None + + +def _check_output(out: str) -> str: + """The directory --out goes into, once --out is known to be a new name there.""" + if os.path.lexists(out): + raise Refusal(EXIT_OUTPUT_REFUSED, + f"{out} exists already; the bound inventory never replaces a file") + directory = os.path.dirname(os.path.abspath(out)) + if not os.path.isdir(directory): + raise Refusal(EXIT_OUTPUT_REFUSED, f"the directory of {out} does not exist") + return directory + + +def _load_binder() -> Tuple[Any, Any]: + try: + from pineforge_codegen.request_feed_inventory import ( + RequestFeedInventoryError, bind_request_feed_inventory) + except Exception as error: # noqa: BLE001 -- whatever stops the pinned package loading + raise Refusal(EXIT_BINDER_MISSING, + "the bundled pineforge-codegen does not provide " + "request_feed_inventory.bind_request_feed_inventory: " + f"{type(error).__name__}: {error}") from None + return bind_request_feed_inventory, RequestFeedInventoryError + + +def _serialize(bound: Any) -> str: + try: + return json.dumps(bound, ensure_ascii=True, allow_nan=False, + separators=(",", ":")) + "\n" + except (TypeError, ValueError) as error: + raise Refusal(EXIT_BINDER_REFUSED, + f"the bound inventory is not serializable JSON: {error}") from None + + +def _publish(directory: str, out: str, text: str) -> None: + """Write text beside out and make it out, never replacing a file named out. + + link(2) is the whole decision: it creates the name only if it is free, so a file + (or dangling symlink) that appears between the early check and here is refused, + where a rename would have replaced it. Once linked, the temporary name and out + are two names of one inode, and both stay until the pipeline's work dir is + removed; nothing is deleted here, a refusal included.""" + try: + descriptor, temporary = tempfile.mkstemp(prefix=".bound-inventory-", suffix=".tmp", + dir=directory) + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + except OSError as error: + raise Refusal(EXIT_OUTPUT_REFUSED, + f"the bound inventory cannot be written: {error}") from None + try: + os.link(temporary, out) + except FileExistsError: + raise Refusal(EXIT_OUTPUT_REFUSED, + f"{out} exists already; the bound inventory never replaces a file" + ) from None + except OSError as error: + raise Refusal(EXIT_OUTPUT_REFUSED, + f"the bound inventory cannot be written: {error}") from None + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser( + prog=PROG, allow_abbrev=False, + description="Bind a source-bound request-feed inventory to the linked strategy " + "library with the pinned pineforge-codegen binder.") + parser.add_argument("--inventory", required=True, + help="source-bound inventory (artifact_sha256 null); only read") + parser.add_argument("--cpp", required=True, help="the C++ file g++ compiled; only read") + parser.add_argument("--so", required=True, help="the library g++ linked; read, not loaded") + parser.add_argument("--out", required=True, help="the bound inventory to create") + args = parser.parse_args(argv) + try: + inventory = _read_inventory(args.inventory) + cpp_bytes = _read_bytes(args.cpp, "compiled C++ source") + artifact_bytes = _read_bytes(args.so, "linked library") + directory = _check_output(args.out) + bind, binder_error = _load_binder() + try: + bound = bind(inventory, cpp_bytes, artifact_bytes) + except binder_error as error: + raise Refusal(EXIT_BINDER_REFUSED, f"the inventory is refused: {error}") from None + except Exception as error: # noqa: BLE001 -- any other binder failure is a refusal too + raise Refusal(EXIT_BINDER_REFUSED, + f"the inventory cannot be bound: {type(error).__name__}: {error}" + ) from None + _publish(directory, args.out, _serialize(bound)) + except Refusal as refusal: + sys.stderr.write(f"{PROG}: {refusal}\n") + return refusal.exit_code + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a62696fe0..294cc0d3a 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -52,10 +52,39 @@ # value is read. An index or feed the harness cannot install fails # the run (exit 4, one {"engine":"pineforge","error":...} line). # +# Optional env vars (selected-window mode; unset PINEFORGE_REPORT_POLICY = the +# unchanged path above, which reads no inventory): +# PINEFORGE_REPORT_POLICY 'selected-window/v1' turns the selected path on. A .pine is +# transpiled with transpile_with_request_inventory, with no chart +# timeframe binding; the generated C++ (exact UTF-8 bytes) and its +# source-bound request-feed inventory are kept in the per-run work dir. +# After g++ links the library, bind_compiled_inventory.py binds that +# inventory to the C++ and library bytes (pinned codegen binder) and the +# bound file goes to run_json.py as --request-feed-inventory. Any other +# non-empty value is forwarded to run_json.py as --report-policy to be +# refused there; no inventory is read or bound for it. +# PINEFORGE_WINDOW_START_MS / PINEFORGE_WINDOW_END_MS / PINEFORGE_PREROLL_BARS / +# PINEFORGE_FED_START_MS forwarded, with the policy set and only when non-empty, as +# --window-start-ms / --window-end-ms / --preroll-bars / --fed-start-ms, +# raw. Nothing is defaulted or checked here: run_json.py validates them. +# PINEFORGE_REQUEST_FEED_INVENTORY Path to the source-bound inventory of a pre-transpiled +# /in/strategy.cpp, from the trusted caller; read only on the selected +# path, and not used with a .pine (that run makes its own). Unset: no +# inventory is passed to run_json.py, never an empty one, and run_json.py +# refuses the run. A supplied file that is unreadable, not strict JSON or +# not that of the C++ bytes compiled ends the run (exit 3). +# On the selected path /in/strategy.cpp is copied once into the per-run +# work dir, and that private copy is what g++ compiles, the binder reads +# and run_json.py gets as --generated-cpp (--transpiled stays false), so a +# change the caller makes to its file afterwards cannot change what was +# bound. A copy that fails ends the run (exit 3). The digest the binder +# checks is the inventory's own, never one read from the strategy code. +# # Exit codes: # 0 success (JSON report, or C++ in transpile-only mode, on stdout) # 2 missing input mount -# 3 compile failure +# 3 compile failure (g++; on the selected path also copying a pre-transpiled strategy.cpp +# into the work dir, or binding the request-feed inventory) # 4 backtest failure # 5 transpile failure (unsupported Pine construct or syntax error) set -euo pipefail @@ -94,6 +123,38 @@ else: PY } +# Selected path: transpile $1 once with transpile_with_request_inventory, passing no +# primary_chart_timeframe (the inventory stays chart-unbound). Writes the generated C++ to +# $2 as exact UTF-8 bytes (the inventory's source_sha256 is over those bytes) and the +# source-bound inventory (artifact_sha256 null) to $3. The C++ is the one run_transpile +# writes. Any failure, an installed codegen without the inventory API included, is exit 5 +# with a message on stderr: there is no fallback to run_transpile. +run_transpile_with_inventory() { + python3 - "$1" "$2" "$3" <<'PY' +import json +import sys + +pine, out, inventory_out = sys.argv[1], sys.argv[2], sys.argv[3] +try: + from pineforge_codegen import transpile_with_request_inventory + from pineforge_codegen.errors import CompileError +except Exception as e: # a codegen without the inventory API + sys.stderr.write(f"[pineforge] transpile error: no request-feed inventory API: {e}\n"); sys.exit(5) +try: + result = transpile_with_request_inventory(open(pine).read(), filename="strategy.pine") + cpp = result["cpp"] + inventory = result["request_feed_inventory"] + with open(out, "wb") as handle: + handle.write(cpp.encode("utf-8")) + with open(inventory_out, "w", encoding="utf-8") as handle: + json.dump(inventory, handle, ensure_ascii=True, allow_nan=False, separators=(",", ":")) +except CompileError as e: + sys.stderr.write(f"[pineforge] transpile error: {e}\n"); sys.exit(5) +except Exception as e: # syntax / unexpected — still a transpile failure + sys.stderr.write(f"[pineforge] transpile error: {e}\n"); sys.exit(5) +PY +} + # --- Transpile-only mode: emit C++ on stdout and exit. --------------------- if [[ "${PINEFORGE_TRANSPILE_ONLY:-}" == "1" || "${PINEFORGE_TRANSPILE_ONLY:-}" == "true" ]]; then if [[ ! -f "${PINE}" ]]; then @@ -104,15 +165,37 @@ if [[ "${PINEFORGE_TRANSPILE_ONLY:-}" == "1" || "${PINEFORGE_TRANSPILE_ONLY:-}" exit 0 fi +# --- Selected-window path on? Only this exact policy value (see the header). - +SELECTED=false +if [[ "${PINEFORGE_REPORT_POLICY:-}" == "selected-window/v1" ]]; then + SELECTED=true +fi +# The source-bound inventory the bind step will read; empty = none to bind. +INVENTORY_IN="" + # --- Resolve the translation unit: prefer .pine, fall back to .cpp. -------- if [[ -f "${PINE}" ]]; then echo "[pineforge] transpiling strategy.pine ..." >&2 - run_transpile "${PINE}" "${GEN}" # set -e aborts (exit 5) on failure + if [[ "${SELECTED}" == "true" ]]; then + INVENTORY_IN="${WORK}/request_feed_inventory.source.json" + run_transpile_with_inventory "${PINE}" "${GEN}" "${INVENTORY_IN}" # set -e aborts (exit 5) + else + run_transpile "${PINE}" "${GEN}" # set -e aborts (exit 5) on failure + fi SRC="${GEN}" TRANSPILED=true elif [[ -f "${SRC_CPP}" ]]; then SRC="${SRC_CPP}" TRANSPILED=false + if [[ "${SELECTED}" == "true" ]]; then + INVENTORY_IN="${PINEFORGE_REQUEST_FEED_INVENTORY:-}" + # Freeze the caller's bytes once into the private work dir (GEN). From here g++, the + # binder and run_json.py's --generated-cpp all use that one file, so the caller's + # mutable file is read only here. A failed copy ends the run before compile and run. + cp -- "${SRC_CPP}" "${GEN}" \ + || { echo "[pineforge] cannot snapshot strategy.cpp into the work dir" >&2; exit 3; } + SRC="${GEN}" + fi else echo "error: missing input — mount /in/strategy.pine (preferred) or /in/strategy.cpp" >&2 exit 2 @@ -140,6 +223,27 @@ g++ -std=c++17 -O2 -ffp-contract=off -fPIC -shared \ -o "${SO}" \ || { echo "[pineforge] compile failed" >&2; exit 3; } +# --- Selected path: bind the inventory to the C++ and library just built. --- +# Reached only after g++ linked (a failed link exited 3 above). A refusal ends the +# run: there is no fallback to the ordinary path and run_json.py is not started. +# Without an inventory (a .cpp the caller gave none for) nothing is bound and none is +# invented; run_json.py gets no --request-feed-inventory and refuses the run. +BOUND_INVENTORY="" +if [[ "${SELECTED}" == "true" ]]; then + if [[ -n "${INVENTORY_IN}" ]]; then + echo "[pineforge] binding request-feed inventory ..." >&2 + BOUND_INVENTORY="${WORK}/request_feed_inventory.json" + python3 "${PREFIX}/bin/bind_compiled_inventory.py" \ + --inventory "${INVENTORY_IN}" \ + --cpp "${SRC}" \ + --so "${SO}" \ + --out "${BOUND_INVENTORY}" \ + || { echo "[pineforge] request-feed inventory binding failed" >&2; exit 3; } + else + echo "[pineforge] no request-feed inventory for this strategy.cpp; none is passed on" >&2 + fi +fi + echo "[pineforge] running backtest ..." >&2 # Optional per-run knobs (mirror scripts/run_strategy.py). Built conditionally so @@ -159,6 +263,26 @@ extra=() if [[ "${PINEFORGE_BENCH:-}" =~ ^(1|true|yes|on)$ ]]; then extra+=(--bench --warmup "${PINEFORGE_WARMUP:-3}" --repeats "${PINEFORGE_REPEATS:-20}") fi +# Selected-window flags (see the header): raw values for run_json.py to validate, none +# defaulted here, and the inventory bound above. +if [[ -n "${PINEFORGE_REPORT_POLICY:-}" ]]; then + extra+=(--report-policy "${PINEFORGE_REPORT_POLICY}") + if [[ -n "${PINEFORGE_WINDOW_START_MS:-}" ]]; then + extra+=(--window-start-ms "${PINEFORGE_WINDOW_START_MS}") + fi + if [[ -n "${PINEFORGE_WINDOW_END_MS:-}" ]]; then + extra+=(--window-end-ms "${PINEFORGE_WINDOW_END_MS}") + fi + if [[ -n "${PINEFORGE_PREROLL_BARS:-}" ]]; then + extra+=(--preroll-bars "${PINEFORGE_PREROLL_BARS}") + fi + if [[ -n "${PINEFORGE_FED_START_MS:-}" ]]; then + extra+=(--fed-start-ms "${PINEFORGE_FED_START_MS}") + fi +fi +if [[ -n "${BOUND_INVENTORY}" ]]; then + extra+=(--request-feed-inventory "${BOUND_INVENTORY}") +fi python3 "${PREFIX}/bin/run_json.py" \ --so "${SO}" \ diff --git a/docker/request_feed_inventory.py b/docker/request_feed_inventory.py new file mode 100644 index 000000000..221cee208 --- /dev/null +++ b/docker/request_feed_inventory.py @@ -0,0 +1,321 @@ +"""Request-feed inventory admission for selected-window mode (the consumer half). + +A selected-window request is admitted only when the request-feed timeframes of +the compiled artifact are known BEFORE any feed is opened, any library is +loaded, any strategy is constructed or any optimizer trial is allocated. The +pinned transpiler emits an inventory beside the artifact; this module reads that +inventory and the artifact's bytes, and nothing else: never a feed or a feed +index, never ctypes, dlopen, an import of the artifact, a strategy or a +subprocess, and no parsing or evaluation of the generated C++. It is a +standalone module and is not wired into run_json here: where it runs (after +request validation, before the first feed or library operation) belongs to the +integration that calls it. + +The inventory is one JSON object with exactly these members: + + {"schema": "pineforge-request-feed-inventory/v1", + "source_sha256": <64 lowercase hex>, "artifact_sha256": <64 lowercase hex>, + "primary_chart_timeframe": , + "entries": [{"kind": "token", "timeframe": }, + {"kind": "input", "key": , "default": }, + {"kind": "unknown"}, ...]} + +A duplicate key, an extra or missing member and a wrong type are refused; nothing +is coerced. An empty "entries" is COMPLETE (the request reads no request feed), +an "unknown" entry is INCOMPLETE. artifact_sha256 must equal the SHA-256 of the +artifact bytes read here (null means unbound); source_sha256 is checked for its +form only, because the generated C++ is not read. A recorded +primary_chart_timeframe must equal the request's canonical primary timeframe. An +"input" entry resolves to validated_inputs[key] when the request validated that +key, else to its declared default; the value must be a string that is a token. + +Timeframe tokens: whole minutes without leading zeros ("1", "240", "100000"), or +D, W, M, S with n a positive whole number without leading zeros +("2D", "10000D", "12M", "30S"). The grammar has no length limit, and digits are +never converted to a number: a spelling is compared as written, after the +normalization below. D and 1D are D, W and 1W are W, a bare M or S is 1M or 1S. +No whitespace, no empty string, no case folding. + +A refusal is InventoryAdmissionError: code window_mode_unsupported and a detail +dict naming the capability. An incomplete inventory carries only +selected_window_request_feed_inventory, never a timeframe. admit_selected_request +checks the primary timeframe against the support policy FIRST and reads no file +for a primary outside it. It never builds a support policy: a policy that is +missing, or is not exactly the three documented members, supports nothing. +""" +from __future__ import annotations + +import hashlib +import json +import os +import re +from collections.abc import Mapping +from typing import Any, List, Optional, Tuple + +__all__ = [ + "InventoryAdmissionError", + "admit_selected_request", + "resolve_request_feed_inventory", +] + +CODE = "window_mode_unsupported" +SCHEMA = "pineforge-request-feed-inventory/v1" +CAPABILITY_CHART = "selected_window_chart_timeframe" +CAPABILITY_FEED = "selected_window_request_feed_timeframe" +CAPABILITY_INVENTORY = "selected_window_request_feed_inventory" + +_INVENTORY_KEYS = frozenset(("schema", "source_sha256", "artifact_sha256", + "primary_chart_timeframe", "entries")) +_POLICY_KEYS = frozenset(("primary_chart_timeframes", "preroll", "request_feed_timeframes")) +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_TIMEFRAME_RE = re.compile(r"(?:(?P[1-9][0-9]*)" + r"|(?P[1-9][0-9]*)?(?P[DWMS]))") +_INVENTORY_MAX_BYTES = 8 * 1024 * 1024 +_READ_CHUNK = 1024 * 1024 + + +class InventoryAdmissionError(Exception): + """A selected-window request this helper refuses. `code` is + window_mode_unsupported; `detail` is the code's typed arguments: + {"capability": "selected_window_chart_timeframe" | "selected_window_request_feed_timeframe", + "timeframe": }, or {"capability": + "selected_window_request_feed_inventory"} with no timeframe member when the + inventory is incomplete. The arguments are `detail` because + BaseException.args is the exception's own tuple.""" + + code = CODE + + def __init__(self, message: str, detail: Mapping[str, str]) -> None: + super().__init__(message) + self.message = message + self.detail = dict(detail) + + +def _incomplete(reason: str) -> InventoryAdmissionError: + return InventoryAdmissionError(f"the request-feed inventory is incomplete: {reason}", + {"capability": CAPABILITY_INVENTORY}) + + +def _canonical_timeframe(value: Any) -> Optional[str]: + """The wire's canonical spelling of value, or None when value is not a string + that spells a timeframe token.""" + if not isinstance(value, str): + return None + match = _TIMEFRAME_RE.fullmatch(value) + if match is None: + return None + if match.group("minutes") is not None: + return value + count, unit = match.group("count"), match.group("unit") + if unit in ("D", "W") and count in (None, "1"): + return unit + return (count or "1") + unit + + +def _policy_token(value: Any) -> Optional[str]: + """value when it is already a canonical token, else None: a policy is written + in the wire's own spelling and an alias in it is a value nobody defined.""" + token = _canonical_timeframe(value) + return token if token is not None and token == value else None + + +def _require_primary(primary_timeframe: Any) -> str: + primary = _canonical_timeframe(primary_timeframe) + if primary is None: + raise ValueError(f"primary_timeframe is not a timeframe token: {primary_timeframe!r:.80}") + return primary + + +def _policy_members(policy: Any) -> Optional[Tuple[Any, Any]]: + """The policy's primary and request-feed members, each the word "all" or a + frozenset of canonical tokens; None when the policy is not a mapping of exactly + primary_chart_timeframes, preroll and request_feed_timeframes with values the + the documented support policy defines. preroll is checked for its form only: a + pre-roll length is no argument of this helper.""" + if not isinstance(policy, Mapping) or set(policy) != _POLICY_KEYS: + return None + preroll = policy["preroll"] + if not isinstance(preroll, str) or preroll not in ("supported", "zero_only"): + return None + members = [] + for key in ("primary_chart_timeframes", "request_feed_timeframes"): + value = policy[key] + if isinstance(value, str): + if value != "all": + return None + members.append("all") + continue + if not isinstance(value, (list, tuple)): + return None + tokens = set() + for item in value: + token = _policy_token(item) + if token is None: + return None + tokens.add(token) + members.append(frozenset(tokens)) + return members[0], members[1] + + +def _read_inventory_bytes(path: Any) -> bytes: + if not isinstance(path, (str, os.PathLike)): + raise _incomplete("no inventory path was given") + try: + with open(path, "rb") as handle: + data = handle.read(_INVENTORY_MAX_BYTES + 1) + except (OSError, ValueError) as error: + raise _incomplete(f"the inventory cannot be read: {error}") from None + if len(data) > _INVENTORY_MAX_BYTES: + raise _incomplete(f"the inventory is over {_INVENTORY_MAX_BYTES} bytes") + return data + + +def _no_duplicate_keys(pairs: List[Tuple[str, Any]]) -> dict: + out = {} + for key, value in pairs: + if key in out: + raise ValueError("duplicate key") + out[key] = value + return out + + +def _parse_inventory(data: bytes) -> Any: + try: + return json.loads(data.decode("utf-8"), object_pairs_hook=_no_duplicate_keys) + except (ValueError, RecursionError): + raise _incomplete("the inventory is not strict JSON (bad UTF-8, syntax or a " + "duplicate key)") from None + + +def _checked_entry(position: int, entry: Any) -> tuple: + keys = set(entry) if isinstance(entry, dict) else None + kind = entry.get("kind") if keys is not None else None + if kind == "token" and keys == {"kind", "timeframe"} and isinstance(entry["timeframe"], str): + return ("token", entry["timeframe"]) + if (kind == "input" and keys == {"kind", "key", "default"} + and isinstance(entry["key"], str) and isinstance(entry["default"], str)): + return ("input", entry["key"], entry["default"]) + if kind == "unknown" and keys == {"kind"}: + return ("unknown",) + raise _incomplete(f"entry {position} is not exactly one of the three entry shapes") + + +def _checked_entries(doc: Any) -> List[tuple]: + """doc's entries as ("token", timeframe), ("input", key, default) or + ("unknown",), after the exact-shape checks of the whole object.""" + if not isinstance(doc, dict) or set(doc) != _INVENTORY_KEYS: + raise _incomplete("the inventory is not one object with exactly the five schema members") + if not isinstance(doc["schema"], str) or doc["schema"] != SCHEMA: + raise _incomplete(f"the inventory's schema is not {SCHEMA}") + if doc["artifact_sha256"] is None: + raise _incomplete("the inventory is unbound (artifact_sha256 is null)") + for member in ("source_sha256", "artifact_sha256"): + value = doc[member] + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise _incomplete(f"{member} is not 64 lowercase hex digits") + binding = doc["primary_chart_timeframe"] + if binding is not None and _canonical_timeframe(binding) is None: + raise _incomplete("primary_chart_timeframe is neither null nor a timeframe token") + if not isinstance(doc["entries"], list): + raise _incomplete("entries is not an array") + return [_checked_entry(position, entry) for position, entry in enumerate(doc["entries"])] + + +def _artifact_digest(path: Any) -> str: + """The SHA-256 of the artifact's bytes, streamed. The artifact is hashed, never + loaded.""" + if not isinstance(path, (str, os.PathLike)): + raise _incomplete("no artifact path was given") + digest = hashlib.sha256() + try: + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(_READ_CHUNK), b""): + digest.update(chunk) + except (OSError, ValueError) as error: + raise _incomplete(f"the artifact cannot be read: {error}") from None + return digest.hexdigest() + + +def resolve_request_feed_inventory(inventory_path: Any, artifact_path: Any, + validated_inputs: Any, *, + primary_timeframe: Any) -> List[str]: + """The sorted, unique canonical request-feed timeframes of one request, from + the artifact's inventory. Reads the inventory file, then the artifact's bytes + for their digest, and nothing else. + + Raises InventoryAdmissionError (code window_mode_unsupported, detail + {"capability": "selected_window_request_feed_inventory"}) when the inventory + is incomplete for this request: absent, unreadable, not strict JSON, not + exactly the schema, unbound, bound to another artifact, bound to another + primary chart timeframe, holding an unknown entry, or holding an input + reference that does not resolve to a token (an input value that is not a + string included: it is not coerced, and a present value never falls back to + the default). ValueError when primary_timeframe is not a timeframe token: the + request was validated before this call.""" + primary = _require_primary(primary_timeframe) + if not isinstance(validated_inputs, Mapping): + raise _incomplete("the request's validated inputs are not a mapping") + doc = _parse_inventory(_read_inventory_bytes(inventory_path)) + entries = _checked_entries(doc) + if _artifact_digest(artifact_path) != doc["artifact_sha256"]: + raise _incomplete("the inventory is bound to another artifact") + binding = doc["primary_chart_timeframe"] + if binding is not None and _canonical_timeframe(binding) != primary: + raise _incomplete("the inventory's chart binding is not the request's primary timeframe") + tokens = set() + for entry in entries: + if entry[0] == "unknown": + raise _incomplete("an entry is not known before the strategy runs") + if entry[0] == "token": + token = _canonical_timeframe(entry[1]) + if token is None: + raise _incomplete("a token entry does not spell a timeframe token") + else: + key, default = entry[1], entry[2] + token = _canonical_timeframe(validated_inputs[key] if key in validated_inputs + else default) + if token is None: + raise _incomplete(f"input {key!r:.80} does not resolve to a timeframe token") + tokens.add(token) + return sorted(tokens) + + +def admit_selected_request(primary_timeframe: Any, inventory_path: Any, artifact_path: Any, + validated_inputs: Any, support_policy: Any) -> Optional[List[str]]: + """Admit a selected-window request against the build's support policy, or + raise InventoryAdmissionError. In this order: canonicalize the primary + timeframe; enforce the policy's primary list (a policy that is missing or not + exactly the three documented members supports no primary: nothing is + synthesized); resolve the inventory; check each resolved token against the + policy's request-feed list. + + A primary outside the policy is {"capability": "selected_window_chart_timeframe", + "timeframe": } and reads no file. A resolved token outside + the list is {"capability": "selected_window_request_feed_timeframe", + "timeframe": }, the first in sorted order when several + are outside. An incomplete inventory is the inventory capability, as in + resolve_request_feed_inventory. Returns the sorted canonical tokens. When the + policy says "all" for request feeds the set does not enter the decision: the + inventory is not read and the result is None, not an empty set. ValueError + when primary_timeframe is not a timeframe token.""" + primary = _require_primary(primary_timeframe) + members = _policy_members(support_policy) + if members is None: + raise InventoryAdmissionError( + "the support policy is missing or is not exactly primary_chart_timeframes, " + f"preroll and request_feed_timeframes, so primary timeframe {primary} is not supported", + {"capability": CAPABILITY_CHART, "timeframe": primary}) + if members[0] != "all" and primary not in members[0]: + raise InventoryAdmissionError( + f"selected-window mode is not supported for primary timeframe {primary}", + {"capability": CAPABILITY_CHART, "timeframe": primary}) + if members[1] == "all": + return None + tokens = resolve_request_feed_inventory(inventory_path, artifact_path, validated_inputs, + primary_timeframe=primary) + for token in tokens: + if token not in members[1]: + raise InventoryAdmissionError( + f"selected-window mode is not supported for request-feed timeframe {token}", + {"capability": CAPABILITY_FEED, "timeframe": token}) + return tokens diff --git a/docker/run_execution_observer.py b/docker/run_execution_observer.py new file mode 100644 index 000000000..0a8d410ee --- /dev/null +++ b/docker/run_execution_observer.py @@ -0,0 +1,344 @@ +"""Python half of the run-only execution observer bridge (observer ABI v1). + +On the success path of a native run the engine calls one registered `before_results` +callback, synchronously and once: after every execution callback ended and the result gate +sealed, before results open. The callback fills a receipt for the run's result_assembly +boundary; anything but return 0 with a valid receipt keeps the generation sealed, with no +result work and no retry. This module is the harness side of +that call for a run that exports its phases with RunPhaseWriter (run_phase_transport.py): +when the engine calls, the callback enters result_assembly on the writer, exactly once, and +reports the bytes the writer really handed to the socket. + + writer = RunPhaseWriter(args.run_phase_fd) + writer.advance("preflight") + ... + writer.advance("execution") # the caller, before any native work + binding = ExecutionObserverBinding(library, state, writer) + binding.attach() # probe the version, bind, register once + ... native run ... + binding.raise_if_failed() # after native returned + binding.detach() # before the state is destroyed + +attach() never advances the writer: the caller has already entered "execution". The receipt +is exactly what the writer did: without a descriptor it is 0/0/0/0 (frame_bytes, +handed_bytes, export_requested, reserved); an exported record of N whole bytes is N/N/1/0 +with 1 <= N <= 1024. Handed bytes are not an acknowledgement: nothing here claims that the +parent received or validated the record. + +Nothing may unwind through native code. The callback catches every BaseException, keeps the +first in `failure`, returns -1 and leaves the receipt as the engine preset it; +raise_if_failed() raises that exception once native code has returned (a PhaseTransportError +stays a PhaseTransportError). A second call of the callback is a failure too: a run has one +observer call, and the record is never written twice. + +The registration borrows what the binding owns: the engine copies the descriptor's values +but keeps calling the callback until it is removed, so the binding holds the descriptor and +the callback strongly. detach() is explicit and comes before the state is destroyed (no +destructor calls native code): it unregisters with a NULL descriptor, checks the result, +never frees a registered callback and never touches the phase descriptor, which the caller +owns. A detach that fails keeps the ownership, for the caller to retry or to leave the +callback alive; only a successful one lets go. A binding is single use: attach() registers +once. + +Errors are ObserverBindingError. `code` is window_mode_unsupported (a library without +execution_observer_v1: a missing export, or a version other than 1), out_of_memory (the +engine could not allocate its consumer storage: registration status -4) or +harness_internal_error (any other refusal, a violation of the callback contract, a misuse). +The code's arguments are `code_args` because BaseException.args is the exception's own +tuple. This module carries no failure catalogue and decides no billing. +""" +from __future__ import annotations + +import ctypes +from typing import Any, Mapping, Optional, Tuple + +__all__ = [ + "BeforeResultsFn", + "BoundaryC", + "CAPABILITY", + "ExecutionObserverBinding", + "MAX_FRAME_BYTES", + "OBSERVER_VERSION", + "ObservationC", + "ObserverBindingError", + "ObserverC", + "ReceiptC", +] + +OBSERVER_VERSION = 1 # pf_execution_observer_version() and every struct's version +MAX_FRAME_BYTES = 1024 # one phase record, LF included: the transport's limit +CAPABILITY = "execution_observer_v1" # the one argument of window_mode_unsupported + +_STATUS_OUT_OF_MEMORY = -4 # registration: the consumer storage could not be allocated +_STATUS_MEANING = { + -1: "invalid size, version, callback or handle", + -2: "unsupported execution contract", + -3: "not quiescent: a run or a callback is in progress", + -4: "out of memory", +} + + +# --- ctypes mirror of --------------------------------------- +# +# Fixed-width scalars at natural alignment, no packing; the LP64 sizes are 24, 32, 24 and 48. + +class BoundaryC(ctypes.Structure): + """pf_execution_boundary_v1: the generation the callback is called for.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("run_generation", ctypes.c_uint64), + ("attempt_serial", ctypes.c_uint64), + ] + + +class ReceiptC(ctypes.Structure): + """pf_boundary_receipt_v1: what the callback fills; the engine presets every field.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("run_generation", ctypes.c_uint64), + ("frame_bytes", ctypes.c_uint32), + ("handed_bytes", ctypes.c_uint32), + ("export_requested", ctypes.c_uint32), + ("reserved", ctypes.c_uint32), + ] + + +# pf_before_results_fn_v1: int (*)(void *context, const pf_execution_boundary_v1 *boundary, +# pf_boundary_receipt_v1 *receipt) +BeforeResultsFn = ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_void_p, ctypes.POINTER(BoundaryC), ctypes.POINTER(ReceiptC)) + + +class ObserverC(ctypes.Structure): + """pf_execution_observer_v1: the descriptor the engine copies at registration.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("context", ctypes.c_void_p), + ("before_results", BeforeResultsFn), + ] + + +class ObservationC(ctypes.Structure): + """pf_execution_observation_v1: the snapshot strategy_execution_observation_v1 fills; + the caller presets struct_size and version.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("run_generation", ctypes.c_uint64), + ("attempt_serial", ctypes.c_uint64), + ("attempt_generation", ctypes.c_uint64), + ("phase", ctypes.c_uint32), + ("fault_stage", ctypes.c_uint32), + ("attempt_outcome", ctypes.c_uint32), + ("boundary_delivered", ctypes.c_uint32), + ] + + +class ObserverBindingError(Exception): + """The observer binding failed on the Python side, before or beside the native run: a + library without the capability or a harness fault, never a strategy outcome. `code` is + window_mode_unsupported (code_args {"capability": "execution_observer_v1"}), + out_of_memory or harness_internal_error (code_args {} for both). The arguments are + code_args because BaseException.args is the exception's own tuple.""" + + def __init__(self, code: str, message: str, + code_args: Optional[Mapping[str, object]] = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.code_args = dict(code_args or {}) + + +def _unsupported(message: str) -> ObserverBindingError: + return ObserverBindingError("window_mode_unsupported", message, {"capability": CAPABILITY}) + + +def _status_error(action: str, status: int) -> ObserverBindingError: + """The error for a nonzero status of strategy_set_execution_observer_v1: -4 is the + engine's allocation failure, every other status a harness fault.""" + meaning = _STATUS_MEANING.get(status, "unknown status") + code = "out_of_memory" if status == _STATUS_OUT_OF_MEMORY else "harness_internal_error" + return ObserverBindingError( + code, f"execution observer {action} was refused with status {status} ({meaning})") + + +def _check_arguments(header: Any, slot: Any) -> None: + """The boundary and the receipt as the engine presets them: this layout and version, a + minted generation and attempt, and the one generation in both.""" + if header.struct_size != ctypes.sizeof(BoundaryC) or header.version != OBSERVER_VERSION: + raise ObserverBindingError( + "harness_internal_error", + f"before_results got a boundary of size {header.struct_size} and version " + f"{header.version}, expected size {ctypes.sizeof(BoundaryC)} and version " + f"{OBSERVER_VERSION}") + if header.run_generation == 0 or header.attempt_serial == 0: + raise ObserverBindingError( + "harness_internal_error", + f"before_results got generation {header.run_generation} and attempt serial " + f"{header.attempt_serial}: both are minted, so nonzero") + if slot.struct_size != ctypes.sizeof(ReceiptC) or slot.version != OBSERVER_VERSION: + raise ObserverBindingError( + "harness_internal_error", + f"before_results got a receipt of size {slot.struct_size} and version " + f"{slot.version}, expected size {ctypes.sizeof(ReceiptC)} and version " + f"{OBSERVER_VERSION}") + if slot.run_generation != header.run_generation: + raise ObserverBindingError( + "harness_internal_error", + f"before_results got a receipt for generation {slot.run_generation} and a " + f"boundary for generation {header.run_generation}") + + +def _handoff(writer: Any) -> Tuple[int, bool]: + """(bytes handed over, exported) of the advance() that just returned, as the writer + reports them, held to the receipt contract: no export is 0 bytes, an export is one whole + record of 1 to MAX_FRAME_BYTES bytes.""" + exported = writer.export_requested + handed = writer.last_handed_frame_bytes + if type(exported) is not bool or type(handed) is not int: + raise ObserverBindingError( + "harness_internal_error", + f"the phase writer reports export_requested {exported!r} and " + f"last_handed_frame_bytes {handed!r}: a bool and an int are required") + valid = 1 <= handed <= MAX_FRAME_BYTES if exported else handed == 0 + if not valid: + raise ObserverBindingError( + "harness_internal_error", + f"the phase writer reports {handed} bytes handed over with export_requested " + f"{exported}: an export is 1 to {MAX_FRAME_BYTES} bytes, no export is 0") + return handed, exported + + +class ExecutionObserverBinding: + """Registers the before_results callback of one strategy handle and answers it from a + RunPhaseWriter (see the module text). `library` is the loaded strategy library, or + anything with its three observer exports; `state` is the pf_strategy_t it was created + with, which the binding borrows; `writer` is the run's RunPhaseWriter, already in phase + "execution" when native work starts. Not thread safe: the thread that calls into native + code drives it.""" + + def __init__(self, library: Any, state: Any, writer: Any) -> None: + self._library = library + self._state = state + self._writer = writer + self._attach_called = False + self._attached = False # the engine may hold the callback + self._entered = False # before_results was called + self._failure: Optional[BaseException] = None + self._set_observer: Any = None + self._callback: Optional[BeforeResultsFn] = None + self._descriptor: Optional[ObserverC] = None + + @property + def failure(self) -> Optional[BaseException]: + """The first exception the callback met, for which it returned -1; None while the + callback met none.""" + return self._failure + + def attach(self) -> None: + """Probe the library, bind its three observer exports and register the callback, + once. The writer is not advanced. Raises ObserverBindingError: window_mode_unsupported + for a library without execution_observer_v1 (nothing is registered), out_of_memory + for registration status -4, harness_internal_error for any other nonzero status and + for a second attach(), which is refused without touching the library.""" + if self._attach_called: + raise ObserverBindingError( + "harness_internal_error", + "execution observer attach() was already called: a binding registers once") + self._attach_called = True + library = self._library + try: + probe = library.pf_execution_observer_version + set_observer = library.strategy_set_execution_observer_v1 + observation = library.strategy_execution_observation_v1 + except AttributeError as error: + raise _unsupported( + f"the strategy library lacks an execution observer export: {error}") from error + probe.argtypes = [] + probe.restype = ctypes.c_uint32 + set_observer.argtypes = [ctypes.c_void_p, ctypes.POINTER(ObserverC)] + set_observer.restype = ctypes.c_int + observation.argtypes = [ctypes.c_void_p, ctypes.POINTER(ObservationC)] + observation.restype = ctypes.c_int + version = probe() + if version != OBSERVER_VERSION: + raise _unsupported( + f"the strategy library reports execution observer version {version}, " + f"not {OBSERVER_VERSION}") + callback = BeforeResultsFn(self._before_results) # the closure captures self + descriptor = ObserverC(ctypes.sizeof(ObserverC), OBSERVER_VERSION, None, callback) + # From here the engine may borrow both, whatever happens next: they stay with the + # binding, and so does the right to unregister. + self._set_observer = set_observer + self._callback = callback + self._descriptor = descriptor + self._attached = True + status = set_observer(self._state, ctypes.pointer(descriptor)) + if status != 0: + self._attached = False # refused: the descriptor the engine had is unchanged + raise _status_error("registration", status) + + def detach(self) -> None: + """Unregister with a NULL descriptor, before the state is destroyed. A binding that + is not registered (never attached, refused, already detached) does nothing. A + nonzero status raises ObserverBindingError and keeps the descriptor and the callback + with the binding: the engine may still hold them, and detach() may be called again. + Only a status 0 lets them go. No phase descriptor is closed or touched.""" + if not self._attached: + return + status = self._set_observer(self._state, None) + if status != 0: + raise _status_error("removal", status) + self._attached = False + self._callback = None + self._descriptor = None + + def raise_if_failed(self) -> None: + """Raise the first exception the callback met, the very object (a + PhaseTransportError stays one). Call it after native code has returned: raised + inside the callback, ctypes would only print it.""" + if self._failure is not None: + raise self._failure + + def _before_results(self, context: Any, boundary: Any, receipt: Any) -> int: + """The native callback: 0 once the receipt is filled, -1 for anything else. Nothing + may unwind through the C caller, so every BaseException is kept, the first one only, + and refused here.""" + try: + self._deliver(boundary, receipt) + except BaseException as error: + if self._failure is None: + self._failure = error + return -1 + return 0 + + def _deliver(self, boundary: Any, receipt: Any) -> None: + if self._entered: + raise ObserverBindingError( + "harness_internal_error", + "before_results was called again: a run has one observer call") + self._entered = True + if not boundary or not receipt: + raise ObserverBindingError( + "harness_internal_error", "before_results got a null boundary or receipt") + # Only the two pointers the engine passed are read, and only when nonnull: a pointer + # that is not what the engine passed is outside the C lifetime contract. + header, slot = boundary.contents, receipt.contents + _check_arguments(header, slot) + writer = self._writer + if writer.phase != "execution": + raise ObserverBindingError( + "harness_internal_error", + f"before_results arrived with the phase writer in phase {writer.phase!r}, " + "not 'execution': the caller enters execution before any native work") + writer.advance("result_assembly") # the one handoff; a refusal raises, typed + handed, exported = _handoff(writer) + # Only now, with the record handed over, is the receipt filled. The boundary and the + # receipt header, generation included, are never written. + slot.frame_bytes = handed + slot.handed_bytes = handed + slot.export_requested = 1 if exported else 0 + slot.reserved = 0 diff --git a/docker/run_failure_codes.json b/docker/run_failure_codes.json index ad5370175..6b2917352 100644 --- a/docker/run_failure_codes.json +++ b/docker/run_failure_codes.json @@ -82,6 +82,40 @@ "retryable": false, "since": "1.4.0" }, + "engine_command_value_invalid": { + "args": { + "field": { + "kind": "string" + }, + "reason": { + "kind": "vocab", + "values": [ + "nonfinite", + "invalid_unicode", + "numeric_overflow" + ] + }, + "stage": { + "kind": "vocab", + "values": [ + "sizing", + "margin", + "fee", + "slippage", + "relative_price", + "risk", + "settlement" + ] + } + }, + "class": "engine_fault", + "description": "The engine derived an invalid command value (non-finite, invalid text or an overflowing finite value) from its own book, equity, margin, sizing, fee, slippage or settlement state: an engine fault, never a strategy failure. stage names where it was derived.", + "english": [ + "The engine derived an invalid execution value." + ], + "retryable": false, + "since": "1.5.0" + }, "engine_invariant": { "args": {}, "class": "engine_fault", @@ -789,6 +823,114 @@ "retryable": false, "since": "1.4.0" }, + "report_platform_unavailable": { + "args": { + "field": { + "kind": "vocab", + "values": [ + "os", + "architecture", + "pointer_bits", + "endianness", + "engine_library_sha256", + "harness_sha256" + ] + } + }, + "class": "engine_fault", + "description": "The runner cannot establish an execution-platform fact its report contract requires. A pre-load backstop for an image that claimed results-digest support, raised before any strategy code loads.", + "english": [ + "The runner cannot establish required execution-platform identity." + ], + "retryable": false, + "since": "1.5.0" + }, + "report_post_execution_failed": { + "args": { + "phase": { + "kind": "vocab", + "values": [ + "result_assembly", + "results_digest", + "serialization" + ] + }, + "reason": { + "kind": "vocab", + "values": [ + "deadline", + "exception", + "io", + "resource" + ] + } + }, + "class": "engine_fault", + "description": "The engine could not finish producing the result after strategy execution completed: a deadline or an otherwise unclassified failure in result assembly, the results digest or serialization. An engine fault, never a strategy failure; a specific typed engine fault keeps its own code.", + "english": [ + "The engine could not finish producing the result after strategy execution completed." + ], + "retryable": false, + "since": "1.5.0" + }, + "report_result_invalid": { + "args": { + "field": { + "kind": "string" + }, + "reason": { + "kind": "vocab", + "values": [ + "nonfinite", + "invalid_unicode", + "column_schema", + "type_schema" + ] + } + }, + "class": "engine_fault", + "description": "The report producer generated an invalid canonical result: an engine or harness fault, never a strategy failure.", + "english": [ + "The report producer generated an invalid canonical result." + ], + "retryable": false, + "since": "1.5.0" + }, + "report_shape_request_invalid": { + "args": { + "option": { + "kind": "vocab", + "values": [ + "report_shape", + "curve_point_budget" + ] + }, + "value": { + "kind": "scalar" + } + }, + "class": "input", + "description": "The report-shape request is invalid: the report shape or the curve point budget carries a value the runner refuses.", + "english": [ + "The report-shape request is invalid." + ], + "retryable": false, + "since": "1.5.0" + }, + "report_shape_unsupported": { + "args": { + "shape": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "The runner does not support the requested report shape. The first selected-window build supports the full shape only.", + "english": [ + "This runner does not support the requested report shape." + ], + "retryable": false, + "since": "1.5.0" + }, "request_symbol_invalid": { "args": {}, "class": "strategy", @@ -877,6 +1019,20 @@ "retryable": false, "since": "1.4.0" }, + "results_digest_unsupported": { + "args": { + "requested": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "The runner does not support the requested results digest. The first selected-window build declares no digest.", + "english": [ + "This runner does not support the requested results digest." + ], + "retryable": false, + "since": "1.5.0" + }, "run_aborted": { "args": {}, "class": "engine_fault", @@ -1262,6 +1418,28 @@ "retryable": false, "since": "1.4.0" }, + "strategy_value_invalid": { + "args": { + "field": { + "kind": "string" + }, + "reason": { + "kind": "vocab", + "values": [ + "nonfinite", + "invalid_unicode", + "numeric_overflow" + ] + } + }, + "class": "strategy", + "description": "The strategy supplied an invalid actionable value (a non-finite number, invalid text or an overflowing finite value) to a declaration, setting or command that needs a valid one.", + "english": [ + "The strategy supplied an invalid execution value." + ], + "retryable": false, + "since": "1.5.0" + }, "stream_input_rejected": { "args": {}, "class": "input", @@ -1462,14 +1640,344 @@ ], "retryable": true, "since": "1.4.0" + }, + "window_boundary_unaligned": { + "args": { + "bound": { + "kind": "vocab", + "values": [ + "start", + "end", + "fed_start" + ] + }, + "next_boundary_ms": { + "kind": "integer" + }, + "previous_boundary_ms": { + "kind": "integer" + }, + "value_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "A selected-window boundary (start, end or fed start) is not aligned to the script calendar; the adjacent boundaries are carried.", + "english": [ + "A window boundary is not aligned to the script calendar." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_calendar_unsupported": { + "args": { + "session": { + "kind": "nullable_string" + }, + "timezone": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "The timezone and session the selected window needs make a calendar the engine does not support; session may be null.", + "english": [ + "The requested window calendar is unsupported." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_feed_identity_changed": { + "args": { + "actual_sha256": { + "kind": "string" + }, + "expected_sha256": { + "kind": "string" + }, + "feed_id": { + "kind": "string" + } + }, + "class": "input", + "description": "The frozen primary feed's bytes differ from the SHA-256 the study recorded. The two digests are 64-character hexadecimal strings supplied by the trusted wrapper; the registry checks only that they are text.", + "english": [ + "The frozen primary feed bytes have changed." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_feed_range_invalid": { + "args": { + "end_ms": { + "kind": "integer" + }, + "fed_start_ms": { + "kind": "integer" + }, + "time_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "A primary-feed bar lies outside the frozen selected-window range.", + "english": [ + "The primary feed lies outside the frozen range." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_input_cap_exceeded": { + "args": { + "actual": { + "kind": "integer" + }, + "cap_name": { + "kind": "vocab", + "values": [ + "run_input_bars", + "run_script_bars", + "native_input_bars", + "hpo_input_bars", + "hpo_script_bars", + "hpo_memory_bytes", + "symbol_feed_input_bars", + "symbol_feeds_total_input_bars" + ] + }, + "limit": { + "kind": "integer" + } + }, + "class": "input", + "description": "The supplied history exceeds a configured input cap; cap_name says which.", + "english": [ + "The supplied history exceeds the configured input cap." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_legacy_flag_conflict": { + "args": { + "option": { + "kind": "string" + } + }, + "class": "input", + "description": "The legacy trade-start option was combined with selected-window mode.", + "english": [ + "The legacy trade-start option cannot be combined with selected-window mode." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_lineage_mismatch": { + "args": { + "actual": { + "kind": "scalar" + }, + "expected": { + "kind": "scalar" + }, + "field": { + "kind": "string" + } + }, + "class": "input", + "description": "A requested window field differs from the study's lineage.", + "english": [ + "The requested window differs from the study lineage." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_mode_unsupported": { + "args": { + "capability": { + "kind": "string" + }, + "requested": { + "kind": "integer", + "optional": true + }, + "timeframe": { + "kind": "string", + "optional": true + } + }, + "class": "unsupported", + "description": "The runner does not support the requested selected-window mode or a limit its build declares on it. capability names what is unsupported; timeframe carries the offending canonical timeframe and requested the offending pre-roll length where the refusal names one. Which optional argument goes with which capability is the trusted raisers' duty, not the registry's.", + "english": [ + "This runner does not support the requested window mode." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_no_observations": { + "args": { + "end_ms": { + "kind": "integer" + }, + "start_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "The selected window holds no observations to score.", + "english": [ + "The study has no selected observations to score." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_preroll_out_of_range": { + "args": { + "requested": { + "kind": "integer" + } + }, + "class": "input", + "description": "The requested pre-roll is outside the range of zero to 5000 script bars.", + "english": [ + "Pre-roll must be between zero and 5000 script bars." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_prestart_fill": { + "args": { + "fill_ms": { + "kind": "integer" + }, + "start_ms": { + "kind": "integer" + } + }, + "class": "engine_fault", + "description": "A fill was dated before the selected window: an engine invariant failure.", + "english": [ + "A fill was dated before the selected window." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_prestart_state_violation": { + "args": { + "position_size": { + "kind": "number" + }, + "resting_orders": { + "kind": "integer" + }, + "time_ms": { + "kind": "integer" + } + }, + "class": "engine_fault", + "description": "A position or a resting order existed before the selected window started: an engine invariant failure.", + "english": [ + "The selected-window pre-start state invariant failed." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_pruner_unsupported": { + "args": { + "pruner": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "A study that requests a selected window together with a trial pruner other than none is refused before any trial is allocated; pruner is the requested pruner's name as the caller sent it. Optimizer-only: run_json never raises it, because a backtest has no pruner.", + "english": [ + "The selected window cannot be combined with this trial pruner." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_range_invalid": { + "args": { + "end_ms": { + "kind": "integer" + }, + "fed_start_ms": { + "kind": "integer" + }, + "start_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "The selected-window range (start, end, fed start) is invalid.", + "english": [ + "The selected-window range is invalid." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_report_mismatch": { + "args": { + "actual": { + "kind": "integer" + }, + "expected": { + "kind": "integer" + }, + "field": { + "kind": "vocab", + "values": [ + "counts.window_input_bars", + "counts.window_script_bars", + "metering.billable_input_bars" + ] + } + }, + "class": "engine_fault", + "description": "The report disagrees with the trusted window counts. Raised by the caller that compares them and never emitted by run_json.", + "english": [ + "The report disagrees with trusted window counts." + ], + "retryable": false, + "since": "1.5.0" + }, + "window_request_invalid": { + "args": { + "option": { + "kind": "vocab", + "values": [ + "report_policy", + "window_start_ms", + "window_end_ms", + "preroll_bars", + "fed_start_ms", + "feed_bytes_sha256", + "input_tf", + "script_tf", + "chart_tz", + "trade_start_ms", + "window", + "schema_version", + "objective" + ] + } + }, + "class": "input", + "description": "The selected-window request is invalid: option names the missing, malformed or conflicting field.", + "english": [ + "The selected-window request is invalid." + ], + "retryable": false, + "since": "1.5.0" } }, "kinds": { "identifier": "A name the compiled strategy spells as a literal (a built-in, an argument).", "integer": "A JSON integer.", "keyword": "A fixed keyword.", + "nullable_string": "UTF-8 text, or null when the request value is absent.", "number": "A JSON number (shortest round-trip double).", "pine_source": "Source text the transpiler emitted as a literal (a call's spelling, an input title).", + "scalar": "A request value: UTF-8 text, an integer, a finite number, true, false or null.", + "string": "UTF-8 text the trusted raiser supplies (a request value or a fixed name).", "symbol": "A symbol the script wrote as a literal.", "timeframe": "A timeframe the run request supplied.", "vocab": "One value of the arg's closed `values` list." diff --git a/docker/run_failure_codes_diff.json b/docker/run_failure_codes_diff.json index dd90d91e8..e2b0c14c7 100644 --- a/docker/run_failure_codes_diff.json +++ b/docker/run_failure_codes_diff.json @@ -8,14 +8,74 @@ "to": { "version": null, "unreleased": true, - "catalogSha256": "bd2b3dcbe1646275cb392e8f3408cccab21186671be4908fb72aa0d54f19d596" + "catalogSha256": "7d61bad51f1f624a83fe90abb64646758c89f6e23c099e52167f9f1c11994d26" }, "catalogSchema": { "before": "pineforge-run-failure-catalog/v1", "after": "pineforge-run-failure-catalog/v1" }, - "metadataChanges": [], + "metadataChanges": [ + { + "path": "kinds.nullable_string", + "beforePresent": false, + "afterPresent": true, + "before": null, + "after": "UTF-8 text, or null when the request value is absent." + }, + { + "path": "kinds.scalar", + "beforePresent": false, + "afterPresent": true, + "before": null, + "after": "A request value: UTF-8 text, an integer, a finite number, true, false or null." + }, + { + "path": "kinds.string", + "beforePresent": false, + "afterPresent": true, + "before": null, + "after": "UTF-8 text the trusted raiser supplies (a request value or a fixed name)." + } + ], "added": [ + { + "code": "engine_command_value_invalid", + "collection": "codes", + "entry": { + "args": { + "field": { + "kind": "string" + }, + "reason": { + "kind": "vocab", + "values": [ + "nonfinite", + "invalid_unicode", + "numeric_overflow" + ] + }, + "stage": { + "kind": "vocab", + "values": [ + "sizing", + "margin", + "fee", + "slippage", + "relative_price", + "risk", + "settlement" + ] + } + }, + "class": "engine_fault", + "description": "The engine derived an invalid command value (non-finite, invalid text or an overflowing finite value) from its own book, equity, margin, sizing, fee, slippage or settlement state: an engine fault, never a strategy failure. stage names where it was derived.", + "english": [ + "The engine derived an invalid execution value." + ], + "retryable": false, + "since": "1.5.0" + } + }, { "code": "outputs_limit", "collection": "codes", @@ -68,6 +128,569 @@ "retryable": false, "since": "1.5.0" } + }, + { + "code": "report_platform_unavailable", + "collection": "codes", + "entry": { + "args": { + "field": { + "kind": "vocab", + "values": [ + "os", + "architecture", + "pointer_bits", + "endianness", + "engine_library_sha256", + "harness_sha256" + ] + } + }, + "class": "engine_fault", + "description": "The runner cannot establish an execution-platform fact its report contract requires. A pre-load backstop for an image that claimed results-digest support, raised before any strategy code loads.", + "english": [ + "The runner cannot establish required execution-platform identity." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "report_post_execution_failed", + "collection": "codes", + "entry": { + "args": { + "phase": { + "kind": "vocab", + "values": [ + "result_assembly", + "results_digest", + "serialization" + ] + }, + "reason": { + "kind": "vocab", + "values": [ + "deadline", + "exception", + "io", + "resource" + ] + } + }, + "class": "engine_fault", + "description": "The engine could not finish producing the result after strategy execution completed: a deadline or an otherwise unclassified failure in result assembly, the results digest or serialization. An engine fault, never a strategy failure; a specific typed engine fault keeps its own code.", + "english": [ + "The engine could not finish producing the result after strategy execution completed." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "report_result_invalid", + "collection": "codes", + "entry": { + "args": { + "field": { + "kind": "string" + }, + "reason": { + "kind": "vocab", + "values": [ + "nonfinite", + "invalid_unicode", + "column_schema", + "type_schema" + ] + } + }, + "class": "engine_fault", + "description": "The report producer generated an invalid canonical result: an engine or harness fault, never a strategy failure.", + "english": [ + "The report producer generated an invalid canonical result." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "report_shape_request_invalid", + "collection": "codes", + "entry": { + "args": { + "option": { + "kind": "vocab", + "values": [ + "report_shape", + "curve_point_budget" + ] + }, + "value": { + "kind": "scalar" + } + }, + "class": "input", + "description": "The report-shape request is invalid: the report shape or the curve point budget carries a value the runner refuses.", + "english": [ + "The report-shape request is invalid." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "report_shape_unsupported", + "collection": "codes", + "entry": { + "args": { + "shape": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "The runner does not support the requested report shape. The first selected-window build supports the full shape only.", + "english": [ + "This runner does not support the requested report shape." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "results_digest_unsupported", + "collection": "codes", + "entry": { + "args": { + "requested": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "The runner does not support the requested results digest. The first selected-window build declares no digest.", + "english": [ + "This runner does not support the requested results digest." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "strategy_value_invalid", + "collection": "codes", + "entry": { + "args": { + "field": { + "kind": "string" + }, + "reason": { + "kind": "vocab", + "values": [ + "nonfinite", + "invalid_unicode", + "numeric_overflow" + ] + } + }, + "class": "strategy", + "description": "The strategy supplied an invalid actionable value (a non-finite number, invalid text or an overflowing finite value) to a declaration, setting or command that needs a valid one.", + "english": [ + "The strategy supplied an invalid execution value." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_boundary_unaligned", + "collection": "codes", + "entry": { + "args": { + "bound": { + "kind": "vocab", + "values": [ + "start", + "end", + "fed_start" + ] + }, + "next_boundary_ms": { + "kind": "integer" + }, + "previous_boundary_ms": { + "kind": "integer" + }, + "value_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "A selected-window boundary (start, end or fed start) is not aligned to the script calendar; the adjacent boundaries are carried.", + "english": [ + "A window boundary is not aligned to the script calendar." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_calendar_unsupported", + "collection": "codes", + "entry": { + "args": { + "session": { + "kind": "nullable_string" + }, + "timezone": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "The timezone and session the selected window needs make a calendar the engine does not support; session may be null.", + "english": [ + "The requested window calendar is unsupported." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_feed_identity_changed", + "collection": "codes", + "entry": { + "args": { + "actual_sha256": { + "kind": "string" + }, + "expected_sha256": { + "kind": "string" + }, + "feed_id": { + "kind": "string" + } + }, + "class": "input", + "description": "The frozen primary feed's bytes differ from the SHA-256 the study recorded. The two digests are 64-character hexadecimal strings supplied by the trusted wrapper; the registry checks only that they are text.", + "english": [ + "The frozen primary feed bytes have changed." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_feed_range_invalid", + "collection": "codes", + "entry": { + "args": { + "end_ms": { + "kind": "integer" + }, + "fed_start_ms": { + "kind": "integer" + }, + "time_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "A primary-feed bar lies outside the frozen selected-window range.", + "english": [ + "The primary feed lies outside the frozen range." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_input_cap_exceeded", + "collection": "codes", + "entry": { + "args": { + "actual": { + "kind": "integer" + }, + "cap_name": { + "kind": "vocab", + "values": [ + "run_input_bars", + "run_script_bars", + "native_input_bars", + "hpo_input_bars", + "hpo_script_bars", + "hpo_memory_bytes", + "symbol_feed_input_bars", + "symbol_feeds_total_input_bars" + ] + }, + "limit": { + "kind": "integer" + } + }, + "class": "input", + "description": "The supplied history exceeds a configured input cap; cap_name says which.", + "english": [ + "The supplied history exceeds the configured input cap." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_legacy_flag_conflict", + "collection": "codes", + "entry": { + "args": { + "option": { + "kind": "string" + } + }, + "class": "input", + "description": "The legacy trade-start option was combined with selected-window mode.", + "english": [ + "The legacy trade-start option cannot be combined with selected-window mode." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_lineage_mismatch", + "collection": "codes", + "entry": { + "args": { + "actual": { + "kind": "scalar" + }, + "expected": { + "kind": "scalar" + }, + "field": { + "kind": "string" + } + }, + "class": "input", + "description": "A requested window field differs from the study's lineage.", + "english": [ + "The requested window differs from the study lineage." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_mode_unsupported", + "collection": "codes", + "entry": { + "args": { + "capability": { + "kind": "string" + }, + "requested": { + "kind": "integer", + "optional": true + }, + "timeframe": { + "kind": "string", + "optional": true + } + }, + "class": "unsupported", + "description": "The runner does not support the requested selected-window mode or a limit its build declares on it. capability names what is unsupported; timeframe carries the offending canonical timeframe and requested the offending pre-roll length where the refusal names one. Which optional argument goes with which capability is the trusted raisers' duty, not the registry's.", + "english": [ + "This runner does not support the requested window mode." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_no_observations", + "collection": "codes", + "entry": { + "args": { + "end_ms": { + "kind": "integer" + }, + "start_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "The selected window holds no observations to score.", + "english": [ + "The study has no selected observations to score." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_preroll_out_of_range", + "collection": "codes", + "entry": { + "args": { + "requested": { + "kind": "integer" + } + }, + "class": "input", + "description": "The requested pre-roll is outside the range of zero to 5000 script bars.", + "english": [ + "Pre-roll must be between zero and 5000 script bars." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_prestart_fill", + "collection": "codes", + "entry": { + "args": { + "fill_ms": { + "kind": "integer" + }, + "start_ms": { + "kind": "integer" + } + }, + "class": "engine_fault", + "description": "A fill was dated before the selected window: an engine invariant failure.", + "english": [ + "A fill was dated before the selected window." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_prestart_state_violation", + "collection": "codes", + "entry": { + "args": { + "position_size": { + "kind": "number" + }, + "resting_orders": { + "kind": "integer" + }, + "time_ms": { + "kind": "integer" + } + }, + "class": "engine_fault", + "description": "A position or a resting order existed before the selected window started: an engine invariant failure.", + "english": [ + "The selected-window pre-start state invariant failed." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_pruner_unsupported", + "collection": "codes", + "entry": { + "args": { + "pruner": { + "kind": "string" + } + }, + "class": "unsupported", + "description": "A study that requests a selected window together with a trial pruner other than none is refused before any trial is allocated; pruner is the requested pruner's name as the caller sent it. Optimizer-only: run_json never raises it, because a backtest has no pruner.", + "english": [ + "The selected window cannot be combined with this trial pruner." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_range_invalid", + "collection": "codes", + "entry": { + "args": { + "end_ms": { + "kind": "integer" + }, + "fed_start_ms": { + "kind": "integer" + }, + "start_ms": { + "kind": "integer" + } + }, + "class": "input", + "description": "The selected-window range (start, end, fed start) is invalid.", + "english": [ + "The selected-window range is invalid." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_report_mismatch", + "collection": "codes", + "entry": { + "args": { + "actual": { + "kind": "integer" + }, + "expected": { + "kind": "integer" + }, + "field": { + "kind": "vocab", + "values": [ + "counts.window_input_bars", + "counts.window_script_bars", + "metering.billable_input_bars" + ] + } + }, + "class": "engine_fault", + "description": "The report disagrees with the trusted window counts. Raised by the caller that compares them and never emitted by run_json.", + "english": [ + "The report disagrees with trusted window counts." + ], + "retryable": false, + "since": "1.5.0" + } + }, + { + "code": "window_request_invalid", + "collection": "codes", + "entry": { + "args": { + "option": { + "kind": "vocab", + "values": [ + "report_policy", + "window_start_ms", + "window_end_ms", + "preroll_bars", + "fed_start_ms", + "feed_bytes_sha256", + "input_tf", + "script_tf", + "chart_tz", + "trade_start_ms", + "window", + "schema_version", + "objective" + ] + } + }, + "class": "input", + "description": "The selected-window request is invalid: option names the missing, malformed or conflicting field.", + "english": [ + "The selected-window request is invalid." + ], + "retryable": false, + "since": "1.5.0" + } } ], "changed": [ diff --git a/docker/run_json.py b/docker/run_json.py index 456f6ed06..8380e7d4d 100755 --- a/docker/run_json.py +++ b/docker/run_json.py @@ -130,6 +130,22 @@ NaN convention: any metric with an empty/zero denominator is null (JSON has no NaN); a real computed 0 stays 0. See the report-schema + metrics reference docs for the per-field meaning of every metrics.* key. + +Selected-window mode (selected-window wire v1.3, full/v1 report, no digest). Opt in +with --report-policy selected-window/v1, --window-start-ms T, --window-end-ms E, +--preroll-bars N, --fed-start-ms F and an explicit --input-tf and --script-tf (the +primary chart token); --request-feed-inventory names the inventory bound to --so. +The request is admitted first, from the inventory and the build's candidate support +constants, before any feed, library or strategy is touched. The native planner then +plans the window; the engine runs the retained rows (pre-roll plus window) and +supplies the selected trades, curve and metrics. The report gains report_window (R), +report_shape, diagnostics.phase_timing, fingerprint.version 2 and +fingerprint.provenance.schema_version 2; R is also applied_runtime.report_window and +fingerprint.provenance.runtime.report_window. --validate-window-only prints +{"validation_only": true, "report_window": R} with applied=false and runs nothing. +--so LIB --capabilities-json prints this runtime's capabilities. --run-phase-fd N +sends the phase records (selected and ordinary runs). Without any of these options a +run is the ordinary run described above, byte for byte. """ from __future__ import annotations @@ -3629,10 +3645,12 @@ def build_report_dict(report: ReportC, ohlcv_path: Path, "input": { "ohlcv": str(ohlcv_path), "bars": n_bars, - "first_ts": int(first_ts), - "last_ts": int(last_ts), - "first_time": fmt_utc(first_ts), - "last_time": fmt_utc(last_ts), + # None only for a selected-window run whose evaluated feed is empty + # (first_ts/last_ts None); an ordinary run always passes integers. + "first_ts": None if first_ts is None else int(first_ts), + "last_ts": None if last_ts is None else int(last_ts), + "first_time": None if first_ts is None else fmt_utc(first_ts), + "last_time": None if last_ts is None else fmt_utc(last_ts), }, "applied_inputs": applied_inputs, "applied_overrides": applied_overrides, @@ -3836,9 +3854,588 @@ def run_failure(lib, strat): return text or RUN_STATUS_FAILED_TEXT, "engine_unclassified_error", {} +# --- Selected-window mode (selected-window/v1: full/v1 report, no digest) ------------------- +# +# A request that names --report-policy selected-window/v1, a window option or +# --validate-window-only is run by _selected_run. Every other request follows the ordinary body +# of _main, which gains only the optional phase records of --run-phase-fd (they change no report +# byte). docker/selected_window_report.py holds the pure pieces. The sibling modules +# run_phase_transport, run_execution_observer, request_feed_inventory and selected_window_plan +# are imported lazily, and only on the paths that need them, so an ordinary run never loads them +# and never reads an inventory, a policy or the planner library. +# +# The order of a selected run; each step ends the run with its typed failure line: +# 1. validate the flags and the request: nothing is read, nothing is loaded; +# 2. open the phase writer (its descriptor is checked before any strategy code loads); +# 3. ADMISSION from the request-feed inventory and the candidate support constants, before +# load_bars, load_symbol_feeds, any ctypes load (the planner included), any strategy +# construction and the first phase record; +# 4. phase preflight; load the primary feed once; plan with the native planner; trim by a +# pointer offset; hash. --validate-window-only ends here, applied=false, no strategy run; +# 5. phase execution, BEFORE the strategy library loads; load it, configure the selected-window +# ABI, attach the observer to the phase writer, run; +# 6. the OBSERVER (never this file) moves the phase to result_assembly during the native call. +# Afterwards the retained callback exception is raised first, the native counts must equal +# the plan, the report is assembled (the engine supplies the selected trades, curve and +# metrics; nothing is recalculated here) and serialized once the serialization phase has +# begun; completed follows the stdout flush. + +_TYPED_ERROR_NAMES = frozenset({"SelectedWindowError", "InventoryAdmissionError", + "PlanBridgeError", "ObserverBindingError", + "PhaseTransportError"}) + + +def _typed_failure(error): + """The RunFailure for an exception raised by one of the selected-window helper modules + (code, message and the code's arguments as that module defines them), or None for any other + exception. Matched by class name so this file imports none of the modules at load.""" + if type(error).__name__ not in _TYPED_ERROR_NAMES: + return None + code = getattr(error, "code", None) + if not isinstance(code, str) or _CODE_RE.fullmatch(code) is None: + return None + code_args = getattr(error, "code_args", None) + if code_args is None: + code_args = getattr(error, "detail", None) # InventoryAdmissionError + text = getattr(error, "message", None) or str(error) + return RunFailure(text, code, dict(code_args) if isinstance(code_args, dict) else {}) + + +def _engine_invariant(text: str) -> RunFailure: + return RunFailure(text, "engine_invariant") + + +def trusted_planner_path() -> str: + """The absolute path of the trusted standalone planner helper of this installation: + /lib/libpineforge_window_plan.so, where is the parent of the directory + this run_json.py is installed in (bin). Never a strategy library and never a path search.""" + return str(Path(__file__).resolve().parent.parent / "lib" / "libpineforge_window_plan.so") + + +def _selected_requested(args) -> bool: + """Whether the command line opts into selected-window mode: the policy, any of the four + window options or the validation-only switch. A window option without the policy is not + legacy: it is refused (window_request_invalid, option report_policy).""" + return (args.report_policy is not None or args.window_start_ms is not None + or args.window_end_ms is not None or args.preroll_bars is not None + or args.fed_start_ms is not None or bool(args.validate_window_only)) + + +class _Phases: + """The run's RunPhaseWriter with its error type: a transport failure of any advance() is + the typed harness fault (harness_internal_error), never a strategy outcome.""" + + def __init__(self, writer, error_type): + self.writer = writer + self._error_type = error_type + + def advance(self, phase: str) -> None: + try: + self.writer.advance(phase) + except self._error_type as error: + raise _typed_failure(error) from None + + +def _open_phases(fd_text, *, always: bool): + """The run's phase session, or None. fd_text is --run-phase-fd as written: None gives + None for an ordinary run (always=False) and a writer that exports nothing but still + tracks the phases for a selected run (always=True). A descriptor that is not a number or + not a usable socket is a preflight harness_internal_error (phase transport v1).""" + if fd_text is None and not always: + return None + import run_phase_transport + fd = None + if fd_text is not None: + if re.fullmatch(r"[0-9]{1,10}", fd_text) is None: + raise RunFailure(f"--run-phase-fd must be a descriptor number from 3 up, got " + f"{_shown(fd_text)}", "harness_internal_error") + fd = int(fd_text) + try: + writer = run_phase_transport.RunPhaseWriter(fd) + except run_phase_transport.PhaseTransportError as error: + raise _typed_failure(error) from None + return _Phases(writer, run_phase_transport.PhaseTransportError) + + +def _advance(phases, phase: str) -> None: + if phases is not None: + phases.advance(phase) + + +def _require_result_assembly(phases) -> None: + """After a successful native run the engine's observer must have moved the writer to + result_assembly. It is never announced from here: anything else is a harness fault, not + a strategy outcome.""" + if phases.writer.phase != "result_assembly": + raise RunFailure( + "the run ended without the engine announcing the end of execution (phase " + f"{phases.writer.phase!r})", "harness_internal_error") + + +# Post-execution classification (the documented selected-window contract). Once the engine's +# observer has moved the REAL phase writer to result_assembly, strategy execution is over: a +# failure that is otherwise unclassified is the engine's, non-billable, and is +# report_post_execution_failed{phase, reason}, never a strategy error or timeout. The phase is +# read from the writer; it is never inferred from Python's _run returning. A specific typed +# fault (a RunFailure, a helper module's typed error, an engine failure with its own code) keeps +# its code, and a failure while the writer is still in `execution` is left to the rules that +# applied before this section existed. +_POST_BOUNDARY_PHASES = ("result_assembly", "results_digest", "serialization") +POST_EXECUTION_TEXT = ("The engine could not finish producing the result after strategy " + "execution completed.") + + +def _boundary_phase(phases): + """The phase of the real writer when the end of execution has been acknowledged + (result_assembly or later), else None (no writer, or still executing).""" + if phases is None: + return None + phase = phases.writer.phase + return phase if phase in _POST_BOUNDARY_PHASES else None + + +def _post_boundary_failure(error, phases): + """report_post_execution_failed for an exception that is neither a RunFailure nor a typed + helper error, raised after the acknowledged boundary: reason resource for MemoryError, io + for OSError, exception for everything else. None when the boundary was not acknowledged + or the error is a specific typed fault.""" + phase = _boundary_phase(phases) + if phase is None or isinstance(error, RunFailure) or _typed_failure(error) is not None: + return None + if isinstance(error, MemoryError): + reason = "resource" + elif isinstance(error, OSError): + reason = "io" + else: + reason = "exception" + return RunFailure(POST_EXECUTION_TEXT, "report_post_execution_failed", + {"phase": phase, "reason": reason}) + + +def _post_boundary_engine_failure(failure, phases): + """The (text, code, args) of an engine failure line. After the acknowledged boundary an + engine out_of_memory (reason resource) and an engine_unclassified_error (reason exception) + are report_post_execution_failed; every other engine code, and every failure + before the boundary (a strategy failure during execution included), is returned unchanged.""" + phase = _boundary_phase(phases) + if phase is None: + return failure + code = failure[1] + if code == "out_of_memory": + reason = "resource" + elif code == "engine_unclassified_error": + reason = "exception" + else: + return failure + return POST_EXECUTION_TEXT, "report_post_execution_failed", {"phase": phase, "reason": reason} + + +def _observer_binding(lib, state, writer): + import run_execution_observer + return run_execution_observer.ExecutionObserverBinding(lib, state, writer) + + +def _detach_observer(binding): + """Unregister the observer in cleanup. The error, if any, is returned instead of raised: + the frees that follow must still run. The binding keeps its callback alive until the + caller lets go of it, after the state is freed.""" + if binding is None: + return None + try: + binding.detach() + except Exception as error: + return error + return None + + +def _read_observation(lib, state) -> dict: + """The identities (run_generation, attempt_serial, attempt_generation) of the latest + attempt from strategy_execution_observation_v1.""" + import run_execution_observer as observer + observation = observer.ObservationC(ctypes.sizeof(observer.ObservationC), + observer.OBSERVER_VERSION) + status = lib.strategy_execution_observation_v1(state, ctypes.byref(observation)) + if status != 0: + raise RunFailure(f"strategy_execution_observation_v1 returned {status}", + "harness_internal_error") + return {"run_generation": int(observation.run_generation), + "attempt_serial": int(observation.attempt_serial), + "attempt_generation": int(observation.attempt_generation)} + + +def _syminfo_clock(syminfo_path) -> tuple: + """(timezone, session) of the --syminfo file as text, "" for an absent or empty member: + read as data, with no library, in the shape apply_syminfo reads it (flat or wrapped under + "syminfo"; a member that is present is str()-ed, as apply_syminfo sends it). The planner + treats "" as UTC and 24x7. A file apply_syminfo would refuse has the same refusal here.""" + def unreadable(text, reason): + return SyminfoError(text, "syminfo_unreadable", {"reason": reason}) + + try: + with open(syminfo_path) as f: + text = f.read() + except OSError as e: + raise unreadable(f"--syminfo: {syminfo_path}: {e.strerror or e}", "io") from None + except ValueError as e: + raise unreadable(f"--syminfo: {syminfo_path} is not JSON: {e}", "not_json") from None + try: + doc = json.loads(text) + except (ValueError, RecursionError) as e: + raise unreadable(f"--syminfo: {syminfo_path} is not JSON: {e}", "not_json") from None + si = doc.get("syminfo", doc) if isinstance(doc, dict) else None + if not isinstance(si, dict): + raise unreadable(f"--syminfo: {syminfo_path}: the syminfo is not a JSON object", + "not_object") + clock = [] + for key in ("timezone", "session"): + value = si.get(key) + value = str(value) if value else "" + try: + value.encode("utf-8") + except UnicodeEncodeError: + raise unreadable(f"syminfo.{key} must be UTF-8 text", "value_type") from None + clock.append(value) + return clock[0], clock[1] + + +def _write_report(text: str, phases=None) -> int: + """Write the whole document to stdout and flush: 0, or 1 when stdout cannot take it. No + failure line (no second JSON document) may follow part of a report, so the reason goes to + stderr, stdout is pointed at /dev/null, the exit status is 1 and the phase writer is left + where it is (serialization): the watchdog sees the terminal status and the last phase. + With a phase session (a run past the boundary) any exception from the write is that + outcome, and the stderr line names it as report_post_execution_failed with the real + phase and the reason (resource, io or exception); without one, only OSError and ValueError + are, and the line is the ordinary one.""" + caught = (OSError, ValueError) if phases is None else Exception + try: + sys.stdout.write(text) + sys.stdout.flush() + except caught as error: + _discard_stdout() + if phases is None: + print(f"run_json: the report could not be written: {error}", file=sys.stderr) + else: + reason = ("resource" if isinstance(error, MemoryError) + else "io" if isinstance(error, (OSError, ValueError)) else "exception") + print("run_json: report_post_execution_failed (phase " + f"{phases.writer.phase}, reason {reason}): the report could not be written " + f"whole: {error}", file=sys.stderr) + return 1 + return 0 + + +def _runtime_supports_selected_window(lib) -> bool: + """Whether this runtime can run selected-window mode: the selected-window ABI and the + execution observer at version 1 on the given library, the phase transport and the trusted + planner helper loadable. False on any missing piece, never an exception for a missing + one; no feed is read and nothing runs.""" + try: + import run_execution_observer + import run_phase_transport # noqa: F401 (present is all the probe asks of it) + import selected_window_plan + import selected_window_report as swr + swr.SelectedWindowAbi(lib) + for name in ("pf_execution_observer_version", "strategy_set_execution_observer_v1", + "strategy_execution_observation_v1"): + if not hasattr(lib, name): + return False + lib.pf_execution_observer_version.argtypes = [] + lib.pf_execution_observer_version.restype = ctypes.c_uint32 + if lib.pf_execution_observer_version() != run_execution_observer.OBSERVER_VERSION: + return False + selected_window_plan.SelectedPrimaryPlanner(trusted_planner_path()) + except (ImportError, OSError): + return False + except Exception as error: + if _typed_failure(error) is not None: + return False + raise + return True + + +def _capabilities_main(args) -> int: + """--so LIB --capabilities-json: what this runtime supports, without OHLCV and without a + run. LIB is the build's trusted canary library; it is loaded like any strategy library + (ABI check, required exports) and no strategy is constructed.""" + import selected_window_report as swr + lib = load_strategy(args.so) + document = swr.capabilities_record(_runtime_supports_selected_window(lib)) + return _write_report(json.dumps(document, separators=(",", ":")) + "\n") + + +def _selected_run(args, inputs, overrides, input_tf, script_tf, bar_magnifier, + magnifier_samples, magnifier_dist) -> int: + import request_feed_inventory + import selected_window_report as swr + + # 1. The request. Nothing is read or loaded until admission has passed. + request = swr.validate_request( + policy=args.report_policy, window_start_ms=args.window_start_ms, + window_end_ms=args.window_end_ms, preroll_bars=args.preroll_bars, + fed_start_ms=args.fed_start_ms, trade_start_ms=args.trade_start_ms, + input_tf=input_tf.decode("utf-8"), script_tf=script_tf.decode("utf-8")) + if args.bench: + raise _request_invalid( + "error: --bench cannot be combined with selected-window mode", "bench") + # 2. The phase transport is checked before any strategy code loads. Validation only runs + # no strategy and has no phases. + phases = None if args.validate_window_only else _open_phases(args.run_phase_fd, always=True) + # 3. Admission: the candidate support constants against the primary chart token and the + # artifact's request-feed inventory (this --so, hashed and never loaded; the inventory + # beside it). Raises InventoryAdmissionError (window_mode_unsupported) before any feed, + # ctypes library, strategy or phase record. The refusal keeps its code and detail. + try: + request_feed_inventory.admit_selected_request( + request.script_tf, args.request_feed_inventory, args.so, inputs, + swr.support_policy()) + except request_feed_inventory.InventoryAdmissionError as error: + raise RunFailure(error.message, error.code, error.detail) from None + + # 4. Preflight: the primary feed is read once and planned by the native planner. + _advance(phases, "preflight") + bars, n, source_values_sha256 = load_bars(args.ohlcv) + source_bytes_sha256 = _sha256_file(args.ohlcv) + if source_bytes_sha256 is None: + raise _bars_unreadable(f"--ohlcv: {args.ohlcv}: the file cannot be read", "io") + timezone_text, session_text = _syminfo_clock(args.syminfo) if args.syminfo else ("", "") + import selected_window_plan + planner = selected_window_plan.SelectedPrimaryPlanner(trusted_planner_path()) + plan = planner.plan( + bars, n, start_ms=request.start_ms, end_ms=request.end_ms, + fed_start_ms=request.fed_start_ms, preroll_bars=request.preroll_bars, + input_tf=request.input_tf, script_tf=request.script_tf, + chart_timezone=args.chart_tz, engine_timezone=timezone_text, session=session_text, + feed_tolerant=True) + refusal = swr.refusal_for_plan( + plan, request, timezone=swr.calendar_timezone(args.chart_tz), + session=swr.calendar_session(session_text)) + if refusal is not None: + raise refusal + # The retained rows are a view of the same allocation; the evaluated hash restarts the + # existing primary hash domain over them (equal to the supplied hash when nothing is cut). + retained, fed_count = swr.retained_rows( + bars, n, plan["trim_index"], plan["fed_input_bars"], BarC) + evaluated_values_sha256 = source_values_sha256 + if plan["trim_index"]: + hasher = _new_source_feed_hasher() + swr.update_rows_hash(hasher, bars, plan["trim_index"], fed_count, BarC, + _SOURCE_FEED_RECORD) + evaluated_values_sha256 = hasher.hexdigest() + feed = { + "input_tf_seconds": request.input_tf_seconds, + "source_bytes_sha256": source_bytes_sha256, + "source_values_sha256": source_values_sha256, + "evaluated_source_values_sha256": evaluated_values_sha256, + } + calendar = swr.calendar_record(args.chart_tz, session_text) + if args.validate_window_only: + # Not execution evidence: the counts are the plan's, no strategy was loaded or run. + window = swr.build_report_window(request, plan, applied=False, feed=feed, + calendar=calendar) + return _write_report(json.dumps( + {"engine": "pineforge", "validation_only": True, "report_window": window}, + separators=(",", ":")) + "\n") + symbol_feeds = load_symbol_feeds(args.symbol_feeds) if args.symbol_feeds else None + + # 5. Execution begins before the strategy library loads. + _advance(phases, "execution") + lib = load_strategy(args.so) + checked = uses_checked_settings(lib) + if args.outputs: + require_outputs(lib) + abi = swr.SelectedWindowAbi(lib) + vw_on = bool(args.magnifier_volume_weighted) and bar_magnifier == 1 + syminfo_applied: dict = {} + + def _make_state(): + """A fresh, fully configured strategy state (the ordinary run's setup without the + legacy trade-start gate, which selected mode refuses); freed again if it fails.""" + nonlocal syminfo_applied + st = create_strategy(lib, checked) + try: + apply_settings(lib, st, inputs, overrides, checked) + if args.syminfo: + r = apply_syminfo(lib, st, args.syminfo) + syminfo_applied = r if isinstance(r, dict) else {} + if symbol_feeds: + install_symbol_feeds(lib, st, symbol_feeds) + if args.chart_tz and hasattr(lib, "strategy_set_chart_timezone"): + lib.strategy_set_chart_timezone(st, args.chart_tz.encode()) + if vw_on and hasattr(lib, "strategy_set_magnifier_volume_weighted"): + lib.strategy_set_magnifier_volume_weighted(st, 1) + if args.outputs: + enable_outputs(lib, st) + except BaseException: + lib.strategy_free(st) + raise + return st + + state = _make_state() + for sym in symbol_feeds or (): + for feed_record in sym["feeds"]: + feed_record["bars"] = feed_record["close_ms"] = None + report = ReportC() + binding = None + started = time.time() + try: + settings_receipt = _release_settings_receipt(lib, state, checked) + abi.configure(state, request.start_ms, request.end_ms) + binding = _observer_binding(lib, state, phases.writer) + binding.attach() + lib.run_backtest_full( + state, retained, fed_count, + input_tf, script_tf, + bar_magnifier, magnifier_samples, magnifier_dist, + ctypes.byref(report)) + elapsed = time.time() - started + # The callback's retained exception comes before any report access. + binding.raise_if_failed() + failure = run_failure(lib, state) + if failure is not None: + # After the acknowledged boundary an engine out_of_memory or unclassified failure + # is report_post_execution_failed; a failure during execution is unchanged. + write_failure(*_post_boundary_engine_failure(failure, phases)) + return 1 + _require_result_assembly(phases) + # The engine's own counts must equal the plan's, and belong to the observed attempt. + swr.check_native_counts(plan, abi.counts(state), _read_observation(lib, state)) + if (int(report.input_bars_processed) != plan["fed_input_bars"] + or int(report.script_bars_processed) != plan["fed_script_bars"]): + raise _engine_invariant( + "the report's processed counts differ from the plan's evaluated-fed counts") + if int(report.input_tf_seconds) != request.input_tf_seconds: + raise _engine_invariant( + f"the engine reports input_tf_seconds {int(report.input_tf_seconds)}, the " + f"request's input timeframe is {request.input_tf_seconds} seconds") + if int(report.equity_curve_len) != plan["window_script_bars"] + 1: + raise _engine_invariant( + f"the engine's selected curve has {int(report.equity_curve_len)} points, the " + f"plan says {plan['window_script_bars'] + 1}") + # The report spells the evaluated first and last bar as UTC dates (fmt_utc). + for ts in (plan["fed_first_data_ms"], plan["fed_last_data_ms"]): + if ts is None: + continue + try: + fmt_utc(ts) + except (ValueError, OverflowError, OSError): + raise _bars_unreadable( + f"--ohlcv: {args.ohlcv}: timestamp {ts} is out of the calendar's range", + "value") from None + window = swr.build_report_window(request, plan, applied=True, feed=feed, + calendar=calendar) + applied_runtime = { + "input_tf": input_tf.decode() if input_tf else "", + "script_tf": script_tf.decode() if script_tf else "", + "input_tf_seconds": int(report.input_tf_seconds), + "script_tf_seconds": int(report.script_tf_seconds), + "script_tf_ratio": int(report.script_tf_ratio), + "needs_aggregation": bool(report.needs_aggregation), + "bar_magnifier": bool(bar_magnifier), + "magnifier_samples": magnifier_samples, + "magnifier_dist": args.magnifier_dist.strip().lower() or "endpoints", + "magnifier_volume_weighted": vw_on, + "trade_start_ms": None, + "chart_tz": args.chart_tz or "", + } + if syminfo_applied: + applied_runtime["syminfo"] = syminfo_applied + if symbol_feeds: + applied_runtime["symbol_feeds"] = symbol_feeds_record(symbol_feeds) + outputs_block = None + if args.outputs: + outputs_block = build_outputs_block(make_outputs_reader(lib, state)) + applied_runtime["outputs"] = True + # ONE R object, placed in all three required places (the third is + # fingerprint.provenance.runtime, which is applied_runtime itself). + applied_runtime["report_window"] = window + incarnation_accessor = getattr(lib, "strategy_closed_trade_entry_incarnation", None) + trade_entry_incarnations = ( + [int(incarnation_accessor(state, i)) for i in range(report.trades_len)] + if incarnation_accessor is not None else None) + out = build_report_dict( + report, args.ohlcv, fed_count, plan["fed_first_data_ms"], plan["fed_last_data_ms"], + elapsed, inputs, overrides, applied_runtime, trade_entry_incarnations) + out["report_window"] = window + out["report_shape"] = swr.report_shape_full(len(out["equity_curve"]), + len(out["trades"])) + if outputs_block is not None: + out["outputs"] = outputs_block + # The selected fingerprint MUST exist, as version 2. There is no best-effort null here + # (the ordinary report keeps its own): any failure below ends the run before anything is + # serialized, as its own typed code, or, when otherwise unclassified, as + # report_post_execution_failed of the except clause at the end of this try. + provenance = build_provenance( + engine_version(lib), + None, + parse_bool(args.transpiled), + inputs, + overrides, + applied_runtime, + source_feed_sha256=source_values_sha256, + ) + provenance["codegen"]["generated_cpp_sha256"] = ( + _sha256_file(args.generated_cpp) if args.generated_cpp else None) + cpp_text = "" + if args.generated_cpp: + with open(args.generated_cpp, encoding="utf-8", errors="replace") as source: + cpp_text = source.read() + provenance = normalize_release_provenance( + provenance, cpp_text, settings_receipt, checked) + provenance["schema_version"] = swr.FINGERPRINT_VERSION + fingerprint = build_fingerprint(provenance) + fingerprint["version"] = swr.FINGERPRINT_VERSION + out["fingerprint"] = fingerprint + # R in all three places, the v2 fingerprint over the same bytes, and the anchor of the + # native curve at the strategy's initial capital: all checked before serialization. + swr.check_report_placements(out, window) + swr.check_selected_curve(out["equity_curve"], request.start_ms, + plan["window_script_bars"], + provenance["strategy"].get("initial_capital")) + # 6. Serialization begins; the timing is known now and is outside the provenance. + _advance(phases, "serialization") + out["diagnostics"]["phase_timing"] = phases.writer.phase_timing() + buffer = io.StringIO() + json.dump(out, buffer, separators=(",", ":")) + buffer.write("\n") + except Exception as error: + # Once the engine has acknowledged result_assembly (the REAL writer phase), an + # unclassified failure is report_post_execution_failed{phase, reason}; typed faults, + # and anything while still executing, are raised unchanged. + classified = _post_boundary_failure(error, phases) + if classified is None: + raise + raise classified from error + finally: + # Unregister the observer, clear the selection and free, on every path; the binding + # (callback and descriptor) stays referenced until after the state is freed. + detach_error = _detach_observer(binding) + try: + abi.clear(state) + except Exception: + pass + lib.report_free(ctypes.byref(report)) + lib.strategy_free(state) + if detach_error is not None and sys.exc_info()[1] is None: + raise _typed_failure(detach_error) or detach_error + status = _write_report(buffer.getvalue(), phases) + if status: + return status + try: + phases.advance("completed") # only after the flush + except RunFailure as error: + # The whole report is already out, so no failure line may follow it, and nothing may + # claim success: this is the harness fault (exit status 1, stderr), never a result. + print(f"run_json: harness_internal_error: the completed phase record could not be " + f"handed over: {error}", file=sys.stderr) + return 1 + return 0 + + def main(argv=None) -> int: """Run the harness. Every failure ends as the one failure line on stdout: - run_json's own (RunFailure) with its code, and anything unexpected as + run_json's own (RunFailure) with its code, a selected-window helper's typed error + (_typed_failure) with its code, and anything unexpected as harness_internal_error (its traceback on stderr). Exit status 1, or 2 for a command line argparse refuses. The one exception is a success report stdout cannot take whole (_main): exit 1 with the reason on stderr and no line after @@ -3849,17 +4446,49 @@ def main(argv=None) -> int: write_failure(str(failure), failure.code, failure.code_args) return failure.exit_status except Exception as error: + typed = _typed_failure(error) + if typed is not None: + write_failure(str(typed), typed.code, typed.code_args) + return typed.exit_status traceback.print_exc(file=sys.stderr) write_failure(f"harness internal error: {type(error).__name__}: {error}", "harness_internal_error") return 1 +def _legacy_abbreviations(argv, legacy_long, all_long): + """argv (sys.argv[1:] when None) as a list, with every FORMERLY UNIQUE legacy long-option + prefix spelled in full. argparse accepts a prefix of a long option when exactly one option + begins with it; the selected-window flags (--report-policy, --window-start-ms, ...) make + some prefixes that were unique ambiguous (--r, --re and --rep for --repeats, --c for + --chart-tz, --w for --warmup), which would turn a working ordinary command line into an + argparse error. A token `--name[=value]` whose name is not itself an option of `all_long` + and is a prefix of exactly one option of `legacy_long` (the long options captured before + the new flags were added) becomes that option, `=value` kept exactly as written. A prefix + of several legacy options (ambiguous before, ambiguous still) and a prefix of none (an + abbreviation of a new flag, left to argparse) are not touched; neither is any token that + does not begin with `--`, any token after a bare `--`, or any value.""" + tokens = list(sys.argv[1:] if argv is None else argv) + out = [] + for index, token in enumerate(tokens): + if token == "--": + out.extend(tokens[index:]) + break + if isinstance(token, str) and token.startswith("--"): + name, equals, value = token.partition("=") + if name not in all_long: + matches = [option for option in legacy_long if option.startswith(name)] + if len(matches) == 1: + token = matches[0] + equals + value + out.append(token) + return out + + def _main(argv=None) -> int: ap = _ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--so", type=Path, required=True, help="strategy.so path") - ap.add_argument("--ohlcv", type=Path, required=True, help="OHLCV CSV path") + ohlcv_option = ap.add_argument("--ohlcv", type=Path, required=True, help="OHLCV CSV path") ap.add_argument("--inputs", default="", help='JSON object overriding input.*() values, e.g. \'{"Fast Length": "8"}\'') ap.add_argument("--overrides", default="", @@ -3914,8 +4543,61 @@ def _main(argv=None) -> int: help="Record the library's outputs (a library compiled to record " "them) and write them as the report's \"outputs\" block; see " "build_outputs_block. A failure on a library that records none.") + # The long options of the command line BEFORE the selected-window flags below; see + # _legacy_abbreviations (the option strings of the parser, captured here and not typed out). + legacy_long = tuple(s for s in ap._option_string_actions if s.startswith("--")) + # Selected-window mode (selected-window/v1; see the section above main). All of these are + # absent from an ordinary command line, which then behaves exactly as before. + ap.add_argument("--report-policy", default=None, + help="selected-window/v1 opts into the selected-window report; absent = " + "the ordinary report, unchanged.") + ap.add_argument("--window-start-ms", default=None, + help="Selected window start T (unix ms, calendar-aligned).") + ap.add_argument("--window-end-ms", default=None, + help="Selected window end E (unix ms, calendar-aligned), T < E.") + ap.add_argument("--preroll-bars", default=None, + help="Requested pre-roll N in script bars (0..5000).") + ap.add_argument("--fed-start-ms", default=None, + help="Frozen start F of the supplied primary feed (unix ms), F <= T.") + ap.add_argument("--capabilities-json", action="store_true", + help="With --so only: print this runtime's selected-window capabilities " + "as JSON. Reads no OHLCV and runs nothing.") + ap.add_argument("--validate-window-only", action="store_true", + help="With the selected-window options: validate and plan the loaded " + "feed and print report_window (applied=false); runs no strategy.") + ap.add_argument("--run-phase-fd", default=None, + help="N >= 3: an inherited AF_UNIX SOCK_STREAM descriptor that receives " + "the run's phase records (phase transport v1).") + ap.add_argument("--request-feed-inventory", type=Path, default=None, + help="The request-feed inventory bound to --so; read only by a " + "selected-window request, for its admission.") + ap.add_argument("--report-shape", default=None, + help="This build offers full/v1 only, for selected-window requests.") + ap.add_argument("--curve-point-budget", default=None, + help="Refused: only the compact shape takes a budget, which this build " + "does not offer.") + ap.add_argument("--results-digest", default=None, + help="Refused: this build offers no results digest.") + all_long = tuple(s for s in ap._option_string_actions if s.startswith("--")) + argv = _legacy_abbreviations(argv, legacy_long, all_long) + # --capabilities-json is a probe of the runtime: it needs --so and no OHLCV. Every other + # command line keeps --ohlcv required, exactly as before. + probe = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + probe.add_argument("--capabilities-json", action="store_true") + if probe.parse_known_args(argv)[0].capabilities_json: + ohlcv_option.required = False args = ap.parse_args(argv) + if args.capabilities_json: + return _capabilities_main(args) + selected = _selected_requested(args) + if (selected or args.report_shape is not None or args.curve_point_budget is not None + or args.results_digest is not None): + import selected_window_report + selected_window_report.check_unsupported_options( + selected=selected, report_shape=args.report_shape, + curve_point_budget=args.curve_point_budget, results_digest=args.results_digest) + inputs = parse_kv_json(args.inputs, "--inputs") overrides = parse_kv_json(args.overrides, "--overrides") input_tf = _utf8_option(args.input_tf.strip(), "--input-tf") @@ -3925,11 +4607,21 @@ def _main(argv=None) -> int: magnifier_samples = max(2, int(args.magnifier_samples)) magnifier_dist = parse_magnifier_dist(args.magnifier_dist) + if selected: + return _selected_run(args, inputs, overrides, input_tf, script_tf, bar_magnifier, + magnifier_samples, magnifier_dist) + + # Ordinary mode. --run-phase-fd alone adds the run's phase records (preflight, execution, + # result_assembly through the engine's observer, serialization, completed after the + # flush); the report and every other output are unchanged. None without the flag. + phases = _open_phases(args.run_phase_fd, always=False) + _advance(phases, "preflight") bars, n, source_feed_sha256 = load_bars(args.ohlcv) if n == 0: raise _bars_unreadable(f"--ohlcv: {args.ohlcv}: no bars", "empty") first_ts, last_ts = bars[0].timestamp, bars[n - 1].timestamp + _advance(phases, "execution") lib = load_strategy(args.so) checked = uses_checked_settings(lib) if args.outputs: @@ -4022,15 +4714,25 @@ def _run(st, rep): for feed in sym["feeds"]: feed["bars"] = feed["close_ms"] = None report = ReportC() + binding = None started = time.time() try: settings_receipt = _release_settings_receipt(lib, state, checked) + if phases is not None: + binding = _observer_binding(lib, state, phases.writer) + binding.attach() _run(state, report) elapsed = time.time() - started + if binding is not None: + binding.raise_if_failed() # the callback's exception, before any report access failure = run_failure(lib, state) if failure is not None: - write_failure(*failure) + # Unchanged without --run-phase-fd; with it, an out_of_memory or unclassified engine + # failure after the acknowledged boundary is report_post_execution_failed. + write_failure(*_post_boundary_engine_failure(failure, phases)) return 1 + if phases is not None: + _require_result_assembly(phases) # The report spells the first and last bar as UTC dates (fmt_utc). # Checked after the run, so the engine's own refusal of the same tape # keeps its line. @@ -4105,25 +4807,38 @@ def _run(st, rep): out["fingerprint"] = build_fingerprint(provenance) except Exception: out["fingerprint"] = None + _advance(phases, "serialization") # phase records only: the report gains no field # Serialized whole before any byte is written (the same json.dump), so # a failure while it is built never follows half a report. buffer = io.StringIO() json.dump(out, buffer, separators=(",", ":")) buffer.write("\n") + except Exception as error: + # Without --run-phase-fd there is no boundary and this re-raises the error unchanged. + classified = _post_boundary_failure(error, phases) + if classified is None: + raise + raise classified from error finally: + detach_error = _detach_observer(binding) # None without --run-phase-fd lib.report_free(ctypes.byref(report)) lib.strategy_free(state) - try: - sys.stdout.write(buffer.getvalue()) - sys.stdout.flush() - except (OSError, ValueError) as error: - # A closed pipe or a full disk while the report is written: a failure - # line now would follow part of a report, so none is written. The reason - # goes to stderr and stdout to /dev/null, so the interpreter's own flush - # at exit cannot fail again. - _discard_stdout() - print(f"run_json: the report could not be written: {error}", file=sys.stderr) - return 1 + if detach_error is not None and sys.exc_info()[1] is None: + raise _typed_failure(detach_error) or detach_error + # A closed pipe or a full disk while the report is written: a failure line now would + # follow part of a report, so none is written (see _write_report). + status = _write_report(buffer.getvalue(), phases) + if status: + return status + if phases is not None: + try: + phases.advance("completed") # only after the flush + except RunFailure as error: + # The whole report is already out: no failure line may follow it, and nothing may + # claim success; this is the harness fault, exit status 1 on stderr. + print(f"run_json: harness_internal_error: the completed phase record could not be " + f"handed over: {error}", file=sys.stderr) + return 1 return 0 diff --git a/docker/run_phase_transport.py b/docker/run_phase_transport.py new file mode 100644 index 000000000..43e59233b --- /dev/null +++ b/docker/run_phase_transport.py @@ -0,0 +1,396 @@ +"""Run-only phase transport writer. + +Phase transport v1 (selected-window wire v1.3) tells the process that started a +run which phase the run has reached, over a channel the strategy cannot write +to, so that an external deadline is classified from the last phase handed over +and never from anything the strategy printed. One RunPhaseWriter serves one run +on one connection: the inherited descriptor named by --run-phase-fd N (N >= 3), +an AF_UNIX SOCK_STREAM socket the caller owns. stdout stays report-only and +stderr diagnostics-only; this module prints nothing. + +The phases, in this order, each at most once, with no restart: + + preflight, execution, result_assembly, results_digest (optional), + serialization, completed + +advance(phase) enters a phase. The caller starts that phase's work only after +advance() returned; an advance() that raised means the phase was not entered +and its work must not start. A boundary is one record: UTF-8 NDJSON, one compact +object per LF, at most 1024 bytes with the LF, at most 6 records per run: + + {"record":"run_phase","version":1,"sequence":1,"scope":"run","trial_id":null,"phase":"preflight","elapsed_ms":0} + +sequence counts the connection's records from 1 without a gap. elapsed_ms is the +monotonic time since the first record, in milliseconds, so preflight reports 0. +completed adds execution_ms, result_assembly_ms, results_digest_ms (null when +the digest phase was skipped) and serialization_ms; serialization_ms is measured +at the completed call, after the caller flushed its output. Nothing here is part +of a report's provenance or results digest. + +Strict handover. A boundary is exactly one nonblocking os.write of the whole +record: no poll, no sleep, no spin, no retry, no wait. A write that does not take +the whole record (short, zero, EAGAIN, a disconnect or any other error) raises +PhaseTransportError, keeps the part the socket did not take in `unsent` (at most +one record, far inside the 1 MiB unsent bound) and ends the writer for good. A +record handed to the socket is not an acknowledgement: nothing here claims that +the parent received or validated it. The optimizer's trial scope and the bounded +wait of its phase-aware mode belong to another writer; this one +never waits for its reader. + +The descriptor is borrowed: it is never closed, duplicated or owned here and no +registry keeps it. RunPhaseWriter(fd) is the check that fails a bad transport in +preflight, so build the writer before any strategy code loads. fd=None still +tracks the phases and their timing, for phase_timing(), but exports nothing. A +writer is not thread safe: one thread drives it. +""" +from __future__ import annotations + +import fcntl +import json +import math +import os +import socket +import time +from typing import Callable, Dict, Mapping, Optional + +__all__ = [ + "MAX_RECORD_BYTES", + "MAX_RECORDS", + "MAX_UNSENT_BYTES", + "PHASES", + "PhaseTransportError", + "RunPhaseWriter", + "encode_record", +] + +PHASES = ("preflight", "execution", "result_assembly", "results_digest", + "serialization", "completed") +MAX_RECORD_BYTES = 1024 # one record, its LF included +MAX_RECORDS = 6 # per scope; a run is one scope +MAX_UNSENT_BYTES = 1024 * 1024 # unsent data a writer may hold; this one holds at most a record + +# What may follow what: results_digest is the only phase a run may skip, a phase +# is entered once, and nothing follows completed. +_SUCCESSORS = { + None: ("preflight",), + "preflight": ("execution",), + "execution": ("result_assembly",), + "result_assembly": ("results_digest", "serialization"), + "results_digest": ("serialization",), + "serialization": ("completed",), + "completed": (), +} + +_ACCESS_MODE_BITS = os.O_WRONLY | os.O_RDWR # the access-mode field of F_GETFL flags +_MAX_DESCRIPTOR = 2 ** 31 - 1 # a C int: nothing above it names a descriptor +_EXACT_INTEGER_LIMIT = 2 ** 53 # whole floats below this go on the wire as integers + + +class PhaseTransportError(Exception): + """The phase transport failed, or was driven outside its contract. For the + harness this is a harness fault, never a strategy outcome: `code` is + harness_internal_error (class engine_fault in run_failure_codes.json). + + `reason` names the cause: invalid_fd, invalid_family, invalid_type, + not_writable, nonblocking_failed (the descriptor); invalid_phase, + phase_order, writer_failed (the call); clock_invalid (the clock); + record_invalid, record_oversize (the record); would_block, short_write, + disconnected, write_failed (the handover); timing_unavailable; unexpected + (any other exception that ended an advance). `phase` is the boundary that + was refused, when there is one.""" + + code = "harness_internal_error" + + def __init__(self, text: str, *, reason: str, phase: Optional[str] = None) -> None: + super().__init__(text) + self.reason = reason + self.phase = phase + + +def encode_record(record: Mapping[str, object]) -> bytes: + """The framing of one record: compact JSON in UTF-8 and the LF that ends the + line, at most MAX_RECORD_BYTES bytes in all. A value JSON cannot hold, a + non-finite number or an oversized record raises PhaseTransportError.""" + try: + text = json.dumps(record, separators=(",", ":"), allow_nan=False) + except (TypeError, ValueError) as error: + raise PhaseTransportError(f"phase record cannot be encoded: {error}", + reason="record_invalid") from error + line = text.encode("utf-8") + b"\n" + if len(line) > MAX_RECORD_BYTES: + raise PhaseTransportError( + f"phase record is {len(line)} bytes, over the {MAX_RECORD_BYTES} byte limit", + reason="record_oversize") + return line + + +def _wire_number(value: Optional[float]) -> Optional[float]: + """A millisecond value as it goes on the wire: a whole number is written as an + integer (elapsed_ms 0, not 0.0, as in the pinned example), any other as the + float. None stays None (results_digest_ms of a skipped digest).""" + if value is not None and value.is_integer() and abs(value) < _EXACT_INTEGER_LIMIT: + return int(value) + return value + + +def _milliseconds(start: float, end: float) -> float: + return (end - start) * 1000.0 + + +def _is_writable_mode(flags: int) -> bool: + """Whether F_GETFL flags carry write access (O_WRONLY or O_RDWR).""" + return (flags & _ACCESS_MODE_BITS) in (os.O_WRONLY, os.O_RDWR) + + +def _borrow_descriptor(fd: object) -> int: + """Check the descriptor named by --run-phase-fd and make it nonblocking: an int + that is not a bool, from 3 up, an open AF_UNIX SOCK_STREAM socket with write + access. The socket is looked at through a temporary wrapper that is detached + again, never closed, and no descriptor is duplicated: on every path, rejected + ones included, the descriptor stays open and the caller's. O_NONBLOCK is set + here, before any write, and left set.""" + if isinstance(fd, bool) or not isinstance(fd, int): + raise PhaseTransportError( + f"run phase descriptor must be an int, not {type(fd).__name__}", reason="invalid_fd") + if fd < 3 or fd > _MAX_DESCRIPTOR: + raise PhaseTransportError( + f"run phase descriptor must be between 3 and {_MAX_DESCRIPTOR}, got {fd}", + reason="invalid_fd") + try: + probe = socket.socket(fileno=fd) + except (OSError, OverflowError, ValueError) as error: + raise PhaseTransportError( + f"run phase descriptor {fd} is not an open socket: {error}", + reason="invalid_fd") from error + try: + family, kind = probe.family, probe.type + finally: + probe.detach() # borrowed: let go of the descriptor without closing it + if family != socket.AF_UNIX: + raise PhaseTransportError( + f"run phase descriptor {fd} is not an AF_UNIX socket", reason="invalid_family") + if kind != socket.SOCK_STREAM: + raise PhaseTransportError( + f"run phase descriptor {fd} is not a SOCK_STREAM socket", reason="invalid_type") + try: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + except (OSError, OverflowError) as error: + raise PhaseTransportError( + f"run phase descriptor {fd} flags cannot be read: {error}", + reason="invalid_fd") from error + if not _is_writable_mode(flags): + raise PhaseTransportError( + f"run phase descriptor {fd} is not writable", reason="not_writable") + try: + os.set_blocking(fd, False) + except OSError as error: + raise PhaseTransportError( + f"run phase descriptor {fd} cannot be made nonblocking: {error}", + reason="nonblocking_failed") from error + return fd + + +class RunPhaseWriter: + """Reports one run's phases over a borrowed descriptor, strictly (see the + module text). + + writer = RunPhaseWriter(args.run_phase_fd) # None without the flag + writer.advance("preflight") + ... + writer.advance("serialization") + timing = writer.phase_timing() # for the report, then serialize + writer.advance("completed") # after the output is flushed + + `clock` returns monotonic seconds. It is read once per advance() and never + by the constructor, and a reading that is not a finite number (a bool, a + string, NaN, infinity) or that is earlier than the last one fails the writer. + """ + + def __init__(self, fd: Optional[int], *, + clock: Callable[[], float] = time.monotonic) -> None: + self._fd = None if fd is None else _borrow_descriptor(fd) + self._clock = clock + self._phase: Optional[str] = None + self._sequence = 0 + self._origin: Optional[float] = None # clock reading of the first record + self._last_reading: Optional[float] = None + self._entered: Dict[str, float] = {} # phase -> clock reading at entry + self._failure: Optional[str] = None + self._unsent = b"" + self._last_handed_frame_bytes = 0 + + @property + def phase(self) -> Optional[str]: + """The last phase entered; None before preflight.""" + return self._phase + + @property + def sequence(self) -> int: + """The sequence of the last record handed over: how many phases were entered.""" + return self._sequence + + @property + def failure(self) -> Optional[str]: + """The reason that ended the writer; None while it is usable.""" + return self._failure + + @property + def unsent(self) -> bytes: + """The part of the failed record the socket did not take; empty when no + handover failed. At most one record.""" + return self._unsent + + @property + def export_requested(self) -> bool: + """Whether this writer exports its records: it holds a descriptor (fd is not + None). The engine's execution observer reports it in its receipt.""" + return self._fd is not None + + @property + def last_handed_frame_bytes(self) -> int: + """The size, LF included, of the one record the last successful advance() handed + to the socket: 0 before any advance and after an advance of a writer without a + descriptor. A refused advance leaves it as it was, since nothing was handed over; + it is read right after the advance() that returned, never after a failure.""" + return self._last_handed_frame_bytes + + def advance(self, phase: str) -> None: + """Enter `phase`. Returns once its whole record is in the socket (one + nonblocking os.write; that is not the parent's receipt). Anything else + raises PhaseTransportError: the phase was NOT entered, the work after it + must not run, and the writer is finished: every later advance() raises + writer_failed without touching the descriptor.""" + if self._failure is not None: + raise PhaseTransportError( + f"phase writer ended ({self._failure}); a failed writer is never reused", + reason="writer_failed", phase=phase if type(phase) is str else None) + try: + self._enter(phase) + except PhaseTransportError as error: + self._failure = error.reason + raise + except BaseException: + self._failure = "unexpected" + raise + + def phase_timing(self) -> Dict[str, object]: + """The report's diagnostics.phase_timing: version 1, execution_ms, + result_assembly_ms and results_digest_ms (None when the digest phase was + skipped), in milliseconds. Known once serialization began, so only after + advance("serialization") returned; the serialization duration is not part + of it, it is in the completed record.""" + if "serialization" not in self._entered: + raise PhaseTransportError( + "phase timing is known only once serialization began", + reason="timing_unavailable") + timing: Dict[str, object] = {"version": 1} + timing.update(self._report_durations()) + return timing + + def _enter(self, phase: str) -> None: + if type(phase) is not str or phase not in PHASES: + shown = repr(phase) if type(phase) is str else f"<{type(phase).__name__}>" + raise PhaseTransportError(f"unknown run phase {shown}", reason="invalid_phase") + if phase not in _SUCCESSORS[self._phase]: + raise PhaseTransportError( + f"run phase {phase} cannot follow {self._phase or 'the start'}", + reason="phase_order", phase=phase) + reading = self._read_clock() + origin = reading if self._origin is None else self._origin + elapsed_ms = _milliseconds(origin, reading) + if not math.isfinite(elapsed_ms): + raise PhaseTransportError( + "phase clock span is not finite", reason="clock_invalid", phase=phase) + handed = 0 + if self._fd is not None: + frame = encode_record(self._record(phase, elapsed_ms, reading)) + self._hand_off(frame, phase) + handed = len(frame) + # The whole record is in the socket (or nothing is exported): now the phase is entered. + self._phase = phase + self._sequence += 1 + self._origin = origin + self._last_reading = reading + self._entered[phase] = reading + self._last_handed_frame_bytes = handed + + def _read_clock(self) -> float: + """One reading, validated: a finite number that is not a bool and not + earlier than the last one.""" + try: + reading = self._clock() + except Exception as error: + raise PhaseTransportError( + f"phase clock failed: {type(error).__name__}: {error}", + reason="clock_invalid") from error + if isinstance(reading, bool) or not isinstance(reading, (int, float)): + raise PhaseTransportError( + f"phase clock returned {type(reading).__name__}, not a number", + reason="clock_invalid") + try: + seconds = float(reading) + except OverflowError as error: + raise PhaseTransportError( + "phase clock reading is too large", reason="clock_invalid") from error + if not math.isfinite(seconds): + raise PhaseTransportError( + "phase clock reading is not finite", reason="clock_invalid") + if self._last_reading is not None and seconds < self._last_reading: + raise PhaseTransportError("phase clock went backward", reason="clock_invalid") + return seconds + + def _report_durations(self) -> Dict[str, Optional[float]]: + """execution_ms, result_assembly_ms and results_digest_ms, known once + serialization began. Each runs from its phase's entry to the next phase's + entry; result_assembly ends at serialization when the digest was skipped.""" + entered = self._entered + digest_at = entered.get("results_digest") + assembly_ends = entered["serialization"] if digest_at is None else digest_at + return { + "execution_ms": _milliseconds(entered["execution"], entered["result_assembly"]), + "result_assembly_ms": _milliseconds(entered["result_assembly"], assembly_ends), + "results_digest_ms": (None if digest_at is None + else _milliseconds(digest_at, entered["serialization"])), + } + + def _record(self, phase: str, elapsed_ms: float, reading: float) -> Dict[str, object]: + record: Dict[str, object] = { + "record": "run_phase", + "version": 1, + "sequence": self._sequence + 1, + "scope": "run", + "trial_id": None, + "phase": phase, + "elapsed_ms": _wire_number(elapsed_ms), + } + if phase == "completed": + durations = self._report_durations() + durations["serialization_ms"] = _milliseconds(self._entered["serialization"], reading) + record.update({name: _wire_number(value) for name, value in durations.items()}) + return record + + def _hand_off(self, line: bytes, phase: str) -> None: + """The boundary's one os.write. Only a write that took the whole record + returns; there is no second attempt for the rest.""" + try: + written = os.write(self._fd, line) + except BlockingIOError as error: + self._unsent = line + raise PhaseTransportError( + f"socket full: the record for phase {phase} was not handed over", + reason="would_block", phase=phase) from error + except (BrokenPipeError, ConnectionResetError, ConnectionAbortedError) as error: + self._unsent = line + raise PhaseTransportError( + f"parent disconnected: the record for phase {phase} was not handed over: {error}", + reason="disconnected", phase=phase) from error + except OSError as error: + self._unsent = line + raise PhaseTransportError( + f"write failed: the record for phase {phase} was not handed over: {error}", + reason="write_failed", phase=phase) from error + if written < len(line): + self._unsent = line[written:] + raise PhaseTransportError( + f"short write: {written} of {len(line)} bytes of the record for phase {phase}", + reason="short_write", phase=phase) diff --git a/docker/selected_window_plan.py b/docker/selected_window_plan.py new file mode 100644 index 000000000..62023b17c --- /dev/null +++ b/docker/selected_window_plan.py @@ -0,0 +1,430 @@ +"""The selected primary planner's standalone C bridge, seen from Python (frozen v1). + +The bridge is the shared library built from the pinned tree (target +pineforge_window_plan, installed as lib/libpineforge_window_plan.so), which +links only the kernel and never strategy code. This module mirrors its two +structs and the engine's bar (RequestC, ResultC, BarC), loads the one library +the caller names by absolute path, calls pf_plan_selected_primary_v1 once per +plan() and returns the planner's own values as a dictionary of the C++ +SelectedPrimaryPlan's field names. It carries no calendar, count, hash or phase +logic of its own, searches no path and never falls back to a strategy library. + +A refusal of the planner is not an error here: it is a nonzero "status" (the +SelectedPlanStatus value) in the returned dictionary, for the caller to map to +its own catalog. PlanBridgeError is only the bridge failing: the capability +missing (window_mode_unsupported), the planner out of memory (out_of_memory), or +a call, return or result that breaks the frozen contract (harness_internal_error). +It does not import run_json; the caller that owns RunFailure maps its code, +message and code_args. +""" + +import ctypes +import os + +__all__ = [ + "BarC", "RequestC", "ResultC", "PlanBridgeError", "SelectedPrimaryPlanner", + "PLAN_ABI_VERSION", "CAPABILITY", "PLAN_FIELDS", "STATUS_NAMES", + "CODE_WINDOW_MODE_UNSUPPORTED", "CODE_OUT_OF_MEMORY", "CODE_HARNESS_INTERNAL_ERROR", +] + +PLAN_ABI_VERSION = 1 +CAPABILITY = "selected_window_planner_v1" + +CODE_WINDOW_MODE_UNSUPPORTED = "window_mode_unsupported" +CODE_OUT_OF_MEMORY = "out_of_memory" +CODE_HARNESS_INTERNAL_ERROR = "harness_internal_error" + +# pf_selected_plan_result_v1.status, by the C++ SelectedPlanStatus value. plan() +# returns the integer; the names are informational and their count bounds the +# status range check. +STATUS_NAMES = ("Ok", "RequestInvalid", "CalendarUnsupported", "BoundaryUnaligned", + "FeedRangeInvalid", "RowsUnordered", "InternalError") + +# No SelectedPlanStatus is negative. plan() presets the result's status to this +# value, after size and version and before the call, so a return 0 that never +# wrote the result reads as a status outside STATUS_NAMES and is refused, not +# passed as an all-zero Ok. +_UNWRITTEN_STATUS = -1 + +_BOUNDS = (-1, 0, 1, 2) # -1 no boundary detail, 0 = T, 1 = E, 2 = F +_COUNT_FIELDS = ( + "supplied_input_bars", "supplied_script_bars", "available_script_bars", + "used_script_bars", "trimmed_script_bars", "trim_index", "fed_input_bars", + "fed_script_bars", "preroll_input_bars", "window_input_bars", "window_script_bars") +# present_mask bit i says the i-th of these eight timestamps is present; an +# absent one is zero in the struct and None in the dictionary. +_OPTIONAL_FIELDS = ( + "preroll_first_bar_ms", "preroll_last_bar_ms", "supplied_first_data_ms", + "supplied_last_data_ms", "fed_first_data_ms", "fed_last_data_ms", + "window_first_data_ms", "window_last_data_ms") +_FLAG_FIELDS = ("shortfall", "complete_pending_preroll_at_horizon") +_STRING_FIELDS = ("input_tf", "script_tf", "chart_timezone", "engine_timezone", "session") +_EXPORTS = ("pf_selected_plan_version", "pf_plan_selected_primary_v1") + +# The keys of the dictionary plan() returns, in the C++ plan's field order. +PLAN_FIELDS = (("status", "option", "bound", "value_ms", "previous_boundary_ms", + "next_boundary_ms") + _COUNT_FIELDS + ("trim_start_ms",) + + _OPTIONAL_FIELDS + _FLAG_FIELDS) + +_INT64_MIN, _INT64_MAX = -(1 << 63), (1 << 63) - 1 +_UINT32_MAX = (1 << 32) - 1 +# The count crosses as uint64_t and must also fit this platform's size_t. +_COUNT_MAX = min((1 << 64) - 1, (1 << (8 * ctypes.sizeof(ctypes.c_size_t))) - 1) + + +# --- ctypes mirror of and pf_bar_t ------- + +class BarC(ctypes.Structure): + """pf_bar_t: five doubles then the int64 timestamp (48 bytes, alignment 8), + the layout of run_json's BarC.""" + _fields_ = [ + ("open", ctypes.c_double), + ("high", ctypes.c_double), + ("low", ctypes.c_double), + ("close", ctypes.c_double), + ("volume", ctypes.c_double), + ("timestamp", ctypes.c_int64), + ] + + +class RequestC(ctypes.Structure): + """pf_selected_plan_request_v1, natural alignment, no packing (LP64: 80 bytes). + The caller presets struct_size to sizeof and version to 1.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("start_ms", ctypes.c_int64), + ("end_ms", ctypes.c_int64), + ("fed_start_ms", ctypes.c_int64), + ("preroll_bars", ctypes.c_uint32), + ("feed_tolerant", ctypes.c_uint32), + ("input_tf", ctypes.c_char_p), + ("script_tf", ctypes.c_char_p), + ("chart_timezone", ctypes.c_char_p), + ("engine_timezone", ctypes.c_char_p), + ("session", ctypes.c_char_p), + ] + + +class ResultC(ctypes.Structure): + """pf_selected_plan_result_v1, natural alignment, no packing (LP64: 248 + bytes). The caller presets struct_size to sizeof and version to 1 (the frozen + contract), then status to -1, which is no planner status; the bridge writes + the whole struct on return 0 and, on a negative return, none of it.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("status", ctypes.c_int32), + ("bound", ctypes.c_int32), + ("value_ms", ctypes.c_int64), + ("previous_boundary_ms", ctypes.c_int64), + ("next_boundary_ms", ctypes.c_int64), + ("supplied_input_bars", ctypes.c_uint64), + ("supplied_script_bars", ctypes.c_uint64), + ("available_script_bars", ctypes.c_uint64), + ("used_script_bars", ctypes.c_uint64), + ("trimmed_script_bars", ctypes.c_uint64), + ("trim_index", ctypes.c_uint64), + ("fed_input_bars", ctypes.c_uint64), + ("fed_script_bars", ctypes.c_uint64), + ("preroll_input_bars", ctypes.c_uint64), + ("window_input_bars", ctypes.c_uint64), + ("window_script_bars", ctypes.c_uint64), + ("trim_start_ms", ctypes.c_int64), + ("preroll_first_bar_ms", ctypes.c_int64), + ("preroll_last_bar_ms", ctypes.c_int64), + ("supplied_first_data_ms", ctypes.c_int64), + ("supplied_last_data_ms", ctypes.c_int64), + ("fed_first_data_ms", ctypes.c_int64), + ("fed_last_data_ms", ctypes.c_int64), + ("window_first_data_ms", ctypes.c_int64), + ("window_last_data_ms", ctypes.c_int64), + ("present_mask", ctypes.c_uint32), + ("shortfall", ctypes.c_uint32), + ("complete_pending_preroll_at_horizon", ctypes.c_uint32), + ("reserved", ctypes.c_uint32), + ("option", ctypes.c_char * 32), + ] + + +_OPTION_START = ResultC.option.offset +_OPTION_END = _OPTION_START + ResultC.option.size + +# (struct, sizeof, alignment, every field's offset): the frozen LP64 layout. +_LAYOUTS = ( + (BarC, 48, 8, (0, 8, 16, 24, 32, 40)), + (RequestC, 80, 8, (0, 4, 8, 16, 24, 32, 36, 40, 48, 56, 64, 72)), + (ResultC, 248, 8, (0, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, + 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, + 204, 208, 212, 216)), +) + + +class PlanBridgeError(Exception): + """The bridge failed; the planner did not refuse (a refusal is a status in + the returned dictionary). code is window_mode_unsupported (code_args + {"capability": "selected_window_planner_v1"}), out_of_memory or + harness_internal_error (both with no code_args); message is the text.""" + + def __init__(self, code, message, code_args=None): + super().__init__(message) + self.code = code + self.message = message + self.code_args = dict(code_args or {}) + + +def _internal(message): + return PlanBridgeError(CODE_HARNESS_INTERNAL_ERROR, message) + + +def _unsupported(message): + return PlanBridgeError(CODE_WINDOW_MODE_UNSUPPORTED, message, {"capability": CAPABILITY}) + + +def _check_layout(): + """The ctypes structs have exactly the frozen size, alignment and offsets.""" + for struct, size, alignment, offsets in _LAYOUTS: + found = (ctypes.sizeof(struct), ctypes.alignment(struct), + tuple(getattr(struct, field[0]).offset for field in struct._fields_)) + if found != (size, alignment, offsets): + raise _internal( + f"the ctypes layout of {struct.__name__} is (size, alignment, offsets) " + f"{found}, the frozen layout is {(size, alignment, offsets)}") + + +def _raw(struct): + return ctypes.string_at(ctypes.addressof(struct), ctypes.sizeof(struct)) + + +def _library_path(library_path): + """The explicit absolute path, as text: no search, no default, no fallback.""" + try: + path = os.fspath(library_path) + except TypeError: + raise _internal("library_path must be the absolute path of the planner library, " + f"got {type(library_path).__name__}") from None + if not isinstance(path, str) or "\0" in path or not os.path.isabs(path): + raise _internal("library_path must be the absolute path of the planner library, " + f"got {path!r}") + return path + + +def _integer(name, value, low, high): + if isinstance(value, bool) or not isinstance(value, int): + raise _internal(f"{name} must be an int, got {type(value).__name__}") + if not low <= value <= high: + raise _internal(f"{name} {value} is outside {low}..{high}") + return value + + +def _utf8(name, value): + if not isinstance(value, str): + raise _internal(f"{name} must be a str, got {type(value).__name__}") + if "\0" in value: + raise _internal(f"{name} holds an embedded NUL") + try: + return value.encode("utf-8") + except UnicodeEncodeError: + raise _internal(f"{name} is not encodable as UTF-8") from None + + +def _bar_layout(element): + """element is BarC or a Structure of exactly its fields, types and offsets.""" + if element is BarC: + return True + if not (isinstance(element, type) and issubclass(element, ctypes.Structure)): + return False + fields = getattr(element, "_fields_", ()) + if len(fields) != len(BarC._fields_): + return False + for field, expected in zip(fields, BarC._fields_): + if (len(field) != 2 or field[0] != expected[0] or field[1] is not expected[1] + or getattr(element, field[0]).offset != getattr(BarC, expected[0]).offset): + return False + return (ctypes.sizeof(element) == ctypes.sizeof(BarC) + and ctypes.alignment(element) == ctypes.alignment(BarC)) + + +def _rows_argument(bars, count): + """(the POINTER(BarC) the call takes, the validated count). bars is a ctypes + array or pointer of BarC-layout rows (run_json's BarC array qualifies), used + where it lies: nothing is copied. A null bars is accepted only with count 0.""" + count = _integer("count", count, 0, _COUNT_MAX) + if bars is None: + if count: + raise _internal(f"bars is None but count is {count}: null rows need count 0") + return None, 0 + if isinstance(bars, ctypes.Array): + if count > len(bars): + raise _internal(f"count {count} is more than the {len(bars)} rows of bars") + elif isinstance(bars, ctypes._Pointer): + if count and not bars: + raise _internal(f"bars is a null pointer but count is {count}: null rows need count 0") + else: + raise _internal("bars must be a ctypes array or pointer of BarC rows, " + f"got {type(bars).__name__}") + if not _bar_layout(bars._type_): + raise _internal("the rows of bars are not the BarC layout (open, high, low, close, " + "volume as c_double, then timestamp as c_int64)") + return ctypes.cast(bars, ctypes.POINTER(BarC)), count + + +def _request_struct(start_ms, end_ms, fed_start_ms, preroll_bars, input_tf, script_tf, + chart_timezone, engine_timezone, session, feed_tolerant): + """(the RequestC, the encoded strings it points into). The strings must stay + referenced until the call has returned.""" + values = { + "start_ms": _integer("start_ms", start_ms, _INT64_MIN, _INT64_MAX), + "end_ms": _integer("end_ms", end_ms, _INT64_MIN, _INT64_MAX), + "fed_start_ms": _integer("fed_start_ms", fed_start_ms, _INT64_MIN, _INT64_MAX), + "preroll_bars": _integer("preroll_bars", preroll_bars, 0, _UINT32_MAX), + } + if not isinstance(feed_tolerant, bool): + raise _internal(f"feed_tolerant must be a bool, got {type(feed_tolerant).__name__}") + texts = (input_tf, script_tf, chart_timezone, engine_timezone, session) + strings = tuple(_utf8(name, text) for name, text in zip(_STRING_FIELDS, texts)) + request = RequestC(struct_size=ctypes.sizeof(RequestC), version=PLAN_ABI_VERSION, + feed_tolerant=1 if feed_tolerant else 0, + **values, **dict(zip(_STRING_FIELDS, strings))) + return request, strings + + +def _option_text(raw): + """The result's option: ASCII, NUL-terminated and zero-padded in its 32 bytes.""" + end = raw.find(b"\0") + if end < 0: + raise _internal("the result option is not NUL-terminated within its 32 bytes") + if any(raw[end:]): + raise _internal("the result option is not zero-padded after its NUL") + if not raw[:end].isascii(): + raise _internal("the result option is not ASCII") + return raw[:end].decode("ascii") + + +def _plan_from(result): + """The result, validated, as the C++ plan's named fields. The planner's own + values pass through unchanged: no count, calendar or hash is recomputed.""" + if result.struct_size != ctypes.sizeof(ResultC) or result.version != PLAN_ABI_VERSION: + raise _internal(f"the result header is (size {result.struct_size}, version " + f"{result.version}), expected ({ctypes.sizeof(ResultC)}, " + f"{PLAN_ABI_VERSION})") + if not 0 <= result.status < len(STATUS_NAMES): + raise _internal(f"the result status {result.status} is not a planner status") + if result.bound not in _BOUNDS: + raise _internal(f"the result bound {result.bound} is not one of {_BOUNDS}") + mask = result.present_mask + if mask >> len(_OPTIONAL_FIELDS): + raise _internal(f"the result present_mask {mask:#x} sets bits past bit " + f"{len(_OPTIONAL_FIELDS) - 1}") + for bit, name in enumerate(_OPTIONAL_FIELDS): + if not (mask >> bit) & 1 and getattr(result, name) != 0: + raise _internal(f"the result {name} is {getattr(result, name)} but absent in " + f"present_mask {mask:#x}") + for name in _FLAG_FIELDS: + if getattr(result, name) not in (0, 1): + raise _internal(f"the result {name} is {getattr(result, name)}, not 0 or 1") + if result.reserved != 0: + raise _internal(f"the result reserved word is {result.reserved}, not 0") + option = _option_text(_raw(result)[_OPTION_START:_OPTION_END]) + plan = {"status": result.status, "option": option, "bound": result.bound} + for name in (("value_ms", "previous_boundary_ms", "next_boundary_ms") + _COUNT_FIELDS + + ("trim_start_ms",)): + plan[name] = getattr(result, name) + for bit, name in enumerate(_OPTIONAL_FIELDS): + plan[name] = getattr(result, name) if (mask >> bit) & 1 else None + for name in _FLAG_FIELDS: + plan[name] = bool(getattr(result, name)) + return plan + + +def _bridge_failure(code, output_changed): + """The PlanBridgeError for a nonzero return of pf_plan_selected_primary_v1.""" + if code < 0 and output_changed: + return _internal(f"pf_plan_selected_primary_v1 returned {code} and still changed " + "the caller's result") + if code == -2: + return PlanBridgeError(CODE_OUT_OF_MEMORY, + "the selected primary planner failed: std::bad_alloc") + if code == -1: + return _internal("pf_plan_selected_primary_v1 refused the call (return -1: a null " + "descriptor, a wrong size or version, a null string, a bad " + "feed_tolerant or an unrepresentable count)") + if code == -3: + return _internal("pf_plan_selected_primary_v1 failed on an exception or invariant " + "(return -3)") + return _internal(f"pf_plan_selected_primary_v1 returned {code}, which is not 0, -1, -2 " + "or -3") + + +class SelectedPrimaryPlanner: + """The trusted standalone planner library, loaded once from the explicit + absolute path of the helper built with the pinned tree (never a strategy + library). The constructor probes the version and binds both functions; the + object keeps the library loaded for every later call.""" + + def __init__(self, library_path): + _check_layout() + path = _library_path(library_path) + try: + library = ctypes.CDLL(path) + except OSError as error: + raise _unsupported(f"cannot load the selected primary planner library " + f"{path}: {error}") from None + missing = [name for name in _EXPORTS if not hasattr(library, name)] + if missing: + raise _unsupported(f"the selected primary planner library {path} has no " + + ", ".join(missing)) + version_function = library.pf_selected_plan_version + version_function.argtypes = [] + version_function.restype = ctypes.c_uint32 + plan_function = library.pf_plan_selected_primary_v1 + plan_function.argtypes = [ctypes.POINTER(RequestC), ctypes.POINTER(BarC), + ctypes.c_uint64, ctypes.POINTER(ResultC)] + plan_function.restype = ctypes.c_int + version = version_function() + if version != PLAN_ABI_VERSION: + raise _unsupported(f"the selected primary planner library {path} reports " + f"version {version}, expected {PLAN_ABI_VERSION}") + self._path = path + self._library = library + self._plan = plan_function + + @property + def library_path(self): + return self._path + + def plan(self, bars, count, *, start_ms, end_ms, fed_start_ms, preroll_bars, + input_tf, script_tf, chart_timezone="", engine_timezone="", session="", + feed_tolerant=True): + """The planner's plan for the first count rows of bars and one selected + window request, as a dict of the C++ SelectedPrimaryPlan's field names: + status (the SelectedPlanStatus integer, 0 = Ok, a refusal otherwise), + option as str, bound, value_ms, previous_boundary_ms, next_boundary_ms, + the counts, trim_start_ms, the eight optional timestamps as int or None, + shortfall and complete_pending_preroll_at_horizon as bool. bars is a + ctypes array or pointer of BarC-layout rows (None with count 0), read in + place. Raises PlanBridgeError when the bridge, not the planner, fails (a + return 0 that wrote no result is such a failure).""" + rows, row_count = _rows_argument(bars, count) + request, strings = _request_struct( + start_ms, end_ms, fed_start_ms, preroll_bars, input_tf, script_tf, + chart_timezone, engine_timezone, session, feed_tolerant) + result = ResultC(struct_size=ctypes.sizeof(ResultC), version=PLAN_ABI_VERSION) + result.status = _UNWRITTEN_STATUS + before = _raw(result) # the baseline a negative return must leave unchanged + # Everything the native call reads or writes stays referenced until it has + # returned: the library, the caller's row object, the encoded strings and + # both structs. + held = (self._library, bars, rows, strings, request, result) + try: + code = self._plan(ctypes.pointer(request), rows, row_count, ctypes.pointer(result)) + except ctypes.ArgumentError as error: + raise _internal("pf_plan_selected_primary_v1 refused its arguments: " + f"{error}") from None + finally: + del held + if isinstance(code, bool) or not isinstance(code, int): + raise _internal(f"pf_plan_selected_primary_v1 returned {code!r}, not an int") + if code != 0: + raise _bridge_failure(code, _raw(result) != before) + return _plan_from(result) diff --git a/docker/selected_window_report.py b/docker/selected_window_report.py new file mode 100644 index 000000000..60b9d1919 --- /dev/null +++ b/docker/selected_window_report.py @@ -0,0 +1,855 @@ +"""Selected-window/v1 helpers of docker/run_json.py (single run, full/v1 report, no digest). + +run_json owns the orchestration (what happens in which order); this module holds the pieces +that are pure data or a thin native wrapper, so each can be controlled on its own: + + * request validation (the documented flags, explicit timeframes, the legacy-flag conflict) and the + typed refusal of the options this build does not offer (compact shape, curve budget, digest); + * the candidate support constants handed to request_feed_inventory.admit_selected_request; + * the mapping of the native primary planner's refusal statuses to the window catalog codes; + * the in-place trim view and the hash of the retained rows in the existing primary hash domain; + * the exact report_window record R (its lookback member is the one triple of a build with no + lookback producer), the full/v1 report_shape and the capability record (its + contract revision is this prerelease build's parity declaration, CONTRACT_REVISIONS); + * the frozen selected_window ABI as a ctypes wrapper, and + the exact comparison of the engine's native counts with the planned counts. + +It never imports run_json (a script run as __main__ would be loaded twice), never loads a strategy +library and carries no calendar, count or hash-domain logic of its own: calendar and count planning +belong to the native planner, the hash domain to run_json's frozen helpers (passed in). + +Errors are SelectedWindowError: `code` is a catalog code, `code_args` its typed arguments and +`message` the documented fixed English text or an invariant description. The +caller (run_json) maps them to RunFailure. +""" +from __future__ import annotations + +import base64 +import binascii +import ctypes +import hashlib +import json +import math +import re +import sys +from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Sequence, Tuple + +import request_feed_inventory as _inventory + +__all__ = [ + "CAPABILITY", "CONTRACT_REVISIONS", "FINGERPRINT_VERSION", "I53_MAX", "MAX_PREROLL_BARS", + "METERING_BASIS", "POLICY", "REPORT_SHAPE", "SelectedWindowAbi", "SelectedWindowConfigC", + "SelectedWindowCountsC", "SelectedWindowError", "WIRE_VERSION", "WindowRequest", + "build_report_window", "calendar_record", "calendar_session", "calendar_timezone", + "canonical_timeframe", "capabilities_record", "check_native_counts", + "check_report_placements", "check_selected_curve", "check_unsupported_options", + "refusal_for_plan", "report_shape_full", + "retained_rows", "support_policy", "timeframe_seconds", "update_rows_hash", + "validate_request", +] + +POLICY = "selected-window/v1" +WIRE_VERSION = 1 +FINGERPRINT_VERSION = 2 +# The contract revision this build declares in its capability record (the member +# selected_window_contract_revisions of --capabilities-json). This is the ONE place it is +# written; capabilities_record is the code that reads it. It is a PRERELEASE PARITY capability: +# the selected window without the value-ownership producers of the full contract, and NOT a +# claim of full 1.3 coverage. A parity build lists ("1.3-parity",) and a full build lists +# ("1.3",), never both: a future full build changes this one tuple and nothing else (no +# feature switch, environment knob or second constant exists). It is a declaration only: no +# request, report_window, phase record or fingerprint member carries it, and the wire version +# (1), the phase version (1) and the fingerprint/schema version (2) are unchanged. +CONTRACT_REVISIONS = ("1.3-parity",) +REPORT_SHAPE = "full/v1" +METERING_BASIS = "window-primary-input-bars/v1" +MAX_PREROLL_BARS = 5000 +I53_MAX = (1 << 53) - 1 +# A request number is at most this many decimal digits (I53_MAX has 16). Every int() of request +# text is preceded by a length test against it, so a number of thousands of digits is a typed +# request error and never Python's integer digit-limit ValueError. +_MAX_INTEGER_DIGITS = len(str(I53_MAX)) +CAPABILITY = "selected_window_v1" # the one argument of window_mode_unsupported for the native ABI + +# The candidate support constants this build hands to admit_selected_request: the eight +# intraday primary tokens, pre-roll supported, request feeds 1 and D. They are the internal +# implementation target, NOT a release declaration: the release policy comes from the release's +# acceptance alone, and run_json never reads or writes one. +_PRIMARY_TIMEFRAMES = ("1", "3", "5", "15", "30", "60", "120", "240") +_REQUEST_FEED_TIMEFRAMES = ("1", "D") + + +def support_policy() -> Dict[str, Any]: + """A fresh copy of the candidate support constants.""" + return {"primary_chart_timeframes": list(_PRIMARY_TIMEFRAMES), + "preroll": "supported", + "request_feed_timeframes": list(_REQUEST_FEED_TIMEFRAMES)} + + +# --- typed errors ----------------------------------------------------------- + +_FIXED_TEXT = { + "window_mode_unsupported": "This runner does not support the requested window mode.", + "window_request_invalid": "The selected-window request is invalid.", + "window_range_invalid": "The selected-window range is invalid.", + "window_preroll_out_of_range": "Pre-roll must be between zero and 5000 script bars.", + "window_legacy_flag_conflict": + "The legacy trade-start option cannot be combined with selected-window mode.", + "window_calendar_unsupported": "The requested window calendar is unsupported.", + "window_boundary_unaligned": "A window boundary is not aligned to the script calendar.", + "window_feed_range_invalid": "The primary feed lies outside the frozen range.", + "report_shape_unsupported": "This runner does not support the requested report shape.", + "report_shape_request_invalid": "The report-shape request is invalid.", + "results_digest_unsupported": "This runner does not support the requested results digest.", +} + + +class SelectedWindowError(Exception): + """A typed refusal or fault of the selected-window path. `code_args` (not + BaseException.args) holds the code's typed arguments.""" + + def __init__(self, code: str, message: str, code_args: Optional[Mapping[str, Any]] = None): + super().__init__(message) + self.code = code + self.message = message + self.code_args = dict(code_args or {}) + + +def _error(code: str, **code_args: Any) -> SelectedWindowError: + return SelectedWindowError(code, _FIXED_TEXT[code], code_args) + + +def _invalid(option: str) -> SelectedWindowError: + return _error("window_request_invalid", option=option) + + +def _internal(text: str) -> SelectedWindowError: + return SelectedWindowError("harness_internal_error", text) + + +def _invariant(text: str) -> SelectedWindowError: + return SelectedWindowError("engine_invariant", text) + + +# --- timeframes ------------------------------------------------------------- + +_TOKEN_UNITS = {"S": 1, "D": 86400, "W": 7 * 86400} +_TOKEN_RE = re.compile(r"([1-9][0-9]*)([DWMS]?)") + + +def canonical_timeframe(value: Any) -> Optional[str]: + """The wire's canonical spelling of value, or None; the common canonicalizer of the + inventory helper, so one spelling rule serves the policy, the inventory and this request.""" + return _inventory._canonical_timeframe(value) + + +def timeframe_seconds(token: Any) -> Optional[int]: + """The nominal length in seconds of a CANONICAL token: n minutes, nS, nD or nW; None for + months, for anything that is not a canonical token and for a length that is not a u53 + number of seconds. Only the report's input_tf_seconds unit conversion: no calendar. + Recognising a token stays unbounded (canonical_timeframe, unchanged); only this numeric + conversion is bounded, by the decimal length of the count before int() is called, so a token + of thousands of digits gives None here and never Python's integer digit-limit error.""" + if not isinstance(token, str): + return None + if token in ("D", "W"): + return _TOKEN_UNITS[token] + match = _TOKEN_RE.fullmatch(token) + if match is None or match.group(2) == "M" or len(match.group(1)) > _MAX_INTEGER_DIGITS: + return None + count, unit = int(match.group(1)), match.group(2) + seconds = count * (60 if unit == "" else _TOKEN_UNITS[unit]) + return seconds if seconds <= I53_MAX else None + + +# --- the request ------------------------------------------------------------ + +class WindowRequest(NamedTuple): + """A validated selected-window request: T, E, N, F and the two explicit timeframes.""" + start_ms: int + end_ms: int + preroll_bars: int + fed_start_ms: int + input_tf: str + script_tf: str # the canonical primary chart token + input_tf_seconds: int + + +_INTEGER_RE = re.compile(r"-?(?:0|[1-9][0-9]*)") + + +def _integer(text: Any, option: str) -> int: + """A decimal integer written as such: no sign other than '-', no leading zeros, no + fraction or exponent, and within the JSON safe-integer range (i53). The text is bounded by + its length (an optional '-' and at most 16 digits) BEFORE the pattern or int() sees it, so + an oversized number is window_request_invalid{option} at once and Python's integer + digit-limit ValueError is unreachable.""" + if (not isinstance(text, str) or len(text) > _MAX_INTEGER_DIGITS + 1 + or _INTEGER_RE.fullmatch(text) is None): + raise _invalid(option) + value = int(text) + if abs(value) > I53_MAX: + raise _invalid(option) + return value + + +def validate_request(*, policy: Any, window_start_ms: Any, window_end_ms: Any, + preroll_bars: Any, fed_start_ms: Any, trade_start_ms: Any, + input_tf: str, script_tf: str) -> WindowRequest: + """Validate the documented flags of a selected-window request, in this order: the policy, the + four window arguments (present, written as integers, i53), the two explicit timeframes + (present and timeframe tokens), the legacy trade-start conflict, the pre-roll range, the + ranges T < E and F <= T. Raises SelectedWindowError with the window catalog code; the + caller refuses before any feed, library or phase work.""" + if policy != POLICY: + raise _invalid("report_policy") + start = _integer(window_start_ms, "window_start_ms") + end = _integer(window_end_ms, "window_end_ms") + preroll = _integer(preroll_bars, "preroll_bars") + fed_start = _integer(fed_start_ms, "fed_start_ms") + if not input_tf: + raise _invalid("input_tf") + if not script_tf: + raise _invalid("script_tf") + primary = canonical_timeframe(script_tf) + if primary is None: + raise _invalid("script_tf") + chart = canonical_timeframe(input_tf) + seconds = timeframe_seconds(chart) + if chart is None or seconds is None: + raise _invalid("input_tf") + if trade_start_ms is not None: + raise _error("window_legacy_flag_conflict", option="trade_start_ms") + if not 0 <= preroll <= MAX_PREROLL_BARS: + raise _error("window_preroll_out_of_range", requested=preroll) + if start >= end or fed_start > start: + raise _error("window_range_invalid", start_ms=start, end_ms=end, fed_start_ms=fed_start) + return WindowRequest(start, end, preroll, fed_start, input_tf, primary, seconds) + + +def check_unsupported_options(*, selected: bool, report_shape: Any, curve_point_budget: Any, + results_digest: Any) -> None: + """Refuse, typed and before any work, what this build does not offer: the compact shape, + an explicit curve point budget (only compact takes one) and any results digest. A selected + request may name full/v1. A legacy request names no shape in this build: with + legacy accounting every shape request is refused (not settled by the documented contract; the + refusal keeps the ordinary report byte for byte).""" + if report_shape is not None: + if selected and report_shape == REPORT_SHAPE: + pass + elif selected and report_shape != "compact/v1": + raise _error("report_shape_request_invalid", option="report_shape", + value=str(report_shape)) + else: + raise _error("report_shape_unsupported", shape=str(report_shape)) + if curve_point_budget is not None: + raise _error("report_shape_request_invalid", option="curve_point_budget", + value=str(curve_point_budget)) + if results_digest is not None: + raise _error("results_digest_unsupported", requested=str(results_digest)) + + +# --- calendar data ---------------------------------------------------------- + +def calendar_timezone(chart_tz: str) -> str: + """R.calendar.timezone: the alignment timezone, --chart-tz, UTC when absent or empty.""" + return chart_tz if chart_tz else "UTC" + + +def calendar_session(session: str) -> Optional[str]: + """R.calendar.session: the resolved session, null for 24x7 (absent, empty or 24x7).""" + return None if session in ("", "24x7") else session + + +def calendar_record(chart_tz: str, session: str) -> Dict[str, Any]: + return {"timezone": calendar_timezone(chart_tz), "session": calendar_session(session)} + + +# --- the native planner's refusals ------------------------------------------ + +_BOUND_NAMES = ("start", "end", "fed_start") + + +def refusal_for_plan(plan: Mapping[str, Any], request: WindowRequest, *, timezone: str, + session: Optional[str]) -> Optional[SelectedWindowError]: + """None for planner status 0 (Ok), else the typed error for the planner's refusal: + + 1 RequestInvalid script_tf / input_tf -> window_request_invalid{option} + start_ms / end_ms / fed_start_ms -> window_range_invalid + preroll_bars -> window_preroll_out_of_range{requested} + any other option (rows, ...) -> harness_internal_error + 2 CalendarUnsupported -> window_calendar_unsupported{timezone, session} + 3 BoundaryUnaligned -> window_boundary_unaligned{bound, value_ms, previous, next} + 4 FeedRangeInvalid -> window_feed_range_invalid{time_ms, fed_start_ms, end_ms} + 5 RowsUnordered -> chart_bars_rejected{field: timestamp, reason: not_increasing} + 6 InternalError -> engine_invariant + anything else -> harness_internal_error + + The option strings are the planner's own. The mapping is this module's reading of them: + the documented contract names the codes, not the planner's options.""" + status = plan["status"] + if status == 0: + return None + option = plan.get("option") + if status == 1: + if option in ("script_tf", "input_tf"): + return _invalid(option) + if option in ("start_ms", "end_ms", "fed_start_ms"): + return _error("window_range_invalid", start_ms=request.start_ms, + end_ms=request.end_ms, fed_start_ms=request.fed_start_ms) + if option == "preroll_bars": + return _error("window_preroll_out_of_range", requested=request.preroll_bars) + return _internal(f"the selected primary planner refused the call as invalid ({option!r})") + if status == 2: + return _error("window_calendar_unsupported", timezone=timezone, session=session) + if status == 3: + bound = plan.get("bound") + if bound in (0, 1, 2): + return _error("window_boundary_unaligned", bound=_BOUND_NAMES[bound], + value_ms=plan["value_ms"], + previous_boundary_ms=plan["previous_boundary_ms"], + next_boundary_ms=plan["next_boundary_ms"]) + return _internal(f"the selected primary planner reported an unaligned boundary with " + f"bound {bound!r}") + if status == 4: + return _error("window_feed_range_invalid", time_ms=plan["value_ms"], + fed_start_ms=request.fed_start_ms, end_ms=request.end_ms) + if status == 5: + return SelectedWindowError( + "chart_bars_rejected", + f"bar timestamps are not increasing (row at {plan.get('value_ms')} after " + f"{plan.get('previous_boundary_ms')})", + {"field": "timestamp", "reason": "not_increasing"}) + if status == 6: + return _invariant(f"the selected primary planner failed on an internal error ({option!r})") + return _internal(f"the selected primary planner returned status {status!r}") + + +# --- trim by pointer offset, hash of the retained rows ---------------------- + +_ROW_FIELDS = ("open", "high", "low", "close", "volume", "timestamp") +_ROW_OFFSETS = (0, 8, 16, 24, 32, 40) +_HASH_CHUNK_ROWS = 1 << 16 + + +def retained_rows(bars: Any, row_count: int, trim_index: int, fed_count: int, + row_type: Any) -> Tuple[Any, int]: + """(rows, count) of the rows[trim_index:] the engine is fed, WITHOUT copying the feed. + + trim_index 0 returns the original array; otherwise a view of the same allocation is + returned (ctypes from_buffer at the byte offset), which holds a reference to the original, + so the original stays alive as long as the view does. fed_count must be exactly + row_count - trim_index (the planner's equality supplied = trimmed + fed); a mismatch is an + engine invariant. With nothing to feed the original array is returned with count 0.""" + for name, value in (("row_count", row_count), ("trim_index", trim_index), + ("fed_count", fed_count)): + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise _internal(f"{name} must be a nonnegative int, got {value!r}") + if trim_index > row_count or fed_count != row_count - trim_index: + raise _invariant(f"the plan keeps {fed_count} rows after trimming {trim_index} of " + f"{row_count} supplied rows") + if trim_index == 0: + return bars, fed_count + if fed_count == 0: + return bars, 0 + view = (row_type * fed_count).from_buffer(bars, trim_index * ctypes.sizeof(row_type)) + return view, fed_count + + +def _row_layout_is_record(row_type: Any, record_size: int) -> bool: + try: + offsets = tuple(getattr(row_type, name).offset for name in _ROW_FIELDS) + except AttributeError: + return False + return offsets == _ROW_OFFSETS and ctypes.sizeof(row_type) == record_size + + +def update_rows_hash(hasher: Any, bars: Any, start: int, count: int, row_type: Any, + record: Any) -> None: + """Feed rows[start:start+count] into hasher in the existing primary hash domain (the + caller's hasher already holds the domain prefix; `record` is the frozen "<5dq" Struct). + On a little-endian host whose rows are exactly those 48 bytes the memory itself is the + record sequence and is hashed in bounded chunks; otherwise each row is packed.""" + if count <= 0: + return + size = ctypes.sizeof(row_type) + if sys.byteorder == "little" and _row_layout_is_record(row_type, record.size): + base = ctypes.addressof(bars) + done = 0 + while done < count: + take = min(_HASH_CHUNK_ROWS, count - done) + hasher.update(ctypes.string_at(base + (start + done) * size, take * size)) + done += take + return + for index in range(start, start + count): + row = bars[index] + hasher.update(record.pack(row.open, row.high, row.low, row.close, row.volume, + row.timestamp)) + + +# --- R = $.report_window ---------------------------------------------------- + +_COUNT_KEYS = ("supplied_input_bars", "supplied_script_bars", "available_script_bars", + "used_script_bars", "trimmed_script_bars", "trim_index", "fed_input_bars", + "fed_script_bars", "preroll_input_bars", "window_input_bars", + "window_script_bars") +_SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +def _u53(plan: Mapping[str, Any], key: str) -> int: + value = plan[key] + if isinstance(value, bool) or not isinstance(value, int) or not 0 <= value <= I53_MAX: + raise _invariant(f"the plan's {key} is {value!r}, not a u53") + return value + + +def _optional_i53(plan: Mapping[str, Any], key: str) -> Optional[int]: + value = plan[key] + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int) or abs(value) > I53_MAX: + raise _invariant(f"the plan's {key} is {value!r}, not an i53 or null") + return value + + +def _lookback_not_produced() -> Dict[str, Any]: + """The lookback record of R in a build that has NO producer for the lookback analysis: + the one triple this build emits, for a run and for validation-only output alike. status + is "unknown", declared_required_chart_bars is null and has_unbounded_state is null, which + means "not produced by this build": it is neither true nor false, and nothing is guessed + or defaulted to either. This build never emits sufficient_declared, insufficient or a + number here, and draws on no lookback analysis, timeframe conversion or input metadata. + A build that has the producer emits a boolean. + + A fresh dict on every call, in the wire table's member order. build_report_window is the + only caller, so this is the only place the triple is written.""" + return { + "declared_required_chart_bars": None, + "status": "unknown", + "has_unbounded_state": None, + } + + +def _lookback_null_pairing_fault(lookback: Any) -> Optional[str]: + """The text of the fault when `lookback` holds a null has_unbounded_state in a pairing that + the documented contract forbids, else None. Null is valid ONLY together with status + "unknown" and a null declared_required_chart_bars, both present (the wire writes null and + never omits a member of an opted-in report): a null beside any other status, or beside a + number, zero included, is an invalid report. + + That is the whole rule. It asks nothing else of the record: a boolean has_unbounded_state + (the shape of a build WITH the producer), the set of statuses, the range of a count, a + member that is absent next to a boolean and a record that is not an object are other + lookback schema questions, which this function does not invent.""" + if (not isinstance(lookback, dict) or "has_unbounded_state" not in lookback + or lookback["has_unbounded_state"] is not None): + return None + if (lookback.get("status") == "unknown" and "declared_required_chart_bars" in lookback + and lookback["declared_required_chart_bars"] is None): + return None + return ("report_window.lookback has has_unbounded_state null beside status " + f"{lookback.get('status')!r} and declared_required_chart_bars " + f"{lookback.get('declared_required_chart_bars', '')!r}: null is valid only " + "beside status 'unknown' and a null declared_required_chart_bars") + + +def build_report_window(request: WindowRequest, plan: Mapping[str, Any], *, applied: bool, + feed: Mapping[str, Any], calendar: Mapping[str, Any]) -> Dict[str, Any]: + """The exact record R (the documented wire table) from the planner's plan. + + `feed` holds input_tf_seconds, source_bytes_sha256, source_values_sha256 and + evaluated_source_values_sha256; `calendar` holds timezone and session. applied is True for + a run, False for validation only (then equity_points and anchor_points are 0, and the counts + are plans, not execution evidence). Every count is the planner's; the wire's counting + equalities are checked here and a failure is an engine invariant. Lookback is not produced + by this build: the record is _lookback_not_produced()'s triple, status + unknown with a null declared count and a null has_unbounded_state, in a run and in + validation-only output alike.""" + counts = {key: _u53(plan, key) for key in _COUNT_KEYS} + window_first = _optional_i53(plan, "window_first_data_ms") + window_last = _optional_i53(plan, "window_last_data_ms") + fed_first = _optional_i53(plan, "fed_first_data_ms") + fed_last = _optional_i53(plan, "fed_last_data_ms") + supplied_first = _optional_i53(plan, "supplied_first_data_ms") + supplied_last = _optional_i53(plan, "supplied_last_data_ms") + preroll_first = _optional_i53(plan, "preroll_first_bar_ms") + preroll_last = _optional_i53(plan, "preroll_last_bar_ms") + trim_start = plan["trim_start_ms"] + if isinstance(trim_start, bool) or not isinstance(trim_start, int) or abs(trim_start) > I53_MAX: + raise _invariant(f"the plan's trim_start_ms is {trim_start!r}, not an i53") + available, used = counts["available_script_bars"], counts["used_script_bars"] + window_script, window_input = counts["window_script_bars"], counts["window_input_bars"] + if (counts["supplied_input_bars"] != counts["trim_index"] + counts["fed_input_bars"] + or counts["supplied_script_bars"] != available + window_script + or counts["fed_input_bars"] != counts["preroll_input_bars"] + window_input + or counts["fed_script_bars"] != used + window_script + or used != min(request.preroll_bars, available) + or counts["trimmed_script_bars"] != available - used + or bool(plan["shortfall"]) != (available < request.preroll_bars)): + raise _invariant("the plan breaks the wire's counting equalities") + for key in ("source_bytes_sha256", "source_values_sha256", "evaluated_source_values_sha256"): + value = feed[key] + if not isinstance(value, str) or _SHA256_RE.fullmatch(value) is None: + raise _internal(f"{key} is not a lowercase SHA-256 digest") + seconds = feed["input_tf_seconds"] + if isinstance(seconds, bool) or not isinstance(seconds, int) or not 0 < seconds <= I53_MAX: + raise _internal("input_tf_seconds is not a positive u53") + nonempty_window = window_script > 0 + return { + "wire_version": WIRE_VERSION, + "policy": POLICY, + "applied": bool(applied), + "window": { + "start_ms": request.start_ms, + "end_ms": request.end_ms, + "first_data_ms": window_first, + "last_data_ms": window_last, + "coverage": "nonempty" if window_input > 0 else "empty", + }, + "feed": { + "requested_start_ms": request.fed_start_ms, + "first_data_ms": fed_first, + "last_data_ms": fed_last, + "supplied_first_data_ms": supplied_first, + "supplied_last_data_ms": supplied_last, + "input_tf_seconds": seconds, + "script_tf": request.script_tf, + "source_bytes_sha256": feed["source_bytes_sha256"], + "source_values_sha256": feed["source_values_sha256"], + "evaluated_source_values_sha256": feed["evaluated_source_values_sha256"], + }, + "calendar": {"timezone": calendar["timezone"], "session": calendar["session"]}, + "preroll": { + "requested_script_bars": request.preroll_bars, + "available_script_bars": available, + "used_script_bars": used, + "trimmed_script_bars": counts["trimmed_script_bars"], + "trim_start_ms": trim_start, + "trimmed_input_bars": counts["trim_index"], + "first_bar_ms": preroll_first, + "last_bar_ms": preroll_last, + "shortfall": bool(plan["shortfall"]), + }, + "lookback": _lookback_not_produced(), + "counts": { + "supplied_input_bars": counts["supplied_input_bars"], + "supplied_script_bars": counts["supplied_script_bars"], + "fed_input_bars": counts["fed_input_bars"], + "fed_script_bars": counts["fed_script_bars"], + "preroll_input_bars": counts["preroll_input_bars"], + "window_input_bars": window_input, + "window_script_bars": window_script, + "equity_points": window_script + 1 if applied else 0, + "anchor_points": 1 if applied else 0, + }, + "metering": { + "billable_input_bars": window_input, + "preroll_billable_input_bars": 0, + "basis": METERING_BASIS, + }, + "indices": { + "fed_script_index_of_window_first": used if nonempty_window else None, + "supplied_script_index_of_window_first": available if nonempty_window else None, + }, + } + + +def report_shape_full(canonical_equity_points: int, canonical_trades: int) -> Dict[str, Any]: + """$.report_shape of a full/v1 report: nothing is projected away.""" + return { + "version": 1, + "name": REPORT_SHAPE, + "curve_point_budget": None, + "canonical_equity_points": canonical_equity_points, + "emitted_equity_points": canonical_equity_points, + "canonical_trades": canonical_trades, + "emitted_trades": canonical_trades, + "retained_point_indices": None, + "curve_decimated": False, + } + + +def _established_number(value: Any) -> Optional[float]: + """value as a finite float when it is a number (not a bool) or text that spells one, else + None: the only forms of an effective initial capital the provenance can hold. None means + the provenance establishes no capital (an unresolved one is recorded as null there); it is + no verdict on the run.""" + if isinstance(value, bool): + return None + if isinstance(value, str): + try: + value = float(value.strip()) + except ValueError: + return None + if not isinstance(value, (int, float)): + return None + try: + number = float(value) + except OverflowError: + return None + return number if math.isfinite(number) else None + + +def _finite_actual(value: Any) -> bool: + """Whether value, as the engine produced it, is a finite number: an int or a float that is + not a bool, not None (the report writes a non-finite double as null), not text, and not NaN + or an infinity. Nothing is converted or read from text.""" + if isinstance(value, bool) or not isinstance(value, (int, float)): + return False + try: + return math.isfinite(float(value)) + except OverflowError: # an int beyond binary64 + return False + + +def check_selected_curve(curve: Sequence[Mapping[str, Any]], start_ms: int, + window_script_bars: int, initial_capital: Any) -> None: + """The serialized selected curve is the native selected curve, unchanged: exactly three + fields per point (time_ms, equity, open_profit) and no other, M + 1 points, position 0 the + anchor at T with open profit 0 and an equity that is a finite number, position i >= 1 the + window bar i - 1. Nothing is added, dropped or recomputed. + + `initial_capital` is the effective strategy setting of the run's provenance, read as the + provenance holds it. The native anchor derives from the engine's own initial capital, so a + provenance that does not establish one (an unresolved capital is null there, as in ordinary + mode) is no evidence that the anchor is wrong: then ONLY the comparison of the anchor equity + with the capital is skipped. No capital is guessed or written back, and every other check + above, the finiteness of the actual anchor equity included, is made in both cases. When the + provenance does establish a finite capital (a number or text spelling one), the anchor + equity must equal it, or the run is an engine invariant failure.""" + if len(curve) != window_script_bars + 1: + raise _invariant(f"the selected curve has {len(curve)} points, the plan says " + f"{window_script_bars + 1}") + for position, point in enumerate(curve): + if set(point) != {"time_ms", "equity", "open_profit"}: + raise _invariant(f"point {position} of the selected curve does not have exactly " + "time_ms, equity and open_profit") + anchor = curve[0] + if anchor["time_ms"] != start_ms or anchor["open_profit"] != 0: + raise _invariant("the selected curve does not start with the window anchor at T with " + "open profit 0") + if not _finite_actual(anchor["equity"]): + raise _invariant(f"the anchor equity {anchor['equity']!r} of the selected curve is " + "not a finite number") + capital = _established_number(initial_capital) + if capital is not None and anchor["equity"] != capital: + raise _invariant(f"the anchor equity {anchor['equity']!r} is not the initial capital " + f"{capital!r}") + + +def check_report_placements(report: Mapping[str, Any], window: Mapping[str, Any]) -> None: + """The selected report carries R in all three required places and a version 2 + fingerprint, or it is not a report: $.report_window, $.applied_runtime.report_window and + $.fingerprint.provenance.runtime.report_window must each equal `window`; the fingerprint has + version 2 and provenance.schema_version 2; and the HASHED bytes (the base64 token, whose + SHA-256 must be the digest) hold schema_version 2 and the same R. R's lookback record is + held to the documented null pairing (a null has_unbounded_state only beside status + "unknown" and a null declared_required_chart_bars) and to nothing else about lookback; the + copies equal `window`, so they carry the same triple. Any failure is an engine invariant + raised before anything is serialized.""" + fault = _lookback_null_pairing_fault(window.get("lookback")) + if fault is not None: + raise _invariant(fault) + fingerprint = report.get("fingerprint") + provenance = fingerprint.get("provenance") if isinstance(fingerprint, dict) else None + if not isinstance(fingerprint, dict) or not isinstance(provenance, dict): + raise _invariant("the selected report has no fingerprint") + applied = report.get("applied_runtime") + runtime = provenance.get("runtime") + places = ( + ("$.report_window", report.get("report_window")), + ("$.applied_runtime.report_window", + applied.get("report_window") if isinstance(applied, dict) else None), + ("$.fingerprint.provenance.runtime.report_window", + runtime.get("report_window") if isinstance(runtime, dict) else None), + ) + for path, placed in places: + if placed != window: + raise _invariant(f"{path} is not the report_window record of the run") + if fingerprint.get("version") != FINGERPRINT_VERSION: + raise _invariant("the selected fingerprint is not version 2") + if provenance.get("schema_version") != FINGERPRINT_VERSION: + raise _invariant("the selected provenance is not schema version 2") + token, digest = fingerprint.get("token"), fingerprint.get("digest") + try: + raw = base64.b64decode(token, validate=True) + hashed = json.loads(raw.decode("utf-8")) + except (TypeError, ValueError, binascii.Error, RecursionError): + raise _invariant("the selected fingerprint token cannot be read back") from None + if digest != "sha256:" + hashlib.sha256(raw).hexdigest(): + raise _invariant("the selected fingerprint digest is not the hash of its token") + hashed_runtime = hashed.get("runtime") if isinstance(hashed, dict) else None + if (not isinstance(hashed, dict) or hashed.get("schema_version") != FINGERPRINT_VERSION + or not isinstance(hashed_runtime, dict) + or hashed_runtime.get("report_window") != window): + raise _invariant("the hashed bytes of the selected fingerprint do not hold schema " + "version 2 and the run's report_window") + + +def capabilities_record(supported: bool) -> Dict[str, Any]: + """The --capabilities-json document. A runtime that cannot establish the native ABIs, the + observer and the planner reports wire version 0 and the other members null or empty; the + members of features this build does not offer (compact shape, digest) are always absent. + + selected_window_contract_revisions is CONTRACT_REVISIONS as a fresh list when supported: + exactly ["1.3-parity"], the prerelease parity declaration (not full 1.3 coverage), and + never "1.3", which only a full build carries. It is [] when the runtime is not supported, + so an unsupported runtime lists no revision at all. No other member of this document + changes with the declaration.""" + return { + "engine": "pineforge", + "capabilities": { + "selected_window_wire_version": WIRE_VERSION if supported else 0, + "selected_window_contract_revisions": list(CONTRACT_REVISIONS) if supported else [], + "report_policy": POLICY if supported else None, + "fingerprint_version": FINGERPRINT_VERSION if supported else None, + "report_shapes": [REPORT_SHAPE] if supported else [], + "curve_point_budget": None, + "results_digest_version": None, + "results_digest_platform": None, + }, + } + + +# --- the frozen selected_window ABI --- + +ABI_VERSION = 1 +_SET_OK = 0 +_SET_STATUS_MEANING = { + -1: "invalid handle, size, version or bounds", + -2: "unsupported execution contract", + -3: "not quiescent", + -4: "out of memory", +} +_COUNTS_NOT_OWNED = -2 + + +class SelectedWindowConfigC(ctypes.Structure): + """pf_selected_window_config_v1 (LP64: 24 bytes): the window [T, E).""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("start_ms", ctypes.c_int64), + ("end_ms", ctypes.c_int64), + ] + + +class SelectedWindowCountsC(ctypes.Structure): + """pf_selected_window_counts_v1 (LP64: 80 bytes): the engine's own counts of one successful + selected generation, with the identities that own it.""" + _fields_ = [ + ("struct_size", ctypes.c_uint32), + ("version", ctypes.c_uint32), + ("run_generation", ctypes.c_uint64), + ("attempt_serial", ctypes.c_uint64), + ("attempt_generation", ctypes.c_uint64), + ("fed_input_bars", ctypes.c_uint64), + ("fed_script_bars", ctypes.c_uint64), + ("preroll_input_bars", ctypes.c_uint64), + ("preroll_script_bars", ctypes.c_uint64), + ("window_input_bars", ctypes.c_uint64), + ("window_script_bars", ctypes.c_uint64), + ] + + +_ABI_EXPORTS = ("pf_selected_window_version", "strategy_set_selected_window_v1", + "strategy_selected_window_counts_v1") + + +def _unsupported_abi(text: str) -> SelectedWindowError: + return SelectedWindowError("window_mode_unsupported", text, {"capability": CAPABILITY}) + + +class SelectedWindowAbi: + """The three exports of the selected-window ABI on one loaded strategy library: probed at + construction (a missing export or a version other than 1 is window_mode_unsupported with + capability selected_window_v1), then configure() before the run, counts() after it and + clear() in cleanup. `library` is the loaded strategy library or anything with the exports.""" + + def __init__(self, library: Any) -> None: + if (ctypes.sizeof(SelectedWindowConfigC) != 24 + or ctypes.sizeof(SelectedWindowCountsC) != 80): + raise _internal("the ctypes layout of the selected-window structs is not the frozen " + "24 and 80 bytes") + try: + version = library.pf_selected_window_version + self._set = library.strategy_set_selected_window_v1 + self._counts = library.strategy_selected_window_counts_v1 + except AttributeError as error: + raise _unsupported_abi( + f"the strategy library lacks a selected-window export: {error}") from error + version.argtypes = [] + version.restype = ctypes.c_uint32 + self._set.argtypes = [ctypes.c_void_p, ctypes.POINTER(SelectedWindowConfigC)] + self._set.restype = ctypes.c_int + self._counts.argtypes = [ctypes.c_void_p, ctypes.POINTER(SelectedWindowCountsC)] + self._counts.restype = ctypes.c_int + found = version() + if found != ABI_VERSION: + raise _unsupported_abi( + f"the strategy library reports selected-window version {found}, " + f"not {ABI_VERSION}") + + def configure(self, state: Any, start_ms: int, end_ms: int) -> None: + """Select [start_ms, end_ms) for the NEXT admitted generation of `state`.""" + config = SelectedWindowConfigC(ctypes.sizeof(SelectedWindowConfigC), ABI_VERSION, + start_ms, end_ms) + status = self._set(state, ctypes.pointer(config)) + if status == _SET_OK: + return + meaning = _SET_STATUS_MEANING.get(status, "unknown status") + text = f"the selected window was refused with status {status} ({meaning})" + if status == -4: + raise SelectedWindowError("out_of_memory", text) + if status == -2: + raise SelectedWindowError("window_mode_unsupported", text, + {"capability": CAPABILITY}) + raise _internal(text) + + def clear(self, state: Any) -> int: + """Remove the configuration (a NULL descriptor) for cleanup; the status is returned + and is never an error here, since the state is about to be freed.""" + return self._set(state, None) + + def counts(self, state: Any) -> Dict[str, int]: + """The engine's counts of the latest attempt, which must own a successful selected + generation: anything else is an engine invariant (the run reported success).""" + counts = SelectedWindowCountsC(ctypes.sizeof(SelectedWindowCountsC), ABI_VERSION) + status = self._counts(state, ctypes.byref(counts)) + if status == _COUNTS_NOT_OWNED: + raise _invariant("the engine reports no successful selected generation for the " + "latest attempt") + if status != 0: + raise _internal(f"strategy_selected_window_counts_v1 returned {status}") + if counts.struct_size != ctypes.sizeof(SelectedWindowCountsC) or counts.version != ABI_VERSION: + raise _internal("the selected-window counts came back with a changed header") + return {name: int(getattr(counts, name)) for name, _ in SelectedWindowCountsC._fields_ + if name not in ("struct_size", "version")} + + +def check_native_counts(plan: Mapping[str, Any], native: Mapping[str, int], + observation: Mapping[str, int]) -> None: + """The engine's counts must equal the planned ones exactly, and belong to the observed + attempt. A mismatch is an engine invariant: the plan's counts are never replaced by the + engine's, and the engine's never by the plan's. + + planned fed_input / fed_script / preroll_input / preroll_script (= used pre-roll bars) / + window_input / window_script versus the same six native counts; the three identities + (run_generation, attempt_serial, attempt_generation) of the counts must equal those of the + execution observation and be nonzero.""" + expected = { + "fed_input_bars": plan["fed_input_bars"], + "fed_script_bars": plan["fed_script_bars"], + "preroll_input_bars": plan["preroll_input_bars"], + "preroll_script_bars": plan["used_script_bars"], + "window_input_bars": plan["window_input_bars"], + "window_script_bars": plan["window_script_bars"], + } + for name, planned in expected.items(): + if native[name] != planned: + raise _invariant(f"the engine counted {native[name]} for {name}, the plan " + f"says {planned}") + for name in ("run_generation", "attempt_serial", "attempt_generation"): + if native[name] == 0 or native[name] != observation[name]: + raise _invariant(f"the selected-window counts name {name} {native[name]}, the " + f"execution observation names {observation[name]}") diff --git a/docs/adr/0001-kernel-adapter-boundary.md b/docs/adr/0001-kernel-adapter-boundary.md index 8fb53ca4d..3fae579d1 100644 --- a/docs/adr/0001-kernel-adapter-boundary.md +++ b/docs/adr/0001-kernel-adapter-boundary.md @@ -37,7 +37,7 @@ Three layers, two of them meant to be optional: `timeframe`, `timezone`, `str_utils`, `c_abi`, `native_c_host`, `reservation_expansion` (`CMakeLists.txt:125`) and `pending_order_mirror` (`CMakeLists.txt:131`) units — plus the headers directly under `include/pineforge/`. That list is what `add_library` - (`CMakeLists.txt:155`) compiles into `pineforge_kernel` / `PineForge::kernel`, and it and + (`CMakeLists.txt:158`) compiles into `pineforge_kernel` / `PineForge::kernel`, and it and `PINEFORGE_SOURCE_LAYER_SOURCES` are disjoint: no file is in both, and none is outside both. **Two stems exist twice; the kernel copy is the generic half** — `ReservationExpansion` (`src/reservation_expansion.cpp:10`) and @@ -78,7 +78,7 @@ summary of it. (`native_host.hpp:893`); also `on_native_input` (`native_host.hpp:875`), `on_native_tick` (`native_host.hpp:878`), `on_native_timeframe_bar` (`native_host.hpp:885`), `on_native_bar_open` — at the modeled opening, before its matching pass - (`native_host.hpp:889`, `on_native_bar_open` `native_execution_consumer.cpp:7434-7436`) — + (`native_host.hpp:889`, `on_native_bar_open` `native_execution_consumer.cpp:7882-7884`) — `on_native_recalculate` for every calculation of the run (`native_host.hpp:913`), `on_native_sub_bar` (`native_host.hpp:928`) and post-fill `on_native_applied` (`native_host.hpp:938`). `on_bar` is `final` (`native_host.hpp:857`). @@ -112,18 +112,18 @@ summary of it. bare host needs no override at all, while `HostSized` hands the units to the host — and an unresolved `HostSized` whose `resolve_execution_terms` (`native_host.hpp:947`) returns no units is a `TermsUnresolved` rejection - (`TermsUnresolved` `native_execution_consumer.cpp:5292`). + (`TermsUnresolved` `native_execution_consumer.cpp:5704`). - **Observing executions.** Implement `on_native_applied` (`native_host.hpp:938`) — notifications drain FIFO after the outer callback returns — and/or poll `native_events(after_ordinal)` (`native_host.hpp:1335`). Do *not* implement `NativeExecutionConsumer`: it is the kernel's internal matcher, declared `final` - (`src/native_execution_consumer.hpp:59`) and bound by the host. + (`src/native_execution_consumer.hpp:85`) and bound by the host. - **The rest of the host surface**, all on `NativeStrategyHost`: | What | Surface | |---|---| | Run state | `native_state()` → `NativeStateView` — kind, phase, `NativeCompletion` BatchComplete / StreamEnded, failure, high water (`NativeCompletion` `native_host.hpp:50`, `NativeStateView` `:282`, `NativeRunPhase` `:41`) | - | Failure model | `NativeFailureCode` (`native_host.hpp:63-77`); `Failed` is latched, nothing resumes. A throwing callback latches `CallbackException` (`CallbackException` `native_execution_consumer.cpp:5265`) | + | Failure model | `NativeFailureCode` (`native_host.hpp:63-77`); `Failed` is latched, nothing resumes. A throwing callback latches `CallbackException` (`CallbackException` `native_execution_consumer.cpp:5677`) | | Cohorts | `cohort_open` (`native_host.hpp:1282`) / `cohort_add` (`:1285`) / `cohort_remove` (`:1288`) — the handle a `BindCohort` owner names | | Mid-callback execution | `current_execution_point` (`native_host.hpp:1087`), `inspect_current_execution` (`native_host.hpp:1099`) / `execute_current` (`native_host.hpp:1104`) | | Margin seams | four, not one: the run spec's own `margin` model (`NativeMarginModel` `native_run_spec.hpp:240`) with its three hooks — `resolve_margin_requirement` (`native_host.hpp:976`), `margin_check_allowed` (`native_host.hpp:988`), `resolve_margin_call_units` (`native_host.hpp:995`) — and `validate_execution_precommit` (`native_host.hpp:960`), whose `AdmitWithHostMargin` verdict hands the opening check to the host | @@ -196,7 +196,7 @@ summary of it. measured equal on macOS/arm64 and Linux/aarch64 at tzdata 2026c). `broker_state_hash()` (`engine.hpp:2043`) folds that hash with position and lot state under the pinned domain - `pineforge-broker-state/v19` (`engine_state_hash.cpp:33`), the closed rows as their count and + `pineforge-broker-state/v19` (`engine_state_hash.cpp:41`), the closed rows as their count and a running digest each final row folds into once; it is the one exported to C (`c_abi.cpp:538`). A host folds its **own** durable state into it through `hash_host_extension` (`engine.hpp:405`), whose deprecated spelling @@ -211,11 +211,11 @@ into kernel driving — never by special-casing the kernel *for a specific scrip kernel and own these quirks, each at its site: - **Seams the adapter overrides.** The kernel is driven, not patched. `PineStrategyHost` overrides - `resolve_execution_terms` (`pine_strategy_host.cpp:802`) for TV sizing and fill spelling; - `validate_execution_precommit` (`pine_strategy_host.cpp:827`), whose - `validate_precommit` (`pine_adapter.cpp:16150`) returns `AdmitWithHostMargin` - (`pine_adapter.cpp:16354`) to own the opening margin decision; `on_native_tick` - (`pine_strategy_host.cpp:533`) / `on_native_applied` (`pine_strategy_host.cpp:601`) for + `resolve_execution_terms` (`pine_strategy_host.cpp:813`) for TV sizing and fill spelling; + `validate_execution_precommit` (`pine_strategy_host.cpp:838`), whose + `validate_precommit` (`pine_adapter.cpp:16158`) returns `AdmitWithHostMargin` + (`pine_adapter.cpp:16362`) to own the opening margin decision; `on_native_tick` + (`pine_strategy_host.cpp:540`) / `on_native_applied` (`pine_strategy_host.cpp:612`) for `calc_on_order_fills` re-entry. It does not own lot excursions: since R5 lane H-THIN it keeps the kernel's `owns_lot_excursions()` default, so the kernel's sampler books every Pine lot's MFE/MAE, and `ab9714be`'s host model (RULING A48) is gone. R5 lane H-MEASURE had measured the @@ -226,10 +226,10 @@ kernel and own these quirks, each at its site: lowering moved the excursion cells of 335 trades in 19 corpus probes: 324 now equal TradingView's, 11 moved closer, none farther (`tests/fixtures/e19_allin_trim` tapes the all-in residual's own entry-bar call). `process_orders_on_close` becomes - `NativeCloseExecution::AfterCalculation` in the projected spec (`pine_adapter.cpp:2751-2752`); + `NativeCloseExecution::AfterCalculation` in the projected spec (`pine_adapter.cpp:2758-2759`); calc cadence and language publication are `PineScheduler`'s (`pine_scheduler.hpp:21`). - **Batching and open-order priority.** Same-bar command batching and its deferred queues - (`PendingSameBarCommand` pine_adapter.hpp:1847, `pending_bracket_legs_` pine_adapter.hpp:2477-2495); the retained parent-before-child ordering of live handles — an + (`PendingSameBarCommand` pine_adapter.hpp:1873, `pending_bracket_legs_` pine_adapter.hpp:2515-2533); the retained parent-before-child ordering of live handles — an exactly-shaped entry and its `from_entry` exit, re-created after a named cancel, on a flat book under `process_orders_on_close` (`src/compat/pine/order_priority.cpp:13-59`, applied by `update_l4c_priority` `pine_adapter.cpp:1275-1345`). Kernel priority is acceptance/incarnation order. @@ -237,14 +237,14 @@ kernel and own these quirks, each at its site: (`select_exit_activation` src/compat/pine/exit_activation.cpp:44, `select_exit_suspension` src/compat/pine/exit_lifecycle.cpp:11, `select_replacement_revival_definition` src/compat/pine/exit_lifecycle.cpp:51) and historical birth reach (`order_birth.cpp` `src/compat/pine/order_birth.cpp:5-14`). - **Trail and tick conventions.** The half-tick arm threshold measured against tick-quantized - extremes while the raw running best is retained (`has_trail_request` `pine_adapter.cpp:12396-12401`); the half-tick + extremes while the raw running best is retained (`has_trail_request` `pine_adapter.cpp:12404-12409`); the half-tick trigger threshold (`source_trigger_threshold` pine_adapter.cpp:343) and raw-vs-booked fill spelling, behind the - adapter's terms seam (`resolve_terms` `pine_adapter.cpp:14623`). + adapter's terms seam (`resolve_terms` `pine_adapter.cpp:14631`). - **Money arithmetic.** Ten-significant-digit half-up money (`source_money_round` `pine_adapter.cpp:506-512`, twin `tv_money_round` `pine_policy_support.hpp:9-15`). - **Quantity dust.** After every applied execution the Pine host erases any lot of at most `kQtyEpsilon` (`1e-10`) from the book it shares with the kernel, with no closing row - (`on_native_applied` `pine_strategy_host.cpp:601`), inside a live book as well as at a flat + (`on_native_applied` `pine_strategy_host.cpp:612`), inside a live book as well as at a flat reset — broader than `ab9714be`, which reset only a whole book at or under that size. Measured by R5 lane H-MEASURE (`tests/test_pine_dust_sweep_paired.cpp`): TradingView's decimal quantities leave no remnant row on a `lab tv` tape, which the Pine host books row for row @@ -256,12 +256,12 @@ kernel and own these quirks, each at its site: restore's second floor) fail tape-pinned rows (`test_pine_dust_sweep_paired`, `test_tv_money_band_l4b`); `quantity_tolerance` is not their rule (its row below). - **Close reservations.** `strategy.close` callsite batching, two-call provenance and the entry-id - ledger (`close_logical_units_` `pine_adapter.hpp:2512-2542`); the POOC reservation-growth population predicate + ledger (`close_logical_units_` `pine_adapter.hpp:2539-2569`); the POOC reservation-growth population predicate (`select_reservation_growth_sources` src/compat/pine/reservation_expansion.cpp:9). - **Margin.** The kernel owns the margin *mechanism* — the level solve, the check points, the kernel request, its re-pricing, the receipt — and the adapter answers its three policy hooks - with TradingView's: `margin_check_allowed` (`pine_adapter.cpp:17601`) for the scheduling, - `resolve_margin_requirement` (`pine_adapter.cpp:17670`) for the ten-significant-digit money, + with TradingView's: `margin_check_allowed` (`pine_adapter.cpp:17609`) for the scheduling, + `resolve_margin_requirement` (`pine_adapter.cpp:17678`) for the ten-significant-digit money, and `resolve_margin_call_units` for the lot-floored 4x restore. That hook answers every call on purpose (R5 lane F7): TradingView floors the restore onto the lot grid before the multiple, so on a gridded tape the kernel's own `ShortfallMultiple 4` books 5 lots where TradingView @@ -272,7 +272,7 @@ kernel and own these quirks, each at its site: margin since F7), while the roll point measures at the remaining path's adverse mark (MG-FX: 0.4116 @ 97 there against TradingView's 0.3996 @ 100). What has no kernel check point at all stays adapter-side: the `process_orders_on_close` chronology exception - (`non_pooc_commissioned_short` `pine_adapter.cpp:23926`) and + (`non_pooc_commissioned_short` `pine_adapter.cpp:23944`) and the 1x-long money call — plus the TV admission scopes (`explicit_pair_scope` `src/compat/pine/market_admission.cpp:34-47`) and the review fold they feed (`awaits_pair_review` `:68-73`, `fold_admission_history` `:83-140`). @@ -333,29 +333,29 @@ kernel and own these quirks, each at its site: projects `NativeCalculationTrigger::BarCloseAndFills` with TradingView's guard literal as `max_recalculations_per_point`. What stays are the specifics COOF adds on top — the language-state snapshot/restore around a recalculation, the waypoint-only refill deferral - (`next_source_path_waypoint` `pine_adapter.cpp:10272`), the first-open execution chain and its - own loop guard (`kFirstOpenLoopGuard` `pine_scheduler_native.cpp:792`), and the two fills Pine refuses to - recalculate on (`suppress_grouped_stop_recalc` `pine_adapter.cpp:6605`). + (`next_source_path_waypoint` `pine_adapter.cpp:10280`), the first-open execution chain and its + own loop guard (`kFirstOpenLoopGuard` `pine_scheduler_native.cpp:848`), and the two fills Pine refuses to + recalculate on (`suppress_grouped_stop_recalc` `pine_adapter.cpp:6613`). - **Pine language state and harness flags.** Series, the tick-level barstate flags and position-view freezing (`PineLanguageState` `pine_language_state.hpp:12`); the session flags, `PineStrategyHost` members since R5 lane F5 (`session_ismarket_` - `pine_strategy_host.hpp:1138`), which `scheduler_update_session_state` - (`pine_strategy_host.cpp:2167`) selects from the kernel's session-day facts before each source + `pine_strategy_host.hpp:1174`), which `scheduler_update_session_state` + (`pine_strategy_host.cpp:2196`) selects from the kernel's session-day facts before each source callback — `session_ismarket_` and the `_regular` pair are those facts as they are, and the plain `session.isfirstbar` / `islastbar` pair widens the regular day by the pre- and post-market bars an extended-hours chart holds (`update_extended_session_day`, Pine policy on TradingView's extended-hours tapes, `tests/test_session_extended_day_tape.cpp`). Generated code reads two of them; a generated chart `session.ismarket` asks the session calendar at the bar's open through the transpiler's own helper, and only one inside a `request.security` payload still calls the - time-of-day predicate (`pine_session_ismarket` `pine_strategy_host.hpp:746`), which on six + time-of-day predicate (`pine_session_ismarket` `pine_strategy_host.hpp:758`), which on six `lab tv` tapes misses every Sunday open of a weekday-masked overnight session and every bar of `0000-2400`, where the kernel's fact matches TradingView on every bar (R5 lane H-MEASURE, `tests/test_session_ismarket_tape.cpp`). A reading at the bar's open also puts a bar that opens inside a break out of market, which TradingView and the kernel's fact hold in market (`tests/test_session_break_open_bar_tape.cpp`); `barstate_islast_`, still a `BacktestEngine` member (`engine.hpp:378`) only this host writes; the live-tail / - probe-suppress overrides (`set_realtime_tail` `pine_strategy_host.hpp:521`, - `set_probe_suppress_tail_logic` `pine_strategy_host.hpp:547`), whose live-probe protocol is + probe-suppress overrides (`set_realtime_tail` `pine_strategy_host.hpp:533`, + `set_probe_suppress_tail_logic` `pine_strategy_host.hpp:559`), whose live-probe protocol is also a Pine policy over the session flags: its batch's final bar reads the kernel's open-ended close. @@ -370,7 +370,7 @@ citations, 264 are in `src/source/` and its headers; 6 sit in kernel files ## Current state vs. target (read before relying on "without Pine") - **The kernel is a separately linkable artifact.** `add_library` - (`CMakeLists.txt:155`) builds `pineforge_kernel` / `PineForge::kernel` from + (`CMakeLists.txt:158`) builds `pineforge_kernel` / `PineForge::kernel` from `PINEFORGE_KERNEL_SOURCES` alone, with the same flags `pineforge` uses, in **both** configurations. `libpineforge.a` still carries the adapter beside the kernel by default; `-DPINEFORGE_BUILD_SOURCE_LAYER=OFF` (`CMakeLists.txt:44`) compiles no source-layer @@ -394,15 +394,15 @@ citations, 264 are in `src/source/` and its headers; 6 sit in kernel files the lines 73-180 earlier drafts cited are and were live code (`ClosedLotExcursionFacts`, `PyramidEntry`, `Trade`, each with its own doc). The arithmetic is adapter-side (`source_money_round` `pine_adapter.cpp:489-507`). `strategy.close` batching and the entry-id ledger - are **gone** from the kernel — the ledger is adapter state (`close_logical_units_` `pine_adapter.hpp:2512-2542`) and + are **gone** from the kernel — the ledger is adapter state (`close_logical_units_` `pine_adapter.hpp:2539-2569`) and only names survive in comments. The TradingView margin-call toggle remains source-owned as - `source::PineStrategyHost::set_margin_call_enabled` (`pine_strategy_host.hpp:499`); + `source::PineStrategyHost::set_margin_call_enabled` (`pine_strategy_host.hpp:511`); what enables the generic model is the presence of `NativeRunSpec::margin`. So is the string-sentinel decoding of adapter-written comments: `closed_trade_close_cause` (`src/engine_trade_accessors.cpp:158`) reads a typed `execution::CloseCause` off the row, compares no string, and answers `3` for a *kernel* liquidation as readily as for the adapter's. What is live is one pair of hashed Pine-shaped lot flags: `skip_entry_bar_high` / `skip_entry_bar_low` (`engine.hpp:153`), hashed - (`skip_entry_bar_high` `src/engine_state_hash.cpp:66`) and, since R5 lane E6, set by no host at + (`skip_entry_bar_high` `src/engine_state_hash.cpp:74`) and, since R5 lane E6, set by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` `src/engine_path_resolve.cpp:95`; no in-tree host has since R5 lane H-THIN, when the Pine host left excursions to the kernel's sampler) and the kernel @@ -423,7 +423,7 @@ citations, 264 are in `src/source/` and its headers; 6 sit in kernel files (`float_band_eq` `ta_compare_band.hpp:38`), the TV-named fields of the frozen `pf_pending_order_v1_t` mirror, and the lot-flag pair above. - **The C API trades.** Two disjoint runtime inventories, both pinned. - `src/c_abi.cpp` implements the **73** codegen-facing runtime `PF_API` symbols that + `src/c_abi.cpp` implements the **80** codegen-facing runtime `PF_API` symbols that `scripts/check_c_abi_runtime.py` pins by name, and `src/native_c_host.cpp` implements the **43** additive symbols of ``. A C host creates a host from a callback table (`strategy_native_host_create_v1` `native_c_api.h:2682`), runs a batch @@ -448,7 +448,7 @@ citations, 264 are in `src/source/` and its headers; 6 sit in kernel files `native_selected_strategy.cpp`), both of which include `native_host.hpp`. Each C host includes ``, which includes `native_c_api.h`. Every one links `PineForge::kernel` and checks its own numbers before it prints its summary line. `PINEFORGE_BUILD_EXAMPLES` - (`CMakeLists.txt:37`, default OFF) builds them (`add_subdirectory` `CMakeLists.txt:344`) and + (`CMakeLists.txt:37`, default OFF) builds them (`add_subdirectory` `CMakeLists.txt:386`) and registers each as a CTest row that runs through `examples/native/run_example.cmake`, which fails on a nonzero exit, a signal, a timeout **or** a missing summary line — so a row asserts both, which a bare `PASS_REGULAR_EXPRESSION` cannot. `scripts/test_example_runner.py` proves @@ -600,20 +600,20 @@ name is usually a plain word. The fourth audit found these gaps (AUDIT4-opus X10 | the margin model's check points on an intrabar path -- `NativeMarginCheckKind::IntrabarSample` (`include/pineforge/native_host.hpp`; C `PF_NATIVE_MARGIN_CHECK_INTRABAR_SAMPLE`), offered at every delivered sample after the script bar's first of a `lower_tf` path matched as `ContinuousSegments`, at that sample's price, immediately before it is matched (`NativeExecutionConsumer::deliver_intrabar_script`, `src/native_execution_consumer.cpp`) | every host with a margin model and such a path, through `margin_check_allowed`; the Pine adapter admits it on a magnified run's leveraged long and, outside `process_orders_on_close`, its short at any margin (R5 lane PAR-MARGIN-2) (`PineExecutionAdapter::intrabar_sample_checked`, `src/source/pine_adapter.cpp`) | **a check point of the kernel's own** (R5 lane PAR-MARGIN, item 3; H-MEASURE's acid finding A). Rule 2 (a), mechanism: a run with no whole-bar waypoint model measures its maintenance requirement where its path is -- at each delivered sample -- the rule the docs and `native_run_spec.hpp` stated and the kernel had not kept (it checked the first sample and after fills only); nothing in it knows a venue, and a host that checks somewhere else suppresses the points it does not share, as for every other kind. Rule 2 (b), knob: none -- the path is the run's own declaration, and a one-price distribution path (`synthesized`, `DistributionSamples`) is not offered the point, since a request armed at a discrete point is matched only at a later one. What moved: a margin run on a continuous intrabar path whose book crosses its line inside a bar now books that call at the crossing sample (H-MEASURE's intrabar probe: 55 points offered, a slice at 105 where none was booked). TradingView's magnified broker does the same at its own intrabars, 2 minutes on a 15-minute chart (`tests/fixtures/intrabar_margin`: the first call at the first lower-timeframe low that crosses on 4 of 4 tapes; a model of the check at TradingView's 2-minute intrabars books every row of all 20 magnified tapes, R5 lane PAR-MARGIN-2). No pinned value moved. Pinned by `tests/test_native_margin_intrabar_samples.cpp` (kernel-only) and the "intrabar margin on tapes" section of `tests/test_adapter_margin_schedule_differential.cpp`. | | the path a post-fill margin check measures -- `NativeExecutionConsumer::margin_segment_origin` (`src/native_execution_consumer.cpp`), the phase the `AfterApplied` re-arm hands `margin_sizing_price`: the waypoint the segment into the fill's driver point starts from, with the fill price as the point's own mark | every host with a margin model on a path walked by its modeled waypoints (no intrabar path), through the `AfterApplied` point `margin_check_allowed` admits | **a mechanism correction** (R5 lane PAR-MARGIN-2). Rule 2 (a), mechanism: a request matched at a driver point was reached on the segment into it, so the book its fill leaves still faces that point's waypoint -- a limit filled on its way down to the bar's low faces the low, a short filled on its way up faces the high -- and the scan that started strictly after the point's phase measured a path the book had not finished; the waypoints the path already passed stay out, as `fx_roll_margin_check_at`'s continuous roll measures from its segment's origin. Nothing in it knows a venue. Rule 2 (b), knob: none. What moved: a post-fill point after a fill matched on a segment now measures that segment's destination waypoint -- on a high-first bar O100 H101 L80 C81 a buy limit at 95 measures 80 where it measured the close 81, and a low-first bar or a short filled on its way up is called where nothing was (`tests/test_native_margin_post_fill_path.cpp`, kernel-only); a fill at a waypoint's own price, the open's included, measures what it did. TradingView books that call on the fill's bar (lab tv tapes `tests/fixtures/margin_entry_bar/pm2-m7-lim-*` at the low, `pm2-m7-slim-*` and `pm2-m7-s1lim-*` at the high, 12 of 12; with the adapter's admissions `pm2-m7-poocl-*` and `pm2-m7b-lim-*` too), as `ab9714be`'s entry-bar suffix did; the differential's M7-B pin moved to that answer, 10 @80. One v19 witness value moved with it: `tests/test_publication_witness.cpp` Config03, a short filled on its way up a bar, now called at that bar's high (461a919ebdcd41e9 -> 4593ef08ab33e045); the corpus is 312/312, moved 0. | | the samples of a retained lower bar -- `NativeExecutionConsumer::deliver_intrabar_script` (`src/native_execution_consumer.cpp`): a bar the direct four-sample walk declines (a leg of zero length, an open at its own low) is walked through its own four turning points, the repeated one included, where the general `ENDPOINTS` sampler filled the missing point uniformly; a bar that traded nothing at one price (volume 0, open = high = low = close) is one discrete point, walked once | every host with a `lower_tf` path; the Pine adapter's magnified path marks a chart bar's own open and close this way (R5 lane MAG-INTRABAR, `source::tradingview_magnifier_bars`, `src/source/magnifier_intrabars.cpp`) | **a mechanism correction** (R5 lane MAG-INTRABAR). Rule 2 (a), mechanism: a retained bar supplies its own four ticks, and every sample is a check point of the margin model since R5 lane PAR-MARGIN, so a sample between them measured -- and a request matched at -- a price the bar did not print at that point of its path; a bar that traded nothing is one quote with no path, and its four coincident samples were three path points, three phases and three margin checks that are not there. Nothing in it knows a venue: the rule reads the bar's own prices and volume. Rule 2 (b), knob: none. TradingView's magnified broker walks an intrabar as its four ticks whatever their values (`tests/fixtures/intrabar_margin/pm2-i3-pooc-mag-0402-2000`: its first call is booked at an intrabar that opens at its low). What moved: an open-at-its-low bar's second mark is its low, 60, where the sampler put 77.5, and the call armed at that open is booked there; a zero-volume one-price bar gives one mark where it gave four (`tests/test_native_intrabar_turning_points.cpp`, kernel-only), and the sampler test's lower-path expectation moved with it (`tests/test_magnifier_ordered_sampler.cpp`: a sub-bar with a zero-length leg walks its corners). No witness value moved; the corpus runs no lower path (312/312, moved 0). | -| the presented bar -- `current_bar_`, `bar_index_`, `prev_bar_timestamp_` (`include/pineforge/engine.hpp`) | the kernel at every callback point: the consumer presents the bar at the point's instant (`src/native_execution_consumer.cpp`). For the body it runs, the Pine host then presents its own script bar. `scheduler_publish_source_bar` and `scheduler_publish_suppressed_tail` write the whole `current_bar_`, stamped at the bar's label (`current_bar_ = bar` `pine_strategy_host.cpp:2304` and `:2417`). They also write the slot's `bar_index_`, restored after a temporary publication, and each published slot's `prev_bar_timestamp_`. `PineScheduler::input` holds `bar_index_` at the input's own index for one input (`InputBarIndexScope` `pine_scheduler_native.cpp:371`) | **retained, the host's view; the fourth audit's E21 check e**. Execution reads its cursor, not this bar: `execute_current` converts at its cursor's rate and neither writes nor restores the host's clock (Section B's row). The broker-state hash folds none of the three, and `scripts/broker_state_hash_waivers.txt` waives `current_bar_` (a convenience copy) and `bar_index_` (a reconstructed cursor). While the Pine body runs, the host's bar is what these read: generated code's `time` and OHLC reads, the presented-clock readers `open_trade_profit` and `active_account_currency_fx`, and `strategy_stream_state_hash`, which folds `current_bar_` and `prev_bar_timestamp_` (`src/engine_stream.cpp`) | -| the run's words on the codegen-ABI class -- `input_tf_`, `script_tf_`, `script_tf_seconds_`, `last_bar_index_`, `last_bar_time_`, `qty_step_` (`include/pineforge/engine.hpp`) | the Pine host alone. `scheduler_prepare_script_run` sets the timeframe words and the feed's last bar from its scheduler's run spec (`script_tf_seconds_` `pine_strategy_host.cpp:1439`). `apply_realtime_tail_horizon` narrows the last bar to a live probe's horizon, and `prepare_native_begin` copies the symbol's `qty_step` | **retained as the Pine host's run facts**. Generated code reads the timeframe words. The kernel reads them for the report (`src/engine_report.cpp`), the implicit chart feed and the chart-day partition (`src/engine_aux_security.cpp`), all of which a bare host reaches with them empty. A bare host's timeframes are its `NativeRunSpec`'s, which the consumer parses into its own. `last_bar_index_`, `last_bar_time_` and `qty_step_` sit in the hashed region and are waived as feed and symbol metadata; the timeframe words sit outside it. `script_tf_seconds_`'s comment used to place `script_tf_`'s writers in `engine_run.cpp`; it now names the one writer (R5 lane H-DOCGATES) | -| the security-evaluator registry the Pine host fills -- `security_eval_states_`, `security_input_tf_`, `security_next_input_ms_` (`include/pineforge/engine.hpp`) | the Pine host. `declare_security_sites_to_kernel` hands a routable run's sites to the kernel and empties the registry (`security_eval_states_` `pine_strategy_host.cpp:1699`). `scheduler_prepare_security_sequence` sets the input timeframe and resets the next-input cursor, and the auxiliary drive and the scheduler advance that cursor (`src/source/pine_aux_security.cpp`, `src/source/pine_scheduler.cpp`) | **retained by design §2.ii row m**: the kernel keeps the registry, the aggregator and one generic step, and the Pine semantics run in `src/source/pine_security_eval.cpp`. `security_next_input_ms_` is waived: the source scheduler's deferred-boundary state is hashed on the source side. `strategy_stream_state_hash` folds the registry | +| the presented bar -- `current_bar_`, `bar_index_`, `prev_bar_timestamp_` (`include/pineforge/engine.hpp`) | the kernel at every callback point: the consumer presents the bar at the point's instant (`src/native_execution_consumer.cpp`). For the body it runs, the Pine host then presents its own script bar. `scheduler_publish_source_bar` and `scheduler_publish_suppressed_tail` write the whole `current_bar_`, stamped at the bar's label (`current_bar_ = bar` `pine_strategy_host.cpp:2333` and `:2463`). They also write the slot's `bar_index_`, restored after a temporary publication, and each published slot's `prev_bar_timestamp_`. `PineScheduler::input` holds `bar_index_` at the input's own index for one input (`InputBarIndexScope` `pine_scheduler_native.cpp:427`) | **retained, the host's view; the fourth audit's E21 check e**. Execution reads its cursor, not this bar: `execute_current` converts at its cursor's rate and neither writes nor restores the host's clock (Section B's row). The broker-state hash folds none of the three, and `scripts/broker_state_hash_waivers.txt` waives `current_bar_` (a convenience copy) and `bar_index_` (a reconstructed cursor). While the Pine body runs, the host's bar is what these read: generated code's `time` and OHLC reads, the presented-clock readers `open_trade_profit` and `active_account_currency_fx`, and `strategy_stream_state_hash`, which folds `current_bar_` and `prev_bar_timestamp_` (`src/engine_stream.cpp`) | +| the run's words on the codegen-ABI class -- `input_tf_`, `script_tf_`, `script_tf_seconds_`, `last_bar_index_`, `last_bar_time_`, `qty_step_` (`include/pineforge/engine.hpp`) | the Pine host alone. `scheduler_prepare_script_run` sets the timeframe words and the feed's last bar from its scheduler's run spec (`script_tf_seconds_` `pine_strategy_host.cpp:1468`). `apply_realtime_tail_horizon` narrows the last bar to a live probe's horizon, and `prepare_native_begin` copies the symbol's `qty_step` | **retained as the Pine host's run facts**. Generated code reads the timeframe words. The kernel reads them for the report (`src/engine_report.cpp`), the implicit chart feed and the chart-day partition (`src/engine_aux_security.cpp`), all of which a bare host reaches with them empty. A bare host's timeframes are its `NativeRunSpec`'s, which the consumer parses into its own. `last_bar_index_`, `last_bar_time_` and `qty_step_` sit in the hashed region and are waived as feed and symbol metadata; the timeframe words sit outside it. `script_tf_seconds_`'s comment used to place `script_tf_`'s writers in `engine_run.cpp`; it now names the one writer (R5 lane H-DOCGATES) | +| the security-evaluator registry the Pine host fills -- `security_eval_states_`, `security_input_tf_`, `security_next_input_ms_` (`include/pineforge/engine.hpp`) | the Pine host. `declare_security_sites_to_kernel` hands a routable run's sites to the kernel and empties the registry (`security_eval_states_` `pine_strategy_host.cpp:1728`). `scheduler_prepare_security_sequence` sets the input timeframe and resets the next-input cursor, and the auxiliary drive and the scheduler advance that cursor (`src/source/pine_aux_security.cpp`, `src/source/pine_scheduler.cpp`) | **retained by design §2.ii row m**: the kernel keeps the registry, the aggregator and one generic step, and the Pine semantics run in `src/source/pine_security_eval.cpp`. `security_next_input_ms_` is waived: the source scheduler's deferred-boundary state is hashed on the source side. `strategy_stream_state_hash` folds the registry | | the auxiliary-feed seams -- `source_aux_security_feed_enabled`, `source_aux_security_input_view` (`include/pineforge/engine.hpp`, behind `PINEFORGE_HAS_AUX_SECURITY_FEED_V1`) -- and `last_error_` | the kernel declares both seams with an inert default (disabled, no bars: `src/engine_consumer.cpp`). It asks them once, in `prepare_native_security_feeds` (`src/engine_aux_security.cpp`), for the auxiliary slice it feeds the `request.security` evaluators. The Pine host overrides both from `strategy_set_aux_security_feed`'s bars (`src/source/pine_aux_security.cpp`). It writes `last_error_` when that setter refuses its input and when its run's preparation throws | **retained with the `auxiliary_feed` adapter-policy ruling above**. The Pine host drives its own auxiliary slice (lane N7's three measurements), and these seams are how that slice reaches the kernel's feed preparation. A bare host declares `NativeRunSpec::auxiliary_feed` and never meets them. `last_error_` is the handle's one error slot, which any layer that refuses a call writes | | `source_stream_entry_comment` (`include/pineforge/engine.hpp`) | nobody. The kernel's default (`src/engine_consumer.cpp`) and the Pine host's override (`src/source/pine_strategy_host.cpp`) both do nothing, so `stream_observe_entry` keeps the lot's own comment (`src/engine_stream.cpp`) | **dead, kept for the script ABI**. It is a virtual of `BacktestEngine`, which every generated strategy derives from, so deleting it moves every generated script's vtable: an `engine_script_run` epoch. It goes at the next one (`docs/native-refactor-progress.md`, "Scheduled for the next script-ABI epoch") | | the magnifier report facts -- `bar_magnifier_enabled_`, `diag_magnifier_sub_bars_processed_`, `diag_magnifier_sample_ticks_processed_` (`include/pineforge/engine.hpp`) | the kernel, from the run spec's intrabar path and its driver statistics (`src/native_execution_consumer.cpp`). The Pine host writes them again at every bar callback, from its scheduler's magnifier switch and the context's driver statistics (`on_native_bar_open`, `on_native_bar`) | **retained, report-only**: the report's `bar_magnifier_enabled` and magnifier totals (`src/engine_report.cpp`), outside the hashed region. The kernel publishes them for a bare host itself (the C-surface row "kernel-owned report facts" above), so the Pine host's second write is an adapter-thinness question, not a boundary one | -| the timeframe report facts -- `diag_script_tf_ratio_`, `diag_needs_aggregation_` (`include/pineforge/engine.hpp`) | the Pine host alone. Its scheduler's run begin stores the ratio of the run spec's script and input timeframes and whether the run aggregates (`diag_script_tf_ratio_` `pine_scheduler_native.cpp:143`, in `PineScheduler::run_begin`), as the retired scheduler's run did before #254. A bare host's report reads 0 for both: the kernel does not write them | **retained, report-only** (R5 lane FIX-E1E2): the report's `script_tf_ratio` and `needs_aggregation` (`src/engine_report.cpp`), outside the hashed region. Execution never reads them | -| `broker_fill_event_seq_` (`include/pineforge/engine.hpp`, in the hashed region) | the Pine host alone, one step per applied broker instruction (`broker_fill_event_seq_` `pine_strategy_host.cpp:696`, in `on_native_applied`); the kernel only resets and folds it | **retained, hashed: the adapter's counter on the kernel's class**. Its placement facts (`signal_close_mc_fill_seq`) compare against it in fill-time gates that cross a bar; the execution authority is the kernel's own ordinal. It is folded into the broker-state hash (`src/engine_state_hash.cpp`), so moving it to the source layer's hash extension moves every run's hash: an epoch decision, not a docs change | -| `broker_state_hashes_` (`include/pineforge/engine.hpp`) | the kernel under `KernelRecorded` (`src/native_execution_consumer.cpp`), and the Pine host at each script point it marks while recording is on (`scheduler_record_broker_hash` `pine_strategy_host.cpp:2444`) | **retained, per driving mode** (design Q4): the report's per-bar hash series, waived as append-only report output that execution never consults | -| the position book's mirror -- `position_qty_`, `position_entry_price_`, `position_entry_count_`, `pyramid_entries_` (`include/pineforge/engine.hpp`, all hashed) | the kernel books and settles them. The Pine host writes them in two places. After every applied event it erases each lot at or below `internal::kQtyEpsilon` and recomputes the position's size, average price and count (`pyramid_entries_` `pine_strategy_host.cpp:660-686`, in `on_native_applied`, restating `ab9714be`'s partial-exit settlement). Before every script body it overwrites `position_entry_count_` with its own entry-slot count (`source_entry_slot_count` `pine_strategy_host.cpp:2365`) | **OPEN, two rulings nobody has made**. The dust sweep erases a lot inside a live book, which the fourth audit found with neither a row nor a tape (its item 32): lane H-THIN's A4-DUST. The count overwrite gives one hashed member two writers with two meanings: the kernel's lots in the current direction, Pine's opened entry slots. It is recorded here first and has no lane yet | -| fields of the kernel's rows the Pine host amends -- `PyramidEntry::time`, `Trade::entry_time`, `Trade::exit_time`, `Trade::close_cause`, `Trade::exit_id`, `Trade::exit_comment`, `Trade::entry_incarnation`, `Trade::exit_from_bracket`, and on a global exit's re-cut rows `Trade::qty`, `Trade::pnl`, `Trade::pnl_pct`, `Trade::commission`, `Trade::max_runup`, `Trade::max_drawdown` (`include/pineforge/engine.hpp`) | the Pine host, inside the applied notification that booked them or at a bar callback. On an aggregated or magnified chart it dates their fills at the chart bar's open (`on_native_applied`; design row AG2), the entry time of a row a later execution of the same pass booked from the lot included (R5 lane FIX-E1E2); their bar indices are the kernel's own script-bar indices, which it books for every host (design row AG1, R5 lane K-IDX), so the host writes no index. It writes no excursion of its own: since R5 lane H-THIN the kernel's sampler books every Pine lot's run-up and drawdown, as it does for any host that keeps the `owns_lot_excursions()` default. The one exception is the re-cut of a global `strategy.exit` fill. When one fill of an exit that names no entry closes the lots of two or more entry ids and leaves the book flat, the host re-cuts that fill's rows into TradingView's children (`order_global_exit_children`, tests/fixtures/global_exit_children). Under `close_entries_rule = "ANY"` it permutes the rows. Otherwise each child is a transaction of its lot's quantity settled oldest lot first, and each slice is a row carrying its share of quantity, P&L, excursions and ten-significant-digit fees; the kernel's book and fills are untouched. It states the close cause from its own labels, clearing the kernel's `Bracket` on its rows (R5 lanes L12 and F3). It relabels an intraday-loss close's `exit_id` and `exit_comment`, swaps `entry_incarnation` on the short-seed report rows (`project_short_seed_report_rows`), and sets `exit_from_bracket` by order family (`exit_from_bracket` `pine_strategy_host.cpp:1238`, in `adapter_label_bracket_trades`) | **retained: the source layer's labels and report dates on the kernel's rows**. A closed row is final for the hash only once its notification returns (`closed_rows_digest`), so the amendments made inside it are folded as the host leaves them. The hash folds a lot's time, entry bar index and two excursions (`src/engine_state_hash.cpp`), and a closed row's entry and exit times, prices, quantity and P&L (`fold_closed_row` `src/native_execution_consumer.cpp`). The labels are not folded. The kernel reads `exit_from_bracket` in one place, `closed_trade_close_cause`, after the row's recorded cause; no kernel path sets it | +| the timeframe report facts -- `diag_script_tf_ratio_`, `diag_needs_aggregation_` (`include/pineforge/engine.hpp`) | the Pine host alone. Its scheduler's run begin stores the ratio of the run spec's script and input timeframes and whether the run aggregates (`diag_script_tf_ratio_` `pine_scheduler_native.cpp:185`, in `PineScheduler::run_begin`), as the retired scheduler's run did before #254. A bare host's report reads 0 for both: the kernel does not write them | **retained, report-only** (R5 lane FIX-E1E2): the report's `script_tf_ratio` and `needs_aggregation` (`src/engine_report.cpp`), outside the hashed region. Execution never reads them | +| `broker_fill_event_seq_` (`include/pineforge/engine.hpp`, in the hashed region) | the Pine host alone, one step per applied broker instruction (`broker_fill_event_seq_` `pine_strategy_host.cpp:707`, in `on_native_applied`); the kernel only resets and folds it | **retained, hashed: the adapter's counter on the kernel's class**. Its placement facts (`signal_close_mc_fill_seq`) compare against it in fill-time gates that cross a bar; the execution authority is the kernel's own ordinal. It is folded into the broker-state hash (`src/engine_state_hash.cpp`), so moving it to the source layer's hash extension moves every run's hash: an epoch decision, not a docs change | +| `broker_state_hashes_` (`include/pineforge/engine.hpp`) | the kernel under `KernelRecorded` (`src/native_execution_consumer.cpp`), and the Pine host at each script point it marks while recording is on (`scheduler_record_broker_hash` `pine_strategy_host.cpp:2742`) | **retained, per driving mode** (design Q4): the report's per-bar hash series, waived as append-only report output that execution never consults | +| the position book's mirror -- `position_qty_`, `position_entry_price_`, `position_entry_count_`, `pyramid_entries_` (`include/pineforge/engine.hpp`, all hashed) | the kernel books and settles them. The Pine host writes them in two places. After every applied event it erases each lot at or below `internal::kQtyEpsilon` and recomputes the position's size, average price and count (`pyramid_entries_` `pine_strategy_host.cpp:660-686`, in `on_native_applied`, restating `ab9714be`'s partial-exit settlement). Before every script body it overwrites `position_entry_count_` with its own entry-slot count (`source_entry_slot_count` `pine_strategy_host.cpp:2400`) | **OPEN, two rulings nobody has made**. The dust sweep erases a lot inside a live book, which the fourth audit found with neither a row nor a tape (its item 32): lane H-THIN's A4-DUST. The count overwrite gives one hashed member two writers with two meanings: the kernel's lots in the current direction, Pine's opened entry slots. It is recorded here first and has no lane yet | +| fields of the kernel's rows the Pine host amends -- `PyramidEntry::time`, `Trade::entry_time`, `Trade::exit_time`, `Trade::close_cause`, `Trade::exit_id`, `Trade::exit_comment`, `Trade::entry_incarnation`, `Trade::exit_from_bracket`, and on a global exit's re-cut rows `Trade::qty`, `Trade::pnl`, `Trade::pnl_pct`, `Trade::commission`, `Trade::max_runup`, `Trade::max_drawdown` (`include/pineforge/engine.hpp`) | the Pine host, inside the applied notification that booked them or at a bar callback. On an aggregated or magnified chart it dates their fills at the chart bar's open (`on_native_applied`; design row AG2), the entry time of a row a later execution of the same pass booked from the lot included (R5 lane FIX-E1E2); their bar indices are the kernel's own script-bar indices, which it books for every host (design row AG1, R5 lane K-IDX), so the host writes no index. It writes no excursion of its own: since R5 lane H-THIN the kernel's sampler books every Pine lot's run-up and drawdown, as it does for any host that keeps the `owns_lot_excursions()` default. The one exception is the re-cut of a global `strategy.exit` fill. When one fill of an exit that names no entry closes the lots of two or more entry ids and leaves the book flat, the host re-cuts that fill's rows into TradingView's children (`order_global_exit_children`, tests/fixtures/global_exit_children). Under `close_entries_rule = "ANY"` it permutes the rows. Otherwise each child is a transaction of its lot's quantity settled oldest lot first, and each slice is a row carrying its share of quantity, P&L, excursions and ten-significant-digit fees; the kernel's book and fills are untouched. It states the close cause from its own labels, clearing the kernel's `Bracket` on its rows (R5 lanes L12 and F3). It relabels an intraday-loss close's `exit_id` and `exit_comment`, swaps `entry_incarnation` on the short-seed report rows (`project_short_seed_report_rows`), and sets `exit_from_bracket` by order family (`exit_from_bracket` `pine_strategy_host.cpp:1267`, in `adapter_label_bracket_trades`) | **retained: the source layer's labels and report dates on the kernel's rows**. A closed row is final for the hash only once its notification returns (`closed_rows_digest`), so the amendments made inside it are folded as the host leaves them. The hash folds a lot's time, entry bar index and two excursions (`src/engine_state_hash.cpp`), and a closed row's entry and exit times, prices, quantity and P&L (`fold_closed_row` `src/native_execution_consumer.cpp`). The labels are not folded. The kernel reads `exit_from_bracket` in one place, `closed_trade_close_cause`, after the row's recorded cause; no kernel path sets it | | the closed-row counters -- `win_trades_count_`, `loss_trades_count_`, `eventrades_count_`, `gross_profit_sum_`, `gross_loss_sum_` (`include/pineforge/engine.hpp`) | the kernel, once per row it books (`engine_orders.cpp`); the Pine host re-counts them over the children's rows when it re-cuts a global exit's fill into more rows than the kernel booked (`order_global_exit_children`), so `strategy.wintrades`, `strategy.losstrades` and `strategy.eventrades` count the rows the report shows | **retained: the counts follow the report's rows**. Hashed (`src/engine_state_hash.cpp`) and folded as the host leaves them; the re-cut runs inside the applied notification, before the next hash read. `net_profit_sum_` is not re-counted: the equity stays the kernel's | | the range-end report -- `range_end_trades_`, the extremes `max_equity_`, `min_equity_`, `max_drawdown_`, `max_runup_`, and `stream_warmup_mode_` (`include/pineforge/engine.hpp`) | the Pine host: `scheduler_record_range_end` produces report-only terminal rows and preserves ordinary stored marks in REALTIME; `PineStrategyHost::present_report` shapes only the caller-owned report copy through the engine's generic default-no-op presentation seam, before metrics are derived. Historical batch keeps its existing terminal re-mark and extreme fold. Complete-prefix stream reports equal batch from the first realtime script bar; an open position at zero realtime bars retains the warmup mark, and mid-bar reads are host presentation, not a pinned batch boundary. The stream hash contract changes: hashed extremes no longer fold hypothetical terminal marks in REALTIME, so stream and batch terminal hashes need not match; restart/replay binds the exact library digest | **adapter-policy, the `report_open_position_at_end` row above** (design RP5 and §3.7 part 2): report shape, not a mark-to-market row, and the kernel's run-end producer is gated out of every Pine run. The extremes are hashed; the report rows are waived as report-only | -| `host_mutation_guard_inert_` (`include/pineforge/engine.hpp`) | the Pine host, once, in its constructor (`host_mutation_guard_inert_ = true` `pine_strategy_host.cpp:154`) | **retained, the host's declared choice**: an in-run call of a frozen C setter is a no-op on a Pine handle, as `ab9714be`'s legacy consumer refused it, and throws on a native host. Outside the hashed region | +| `host_mutation_guard_inert_` (`include/pineforge/engine.hpp`) | the Pine host, once, in its constructor (`host_mutation_guard_inert_ = true` `pine_strategy_host.cpp:156`) | **retained, the host's declared choice**: an in-run call of a frozen C setter is a no-op on a Pine handle, as `ab9714be`'s legacy consumer refused it, and throws on a native host. Outside the hashed region | | `fold_exit_trail_peak_` (`include/pineforge/engine.hpp`, hashed) | nobody but the per-run reset to NaN (`reset_run_state`, `src/engine_run.cpp`) | **dead, kept by the hash** (design §3.7, "A dead branch found and reported"). Its readers test `isnan` and never take the branch, and the member's own comment says so. The broker-state hash folds it (`src/engine_state_hash.cpp`), so deleting it moves every run's hash and `BacktestEngine`'s layout: an epoch item | | `NativeHostCache` (`src/native_execution_consumer.hpp`) | the Pine adapter adopts its own subclass per run (`PineRunCache` in `src/source/pine_host_reads.hpp`, through `adopt_host_cache`). The consumer owns it, drops it at every run begin and never reads it | **retained as generic** (R5 lane PERF-P7): lookup state a host derives from its own bookkeeping and cannot keep in its own object, because a generated script's ABI fixes that object's layout. Nothing in it is hashed, recorded or read by the kernel, and the host verifies what it reads back. The header is not installed, so the in-tree source layer is its only user | @@ -710,13 +710,13 @@ Every line number in this section names its symbol on this tree, which | `NativeRunSpec` field | ruling | what the adapter runs instead | measurement and ruling of record | executed native consumers | |---|---|---|---|---| | `price_grid`, `grid_rounding` | **native-only** | TradingView's per-order-kind tick rules on top of `None`: `source_trigger_threshold` (`pine_adapter.cpp:343`), `source_level_on_price_grid` (`:329`), `nearest_tick` (`:250`) / `source_bar_fill_tick` (`:271`) / `directional_tick` (`:295`), behind the terms seam | R5-3 and design risk E7 ruled `None` for the adapter before the lane ran; lanes R7 and N13 measured the alternative anyway (raw levels submitted, `QuantizeFillsAndTriggers` with `HalfUp` declared). L8b closed the first blocker (no run aborts). The second has no remedy on either side: TradingView quantizes per order kind (stop and limit legs and a trail's activation on the quantized bar; the trail stop, the running best, stop-limit entries and the `calc_on_order_fills` cursors raw), the grid is one rule for the run, and the old N13 trial moved 30 pinned checks in 4 units; this is historical evidence, not a current-head result; a per-kind mask would spell that inconsistency into the kernel. The corpus cannot arbitrate (every probe runs a 0.01 tick on an on-grid feed; 5 of 312 differ in an engine-only column). Permanent witness: `tests/test_adapter_grid_relower.cpp`, whose section 5 pins the trail stop the grid fires a bar early. Design row PG and §3.6 | `examples/native/native_price_grid_strategy.cpp`, `examples/native/native_price_grid_c.c`, `tests/test_native_price_grid.cpp` | -| `risk` | **native-only** | all of `strategy.risk.*` but the direction: `update_risk_state` (`pine_adapter.cpp:17260`), `SourceDayLedger`, `submit_intraday_loss_close` (`pine_adapter.cpp:20627`), `chart_day_key` (`pine_adapter.cpp:17157`), `compat::pine::IntradayCap` with `IntradayOrderBudget` (337 lines) | Structural first: Pine's risk calls are per-bar statements, so a limit reaches the adapter on script bar 0, after `project()` (`pine_strategy_host.cpp:333`) and the kernel's `configure` (`pine_strategy_host.cpp:342`) have fixed and digested the spec. In substance (lane N12, `tests/test_adapter_risk_relower.cpp`, 62 checks over nine paired scenarios): the drawdown latch samples at the close only and still admits a reversal; the loss-day streak counts trades, not days; the intraday loss closes at the path's adverse extreme, refuses every placement and withdraws the book; the fill cap charges slots, transfers quota and closes at the bar's better extreme on the chart timezone's day. With the kernel seeded on the corpus, 3 of 4 cap probes diverge (3840 of 3916, 312 of 604, 2370 of 2384 rows; re-derived by `scripts/check_seeded_risk_experiment.sh` since R5 lane H-MEASURE). `strategy.risk.allow_entry_in` is the one rule that is the kernel's already (`allowed_open_directions`). Design §3.6 | `examples/native/native_risk_limits_strategy.cpp`, `examples/native/native_trail_risk_strategy.cpp`, `tests/test_native_risk_limits.cpp`, `tests/test_native_c_api.c` | -| `max_abs_units` | **adapter-policy** | `strategy.risk.max_position_size` as a gate on the LIVE book before the fill (`max_position_size` `pine_adapter.cpp:16268-16269`): an entry is refused once the book already holds the limit | design row MG2 (R5-1): the resulting-book cap is the generic one. Measured by N12's scenario `PS` in `tests/test_adapter_risk_relower.cpp`: two-unit entries against a limit of 3 leave the adapter at 4 and the kernel cap at 2 | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_run_spec.cpp` | -| `max_open_lots` | **adapter-policy** | Pine pyramiding is a per-cycle entry count in the adapter's command policy; a resting source entry must not consume a physical-lot cap before it fills, so `project()` leaves the cap unset (`max_open_lots` `pine_adapter.cpp:2792-2794`) | design row MG3 (R5-1); the contract comment in `project()`. Measured against TradingView by R5 lane H-MEASURE (`tests/test_pyramiding_count_differential.cpp`, 15 scenarios on three `lab tv` tapes): TradingView checks an entry once, at its first eligible point, against the trades then open, and does not check a resting entry again at its fill; the kernel's cap books 14 of the 15 as TradingView does, the adapter's per-cycle count 13 (it keeps a slot a close or another entry's exit drained; since lane W8A-SIGSTATE-1's R-B a resting LIMIT entry no longer counts, and one the count admitted fills over the cap, and since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap, as on TradingView's tapes). A recorded divergence: the retention stands pending a re-lowering ruling. R5 lane H-THIN tried the lowering and kept the count: on the kernel's cap four corpus probes moved away from TradingView, for want of the first-eligible-point rule (H-MEASURE's P4b); it fixed the reservation divergence measured beside it (P10), so a default `strategy.exit(from_entry)` under FIFO now reserves its entry's own quantity, the oldest lots first, as TradingView does | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_margin_model.cpp` | -| `initial_margin_fraction` | **adapter-policy** | TradingView's ten-significant-digit money admission against the signal-time tuple, answered as `AdmitWithHostMargin`; the `margin` model the adapter does declare is maintenance-only (`NativeMarginModel` `pine_adapter.cpp:2815-2823`, `margin.maintenance_long` `pine_adapter.cpp:2835`) | design row MG4 and the wave-4 ruling recorded in `project()`: a positive initial requirement would decline openings TradingView takes. Measured both ways by R5 lane H-MEASURE (`tests/test_adapter_margin_schedule_differential.cpp`, M11): the kernel's gate declines a 1x long whose entry fee the adapter admits (`InitialMargin`), and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times | `tests/test_native_precommit_view.cpp`, `tests/test_native_margin_model.cpp` | -| `report_open_position_at_end` | **adapter-policy** | The Pine host owns TradingView's range-end shape: batch re-marks the stored final point and re-folds extremes, while REALTIME retains ordinary marks and `PineStrategyHost::present_report` re-marks only the caller-owned copy before generic metric derivation; the stream hash excludes that terminal extreme fold (`scheduler_record_range_end` pine_strategy_host.cpp:1932): report shape, not a mark-to-market row (`KernelRecordedAtHostMarks` pine_adapter.cpp:2790) | design row RP5; the kernel reads the field under `KernelRecorded` only, which `scripts/check_adapter_spec_shadowing.py` gates | `examples/native/native_sized_report_strategy.cpp`, `tests/test_native_report_truth.cpp` | +| `risk` | **native-only** | all of `strategy.risk.*` but the direction: `update_risk_state` (`pine_adapter.cpp:17268`), `SourceDayLedger`, `submit_intraday_loss_close` (`pine_adapter.cpp:20635`), `chart_day_key` (`pine_adapter.cpp:17165`), `compat::pine::IntradayCap` with `IntradayOrderBudget` (337 lines) | Structural first: Pine's risk calls are per-bar statements, so a limit reaches the adapter on script bar 0, after `project()` (`pine_strategy_host.cpp:335`) and the kernel's `configure` (`pine_strategy_host.cpp:344`) have fixed and digested the spec. In substance (lane N12, `tests/test_adapter_risk_relower.cpp`, 62 checks over nine paired scenarios): the drawdown latch samples at the close only and still admits a reversal; the loss-day streak counts trades, not days; the intraday loss closes at the path's adverse extreme, refuses every placement and withdraws the book; the fill cap charges slots, transfers quota and closes at the bar's better extreme on the chart timezone's day. With the kernel seeded on the corpus, 3 of 4 cap probes diverge (3840 of 3916, 312 of 604, 2370 of 2384 rows; re-derived by `scripts/check_seeded_risk_experiment.sh` since R5 lane H-MEASURE). `strategy.risk.allow_entry_in` is the one rule that is the kernel's already (`allowed_open_directions`). Design §3.6 | `examples/native/native_risk_limits_strategy.cpp`, `examples/native/native_trail_risk_strategy.cpp`, `tests/test_native_risk_limits.cpp`, `tests/test_native_c_api.c` | +| `max_abs_units` | **adapter-policy** | `strategy.risk.max_position_size` as a gate on the LIVE book before the fill (`max_position_size` `pine_adapter.cpp:16276-16277`): an entry is refused once the book already holds the limit | design row MG2 (R5-1): the resulting-book cap is the generic one. Measured by N12's scenario `PS` in `tests/test_adapter_risk_relower.cpp`: two-unit entries against a limit of 3 leave the adapter at 4 and the kernel cap at 2 | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_run_spec.cpp` | +| `max_open_lots` | **adapter-policy** | Pine pyramiding is a per-cycle entry count in the adapter's command policy; a resting source entry must not consume a physical-lot cap before it fills, so `project()` leaves the cap unset (`max_open_lots` `pine_adapter.cpp:2798-2800`) | design row MG3 (R5-1); the contract comment in `project()`. Measured against TradingView by R5 lane H-MEASURE (`tests/test_pyramiding_count_differential.cpp`, 15 scenarios on three `lab tv` tapes): TradingView checks an entry once, at its first eligible point, against the trades then open, and does not check a resting entry again at its fill; the kernel's cap books 14 of the 15 as TradingView does, the adapter's per-cycle count 13 (it keeps a slot a close or another entry's exit drained; since lane W8A-SIGSTATE-1's R-B a resting LIMIT entry no longer counts, and one the count admitted fills over the cap, and since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap, as on TradingView's tapes). A recorded divergence: the retention stands pending a re-lowering ruling. R5 lane H-THIN tried the lowering and kept the count: on the kernel's cap four corpus probes moved away from TradingView, for want of the first-eligible-point rule (H-MEASURE's P4b); it fixed the reservation divergence measured beside it (P10), so a default `strategy.exit(from_entry)` under FIFO now reserves its entry's own quantity, the oldest lots first, as TradingView does | `tests/test_native_resting_matching_contract.cpp`, `tests/test_native_margin_model.cpp` | +| `initial_margin_fraction` | **adapter-policy** | TradingView's ten-significant-digit money admission against the signal-time tuple, answered as `AdmitWithHostMargin`; the `margin` model the adapter does declare is maintenance-only (`NativeMarginModel` `pine_adapter.cpp:2826-2834`, `margin.maintenance_long` `pine_adapter.cpp:2841`) | design row MG4 and the wave-4 ruling recorded in `project()`: a positive initial requirement would decline openings TradingView takes. Measured both ways by R5 lane H-MEASURE (`tests/test_adapter_margin_schedule_differential.cpp`, M11): the kernel's gate declines a 1x long whose entry fee the adapter admits (`InitialMargin`), and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times | `tests/test_native_precommit_view.cpp`, `tests/test_native_margin_model.cpp` | +| `report_open_position_at_end` | **adapter-policy** | The Pine host owns TradingView's range-end shape: batch re-marks the stored final point and re-folds extremes, while REALTIME retains ordinary marks and `PineStrategyHost::present_report` re-marks only the caller-owned copy before generic metric derivation; the stream hash excludes that terminal extreme fold (`scheduler_record_range_end` pine_strategy_host.cpp:1961): report shape, not a mark-to-market row (`KernelRecordedAtHostMarks` pine_adapter.cpp:2796) | design row RP5; the kernel reads the field under `KernelRecorded` only, which `scripts/check_adapter_spec_shadowing.py` gates | `examples/native/native_sized_report_strategy.cpp`, `tests/test_native_report_truth.cpp` | | `open_bar_view` | **adapter-policy** | `Complete`: TradingView's bar-open scheduling and its `calc_on_order_fills` callback read the whole script bar (`pine_adapter.cpp:2758-2760`) | design rows CT4 and E5: the open-only view is opt-in because the adapter needs the full bar | `examples/native/native_calc_on_fills_strategy.cpp`, `tests/test_native_calc_timing.cpp` | -| `subscriptions` | **adapter-hook** | `declare_timeframe_subscriptions` from the begin-time hook (`pine_strategy_host.cpp:1667`): a plain `request.security` site is a kernel subscription | lane R3b over the L6c hook: 21 of the corpus's 23 `request.security` probes run their sites on the kernel, byte-identical. The predicate (`declare_security_sites_to_kernel` `pine_strategy_host.cpp:1658`) declares every site of a run or none. A run keeps all of its sites on the source evaluator when it is a stream's warmup, runs the bar magnifier, cuts its range start to na (KI-55), projects historical lookahead, installs an auxiliary feed, aggregates its chart or has no detected timeframe. It keeps them there too when any one site uses `lookahead_on`, `ticker.heikinashi`, a lower timeframe (requested, emulated, read from the input, or as an array), a publication gate, or a calling-bar close or open latch; when a forex or CFD intraday chart without a native feed asks for `D`; when a site asks for N days, whose bars TradingView counts in the symbol's trading dates from each year's first (`tests/test_security_multi_day_buckets_tapes.cpp`), not on the kernel's calendar-day grid; when the registry's `sec_id`s are not dense from 0; and when the kernel refuses the declaration. R5 lane H-MEASURE pins each reachable condition with what it changes (`tests/test_adapter_security_route_conditions.cpp`, a site through the evaluator against the same site through a bare kernel host): on the feeds measured the magnifier and an aggregated chart change no delivered value, while the range-start cut, the projection's incomplete tail, `lookahead_on` and the OTC pins do; the four lower-timeframe flags, the publish gate and the calling-bar latches are subsumed by other clauses, and the empty or undetected timeframe checks are unreachable after begin-time validation | `examples/native/native_htf_strategy.cpp`, `tests/test_native_htf_subscriptions.cpp` | +| `subscriptions` | **adapter-hook** | `declare_timeframe_subscriptions` from the begin-time hook (`pine_strategy_host.cpp:1696`): a plain `request.security` site is a kernel subscription | lane R3b over the L6c hook: 21 of the corpus's 23 `request.security` probes run their sites on the kernel, byte-identical. The predicate (`declare_security_sites_to_kernel` `pine_strategy_host.cpp:1687`) declares every site of a run or none. A run keeps all of its sites on the source evaluator when it is a stream's warmup, runs the bar magnifier, cuts its range start to na (KI-55), projects historical lookahead, installs an auxiliary feed, aggregates its chart or has no detected timeframe. It keeps them there too when any one site uses `lookahead_on`, `ticker.heikinashi`, a lower timeframe (requested, emulated, read from the input, or as an array), a publication gate, or a calling-bar close or open latch; when a forex or CFD intraday chart without a native feed asks for `D`; when a site asks for N days, whose bars TradingView counts in the symbol's trading dates from each year's first (`tests/test_security_multi_day_buckets_tapes.cpp`), not on the kernel's calendar-day grid; when the registry's `sec_id`s are not dense from 0; and when the kernel refuses the declaration. R5 lane H-MEASURE pins each reachable condition with what it changes (`tests/test_adapter_security_route_conditions.cpp`, a site through the evaluator against the same site through a bare kernel host): on the feeds measured the magnifier and an aggregated chart change no delivered value, while the range-start cut, the projection's incomplete tail, `lookahead_on` and the OTC pins do; the four lower-timeframe flags, the publish gate and the calling-bar latches are subsumed by other clauses, and the empty or undetected timeframe checks are unreachable after begin-time validation | `examples/native/native_htf_strategy.cpp`, `tests/test_native_htf_subscriptions.cpp` | | `auxiliary_feed` | **adapter-policy** | the adapter's own auxiliary drive: the chart-slice mapping and the deferred first bucket (`src/source/pine_aux_security.cpp`) | lane N7, retained on three measurements: 0 of 312 corpus probes install an auxiliary feed; TradingView's chart slice leaves pre-range coverage inert where the kernel folds it by time, and evaluates after the bar's matching pass where the kernel delivers before it (`tests/test_native_auxiliary_feed_twin.cpp`, rows B and C). Design §2.iv | `tests/test_native_auxiliary_feed.cpp`, `tests/test_native_auxiliary_feed_stream.cpp` | | `quantity_tolerance` | **adapter-policy** | TradingView's `1e-10` quantity rules on top of the exact default: a close within `1e-10` of the held position is a `Flatten`, a batched process-orders-on-close (POOC) close ending within `internal::kQtyEpsilon` (`1e-10`) of an interior lot boundary is one selected `Flatten` of whole lots (`source_fifo_prefix_openings`); an ambiguous ordinary deferred FIFO `strategy.close` instead uses `HostSized` to authenticate the exact physical FIFO prefix sum for its `Reduce` at fill time, with that same tolerance (`source_fifo_close_units`, `src/source/pine_adapter.cpp`). Detection checks only the current prefix and prefixes reachable after already-pending same-bar closes, so non-ambiguous request shapes, hashes and readback remain unchanged. Same-bar market transactions, `immediately` closes and the ShortSeed placeholder remain out of scope, and after every fill `PineStrategyHost::on_native_applied` erases lots of at most `kQtyEpsilon` without a closing row (`src/source/pine_strategy_host.cpp`) | lane K-ULP4 (the owner-level ruling of audit X1, part b): opt-in and absent by default, so every adapter run is byte-identical by construction. The tolerance charges a snapped `Reduce` its request and books every dust lot it closes as a row, where the adapter's prefix `Flatten` is charged the lots it holds and its sweep books no row; declaring it would move the adapter's fills, ledgers and hashes, so whether the adapter can move onto it is a later lane's measurement. R5 lane H-THIN measured each of those comparisons (design §3.9b) and kept them: made exact, the sweep fails its own tape row (`test_pine_dust_sweep_paired`) | `tests/test_native_quantity_tolerance.cpp` | @@ -754,17 +754,17 @@ number names its symbol on this tree. | surface | ruling | reason | |---|---|---| | `cancel_all` (`native_host.hpp:1267`) / `cancel_where` answer a count, not a `CancelResult` per request (E12 f7) | **retained** | A bulk cancel is many commands in one call, and each withdrawn request records its own `CancelledEvent` with its own `CancelReason` — dependants of a cancelled owner included — in the history a host already polls (`strategy_native_events_v1`). That is the per-request answer; the count is the call's summary. A result vector would restate the history in a second, allocating shape that C could not take without a caller-sized array. The C spellings answer the same count (`strategy_native_cancel_all_v1` `native_c_api.h:2808`). | -| `cohort_add` (`native_host.hpp:1285`) / `cohort_remove` answer `void` (E12 f7) | **retained; the typed receipt is a named follow-up** | The kernel judges every enrolment into a `CohortReceipt` (Applied, InvalidHandle, UnknownOrigin, TerminalOrigin) and folds it into the continuation (`hash_cohort_receipt` `native_execution_consumer.cpp:906`, since v19 once, at the enrolment, into the running digest the continuation carries), so a replay that diverges there diverges in the hash. Nothing that matches or settles reads it: a roster is a relation the close reads at its match, and a refused enrolment is a member the close does not take. A host that needs the receipt AT the call needs `NativeStrategyHost::cohort_add_result` first, a kernel-header change outside this lane; its C spelling would then be an `_ext_v1` with a receipt word. Until then `strategy_native_cohort_add_v1`'s `PF_NATIVE_OK` means the enrolment was issued. | +| `cohort_add` (`native_host.hpp:1285`) / `cohort_remove` answer `void` (E12 f7) | **retained; the typed receipt is a named follow-up** | The kernel judges every enrolment into a `CohortReceipt` (Applied, InvalidHandle, UnknownOrigin, TerminalOrigin) and folds it into the continuation (`hash_cohort_receipt` `native_execution_consumer.cpp:907`, since v19 once, at the enrolment, into the running digest the continuation carries), so a replay that diverges there diverges in the hash. Nothing that matches or settles reads it: a roster is a relation the close reads at its match, and a refused enrolment is a member the close does not take. A host that needs the receipt AT the call needs `NativeStrategyHost::cohort_add_result` first, a kernel-header change outside this lane; its C spelling would then be an `_ext_v1` with a receipt word. Until then `strategy_native_cohort_add_v1`'s `PF_NATIVE_OK` means the enrolment was issued. | | `native_sized_units` (`native_host.hpp:1314`) / `native_liquidation_price` answer `std::optional` (E12 f7) | **retained** | Observation queries, not commands: each empty is documented cause by cause in its own comment (unconfigured, non-positive money or denominator, a below-one-step quotient; no margin model, no maintenance fraction for the side, a flat book, no finite solution), and every cause is a fact the host can read itself. A typed reason would name what the caller already knows. C answers `PF_NATIVE_ABSENT` and NaN (`strategy_native_liquidation_price_v1` `native_c_api.h:3087`). | | `pf_native_working_v1` without the leg's anchor or owner relation (E13 f4) | **carried** | The C++ host reads them off `NativeWorkingRequest::definition`, so the C readout's relation tail carries them: `anchor` (`native_c_api.h:1718`) through `arm_scope`, the request as the kernel holds it now (an armed leg reads ABSOLUTE with its installed level). | | `hash_source_extension` (`engine.hpp:411`) | **C++-only** | The deprecated spelling of `hash_host_extension`, kept so an existing C++ subclass compiles and folds unchanged. A C host has only ever had the current spelling, `on_hash_extension`. | | `NativeReportPolicy::KernelRecordedAtHostMarks` (`native_run_spec.hpp:64`) | **C++-only** (lane E22's ruling, restated) | Under it the host names each report point from inside its own callbacks, and the C table has no call that marks one: a C host could only declare a series nobody records. `pf_native_report_policy_e` (`native_c_api.h:661`) leaves its value unnamed, and the enum guard holds that exclusion. A C++ host now reaches the same kernel producer through `NativeStrategyHost::mark_native_report_point`; a mark outside a running HostMarks run returns false. | -| `strategy_set_path_order` (`pineforge.h:941`) clamps an out-of-range mode to AUTO | **retained** | A `void` ABI v4 setter cannot answer a refusal. Its values are now documented as `pf_native_path_order_e`'s (`native_c_api.h:904`); a C host that wants an unknown word refused declares `path_order` in `pf_native_run_spec_ext_v1`, which answers `PF_NATIVE_E_TAG`. | -| the enumerated `int32` fields of `pf_pending_order_v1_t` (`strategy_pending_order_get` `pineforge.h:1036`) | **no C names in the header; generator-owned** | The mirror is generated from the Pine source host's resting-order record by `scripts/gen_pending_order_mirror.py` (held by its `--check` source guard) and self-described at run time by `strategy_pending_order_layout`. Its enumerated fields are the source layer's order vocabulary, not a kernel enumeration. A C name for them belongs in that generator, which owns the struct, so that it cannot drift from the mirror — never hand-written beside it. | -| `strategy_configure_native_v1` (`c_abi.cpp:930`) versus `strategy_configure_native_ext_result_v1` (`native_c_host.cpp:4066`) | **retained legacy base plus typed additive route** | A base specification the kernel rejects still returns `-1` and latches `Failed`, preserving its frozen caller contract; a Ready misuse retains that Contract latch. The C boundary refuses arguments, a Running phase and a Completed hash hook without entering kernel configure, so a read cannot reconfigure its host. The extended result call validates before its first configure, writes `pf_native_spec_error_t` / `pf_native_spec_field_t`, and names reused-host session-key and run-number refusals; no old signature or layout moved. | +| `strategy_set_path_order` (`pineforge.h:948`) clamps an out-of-range mode to AUTO | **retained** | A `void` ABI v4 setter cannot answer a refusal. Its values are now documented as `pf_native_path_order_e`'s (`native_c_api.h:904`); a C host that wants an unknown word refused declares `path_order` in `pf_native_run_spec_ext_v1`, which answers `PF_NATIVE_E_TAG`. | +| the enumerated `int32` fields of `pf_pending_order_v1_t` (`strategy_pending_order_get` `pineforge.h:1096`) | **no C names in the header; generator-owned** | The mirror is generated from the Pine source host's resting-order record by `scripts/gen_pending_order_mirror.py` (held by its `--check` source guard) and self-described at run time by `strategy_pending_order_layout`. Its enumerated fields are the source layer's order vocabulary, not a kernel enumeration. A C name for them belongs in that generator, which owns the struct, so that it cannot drift from the mirror — never hand-written beside it. | +| `strategy_configure_native_v1` (`c_abi.cpp:980`) versus `strategy_configure_native_ext_result_v1` (`native_c_host.cpp:4066`) | **retained legacy base plus typed additive route** | A base specification the kernel rejects still returns `-1` and latches `Failed`, preserving its frozen caller contract; a Ready misuse retains that Contract latch. The C boundary refuses arguments, a Running phase and a Completed hash hook without entering kernel configure, so a read cannot reconfigure its host. The extended result call validates before its first configure, writes `pf_native_spec_error_t` / `pf_native_spec_field_t`, and names reused-host session-key and run-number refusals; no old signature or layout moved. | | `strategy_configure_native_fx_curve_v1` (`pineforge.h:546`) | **typed additive route** | `strategy_configure_native_fx_curve_ext_v1` writes the C twin of `NativeFxCurveValidation` and its offending index. The old `-1` route remains unchanged, while a C caller can now distinguish a decreasing timestamp, non-positive rate, allocation failure or wrong phase. | | `on_native_timeframe_bar` (`native_host.hpp:885`) and `NativeTimeframeBarContext::interval` (`native_host.hpp:809`) | **query routed** | The callback signature stays frozen. `strategy_native_timeframe_bar_interval_v1` is valid only during that callback and copies the kernel calendar interval without deriving it from the delivered bar or delivery timestamp. | -| `magnifier_sub_bars_total` / `magnifier_sample_ticks_total` (`engine_report.cpp:65-66`) | **kernel-owned report facts** | The generic intrabar driver already increments `NativeDriverStatistics::sub_bars_processed` / `sample_ticks_processed` for every retained path (`native_execution_consumer.cpp:8146-8158`); the kernel now publishes those same counters for bare hosts. They are report fields, not new durable state: the existing `native_continuation_hash` fold of `NativeDriverStatistics` (`native_execution_consumer.cpp:1098-1103`) sees the same values, so no hash contract moves. | +| `magnifier_sub_bars_total` / `magnifier_sample_ticks_total` (`engine_report.cpp:75-76`) | **kernel-owned report facts** | The generic intrabar driver already increments `NativeDriverStatistics::sub_bars_processed` / `sample_ticks_processed` for every retained path (`native_execution_consumer.cpp:8598-8610`); the kernel now publishes those same counters for bare hosts. They are report fields, not new durable state: the existing `native_continuation_hash` fold of `NativeDriverStatistics` (`native_execution_consumer.cpp:1098-1103`) sees the same values, so no hash contract moves. | | `pf_native_lot_excursion_v1` (`native_c_api.h:1912`) | **additive fee tail** | `entry_commission` carries the C++ `ClosedLotExcursionFacts::entry_commission` in account currency. The callback table publishes a trailing layout marker so F4 callers still receive the base facts length; current callers receive the fee tail without changing `PF_NATIVE_API_VERSION`. | | `strategy_outputs_set_enabled` and the eight `strategy_outputs_*` readers (`pineforge.h`, group `pf_outputs`) | **additive runtime exports** | Recording is off by default, and nothing a run computes reads the record. The event copy is size-prefixed like `strategy_pending_order_get`, and `PF_ABI_VERSION` stays 4. The writers are C++ only in this version (the BASE-CLASS SEAMS rows of `native_c_api.h`): a C host declares no outputs, so its handle answers every reader as empty. | @@ -823,7 +823,7 @@ The table records, at the tree the doc wave read, what each block described and | `ta::ATR` `include/pineforge/ta.hpp:321` (the `TradingView` comment `sha256:4f6f6939aec2773684b114a13dd002eb58990d6d96c3056ad87361753faf7a0a` :333-340) | average true range over Wilder's `RMA` | Wilder's ATR is the textbook definition; the comment records which convention and the tape that pinned the recompute behaviour | a seeding/averaging field on an indicator whose definition is the convention | `tests/test_ta_rma_warmup.cpp`, `tests/test_recompute.cpp` | | `ta::StdDev` `include/pineforge/ta.hpp:430` (the `TradingView` comment `sha256:7cc1bbcd32dfe673285d51a8ca7080890ac551858099b523480d998f46408666` :420-429) | the population (biased) standard deviation | biased vs sample is a textbook choice, and the kernel states which | a per-instance switch is the *right* answer where two conventions genuinely compete, which is why `ta::EmaSeeding` exists and this one does not need it | `tests/test_ta_indicators_extras.cpp` | | `float_band_eq` `include/pineforge/ta_compare_band.hpp:38` | an absolute 1e-10 float comparison for indicator equality | an epsilon band is needed by any float comparison; the width is the calibration | a band-width field on a comparison the indicators share | `tests/test_dmi_parity.cpp`, `tests/test_ta_osc_edge.cpp` | -| the two delivery rules, `projected_first_index` `src/native_execution_consumer.cpp:9181-9184` (lookahead) and `clear_if_gapped` `:9170-9173` (gaps) | deliver a completed bucket at its **first** contributing bar (lookahead), and **clear** the series on a bar it publishes nothing on (gaps) | both are delivery rules over a bucket the aggregator already closed; neither reads a platform. They are spec fields: `NativeTimeframeSubscription::lookahead` / `::gaps` | already spelled, and both default to the rule that moves no identity | `tests/test_native_htf_subscriptions.cpp` | +| the two delivery rules, `projected_first_index` `src/native_execution_consumer.cpp:9690-9693` (lookahead) and `clear_if_gapped` `:9679-9682` (gaps) | deliver a completed bucket at its **first** contributing bar (lookahead), and **clear** the series on a bar it publishes nothing on (gaps) | both are delivery rules over a bucket the aggregator already closed; neither reads a platform. They are spec fields: `NativeTimeframeSubscription::lookahead` / `::gaps` | already spelled, and both default to the rule that moves no identity | `tests/test_native_htf_subscriptions.cpp` | | `pf_native_subscription_v1::lookahead` `include/pineforge/native_c_api.h:2271`, `::gaps` `include/pineforge/native_c_api.h:2275` | the C spelling of those two rules | since R5 lane E7 both words are typed and named after the kernel's own delivery rules (`pf_native_lookahead_e` `include/pineforge/native_c_api.h:799`, `pf_native_gaps_e` `:810`); the C header no longer names `barmerge` at all, and the migration page maps Pine's words onto them — a *pointer to the reader's vocabulary*, not a justification | — | `tests/test_native_c_api.c` | | the authoritative-feed partition `src/engine_aux_security.cpp` | a feed is the venue's own bars of one timeframe; its stamps are the period partition | already ruled: design §2.ii row s, and the residual table's own row | the policy knob is installing the feed or not | `tests/test_native_htf_subscriptions.cpp`, `tests/test_native_security_feed.cpp` | | `session_template_knows_early_close` `src/engine_security.cpp` | the instrument-class classification | already ruled: design §2.ii row t; `SymInfo::type`'s vocabulary is fixed by the frozen C ABI | — | `tests/test_native_wm_buckets.cpp` | @@ -837,12 +837,12 @@ The table records, at the tree the doc wave read, what each block described and | the indicator library -- `include/pineforge/ta.hpp`, `src/ta_moving_averages.cpp`, `src/ta_oscillators.cpp`, `src/ta_extremes_volume.cpp`, `src/ta_volatility_trend.cpp`, `src/ta_misc.cpp`, `include/pineforge/window_sum.hpp`, `include/pineforge/math.hpp` | each indicator implements its Pine `ta.*` / `math.*` definition -- warm-up and na rules, the window call-site ring, the compensated window sum, tie and epsilon rules -- and its comments record the TradingView tape or export that calibrated each number (the `ta::ATR` and standard-deviation rows above are two of them) | an indicator is a function of its inputs, the same for every venue; generated strategies call exactly these classes, and where two textbook conventions genuinely compete the choice is a per-instance option (`ta::EmaSeeding`), never a branch on the platform | a convention switch on every indicator, for a choice the script already made by calling that Pine function | the kernel-only TA suites `tests/test_ta_*.cpp` and the corpus parity gate | | the formatting library -- `include/pineforge/str_utils.hpp`, `src/str_utils.cpp` (`str_tostring`, `str_format_values`, `str_format`) | the number text of Pine's `str.tostring` / `str.format` built-ins: a number's shortest round-trip decimal spelling, scaled and rounded half-up on its digits, the default, `percent`, `volume` (K / M / B / T) and pattern modes, MessageFormat placeholders and quoting, `NaN` and `Infinity` -- the published rendering TradingView's exported tapes pin (R5 lane C6), ported from the codegen formatter (lane C6b) by R5 lane B-ENGINE; the `mintick` mode, which generated code delegates to it, is kept as it was | a formatting library is a function of its inputs, the same for every venue, as the indicator library is: it spells a number the way the source language's own `str.*` library defines, and no kernel order, fill, settlement, margin or hash code calls it -- of the thirty-five kernel translation units only `src/str_utils.cpp` names these functions | a rendering switch on functions whose definition the calling script chose by calling them | `tests/test_str_number_format.cpp` (lane C6b's 42-case edge battery, every numeric string of lane C6's TradingView exports, byte-identical output on macOS arm64 and Linux x86-64) and `tests/test_str_utils.cpp` | | the language calendar functions -- `include/pineforge/session_time.hpp`, `src/session_time.cpp` (`timeframe_time`, `local_hour` and its siblings, `session_in_market`, `session_trading_day_open_ms`, ...) | the definitions of Pine's `time()` / `time_close()` / bare-time / `session.is*` built-ins for generated code: session and timezone parsing, the invalid- and 24-hour-session spellings, `time("60")` day keying, `session.is*` on daily charts, each comment naming the tape that pinned it | the inputs are data -- a session string, a timezone, a timeframe -- so another venue is driven by its own session template; nothing branches on a platform, and a native host's calendar is the separate generic `native_calendar` | a rules switch on functions whose definition the generated script chose by calling them | `tests/test_session_time.cpp`, `tests/test_pine_time_day_stamp_grid.cpp` and the corpus parity gate | -| `present_session_day` `src/native_execution_consumer.cpp:7636`, presenting `NativeDecisionContext::in_session` `include/pineforge/market_driver.hpp:161` and the three facts after it | a script bar's session-day facts: in session on the run's calendar, read at the bar's label, or at its interval's first eligible instant when that is later (a bar opening in a declared break), never at an instant before the label; it opens / closes its session day when the bar before / after it is out of session or on another session day, that bar being the one the run holds (batch input, stream warmup) and otherwise the calendar's previous or next eligible input slot across declared breaks; the run's first bar opens its day, a batch's final bar closes it, a D/W/M bar holds whole days | the session, the timezone and the session day are the run's own calendar (`spec.session`, `spec.timezone`, `native_calendar::session_day_ordinal`), the neighbours are the run's own input, and a second venue drives it through that data alone. The reading is TradingView's session DAY, measured on its tapes by lanes E25/E26 (`tests/fixtures/session_islastbar`: 255/255 NYSE:F last bars, 255/255 first, 370/370 ETH last), which a bare host now reproduces from the kernel alone. Reading a bar at its own label is generic too: an instant before the label is not the bar's, whatever grid the calendar keys its interval to (R5 lane K-SESSION-WINDOWS: NASDAQ:AAPL with extended hours opens its bars on the hour, and the 16:00 bar, which the 09:30-anchored grid files under 15:30, read in session and closed the regular day in place of the 15:00 bar; TSE:7203's and CBOT:ZC1!'s 60-minute bars that open inside a break are in session on TradingView's tapes, which reading at the label alone would lose) | a switch over the run-end convention or the neighbour rule would put a hash-visible choice on the surface for what the run's own input already decides. The one host that reads its batch end differently — a live probe recomputing a batch whose last input is still forming — reads a second fact, `closes_session_day_open_ended`, not a switch; the facts are presentation and fold into no digest | `tests/test_native_session_day_facts.cpp` (kernel-only: the tapes, the audit's Tokyo `2230-0500` session, streams, a fill recalculation), `tests/test_native_session_day_bar_open.cpp` (kernel-only: TradingView's extended-hours and break-open tapes, `tests/fixtures/session_windows`), the C scenario of `tests/test_native_c_api.c`, `tests/test_session_day_facts_adapter.cpp` (the adapter selects them flag for flag), `tests/test_native_session_day_utc.cpp` (kernel-only: on a UTC calendar the facts read an in-session instant without resolving its interval, and match the zone path at every decision point), and `tests/test_native_session_day_zoned.cpp` (kernel-only: on a certified cycle of an exchange zone, which `native_calendar::cycle_certificate` decides and the witness checks at real transitions, the facts read an instant off its session day, and match the zone's POSIX twin at every decision point) | -| `NativeExecutionConsumer::script_bucket_completions` `src/native_execution_consumer.cpp:8391-8407` | before a run, how an input span aggregates into script bars: which input bars complete one, and which complete one on a period boundary, under the run's own timeframes, zone and session (R5 lane B-ADAPTER, item 4) | any host that aggregates a finer feed asks it before its first input; the answer is the run's own `TimeframeAggregator` over the span and spec fields -- the real-end / chart-close completion and the session-day split this table already rules generic -- and nothing branches on a platform | none: a const query with no spec field, nothing hashed and no state moved; the host decides what the answer means for it (the Pine scheduler reads it instead of running its own preview aggregator) | `tests/test_native_script_bucket_completions.cpp` (kernel-only: the Pine scheduler's preview on the K1 zones, across DST, windowed sessions and 1D/1W/1M) | -| `PhysicalExecutionContext::account_fx` `include/pineforge/execution.hpp:92`, read by `build_close_trade_with_costs` `src/engine_orders.cpp:70-87` and handed by `append_open_position_report_rows` `src/native_execution_consumer.cpp:8348-8369` | a closing or mark-to-market row converts at the account-currency rate its execution names -- a mark at a report point's instant, a fill at its own cursor -- and at the presented clock's rate when it names none, as before (R5 lane B-ADAPTER, item 3) | the rate is data a host of any venue supplies with the row it asks for; nothing branches on a platform, and the row's currency is the basis its owner's magnitudes are recorded in | none: no spec field; a caller names the rate per call, and a call naming none converts exactly as before | `tests/test_adapter_range_end_fx.cpp` (a stepped FX curve at seven instants around the range end, plain and aggregated charts) | -| `NativeExecutionConsumer::execute_current` `src/native_execution_consumer.cpp:6937` | a current execution converts at its cursor's rate, threaded through its terms facts, inspection, preview and settlement contexts, and neither writes nor restores the host's presented clock, so its own hooks and the host after it see the frame's clock; its preview `inspect_current_execution` still converts at the presented clock (R5 lane B-ADAPTER, item 5) | an execution converting at its own instant's rate is the same for every host and venue, and the host's presented clock stays its own | none: the rate is the one the run already chose (the cursor's instant), carried as data | `tests/test_native_current_execution_rate.cpp` (kernel-only) | -| `NativeExecutionConsumer::validate_current_execution` `src/native_execution_consumer.cpp:6662` | a current execution refuses a request bound to a host roster (`BindCohort`) as `UnsupportedRequest`, before its point is taken and whatever the roster holds: it admits exactly the owners the evaluation's current shape executes (`current_shape` `src/native_order.cpp:383`: `Independent`, `BindOpening`, `BindOpenings`). Since R4 slice C it had admitted `BindCohort` as well, which the evaluation then left unevaluated, so a roster holding a live lot failed the run with "native current evaluated allowance mismatch" and an empty one answered `UnreadyOwner`; its read-only preview (`inspect_current_execution`) settled what the execution could not (R5 lane PAR-ORDERS-2) | one legality rule, the evaluation's own, decides a command before any point is taken, the same for every host and for the C surface (`strategy_native_execute_current_v1` answers through this validation); a roster read at the match is not a membership fixed at the command, which is what the documented contract names (`docs/pages/native-engine.md` "Selected exposure and current execution") | none: no spec field or hook. A refused request stays queued and fills at the next match like any market close; a host that wants a cohort's lots closed at once fixes its roster first (`BindOpenings`), as the Pine adapter's immediate closes do | `tests/test_native_current_cohort_refusal.cpp` (kernel-only: a live and an empty roster refused, then filled as queued closes; a fixed roster executing at once) , `tests/test_native_cohort_sliced_close.cpp` (its preview rows: the roster close's preview is refused, the same selection's settles) and `tests/test_adapter_live_state_equivalence.cpp` (reversals seed 3246599, which the failure used to end) | -| the chart symbol's daily partition -- `prepare_chart_day_partition` `src/engine_aux_security.cpp:358`, kept in `chart_day_partition_` `include/pineforge/engine.hpp:1218` and presented to the language runtime through `RuntimeAmbient::day_partition` `src/runtime_ambient.hpp:38` (the TradingView comment above `NativeDayPartition` in `include/pineforge/timeframe.hpp`) | on an intraday chart whose run installed its own symbol's `D` feed, the chart-level D period follows that feed's stamps: the exchange's trade-date bars, a holiday-merged day included. It governs `time("D")`, `timeframe.change("1D")`, `ta.change(time("D"))`, the default-anchored `ta.vwap` and `crosses_boundary(DAY)`. Without the feed, on a 1D chart or on another symbol's clock every rule is the nominal session day | the stamps are the venue's own data, as for the authoritative-feed partition above: another venue is driven by installing its own daily bars, and nothing branches on a platform | the policy knob is installing the feed or not, so a "which daily partition" field would restate it. Only the Pine host builds it (`prepare_chart_day_partition` `pine_strategy_host.cpp:1601`, `:1642`) and presents it for each bar it publishes (`AmbientDayPartitionScope` `pine_strategy_host.cpp:2333`). A bare host never calls the builder, so its calendar stays nominal | `tests/test_o_close_pct_day_anchor.cpp` (the partition is built from the D feed, and none without it) and `tests/test_chart_day_memo.cpp`; the tape is `lab tv` `o-cme-dayanchor-full`, recorded at the site | +| `present_session_day` `src/native_execution_consumer.cpp:8088`, presenting `NativeDecisionContext::in_session` `include/pineforge/market_driver.hpp:161` and the three facts after it | a script bar's session-day facts: in session on the run's calendar, read at the bar's label, or at its interval's first eligible instant when that is later (a bar opening in a declared break), never at an instant before the label; it opens / closes its session day when the bar before / after it is out of session or on another session day, that bar being the one the run holds (batch input, stream warmup) and otherwise the calendar's previous or next eligible input slot across declared breaks; the run's first bar opens its day, a batch's final bar closes it, a D/W/M bar holds whole days | the session, the timezone and the session day are the run's own calendar (`spec.session`, `spec.timezone`, `native_calendar::session_day_ordinal`), the neighbours are the run's own input, and a second venue drives it through that data alone. The reading is TradingView's session DAY, measured on its tapes by lanes E25/E26 (`tests/fixtures/session_islastbar`: 255/255 NYSE:F last bars, 255/255 first, 370/370 ETH last), which a bare host now reproduces from the kernel alone. Reading a bar at its own label is generic too: an instant before the label is not the bar's, whatever grid the calendar keys its interval to (R5 lane K-SESSION-WINDOWS: NASDAQ:AAPL with extended hours opens its bars on the hour, and the 16:00 bar, which the 09:30-anchored grid files under 15:30, read in session and closed the regular day in place of the 15:00 bar; TSE:7203's and CBOT:ZC1!'s 60-minute bars that open inside a break are in session on TradingView's tapes, which reading at the label alone would lose) | a switch over the run-end convention or the neighbour rule would put a hash-visible choice on the surface for what the run's own input already decides. The one host that reads its batch end differently — a live probe recomputing a batch whose last input is still forming — reads a second fact, `closes_session_day_open_ended`, not a switch; the facts are presentation and fold into no digest | `tests/test_native_session_day_facts.cpp` (kernel-only: the tapes, the audit's Tokyo `2230-0500` session, streams, a fill recalculation), `tests/test_native_session_day_bar_open.cpp` (kernel-only: TradingView's extended-hours and break-open tapes, `tests/fixtures/session_windows`), the C scenario of `tests/test_native_c_api.c`, `tests/test_session_day_facts_adapter.cpp` (the adapter selects them flag for flag), `tests/test_native_session_day_utc.cpp` (kernel-only: on a UTC calendar the facts read an in-session instant without resolving its interval, and match the zone path at every decision point), and `tests/test_native_session_day_zoned.cpp` (kernel-only: on a certified cycle of an exchange zone, which `native_calendar::cycle_certificate` decides and the witness checks at real transitions, the facts read an instant off its session day, and match the zone's POSIX twin at every decision point) | +| `NativeExecutionConsumer::script_bucket_completions` `src/native_execution_consumer.cpp:8843-8859` | before a run, how an input span aggregates into script bars: which input bars complete one, and which complete one on a period boundary, under the run's own timeframes, zone and session (R5 lane B-ADAPTER, item 4) | any host that aggregates a finer feed asks it before its first input; the answer is the run's own `TimeframeAggregator` over the span and spec fields -- the real-end / chart-close completion and the session-day split this table already rules generic -- and nothing branches on a platform | none: a const query with no spec field, nothing hashed and no state moved; the host decides what the answer means for it (the Pine scheduler reads it instead of running its own preview aggregator) | `tests/test_native_script_bucket_completions.cpp` (kernel-only: the Pine scheduler's preview on the K1 zones, across DST, windowed sessions and 1D/1W/1M) | +| `PhysicalExecutionContext::account_fx` `include/pineforge/execution.hpp:92`, read by `build_close_trade_with_costs` `src/engine_orders.cpp:70-87` and handed by `append_open_position_report_rows` `src/native_execution_consumer.cpp:8800-8821` | a closing or mark-to-market row converts at the account-currency rate its execution names -- a mark at a report point's instant, a fill at its own cursor -- and at the presented clock's rate when it names none, as before (R5 lane B-ADAPTER, item 3) | the rate is data a host of any venue supplies with the row it asks for; nothing branches on a platform, and the row's currency is the basis its owner's magnitudes are recorded in | none: no spec field; a caller names the rate per call, and a call naming none converts exactly as before | `tests/test_adapter_range_end_fx.cpp` (a stepped FX curve at seven instants around the range end, plain and aggregated charts) | +| `NativeExecutionConsumer::execute_current` `src/native_execution_consumer.cpp:7367` | a current execution converts at its cursor's rate, threaded through its terms facts, inspection, preview and settlement contexts, and neither writes nor restores the host's presented clock, so its own hooks and the host after it see the frame's clock; its preview `inspect_current_execution` still converts at the presented clock (R5 lane B-ADAPTER, item 5) | an execution converting at its own instant's rate is the same for every host and venue, and the host's presented clock stays its own | none: the rate is the one the run already chose (the cursor's instant), carried as data | `tests/test_native_current_execution_rate.cpp` (kernel-only) | +| `NativeExecutionConsumer::validate_current_execution` `src/native_execution_consumer.cpp:7092` | a current execution refuses a request bound to a host roster (`BindCohort`) as `UnsupportedRequest`, before its point is taken and whatever the roster holds: it admits exactly the owners the evaluation's current shape executes (`current_shape` `src/native_order.cpp:383`: `Independent`, `BindOpening`, `BindOpenings`). Since R4 slice C it had admitted `BindCohort` as well, which the evaluation then left unevaluated, so a roster holding a live lot failed the run with "native current evaluated allowance mismatch" and an empty one answered `UnreadyOwner`; its read-only preview (`inspect_current_execution`) settled what the execution could not (R5 lane PAR-ORDERS-2) | one legality rule, the evaluation's own, decides a command before any point is taken, the same for every host and for the C surface (`strategy_native_execute_current_v1` answers through this validation); a roster read at the match is not a membership fixed at the command, which is what the documented contract names (`docs/pages/native-engine.md` "Selected exposure and current execution") | none: no spec field or hook. A refused request stays queued and fills at the next match like any market close; a host that wants a cohort's lots closed at once fixes its roster first (`BindOpenings`), as the Pine adapter's immediate closes do | `tests/test_native_current_cohort_refusal.cpp` (kernel-only: a live and an empty roster refused, then filled as queued closes; a fixed roster executing at once) , `tests/test_native_cohort_sliced_close.cpp` (its preview rows: the roster close's preview is refused, the same selection's settles) and `tests/test_adapter_live_state_equivalence.cpp` (reversals seed 3246599, which the failure used to end) | +| the chart symbol's daily partition -- `prepare_chart_day_partition` `src/engine_aux_security.cpp:358`, kept in `chart_day_partition_` `include/pineforge/engine.hpp:1218` and presented to the language runtime through `RuntimeAmbient::day_partition` `src/runtime_ambient.hpp:38` (the TradingView comment above `NativeDayPartition` in `include/pineforge/timeframe.hpp`) | on an intraday chart whose run installed its own symbol's `D` feed, the chart-level D period follows that feed's stamps: the exchange's trade-date bars, a holiday-merged day included. It governs `time("D")`, `timeframe.change("1D")`, `ta.change(time("D"))`, the default-anchored `ta.vwap` and `crosses_boundary(DAY)`. Without the feed, on a 1D chart or on another symbol's clock every rule is the nominal session day | the stamps are the venue's own data, as for the authoritative-feed partition above: another venue is driven by installing its own daily bars, and nothing branches on a platform | the policy knob is installing the feed or not, so a "which daily partition" field would restate it. Only the Pine host builds it (`prepare_chart_day_partition` `pine_strategy_host.cpp:1630`, `:1671`) and presents it for each bar it publishes (`AmbientDayPartitionScope` `pine_strategy_host.cpp:2362`). A bare host never calls the builder, so its calendar stays nominal | `tests/test_o_close_pct_day_anchor.cpp` (the partition is built from the D feed, and none without it) and `tests/test_chart_day_memo.cpp`; the tape is `lab tv` `o-cme-dayanchor-full`, recorded at the site | | the bar-time accessors -- `_decompose_bar_time` and `_bar_hour` through `_bar_weekofyear` `include/pineforge/engine.hpp:1066-1093` | the presented bar's stamp broken into calendar fields in UTC, memoized per stamp | a function of the stamp alone, and UTC is the engine's storage timezone | none: they read no spec field and fold into no digest. They are protected `BacktestEngine` members, part of the codegen contract. Their comment claimed that generated code reads them for Pine's bare `hour` / `minute` / `dayofweek`. It does not: the codegen transpiler emits `pine_hour(current_bar_.timestamp, syminfo_.timezone)` and its siblings on the exchange timezone (`pineforge_codegen/codegen/tables.py` at the corpus's pinned commit `f3285d79`), which are the language calendar functions above. No generated strategy in `corpus/` names a `_bar_*` accessor. The comment now says so (R5 lane H-DOCGATES) | `tests/test_chart_timezone.cpp` (`_bar_hour()` is the UTC hour) and `tests/test_intraday_rollover_chart_tz_l4a.cpp` | | the conventional offset spellings of `timezone_accepted` `include/pineforge/native_calendar.hpp:208` (its comment's "TV sign" line) | `UTC+05:30` and `GMT-4` mean what they say: five and a half hours ahead of UTC, four behind (ISO 8601's sign), where a POSIX `TZ` string of the same shape means the opposite; a POSIX rule string keeps POSIX's own sign | a zone spelling is data, and the sign is the one a venue printing `UTC+8` means; nothing branches on a platform | a sign-convention switch over a spelling whose meaning is fixed | `tests/test_native_calendar.cpp` (`resolve_civil("UTC+05:30", …)` and `resolve_civil("GMT-4", …)` against their fixed instants) | | the grid and lot helpers -- `tick_grid_price` `include/pineforge/engine.hpp:799`, `price_grid_decimals` `include/pineforge/engine.hpp:833`, `level_on_price_grid` `include/pineforge/engine.hpp:843`, `apply_qty_step` `include/pineforge/engine.hpp:932`, `apply_exit_qty_step` `include/pineforge/engine.hpp:952` (their TradingView calibration comments sit above each) | nearest-tick and price-grid quantization and the lot-increment floors, each calibrated on TradingView tapes | **not a kernel mechanism**: no kernel or source-layer code calls any of them (`tests/test_f6_dead_kernel_members.cpp`, witness 2). The adapter runs its own copies of TradingView's per-order-kind tick rules (ruled native-only in the `price_grid` row above) and sizes its own lots; a native host declares a price grid and a quantity grid in its run spec | nothing to spell. They are rule 5's legacy, kept only because the twin-parity freeze (`scripts/check_twin_parity.py`, base `ab9714be`) holds the checks of the suites that reach them, and they leave with those checks | `tests/test_stop_tick_rounding_l4d.cpp`, `tests/test_level_grid_snap_l4d.cpp`, `tests/test_qty_step_epsilon_floor_l4b.cpp` | @@ -941,7 +941,7 @@ questions: the mechanism, why it is generic, and why no knob is needed. | Kernel capability | Generic mechanism | Why no platform knob is added | Executed witness | |---|---|---|---| -| A daily bar the installed daily feed dates: `NativeExecutionConsumer::prepare_day_labels` `src/native_execution_consumer.cpp:2079`, called at the end of `begin_ready`, once every feed is installed (`prepare_day_labels` `src/native_execution_consumer.cpp:2674`) and read where a script interval resolves (`script_interval_resolved` `src/native_execution_consumer.cpp:2029`); a lookup by a moved label is its bar's own (`day_label_origin` `src/native_execution_consumer.cpp:2132`) | (a) The venue's own daily bars date its days. When an intraday input aggregates into `"D"` script bars and a daily feed is installed -- the feed store's (`set_native_security_feed`) or a declared series' authoritative bars -- each stamp labels the D bar holding the session instant it covers: the stamp itself inside a session, else the next session's open, the instant the store already keys the feed's bars by. Only the label moves; the bar's span, closes, successor and OHLCV stay the calendar's aggregate, and every lookup by the label -- the bar's open and close, its sub-bars on a magnified run, its execution anchors, its risk day and a C host's session day -- reads the bar's own interval and day, not the day before the stamp. A bar no stamp covers keeps the calendar's label, a bar two stamps cover takes the earlier, and a label never passes an input bar the run holds: a stamp with an input between it and its bar's calendar open labels nothing, so a label never follows its bar's first input nor precedes another bar's. A stamp at the calendar's own open (09:30 ET, 17:00 CT, 00:00 UTC, OANDA:EURUSD's 17:00 ET) changes nothing; a venue that dates its days elsewhere -- OANDA's XAUUSD at 17:00 ET in the break of its 1800-1700 session, NSE's Muhurat day at its 13:45 open -- is driven by installing its own daily bars, and the kernel holds no venue's day-stamp convention | (b) None: the policy knob is the authoritative-feed ruling's, installing the daily feed or not (`engine_aux_security.cpp`'s contract), exactly as W15's late opens are; a "day label" field would restate the stamps the run installed. No input moves without the feed, and a calendar input's labels (the venue's bars themselves) never move. Calibration: TradingView dates every order of OANDA:XAUUSD 1D, backtested with the bar magnifier, by the 17:00 ET stamp -- 22:00 UTC in EST, 21:00 UTC in EDT -- where the 1m aggregate's session opens at 18:00 (`lab tv` `xc-dmag-xau1d`, `xc-dmag-dip-xau1d` and their `-off` twins, `tests/fixtures/daily_magnifier`) | `tests/test_native_day_labels.cpp` (kernel-only: break stamps across a DST switch, the bar's open reading its close's interval and input slot, a magnified run's sub-bars, a declared series' authoritative bars, stamps at the open, a missing, a doubled and a late stamp, an input in the break before a stamp's bar, NSE's late open, a daily, weekly and intraday script control, a reused host), `tests/test_native_c_api.c` (a C host's session day of a dated bar), `tests/test_daily_magnifier_tapes.cpp` (the four tapes through the Pine adapter: 20/20, 18/18, 31/31, 31/31; without the daily feed every trade an hour late) | +| A daily bar the installed daily feed dates: `NativeExecutionConsumer::prepare_day_labels` `src/native_execution_consumer.cpp:2390`, called at the end of `begin_ready`, once every feed is installed (`prepare_day_labels` `src/native_execution_consumer.cpp:3007`) and read where a script interval resolves (`script_interval_resolved` `src/native_execution_consumer.cpp:2340`); a lookup by a moved label is its bar's own (`day_label_origin` `src/native_execution_consumer.cpp:2443`) | (a) The venue's own daily bars date its days. When an intraday input aggregates into `"D"` script bars and a daily feed is installed -- the feed store's (`set_native_security_feed`) or a declared series' authoritative bars -- each stamp labels the D bar holding the session instant it covers: the stamp itself inside a session, else the next session's open, the instant the store already keys the feed's bars by. Only the label moves; the bar's span, closes, successor and OHLCV stay the calendar's aggregate, and every lookup by the label -- the bar's open and close, its sub-bars on a magnified run, its execution anchors, its risk day and a C host's session day -- reads the bar's own interval and day, not the day before the stamp. A bar no stamp covers keeps the calendar's label, a bar two stamps cover takes the earlier, and a label never passes an input bar the run holds: a stamp with an input between it and its bar's calendar open labels nothing, so a label never follows its bar's first input nor precedes another bar's. A stamp at the calendar's own open (09:30 ET, 17:00 CT, 00:00 UTC, OANDA:EURUSD's 17:00 ET) changes nothing; a venue that dates its days elsewhere -- OANDA's XAUUSD at 17:00 ET in the break of its 1800-1700 session, NSE's Muhurat day at its 13:45 open -- is driven by installing its own daily bars, and the kernel holds no venue's day-stamp convention | (b) None: the policy knob is the authoritative-feed ruling's, installing the daily feed or not (`engine_aux_security.cpp`'s contract), exactly as W15's late opens are; a "day label" field would restate the stamps the run installed. No input moves without the feed, and a calendar input's labels (the venue's bars themselves) never move. Calibration: TradingView dates every order of OANDA:XAUUSD 1D, backtested with the bar magnifier, by the 17:00 ET stamp -- 22:00 UTC in EST, 21:00 UTC in EDT -- where the 1m aggregate's session opens at 18:00 (`lab tv` `xc-dmag-xau1d`, `xc-dmag-dip-xau1d` and their `-off` twins, `tests/fixtures/daily_magnifier`) | `tests/test_native_day_labels.cpp` (kernel-only: break stamps across a DST switch, the bar's open reading its close's interval and input slot, a magnified run's sub-bars, a declared series' authoritative bars, stamps at the open, a missing, a doubled and a late stamp, an input in the break before a stamp's bar, NSE's late open, a daily, weekly and intraday script control, a reused host), `tests/test_native_c_api.c` (a C host's session day of a dated bar), `tests/test_daily_magnifier_tapes.cpp` (the four tapes through the Pine adapter: 20/20, 18/18, 31/31, 31/31; without the daily feed every trade an hour late) | ## Boundary rules (for contributors) diff --git a/docs/coverage.md b/docs/coverage.md index 6a06866f4..712ca8e83 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -62,7 +62,7 @@ ## Public C ABI `` is the **single canonical consumer header**. It has -exactly 93 public `PF_API` declarations: 73 runtime implementations and twenty +exactly 94 public `PF_API` declarations: 74 of the 80 runtime implementations (the other six are declared in `execution_observer.h` and `selected_window.h`) and twenty per-strategy generated exports. Every compiled PineForge strategy `.so` exports that public set, except that `strategy_declares_bar_magnifier` is exported only by a script that declares `use_bar_magnifier = true` and the three [outputs](outputs.md) exports only by a module that records outputs; older modules lack the six @@ -132,7 +132,7 @@ single `.hpp`): | Module | Header | Source | Pine-facing role | | ------------------ | ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Public C ABI | `pineforge.h` | `c_abi.cpp` (+ layout `static_assert`s) | 93 public `PF_API` declarations: 73 runtime implementations plus twenty per-strategy generated exports. `strategy_configure_native_fx_curve_v1` stages the additive native FX curve. | +| Public C ABI | `pineforge.h` | `c_abi.cpp` (+ layout `static_assert`s) | 94 public `PF_API` declarations: 74 of the 80 runtime implementations plus twenty per-strategy generated exports. `strategy_configure_native_fx_curve_v1` stages the additive native FX curve. | | Engine | `engine.hpp` | `engine_run.cpp`, `engine_stream.cpp`, `engine_execution.cpp`, `engine_orders.cpp`, `engine_path_resolve.cpp`, `engine_trade_accessors.cpp`, `engine_security.cpp`, `engine_lower_tf.cpp`, `engine_report.cpp`, `native_execution_consumer.cpp` | One-shot and continuous lifecycle, native request matching/settlement, orders, reports, inputs / syminfo, magnifier, TF aggregation, and `request.security` plumbing. | | Engine internals | `engine_internal.hpp` | (private cross-TU header) | `pineforge::internal::*` types and helpers shared between engine `.cpp` partitions; not part of the public ABI. | | Technical analysis | `ta.hpp` | `ta_moving_averages.cpp`, `ta_oscillators.cpp`, `ta_volatility_trend.cpp`, `ta_extremes_volume.cpp`, `ta_misc.cpp` | Official `ta.*` functions and series variables backed by stateful runtime classes with `compute` / `recompute`, plus `pivot_point_levels(...)` free function. | diff --git a/docs/design/native-feature-parity.md b/docs/design/native-feature-parity.md index 76be5806a..a8122e65e 100644 --- a/docs/design/native-feature-parity.md +++ b/docs/design/native-feature-parity.md @@ -22,7 +22,7 @@ Most status differences are row granularity (one input folds two features into o | # | Ruling | |---|---| | R5-1 | **Classification.** Generic execution feature → kernel native API. TradingView quirk → adapter. Mixed feature → kernel gets the generic mechanism with a policy knob, the adapter selects the TV policy, the bare-host default is the plain generic behaviour. | -| R5-2 | **Brackets are not a new kernel object.** `WaitForApplied{RequestHandle parent}` (native_order.hpp:436-438) and `waiting_children(parent)` (native_order.hpp:1895, *fresh*) exist; L7 delivers a builder/toolkit over them plus relative anchors and the missing offsets. Size M. | +| R5-2 | **Brackets are not a new kernel object.** `WaitForApplied{RequestHandle parent}` (native_order.hpp:436-438) and `waiting_children(parent)` (native_order.hpp:1940, *fresh*) exist; L7 delivers a builder/toolkit over them plus relative anchors and the missing offsets. Size M. | | R5-3 | **Price grid is its own kernel lane (L8).** Quantity grid is native already; the price grid (booked fill on the tick, optional quantized trigger test) is generic and missing. TV's exact half-tick rule stays in the adapter on top. | | R5-4 | **Close reservations split.** Fractional reduce + reservation against sibling exits → kernel, inside the sizing lane. Callsite batching, two-call provenance, the POOC population predicate → adapter. | | R5-5 | **Calc-timing is v1.** Sub-bar hook + callback chronology contract; the adapter keeps its current path (zero drift). | @@ -47,123 +47,123 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| OL1 | Market / limit / stop / stop-limit / trail triggers (`trigger_for` pine_adapter.cpp:3293-3304; `entry()` pine_adapter.cpp:6088, `order()` pine_adapter.cpp:14395) | **yes** — `Trigger` native_order.hpp:296-315; geometric matching `first_region_entry` native_execution_consumer.cpp:5975-6022; `submit` native_host.hpp:1213 | K | — | F:A1 O:A1 S:O2 | — | -| OL2 | Intents: flatten, reduce by units, reduce owner-opened units, transact, reverse-to (`close_all()` pine_adapter.cpp:10349 → `Flatten{}` pine_adapter.cpp:10382-10383; `Reduce{ExplicitUnits}` pine_adapter.cpp:9198-9207; `ReverseTo` pine_adapter.cpp:7668-7669) | **yes** — `OrderIntent` native_order.hpp:229, reduce sizes `ReductionSize` native_order.hpp:215, `ReverseTo` native_order.hpp:60-62, `OpeningShape` native_order.hpp:910-914, `Flatten` native_order.hpp:36 | K | — | F:A4 F:A8 O:A2 O:A3 S:O1 | — | -| OL3 | Amend / cancel one working order (`submit_or_replace` pine_adapter.cpp:2809-4044 → `host.replace` pine_adapter.cpp:3259-3356) | **yes** — `replace` native_host.hpp:1218-1220, `cancel` native_host.hpp:1249; `ReplacedEvent` native_order.hpp:1017-1031; result and event types `SubmitStatus` native_order.hpp:835-861, `ReplacedEvent` native_order.hpp:1001-1067 | K | — | F:A2 O:A4 S:O3 | id → handle map is host bookkeeping (F) → OL4 | -| OL4 | Cancel all / cancel by label; label → handle book (`cancel(id)` pine_adapter.cpp:5349-5395, `cancel_all` pine_adapter.cpp:25555-25570; `live_by_source_key_` pine_adapter.hpp:2486; predecessor state hand-carried `projection_predecessor` pine_adapter.cpp:4595-4669) | **partial** — single-handle `cancel` native_host.hpp:1249; no enumeration (`WorkingRequestCore::live()` is consumer-internal native_order.hpp:1703); labels exist but are unindexed (`Request::label` native_order.hpp:485) **Closed:** `cancel_all` native_host.hpp:1267, `cancel_where(comment)` native_host.hpp:1272 and `cancel_where(text, NativeRequestField::Label)` native_host.hpp:1272 withdraw by predicate and answer how many left the book; `native_working_requests()` native_host.hpp:1263 is the enumeration; the id → handle book is `native_toolkit::OrderBook`, host-side and Pine-free (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | F:A3 F:A2 O:A5 O:A14 S:O3 | **Disagree:** F partial / O no (A5) + partial (A14) / S yes (folded into O3). R5-1: cancel-all / by-label is generic → kernel. TV state inheritance across a same-id re-issue stays A. The adapter's `strategy.cancel` / `strategy.cancel_all` keep their per-handle loops, measured by R5 lane F7 (audit M27): TradingView cancels in the SOURCE book, which holds commands of the same script call that have not reached the kernel yet (and a live `strategy.close` is labelled `__close__` there), so at the same point `cancel_where(id, NativeRequestField::Label)` and `cancel_all()` withdraw 0 — the close flattens at the next open, the reversal lands — where the source cancels withdraw them (`tests/test_adapter_brackets_relower.cpp`, M27 rows); each withdrawn source handle is also retired one by one. | -| OL5 | Per-point fill capacity, partial fills, residual working quantity (Pine uses only `ImmediateRemaining` pine_adapter.cpp:4802; residual closes `enqueue_pooc_fifo_close` pine_adapter.cpp:8750-8983) | **yes** — `Capacity` / `PointBudget` native_order.hpp:379-383, `Remaining` native_order.hpp:557-579, allowances `Allowance` native_order.hpp:662-675 | K | — | O:A16 S:O4 | — | -| OL6 | Ownership: legs bound to a parent / opening(s) / cohort; close the lots of a given entry, FIFO or ANY (`owner_for_close` pine_adapter.cpp:6407-6442, `cohort_for` pine_adapter.cpp:2945; FIFO prefix computed adapter-side `source_fifo_prefix_openings` pine_adapter.cpp:9169-9211) | **yes** — `Owner` native_order.hpp:464-479, `Authority` native_order.hpp:620-621, `CohortClose` native_order.hpp:617-619, `SelectedExposure` native_order.hpp:625-629, `cohort_open` native_host.hpp:1282, `CancelReason::OwnerGone` native_order.hpp:949 | K | — | F:A6 O:A8 O:A9 S:O5 | FIFO *ordering* is host-computed (O) | -| OL7 | OCA cancel / reduce groups (`group_for` pine_adapter.cpp:3305-3311) | **yes** — `Group` / `GroupEffect` native_order.hpp:466-473, `CancelReason::Group` native_order.hpp:948-953, `ReservationReducedEvent` native_order.hpp:1191-1201 (deferred form `DeferredGroupAdjustmentEvent` native_order.hpp:1211-1221) | K | — | F:A12 O:A6 O:A7 S:O7 | — | +| OL1 | Market / limit / stop / stop-limit / trail triggers (`trigger_for` pine_adapter.cpp:3301-3312; `entry()` pine_adapter.cpp:6096, `order()` pine_adapter.cpp:14403) | **yes** — `Trigger` native_order.hpp:296-315; geometric matching `first_region_entry` native_execution_consumer.cpp:6395-6442; `submit` native_host.hpp:1213 | K | — | F:A1 O:A1 S:O2 | — | +| OL2 | Intents: flatten, reduce by units, reduce owner-opened units, transact, reverse-to (`close_all()` pine_adapter.cpp:10357 → `Flatten{}` pine_adapter.cpp:10390-10391; `Reduce{ExplicitUnits}` pine_adapter.cpp:9206-9215; `ReverseTo` pine_adapter.cpp:7676-7677) | **yes** — `OrderIntent` native_order.hpp:229, reduce sizes `ReductionSize` native_order.hpp:215, `ReverseTo` native_order.hpp:60-62, `OpeningShape` native_order.hpp:910-914, `Flatten` native_order.hpp:36 | K | — | F:A4 F:A8 O:A2 O:A3 S:O1 | — | +| OL3 | Amend / cancel one working order (`submit_or_replace` pine_adapter.cpp:2809-4044 → `host.replace` pine_adapter.cpp:3267-3364) | **yes** — `replace` native_host.hpp:1218-1220, `cancel` native_host.hpp:1249; `ReplacedEvent` native_order.hpp:1017-1031; result and event types `SubmitStatus` native_order.hpp:835-861, `ReplacedEvent` native_order.hpp:1001-1067 | K | — | F:A2 O:A4 S:O3 | id → handle map is host bookkeeping (F) → OL4 | +| OL4 | Cancel all / cancel by label; label → handle book (`cancel(id)` pine_adapter.cpp:5357-5403, `cancel_all` pine_adapter.cpp:25573-25588; `live_by_source_key_` pine_adapter.hpp:2513; predecessor state hand-carried `projection_predecessor` pine_adapter.cpp:4603-4677) | **partial** — single-handle `cancel` native_host.hpp:1249; no enumeration (`WorkingRequestCore::live()` is consumer-internal native_order.hpp:1703); labels exist but are unindexed (`Request::label` native_order.hpp:485) **Closed:** `cancel_all` native_host.hpp:1267, `cancel_where(comment)` native_host.hpp:1272 and `cancel_where(text, NativeRequestField::Label)` native_host.hpp:1272 withdraw by predicate and answer how many left the book; `native_working_requests()` native_host.hpp:1263 is the enumeration; the id → handle book is `native_toolkit::OrderBook`, host-side and Pine-free (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | F:A3 F:A2 O:A5 O:A14 S:O3 | **Disagree:** F partial / O no (A5) + partial (A14) / S yes (folded into O3). R5-1: cancel-all / by-label is generic → kernel. TV state inheritance across a same-id re-issue stays A. The adapter's `strategy.cancel` / `strategy.cancel_all` keep their per-handle loops, measured by R5 lane F7 (audit M27): TradingView cancels in the SOURCE book, which holds commands of the same script call that have not reached the kernel yet (and a live `strategy.close` is labelled `__close__` there), so at the same point `cancel_where(id, NativeRequestField::Label)` and `cancel_all()` withdraw 0 — the close flattens at the next open, the reversal lands — where the source cancels withdraw them (`tests/test_adapter_brackets_relower.cpp`, M27 rows); each withdrawn source handle is also retired one by one. | +| OL5 | Per-point fill capacity, partial fills, residual working quantity (Pine uses only `ImmediateRemaining` pine_adapter.cpp:4810; residual closes `enqueue_pooc_fifo_close` pine_adapter.cpp:8750-8983) | **yes** — `Capacity` / `PointBudget` native_order.hpp:379-383, `Remaining` native_order.hpp:557-579, allowances `Allowance` native_order.hpp:662-675 | K | — | O:A16 S:O4 | — | +| OL6 | Ownership: legs bound to a parent / opening(s) / cohort; close the lots of a given entry, FIFO or ANY (`owner_for_close` pine_adapter.cpp:6415-6450, `cohort_for` pine_adapter.cpp:2951; FIFO prefix computed adapter-side `source_fifo_prefix_openings` pine_adapter.cpp:9177-9219) | **yes** — `Owner` native_order.hpp:464-479, `Authority` native_order.hpp:620-621, `CohortClose` native_order.hpp:617-619, `SelectedExposure` native_order.hpp:625-629, `cohort_open` native_host.hpp:1282, `CancelReason::OwnerGone` native_order.hpp:949 | K | — | F:A6 O:A8 O:A9 S:O5 | FIFO *ordering* is host-computed (O) | +| OL7 | OCA cancel / reduce groups (`group_for` pine_adapter.cpp:3313-3319) | **yes** — `Group` / `GroupEffect` native_order.hpp:466-473, `CancelReason::Group` native_order.hpp:948-953, `ReservationReducedEvent` native_order.hpp:1191-1201 (deferred form `DeferredGroupAdjustmentEvent` native_order.hpp:1211-1221) | K | — | F:A12 O:A6 O:A7 S:O7 | — | | OL8 | Immediate execution at the current point (`immediately=true`) | **yes** — `inspect_current_execution` / `execute_current` native_host.hpp:1104-1105 | K | — | F:A9 | (single-source row; O:F2 cites the same call) | | OL9 | Birth eligibility: an order accepted at bar N cannot fill on N's already-delivered points | **yes** — `point_eligible` native_order.hpp:786-790, the generic birth record `OrderBirth` include/pineforge/order_birth.hpp:59, contract `docs/pages/native-engine.md` ("Acceptance is not a fill") | K+A | — | F:A13 O:A10 S:R4 | **Disagree:** F yes / O partial / S "no, TV quirk". R5-1: the generic core (birth cursor) is native; TV's bar-granular first-bar rule is the A half → OL14. | -| OL10 | TP/SL bracket per entry, armed before the entry fills, OCA between legs (`exit()` pine_adapter.cpp:10842; lowering `submit_one` pine_adapter.cpp:11699-11711, `BindCohort` pine_adapter.cpp:12161-12168, `submit_leg` pine_adapter.cpp:12248-12272; lifecycle `select_exit_suspension` exit_lifecycle.cpp:10-66) | **partial** — every primitive is native (`WaitForApplied{parent}` native_order.hpp:436-438, `Reduce{OwnerOpenedUnits}` native_order.hpp:220-224, `Member{…, Cancel}` native_order.hpp:468-475, `waiting_children` native_order.hpp:1895); no builder **Closed:** the builder is `native_toolkit::submit_bracket(host, BracketSpec)` — take-profit, stop-loss and trail as one parent's `WaitForApplied` children in one OCA group, with `anchor_rounding` and `visibility` knobs — and it adds no kernel behaviour (`tests/test_native_toolkit_bracket.cpp`). | K (toolkit) | L7 | F:A10 O:A8 S:O6 | **Disagree:** F "yes (primitives)" + toolkit / S partial + new kernel bracket object, lane size L and prerequisite for everything. R5-2: builder over the existing primitives, size M. | -| OL11 | Bracket levels relative to the parent's fill (profit / loss ticks; derived `profit_ticks` pine_adapter.cpp:11099-11102; `materialize_relative_exits` pine_adapter.cpp:24641-24683, pine_adapter.hpp:2222-2234) | **partial** — composable: the host computes levels in `on_native_applied` and submits; suffix eligibility makes that same-bar correct `born_on_remaining_path` native_execution_consumer.cpp:5916-5950. `Trigger` itself is absolute-only native_order.hpp:296 **Closed:** `Request::anchor = FromOwnerFill{offset, ticks, rounding}` (`TriggerAnchor` native_order.hpp:336) defers the level to the owner's fill, the kernel materializes it once at the arm under `NativeAnchorRounding`, and `resolve_anchored_level` native_host.hpp:1012 is where a source language's own level arithmetic lives (`tests/test_native_anchored_legs.cpp`, `tests/test_native_arm_options.cpp`). | K | L7 | F:A11 O:A11 S:O6 | **Disagree:** F partial (toolkit is enough) / O no (kernel `TriggerAnchor`). R5-2: L7 delivers relative anchors; kernel anchor primary, toolkit materialization is the zero-kernel fallback. | -| OL12 | Working-order view for the host and outside observers (`strategy_pending_order_*`; `PendingIntentView` pine_adapter.hpp:1449-1465; the POD is filled only by the adapter `pine_birth_reach` pine_adapter.cpp:26787-26816) | **no** — base returns 0 / -1 `observe_last_bar_dual_entry_path_v1` engine_consumer.cpp:124-133; the POD is TV-shaped (`tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135) **Closed for a bare host:** `native_working_requests()` native_host.hpp:1263 returns one owning `NativeWorkingRequest` native_host.hpp:681 per live request — definition, remaining, trigger state — with the C spelling `strategy_native_working_len_v1` / `_get_v1`. The TV-shaped POD stays frozen and stays the source layer's projection (`tests/test_native_limit_fill_through.cpp`, `tests/test_native_arm_options.cpp`, `tests/test_native_c_api.c`). | K+A | L7 (C++), L13 (C) | F:J5 O:J6 S:X4 | **Disagree:** O "partial + TV-shaped" / F, S no. A bare host reads zeros → no. R5-1: neutral view → kernel; POD shape and Pine journal retention stay A (POD frozen, §2.ii j). | -| OL13 | *(quirk)* Same-bar command batching and the seven deferred queues (`pending_bracket_legs_` pine_adapter.hpp:2480-2499); KI-62 open-order priority (`select` order_priority.cpp:13-60, selected `update_l4c_priority` pine_adapter.cpp:1226-1305); per-bar priced-entry throttle, BUY-before-SELL batch, short-seed plan / collision (`maybe_activate_short_seed_plan` pine_adapter.cpp:3896-4063, `materialize_snapshot` pine_adapter.cpp:13733-13757), reversal-gap bracket policy (`ShortSeedPlan` pine_adapter.hpp:1357-1384, `DeferredOpenMarketableSell` pine_adapter.hpp:1797-1820, `CloseCallsiteState` pine_adapter.hpp:1949-1963; `apply_reversal_gap_bracket_policy` pine_adapter.cpp:21386, `reaccept_gapped_bracket_behind_same_id_add` pine_adapter.cpp:21620) | n/a — native priority is acceptance / incarnation order `native_events` native-engine.md:676-678 | A | — | F:A15 O:A15 S:O9 S:R3 | R5-1 names KI-62. S: the kernel exposes deterministic insertion order only. | -| OL14 | *(quirk)* Exit activation / suspension / revival barriers, historical birth reach (`select_exit_activation` exit_activation.cpp:44-90, enforced `leg_activation` pine_adapter.cpp:1197-1202; `select_exit_activation` exit_activation.cpp:20-90, `select_historical_birth_reach` order_birth.cpp:5-14, `capture_order_birth` pine_adapter.cpp:986-1019) | n/a | A | — | S:R4 O:A10 | TV half of OL9. **V19D-P1 (R5 lanes B-ADAPTER, V19-FIX):** a margin-call revival never restores an exit the script cancelled: `strategy.cancel` / `cancel_all` retire every leg of each exit row they name — the live ones they take off the book (caused at the Cancelled event, after anything the same callback did to the leg), the dormant ones, and the ones the adapter took off the book itself — and drop a gapped stop a declined reversal parked for the next margin-call slice (`retire_cancelled_exits` pine_adapter.cpp:2061); the revival skips a withdrawn exit (`revive_brackets_after_margin` pine_adapter.cpp:2135). What `tests/test_adapter_margin_revival_cancel.cpp` replays on the NYSE:F 15m chart of seven lab tv tapes: badapter-v19dp1-control (no cancel: the margin call closes 928 through X) and v19fix-gapped-stop-control (X's stop at 10.15, gapped by the 05-01 open and parked: the same close), and badapter-v19dp1-cancel, v19fix-cancel-before-pair, v19fix-cancel-at-1000, v19fix-gapped-stop-cancel and v19fix-gapped-stop-cancel-after-pair (a cancel on 05-01 09:30 while X is dormant, on 04-30 15:45 before the reversal pair and on 04-30 10:00 while X is live, and on 05-01 09:30 and after the pair while X is parked; one tape file: margin calls 12 @10.39 and 92 @10.57, 836 open). Untaped, the same row holds the tapes' trades for `strategy.cancel_all` on 05-01 and before the pair and for `strategy.cancel` after the pair, and on the same bars with no quantity grid — where the pair holds X in the cancel's own callback — every spelling books what the cancel on the dormant exit books | +| OL10 | TP/SL bracket per entry, armed before the entry fills, OCA between legs (`exit()` pine_adapter.cpp:10850; lowering `submit_one` pine_adapter.cpp:11707-11719, `BindCohort` pine_adapter.cpp:12169-12176, `submit_leg` pine_adapter.cpp:12256-12280; lifecycle `select_exit_suspension` exit_lifecycle.cpp:10-66) | **partial** — every primitive is native (`WaitForApplied{parent}` native_order.hpp:436-438, `Reduce{OwnerOpenedUnits}` native_order.hpp:220-224, `Member{…, Cancel}` native_order.hpp:468-475, `waiting_children` native_order.hpp:1940); no builder **Closed:** the builder is `native_toolkit::submit_bracket(host, BracketSpec)` — take-profit, stop-loss and trail as one parent's `WaitForApplied` children in one OCA group, with `anchor_rounding` and `visibility` knobs — and it adds no kernel behaviour (`tests/test_native_toolkit_bracket.cpp`). | K (toolkit) | L7 | F:A10 O:A8 S:O6 | **Disagree:** F "yes (primitives)" + toolkit / S partial + new kernel bracket object, lane size L and prerequisite for everything. R5-2: builder over the existing primitives, size M. | +| OL11 | Bracket levels relative to the parent's fill (profit / loss ticks; derived `profit_ticks` pine_adapter.cpp:11107-11110; `materialize_relative_exits` pine_adapter.cpp:24659-24701, pine_adapter.hpp:2259-2271) | **partial** — composable: the host computes levels in `on_native_applied` and submits; suffix eligibility makes that same-bar correct `born_on_remaining_path` native_execution_consumer.cpp:6336-6370. `Trigger` itself is absolute-only native_order.hpp:296 **Closed:** `Request::anchor = FromOwnerFill{offset, ticks, rounding}` (`TriggerAnchor` native_order.hpp:336) defers the level to the owner's fill, the kernel materializes it once at the arm under `NativeAnchorRounding`, and `resolve_anchored_level` native_host.hpp:1012 is where a source language's own level arithmetic lives (`tests/test_native_anchored_legs.cpp`, `tests/test_native_arm_options.cpp`). | K | L7 | F:A11 O:A11 S:O6 | **Disagree:** F partial (toolkit is enough) / O no (kernel `TriggerAnchor`). R5-2: L7 delivers relative anchors; kernel anchor primary, toolkit materialization is the zero-kernel fallback. | +| OL12 | Working-order view for the host and outside observers (`strategy_pending_order_*`; `PendingIntentView` pine_adapter.hpp:1449-1484; the POD is filled only by the adapter `pine_birth_reach` pine_adapter.cpp:26836-26865) | **no** — base returns 0 / -1 `observe_last_bar_dual_entry_path_v1` engine_consumer.cpp:124-133; the POD is TV-shaped (`tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135) **Closed for a bare host:** `native_working_requests()` native_host.hpp:1263 returns one owning `NativeWorkingRequest` native_host.hpp:681 per live request — definition, remaining, trigger state — with the C spelling `strategy_native_working_len_v1` / `_get_v1`. The TV-shaped POD stays frozen and stays the source layer's projection (`tests/test_native_limit_fill_through.cpp`, `tests/test_native_arm_options.cpp`, `tests/test_native_c_api.c`). | K+A | L7 (C++), L13 (C) | F:J5 O:J6 S:X4 | **Disagree:** O "partial + TV-shaped" / F, S no. A bare host reads zeros → no. R5-1: neutral view → kernel; POD shape and Pine journal retention stay A (POD frozen, §2.ii j). | +| OL13 | *(quirk)* Same-bar command batching and the seven deferred queues (`pending_bracket_legs_` pine_adapter.hpp:2515-2534); KI-62 open-order priority (`select` order_priority.cpp:13-60, selected `update_l4c_priority` pine_adapter.cpp:1226-1305); per-bar priced-entry throttle, BUY-before-SELL batch, short-seed plan / collision (`maybe_activate_short_seed_plan` pine_adapter.cpp:3904-4071, `materialize_snapshot` pine_adapter.cpp:13741-13765), reversal-gap bracket policy (`ShortSeedPlan` pine_adapter.hpp:1357-1384, `DeferredOpenMarketableSell` pine_adapter.hpp:1845-1868, `CloseCallsiteState` pine_adapter.hpp:1983-1997; `apply_reversal_gap_bracket_policy` pine_adapter.cpp:21394, `reaccept_gapped_bracket_behind_same_id_add` pine_adapter.cpp:21628) | n/a — native priority is acceptance / incarnation order `native_events` native-engine.md:676-678 | A | — | F:A15 O:A15 S:O9 S:R3 | R5-1 names KI-62. S: the kernel exposes deterministic insertion order only. | +| OL14 | *(quirk)* Exit activation / suspension / revival barriers, historical birth reach (`select_exit_activation` exit_activation.cpp:44-90, enforced `leg_activation` pine_adapter.cpp:1197-1202; `select_exit_activation` exit_activation.cpp:20-90, `select_historical_birth_reach` order_birth.cpp:5-14, `capture_order_birth` pine_adapter.cpp:986-1019) | n/a | A | — | S:R4 O:A10 | TV half of OL9. **V19D-P1 (R5 lanes B-ADAPTER, V19-FIX):** a margin-call revival never restores an exit the script cancelled: `strategy.cancel` / `cancel_all` retire every leg of each exit row they name — the live ones they take off the book (caused at the Cancelled event, after anything the same callback did to the leg), the dormant ones, and the ones the adapter took off the book itself — and drop a gapped stop a declined reversal parked for the next margin-call slice (`retire_cancelled_exits` pine_adapter.cpp:2064); the revival skips a withdrawn exit (`revive_brackets_after_margin` pine_adapter.cpp:2141). What `tests/test_adapter_margin_revival_cancel.cpp` replays on the NYSE:F 15m chart of seven lab tv tapes: badapter-v19dp1-control (no cancel: the margin call closes 928 through X) and v19fix-gapped-stop-control (X's stop at 10.15, gapped by the 05-01 open and parked: the same close), and badapter-v19dp1-cancel, v19fix-cancel-before-pair, v19fix-cancel-at-1000, v19fix-gapped-stop-cancel and v19fix-gapped-stop-cancel-after-pair (a cancel on 05-01 09:30 while X is dormant, on 04-30 15:45 before the reversal pair and on 04-30 10:00 while X is live, and on 05-01 09:30 and after the pair while X is parked; one tape file: margin calls 12 @10.39 and 92 @10.57, 836 open). Untaped, the same row holds the tapes' trades for `strategy.cancel_all` on 05-01 and before the pair and for `strategy.cancel` after the pair, and on the same bars with no quantity grid — where the pair holds X in the cancel's own callback — every spelling books what the cancel on the dormant exit books | ### 1.2 Trailing | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| TR1 | Trail arm → track → trigger, with a state query | **yes** — `Trail{offset, arm_price}` native_order.hpp:290-293; states `TrailWaitArm` native_order.hpp:642-660; arm / track / fire `TrailWaitArm` native_execution_consumer.cpp:5886-5908; `NativeTrailState` native_host.hpp:669-674, `trail_state` native_host.hpp:1092 | K | — | F:B1 O:B1 S:T1 | — | +| TR1 | Trail arm → track → trigger, with a state query | **yes** — `Trail{offset, arm_price}` native_order.hpp:290-293; states `TrailWaitArm` native_order.hpp:642-660; arm / track / fire `TrailWaitArm` native_execution_consumer.cpp:6306-6328; `NativeTrailState` native_host.hpp:669-674, `trail_state` native_host.hpp:1092 | K | — | F:B1 O:B1 S:T1 | — | | TR2 | Trail activation / offset in ticks or points, relative to the entry fill (`trail_points_to_ticks` include/pineforge/compat/pine/trail_ticks.hpp:44 + grid snap exit_lifecycle.cpp:22-27; `trail_activation_level` pine_adapter.hpp:350-352) | **partial** — absolute prices and a raw price distance only **Closed:** `TrailTicks{n}` (`Trail::ticks` native_order.hpp:293) is resolved against `NativeRunSpec::price_tick` at acceptance, so a tick-spelled trail and its price-spelled equal behave identically bar for bar; a relative arm threshold is the `FromOwnerFill` anchor of OL11 (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K | L7 | F:B2 O:B2 S:T2 | **Disagree:** S classes all trail operand conversion as TV quirk. R5-1: tick / relative spelling is generic. | -| TR3 | Zero-distance trail, "ride the best, exit on any adverse tick" (adapter substitutes `tick * 0.5` pine_adapter.cpp:14774-14786) | **no** — the kernel fails the run when `offset <= 0` (`checked_trail_stop` → native_execution_consumer.cpp:4774-4779) **Closed:** a zero offset is legal and means *ride the best* — the level is the running best itself and the exit is the first print strictly past it; negative and nonfinite offsets stay rejected (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B3 F:B3 S:T2 | **Disagree:** O kernel / F, S quirk. R5-1: accepting a zero offset is generic; TV zero-offset *pricing* (`zero_trail_sibling_stop` pine_adapter.cpp:14510-14698) stays A. | -| TR4 | Re-price a trail without losing the running extreme (adapter skips `replace` and reads `trail_state()` pine_adapter.cpp:4574-4622) | **partial** — the read exists `trail_state` native_host.hpp:1092; `replace` always rebuilds trigger state **Closed:** `replace(handle, request, ReplaceOptions{/*retain_trigger_state=*/true})` carries the predecessor's live trigger state — a tracking trail's best, an already active stop — into the successor, refusing the replacement when the retained best leaves no representable level (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B4 F:B3 | Status is single-source (O). R5-1: retain-trail replace is generic; TV's "retained best across re-issue" convention (`retained_trail_best` pine_adapter.hpp:353-355) is A. | -| TR5 | *(quirk)* TV trail conventions: half-tick arm threshold against tick-quantized extremes `has_trail_request` pine_adapter.cpp:12398-12403, zero-offset pricing `zero_trail_sibling_stop` pine_adapter.cpp:14510-14698, retained best `retained_trail_best` pine_adapter.hpp:353-355, operand / restart rules `source_trail_points` pine_adapter.cpp:10450-10966, `trail_one_shot` pine_adapter.cpp:12206-12295 | n/a | A | — | F:B3 S:T2 | R5-1 | +| TR3 | Zero-distance trail, "ride the best, exit on any adverse tick" (adapter substitutes `tick * 0.5` pine_adapter.cpp:14788-14800) | **no** — the kernel fails the run when `offset <= 0` (`checked_trail_stop` → native_execution_consumer.cpp:5186-5191) **Closed:** a zero offset is legal and means *ride the best* — the level is the running best itself and the exit is the first print strictly past it; negative and nonfinite offsets stay rejected (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B3 F:B3 S:T2 | **Disagree:** O kernel / F, S quirk. R5-1: accepting a zero offset is generic; TV zero-offset *pricing* (`zero_trail_sibling_stop` pine_adapter.cpp:14518-14706) stays A. | +| TR4 | Re-price a trail without losing the running extreme (adapter skips `replace` and reads `trail_state()` pine_adapter.cpp:4582-4630) | **partial** — the read exists `trail_state` native_host.hpp:1092; `replace` always rebuilds trigger state **Closed:** `replace(handle, request, ReplaceOptions{/*retain_trigger_state=*/true})` carries the predecessor's live trigger state — a tracking trail's best, an already active stop — into the successor, refusing the replacement when the retained best leaves no representable level (`tests/test_native_order_ergonomics.cpp`, `tests/test_native_anchored_legs.cpp`). | K+A | L7 | O:B4 F:B3 | Status is single-source (O). R5-1: retain-trail replace is generic; TV's "retained best across re-issue" convention (`retained_trail_best` pine_adapter.hpp:353-355) is A. | +| TR5 | *(quirk)* TV trail conventions: half-tick arm threshold against tick-quantized extremes `has_trail_request` pine_adapter.cpp:12406-12411, zero-offset pricing `zero_trail_sibling_stop` pine_adapter.cpp:14518-14706, retained best `retained_trail_best` pine_adapter.hpp:353-355, operand / restart rules `source_trail_points` pine_adapter.cpp:10458-10974, `trail_one_shot` pine_adapter.cpp:12214-12303 | n/a | A | — | F:B3 S:T2 | R5-1 | ### 1.3 Sizing bases and reservations | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| SZ1 | Explicit units (`Transact{signed}` after a grid floor pine_adapter.cpp:7049-7053, pine_adapter.cpp:7053) | **yes** — `Transact` in `OrderIntent` native_order.hpp:229 | K | — | F:C1 O:C1 S:S1 | — | -| SZ2 | Host override of the resolved price / units (`resolve_terms` pine_adapter.cpp:13726-14954) | **yes** — `HostSized` native_order.hpp:73-78 + `resolve_execution_terms` native_host.hpp:945-949, invoked `resolve_execution_terms` native_execution_consumer.cpp:5263, guarded by `consume_matched_request`'s `try` native_execution_consumer.cpp:5229-5263 | K | — | F:C2 O:H9 S:S2 | **Disagree:** S "partial" (a hook is not a basis). The hook is native; bases are SZ3-SZ5. It stays the custom-policy seam after L3. | -| SZ3 | Percent-of-equity sizing (`default_sizing_units` pine_adapter.cpp:3030-3094) | **no** — `HostSized` without host units is `TermsUnresolved` native_execution_consumer.cpp:5292-5294 **Closed:** `Sized{Side, EquityFraction{f}}` (`OrderIntent` native_order.hpp:229) is the sixth intent — the kernel resolves the units from a fraction of marked equity at the sizing point, so a bare host needs no terms override (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:C3 O:C2 S:S3 | TV parts stay A: the percentage's money rounded `source_money_round` pine_adapter.cpp:3024 (not the equity: R5 lane W5B-ENG-MARGIN-RESIDUAL, `tests/fixtures/margin_residual` w5b-sz-*), money floor-to-lot `source_money_floor_lot` pine_adapter.cpp:606 (→ SZ10) | -| SZ4 | Cash-value sizing (`default_qty_type` pine_adapter.cpp:2996-3005) | **no** — same **Closed:** `Sized{Side, CashValue{cash}}`, the same resolution over an account-currency amount (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:C2 O:C3 S:S4 | A per-call `qty_type` (cash or percent of equity on one `strategy.entry`) is not Pine — TradingView's compiler refuses it, so only generated or C++ callers reach it — and since R5 lane B-ADAPTER it converts through the kernel's `native_sized_units` exactly as the declared default does (`typed_quantity_units` pine_adapter.cpp:3100, `typed_entry_units` pine_adapter.cpp:3118; `tests/test_adapter_typed_entry_admission.cpp`) | -| SZ5 | Fee-net sizing (commission reserve `/(1+c)`, asked of the kernel as `reserve_percent_fee` pine_adapter.cpp:3079; `percent_commission_live_equity` pine_adapter.cpp:4899-4954) | **no** — only a dead primitive, `reserve_percent_commission` (since deleted), with no caller in `src/` **Closed:** `Sized::reserve_percent_fee` divides the sizing cash by `1 + fee_value / 100` under `NativeFeeKind::Percent` and is an exact no-op for every other fee kind; the dead `reserve_percent_commission` primitive is gone (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K | L3 | F:C3 O:C6 S:S3 | **Disagree:** F lists the reserve under quirk yet offers a knob / O kernel, "partial" / S knob. R5-1: fee-net option is generic. TradingView's cash-commission reserve stays adapter policy over the kernel's `CashValue`: a percent-of-equity default quantity under a cash-per-order or cash-per-contract fee takes its percentage of `strategy.equity` (`PineSizingSnapshot::strategy_equity`, the open entries' cash fees charged) and leaves out the fee its own order pays (`default_sizing_cash`, R5 lane PAR-CASHFEE, §3.10; `tests/test_cash_fee_sizing_tapes.cpp` on the tapes in `tests/fixtures/cash_fee_sizing`) | -| SZ6 | Placement-time vs fill-time sizing (`sizing_snapshot` pine_adapter.cpp:2942-2960, `PineSizingSnapshot` pine_adapter.hpp:157-169; fill-time exceptions `default_sized` pine_adapter.cpp:8066-8120) | **no** **Closed:** `SizeTime::AtMatch` (default) resolves at the matching candidate and `SizeTime::AtAcceptance` freezes the units against the command point, where the run's own opening admission also runs against the frozen quantity (`RequestRejectReason::PlacementAdmission`). Which price the basis converts at is `SizePrice::{Resolved, Signal, SignalOnTick}` (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K+A | L3 | F:C3 O:C7 S:S3 | **Disagree:** O adapter-only / F, S generic timing knob. R5-11 (L3 scope) + R5-1: the knob is kernel; TV's frozen tuple stays A through `HostSized`. | -| SZ7 | Quantity step | **yes** — `quantity_grid` admission native_run_spec.hpp:660, native_order.hpp:1421-1438; host-sized terms snap (`ExecutionGridPolicy` native_order.hpp:71-85); exact grid validation `execution_terms_grid_representable` native_execution_consumer.cpp:5309-5312; the book's own quantities are on the grid -- a whole-scope `ScopeFraction` and a `Reduce` of a FIFO boundary of its scope (R5 lane K-ULP4) | K | — | F:C4 O:C5 S:S1 | **Disagree:** F, O partial / S yes. R5-3: the quantity grid is native already. L3's `Sized` intent reuses the snap policy. | -| SZ8 | Fractional reduce of the position / a cohort (`quantize_close_units` pine_adapter.cpp:15482-15510, `compute_exit_reservation` pine_adapter.cpp:4988-5117) | **no** — sizes are `ExplicitUnits` / `OwnerOpenedUnits` / `HostSized` only native_order.hpp:60-74 **Closed:** `Reduce{ScopeFraction{fraction, claim, basis}}` takes a fraction in `(0, 1]` of the scope the reduce is bound to — the book, one opening, or a cohort roster (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:A7 O:A12 O:C4 S:S5 | F's "partial" counts the units-reduce of OL2. R5-4 → L3. | -| SZ9 | Exit-quantity reservation against sibling exits (`compute_exit_reservation` pine_adapter.cpp:4747-5111, `reserved_exit_qty` pine_adapter.cpp:10886-11362) | **partial** — `PointBudget` native_order.hpp:379-382 and group `Reduce` native_order.hpp:220 give a generic reservation; no net-of-siblings claim for fraction sizes **Closed:** `ScopeClaim::NetOfSiblings` subtracts the units already claimed by live sibling reduces bound to the same scope, and `ScopeBasis::{AtMatch, AtAcceptance}` chooses when that scope is measured (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:A14 S:S5 S:O10 O:A12 | F proposed nothing. R5-4: generic half → L3; the Pine ledger → SZ11, SZ12. | -| SZ10 | *(quirk)* Ten-significant-digit money rule (`source_money_round` pine_adapter.cpp:499-506, `source_money_floor_lot` pine_adapter.cpp:606; twin `tv_money_round` pine_policy_support.hpp:9-15); exact percent-exit rounding (`quantize_percent_exit_units` pine_adapter.cpp:5075-5090) | n/a — kernel money is plain binary64 `marked_equity` engine_execution.cpp:1638-1657 | A | — | O:C8 F:C3 S:M4 S:S5 | R5-1 | -| SZ10a | *(quirk, measured retention)* The money band's affordable price, `sig10(sig10(E) / notional per unit of price)` against the sizing price — at placement (`entry`, `affordable_price` pine_adapter.cpp:8333), in the paired-reversal whole drop (`close`, `affordable_price` pine_adapter.cpp:9845), at the terms boundary (`resolve_terms`, `affordable_price` pine_adapter.cpp:15991), in the precommit money scopes (`validate_precommit`, `affordable_price` pine_adapter.cpp:16522, pine_adapter.cpp:16575) and in the carried-money check (`apply_reversal_gap_bracket_policy`, `affordable_price` pine_adapter.cpp:21430) | n/a — not a units conversion. Every money-to-units conversion in the adapter is the kernel's `native_sized_units` under the source's floors since R5 lane B-ADAPTER: the default at placement and at the fill (`default_sizing_units` pine_adapter.cpp:3087), the typed per-call quantity at placement, at the fill and in the probe (`typed_quantity_units` pine_adapter.cpp:3100, `typed_entry_units` pine_adapter.cpp:3118), and the pending-order fill-quantity probe (`probe_fill_qty` pine_adapter.cpp:26379) | A | — | AUDIT3-opus2 H4 (G2 R2, R3) | **Measured:** the quotient is TradingView's whole-drop PRICE, pinned by 445 famr3 tapes (`tests/test_tv_money_band_l4b.cpp`) and 202 famr tapes (`tests/test_tv_money_precision_l4b.cpp`); on a seven-digit tie (E 9998.99999996, 10 000 lots at tick(close) 0.9999) the rule admits while the kernel's conversion buys 9999.99999996 lots and binary64 gives E/Q 0.999899999996 (`tests/test_adapter_fill_qty_probe.cpp`, section 2). | -| SZ11 | *(quirk)* Same-bar `strategy.close` callsite batching, two-call provenance, entry-id ledger (`enqueue_pooc_fifo_close` pine_adapter.cpp:8743-8982; ledger `close_batch_bar_` pine_adapter.hpp:2560-2566; the kernel's comment-only residue was deleted by R5 lanes E6 and E10) | n/a | A | — | S:O8 F:A15 O:A15 | **Disagree:** S "generic → kernel" (its P2 `ReplaceByKey` / `Fifo`). R5-4: adapter. | +| SZ1 | Explicit units (`Transact{signed}` after a grid floor pine_adapter.cpp:7057-7061, pine_adapter.cpp:7061) | **yes** — `Transact` in `OrderIntent` native_order.hpp:229 | K | — | F:C1 O:C1 S:S1 | — | +| SZ2 | Host override of the resolved price / units (`resolve_terms` pine_adapter.cpp:13726-14954) | **yes** — `HostSized` native_order.hpp:73-78 + `resolve_execution_terms` native_host.hpp:945-949, invoked `resolve_execution_terms` native_execution_consumer.cpp:5675, guarded by `consume_matched_request`'s `try` native_execution_consumer.cpp:5641-5675 | K | — | F:C2 O:H9 S:S2 | **Disagree:** S "partial" (a hook is not a basis). The hook is native; bases are SZ3-SZ5. It stays the custom-policy seam after L3. | +| SZ3 | Percent-of-equity sizing (`default_sizing_units` pine_adapter.cpp:3030-3094) | **no** — `HostSized` without host units is `TermsUnresolved` native_execution_consumer.cpp:5704-5706 **Closed:** `Sized{Side, EquityFraction{f}}` (`OrderIntent` native_order.hpp:229) is the sixth intent — the kernel resolves the units from a fraction of marked equity at the sizing point, so a bare host needs no terms override (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:C3 O:C2 S:S3 | TV parts stay A: the percentage's money rounded `source_money_round` pine_adapter.cpp:3030 (not the equity: R5 lane W5B-ENG-MARGIN-RESIDUAL, `tests/fixtures/margin_residual` w5b-sz-*), money floor-to-lot `source_money_floor_lot` pine_adapter.cpp:606 (→ SZ10) | +| SZ4 | Cash-value sizing (`default_qty_type` pine_adapter.cpp:3002-3011) | **no** — same **Closed:** `Sized{Side, CashValue{cash}}`, the same resolution over an account-currency amount (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:C2 O:C3 S:S4 | A per-call `qty_type` (cash or percent of equity on one `strategy.entry`) is not Pine — TradingView's compiler refuses it, so only generated or C++ callers reach it — and since R5 lane B-ADAPTER it converts through the kernel's `native_sized_units` exactly as the declared default does (`typed_quantity_units` pine_adapter.cpp:3106, `typed_entry_units` pine_adapter.cpp:3124; `tests/test_adapter_typed_entry_admission.cpp`) | +| SZ5 | Fee-net sizing (commission reserve `/(1+c)`, asked of the kernel as `reserve_percent_fee` pine_adapter.cpp:3085; `percent_commission_live_equity` pine_adapter.cpp:4907-4962) | **no** — only a dead primitive, `reserve_percent_commission` (since deleted), with no caller in `src/` **Closed:** `Sized::reserve_percent_fee` divides the sizing cash by `1 + fee_value / 100` under `NativeFeeKind::Percent` and is an exact no-op for every other fee kind; the dead `reserve_percent_commission` primitive is gone (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K | L3 | F:C3 O:C6 S:S3 | **Disagree:** F lists the reserve under quirk yet offers a knob / O kernel, "partial" / S knob. R5-1: fee-net option is generic. TradingView's cash-commission reserve stays adapter policy over the kernel's `CashValue`: a percent-of-equity default quantity under a cash-per-order or cash-per-contract fee takes its percentage of `strategy.equity` (`PineSizingSnapshot::strategy_equity`, the open entries' cash fees charged) and leaves out the fee its own order pays (`default_sizing_cash`, R5 lane PAR-CASHFEE, §3.10; `tests/test_cash_fee_sizing_tapes.cpp` on the tapes in `tests/fixtures/cash_fee_sizing`) | +| SZ6 | Placement-time vs fill-time sizing (`sizing_snapshot` pine_adapter.cpp:2948-2966, `PineSizingSnapshot` pine_adapter.hpp:157-169; fill-time exceptions `default_sized` pine_adapter.cpp:8066-8120) | **no** **Closed:** `SizeTime::AtMatch` (default) resolves at the matching candidate and `SizeTime::AtAcceptance` freezes the units against the command point, where the run's own opening admission also runs against the frozen quantity (`RequestRejectReason::PlacementAdmission`). Which price the basis converts at is `SizePrice::{Resolved, Signal, SignalOnTick}` (`tests/test_native_sizing_bases.cpp`, `tests/test_native_sizing_price_rule.cpp`). | K+A | L3 | F:C3 O:C7 S:S3 | **Disagree:** O adapter-only / F, S generic timing knob. R5-11 (L3 scope) + R5-1: the knob is kernel; TV's frozen tuple stays A through `HostSized`. | +| SZ7 | Quantity step | **yes** — `quantity_grid` admission native_run_spec.hpp:660, native_order.hpp:1421-1438; host-sized terms snap (`ExecutionGridPolicy` native_order.hpp:71-85); exact grid validation `execution_terms_grid_representable` native_execution_consumer.cpp:5721-5724; the book's own quantities are on the grid -- a whole-scope `ScopeFraction` and a `Reduce` of a FIFO boundary of its scope (R5 lane K-ULP4) | K | — | F:C4 O:C5 S:S1 | **Disagree:** F, O partial / S yes. R5-3: the quantity grid is native already. L3's `Sized` intent reuses the snap policy. | +| SZ8 | Fractional reduce of the position / a cohort (`quantize_close_units` pine_adapter.cpp:15490-15518, `compute_exit_reservation` pine_adapter.cpp:4996-5125) | **no** — sizes are `ExplicitUnits` / `OwnerOpenedUnits` / `HostSized` only native_order.hpp:60-74 **Closed:** `Reduce{ScopeFraction{fraction, claim, basis}}` takes a fraction in `(0, 1]` of the scope the reduce is bound to — the book, one opening, or a cohort roster (`tests/test_native_sizing_bases.cpp`). | K | L3 | F:A7 O:A12 O:C4 S:S5 | F's "partial" counts the units-reduce of OL2. R5-4 → L3. | +| SZ9 | Exit-quantity reservation against sibling exits (`compute_exit_reservation` pine_adapter.cpp:4755-5119, `reserved_exit_qty` pine_adapter.cpp:10894-11370) | **partial** — `PointBudget` native_order.hpp:379-382 and group `Reduce` native_order.hpp:220 give a generic reservation; no net-of-siblings claim for fraction sizes **Closed:** `ScopeClaim::NetOfSiblings` subtracts the units already claimed by live sibling reduces bound to the same scope, and `ScopeBasis::{AtMatch, AtAcceptance}` chooses when that scope is measured (`tests/test_native_sizing_bases.cpp`). | K+A | L3 | F:A14 S:S5 S:O10 O:A12 | F proposed nothing. R5-4: generic half → L3; the Pine ledger → SZ11, SZ12. | +| SZ10 | *(quirk)* Ten-significant-digit money rule (`source_money_round` pine_adapter.cpp:499-506, `source_money_floor_lot` pine_adapter.cpp:606; twin `tv_money_round` pine_policy_support.hpp:9-15); exact percent-exit rounding (`quantize_percent_exit_units` pine_adapter.cpp:5083-5098) | n/a — kernel money is plain binary64 `marked_equity` engine_execution.cpp:1638-1657 | A | — | O:C8 F:C3 S:M4 S:S5 | R5-1 | +| SZ10a | *(quirk, measured retention)* The money band's affordable price, `sig10(sig10(E) / notional per unit of price)` against the sizing price — at placement (`entry`, `affordable_price` pine_adapter.cpp:8341), in the paired-reversal whole drop (`close`, `affordable_price` pine_adapter.cpp:9853), at the terms boundary (`resolve_terms`, `affordable_price` pine_adapter.cpp:15999), in the precommit money scopes (`validate_precommit`, `affordable_price` pine_adapter.cpp:16530, pine_adapter.cpp:16583) and in the carried-money check (`apply_reversal_gap_bracket_policy`, `affordable_price` pine_adapter.cpp:21438) | n/a — not a units conversion. Every money-to-units conversion in the adapter is the kernel's `native_sized_units` under the source's floors since R5 lane B-ADAPTER: the default at placement and at the fill (`default_sizing_units` pine_adapter.cpp:3093), the typed per-call quantity at placement, at the fill and in the probe (`typed_quantity_units` pine_adapter.cpp:3106, `typed_entry_units` pine_adapter.cpp:3124), and the pending-order fill-quantity probe (`probe_fill_qty` pine_adapter.cpp:26423) | A | — | AUDIT3-opus2 H4 (G2 R2, R3) | **Measured:** the quotient is TradingView's whole-drop PRICE, pinned by 445 famr3 tapes (`tests/test_tv_money_band_l4b.cpp`) and 202 famr tapes (`tests/test_tv_money_precision_l4b.cpp`); on a seven-digit tie (E 9998.99999996, 10 000 lots at tick(close) 0.9999) the rule admits while the kernel's conversion buys 9999.99999996 lots and binary64 gives E/Q 0.999899999996 (`tests/test_adapter_fill_qty_probe.cpp`, section 2). | +| SZ11 | *(quirk)* Same-bar `strategy.close` callsite batching, two-call provenance, entry-id ledger (`enqueue_pooc_fifo_close` pine_adapter.cpp:8743-8982; ledger `close_batch_bar_` pine_adapter.hpp:2587-2593; the kernel's comment-only residue was deleted by R5 lanes E6 and E10) | n/a | A | — | S:O8 F:A15 O:A15 | **Disagree:** S "generic → kernel" (its P2 `ReplaceByKey` / `Fifo`). R5-4: adapter. | | SZ12 | *(quirk)* POOC reservation-growth market-entry population predicate (`select_reservation_growth_sources` src/compat/pine/reservation_expansion.cpp:9-34, called from the adapter, `select_reservation_growth_sources` pine_adapter.cpp:1263) | n/a | A | — | S:O10 | (single-source) R5-4 confirms S's own split: concept → SZ9, predicate A. | ### 1.4 Margin, admission, liquidation, risk | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| MG1 | Allowed opening directions (`allowed_open_directions` pine_adapter.cpp:2794) | **yes** — `allowed_open_directions` native_run_spec.hpp:665 → `MatchRejectReason::OpeningDirection` native_execution_consumer.cpp:3259-3267 | K | — | F:D4 O:D7 S:M1 | the one risk rule the adapter delegates (O) | -| MG2 | Max absolute position size | **yes** — `max_abs_units` native_run_spec.hpp:663 tests the resulting book native_execution_consumer.cpp:3273-3276 | K+A | — | F:D5 O:D5 S:M1 S:M5 | **Disagree:** O, S:M5 partial. R5-1: the generic cap is native; the adapter's live-position `>= held` variant (`max_position_size` pine_adapter.cpp:16267-16270) is TV. | -| MG3 | Open-lot cap (pyramiding) | **yes** — `max_open_lots` native_run_spec.hpp:664 → `MaxOpenLots` native_execution_consumer.cpp:3278-3281 | K+A | — | F:A5 O:A13 O:D6 S:M1 | **Disagree:** F, O:A13 partial / O:D6, S yes. R5-1: Pine's per-cycle entry count (`accepted_in_cycle` pine_adapter.cpp:7504-7525, cap left unset `max_open_lots` pine_adapter.cpp:2792-2794) is TV. **Measured (R5 lane H-MEASURE), and not TradingView's rule:** on 15 scenarios of three `lab tv` tapes (`tests/test_pyramiding_count_differential.cpp`) TradingView checks an entry once, at its first eligible point, against the trades then open; the kernel's `max_open_lots` books 14 as TradingView does, the adapter's count 13 (it keeps a slot a close or another entry's exit drained; since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap). The fifteenth, a resting entry accepted below the cap and filled at it, the kernel's cap refuses; the adapter books it since lane W8A-SIGSTATE-1's R-B, which also stopped the count from counting a resting LIMIT entry (a pending LIMIT entry does not count at a later call, and one the cap admitted fills over it: `tests/fixtures/pyramiding_open_order`). A recorded divergence, retained pending a re-lowering ruling. **Re-lowering tried by R5 lane H-THIN and not taken:** on the kernel's cap four corpus probes moved away from TradingView, for want of that first-eligible-point rule, so the count stays. The lane fixed the reservation divergence H-MEASURE measured beside it (P10): a default `strategy.exit(from_entry)` under FIFO reserved the whole book (200 in the adapter, 100 on TradingView's tape); it now reserves the entry's own quantity, the oldest lots first (`fifo_named_entry` pine_adapter.cpp:11346-11385). | -| MG4 | Initial-margin opening gate (adapter answers `AdmitWithHostMargin` from frozen signal-time equity pine_adapter.cpp:16143-16356 and leaves the kernel gate unset pine_adapter.cpp:16352-16354) | **yes** — `initial_margin_fraction` native_run_spec.hpp:666; the requirement `resulting_abs_notional * fraction` native_execution_consumer.cpp:3296 against marked equity. **Conversion instant, R5 follow-up lane E21:** both sites of the gate convert at their own point's rate — the candidate at its cursor, the placement gate at the acceptance point where a `Sized{SizeTime::AtAcceptance}` freezes its units — for the notional, the equity and a percent ticket, and the freeze's equity basis with them. The rate is threaded, never the clock: `make_command_context` native_execution_consumer.cpp:3142 hands the point's rate to the freeze, to `marked_equity_at` and to the engine's rate-explicit settlement inspection `inspect_native_settlement_scoped_at` engine.hpp:649, and the engine's presented clock is never written (`tests/test_native_margin_fx_clock.cpp` sections 8, 10 and 11) | K | — | F:D1 O:D1 S:M2 | **Disagree:** F, O partial (they fold per-side in) / S yes. Row split → MG5. **Measured (R5 lane H-MEASURE, M11):** the two gates part both ways on the same bars — the kernel's declines a 1× long whose entry fee the adapter admits, and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times (`tests/test_adapter_margin_schedule_differential.cpp`). **Since R5 lane TAIL-C** an explicit-quantity market buy from flat is costed at its signal close's tick plus the slippage ticks its fill carries, off `process_orders_on_close` (`tests/test_explicit_market_admission_tapes.cpp`), and an infinite `strategy.entry` quantity trades the default quantity, as na does (`tests/test_nonfinite_entry_qty_tape.cpp`). | -| MG5 | Per-side initial margin (`margin_pct` pine_adapter.cpp:17092-17093, `margin_pct` pine_adapter.cpp:16534) | **no** — one `std::optional` `initial_margin_fraction` native_run_spec.hpp:666 **Closed:** `NativeMarginModel::initial_long` / `initial_short` native_run_spec.hpp:242, opt-in and mutually exclusive with `initial_margin_fraction`; `initial_ == 0.0` is the maintenance-only spelling that leaves opening admission to the host at both of its gates (`tests/test_native_margin_model.cpp`). | K | L4 | F:D1 O:D2 | R5-1 | -| MG6 | Maintenance margin + forced liquidation (`submit_margin_call_slice` pine_adapter.cpp:17076-17359, `submit_margin_call_units` pine_adapter.cpp:18396-18444, `schedule_margin_call_path` pine_adapter.cpp:25425-25541) | **no** — "no maintenance liquidation" `initial_margin_fraction` native_run_spec.hpp:666-667; margin is checked only when `inspect.would_open` native_execution_consumer.cpp:3255 **Closed:** `maintenance_long` / `maintenance_short` with `NativeLiquidationCheck::{PathAdverseExtreme, PathAdverseExtremeMark, CalculationOnly}`; the kernel solves the level, rests its own `Reduce`/`Flatten` bound to the live book, re-prices it with `CancelReason::Superseded` and books a `MarginCallEvent` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | F:D2 O:D3 S:M3 | TV parts → MG15 | -| MG7 | Liquidation slice sizing (grid floor, ×4 restore, one-contract fallback `source_margin_units` pine_adapter.cpp:17303-17374) | **no** **Closed:** `NativeLiquidationSizing::{RestoreMinimum, ShortfallMultiple, Flatten}` with `shortfall_multiple` and `liquidation_min_units`, capped at the held quantity and floored onto `quantity_grid`; `resolve_margin_call_units` native_host.hpp:995 gives a host the last word, clamped into `(0, held]` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | O:D4 F:D2 S:M4 | **Disagree:** F generic knob `ShortfallMultiple` / O "×4 is A" / S quirk, yet a 4.0 default inside its generic model. R5-1: policy knob in the kernel, plain default (restore-minimum). **Decided by R5 lane F7 (the D2 question):** the adapter selects NO kernel knob — `project()` leaves `sizing` / `shortfall_multiple` / `liquidation_min_units` unset and its `resolve_margin_call_units` answers every call with TradingView's slice (the restore floored onto the lot grid BEFORE the ×4, floored again, the one-contract band, the dust gate). Measured: on a one-unit grid, 20 long at 100 on 1000 at 50 % reaching 93.9, TradingView books 4 lots and the kernel's own `ShortfallMultiple 4` books 5 (`tests/test_native_margin_hooks_twin.cpp` MG-F3, an executed run, no longer a comment); without a grid the two agree bit for bit (MG-F/H, 4.2105263157894735), but declaring the knob for those runs alone folds `NativeMarginModel::sizing` into every margin run's identity — an epoch decision F7 does not take. Since R5 lane H-THIN the pre-open admission slice (`schedule_preopen_margin_slice`) sizes through the same `source_margin_units`, where it kept its own copy of the ×4 without the band or the dust gates (AUDIT4 §6 G2-12; `tests/test_adapter_margin_schedule_differential.cpp` M10: band and fudge now 1 and 1, the kernel's and `ab9714be`'s). A four-times call on a whole-unit grid whose book's oldest lot is one unit is followed by one more unit at the next check point (`whole_unit_follow_up_due` pine_adapter.cpp:18900; R5 lane W5B-ENG-MARGIN-RESIDUAL, `tests/fixtures/margin_residual` w5b-es-*, w5b-sb-es-*, w5b-f-q1x12). With slippage the one-contract band takes its unit only where that unit restores the book at its own fill: the position's requirement at the liquidation's slipped print, less the equity at the mark, under one unit's margin (`source_margin_units` pine_adapter.cpp:17372; lane INT28-FIX rule OU, `tests/fixtures/slipped_short`). A one-unit call that leaves the book short at its own fill, by more than one unit's margin and two slippage steps with the book re-marked there, is followed by one more unit at the bar's next path point, where that point's own check calls nothing and its print is inside the call's fill (lane INT28-FIX rule OF). Under a percent commission a call of any size is followed so, on a fractional lot grid too: while the book the call left, re-marked at its fill, is short by more than two slippage steps for each unit called and the exit fees of those units and of one more, the next path point inside that fill takes four times the shortfall's lot floor over the fill, or one unit where that floors to none (`margin_follow_up_units` pine_adapter.cpp:18942, `follow_margin_call` pine_adapter.cpp:18991; R5 lane TAIL-I, `tests/fixtures/margin_open_print`). | +| MG1 | Allowed opening directions (`allowed_open_directions` pine_adapter.cpp:2800) | **yes** — `allowed_open_directions` native_run_spec.hpp:665 → `MatchRejectReason::OpeningDirection` native_execution_consumer.cpp:3592-3600 | K | — | F:D4 O:D7 S:M1 | the one risk rule the adapter delegates (O) | +| MG2 | Max absolute position size | **yes** — `max_abs_units` native_run_spec.hpp:663 tests the resulting book native_execution_consumer.cpp:3606-3609 | K+A | — | F:D5 O:D5 S:M1 S:M5 | **Disagree:** O, S:M5 partial. R5-1: the generic cap is native; the adapter's live-position `>= held` variant (`max_position_size` pine_adapter.cpp:16275-16278) is TV. | +| MG3 | Open-lot cap (pyramiding) | **yes** — `max_open_lots` native_run_spec.hpp:664 → `MaxOpenLots` native_execution_consumer.cpp:3611-3614 | K+A | — | F:A5 O:A13 O:D6 S:M1 | **Disagree:** F, O:A13 partial / O:D6, S yes. R5-1: Pine's per-cycle entry count (`accepted_in_cycle` pine_adapter.cpp:7512-7533, cap left unset `max_open_lots` pine_adapter.cpp:2798-2800) is TV. **Measured (R5 lane H-MEASURE), and not TradingView's rule:** on 15 scenarios of three `lab tv` tapes (`tests/test_pyramiding_count_differential.cpp`) TradingView checks an entry once, at its first eligible point, against the trades then open; the kernel's `max_open_lots` books 14 as TradingView does, the adapter's count 13 (it keeps a slot a close or another entry's exit drained; since lane W5B-ENG-MARGIN-RESIDUAL's SB the later market entries of a flat bar are refused at the fill once the book holds the cap). The fifteenth, a resting entry accepted below the cap and filled at it, the kernel's cap refuses; the adapter books it since lane W8A-SIGSTATE-1's R-B, which also stopped the count from counting a resting LIMIT entry (a pending LIMIT entry does not count at a later call, and one the cap admitted fills over it: `tests/fixtures/pyramiding_open_order`). A recorded divergence, retained pending a re-lowering ruling. **Re-lowering tried by R5 lane H-THIN and not taken:** on the kernel's cap four corpus probes moved away from TradingView, for want of that first-eligible-point rule, so the count stays. The lane fixed the reservation divergence H-MEASURE measured beside it (P10): a default `strategy.exit(from_entry)` under FIFO reserved the whole book (200 in the adapter, 100 on TradingView's tape); it now reserves the entry's own quantity, the oldest lots first (`fifo_named_entry` pine_adapter.cpp:11354-11393). | +| MG4 | Initial-margin opening gate (adapter answers `AdmitWithHostMargin` from frozen signal-time equity pine_adapter.cpp:16143-16356 and leaves the kernel gate unset pine_adapter.cpp:16352-16354) | **yes** — `initial_margin_fraction` native_run_spec.hpp:666; the requirement `resulting_abs_notional * fraction` native_execution_consumer.cpp:3629 against marked equity. **Conversion instant, R5 follow-up lane E21:** both sites of the gate convert at their own point's rate — the candidate at its cursor, the placement gate at the acceptance point where a `Sized{SizeTime::AtAcceptance}` freezes its units — for the notional, the equity and a percent ticket, and the freeze's equity basis with them. The rate is threaded, never the clock: `make_command_context` native_execution_consumer.cpp:3475 hands the point's rate to the freeze, to `marked_equity_at` and to the engine's rate-explicit settlement inspection `inspect_native_settlement_scoped_at` engine.hpp:649, and the engine's presented clock is never written (`tests/test_native_margin_fx_clock.cpp` sections 8, 10 and 11) | K | — | F:D1 O:D1 S:M2 | **Disagree:** F, O partial (they fold per-side in) / S yes. Row split → MG5. **Measured (R5 lane H-MEASURE, M11):** the two gates part both ways on the same bars — the kernel's declines a 1× long whose entry fee the adapter admits, and admits a gap-up add the adapter refuses against its signal-time equity; `ab9714be` books the adapter's answer both times (`tests/test_adapter_margin_schedule_differential.cpp`). **Since R5 lane TAIL-C** an explicit-quantity market buy from flat is costed at its signal close's tick plus the slippage ticks its fill carries, off `process_orders_on_close` (`tests/test_explicit_market_admission_tapes.cpp`), and an infinite `strategy.entry` quantity trades the default quantity, as na does (`tests/test_nonfinite_entry_qty_tape.cpp`). | +| MG5 | Per-side initial margin (`margin_pct` pine_adapter.cpp:17100-17101, `margin_pct` pine_adapter.cpp:16542) | **no** — one `std::optional` `initial_margin_fraction` native_run_spec.hpp:666 **Closed:** `NativeMarginModel::initial_long` / `initial_short` native_run_spec.hpp:242, opt-in and mutually exclusive with `initial_margin_fraction`; `initial_ == 0.0` is the maintenance-only spelling that leaves opening admission to the host at both of its gates (`tests/test_native_margin_model.cpp`). | K | L4 | F:D1 O:D2 | R5-1 | +| MG6 | Maintenance margin + forced liquidation (`submit_margin_call_slice` pine_adapter.cpp:17076-17359, `submit_margin_call_units` pine_adapter.cpp:18404-18452, `schedule_margin_call_path` pine_adapter.cpp:25443-25559) | **no** — "no maintenance liquidation" `initial_margin_fraction` native_run_spec.hpp:666-667; margin is checked only when `inspect.would_open` native_execution_consumer.cpp:3588 **Closed:** `maintenance_long` / `maintenance_short` with `NativeLiquidationCheck::{PathAdverseExtreme, PathAdverseExtremeMark, CalculationOnly}`; the kernel solves the level, rests its own `Reduce`/`Flatten` bound to the live book, re-prices it with `CancelReason::Superseded` and books a `MarginCallEvent` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | F:D2 O:D3 S:M3 | TV parts → MG15 | +| MG7 | Liquidation slice sizing (grid floor, ×4 restore, one-contract fallback `source_margin_units` pine_adapter.cpp:17311-17382) | **no** **Closed:** `NativeLiquidationSizing::{RestoreMinimum, ShortfallMultiple, Flatten}` with `shortfall_multiple` and `liquidation_min_units`, capped at the held quantity and floored onto `quantity_grid`; `resolve_margin_call_units` native_host.hpp:995 gives a host the last word, clamped into `(0, held]` (`tests/test_native_margin_model.cpp`, `tests/test_native_margin_hooks.cpp`). | K+A | L4 | O:D4 F:D2 S:M4 | **Disagree:** F generic knob `ShortfallMultiple` / O "×4 is A" / S quirk, yet a 4.0 default inside its generic model. R5-1: policy knob in the kernel, plain default (restore-minimum). **Decided by R5 lane F7 (the D2 question):** the adapter selects NO kernel knob — `project()` leaves `sizing` / `shortfall_multiple` / `liquidation_min_units` unset and its `resolve_margin_call_units` answers every call with TradingView's slice (the restore floored onto the lot grid BEFORE the ×4, floored again, the one-contract band, the dust gate). Measured: on a one-unit grid, 20 long at 100 on 1000 at 50 % reaching 93.9, TradingView books 4 lots and the kernel's own `ShortfallMultiple 4` books 5 (`tests/test_native_margin_hooks_twin.cpp` MG-F3, an executed run, no longer a comment); without a grid the two agree bit for bit (MG-F/H, 4.2105263157894735), but declaring the knob for those runs alone folds `NativeMarginModel::sizing` into every margin run's identity — an epoch decision F7 does not take. Since R5 lane H-THIN the pre-open admission slice (`schedule_preopen_margin_slice`) sizes through the same `source_margin_units`, where it kept its own copy of the ×4 without the band or the dust gates (AUDIT4 §6 G2-12; `tests/test_adapter_margin_schedule_differential.cpp` M10: band and fudge now 1 and 1, the kernel's and `ab9714be`'s). A four-times call on a whole-unit grid whose book's oldest lot is one unit is followed by one more unit at the next check point (`whole_unit_follow_up_due` pine_adapter.cpp:18908; R5 lane W5B-ENG-MARGIN-RESIDUAL, `tests/fixtures/margin_residual` w5b-es-*, w5b-sb-es-*, w5b-f-q1x12). With slippage the one-contract band takes its unit only where that unit restores the book at its own fill: the position's requirement at the liquidation's slipped print, less the equity at the mark, under one unit's margin (`source_margin_units` pine_adapter.cpp:17380; lane INT28-FIX rule OU, `tests/fixtures/slipped_short`). A one-unit call that leaves the book short at its own fill, by more than one unit's margin and two slippage steps with the book re-marked there, is followed by one more unit at the bar's next path point, where that point's own check calls nothing and its print is inside the call's fill (lane INT28-FIX rule OF). Under a percent commission a call of any size is followed so, on a fractional lot grid too: while the book the call left, re-marked at its fill, is short by more than two slippage steps for each unit called and the exit fees of those units and of one more, the next path point inside that fill takes four times the shortfall's lot floor over the fill, or one unit where that floors to none (`margin_follow_up_units` pine_adapter.cpp:18950, `follow_margin_call` pine_adapter.cpp:18999; R5 lane TAIL-I, `tests/fixtures/margin_open_print`). | | MG8 | Liquidation price query (`margin_liquidation_price` pine_strategy_host.cpp:168-184) | **no** **Closed:** `native_liquidation_price()` native_host.hpp:1320 answers the exact solved level, `nullopt` where none exists; `NativeLiquidationLevelBase` chooses whether the solve answers for the open entries' fees (`tests/test_native_margin_model.cpp`, `tests/test_native_open_lots.cpp`). | K+A | L4 | F:D3 S:R5 | **Disagree:** F generic / S adapter-only. R5-1: liquidation price is generic; the Pine spelling stays A. **Lowered by R5 lane H-THIN:** the Pine host's second solve (`compute_liquidation_price`) is gone; `strategy.margin_liquidation_price` reads `native_liquidation_price()`, which the adapter configures with `NativeLiquidationLevelBase::RealizedOnly` (the Pine formula's closed-money solve, bit for bit at every bar of the lane's 1,021 seeded margin-call runs), and keeps only TradingView's tick spelling: floored for a long, ceiled for a short. A run whose margin calls are switched off declares no margin model, so it reads na. | -| MG9 | FX revaluation of margin (`apply_fx_open_margin_slice` pine_adapter.cpp:22182-22258, `apply_fx_opening_margin_slice` pine_adapter.cpp:25587-25615) | **no** — staged FX curve `configure_native_fx_curve` native_host.hpp:1202 but no margin re-check. **Closed by audit lane N6:** `NativeMarginCheckKind::FxRoll` — a step of the declared curve is a check point of its own, measured at the unchanged price before the first point the new rate converts (`tests/test_native_margin_fx_roll.cpp`). The adapter refuses the point by kind, for a measured reason since R5 lane F7: TradingView's rollover is a broker-open checkpoint at the open price, now at any positive margin (`apply_fx_open_margin_slice`, on the one TradingView money / units rule every other checkpoint uses), while the roll point measures at the remaining path's adverse mark and rests the slice there — a carried 1x long on a step bar O 100 L 97 books 0.4116 @ 97 on the adapter's run shape and 0.4036 @ 99, measured a bar early, on a strict calendar, against TradingView's 0.3996 @ 100 (`tests/test_native_margin_hooks_twin.cpp` MG-FX). Both N6 follow-ups are closed (§4.4). **Completed by R5 follow-up lane E3:** the conversion INSTANT is now generic too — every check kind converts at its own cursor's rate (`account_currency_fx_at(cursor.point.effective_time_ms)`) for the requirement, the equity and the level, in batch and stream, so a step landing between a fill's cursor and the clock the engine presents no longer breaches at the fill (`tests/test_native_margin_fx_clock.cpp`; N6's local clock swap inside `fx_roll_margin_check` is removed as redundant). **Extended by R5 follow-up lane E21** to the opening gate's placement site (MG4), and the rate-explicit equity E3 restated beside the engine's accessor is now that accessor's one implementation, `BacktestEngine::marked_equity_at` engine.hpp:729 | K | L4 → N6 → E3 → E21 | O:D8 | (single-source) | +| MG9 | FX revaluation of margin (`apply_fx_open_margin_slice` pine_adapter.cpp:22200-22276, `apply_fx_opening_margin_slice` pine_adapter.cpp:25605-25633) | **no** — staged FX curve `configure_native_fx_curve` native_host.hpp:1202 but no margin re-check. **Closed by audit lane N6:** `NativeMarginCheckKind::FxRoll` — a step of the declared curve is a check point of its own, measured at the unchanged price before the first point the new rate converts (`tests/test_native_margin_fx_roll.cpp`). The adapter refuses the point by kind, for a measured reason since R5 lane F7: TradingView's rollover is a broker-open checkpoint at the open price, now at any positive margin (`apply_fx_open_margin_slice`, on the one TradingView money / units rule every other checkpoint uses), while the roll point measures at the remaining path's adverse mark and rests the slice there — a carried 1x long on a step bar O 100 L 97 books 0.4116 @ 97 on the adapter's run shape and 0.4036 @ 99, measured a bar early, on a strict calendar, against TradingView's 0.3996 @ 100 (`tests/test_native_margin_hooks_twin.cpp` MG-FX). Both N6 follow-ups are closed (§4.4). **Completed by R5 follow-up lane E3:** the conversion INSTANT is now generic too — every check kind converts at its own cursor's rate (`account_currency_fx_at(cursor.point.effective_time_ms)`) for the requirement, the equity and the level, in batch and stream, so a step landing between a fill's cursor and the clock the engine presents no longer breaches at the fill (`tests/test_native_margin_fx_clock.cpp`; N6's local clock swap inside `fx_roll_margin_check` is removed as redundant). **Extended by R5 follow-up lane E21** to the opening gate's placement site (MG4), and the rate-explicit equity E3 restated beside the engine's accessor is now that accessor's one implementation, `BacktestEngine::marked_equity_at` engine.hpp:729 | K | L4 → N6 → E3 → E21 | O:D8 | (single-source) | | MG10 | Max drawdown halt (`update_risk_state` pine_adapter.cpp:23845-23905) | **no** — observation field only `drawdown_limit` include/pineforge/source/market_admission.hpp:50; composable today from `native_marked_equity` native_host.hpp:1301 **Closed:** `NativeRiskLimits::max_drawdown` native_run_spec.hpp:311, a `NativeLossLimit{value, percent}` measured against the running peak equity at three points of every script bar, blocking openings for the rest of the run (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E1 S:M6 | R5-9 | -| MG11 | Max consecutive losing days (`SourceDayLedger` pine_adapter.hpp:1419-1427; gate `max_cons_loss_days` pine_adapter.cpp:16563-16566) | **no** — observation only `loss_days_limit` include/pineforge/source/market_admission.hpp:49 **Closed:** `NativeRiskLimits::max_consecutive_loss_days`, settled from each day's realized result at the next day's open (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E2 S:M6 | R5-9 | -| MG12 | Max intraday loss → flatten + block (`intraday_loss_breached` pine_adapter.cpp:17275-17289, forced flatten `submit_intraday_loss_close` pine_adapter.cpp:20597-20627) | **no** — observation only `intraday_loss_limit` include/pineforge/source/market_admission.hpp:51 **Closed:** `NativeRiskLimits::max_intraday_loss` with `NativeRiskAction::FlattenAndBlock`, under which the kernel issues one `Flatten` of its own at the breach point (`RequestOrigin::KernelRisk`) and latches the block before that flatten settles (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D7 O:E3 S:M6 | R5-9 | -| MG13 | Max filled orders per day (`IntradayCap` intraday_cap.hpp:84-263, `IntradayOrderBudget` intraday_order_budget.hpp:27-92; `execute_due_cap_close` pine_adapter.cpp:20046-20670) | **no** **Closed:** `NativeRiskLimits::max_fills_per_day` counts every applied execution of the day as it settles, whatever issued it (`tests/test_native_risk_limits.cpp`). | K+A | L9 | F:D8 O:E4 S:M7 | Count cap K; quota transfer, POOC deferral, TV close price A | -| MG14 | Risk-day boundary (`chart_day_key` pine_adapter.cpp:17036-17109) | **no** — the calendar exists `session_day_ordinal` native_calendar.hpp:416-419, there is no risk-day ledger **Closed:** `NativeRiskDay::{SessionDay, CalendarDayInTimezone}` keys the risk day, and `native_risk_state()` native_host.hpp:1329 reads the whole ledger back — block and reason, day, fills counted, loss-day streak, peak equity, day-opening equity (`tests/test_native_risk_limits.cpp`). | K+A | L9 | O:E5 S:M7 | The chart-day shortcut stays A; the kernel keys on session / calendar day | -| MG15 | *(quirk)* TV margin call: money rounding `source_margin_money` pine_adapter.cpp:17317, 4× shortfall `4.0 * minimum` pine_adapter.cpp:17406, POOC chronology exception `resting_order` pine_adapter.cpp:20001-20016 (the commissioned carried short; the competing-entry and rounded-money exceptions retired by R5 lane W5B-ENG-MARGIN-RESIDUAL, `tests/fixtures/margin_residual` w5b-pc-*), 1×-long money call `submit_tv_money_long_margin_call` pine_adapter.cpp:17563-17693, fill pricing (the slice books at its waypoint's mark, `submit_tv_money_long_margin_call` pine_adapter.cpp:18310) | n/a | A | — | F:D2 F:G4 O:D4 S:M4 | R5-1. **Measured (R5 lane H-MEASURE)** against the kernel's own points on the same bars and orders, the kernel's hooks answering TradingView's money and slice (`tests/test_adapter_margin_schedule_differential.cpp`): a gap-open breach executes at the open, sized on the open's money, where the kernel's mark check rests at the path's adverse extreme and `CalculationOnly` executes at the calculation (M8); the pre-open slice parts from the kernel's `AfterApplied` point, and from `ab9714be`, on the one-contract band and the restore's `+1e-6` floor (M10; the frozen units it also parted on are TradingView's book since R5 lane PAR-MARGIN froze a default-percent stop above 100 % as eight `lab tv` tapes, `tests/fixtures/margin_entry_bar/pm-m10-*`, show TradingView does); the entry bar of a leveraged opening, where four `lab tv` tapes (`tests/fixtures/margin_entry_bar`) and the kernel's `AfterApplied` point book the margin call at that bar's low, is admitted by `on_applied` since R5 lane PAR-MARGIN on the run shapes `ab9714be` checked it on (the pre-open slice's own opening keeps that slice's verdict), and the adapter books the tapes' rows (M7; H-MEASURE had recorded it as a divergence); R5 lane PAR-MARGIN-2 extended it to limit openings, `process_orders_on_close`, `calc_on_order_fills` and adds to a carried book, and made the kernel's post-fill point reach the low a limit was filled on the way down to, or a short's high on its way up (twenty-eight `lab tv` tapes, `tests/fixtures/margin_entry_bar/pm2-*`). Under the magnifier the kernel's `IntrabarSample` point (R5 lane PAR-MARGIN) is admitted on a leveraged long, plain, under `process_orders_on_close` or `calc_on_order_fills`, and on a short outside `process_orders_on_close` (PAR-MARGIN-2), where TradingView books its call at the first lower-timeframe low that crosses -- at its own 2-minute intrabars on a 15-minute chart, which a model reproduces on all 20 magnified tapes (`tests/fixtures/intrabar_margin`); six tapes part on the one-minute samples the host feeds, a recorded divergence of the magnifier's sampling. A margin call books the nearest tick of its print, as TradingView's market fills do (12.105 books 12.11; `tests/fixtures/half_tick_rounding`, PAR-MARGIN-2). | -| MG16 | *(quirk)* Historical market-admission review / retention journal (`explicit_pair_scope` src/compat/pine/market_admission.cpp:34 and the review fold it feeds (`original_pair_call` src/compat/pine/market_admission.cpp:48); `admission_allocation` pine_adapter.cpp:4344-4412, `record_market_review` pine_adapter.cpp:20866-20926) | n/a | A | — | S:R2 | (single-source) | +| MG11 | Max consecutive losing days (`SourceDayLedger` pine_adapter.hpp:1419-1427; gate `max_cons_loss_days` pine_adapter.cpp:16571-16574) | **no** — observation only `loss_days_limit` include/pineforge/source/market_admission.hpp:49 **Closed:** `NativeRiskLimits::max_consecutive_loss_days`, settled from each day's realized result at the next day's open (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D6 O:E2 S:M6 | R5-9 | +| MG12 | Max intraday loss → flatten + block (`intraday_loss_breached` pine_adapter.cpp:17283-17297, forced flatten `submit_intraday_loss_close` pine_adapter.cpp:20605-20635) | **no** — observation only `intraday_loss_limit` include/pineforge/source/market_admission.hpp:51 **Closed:** `NativeRiskLimits::max_intraday_loss` with `NativeRiskAction::FlattenAndBlock`, under which the kernel issues one `Flatten` of its own at the breach point (`RequestOrigin::KernelRisk`) and latches the block before that flatten settles (`tests/test_native_risk_limits.cpp`). | K | L9 | F:D7 O:E3 S:M6 | R5-9 | +| MG13 | Max filled orders per day (`IntradayCap` intraday_cap.hpp:84-263, `IntradayOrderBudget` intraday_order_budget.hpp:27-92; `execute_due_cap_close` pine_adapter.cpp:20054-20678) | **no** **Closed:** `NativeRiskLimits::max_fills_per_day` counts every applied execution of the day as it settles, whatever issued it (`tests/test_native_risk_limits.cpp`). | K+A | L9 | F:D8 O:E4 S:M7 | Count cap K; quota transfer, POOC deferral, TV close price A | +| MG14 | Risk-day boundary (`chart_day_key` pine_adapter.cpp:17044-17117) | **no** — the calendar exists `session_day_ordinal` native_calendar.hpp:416-419, there is no risk-day ledger **Closed:** `NativeRiskDay::{SessionDay, CalendarDayInTimezone}` keys the risk day, and `native_risk_state()` native_host.hpp:1329 reads the whole ledger back — block and reason, day, fills counted, loss-day streak, peak equity, day-opening equity (`tests/test_native_risk_limits.cpp`). | K+A | L9 | O:E5 S:M7 | The chart-day shortcut stays A; the kernel keys on session / calendar day | +| MG15 | *(quirk)* TV margin call: money rounding `source_margin_money` pine_adapter.cpp:17325, 4× shortfall `4.0 * minimum` pine_adapter.cpp:17414, POOC chronology exception `resting_order` pine_adapter.cpp:20009-20024 (the commissioned carried short; the competing-entry and rounded-money exceptions retired by R5 lane W5B-ENG-MARGIN-RESIDUAL, `tests/fixtures/margin_residual` w5b-pc-*), 1×-long money call `submit_tv_money_long_margin_call` pine_adapter.cpp:17571-17701, fill pricing (the slice books at its waypoint's mark, `submit_tv_money_long_margin_call` pine_adapter.cpp:18318) | n/a | A | — | F:D2 F:G4 O:D4 S:M4 | R5-1. **Measured (R5 lane H-MEASURE)** against the kernel's own points on the same bars and orders, the kernel's hooks answering TradingView's money and slice (`tests/test_adapter_margin_schedule_differential.cpp`): a gap-open breach executes at the open, sized on the open's money, where the kernel's mark check rests at the path's adverse extreme and `CalculationOnly` executes at the calculation (M8); the pre-open slice parts from the kernel's `AfterApplied` point, and from `ab9714be`, on the one-contract band and the restore's `+1e-6` floor (M10; the frozen units it also parted on are TradingView's book since R5 lane PAR-MARGIN froze a default-percent stop above 100 % as eight `lab tv` tapes, `tests/fixtures/margin_entry_bar/pm-m10-*`, show TradingView does); the entry bar of a leveraged opening, where four `lab tv` tapes (`tests/fixtures/margin_entry_bar`) and the kernel's `AfterApplied` point book the margin call at that bar's low, is admitted by `on_applied` since R5 lane PAR-MARGIN on the run shapes `ab9714be` checked it on (the pre-open slice's own opening keeps that slice's verdict), and the adapter books the tapes' rows (M7; H-MEASURE had recorded it as a divergence); R5 lane PAR-MARGIN-2 extended it to limit openings, `process_orders_on_close`, `calc_on_order_fills` and adds to a carried book, and made the kernel's post-fill point reach the low a limit was filled on the way down to, or a short's high on its way up (twenty-eight `lab tv` tapes, `tests/fixtures/margin_entry_bar/pm2-*`). Under the magnifier the kernel's `IntrabarSample` point (R5 lane PAR-MARGIN) is admitted on a leveraged long, plain, under `process_orders_on_close` or `calc_on_order_fills`, and on a short outside `process_orders_on_close` (PAR-MARGIN-2), where TradingView books its call at the first lower-timeframe low that crosses -- at its own 2-minute intrabars on a 15-minute chart, which a model reproduces on all 20 magnified tapes (`tests/fixtures/intrabar_margin`); six tapes part on the one-minute samples the host feeds, a recorded divergence of the magnifier's sampling. A margin call books the nearest tick of its print, as TradingView's market fills do (12.105 books 12.11; `tests/fixtures/half_tick_rounding`, PAR-MARGIN-2). | +| MG16 | *(quirk)* Historical market-admission review / retention journal (`explicit_pair_scope` src/compat/pine/market_admission.cpp:34 and the review fold it feeds (`original_pair_call` src/compat/pine/market_admission.cpp:48); `admission_allocation` pine_adapter.cpp:4352-4420, `record_market_review` pine_adapter.cpp:20874-20934) | n/a | A | — | S:R2 | (single-source) | ### 1.5 Calculation timing | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| CT1 | Calculate at script-bar close | **yes** — `on_native_bar` native_host.hpp:893, dispatched by `invoke_callback` native_execution_consumer.cpp:7533-7566 (call site `on_native_recalculate` native_execution_consumer.cpp:7585) | K | — | F:E1 O:F1 | — | -| CT2 | Process orders on close (`close_execution` pine_adapter.cpp:2750-2751) | **yes** — `NativeCloseExecution::AfterCalculation` native_run_spec.hpp:35-38; `AfterCalculation` native_execution_consumer.cpp:7996-8000 | K | — | F:E2 O:F6 S:C1 | — | -| CT3 | Pre-match hook at bar open (`on_bar_open` pine_adapter.cpp:22011) | **yes** — `on_native_bar_open` native_host.hpp:889 runs before the open match native_execution_consumer.cpp:7434-7438; `execute_current` legal native_host.hpp:1104 | K | — | F:E3 O:F2 | hazard → CT4 | -| CT4 | Partial-bar view / open-bar lookahead guard | **no** — the open hook receives the *complete* script bar: `invoke_bar_open_callback(engine, bar, point)` native_execution_consumer.cpp:7377 with `engine.current_bar_ = open_view` native_execution_consumer.cpp:7428 (the complete bar unless `OpenOnly`) **Closed:** `current_partial_bar()` native_host.hpp:1073 is the lookahead-free bar so far, through the last path point consumed (a fill inside a segment reads it at the segment's origin), and `NativeOpenBarView::OpenOnly` native_run_spec.hpp:109 masks `on_native_bar_open`'s bar to `H = L = C = open` as presentation only — the complete bar is restored before the open match (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E3 | (single-source) The adapter needs the full bar, so the guard is opt-in. | +| CT1 | Calculate at script-bar close | **yes** — `on_native_bar` native_host.hpp:893, dispatched by `invoke_callback` native_execution_consumer.cpp:7981-8014 (call site `on_native_recalculate` native_execution_consumer.cpp:8036) | K | — | F:E1 O:F1 | — | +| CT2 | Process orders on close (`close_execution` pine_adapter.cpp:2756-2757) | **yes** — `NativeCloseExecution::AfterCalculation` native_run_spec.hpp:35-38; `AfterCalculation` native_execution_consumer.cpp:8448-8452 | K | — | F:E2 O:F6 S:C1 | — | +| CT3 | Pre-match hook at bar open (`on_bar_open` pine_adapter.cpp:22029) | **yes** — `on_native_bar_open` native_host.hpp:889 runs before the open match native_execution_consumer.cpp:7882-7886; `execute_current` legal native_host.hpp:1104 | K | — | F:E3 O:F2 | hazard → CT4 | +| CT4 | Partial-bar view / open-bar lookahead guard | **no** — the open hook receives the *complete* script bar: `invoke_bar_open_callback(engine, bar, point)` native_execution_consumer.cpp:7825 with `engine.current_bar_ = open_view` native_execution_consumer.cpp:7876 (the complete bar unless `OpenOnly`) **Closed:** `current_partial_bar()` native_host.hpp:1073 is the lookahead-free bar so far, through the last path point consumed (a fill inside a segment reads it at the segment's origin), and `NativeOpenBarView::OpenOnly` native_run_spec.hpp:109 masks `on_native_bar_open`'s bar to `H = L = C = open` as presentation only — the complete bar is restored before the open match (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E3 | (single-source) The adapter needs the full bar, so the guard is opt-in. | | CT5 | Raw input observation before aggregation | **yes** — `on_native_input` native_host.hpp:875, `NativeInputContext` native_host.hpp:773-778 | K | — | O:F3 | (single-source row) | -| CT6 | Calculate on order fills (scheduler `PineScheduler::recalculate` pine_scheduler_native.cpp:671, `coof_recalculation_due` pine_scheduler_native.cpp:637, the first-open chain `kFirstOpenLoopGuard` pine_scheduler_native.cpp:792; `begin_coof_recalc` pine_adapter.cpp:6535, `flush_coof_tail` pine_adapter.cpp:22235, `end_coof_recalc` pine_adapter.cpp:6592) | **partial** — `on_native_applied` native_host.hpp:938 fires mid-path after each fill native_execution_consumer.cpp:7281, native_execution_consumer.cpp:7267-7297; commands are legal there `commands_allowed` native_execution_consumer.cpp:1512-1520; a request born there is eligible on the unconsumed rest of the bar `born_on_remaining_path` native_execution_consumer.cpp:5916-5950, `remaining_path_delivery` native_order.cpp:2061-2069. Missing: calculation re-entry, a documented chronology, a bar-so-far view **Closed:** `NativeCalculationTrigger::BarCloseAndFills` native_run_spec.hpp:96 drives one `OrderFill` recalculation per applied execution at that event's own cursor, delivered as `on_native_recalculate(..., OrderFill, cause)` and bounded by `max_recalculations_per_point` (`tests/test_native_calc_timing.cpp`). | K+A | L5 | F:E4 O:F5 S:C2 | **Disagree:** F partial / O, S no; F, S v1 / O deferrable. R5-5: v1, sub-bar hook + chronology contract, adapter path unchanged. TV parts → CT11. | -| CT7 | Calculate on every tick | **partial** — `on_native_tick` native_host.hpp:878 fires per accepted realtime print native_execution_consumer.cpp:7516-7520; no per-tick calculation in batch; the guide pinned "close-only" (`docs/pages/native-engine.md`, "Batch OHLCV vs ticks vs quiet") **Closed:** `NativeCalculationTrigger::EveryModeledPoint` native_run_spec.hpp:97 adds one `Tick` recalculation at every modeled point of the delivered path and at every observed print, in batch and in a stream alike (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E5 O:F4 S:C3 | **Disagree:** F yes / O partial / S no. R5-5: v1. Not an adapter feature either: absent from `PineStrategyConfig` pine_adapter.hpp:62-76, the runner rejects it (`calc_on_every_tick` main.cpp:259), the Pine stream route refuses `calc_on_order_fills` pine_strategy_host.cpp:277-279. | -| CT8 | Historical intrabar calculation (per lower-TF sub-bar / magnifier sample) | **no** — with an `IntrabarPath` the callback still fires once per script bar native_execution_consumer.cpp:7947-8089 **Closed:** `EveryModeledPoint` recalculates at each intrabar sample, and `on_native_sub_bar` native_host.hpp:928 fires once after each retained lower-timeframe sub-bar's whole matching path (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E6 O:F8 S:C4 | — | -| CT9 | Intrabar (magnifier) matching path + path-order policy (`volume_weighted_cap` pine_adapter.cpp:2814-2870) | **yes** — `IntrabarPath` native_run_spec.hpp:362-403, delivery `deliver_confirmed_script` native_execution_consumer.cpp:7732-7998; `NativePathOrder` native_run_spec.hpp:335-339 | K+A | — | F:E7 F:G5 O:F7 S:C4 | **Disagree:** S partial (it wants callback cadence → CT8). TV sampler choice is A: TradingView's own intrabars at its table's timeframe, each owned by the chart bar of its last minute, built from the finer feed (`tradingview_magnifier_bars` magnifier_intrabars.hpp:44, R5 lane MAG-INTRABAR). | -| CT10 | Batch → warmup → realtime forward execution | **yes** — `stream_begin` engine.hpp:1771, `stream_push_bar` engine.hpp:1777, `stream_end` engine.hpp:1781, `NativeRunPhase` native_host.hpp:41, C ABI `strategy_stream_begin` c_abi.cpp:658 | K+A | — | F:E8 O:J3 S:C5 | **Disagree:** S partial. R5-1: the lifecycle is native; Pine trade-start suppression and realtime-tail / probe rules (`trading_window_active` pine_strategy_commands.cpp:15-21, `prepare_native_begin` pine_strategy_host.cpp:206-307) are A. S's "neutral tail + trade-window policy" is single-source and unassigned (§4 Q8). | -| CT11 | *(quirk)* COOF TV specifics: waypoint-only refill (`coof_next_waypoint` pine_adapter.cpp:6808), two-fills-at-open rule, script-state rollback (`PineScheduler` pine_scheduler_native.cpp:54-106; hooks `snapshot_script_state` engine.hpp:1332-1333), the first-open guard `kFirstOpenLoopGuard` pine_scheduler_native.cpp:792, deferral queue of `PendingCoofRequest` pine_adapter.hpp:1906-1936 | n/a | A | — | F:E4 O:F5 S:C2 | R5-5: the adapter keeps its path | +| CT6 | Calculate on order fills (scheduler `PineScheduler::recalculate` pine_scheduler_native.cpp:727, `coof_recalculation_due` pine_scheduler_native.cpp:693, the first-open chain `kFirstOpenLoopGuard` pine_scheduler_native.cpp:848; `begin_coof_recalc` pine_adapter.cpp:6543, `flush_coof_tail` pine_adapter.cpp:22253, `end_coof_recalc` pine_adapter.cpp:6600) | **partial** — `on_native_applied` native_host.hpp:938 fires mid-path after each fill native_execution_consumer.cpp:7729, native_execution_consumer.cpp:7729-7759; commands are legal there `commands_allowed` native_execution_consumer.cpp:1807-1819; a request born there is eligible on the unconsumed rest of the bar `born_on_remaining_path` native_execution_consumer.cpp:6336-6370, `remaining_path_delivery` native_order.cpp:2082-2090. Missing: calculation re-entry, a documented chronology, a bar-so-far view **Closed:** `NativeCalculationTrigger::BarCloseAndFills` native_run_spec.hpp:96 drives one `OrderFill` recalculation per applied execution at that event's own cursor, delivered as `on_native_recalculate(..., OrderFill, cause)` and bounded by `max_recalculations_per_point` (`tests/test_native_calc_timing.cpp`). | K+A | L5 | F:E4 O:F5 S:C2 | **Disagree:** F partial / O, S no; F, S v1 / O deferrable. R5-5: v1, sub-bar hook + chronology contract, adapter path unchanged. TV parts → CT11. | +| CT7 | Calculate on every tick | **partial** — `on_native_tick` native_host.hpp:878 fires per accepted realtime print native_execution_consumer.cpp:7964-7968; no per-tick calculation in batch; the guide pinned "close-only" (`docs/pages/native-engine.md`, "Batch OHLCV vs ticks vs quiet") **Closed:** `NativeCalculationTrigger::EveryModeledPoint` native_run_spec.hpp:97 adds one `Tick` recalculation at every modeled point of the delivered path and at every observed print, in batch and in a stream alike (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E5 O:F4 S:C3 | **Disagree:** F yes / O partial / S no. R5-5: v1. Not an adapter feature either: absent from `PineStrategyConfig` pine_adapter.hpp:62-76, the runner rejects it (`calc_on_every_tick` main.cpp:259), the Pine stream route refuses `calc_on_order_fills` pine_strategy_host.cpp:279-281. | +| CT8 | Historical intrabar calculation (per lower-TF sub-bar / magnifier sample) | **no** — with an `IntrabarPath` the callback still fires once per script bar native_execution_consumer.cpp:8399-8541 **Closed:** `EveryModeledPoint` recalculates at each intrabar sample, and `on_native_sub_bar` native_host.hpp:928 fires once after each retained lower-timeframe sub-bar's whole matching path (`tests/test_native_calc_timing.cpp`). | K | L5 | F:E6 O:F8 S:C4 | — | +| CT9 | Intrabar (magnifier) matching path + path-order policy (`volume_weighted_cap` pine_adapter.cpp:2820-2876) | **yes** — `IntrabarPath` native_run_spec.hpp:362-403, delivery `deliver_confirmed_script` native_execution_consumer.cpp:8184-8450; `NativePathOrder` native_run_spec.hpp:335-339 | K+A | — | F:E7 F:G5 O:F7 S:C4 | **Disagree:** S partial (it wants callback cadence → CT8). TV sampler choice is A: TradingView's own intrabars at its table's timeframe, each owned by the chart bar of its last minute, built from the finer feed (`tradingview_magnifier_bars` magnifier_intrabars.hpp:44, R5 lane MAG-INTRABAR). | +| CT10 | Batch → warmup → realtime forward execution | **yes** — `stream_begin` engine.hpp:1771, `stream_push_bar` engine.hpp:1777, `stream_end` engine.hpp:1781, `NativeRunPhase` native_host.hpp:41, C ABI `strategy_stream_begin` c_abi.cpp:698 | K+A | — | F:E8 O:J3 S:C5 | **Disagree:** S partial. R5-1: the lifecycle is native; Pine trade-start suppression and realtime-tail / probe rules (`trading_window_active` pine_strategy_commands.cpp:15-21, `prepare_native_begin` pine_strategy_host.cpp:206-307) are A. S's "neutral tail + trade-window policy" is single-source and unassigned (§4 Q8). | +| CT11 | *(quirk)* COOF TV specifics: waypoint-only refill (`coof_next_waypoint` pine_adapter.cpp:6816), two-fills-at-open rule, script-state rollback (`PineScheduler` pine_scheduler_native.cpp:54-106; hooks `snapshot_script_state` engine.hpp:1332-1333), the first-open guard `kFirstOpenLoopGuard` pine_scheduler_native.cpp:848, deferral queue of `PendingCoofRequest` pine_adapter.hpp:1961-1991 | n/a | A | — | F:E4 O:F5 S:C2 | R5-5: the adapter keeps its path | ### 1.6 Higher timeframes and auxiliary feeds | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| HT1 | HTF series of the same symbol (`request.security`): evaluators registered `scheduler_configure_security_evaluators` pine_strategy_host.cpp:1474-1477, pumped from the scheduler `scheduler_prepare_security_sequence` pine_strategy_host.cpp:1529-1828; state lives in `BacktestEngine` engine.hpp:356-558, engine.hpp:356-402, engine_security.cpp:27 | **no** sanctioned path — `configure_security_evaluators` is an empty virtual engine.hpp:1322 whose only caller is pine_strategy_host.cpp:1475 **Closed:** `NativeRunSpec::subscriptions` (`NativeTimeframeSubscription` native_run_spec.hpp:491), or `declare_timeframe_subscriptions` native_host.hpp:1136 from inside `on_native_run_begin`, with `on_native_timeframe_bar` native_host.hpp:885 and `native_series_bar(index)` native_host.hpp:1119 reading them back; the kernel registers the declared series after that callback returns, so a host's own evaluators survive (`tests/test_native_htf_subscriptions.cpp`, `tests/test_native_htf_subscriptions_stream.cpp`). | K | L6 | F:F1 O:G-1 S:H1 | **Disagree:** S partial (it counts kernel-side storage → HT2); F, S v1 / O deferrable. R5-6: v1. | -| HT2 | Authoritative (exchange) HTF bars substituted into a series (`set_native_security_feed` engine.hpp:1731, engine_aux_security.cpp:79-133, C ABI `strategy_set_native_security_feed` c_abi.cpp:887) | **partial (inert)** — pre-run staging is accepted `refuse_source_mutation` native_execution_consumer.cpp:1535-1551 but never consumed: `prepare_native_security_feeds` is protected engine.hpp:1639, its sole call `prepare_native_security_feeds` pine_strategy_host.cpp:1606; in-run it latches `UnsupportedSource` native_execution_consumer.cpp:1568-1573 **Closed:** a bare host's subscriptions consume authoritative bars: `NativeTimeframeSubscription::authoritative_bars` native_run_spec.hpp:493 is staged into the feed store by the kernel's own `prepare_native_security_feeds` native_execution_consumer.cpp:8838 call at begin, so a declared series is the venue's own bars (`tests/test_native_htf_subscriptions.cpp`). | K | L6 | F:F2 O:G-2 S:H1 | F says "no" on the same facts. "native" in these names means TradingView's native-timeframe bars, not the native host (the feed store's comment `never from the chart's intraday prints` `sha256:3cc5174fa9a60fc0e2ca7870ef1cccd875abaf5d8e11248e633922612c18d9d8` engine_aux_security.cpp:51-61). | -| HT3 | Lower-timeframe arrays (`request.security_lower_tf`; `register_security_lower_tf_eval` include/pineforge/source/pine_strategy_host.hpp:808) | **no** — `IntrabarPath::lower_tf` drives matching only **Closed:** `on_native_sub_bar` native_host.hpp:928 delivers every retained `lower_tf` sub-bar after its matching path, so a host that put the lower bars in the path reads them bar by bar (`tests/test_native_calc_timing.cpp`); the series form is the auxiliary feed of HT4 (`tests/test_native_auxiliary_feed.cpp`). | K | L5 (sub-bar hook); series form optional in L6 | F:F4 O:G-3 S:H2 | **Disagree on lane:** F via `on_native_sub_bar` (the host already owns the lower bars it put in the path) / O, S via the series API. v1 = the L5 hook (R5-5), landed; the array form Pine returns stays the adapter's. | -| HT4 | Auxiliary finer feed driving `request.security` (`set_aux_security_feed` pine_aux_security.cpp:23-241, `feed_aux_security_for_chart_bar` pine_aux_security.cpp:262-488; the kernel base returns false `set_aux_security_feed` engine_consumer.cpp:158-161) | **no** **Closed:** `NativeRunSpec::auxiliary_feed` (`NativeAuxiliaryFeed` native_run_spec.hpp:524) plus `NativeSeriesSource::AuxiliaryFeed` build a series from the host's finer bars, routed by time alone, with `declare_auxiliary_feed` at begin and `append_auxiliary_bars` on a realtime stream (`tests/test_native_auxiliary_feed.cpp`, `tests/test_native_auxiliary_feed_stream.cpp`). | K+A | L6 | F:F3 S:H2 | **Disagree:** F quirk (TV split-feed harness construct) / S generic. R5-1 + R5-6: aux feed registration is generic; chart-slice mapping and deferred first-bucket publication stay A. **Closed (lane INT28-FIX, rule RC):** a `calc_on_order_fills` recalculation reads the calling bar's own slice, as the bar's close calculation does and as TradingView's does after the open fill or a later fill (a lookahead_on site through the slice's first bucket; `lab tv` tapes `tests/fixtures/coof_recalc_security`): the bar's first recalculation before its close callback feeds it (`scheduler_feed_aux_security` pine_scheduler_native.cpp:716). The close callback fed it alone (`scheduler_feed_aux_security` pine_scheduler_native.cpp:578, now only when no recalculation did), after the bar's recalculations had read the previous bar's; a recalculation after that callback feeds nothing. | +| HT1 | HTF series of the same symbol (`request.security`): evaluators registered `scheduler_configure_security_evaluators` pine_strategy_host.cpp:1503-1506, pumped from the scheduler `scheduler_prepare_security_sequence` pine_strategy_host.cpp:1529-1828; state lives in `BacktestEngine` engine.hpp:356-558, engine.hpp:356-402, engine_security.cpp:27 | **no** sanctioned path — `configure_security_evaluators` is an empty virtual engine.hpp:1322 whose only caller is pine_strategy_host.cpp:1504 **Closed:** `NativeRunSpec::subscriptions` (`NativeTimeframeSubscription` native_run_spec.hpp:491), or `declare_timeframe_subscriptions` native_host.hpp:1136 from inside `on_native_run_begin`, with `on_native_timeframe_bar` native_host.hpp:885 and `native_series_bar(index)` native_host.hpp:1119 reading them back; the kernel registers the declared series after that callback returns, so a host's own evaluators survive (`tests/test_native_htf_subscriptions.cpp`, `tests/test_native_htf_subscriptions_stream.cpp`). | K | L6 | F:F1 O:G-1 S:H1 | **Disagree:** S partial (it counts kernel-side storage → HT2); F, S v1 / O deferrable. R5-6: v1. | +| HT2 | Authoritative (exchange) HTF bars substituted into a series (`set_native_security_feed` engine.hpp:1731, engine_aux_security.cpp:79-133, C ABI `strategy_set_native_security_feed` c_abi.cpp:937) | **partial (inert)** — pre-run staging is accepted `refuse_source_mutation` native_execution_consumer.cpp:1834-1850 but never consumed: `prepare_native_security_feeds` is protected engine.hpp:1639, its sole call `prepare_native_security_feeds` pine_strategy_host.cpp:1635; in-run it latches `UnsupportedSource` native_execution_consumer.cpp:1867-1872 **Closed:** a bare host's subscriptions consume authoritative bars: `NativeTimeframeSubscription::authoritative_bars` native_run_spec.hpp:493 is staged into the feed store by the kernel's own `prepare_native_security_feeds` native_execution_consumer.cpp:9347 call at begin, so a declared series is the venue's own bars (`tests/test_native_htf_subscriptions.cpp`). | K | L6 | F:F2 O:G-2 S:H1 | F says "no" on the same facts. "native" in these names means TradingView's native-timeframe bars, not the native host (the feed store's comment `never from the chart's intraday prints` `sha256:3cc5174fa9a60fc0e2ca7870ef1cccd875abaf5d8e11248e633922612c18d9d8` engine_aux_security.cpp:51-61). | +| HT3 | Lower-timeframe arrays (`request.security_lower_tf`; `register_security_lower_tf_eval` include/pineforge/source/pine_strategy_host.hpp:820) | **no** — `IntrabarPath::lower_tf` drives matching only **Closed:** `on_native_sub_bar` native_host.hpp:928 delivers every retained `lower_tf` sub-bar after its matching path, so a host that put the lower bars in the path reads them bar by bar (`tests/test_native_calc_timing.cpp`); the series form is the auxiliary feed of HT4 (`tests/test_native_auxiliary_feed.cpp`). | K | L5 (sub-bar hook); series form optional in L6 | F:F4 O:G-3 S:H2 | **Disagree on lane:** F via `on_native_sub_bar` (the host already owns the lower bars it put in the path) / O, S via the series API. v1 = the L5 hook (R5-5), landed; the array form Pine returns stays the adapter's. | +| HT4 | Auxiliary finer feed driving `request.security` (`set_aux_security_feed` pine_aux_security.cpp:23-241, `feed_aux_security_for_chart_bar` pine_aux_security.cpp:262-488; the kernel base returns false `set_aux_security_feed` engine_consumer.cpp:158-161) | **no** **Closed:** `NativeRunSpec::auxiliary_feed` (`NativeAuxiliaryFeed` native_run_spec.hpp:524) plus `NativeSeriesSource::AuxiliaryFeed` build a series from the host's finer bars, routed by time alone, with `declare_auxiliary_feed` at begin and `append_auxiliary_bars` on a realtime stream (`tests/test_native_auxiliary_feed.cpp`, `tests/test_native_auxiliary_feed_stream.cpp`). | K+A | L6 | F:F3 S:H2 | **Disagree:** F quirk (TV split-feed harness construct) / S generic. R5-1 + R5-6: aux feed registration is generic; chart-slice mapping and deferred first-bucket publication stay A. **Closed (lane INT28-FIX, rule RC):** a `calc_on_order_fills` recalculation reads the calling bar's own slice, as the bar's close calculation does and as TradingView's does after the open fill or a later fill (a lookahead_on site through the slice's first bucket; `lab tv` tapes `tests/fixtures/coof_recalc_security`): the bar's first recalculation before its close callback feeds it (`scheduler_feed_aux_security` pine_scheduler_native.cpp:772). The close callback fed it alone (`scheduler_feed_aux_security` pine_scheduler_native.cpp:634, now only when no recalculation did), after the bar's recalculations had read the previous bar's; a recalculation after that callback feeds nothing. | | HT5 | Publication mode: lookahead, gaps, session / early-close boundaries (`prepare_historical_security_lookahead_projections` pine_scheduler.cpp:42-132, `calling_bar_complete` pine_aux_security.cpp:282-397) | **no** for a bare host **Closed for a bare host:** `NativeTimeframeSubscription::lookahead` and `::gaps` are the two delivery rules, and the calendar boundaries — session close, early close by instrument class, the lazy seal of a clipped script interval — are the kernel's own (`tests/test_native_htf_subscriptions.cpp`). | K+A | L6 | S:H3 F:F1 O:G-1 | **Disagree:** F "no `lookahead_on`, by construction" / O `lookahead = false` default, available / S `lookahead_on` field, status partial. R5-6: user question (§4 U1); the exact TV projection stays A. | -| HT6 | Self-aggregation recipe (interim) | **yes** — `TimeframeAggregator` is public and engine-free timeframe.hpp:288-486, fed from `on_native_input` native_host.hpp:875; the Pine scheduler asks the kernel's `script_bucket_completions` native_execution_consumer.cpp:8391-8408, which feeds the same class `TimeframeAggregator` native_execution_consumer.cpp:8398, how its input span buckets (R5 lane B-ADAPTER) | K | L0 (document) | O:G-4 F:F1 | R5-6 | +| HT6 | Self-aggregation recipe (interim) | **yes** — `TimeframeAggregator` is public and engine-free timeframe.hpp:288-486, fed from `on_native_input` native_host.hpp:875; the Pine scheduler asks the kernel's `script_bucket_completions` native_execution_consumer.cpp:8843-8860, which feeds the same class `TimeframeAggregator` native_execution_consumer.cpp:8850, how its input span buckets (R5 lane B-ADAPTER) | K | L0 (document) | O:G-4 F:F1 | R5-6 | ### 1.7 Fill price, slippage, commission, FX | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| FP1 | Trigger geometry on the modeled path; fill at the level or at the gap price; market-if-touched | **yes** — `native_matching` native_execution_consumer.cpp:5837-5884, `first_region_entry` native_matching.hpp:229-257; kind `NativeCandidatePriceKind` native_execution_consumer.cpp:6593-6601; `Limit::fill_through` native_order.hpp:239-244 enforced where the matched price is limit-protected (`protect_limit` native_execution_consumer.cpp:6596-6606) and re-checked after host terms (`InvalidTerms` native_execution_consumer.cpp:5274-5292) | K | — | F:G3 O:H1 O:H2 O:H3 | — | -| FP2 | Slippage in ticks, limit-clamped | **yes** — `slippage_ticks` native_run_spec.hpp:652 applied once native_execution_consumer.cpp:4925-4926, clamped by `protect_limit` native_execution_consumer.cpp:6596 | K+A | — | F:G1 O:H7 S:F2 | TV's round-then-slip order (`source_bar_fill` pine_adapter.cpp:14938-14947) is A | +| FP1 | Trigger geometry on the modeled path; fill at the level or at the gap price; market-if-touched | **yes** — `native_matching` native_execution_consumer.cpp:6257-6304, `first_region_entry` native_matching.hpp:229-257; kind `NativeCandidatePriceKind` native_execution_consumer.cpp:7023-7031; `Limit::fill_through` native_order.hpp:239-244 enforced where the matched price is limit-protected (`protect_limit` native_execution_consumer.cpp:7026-7036) and re-checked after host terms (`InvalidTerms` native_execution_consumer.cpp:5686-5704) | K | — | F:G3 O:H1 O:H2 O:H3 | — | +| FP2 | Slippage in ticks, limit-clamped | **yes** — `slippage_ticks` native_run_spec.hpp:652 applied once native_execution_consumer.cpp:5337-5338, clamped by `protect_limit` native_execution_consumer.cpp:7026 | K+A | — | F:G1 O:H7 S:F2 | TV's round-then-slip order (`source_bar_fill` pine_adapter.cpp:14946-14955) is A | | FP3 | Commission: percent / cash per unit / cash per execution (`fee_kind_for` pine_adapter.cpp:616-623) | **yes** — `NativeFeeKind` native_run_spec.hpp:19-23; `quote_execution_commissions` engine_execution.cpp:1575-1611 | K | — | F:G2 O:H8 S:F1 | — | -| FP4 | Booked fill quantized to the instrument tick, nearest or directional (`nearest_tick` pine_adapter.cpp:249-252, `source_bar_fill` pine_adapter.cpp:14938-14947, `directional_tick` pine_adapter.cpp:294-298) | **no** — `price_tick` is only a slippage multiplier native_execution_consumer.cpp:4387; `bar_fill_price` engine.hpp:771 is reachable only through `NativeCurrentPriceRule::NearestTick` native_execution_consumer.cpp:6682-6683; the reference helpers were dead (since deleted) **Closed:** `NativeRunSpec::price_grid` (`NativePriceGrid` native_run_spec.hpp:118) with `grid_rounding` (`NativeGridRounding` native_run_spec.hpp:128) rounds the fill basis onto `price_tick` before slippage, `HalfUp` or `Directional` (`tests/test_native_price_grid.cpp`). | K | L8 | O:H4 O:H6 F:G4 S:F3 | **Disagree:** O kernel / F, S "stays adapter, no promotion". R5-3: own kernel lane. | +| FP4 | Booked fill quantized to the instrument tick, nearest or directional (`nearest_tick` pine_adapter.cpp:249-252, `source_bar_fill` pine_adapter.cpp:14946-14955, `directional_tick` pine_adapter.cpp:294-298) | **no** — `price_tick` is only a slippage multiplier native_execution_consumer.cpp:4722; `bar_fill_price` engine.hpp:771 is reachable only through `NativeCurrentPriceRule::NearestTick` native_execution_consumer.cpp:7112-7113; the reference helpers were dead (since deleted) **Closed:** `NativeRunSpec::price_grid` (`NativePriceGrid` native_run_spec.hpp:118) with `grid_rounding` (`NativeGridRounding` native_run_spec.hpp:128) rounds the fill basis onto `price_tick` before slippage, `HalfUp` or `Directional` (`tests/test_native_price_grid.cpp`). | K | L8 | O:H4 O:H6 F:G4 S:F3 | **Disagree:** O kernel / F, S "stays adapter, no promotion". R5-3: own kernel lane. | | FP5 | Trigger tested against the tick-quantized bar | **no** — the kernel tests raw prices against raw levels **Closed:** `NativePriceGrid::QuantizeFillsAndTriggers` additionally tests and re-validates every activation against the tick-quantized path, one rule for every trigger kind (`tests/test_native_price_grid.cpp`). | K+A | L8 | O:H5 F:G4 S:F3 | R5-3: the optional quantized trigger test is K; TV's exact half-tick rule (`source_trigger_threshold` pine_adapter.cpp:325-373) is A on top. | | PG | The Pine adapter on the kernel price grid: raw levels submitted, `project()` declaring `QuantizeFillsAndTriggers` + `HalfUp`, `source_trigger_threshold` deleted (lane R7; re-tested after L8b by gap lane N13) | **measured-infeasible for the Pine adapter** — B1 (the sub-tick cursor-print abort) is closed by L8b's `ActivationGrid`: 0 aborted runs under the trial (R7: 9 zero-offset-trail tapes + 6 POOC panels; now `test_zero_offset_trail_rides_l4c` 449/450, `test_pooc_short_close_tick_l4d` 183/183). B2 (TradingView quantizes per order kind, the grid is one run-wide rule) moved 30 pinned checks in 4 units when N13 last ran the trial, before lanes E14 and E16, with no adapter-side remedy: `test_coof_market_limit_recross_l4c` 24, `test_stop_tick_rounding_l4d` 3, `test_adapter_grid_relower` 2, `test_zero_offset_trail_rides_l4c` 1; 12 more checks in 3 units have an adapter-side cause (`test_trail_fill_snap_l4c` 2, `test_native_margin_hooks` 2, `test_adapter_brackets_relower` 8 book digests). Corpus: 5/312 probes differ, in the engine-only entry-incarnation column only (every probe runs a 0.01 tick on an on-grid feed, so it cannot arbitrate) | A | waived (R7, N13) | — | **Waiver.** No class of triggers is byte-identical on its own: the grid is a run-wide switch and a per-kind mask would spell TradingView's inconsistency into the kernel (not generic, R5-3). The adapter stays on `NativePriceGrid::None` and keeps `source_trigger_threshold` and its call sites; native-engine.md "The Pine adapter's grid re-lowering is waived". **Native-only by ruling (audit lane P6): §3.6.2**, with `examples/native/native_price_grid_strategy.cpp` and its C twin as the feature's hosts. | -| FP6 | Account-currency FX | **partial** — scalar `account_fx` + immutable `NativeFxCurve` native_host.hpp:1202 in batch; the stream then refused a non-empty curve (today only tick-driven input is refused under one, `refuse_fx_curve_tick_input` native_execution_consumer.cpp:9282). **Closed by audit lane N6:** the declared curve is the run's immutable FX epoch and a confirmed-bar stream runs under it exactly as a batch does (stream twin in `tests/test_native_margin_fx_roll.cpp`); no realtime rate ingestion and no tick-driven input under a curve are stated as permanent limits with their reasons (native-engine.md, "Stream FX"). **Re-examined by R5 follow-up lane E3 and KEPT:** the tick refusal was never only about the margin check — it guards the SCRIPT-VISIBLE conversions (`open_trade_profit()`, `marked_equity()`, the equity curve), which read `active_account_currency_fx()`, i.e. the presented clock the observation hook has not yet moved (re-measured: a step at T+90s left the T+100s and T+110s prints converting at the pre-step rate). E3's cursor rule reaches the margin model only, so the refusal stands and is pinned in `tests/test_native_margin_fx_clock.cpp` | K | unassigned → N6 (E3: refusal kept) | F:D9 S:F4 | **Disagree:** F out of scope ("O7 clock") / S its P10, a stream-safe FX epoch. No R5 lane → §4 U4 (answered by N6: S's "same curve in batch and stream", without a new clock or callback). | -| FP7 | *(quirk)* TV tick conventions: half-tick threshold `source_bar_fill_tick` pine_adapter.cpp:332-380 (used `source_trigger_threshold` pine_adapter.cpp:11518-11523, `exit_limit_trigger` pine_adapter.cpp:12248-12269), `source_bar_fill_tick` pine_adapter.cpp:259-272, raw-vs-booked spelling `source_same_point` pine_adapter.cpp:288-292, margin-call fill pricing `source_margin_fill_price` pine_adapter.cpp:17535-17541 | n/a — already isolated behind the terms seam (adapter `resolve_terms` pine_adapter.cpp:14623) | A | — | F:G4 O:H5 S:F3 | R5-1, R5-3 | +| FP6 | Account-currency FX | **partial** — scalar `account_fx` + immutable `NativeFxCurve` native_host.hpp:1202 in batch; the stream then refused a non-empty curve (today only tick-driven input is refused under one, `refuse_fx_curve_tick_input` native_execution_consumer.cpp:9791). **Closed by audit lane N6:** the declared curve is the run's immutable FX epoch and a confirmed-bar stream runs under it exactly as a batch does (stream twin in `tests/test_native_margin_fx_roll.cpp`); no realtime rate ingestion and no tick-driven input under a curve are stated as permanent limits with their reasons (native-engine.md, "Stream FX"). **Re-examined by R5 follow-up lane E3 and KEPT:** the tick refusal was never only about the margin check — it guards the SCRIPT-VISIBLE conversions (`open_trade_profit()`, `marked_equity()`, the equity curve), which read `active_account_currency_fx()`, i.e. the presented clock the observation hook has not yet moved (re-measured: a step at T+90s left the T+100s and T+110s prints converting at the pre-step rate). E3's cursor rule reaches the margin model only, so the refusal stands and is pinned in `tests/test_native_margin_fx_clock.cpp` | K | unassigned → N6 (E3: refusal kept) | F:D9 S:F4 | **Disagree:** F out of scope ("O7 clock") / S its P10, a stream-safe FX epoch. No R5 lane → §4 U4 (answered by N6: S's "same curve in batch and stream", without a new clock or callback). | +| FP7 | *(quirk)* TV tick conventions: half-tick threshold `source_bar_fill_tick` pine_adapter.cpp:332-380 (used `source_trigger_threshold` pine_adapter.cpp:11526-11531, `exit_limit_trigger` pine_adapter.cpp:12256-12277), `source_bar_fill_tick` pine_adapter.cpp:259-272, raw-vs-booked spelling `source_same_point` pine_adapter.cpp:288-292, margin-call fill pricing `source_margin_fill_price` pine_adapter.cpp:17543-17549 | n/a — already isolated behind the terms seam (adapter `resolve_terms` pine_adapter.cpp:14631) | A | — | F:G4 O:H5 S:F3 | R5-1, R5-3 | ### 1.8 Excursion, metrics, report, hash | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| -| RP1 | Per-lot MFE / MAE, kernel-sampled or host-owned | **yes** — sampler `apply_excursion` native_execution_consumer.cpp:2914-2927, closing-row fold `max_runup` engine_orders.cpp:157-171; `owns_lot_excursions` native_host.hpp:1021-1025 | K+A | — | F:H1 O:I1 O:I2 S:X1 | **Disagree:** S partial + its P8 excursion-mode enum (single-source, not adopted, Appendix A.9). **Lowered by R5 lane H-THIN:** the Pine host keeps the kernel's `owns_lot_excursions()` default, so the sampler is the Pine excursion; the host model, its entry-bar masks and its caches are deleted, and the source layer's fold no longer carries them. **Measured against TradingView first (R5 lane H-MEASURE, `tests/test_e19_excursion_tape.cpp`):** the host-owned model was `ab9714be`'s, not TradingView's — where it and the kernel sampler differ the kernel's number is TradingView's (entry-bar over-mask, exit-bar over-fold after an at-open exit, a `calc_on_order_fills` scratch TradingView reports as 0 / 0); both miss a bar's open print ahead of a same-bar stop fill. On the corpus the two models differ on 341 of 431 390 trades, and of the 326 of those a TradingView export identifies, the kernel's number is TradingView's on 315 and the host's on none. The lowering moved the cells of 335 trades in 19 corpus probes: 324 now equal TradingView's, 11 moved closer, none farther; the all-in residual's own entry-bar call is tape `tests/fixtures/e19_allin_trim`. | +| RP1 | Per-lot MFE / MAE, kernel-sampled or host-owned | **yes** — sampler `apply_excursion` native_execution_consumer.cpp:3247-3260, closing-row fold `max_runup` engine_orders.cpp:157-171; `owns_lot_excursions` native_host.hpp:1021-1025 | K+A | — | F:H1 O:I1 O:I2 S:X1 | **Disagree:** S partial + its P8 excursion-mode enum (single-source, not adopted, Appendix A.9). **Lowered by R5 lane H-THIN:** the Pine host keeps the kernel's `owns_lot_excursions()` default, so the sampler is the Pine excursion; the host model, its entry-bar masks and its caches are deleted, and the source layer's fold no longer carries them. **Measured against TradingView first (R5 lane H-MEASURE, `tests/test_e19_excursion_tape.cpp`):** the host-owned model was `ab9714be`'s, not TradingView's — where it and the kernel sampler differ the kernel's number is TradingView's (entry-bar over-mask, exit-bar over-fold after an at-open exit, a `calc_on_order_fills` scratch TradingView reports as 0 / 0); both miss a bar's open print ahead of a same-bar stop fill. On the corpus the two models differ on 341 of 431 390 trades, and of the 326 of those a TradingView export identifies, the kernel's number is TradingView's on 315 and the host's on none. The lowering moved the cells of 335 trades in 19 corpus probes: 324 now equal TradingView's, 11 moved closer, none farther; the all-in residual's own entry-bar call is tape `tests/fixtures/e19_allin_trim`. | | RP2 | Closed-trade list, trade statistics, report struct | **yes** — `fill_report` engine_report.cpp:39-85, `get_trade` engine.hpp:1792-1829, `compute_trade_stats` engine_metrics.cpp:122 | K | — | F:H2 O:I3 S:X2 | — | | RP3 | Equity curve | **no** — `update_equity_extremes` engine.hpp:1362 / `record_equity_point` engine.hpp:1380 are protected and were called only by the adapter **Closed:** `NativeReportPolicy::KernelRecorded` native_run_spec.hpp:63 has the consumer fold the extremes and append one point per script calculation, labelled with the script interval's open; `KernelRecordedAtHostMarks` records the same series at the host's own marks (`tests/test_native_report_truth.cpp`). | K | L2 | F:H3 O:I4 S:X2 | S's "partial" folds RP2 in | | RP4 | Equity metrics (drawdown, run-up, Sharpe / Sortino, CAGR, time in market) | **no** — `compute_equity_stats` exits early on an empty curve engine_metrics.cpp:203; `fill_metrics_section` engine_report.cpp:138-145 **Closed:** with a recorded curve the drawdown / run-up walk is finite and every equity metric is computed over a real series; `report_open_position_at_end` adds the still-open position as a mark-to-market row (`tests/test_native_report_truth.cpp`). | K | L2 | F:H3 O:I5 | silent degenerate metrics, no error (O) | -| RP5 | Range-end row for a position still open at the end (`scheduler_record_range_end` pine_strategy_host.cpp:1932) | **yes, shared** — one generic producer `NativeExecutionConsumer::append_open_position_report_rows`, called by the kernel's run-end `record_open_position_report_rows` (`KernelRecorded` + `report_open_position_at_end`) and by the adapter at TradingView's mark; `report_trade_count()` engine.hpp:1831 includes the rows | K + A report shape | L2, Q6 | F:H4 O:I6 S:X2 | **Disagree:** S says kernel range-end rows exist (storage only). The ROWS are now the kernel's (§3.7 part 1); TV report shape — the equity re-mark, the extreme re-fold, the same-bar re-sort, three marks — stays A, measured in §3.7 part 2. The short-seed swaps (`project_short_seed_report_rows` pine_strategy_host.cpp:1399) stay A as well: a projection of two rows' `entry_incarnation` onto `ab9714be`'s numbering, which TradingView does not export; measured against the kernel's own lot identities by `tests/test_short_seed_report_swap.cpp` (R5 lane H-MEASURE), and pinned by `test_native_oracle_short_seed_percent_full_l2`, whose two incarnation checks fail when the swap is skipped. The swap runs only when a later close of the seed id closes a remnant; a flat collision keeps the kernel's numbering. | +| RP5 | Range-end row for a position still open at the end (`scheduler_record_range_end` pine_strategy_host.cpp:1961) | **yes, shared** — one generic producer `NativeExecutionConsumer::append_open_position_report_rows`, called by the kernel's run-end `record_open_position_report_rows` (`KernelRecorded` + `report_open_position_at_end`) and by the adapter at TradingView's mark; `report_trade_count()` engine.hpp:1831 includes the rows | K + A report shape | L2, Q6 | F:H4 O:I6 S:X2 | **Disagree:** S says kernel range-end rows exist (storage only). The ROWS are now the kernel's (§3.7 part 1); TV report shape — the equity re-mark, the extreme re-fold, the same-bar re-sort, three marks — stays A, measured in §3.7 part 2. The short-seed swaps (`project_short_seed_report_rows` pine_strategy_host.cpp:1428) stay A as well: a projection of two rows' `entry_incarnation` onto `ab9714be`'s numbering, which TradingView does not export; measured against the kernel's own lot identities by `tests/test_short_seed_report_swap.cpp` (R5 lane H-MEASURE), and pinned by `test_native_oracle_short_seed_percent_full_l2`, whose two incarnation checks fail when the swap is skipped. The swap runs only when a later close of the seed id closes a remnant; a flat collision keeps the kernel's numbering. | | RP6 | Per-trade accessors | **partial** — `get_trade` is public; the `closed_trade_*` family is protected engine.hpp:1391 **Closed:** `closed_trade_count()` / `closed_trade(i)` and `report_trade_count()` / `get_report_trade(i)` are public, and `closed_trade_close_cause(i)` engine.hpp:1849 answers the typed `execution::CloseCause` the closer recorded (`tests/test_native_report_truth.cpp`). | K | L2 | F:H6 | (single-source) | -| RP7 | Live order-action stream (runner ledger / webhook) | **yes** — `stream_observe_actions_` engine_execution.cpp:1293, enabled `stream_observe_actions_` native_execution_consumer.cpp:9958, C ABI `strategy_stream_order_actions_len` c_abi.cpp:697-714 | K | — | F:H5 | (single-source) | -| RP8 | Continuation hash | **yes** — `native_continuation_hash` native_host.hpp:1376, native_execution_consumer.cpp:11473; the spec is folded by `hash_spec` native_execution_consumer.cpp:238-260 | K | — | F:I1 O:J2 S:X3 | — | -| RP9 | Broker-state hash: scalar + per-bar recording | **partial** — scalar `broker_state_hash` engine_state_hash.cpp:22-118, C ABI `strategy_broker_state_hash` c_abi.cpp:572; per-bar rows are appended only by the source scheduler `scheduler_record_broker_hash` pine_strategy_host.cpp:2444 **Closed:** under `KernelRecorded` with `set_broker_state_hash_recording(true)` the consumer appends one `broker_state_hash()` row after each report point, so `broker_state_hash_len == equity_curve_len == script_bars_processed` on a completed run in batch and across a stream's warmup and realtime legs. A Failed or Aborted run can expose a shorter prefix. A row is the run's continuation identity at that bar, per driving mode — the broker half alone is mode-invariant (`tests/test_native_report_truth.cpp`). | K | L2 | F:I2 O:J1 S:X3 | O's "yes" covers the scalar only. §3.9 records the old quadratic adapter fold and closed-row kernel walk as history; V19-E and V19-A replaced them with running digests. | +| RP7 | Live order-action stream (runner ledger / webhook) | **yes** — `stream_observe_actions_` engine_execution.cpp:1293, enabled `stream_observe_actions_` native_execution_consumer.cpp:10514, C ABI `strategy_stream_order_actions_len` c_abi.cpp:737-754 | K | — | F:H5 | (single-source) | +| RP8 | Continuation hash | **yes** — `native_continuation_hash` native_host.hpp:1376, native_execution_consumer.cpp:12077; the spec is folded by `hash_spec` native_execution_consumer.cpp:238-260 | K | — | F:I1 O:J2 S:X3 | — | +| RP9 | Broker-state hash: scalar + per-bar recording | **partial** — scalar `broker_state_hash` engine_state_hash.cpp:22-118, C ABI `strategy_broker_state_hash` c_abi.cpp:593; per-bar rows are appended only by the source scheduler `scheduler_record_broker_hash` pine_strategy_host.cpp:2742 **Closed:** under `KernelRecorded` with `set_broker_state_hash_recording(true)` the consumer appends one `broker_state_hash()` row after each report point, so `broker_state_hash_len == equity_curve_len == script_bars_processed` on a completed run in batch and across a stream's warmup and realtime legs. A Failed or Aborted run can expose a shorter prefix. A row is the run's continuation identity at that bar, per driving mode — the broker half alone is mode-invariant (`tests/test_native_report_truth.cpp`). | K | L2 | F:I2 O:J1 S:X3 | O's "yes" covers the scalar only. §3.9 records the old quadratic adapter fold and closed-row kernel walk as history; V19-E and V19-A replaced them with running digests. | | RP10 | Fold the host's own durable state into the hash | **partial** — the protected virtual `hash_source_extension` engine.hpp:411 is overridable but source-named; the kernel default folds `"source:none"` engine_state_hash.cpp:19-21 **Closed:** `hash_host_extension(BrokerStateHashSink&)` engine.hpp:405 is the generic seam — protected virtual on `BacktestEngine`, called once per hash, last — and `hash_source_extension` engine.hpp:411 is its deprecated spelling, which the generic default forwards to (`tests/test_native_host_hash_extension.cpp`). | K | L2 (the virtual rides the shared bump) | F:I3 O:J1 S:X3 | — | ### 1.9 Other runtime surface @@ -171,13 +171,13 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | ID | Feature (adapter mechanism) | Native today | Class | Lane | Sources | Notes | |---|---|---|---|---|---|---| | OT1 | Key / value run parameters (`set_input` engine.hpp:1871, refused in-run by `guard_native_mutation` engine.hpp:1876) | **partial** — staging allowed pre-run, getters protected | K (low) | — | F:J1 | (single-source) Nothing for v1: native hosts take constructor parameters. | -| OT2 | Symbol info, sessions, calendars, timezones (`timezone` pine_adapter.cpp:2671-2672, `chart_timezone` pine_adapter.cpp:2713-2715) | **yes** — spec fields `ticker` native_run_spec.hpp:636-646; `parse_session` native_calendar.hpp:180-467; intervals on every decision context `input_interval` market_driver.hpp:123-124 | K | — | F:J2 O:G-5 | — | +| OT2 | Symbol info, sessions, calendars, timezones (`timezone` pine_adapter.cpp:2677-2678, `chart_timezone` pine_adapter.cpp:2719-2721) | **yes** — spec fields `ticker` native_run_spec.hpp:636-646; `parse_session` native_calendar.hpp:180-467; intervals on every decision context `input_interval` market_driver.hpp:123-124 | K | — | F:J2 O:G-5 | — | | OT3 | Indicators, series, math, matrix, map, string utils | **yes** — `ta.hpp` includes only `na/series/window_sum` `window_sum` ta.hpp:2-8 | K | L11 (naming only) | F:J3 | (single-source row) TV-calibrated numerics → §2.ii b | | OT4 | Cooperative abort | **yes** — `request_abort` engine.hpp:2174, `NativeAbortReporting` native_run_spec.hpp:41-44 | K | — | F:J4 | (single-source) | -| OT5 | C-level run configuration, stream, FX curve, contract probe | **yes** — `strategy_configure_native_v1` c_abi.cpp:930, FX curve `strategy_configure_native_fx_curve_v1` c_abi.cpp:1001, probe `strategy_execution_contract` c_abi.cpp:921, stream `strategy_stream_begin` c_abi.cpp:658 **Extended:** `pf_native_run_spec_ext_v1`'s policy tail now carries `intrabar`, `path_order`, `abort_reporting` and the slot-label / feed-tolerance policies (`PF_NATIVE_SPEC_EXT_INTRABAR`, `PF_NATIVE_SPEC_EXT_FEED_POLICY`), so nothing of `NativeRunSpec` but `identity` is unreachable from C (`tests/test_native_c_api.c`). | K | — | O:J4 | The v1 C spec `pf_native_run_spec_v1` pineforge.h:510 omitted `intrabar`, `path_order` and `abort_reporting`; they now travel in the extension's policy tail, `pf_native_abort_reporting_e` native_c_api.h:911 among them (F §2.iii) | -| OT6 | C-level order submission + strategy callbacks | **no** — none of the original runtime `PF_API` symbols; then listed as a refusal on native-engine.md **Closed:** `strategy_native_host_create_v1` takes a C callback table and `strategy_native_submit_v1` / `_replace_v1` / `_cancel_v1` / `_cancel_all_v1` / `_cancel_where_v1` / `_execute_current_v1` command the same kernel under the same legality rule; the 41 `strategy_native_*` declarations plus two `strategy_configure_native_ext_*` declarations are a second, disjoint inventory beside the 73 runtime exports (and eight generated per-strategy exports) (`tests/test_native_c_api.c`, `tests/test_native_c_api_frozen_header.cpp`). | K | L13 | F:J6 O:J5 | R5-7: post-v1. S covers it in its §2.3 without a row. | -| OT7 | *(quirk)* Pine language state: series, barstate / session flags, position-view freezing (`PineLanguageState` pine_language_state.hpp:12-68; `publish_series` pine_scheduler_native.cpp:267-302, `chart_index` pine_scheduler_native.cpp:570-703) | n/a | A | — | S:R1 | (single-source) **Closed (E25, lane E20's finding 2; E26, E25's findings 1 and 3):** `session.islastbar` reads one script bar ahead on every driving path: the chart timeframe's next retained bar, an aggregated chart's next bucket (a scheduler lookahead, deleted when the rule moved into the kernel), the calendar for the live tail's bar and for every bar of a stream, warmup included — a stream continues past its replay, so only a batch run's final bar reads "last" from having no successor. Since R4 slice C an aggregated or realtime bar read every in-session bar as last. **What TradingView does:** it ends a session at the session DAY — the flag belongs to the last chart bar whose successor belongs to another session day, in session or not, and `session.isfirstbar` is its dual, the bar after a session's last one. The ordinal is `internal::session_trading_day_index` timeframe.hpp:145, the exchange-timezone day rolling at the symbol's day stamp (09:30 ET on NYSE RTH, midnight on 24x7, 17:00 ET on a 1700-1700 forex session), not local midnight; the unmerged form, since a fused holiday daily bar still holds two intraday sessions. The three `lab tv` tapes (`tests/fixtures/session_islastbar`) give that rule bar for bar with no disagreement, and the engine now holds 255/255 NYSE:F last bars, 255/255 first bars and 370/370 ETH last bars on the chart-timeframe AND the aggregated path (it held 1, 0 and 1 before). Early closes come with it: the three NYSE:F half days flag 12:45 ET without a holiday calendar, because the next bar is the next session day's. **Generic since R5 lane F5:** the kernel computes that reading for every host — `NativeDecisionContext::in_session` market_driver.hpp:161 and the facts after it (`opens_session_day`, `closes_session_day`, `closes_session_day_open_ended`), in C the session bytes of `pf_native_decision_v1` native_c_api.h:1485 — from the run's own calendar and input (`present_session_day` native_execution_consumer.cpp:7636), so a bare host reproduces the tapes flag for flag (`tests/test_native_session_day_facts.cpp`, kernel-only). The adapter computes no regular-day rule of its own: `scheduler_update_session_state` pine_strategy_host.cpp:2167 selects the kernel's facts into the flags, and only widens the plain `session.isfirstbar` / `islastbar` pair to the chart's day on an extended-hours chart (R5 lane K-SESSION-WINDOWS F2: TradingView's `isfirstbar` / `islastbar` span the pre- and post-market bars there while the `_regular` pair keeps the regular day, `tests/test_session_extended_day_tape.cpp`), which are `PineStrategyHost` members now (`session_ismarket_` pine_strategy_host.hpp:1138; `BacktestEngine` holds no Pine session state), and only its live-probe tail reads the open-ended close at its batch's final bar (`tests/test_session_day_facts_adapter.cpp`). Selecting the kernel also ended a calc_on_order_fills defect of the adapter's lookahead, which read two bars ahead on a bar a fill recalculation had already published (a fill on a day's second-to-last bar flagged that bar last and the day's real last bar first). **Ruled, not open:** a bar with nothing held after it — a stream's bar, a live-tail bar — steps the calendar one width on, so it cannot see an early close the session string does not declare (ADR-0001, "Session-day facts at a bar with nothing held after it"). **Closed (F1):** a `calc_on_order_fills` recalculation reads the flags of the bar it runs on, not the previous bar's (the kernel presents the current bar's facts on that recalculation), and the close callback after it no longer looks one retained bar too far ahead (§3.8, AG3). | -| OT8 | *(quirk)* Live-tail / probe-suppress harness flags (`realtime_tail_` pine_strategy_host.cpp:282-284) | n/a — **contained since N14** (the flags were `BacktestEngine` members until then; now `source::PineStrategyHost` state behind two kernel virtual seams, §2.ii q) | A | N14 | F:J7 S:C5 | parity-campaign tooling | +| OT5 | C-level run configuration, stream, FX curve, contract probe | **yes** — `strategy_configure_native_v1` c_abi.cpp:980, FX curve `strategy_configure_native_fx_curve_v1` c_abi.cpp:1051, probe `strategy_execution_contract` c_abi.cpp:971, stream `strategy_stream_begin` c_abi.cpp:698 **Extended:** `pf_native_run_spec_ext_v1`'s policy tail now carries `intrabar`, `path_order`, `abort_reporting` and the slot-label / feed-tolerance policies (`PF_NATIVE_SPEC_EXT_INTRABAR`, `PF_NATIVE_SPEC_EXT_FEED_POLICY`), so nothing of `NativeRunSpec` but `identity` is unreachable from C (`tests/test_native_c_api.c`). | K | — | O:J4 | The v1 C spec `pf_native_run_spec_v1` pineforge.h:510 omitted `intrabar`, `path_order` and `abort_reporting`; they now travel in the extension's policy tail, `pf_native_abort_reporting_e` native_c_api.h:911 among them (F §2.iii) | +| OT6 | C-level order submission + strategy callbacks | **no** — none of the original runtime `PF_API` symbols; then listed as a refusal on native-engine.md **Closed:** `strategy_native_host_create_v1` takes a C callback table and `strategy_native_submit_v1` / `_replace_v1` / `_cancel_v1` / `_cancel_all_v1` / `_cancel_where_v1` / `_execute_current_v1` command the same kernel under the same legality rule; the 41 `strategy_native_*` declarations plus two `strategy_configure_native_ext_*` declarations are a second, disjoint inventory beside the 80 runtime exports (and eight generated per-strategy exports) (`tests/test_native_c_api.c`, `tests/test_native_c_api_frozen_header.cpp`). | K | L13 | F:J6 O:J5 | R5-7: post-v1. S covers it in its §2.3 without a row. | +| OT7 | *(quirk)* Pine language state: series, barstate / session flags, position-view freezing (`PineLanguageState` pine_language_state.hpp:12-68; `publish_series` pine_scheduler_native.cpp:323-358, `chart_index` pine_scheduler_native.cpp:570-703) | n/a | A | — | S:R1 | (single-source) **Closed (E25, lane E20's finding 2; E26, E25's findings 1 and 3):** `session.islastbar` reads one script bar ahead on every driving path: the chart timeframe's next retained bar, an aggregated chart's next bucket (a scheduler lookahead, deleted when the rule moved into the kernel), the calendar for the live tail's bar and for every bar of a stream, warmup included — a stream continues past its replay, so only a batch run's final bar reads "last" from having no successor. Since R4 slice C an aggregated or realtime bar read every in-session bar as last. **What TradingView does:** it ends a session at the session DAY — the flag belongs to the last chart bar whose successor belongs to another session day, in session or not, and `session.isfirstbar` is its dual, the bar after a session's last one. The ordinal is `internal::session_trading_day_index` timeframe.hpp:145, the exchange-timezone day rolling at the symbol's day stamp (09:30 ET on NYSE RTH, midnight on 24x7, 17:00 ET on a 1700-1700 forex session), not local midnight; the unmerged form, since a fused holiday daily bar still holds two intraday sessions. The three `lab tv` tapes (`tests/fixtures/session_islastbar`) give that rule bar for bar with no disagreement, and the engine now holds 255/255 NYSE:F last bars, 255/255 first bars and 370/370 ETH last bars on the chart-timeframe AND the aggregated path (it held 1, 0 and 1 before). Early closes come with it: the three NYSE:F half days flag 12:45 ET without a holiday calendar, because the next bar is the next session day's. **Generic since R5 lane F5:** the kernel computes that reading for every host — `NativeDecisionContext::in_session` market_driver.hpp:161 and the facts after it (`opens_session_day`, `closes_session_day`, `closes_session_day_open_ended`), in C the session bytes of `pf_native_decision_v1` native_c_api.h:1485 — from the run's own calendar and input (`present_session_day` native_execution_consumer.cpp:8088), so a bare host reproduces the tapes flag for flag (`tests/test_native_session_day_facts.cpp`, kernel-only). The adapter computes no regular-day rule of its own: `scheduler_update_session_state` pine_strategy_host.cpp:2196 selects the kernel's facts into the flags, and only widens the plain `session.isfirstbar` / `islastbar` pair to the chart's day on an extended-hours chart (R5 lane K-SESSION-WINDOWS F2: TradingView's `isfirstbar` / `islastbar` span the pre- and post-market bars there while the `_regular` pair keeps the regular day, `tests/test_session_extended_day_tape.cpp`), which are `PineStrategyHost` members now (`session_ismarket_` pine_strategy_host.hpp:1174; `BacktestEngine` holds no Pine session state), and only its live-probe tail reads the open-ended close at its batch's final bar (`tests/test_session_day_facts_adapter.cpp`). Selecting the kernel also ended a calc_on_order_fills defect of the adapter's lookahead, which read two bars ahead on a bar a fill recalculation had already published (a fill on a day's second-to-last bar flagged that bar last and the day's real last bar first). **Ruled, not open:** a bar with nothing held after it — a stream's bar, a live-tail bar — steps the calendar one width on, so it cannot see an early close the session string does not declare (ADR-0001, "Session-day facts at a bar with nothing held after it"). **Closed (F1):** a `calc_on_order_fills` recalculation reads the flags of the bar it runs on, not the previous bar's (the kernel presents the current bar's facts on that recalculation), and the close callback after it no longer looks one retained bar too far ahead (§3.8, AG3). | +| OT8 | *(quirk)* Live-tail / probe-suppress harness flags (`realtime_tail_` pine_strategy_host.cpp:284-286) | n/a — **contained since N14** (the flags were `BacktestEngine` members until then; now `source::PineStrategyHost` state behind two kernel virtual seams, §2.ii q) | A | N14 | F:J7 S:C5 | parity-campaign tooling | --- @@ -185,7 +185,7 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa ### 2.i Kernel-only build target (L1) -- **Before L1:** one static library always compiled the source layer and installed every header, `include/pineforge/source/**` and `include/pineforge/compat/pine/**` included, and `src/compat/pine/market_admission.cpp` sat in the *main* list for historical reasons (added by `3b13183`, before the source-layer list existed — F). **Since L1:** the source layer is its own list (`PINEFORGE_SOURCE_LAYER_SOURCES` CMakeLists.txt:91, `market_admission.cpp` at :94), appended to `pineforge` only when `PINEFORGE_BUILD_SOURCE_LAYER` is ON (`add_library(pineforge STATIC` CMakeLists.txt:166-169), beside the kernel-only archive (`pineforge_kernel` CMakeLists.txt:155), whose header install excludes `source` and `compat` (`PATTERN` CMakeLists.txt:428-433). +- **Before L1:** one static library always compiled the source layer and installed every header, `include/pineforge/source/**` and `include/pineforge/compat/pine/**` included, and `src/compat/pine/market_admission.cpp` sat in the *main* list for historical reasons (added in an earlier commit, before the source-layer list existed — F). **Since L1:** the source layer is its own list (`PINEFORGE_SOURCE_LAYER_SOURCES` CMakeLists.txt:91, `market_admission.cpp` at :94), appended to `pineforge` only when `PINEFORGE_BUILD_SOURCE_LAYER` is ON (`add_library(pineforge STATIC` CMakeLists.txt:167-170), beside the kernel-only archive (`pineforge_kernel` CMakeLists.txt:158), whose header install excludes `source` and `compat` (`PATTERN` CMakeLists.txt:471-476). - **Link feasibility:** no kernel TU and no public kernel header includes `source/` or `compat/pine/` (F, O). The only code coupling is the forward-declared opaque `source::StrategyOverrides*` (§2.ii a), which needs no definition. - **Risk:** the archive member order of `libpineforge.a` changes. Harmless for static linking, but check `PINEFORGE_REQUIRE_ABI_RECEIPTS` (CMakeLists.txt:65); what the receipts pin is unverified (F). @@ -202,18 +202,18 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa | # | Item | Where / what it is used for | Class | Plan | Src | |---|---|---|---|---|---| -| a | `source::StrategyOverrides` opaque pointer | Forward declarations in the kernel headers (since removed: no kernel header names the adapter type now); parameter of `run_rich` (execution_consumer.hpp:46-55, native_execution_consumer.hpp:83-92) and of the rich `BacktestEngine::run` overload (engine.hpp:1796-1801, engine_consumer.cpp:81-93). The kernel only forwards it as `NativeBeginArgs::overrides_opaque` native_host.hpp:758; the source host casts it back `overrides_opaque` pine_strategy_host.cpp:285-290 | NEUTRAL in behaviour; a consumer-vtable signature change, so it rides the shared `engine_script_run` bump | `const void*` in the kernel signatures (F `host_begin_extras`, O `overrides_opaque`), typed convenience overload in `source/pine_strategy_host.hpp`; then drop the checker's single whitelist entry `test_native_example_batch` check_native_include_independence.py:42-46. S's alternative (`NativeRunOverrides`, or removing the rich overload) not adopted. Unverified: generated code calls the typed overload (codegen repo is not in this tree — F). **Status (L11, `3f4fff5b`): done** — `run_rich` and the rich `BacktestEngine::run` take `const void*` overrides, the rich begin carries them as `overrides_opaque`, and the independence checker whitelists nothing (§2.vi row 6) | F O S | +| a | `source::StrategyOverrides` opaque pointer | Forward declarations in the kernel headers (since removed: no kernel header names the adapter type now); parameter of `run_rich` (execution_consumer.hpp:46-55, native_execution_consumer.hpp:110-119) and of the rich `BacktestEngine::run` overload (engine.hpp:1796-1801, engine_consumer.cpp:81-93). The kernel only forwards it as `NativeBeginArgs::overrides_opaque` native_host.hpp:758; the source host casts it back `overrides_opaque` pine_strategy_host.cpp:287-292 | NEUTRAL in behaviour; a consumer-vtable signature change, so it rides the shared `engine_script_run` bump | `const void*` in the kernel signatures (F `host_begin_extras`, O `overrides_opaque`), typed convenience overload in `source/pine_strategy_host.hpp`; then drop the checker's single whitelist entry `test_native_example_batch` check_native_include_independence.py:42-46. S's alternative (`NativeRunOverrides`, or removing the rich overload) not adopted. Unverified: generated code calls the typed overload (codegen repo is not in this tree — F). **Status (L11, `3f4fff5b`): done** — `run_rich` and the rich `BacktestEngine::run` take `const void*` overrides, the rich begin carries them as `overrides_opaque`, and the independence checker whitelists nothing (§2.vi row 6) | F O S | | b | `pine_float_compare.hpp` | 1e-10 absolute equality band, used by kernel TA only: `float_band_le` ta_misc.cpp:30 (percentrank), `float_band_gt` ta_oscillators.cpp:462-463 (MFI), `plus_dm` ta_volatility_trend.cpp:337-340 (DMI); no execution TU includes it (O) | NEUTRAL (rename only) | Rename to a neutral header (F `float_band_compare.hpp`, O `ta_compare_band.hpp`), keep `pine_float_*` inline aliases in a source-layer header for generated code. Numerics must not move. S's behavioural `ComparePolicy` split is not neutral → not adopted (§4 U6). **Status (L11 + N14): done** — `ta_compare_band.hpp` is the kernel header; the alias shim is `include/pineforge/source/pine_float_compare.hpp` (N14 moved it off the kernel include root; it has no in-tree includer, generated code emits its own comparator) | F O S | -| c | `NativeSlotLabelPolicy::LegacyTolerant`, `NativeLegacyTolerance` (now `NativeSlotLabelPolicy::FeedTolerant`, `NativeFeedTolerance`) | `NativeSlotLabelPolicy` native_run_spec.hpp:324-353; read by `legacy_tolerant_slot_labels` market_driver.cpp:55-56 and `native_feed_tolerance_enabled` market_driver.cpp:43-49, and by `legacy_tolerant_slot_labels` native_execution_consumer.hpp:960-962 (outside a run it answers the spec's `spec_tolerant_slot_labels` native_execution_consumer.cpp:1985-1988; the reads are `legacy_tolerant_slot_labels` native_execution_consumer.cpp:2002, `:9412`); set only by the adapter `slot_label_policy` pine_adapter.cpp:2740-2741. A feed-tolerance policy (raw strictly-increasing labels, zero / NaN-volume legacy bars), not TV semantics (O) | NEUTRAL | Rename with deprecated aliases (F `RawIncreasing` + `NativeInputTolerance::{StructuralBars, WarmupNonNegativeOHLC}`; O `NativeLabelPolicy::ProviderLabels`). Enumerator values stay 0/1 and 1/2: they are hashed `slot_label_policy` native_execution_consumer.cpp:269-270. **Status:** renamed as `FeedTolerant` / `NativeFeedTolerance` (the enumerator values unchanged), and the C transport carries them now, in the extension's feed-policy block (`PF_NATIVE_SPEC_EXT_FEED_POLICY` native_c_api.h:745) | F O S | +| c | `NativeSlotLabelPolicy::LegacyTolerant`, `NativeLegacyTolerance` (now `NativeSlotLabelPolicy::FeedTolerant`, `NativeFeedTolerance`) | `NativeSlotLabelPolicy` native_run_spec.hpp:324-353; read by `legacy_tolerant_slot_labels` market_driver.cpp:55-56 and `native_feed_tolerance_enabled` market_driver.cpp:43-49, and by `legacy_tolerant_slot_labels` native_execution_consumer.hpp:1027-1029 (outside a run it answers the spec's `spec_tolerant_slot_labels` native_execution_consumer.cpp:2296-2299; the reads are `legacy_tolerant_slot_labels` native_execution_consumer.cpp:2313, `:9921`); set only by the adapter `slot_label_policy` pine_adapter.cpp:2747-2748. A feed-tolerance policy (raw strictly-increasing labels, zero / NaN-volume legacy bars), not TV semantics (O) | NEUTRAL | Rename with deprecated aliases (F `RawIncreasing` + `NativeInputTolerance::{StructuralBars, WarmupNonNegativeOHLC}`; O `NativeLabelPolicy::ProviderLabels`). Enumerator values stay 0/1 and 1/2: they are hashed `slot_label_policy` native_execution_consumer.cpp:269-270. **Status:** renamed as `FeedTolerant` / `NativeFeedTolerance` (the enumerator values unchanged), and the C transport carries them now, in the extension's feed-policy block (`PF_NATIVE_SPEC_EXT_FEED_POLICY` native_c_api.h:745) | F O S | | d | Orphaned comments | The KI-62 block of `engine_orders.cpp` (*fresh*: comment-only, the bodies are deleted; S read it as live code) and a comment of `engine.hpp`; wider deleted-declaration blocks in `engine.hpp`; the money-rule essay in `engine.hpp` (the code is `source_money_round` pine_adapter.cpp:505-511) | NEUTRAL | Delete, or move to `docs/` next to the adapter code that implements them. **Status: done** — R5 lanes E6 and E10 deleted the detached comment residue (a census of 0 over the kernel's compile closure) | F O | -| e | Dead helpers in the public kernel header | **Done.** All four are gone from `engine.hpp`: `round_to_mintick_directional` and `apply_slippage` were the pre-R5 spelling of the directional snap and the slippage step and the kernel now fills on `native_matching::grid_round_directional` / `native_matching::apply_slippage` src/native_matching.hpp:259 (the engine.hpp comment that named them went with R5 lane E10's residue sweep); `apply_limit_fill` and `reserve_percent_commission` are deleted outright, and only the adapter still spells the first two names, in its own comments. `round_to_mintick` engine.hpp:753 and `bar_fill_price` engine.hpp:771 stay, live through the call `bar_fill_price` native_execution_consumer.cpp:6752 | NEUTRAL | Reuse as the reference arithmetic of the price-grid and sizing lanes, delete the rest | O (single-source) | +| e | Dead helpers in the public kernel header | **Done.** All four are gone from `engine.hpp`: `round_to_mintick_directional` and `apply_slippage` were the pre-R5 spelling of the directional snap and the slippage step and the kernel now fills on `native_matching::grid_round_directional` / `native_matching::apply_slippage` src/native_matching.hpp:259 (the engine.hpp comment that named them went with R5 lane E10's residue sweep); `apply_limit_fill` and `reserve_percent_commission` are deleted outright, and only the adapter still spells the first two names, in its own comments. `round_to_mintick` engine.hpp:753 and `bar_fill_price` engine.hpp:771 stay, live through the call `bar_fill_price` native_execution_consumer.cpp:7182 | NEUTRAL | Reuse as the reference arithmetic of the price-grid and sizing lanes, delete the rest | O (single-source) | | f | Dead legacy stream path | **Done.** `stream_feed_input_bar` and `dispatch_source_stream_script_bar` are both gone — neither name is in `src/` or `include/` — and the virtual left with the epoch that carried it. `stream_state_hash` engine.hpp:1790 stays, and the declaration/definition pairing is still enforced by `scripts/test_native_source_guard.py` | NEUTRAL (removing the virtual changes the vtable → shared bump) | Delete both | O; F asked (its Q6) | | g | Legacy path resolver | **Done, and then deleted.** `src/engine_path_resolve.cpp` is 118 lines with five functions: `bar_path_uses_high_first`, both `first_touch_position` overloads, `fill_bar_path_points_ordered`, and `BacktestEngine::declare_opened_lot_entry_bar_mask`. Ten test files still mention `resolve_exit_path_fill` in historical comments; the resolver body and test-only oracle are gone. | NEUTRAL (pure move; verify with the link gate) | Keep the generic path helpers in the kernel TU; the old source resolver is not a live counterpart | F (single-source) | | | h | `pine_*`-named generic API | `timeframe_time` session_time.hpp:35-211 (live, e.g. `session_in_market`, which generated code reaches through `pine_session_ismarket` session_time.hpp:263), `str_utils.hpp`, `deterministic_random` math.hpp:9, `tz_util` timezone.hpp:8-30; the guard bans `pine_*` only inside `IDENTIFIER_ROOTS` (test_native_source_guard.py:34, test_native_source_guard.py:34-42) | NEUTRAL, M | Neutral aliases; `pine_*` kept as deprecated inlines for codegen. Large blast radius in generated code. **Status (L11 + N14): done** — session_time / str_utils / math / timezone in L11; N14 applied the same treatment to the two public matrix types: `PineMatrix` → `NumericMatrix`, `PineGenericMatrix` → `GenericMatrix`, old spellings kept as exact aliases (no version script pins either name) | O (single-source) | -| i | `pf_pending_order_v1_t` Pine-named fields | `tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135, `pine_exit_activation_owner_cycle_at_birth` pending_order_mirror.hpp:147-156, `pine_frozen_market_instruction_kind` pending_order_mirror.hpp:164-169; enumerated by `PF_PO_FIELD` pending_order_mirror.cpp:144-178; written only by `pine_birth_reach` pine_adapter.cpp:26787-26816 | no change | Freeze (ABI). The neutral view arrives beside it (L7 / L13) | O (single-source) | -| j | Pine-shaped lot flags | **Mostly done.** `market_pyramid_add` moved to the adapter side table the row asked for — `market_pyramid_adds_` src/source/pine_state_hash.cpp:691, whose own comment records that it was `PyramidEntry::market_pyramid_add` — and `ordinary_market_open`, `pooc_terminal_market_entry`, `ordinary_stop_open`, `bracket_slot_shadowed` and `entry_path_position` are all deleted. What is left on the lot is the intrabar-fill excursion mask, `skip_entry_bar_high` / `skip_entry_bar_low` engine.hpp:153, hashed (`skip_entry_bar_high` engine_state_hash.cpp:66) and, since R5 lane E6, written by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the Pine host did until R5 lane H-THIN left its excursions to the kernel's sampler) and the kernel derives the pair | HASH-VISIBLE | Move the live flag to an adapter side table keyed by `entry_incarnation` (the adapter already keeps `PlacementTable` pine_adapter.hpp:355-675) and fold it through `hash_source_extension`; delete the dead ones; update the hash-coverage waivers. Whether a kernel TU beyond the hash reads them is unverified (O) | F O S | -| k | `margin_call_enabled_` | **Done.** The generic kernel model is enabled by `NativeRunSpec::margin` native_run_spec.hpp:672. TradingView-specific toggling remains `source::PineStrategyHost::set_margin_call_enabled` pine_strategy_host.hpp:499; the frozen C symbol remains source-owned. | HASH-VISIBLE (R5-8) | Keep the source setter in the adapter; the kernel toggle is the presence of `spec.margin` | F O | -| l | Kernel decodes adapter strings | **Done.** `closed_trade_close_cause` engine_trade_accessors.cpp:158 branches on no string at all: it reads `Trade::open_at_end`, then the typed `execution::CloseCause` the closer recorded on the row, then `exit_from_bracket`. A kernel-originated liquidation or risk flatten states its own cause through the settling fill, so a bare host reads `3` / `4` where the retired string derivation answered `1`. The public C contract it implements is `strategy_closed_trade_close_cause` pineforge.h:1249 | HASH-VISIBLE (adds durable state) | Kernel `enum class CloseCause` recorded on the `Trade` at settlement (the kernel sets `MarginCall` when L4 liquidates; a source virtual covers adapter causes). C numbers unchanged. After L4 | O (single-source) | +| i | `pf_pending_order_v1_t` Pine-named fields | `tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135, `pine_exit_activation_owner_cycle_at_birth` pending_order_mirror.hpp:147-156, `pine_frozen_market_instruction_kind` pending_order_mirror.hpp:164-169; enumerated by `PF_PO_FIELD` pending_order_mirror.cpp:144-178; written only by `pine_birth_reach` pine_adapter.cpp:26836-26865 | no change | Freeze (ABI). The neutral view arrives beside it (L7 / L13) | O (single-source) | +| j | Pine-shaped lot flags | **Mostly done.** `market_pyramid_add` moved to the adapter side table the row asked for — `market_pyramid_adds_` src/source/pine_state_hash.cpp:692, whose own comment records that it was `PyramidEntry::market_pyramid_add` — and `ordinary_market_open`, `pooc_terminal_market_entry`, `ordinary_stop_open`, `bracket_slot_shadowed` and `entry_path_position` are all deleted. What is left on the lot is the intrabar-fill excursion mask, `skip_entry_bar_high` / `skip_entry_bar_low` engine.hpp:153, hashed (`skip_entry_bar_high` engine_state_hash.cpp:74) and, since R5 lane E6, written by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the Pine host did until R5 lane H-THIN left its excursions to the kernel's sampler) and the kernel derives the pair | HASH-VISIBLE | Move the live flag to an adapter side table keyed by `entry_incarnation` (the adapter already keeps `PlacementTable` pine_adapter.hpp:355-675) and fold it through `hash_source_extension`; delete the dead ones; update the hash-coverage waivers. Whether a kernel TU beyond the hash reads them is unverified (O) | F O S | +| k | `margin_call_enabled_` | **Done.** The generic kernel model is enabled by `NativeRunSpec::margin` native_run_spec.hpp:672. TradingView-specific toggling remains `source::PineStrategyHost::set_margin_call_enabled` pine_strategy_host.hpp:511; the frozen C symbol remains source-owned. | HASH-VISIBLE (R5-8) | Keep the source setter in the adapter; the kernel toggle is the presence of `spec.margin` | F O | +| l | Kernel decodes adapter strings | **Done.** `closed_trade_close_cause` engine_trade_accessors.cpp:158 branches on no string at all: it reads `Trade::open_at_end`, then the typed `execution::CloseCause` the closer recorded on the row, then `exit_from_bracket`. A kernel-originated liquidation or risk flatten states its own cause through the settling fill, so a bare host reads `3` / `4` where the retired string derivation answered `1`. The public C contract it implements is `strategy_closed_trade_close_cause` pineforge.h:1309 | HASH-VISIBLE (adds durable state) | Kernel `enum class CloseCause` recorded on the `Trade` at settlement (the kernel sets `MarginCall` when L4 liquidates; a source virtual covers adapter causes). C numbers unchanged. After L4 | O (single-source) | | m | `request.security` machinery | `register_security_eval` engine_security.cpp:27, `set_native_security_feed` engine_aux_security.cpp:77-131, `SecurityEvalState` engine.hpp:1156-1360 carry Pine semantics (lookahead, gaps, Heikin-Ashi, KI-55 range start) and are driven only by the source layer | HASH-VISIBLE / measured path | F planned to move both TUs and the state into the source host after native HTF. **R5-6 narrows this:** L6 reuses the feed store and routing, so those stay kernel; only the Pine-only evaluator semantics remain candidates. Do last. **Status (lane L12b): done** — the evaluator semantics are `source::PineSecurityEvalState` + `src/source/pine_security_eval.cpp`; the kernel keeps the registry, aggregator, feed store and one generic step | F S | | n | S-only proposals, not adopted | A generic `FillModel` interface with a TV model installed by the adapter (for the TradingView money / tick / path comment blocks in `engine.hpp` and the dead helpers of item e); "emit generic FIFO facts and let an adapter policy produce the KI-62 scratch" | — | Superseded by R5-3 (price-grid lane) and R5-1 (TV fill specifics stay behind `resolve_execution_terms`); the dead-helper part is item e; there is no scratch code left to re-lower (item d) | S | | o | `pineforge::admission` (market-admission journal) — **N14 addition** | was `include/pineforge/market_admission.hpp` (275 lines, installed by a kernel-only build) + `src/market_admission.cpp` (kernel TU) until N14 moved both under `source/` (`include/pineforge/source/market_admission.hpp`, `src/source/market_admission.cpp`): the observation journal of TradingView admission reviews; every `Configuration` field is a `strategy()` declaration parameter. No kernel TU consumed it (engine.hpp included it and used nothing; the hash helper in `src/broker_state_hash_internal.hpp` had one caller, `src/source/pine_state_hash.cpp`) | NEUTRAL (pure move; the fold is unchanged, so no hash domain moves) | **Status (N14): done** — moved whole to `include/pineforge/source/market_admission.hpp` + `src/source/market_admission.cpp` (`PINEFORGE_SOURCE_LAYER_SOURCES`); `hash_admission_field` moved verbatim into the source hash TU; `check_market_admission_schema.py` re-pointed; `MarketAdmissionDraft` / `MarketAdmissionJournal` spellings and the `market_admission_v2` inline namespace unchanged. ADR-0001 no longer calls it a generic stem | audit (Opus N14) | @@ -230,7 +230,7 @@ Columns: **Feature** (adapter mechanism, cited) · **Native today** (`yes` / `pa 1.0 C surface still does not expose is the 1.0 C boundary table of `docs/pages/native-engine.md`. "Today" below is the tree L13 read.* -Today a C host can configure (`strategy_configure_native_v1` c_abi.cpp:895-960), stream (`strategy_stream_*` c_abi.cpp:623-732) and read results, but strategy logic is C++-only and the batch entry points (`strategy_create`, `run_backtest*`) are per-strategy symbols a native author hand-writes. Design retained from F (most complete), with O's event polling and S's hardening rules: +Today a C host can configure (`strategy_configure_native_v1` c_abi.cpp:980-1045), stream (`strategy_stream_*` c_abi.cpp:623-732) and read results, but strategy logic is C++-only and the batch entry points (`strategy_create`, `run_backtest*`) are per-strategy symbols a native author hand-writes. Design retained from F (most complete), with O's event polling and S's hardening rules: ```c typedef struct pf_native_decision_v1 { uint64_t ordinal; int32_t interval_index; /* SCRIPT-bar index */ int64_t effective_time_ms, @@ -267,8 +267,8 @@ PF_API int pf_native_events_v1(pf_strategy_t, uint64_t after_ordinal, pf_native_ | Point | Decision | Src | |---|---|---| | Implementation | One new kernel TU (F `src/native_c_host.cpp`, O `c_native_host.cpp`, S `native_c_api.cpp`): `class CCallbackHost final : public NativeStrategyHost` forwarding each virtual to the table. Streaming reuses the existing `strategy_stream_*` symbols unchanged (they take any `BacktestEngine*`) | F O S | -| Errors | A non-zero callback return maps to `NativeFailureCode::CallbackException`; C callbacks must not unwind. The C++ path already latches exceptions as `CallbackException` native_execution_consumer.cpp:7592 | F O | -| Legality | The existing rule (`commands_allowed` native_execution_consumer.cpp:1512-1519): inside a callback, or between realtime inputs | F | +| Errors | A non-zero callback return maps to `NativeFailureCode::CallbackException`; C callbacks must not unwind. The C++ path already latches exceptions as `CallbackException` native_execution_consumer.cpp:8043 | F O | +| Legality | The existing rule (`commands_allowed` native_execution_consumer.cpp:1807-1814): inside a callback, or between realtime inputs | F | | Hardening | Tagged, size-prefixed PODs; `struct_size` + version; unknown-tag refusal; never cast the C request to a C++ variant; a C fixture compiles against the frozen previous header. S puts the API in a separate header `include/pineforge/native_c_api.h` | S | | SOP | Every symbol goes into `src/c_abi.cpp`, `pineforge.h`, `EXPECTED_RUNTIME` + both counts (check_c_abi_runtime.py:19-79), the README table and the Python harnesses, or the `source-guard-c-abi` stage fails, in preflight and every `ci_verify` profile (`CLAUDE.md`) | F O S | | Not in v1 of the C API | `resolve_execution_terms` / `validate_execution_precommit` are the adapter's seam; a C host that needs sizing uses `Sized` (F). O offers an optional `resolve_terms` callback: deferred. *History: R5 lane F4 routed both (`on_execution_terms`, `on_precommit`); the terms hook's units answer outside a host-sized close is still C++-only (the 1.0 C boundary table)* | F O | @@ -281,16 +281,16 @@ Depends on L7 (working view), L3 (so `Sized` is in v1 of the C request) and the | | F (its §2.iv A) | O (its G10) | S (its P7) | Chosen | |---|---|---|---|---| | API shape | spec `subscriptions` + push callback `on_native_timeframe_bar` | `register_native_series` + pull `native_series_bar(id)` / `native_series_slot_is_new(id)` | spec `security_requests` + `configure_native_security_feed` + `native_security(id)` view | F's subscription in the spec + push callback, plus O's pull accessor | -| Engine reuse | none: generalize the consumer's one script bucket (`ScriptBucket` native_execution_consumer.hpp:678-693, `contribute_input` native_execution_consumer.cpp:8527, `seal_script` native_execution_consumer.cpp:8492) to N; rejects exposing the `request.security` path as Pine-shaped end to end | promote the existing evaluator + feed machinery; edits `engine_security.cpp`, `engine_aux_security.cpp`, `engine_lower_tf.cpp` | same TUs + `native_calendar.cpp` | The existing feed machinery, **called, not edited**; `engine_security.cpp` untouched | -| Pump | consumer seal logic | accepted-input path (`invoke_input_callback` native_execution_consumer.cpp:7466), only for natively registered series | copy at Ready, evaluators created before the first input | O's registration-gated pump + S's copy-at-Ready | +| Engine reuse | none: generalize the consumer's one script bucket (`ScriptBucket` native_execution_consumer.hpp:755-770, `contribute_input` native_execution_consumer.cpp:9034, `seal_script` native_execution_consumer.cpp:8944) to N; rejects exposing the `request.security` path as Pine-shaped end to end | promote the existing evaluator + feed machinery; edits `engine_security.cpp`, `engine_aux_security.cpp`, `engine_lower_tf.cpp` | same TUs + `native_calendar.cpp` | The existing feed machinery, **called, not edited**; `engine_security.cpp` untouched | +| Pump | consumer seal logic | accepted-input path (`invoke_input_callback` native_execution_consumer.cpp:7914), only for natively registered series | copy at Ready, evaluators created before the first input | O's registration-gated pump + S's copy-at-Ready | | Authoritative bars | `authoritative_bars` on the subscription | `Source::ExchangeFeed` through `set_native_security_feed` | `configure_native_security_feed(id, bars, tf)` | F's field, installed through the existing store | | Lookahead | none, by construction | `lookahead = false` default, available | `lookahead_on = false`, `gaps_on = true` | open then → §4 U1; landed as `lookahead` and `gaps`, both default false (HT5) | 1. **Spec:** `std::vector subscriptions` (Appendix A.5), hashed only when non-empty (digest like `native_intrabar_path_digest`). 2. **Wiring**, all inside the native consumer at Ready → begin and only when `subscriptions` is non-empty (zero work for the adapter): register one evaluator state per subscription through the existing `register_security_eval` (engine.hpp:1295-1297, *fresh*); install `authoritative_bars` into the existing store (`set_native_security_feed` engine.hpp:1731, engine_aux_security.cpp:79-133); call `prepare_native_security_feeds` (engine.hpp:1639, defined `BacktestEngine::prepare_native_security_feeds` engine_aux_security.cpp:136, *fresh*) — the "made native-callable" step: reachable from the native consumer, still refused from host code in-run; pump `feed_security_eval_state` (engine.hpp:1312-1314, *fresh*) from the accepted-input path; deliver each completed bucket through `on_native_timeframe_bar`. -3. **Untouched:** `engine_security.cpp`, the Pine scheduler sequence pine_strategy_host.cpp:1593-1677 and its call order. The per-run evaluator init is today a *source-layer* member (`PineStrategyHost::init_security_eval_states_for_run` pine_scheduler.cpp:9-39, declared `init_security_eval_states_for_run` pine_strategy_host.hpp:932, *fresh*), so L6 adds a kernel-side twin instead of moving it. +3. **Untouched:** `engine_security.cpp`, the Pine scheduler sequence pine_strategy_host.cpp:1593-1677 and its call order. The per-run evaluator init is today a *source-layer* member (`PineStrategyHost::init_security_eval_states_for_run` pine_scheduler.cpp:9-39, declared `init_security_eval_states_for_run` pine_strategy_host.hpp:944, *fresh*), so L6 adds a kernel-side twin instead of moving it. 4. **Delivery rule (F):** a subscription bar is delivered on the input bar that completes it — its last contributing input bar, or, for a `LazyComplete` bucket, the next period's first input bar — immediately before that input's script calculation, never earlier — and never before an earlier script interval's lazily sealed calculation has run: the pump is ordered against the script interval, `seal(k) → deliver(i+1) → calc(i+1)` (L6d). -5. **Interim, documented in L0:** `on_native_input` + `TimeframeAggregator` (HT6). Doc notes (O): `set_native_security_feed` is public but inert without evaluators and throws in-run (`guard_native_mutation` engine_aux_security.cpp:83, which latches `UnsupportedSource` native_execution_consumer.cpp:1568); `set_aux_security_feed` returns false in the kernel engine_consumer.cpp:158-160. +5. **Interim, documented in L0:** `on_native_input` + `TimeframeAggregator` (HT6). Doc notes (O): `set_native_security_feed` is public but inert without evaluators and throws in-run (`guard_native_mutation` engine_aux_security.cpp:83, which latches `UnsupportedSource` native_execution_consumer.cpp:1867); `set_aux_security_feed` returns false in the kernel engine_consumer.cpp:158-160. 6. **Entanglement to put in the lane brief** (*fresh* read of `set_native_security_feed` `sha256:eca3dbea0baa461982ba89f13d0a5c43277854069fb6eb2dd2f5c5471313204d` engine_aux_security.cpp:51-83): the feed machinery is TV-calibrated. "W" / "M" buckets are built from the installed native daily bars, and the native stamps define the period partition (a holiday session folds into the next trade date's bar). A bare host that supplies `authoritative_bars` inherits those rules. Document them as the contract or gate them behind a partition policy — decide in the lane, not silently (§4 E8). 7. **HT3 / HT4:** lower-TF arrays come from the L5 sub-bar hook in v1. An auxiliary feed is a subscription whose bars come from the host; the Pine chart-slice mapping is not exposed. Whether v1 accepts a feed *finer* than the input is left to the lane brief (S wants it, F calls it a TV harness construct). **Status (audit lane N7): HT4 done for a bare host; the adapter's auxiliary path retained, measured.** `NativeRunSpec::auxiliary_feed` (`NativeAuxiliaryFeed{tf, bars}`, strictly finer than `input_tf`, opt-in and hashed only when present) plus `NativeTimeframeSubscription::source = NativeSeriesSource::AuxiliaryFeed` build a series from the host's finer bars through the existing evaluator step (`register_security_eval` against the feed's timeframe, `feed_security_eval_state` per feed bar — called, not edited); routing is by time alone (a feed bar rides on the first accepted input whose period it opened before), with `declare_auxiliary_feed` at begin, `append_auxiliary_bars` on a realtime stream and a C spelling (`PF_NATIVE_SPEC_EXT_AUXILIARY_FEED` tail, `strategy_native_append_auxiliary_bars_v1`). No epoch moved: `native_run_spec_v3` and `engine_script_run_v18` were open then (no release tag contained them, no frozen provider fixture of either existed), exactly as L6c's `gaps` and begin-time hook were added. `BacktestEngine::set_aux_security_feed` stays the source host's door and still answers a silent false on a bare host (the pre-begin setter pins, `tests/test_native_example_batch.cpp` among them, hold it to that). **Retained, why:** the adapter does not re-lower its auxiliary sites, on three measurements — 0 of 312 corpus probes install an auxiliary feed, so corpus byte-identity cannot witness the move; TradingView's chart slice leaves pre-range feed coverage inert where the kernel folds it by time (two hourly buckets against three over one feed, `tests/test_native_auxiliary_feed_twin.cpp` row B); and the adapter evaluates a chart bar's slice at its calculation, after that bar's matching pass, where the kernel delivers before it (position 1 against 0 at the same bucket, row C), which no predicate over an opaque generated `evaluate_security` can prove neutral. The congruent shape is the same series on both (row A) and can move once a population exercises it. Full contract: `docs/pages/native-engine.md`, "The auxiliary finer feed". @@ -299,7 +299,7 @@ Depends on L7 (working view), L3 (so `Sized` is in v1 of the C request) and the **Status (L10): done.** `examples/native/` holds eighteen hosts, sixteen C++ and two C, built by `PINEFORGE_BUILD_EXAMPLES`; `include/pineforge/native_module.hpp` defines `PINEFORGE_EXPORT_NATIVE_STRATEGY(Class)`. The bullets below are the plan L10 worked from. -- Move `runner/examples/native_market_strategy.cpp` and `native_selected_strategy.cpp` to a top-level `examples/native/`, built by `PINEFORGE_BUILD_EXAMPLES` (then "none yet" and guarding nothing CMakeLists.txt:37; the examples build only under `PINEFORGE_BUILD_LIVE_RUNNER` CMakeLists.txt:466-467, with `native_market_example` runner/CMakeLists.txt:54). Link the kernel target (S), or `PineForge::pineforge` during migration (O) — no SQLite / curl / OpenSSL (runner/CMakeLists.txt:1-4). +- Move `runner/examples/native_market_strategy.cpp` and `native_selected_strategy.cpp` to a top-level `examples/native/`, built by `PINEFORGE_BUILD_EXAMPLES` (then "none yet" and guarding nothing CMakeLists.txt:37; the examples build only under `PINEFORGE_BUILD_LIVE_RUNNER` CMakeLists.txt:508-509, with `native_market_example` runner/CMakeLists.txt:54). Link the kernel target (S), or `PineForge::pineforge` during migration (O) — no SQLite / curl / OpenSSL (runner/CMakeLists.txt:1-4). - Build (1) standalone executables with a `main()` that runs batch + stream on embedded bars (the guide already contains that `main` native-engine.md:3237-3305) and (2) the same MODULE targets the runner tests load; keep the C-ABI shims so the live runner can still `dlopen` them (F, O, S). - Update the three places that pin the paths: `native_market_example` runner/CMakeLists.txt:54, `NATIVE_EXAMPLES` check_native_include_independence.py:49, the tests at `test_native_example_batch` runner/CMakeLists.txt:158. - Add a minimal "hello, kernel" host (~60 lines, no C ABI — O) and one example per v1 feature lane as it lands; each doubles as that lane's twin host (F). @@ -326,11 +326,11 @@ Union of F §2.vi (9), O §2.vi (5 + 1) and S §2.6 (7), deduplicated. S had no | 4 | HTF "comes from the kernel feed `set_native_security_feed` / `prepare_native_security_feeds`" (ADR L84-89) | Not usable by a bare host: a partial boundary, not an absent kernel. `prepare_native_security_feeds` is protected with one caller; `configure_security_evaluators` has one caller; the public setter is inert pre-run and refused in-run | `prepare_native_security_feeds` engine.hpp:1639, `configure_security_evaluators` engine.hpp:1322, `set_native_security_feed` engine.hpp:1731, `guard_native_mutation` engine_aux_security.cpp:83 | F O S | | 5 | "`engine_aux_security.cpp` includes `source/pine_strategy_host.hpp`" (ADR L129-132) | False at `73817c1`: its includes are `engine_internal.hpp`, `ta.hpp` and std headers (*fresh* re-read; the run-failure codes added `run_failure.hpp`, a kernel header). No kernel TU includes a source header | `sha256:560a7f68b243d921382a69ed0390c398f28370ac066b84f30b5f96cbb385fb60` engine_aux_security.cpp:6-14 | F | | 6 | "Kernel TUs reference adapter types (`source::StrategyOverrides` …)" (ADR L129-131) | True only as a forward-declared opaque pointer the kernel never dereferences; the independence checker whitelists exactly that symbol | at `73817c1`; overtaken by `3f4fff5b` (§2.ii a), after which the rich begin carries `overrides_opaque` native_host.hpp:758 and the independence checker whitelists nothing | F O | -| 7 | `engine.hpp` hosts "the ten-significant-digit money rule, KI-62, the `strategy.close` batching + entry-id ledger, a TradingView margin-call toggle" (ADR L131-134) | Two of four survive as code. Money rule: a comment block; the arithmetic is in the adapter. KI-62: orphaned comments + one live, hashed lot flag. Close batching + id ledger: gone from the kernel (the ledger is in the adapter; the old names survive only in comments). Live: `margin_call_enabled_`, the KI-62 flag and — unmentioned by the ADR — string-sentinel decoding of adapter comments | the money-rule block was lines 927-961 of `engine.hpp` and R5 lane E6 deleted it (lines 73-180 were and are live code), the arithmetic `source_money_round` pine_adapter.cpp:489-507 and its twin `tv_money_round` pine_policy_support.hpp:9-15; `skip_entry_bar_high` engine.hpp:152, `skip_entry_bar_low` engine_state_hash.cpp:66, `declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the ledger `close_logical_units_` pine_adapter.hpp:2512-2542; `closed_trade_close_cause` engine_trade_accessors.cpp:158. **Overtaken:** the margin-call toggle and the string decoding are both gone — see ADR-0001, "Current state vs. target" | F O | -| 8 | "No kernel-level C API. `c_abi.cpp` exports only version/descriptor symbols" (ADR L136-138) | 73 runtime `PF_API` symbols: native configure, the 12-symbol stream family, pending-order views, broker-state hash, security feeds, FX curve, reports. The 73 compiled-strategy runtime symbols in `c_abi.cpp` are distinct from the native C host API in `native_c_host.cpp`: the latter exposes submit, replace, cancel and a callback table to a C host. | `sha256:70be189ab83fc6c7586c9f647c39fbf76a4f2dfac09627039b730dcb3084d175` c_abi.cpp:308-963, `sha256:4b7e398c78b276b34c4247e8271242b48aa195cea29126f4e809073f2665e1ea` check_c_abi_runtime.py:20-80, c_abi.cpp:896, `sha256:3b43bbaa40f1bab3c7ffa4e20d45fd8ea2ba321e8a54b7cbe45dc625dfca59f6` c_abi.cpp:624-743, `sha256:db54076d46a71efb0c7556c97494317e7fca443278b6963c3edf79ce918d72a1` c_abi.cpp:549-601, `sha256:f18d46ac323234f7e3082367d30840b704b8a7548f7ae95ee8cff8714f29fd50` c_abi.cpp:528-545, `sha256:2cb41f4006caae992ee41605054a3397689ae00bf3080011db1bba6412ffcd87` c_abi.cpp:836-867, c_abi.cpp:963 | F O S | +| 7 | `engine.hpp` hosts "the ten-significant-digit money rule, KI-62, the `strategy.close` batching + entry-id ledger, a TradingView margin-call toggle" (ADR L131-134) | Two of four survive as code. Money rule: a comment block; the arithmetic is in the adapter. KI-62: orphaned comments + one live, hashed lot flag. Close batching + id ledger: gone from the kernel (the ledger is in the adapter; the old names survive only in comments). Live: `margin_call_enabled_`, the KI-62 flag and — unmentioned by the ADR — string-sentinel decoding of adapter comments | the money-rule block was lines 927-961 of `engine.hpp` and R5 lane E6 deleted it (lines 73-180 were and are live code), the arithmetic `source_money_round` pine_adapter.cpp:489-507 and its twin `tv_money_round` pine_policy_support.hpp:9-15; `skip_entry_bar_high` engine.hpp:152, `skip_entry_bar_low` engine_state_hash.cpp:74, `declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the ledger `close_logical_units_` pine_adapter.hpp:2539-2569; `closed_trade_close_cause` engine_trade_accessors.cpp:158. **Overtaken:** the margin-call toggle and the string decoding are both gone — see ADR-0001, "Current state vs. target" | F O | +| 8 | "No kernel-level C API. `c_abi.cpp` exports only version/descriptor symbols" (ADR L136-138) | 80 runtime `PF_API` symbols: native configure, the 12-symbol stream family, pending-order views, broker-state hash, security feeds, FX curve, reports. The 80 compiled-strategy runtime symbols in `c_abi.cpp` are distinct from the native C host API in `native_c_host.cpp`: the latter exposes submit, replace, cancel and a callback table to a C host. | `sha256:733786ed81c9137223b98e1887e28d01afdb3c64ba5e8ca52ce5161f2caf3f75` c_abi.cpp:325-1013, `sha256:6327f1829019fdd172cd76a96e3d8c9c0af1610ad0dc9878640dda0ca21a686b` check_c_abi_runtime.py:20-81, c_abi.cpp:946, `sha256:5ff4c65a56410fe29b21cd68bc21ac87bdb34b77344e4f75f90efa8887fe48e8` c_abi.cpp:664-788, `sha256:051fe087eb9d30e9960dc4f0ab3f7cc837531ebeaa71b5871a3f1cdc22fb37c2` c_abi.cpp:566-641, `sha256:f18d46ac323234f7e3082367d30840b704b8a7548f7ae95ee8cff8714f29fd50` c_abi.cpp:545-562, `sha256:6a9ef4e688f33dc10ea8297d9eaa8144ea7923ef6ee6f8fe9ff20f6af4e9acb3` c_abi.cpp:881-917, c_abi.cpp:1013 | F O S | | 9 | "No examples. `PINEFORGE_BUILD_EXAMPLES` is 'none yet'" (ADR L139) | The option is inert, but two Pine-free `NativeStrategyHost` examples exist, are compiled by the independence checker and tested through the live runner, only under `PINEFORGE_BUILD_LIVE_RUNNER`. Wording: "no top-level examples target" | at `73817c1` (overtaken; ADR-0001, "Current state vs. target") | F O S | | 10 | Follow-ups (1)-(4) (ADR L168-170) | Add native reporting, sizing, margin, HTF — and, per R5, calc-timing, price grid and order ergonomics. Without them "usable without Pine" is a build-system claim, not a feature claim | §3.5 | F | -| 11 | Build layout | One `pineforge` library always appends the source list; a kernel-only target does not exist yet; `compat/pine/market_admission.cpp` sits in the main list, a boundary bug. **Overtaken:** `PINEFORGE_KERNEL_SOURCES` CMakeLists.txt:114 and `add_library` CMakeLists.txt:155 build `PineForge::kernel`, and the compat unit is in `PINEFORGE_SOURCE_LAYER_SOURCES` CMakeLists.txt:91 | `add_library` CMakeLists.txt:155 | S (F, O agree in §2.i) | +| 11 | Build layout | One `pineforge` library always appends the source list; a kernel-only target does not exist yet; `compat/pine/market_admission.cpp` sits in the main list, a boundary bug. **Overtaken:** `PINEFORGE_KERNEL_SOURCES` CMakeLists.txt:114 and `add_library` CMakeLists.txt:158 build `PineForge::kernel`, and the compat unit is in `PINEFORGE_SOURCE_LAYER_SOURCES` CMakeLists.txt:91 | `add_library` CMakeLists.txt:158 | S (F, O agree in §2.i) | | 12 | Host constructors | `NativeStrategyHost` is a zero-argument C++ host with the callbacks and the request API; `PineStrategyHost` is the class that takes `CapAttachment` — do not attribute it to the native constructor | `NativeStrategyHost` native_host.hpp:845, `PineStrategyHost` pine_strategy_host.hpp:287 | S (single-source) | `docs/pages/native-engine.md` fixes, all applied (F unless noted). The page is cited by @@ -343,7 +343,7 @@ the campaign's own anchor audit found that the first thing to rot: | the request-epoch sentence | `native_order_v4` | `native_order_v7` native_order.hpp:25 | | the two host-epoch sentences | host epoch v16 | `engine_script_run_v19` native_host.hpp:20 | | "What still requires Pine compatibility to build" | `BacktestEngine` still holds `CapAttachment` / `OrderPriority` / `IntradayCap` | `engine.hpp` has zero references to them; the cap type is `CapAttachment` intraday_cap.hpp:19 and the constructor is `PineStrategyHost`'s pine_strategy_host.hpp:289 | -| the rich-`run` paragraph | the rich `run` overload fails a native host | `run_rich` native_execution_consumer.cpp:9810-9852 shows no such refusal (still unpinned by a test; the page says so) | +| the rich-`run` paragraph | the rich `run` overload fails a native host | `run_rich` native_execution_consumer.cpp:10354-10396 shows no such refusal (still unpinned by a test; the page says so) | | added | — | the self-aggregation recipe beside the subscriptions it is an alternative to; the open-bar lookahead warning with `current_partial_bar` native_host.hpp:1073; and `set_native_security_feed` documented as the pre-run door for a series' authoritative bars, refused in-run | --- @@ -352,7 +352,7 @@ the campaign's own anchor audit found that the first thing to rot: ### 3.1 Parity discipline for every lane (R5-10) — stated once, referenced per lane -- **(a) NEUTRALITY.** Every new behaviour is opt-in by a spec field or a new request kind. The adapter's `project()` (pine_adapter.cpp:2524-2791) never sets the field and never emits the kind, so adapter runs are byte-identical by construction. The four feed-shape fields are folded into `hash_spec` unconditionally today, including defaults (native_execution_consumer.cpp:238-260), so even a defaulted new field would move every continuation hash; the precedent for a conditional digest is `precommit_digest_` (native_execution_consumer.hpp:1812-1816). Each lane adds a test that a default spec hashes to the pre-change constant. Every new durable field is hashed (`check_broker_state_hash_coverage.py`, fail-closed, waivers in `broker_state_hash_waivers.txt`). Proof per PR: `scripts/run_corpus.sh` + `scripts/verify_corpus.py` locally, then ONE campaign sweep on the final tree, run by the maintainers (fixed population, 4190 probes when this plan was written, zero tolerance on the hard band — native-refactor-progress.md:42-48; a fresh exact-HEAD `PASS` verdict is required, `sha256:59adeba8aabaefa740633d52e9216b49065b28b397b87f22c57ad5e19f2948c2` AGENTS.md:144-145). A lane that cannot be shown neutral by construction does not merge. +- **(a) NEUTRALITY.** Every new behaviour is opt-in by a spec field or a new request kind. The adapter's `project()` (pine_adapter.cpp:2524-2791) never sets the field and never emits the kind, so adapter runs are byte-identical by construction. The four feed-shape fields are folded into `hash_spec` unconditionally today, including defaults (native_execution_consumer.cpp:238-260), so even a defaulted new field would move every continuation hash; the precedent for a conditional digest is `precommit_digest_` (native_execution_consumer.hpp:1919-1923). Each lane adds a test that a default spec hashes to the pre-change constant. Every new durable field is hashed (`check_broker_state_hash_coverage.py`, fail-closed, waivers in `broker_state_hash_waivers.txt`). Proof per PR: `scripts/run_corpus.sh` + `scripts/verify_corpus.py` locally, then ONE campaign sweep on the final tree, run by the maintainers (fixed population, 4190 probes when this plan was written, zero tolerance on the hard band — native-refactor-progress.md:42-48; a fresh exact-HEAD `PASS` verdict is required, `sha256:59adeba8aabaefa740633d52e9216b49065b28b397b87f22c57ad5e19f2948c2` AGENTS.md:144-145). A lane that cannot be shown neutral by construction does not merge. - **(b) TWIN.** One chosen probe per lane runs through the adapter AND a hand-written `NativeStrategyHost` on the same bars and spec. Trade lists are diffed **by identity** (entry / exit time, price, qty, pnl, commission), never by trade number; then event, ownership and hash detail (S's D → E order). A difference is allowed only where a named TV-quirk row of §1 is active, and each one is itemized. Because 430 test files drove the adapter and 14 the native host when this plan was written (O; on this tree 543 `tests/*.cpp` name `source::PineStrategyHost` and 110 name `NativeStrategyHost`), every lane also adds native-only tests: a lane is not accepted on the corpus alone, nor on a unit test alone (S). - **(c) EPOCH BUDGET.** Three inline-namespace epochs move — the request epoch, the host epoch and (*fresh*, not in the inputs) the run-spec epoch, which every new spec field moves. They stood at `native_order_v5`, `engine_script_run_v17` and `native_run_spec_v2` when this plan was written; on this tree they are `native_order_v7` (native_order.hpp:25), `engine_script_run_v19` (native_host.hpp:20) and `native_run_spec_v3` (native_run_spec.hpp:15): the budget below was spent exactly once, and the v19 value epoch (R5 lane V19-A) took the host epoch one step past it. One bump each per release batch: `native_order_v6` = L3 + L7 + L4's origin / event; `engine_script_run_v18` = L4's host hooks + L5 + L6 virtuals + L2's `hash_host_extension` + §2.ii a, f; `native_run_spec_v3` = the spec blocks of L2, L4, L5, L6, L8 (L9 if it ships in the batch). A lane that lands before its batch closes stages behind the open epoch, so codegen-built strategies rebuild once per batch. Epoch lanes are not neutral-refactor lanes: the brief lists the header extension explicitly; the frozen C++ ABI fixtures (`tests/fixtures/native_cpp_abi/host-*`) and the ABI checkers (`check_native_cpp_abi.py`, `check_settlement_cpp_abi.py`) move with them; `variant_size` pins change (`OrderIntent` native_order.hpp:229, `CommandEvent` native_order.hpp:1308). Every new test unit compiles first against the frozen previous-epoch header closure (fail-before, `CLAUDE.md`). - **Worker verification** (`CLAUDE.md`): `ci_preflight.py`, then `ci_verify.py release`; plus `check_c_abi_runtime.py` and `check_native_include_independence.py`. No campaign sweep from a worker. @@ -410,17 +410,17 @@ closure markers and `scripts/check_design_inventory.py`). |---|---|---|---| | L0 | — | docs only | — | | L1 | option `PINEFORGE_BUILD_SOURCE_LAYER`, targets `pineforge_kernel` + unchanged `pineforge` (§2.i) | same TUs, same flags; ctest + corpus | kernel-only link of the two examples + the 20 source-free native tests; installed-header closure | -| L2 | `NativeReportPolicy { HostRecorded, KernelRecorded }` (R1 appended a third value, `KernelRecordedAtHostMarks` native_run_spec.hpp:64), `report_open_position_at_end`, `hash_host_extension`, `native_metrics()` → A.1 | at L2 the adapter stayed `HostRecorded`; since R1 it declares `KernelRecordedAtHostMarks` pine_adapter.cpp:2790, so the kernel records the same series at the adapter's own report marks — still one recorder, no double recording, `fill_report` bytes unchanged on the corpus | native example: curve length == script bars, drawdown equals a hand walk (F); 50-bar host asserts a non-empty curve, a finite max drawdown and a range-end row (O); same generic strategy through both hosts → equal rows and curve (S) | +| L2 | `NativeReportPolicy { HostRecorded, KernelRecorded }` (R1 appended a third value, `KernelRecordedAtHostMarks` native_run_spec.hpp:64), `report_open_position_at_end`, `hash_host_extension`, `native_metrics()` → A.1 | at L2 the adapter stayed `HostRecorded`; since R1 it declares `KernelRecordedAtHostMarks` pine_adapter.cpp:2796, so the kernel records the same series at the adapter's own report marks — still one recorder, no double recording, `fill_report` bytes unchanged on the corpus | native example: curve length == script bars, drawdown equals a hand walk (F); 50-bar host asserts a non-empty curve, a finite max drawdown and a range-end row (O); same generic strategy through both hosts → equal rows and curve (S) | | L3 | `Sized{side, basis, time, grid_policy, reserve_percent_fee}` as the 6th `OrderIntent`; `ScopeFraction{fraction, claim}` in `ReductionSize` → A.2 | at L3 the adapter emitted neither. Since R2 it lowers a default cash / percent opening frozen at the signal onto `Sized{CashValue, AtAcceptance, SignalOnTick, ExplicitUnits}`, since N11 its placement-time quantity is the kernel's `native_sized_units`, and since R5 lane F7 a typed quantity (`qty_type` cash / percent) converts through the same query and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (`tests/test_adapter_sizing_relower.cpp` cases 1-4, 8, 11, 12). It still never emits `ScopeFraction` (case 5) | percent-of-equity probe, zero fee, no grid: exact; with grid / fee: differences explained by SZ10 only (F). Differential: `Sized` vs a host-resolved `HostSized` on the same facts → identical `ExecutionAppliedEvent` (O). Gap, reversal, fee-bearing first lot; two 50 % siblings, partial close, pending parent, replacement (S) | | L4 | `NativeMarginModel`, `native_liquidation_price()`, `resolve_margin_call_units`, `on_native_margin_call`, `RequestDefinition::origin`, `MarginCallEvent` → A.3 | at L4 the adapter left `margin` unset. Since R5 `project()` declares a maintenance-only model (per-side `initial_*` 0, `maintenance_*` = `margin_long` / `margin_short`, `PathAdverseExtremeMark`, TradingView's equity basis) and answers its three hooks with TradingView's scheduling, money and slice (MG6, MG7), while every opening is still `AdmitWithHostMargin` (MG4) | first a plain maintenance breach in both hosts (S); then a margin-call probe reproduced with `ShortfallMultiple 4`: same bar, same side; quantity deltas itemized against MG15 (F, O) | -| L5 | `NativeCalculationTrigger`, `on_native_recalculate(…, reason, cause)`, `on_native_sub_bar`, `current_partial_bar()`, `NativeOpenBarView` → A.4 | the adapter keeps `BarClose` + its own cascade; `born_on_remaining_path` (native_execution_consumer.cpp:5944-5948) and `drain_applied_notifications` (native_execution_consumer.cpp:6609-6622) are frozen; accessors are inert | COOF probe re-expressed natively: same fills where the TV waypoint rule is inactive (F); a host that flips on its own fill produces the adapter's cascade in shape (O); callback reason, cursor, newborn eligibility and event ordinals compared (S) | +| L5 | `NativeCalculationTrigger`, `on_native_recalculate(…, reason, cause)`, `on_native_sub_bar`, `current_partial_bar()`, `NativeOpenBarView` → A.4 | the adapter keeps `BarClose` + its own cascade; `born_on_remaining_path` (native_execution_consumer.cpp:6364-6368) and `drain_applied_notifications` (native_execution_consumer.cpp:7039-7052) are frozen; accessors are inert | COOF probe re-expressed natively: same fills where the TV waypoint rule is inactive (F); a host that flips on its own fill produces the adapter's cascade in shape (O); callback reason, cursor, newborn eligibility and event ordinals compared (S) | | L6 | `NativeTimeframeSubscription`, `on_native_timeframe_bar`, `native_series_bar` → A.5, §2.iv | `subscriptions` empty for the adapter; registration-gated pump; `engine_security.cpp` and pine_strategy_host.cpp:1086-1170 untouched; corpus diff before / after (O: highest parity risk of any lane) | HTF-filter probe on a 24x7 symbol: exact (F); `"D"` over a 15m feed equals a `TimeframeAggregator` baseline bar for bar (O); installed-feed and no-feed cases (S) | | L7 | `native_working_requests()`, `cancel_all()`, `cancel_where(label)`, `TriggerAnchor`, `TrailTicks`, zero offset, `ReplaceOptions`; toolkit `submit_bracket`, `OrderBook` → A.6 | new kinds unused by the adapter; anchor defaults to `Absolute`; the adapter's `tick * 0.5` sentinel keeps working | bracket + trailing probe through the builder: exact where TR5 / FP7 are inactive (on-grid levels) (F); the zero-offset trail test asserts the run no longer fails (O); parent-limit + stop sibling, parent rejection, replacement, parent-cycle revival (S) | | L8 | `NativePriceGrid { None, QuantizeFills, QuantizeFillsAndTriggers }`, `NativeGridRounding` → A.7 | `None` for the adapter, permanently — **native-only by ruling, audit lane P6: §3.6.2** | a host with `QuantizeFills` books on-grid prices; a second test pins that `QuantizeFillsAndTriggers` still differs from TV's half-tick threshold (FP5) (O) | | L9 | `NativeRiskLimits`, `NativeRiskEvent`, `MatchRejectReason::RiskLimit` → A.8 | `risk` unset for the adapter; its ledger (`SourceDayLedger` pine_adapter.hpp:1423) untouched — **retained after measurement, audit lane N12; native-only by ruling, audit lane P6: §3.6.1** | risk probes: same halt bar (F); each limit + the day-boundary basis (O); breach, forced close, cancellation, next-day reset (S) | | L10 | `PINEFORGE_EXPORT_NATIVE_STRATEGY(Class)` (§2.v) | build-only | the examples run in ctest; the independence checker compiles the relocated examples | | L11 | — (renames / moves, §2.ii a-i) | rename / move only; hashed enumerator values pinned; sweep unchanged | — | -| L12 | — (§2.ii j-m) | **not neutral**: coordinated sweep, waiver updates, hash-domain plan (the `"pineforge-broker-state/v17"` literal of that wave, since moved to `"pineforge-broker-state/v19"` engine_state_hash.cpp:33 and pinned to one occurrence by check_broker_state_hash_coverage.py:223) | trades identical, hashes re-baselined once | +| L12 | — (§2.ii j-m) | **not neutral**: coordinated sweep, waiver updates, hash-domain plan (the `"pineforge-broker-state/v17"` literal of that wave, since moved to `"pineforge-broker-state/v19"` engine_state_hash.cpp:41 and pinned to one occurrence by check_broker_state_hash_coverage.py:223) | trades identical, hashes re-baselined once | | L13 | `pf_native_*` symbols (§2.iii) | additive symbols; `check_c_abi_runtime.py` exits 0 | a pure-C twin of the market example reproduces the C++ trade rows (O) and the event-history hash (F); C test: submit a market, replace a limit, cancel a child, read the Applied event (S) | ### 3.4 Input dependency notes against the R5-11 order @@ -446,7 +446,7 @@ Flagged: | Tier | Lanes | Why | |---|---|---| -| Day-one blockers (O) | L1, L2, L3, L10 | Without L2 `fill_report` silently returns an empty curve and degenerate equity metrics and drops an open final position. Without L3 every non-unit order needs a hand-written `resolve_execution_terms`, and a mistake is a run-terminating `TermsUnresolved` (native_execution_consumer.cpp:5292-5294). Without L1 "kernel" is a claim, not an artifact. L10 is the minimum buildable proof. | +| Day-one blockers (O) | L1, L2, L3, L10 | Without L2 `fill_report` silently returns an empty curve and degenerate equity metrics and drops an open final position. Without L3 every non-unit order needs a hand-written `resolve_execution_terms`, and a mistake is a run-terminating `TermsUnresolved` (native_execution_consumer.cpp:5704-5706). Without L1 "kernel" is a claim, not an artifact. L10 is the minimum buildable proof. | | Required by the promise (R5-11) | L0, L4, L5, L6, L7, L8 | A kernel that cannot liquidate is not a broker model (O); bracket / trailing corner cases are hit immediately (O); calc-timing and HTF are Pine features the user wants natively (R5-5, R5-6); the price grid is generic and missing (R5-3). | | Wanted with v1, not blocking | L11 a, d | removes the last `source::` name from kernel signatures and the orphaned TV commentary (F) | | In scope, 1.x | L9 | R5-9; a toolkit stand-in is acceptable meanwhile (F) | @@ -502,12 +502,12 @@ inside the per-bar body (corpus 442d497, `set_pine_risk_max_intraday_filled_orders(3)`), so the adapter first sees a limit on script bar 0, from `PineStrategyHost::set_pine_risk_*` (pine_strategy_host.cpp:846-868) — after `prepare_native_begin` has already -projected the spec (pine_strategy_host.cpp:227) and `configure_native` has +projected the spec (pine_strategy_host.cpp:229) and `configure_native` has folded it into the continuation digest. Measured: an env-gated probe printing the adapter's risk configuration inside `project()` on the corpus reads `max_drawdown=0 cap.active=0` for every risk probe. `NativeRunSpec::risk` is a begin-time declaration (`native_run_spec.hpp:677-684,504`, folded at -native_execution_consumer.cpp:236-237); a statement-time limit has nowhere to +native_execution_consumer.cpp:332-333); a statement-time limit has nowhere to land. Routing any rule needs either the transpiler to hoist constant risk statements into the constructor (out of this repository) or a kernel re-declaration API mid-run (a hash-visible mutation of a digested field: an @@ -518,17 +518,17 @@ adapter's own rule acts. **Per rule (STEP 1), assuming the values were known at begin time.** Adapter line numbers are re-anchored to this tree (the measurement ran on 683a82f); kernel semantics are -`NativeRiskLimits` native_run_spec.hpp:310 and `risk_evaluate` native_execution_consumer.cpp:3997 +`NativeRiskLimits` native_run_spec.hpp:310 and `risk_evaluate` native_execution_consumer.cpp:4330 (evaluated at the script bar's open, its close calculation and after each applied drain; fills counted at the corresponding adapter checkpoints; the block refuses `would_open`). | Pine rule | adapter mechanism | kernel counterpart | class | measured divergence (test scenario) | ruling | |---|---|---|---|---|---| -| `strategy.risk.allow_entry_in` | `risk_.direction`; `project()` already declares `spec.allowed_open_directions` (pine_adapter.cpp:2794); the forbidden opposite entry is reshaped to `CloseOpposite` in the adapter's own terms branch | per-opening cap `allowed_open_directions` | (i)+(ii), **already re-lowered** | none to measure: the cap is the kernel's, the close-only reshaping is the retained TV policy | done before N12; nothing to move | -| `strategy.risk.max_position_size` | precommit refuses a flat/same-side ENTRY at its fill when the LIVE book already holds ≥ the limit (`max_position_size` pine_adapter.cpp:16268-16269; the retired legacy rule is `git show ab9714be:src/source/pine_risk.cpp`, lines 115 and 2531) | `max_abs_units`: refuses a fill whose RESULTING book would exceed it | (iv) | `PS`: two-unit entries against 3 — adapter admits the second (live 2 < 3, book 4) and refuses the third; kernel refuses the second (`MaxAbsUnits` at bars 1, 2), book 2 | retained: a pre-fill-book gate is a TV emulation fact, not a generic cap | -| `strategy.risk.max_drawdown` | `update_risk_state(bar.close)` once per script bar at the close mark (`update_risk_state(bar.close)` pine_adapter.cpp:23878, `update_risk_state` pine_adapter.cpp:17233-17260): peak / running max drawdown at close marks, `>=` exact, percent of the current peak; the latch `risk_.halted` gates flat/same-side entries at precommit (`risk_.halted` pine_adapter.cpp:16262), never an opposite entry | `max_drawdown`: current drawdown vs percent-of-peak, at all three points; `BlockOpenings` refuses every opening | (ii) cadence + (iv) scope | `MG10-a`: bar 1 opens 600 under the entry and closes 100 under — kernel breaches at the open mark and refuses bar 2's add (book 100), adapter admits it (book 200). `MG10-b`: latched at bar 1's close, the adapter still reverses twice (`L→S` −600, `S→L`), the kernel refuses both reversals (`RiskLimit` at bars 2, 3) and holds. Corpus: `ta-closedtrades-risk-introspection-01` (20 % of peak) identical 1502/1502 rows with the kernel seeded — the rule never fires there | retained: the latched-reversal exemption cannot be composed on a kernel block; a close-only cadence knob alone would not close it | -| `strategy.risk.max_cons_loss_days` | `SourceDayLedger` (pine_adapter.hpp:1419-1427): +1 per losing TRADE on a new chart day, reset to 0 by any winning trade at its fill (`consecutive_loss_days` pine_adapter.cpp:25467-25477); gated immediately at precommit and latched at the close | `max_consecutive_loss_days`: a day's NET realized result settles the streak when the next day opens | (iv) | `MG11-a` (loss; win then loss; loss): adapter latches at day 3's first loss (4 rows), kernel's netted streak never reaches 2 (6 rows, no event). `MG11-b` (loss then smaller win, twice): adapter resets daily (5 rows), kernel blocks at day 3's open (4 rows, one event). No corpus probe declares the rule | retained: an order-dependent per-trade streak is not a generic day result | -| `strategy.risk.max_intraday_loss` | day-open equity at the day's first bar (`intraday_start_equity` pine_adapter.cpp:22251); checked at the bar open (`submit_intraday_loss_close(bar.open, context, true)` pine_adapter.cpp:22651), along the bar's PATH at its adverse extreme with a resting `Stop` flatten (`schedule_intraday_loss_path` pine_adapter.cpp:20651-20660, `submit_intraday_loss_close` pine_adapter.cpp:20600-20628), and at a closing fill with that fill's own P&L unbooked (`preclose_intraday_loss` pine_adapter.cpp:24435); epsilon `1e-9·max(1,|limit|)`; ticket `"Close Position (Max intraday Loss)"` → `CloseCause::RiskLimit`; the block refuses ALL placements for the chart day (`entry`/`order`/`close`/`close_all`/`exit`, `intraday_loss_orders_blocked` pine_adapter.cpp:7125, `:9441`, `:10350`, `:10849`, `:14398`) and `cancel_all()` withdraws the working book (`cancel_all()` pine_adapter.cpp:25568-25571) | `max_intraday_loss` at the three points; `FlattenAndBlock` closes `AsPresented` under `"__kernel_risk__"` / `"Risk limit"`, blocks openings only, leaves working orders | (ii) ticket, day basis + (iv) path check, placement scope, unbooked fill, epsilon | `MG12-a`: day 2's first bar 100→85→95 — adapter closes at the extreme (85, −1500, cause 4) and refuses the day's later entry, kernel measures 0 at the open and 500 at the close, never breaches, ends with 300 units. `MG12-b`: breach at a bar open inside the day — same money on both sides (100 @ 88, −1200) under different tickets; the adapter withdraws a resting limit entry, the kernel leaves it working and it fills on day 3. Both sides mark the day's opening equity at its first bar: a gap there is not an intraday loss for either | retained: the path-extreme check is the rule's substance and the kernel has no path evaluation point; a risk ticket field and a `PathAdverseExtreme` check kind would be generic additions (mirroring `NativeMarginModel`) but would still leave the placement scope, the unbooked fill and the epsilon | -| `strategy.risk.max_intraday_filled_orders` | `compat::pine::IntradayCap` + `IntradayOrderBudget` (243 + 94 lines): charged slots per risk day, quota transfer, POOC deferral, noop-market skip, the close at the fill price or the bar's better extreme (`post_dispatch`), latch → all placements denied (`cap_placement_denied` pine_adapter.cpp:17249-17251); ticket `"Close Position (Max number of filled orders in one day)"` → `CloseCause::FillCap`; day = `session_trading_day_index` for a `HHMM-HHMM` session, else the chart day (`chart_day_key` pine_adapter.cpp:17088-17109, `mday·100+month` in the CHART timezone) | `max_fills_per_day`: applied fills counted as they settle, evaluated at the three points; `FlattenAndBlock` at the drain's cursor; `SessionDay` / `CalendarDayInTimezone` on the SPEC timezone | (i) count + (ii) ticket + (iv) close price, charged-slot budget, placement scope, chart-day key | `MG13`: the second fill reaches the cap on a bar that closes above its open — adapter closes at that bar's HIGH (103) under its ticket, kernel at the close (101) under its own. `MG14`: chart timezone `America/New_York` over a UTC symbol — bar 4 (04:00 UTC) is a new chart day for the adapter and the same UTC day for the kernel. Corpus, kernel seeded with the declared cap: `cap-risk-gates-allow-max-intraday-01` identical 1464/1464 (the third fill of each day flattens the book, so block and latch coincide); `cap-max-intraday-filled-orders-isolate-01` 3840 of 3916 rows differ from row 77 (2025-04-08); `cap-gatekeeper-intraday-risk-01` 312 of 604 differ from row 293 (2025-10-25); `composite-bracket-cap-range-pending-stop-01` 2370 of 2384 differ from row 17 (2025-04-02 02:45), two extra rows | retained: the count alone is expressible, but the kernel's block would refuse fills the transfer admits and the close price is TV's; a report-only action would give the adapter a count it cannot use (charged slots ≠ applied fills) | +| `strategy.risk.allow_entry_in` | `risk_.direction`; `project()` already declares `spec.allowed_open_directions` (pine_adapter.cpp:2800); the forbidden opposite entry is reshaped to `CloseOpposite` in the adapter's own terms branch | per-opening cap `allowed_open_directions` | (i)+(ii), **already re-lowered** | none to measure: the cap is the kernel's, the close-only reshaping is the retained TV policy | done before N12; nothing to move | +| `strategy.risk.max_position_size` | precommit refuses a flat/same-side ENTRY at its fill when the LIVE book already holds ≥ the limit (`max_position_size` pine_adapter.cpp:16276-16277; the retired legacy rule is `git show ab9714be:src/source/pine_risk.cpp`, lines 115 and 2531) | `max_abs_units`: refuses a fill whose RESULTING book would exceed it | (iv) | `PS`: two-unit entries against 3 — adapter admits the second (live 2 < 3, book 4) and refuses the third; kernel refuses the second (`MaxAbsUnits` at bars 1, 2), book 2 | retained: a pre-fill-book gate is a TV emulation fact, not a generic cap | +| `strategy.risk.max_drawdown` | `update_risk_state(bar.close)` once per script bar at the close mark (`update_risk_state(bar.close)` pine_adapter.cpp:23896, `update_risk_state` pine_adapter.cpp:17241-17268): peak / running max drawdown at close marks, `>=` exact, percent of the current peak; the latch `risk_.halted` gates flat/same-side entries at precommit (`risk_.halted` pine_adapter.cpp:16270), never an opposite entry | `max_drawdown`: current drawdown vs percent-of-peak, at all three points; `BlockOpenings` refuses every opening | (ii) cadence + (iv) scope | `MG10-a`: bar 1 opens 600 under the entry and closes 100 under — kernel breaches at the open mark and refuses bar 2's add (book 100), adapter admits it (book 200). `MG10-b`: latched at bar 1's close, the adapter still reverses twice (`L→S` −600, `S→L`), the kernel refuses both reversals (`RiskLimit` at bars 2, 3) and holds. Corpus: `ta-closedtrades-risk-introspection-01` (20 % of peak) identical 1502/1502 rows with the kernel seeded — the rule never fires there | retained: the latched-reversal exemption cannot be composed on a kernel block; a close-only cadence knob alone would not close it | +| `strategy.risk.max_cons_loss_days` | `SourceDayLedger` (pine_adapter.hpp:1419-1427): +1 per losing TRADE on a new chart day, reset to 0 by any winning trade at its fill (`consecutive_loss_days` pine_adapter.cpp:25485-25495); gated immediately at precommit and latched at the close | `max_consecutive_loss_days`: a day's NET realized result settles the streak when the next day opens | (iv) | `MG11-a` (loss; win then loss; loss): adapter latches at day 3's first loss (4 rows), kernel's netted streak never reaches 2 (6 rows, no event). `MG11-b` (loss then smaller win, twice): adapter resets daily (5 rows), kernel blocks at day 3's open (4 rows, one event). No corpus probe declares the rule | retained: an order-dependent per-trade streak is not a generic day result | +| `strategy.risk.max_intraday_loss` | day-open equity at the day's first bar (`intraday_start_equity` pine_adapter.cpp:22269); checked at the bar open (`submit_intraday_loss_close(bar.open, context, true)` pine_adapter.cpp:22669), along the bar's PATH at its adverse extreme with a resting `Stop` flatten (`schedule_intraday_loss_path` pine_adapter.cpp:20659-20668, `submit_intraday_loss_close` pine_adapter.cpp:20608-20636), and at a closing fill with that fill's own P&L unbooked (`preclose_intraday_loss` pine_adapter.cpp:24453); epsilon `1e-9·max(1,|limit|)`; ticket `"Close Position (Max intraday Loss)"` → `CloseCause::RiskLimit`; the block refuses ALL placements for the chart day (`entry`/`order`/`close`/`close_all`/`exit`, `intraday_loss_orders_blocked` pine_adapter.cpp:7133, `:9449`, `:10358`, `:10857`, `:14406`) and `cancel_all()` withdraws the working book (`cancel_all()` pine_adapter.cpp:25586-25589) | `max_intraday_loss` at the three points; `FlattenAndBlock` closes `AsPresented` under `"__kernel_risk__"` / `"Risk limit"`, blocks openings only, leaves working orders | (ii) ticket, day basis + (iv) path check, placement scope, unbooked fill, epsilon | `MG12-a`: day 2's first bar 100→85→95 — adapter closes at the extreme (85, −1500, cause 4) and refuses the day's later entry, kernel measures 0 at the open and 500 at the close, never breaches, ends with 300 units. `MG12-b`: breach at a bar open inside the day — same money on both sides (100 @ 88, −1200) under different tickets; the adapter withdraws a resting limit entry, the kernel leaves it working and it fills on day 3. Both sides mark the day's opening equity at its first bar: a gap there is not an intraday loss for either | retained: the path-extreme check is the rule's substance and the kernel has no path evaluation point; a risk ticket field and a `PathAdverseExtreme` check kind would be generic additions (mirroring `NativeMarginModel`) but would still leave the placement scope, the unbooked fill and the epsilon | +| `strategy.risk.max_intraday_filled_orders` | `compat::pine::IntradayCap` + `IntradayOrderBudget` (243 + 94 lines): charged slots per risk day, quota transfer, POOC deferral, noop-market skip, the close at the fill price or the bar's better extreme (`post_dispatch`), latch → all placements denied (`cap_placement_denied` pine_adapter.cpp:17257-17259); ticket `"Close Position (Max number of filled orders in one day)"` → `CloseCause::FillCap`; day = `session_trading_day_index` for a `HHMM-HHMM` session, else the chart day (`chart_day_key` pine_adapter.cpp:17096-17117, `mday·100+month` in the CHART timezone) | `max_fills_per_day`: applied fills counted as they settle, evaluated at the three points; `FlattenAndBlock` at the drain's cursor; `SessionDay` / `CalendarDayInTimezone` on the SPEC timezone | (i) count + (ii) ticket + (iv) close price, charged-slot budget, placement scope, chart-day key | `MG13`: the second fill reaches the cap on a bar that closes above its open — adapter closes at that bar's HIGH (103) under its ticket, kernel at the close (101) under its own. `MG14`: chart timezone `America/New_York` over a UTC symbol — bar 4 (04:00 UTC) is a new chart day for the adapter and the same UTC day for the kernel. Corpus, kernel seeded with the declared cap: `cap-risk-gates-allow-max-intraday-01` identical 1464/1464 (the third fill of each day flattens the book, so block and latch coincide); `cap-max-intraday-filled-orders-isolate-01` 3840 of 3916 rows differ from row 77 (2025-04-08); `cap-gatekeeper-intraday-risk-01` 312 of 604 differ from row 293 (2025-10-25); `composite-bracket-cap-range-pending-stop-01` 2370 of 2384 differ from row 17 (2025-04-02 02:45), two extra rows | retained: the count alone is expressible, but the kernel's block would refuse fills the transfer admits and the close price is TV's; a report-only action would give the adapter a count it cannot use (charged slots ≠ applied fills) | **Proposed generic additions (not implemented — no consumer today).** A risk ticket (`label` / `comment` on `NativeRiskLimits`, folded only when set, @@ -557,7 +557,7 @@ Pine run needs the source change the patch makes, and the in-process pins remain the five probes' baseline rows and the `SW` check above. **The P6 decision: native-only.** Re-verified on main `b8e7976e`: the ordering -stands (`project()` at pine_strategy_host.cpp:333, the kernel's `configure` at :342, +stands (`project()` at pine_strategy_host.cpp:335, the kernel's `configure` at :344, the `set_pine_risk_*` setters (`set_pine_risk_direction` pine_strategy_host.cpp:854-877), and corpus 442d497 `corpus/validation/cap-risk-gates-allow-max-intraday-01`'s generated strategy still emits the three calls inside `on_source_bar`); `rg "spec\.risk|NativeRiskLimits" src/source @@ -851,7 +851,7 @@ so the gross open profit is 800. | **the equity re-mark** | `last.open_profit = 0; last.equity = initial_capital_ + net_profit_sum_ + range_end_pnl` | none — the kernel leaves the curve exactly as the run marked it | `RE1` (no fee): the adapter reports the curve's last point as `eq=100800 op=0`, the kernel as `eq=100800 op=800` — same equity, the open profit moved into the realized side. `RE2` (0.1 %): `eq=100758.8 op=0` against `eq=100800 op=800`; the adapter's equity is **41.2 lower**, exactly the round-trip commission of both lots (20.5 + 20.7), because the re-mark carries the NET row P&L where the run's own mark carried the GROSS open profit | retained: a report that restates the last equity point off net row P&L is TradingView's report shape. Re-marking the curve from the kernel would move a figure every bare host already reads | | **the extreme re-fold** | `max_equity_`/`min_equity_` reset to `initial_capital_`, `max_drawdown_`/`max_runup_` to 0, then `fold_equity_extreme` over the whole curve | none — the extremes stay as `update_equity_extremes` sampled them | `RE3` (a dip to 90 at bar 1, a peak of 108 at bar 3, 0.1 %): both sides agree on `maxeq=101400 mineq=99000 maxdd=2400`, and the run-up diverges — adapter `maxru=1758.8000000000029`, kernel `maxru=1800`. The re-fold is a no-op **except** where the re-marked last point differs, so the divergence is again the 41.2 round trip, modulo the binary fold order | retained: it is a consequence of the re-mark above, not a mechanism of its own; it cannot be re-lowered while the re-mark is adapter policy | | **the same-bar exit re-sort** | `sort_same_bar_exit_trades(trades_, adapter_)` — same-bar bracket exits ordered by script command sequence | none — the kernel never reorders `trades_` | not exercised by the four scenarios (it reorders CLOSED rows, not range-end rows); it is TradingView command-sequence ordering by construction (`git show ab9714be:src/source/pine_fills.cpp`, lines 664-670) | retained: script command sequence is a source-language fact | -| **the mark cadence** | three call sites, each clearing and rebuilding: the terminal sub-bar of the last source bar (`scheduler_record_range_end` pine_strategy_host.cpp:584), after every applied execution once terminal (`record_applied_range_end`, :775), and the scheduler's terminal-source-bar hook (`PineScheduler::applied` pine_scheduler_native.cpp:632, before the bar's broker hash) | once, after the last point of the run (`run_batch` ×3, `stream_end`) | structural, not a number: the adapter's rows are re-derived at each mark so the report reflects the book as of the last event on the terminal bar, and its re-marked equity is visible to that bar's broker hash. A one-shot run-end producer cannot occupy those three points | retained: three marks on a terminal bar is a TradingView report cadence, not a generic run-end fact | +| **the mark cadence** | three call sites, each clearing and rebuilding: the terminal sub-bar of the last source bar (`scheduler_record_range_end` pine_strategy_host.cpp:595), after every applied execution once terminal (`record_applied_range_end`, :786), and the scheduler's terminal-source-bar hook (`PineScheduler::applied` pine_scheduler_native.cpp:688, before the bar's broker hash) | once, after the last point of the run (`run_batch` ×3, `stream_end`) | structural, not a number: the adapter's rows are re-derived at each mark so the report reflects the book as of the last event on the terminal bar, and its re-marked equity is visible to that bar's broker hash. A one-shot run-end producer cannot occupy those three points | retained: three marks on a terminal bar is a TradingView report cadence, not a generic run-end fact | | **the excursion projection gate** | `excursion_range_end_projection_` suppressed the owner's exit-bar path fold for the projection | none — the generic builder folds what it folds | `RE1`: identical run-up/draw-down on both halves (`ru=500 dd=0`, `ru=300 dd=0`) | retired by R5 lane H-THIN: the Pine host owns no lot excursion since, so no owner fold is left to suppress; `RE1` still reads the same magnitudes, now the kernel's | | **the tail / warm-up suppression** | `if (stream_warmup_mode_ \|\| realtime_tail_ \|\| equity_curve_.empty()) return;` | none — the kernel emits at `stream_end` whatever the run left open | structural: the kernel has no warm-up or realtime-tail notion, and `run_corpus`'s probe-tail suppression (`suppress_probe_tail`) is a harness fact | retained: a tail is a source-layer replay concept | @@ -913,7 +913,7 @@ this lane re-pins the affected v19 witnesses once. |---|---|---|---|---| | AG1 | A trade row counts **chart** bars: its entry and exit bar are the `bar_index` the script read there, so `bar_index - strategy.opentrades.entry_bar_index(0)` is the bars held, and a tape row one chart bar long says "Duration (bars)" 1 | The pre-K-IDX kernel used the first contributing INPUT slot as its public interval index on an aggregated run. That made a two-script-bar trade report input indices and made `avg_bars_in_trade` count input slots. | **Kernel (Option A).** `NativeExecutionConsumer::ScriptBucket::script_index` assigns the sequential script index when a bucket seals. `NativeCoordinate::interval_index`, `PyramidEntry::entry_bar_index`, `Trade::exit_bar_index`, open-lot rows and the metrics calculation all consume that script index. `NativeCoordinate::input_interval_index` preserves the first input slot. `PineStrategyHost::on_native_applied` no longer rewrites lots or rows; Pine's cohort sampling still passes its explicit chart source index to the source-layer sampler (its excursion sampling is the kernel's since R5 lane H-THIN). | `kc_a310` / `kc_a310_c` in the K-IDX scratch: batch, stream and both magnified modes report `0/1/2`, entry/exit `1/2`, avg `2`; `tests/test_aggregated_path_regressions.cpp` and `tests/test_pooc_fill_stamp_aggregation.cpp` remain green. TradingView: `tests/test_aggregated_entry_bar_index_tape.cpp` (R5 lane H-MEASURE), a tape whose exits read the indices, booked field for field on all three paths. | | AG2 | Every fill of a chart bar is dated at the bar's **open**: the e25-f-islastbar tape dates all 510 of its `process_orders_on_close` rows there, and `ab9714be`'s aggregated rows equal its chart rows | The kernel dates a close point (the calculation, the close after it, a modeled bar's close leg) at `calculation_time()`, the bucket's close, and a fill the host executes at the current point (a `calc_on_order_fills` first-open chain, a same-bar close-then-entry, the KI-62 cover) at its decision floor, which on the no-path route the bucket's inputs have already carried to the close. The chart timeframe's zero-width interval hides both, since every point of a bar sits at its label | **Adapter, at projection time.** On a chart whose script bars the kernel gathers from the input (`native_aggregates_input_bars()`, the kernel's `pairing_aggregates` of the resolved pairing, since R5 lane H-THIN; the adapter's own literal test parted from it on one-bucket pairings such as `1D` on `D`), `on_native_applied` dates the lot a fill opened (`PyramidEntry::time`, hence `Trade::entry_time`) and the rows it closed (`Trade::exit_time`) at the chart bar's open: on an aggregated chart every point of a confirmed bar's modeled path (its open, the segments, the calculation, the close after it), where the chart timeframe's own points all sit, and under the magnifier every sub-bar point too (each sub-bar's open, path and `ModeledOHLCClose`; lane E27 had cut it to the close points, R5 lane PAR-ORDERS widened it, TradingView's dating). A realtime print keeps its own instant, as the chart-timeframe stream keeps it, and so does a stream's quiet carried interval | the audit's pooc probe rows (`ab9714be`'s 19:45Z / 13:30Z on all three paths), row 4, the KI-62 row and the stream row (a chart-timeframe stream and the same stream fed 1m agree on every row, a 10:30:01 print fill and an 11:00 carried fill included); E27's stamp rows flipped. Measured: the chart path unchanged byte for byte, the magnified run moves only its POOC close-fill instants. TradingView itself dates a magnified fill at its chart bar's open too (218 of 218 fills on sixteen `lab tv` tapes, R5 lane H-MEASURE); the magnified run kept a sub-bar fill's own instant (71 of the 152 instants `tests/test_magnified_aggregated_tape.cpp` replays sat inside their bar) until R5 lane PAR-ORDERS, and the row now asserts every instant exactly | -| AG3 | A `calc_on_order_fills` recalculation runs the script **on the bar the fill is on** and reads that bar's `session.ismarket` / `isfirstbar` / `islastbar`; `ab9714be` set them before any fill of the bar | The scheduler set the flags in the close callback only, so a recalculation published the PREVIOUS bar's: a fill at the next session day's 09:30 read `islastbar`. On the chart timeframe the close then read its lookahead from a count the recalculation had already advanced, one retained bar too far: a fill on 15:30 flagged 15:30 `islastbar` and, through E26's dual, 15:45 `isfirstbar` | **Adapter** (Pine language state, OT7). `scheduler_update_session_state` stays the one writer and runs once per script bar, at the bar's first publication: `PineScheduler::recalculate` asks before the script runs, `PineScheduler::bar` pine_scheduler_native.cpp:526 only when no recalculation published the bar. Since R5 lane F5 it selects the kernel's session-day facts of that bar, which every callback of the bar carries alike, so the scheduler's lookahead is gone. E26's session-day rule and its `isfirstbar` dual are unchanged | the audit's coof_session probes and the 15:30 case in `tests/test_aggregated_path_regressions.cpp`; section 7 of `tests/test_session_islastbar_aggregation.cpp` (TradingView's NYSE:F days with `calc_on_order_fills`, chart, aggregated and magnified: every callback of a bar reads the tape's flags) | +| AG3 | A `calc_on_order_fills` recalculation runs the script **on the bar the fill is on** and reads that bar's `session.ismarket` / `isfirstbar` / `islastbar`; `ab9714be` set them before any fill of the bar | The scheduler set the flags in the close callback only, so a recalculation published the PREVIOUS bar's: a fill at the next session day's 09:30 read `islastbar`. On the chart timeframe the close then read its lookahead from a count the recalculation had already advanced, one retained bar too far: a fill on 15:30 flagged 15:30 `islastbar` and, through E26's dual, 15:45 `isfirstbar` | **Adapter** (Pine language state, OT7). `scheduler_update_session_state` stays the one writer and runs once per script bar, at the bar's first publication: `PineScheduler::recalculate` asks before the script runs, `PineScheduler::bar` pine_scheduler_native.cpp:582 only when no recalculation published the bar. Since R5 lane F5 it selects the kernel's session-day facts of that bar, which every callback of the bar carries alike, so the scheduler's lookahead is gone. E26's session-day rule and its `isfirstbar` dual are unchanged | the audit's coof_session probes and the 15:30 case in `tests/test_aggregated_path_regressions.cpp`; section 7 of `tests/test_session_islastbar_aggregation.cpp` (TradingView's NYSE:F days with `calc_on_order_fills`, chart, aggregated and magnified: every callback of a bar reads the tape's flags) | **Recorded, not changed here.** @@ -1082,13 +1082,13 @@ row C). | **D** | the kernel's cohort rosters on a Pine host: O(openings) per row (V19-A's finding) | ×2.45, ×2.80, ×3.06 per doubling from 4,000 to 32,000 bars of a flip recorded per bar on `91d65ad6` (0.206 s to 4.33 s), against ×2.02 with recording off | **resolved** by R5 lane V19-FIX: the adapter takes an origin that can no longer be bound off its kernel roster, so the continuation folds the openings that still work or hold a lot | **A. The adapter's recording fold.** Until R5 lane V19-E (below), -`PineExecutionAdapter::hash_state` pine_state_hash.cpp:269 folded, at every +`PineExecutionAdapter::hash_state` pine_state_hash.cpp:270 folded, at every row, every placement snapshot the run had accepted (a sorted walk of every key, since replaced by the walk of the retained rows, `placement_` -pine_state_hash.cpp:329) — one per accepted request, cleared only by -`reset_for_run` pine_adapter.cpp:2331 — and reflected the whole admission +pine_state_hash.cpp:330) — one per accepted request, cleared only by +`reset_for_run` pine_adapter.cpp:2337 — and reflected the whole admission journal (since replaced by a running digest of its events, `admission_journal` -pine_state_hash.cpp:758). `PineScheduler::hash_state` pine_state_hash.cpp:796 +pine_state_hash.cpp:759). `PineScheduler::hash_state` pine_state_hash.cpp:797 re-folded the consumed source prefix too (`consumed` pine_state_hash.cpp:783-805). Measured with recording on and off, in two shapes: @@ -1133,10 +1133,10 @@ moving a single value, but not from inside lane F9's files: 1,339 rows each one callback old, an OCA re-price and a trail attach rows two callbacks old, and the row of an entry that filled at bar 1 is rewritten at bar 60, when its first exit is attached - (`has_full_entry_bracket` pine_adapter.cpp:11216). Full-table scans + (`has_full_entry_bracket` pine_adapter.cpp:11224). Full-table scans rewrite rows as well (`suspend_brackets_for_reversal` pine_adapter.cpp:1906, - `revive_brackets_after_margin` pine_adapter.cpp:2135, - `preserved_by_close_all` pine_adapter.cpp:14429), and + `revive_brackets_after_margin` pine_adapter.cpp:2141, + `preserved_by_close_all` pine_adapter.cpp:14437), and `admission::Journal::retain` market_admission.hpp:205 drops events from the middle of the journal. So "a row stops changing once its request is done" is false, and a cache resting on it would move a value. @@ -1221,16 +1221,16 @@ doubling from 4,000 to 32,000 bars (11.2 s at 32,000), and standing one (OL14), and a gapped stop a declined reversal had parked for the next margin-call slice came back after a cancel too. A script cancel now retires every leg of each exit row it names and drops such a parked stop -(`retire_cancelled_exits` pine_adapter.cpp:2061), the revival skips a withdrawn +(`retire_cancelled_exits` pine_adapter.cpp:2064), the revival skips a withdrawn exit, and K1 holds a withdrawn leg only while its origin is live or opened -- the one reader left is `origin_leg_consumed`, and it asks about such origins alone (`erase_retired_rows` -pine_adapter.cpp:3505). Both shapes now cost ×1.89 to ×2.12 per doubling up +pine_adapter.cpp:3513). Both shapes now cost ×1.89 to ×2.12 per doubling up to 64,000 bars and retain 3 rows. Row D was the kernel's half: the adapter enrolled every opening in its id's kernel roster (`cohort_add`) and never took one off, and `hash_cohorts` folds every member at every read. An origin whose request chain no longer works and that holds no open lot can never be bound again, so at every bar open that follows a command event the adapter takes -it off (`release_closed_cohort_origins` pine_adapter.cpp:3953); a lot opened +it off (`release_closed_cohort_origins` pine_adapter.cpp:3961); a lot opened by a replace successor keeps every older member of its roster while it is open, because its root is not a public fact. The flip then costs ×1.99, ×2.01, ×2.02 per doubling, and its rosters end with one member where they @@ -1266,8 +1266,8 @@ receipts of older rows (first at each cancel or re-price), and a consumed-prefix cache that stops at 16 bars (row 16 in all six scenarios). **B. The kernel recorder's closed rows.** -`broker_state_hash_from_execution_hash` engine_state_hash.cpp:30 folds six -fields of every closed row (`trades_` engine_state_hash.cpp:112-113) at every +`broker_state_hash_from_execution_hash` engine_state_hash.cpp:38 folds six +fields of every closed row (`trades_` engine_state_hash.cpp:120-121) at every recorded row, so the recorder costs O(closed rows) per row. The audit's bare host — one round trip per 20 bars, one request accepted and cancelled per bar — from 1,500 to 24,000 bars and 75 to 1,200 closed rows: recording 0.0087 / @@ -1288,7 +1288,7 @@ engine.hpp:1667) is not. The other is a contract that the six folded fields of a booked row are never rewritten. In the tree they are not; the only writer after booking, the Pine host, rewrites `close_cause`, `exit_from_bracket`, `exit_comment` and `entry_incarnation` only; its -writes are `close_cause` pine_strategy_host.cpp:612, `exit_comment` :742, `close_cause` :754-759, `exit_from_bracket` :1238 and `entry_incarnation` :1410-1418. +writes are `close_cause` pine_strategy_host.cpp:623, `exit_comment` :753, `close_cause` :765-770, `exit_from_bracket` :1267 and `entry_incarnation` :1439-1447. But `trades_` engine.hpp:546 is a protected member any host can write, so a lane that takes the consumer-side digest has to make that contract explicit. Folding `(count, digest)` in place of the rows saves the 256-entry table but @@ -1320,8 +1320,8 @@ the bars cost 3.8 times the CPU with a round trip every ten bars script point, not after the run's teardown, so that it is one value with recording on or off (`broker_state_hash_projection` pine_strategy_host.cpp:157-165). `capture_script_continuation_hash` -pine_strategy_host.cpp:503-511 takes that snapshot at the last batch bar -(`last_batch` pine_strategy_host.cpp:580-590), and taking it is one full +pine_strategy_host.cpp:510-518 takes that snapshot at the last batch bar +(`last_batch` pine_strategy_host.cpp:591-601), and taking it is one full continuation fold per run. Measured on the gated replay — `ReissueReplay` on 43,008 bars, `run()` process CPU, best of 5, three rounds — it is 0.2116 / 0.2137 / 0.2141 s with the capture and 0.1632 / 0.1614 / 0.1652 s with it @@ -1599,14 +1599,14 @@ the rule gives 9.9 + 9.89). | # | Risk | Why it bites | Containment | Src | |---|---|---|---|---| -| E1 | **Spec hashing.** `hash_spec` folded every field unconditionally (native_execution_consumer.cpp:234-256); every block added since folds only when set | Any new field, even defaulted, changes every continuation / broker-state hash, adapter runs included | Fold new blocks only when set, as a tagged extension (precedent `precommit_digest_` native_execution_consumer.hpp:1812); test that a default spec hashes to the pre-change constant | F | -| E2 | **Margin.** The TV margin call is built host-side from kernel primitives (a Stop-triggered `Reduce` bound to openings, or `execute_current`, which survives in `submit_margin_call_slice` pine_adapter.cpp:17715); its chronology lives at `schedule_margin_call_path` pine_adapter.cpp:19993 and its excursion sample differed by chronology (the Pine host's own excursion model, retired by R5 lane H-THIN: the kernel's sampler books it) | A kernel model under an adapter run would liquidate twice or reorder settlement; a checkpoint that reduces mid-path changes lot state exactly where the adapter's chronology lives | Superseded by R5: the adapter declares `spec.margin` — maintenance-only, so no opening is decided twice (`AdmitWithHostMargin`, MG4) — and the kernel owns the mechanism (check points, resting liquidation, receipt) while the adapter's hooks answer TradingView's scheduling, money and slice (ADR-0001 "Margin"). What stays host-side is named there: the chronology exceptions, the pre-open slice, the 1x-long money call and the broker-open FX rollover, whose kernel `FxRoll` point measures elsewhere (MG9). Corrected by R5 lane F7 | F O S | -| E3 | **Sizing.** TV sizes from the tick-rounded signal close, 10-digit-rounded equity and fee-reserved cash, frozen at signal, with fill-time exceptions in COOF paths (`default_sizing_shape` pine_adapter.cpp:3070, the COOF `at_fill` exception pine_adapter.cpp:8217) | One "generic" percent sizing serving both would break parity or import TV rounding into the kernel; the last ulp differs on lot-stepped instruments | Two paths on purpose: `Sized` (kernel, plain arithmetic, timing as a first-class field) vs `HostSized` + the adapter resolver. Never "simplify" the adapter onto the kernel basis. What the two paths share is the conversion: since R5 lanes N11 and F7 the division by price × point value × FX and the percentage fee reserve exist once, in the kernel (`native_sized_units`), for a default AND a typed (`qty_type` cash / percent) quantity. The adapter keeps what is measured TradingView: the money basis (signal-time equity, TradingView's margin equity at a typed fill, the hypothetical Flatten's balance for a typed reversal), the freeze, the fill-time exceptions and both lot floors (`tests/test_adapter_sizing_relower.cpp` cases 8, 9, 11: kernel 0.0392 against the percent floor's 0.0391 on a 0.0001 lot), and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (case 12) | F O S | -| E4 | **Calc on fill.** Kernel suffix eligibility is continuous along the segment; TV refills only at O/H/L/C waypoints -- on a magnified path at each intrabar's open, twice, and its two extremes, never its close (R5 lane MAG-INTRABAR) (`PineScheduler::recalculate` pine_scheduler_native.cpp:671, `coof_next_waypoint` pine_adapter.cpp:6808) | Changing `born_on_remaining_path` or the notification order shifts adapter fills | Freeze `born_on_remaining_path` native_execution_consumer.cpp:5948 and `drain_applied_notifications` (native_execution_consumer.cpp:6609-6622); new behaviour only behind new opt-ins | F | -| E5 | **Open-bar lookahead.** The adapter reads the full bar in `on_bar_open` (margin path scheduling uses H/L, pine_adapter.cpp:21956-22016) | The complete bar cannot be masked globally | Opt-in `OpenOnly` view (A.4); loud doc warning in L0 | F | -| E6 | **Double equity recording.** The source host already records per bar (history: only its cadence remains, `scheduler_mark_report_point` pine_strategy_host.cpp:2440) | Kernel recording would duplicate points and shift drawdown | Report policy / ownership (A.1); the adapter declares `KernelRecordedAtHostMarks` (R1): one recorder, at the adapter's own marks | F O | +| E1 | **Spec hashing.** `hash_spec` folded every field unconditionally (native_execution_consumer.cpp:234-256); every block added since folds only when set | Any new field, even defaulted, changes every continuation / broker-state hash, adapter runs included | Fold new blocks only when set, as a tagged extension (precedent `precommit_digest_` native_execution_consumer.hpp:1919); test that a default spec hashes to the pre-change constant | F | +| E2 | **Margin.** The TV margin call is built host-side from kernel primitives (a Stop-triggered `Reduce` bound to openings, or `execute_current`, which survives in `submit_margin_call_slice` pine_adapter.cpp:17723); its chronology lives at `schedule_margin_call_path` pine_adapter.cpp:20001 and its excursion sample differed by chronology (the Pine host's own excursion model, retired by R5 lane H-THIN: the kernel's sampler books it) | A kernel model under an adapter run would liquidate twice or reorder settlement; a checkpoint that reduces mid-path changes lot state exactly where the adapter's chronology lives | Superseded by R5: the adapter declares `spec.margin` — maintenance-only, so no opening is decided twice (`AdmitWithHostMargin`, MG4) — and the kernel owns the mechanism (check points, resting liquidation, receipt) while the adapter's hooks answer TradingView's scheduling, money and slice (ADR-0001 "Margin"). What stays host-side is named there: the chronology exceptions, the pre-open slice, the 1x-long money call and the broker-open FX rollover, whose kernel `FxRoll` point measures elsewhere (MG9). Corrected by R5 lane F7 | F O S | +| E3 | **Sizing.** TV sizes from the tick-rounded signal close, 10-digit-rounded equity and fee-reserved cash, frozen at signal, with fill-time exceptions in COOF paths (`default_sizing_shape` pine_adapter.cpp:3076, the COOF `at_fill` exception pine_adapter.cpp:8225) | One "generic" percent sizing serving both would break parity or import TV rounding into the kernel; the last ulp differs on lot-stepped instruments | Two paths on purpose: `Sized` (kernel, plain arithmetic, timing as a first-class field) vs `HostSized` + the adapter resolver. Never "simplify" the adapter onto the kernel basis. What the two paths share is the conversion: since R5 lanes N11 and F7 the division by price × point value × FX and the percentage fee reserve exist once, in the kernel (`native_sized_units`), for a default AND a typed (`qty_type` cash / percent) quantity. The adapter keeps what is measured TradingView: the money basis (signal-time equity, TradingView's margin equity at a typed fill, the hypothetical Flatten's balance for a typed reversal), the freeze, the fill-time exceptions and both lot floors (`tests/test_adapter_sizing_relower.cpp` cases 8, 9, 11: kernel 0.0392 against the percent floor's 0.0391 on a 0.0001 lot), and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (case 12) | F O S | +| E4 | **Calc on fill.** Kernel suffix eligibility is continuous along the segment; TV refills only at O/H/L/C waypoints -- on a magnified path at each intrabar's open, twice, and its two extremes, never its close (R5 lane MAG-INTRABAR) (`PineScheduler::recalculate` pine_scheduler_native.cpp:727, `coof_next_waypoint` pine_adapter.cpp:6816) | Changing `born_on_remaining_path` or the notification order shifts adapter fills | Freeze `born_on_remaining_path` native_execution_consumer.cpp:6368 and `drain_applied_notifications` (native_execution_consumer.cpp:7039-7052); new behaviour only behind new opt-ins | F | +| E5 | **Open-bar lookahead.** The adapter reads the full bar in `on_bar_open` (margin path scheduling uses H/L, pine_adapter.cpp:22029-22089) | The complete bar cannot be masked globally | Opt-in `OpenOnly` view (A.4); loud doc warning in L0 | F | +| E6 | **Double equity recording.** The source host already records per bar (history: only its cadence remains, `scheduler_mark_report_point` pine_strategy_host.cpp:2738) | Kernel recording would duplicate points and shift drawdown | Report policy / ownership (A.1); the adapter declares `KernelRecordedAtHostMarks` (R1): one recorder, at the adapter's own marks | F O | | E7 | **Price grid.** TV does not merely quantize: it tests triggers against a quantized bar while booking the fill on the *unrounded* level, so the adapter shifts the threshold (`source_trigger_threshold` pine_adapter.cpp:343) and books elsewhere | A kernel grid that quantizes both sides is correct and not TV; moving rounding into the kernel would change both native results and the adapter | `NativePriceGrid::None` for the adapter permanently; never "unify". Before L8: a byte-level experiment that the default price spelling on decimal ticks (one-ULP representations) does not move (S) | O S | -| E8 | **HTF.** The evaluator machinery carries TV publication semantics (lookahead, boundary deferral, early close, `syminfo.type` branching engine_security.cpp:67-73). The feed store is TV-calibrated too: "W" / "M" from native daily bars, native stamps as the period partition (`set_native_security_feed` `sha256:eca3dbea0baa461982ba89f13d0a5c43277854069fb6eb2dd2f5c5471313204d` engine_aux_security.cpp:51-83, *fresh*). Two aggregators exist: script buckets use `native_calendar`, `request.security` uses `TimeframeAggregator`, whose completion rules are TV-pinned (timeframe.hpp:288-330) | Reuse risks changing the *order* of the Pine scheduler's pumps; a bare host inherits TV partition rules; native HTF may differ from Pine HTF on early-close sessions | Registration-gated pump; no change to `scheduler_feed_security_input` pine_strategy_host.cpp:1766; full corpus diff; supplied-bar provenance preserved, never silently replaced by chart aggregation (S); differences on early-close sessions are recorded, not "fixed" (F). The Pine scheduler keeps one more `TimeframeAggregator` pass, before the run (`PineScheduler::run_begin`, audit M26): the script-bar count it yields (`terminal_source_bar`, the probe-tail suppression) and the per-input completion flags it hands `on_native_input` are needed before the first input arrives, while the kernel seals script intervals only as inputs arrive and offers no pre-run interval query (`NativeExecutionConsumer::script_interval_at` is consumer-private). Retained by R5 lane F7: replacing it needs that kernel capability | F O S | +| E8 | **HTF.** The evaluator machinery carries TV publication semantics (lookahead, boundary deferral, early close, `syminfo.type` branching engine_security.cpp:67-73). The feed store is TV-calibrated too: "W" / "M" from native daily bars, native stamps as the period partition (`set_native_security_feed` `sha256:eca3dbea0baa461982ba89f13d0a5c43277854069fb6eb2dd2f5c5471313204d` engine_aux_security.cpp:51-83, *fresh*). Two aggregators exist: script buckets use `native_calendar`, `request.security` uses `TimeframeAggregator`, whose completion rules are TV-pinned (timeframe.hpp:288-330) | Reuse risks changing the *order* of the Pine scheduler's pumps; a bare host inherits TV partition rules; native HTF may differ from Pine HTF on early-close sessions | Registration-gated pump; no change to `scheduler_feed_security_input` pine_strategy_host.cpp:1795; full corpus diff; supplied-bar provenance preserved, never silently replaced by chart aggregation (S); differences on early-close sessions are recorded, not "fixed" (F). The Pine scheduler keeps one more `TimeframeAggregator` pass, before the run (`PineScheduler::run_begin`, audit M26): the script-bar count it yields (`terminal_source_bar`, the probe-tail suppression) and the per-input completion flags it hands `on_native_input` are needed before the first input arrives, while the kernel seals script intervals only as inputs arrive and offers no pre-run interval query (`NativeExecutionConsumer::script_interval_at` is consumer-private). Retained by R5 lane F7: replacing it needs that kernel capability | F O S | | E9 | **TA numerics are TV-calibrated** (1e-10 band in DMI / MFI / percentrank; conditional-call extremum ring `ExtremeRing` ta.hpp:134) | Non-Pine users may expect IEEE semantics; changing them breaks parity | Rename only (§2.ii b). For an every-bar caller the ring equals a positional window (the ring comment `sha256:fd1f92206558863ad8df33fd3ef3b8cc8b9bd6678a25fc98ca866168c1a81aca` ta.hpp:87-88). Document. → U6 | F O S | | E10 | **Dead-but-hashed lot flags** (§2.ii j) | Deleting them changes `broker_state_hash` | Separate, explicitly non-neutral lane (L12) | F O | | E11 | **OCA and ownership.** Pine's default-sized OCA and `from_entry` reservation rules are source policies | Source IDs, callsite tokens or Pine labels leaking into `WorkingRequestCore` | Opaque keys and receipts only (R5-4) | S | @@ -1619,11 +1619,11 @@ the rule gives 9.9 + 9.89). | P1 | **Epoch bumps** move the frozen C++ ABI fixtures and the settlement ABI checker `CLAUDE.md` names; a neutral-refactor worker may not edit frozen headers | Bump lanes are not neutral-refactor lanes; the brief lists the header extension (§3.1 c) | F O S | | P2 | **Stale fence.** `CLAUDE.md` names "measured sources" `engine_strategy_commands.cpp`, `engine_fills.cpp`, `engine_risk.cpp`, `engine_market_admission.cpp`; none exists in `src/` at `73817c1` (only `engine_orders.cpp` and `engine_run.cpp` do) | Done: `CLAUDE.md` now names the post-R4-C measured sources (`src/native_execution_consumer.cpp`, `src/native_order.cpp`, `src/engine_execution.cpp`, `src/engine_orders.cpp`, `src/engine_run.cpp`, `src/engine_path_resolve.cpp`, `src/source/*`) | F | | P3 | **Corpus gate state.** S's audit found `scripts/run_corpus.sh` stopping at the corpus strategy compile: committed `generated.cpp` files no longer match the engine (8/8 drifted from the pinned codegen commit under `VERIFY=1 ONLY=validation/analyzer scripts/regen_corpus_cpp.sh`) | §3.1 a relies on that script. Re-pin the engine / codegen corpus tuple before L1. Not re-verified in this merge (no builds were run). Resolved since: the corpus was re-pinned (the submodule is at `b40aa8e`) and the sweep runs on `main` | S (single-source) | -| P4 | **Hash compatibility.** The source hash covers hundreds of adapter and scheduler fields (`PineExecutionAdapter::hash_state` pine_state_hash.cpp:269, `PineScheduler::hash_state` pine_state_hash.cpp:796); L4 / L9 add hashed state, L12 removes some | A hash mismatch is a diagnostic until the trade diff is known, never a reason to relax parity (S). Open: does the campaign gate compare broker-state hashes across releases, or only trades (O)? → Q3 | O S | +| P4 | **Hash compatibility.** The source hash covers hundreds of adapter and scheduler fields (`PineExecutionAdapter::hash_state` pine_state_hash.cpp:270, `PineScheduler::hash_state` pine_state_hash.cpp:797); L4 / L9 add hashed state, L12 removes some | A hash mismatch is a diagnostic until the trade diff is known, never a reason to relax parity (S). Open: does the campaign gate compare broker-state hashes across releases, or only trades (O)? → Q3 | O S | | P5 | **Test asymmetry:** 430 adapter test files vs 14 native when this plan was written (on this tree 543 `tests/*.cpp` name `source::PineStrategyHost`, 110 `NativeStrategyHost`) | native-only tests in every lane (§3.1 b) | O | | P6 | **Stream vs batch.** The stream refuses monthly input, mixed modes, in-session gaps, security feeds and source setters, and tick input under an FX curve (any non-empty curve until audit lane N6) (`native-engine.md`, "Stream limitations (current)") | L5 and L6 must state which refusals are generic and which are deliberate source limits | S | | P7 | **C ABI variant growth** | §2.iii hardening rules | S | -| P8 | **Unverified in the inputs:** whether `execution::LifecycleEffects` (execution.hpp:136-139) is ever non-empty from the Pine path, else `PlacementSnapshot::legs` (pine_adapter.hpp:364) is the only live lifecycle (O); the `strategy.risk.allow_entry_in` setter chain beyond `set_pine_risk_direction` pine_strategy_host.cpp:854 → `set_risk_direction` pine_adapter.cpp:26220 (O); what the ABI receipts pin (F); the probe corpus itself lives outside this tree (F) | verify inside the lane that touches each | F O | +| P8 | **Unverified in the inputs:** whether `execution::LifecycleEffects` (execution.hpp:136-139) is ever non-empty from the Pine path, else `PlacementSnapshot::legs` (pine_adapter.hpp:364) is the only live lifecycle (O); the `strategy.risk.allow_entry_in` setter chain beyond `set_pine_risk_direction` pine_strategy_host.cpp:868 → `set_risk_direction` pine_adapter.cpp:26238 (O); what the ABI receipts pin (F); the probe corpus itself lives outside this tree (F) | verify inside the lane that touches each | F O | ### 4.3 Open design questions (supervisor level) @@ -1632,7 +1632,7 @@ the rule gives 9.9 + 9.89). | Q1 | Does `engine_script_run_v18` open at L4 and close after L5 ∥ L6, or does L4's sizing hook ship with that batch (§3.4 row 4)? | — (raised by this merge) | | Q2 | Is a hash-domain bump (needed by §2.ii j) acceptable before the R5 final audit (phase table, native-refactor-progress.md:24, *fresh*), or must L12 wait? | F asks; none | | Q3 | Does the fixed-population gate (4190 probes then) compare broker-state hashes across releases, or only trades? If hashes, L4 and L12 need a hash-version plan | O asks; none | -| Q4 | `SizeTime::AtAcceptance` needs a current execution point (`current_execution_point` native_execution_consumer.cpp:3179); between realtime inputs there is none — reject or fall back to `AtMatch`? | F: reject with `InvalidQuantityBasis` | +| Q4 | `SizeTime::AtAcceptance` needs a current execution point (`current_execution_point` native_execution_consumer.cpp:3512); between realtime inputs there is none — reject or fall back to `AtMatch`? | F: reject with `InvalidQuantityBasis` | | Q5 | Liquidation check granularity: `PathAdverseExtreme` default; is a cheaper `CalculationOnly` worth shipping in v1? Pure spec value or injectable policy object (S)? | F: path-accurate default; merged A.3 = spec value + host sizing hook | | Q6 | Does a native series return the current value, a retained series object, or both? It decides C ABI ownership and hash size | S asks; A.5 offers push + current-bar pull | | Q7 | Is a close-reservation key stable across host restarts or only within one `RunIdentity`? | S: identity rules suggest run scope | @@ -1650,7 +1650,7 @@ All six are settled by what landed: U1 (native subscriptions take `lookahead` an | **U3** | **Do risk limits ship in v1?** | R5-9: in scope, not a blocker. S wanted them as a prerequisite; F accepts a toolkit stand-in; O defers. | | **U4** | **Stream-safe FX (FP6): in the roadmap or not?** It has no R5 lane. | F: out of scope here ("O7 clock"). S: its P10, an FX epoch consumed while flat, same curve in batch and stream. | | — | **U4, answered by audit lane N6.** In: the curve declared through `configure_native_fx_curve` is the FX epoch of the run, batch or stream; the account converts at the rate of each driver point's effective time, so the epoch is consumed by the walk itself, flat or not, and a confirmed-bar stream is the batch of the same bars. No `NativeFxEpoch` type and no `on_native_fx_epoch` callback were needed. Out, permanently: realtime rate ingestion (a new rate is a new run) and tick-driven input under a curve (its hooks and partial slots read a stale conversion clock — measured). | — | -| — | **N6 follow-ups (measured on the adapter; both closed by R5 lane F7).** (1) `apply_fx_open_margin_slice` (pine_adapter.cpp:22182-22258) throws on a rate step over a carried position whose margin is not 100 %: a 2× long carried through a 1.0 → 1.5 step ends the whole run with status 1 and an empty report, where the same book without a series, or with a series that never steps, completes. The kernel's `FxRoll` point is the generic mechanism that case can re-lower onto. (2) `default_sizing_intent` (pine_adapter.cpp:3137) returns nullopt whenever any FX series is staged — even one point that restates the scalar rate — so an FX run silently loses the kernel `Sized` path R2b landed (measured: 1 kernel `Sized` request without a series, 0 with a one-point 1.0 series, same 50-unit book). **Closed by R5 lane F7.** (1) The throw is gone: the carried roll takes TradingView's broker-open checkpoint at any positive margin — its money at the open under the new rate and the side's own maintenance ratio, its lot-floored 4x restore, executed at the open before the step bar's script — and a survivor that is not a 1x long is re-checked over the rest of the bar by the kernel's liquidation. A 2x long or short of 15 at 100 on 1000 across 1 → 1.5 takes 6.666666666666667 at the open and keeps 8.33; at a 25 % side ratio it takes nothing; a constant curve never rolls (`tests/test_affordability_fx.cpp` F7 rows; G6's leveraged pin and its L0 oracle twin moved with an `expectation corrected:` note). The kernel's `FxRoll` point stays refused, measured (MG9). (2) `Sized` is withdrawn only where the kernel would convert at another rate than the source sized at — a curve step between the sub-bar open and the acceptance coordinate — so a one-point series lowers exactly as no series does, and on a stepped curve only the step's own bar keeps its host-resolved intent (`tests/test_adapter_sizing_relower.cpp` case 12). | — | +| — | **N6 follow-ups (measured on the adapter; both closed by R5 lane F7).** (1) `apply_fx_open_margin_slice` (pine_adapter.cpp:22200-22276) throws on a rate step over a carried position whose margin is not 100 %: a 2× long carried through a 1.0 → 1.5 step ends the whole run with status 1 and an empty report, where the same book without a series, or with a series that never steps, completes. The kernel's `FxRoll` point is the generic mechanism that case can re-lower onto. (2) `default_sizing_intent` (pine_adapter.cpp:3143) returns nullopt whenever any FX series is staged — even one point that restates the scalar rate — so an FX run silently loses the kernel `Sized` path R2b landed (measured: 1 kernel `Sized` request without a series, 0 with a one-point 1.0 series, same 50-unit book). **Closed by R5 lane F7.** (1) The throw is gone: the carried roll takes TradingView's broker-open checkpoint at any positive margin — its money at the open under the new rate and the side's own maintenance ratio, its lot-floored 4x restore, executed at the open before the step bar's script — and a survivor that is not a 1x long is re-checked over the rest of the bar by the kernel's liquidation. A 2x long or short of 15 at 100 on 1000 across 1 → 1.5 takes 6.666666666666667 at the open and keeps 8.33; at a 25 % side ratio it takes nothing; a constant curve never rolls (`tests/test_affordability_fx.cpp` F7 rows; G6's leveraged pin and its L0 oracle twin moved with an `expectation corrected:` note). The kernel's `FxRoll` point stays refused, measured (MG9). (2) `Sized` is withdrawn only where the kernel would convert at another rate than the source sized at — a curve step between the sub-bar open and the acceptance coordinate — so a one-point series lowers exactly as no series does, and on a stepped curve only the step's own bar keeps its host-resolved intent (`tests/test_adapter_sizing_relower.cpp` case 12). | — | | **U5** | **Open-bar view default.** Should `OpenOnly` be the default for hosts that are not `LegacyTolerant`? | F: safer for users, but a behaviour change for the ~20 existing native tests; default `Complete` proposed. | | **U6** | **TA comparison band.** Is TradingView's 1e-10 tie-breaking a feature of the TA library, or a default to make configurable? | F, O: rename only; a switch changes indicator outputs and needs its own lane. S: separate policies, hash the policy identity. | @@ -1672,11 +1672,11 @@ virtual void hash_host_extension(BrokerStateHashSink&) const {} // F (S: hash_ NativeMetricsView native_metrics() const; // S: net_profit, equity, closed_trades, open_lots; non-virtual ``` -- `KernelRecorded`: the consumer calls the existing `update_equity_extremes()` (engine.hpp:1362) + `record_equity_point(script_open_ms)` (engine.hpp:1380) once per script calculation (after `invoke_callback` native_execution_consumer.cpp:7998, :8259, :8455, :10232) and after the `AfterCalculation` close; appends the per-bar broker hash when recording is on; synthesizes the open-position row at run end / `stream_end` with the existing `build_close_trade_with_costs` (engine_orders.cpp:70). -- `KernelRecordedAtHostMarks` (R1, KERNEL-EDGE): the same recording, at the points a C++ host names with `NativeStrategyHost::mark_native_report_point` instead of once per calculation; the consumer never records on its own initiative under it, so the continuation identity stays where `HostRecorded` leaves it. The Pine adapter's existing `mark_script_report_point` route has the same kernel producer. It is what the adapter declares (`KernelRecordedAtHostMarks` pine_adapter.cpp:2790), with `report_open_position_at_end == false`. +- `KernelRecorded`: the consumer calls the existing `update_equity_extremes()` (engine.hpp:1362) + `record_equity_point(script_open_ms)` (engine.hpp:1380) once per script calculation (after `invoke_callback` native_execution_consumer.cpp:8450, :8711, :8907, :10796) and after the `AfterCalculation` close; appends the per-bar broker hash when recording is on; synthesizes the open-position row at run end / `stream_end` with the existing `build_close_trade_with_costs` (engine_orders.cpp:70). +- `KernelRecordedAtHostMarks` (R1, KERNEL-EDGE): the same recording, at the points a C++ host names with `NativeStrategyHost::mark_native_report_point` instead of once per calculation; the consumer never records on its own initiative under it, so the continuation identity stays where `HostRecorded` leaves it. The Pine adapter's existing `mark_script_report_point` route has the same kernel producer. It is what the adapter declares (`KernelRecordedAtHostMarks` pine_adapter.cpp:2796), with `report_open_position_at_end == false`. - Publish read-only `closed_trade_*` accessors (RP6). - Alternative: F's virtual `owns_equity_recording()` (the A48 ownership pattern of `owns_lot_excursions` native_host.hpp:1023). Same semantics, but it costs a host-epoch bump, so the spec form was chosen; the virtual was never added, and `NativeReportPolicy` is what landed (§3.4 row 3). -- TUs: `native_execution_consumer.cpp` (seal path, `seal_script` native_execution_consumer.cpp:8492-8515), `engine_run.cpp`, `engine_report.cpp`, `engine_state_hash.cpp`, `native_run_spec.{hpp,cpp}`. +- TUs: `native_execution_consumer.cpp` (seal path, `seal_script` native_execution_consumer.cpp:8944-8967), `engine_run.cpp`, `engine_report.cpp`, `engine_state_hash.cpp`, `native_run_spec.{hpp,cpp}`. ### A.2 L3 — sizing, fractional reduce, sibling reservation (F P1, P7 + O G2 + S P2, P3) @@ -1699,7 +1699,7 @@ struct ScopeFraction { double fraction = 1.0; ScopeClaim claim = ScopeClaim::Gro using ReductionSize = std::variant; // F ``` -- Resolution: `units = cash / (price * point_value * fx)`, `cash` = the value or `fraction * marked_equity(price)`, `price` = the candidate's `default_resolved_price`. It plugs in where `HostSized` units resolve today (native_execution_consumer.cpp:709; terms resolution `resolve_execution_terms` native_execution_consumer.cpp:5263). A host override of `resolve_execution_terms` keeps the last word. Non-representable → `MatchRejectReason::TermsUnresolved` (native_order.hpp:877); validation reuses `RequestRejectReason::InvalidQuantityBasis` (native_order.hpp:827). +- Resolution: `units = cash / (price * point_value * fx)`, `cash` = the value or `fraction * marked_equity(price)`, `price` = the candidate's `default_resolved_price`. It plugs in where `HostSized` units resolve today (native_execution_consumer.cpp:710; terms resolution `resolve_execution_terms` native_execution_consumer.cpp:5675). A host override of `resolve_execution_terms` keeps the last word. Non-representable → `MatchRejectReason::TermsUnresolved` (native_order.hpp:877); validation reuses `RequestRejectReason::InvalidQuantityBasis` (native_order.hpp:827). - `NetOfSiblings`: a fraction-sized reduce claims the bound scope minus the units already claimed by live sibling reduces on it; keys are opaque, never source IDs (S). - Not adopted: S's spec-level `default_sizing` + virtual `resolve_native_sizing` (duplicates the per-request intent, costs an epoch); S's `PercentFreeMargin` (needs L4's free-margin definition) and `EveryCalculation` timing; O's spec-level `size_net_of_fees` (per-request flag, 2 of 3); O's `PercentOfScope` basis (expressed as `ScopeFraction`); S's `ReplaceByKey` / `Fifo` modes (callsite semantics → adapter, R5-4). - TUs: `native_order.{hpp,cpp}`, `native_execution_consumer.cpp`, `native_run_spec.cpp`, the definition hash. @@ -1726,9 +1726,9 @@ virtual std::optional resolve_margin_call_units(const NativeMarginCallVi virtual void on_native_margin_call(const native_order::MarginCallEvent&) {} // O, S ``` -- Mechanism (F): a **kernel-originated request**. At each script-bar open and after each applied fill the kernel computes the liquidation level and rests a `Reduce` / `Flatten` with `Stop{level}` bound to the live openings — the shape the adapter builds host-side today (pine_adapter.cpp:16163-16184). New hashed `RequestDefinition::origin { Host, KernelLiquidation, KernelRisk }`, new `CancelReason::Superseded` on re-price, `MarginCallEvent` joins `CommandEvent` (17 → 18, native_order.hpp:1308). O's alternative, a checkpoint on the excursion walk (`apply_excursion` native_execution_consumer.cpp:2925), is not chosen: a reduction mid-path is the entanglement O itself flags (E2). +- Mechanism (F): a **kernel-originated request**. At each script-bar open and after each applied fill the kernel computes the liquidation level and rests a `Reduce` / `Flatten` with `Stop{level}` bound to the live openings — the shape the adapter builds host-side today (pine_adapter.cpp:16171-16192). New hashed `RequestDefinition::origin { Host, KernelLiquidation, KernelRisk }`, new `CancelReason::Superseded` on re-price, `MarginCallEvent` joins `CommandEvent` (17 → 18, native_order.hpp:1308). O's alternative, a checkpoint on the excursion walk (`apply_excursion` native_execution_consumer.cpp:3258), is not chosen: a reduction mid-path is the entanglement O itself flags (E2). - FX revaluation (MG9) falls out once `native_fx_curve` drives the mark (`native_fx_curve.cpp`). -- TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:4989-5002 per side; scheduling next to `invoke_bar_open_callback` / `drain_after_applied`), `native_order.*`, `engine_state_hash.cpp` + the coverage checker, `c_abi.cpp` + `pineforge.h` (`pf_native_run_spec_v2` or an extension struct; `CLAUDE.md` SOP). +- TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:5401-5414 per side; scheduling next to `invoke_bar_open_callback` / `drain_after_applied`), `native_order.*`, `engine_state_hash.cpp` + the coverage checker, `c_abi.cpp` + `pineforge.h` (`pf_native_run_spec_v2` or an extension struct; `CLAUDE.md` SOP). ### A.4 L5 — calculation timing (F P4 + O G8, G9 + S P6) @@ -1748,10 +1748,10 @@ std::optional current_partial_bar() const; Callback chronology contract (the normative part of the lane; F P4.1 + O G8): -1. At one point: match and settle → `on_native_applied` per applied event, FIFO (rule already at `on_native_applied` native-engine.md:1729-1730) → with `BarCloseAndFills`, one recalculation at the fill cursor (reason `OrderFill`), driven from the existing notification drain (native_execution_consumer.cpp:7345-7357) under its re-entrancy guard, bounded by `max_recalculations_per_point`. S would run it *before* the drain; not chosen, the drain order is frozen (E4). -2. Requests born in any of these callbacks follow the existing birth rule: eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:5948). `born_on_remaining_path` (native_execution_consumer.cpp:5944-5948) is not touched — the adapter's COOF deferral (`pending_coof_requests_`, element type `PendingCoofRequest` pine_adapter.hpp:1906-1936) is calibrated against it. -3. `EveryModeledPoint`: a calculation at each magnifier sample / observed print, in batch as well as stream (every delivery loop asks the per-point guard native_execution_consumer.hpp:1199-1206, whose body is `recalculate_at_modeled_point` native_execution_consumer.cpp:7191-7198). `on_native_tick` stays the observation hook (S). -4. `current_partial_bar()` derives from the driver points already recorded (`record_driver` native_execution_consumer.cpp:2869); valid in bar-open, applied and tick callbacks. +1. At one point: match and settle → `on_native_applied` per applied event, FIFO (rule already at `on_native_applied` native-engine.md:1729-1730) → with `BarCloseAndFills`, one recalculation at the fill cursor (reason `OrderFill`), driven from the existing notification drain (native_execution_consumer.cpp:7793-7805) under its re-entrancy guard, bounded by `max_recalculations_per_point`. S would run it *before* the drain; not chosen, the drain order is frozen (E4). +2. Requests born in any of these callbacks follow the existing birth rule: eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:6368). `born_on_remaining_path` (native_execution_consumer.cpp:6364-6368) is not touched — the adapter's COOF deferral (`pending_coof_requests_`, element type `PendingCoofRequest` pine_adapter.hpp:1961-1991) is calibrated against it. +3. `EveryModeledPoint`: a calculation at each magnifier sample / observed print, in batch as well as stream (every delivery loop asks the per-point guard native_execution_consumer.hpp:1283-1290, whose body is `recalculate_at_modeled_point` native_execution_consumer.cpp:7639-7646). `on_native_tick` stays the observation hook (S). +4. `current_partial_bar()` derives from the driver points already recorded (`record_driver` native_execution_consumer.cpp:3202); valid in bar-open, applied and tick callbacks. 5. The kernel never attempts language-state rollback; that stays Pine's (`snapshot / restore / commit_coof_script_state`, pine_scheduler_native.cpp:54-106) (O). ### A.5 L6 — native HTF subscriptions (F §2.iv + O G10 + S P7; design in §2.iv) @@ -1789,7 +1789,7 @@ struct FromOwnerFill { double offset = 0.0; bool ticks = false; }; // O: signe using TriggerAnchor = std::variant; // Request::anchor = Absolute{} struct TrailTicks { double ticks; }; // O: offset spelling resolved against price_tick struct ReplaceOptions { bool retain_trigger_state = false; }; // O: keeps `TrailTrack::best` (native_order.hpp:651) -// Trail::offset == 0.0 is accepted: "exit on the first adverse move past the best" (now accepted: `valid_trail_offset` native_order.cpp:496) +// Trail::offset == 0.0 is accepted: "exit on the first adverse move past the best" (now accepted: `valid_trail_offset` native_order.cpp:511) // include/pineforge/native_toolkit.hpp — header-only, Pine-free, covered by the include-independence checker (F; shapes from S P1) struct BracketSpec { native_order::RequestHandle parent; std::optional take_profit, stop_loss, trail; @@ -1817,7 +1817,7 @@ NativeGridRounding grid_rounding = NativeGridRounding::HalfUp; ``` - `QuantizeFills`: the kernel rounds `resolved_price` onto `price_tick` **before** slippage (`HalfUp`) or in the direction that favours the resting order (`Directional`). `QuantizeFillsAndTriggers`: the matcher additionally compares the quantized path against the raw level. -- TUs: `native_run_spec.cpp` (validation), `native_execution_consumer.cpp` (resolve path native_execution_consumer.cpp:4099-4121, `execute_current` path native_execution_consumer.cpp:4096-4106), `native_matching.hpp` (threshold form). The dead helpers were the reference arithmetic and are deleted (§2.ii e); the kernel's own step is `native_matching::apply_slippage` native_matching.hpp:259. +- TUs: `native_run_spec.cpp` (validation), `native_execution_consumer.cpp` (resolve path native_execution_consumer.cpp:4099-4121, `execute_current` path native_execution_consumer.cpp:4429-4439), `native_matching.hpp` (threshold form). The dead helpers were the reference arithmetic and are deleted (§2.ii e); the kernel's own step is `native_matching::apply_slippage` native_matching.hpp:259. ### A.8 L9 — risk limits (F P3 + O G7 + S P5) @@ -1836,8 +1836,8 @@ std::optional risk; // NativeRunSpec - Breach: `MatchRejectReason::RiskLimit` for openings (O; F: `RiskHalted`), an optional kernel-issued flatten through A.3's kernel-originated request (`origin = KernelRisk`, F), and a `NativeRiskEvent` in `native_events` (native_host.hpp:1335) (O) or a callback `on_native_risk_event` (S — a virtual, so only inside an epoch batch). The day key uses the existing calendar (`session_day_ordinal` native_calendar.hpp:416), not a chart clock. - S's `directions` / `max_units` / `max_position_units` are existing spec fields (MG1-MG3), not duplicated. -- Interim (F): a C++ host can build all four today from `native_marked_equity` + `on_native_applied`, which is what the adapter does (pine_adapter.cpp:16998-17090); a toolkit helper is an acceptable stand-in until L9. -- TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:4989-5002 and the per-point walk), `native_order.hpp`, the state hash. +- Interim (F): a C++ host can build all four today from `native_marked_equity` + `on_native_applied`, which is what the adapter does (pine_adapter.cpp:17006-17098); a toolkit helper is an acceptable stand-in until L9. +- TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:5401-5414 and the per-point walk), `native_order.hpp`, the state hash. ### A.9 Single-source proposals not adopted diff --git a/docs/native-refactor-progress.md b/docs/native-refactor-progress.md index 799a78b03..2694e572a 100644 --- a/docs/native-refactor-progress.md +++ b/docs/native-refactor-progress.md @@ -173,18 +173,18 @@ them for the record only. "Linux aarch64" is the maintainers' Cortex-A725 host. | Item | What it is | Measured size | Deferred by | Prototype and evidence | Reopens when | | --- | --- | --- | --- | --- | --- | -| D2-B, the quiet-bar protocol and one receipt read per bar | PERF-D2's lane: a per-bar bar-open request, a bulk fold of quiet driver points, the adapter's quiet-open predicate with lazy open bookkeeping, and a gate before the receipt poll. D2-A landed the hook declarations and the O(1) `has_command_after` (`src/native_execution_consumer.hpp:295`). PERF-L4 gated the read inside `observe_terminal_receipts` (`src/source/pine_adapter.cpp:6307`), which still runs about three times a bar | PERF-D2, Linux aarch64 (Cortex-A725), callgrind and GBench: about ×0.88 on the 14 probes (an estimate from a measured −13 to −14 % floor bound for the bar-open request alone); after PB the receipt poll still costs about 250 instructions a bar on the no-order floor. On this tree the quiet-bar probe asks the receipt gate 40,996 times and each once-per-bar-open gate 13,248 times: 3.09 reads a bar, 82 % of them stopped at the gate (macOS arm64, AppleClang 17, Release) | "R5 PERF-D2 DONE 2026-09-24 19:56" (D2-B after V19-D); "R5 SUPERVISOR RULING 2026-09-25 00:15 Taipei": "D2-B (quiet-bar protocol) waits for PERF-D3's verdict"; TRIAGE1 §D | An unpublished prototype branch and its patches. PERF-D3's S1 (next row) prototyped the request and the lazy opening | A post-release performance epoch. Done when the receipt gate is asked as often as the once-per-bar gates | +| D2-B, the quiet-bar protocol and one receipt read per bar | PERF-D2's lane: a per-bar bar-open request, a bulk fold of quiet driver points, the adapter's quiet-open predicate with lazy open bookkeeping, and a gate before the receipt poll. D2-A landed the hook declarations and the O(1) `has_command_after` (`src/native_execution_consumer.hpp:321`). PERF-L4 gated the read inside `observe_terminal_receipts` (`src/source/pine_adapter.cpp:6315`), which still runs about three times a bar | PERF-D2, Linux aarch64 (Cortex-A725), callgrind and GBench: about ×0.88 on the 14 probes (an estimate from a measured −13 to −14 % floor bound for the bar-open request alone); after PB the receipt poll still costs about 250 instructions a bar on the no-order floor. On this tree the quiet-bar probe asks the receipt gate 40,996 times and each once-per-bar-open gate 13,248 times: 3.09 reads a bar, 82 % of them stopped at the gate (macOS arm64, AppleClang 17, Release) | "R5 PERF-D2 DONE 2026-09-24 19:56" (D2-B after V19-D); "R5 SUPERVISOR RULING 2026-09-25 00:15 Taipei": "D2-B (quiet-bar protocol) waits for PERF-D3's verdict"; TRIAGE1 §D | An unpublished prototype branch and its patches. PERF-D3's S1 (next row) prototyped the request and the lazy opening | A post-release performance epoch. Done when the receipt gate is asked as often as the once-per-bar gates | | S1a, kernel quiet-bar driving (PERF-D3) | A bar with nothing live, a flat book and a declined opening is driven in a quiet form: the kernel writes the bookkeeping its four price points would leave (ordinals, the last two driver marks, the floor, the point epochs) without walking them or calling the bar-open callback. Additive host cadence declarations; C hosts declare from their callback table; one C symbol under the PF_API checklist | PERF-D3, Linux aarch64 (Cortex-A725), GBench time and `perf stat` instructions, on the L3 tree `b33ea659`, S1a with S1b: no-order floor 0.764 / 0.660 in time (0.718 / 0.656 in instructions), 14 probes 0.958 / 0.921 (0.892 / 0.856), 100 slots 0.972 / 0.956 per slot. A bare kernel host that opts in: idle 0.545× (35 % fewer instructions a bar), market 0.81×, bracket 0.92×. Byte-identical: corpus 312/312, the 132-configuration battery with its hash columns | "FINDING PERF-D3 (filed for a post-release epoch per the owner's cap)" (ledger, 2026-09-24 16:58 UTC); "OWNER DECISION 2026-09-25 00:20 Taipei"; TRIAGE1 §D: "PERF-D3 S1/S2 (post-release epoch)" | An unpublished prototype branch (five commits on L3's tree) and its patches | The post-release epoch, with the owner calls PERF-D3 lists (the API under the existing ruling, the restated gate count) | | S1b, the Pine lazy opening | The Pine host records what its skipped opening and its uncalled input callback would have written, at the bar's first host callback. Member functions only, no layout change | Measured with S1a. A quiet bar then asks 15 gates, not 30, so the quiet-bar cost check `CHECK(asked >= 20 * o.boundary.size())` (`tests/test_adapter_quiet_bar.cpp:1054`) must be restated | As S1a | As S1a. The in-script hash witnesses caught the prototype's one miss (`awaiting_legacy_script_open_ms_`) | As S1a | | S1c, the held-position wake band | S1 for bars that hold a position with nothing live, 41 % of the median public slot's bars. The host declares a price band, and the kernel calls the opening only when the bar's range leaves it. The adapter derives the band from its margin model and declines when it cannot bound it | PERF-D3, a measurement-only upper bound without the band, which is not sound in general: 100 slots 0.776 / 0.716 per slot, 14 probes 0.830 / 0.761, cumulative against L3 with S1, S2 and D2-A | As S1a; the owner call "Fund the price band" | An unpublished measurement-only script | As S1a. Size L, parity risk high: TradingView's margin money rules | | S2 and S2e, the codegen feature manifest | Codegen states which per-bar hooks and publications a strategy can reach, and the engine skips the others; no manifest means every feature. A median 39 of 52 are statically unreachable, over the 312 corpus strategies and the 100 slots alike. S2e adds two gates that stop the Pine risk ledgers of a strategy with no risk rule | PERF-D3, on top of S1: floor −14 % / −15 % in time (−6.4 % in instructions), 14 probes 0.943 / 0.934, 100 slots 0.942 / 0.945. The nine S2 gates are byte-identical. S2e adds about 2–3 % and moves the adapter hash domain `pineforge-source-adapter/v4` to its next version | As S1a; the supervisor's cap: "any structural S1/S2 redesign is a post-release epoch" | In PERF-D3's unpublished prototype (a regex-based manifest scanner) | A codegen and engine co-change with a corpus re-emission; S2e also needs an owner decision on the hash-domain step | | The kernel quiet-run loop and the Pine quiet calculation | A loop for consecutive quiet bars of a pumped batch at the chart's own timeframe (label and gap checks in the batch preflight, precomputed intervals and session facts, one calculation frame); and skipping a flat bar's post-script hooks when its script issued nothing | Estimated, not prototyped (PERF-D3's report, section 3.7): about 1,000 instructions a bar for the loop and 800 for the quiet calculation, taking the floor from 3,726 instructions (S1, S2 and D2-A) to about 1,900, or 180–200 ns a bar | As S1a (PERF-D3's follow-up row) | The PERF-D3 report only | After S1a and S1b | | PGO | Profile-guided optimisation of the library objects: an instrumented build, a training run over the bench strategies, a `-fprofile-use` rebuild and the identity gates, per compiler and target, with a staleness gate on the profile | LAYOUT1, Linux aarch64 (Cortex-A725), GCC 13.3, `perf stat`, trained on 86 slots and timed on the 14 held-out probes: 14.5 % fewer cycles, CPU time 0.865, 7 % fewer instructions, 26 % fewer L1i refills, 36 % fewer front-end stalls, layout spread 9.0 % → 5.1 %, fingerprints identical | "R5 LAYOUT1 DONE + ACCEPTED 2026-09-24 21:45 Taipei": "PGO APPROVED as a lane (PGO1) scheduled LAST in the perf wave after the code lanes (profile staleness)"; the supervisor's cap: "PGO folds into the release lane"; the owner: "release with PGO in the release build"; TRIAGE1 §D | Unpublished. The tree carries only LAYOUT1's `-falign-functions=64` | The release lane. BENCH4 timed the standard Release build without PGO, and neither BENCH4's nor REL10-ENGINE's brief names it | -| Compact journal records and a history ring buffer (L3) | Smaller command records and a ring buffer for the request core's history. `WorkingRequestCore::history` (`include/pineforge/native_order.hpp:1709`) hands out the whole vector and the core holds `history_` (`include/pineforge/native_order.hpp:2197`) inside `native_order_v7`, so both are a layout and API change | L3, Linux aarch64 (Cortex-A725), `perf` call graphs, inclusive shares: at most 3.68 % of bracket, 11.99 % of re-issue, 4.26 % of market and 3.42 % of the 14 probes (retiring the history alone: 1.64, 4.51, 2.53 and 1.00 %); only part is realisable, since records must still be written | "R5 L3 DONE + ACCEPTED 2026-09-24 22:10 Taipei", ruling 1: "DEFERRED to the PERF-D3 structural plan"; TRIAGE1 §D: "compact journal records" | Lane L3's code, merged by INT22 (#285); its profiles are unpublished | A `native_order` layout epoch | -| A checked-values handoff for executions (L3) | `execution_values` runs twice, in `check_execution` (`src/native_order.cpp:4791`) and again in `apply_execution` (`src/native_order.cpp:4805`); handing the checked values across needs a public type, an API addition with no layout change | L3, as above: 2.21 % of bracket for the two together, about 1.1 % recoverable | L3 ruling 2: "a public checked-values handoff for executions (~1.1% bracket) -> DEFERRED" | As above | An owner decision on the API addition | +| Compact journal records and a history ring buffer (L3) | Smaller command records and a ring buffer for the request core's history. `WorkingRequestCore::history` (`include/pineforge/native_order.hpp:1709`) hands out the whole vector and the core holds `history_` (`include/pineforge/native_order.hpp:2280`) inside `native_order_v7`, so both are a layout and API change | L3, Linux aarch64 (Cortex-A725), `perf` call graphs, inclusive shares: at most 3.68 % of bracket, 11.99 % of re-issue, 4.26 % of market and 3.42 % of the 14 probes (retiring the history alone: 1.64, 4.51, 2.53 and 1.00 %); only part is realisable, since records must still be written | "R5 L3 DONE + ACCEPTED 2026-09-24 22:10 Taipei", ruling 1: "DEFERRED to the PERF-D3 structural plan"; TRIAGE1 §D: "compact journal records" | Lane L3's code, merged by INT22 (#285); its profiles are unpublished | A `native_order` layout epoch | +| A checked-values handoff for executions (L3) | `execution_values` runs twice, in `check_execution` (`src/native_order.cpp:4887`) and again in `apply_execution` (`src/native_order.cpp:4910`); handing the checked values across needs a public type, an API addition with no layout change | L3, as above: 2.21 % of bracket for the two together, about 1.1 % recoverable | L3 ruling 2: "a public checked-values handoff for executions (~1.1% bracket) -> DEFERRED" | As above | An owner decision on the API addition | | A chunked `trades_` store (L3) | Closed trades in chunks rather than one vector; `trades_` (`include/pineforge/engine.hpp:546`) is protected `BacktestEngine` state, so script ABI | L3, as above: on bracket, vector growth 0.64 %, `Trade` moves 0.74 % and `record_close_trade` 0.43 % (at most about 1.4 %); at most 0.35 % on the 14 probes | L3 ruling 4: "chunked trades_ store (ABI) -> DEFERRED" | As above | An `engine_script_run` epoch that changes the engine's protected layout | -| A zoned chart-day memo (D2-D finding 4) | On a chart with a timezone, `libc_chart_day_key` (`src/source/pine_adapter.cpp:17130`) takes the process timezone lock through `ScopedTimezone` and calls `localtime_r` on every bar, because `on_bar_open` (`src/source/pine_adapter.cpp:22011`) reads `chart_day_key` (`src/source/pine_adapter.cpp:22248`) unconditionally. The non-UTC monthly Sharpe/Sortino walk, `month_key_local` (`src/engine_metrics.cpp:33`), pays one `localtime_r` per equity point. UTC charts use D2-D's memo, `civil_chart_day_key` (`src/source/pine_adapter.cpp:17114`) | Not measured on a zoned chart. The UTC path cost 134 instructions a bar before D2-D's memo and 68 after (D2-D, Linux aarch64, callgrind on the floor). PERF-K1: `ScopedTimezone` copies the zone name, which allocates for names of 16 or more characters under libstdc++ | TRIAGE1 §D: "D2-D finding 4 (the timezone `localtime_r` memo)", under the owner's cap. D2-D's report: an exact zoned memo "would need the zone's transition table, which belongs in `timezone.cpp`" | None; the UTC memo's witness is `tests/test_chart_day_memo.cpp` | A lane that owns `src/timezone.cpp` and builds an exact zone-transition memo (PERF-K1's Sitka case shows why a shortcut is not exact), byte-identical, with the zoned keys of `test_chart_day_memo` still equal to the computed ones | -| Thread-local reads that remain per call | D2-C moved the three scopes' writes onto the pump's block, `pump_ambient` (`src/native_execution_consumer.hpp:470`). A reader still reaches the block through `tl_runtime_ambient` (`src/ta_extremes_volume.cpp:29`): `bar_context` (`src/ta_extremes_volume.cpp:82`), read by every `ExtremeRing::update` (`src/ta_extremes_volume.cpp:117`), which backs `ta.highest`, `ta.lowest`, `ta.highestbars` and `ta.lowestbars` and, through an embedded `Highest` and `Lowest`, `ta.stoch`, `ta.wpr` and `ta.range`; and `matching_day_partition` (`src/timeframe.cpp:628`), read by the default-anchored `ta.vwap` through `session_day_index`, by `crosses_boundary` on a DAY period (`timeframe.change("D")`) and by the session-period helpers. `decompose_ms_local` (`src/session_time.cpp:250`) reads two thread-locals per call on a zoned clock, and the prepared order storage's `free_blocks` (`src/native_order.cpp:605`) one per prepared mutation. `ema_na_warmup_flag` is read once per EMA, at its first compute | One thread-local access per reader call. AUDIT4, macOS arm64, a dlopen'd tutorial-MACD probe, counted by `_tlv_get_addr` breakpoints and by descriptor patching: 1.000 a bar for one `ta::Highest` and for one default `ta::VWAP`, 1.999 for one `pine_hour` on a New York clock, 0.000 on the no-order floor and with orders. On Linux a TLS-descriptor call costs about 20 instructions: D2-C's 13 → 4 calls a bar was 260 → 80 instructions (Linux aarch64, GCC 13, callgrind). INT23: 2.00 a bar on the zoned probe 064, 0.08 on probe 008 | None for the TA and day-partition readers: "RULING: INT23 ACCEPTED" (ledger, 2026-09-24 23:30 UTC) reads "dlopen TLS 13->0/bar", and INT23's report names only the timezone and order-core residuals. This row is their record | Unpublished (AUDIT4's scratch) | A byte-identical lane that hands these readers the pump's block without a thread-local read. `ta::bar_context()` is script ABI, so its public declaration stays as it is | +| A zoned chart-day memo (D2-D finding 4) | On a chart with a timezone, `libc_chart_day_key` (`src/source/pine_adapter.cpp:17138`) takes the process timezone lock through `ScopedTimezone` and calls `localtime_r` on every bar, because `on_bar_open` (`src/source/pine_adapter.cpp:22029`) reads `chart_day_key` (`src/source/pine_adapter.cpp:22266`) unconditionally. The non-UTC monthly Sharpe/Sortino walk, `month_key_local` (`src/engine_metrics.cpp:33`), pays one `localtime_r` per equity point. UTC charts use D2-D's memo, `civil_chart_day_key` (`src/source/pine_adapter.cpp:17122`) | Not measured on a zoned chart. The UTC path cost 134 instructions a bar before D2-D's memo and 68 after (D2-D, Linux aarch64, callgrind on the floor). PERF-K1: `ScopedTimezone` copies the zone name, which allocates for names of 16 or more characters under libstdc++ | TRIAGE1 §D: "D2-D finding 4 (the timezone `localtime_r` memo)", under the owner's cap. D2-D's report: an exact zoned memo "would need the zone's transition table, which belongs in `timezone.cpp`" | None; the UTC memo's witness is `tests/test_chart_day_memo.cpp` | A lane that owns `src/timezone.cpp` and builds an exact zone-transition memo (PERF-K1's Sitka case shows why a shortcut is not exact), byte-identical, with the zoned keys of `test_chart_day_memo` still equal to the computed ones | +| Thread-local reads that remain per call | D2-C moved the three scopes' writes onto the pump's block, `pump_ambient` (`src/native_execution_consumer.hpp:496`). A reader still reaches the block through `tl_runtime_ambient` (`src/ta_extremes_volume.cpp:29`): `bar_context` (`src/ta_extremes_volume.cpp:82`), read by every `ExtremeRing::update` (`src/ta_extremes_volume.cpp:117`), which backs `ta.highest`, `ta.lowest`, `ta.highestbars` and `ta.lowestbars` and, through an embedded `Highest` and `Lowest`, `ta.stoch`, `ta.wpr` and `ta.range`; and `matching_day_partition` (`src/timeframe.cpp:628`), read by the default-anchored `ta.vwap` through `session_day_index`, by `crosses_boundary` on a DAY period (`timeframe.change("D")`) and by the session-period helpers. `decompose_ms_local` (`src/session_time.cpp:250`) reads two thread-locals per call on a zoned clock, and the prepared order storage's `free_blocks` (`src/native_order.cpp:620`) one per prepared mutation. `ema_na_warmup_flag` is read once per EMA, at its first compute | One thread-local access per reader call. AUDIT4, macOS arm64, a dlopen'd tutorial-MACD probe, counted by `_tlv_get_addr` breakpoints and by descriptor patching: 1.000 a bar for one `ta::Highest` and for one default `ta::VWAP`, 1.999 for one `pine_hour` on a New York clock, 0.000 on the no-order floor and with orders. On Linux a TLS-descriptor call costs about 20 instructions: D2-C's 13 → 4 calls a bar was 260 → 80 instructions (Linux aarch64, GCC 13, callgrind). INT23: 2.00 a bar on the zoned probe 064, 0.08 on probe 008 | None for the TA and day-partition readers: "RULING: INT23 ACCEPTED" (ledger, 2026-09-24 23:30 UTC) reads "dlopen TLS 13->0/bar", and INT23's report names only the timezone and order-core residuals. This row is their record | Unpublished (AUDIT4's scratch) | A byte-identical lane that hands these readers the pump's block without a thread-local read. `ta::bar_context()` is script ABI, so its public declaration stays as it is | | "audit-off mode", "LTO", "data-oriented layout" | Named in "OWNER DECISION 2026-09-25 00:20 Taipei" among the post-release structural items: "S1 bulk quiet bars, S2 per-strategy specialization, audit-off mode, LTO+PGO, data-oriented layout". No engine or campaign document defines, designs or measures the three | None | The owner decision | None | Struck from this inventory until a design note defines one of them | ### Recorded witness drift @@ -194,12 +194,12 @@ them for the record only. "Linux aarch64" is the maintainers' Cortex-A725 host. K4, the batch log pre-size. Since V19-B's step 6 (`08e71bae`) a run under the default `NativeEventRetention::Window` keeps no driver point and retires its journal at every script-bar boundary, so K4 has nothing to size. -`reserve_driver_log` (`src/native_execution_consumer.cpp:2884-2889`) returns +`reserve_driver_log` (`src/native_execution_consumer.cpp:3217-3222`) returns unless the retention is Full, and `presize_logs` -(`src/native_execution_consumer.cpp:9669-9680`) sizes no journal under Window. +(`src/native_execution_consumer.cpp:10192-10203`) sizes no journal under Window. `NativeRunSpec::event_retention` (`include/pineforge/native_run_spec.hpp:714`) defaults to Window, and the Pine adapter declares -`NativeEventRetention::Window` (`src/source/pine_adapter.cpp:2931`). K4 still +`NativeEventRetention::Window` (`src/source/pine_adapter.cpp:2937`). K4 still sizes Full runs (driver log and journal) and Commands runs (journal), and its witness, `tests/test_native_batch_log_presize.cpp`, runs under Full. PERF-K24's replay figure (14.9 % less CPU on Linux aarch64) describes the tree before V19-B. In @@ -210,7 +210,7 @@ or more, and the same run under Full holds 66,000 points in one block of **PERF-L4 reads the receipts about three times a bar, not once.** PERF-L4's brief asked it to "read the receipts ONCE per bar instead of three times". The lane gated the read instead: `observe_terminal_receipts` -(`src/source/pine_adapter.cpp:6307`) returns at its receipt gate when no +(`src/source/pine_adapter.cpp:6315`) returns at its receipt gate when no command lies above its cursor. `every_gate_is_taken_both_ways` (`tests/test_adapter_quiet_bar.cpp:1061`) prints `gate ReceiptRead asked 40996 quiet 33552 live 7444`, against `asked 13248` @@ -241,7 +241,7 @@ Mac's 489 and 412 and 1003 of 1249, and no Mac output moved. - **PERF-L4's retired reader slots.** `event_high_water_reader_` and `terminal_receipt_high_water_reader_` - (`include/pineforge/source/pine_adapter.hpp:2355-2356`), with their type + (`include/pineforge/source/pine_adapter.hpp:2381-2382`), with their type `ReceiptHighWaterReader`, are storage nothing reads or writes since PERF-L4: they keep `PineExecutionAdapter`'s layout, which is part of `PineStrategyHost`'s and so of the generated script's ABI. They go at the next diff --git a/docs/pages/abi-stability.md b/docs/pages/abi-stability.md index 51d342722..ff434fa88 100644 --- a/docs/pages/abi-stability.md +++ b/docs/pages/abi-stability.md @@ -17,7 +17,14 @@ From 1.0.0, within one `PINEFORGE_VERSION_MAJOR` (the - **`extern "C"` functions are append-only.** New functions may be added; existing ones are never removed or renamed. - **Enum values** are **stable**. New enumerators may be added; existing - values never change. + values never change. One exception, unreleased and targeted for 1.5.0: + `RunFailureCode` in the installed C++ header + `include/pineforge/run_failure_codes.hpp` is renumbered, and 36 of the 38 + codes released in 1.4.0 get new numeric values (`none` and the first two + codes keep theirs). A run-failure code crosses the C boundary by its name, + never by its number, so a caller that reads code names is unaffected; C++ + code that stores or compares the numbers must be rebuilt against the new + header. Before 1.0 a MINOR release could grow a caller-visible struct: v0.13.0 appended `pf_trade_t::open_at_end`, and the ABI v4 fields of `pf_report_t` @@ -62,9 +69,9 @@ Three layers: The counts below include recorded outputs and describe the unreleased API targeted for 1.5.0; recorded outputs are not part of 1.4.0. -A built strategy `.so` exposes 89 compiled-strategy `PF_API` declarations -(73 runtime implementations plus sixteen generated exports) plus 43 native-host -declarations: 132 `PF_API` exports in total; a script that declares +A built strategy `.so` exposes 96 compiled-strategy `PF_API` declarations +(80 runtime implementations plus sixteen generated exports) plus 43 native-host +declarations: 139 `PF_API` exports in total; a script that declares `use_bar_magnifier = true` also exports `strategy_declares_bar_magnifier`, and a module that records outputs its three outputs exports. `nm -gU` also reports libc++'s `std::piecewise_construct`; no project-internal C++ symbol is exported. The grouped table below is a guide, @@ -117,7 +124,7 @@ not the complete inventory: | `strategy_signal_safety_receipt` | @ref pf_outputs (a recording module's own) | Sixteen per-strategy exports include the five create/run/free lifecycle -functions. The remaining 73 runtime implementations, including the +functions. The remaining 80 runtime implementations, including the closed-trade incarnation accessor, are force-linked into each strategy library, so consumers resolve the same complete ABI from the strategy `.so`. All additions remain covered by the minor-version append-only guarantee. diff --git a/docs/pages/contributing-llm.md b/docs/pages/contributing-llm.md index a1f9dbad3..84ae3e89e 100644 --- a/docs/pages/contributing-llm.md +++ b/docs/pages/contributing-llm.md @@ -261,7 +261,7 @@ decision, and removing one is a regression: **kernel** — the Pine-agnostic half of the engine: matching, fills, sizing, margin, settlement, indicators, calendars. Buildable alone as -`PineForge::kernel` CMakeLists.txt:156 with `PINEFORGE_BUILD_SOURCE_LAYER` +`PineForge::kernel` CMakeLists.txt:158 with `PINEFORGE_BUILD_SOURCE_LAYER` CMakeLists.txt:43 off. **adapter** — `src/source/` and `src/compat/pine/`: the TradingView-parity diff --git a/docs/pages/index.md b/docs/pages/index.md index 6e2867553..b6d95ed88 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -137,9 +137,9 @@ directly, so a Pine-layer symbol reaching one is a link error. The counts below include recorded outputs and describe the unreleased API targeted for 1.5.0; recorded outputs are not part of 1.4.0. -The public C surface is **136 `PF_API` declarations** across two headers: +The public C surface is **145 `PF_API` declarations** across the five installed headers that declare any: -- `` — **93**: 73 runtime implementations plus twenty +- `` — **94**: 74 of the 80 runtime implementations (the other six are declared in `execution_observer.h` and `selected_window.h`) plus twenty per-strategy generated exports (`strategy_declares_bar_magnifier` only in a script that declares `use_bar_magnifier = true`, the three recorded-outputs exports only in a module that records outputs). This is what a compiled strategy `.so` exports and what a harness calls. Older modules lack the six @@ -149,8 +149,15 @@ The public C surface is **136 `PF_API` declarations** across two headers: direction, where the host drives the kernel itself. Submit, replace, cancel, execute, read the book, read the lots. Additive: no symbol, struct or behaviour of the first set changes. - -`scripts/check_c_abi_runtime.py` pins both symbol inventories (81 and 43); +- `` and `` — + three each: the six of the 80 runtime implementations that `pineforge.h` + does not declare (each header's version reader, the execution observer's + setter and observation reader, the selected window's setter and counts + reader). +- `` — two: the selected-window planner's + version reader and `pf_plan_selected_primary_v1`. + +`scripts/check_c_abi_runtime.py` pins both symbol inventories (80 and 43); `scripts/check_native_c_api_surface.py` separately pins the host COVERAGE block. | Group | Symbols | Reference | diff --git a/docs/pages/native-engine.md b/docs/pages/native-engine.md index 8940a9580..dc7865327 100644 --- a/docs/pages/native-engine.md +++ b/docs/pages/native-engine.md @@ -470,7 +470,7 @@ spec fields. The rich `run(bars, n, input_tf, script_tf, inputs, syminfo, overrides, …)` overload (`engine.hpp:1800-1812`) is **not** refused as a source mutation: it reaches `NativeExecutionConsumer::run_rich` -(`native_execution_consumer.cpp:9810-9852`), which admits the begin, checks the +(`native_execution_consumer.cpp:10354-10396`), which admits the begin, checks the timeframe arguments against the spec, preflights and pumps the batch exactly like the plain overload. `inputs` / `syminfo` / `overrides` are carried only as `NativeBeginArgs` fields to `prepare_native_begin` — the overrides as the @@ -638,11 +638,11 @@ a host reacts to its own execution and may submit again. A request born there, mid-bar on a continuous segment, is eligible on the **remaining path suffix** of that segment — the birth is admitted at the current cursor and the geometric search then sees only the unconsumed suffix (`born_on_remaining_path`, -`native_execution_consumer.cpp:5944-5948`). Requests accepted before the +`native_execution_consumer.cpp:6364-6368`). Requests accepted before the segment, and discrete points, keep the ordinary birth gate above. `on_native_bar_open` fires at the modeled opening, before that point's matching -pass (`native_execution_consumer.cpp:7434-7436`). **Lookahead warning:** the +pass (`native_execution_consumer.cpp:7882-7884`). **Lookahead warning:** the `Bar` it receives is the *complete* script bar — the consumer has already set `engine.current_bar_ = open_view` (`native_execution_consumer.cpp:7428`), the complete bar unless the spec asks for `NativeOpenBarView::OpenOnly` — so its @@ -2112,7 +2112,7 @@ default, set while no run is active — because each row is a full the live state, not the run's length: the closed rows enter through a running digest). With the switch on, one row follows each point, after the extremes that point just folded -(`record_script_report_point`, `native_execution_consumer.cpp:8316`), so +(`record_script_report_point`, `native_execution_consumer.cpp:8768`), so ```text broker_state_hash_len == equity_curve_len == script_bars_processed @@ -2239,13 +2239,13 @@ under the ticket the model or the run named. A host running its own forced close states the cause on the row it produced. `closed_trade_close_cause(i)` (`engine.hpp:1849`) is the C++ read and -`strategy_closed_trade_close_cause` (`pineforge.h:1249`) the C one, with the +`strategy_closed_trade_close_cause` (`pineforge.h:1309`) the C one, with the same numbering: `-1` for a bad index or a NULL handle, `0` UNKNOWN, `1` SCRIPT, `2` BRACKET, `3` MARGIN_CALL, `4` INTRADAY_LOSS_CAP, `5` INTRADAY_FILL_CAP, `6` RANGE_END. A row closed at the end of the run (`open_at_end`) always answers `6`, ahead of every other cause. The ticket a row was booked under is `strategy_closed_trade_entry_id` / -`_exit_id` / `_exit_comment` (`pineforge.h:1167-1182`), which index exactly the +`_exit_id` / `_exit_comment` (`pineforge.h:1242-1257`), which index exactly the rows of `fill_report`'s trade array and take any handle this engine produces — including a `pf_strategy_t` from `strategy_native_host_create_v1`, which is how a C host reads back the ticket its own margin model declared. @@ -3086,7 +3086,7 @@ Only completed buckets are published, so this recipe has no lookahead by construction. It is the same class the kernel's own subscription evaluator aggregates with, and the one the kernel's `script_bucket_completions` query feeds when the Pine scheduler asks how its input span buckets -(`TimeframeAggregator` `native_execution_consumer.cpp:8398`). What it does +(`TimeframeAggregator` `native_execution_consumer.cpp:8850`). What it does **not** give you is what a declared subscription does: an `authoritative_bars` feed, the `gaps` and `lookahead` delivery rules, the lazy-seal chronology, a C spelling, and the @@ -3150,7 +3150,7 @@ These are existing refusals, not implied future features: - In-session gaps on a realtime stream, and in a `Canonical` stream warmup - Source `calc_on_every_tick` / `calc_on_order_fills` enabled (the runner rejects an explicit true override, and the Pine host refuses a stream begin - with `calc_on_order_fills`, `pine_strategy_host.cpp:276-279`). This is a + with `calc_on_order_fills`, `pine_strategy_host.cpp:278-281`). This is a **source-route** refusal, not a limit on the native hooks: `on_native_tick` and `on_native_applied` are delivered on a stream, and a native host's own `NativeRunSpec::calculation` is accepted there, where `EveryModeledPoint` @@ -4550,7 +4550,7 @@ Nor is there a build-level one. The two source sets are disjoint: `PINEFORGE_SOURCE_LAYER_SOURCES` (`CMakeLists.txt:91`) holds all six `src/compat/pine/` units and all twelve `src/source/` ones, and `PINEFORGE_KERNEL_SOURCES` (`CMakeLists.txt:114`) holds the kernel's own -thirty-five, which are what `add_library` (`CMakeLists.txt:155`) compiles into +thirty-five, which are what `add_library` (`CMakeLists.txt:158`) compiles into `pineforge_kernel`. `libpineforge.a` still carries both sets when `PINEFORGE_BUILD_SOURCE_LAYER` is ON, which is the default; the kernel archive exists either way. The installed-header closure is clean too, which the diff --git a/docs/pages/pine-to-native.md b/docs/pages/pine-to-native.md index 6d376e26d..99cc844a2 100644 --- a/docs/pages/pine-to-native.md +++ b/docs/pages/pine-to-native.md @@ -126,9 +126,9 @@ rather than defaulted. | `currency` | `currency` native_run_spec.hpp:639 | `currency` pineforge.h:499 | `hello_kernel.cpp` | With `ticker` native_run_spec.hpp:636, `tickerid` native_run_spec.hpp:637, `type` native_run_spec.hpp:638, `basecurrency` native_run_spec.hpp:640, `description` native_run_spec.hpp:641 and `volumetype` native_run_spec.hpp:642 this is Pine's whole `syminfo` block, named rather than inherited from a chart. | | `commission_type` | `NativeFeeKind` native_run_spec.hpp:19 | `fee_kind` pineforge.h:502 | `native_open_lots_strategy.cpp` | `Percent`, `CashPerUnit`, `CashPerExecution` — the same three TradingView offers. | | `commission_value` | `fee_value` native_run_spec.hpp:658 | `fee_value` pineforge.h:502 | `native_open_lots_strategy.cpp` | Quoted per execution by `quote_execution_commissions` engine_execution.cpp:1588. A cash-per-execution ticket is split by units over every slice of that execution. | -| `slippage` | `slippage_ticks` native_run_spec.hpp:652 | `slippage_ticks` pineforge.h:502 | `native_price_grid_strategy.cpp` | Raw `ticks * price_tick`, applied once native_execution_consumer.cpp:6578-6582. TradingView's round-then-slip order is adapter-only. | -| `process_orders_on_close` | `NativeCloseExecution` native_run_spec.hpp:33 | `close_execution` pineforge.h:503 | `native_open_lots_strategy.cpp` | `AfterCalculation` emits the post-calculation close point native_execution_consumer.cpp:8000-8001, so a request submitted in bar *k*'s calculation fills at bar *k*'s own close. | -| `pyramiding` | `max_open_lots` native_run_spec.hpp:664 | `max_open_lots` pineforge.h:505 | `native_open_lots_strategy.cpp` | Caps surviving+new physical lots (`resulting_lot_count` native_execution_consumer.cpp:3277). The Pine adapter keeps its own per-cycle *entry count*, ruled **adapter-policy** in ADR-0001's table row `max_open_lots`; measured against TradingView it is not TradingView's rule, and this cap is the closer one (14 of 15 tape scenarios against 13, `tests/test_pyramiding_count_differential.cpp`). Lowered onto the cap, four corpus probes move away from TradingView, for want of its first-eligible-point rule, so the adapter keeps its count. | +| `slippage` | `slippage_ticks` native_run_spec.hpp:652 | `slippage_ticks` pineforge.h:502 | `native_price_grid_strategy.cpp` | Raw `ticks * price_tick`, applied once native_execution_consumer.cpp:7008-7012. TradingView's round-then-slip order is adapter-only. | +| `process_orders_on_close` | `NativeCloseExecution` native_run_spec.hpp:33 | `close_execution` pineforge.h:503 | `native_open_lots_strategy.cpp` | `AfterCalculation` emits the post-calculation close point native_execution_consumer.cpp:8452-8453, so a request submitted in bar *k*'s calculation fills at bar *k*'s own close. | +| `pyramiding` | `max_open_lots` native_run_spec.hpp:664 | `max_open_lots` pineforge.h:505 | `native_open_lots_strategy.cpp` | Caps surviving+new physical lots (`resulting_lot_count` native_execution_consumer.cpp:3610). The Pine adapter keeps its own per-cycle *entry count*, ruled **adapter-policy** in ADR-0001's table row `max_open_lots`; measured against TradingView it is not TradingView's rule, and this cap is the closer one (14 of 15 tape scenarios against 13, `tests/test_pyramiding_count_differential.cpp`). Lowered onto the cap, four corpus probes move away from TradingView, for want of its first-eligible-point rule, so the adapter keeps its count. | | `default_qty_type` | `Sized` native_order.hpp:164 with `CashValue` native_order.hpp:91 or `EquityFraction` native_order.hpp:97 | `PF_NATIVE_INTENT_SIZED` native_c_api.h:348 with `size_basis` native_c_api.h:2147 | `native_sized_report_strategy.cpp` | The size is a property of the *request*, not of the run: a native host may size one order by cash and the next by equity fraction. `Transact` native_order.hpp:41 is the fixed-units form. Pine itself has no per-call `qty_type` — TradingView's compiler refuses one — so a script's `strategy.entry(qty = ...)` is always units; the source layer's typed per-call entry, which only generated or C++ callers reach, converts its money exactly as the declared default does. | | `default_qty_value` | the basis's own scalar: `CashValue::cash` native_order.hpp:92, `EquityFraction::fraction` native_order.hpp:98 | `intent_value` native_c_api.h:2151 | `native_sized_report_strategy.cpp` | `SizeTime` native_order.hpp:106 chooses *when* the basis resolves and `SizePrice` native_order.hpp:146 *which* price it converts at; both compose. | | `calc_on_order_fills` | `NativeCalculationTrigger` native_run_spec.hpp:94 | `calculation` native_c_api.h:2321 | `native_calc_on_fills_strategy.cpp` | `BarCloseAndFills` recalculates once at the cursor of each applied execution, bounded by `max_recalculations_per_point` native_run_spec.hpp:695. The generic cadence only: TradingView's waypoint-only refill, its two-fills-at-open rule and its script-state rollback stay in the adapter. | @@ -296,7 +296,7 @@ is `strategy_native_submit_v1` native_c_api.h:2742 over a | `strategy.oca.reduce` | `GroupEffect::Reduce` native_order.hpp:466 | `PF_NATIVE_GROUP_REDUCE` native_c_api.h:479 | `tests/test_native_resting_group_contract.cpp` | The sibling's quantity is reduced instead of cancelled. | | `strategy.oca.none` | `NoGroup` native_order.hpp:467 | `PF_NATIVE_GROUP_NONE` native_c_api.h:472 | `hello_kernel.cpp` | The default: no group, no sibling effect. | | `strategy.direction.all` | `NativeOpenDirections::Both` native_run_spec.hpp:76 | `allowed_open_directions` pineforge.h:503 | `native_risk_limits_strategy.cpp` | The default. | -| `strategy.direction.long` | `NativeOpenDirections::Long` native_run_spec.hpp:74 | `allowed_open_directions` pineforge.h:503 | `tests/test_native_run_spec.cpp` | An opening the direction forbids is refused with `OpeningDirection` native_execution_consumer.cpp:3266. | +| `strategy.direction.long` | `NativeOpenDirections::Long` native_run_spec.hpp:74 | `allowed_open_directions` pineforge.h:503 | `tests/test_native_run_spec.cpp` | An opening the direction forbids is refused with `OpeningDirection` native_execution_consumer.cpp:3599. | | `strategy.direction.short` | `NativeOpenDirections::Short` native_run_spec.hpp:75 | `allowed_open_directions` pineforge.h:503 | `tests/test_native_run_spec.cpp` | Same, the other way. | | `strategy.commission.percent` | `NativeFeeKind::Percent` native_run_spec.hpp:20 | `fee_kind` pineforge.h:502 | `native_sized_report_strategy.cpp` | `fee_value` is percent ÷ 100 of the absolute account notional. | | `strategy.commission.cash_per_contract` | `NativeFeeKind::CashPerUnit` native_run_spec.hpp:21 | `fee_kind` pineforge.h:502 | `tests/test_native_selected_requests.cpp` | Account currency per unit. | @@ -405,7 +405,7 @@ block is the `PF_NATIVE_SPEC_EXT_RISK` native_c_api.h:739 tail. | Pine | C++ | C | Runs in | Notes | | --- | --- | --- | --- | --- | -| `strategy.risk.allow_entry_in()` | `allowed_open_directions` native_run_spec.hpp:665 | `allowed_open_directions` pineforge.h:503 | `native_risk_limits_strategy.cpp` | The one `strategy.risk.*` rule that is already the kernel's, per ADR-0001's `risk` ruling row. Rejects with `OpeningDirection` native_execution_consumer.cpp:3266. | +| `strategy.risk.allow_entry_in()` | `allowed_open_directions` native_run_spec.hpp:665 | `allowed_open_directions` pineforge.h:503 | `native_risk_limits_strategy.cpp` | The one `strategy.risk.*` rule that is already the kernel's, per ADR-0001's `risk` ruling row. Rejects with `OpeningDirection` native_execution_consumer.cpp:3599. | | `strategy.risk.max_position_size()` | `max_abs_units` native_run_spec.hpp:663 | `max_abs_units` pineforge.h:504 | `tests/test_native_run_spec.cpp` | Tests the *resulting* book. TradingView gates the *live* book before the fill, so an entry is refused only once the book already holds the limit — ruled **adapter-policy** in ADR-0001's `max_abs_units` row, with lane N12's measurement (the adapter ends at 4 where the kernel cap ends at 2). | | `strategy.risk.max_drawdown()` | `NativeRiskLimits::max_drawdown` native_run_spec.hpp:311 | `risk_max_drawdown` native_c_api.h:2357 | `native_risk_limits_strategy.cpp` | `NativeLossLimit` native_run_spec.hpp:259 is account currency, or a percent of the running peak when `percent` native_run_spec.hpp:261 is set. Blocks openings to the end of the run. | | `strategy.risk.max_intraday_loss()` | `NativeRiskLimits::max_intraday_loss` native_run_spec.hpp:312 | `risk_max_intraday_loss` native_c_api.h:2360 | `native_risk_limits_strategy.cpp` | Percent is of the day's opening equity. Blocks to the end of its own day. | @@ -463,7 +463,7 @@ part both ways (`tests/test_adapter_margin_schedule_differential.cpp`, M11). | `barstate.isrealtime` | `NativeStateView::phase` native_host.hpp:285 with `NativeRunPhase` native_host.hpp:41 | `pf_native_state_v1` native_c_api.h:1791 | `native_market_strategy.cpp` | `Batch`, `Warmup`, `Realtime`. | | bar-open pre-pass | `on_native_bar_open` native_host.hpp:889 | `on_bar_open` native_c_api.h:2506 | `native_calc_on_fills_strategy.cpp` | Receives the whole script bar by default; `NativeOpenBarView::OpenOnly` native_run_spec.hpp:109 masks that lookahead (H = L = C = open, volume 0) for a host that must decide at the open with open-only information. | | the bar so far, mid-bar | `current_partial_bar` native_host.hpp:1073 | `strategy_native_partial_bar_v1` native_c_api.h:3039 | `native_calc_on_fills_strategy.cpp` | Never runs ahead of the cursor: at a fill inside a leg it holds the points already reached and never the later high. | -| a fill, as it happens | `on_native_applied` native_host.hpp:938 | `on_applied` native_c_api.h:2509 | `native_calc_on_fills_strategy.cpp` | Fires mid-path after each fill; a request born there is eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:5948-5952). | +| a fill, as it happens | `on_native_applied` native_host.hpp:938 | `on_applied` native_c_api.h:2509 | `native_calc_on_fills_strategy.cpp` | Fires mid-path after each fill; a request born there is eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:6368-6372). | | raw input before aggregation | `on_native_input` native_host.hpp:875 | `on_input` native_c_api.h:2504 | `native_auxiliary_feed_strategy.cpp` | Every accepted confirmed input bar, before aggregation or matching. | | a lower-timeframe sub-bar | `on_native_sub_bar` native_host.hpp:928 | `on_sub_bar` native_c_api.h:2545 | `tests/test_native_calc_timing.cpp` | `NativeCalculationReason::SubBar` is reserved and never delivered to the recalculate hook — a sub-bar has its own callback. | | `time`, `bar_index`, the bar's calendar interval, session facts | `NativeDecisionContext` market_driver.hpp:120 with `NativeCoordinate` market_driver.hpp:72 | `pf_native_decision_v1` native_c_api.h:1485 | `hello_kernel.cpp` | `NativeCoordinate::interval_index` is the sequential SCRIPT-bar index on every run, including aggregation; `input_interval_index` names the input slot for a host that needs input cadence. Both are copied onto the callback stack, and mutating them changes nothing. In C++ `script_interval` market_driver.hpp:124 and `input_interval` market_driver.hpp:123 add the calendar facts (eligible open, last traded close, next period and next input opens); the C decision carries the script interval's five instants and the session days of its open and of the next input in its tail, presented to a callback table at `PF_NATIVE_CALLBACKS_V1_POLICY_SIZE` or later, and no input interval. The C `on_input` callback receives the input index separately. The session-day facts are the three rows below. | @@ -534,9 +534,9 @@ range-end reporting is off. | --- | --- | --- | --- | --- | | `strategy.account_currency` | `currency` native_run_spec.hpp:639 | `currency` pineforge.h:499 | `hello_kernel.cpp` / `native_fx_roll_strategy.cpp` | Declared, never inferred. `account_fx` native_run_spec.hpp:650 is the one positive scalar that converts quote to account; a timestamped curve is `configure_native_fx_curve` native_host.hpp:1202. The FX-roll example runs a JPY account on a USD-quoted stock through such a curve, and a step of it is a margin check point of its own. | | `strategy.initial_capital` | `initial_capital` native_run_spec.hpp:648 | `initial_capital` pineforge.h:501 | `hello_kernel.cpp` | The value you declared, unchanged by the run. | -| `strategy.equity` | `native_marked_equity(mark)` native_host.hpp:1301; on the Pine host `current_equity() + open_profit(mark)` pine_strategy_host.hpp:664 | `strategy_native_marked_equity_v1` native_c_api.h:3079 | `native_open_lots_strategy.cpp` | Pine marks at the current `close`. The native marked call takes the mark explicitly and returns balance plus open lots' **net** `unrealized_pnl` native_host.hpp:341, which is Pine's equity: TradingView charges the open lots' paid entry fees. | -| `strategy.netprofit` | on the Pine host `net_profit` pine_strategy_host.hpp:663; `net_profit` engine.hpp:967 is the closed trades' figure | `net_profit` pineforge.h:374, less open lots' `entry_commission` | `native_open_lots_strategy.cpp` | The closed trades' net profit less the entry fees the open lots have paid. The report's figure is the closed trades'. | -| `strategy.netprofit_percent` | derive: `net_profit` pine_strategy_host.hpp:663 over `initial_capital` native_run_spec.hpp:648 | `pf_metrics_t` pineforge.h:320 | `native_sized_report_strategy.cpp` | The kernel keeps the facts, not the ratio. | +| `strategy.equity` | `native_marked_equity(mark)` native_host.hpp:1301; on the Pine host `current_equity() + open_profit(mark)` pine_strategy_host.hpp:676 | `strategy_native_marked_equity_v1` native_c_api.h:3079 | `native_open_lots_strategy.cpp` | Pine marks at the current `close`. The native marked call takes the mark explicitly and returns balance plus open lots' **net** `unrealized_pnl` native_host.hpp:341, which is Pine's equity: TradingView charges the open lots' paid entry fees. | +| `strategy.netprofit` | on the Pine host `net_profit` pine_strategy_host.hpp:675; `net_profit` engine.hpp:967 is the closed trades' figure | `net_profit` pineforge.h:374, less open lots' `entry_commission` | `native_open_lots_strategy.cpp` | The closed trades' net profit less the entry fees the open lots have paid. The report's figure is the closed trades'. | +| `strategy.netprofit_percent` | derive: `net_profit` pine_strategy_host.hpp:675 over `initial_capital` native_run_spec.hpp:648 | `pf_metrics_t` pineforge.h:320 | `native_sized_report_strategy.cpp` | The kernel keeps the facts, not the ratio. | | `strategy.grossprofit` | `gross_profit` engine.hpp:968 | `pf_trade_stats_t` pineforge.h:269 | `native_open_lots_strategy.cpp` | | | `strategy.grossprofit_percent` | `grossprofit_percent` engine.hpp:975 | `pf_metrics_t` pineforge.h:320 | `native_sized_report_strategy.cpp` | Of initial capital. | | `strategy.grossloss` | `gross_loss` engine.hpp:969 | `pf_trade_stats_t::gross_loss` pineforge.h:224 | `native_open_lots_strategy.cpp` | For booked closed trades the protected C++ accessor is signed (negative for losses), while the C report stores a positive loss magnitude. With range-end reporting off, `-C.gross_loss == gross_loss()` and `netprofit == grossprofit - C.gross_loss`. With it on, C trade statistics include the report-only range-end rows; filter `open_at_end == 0` before comparing to Pine's closed-trade statistics. | @@ -618,22 +618,22 @@ span the same rows followed by the range-end rows `report_open_position_at_end` native_run_spec.hpp:683 adds. In C the report's `pf_report_t::trades` pineforge.h:371 carries the numeric fields, and the strings and the cause come from `strategy_closed_trade_entry_id` -pineforge.h:1167, `strategy_closed_trade_exit_id` pineforge.h:1180, -`strategy_closed_trade_exit_comment` pineforge.h:1182 and -`strategy_closed_trade_close_cause` pineforge.h:1249, which index exactly the +pineforge.h:1227, `strategy_closed_trade_exit_id` pineforge.h:1240, +`strategy_closed_trade_exit_comment` pineforge.h:1242 and +`strategy_closed_trade_close_cause` pineforge.h:1309, which index exactly the rows of that array. | Pine | C++ | C | Runs in | Notes | | --- | --- | --- | --- | --- | | `strategy.closedtrades` | `closed_trade_count` engine.hpp:1823 | `total_trades` pineforge.h:368 | `native_open_lots_strategy.cpp` | `closed_trade_count` counts booked closed rows. The C report's `total_trades` also includes range-end rows when `report_open_position_at_end` is enabled; filter `pf_trade_t::open_at_end == 0` to match Pine `strategy.closedtrades`. | | `strategy.closedtrades.first_index` | none — the kernel keeps every row | — | `native_open_lots_strategy.cpp` | TradingView drops old rows past a 9000-trade cap and advances `first_index` when it does. The kernel caps nothing, so the first index is always 0 and codegen emits the literal. | -| `strategy.closedtrades.entry_id()` | `entry_id` engine.hpp:183 | `strategy_closed_trade_entry_id` pineforge.h:1167 | `native_open_lots_strategy.cpp` | The same string the lot carried as `entry_label` native_host.hpp:333. | -| `strategy.closedtrades.entry_comment()` | `entry_comment` engine.hpp:184 | — | `native_open_lots_strategy.cpp` | No C accessor: `strategy_closed_trade_entry_id` pineforge.h:1167 answers the id, not the comment, and `pf_trade_t` carries no strings. A C host that needs it reads the open lot's `entry_comment` (`pf_native_open_lot_v1` native_c_api.h:1767) before the close. | +| `strategy.closedtrades.entry_id()` | `entry_id` engine.hpp:183 | `strategy_closed_trade_entry_id` pineforge.h:1227 | `native_open_lots_strategy.cpp` | The same string the lot carried as `entry_label` native_host.hpp:333. | +| `strategy.closedtrades.entry_comment()` | `entry_comment` engine.hpp:184 | — | `native_open_lots_strategy.cpp` | No C accessor: `strategy_closed_trade_entry_id` pineforge.h:1227 answers the id, not the comment, and `pf_trade_t` carries no strings. A C host that needs it reads the open lot's `entry_comment` (`pf_native_open_lot_v1` native_c_api.h:1767) before the close. | | `strategy.closedtrades.entry_bar_index()` | `entry_bar_index` engine.hpp:181 | `pf_trade_t` pineforge.h:203 | `native_open_lots_strategy.cpp` | Copied from the lot. | | `strategy.closedtrades.entry_time()` | `entry_time` engine.hpp:173 | `pf_trade_t` pineforge.h:203 | `native_open_lots_strategy.cpp` | Unix ms. | | `strategy.closedtrades.entry_price()` | `entry_price` engine.hpp:175 | `pf_trade_t` pineforge.h:203 | `native_open_lots_strategy.cpp` | | -| `strategy.closedtrades.exit_id()` | `exit_id` engine.hpp:186 | `strategy_closed_trade_exit_id` pineforge.h:1180 | `native_margin_strategy.cpp` | For a kernel liquidation this is the model's own `liquidation_label` native_run_spec.hpp:251, which is how a reporting layer classifies the row. | -| `strategy.closedtrades.exit_comment()` | `exit_comment` engine.hpp:185 | `strategy_closed_trade_exit_comment` pineforge.h:1182 | `native_margin_strategy.cpp` | | +| `strategy.closedtrades.exit_id()` | `exit_id` engine.hpp:186 | `strategy_closed_trade_exit_id` pineforge.h:1240 | `native_margin_strategy.cpp` | For a kernel liquidation this is the model's own `liquidation_label` native_run_spec.hpp:251, which is how a reporting layer classifies the row. | +| `strategy.closedtrades.exit_comment()` | `exit_comment` engine.hpp:185 | `strategy_closed_trade_exit_comment` pineforge.h:1242 | `native_margin_strategy.cpp` | | | `strategy.closedtrades.exit_bar_index()` | `exit_bar_index` engine.hpp:182 | `pf_trade_t` pineforge.h:203 | `native_open_lots_strategy.cpp` | | | `strategy.closedtrades.exit_time()` | `exit_time` engine.hpp:174 | `pf_trade_t` pineforge.h:203 | `native_open_lots_strategy.cpp` | | | `strategy.closedtrades.exit_price()` | `exit_price` engine.hpp:176 | `pf_trade_t` pineforge.h:203 | `native_open_lots_strategy.cpp` | | @@ -648,7 +648,7 @@ rows of that array. Pine has no word for *why* a row closed. The native row does: `closed_trade_close_cause` engine.hpp:1849 (C: -`strategy_closed_trade_close_cause` pineforge.h:1249) distinguishes a script +`strategy_closed_trade_close_cause` pineforge.h:1309) distinguishes a script close, a liquidation, a risk flatten and the range end — the kernel states the last three itself. The bracket-leg cause (`2`) is the kernel's too for a close its owner's fill armed — a `WaitForApplied` owner relation with an intent that @@ -669,7 +669,7 @@ classifies its own rows by order family instead. Which leg closed a row is its | `barmerge.lookahead_off` | `NativeTimeframeSubscription::lookahead` native_run_spec.hpp:494 set false | `pf_native_subscription_v1::lookahead` = `PF_NATIVE_LOOKAHEAD_AT_COMPLETION` native_c_api.h:800 | `native_htf_strategy.cpp` | The default and the honest one: a bucket is delivered when it completes. | | `barmerge.lookahead_on` | `NativeTimeframeSubscription::lookahead` native_run_spec.hpp:494 set true | `pf_native_subscription_v1::lookahead` = `PF_NATIVE_LOOKAHEAD_AT_FIRST_INPUT` native_c_api.h:802 | `tests/test_native_htf_subscriptions.cpp` | Delivers a bucket's final values on the input that held its first bar. It is lookahead: use it to reproduce a chart, never to trade. | | `request.currency_rate()` | none — no FX data feed | `strategy_configure_native_fx_curve_v1` pineforge.h:546 | `native_fx_roll_strategy.cpp` / `tests/test_native_fx_curve.cpp` | The kernel converts with the curve you stage; it fetches nothing. | -| `request.dividends()` | none — no fundamentals feed | — | — | For a Pine script, pineforge-codegen 1.0.0 (0.10.4 rejects the call at transpile) reads TradingView's recorded answers by request key from the source host (`strategy_set_recorded_series` pineforge.h:1493), reads `na` when the value reaches only plots and alerts, and otherwise stops the run where the value is read if nothing is recorded. A native host fetches and models it itself. | +| `request.dividends()` | none — no fundamentals feed | — | — | For a Pine script, pineforge-codegen 1.0.0 (0.10.4 rejects the call at transpile) reads TradingView's recorded answers by request key from the source host (`strategy_set_recorded_series` pineforge.h:1553), reads `na` when the value reaches only plots and alerts, and otherwise stops the run where the value is read if nothing is recorded. A native host fetches and models it itself. | | `request.earnings()` | none — no fundamentals feed | — | — | Same. | | `request.financial()` | none — no fundamentals feed | — | — | Same. | | `request.splits()` | none — no corporate-actions feed | — | — | Same. | @@ -1025,7 +1025,7 @@ cmake -S . -B build-kernel -DPINEFORGE_BUILD_SOURCE_LAYER=OFF -DPINEFORGE_BUILD_ ``` `PINEFORGE_BUILD_SOURCE_LAYER` CMakeLists.txt:43 OFF leaves `libpineforge.a` -holding exactly the objects of `PineForge::kernel` CMakeLists.txt:156, and +holding exactly the objects of `PineForge::kernel` CMakeLists.txt:158, and every example already links that target, so a source-layer symbol reaching an example is a link error in the default build too. diff --git a/docs/pages/run-failure-codes.md b/docs/pages/run-failure-codes.md index 80720f439..b45aea2e5 100644 --- a/docs/pages/run-failure-codes.md +++ b/docs/pages/run-failure-codes.md @@ -40,9 +40,11 @@ if (code != NULL && code[0] != '\0') { `engine_unclassified_error`, never `""`. - #strategy_get_last_error_args answers the arguments as one canonical JSON object: UTF-8, keys sorted, no whitespace; an `integer` argument is a JSON - integer, a `number` the shortest round-trip double, every other kind a JSON - string. It is `"{}"` for a code without arguments and `""` when there is no - failure. + integer, a `number` the shortest round-trip double, a `nullable_string` a + JSON string or `null`, a `scalar` a JSON string, integer, finite number, + `true`, `false` or `null` (a boolean or a null is never written as a + string), every other kind a JSON string. It is `"{}"` for a code without + arguments and `""` when there is no failure. - Both pointers live as long as #strategy_get_last_error's: until the next run or setter on the handle. - Run each request on a fresh handle. A handle whose run failed, a script @@ -91,7 +93,7 @@ codes it tested the text alone, so a script stopped by an empty The tables below list the catalog at this tree. -The `outputs_limit` and `outputs_rejected` rows below are unreleased, targeted for 1.5.0, and not part of 1.4.0; the other 38 codes retain their released 1.4.0 status. The `strategy_library_incompatible` row counts the two unreleased 1.5.0 outputs reasons, `outputs_api_mismatch` and `outputs_manifest_invalid`, in addition to its six released 1.4.0 reasons. +The `outputs_limit` and `outputs_rejected` rows below are unreleased, targeted for 1.5.0, and not part of 1.4.0; so are the 24 selected-window and report-contract rows described [after the tables](#run_failure_codes_selected_window). The other 38 codes retain their released 1.4.0 status. The `strategy_library_incompatible` row counts the two unreleased 1.5.0 outputs reasons, `outputs_api_mismatch` and `outputs_manifest_invalid`, in addition to its six released 1.4.0 reasons. @@ -115,6 +117,7 @@ The `outputs_limit` and `outputs_rejected` rows below are unreleased, targeted f | --- | --- | --- | --- | --- | | `chart_bars_rejected` | `input` | no | `field` (vocab, 6 values, optional), `index` (integer, optional), `reason` (vocab, 12 values) | The run's chart bars (the request's OHLCV) were refused. | | `chart_bars_unreadable` | `input` | no | `reason` (vocab, 4 values) | The harness could not read the run's OHLCV file. | +| `engine_command_value_invalid` | `engine_fault` | no | `field` (string), `reason` (vocab, 3 values), `stage` (vocab, 7 values) | The engine derived an invalid command value (non-finite, invalid text or an overflowing finite value) from its own book, equity, margin, sizing, fee, slippage or settlement state: an engine fault, never a strategy failure. stage names where it was derived. | | `engine_invariant` | `engine_fault` | no | none | An engine invariant failed: a PineForge bug. | | `engine_unclassified_error` | `engine_fault` | no | none | An error no coded site reported: a PineForge bug. | | `harness_internal_error` | `engine_fault` | no | none | The run harness failed: a PineForge bug. | @@ -134,10 +137,16 @@ The `outputs_limit` and `outputs_rejected` rows below are unreleased, targeted f | `recalc_cap` | `strategy_limit` | no | none | The script's calc_on_order_fills recalculation loop reached its cap. | | `recorded_request_missing` | `no_data` | no | none | No recorded series is installed for a request the script reads. | | `recorded_request_refused` | `input` | no | `reason` (vocab, 5 values) | A recorded request series was refused. | +| `report_platform_unavailable` | `engine_fault` | no | `field` (vocab, 6 values) | The runner cannot establish an execution-platform fact its report contract requires. A pre-load backstop for an image that claimed results-digest support, raised before any strategy code loads. | +| `report_post_execution_failed` | `engine_fault` | no | `phase` (vocab, 3 values), `reason` (vocab, 4 values) | The engine could not finish producing the result after strategy execution completed: a deadline or an otherwise unclassified failure in result assembly, the results digest or serialization. An engine fault, never a strategy failure; a specific typed engine fault keeps its own code. | +| `report_result_invalid` | `engine_fault` | no | `field` (string), `reason` (vocab, 4 values) | The report producer generated an invalid canonical result: an engine or harness fault, never a strategy failure. | +| `report_shape_request_invalid` | `input` | no | `option` (vocab, 2 values), `value` (scalar) | The report-shape request is invalid: the report shape or the curve point budget carries a value the runner refuses. | +| `report_shape_unsupported` | `unsupported` | no | `shape` (string) | The runner does not support the requested report shape. The first selected-window build supports the full shape only. | | `request_symbol_invalid` | `strategy` | no | none | A request.security symbol is not a valid symbol. | | `request_timeframe_invalid` | `strategy` | no | `api` (vocab, 2 values) | A request timeframe literal does not parse. | | `request_timeframe_unsupported` | `unsupported` | no | `input_tf` (timeframe, optional), `reason` (vocab, 7 values), `script_tf` (timeframe, optional) | A request timeframe cannot be served for this run's chart timeframe. | | `request_unsupported` | `unsupported` | no | `line` (integer, optional), `reason` (vocab, 2 values) | A request shape PineForge does not support. | +| `results_digest_unsupported` | `unsupported` | no | `requested` (string) | The runner does not support the requested results digest. The first selected-window build declares no digest. | | `run_aborted` | `engine_fault` | yes | none | The run was aborted on request. | | `run_mode_unsupported` | `unsupported` | no | `feature` (vocab, 10 values) | A feature the requested run mode does not support. | | `run_options_rejected` | `input` | no | `input_tf` (timeframe, optional), `option` (vocab, 11 values), `script_tf` (timeframe, optional) | The run's own options (timeframes, intrabar path, magnifier, path order) were refused. | @@ -149,10 +158,27 @@ The `outputs_limit` and `outputs_rejected` rows below are unreleased, targeted f | `strategy_library_incompatible` | `engine_fault` | no | `abi` (integer, optional), `missing` (vocab, 13 values, optional), `reason` (vocab, 8 values) | The compiled strategy library does not match the harness. | | `strategy_runtime_error` | `strategy` | no | none | The script stopped itself with runtime.error(); the message, possibly empty, is the text. | | `strategy_settings_rejected` | `strategy` | no | `field` (vocab, 30 values) | A strategy() declaration setting (or its override) is not a value the engine accepts. | +| `strategy_value_invalid` | `strategy` | no | `field` (string), `reason` (vocab, 3 values) | The strategy supplied an invalid actionable value (a non-finite number, invalid text or an overflowing finite value) to a declaration, setting or command that needs a valid one. | | `stream_input_rejected` | `input` | no | none | A streaming input (bar, tick or auxiliary bar) was refused. | | `symbol_feeds_refused` | `symbol_feeds` | no | `field` (vocab, 6 values, optional), `input_tf` (timeframe, optional), `reason` (vocab, 41 values), `script_tf` (timeframe, optional) | Other symbols' bars installed for request.security were refused. | | `symbol_metadata_rejected` | `symbol_metadata` | yes | `field` (vocab, 14 values) | The symbol's catalog metadata (syminfo) is not a value the engine accepts. | | `syminfo_unreadable` | `symbol_metadata` | yes | `reason` (vocab, 4 values) | The harness could not read the run's syminfo file. | +| `window_boundary_unaligned` | `input` | no | `bound` (vocab, 3 values), `next_boundary_ms` (integer), `previous_boundary_ms` (integer), `value_ms` (integer) | A selected-window boundary (start, end or fed start) is not aligned to the script calendar; the adjacent boundaries are carried. | +| `window_calendar_unsupported` | `unsupported` | no | `session` (nullable_string), `timezone` (string) | The timezone and session the selected window needs make a calendar the engine does not support; session may be null. | +| `window_feed_identity_changed` | `input` | no | `actual_sha256` (string), `expected_sha256` (string), `feed_id` (string) | The frozen primary feed's bytes differ from the SHA-256 the study recorded. The two digests are 64-character hexadecimal strings supplied by the trusted wrapper; the registry checks only that they are text. | +| `window_feed_range_invalid` | `input` | no | `end_ms` (integer), `fed_start_ms` (integer), `time_ms` (integer) | A primary-feed bar lies outside the frozen selected-window range. | +| `window_input_cap_exceeded` | `input` | no | `actual` (integer), `cap_name` (vocab, 8 values), `limit` (integer) | The supplied history exceeds a configured input cap; cap_name says which. | +| `window_legacy_flag_conflict` | `input` | no | `option` (string) | The legacy trade-start option was combined with selected-window mode. | +| `window_lineage_mismatch` | `input` | no | `actual` (scalar), `expected` (scalar), `field` (string) | A requested window field differs from the study's lineage. | +| `window_mode_unsupported` | `unsupported` | no | `capability` (string), `requested` (integer, optional), `timeframe` (string, optional) | The runner does not support the requested selected-window mode or a limit its build declares on it. capability names what is unsupported; timeframe carries the offending canonical timeframe and requested the offending pre-roll length where the refusal names one. Which optional argument goes with which capability is the trusted raisers' duty, not the registry's. | +| `window_no_observations` | `input` | no | `end_ms` (integer), `start_ms` (integer) | The selected window holds no observations to score. | +| `window_preroll_out_of_range` | `input` | no | `requested` (integer) | The requested pre-roll is outside the range of zero to 5000 script bars. | +| `window_prestart_fill` | `engine_fault` | no | `fill_ms` (integer), `start_ms` (integer) | A fill was dated before the selected window: an engine invariant failure. | +| `window_prestart_state_violation` | `engine_fault` | no | `position_size` (number), `resting_orders` (integer), `time_ms` (integer) | A position or a resting order existed before the selected window started: an engine invariant failure. | +| `window_pruner_unsupported` | `unsupported` | no | `pruner` (string) | A study that requests a selected window together with a trial pruner other than none is refused before any trial is allocated; pruner is the requested pruner's name as the caller sent it. Optimizer-only: run_json never raises it, because a backtest has no pruner. | +| `window_range_invalid` | `input` | no | `end_ms` (integer), `fed_start_ms` (integer), `start_ms` (integer) | The selected-window range (start, end, fed start) is invalid. | +| `window_report_mismatch` | `engine_fault` | no | `actual` (integer), `expected` (integer), `field` (vocab, 3 values) | The report disagrees with the trusted window counts. Raised by the caller that compares them and never emitted by run_json. | +| `window_request_invalid` | `input` | no | `option` (vocab, 13 values) | The selected-window request is invalid: option names the missing, malformed or conflicting field. | ### Argument kinds @@ -161,8 +187,11 @@ The `outputs_limit` and `outputs_rejected` rows below are unreleased, targeted f | `identifier` | A name the compiled strategy spells as a literal (a built-in, an argument). | | `integer` | A JSON integer. | | `keyword` | A fixed keyword. | +| `nullable_string` | UTF-8 text, or null when the request value is absent. | | `number` | A JSON number (shortest round-trip double). | | `pine_source` | Source text the transpiler emitted as a literal (a call's spelling, an input title). | +| `scalar` | A request value: UTF-8 text, an integer, a finite number, true, false or null. | +| `string` | UTF-8 text the trusted raiser supplies (a request value or a fixed name). | | `symbol` | A symbol the script wrote as a literal. | | `timeframe` | A timeframe the run request supplied. | | `vocab` | One value of the arg's closed `values` list. | @@ -184,6 +213,91 @@ configure refusal above) keeps its own text and carries that failure's code: nothing was recorded. A consumer reads the code and its arguments, never the English. +## The selected-window and report-contract codes {#run_failure_codes_selected_window} + +Twenty-four rows are the failure vocabulary of the documented selected-window +wire contract (v1.3). Twenty-one are its base table: the fifteen `window_*` +codes, `report_platform_unavailable`, `report_result_invalid`, +`report_shape_request_invalid`, `report_shape_unsupported`, +`results_digest_unsupported` and `strategy_value_invalid`. The contract adds +three: `report_post_execution_failed` (the phases after the strategy has +finished), `engine_command_value_invalid` (an invalid command value the engine +derived itself) and `window_pruner_unsupported` (the optimizer's trial pruner). +Their classes, argument kinds and English are the contract's; all are +`retryable: false` and `since: 1.5.0`, and none is released. The registry +declares them and validates their arguments, which is all this page states +about them: whether a build raises one is the build's own, and a row in the +catalog is not proof that the source paths it names enforce anything. + +- **Argument kinds.** `string`, `nullable_string` and `scalar` are generic + kinds added for them (the contract's string, nullable string and scalar). An + `integer` argument is any 64-bit integer; the wire's i53 and u53 bounds are + enforced where the request is read, before a code is raised. The + 64-hexadecimal-character digests of `window_feed_identity_changed` are plain + `string` arguments: the registry checks that they are text and the trusted + wrapper that raises the code owns their shape. A vocab value may hold dots + (`counts.window_input_bars`, `metering.billable_input_bars`). +- **`window_mode_unsupported`** takes the required `capability` and the optional + `timeframe` and `requested`. The contract names four capabilities: for + `selected_window_chart_timeframe` and `selected_window_request_feed_timeframe` + `timeframe` is the canonical timeframe token refused; for + `selected_window_request_feed_inventory` there is no `timeframe` (an + incomplete inventory names no token, and none is invented); for + `selected_window_preroll` `requested` is the pre-roll length. The registry + does not tie an optional argument to a capability: the trusted wrappers that + raise the code do. +- **`window_calendar_unsupported`** takes `session` as a `nullable_string`: a + JSON string, or `null` when the request names no session. `window_lineage_mismatch` + and `report_shape_request_invalid` carry request values as `scalar`. +- **Caller-only.** `window_report_mismatch` is raised by the caller that + compares a report with its trusted window counts. `run_json` never emits it. +- **Who raises the others.** The contract's route for each code is: the caller + or the deployed capability for `window_mode_unsupported`, + `report_shape_unsupported` and `results_digest_unsupported`; the caller's + request, bounds, calendar, feed or lineage for `window_request_invalid`, + `window_range_invalid`, `window_preroll_out_of_range`, + `window_legacy_flag_conflict`, `window_calendar_unsupported`, + `window_boundary_unaligned`, `window_feed_range_invalid`, + `window_no_observations`, `window_lineage_mismatch`, + `window_feed_identity_changed` and `report_shape_request_invalid`; the + caller's study configuration for `window_pruner_unsupported`; the + supplied-input cap for `window_input_cap_exceeded`; the engine's window + invariants for `window_prestart_state_violation` and `window_prestart_fill`; + a broken image before any strategy code loads for + `report_platform_unavailable`; a producer invariant defect for + `report_result_invalid`; and a value the script itself supplied to a source + sink for `strategy_value_invalid`. Two codes belong to the engine although a + valid script can lead up to them, because reachability does not confer + ownership: `report_post_execution_failed` (time or failure after the + strategy has finished, in result assembly, the digest or serialization) and + `engine_command_value_invalid` (a command value the engine derived from its + own book, equity, margin, sizing, fee, slippage or settlement state, as + opposed to one the script supplied). +- **Producers in this tree.** Twelve of the 24 have a raise site here, all in + the Python wrappers under `docker/`: `window_mode_unsupported`; the + selected-window request refusals `window_request_invalid`, + `window_range_invalid`, `window_preroll_out_of_range`, + `window_legacy_flag_conflict`, `window_calendar_unsupported`, + `window_boundary_unaligned` and `window_feed_range_invalid`; the report-option + refusals `report_shape_request_invalid`, `report_shape_unsupported` and + `results_digest_unsupported`; and `report_post_execution_failed`, for a + failure after the strategy has finished. The other twelve have none: + `window_report_mismatch` is the caller's (above), and the producers of + `window_no_observations`, `window_lineage_mismatch`, + `window_feed_identity_changed`, `window_input_cap_exceeded`, + `window_prestart_state_violation`, `window_prestart_fill`, + `report_platform_unavailable`, `report_result_invalid`, + `strategy_value_invalid`, `engine_command_value_invalid` and + `window_pruner_unsupported` are not in this tree. `report_shape_unsupported` + and `results_digest_unsupported` are the typed refusals this build raises + for the compact shape and the digest it does not declare; + `report_platform_unavailable` is meaningful only in an image that claims the + digest. `window_pruner_unsupported` is optimizer-only: + `run_json` never raises it, because a backtest has no pruner. The contract + has both study admissions (the Python study specification and the native + command line) refuse a selected window combined with a pruner other than + `none` before any trial is allocated; no raise site for it is in this tree. + ## The catalog {#run_failure_codes_catalog} `docker/run_failure_codes.json` is canonical JSON (keys sorted, one-space diff --git a/docs/report-terminal-quote.md b/docs/report-terminal-quote.md index 24288cd3d..c9db58a12 100644 --- a/docs/report-terminal-quote.md +++ b/docs/report-terminal-quote.md @@ -24,6 +24,21 @@ manifest. No runtime C ABI export or codegen change is needed; reserved numeric metadata keys bridge generated strategies, and direct hosts have an explicit setter and clearer. +A selected-window run (`strategy_set_selected_window_v1`) takes the quote as a +pre-begin input only. Before its first admitted begin, and on every handle +without a selected window, the writers behave as above. Once the selected run +has sealed (completed, failed or aborted), none of them stores: +`set_report_terminal_quote` returns false, and `set_syminfo_metadata` for the two +reserved keys and `clear_report_terminal_quote` throw +`SelectedWindowQueryAfterSeal` (`selected_window_query_after_seal`) before any +store, whatever the value. The C export `strategy_set_syminfo_metadata` is void +and swallows that exception, so a caller checks `strategy_state_query_status_v1`, +which answers `PF_STATE_QUERY_SELECTED_WINDOW_AFTER_SEAL_V1` for exactly this +condition. The refusal latches no run failure, changes no error, status or first +cause, and leaves the captured report as it was, so a repeated read presents the +quote the run was given. `tests/test_selected_quote_seal.cpp` forces every route +and compares the host object's bytes across the rejected call. + The accepted window has two chart-grid points: the completed mark itself or exactly one chart interval later. Off-grid timestamps and quotes beyond a session gap are inert. Direct hosts can inspect diff --git a/include/pineforge/compat/pine/intraday_cap.hpp b/include/pineforge/compat/pine/intraday_cap.hpp index 6c7ffef36..19f2689d2 100644 --- a/include/pineforge/compat/pine/intraday_cap.hpp +++ b/include/pineforge/compat/pine/intraday_cap.hpp @@ -143,6 +143,18 @@ class IntradayCap { configuration_.count_pooc_full_close = true; } std::uint8_t declared() const { return declared_; } + // Selected-window replay seam (Pine compatibility, not a kernel policy): + // writes back exactly the three configuration-owner fields and nothing + // else. It is not attach(), adopt_script_count() or operator=(int), and it + // never touches budget_, due_cause_ or next_action_, so no consumed-count + // fact is rewound. + void restore_replay_configuration(CapAttachment attachment, + const CapConfiguration& configuration, + std::uint8_t declared) noexcept { + attachment_ = attachment; + configuration_ = configuration; + declared_ = declared; + } static bool uses_chart_clock(const std::string& session) { return !(session.size() >= 9 && session[4] == '-' && hhmm_to_minutes(session.substr(0, 4)) >= 0 diff --git a/include/pineforge/execution_observer.h b/include/pineforge/execution_observer.h new file mode 100644 index 000000000..ad25006bf --- /dev/null +++ b/include/pineforge/execution_observer.h @@ -0,0 +1,148 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * execution_observer.h - public C ABI for the execution observer, version 1. + * + * Declarations only. The runtime supplies the definitions. + * + * Layout: natural C alignment, fixed-width scalars, no struct packing. Each + * struct_size must equal sizeof its declaration, and each version must be 1. + * Expected LP64 sizes are 24 (boundary), 32 (receipt), 24 (observer) and 48 + * (observation). + * + * Pointers are borrowed for the lifetime stated on each declaration; the + * runtime never takes ownership. A pf_strategy_t must be a valid, nonnull + * handle from the matched runtime. Forged or dangling handles are outside the + * contract. + */ + +#ifndef PINEFORGE_EXECUTION_OBSERVER_H +#define PINEFORGE_EXECUTION_OBSERVER_H + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Boundary handed to the observer. The kernel fills every field before the + * callback. run_generation is the latest admitted epoch: nonzero at + * admission, monotonic, never wraps. attempt_serial is the serial of the + * public run or stream-begin entry that owns that epoch. */ +typedef struct pf_execution_boundary_v1 { + uint32_t struct_size; + uint32_t version; + uint64_t run_generation; + uint64_t attempt_serial; +} pf_execution_boundary_v1; + +/** Receipt the callback fills. Before the callback the kernel sets the + * receipt's struct_size, version and run_generation, frame_bytes to + * UINT32_MAX, and handed_bytes, export_requested and reserved to 0. + * + * The callback must overwrite the UINT32_MAX frame_bytes sentinel even when + * nothing is exported. It succeeds only if it returns 0 with no exception and + * no earlier failure cause, and then: + * - struct_size, version and run_generation are unchanged; + * - reserved is 0; + * - export_requested is exactly 0 or exactly 1; + * - export_requested 0: frame_bytes and handed_bytes are both 0; + * - export_requested 1: 1 <= frame_bytes <= 1024 (the whole frame), and + * handed_bytes equals frame_bytes. + * + * handed_bytes does not claim a parent ACK. Any failure leaves the generation + * sealed, with no result work and no retry. */ +typedef struct pf_boundary_receipt_v1 { + uint32_t struct_size; + uint32_t version; + uint64_t run_generation; + uint32_t frame_bytes; + uint32_t handed_bytes; + uint32_t export_requested; + uint32_t reserved; +} pf_boundary_receipt_v1; + +/** Observer callback. The runtime calls it once, synchronously, after the + * generation seals on a successful terminal path and before results open. + * A failed or aborted generation closes without a successful callback. An + * exception thrown here is a failure and never unwinds through C. boundary + * and receipt are valid for this call only; context follows the registration + * lifetime. */ +typedef int (*pf_before_results_fn_v1)( + void *context, const pf_execution_boundary_v1 *boundary, + pf_boundary_receipt_v1 *receipt); + +/** Observer descriptor for strategy_set_execution_observer_v1(). */ +typedef struct pf_execution_observer_v1 { + uint32_t struct_size; + uint32_t version; + void *context; + pf_before_results_fn_v1 before_results; +} pf_execution_observer_v1; + +/** Snapshot of the execution state, filled by + * strategy_execution_observation_v1(). + * + * phase: 0 Idle, 1 Executing, 2 Capturing, 3 Sealed, 4 Results. + * fault_stage: 0 None, 1 Execution, 2 AfterExecution. + * attempt_outcome: 0 NotCompleted, 1 Refused, 2 Failed, 3 Aborted, + * 4 ResultsOpen, 5 Live. + * run_generation: latest admitted epoch (see pf_execution_boundary_v1). + * attempt_serial: latest minting public run or stream-begin entry. + * attempt_generation: 0 if that attempt admitted no generation. + * + * The caller correlates its own serial and generation. A rejected later + * attempt never turns an older Results snapshot into permission for itself. + * Registration or removal does not reopen a closed generation. A refused + * attempt before admission mints no generation, so the previous closure + * stays. */ +typedef struct pf_execution_observation_v1 { + uint32_t struct_size; + uint32_t version; + uint64_t run_generation; + uint64_t attempt_serial; + uint64_t attempt_generation; + uint32_t phase; + uint32_t fault_stage; + uint32_t attempt_outcome; + uint32_t boundary_delivered; +} pf_execution_observation_v1; + +/** Returns the execution observer ABI version, 1. */ +PF_API uint32_t pf_execution_observer_version(void); + +/** Registers, replaces or removes the strategy's execution observer. + * + * Returns 0 when accepted; -1 for an invalid struct_size or version, a null + * before_results or a null strategy handle; -2 for an unsupported execution + * contract (not a purity assertion or an unknown-override detector); -3 for a + * nonquiescent or reentrant call; -4 when the private consumer storage cannot + * be allocated. No exception crosses C. + * + * Registration copies the descriptor's values. context and before_results + * stay borrowed until a successful replacement or removal, or until the + * strategy is destroyed. A NULL descriptor removes the registration, but only + * when quiescent. A refusal leaves the installed descriptor unchanged. A + * forbidden reentry keeps its first cause latched, as the existing entry + * contract requires; an ordinary busy refusal does not rewrite a run outcome. + * Destroying the strategy never calls the observer. */ +PF_API int strategy_set_execution_observer_v1( + pf_strategy_t, const pf_execution_observer_v1 *); + +/** Copies one complete observation into *out. The caller first sets + * out->struct_size and out->version. + * + * Returns 0 with one complete snapshot, or -1 with *out unchanged. Allowed + * inside the observer; it dispatches no host code and allocates no missing + * consumer. A fresh or copied consumer reports generation 0 and no owned + * result. This is observation only, not permission to publish stale output. */ +PF_API int strategy_execution_observation_v1( + pf_strategy_t, pf_execution_observation_v1 *); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* PINEFORGE_EXECUTION_OBSERVER_H */ diff --git a/include/pineforge/metrics.hpp b/include/pineforge/metrics.hpp index f074309a2..c57e3cccc 100644 --- a/include/pineforge/metrics.hpp +++ b/include/pineforge/metrics.hpp @@ -25,5 +25,21 @@ pf_equity_stats_t compute_equity_stats(const pf_equity_point_t* curve, int64_t n double first_open, double last_close, int64_t bars_in_market, double net_profit); +// The same computation with the observation count stated, not read from the +// curve length. observation_count is the denominator of time_in_market_pct +// (bars_in_market / observation_count; NaN when it is not positive) and +// nothing else: every walk, guard and formula runs over the n curve points as +// above. The ordinary overload is this one with observation_count = n. A +// selected-window curve is its window anchor followed by M script-bar +// observations, so its caller passes n = M + 1 and observation_count = M: the +// anchor is a curve point and the base of the first return, not a bar. Same +// timezone-lock rule as above. +pf_equity_stats_t compute_equity_stats(const pf_equity_point_t* curve, int64_t n, + double initial_capital, + const std::string& chart_tz, + double first_open, double last_close, + int64_t bars_in_market, double net_profit, + int64_t observation_count); + } // namespace metrics } // namespace pineforge diff --git a/include/pineforge/native_order.hpp b/include/pineforge/native_order.hpp index 2afe71f32..20a2415ee 100644 --- a/include/pineforge/native_order.hpp +++ b/include/pineforge/native_order.hpp @@ -1807,6 +1807,23 @@ class WorkingRequestCore { const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal); + /// The same preparation under an explicit open activation grant: the caller + /// vouches, for this handle alone, that the request is to be read as born + /// before this point. The grant satisfies only the strict ordinal half of + /// the birth gate, and only at an Open cursor whose effective time is not + /// before the request's decision-time lower bound. It relaxes no time floor + /// and no trigger, driver, quantity, margin or price check, it leaves the + /// other birth exceptions as they stand, and it gives + /// EvaluationContext::pre_open_birth_eligible no new meaning (that bool stays + /// market/immediate only). Nothing retains it -- no request, plan or prepared + /// token -- so every call that tests the gate takes it afresh; the check and + /// the apply of an execution each do. The form above is this one with the + /// grant false. + Preparation prepare_evaluation(const RequestHandle& target, + const EvaluationContext& context, + const TargetObservation& observation, + uint64_t& next_timeline_ordinal, + bool open_activation_grant); /// `grid` carries the run's activation grid (ActivationGrid); the default /// is the raw rule, so every existing caller keeps its meaning. Preparation prepare_trigger(const RequestHandle& target, @@ -1815,11 +1832,28 @@ class WorkingRequestCore { uint64_t& next_timeline_ordinal, std::optional cohort_side = std::nullopt, const ActivationGrid& grid = {}); + /// prepare_trigger under the open activation grant of prepare_evaluation: the + /// cursor test every trigger transition makes (Stop, StopLimit and Trail + /// activation among them) takes the grant too. Every parameter is explicit, + /// so no earlier call can resolve to this form. + Preparation prepare_trigger(const RequestHandle& target, + const TriggerTransition& transition, + DriverEligibilityClass driver_class, + uint64_t& next_timeline_ordinal, + std::optional cohort_side, + const ActivationGrid& grid, + bool open_activation_grant); InstallResult install_mutation(PreparedMutation&& prepared) noexcept; Preparation prepare_execution(const RequestHandle& target, const ExecutionProposal& proposal, uint64_t& next_timeline_ordinal); + /// prepare_execution under the open activation grant of prepare_evaluation. + /// The token holds no grant: install_execution does not read the birth gate. + Preparation prepare_execution(const RequestHandle& target, + const ExecutionProposal& proposal, + uint64_t& next_timeline_ordinal, + bool open_activation_grant); InstallResult install_execution(PreparedExecution&& prepared, const CommittedExecutionFacts& facts) noexcept; Preparation prepare_no_effect(const RequestHandle& target, @@ -1862,6 +1896,11 @@ class WorkingRequestCore { bool refresh_allowance(const RequestHandle& target, const EvaluationContext& context, const TargetObservation& observation); + /// refresh_allowance under the open activation grant of prepare_evaluation. + bool refresh_allowance(const RequestHandle& target, + const EvaluationContext& context, + const TargetObservation& observation, + bool open_activation_grant); /// The same refresh, handed the run's timeline: an unbound close that /// carried a binding the book still has (ReplaceOptions::keep_binding) is /// bound here too, to the BookClose its CloseBoundEvent would install, and @@ -1871,6 +1910,12 @@ class WorkingRequestCore { const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal); + /// The timeline-handed refresh under the open activation grant. + bool refresh_allowance(const RequestHandle& target, + const EvaluationContext& context, + const TargetObservation& observation, + uint64_t& next_timeline_ordinal, + bool open_activation_grant); bool refresh_cohort_allowance(const RequestHandle& target, const EvaluationContext& context, const TargetObservation& observation); @@ -1961,12 +2006,27 @@ class WorkingRequestCore { const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal); + /// apply_evaluation under the open activation grant of prepare_evaluation. + Preparation apply_evaluation(const RequestHandle& target, + const EvaluationContext& context, + const TargetObservation& observation, + uint64_t& next_timeline_ordinal, + bool open_activation_grant); Preparation apply_trigger(const RequestHandle& target, const TriggerTransition& transition, DriverEligibilityClass driver_class, uint64_t& next_timeline_ordinal, std::optional cohort_side = std::nullopt, const ActivationGrid& grid = {}); + /// apply_trigger under the open activation grant of prepare_evaluation; every + /// parameter is explicit, like prepare_trigger's grant form. + Preparation apply_trigger(const RequestHandle& target, + const TriggerTransition& transition, + DriverEligibilityClass driver_class, + uint64_t& next_timeline_ordinal, + std::optional cohort_side, + const ActivationGrid& grid, + bool open_activation_grant); Preparation apply_no_effect(const RequestHandle& target, const EvaluationContext& context, uint64_t& next_timeline_ordinal); @@ -2001,15 +2061,36 @@ class WorkingRequestCore { Preparation check_execution(const RequestHandle& target, const ExecutionProposal& proposal, uint64_t& next_timeline_ordinal); + /// check_execution and apply_execution under the open activation grant of + /// prepare_evaluation. The two make the same birth test, so a caller hands + /// both the grant its scope holds; an apply given a different one answers + /// StalePreparation like any check that no longer passes. + Preparation check_execution(const RequestHandle& target, + const ExecutionProposal& proposal, + uint64_t& next_timeline_ordinal, + bool open_activation_grant); InstallResult apply_execution(const RequestHandle& target, const ExecutionProposal& proposal, const CommittedExecutionFacts& facts, uint64_t& next_timeline_ordinal); + InstallResult apply_execution(const RequestHandle& target, + const ExecutionProposal& proposal, + const CommittedExecutionFacts& facts, + uint64_t& next_timeline_ordinal, + bool open_activation_grant); EligibilityFacts eligibility_facts(const LiveRequest& live, const EvaluationContext& context) const noexcept; + /// eligibility_facts and evaluation_eligible under the open activation grant + /// of prepare_evaluation: the grant can only set birth_ok. + EligibilityFacts eligibility_facts(const LiveRequest& live, + const EvaluationContext& context, + bool open_activation_grant) const noexcept; bool evaluation_eligible(const LiveRequest& live, const EvaluationContext& context) const noexcept; + bool evaluation_eligible(const LiveRequest& live, + const EvaluationContext& context, + bool open_activation_grant) const noexcept; bool trigger_permits_driver(const Trigger& trigger, const TriggerState& state, DriverEligibilityClass driver_class, @@ -2079,10 +2160,11 @@ class WorkingRequestCore { /// queue order (the priority repriced_ names, or the handle's own number). std::size_t repriced_position(uint64_t incarnation) const noexcept; /// refresh_allowance's body, and its bind for an unbound close that - /// carried a binding. + /// carried a binding, under the open activation grant (false for every + /// form that does not name one). bool refresh_allowance(const RequestHandle& target, const EvaluationContext& context, const TargetObservation& observation, - uint64_t* next_timeline_ordinal); + uint64_t* next_timeline_ordinal, bool open_activation_grant); bool refresh_kept_binding(LiveRequest& live, const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal); @@ -2189,7 +2271,8 @@ class WorkingRequestCore { }; Preparation execution_values(const RequestHandle& target, const ExecutionProposal& proposal, - uint64_t& next_timeline_ordinal) const; + uint64_t& next_timeline_ordinal, + bool open_activation_grant) const; RunIdentity identity_; std::shared_ptr instance_; diff --git a/include/pineforge/pineforge.h b/include/pineforge/pineforge.h index 78e39c604..6131e3fdc 100644 --- a/include/pineforge/pineforge.h +++ b/include/pineforge/pineforge.h @@ -788,7 +788,14 @@ PF_API void strategy_stream_order_actions_clear(pf_strategy_t s); * build and configuration. The fingerprint's recipe belongs to the script * ABI epoch: from 1.0.0 it is fixed for every 1.x engine * (engine_script_run_v19), and a new recipe is a new epoch; a value from a - * build before 1.0.0 is not comparable. Returns 0 for NULL. */ + * build before 1.0.0 is not comparable. Returns 0 for NULL. + * + * A refused read also returns 0, so a 0 alone does not tell it from an + * answer: a handle that carries selected-window provenance refuses this + * query once its run has sealed, and #strategy_state_query_status_v1 names + * that refusal. The recorded broker-state rows + * (#strategy_set_broker_state_hash_recording) are not values of this token: + * it has no recorded counterpart, so a refused read leaves no witness. */ PF_API uint64_t strategy_stream_state_hash(pf_strategy_t s); /** Push one normalized realtime trade. Returns 0 on success, -1 on failure. */ @@ -1011,8 +1018,61 @@ PF_API void strategy_set_broker_state_hash_recording(pf_strategy_t s, int on); * configuration: the recipe belongs to the script ABI epoch, fixed from 1.0.0 * for every 1.x engine (engine_script_run_v19), so two values differ only * when the states they fold differ. This is not a serialized checkpoint. - * Returns 0 when @p s is NULL. */ + * Returns 0 when @p s is NULL. + * + * A refused read also returns 0, so a 0 alone does not tell it from an + * answer: once a handle that carries selected-window provenance has sealed + * its run, this query is refused and #strategy_state_query_status_v1 names + * it (#PF_STATE_QUERY_SELECTED_WINDOW_AFTER_SEAL_V1). A caller that needs + * that run's terminal witness enables + * #strategy_set_broker_state_hash_recording BEFORE the run and reads the last + * pf_report_t::broker_state_hash row; no row means no witness. */ PF_API uint64_t strategy_broker_state_hash(pf_strategy_t s); + +/** What #strategy_state_query_status_v1 answers: whether the two state queries, + * #strategy_broker_state_hash and #strategy_stream_state_hash, may answer a + * handle now, and whether the report-only terminal-quote writers may still + * store. A fixed-width word rather than an enum, so its width does not + * depend on the consumer's compiler flags. */ +typedef int32_t pf_state_query_status_v1_t; +#define PF_STATE_QUERY_ALLOWED_V1 0 /**< Both queries may answer, and the quote writers store. */ +#define PF_STATE_QUERY_SELECTED_WINDOW_AFTER_SEAL_V1 1 /**< Both are refused, and so are the quote writers: a selected window's run has sealed. */ +#define PF_STATE_QUERY_INVALID_HANDLE_V1 2 /**< The handle is NULL. */ + +/** The current eligibility of BOTH #strategy_broker_state_hash and + * #strategy_stream_state_hash on @p s, as a #pf_state_query_status_v1_t: + * #PF_STATE_QUERY_ALLOWED_V1 when either may answer, + * #PF_STATE_QUERY_SELECTED_WINDOW_AFTER_SEAL_V1 when the handle carries + * selected-window provenance and its run has sealed (a refused query returns + * 0, which is also a legal hash value: this tells the two apart), and + * #PF_STATE_QUERY_INVALID_HANDLE_V1 for a NULL @p s. A handle answers + * #PF_STATE_QUERY_ALLOWED_V1 unless it was admitted as a selected-window run, + * which only a separately documented export can do. This is the state of the + * handle at the call, not a last-error slot: it allocates nothing, changes + * nothing, runs no host code and never records, replaces or clears + * #strategy_get_last_error or its code, so a run's first cause stays what the + * run made it. To keep a selected run's terminal witness, enable + * #strategy_set_broker_state_hash_recording BEFORE the run and read the last + * pf_report_t::broker_state_hash row. A broker-state row is not a value of + * the #strategy_stream_state_hash token, and no row means no witness. + * + * The same status is the observable refusal of the report-only terminal-quote + * writers (docs/report-terminal-quote.md), under the same condition: once a + * selected-window run has sealed (completed, failed or aborted), the Pine + * host stores neither reserved syminfo metadata key, + * report_terminal_quote_time_ms and report_terminal_quote_close, whatever the + * value, nor a clear of the quote. In C++ the host throws + * pineforge::SelectedWindowQueryAfterSeal before any store; the C export + * #strategy_set_syminfo_metadata keeps its void signature and swallows every + * exception, so a caller learns of the refusal from this status: + * #PF_STATE_QUERY_SELECTED_WINDOW_AFTER_SEAL_V1 means a quote-key write stored + * nothing. Like the state queries, the refusal records, replaces and clears no + * #strategy_get_last_error or its code, latches no run failure and leaves the + * captured report as it was. A handle without selected-window provenance, and + * a selected handle before its first admitted begin, accept the writes as + * before. Proof: tests/test_selected_quote_seal.cpp. */ +PF_API pf_state_query_status_v1_t strategy_state_query_status_v1(pf_strategy_t s); + /** Number of orders resting in the engine's pending-order book after the * most recent run() (ABI v4 live-runtime surface, task 7, spec 3.6): the * book in force for the NEXT bar. 0 when @p s is NULL. Read-only; a diff --git a/include/pineforge/query_refusal.hpp b/include/pineforge/query_refusal.hpp new file mode 100644 index 000000000..cf6d8c3a7 --- /dev/null +++ b/include/pineforge/query_refusal.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include + +namespace pineforge { + +/// The refusal of a read-only state query, or of a report-only write, that may +/// no longer be answered. +/// +/// BacktestEngine::broker_state_hash() and BacktestEngine::stream_state_hash() +/// throw it, before they project or fold anything, when the engine carries +/// selected-window provenance and its selected run has sealed: a post-seal read +/// is refused by name instead of answering from state the window no longer +/// owns. An engine without that provenance never throws it, and an engine has +/// it only when its run was admitted as a selected window. The check reads +/// only the engine's own execution slot and its consumer's open flag, so it +/// allocates nothing, runs no host code and is no input of any hash. +/// +/// A caller that needs a selected run's terminal witness asks for it before the +/// run: enable set_broker_state_hash_recording(true), run, and read the last +/// recorded broker-state row (pf_report_t::broker_state_hash). A broker-state +/// row is not a stream-token value: stream_state_hash() has no recorded +/// counterpart, and no row means no witness. +/// +/// Report-only writes. The same condition, the same predicate and this same +/// exception also fence the writers of the report-only terminal quote +/// (docs/report-terminal-quote.md), whose values the sealed capture can no +/// longer use: once the selected run has sealed (completed, failed or aborted), +/// source::PineStrategyHost::set_syminfo_metadata for the two reserved keys +/// report_terminal_quote_time_ms and report_terminal_quote_close, and +/// source::PineStrategyHost::clear_report_terminal_quote, throw it before any +/// store, whatever the value; the bool +/// source::PineStrategyHost::set_report_terminal_quote returns false instead, +/// also with no store. A preset before the first admitted begin, and every +/// handle without selected-window provenance, keep their original behaviour. The +/// refusal latches no run failure, leaves the captured report as it was and +/// changes nothing a read of the run's error, status or first cause would show. +/// +/// The C exports never throw it. strategy_state_query_status_v1() reports +/// whether both queries would be refused, and strategy_broker_state_hash() and +/// strategy_stream_state_hash() return 0 for a refused read without entering +/// the C++ path. The C export strategy_set_syminfo_metadata() is void and +/// swallows every exception, so its refusal of a quote key is observed through +/// strategy_state_query_status_v1(): SELECTED_WINDOW_AFTER_SEAL means the write +/// stored nothing. Neither route changes a run's error, status or first cause. +/// Proof: tests/test_selected_quote_seal.cpp forces every writer after a +/// selected seal and compares the host object's bytes across the rejected call. +/// +/// what() is the fixed literal "selected_window_query_after_seal"; the class +/// holds no string and allocates nothing. +class SelectedWindowQueryAfterSeal : public std::exception { +public: + const char* what() const noexcept override { + return "selected_window_query_after_seal"; + } +}; + +} // namespace pineforge diff --git a/include/pineforge/run_failure.hpp b/include/pineforge/run_failure.hpp index a62616caf..5eefbaa00 100644 --- a/include/pineforge/run_failure.hpp +++ b/include/pineforge/run_failure.hpp @@ -117,6 +117,7 @@ #include +#include #include #include #include @@ -158,12 +159,22 @@ enum class RunFailureArgKind : std::uint8_t { pine_source, symbol, timeframe, + // The kinds below are appended, so every earlier kind keeps its value. + // string: text. nullable_string: text or a JSON null. scalar: text, an + // integer, a finite number, a boolean or a JSON null. None takes an array + // or an object. + string, + nullable_string, + scalar, }; /// One typed argument as a raise site spells it. `name` is a literal; a text /// value is copied. struct RunFailureArg { - enum class Type : std::uint8_t { text, integer, number }; + // boolean and null_value are appended, so the earlier values keep theirs. A + // boolean is held in `integer` (0 or 1) and serializes as true or false; a + // null_value holds nothing and serializes as null. Neither is ever a string. + enum class Type : std::uint8_t { text, integer, number, boolean, null_value }; const char* name = ""; Type type = Type::text; std::string text; @@ -175,6 +186,16 @@ struct RunFailureArg { RunFailureArg(const char* arg_name, const char* value); RunFailureArg(const char* arg_name, std::string value) : name(arg_name), text(std::move(value)) {} + // A JSON null, spelled with the literal nullptr. A `const char*` variable + // that happens to be null still reads as the empty text, as it always did. + RunFailureArg(const char* arg_name, std::nullptr_t) + : name(arg_name), type(Type::null_value) {} + // A JSON true or false. Only an argument of exactly type bool takes it: no + // pointer or integer converts to a boolean here. + template ::value, int>::type = 0> + RunFailureArg(const char* arg_name, Boolean value) + : name(arg_name), type(Type::boolean), integer(value ? 1 : 0) {} template ::value && !std::is_same::value, int>::type = 0> diff --git a/include/pineforge/run_failure_codes.hpp b/include/pineforge/run_failure_codes.hpp index 24d580518..0b7e2b852 100644 --- a/include/pineforge/run_failure_codes.hpp +++ b/include/pineforge/run_failure_codes.hpp @@ -15,46 +15,70 @@ enum class RunFailureCode : std::uint16_t { none = 0, chart_bars_rejected = 1, chart_bars_unreadable = 2, - engine_invariant = 3, - engine_unclassified_error = 4, - harness_internal_error = 5, - lot_grid_rejected = 6, - no_data_request = 7, - other_symbol_feed_missing = 8, - other_symbol_request = 9, - out_of_memory = 10, - outputs_limit = 11, - outputs_rejected = 12, - pine_array_error = 13, - pine_invalid_argument = 14, - pine_matrix_error = 15, - pine_na_reference = 16, - pine_runtime_limit = 17, - pine_string_error = 18, - recalc_cap = 19, - recorded_request_missing = 20, - recorded_request_refused = 21, - request_symbol_invalid = 22, - request_timeframe_invalid = 23, - request_timeframe_unsupported = 24, - request_unsupported = 25, - run_aborted = 26, - run_mode_unsupported = 27, - run_options_rejected = 28, - run_request_invalid = 29, - security_feed_refused = 30, - setting_rejected = 31, - setting_unsupported = 32, - strategy_create_failed = 33, - strategy_library_incompatible = 34, - strategy_runtime_error = 35, - strategy_settings_rejected = 36, - stream_input_rejected = 37, - symbol_feeds_refused = 38, - symbol_metadata_rejected = 39, - syminfo_unreadable = 40, + engine_command_value_invalid = 3, + engine_invariant = 4, + engine_unclassified_error = 5, + harness_internal_error = 6, + lot_grid_rejected = 7, + no_data_request = 8, + other_symbol_feed_missing = 9, + other_symbol_request = 10, + out_of_memory = 11, + outputs_limit = 12, + outputs_rejected = 13, + pine_array_error = 14, + pine_invalid_argument = 15, + pine_matrix_error = 16, + pine_na_reference = 17, + pine_runtime_limit = 18, + pine_string_error = 19, + recalc_cap = 20, + recorded_request_missing = 21, + recorded_request_refused = 22, + report_platform_unavailable = 23, + report_post_execution_failed = 24, + report_result_invalid = 25, + report_shape_request_invalid = 26, + report_shape_unsupported = 27, + request_symbol_invalid = 28, + request_timeframe_invalid = 29, + request_timeframe_unsupported = 30, + request_unsupported = 31, + results_digest_unsupported = 32, + run_aborted = 33, + run_mode_unsupported = 34, + run_options_rejected = 35, + run_request_invalid = 36, + security_feed_refused = 37, + setting_rejected = 38, + setting_unsupported = 39, + strategy_create_failed = 40, + strategy_library_incompatible = 41, + strategy_runtime_error = 42, + strategy_settings_rejected = 43, + strategy_value_invalid = 44, + stream_input_rejected = 45, + symbol_feeds_refused = 46, + symbol_metadata_rejected = 47, + syminfo_unreadable = 48, + window_boundary_unaligned = 49, + window_calendar_unsupported = 50, + window_feed_identity_changed = 51, + window_feed_range_invalid = 52, + window_input_cap_exceeded = 53, + window_legacy_flag_conflict = 54, + window_lineage_mismatch = 55, + window_mode_unsupported = 56, + window_no_observations = 57, + window_preroll_out_of_range = 58, + window_prestart_fill = 59, + window_prestart_state_violation = 60, + window_pruner_unsupported = 61, + window_range_invalid = 62, + window_report_mismatch = 63, + window_request_invalid = 64, }; -inline constexpr std::uint16_t kRunFailureCodeCount = 41; +inline constexpr std::uint16_t kRunFailureCodeCount = 65; } // namespace pineforge diff --git a/include/pineforge/selected_window.h b/include/pineforge/selected_window.h new file mode 100644 index 000000000..283f9eea9 --- /dev/null +++ b/include/pineforge/selected_window.h @@ -0,0 +1,106 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * selected_window.h - public C ABI for the selected-window configuration and + * counters, version 1. + * + * Declarations only. The runtime supplies the definitions. + * + * Layout: natural C alignment, fixed-width scalars, no struct packing. Each + * struct_size must equal sizeof its declaration, and each version must be 1. + * Expected LP64 sizes are 24 (configuration) and 80 (counts). + */ + +#ifndef PINEFORGE_SELECTED_WINDOW_H +#define PINEFORGE_SELECTED_WINDOW_H + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** Selected-window configuration, handed to strategy_set_selected_window_v1(). + * start_ms and end_ms are signed JSON-safe integer milliseconds with + * start_ms < end_ms. The selected window is [start_ms, end_ms); preroll is + * before start_ms. */ +typedef struct pf_selected_window_config_v1 { + uint32_t struct_size; + uint32_t version; + int64_t start_ms; + int64_t end_ms; +} pf_selected_window_config_v1; + +/** Native-owned counts for the latest attempt, filled by + * strategy_selected_window_counts_v1(). run_generation, attempt_serial and + * attempt_generation are the three identities the caller correlates with + * pf_execution_observation_v1 (see execution_observer.h). + * + * Counts are facts of actual successful primary input and script consumption, + * not caller-provided expectations, and a host's diagnostic presentation does + * not rewrite them. On success, fed_input_bars equals preroll_input_bars plus + * window_input_bars, and fed_script_bars equals preroll_script_bars plus + * window_script_bars. */ +typedef struct pf_selected_window_counts_v1 { + uint32_t struct_size; + uint32_t version; + uint64_t run_generation; + uint64_t attempt_serial; + uint64_t attempt_generation; + uint64_t fed_input_bars; + uint64_t fed_script_bars; + uint64_t preroll_input_bars; + uint64_t preroll_script_bars; + uint64_t window_input_bars; + uint64_t window_script_bars; +} pf_selected_window_counts_v1; + +/** Returns the selected-window ABI version, 1. */ +PF_API uint32_t pf_selected_window_version(void); + +/** Sets the pending selected-window configuration, or clears it with a NULL + * descriptor. A NULL descriptor disables selection for the next admitted + * generation. The setter copies only start_ms and end_ms. + * + * Returns 0 when accepted; -1 for an invalid handle, struct_size, version or + * bounds; -2 for an unsupported execution contract (not a purity declaration + * or an override detector, and not permission to reject a custom presentation + * host); -3 for a nonquiescent or reentrant call; -4 when required private + * storage cannot be allocated. Every refusal leaves the configuration + * unchanged, and no exception crosses C. + * + * The setter is quiescent only outside execution, capture and observer + * dispatch, including outside the existing preparing_begin, in_callback and + * processing_input barriers. Setting or clearing changes pending configuration + * only: it never reopens the provenance or results of an existing generation, + * rewrites the latest attempt or changes an existing capture. Configuration is + * copied into admitted state after the last begin refusal or reset and before + * the first admitted host callback, and a refused begin consumes no new + * selected generation. Copies and moves that discard the private consumer also + * discard pending configuration and any owned result, but preserve closed + * provenance. */ +PF_API int strategy_set_selected_window_v1( + pf_strategy_t, const pf_selected_window_config_v1 *); + +/** Copies one complete counts snapshot for the latest attempt into *out. The + * caller first sets out->struct_size and out->version. + * + * Returns 0 with one complete snapshot only when the latest attempt owns a + * successful selected Results generation. Returns -1 for a null handle or + * output, or a wrong struct_size or version. Returns -2 when no such result + * belongs to the latest attempt (fresh, copied, OFF, live, refused, failed, + * aborted or observer-faulted). On failure, valid caller storage is unchanged. + * + * The getter never executes host code, allocates a missing consumer or + * materializes a report. Counts do not open early inside the observer, and + * these counters never authorize charges. */ +PF_API int strategy_selected_window_counts_v1( + pf_strategy_t, pf_selected_window_counts_v1 *); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* PINEFORGE_SELECTED_WINDOW_H */ diff --git a/include/pineforge/selected_window_plan.h b/include/pineforge/selected_window_plan.h new file mode 100644 index 000000000..a3c1215a3 --- /dev/null +++ b/include/pineforge/selected_window_plan.h @@ -0,0 +1,139 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * selected_window_plan.h - public C bridge to the selected primary planner + * (pineforge::plan_selected_primary, selected_window_plan.hpp), version 1. + * + * The definitions live in the standalone window-plan shared library (a CMake + * target named in CMakeLists.txt, installed as a shared object on + * Linux). It links only the kernel archive: it is not a strategy library, loads + * no strategy, runs no host or library callback and exports nothing of the + * compiled-strategy ABI in pineforge.h. A caller names the helper by an absolute + * path from its own pinned build. C++ callers may keep linking PineForge::kernel + * and call the C++ planner directly. + * + * Layout: natural C alignment, fixed-width scalars, no struct packing. The + * caller presets struct_size to sizeof of its declaration and version to 1 in + * both descriptors. Expected LP64 sizes are 80 (request) and 248 (result). + * + * Rows: a read-only array of pf_bar_t (pineforge.h: five doubles, then the int64 + * Unix-millisecond timestamp), which the bridge reads in place as the identical + * pineforge::Bar. Only the timestamps are read; no row is copied. The array and + * all five strings need only stay valid for the duration of the call; the bridge + * keeps nothing, owns nothing it hands back, and has no free function. + */ + +#ifndef PINEFORGE_SELECTED_WINDOW_PLAN_H +#define PINEFORGE_SELECTED_WINDOW_PLAN_H + +#include + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** One selected-window planning request. Field meanings are those of the C++ + * SelectedPlanRequest: start_ms (T), end_ms (E) and fed_start_ms (F) are + * Unix milliseconds, preroll_bars is N, and the five strings are the input + * timeframe, the script timeframe, the chart and engine timezones and the + * session. + * + * All five strings must be non-NULL, NUL-terminated UTF-8. They are copied + * during the call. An empty chart_timezone, engine_timezone or session keeps + * the C++ default meaning; an empty timeframe is a planner refusal, not a + * bridge failure. feed_tolerant is exactly 0 or 1. */ +typedef struct pf_selected_plan_request_v1 { + uint32_t struct_size; + uint32_t version; + int64_t start_ms; + int64_t end_ms; + int64_t fed_start_ms; + uint32_t preroll_bars; + uint32_t feed_tolerant; + const char *input_tf; + const char *script_tf; + const char *chart_timezone; + const char *engine_timezone; + const char *session; +} pf_selected_plan_request_v1; + +/** One complete planner result, plain data, written whole by a successful call. + * + * status is the C++ SelectedPlanStatus mapped explicitly, not inferred from the + * enumerator's width: 0 Ok, 1 RequestInvalid, 2 CalendarUnsupported, + * 3 BoundaryUnaligned, 4 FeedRangeInvalid, 5 RowsUnordered, 6 InternalError. + * bound is -1 (no boundary detail), 0 (T), 1 (E) or 2 (F). Every other field + * has the meaning and the equalities of the C++ SelectedPrimaryPlan. + * + * present_mask bit i (0..7) says whether the i-th of the eight optional + * timestamps is present, in their field order below: preroll_first_bar_ms, + * preroll_last_bar_ms, supplied_first_data_ms, supplied_last_data_ms, + * fed_first_data_ms, fed_last_data_ms, window_first_data_ms, + * window_last_data_ms. An absent timestamp is zero, and a zero with its bit set + * is a real timestamp. shortfall and complete_pending_preroll_at_horizon are + * exactly 0 or 1; reserved is 0. option is ASCII, NUL-terminated and + * zero-padded (the C++ option string, at most 31 characters). */ +typedef struct pf_selected_plan_result_v1 { + uint32_t struct_size; + uint32_t version; + int32_t status; + int32_t bound; + int64_t value_ms; + int64_t previous_boundary_ms; + int64_t next_boundary_ms; + uint64_t supplied_input_bars; + uint64_t supplied_script_bars; + uint64_t available_script_bars; + uint64_t used_script_bars; + uint64_t trimmed_script_bars; + uint64_t trim_index; + uint64_t fed_input_bars; + uint64_t fed_script_bars; + uint64_t preroll_input_bars; + uint64_t window_input_bars; + uint64_t window_script_bars; + int64_t trim_start_ms; + int64_t preroll_first_bar_ms; + int64_t preroll_last_bar_ms; + int64_t supplied_first_data_ms; + int64_t supplied_last_data_ms; + int64_t fed_first_data_ms; + int64_t fed_last_data_ms; + int64_t window_first_data_ms; + int64_t window_last_data_ms; + uint32_t present_mask; + uint32_t shortfall; + uint32_t complete_pending_preroll_at_horizon; + uint32_t reserved; + char option[32]; +} pf_selected_plan_result_v1; + +/** Returns the selected-planner C bridge version, 1. */ +PF_API uint32_t pf_selected_plan_version(void); + +/** Plans the primary rows of one selected-window request. + * + * Returns 0 when one whole result was written to *out, INCLUDING every planner + * refusal (a refusal is carried in out->status, never in the return value). + * Returns -1 for a NULL request or output, a wrong struct_size or version in + * either descriptor, a NULL string pointer, a feed_tolerant other than 0 or 1, + * or a count that does not fit size_t; -2 when memory could not be allocated; + * -3 for any other bridge failure or broken bridge invariant (for example a + * planner option longer than 31 characters, which is never truncated). On every + * negative return a valid caller output is byte-unchanged. + * + * rows may be NULL only with count 0. A NULL rows with a positive count is not + * dereferenced by the bridge: the planner refuses it as RequestInvalid. The + * bridge builds a local result and commits it to *out with one assignment, calls + * no host or library callback, and lets no exception cross C. */ +PF_API int pf_plan_selected_primary_v1(const pf_selected_plan_request_v1 *, + const pf_bar_t *rows, uint64_t count, + pf_selected_plan_result_v1 *); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* PINEFORGE_SELECTED_WINDOW_PLAN_H */ diff --git a/include/pineforge/selected_window_plan.hpp b/include/pineforge/selected_window_plan.hpp new file mode 100644 index 000000000..833e3836a --- /dev/null +++ b/include/pineforge/selected_window_plan.hpp @@ -0,0 +1,160 @@ +#pragma once + +#include + +#include +#include +#include +#include + +// The selected primary planner: from the supplied primary rows and one +// selected-window request, which rows a selected run keeps and what the native +// consumer will execute over them. Pure and read-only: it constructs no +// strategy or engine, runs no host callback, hashes nothing, opens no feed or +// artifact, and reads only the rows' timestamps. The caller owns the row array; +// the result owns its values. An allocation failure (std::bad_alloc) +// propagates; any other exception of the native calendar becomes a status, +// never a guessed value. +// +// Grouping is the consumer's own (NativeExecutionConsumer for a batch run): the +// input interval at the row's timestamp, then the script interval at that input +// interval's open, both from native_calendar and its session-day memo. A +// group's key and its published label are both the script interval's open_ms +// and are held separately. A FeedTolerant request on a Passthrough pairing +// groups on each row's own timestamp; otherwise a failed interval lookup falls +// back to that raw partition under FeedTolerant and is a refusal without it. The +// groups a run executes follow the consumer's three seal rules: an input that +// reaches the script interval's next_period_open, the first input of a later +// group, and, at the end of the batch, a final input that reaches the script +// interval's last_traded_close. A final window group none of them seals is +// never executed and is not counted. +// +// Admission order: the script token (one of the first build's eight intraday +// primary tokens) and an explicit input timeframe; T/E/F as signed 53-bit +// integers with T < E and F <= T; preroll_bars <= 5000; the row array; the +// conservative clock identity (after the empty default only the exact spellings +// UTC, GMT, Etc/UTC and Etc/GMT fold together, every other zone string is +// compared exactly, and the one session string serves both clocks); the native +// session, timezone and timeframe grammar and the pairing; +// T, E and F each equal to the nominal open of the native script interval that +// holds them, independent of the rows; then the rows (strictly increasing, each +// F <= timestamp < E). Nothing snaps a boundary. +// +// On a refusal only status, option, bound, value_ms, previous_boundary_ms and +// next_boundary_ms are set; every count is zero. `option` names the faulty +// request member: +// RequestInvalid script_tf, input_tf, start_ms, end_ms, fed_start_ms, +// preroll_bars (value_ms = the count), rows. A bound +// carries bound 0/1/2 and value_ms; end_ms <= start_ms +// carries previous_boundary_ms = T, fed_start_ms > T +// carries next_boundary_ms = T. +// CalendarUnsupported timezone (the clock identities differ), engine_timezone +// (grammar), session (grammar), calendar (a native +// calendar exception), timeframes (a pairing the +// consumer does not group), start_ms/end_ms/fed_start_ms +// (no native interval holds the boundary), rows (no input +// or script interval holds a row, value_ms = its +// timestamp), and start_ms with bound 0 when one group +// would hold rows on both sides of T (previous_boundary_ms +// and next_boundary_ms = that group's open and next open). +// BoundaryUnaligned start_ms, end_ms or fed_start_ms with bound 0, 1 or 2, +// value_ms = the boundary, previous_boundary_ms and +// next_boundary_ms = the open and next_period_open of the +// native script interval that holds it. +// FeedRangeInvalid rows; bound 2 below F, bound 1 at or after E, value_ms = +// the row's timestamp, previous_boundary_ms = F, +// next_boundary_ms = E. +// RowsUnordered rows; value_ms = the row's timestamp, +// previous_boundary_ms = the row before it. +// InternalError a native exception outside the admission step, or a +// state the plan's own two passes disagree on. +// +// The plan is a prediction, not a count: the actual executed counts are the +// consumer's observed counters, which must later equal it. A pending PRE-T +// group that nothing seals is completed once at the accepted T from its own +// retained aggregate (the consumer integration, a separate change, must do that +// without synthetic prices, orders or fills and before capture); +// complete_pending_preroll_at_horizon says it is needed. An OFF run never +// uses this planner. + +namespace pineforge { +inline namespace selected_window_plan_v1 { + +enum class SelectedPlanStatus { Ok, RequestInvalid, CalendarUnsupported, + BoundaryUnaligned, FeedRangeInvalid, RowsUnordered, InternalError }; +struct SelectedPlanRequest { + std::int64_t start_ms, end_ms, fed_start_ms; + std::uint32_t preroll_bars; + std::string input_tf, script_tf, chart_timezone, engine_timezone, session; + bool feed_tolerant = true; +}; +struct SelectedPrimaryPlan { + SelectedPlanStatus status = SelectedPlanStatus::InternalError; + std::string option; + int bound = -1; // 0=T, 1=E, 2=F, -1=no boundary detail + std::int64_t value_ms = 0, previous_boundary_ms = 0, next_boundary_ms = 0; + std::uint64_t supplied_input_bars = 0, supplied_script_bars = 0; + std::uint64_t available_script_bars = 0, used_script_bars = 0; + std::uint64_t trimmed_script_bars = 0, trim_index = 0; + std::uint64_t fed_input_bars = 0, fed_script_bars = 0; + std::uint64_t preroll_input_bars = 0, window_input_bars = 0, window_script_bars = 0; + std::int64_t trim_start_ms = 0; + std::optional preroll_first_bar_ms, preroll_last_bar_ms; + std::optional supplied_first_data_ms, supplied_last_data_ms; + std::optional fed_first_data_ms, fed_last_data_ms; + std::optional window_first_data_ms, window_last_data_ms; + bool shortfall = false; + bool complete_pending_preroll_at_horizon = false; +}; +SelectedPrimaryPlan plan_selected_primary(const Bar* rows, std::size_t count, + const SelectedPlanRequest& request); + +// The retained primary count, for the source layer's tail expectations: over +// rows the caller has ALREADY retained (any pre-roll trimming happened before +// them), the number of script bars a selected batch run executes. It is +// plan_selected_primary's fed_script_bars for a feed that plan would not trim +// further, taken from the same clock admission, boundary check and consumer +// bucket walk (no second calendar), with no F boundary, no pre-roll request and +// no 5000 cap: a native caller's retained prehistory is its own. Every group +// that opens before T is counted (each is executed, a last pending one by the +// completion at the accepted T) together with every window group the consumer's +// seal rules execute; an unsealed window tail adds none. Pure and read-only like +// the planner: it constructs no strategy or consumer and reads only timestamps. +// +// Admission order: the script token (one of the eight intraday primary tokens) +// and an explicit input timeframe; T and E as signed 53-bit integers with +// T < E; the row array; the conservative clock identity, the native session, +// timezone and timeframe grammar and the pairing, exactly as the planner +// admits them; T and E each equal to the nominal open of the native script +// interval that holds them; then the rows (strictly increasing, each < E). +// Nothing snaps a boundary. A refusal sets `status` and `option` with the +// planner's names for the same fault (script_tf, input_tf, start_ms, end_ms, +// rows, timezone, engine_timezone, session, calendar, timeframes) and leaves +// script_bars zero; a group that would hold rows on both sides of T is +// CalendarUnsupported on start_ms. Allocation failure behaves as in the +// planner: std::bad_alloc propagates and any other native calendar exception +// becomes a status. +struct SelectedRetainedRequest { + std::int64_t start_ms, end_ms; + std::string input_tf, script_tf, chart_timezone, engine_timezone, session; + bool feed_tolerant = true; +}; +struct SelectedRetainedCount { + SelectedPlanStatus status = SelectedPlanStatus::InternalError; + std::string option; + std::uint64_t script_bars = 0; +}; +SelectedRetainedCount count_selected_retained_primary(const Bar* rows, std::size_t count, + const SelectedRetainedRequest& request); + +// The script bars an admitted selected batch run executes over its retained rows: +// count_selected_retained_primary with E unchecked and unbounded. The selected +// window setter admits any T < E, the consumer executes the same groups whatever E +// is, and the count above refuses only on E's own alignment or a row at or after +// it, so a run reads the one answer every such E gives. T, the clock and the rows +// are admitted exactly as above. +SelectedRetainedCount count_selected_executed_primary(const Bar* rows, std::size_t count, + const SelectedRetainedRequest& request); + +} // inline namespace selected_window_plan_v1 +} // namespace pineforge diff --git a/include/pineforge/source/pine_adapter.hpp b/include/pineforge/source/pine_adapter.hpp index 34cd4e1e6..5d31bafa6 100644 --- a/include/pineforge/source/pine_adapter.hpp +++ b/include/pineforge/source/pine_adapter.hpp @@ -1452,6 +1452,25 @@ struct PineRiskState { bool intraday_cancel_pending = false; }; +// The source-risk CONFIGURATION alone, for the selected-window replay bracket +// (PineExecutionAdapter::capture_replay_risk_config and +// restore_replay_risk_config). It holds no runtime fact of PineRiskState +// (halted, observed peak or drawdown, intraday block, cancel latch) and none +// of the cap's consumed state. The percent flags are kept exactly because the +// setters make them sticky. +struct PineReplayRiskConfig { + int direction = 0; // 0 both, >0 long, <0 short + int max_cons_loss_days = 0; + double max_drawdown = 0.0; + bool max_drawdown_percent = false; + double max_intraday_loss = 0.0; + bool max_intraday_loss_percent = false; + double max_position_size = 0.0; + compat::pine::CapAttachment cap_attachment = compat::pine::CapAttachment::None; + compat::pine::CapConfiguration cap_configuration{}; + std::uint8_t cap_declared = 0; +}; + class PineExecutionAdapter; // Allocation-free view facade for Appendix C's later C projection. L2 does @@ -1664,6 +1683,13 @@ class PineExecutionAdapter { void set_risk_max_drawdown(double value, bool percent) noexcept; void set_risk_max_intraday_loss(double value, bool percent) noexcept; void set_risk_max_position_size(double value) noexcept; + // Exact capture/restore of the source-risk configuration (replay seam). + // Restore writes the seven risk fields directly, because the setters above + // keep percent flags sticky, and restores the cap's three configuration + // fields through IntradayCap::restore_replay_configuration. No risk runtime + // fact and no cap accounting is read or written. + PineReplayRiskConfig capture_replay_risk_config() const noexcept; + void restore_replay_risk_config(const PineReplayRiskConfig&) noexcept; bool allows_risk_direction(bool is_long) const noexcept { return risk_.direction == 0 || (is_long ? risk_.direction > 0 : risk_.direction < 0); } @@ -2440,6 +2466,7 @@ class PineExecutionAdapter { const Bar&, const NativeDecisionContext&); void order_open_marketable_limit_entries(const Bar&, const NativeDecisionContext&); void apply_terminal_explicit_market_policy(const NativeDecisionContext&); + void apply_terminal_explicit_market_policy(const NativeDecisionContext&, bool execute_now); bool enqueue_pooc_fifo_close(const SourceId&, const std::string&, std::uint64_t, std::uint64_t); void observe_close_ledger(const native_order::ExecutionAppliedEvent&, diff --git a/include/pineforge/source/pine_strategy_host.hpp b/include/pineforge/source/pine_strategy_host.hpp index 3050723b1..7bf51d77b 100644 --- a/include/pineforge/source/pine_strategy_host.hpp +++ b/include/pineforge/source/pine_strategy_host.hpp @@ -407,6 +407,18 @@ class PineStrategyHost : public NativeStrategyHost, public BrokerStateHashProvid int short_seed_collision_role_v1(native_order::RequestHandle) const noexcept; void enable_pine_intraday_cap(); void attach_pine_execution_adapter(); + // The report-only terminal quote (docs/report-terminal-quote.md) has three + // writers, and none stores once a selected-window run has sealed + // (selected_window.h; the fence of query_refusal.hpp): set_syminfo_metadata, + // for the two reserved keys report_terminal_quote_time_ms and + // report_terminal_quote_close, and clear_report_terminal_quote throw + // pineforge::SelectedWindowQueryAfterSeal before any store, and + // set_report_terminal_quote returns false. The C export + // strategy_set_syminfo_metadata is void and swallows the throw, so its + // refusal is read through strategy_state_query_status_v1. The refusal latches + // no run failure and leaves the error, the status, the first cause and the + // captured report alone. A preset before the first admitted begin, and every + // handle without selected-window provenance, keep their original behaviour. void set_syminfo_metadata(const std::string&, double) override; bool set_report_terminal_quote(std::int64_t time_ms, double close); void clear_report_terminal_quote(); @@ -1003,6 +1015,30 @@ class PineStrategyHost : public NativeStrategyHost, public BrokerStateHashProvid void scheduler_publish_source_bar(const Bar&, bool first_tick, bool advance_source_index = true); void scheduler_publish_suppressed_tail(const Bar&); + // The selected window's pre-roll capture hooks (src/source/pine_intent_state.hpp): + // nonvirtual, and no member of this class; the state is the consumer's host + // state. Begin resets the buffer to the evaluation labelled `label_ms` and + // opens its capture bracket; it answers false, doing nothing, unless the run is + // selected and the label precedes the window. Finish records the evaluation's + // end configuration and closes the bracket. A suppressed pre-roll tail runs no + // body and clears the buffer to an empty evaluation. + bool selected_begin_evaluation(std::int64_t label_ms); + void selected_finish_evaluation() noexcept; + void selected_clear_suppressed_evaluation(std::int64_t label_ms); + // The selected window's replay of that buffer, nonvirtual like the hooks above. + // At the first Open labelled inside the window, ahead of adapter_.on_bar_open, + // the last pre-roll evaluation's rows go to the adapter's own doors once, in + // their original order, each under its own configuration and retained point, + // inside the consumer's open activation scope; then the same drains a source + // evaluation ends with and the terminal explicit-market policy, submit-only. + // It does nothing, and allocates nothing, for a run that is not selected and + // for a label before the window. A broken precondition or a native failure + // throws engine_invariant: the replay is never skipped quietly. + void selected_replay_at_open(const Bar&, const NativeDecisionContext&); + // The drains that close a source evaluation once its body has run: pending + // closes, pending entries, bracket legs, relative exits and the same-point + // entry order. The one list scheduler_publish_source_bar and the replay share. + void finish_adapter_evaluation(); void project_short_seed_report_rows(const native_order::ExecutionAppliedEvent&); void order_global_exit_children(const native_order::ExecutionAppliedEvent&); bool scheduler_coof_enabled() const noexcept { return config_.calc_on_order_fills; } diff --git a/scripts/check_c_abi_runtime.py b/scripts/check_c_abi_runtime.py index a8d42e7ed..143071835 100644 --- a/scripts/check_c_abi_runtime.py +++ b/scripts/check_c_abi_runtime.py @@ -3,11 +3,11 @@ Two inventories, pinned independently so neither can drift into the other: - 1. The compiled-strategy surface. include/pineforge/pineforge.h lists the - harness-facing symbols; only those in EXPECTED_RUNTIME are defined in the - static runtime (src/c_abi.cpp) — the rest are emitted per-strategy by the - transpiler (see comment in src/c_abi.cpp). If that split changes, update - EXPECTED_RUNTIME below and the comment block in c_abi.cpp together. + 1. The compiled-strategy surface. pineforge.h plus the v1 headers named in + RUNTIME_HEADERS list the harness-facing symbols; only those in + EXPECTED_RUNTIME are defined in the static runtime (src/c_abi.cpp) — the + rest are emitted per-strategy by the transpiler. If that split changes, + update EXPECTED_RUNTIME below and the comment block in c_abi.cpp together. 2. The C-level native host API (R5 lane L13). Its declarations live in include/pineforge/native_c_api.h and its implementations in @@ -62,6 +62,7 @@ "strategy_last_bar_dual_entry_path", "strategy_set_broker_state_hash_recording", "strategy_broker_state_hash", + "strategy_state_query_status_v1", "strategy_pending_orders_len", "strategy_pending_order_get", "strategy_pending_order_layout", @@ -104,10 +105,28 @@ "strategy_outputs_event_get", "strategy_outputs_events_clear", "strategy_outputs_constants_copy", + # Execution observer v1 (execution_observer.h) and selected window v1 + # (selected_window.h): each header's version getter, setter and reader. + "pf_execution_observer_version", + "strategy_set_execution_observer_v1", + "strategy_execution_observation_v1", + "pf_selected_window_version", + "strategy_set_selected_window_v1", + "strategy_selected_window_counts_v1", }) -EXPECTED_PUBLIC_DECLARATIONS = 93 -EXPECTED_RUNTIME_IMPLEMENTATIONS = 73 +# The headers under include/pineforge/ whose PF_API declarations are read as one +# set for the two counts below: pineforge.h plus the two additive v1 headers, +# every symbol of which is defined in src/c_abi.cpp. native_c_api.h is not one +# of them: it is the independent inventory pinned by EXPECTED_NATIVE_C_API. +RUNTIME_HEADERS = ( + "pineforge.h", + "execution_observer.h", + "selected_window.h", +) + +EXPECTED_PUBLIC_DECLARATIONS = 100 +EXPECTED_RUNTIME_IMPLEMENTATIONS = 80 # Runtime setters whose data a stream cannot take: the source host refuses # stream_begin() while any of them holds data (the native daily feed since its @@ -124,7 +143,7 @@ }) HISTORICAL_ONLY_PHRASE = "stream_begin() fails closed" -# The C-level native host API. Additive to the 73 above: every symbol here is +# The C-level native host API. Additive to the 80 above: every symbol here is # declared in include/pineforge/native_c_api.h and implemented in # src/native_c_host.cpp, and neither file contributes to the two counts above. EXPECTED_NATIVE_C_API = frozenset({ @@ -189,13 +208,16 @@ def _pf_api_names(path: Path) -> list[str]: def main() -> int: - header = ROOT / "include" / "pineforge" / "pineforge.h" + include_dir = ROOT / "include" / "pineforge" + header = include_dir / "pineforge.h" + headers = [include_dir / name for name in RUNTIME_HEADERS] + declared_in = ", ".join(RUNTIME_HEADERS) c_abi = ROOT / "src" / "c_abi.cpp" - if not header.is_file() or not c_abi.is_file(): - print("check_c_abi_runtime: missing pineforge.h or c_abi.cpp", file=sys.stderr) + if not all(path.is_file() for path in headers) or not c_abi.is_file(): + print(f"check_c_abi_runtime: missing {declared_in} or c_abi.cpp", file=sys.stderr) return 2 - header_funcs = _pf_api_names(header) + header_funcs = [name for path in headers for name in _pf_api_names(path)] c_abi_funcs = _pf_api_names(c_abi) hdr_set = set(header_funcs) @@ -210,7 +232,8 @@ def main() -> int: return 1 if len(header_funcs) != len(hdr_set): - print("check_c_abi_runtime: duplicate PF_API lines in pineforge.h", file=sys.stderr) + print(f"check_c_abi_runtime: duplicate PF_API lines across {declared_in}", + file=sys.stderr) return 1 if len(c_abi_funcs) != len(abi_set): @@ -219,7 +242,7 @@ def main() -> int: if len(header_funcs) != EXPECTED_PUBLIC_DECLARATIONS: print( - "check_c_abi_runtime: pineforge.h PF_API declaration count " + f"check_c_abi_runtime: {declared_in} PF_API declaration count " f"{len(header_funcs)} != {EXPECTED_PUBLIC_DECLARATIONS}", file=sys.stderr, ) @@ -244,7 +267,7 @@ def main() -> int: if not abi_set <= hdr_set: print( "check_c_abi_runtime: c_abi.cpp implements PF_API symbols " - f"not declared in pineforge.h: {sorted(abi_set - hdr_set)}", + f"not declared in {declared_in}: {sorted(abi_set - hdr_set)}", file=sys.stderr, ) return 1 diff --git a/scripts/check_run_failure_codes.py b/scripts/check_run_failure_codes.py index 61a9fef90..d966084a2 100644 --- a/scripts/check_run_failure_codes.py +++ b/scripts/check_run_failure_codes.py @@ -13,9 +13,12 @@ This guard fails closed when: 1. the catalog is not canonical (sorted keys, one-space indent, a final - newline), names an unknown class or kind, a code or value outside - ^[a-z][a-z0-9_]{2,47}$ (a vocab value may also hold dots), a vocab arg - without values, or an arg kind/value list a code cannot carry; + newline), names an unknown class or kind, a code outside + ^[a-z][a-z0-9_]{2,47}$, a vocab value that is not lower snake_case + identifiers optionally joined by dots (VALUE, at most 64 characters), a + vocab arg without values, or an arg kind/value list a code cannot carry + (only a vocab arg carries values; string, nullable_string and scalar + take none); 2. a generated file differs from what the catalog generates (run `--write-registry` and commit both); 3. against the catalog of the newest release tag (`git show`), a code was @@ -89,7 +92,7 @@ CLASSES = ("strategy", "strategy_limit", "no_data", "symbol_metadata", "symbol_feeds", "input", "unsupported", "resource", "engine_fault") KINDS = ("identifier", "keyword", "vocab", "integer", "number", "pine_source", "symbol", - "timeframe") + "timeframe", "string", "nullable_string", "scalar") NAME = re.compile(r"^[a-z][a-z0-9_]{2,47}$") VALUE = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*$") CODE_FIELDS = {"class", "retryable", "args", "english", "since", "description", @@ -742,6 +745,36 @@ def diff_to(tree: dict, old: bytes | None, tag: str) -> dict: registry = render_registry(good) if '"one",' not in registry or "RunFailureArgKind::vocab, false, 0, 2" not in registry: failures.append("the registry is wrong") + # the generic text kinds and dotted vocab values + kinds_ok = json.loads(json.dumps(good)) + kinds_ok["codes"]["alpha_code"]["args"].update({ + "label": {"kind": "string"}, + "session": {"kind": "nullable_string", "optional": True}, + "value": {"kind": "scalar"}, + "path": {"kind": "vocab", "values": ["counts.window_input_bars", "plain_value"]}, + }) + found = validate_catalog(kinds_ok, canonical(kinds_ok)) + if found: + failures.append("the string, nullable_string and scalar kinds or a dotted vocab value " + "were refused: " + repr(found)) + for bad_value in ("Counts.window", "counts..window", "counts.", ".counts", "1counts", "a-b"): + bad = json.loads(json.dumps(kinds_ok)) + bad["codes"]["alpha_code"]["args"]["path"]["values"] = [bad_value] + if not validate_catalog(bad): + failures.append(f"the vocab value {bad_value!r} passed") + bad = json.loads(json.dumps(kinds_ok)) + bad["codes"]["alpha_code"]["args"]["label"]["values"] = ["x"] + if not validate_catalog(bad): + failures.append("values on a string arg passed") + bad = json.loads(json.dumps(kinds_ok)) + bad["codes"]["alpha_code"]["args"]["label"]["kind"] = "text" + if not validate_catalog(bad): + failures.append("an unknown arg kind passed") + registry = render_registry(kinds_ok) + for needle in ("RunFailureArgKind::string, false", "RunFailureArgKind::nullable_string, true", + "RunFailureArgKind::scalar, false"): + if needle not in registry: + failures.append("the registry lacks " + needle) # throw scan with tempfile.TemporaryDirectory(prefix="pf-run-failure-codes-") as temporary: root = Path(temporary) diff --git a/scripts/ci_verify.py b/scripts/ci_verify.py index f7c8762d1..1d9fa4367 100644 --- a/scripts/ci_verify.py +++ b/scripts/ci_verify.py @@ -318,7 +318,7 @@ # tape converter, the loader, the harness's syminfo-metadata transport) # Both register in release too. 300 registered, 299 run after parser removal: the WebSocket row # still skips on a system libcurl. Lane SM-E1's rows follow EXCLUDED_REGISTERED_MIN below. -KERNEL_MIN_TESTS = 305 +KERNEL_MIN_TESTS = 325 # Release-row floor, the same gate for the default profile. Before lane P7 # only the kernel profile had one, so a row that left release alone (a # source-bound TU dropped from TEST_SOURCES, a deleted twin or ABI row) left a @@ -592,7 +592,7 @@ # +1 FIX-E1E2 test_aggregated_admission_receipt_tapes (an aggregated # chart's admission receipts in their command's bar space) # No release row skips, so 792 registered is 792 run (SM-E1: below EXCLUDED_REGISTERED_MIN). -RELEASE_MIN_TESTS = 799 +RELEASE_MIN_TESTS = 895 # ADR-0001 ruled-count floors, beside the ctest floors (R5 lane H-DOCGATES, # AUDIT4-opus X13 / docs-a N7). check_kernel_residuals.py counts the rulings # its vocabulary reads -- 174 identifiers and 45 texts on the lane's tree: the @@ -641,7 +641,7 @@ # counted the same way; 773/773/782 with lane FIX-E1E2's two rows of # RELEASE_MIN_TESTS in each, counted the same way. # An excluded run must still discover at least this many rows before -LE. -EXCLUDED_REGISTERED_MIN = {'debug': 780, 'sanitizers': 779, 'native': 789} +EXCLUDED_REGISTERED_MIN = {'debug': 876, 'sanitizers': 867, 'native': 963} # Lane SM-E1 (recorded outputs, docs/outputs.md) raises the three floors # above, its comments kept here so that no line cited by line number moves. # Counted with ctest -N on the lane's tree: @@ -661,6 +661,41 @@ # EXCLUDED_REGISTERED_MIN 780/779/789 = 773/773/782 plus the seven, counted # the same way; the sanitizers profile builds no module that loads into # the interpreter, so it registers six of them. +# The 1.5.0-rc.1 selected-window backtests add fifteen rows, +# counted with ctest -N, configure only, on 873b25da and on 9f53a56ab: each +# profile configured with the definitions this file gives it, the runner-built +# ones with the pinned curl 8.14.1 install (--curl-dir) and the corpus +# submodule at its gitlink b40aa8ec, which both commits share: +# +1 test_compile_inventory_pipeline +# +1 test_request_feed_inventory_admission +# +1 test_run_execution_observer +# +1 test_run_json_selected_window +# +1 test_run_phase_transport +# +1 test_selected_primary_plan +# +1 test_selected_production_routes +# +1 test_selected_quote_seal +# +1 test_selected_report_view +# +1 test_selected_retained_count +# +1 test_selected_window_native_counts +# +1 test_selected_window_plan_c +# +1 test_selected_window_plan_python +# +1 test_selected_window_report_integration +# +1 test_selected_window_routes +# All fifteen register in release, debug, sanitizers and native; the kernel +# profile registers thirteen, every one but test_selected_quote_seal and +# test_selected_window_routes, which are source-bound. The runner's own CTest +# tree gains none, so its floor is unchanged. The other floors stood below the +# counts registered at 873b25da (kernel 312 run, release 880, +# debug/sanitizers/native 861/852/948), so each is set to its count at +# 9f53a56ab, not summed onto its old value: +# RELEASE_MIN_TESTS 895 = the 880 rows registered at 873b25da plus the +# fifteen; no release row skips, so 895 registered is 895 run. +# KERNEL_MIN_TESTS 325 run = the 313 registered at 873b25da plus the +# thirteen = 326 registered, less the WebSocket row that still skips on a +# system libcurl. +# EXCLUDED_REGISTERED_MIN 876/867/963 = the 861/852/948 registered at +# 873b25da plus the fifteen, counted before -LE (-LE slow selects +# 847/838/934). # The ctest stage's bound. A full sanitizers run (push to main, a manual # dispatch, the maintainers' verification) ran out of its 30 minutes twice on # main's four-core runner before every row had finished, so it gets an hour; a diff --git a/scripts/run_failure_throw_allowlist.txt b/scripts/run_failure_throw_allowlist.txt index 4abcef93e..09a48e1fd 100644 --- a/scripts/run_failure_throw_allowlist.txt +++ b/scripts/run_failure_throw_allowlist.txt @@ -34,3 +34,4 @@ include/pineforge/source/pine_adapter.hpp | throw; | 2 | rethrows (PlacementTabl src/source/pine_adapter.cpp | throw; | 1 | rethrow after a relative exit's materialization resets its in-progress flags: the exception from exit() keeps its own code src/source/pine_scheduler_native.cpp | throw; | 1 | rethrow after a calc_on_order_fills recalculation restores the previous bar timestamp and ends the recalculation: the script's or the engine's exception keeps its own code src/source/pine_strategy_host.cpp | throw; | 5 | rethrows: rethrow_latched_setting, a checked_settings::LatchedSettingsFailure (the generated setter refusal) that already carries a code (called from prepare_native_begin and on_native_run_begin); the coded run_options_rejected script-timeframe refusal through the catch-all around tf_ratio (prepare_native_begin); a failed projection and a failed configure, once the lent symbol-feed bytes are handed back (prepare_native_begin); a stream run's script-preparation failure, which the consumer classifies (on_native_run_begin) +src/source/pine_strategy_host.cpp | throw SelectedWindowQueryAfterSeal(); | 3 | a report-only terminal-quote write refused after the selected run sealed (set_syminfo_metadata for the two reserved quote keys, and clear_report_terminal_quote): the named refusal SelectedWindowQueryAfterSeal of include/pineforge/query_refusal.hpp, thrown before any store to the caller of the writer once the run is over, so it never reaches a run; the void C export swallows it, and it latches no run failure and changes no error, status or first cause (docs/report-terminal-quote.md) diff --git a/scripts/test_ci_verify.py b/scripts/test_ci_verify.py index ff8346e46..53eab78db 100644 --- a/scripts/test_ci_verify.py +++ b/scripts/test_ci_verify.py @@ -1049,12 +1049,16 @@ def test_public_and_runtime_export_counts_are_exact(self): EXPECTED_PUBLIC_DECLARATIONS, EXPECTED_RUNTIME, EXPECTED_RUNTIME_IMPLEMENTATIONS, + RUNTIME_HEADERS, _pf_api_names, ) - header = _pf_api_names(ROOT / 'include/pineforge/pineforge.h') + # The public count is read the way the script reads it: pineforge.h + # plus the two additive v1 headers. + header = [name for relative in RUNTIME_HEADERS + for name in _pf_api_names(ROOT / 'include/pineforge' / relative)] runtime = _pf_api_names(ROOT / 'src/c_abi.cpp') - self.assertEqual(EXPECTED_PUBLIC_DECLARATIONS, 93) - self.assertEqual(EXPECTED_RUNTIME_IMPLEMENTATIONS, 73) + self.assertEqual(EXPECTED_PUBLIC_DECLARATIONS, 100) + self.assertEqual(EXPECTED_RUNTIME_IMPLEMENTATIONS, 80) self.assertEqual(len(EXPECTED_RUNTIME), EXPECTED_RUNTIME_IMPLEMENTATIONS) self.assertEqual(len(header), EXPECTED_PUBLIC_DECLARATIONS) self.assertEqual(len(header), len(set(header))) @@ -1504,7 +1508,7 @@ def test_ctest_label_exclusion_is_forwarded(self): def test_pr_exclusion_proves_registered_minus_labelled_equals_ran(self): self.assertEqual(ci_verify.EXCLUDED_REGISTERED_MIN, - {'debug': 780, 'sanitizers': 779, 'native': 789}) + {'debug': 876, 'sanitizers': 867, 'native': 963}) for profile, registered in ci_verify.EXCLUDED_REGISTERED_MIN.items(): with self.subTest(profile=profile): code, summary, scripted, _ = self.run_profile( diff --git a/src/c_abi.cpp b/src/c_abi.cpp index 0e3872339..5ca7ea09c 100644 --- a/src/c_abi.cpp +++ b/src/c_abi.cpp @@ -17,6 +17,7 @@ * strategy_set_realtime_tail, strategy_set_probe_suppress_tail_logic, * strategy_set_path_order, strategy_last_bar_dual_entry_path, * strategy_set_broker_state_hash_recording, strategy_broker_state_hash, + * strategy_state_query_status_v1, * strategy_pending_orders_len, strategy_pending_order_get, * strategy_pending_order_layout, strategy_pending_order_fill_qty, * strategy_pending_order_level_resolved, @@ -29,7 +30,10 @@ * pf_version_get/pf_version_string, * pf_abi_version, strategy_execution_contract, * strategy_configure_native_v1, strategy_configure_native_fx_curve_v1, - * strategy_configure_native_fx_curve_ext_v1 — the + * strategy_configure_native_fx_curve_ext_v1, + * pf_execution_observer_version, strategy_set_execution_observer_v1, + * strategy_execution_observation_v1, pf_selected_window_version, + * strategy_set_selected_window_v1, strategy_selected_window_counts_v1 — the * authoritative list is EXPECTED_RUNTIME in * scripts/check_c_abi_runtime.py, enforced by CI). The other * `extern "C"` symbols listed in pineforge.h (strategy_create, @@ -39,8 +43,11 @@ // Include order is load-bearing: pineforge.h BEFORE engine.hpp keeps the // per-strategy declarations visible so definitions here are prototype-checked. -// engine.hpp defines PINEFORGE_NO_STRATEGY_DECLS, which suppresses them. +// engine.hpp defines PINEFORGE_NO_STRATEGY_DECLS, which suppresses them. The +// two v1 headers follow pineforge.h for the same reason. #include +#include +#include #include #include #include @@ -58,6 +65,16 @@ namespace pineforge { void clear_native_c_host_refusal_error(BacktestEngine* engine) noexcept; bool native_c_host_in_postrun_hook_frame(BacktestEngine* engine) noexcept; inline namespace engine_script_run_v19 { void native_stream_snapshot_report(BacktestEngine* engine, ReportC* out); } +inline namespace engine_script_run_v19 { bool native_selected_query_closed(const BacktestEngine* engine) noexcept; } +inline namespace engine_script_run_v19 { +// The private bridges behind the execution observer and selected-window +// exports at the end of this file. The native consumer defines them and owns +// every null, size, version and state check; the exports only delegate. +int native_set_execution_observer_v1(BacktestEngine*, const pf_execution_observer_v1*) noexcept; +int native_execution_observation_v1(const BacktestEngine*, pf_execution_observation_v1*) noexcept; +int native_set_selected_window_v1(BacktestEngine*, const pf_selected_window_config_v1*) noexcept; +int native_selected_window_counts_v1(const BacktestEngine*, pf_selected_window_counts_v1*) noexcept; +} } namespace { @@ -568,12 +585,35 @@ PF_API void strategy_set_broker_state_hash_recording(pf_strategy_t s, int on) { /* ABI v4 live-runtime surface (task 6): the broker-state hash of the FINAL * state after the most recent run(), regardless of whether per-bar - * recording was enabled. Returns 0 when @p s is NULL. */ + * recording was enabled. Returns 0 when @p s is NULL, and 0 for a read the + * selected-window fence refuses (strategy_state_query_status_v1 names it): + * the C++ path throws that refusal, so it is prechecked here and never + * unwinds through C. A caller that needs a sealed selected run's terminal + * witness enables recording before the run and reads the last recorded row. */ PF_API uint64_t strategy_broker_state_hash(pf_strategy_t s) { if (!s) return 0; + if (strategy_state_query_status_v1(s) != PF_STATE_QUERY_ALLOWED_V1) return 0; return static_cast(s)->broker_state_hash(); } +/* The current eligibility of both state queries on @p s (broker-state hash + * above, stream_state_hash below): the selected-window fence, read in place. + * NULL is INVALID_HANDLE, a handle whose selected window has sealed is + * SELECTED_WINDOW_AFTER_SEAL, every other handle is ALLOWED. A pure predicate + * on the handle: no allocation, no mutation, no host dispatch, and no touch of + * the last-error slot or the run's first cause. It is also the observable + * refusal of the report-only terminal-quote writers: strategy_set_syminfo_metadata + * (below) is void and pf_cabi_void swallows the SelectedWindowQueryAfterSeal the + * Pine host throws, before any store, for the two quote keys once the selected run + * has sealed, so a caller reads this status to learn that such a write stored + * nothing. */ +PF_API pf_state_query_status_v1_t strategy_state_query_status_v1(pf_strategy_t s) { + if (!s) return PF_STATE_QUERY_INVALID_HANDLE_V1; + return pineforge::native_selected_query_closed(static_cast(s)) + ? PF_STATE_QUERY_SELECTED_WINDOW_AFTER_SEAL_V1 + : PF_STATE_QUERY_ALLOWED_V1; +} + /* ABI v4 live-runtime surface (task 7, spec 3.6): the resting-order book * after the most recent run(), read through the generated POD mirror * (pf_pending_order_v1_t, include/pineforge/pending_order_mirror.hpp; @@ -715,8 +755,13 @@ PF_API void strategy_stream_order_actions_clear(pf_strategy_t s) { if (s) static_cast(s)->stream_order_actions_clear(); } +/* Returns 0 for NULL and for a read the selected-window fence refuses, the + * same precheck as strategy_broker_state_hash: the refusal is a C++ exception + * and never crosses this boundary. A broker-state row is not a stream-token + * value, so a refused read of this token leaves no recorded witness. */ PF_API uint64_t strategy_stream_state_hash(pf_strategy_t s) { if (!s) return 0; + if (strategy_state_query_status_v1(s) != PF_STATE_QUERY_ALLOWED_V1) return 0; return static_cast(s)->stream_state_hash(); } @@ -844,7 +889,12 @@ PF_API void strategy_set_syminfo_pointvalue(pf_strategy_t s, double pointvalue) /* Inject a fundamental/exchange metadata value (shares_outstanding_total, * recommendations_*, target_price_*, …) by Pine member name. Without an - * injection the corresponding syminfo.* read returns na. NULL key ignored. */ + * injection the corresponding syminfo.* read returns na. NULL key ignored. + * After a selected window's run has sealed, the Pine host refuses its two + * report-only quote keys (report_terminal_quote_time_ms, report_terminal_quote_close) + * by throwing SelectedWindowQueryAfterSeal before any store; this export keeps its + * void signature, swallows that exception and records nothing, and the refusal is + * observed through strategy_state_query_status_v1 (above). */ PF_API void strategy_set_syminfo_metadata(pf_strategy_t s, const char* key, double value) { pf_cabi_void([&] { @@ -1170,4 +1220,85 @@ PF_API int strategy_outputs_constants_copy(pf_strategy_t s, double* out, int cap }); } +/* Execution observer and selected window, version 1 (execution_observer.h and + * selected_window.h). Natural alignment, fixed-width scalars, no packing. The + * sizes are the frozen LP64 numbers, and the observer descriptor holds two + * pointers, so these pins apply to 64-bit-pointer builds only. They pin the + * installed layouts and change none: a header edit that moves a size or an + * offset fails this build. */ +#if UINTPTR_MAX == UINT64_MAX +static_assert(sizeof(pf_execution_boundary_v1) == 24, "pf_execution_boundary_v1 size moved"); +static_assert(sizeof(pf_boundary_receipt_v1) == 32, "pf_boundary_receipt_v1 size moved"); +static_assert(sizeof(pf_execution_observer_v1) == 24, "pf_execution_observer_v1 size moved"); +static_assert(sizeof(pf_execution_observation_v1) == 48, "pf_execution_observation_v1 size moved"); +static_assert(sizeof(pf_selected_window_config_v1) == 24, "pf_selected_window_config_v1 size moved"); +static_assert(sizeof(pf_selected_window_counts_v1) == 80, "pf_selected_window_counts_v1 size moved"); +static_assert(offsetof(pf_execution_boundary_v1, run_generation) == 8 + && offsetof(pf_execution_boundary_v1, attempt_serial) == 16, + "pf_execution_boundary_v1 field offset moved"); +static_assert(offsetof(pf_boundary_receipt_v1, run_generation) == 8 + && offsetof(pf_boundary_receipt_v1, frame_bytes) == 16 + && offsetof(pf_boundary_receipt_v1, handed_bytes) == 20 + && offsetof(pf_boundary_receipt_v1, export_requested) == 24 + && offsetof(pf_boundary_receipt_v1, reserved) == 28, + "pf_boundary_receipt_v1 field offset moved"); +static_assert(offsetof(pf_execution_observer_v1, context) == 8 + && offsetof(pf_execution_observer_v1, before_results) == 16, + "pf_execution_observer_v1 field offset moved"); +static_assert(offsetof(pf_execution_observation_v1, run_generation) == 8 + && offsetof(pf_execution_observation_v1, attempt_serial) == 16 + && offsetof(pf_execution_observation_v1, attempt_generation) == 24 + && offsetof(pf_execution_observation_v1, phase) == 32 + && offsetof(pf_execution_observation_v1, fault_stage) == 36 + && offsetof(pf_execution_observation_v1, attempt_outcome) == 40 + && offsetof(pf_execution_observation_v1, boundary_delivered) == 44, + "pf_execution_observation_v1 field offset moved"); +static_assert(offsetof(pf_selected_window_config_v1, start_ms) == 8 + && offsetof(pf_selected_window_config_v1, end_ms) == 16, + "pf_selected_window_config_v1 field offset moved"); +static_assert(offsetof(pf_selected_window_counts_v1, run_generation) == 8 + && offsetof(pf_selected_window_counts_v1, attempt_serial) == 16 + && offsetof(pf_selected_window_counts_v1, attempt_generation) == 24 + && offsetof(pf_selected_window_counts_v1, fed_input_bars) == 32 + && offsetof(pf_selected_window_counts_v1, fed_script_bars) == 40 + && offsetof(pf_selected_window_counts_v1, preroll_input_bars) == 48 + && offsetof(pf_selected_window_counts_v1, preroll_script_bars) == 56 + && offsetof(pf_selected_window_counts_v1, window_input_bars) == 64 + && offsetof(pf_selected_window_counts_v1, window_script_bars) == 72, + "pf_selected_window_counts_v1 field offset moved"); +#endif + +/* The version getters return 1. Every other export below hands its opaque + * handle to one private bridge, declared at the top of this file, and returns + * the bridge's code. The bridge owns every null, size, version and state check + * and the transactional output, so these exports validate, allocate, catch and + * relabel nothing and keep no state of their own. */ +PF_API uint32_t pf_execution_observer_version(void) { return 1; } + +PF_API int strategy_set_execution_observer_v1( + pf_strategy_t s, const pf_execution_observer_v1* observer) { + return pineforge::native_set_execution_observer_v1( + static_cast(s), observer); +} + +PF_API int strategy_execution_observation_v1( + pf_strategy_t s, pf_execution_observation_v1* out) { + return pineforge::native_execution_observation_v1( + static_cast(s), out); +} + +PF_API uint32_t pf_selected_window_version(void) { return 1; } + +PF_API int strategy_set_selected_window_v1( + pf_strategy_t s, const pf_selected_window_config_v1* config) { + return pineforge::native_set_selected_window_v1( + static_cast(s), config); +} + +PF_API int strategy_selected_window_counts_v1( + pf_strategy_t s, pf_selected_window_counts_v1* out) { + return pineforge::native_selected_window_counts_v1( + static_cast(s), out); +} + } /* extern "C" */ diff --git a/src/engine_internal.hpp b/src/engine_internal.hpp index ddb8852fd..31d4a9b14 100644 --- a/src/engine_internal.hpp +++ b/src/engine_internal.hpp @@ -189,6 +189,41 @@ void set_fused_settlement(bool enabled) noexcept; void count_settlement_paths(bool enabled) noexcept; SettlementPathCounts settlement_path_counts() noexcept; +// ── Report metrics (defined in engine_report.cpp) ── +// The four metric blocks of a report, computed from explicit values and no +// engine or host pointer, so one computation serves both readers of a presented +// report: BacktestEngine::fill_metrics_section, right after present_report, and +// NativeExecutionConsumer::fill_selected_report, over a captured presentation +// after the selected seal. The trade blocks (all, longs, shorts) read `trades` +// over `trades_len`; the equity block reads `equity_curve` over `equity_len` +// points -- the length the engine's equity vector had when the curve was copied +// out, not script_bars_processed -- and `net_profit` as the presentation left +// it. The blocks are written in that order, as fill_metrics_section always did. +// compute_equity_stats takes the non-recursive global timezone lock when +// `chart_timezone` is non-UTC: never call this while holding a ScopedTimezone. +void compute_report_metrics(pf_metrics_t& result, + const TradeC* trades, int trades_len, + const pf_equity_point_t* equity_curve, int64_t equity_len, + double net_profit, double initial_capital, + const std::string& chart_timezone, + double first_bar_open, double last_close, + int64_t bars_in_market); +// The same four blocks with the equity block's observation count stated +// (metrics::compute_equity_stats' final parameter): the denominator of +// time_in_market_pct and nothing else. The signature above is this one with +// observation_count = equity_len, so every ordinary report computes exactly what +// it computed. A selected-window capture is its window anchor followed by M +// script-bar observations; its reader passes equity_len = M + 1 and +// observation_count = M, both held by the consumer apart from the presented +// report (NativeExecutionConsumer::fill_selected_report). +void compute_report_metrics(pf_metrics_t& result, + const TradeC* trades, int trades_len, + const pf_equity_point_t* equity_curve, int64_t equity_len, + double net_profit, double initial_capital, + const std::string& chart_timezone, + double first_bar_open, double last_close, + int64_t bars_in_market, int64_t observation_count); + } // namespace internal } // namespace pineforge diff --git a/src/engine_metrics.cpp b/src/engine_metrics.cpp index 258d6b2a0..34faf0b53 100644 --- a/src/engine_metrics.cpp +++ b/src/engine_metrics.cpp @@ -204,7 +204,8 @@ pf_equity_stats_t compute_equity_stats(const pf_equity_point_t* curve, int64_t n double initial_capital, const std::string& chart_tz, double first_open, double last_close, - int64_t bars_in_market, double net_profit) { + int64_t bars_in_market, double net_profit, + int64_t observation_count) { pf_equity_stats_t e{}; e.sharpe_monthly = kNaN; e.sortino_monthly = kNaN; e.sharpe_bar = kNaN; e.sortino_bar = kNaN; e.cagr = kNaN; e.calmar = kNaN; e.recovery_factor = kNaN; @@ -218,7 +219,11 @@ pf_equity_stats_t compute_equity_stats(const pf_equity_point_t* curve, int64_t n if (n <= 0 || curve == nullptr) return e; e.open_pl = curve[n - 1].open_profit; - e.time_in_market_pct = (double)bars_in_market / (double)n * 100.0; + // Exposure is over observations, not curve points: a selected-window curve + // carries its window anchor as point 0 (metrics.hpp), and the anchor is no + // bar. Every other statistic below walks the n curve points. + e.time_in_market_pct = observation_count > 0 + ? (double)bars_in_market / (double)observation_count * 100.0 : kNaN; // --- Drawdown / runup walk. MUST mirror update_equity_extremes // (engine.hpp): trough resets to eq on every new peak. The walk over the @@ -303,5 +308,15 @@ pf_equity_stats_t compute_equity_stats(const pf_equity_point_t* curve, int64_t n return e; } +// The ordinary entry: every point of the curve is an observation. +pf_equity_stats_t compute_equity_stats(const pf_equity_point_t* curve, int64_t n, + double initial_capital, + const std::string& chart_tz, + double first_open, double last_close, + int64_t bars_in_market, double net_profit) { + return compute_equity_stats(curve, n, initial_capital, chart_tz, first_open, last_close, + bars_in_market, net_profit, n); +} + } // namespace metrics } // namespace pineforge diff --git a/src/engine_report.cpp b/src/engine_report.cpp index a88a6c00c..713f6669a 100644 --- a/src/engine_report.cpp +++ b/src/engine_report.cpp @@ -54,10 +54,20 @@ void BacktestEngine::trace(const std::string& name, double value) { // function asking for the rows' own owner (NativeExecutionConsumer:: // fill_snapshot_report), so a live stream's rows survive such a refusal. void BacktestEngine::fill_report(ReportC* out) const { + // A read from inside a capture or an observer call is refused before any + // gate: the empty report, latched for the terminal helper, and no recursive + // present_report. + if (NativeExecutionConsumer::refuse_report_in_dispatch(*this, out)) return; if (!NativeExecutionConsumer::rows_are_current(*this)) { *out = ReportC{}; return; } + // A selected window that has sealed, and a generation admitted with an + // observer that has sealed, answer from the captured presentation (or the + // empty report) and never reach present_report again; an OFF engine, a + // selected run still admitted-open and a generation admitted with neither a + // window nor an observer fall through unchanged. + if (NativeExecutionConsumer::fill_selected_report(*this, out)) return; fill_trades_section(out); out->input_bars_processed = diag_input_bars_processed_; @@ -128,6 +138,49 @@ void BacktestEngine::fill_trades_section(ReportC* out) const { } +namespace internal { + +// The metric blocks of a report from explicit values; see engine_internal.hpp. +// The statements are the ones fill_metrics_section ran inline after +// present_report, in the same order. +void compute_report_metrics(pf_metrics_t& result, + const TradeC* trades, int trades_len, + const pf_equity_point_t* equity_curve, int64_t equity_len, + double net_profit, double initial_capital, + const std::string& chart_timezone, + double first_bar_open, double last_close, + int64_t bars_in_market, int64_t observation_count) { + using metrics::TradeFilter; + result.all = metrics::compute_trade_stats( + trades, trades_len, TradeFilter::ALL, initial_capital); + result.longs = metrics::compute_trade_stats( + trades, trades_len, TradeFilter::LONG, initial_capital); + result.shorts = metrics::compute_trade_stats( + trades, trades_len, TradeFilter::SHORT, initial_capital); + result.equity = metrics::compute_equity_stats( + equity_curve, equity_len, initial_capital, chart_timezone, + first_bar_open, last_close, bars_in_market, + net_profit, // includes the range-end rows, like the trades + observation_count); +} + +// The ordinary entry: every point of the curve is an observation, which is the +// expression compute_equity_stats' own ordinary overload evaluates. +void compute_report_metrics(pf_metrics_t& result, + const TradeC* trades, int trades_len, + const pf_equity_point_t* equity_curve, int64_t equity_len, + double net_profit, double initial_capital, + const std::string& chart_timezone, + double first_bar_open, double last_close, + int64_t bars_in_market) { + compute_report_metrics(result, trades, trades_len, equity_curve, equity_len, + net_profit, initial_capital, chart_timezone, + first_bar_open, last_close, bars_in_market, equity_len); +} + +} // namespace internal + + // Copy the equity curve out and compute all metric blocks. Must run AFTER // fill_trades_section (reads out->trades). Owns the curve allocation; // freed by free_report (which expects new pf_equity_point_t[n]). @@ -146,17 +199,10 @@ void BacktestEngine::fill_metrics_section(ReportC* out) const { out->equity_curve = nullptr; } present_report(out); - using metrics::TradeFilter; - out->metrics.all = metrics::compute_trade_stats( - out->trades, out->trades_len, TradeFilter::ALL, initial_capital_); - out->metrics.longs = metrics::compute_trade_stats( - out->trades, out->trades_len, TradeFilter::LONG, initial_capital_); - out->metrics.shorts = metrics::compute_trade_stats( - out->trades, out->trades_len, TradeFilter::SHORT, initial_capital_); - out->metrics.equity = metrics::compute_equity_stats( - out->equity_curve, n, initial_capital_, chart_timezone_, - first_bar_open_, current_bar_.close, bars_in_market_, - out->net_profit); // includes the range-end rows, like the trades + compute_report_metrics(out->metrics, out->trades, out->trades_len, + out->equity_curve, n, out->net_profit, + initial_capital_, chart_timezone_, + first_bar_open_, current_bar_.close, bars_in_market_); } diff --git a/src/engine_state_hash.cpp b/src/engine_state_hash.cpp index 6e843b09d..170c9e1e2 100644 --- a/src/engine_state_hash.cpp +++ b/src/engine_state_hash.cpp @@ -5,6 +5,8 @@ #include "broker_state_hash_internal.hpp" #include "native_execution_consumer.hpp" +#include + namespace pineforge { // A host that overrides nothing keeps the bytes it always folded: the generic @@ -20,6 +22,12 @@ void BacktestEngine::hash_source_extension(BrokerStateHashSink& sink) const { } uint64_t BacktestEngine::broker_state_hash() const { + // The selected-window fence comes first, ahead of the projection (a + // virtual, so a host override) and of any fold: a read the window no longer + // owns is refused by name, never answered from state it left behind. + if (NativeExecutionConsumer::selected_query_closed(*this)) { + throw SelectedWindowQueryAfterSeal(); + } return broker_state_hash_projection(); } diff --git a/src/engine_stream.cpp b/src/engine_stream.cpp index dfc53ca99..f4c1b00cf 100644 --- a/src/engine_stream.cpp +++ b/src/engine_stream.cpp @@ -3,6 +3,9 @@ */ #include "engine_internal.hpp" +#include "native_execution_consumer.hpp" + +#include #include #include @@ -88,6 +91,11 @@ uint64_t BacktestEngine::stream_state_hash() const { // deliberately excludes the consumable action queue. Recovery replays the // same externally supplied input sequence into the same strategy/config/ // version and checks every step. This does not predict live executions. + // A sealed selected window refuses the read before anything is folded + // (see broker_state_hash(), which this token also folds). + if (NativeExecutionConsumer::selected_query_closed(*this)) { + throw SelectedWindowQueryAfterSeal(); + } uint64_t hash = 1469598103934665603ULL; auto bytes = [&hash](const void* data, size_t count) { const auto* p = static_cast(data); diff --git a/src/native_execution_consumer.cpp b/src/native_execution_consumer.cpp index fce22a805..1f14ad446 100644 --- a/src/native_execution_consumer.cpp +++ b/src/native_execution_consumer.cpp @@ -2,6 +2,7 @@ #include "engine_internal.hpp" #include "engine_settlement_stage.hpp" #include "native_matching.hpp" +#include "selected_report_view.hpp" #include #include @@ -1351,6 +1352,300 @@ NativeExecutionConsumer::AttemptScope::~AttemptScope() { c_.batch_current_ = !nested_ && !over_live_ && c_.rows_current_; } +namespace { + +// Holds the epoch's dispatch barrier for the length of one host call the kernel +// itself makes -- the capture's present_report and the observer -- and releases +// it on every exit, normal or unwinding, before anything is validated or +// rendered. +class EpochDispatchScope { +public: + explicit EpochDispatchScope(NativeSelectedEpoch& epoch) noexcept : epoch_(epoch) { + epoch_.dispatching = true; + } + ~EpochDispatchScope() { epoch_.dispatching = false; } + EpochDispatchScope(const EpochDispatchScope&) = delete; + EpochDispatchScope& operator=(const EpochDispatchScope&) = delete; + +private: + NativeSelectedEpoch& epoch_; +}; + +// The frozen receipt contract (execution_observer.h), judged after the +// callback returned 0 without an exception: the boundary and the receipt keep +// the size, version and generation the kernel gave them (and the boundary its +// owner serial), reserved is 0, export_requested is exactly 0 or 1, the +// callback overwrote the UINT32_MAX frame_bytes sentinel, a request for nothing +// leaves both byte counts 0, and a request for a frame is a whole frame of +// 1..1024 bytes all of which was handed over. +bool observer_receipt_accepted(const pf_execution_boundary_v1& issued, + const pf_execution_boundary_v1& boundary, + const pf_boundary_receipt_v1& receipt) noexcept { + if (boundary.struct_size != issued.struct_size || boundary.version != issued.version + || boundary.run_generation != issued.run_generation + || boundary.attempt_serial != issued.attempt_serial) { + return false; + } + if (receipt.struct_size != sizeof(pf_boundary_receipt_v1) || receipt.version != 1 + || receipt.run_generation != issued.run_generation || receipt.reserved != 0) { + return false; + } + if (receipt.export_requested == 0) { + return receipt.frame_bytes == 0 && receipt.handed_bytes == 0; + } + if (receipt.export_requested != 1) return false; + return receipt.frame_bytes >= 1 && receipt.frame_bytes <= 1024 + && receipt.handed_bytes == receipt.frame_bytes; +} + +} // namespace + +// The selected epoch's lifecycle (declared after selected_epoch_ in the header, +// with EntryExitGuard). Everything here is gated on selected_epoch_tracked(), so +// a handle that never opted in takes none of it. + +bool NativeExecutionConsumer::mint_attempt(BacktestEngine& engine, bool nested) { + if (nested || !selected_epoch_tracked()) return true; + NativeSelectedEpoch& epoch = selected_epoch_; + if (epoch.attempt_counter == std::numeric_limits::max()) { + present_refusal(engine, "native attempt serial counter exhausted"); + return false; + } + pf_execution_observation_v1& seen = epoch.observation; + seen.attempt_serial = ++epoch.attempt_counter; + seen.attempt_generation = 0; + seen.attempt_outcome = kEpochOutcomeRefused; + return true; +} + +void NativeExecutionConsumer::admit_selected_generation(BacktestEngine& engine) noexcept { + if (!selected_epoch_tracked()) return; + NativeSelectedEpoch& epoch = selected_epoch_; + // A new generation never inherits the capture, the latch or the counts of + // an older one. + drop_presentation(); + epoch.admitted_selected = epoch.window_configured; + epoch.admitted_window = epoch.admitted_selected ? epoch.pending_window + : pf_selected_window_config_v1{}; + epoch.phase_observed = epoch.observer_registered; + epoch.dispatch_reentry_refused = false; + pf_execution_observation_v1& seen = epoch.observation; + epoch.owner_serial = seen.attempt_serial; + seen.run_generation = ++epoch.generation_counter; + seen.attempt_generation = seen.run_generation; + seen.phase = kEpochExecuting; + seen.fault_stage = kEpochFaultNone; + seen.attempt_outcome = kEpochOutcomeLive; + seen.boundary_delivered = 0; + epoch.counts = pf_selected_window_counts_v1{}; + epoch.counts.struct_size = static_cast(sizeof epoch.counts); + epoch.counts.version = 1; + // The selected-only buy-and-hold facts start empty with the counts. + selected_has_window_input_ = false; + selected_first_open_ = std::numeric_limits::quiet_NaN(); + selected_last_close_ = std::numeric_limits::quiet_NaN(); + selected_query_begin_admitted(engine, epoch.admitted_selected); +} + +void NativeExecutionConsumer::close_epoch_failed(std::uint32_t fault_stage) noexcept { + pf_execution_observation_v1& seen = selected_epoch_.observation; + selected_query_seal(); + drop_presentation(); + seen.phase = kEpochSealed; + seen.fault_stage = fault_stage; + seen.boundary_delivered = 0; + // The observation's outcome row is the LATEST attempt's: a later refused + // attempt keeps its own Refused, and only the generation's phase, fault and + // boundary move. + if (seen.attempt_serial == selected_epoch_.owner_serial) { + seen.attempt_outcome = recoverable_abort() ? kEpochOutcomeAborted : kEpochOutcomeFailed; + } +} + +void NativeExecutionConsumer::close_open_epoch_at_exit(BacktestEngine& engine, bool unwinding, + bool allow_live) noexcept { + // Keyed on the generation's own open phase, never on state_: a refused + // attempt can latch NativeFailed before any admission, and must leave an + // earlier Sealed or Results generation as it was. + const std::uint32_t phase = selected_epoch_.observation.phase; + if (phase != kEpochExecuting && phase != kEpochCapturing) return; + // The one open exit: an admitted stream that returned normally still Running. + if (!unwinding && allow_live && std::holds_alternative(state_)) return; + close_epoch_failed(kEpochFaultExecution); + engine.last_run_status_ = 1; + // A close with no failure latched and no text (nothing reached fail() or + // render) still must not read as success: an empty-text record with a code + // is what failure_code_of answers as engine_unclassified_error. A failure + // that is already there -- a latched NativeFailed (an abort included), a + // code or any text -- keeps its code and text. + if (!failed() && run_failure_.code == RunFailureCode::none && engine.last_error_.empty()) { + run_failure_.code = RunFailureCode::engine_unclassified_error; + run_failure_.args.reset(); + run_failure_.text.clear(); + } +} + +void NativeExecutionConsumer::terminal_success(BacktestEngine& engine, + NativeFailureOperation operation) { + NativeSelectedEpoch& epoch = selected_epoch_; + pf_execution_observation_v1& seen = epoch.observation; + if (seen.phase == kEpochSealed && seen.fault_stage != kEpochFaultNone) { + // The generation closed failed before its run completed: a stream_end + // that was refused without failing the run left the stream Running and + // its epoch closed (the exit guard), and a retry lands here. It can open + // no results, and the call must not acknowledge success. + if (!failed()) { + fail(engine, NativeFailure{NativeFailureCode::Contract, operation}); + render(engine, "native generation was closed before its run completed"); + } + return; + } + if (seen.phase != kEpochExecuting) return; + // Only a generation with a window or an observer is captured; any other + // tracked generation just seals and its original report path remains. + const bool capture = epoch.admitted_selected || epoch.phase_observed; + if (capture) { + seen.phase = kEpochCapturing; + { + const EpochDispatchScope dispatch(epoch); + capture_presentation(engine); + } + // The barrier is released. Abort and first cause first (check_abort + // answers false for either), then the reentry latch, which is consumed + // here whatever the answer. + const bool proceed = check_abort(engine, operation); + const bool reentered = epoch.dispatch_reentry_refused; + epoch.dispatch_reentry_refused = false; + if (!proceed) { + close_epoch_failed(kEpochFaultExecution); + return; + } + if (reentered) { + fail(engine, NativeFailure{NativeFailureCode::Contract, operation}); + close_epoch_failed(kEpochFaultExecution); + render(engine, "native entry refused during the results dispatch"); + return; + } + } + selected_query_seal(); + seen.phase = kEpochSealed; + if (epoch.phase_observed && !call_execution_observer(engine, operation)) return; + seen.phase = kEpochResults; + if (seen.attempt_serial == epoch.owner_serial) seen.attempt_outcome = kEpochOutcomeResultsOpen; + if (capture) selected_results_open_ = true; +} + +bool NativeExecutionConsumer::call_execution_observer(BacktestEngine& engine, + NativeFailureOperation operation) { + NativeSelectedEpoch& epoch = selected_epoch_; + pf_execution_observation_v1& seen = epoch.observation; + const pf_before_results_fn_v1 callback = epoch.observer.before_results; + void* const context = epoch.observer.context; + + // The exact frozen initialisation: the boundary carries the serial that + // OWNS the epoch (never a later refused attempt's), and the receipt starts + // with the sentinel the callback must overwrite. + pf_execution_boundary_v1 boundary{}; + boundary.struct_size = static_cast(sizeof boundary); + boundary.version = 1; + boundary.run_generation = seen.run_generation; + boundary.attempt_serial = epoch.owner_serial; + const pf_execution_boundary_v1 issued = boundary; + pf_boundary_receipt_v1 receipt{}; + receipt.struct_size = static_cast(sizeof receipt); + receipt.version = 1; + receipt.run_generation = seen.run_generation; + receipt.frame_bytes = std::numeric_limits::max(); + receipt.handed_bytes = 0; + receipt.export_requested = 0; + receipt.reserved = 0; + + int rc = -1; + std::exception_ptr thrown; + if (callback != nullptr) { + const EpochDispatchScope dispatch(epoch); + try { + rc = callback(context, &boundary, &receipt); + } catch (...) { + thrown = std::current_exception(); + } + } + // The barrier is released; nothing below runs host code. + const bool reentered = epoch.dispatch_reentry_refused; + epoch.dispatch_reentry_refused = false; + + // The first cause wins, in the order it happened: a failure already latched, + // then a reentry the barrier refused during the call, then what the call + // itself did. `refusal` is the English of an engine_invariant refusal; + // `render_thrown` renders the callback's own exception, classified. + const char* refusal = nullptr; + bool render_thrown = false; + NativeFailure failure{NativeFailureCode::Contract, operation}; + if (failed()) { + // An earlier cause is latched: its code and text stay. + } else if (reentered) { + refusal = "native entry refused during the results dispatch"; + } else if (thrown) { + failure = NativeFailure{NativeFailureCode::CallbackException, + NativeFailureOperation::Callback}; + render_thrown = true; + } else if (rc != 0) { + failure = NativeFailure{NativeFailureCode::CallbackException, + NativeFailureOperation::Callback}; + refusal = "native execution observer returned non-zero"; + } else if (!observer_receipt_accepted(issued, boundary, receipt)) { + refusal = "native execution observer receipt rejected"; + } else { + seen.boundary_delivered = 1; + return true; + } + // Latch and close first (neither can throw), render last: a render that + // throws leaves the epoch already Sealed, faulted and without a capture. + fail(engine, failure); + close_epoch_failed(kEpochFaultAfterExecution); + if (refusal != nullptr) { + render(engine, refusal); + } else if (render_thrown) { + try { + std::rethrow_exception(thrown); + } catch (const std::exception& e) { + render_exception(engine, e); + } catch (...) { + render_unclassified(engine, "native execution observer exception"); + } + } + return false; +} + +void NativeExecutionConsumer::fail_from_exception(BacktestEngine& engine, + NativeFailureOperation operation, + const std::exception& error) { + if (selected_epoch_tracked()) { + // processing_input_ is false at every try's entry (admission refuses + // otherwise), so a throw that left it set is put back, and the + // nesting predicate of the next entry reads true again. + processing_input_ = false; + // The first latched cause stays: its code and text are not overwritten. + if (failed()) { + engine.last_run_status_ = 1; + return; + } + } + fail(engine, NativeFailure{NativeFailureCode::Unexpected, operation}); + render_exception(engine, error); +} + +void NativeExecutionConsumer::fail_unclassified(BacktestEngine& engine, + NativeFailureOperation operation) { + processing_input_ = false; + if (failed()) { + engine.last_run_status_ = 1; + return; + } + fail(engine, NativeFailure{NativeFailureCode::Unexpected, operation}); + render_unclassified(engine, "native non-standard exception"); +} + void NativeExecutionConsumer::present_refusal(BacktestEngine& engine, const char* text, RunFailureCode code, const RunFailureArgs& args) { @@ -1517,6 +1812,10 @@ bool NativeExecutionConsumer::path_high_first(const Bar& bar) const { } bool NativeExecutionConsumer::commands_allowed() const { + // A command asked for from inside a capture or an observer call is refused + // and latched, so the terminal helper rejects that generation; every + // ordinary admission below is unchanged. + if (refuse_dispatch_reentry()) return false; if (failed() || consuming_request_) return false; const auto* running = std::get_if(&state_); if (!running) return false; @@ -1697,6 +1996,18 @@ uint64_t NativeExecutionConsumer::continuation_hash() const noexcept { f.i(pre_open_birth_time_ms_); f.u(pre_open_births_.size()); for (const auto& handle : pre_open_births_) hash_handle(f, handle); + // The open activation grants fold only while one is held, a tag word first + // so their bits can never read as another optional fold's word: a run that + // never opens the scope, and every state after the Open the grants were + // recorded for, keeps its pre-scope continuation identity. The scope's own + // open bit is call-stack state and never folds. + if (!activation_births_.empty()) { + f.u(0x4f50454e41435456ULL); // "OPENACTV" + f.u(activation_birth_point_ordinal_); + f.i(activation_birth_time_ms_); + f.u(activation_births_.size()); + for (const auto& handle : activation_births_) hash_handle(f, handle); + } f.u(applied_notifications_.size() - notification_head_); for (std::size_t i = notification_head_; i < applied_notifications_.size(); ++i) { const auto& notification = applied_notifications_[i]; @@ -2492,9 +2803,22 @@ bool NativeExecutionConsumer::begin_ready(BacktestEngine& engine, NativeRunPhase RunFailureCode::symbol_metadata_rejected, {{"field", "timezone"}}); return false; } + // The selected epoch's last refusal, before anything is reset: a tracked + // handle whose generation counter is spent admits no run, so the counter + // never wraps and the previous closure stays as it was. + if (selected_epoch_tracked() + && selected_epoch_.generation_counter == std::numeric_limits::max()) { + fail(engine, + NativeFailure{NativeFailureCode::CounterExhausted, NativeFailureOperation::Begin}); + render(engine, "native run generation counter exhausted"); + return false; + } engine.reset_run_state(); // The rows are this attempt's from here on, whether it completes or fails. rows_current_ = batch_current_ = true; + // The generation is admitted here: after the reset and the last refusal, + // before the first host callback of the run. + admit_selected_generation(engine); // RULING A48: one generic capability, wired once per run. A host that // declares ownership supplies the closing-row magnitudes; the kernel then // keeps no excursion model of its own for this run. @@ -2533,6 +2857,10 @@ bool NativeExecutionConsumer::begin_ready(BacktestEngine& engine, NativeRunPhase pre_open_birth_point_ordinal_ = 0; pre_open_birth_time_ms_ = 0; pre_open_births_.clear(); + open_activation_scope_ = false; + activation_birth_point_ordinal_ = 0; + activation_birth_time_ms_ = 0; + activation_births_.clear(); applied_notifications_.clear(); notification_head_ = 0; consuming_request_ = false; @@ -2623,6 +2951,11 @@ bool NativeExecutionConsumer::begin_ready(BacktestEngine& engine, NativeRunPhase state_ = NativeRunning{std::move(ready->spec), phase}; cache_running_policy(); if (!check_abort_or_projection(engine, NativeFailureOperation::Begin)) return false; + // The host's opaque state is dropped here: after the final pre-host check has + // passed and before the run-begin callback, so state the host adopts from + // on_native_run_begin or later survives this run. A begin that returns before + // this line never reaches it. Not hashed; no switch. + host_state_.reset(); // The previous run's declared series are torn down BEFORE the host's // run-begin callback: whatever evaluator states that callback registers // -- even ones identical to the kernel's last registration, as a host @@ -4161,8 +4494,10 @@ bool NativeExecutionConsumer::install_mutation( bool NativeExecutionConsumer::apply_execution( BacktestEngine& engine, const native_order::RequestHandle& target, const native_order::ExecutionProposal& proposal, - const native_order::CommittedExecutionFacts& facts, uint64_t ordinal) { - const auto result = requests_.apply_execution(target, proposal, facts, next_timeline_ordinal_); + const native_order::CommittedExecutionFacts& facts, uint64_t ordinal, + bool open_activation_grant) { + const auto result = requests_.apply_execution(target, proposal, facts, next_timeline_ordinal_, + open_activation_grant); if (const auto* err = std::get_if(&result)) { fail(engine, NativeFailure{NativeFailureCode::Contract, NativeFailureOperation::Settlement, ordinal, @@ -4509,7 +4844,10 @@ void NativeExecutionConsumer::observe_trails( && !evaluation.cohort_side) { continue; } - if (!requests_.evaluation_eligible(*live, evaluation)) continue; + // The open activation grant of this handle at this cursor, for the + // eligibility test and the trigger transition below. + const bool open_grant = open_activation_granted(handle, cursor.point); + if (!requests_.evaluation_eligible(*live, evaluation, open_grant)) continue; const auto* track = std::get_if(&live->trigger_state); if (!track) continue; const bool buy = scratch_request_is_buy(engine, *live); @@ -4522,10 +4860,10 @@ void NativeExecutionConsumer::observe_trails( step = direct_mutation_ ? core_step(requests_.apply_trigger( handle, extremum, evaluation.driver_class, next_timeline_ordinal_, - evaluation.cohort_side, trigger_grid)) + evaluation.cohort_side, trigger_grid, open_grant)) : core_step(requests_.prepare_trigger( handle, extremum, evaluation.driver_class, next_timeline_ordinal_, - evaluation.cohort_side, trigger_grid)); + evaluation.cohort_side, trigger_grid, open_grant)); } catch (const std::exception& e) { fail(engine, NativeFailure{NativeFailureCode::Allocation, NativeFailureOperation::Settlement, cursor.point.ordinal}); @@ -4556,6 +4894,13 @@ void NativeExecutionConsumer::match_discrete(BacktestEngine& engine, const Nativ pre_open_birth_time_ms_ = 0; pre_open_births_.clear(); } + // The open activation grants end with the match of the Open they were + // recorded for, exactly as the pre-open births above do. + if (activation_birth_point_ordinal_ == point.coordinate.ordinal) { + activation_birth_point_ordinal_ = 0; + activation_birth_time_ms_ = 0; + activation_births_.clear(); + } } bool NativeExecutionConsumer::pre_open_birth_eligible( @@ -4585,6 +4930,73 @@ void NativeExecutionConsumer::record_pre_open_birth( } } +bool NativeExecutionConsumer::begin_open_activation(BacktestEngine& engine) { + // Every refusal is made before anything is written; the caller owns the + // typed failure. `current_point` is non-null exactly for a running + // consumer inside a callback with a frame, and the engine must be the one + // this run began on. Nothing here reaches the host. + const auto* point = current_point(); + // One scope per Open: the first begin at this point set the marker, even when + // no command followed, and end_open_activation leaves it, so a second begin + // at the same Open is refused here -- before anything is written -- and the + // first scope's grants stay for the match of this Open, which clears the + // marker with them (match_discrete; begin_ready resets both for a new run). + // A later Open has another ordinal and begins normally. + if (open_activation_scope_ || point == nullptr || &engine != downcast_engine_ + || callback_phase_ != CallbackPhase::PreOpen + || point->decision.coordinate.path_phase != NativePathPhase::Open + || activation_birth_point_ordinal_ == point->decision.coordinate.ordinal) { + return false; + } + // The grants of any earlier scope end here; the point is the one the open + // match will present, so the helper can match it exactly. + activation_births_.clear(); + activation_birth_point_ordinal_ = point->decision.coordinate.ordinal; + activation_birth_time_ms_ = point->decision.coordinate.effective_time_ms; + open_activation_scope_ = true; + return true; +} + +void NativeExecutionConsumer::end_open_activation() noexcept { + // The capture scope alone. The grants it recorded stay for the Open match + // that follows (match_discrete clears them at its end). + open_activation_scope_ = false; +} + +bool NativeExecutionConsumer::open_activation_granted( + const native_order::RequestHandle& handle, const NativeCoordinate& at) const noexcept { + if (activation_births_.empty() || activation_birth_point_ordinal_ != at.ordinal + || activation_birth_time_ms_ != at.effective_time_ms + || at.path_phase != NativePathPhase::Open) { + return false; + } + return std::find(activation_births_.begin(), activation_births_.end(), handle) + != activation_births_.end(); +} + +void NativeExecutionConsumer::record_activation_birth( + BacktestEngine& engine, const native_order::RequestHandle& handle) { + if (!open_activation_scope_) return; + if (std::find(activation_births_.begin(), activation_births_.end(), handle) + != activation_births_.end()) { + return; + } + try { + activation_births_.push_back(handle); + } catch (const std::exception& e) { + fail(engine, NativeFailure{NativeFailureCode::Allocation, NativeFailureOperation::Command}); + render_exception(engine, e); + throw; + } +} + +void NativeExecutionConsumer::revoke_activation_birth(const native_order::RequestHandle& handle) { + if (activation_births_.empty()) return; + activation_births_.erase( + std::remove(activation_births_.begin(), activation_births_.end(), handle), + activation_births_.end()); +} + void NativeExecutionConsumer::match_segment( BacktestEngine& engine, const NativeDriverPoint& dest, double from_price) { if (!match_quiet_point(engine, dest, true, dest.raw_price)) @@ -5502,6 +5914,12 @@ std::optional NativeExecutionConsumer::consume_mat native_order::ExecutionProposal proposal; proposal.cursor = evaluation.cursor; proposal.pre_open_birth_eligible = evaluation.pre_open_birth_eligible; + // The handle's open activation grant at this cursor, read once: the + // direct form's check and its apply, and the prepared form's prepare, + // each make the birth test, and each is handed this same bool (the + // token holds none: install_execution never reads the birth gate). A + // current execution's cursor is a point of its own, so it reads false. + const bool open_grant = open_activation_granted(handle, evaluation.cursor.point); proposal.raw_price = raw_price; proposal.resolved_price = resolved_price; proposal.physical_action = candidate.physical; @@ -5527,7 +5945,8 @@ std::optional NativeExecutionConsumer::consume_mat return error.code == native_order::CoreFailure::NonrepresentableQuantity; }; if (direct_mutation_) { - auto checked = requests_.check_execution(handle, proposal, next_timeline_ordinal_); + auto checked = requests_.check_execution(handle, proposal, next_timeline_ordinal_, + open_grant); if (const auto* error = std::get_if(&checked)) { if (unrepresentable(*error)) { return terminal(native_order::MatchRejectReason::UnrepresentableQuantity, @@ -5538,7 +5957,8 @@ std::optional NativeExecutionConsumer::consume_mat } ready = std::holds_alternative(checked); } else { - auto prepared = requests_.prepare_execution(handle, proposal, next_timeline_ordinal_); + auto prepared = requests_.prepare_execution(handle, proposal, next_timeline_ordinal_, + open_grant); if (const auto* error = std::get_if(&prepared)) { if (unrepresentable(*error)) { return terminal(native_order::MatchRejectReason::UnrepresentableQuantity, @@ -5707,7 +6127,7 @@ std::optional NativeExecutionConsumer::consume_mat committed.post_target = std::move(candidate.target); committed.committed_action = candidate.physical; const bool installed = direct_mutation_ - ? apply_execution(engine, handle, proposal, committed, P) + ? apply_execution(engine, handle, proposal, committed, P, open_grant) : install_execution(engine, std::move(*token), committed, P); if (!installed) return std::nullopt; const auto& applied = std::get( @@ -6296,7 +6716,8 @@ void NativeExecutionConsumer::match_path( candidate_eval.pre_open_birth_eligible = pre_open_birth_eligible(handle, point) || born_on_remaining_path(*live); candidate_eval.cohort_side = cohort_side; - const auto facts = requests_.eligibility_facts(*live, candidate_eval); + const auto facts = requests_.eligibility_facts( + *live, candidate_eval, open_activation_granted(handle, point.coordinate)); if (!facts.birth_ok || facts.waiting || !facts.driver_ok) { erase_provenance_for(handle); continue; @@ -6425,6 +6846,10 @@ void NativeExecutionConsumer::match_path( if (!live) continue; eval.pre_open_birth_eligible = pre_open_birth_eligible(winner->handle, point) || born_on_remaining_path(*live); + // The winner's open activation grant, read once for every core call + // this row makes below (refresh, evaluation, trigger, execution): the + // same bool each time, so a check and its apply never disagree. + const bool open_grant = open_activation_granted(winner->handle, point.coordinate); const auto* winner_target = cached_cohort_target(engine, *live); eval.cohort_side = winner_target ? side_from_target(*winner_target) : scratch_cohort_side(engine, *live); @@ -6438,14 +6863,18 @@ void NativeExecutionConsumer::match_path( const bool reuse_after_refresh = keep_rows && cursor_unmoved; try { if (winner_target) { - if (requests_.refresh_allowance(winner->handle, eval, *winner_target)) { + // The cohort refresh: the core's context-bearing form under + // the grant (refresh_cohort_allowance forwards to the same + // body without one). + if (requests_.refresh_allowance(winner->handle, eval, *winner_target, + open_grant)) { rescan_winner_only = reuse_after_refresh; continue; } } else if (std::holds_alternative(live->authority)) { native_order::TargetObservation obs; obs.current_position = read_position(engine); - if (requests_.refresh_allowance(winner->handle, eval, obs)) { + if (requests_.refresh_allowance(winner->handle, eval, obs, open_grant)) { rescan_winner_only = reuse_after_refresh; continue; } @@ -6455,7 +6884,7 @@ void NativeExecutionConsumer::match_path( // binding the book still has binds here, taking an // ordinal (ReplaceOptions::keep_binding, R5 lane V19-D). if (requests_.refresh_allowance(winner->handle, eval, match_target_, - next_timeline_ordinal_)) { + next_timeline_ordinal_, open_grant)) { catch_up_timeline(); rescan_winner_only = reuse_after_refresh; continue; @@ -6476,9 +6905,9 @@ void NativeExecutionConsumer::match_path( winner_target ? *winner_target : match_target_; step = direct_mutation_ ? core_step(requests_.apply_evaluation( - winner->handle, eval, observed, next_timeline_ordinal_)) + winner->handle, eval, observed, next_timeline_ordinal_, open_grant)) : core_step(requests_.prepare_evaluation( - winner->handle, eval, observed, next_timeline_ordinal_)); + winner->handle, eval, observed, next_timeline_ordinal_, open_grant)); } catch (const std::exception& e) { fail(engine, NativeFailure{NativeFailureCode::Allocation, NativeFailureOperation::Settlement, P}); @@ -6528,10 +6957,11 @@ void NativeExecutionConsumer::match_path( step = direct_mutation_ ? core_step(requests_.apply_trigger(winner->handle, transition, driver_class, next_timeline_ordinal_, eval.cohort_side, - trigger_grid)) + trigger_grid, open_grant)) : core_step(requests_.prepare_trigger(winner->handle, transition, driver_class, next_timeline_ordinal_, - eval.cohort_side, trigger_grid)); + eval.cohort_side, trigger_grid, + open_grant)); } catch (const std::exception& e) { fail(engine, NativeFailure{NativeFailureCode::Allocation, NativeFailureOperation::Settlement, P}); @@ -6959,6 +7389,24 @@ NativeCurrentExecutionResult NativeExecutionConsumer::execute_current( || !check_projection(engine, NativeFailureOperation::Command)) throw coded(recorded_cause(), "native current execution projection/abort failure"); + // The open activation scope (begin_open_activation) is a capture window, not + // a point to execute at: a current execution inside it would take an ordinal + // and settle against a book the window has not finished building. That is + // the caller's sequencing violation, not a refusal it may ignore, so the run + // fails and the call throws, before an ordinal is taken or the book is + // touched. A cause the run already carries stays the first: it is not + // rendered over, and it is the one the thrown error carries. + if (open_activation_scope_) { + const bool first_cause = !failed(); + fail(engine, NativeFailure{NativeFailureCode::Contract, NativeFailureOperation::Command}); + if (first_cause) { + render(engine, + "native current execution is refused while the open activation scope is open", + RunFailureCode::engine_invariant); + } + throw coded( + recorded_cause(), "native current execution inside the open activation scope"); + } try { if (auto refusal = validate_current_execution(engine, command)) return *refusal; consuming_request_ = true; @@ -7571,6 +8019,9 @@ void NativeExecutionConsumer::invoke_callback(BacktestEngine& engine, const Bar& if (callback_context_.sub_index < 0) callback_context_.sub_index = 0; if (callback_context_.sub_bar_open_ms == 0) callback_context_.sub_bar_open_ms = coordinate.open_ms; if (callback_context_.script_bar_open_ms == 0) callback_context_.script_bar_open_ms = coordinate.open_ms; + // The label this script bar is published under, read before the host runs: + // the selected window's script counts split on it. + const std::int64_t published_label = callback_context_.script_bar_open_ms; // The quote kind and origin are the view's defaults, as they were. current_frame_.emplace(callback_context_, bar.close, NativeCurrentQuoteKind::MarketDecision, uint64_t{0}, next_timeline_ordinal_ - 1); @@ -7608,6 +8059,7 @@ void NativeExecutionConsumer::invoke_callback(BacktestEngine& engine, const Bar& return; } ++engine.diag_script_bars_processed_; + count_selected_script(published_label); calculation_margin_check(engine, coordinate, bar.close); // L9: the third risk evaluation point is the script bar's own close // calculation, inside this still-open frame, so a breach that first @@ -8524,6 +8976,61 @@ bool NativeExecutionConsumer::final_script_session_closed() const noexcept { >= script_.interval.last_traded_close_ms; } +// The selected window's consumption counts (declared with the seal helpers). +// The flag load is all an OFF handle pays. The admission reset in +// admit_selected_generation zeroes the counts per generation and the counts +// getter alone attaches the three identities, so nothing here is copied from a +// planner's expectation: each increment is one primary input or script callback +// that actually completed. The window starts at T, the row or label at or after +// it is window, any before it pre-roll. +void NativeExecutionConsumer::count_selected_input(const Bar& bar) noexcept { + if (!selected_epoch_.admitted_selected) return; + pf_selected_window_counts_v1& counts = selected_epoch_.counts; + ++counts.fed_input_bars; + if (bar.timestamp < selected_epoch_.admitted_window.start_ms) { + ++counts.preroll_input_bars; + } else { + ++counts.window_input_bars; + // The selected report's buy-and-hold basis: the first window input's + // open and the latest window input's close, never a pre-roll price. + if (!selected_has_window_input_) { + selected_has_window_input_ = true; + selected_first_open_ = bar.open; + } + selected_last_close_ = bar.close; + } +} + +void NativeExecutionConsumer::count_selected_script(std::int64_t label) noexcept { + if (!selected_epoch_.admitted_selected) return; + pf_selected_window_counts_v1& counts = selected_epoch_.counts; + ++counts.fed_script_bars; + if (label < selected_epoch_.admitted_window.start_ms) { + ++counts.preroll_script_bars; + } else { + ++counts.window_script_bars; + } +} + +// The accepted pre-roll horizon. A pre-T script bucket is sealed by its own +// last input or by the first input of a later interval; a feed that stops +// short of the interval's end leaves the last one pending, so the source would +// never see its final pre-roll intent (no window row at all), or only after the +// first window input's own work. This seals it once at the accepted T from the +// real aggregate its own inputs left, LazyComplete as the native lazy seal +// does, and resets the bucket: the calculation reads no later input, price, +// fill or order, a natural seal that already ran left nothing to seal here, and +// a second call finds an empty bucket. Only a bucket whose label precedes T is +// completed; a WINDOW bucket that is still incomplete is never forced. +bool NativeExecutionConsumer::complete_selected_preroll_horizon(BacktestEngine& engine) { + if (!selected_epoch_.admitted_selected || !script_.has_data || script_.sealed) return true; + if (script_.interval.open_ms >= selected_epoch_.admitted_window.start_ms) return true; + seal_script(engine, NativeCompletionKind::LazyComplete); + if (failed()) return false; + script_ = ScriptBucket{}; + return true; +} + bool NativeExecutionConsumer::contribute_input( BacktestEngine& engine, const Bar& bar, const native_calendar::NativeInterval& interval, @@ -8589,8 +9096,10 @@ bool NativeExecutionConsumer::contribute_input( engine.bar_index_ = script_.has_data ? script_.script_index : std::max(0, next_script_index_ - 1); next_interval_index_ = index + 1; - if (!failed() && kind != InputContribution::QuietCarried) + if (!failed() && kind != InputContribution::QuietCarried) { ++engine.diag_input_bars_processed_; + count_selected_input(bar); + } return !failed(); } @@ -9553,6 +10062,20 @@ bool NativeExecutionConsumer::consume_confirmed_input(BacktestEngine& engine, co input_context.input_index = index; input_context.completes_script_interval = interval->next_period_open_ms >= script_interval->next_period_open_ms; + // The accepted pre-roll horizon of a selected batch: the first input at or + // after T finds the last pre-T bucket still pending when its own inputs + // stopped short of the script interval's end. It is completed here, ahead + // of this input's callback, bar open and matching work, so the source has + // produced its last pre-roll intent first. A bucket this very input would + // continue (the same script key) is never split, and a stream is untouched. + if (selected_epoch_.admitted_selected && running + && running->phase == NativeRunPhase::Batch + && bar.timestamp >= selected_epoch_.admitted_window.start_ms + && script_.has_data && script_.key != script_interval->open_ms + && !complete_selected_preroll_horizon(engine)) { + processing_input_ = false; + return false; + } if (!invoke_input_callback(engine, bar, input_context)) { processing_input_ = false; return false; @@ -9725,12 +10248,24 @@ void NativeExecutionConsumer::pump_batch(BacktestEngine& engine, const Bar* bars if (failed()) return; script_ = ScriptBucket{}; } + // The accepted pre-roll horizon of a selected batch: with no later + // input to seal it, the last pending pre-T bucket is completed now, + // before the terminal capture, once the natural final seal above has + // had its say. An incomplete window bucket stays unsealed. + if (running && running->phase == NativeRunPhase::Batch + && selected_epoch_.admitted_selected + && !complete_selected_preroll_horizon(engine)) { + return; + } } (void)check_abort_or_projection(engine, NativeFailureOperation::Input); } void NativeExecutionConsumer::run_simple(BacktestEngine& engine, const Bar* bars, int n) { + const EntryExitGuard exit_scope(*this, engine, false); + if (refuse_dispatch_reentry()) return; const AttemptScope attempt(*this); + if (!mint_attempt(engine, exit_scope.nested())) return; NativeBeginArgs args{bars, n, {}, {}, false, 4, MagnifierDistribution::ENDPOINTS, engine.magnifier_volume_weighted_, 2}; args.simple_run = true; @@ -9753,9 +10288,12 @@ void NativeExecutionConsumer::run_simple(BacktestEngine& engine, const Bar* bars state_ = NativeCompleted{std::move(running->spec), NativeCompletion::BatchComplete}; verify_closed_rows(engine); + terminal_success(engine, NativeFailureOperation::Input); } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Input}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Input, e); + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Input); } } @@ -9765,7 +10303,10 @@ void NativeExecutionConsumer::run_tf(BacktestEngine& engine, const std::string& script_tf, bool bar_magnifier, int magnifier_samples, MagnifierDistribution magnifier_dist) { + const EntryExitGuard exit_scope(*this, engine, false); + if (refuse_dispatch_reentry()) return; const AttemptScope attempt(*this); + if (!mint_attempt(engine, exit_scope.nested())) return; const NativeBeginArgs args{input_bars, n_input, input_tf, script_tf, bar_magnifier, magnifier_samples, magnifier_dist, engine.magnifier_volume_weighted_, 2}; if (!prepare_public_begin(engine, args)) return; @@ -9801,9 +10342,12 @@ void NativeExecutionConsumer::run_tf(BacktestEngine& engine, state_ = NativeCompleted{std::move(running->spec), NativeCompletion::BatchComplete}; verify_closed_rows(engine); + terminal_success(engine, NativeFailureOperation::Input); } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Input}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Input, e); + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Input); } } @@ -9815,7 +10359,10 @@ void NativeExecutionConsumer::run_rich(BacktestEngine& engine, const void* overrides, bool bar_magnifier, int magnifier_samples, MagnifierDistribution magnifier_dist) { + const EntryExitGuard exit_scope(*this, engine, false); + if (refuse_dispatch_reentry()) return; const AttemptScope attempt(*this); + if (!mint_attempt(engine, exit_scope.nested())) return; NativeBeginArgs args{input_bars, n_input, input_tf, script_tf, bar_magnifier, magnifier_samples, magnifier_dist, engine.magnifier_volume_weighted_, 2}; args.inputs = &inputs; @@ -9846,9 +10393,12 @@ void NativeExecutionConsumer::run_rich(BacktestEngine& engine, state_ = NativeCompleted{std::move(running->spec), NativeCompletion::BatchComplete}; verify_closed_rows(engine); + terminal_success(engine, NativeFailureOperation::Input); } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Input}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Input, e); + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Input); } } @@ -9856,7 +10406,13 @@ bool NativeExecutionConsumer::stream_begin(BacktestEngine& engine, const Bar* warmup_bars, int n_warmup, const std::string& input_tf, const std::string& script_tf) { + const EntryExitGuard exit_scope(*this, engine, true); + if (refuse_dispatch_reentry()) return false; const AttemptScope attempt(*this); + // The latest attempt is minted before any check below, so a refusal over a + // live stream is a later attempt of its own (owner_serial stays the live + // generation's). + if (!mint_attempt(engine, exit_scope.nested())) return false; // Preserve the live stream before asking the provider to stage/configure a // new run. The legacy route diagnoses this state first; in particular, // no warmup copy, adapter reset, or broker/spec mutation may occur. @@ -9964,13 +10520,18 @@ bool NativeExecutionConsumer::stream_begin(BacktestEngine& engine, } return true; } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Stream}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Stream, e); + return false; + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Stream); return false; } } bool NativeExecutionConsumer::stream_push_bar(BacktestEngine& engine, const Bar& bar) { + const EntryExitGuard exit_scope(*this, engine, true); + if (refuse_dispatch_reentry()) return false; if (!admit_public_stream_input(engine, NativeFailureOperation::Input)) return false; clear_rendered(engine); engine.last_run_status_ = 0; @@ -9997,8 +10558,11 @@ bool NativeExecutionConsumer::stream_push_bar(BacktestEngine& engine, const Bar& if (!check_abort_or_projection(engine, NativeFailureOperation::Input)) return false; return !failed(); } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Input}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Input, e); + return false; + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Input); return false; } } @@ -10390,6 +10954,8 @@ bool NativeExecutionConsumer::deliver_tick(BacktestEngine& engine, const TradeTi } bool NativeExecutionConsumer::stream_push_tick(BacktestEngine& engine, const TradeTick& tick) { + const EntryExitGuard exit_scope(*this, engine, true); + if (refuse_dispatch_reentry()) return false; if (!admit_public_stream_input(engine, NativeFailureOperation::Input)) return false; clear_rendered(engine); engine.last_run_status_ = 0; @@ -10403,13 +10969,18 @@ bool NativeExecutionConsumer::stream_push_tick(BacktestEngine& engine, const Tra return check_abort_or_projection(engine, NativeFailureOperation::Input); } catch (const std::exception& e) { processing_input_ = false; - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Input}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Input, e); + return false; + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Input); return false; } } bool NativeExecutionConsumer::stream_push_ticks(BacktestEngine& engine, const TradeTick* ticks, int n) { + const EntryExitGuard exit_scope(*this, engine, true); + if (refuse_dispatch_reentry()) return false; if (!admit_public_stream_input(engine, NativeFailureOperation::Input)) return false; clear_rendered(engine); engine.last_run_status_ = 0; @@ -10427,13 +10998,18 @@ bool NativeExecutionConsumer::stream_push_ticks(BacktestEngine& engine, const Tr return !failed(); } catch (const std::exception& e) { processing_input_ = false; - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Input}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Input, e); + return false; + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Input); return false; } } bool NativeExecutionConsumer::stream_advance_time(BacktestEngine& engine, int64_t timestamp_ms) { + const EntryExitGuard exit_scope(*this, engine, true); + if (refuse_dispatch_reentry()) return false; if (!admit_public_stream_input(engine, NativeFailureOperation::Stream)) return false; clear_rendered(engine); engine.last_run_status_ = 0; @@ -10473,13 +11049,18 @@ bool NativeExecutionConsumer::stream_advance_time(BacktestEngine& engine, int64_ raise_floor(timestamp_ms); return !failed(); } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Stream}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Stream, e); + return false; + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Stream); return false; } } bool NativeExecutionConsumer::stream_end(BacktestEngine& engine, bool finalize_partial_input_bar) { + const EntryExitGuard exit_scope(*this, engine, false); + if (refuse_dispatch_reentry()) return false; if (!admit_public_stream_input(engine, NativeFailureOperation::Stream)) return false; clear_rendered(engine); engine.last_run_status_ = 0; @@ -10522,10 +11103,14 @@ bool NativeExecutionConsumer::stream_end(BacktestEngine& engine, bool finalize_p state_.emplace(NativeCompleted{std::move(spec), NativeCompletion::StreamEnded}); engine.stream_phase_ = BacktestEngine::StreamPhase::IDLE; verify_closed_rows(engine); + terminal_success(engine, NativeFailureOperation::Stream); return !failed(); } catch (const std::exception& e) { - fail(engine, NativeFailure{NativeFailureCode::Unexpected, NativeFailureOperation::Stream}); - render_exception(engine, e); + fail_from_exception(engine, NativeFailureOperation::Stream, e); + return false; + } catch (...) { + if (!selected_epoch_tracked()) throw; + fail_unclassified(engine, NativeFailureOperation::Stream); return false; } } @@ -10572,7 +11157,12 @@ native_order::SubmitResult NativeExecutionConsumer::submit_with_surface( note_install(ok.events); if (ok.result.status == native_order::SubmitStatus::Accepted) { ++engine.next_order_incarnation_; - if (ok.result.handle) record_pre_open_birth(request, *ok.result.handle); + if (ok.result.handle) { + record_pre_open_birth(request, *ok.result.handle); + // Under the open activation scope every accepted request is + // granted, whatever its trigger kind. + record_activation_birth(engine, *ok.result.handle); + } } return std::move(ok.result); } @@ -10640,6 +11230,14 @@ native_order::ReplaceResult NativeExecutionConsumer::replace_with_surface( pre_open_births_.erase(std::remove(pre_open_births_.begin(), pre_open_births_.end(), target), pre_open_births_.end()); } + // A replace is a new definition and a renewed birth, so a replace that + // lands outside the open activation scope ends the grant its old handle + // held, a kept handle's included. Inside the scope the old handle keeps + // its grant and the resulting handle is recorded below. A refused replace + // changed nothing, so it leaves the grant as it was. + if (predicted_status == native_order::ReplaceStatus::Replaced && !open_activation_scope_) { + revoke_activation_birth(target); + } if (predicted_status == native_order::ReplaceStatus::Replaced && ok.result.successor) { const auto* successor = requests_.find_live(*ok.result.successor); if (successor && std::holds_alternative(successor->authority)) { @@ -10659,7 +11257,10 @@ native_order::ReplaceResult NativeExecutionConsumer::replace_with_surface( // A re-price takes the number a successor would have been issued, so // the counter moves either way. ++engine.next_order_incarnation_; - if (ok.result.successor) record_pre_open_birth(request, *ok.result.successor); + if (ok.result.successor) { + record_pre_open_birth(request, *ok.result.successor); + record_activation_birth(engine, *ok.result.successor); + } // The request a re-price kept is still the parent its waiting // children wait on. if (!kept_handle) { @@ -10746,6 +11347,9 @@ native_order::CancelResult NativeExecutionConsumer::cancel( const auto predicted_status = ok.result.status; note_install(ok.events); if (predicted_status == native_order::CancelStatus::Cancelled) { + // A cancelled request is a dead handle: its open activation grant, if + // it held one, goes with it. + revoke_activation_birth(target); try { drain_parent_terminal(engine, predicted, target, NativeFailureOperation::Command); } catch (const std::exception& e) { @@ -11536,6 +12140,508 @@ void native_stream_snapshot_report(BacktestEngine* engine, ReportC* out) { NativeExecutionConsumer::fill_snapshot_report(*engine, out); } +// The selected-window query fence (include/pineforge/query_refusal.hpp). The +// slot's boolean keeps its inline bodies and canonical values: true is no +// selected-run provenance (the constructor's value, and every OFF path's), false +// is selected provenance, and a selected value is query-open only while its +// current consumer holds admitted-open evidence. The predicate reads the slot +// and that one flag in place, so it allocates nothing and no host can see or +// steer it; a null pointer (a copy, which discards the consumer) is closed. +bool NativeExecutionConsumer::selected_query_closed(const BacktestEngine& engine) noexcept { + const auto& slot = engine.execution_consumer_slot_; + if (slot.native) return false; + const auto* consumer = static_cast(slot.ptr.get()); + return consumer == nullptr || !consumer->selected_open_; +} + +// Called on the engine's own bound consumer, after admission has accepted the +// run and before its first execution hook: the slot's provenance and this +// consumer's evidence change together, here and nowhere else. +void NativeExecutionConsumer::selected_query_begin_admitted(BacktestEngine& engine, + bool selected) noexcept { + engine.execution_consumer_slot_.native = !selected; + selected_open_ = selected; +} + +// The terminal seal (Completed, Failed or Aborted): the evidence closes and the +// provenance stays, so a selected engine keeps reading closed. +void NativeExecutionConsumer::selected_query_seal() noexcept { + selected_open_ = false; +} + +// The C exports' read of the fence, declared in c_abi.cpp beside +// native_stream_snapshot_report, in the engine epoch. +bool native_selected_query_closed(const BacktestEngine* engine) noexcept { + return NativeExecutionConsumer::selected_query_closed(*engine); +} + +// The destructor frees the presentation capture, the only state this consumer +// owns outside its members' own destructors. +NativeExecutionConsumer::~NativeExecutionConsumer() { + drop_presentation(); +} + +// Frees through BacktestEngine::free_report, which deletes exactly what +// capture_presentation (and fill_trades_section before it) allocated, then +// zeroes what free_report leaves (the scalars, the metrics, the lengths of +// arrays that were never allocated) and the kept metric equity length, and +// closes both flags. +void NativeExecutionConsumer::drop_presentation() noexcept { + BacktestEngine::free_report(&presentation_); + presentation_ = ReportC{}; + presentation_metric_equity_len_ = 0; + presentation_metric_observations_ = 0; + presentation_ready_ = false; + selected_results_open_ = false; +} + +// A selected capture the engine's own records contradict is a call the engine +// got wrong -- a coded engine_invariant, the way a broken recorder contract is +// coded -- and not a strategy refusal. The five sites in capture_presentation +// (a trade or an open lot entered before T, a pre-T observation that is not the +// untouched account, observations that are not one run, a count that differs +// from the native window script count) use it until the final catalog supplies +// specialized codes for them. +[[noreturn]] static void refuse_selected_capture(const char* text) { + throw coded(RunFailureCode::engine_invariant, {}, text); +} + +// The input BacktestEngine::present_report takes, and nothing past it: the same +// trades section, nine diagnostic scalars and equity copy, in the order +// fill_report and fill_metrics_section write them, then the one virtual call. A +// local report holds the work, so a throw (an allocation, or the host's own +// override) frees it and rethrows with presentation_ as it was; only the final +// move, which cannot throw, replaces the capture. The equity length the metrics +// will read is fill_metrics_section's `n`, taken before the one virtual call and +// held in a local the callback cannot reach; it is published beside the capture +// after the last step that can throw, never read back from the report. +// +// A selected generation takes the second form of the equity copy below, and +// nothing else changes order: the trades, the nine scalars, the curve, then the +// one hook. The curve is the window's own, built from the engine's recorded +// observations before the hook and never cut out of what the host presents: the +// anchor at T with the initial capital, then the observations labelled in +// [T, E) exactly as they were recorded (report::build_selected_equity_curve), +// whose one new[] array is handed to the local report by release() and freed +// the way any report's curve is, by BacktestEngine::free_report. Before the +// build the engine's own records are held to the window, and a record that +// contradicts it is refused (refuse_selected_capture), never filtered into an +// apparent success: a trade or open lot entered before T, a pre-T observation +// that is not the untouched account, an observation run that is not contiguous, +// or an observation count that is not the native window script count. +void NativeExecutionConsumer::capture_presentation(BacktestEngine& engine) { + ReportC local{}; + int64_t metric_equity_len = 0; + int64_t metric_observations = 0; + try { + engine.fill_trades_section(&local); + + local.input_bars_processed = engine.diag_input_bars_processed_; + local.script_bars_processed = engine.diag_script_bars_processed_; + local.magnifier_sub_bars_total = engine.diag_magnifier_sub_bars_processed_; + local.magnifier_sample_ticks_total = engine.diag_magnifier_sample_ticks_processed_; + local.input_tf_seconds = tf_to_seconds(engine.input_tf_); + local.script_tf_seconds = engine.script_tf_seconds_; + local.script_tf_ratio = engine.diag_script_tf_ratio_; + local.needs_aggregation = engine.diag_needs_aggregation_ ? 1 : 0; + local.bar_magnifier_enabled = engine.bar_magnifier_enabled_ ? 1 : 0; + + if (selected_epoch_.admitted_selected) { + const int64_t window_start = selected_epoch_.admitted_window.start_ms; + const int64_t window_end = selected_epoch_.admitted_window.end_ms; + const std::vector& recorded = engine.equity_curve_; + // No position, fill or account change may precede T: a trade row + // (closed, or the range-end row of an open lot) or an open lot + // entered before the window is a carry-in. + for (int i = 0; i < local.trades_len; ++i) { + if (local.trades[i].entry_time < window_start) + refuse_selected_capture( + "native selected report: a trade was entered before the window start"); + } + for (const auto& lot : engine.pyramid_entries_) { + if (lot.time < window_start) + refuse_selected_capture( + "native selected report: a position was carried in before the window start"); + } + // The observations labelled in [T, E), in recorded order, are one + // contiguous run; every earlier one is the untouched account. + std::size_t first = 0; + int64_t observed = 0; + bool contiguous = true; + for (std::size_t i = 0; i < recorded.size(); ++i) { + const pf_equity_point_t& point = recorded[i]; + if (point.time_ms < window_start) { + if (point.equity != engine.initial_capital_ || point.open_profit != 0.0) + refuse_selected_capture( + "native selected report: the account changed before the window start"); + } else if (point.time_ms < window_end) { + if (observed == 0) first = i; + else if (i != first + static_cast(observed)) contiguous = false; + ++observed; + } + } + if (!contiguous) + refuse_selected_capture( + "native selected report: the window's equity observations are not one run"); + // The native script count and the observations are two records of the + // same bars, and must agree. + if (static_cast(observed) != selected_epoch_.counts.window_script_bars) + refuse_selected_capture( + "native selected report: the window's equity observations differ from its script bars"); + report::SelectedEquityCurve curve = report::build_selected_equity_curve( + window_start, engine.initial_capital_, + observed > 0 ? recorded.data() + first : nullptr, observed); + metric_equity_len = curve.count; + metric_observations = observed; + local.equity_curve_len = curve.count; + // One transfer, after the last step that can throw: from here the + // local report owns the array and a throw below frees it. + local.equity_curve = curve.points.release(); + } else { + const int64_t n = (int64_t)engine.equity_curve_.size(); + metric_equity_len = n; + metric_observations = n; + local.equity_curve_len = n; + if (n > 0) { + local.equity_curve = new pf_equity_point_t[n]; + std::copy(engine.equity_curve_.begin(), engine.equity_curve_.end(), + local.equity_curve); + } else { + local.equity_curve = nullptr; + } + } + engine.present_report(&local); + } catch (...) { + BacktestEngine::free_report(&local); + throw; + } + drop_presentation(); + presentation_ = local; + presentation_metric_equity_len_ = metric_equity_len; + presentation_metric_observations_ = metric_observations; + presentation_ready_ = true; +} + +// The report read's reentry barrier (declared beside rows_are_current): asked +// before the row gate. The slot is read in place and nothing is created; a read +// from inside a capture or an observer call is refused whole, latched for the +// terminal helper, and never reaches present_report again. +bool NativeExecutionConsumer::refuse_report_in_dispatch(const BacktestEngine& engine, + ReportC* out) noexcept { + const auto* consumer = static_cast( + engine.execution_consumer_slot_.ptr.get()); + if (consumer == nullptr || !consumer->selected_epoch_.dispatching) return false; + consumer->selected_epoch_.dispatch_reentry_refused = true; + *out = ReportC{}; + return true; +} + +// A generation admitted with an observer that has closed: Sealed (it failed, or +// the observer or the receipt did) or Results. Its boundary is behind it, so a +// report read must never dispatch the host's present_report again, selected +// window or not. +bool NativeExecutionConsumer::observed_epoch_sealed() const noexcept { + const std::uint32_t phase = selected_epoch_.observation.phase; + return selected_epoch_.phase_observed && (phase == kEpochSealed || phase == kEpochResults); +} + +// The capture may be published only for the generation that opened its results: +// phase Results with no fault, the capture ready and its results open. A +// failed, aborted or observer-faulted generation is Sealed and holds neither. +bool NativeExecutionConsumer::capture_readable() const noexcept { + return presentation_ready_ && selected_results_open_ + && selected_epoch_.observation.phase == kEpochResults + && selected_epoch_.observation.fault_stage == kEpochFaultNone; +} + +// The selected-window report (declared beside rows_are_current). The slot is +// read in place, as selected_query_closed reads it, and no consumer is ever +// created or asked for: a copied value has none and reads the empty report. +// Two engines are served from here: one with closed selected provenance, and one +// with no provenance (the OFF value of the slot) whose latest generation was +// admitted with an observer and has closed -- observer-only, whose state-hash +// queries stay open. Everything else, a handle that never opted in and a +// generation admitted with neither a window nor an observer included, returns +// false and reports exactly as it always did. +bool NativeExecutionConsumer::fill_selected_report(const BacktestEngine& engine, + ReportC* out) { + const auto& slot = engine.execution_consumer_slot_; + const auto* consumer = static_cast(slot.ptr.get()); + if (slot.native) { + if (consumer == nullptr || !consumer->observed_epoch_sealed()) return false; + } else if (!selected_query_closed(engine)) { + return false; + } + if (consumer == nullptr || !consumer->capture_readable()) { + *out = ReportC{}; + return true; + } + + // The capture's scalars come across by value; every array pointer starts + // null, so what a throw frees below is only what this call allocated and + // never the capture's own. The two arrays the capture owns are copied; the + // others are the suffix's to build, as fill_report builds them after + // present_report. + const ReportC& captured = consumer->presentation_; + // Two equity lengths, as OFF has them: the presented one + // (captured.equity_curve_len, whatever present_report left, published as it + // stands) and the metrics' own, the engine vector's size from before that + // call, kept beside the capture. OFF's buffer holds the second and a reader + // may take either, so the copy covers the larger. + const int64_t metric_equity_len = consumer->presentation_metric_equity_len_; + const int64_t equity_storage_len = std::max(captured.equity_curve_len, metric_equity_len); + ReportC local = captured; + local.trades = nullptr; + local.security_diag = nullptr; + local.trace = nullptr; + local.trace_names = nullptr; + local.equity_curve = nullptr; + local.broker_state_hash = nullptr; + try { + if (captured.trades != nullptr && captured.trades_len > 0) { + local.trades = new TradeC[captured.trades_len]; + std::copy(captured.trades, captured.trades + captured.trades_len, local.trades); + } + if (captured.equity_curve != nullptr && equity_storage_len > 0) { + local.equity_curve = new pf_equity_point_t[equity_storage_len]; + std::copy(captured.equity_curve, captured.equity_curve + equity_storage_len, + local.equity_curve); + } + + // The suffix, in fill_report's order. The metrics get the original + // length, as fill_metrics_section passes its `n`: never the presented + // length, script_bars_processed, or the engine's vector read now. + if (consumer->selected_epoch_.admitted_selected) { + // A selected window's own view: the curve is M + 1 points (the + // anchor and M observations) and exposure divides by the M the + // capture held, both from before the presentation; buy-and-hold + // runs from the first window input's open to the latest window + // input's close, undefined (NaN) while the window has none. The + // trade blocks and every other equity formula are the ordinary ones. + internal::compute_report_metrics(local.metrics, local.trades, local.trades_len, + local.equity_curve, metric_equity_len, + local.net_profit, engine.initial_capital_, + engine.chart_timezone_, + consumer->selected_first_open_, + consumer->selected_last_close_, + engine.bars_in_market_, + consumer->presentation_metric_observations_); + } else { + internal::compute_report_metrics(local.metrics, local.trades, local.trades_len, + local.equity_curve, metric_equity_len, + local.net_profit, engine.initial_capital_, + engine.chart_timezone_, engine.first_bar_open_, + engine.current_bar_.close, engine.bars_in_market_); + } + engine.fill_security_diag_section(&local); + engine.fill_trace_section(&local); + + const int64_t hn = (int64_t)engine.broker_state_hashes_.size(); + local.broker_state_hash_len = hn; + local.broker_state_hash = hn > 0 ? new uint64_t[hn] : nullptr; + if (hn > 0) { + std::copy(engine.broker_state_hashes_.begin(), engine.broker_state_hashes_.end(), + local.broker_state_hash); + } + } catch (...) { + BacktestEngine::free_report(&local); + throw; + } + *out = local; + return true; +} + +// The observer and selected-window accessors (declared beside +// fill_selected_report; pineforge/execution_observer.h and selected_window.h +// are their C faces). This is the storage and the four bridges, not the +// lifecycle: nothing here mints a counter, moves a phase, admits a run, +// captures, seals or calls an observer, so a consumer answers as a fresh one +// does until the lifecycle writes selected_epoch_. +// +// Whether a setter must refuse as nonquiescent: the barriers the begin and +// attempt paths already keep, the epoch's own dispatch flag, and the persisted +// phases that say a run is Executing (1) or Capturing (2). A Sealed (3) or +// Results (4) phase alone is not busy: that generation is over, and a setter +// writes only the pending state. A refusal on this answer is an ordinary +// accessor refusal -- it latches no failure and rewrites no outcome. +bool NativeExecutionConsumer::selected_epoch_busy() const noexcept { + return preparing_begin_ || in_callback_ || processing_input_ || in_attempt_call_ + || running_spec_ != nullptr || std::holds_alternative(state_) + || selected_epoch_.dispatching + || selected_epoch_.observation.phase == 1 || selected_epoch_.observation.phase == 2; +} + +// The setters' gate (declared with selected_epoch_). The consumer is looked up +// in the slot first; only a descriptor to store may bind a missing one, through +// the engine's own lazy path, and a throw there leaves the slot as it was. +int NativeExecutionConsumer::gate_selected_setter(BacktestEngine* engine, bool bind_missing, + NativeExecutionConsumer** out) noexcept { + *out = nullptr; + if (engine->execution_contract() != PF_EXECUTION_CONTRACT_NATIVE_MARKET_V1) return -2; + auto* consumer = + static_cast(engine->execution_consumer_slot_.ptr.get()); + if (consumer == nullptr) { + if (!bind_missing) return 0; + try { + consumer = static_cast(&engine->execution_consumer()); + } catch (...) { + return -4; + } + } + if (consumer->selected_epoch_busy()) return -3; + *out = consumer; + return 0; +} + +// Only a descriptor of exactly this layout is copied (size and version are read +// first), the copy is judged whole, and the stored descriptor is replaced by one +// assignment of that same copy after every refusal has been passed, so a refusal +// leaves the registration as it was. A null descriptor stores a zero one and +// clears the flag; it never touches phase_observed. +int NativeExecutionConsumer::set_execution_observer_v1( + BacktestEngine* engine, const pf_execution_observer_v1* observer) noexcept { + if (engine == nullptr) return -1; + pf_execution_observer_v1 next{}; + if (observer != nullptr) { + if (observer->struct_size != sizeof(pf_execution_observer_v1) || observer->version != 1) + return -1; + next = *observer; + if (next.before_results == nullptr) return -1; + } + NativeExecutionConsumer* consumer = nullptr; + const int refusal = gate_selected_setter(engine, observer != nullptr, &consumer); + if (refusal != 0) return refusal; + // No consumer and nothing to store: a clear of nothing, accepted as it is. + if (consumer == nullptr) return 0; + consumer->selected_epoch_.observer = next; + consumer->selected_epoch_.observer_registered = observer != nullptr; + return 0; +} + +// The slot is read in place: a missing consumer reads as a fresh one, and no +// consumer is created. The snapshot is built in a local and assigned once. +int NativeExecutionConsumer::execution_observation_v1( + const BacktestEngine* engine, pf_execution_observation_v1* out) noexcept { + if (engine == nullptr || out == nullptr) return -1; + if (out->struct_size != sizeof(pf_execution_observation_v1) || out->version != 1) return -1; + pf_execution_observation_v1 local{}; + local.struct_size = static_cast(sizeof local); + local.version = 1; + const auto* consumer = + static_cast(engine->execution_consumer_slot_.ptr.get()); + if (consumer != nullptr) local = consumer->selected_epoch_.observation; + *out = local; + return 0; +} + +// Same discipline as the observer setter, with the window's structural bounds +// (calendar alignment and supplied feeds are the harness's preflight). Only T +// and E are taken from the caller; the stored descriptor's header is the native +// one. Clearing never touches admitted_selected. +int NativeExecutionConsumer::set_selected_window_v1( + BacktestEngine* engine, const pf_selected_window_config_v1* config) noexcept { + if (engine == nullptr) return -1; + pf_selected_window_config_v1 next{}; + if (config != nullptr) { + if (config->struct_size != sizeof(pf_selected_window_config_v1) || config->version != 1) + return -1; + // JSON-safe integer milliseconds, T < E: 2^53 - 1 is the largest integer + // a double, and so a JSON number, holds exactly. + constexpr std::int64_t kJsonSafeMs = 9007199254740991LL; + const std::int64_t start_ms = config->start_ms; + const std::int64_t end_ms = config->end_ms; + if (start_ms < -kJsonSafeMs || start_ms >= end_ms || end_ms > kJsonSafeMs) return -1; + next.struct_size = static_cast(sizeof next); + next.version = 1; + next.start_ms = start_ms; + next.end_ms = end_ms; + } + NativeExecutionConsumer* consumer = nullptr; + const int refusal = gate_selected_setter(engine, config != nullptr, &consumer); + if (refusal != 0) return refusal; + if (consumer == nullptr) return 0; + consumer->selected_epoch_.pending_window = next; + consumer->selected_epoch_.window_configured = config != nullptr; + return 0; +} + +// Latest-attempt access, not the explicit stream snapshot. The counts are +// readable only while the latest attempt owns successful selected results: the +// configuration was admitted, the capture is ready and open, the latest call +// began, and the observation reads phase Results with outcome ResultsOpen for a +// generation that is both the latest and this attempt's, under the serial that +// owns the epoch. Anything else is -2 with the caller's output untouched. The +// three identities are overwritten from the same observation that was judged, +// so the snapshot is one. +int NativeExecutionConsumer::selected_window_counts_v1( + const BacktestEngine* engine, pf_selected_window_counts_v1* out) noexcept { + if (engine == nullptr || out == nullptr) return -1; + if (out->struct_size != sizeof(pf_selected_window_counts_v1) || out->version != 1) return -1; + const auto* consumer = + static_cast(engine->execution_consumer_slot_.ptr.get()); + if (consumer == nullptr) return -2; + const NativeSelectedEpoch& epoch = consumer->selected_epoch_; + const pf_execution_observation_v1& seen = epoch.observation; + if (!epoch.admitted_selected || !consumer->selected_results_open_ + || !consumer->presentation_ready_ || !consumer->batch_current_ + || seen.phase != 4 || seen.attempt_outcome != 4 + || seen.run_generation == 0 || seen.attempt_generation != seen.run_generation + || seen.attempt_serial != epoch.owner_serial) { + return -2; + } + pf_selected_window_counts_v1 local = epoch.counts; + local.run_generation = seen.run_generation; + local.attempt_serial = seen.attempt_serial; + local.attempt_generation = seen.attempt_generation; + *out = local; + return 0; +} + +// The source adapter's reads (declared with the accessors above). The slot is +// read in place; the configuration is copied only when its flag says it exists. +bool NativeExecutionConsumer::selected_window_pending( + const BacktestEngine& engine, pf_selected_window_config_v1* out) noexcept { + const auto* consumer = + static_cast(engine.execution_consumer_slot_.ptr.get()); + if (out == nullptr || consumer == nullptr || !consumer->selected_epoch_.window_configured) + return false; + *out = consumer->selected_epoch_.pending_window; + return true; +} + +bool NativeExecutionConsumer::selected_window_admitted( + const BacktestEngine& engine, pf_selected_window_config_v1* out) noexcept { + const auto* consumer = + static_cast(engine.execution_consumer_slot_.ptr.get()); + if (out == nullptr || consumer == nullptr || !consumer->selected_epoch_.admitted_selected) + return false; + *out = consumer->selected_epoch_.admitted_window; + return true; +} + +// The C exports' bridges, in the engine epoch for c_abi.cpp to declare as it +// does native_selected_query_closed. The shims validate, allocate and catch +// nothing: all of it is above, so no exception crosses C. +int native_set_execution_observer_v1(BacktestEngine* engine, + const pf_execution_observer_v1* observer) noexcept { + return NativeExecutionConsumer::set_execution_observer_v1(engine, observer); +} + +int native_execution_observation_v1(const BacktestEngine* engine, + pf_execution_observation_v1* out) noexcept { + return NativeExecutionConsumer::execution_observation_v1(engine, out); +} + +int native_set_selected_window_v1(BacktestEngine* engine, + const pf_selected_window_config_v1* config) noexcept { + return NativeExecutionConsumer::set_selected_window_v1(engine, config); +} + +int native_selected_window_counts_v1(const BacktestEngine* engine, + pf_selected_window_counts_v1* out) noexcept { + return NativeExecutionConsumer::selected_window_counts_v1(engine, out); +} + } // inline namespace engine_script_run_v19 // The portable run-spec digest lives here, in the one translation unit that diff --git a/src/native_execution_consumer.hpp b/src/native_execution_consumer.hpp index 7ec496cd5..3905cd75e 100644 --- a/src/native_execution_consumer.hpp +++ b/src/native_execution_consumer.hpp @@ -5,10 +5,12 @@ #include #include "native_calendar_memo.hpp" +#include "native_selected_epoch.hpp" #include "runtime_ambient.hpp" #include #include +#include #include #include #include @@ -44,6 +46,30 @@ class NativeHostCache { private: const void* kind_ = nullptr; }; +// Opaque run state a host cannot rebuild by scanning, parked with the consumer +// that serves it. Unlike NativeHostCache, which only shortcuts a scan the host +// can repeat, the slot is mandatory: no switch, no fallback, and the host +// allocates before it adopts, so an allocation failure surfaces in the host and +// never silently drops the state. The consumer owns it through an ordinary +// unique_ptr (no callback), drops it at every admitted run begin, and never +// reads, hashes or records it: a host whose state is trading state folds it +// itself. +class NativeHostState { +public: + // `kind` names the host's state type, so the host recognizes its own state + // with one comparison instead of a dynamic_cast. Any address the host owns + // will do; the consumer never reads it. + explicit NativeHostState(const void* kind) noexcept : kind_(kind) {} + NativeHostState(const NativeHostState&) = delete; + NativeHostState& operator=(const NativeHostState&) = delete; + NativeHostState(NativeHostState&&) = delete; + NativeHostState& operator=(NativeHostState&&) = delete; + virtual ~NativeHostState() noexcept = default; + const void* kind() const noexcept { return kind_; } + +private: + const void* kind_; +}; // The journal window's stress switch (NativeExecutionConsumer:: // set_retire_every_point): a build compiled with // -DPINEFORGE_STRESS_RETIRE_EVERY_POINT starts every consumer with it on, so a @@ -517,6 +543,47 @@ class NativeExecutionConsumer final : public IExecutionConsumer { host_cache_enabled_ = enabled; if (!enabled) host_cache_.reset(); } + // The host's opaque state (NativeHostState), or null. A host adopts one per + // run; it lives until the next adoption, the consumer's destruction, or the + // reset in begin_ready. That reset runs only after the final pre-host + // abort/projection check has passed, and before on_native_run_begin, so a + // host adopts from the run-begin hook or later. A state adopted earlier, + // including in prepare_native_begin, is dropped by that reset if the begin + // reaches it. Independent of set_host_cache: no switch, no fallback, and + // neither call allocates (the host allocates first, so a failure surfaces + // there). The consumer never reads, hashes or records it. + NativeHostState* host_state() const noexcept { return host_state_.get(); } + NativeHostState* adopt_host_state(std::unique_ptr state) noexcept { + host_state_ = std::move(state); + return host_state_.get(); + } + // The open activation scope: a window inside one PreOpen callback, at an + // Open current frame, in which every request a successful submit or replace + // leaves live -- of any trigger kind -- is granted its activation at that + // Open. The grant is explicit, per handle and per point: it lives from the + // command that recorded it through the immediately following match of that + // exact Open (match_discrete clears it at the match's end), and the helper + // that reads it checks the exact handle, the point's ordinal and effective + // time and the Open phase, so it never applies at a later point. A + // successful replace outside the scope revokes the replaced handle's grant, + // a kept handle's included; a refused replace leaves it. Nothing but this + // consumer holds it: every core call that tests a birth gate is handed it + // afresh as a trailing bool (open_activation_granted answers it), and no + // installed member, vtable or epoch carries one. + // + // begin_open_activation answers false, and changes nothing, for anything + // but a running consumer inside its own PreOpen callback at an Open current + // frame, and for a scope already open: the caller turns that false into its + // typed failure. It resets the grants of any earlier scope. The three + // methods never dispatch to the host. end_open_activation closes the + // capture scope and only that: the grants it recorded stay for the Open + // match that follows. While the scope is open execute_current latches a + // Contract failure of the Command operation (engine_invariant) and throws, + // before it takes an ordinal or touches the book: a sequencing violation, + // so that no caller can carry on past it. + bool begin_open_activation(BacktestEngine& engine); + void end_open_activation() noexcept; + bool open_activation_active() const noexcept { return open_activation_scope_; } // V19-B: under NativeEventRetention::Window the journal retires what its // host acknowledged at every script-bar boundary. This switch retires at // every driver point instead -- the most any reader can lose -- so a run @@ -1078,6 +1145,23 @@ class NativeExecutionConsumer final : public IExecutionConsumer { bool seal_stale_script(BacktestEngine& engine, std::int64_t script_key); void seal_script(BacktestEngine& engine, NativeCompletionKind kind); bool final_script_session_closed() const noexcept; + // The selected window's consumption counts (pineforge/selected_window.h): + // facts of the primary input and script callbacks that actually completed, + // written into selected_epoch_.counts beside the two engine diagnostics they + // mirror, and only for an admitted selected generation (an OFF handle reads + // one flag and does nothing else). The input splits on the row's own + // timestamp against T, the script bar on its published label against T. + // Report bookkeeping like selected_epoch_: never hashed, never read by the + // run, and no setter or host can reach them. + void count_selected_input(const Bar& bar) noexcept; + void count_selected_script(std::int64_t label) noexcept; + // The accepted pre-roll horizon of a selected batch: seals the last pending + // script bucket whose label precedes T once, from the real aggregate its own + // inputs left (LazyComplete, the native lazy seal), and resets it. A window + // bucket is never forced and a natural seal that already ran is never + // repeated. True when nothing was pending or the sealed calculation + // succeeded. + bool complete_selected_preroll_horizon(BacktestEngine& engine); void deliver_confirmed_script(BacktestEngine& engine, const Bar& bar, const NativeCoordinate& base); void deliver_intrabar_script(BacktestEngine& engine, const Bar& bar, const NativeCoordinate& base); @@ -1437,10 +1521,12 @@ class NativeExecutionConsumer final : public IExecutionConsumer { uint64_t ordinal); // install_execution for the core's direct form (R5 lane L3): the second // half of check_execution, after the settlement, failing the run exactly - // as an install that fails does. + // as an install that fails does. `open_activation_grant` is the one the + // check was handed: the apply makes the same birth test. bool apply_execution(BacktestEngine& engine, const native_order::RequestHandle& target, const native_order::ExecutionProposal& proposal, - const native_order::CommittedExecutionFacts& facts, uint64_t ordinal); + const native_order::CommittedExecutionFacts& facts, uint64_t ordinal, + bool open_activation_grant); native_order::SubmitResult submit_with_surface( BacktestEngine& engine, const native_order::Request& request, native_order::CommandSurface surface); @@ -1545,6 +1631,19 @@ class NativeExecutionConsumer final : public IExecutionConsumer { bool pre_open_birth_eligible(const native_order::RequestHandle&, const NativeDriverPoint&) const noexcept; void record_pre_open_birth(const native_order::Request&, const native_order::RequestHandle&); + // The open activation grant of one handle at one point (begin_open_ + // activation): true only for a handle recorded under the scope, at the + // exact Open point the scope began at -- ordinal, effective time and phase. + // It is a separate list from pre_open_births_ and leaves that one's + // Market/Immediate meaning alone. The bool every core call that tests a + // birth gate is handed; false whenever no scope ever recorded a handle. + bool open_activation_granted(const native_order::RequestHandle&, + const NativeCoordinate&) const noexcept; + // Records the handle a submit or replace left live while the scope is open, + // whatever its trigger kind; a no-op outside the scope. + void record_activation_birth(BacktestEngine&, const native_order::RequestHandle&); + // Drops one handle's grant; a no-op for a handle holding none. + void revoke_activation_birth(const native_order::RequestHandle&); void note_committed_events(const native_order::EventRange& events) noexcept; void note_cohort_receipts() noexcept; void verify_closed_rows(const BacktestEngine& engine) const noexcept; @@ -1606,6 +1705,14 @@ class NativeExecutionConsumer final : public IExecutionConsumer { uint64_t pre_open_birth_point_ordinal_ = 0; int64_t pre_open_birth_time_ms_ = 0; std::vector pre_open_births_; + // The open activation scope (begin_open_activation). open_activation_scope_ + // is call-stack state, never folded; the point and the handles recorded + // under the scope are durable until the Open's match ends, and fold into + // the continuation hash only while a handle is held. + bool open_activation_scope_ = false; + uint64_t activation_birth_point_ordinal_ = 0; + int64_t activation_birth_time_ms_ = 0; + std::vector activation_births_; std::vector applied_notifications_; std::size_t notification_head_ = 0; bool processing_input_ = false; @@ -1937,6 +2044,327 @@ class NativeExecutionConsumer final : public IExecutionConsumer { bool settlement_carry_ = true; uint64_t carried_settlements_ = 0; uint64_t settlement_generation_ = 0; + +public: + // The selected-window query fence (pineforge/query_refusal.hpp): whether + // `engine` must refuse broker_state_hash(), stream_state_hash() and the two + // C exports that wrap them. It reads the engine's execution slot and, for a + // slot with selected provenance (`native` false), the open flag below -- + // nothing else: no bound(), no execution_consumer() (which would create a + // consumer), no virtual, no hook, no allocation. `native` true is no + // selected-run provenance and is never closed. A selected engine is closed + // unless its current consumer holds positive open evidence, so a copy, which + // loses the consumer, reads closed and never reopens as OFF. Query and + // report-control eligibility only; no hash folds it. + static bool selected_query_closed(const BacktestEngine& engine) noexcept; + + // The reentry barrier of a report read, asked by BacktestEngine::fill_report + // BEFORE its rows_are_current gate. A report read from inside a capture or + // an observer call (the epoch's `dispatching` is held) is refused: the + // helper latches dispatch_reentry_refused, writes the empty report whole and + // answers true, so no present_report recurses and the first cause stands. It + // answers false for every other read, including every OFF engine and a + // copied value. It reads the slot's pointer in place, allocates nothing and + // never creates a consumer. The observation getter is not a report read and + // stays allowed inside the dispatch. + static bool refuse_report_in_dispatch(const BacktestEngine& engine, + ReportC* out) noexcept; + + // The selected-window report, asked by BacktestEngine::fill_report right + // after its rows_are_current gate. False when the caller must fill the report + // exactly as before: an OFF engine, a selected run still admitted-open, and a + // handle whose latest generation was admitted with neither a window nor an + // observer. True when the generation behind the engine is closed -- a + // selected window that has sealed, or a generation admitted with an observer + // (a window or not) that has sealed -- with `*out` written whole: the empty + // report (every array NULL, every count zero, never freed) unless the + // engine's own bound consumer holds a captured presentation whose results are + // open, which a failed, aborted or observer-faulted generation never does, so + // no present_report runs after any boundary. Then the capture is deep-copied + // into a local report, only the output-only suffix runs over it (metrics, + // security diagnostics, trace, recorded hashes: engine code that is handed + // no host pointer and dispatches nothing virtual), and the finished report + // is published by one struct assignment, so a throw frees the local and + // leaves `*out` alone. Repeated reads reuse the capture. It reads the slot's + // pointer in place and never creates a consumer: a copied value, which lost + // its consumer, reads empty. + static bool fill_selected_report(const BacktestEngine& engine, ReportC* out); + + // The private bridges behind the four C accessors of + // pineforge/execution_observer.h and pineforge/selected_window.h. The free + // functions native_set_execution_observer_v1, native_execution_observation_v1, + // native_set_selected_window_v1 and native_selected_window_counts_v1 of this + // epoch delegate here, so the C shims neither allocate nor catch. Each one + // validates its raw pointers, struct_size and version before it reads + // another field or changes anything, and answers -1 for a failure of that + // kind: a null engine or output, a wrong size or version, a null + // before_results, or window bounds outside T < E within +-(2^53 - 1). + // + // A getter reads the engine's execution slot in place: it creates no + // consumer, calls no host code, builds its answer in a local and writes the + // caller's output once, whole, on success; a refusal leaves that output + // alone. execution_observation_v1 answers a fresh snapshot (struct_size and + // version set, everything else zero) when no consumer is bound, and is + // allowed inside a dispatch. selected_window_counts_v1 answers 0 only while + // the latest attempt owns successful selected results, and -2 otherwise, + // with no consumer bound included. + // + // A setter answers -2 for an execution contract other than the 2 that + // BacktestEngine::execution_contract() reports, -3 for a consumer that is + // busy (selected_epoch_busy) and -4 when a descriptor needs a consumer and + // none can be allocated. It copies or clears only its pending descriptor and + // registration flag (never phase_observed or admitted_selected), and a + // refusal changes nothing: the first cause, the run failure record and the + // slot's provenance included. A null descriptor with no consumer bound is + // accepted without allocating one. No exception leaves any of them. + static int set_execution_observer_v1(BacktestEngine* engine, + const pf_execution_observer_v1* observer) noexcept; + static int execution_observation_v1(const BacktestEngine* engine, + pf_execution_observation_v1* out) noexcept; + static int set_selected_window_v1(BacktestEngine* engine, + const pf_selected_window_config_v1* config) noexcept; + static int selected_window_counts_v1(const BacktestEngine* engine, + pf_selected_window_counts_v1* out) noexcept; + + // The source adapter's read of the selected window, never a C export: the + // pending configuration (what the next admitted generation will take, read + // at prepare_native_begin) and the admitted one (what the running generation + // took, read inside the adapter's callbacks). Each reads the slot's pointer + // in place and copies the configuration to *out only when it exists, + // answering true; a null `out`, no consumer or an unset configuration + // answers false and leaves *out alone. No allocation, callback, mint, + // validation or mutation, and a true answer does not make a run admitted. + static bool selected_window_pending(const BacktestEngine& engine, + pf_selected_window_config_v1* out) noexcept; + static bool selected_window_admitted(const BacktestEngine& engine, + pf_selected_window_config_v1* out) noexcept; + + // Frees the presentation capture (drop_presentation). + ~NativeExecutionConsumer() override; + +private: + // The lifecycle primitives, called only on `engine`'s own bound consumer. + // begin_admitted runs once admission has accepted the run and before its + // first execution hook, writes the slot's provenance (`native` is + // !selected) and this consumer's open flag together, and is never reached + // by a refused begin or configure. seal runs when the run ends Completed, + // Failed or Aborted: it clears the flag and leaves provenance alone, so a + // selected engine reads closed from then on. + void selected_query_begin_admitted(BacktestEngine& engine, bool selected) noexcept; + void selected_query_seal() noexcept; + // Positive evidence that this consumer's selected run is open. Never + // hashed, never a broker or trading-state input. Declared last so no member + // the consumer reads at every point changes offset. + bool selected_open_ = false; + // The presentation capture of a selected run. A selected success tail calls + // capture_presentation once, before the seal: it builds a local report with + // the trades, the nine diagnostic scalars and the equity copy -- the input + // BacktestEngine::present_report takes, in the order fill_report gives it -- + // calls that virtual exactly once, frees the local if anything throws, and + // moves it into presentation_ only on success (replacing, and so freeing, any + // earlier capture), together with the equity length it took before that call + // (presentation_metric_equity_len_). Nothing after present_report is + // captured: the metrics, security diagnostics, trace and recorded hashes are + // output-only and run after the seal, in fill_selected_report. The terminal + // helper (terminal_success) is its one caller, under the epoch's dispatch + // barrier. + // + // For a selected generation the equity copy is the window's own curve, built + // here before the one presentation hook: the engine's recorded observations + // labelled in [T, E), in the order they were recorded, behind the anchor + // (report::build_selected_equity_curve), handed to the local report by one + // owned-array transfer. Their count M must equal the native window script + // count, and a position, fill or account change before T is a failure, never + // filtered out; each is a coded engine_invariant thrown before the hook. + void capture_presentation(BacktestEngine& engine); + // Frees and zeroes the capture and its metric equity length and closes both + // flags. The destructor calls it, and the selected lifecycle will at + // admission, so a new generation never inherits an old capture. + void drop_presentation() noexcept; + // The capture. Its arrays are the ones BacktestEngine::free_report deletes + // (new TradeC[] / new pf_equity_point_t[]), and it is freed only through it. + ReportC presentation_{}; + // The equity length the metrics read: the engine's equity vector size taken + // before present_report ran (fill_metrics_section's `n`), which a + // presentation may leave behind in presentation_.equity_curve_len, the length + // a report reader sees. Published with the capture, only once the capture + // succeeded, and zeroed with it; the assembly never reads the engine's vector + // for it. A selected capture's is M + 1: the window anchor and its M + // observations. Its observation count, presentation_metric_observations_, + // is declared with the selected report facts at the end of the members. + int64_t presentation_metric_equity_len_ = 0; + bool presentation_ready_ = false; + // Whether the capture may be read: granted only after the execution observer + // has validated, and only by the lifecycle that owns the seal. Report-control + // state like selected_open_: never hashed, never a broker or trading-state + // input. Declared last, like the members above, so no member the consumer + // reads at every point changes offset. + bool selected_results_open_ = false; + // The setters' shared gate (set_execution_observer_v1 and + // set_selected_window_v1), run once their descriptors validated: -2 for an + // execution contract other than 2, then -3 when a bound consumer is busy. + // With no consumer bound it binds one through the engine's own + // execution_consumer() only when `bind_missing` (a descriptor is to be + // stored), answering -4 if that allocation throws; without it the gate + // answers 0 and leaves *out null, so a clear of nothing allocates nothing. + // On 0 a nonnull *out is the consumer to write. Nothing is written here. + static int gate_selected_setter(BacktestEngine* engine, bool bind_missing, + NativeExecutionConsumer** out) noexcept; + // Whether a setter must refuse as nonquiescent: the consumer is preparing a + // begin, inside a callback or an input, in an attempt call, running, held + // by a dispatch, or last observed Executing or Capturing (phase 1 or 2). A + // persisted Sealed or Results phase alone is not busy. Reads only. + bool selected_epoch_busy() const noexcept; + // The observer and selected-window bookkeeping (native_selected_epoch.hpp). + // Report and control state like selected_open_: never hashed, never a broker + // or trading-state input. Declared last, like the members above, so no + // member the consumer reads at every point changes offset. + NativeSelectedEpoch selected_epoch_{}; + // The host's opaque state (adopt_host_state): dropped by the reset in + // begin_ready, which a begin reaches only after its final pre-host check has + // passed (see host_state()). Never read, hashed or recorded by the consumer. + // Declared last, like the members above, so no member the consumer reads at + // every point changes offset. + std::unique_ptr host_state_; + // The selected report's private facts, declared after every member above so + // none of them changes offset. Report bookkeeping like selected_epoch_: never + // hashed, never a broker or trading-state input, and no host can write them. + // The observation count the equity block divides exposure by, held beside + // presentation_metric_equity_len_ and apart from the presented report in the + // same way: M for a selected capture, the length itself for every other. + // Published and zeroed with the capture. + int64_t presentation_metric_observations_ = 0; + // A selected generation's buy-and-hold facts, written beside the native + // input count (count_selected_input) and reset at admission: the open of + // the first primary input at or after T and the close of the latest one. NaN + // until a window input is consumed, so an empty selected coverage never + // borrows a pre-roll price. + bool selected_has_window_input_ = false; + double selected_first_open_ = std::numeric_limits::quiet_NaN(); + double selected_last_close_ = std::numeric_limits::quiet_NaN(); + + // The selected epoch's lifecycle: the admission, terminal and exit halves + // that write selected_epoch_.observation. The nine public entries are their + // only callers (run_simple, run_tf, run_rich, stream_begin, stream_push_bar, + // stream_push_tick, stream_push_ticks, stream_advance_time, stream_end), and + // begin_ready admits. + // + // A handle is tracked when a window or an observer is pending, or once it has + // minted an attempt or a generation (a counter is nonzero, and stays so). + // Every path below is gated on it: a handle that never was opted in keeps + // its callbacks, their order and its rendered texts exactly as before. + bool selected_epoch_tracked() const noexcept { + return selected_epoch_.window_configured || selected_epoch_.observer_registered + || selected_epoch_.attempt_counter != 0 || selected_epoch_.generation_counter != 0; + } + + // The exit half, opened FIRST by every public entry, before AttemptScope, so + // it is destroyed after the AttemptScope and every pump and callback frame + // have unwound. The constructor records, before the entry runs anything, + // whether the handle is tracked, whether this call is nested under an + // attempt, a begin, a callback, an input or a dispatch (meaningful for a + // tracked handle only), and std::uncaught_exceptions(). `allow_live` lets a + // NORMAL return that leaves the run Running keep its generation open: true + // for stream_begin and the pushes and the time advance, false for the batch + // runs and stream_end. The destructor is noexcept and renders, dispatches, + // allocates and moves a NativeRunSpec never. It does nothing for an untracked + // handle or a nested call. Otherwise it closes an epoch that is STILL + // Executing or Capturing (close_open_epoch_at_exit) and leaves a Sealed or + // Results one alone, so a refused attempt that merely latched NativeFailed + // before admission never rewrites an earlier generation. + class EntryExitGuard { + NativeExecutionConsumer& c_; + BacktestEngine& engine_; + const bool tracked_; + const int uncaught_; + const bool nested_; + const bool allow_live_; + public: + // Inline: the stream pushes are per-input entries, and an untracked + // handle must pay only these loads. + EntryExitGuard(NativeExecutionConsumer& c, BacktestEngine& engine, + bool allow_live) noexcept + : c_(c), engine_(engine), tracked_(c.selected_epoch_tracked()), + uncaught_(tracked_ ? std::uncaught_exceptions() : 0), + nested_(tracked_ && (c.in_attempt_call_ || c.preparing_begin_ || c.in_callback_ + || c.processing_input_ || c.selected_epoch_.dispatching)), + allow_live_(allow_live) {} + ~EntryExitGuard() { + if (!tracked_ || nested_) return; + c_.close_open_epoch_at_exit(engine_, std::uncaught_exceptions() > uncaught_, + allow_live_); + } + EntryExitGuard(const EntryExitGuard&) = delete; + EntryExitGuard& operator=(const EntryExitGuard&) = delete; + bool nested() const noexcept { return nested_; } + }; + // The reentry barrier of a public entry, checked after the guard and BEFORE + // AttemptScope in the four begin entries and before stream admission in the + // other five: while the epoch's `dispatching` is held (a capture or an + // observer call) it latches dispatch_reentry_refused and answers true, and + // the entry returns without an AttemptScope, a cache write, clear_rendered or + // a provider callback. rows_current_ and batch_current_ are not touched. + bool refuse_dispatch_reentry() const noexcept { + if (!selected_epoch_.dispatching) return false; + selected_epoch_.dispatch_reentry_refused = true; + return true; + } + // Mints the latest attempt serial once per outermost batch-run or + // stream-begin entry of a tracked handle, right after AttemptScope opens: + // attempt_generation 0 and outcome Refused until admission, owner_serial + // unchanged. A call over a live stream is a later attempt, not a new + // generation. At UINT64_MAX it refuses (engine_invariant, rendered here) + // before the provider is dispatched or anything is reset, and never wraps. + // False means the entry must return. + bool mint_attempt(BacktestEngine& engine, bool nested); + // Admission, called by begin_ready right after reset_run_state and the + // rows_current_/batch_current_ restore: drops the old capture, copies the + // pending window and the observer flag into the admitted state, mints the + // generation, makes the latest attempt its owner and opens the epoch + // (Executing, fault None, Live, boundary not delivered, counts reset), then + // tells the query fence. Only a tracked handle does anything. + void admit_selected_generation(BacktestEngine& engine) noexcept; + // The terminal-success helper: the last step of the four success tails + // (run_simple, run_tf, run_rich, stream_end), after verify_closed_rows and + // inside their try arms. Nothing for an epoch that is not Executing. For a + // generation with a window or an observer it captures the presentation once + // under the dispatch barrier, rechecks the abort, the first cause and the + // reentry latch, seals, calls the observer once and validates its receipt; + // only then does the phase become Results and the capture readable. A + // failure at any step latches the run, closes the epoch (Execution before + // the seal, AfterExecution after) and leaves no capture. `operation` is the + // entry's own (Input for the batch runs, Stream for stream_end). + void terminal_success(BacktestEngine& engine, NativeFailureOperation operation); + // The observer call and its frozen receipt contract (execution_observer.h). + // True when the receipt validated and boundary_delivered is 1. False after it + // has latched the failure, closed the epoch AfterExecution and rendered; an + // earlier cause keeps its code and text. + bool call_execution_observer(BacktestEngine& engine, NativeFailureOperation operation); + // Closes the open epoch as failed: query evidence sealed, capture dropped, + // phase Sealed, `fault_stage`, boundary not delivered, and the latest + // attempt's outcome Aborted (a recoverable abort) or Failed -- only when + // that attempt owns the epoch. Never renders or allocates. + void close_epoch_failed(std::uint32_t fault_stage) noexcept; + // The destructor's work (EntryExitGuard), also noexcept. A defensive close + // with no failure latched and no error text sets the run failure record to + // engine_unclassified_error with empty text, which failure_code_of reads, so + // the closed generation never reports success. + void close_open_epoch_at_exit(BacktestEngine& engine, bool unwinding, + bool allow_live) noexcept; + // The tracked catch arms of the nine entries. A tracked handle restores + // processing_input_ (false at every try's entry) and keeps a failure that is + // already latched -- its code and text stay -- where an untracked handle runs + // the two statements it always ran. fail_unclassified is the catch (...) arm + // for a tracked handle: engine_unclassified_error. An untracked handle's + // catch (...) rethrows. + void fail_from_exception(BacktestEngine& engine, NativeFailureOperation operation, + const std::exception& error); + void fail_unclassified(BacktestEngine& engine, NativeFailureOperation operation); + // The two report-routing reads of fill_selected_report. A generation that + // was admitted with an observer and has closed (Sealed or Results), and a + // capture that may be read (Results, no fault, ready and open). + bool observed_epoch_sealed() const noexcept; + bool capture_readable() const noexcept; }; inline NativeExecutionConsumer& as_native_consumer(IExecutionConsumer& consumer) { diff --git a/src/native_order.cpp b/src/native_order.cpp index ac44081a2..a338c6147 100644 --- a/src/native_order.cpp +++ b/src/native_order.cpp @@ -479,11 +479,26 @@ bool same_p_continuation(const LiveRequest& live, const MatchCursor& cursor) noe return false; } -bool trigger_cursor_eligible(const LiveRequest& live, const MatchCursor& cursor) noexcept { +// The explicit open activation grant: a caller-scoped authorization that no +// request, plan or token retains. It satisfies only the strict ordinal half of +// the birth gate, as if the request had been born before this point, and only +// at an Open cursor; the time half stands, so the point's effective time must +// not precede the request's decision-time lower bound. Every other birth +// exception stays as it was, and this adds one to them. +bool open_grant_birth_ok(const LiveRequest& live, const MatchCursor& cursor, + bool open_activation_grant) noexcept { + return open_activation_grant + && cursor.point.path_phase == NativePathPhase::Open + && cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound; +} + +bool trigger_cursor_eligible(const LiveRequest& live, const MatchCursor& cursor, + bool open_activation_grant) noexcept { if (point_eligible(live.birth(), cursor.point.ordinal, cursor.point.effective_time_ms)) { return true; } - return same_p_continuation(live, cursor); + return same_p_continuation(live, cursor) + || open_grant_birth_ok(live, cursor, open_activation_grant); } const ExecutionAppliedEvent* as_applied(const CommandEvent& event) noexcept { @@ -2033,6 +2048,12 @@ bool WorkingRequestCore::working_is_buy(const LiveRequest& live, EligibilityFacts WorkingRequestCore::eligibility_facts( const LiveRequest& live, const EvaluationContext& context) const noexcept { + return eligibility_facts(live, context, false); +} + +EligibilityFacts WorkingRequestCore::eligibility_facts( + const LiveRequest& live, const EvaluationContext& context, + bool open_activation_grant) const noexcept { EligibilityFacts facts; facts.trigger = &live.request().trigger; facts.trigger_state = &live.trigger_state; @@ -2066,7 +2087,8 @@ EligibilityFacts WorkingRequestCore::eligibility_facts( || (evaluated_at_point && context.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound) || ((pre_open_delivery || remaining_path_delivery) - && context.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound); + && context.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound) + || open_grant_birth_ok(live, context.cursor, open_activation_grant); if (facts.waiting) { facts.driver_ok = false; facts.ready_to_match = false; @@ -2096,7 +2118,13 @@ EligibilityFacts WorkingRequestCore::eligibility_facts( bool WorkingRequestCore::evaluation_eligible(const LiveRequest& live, const EvaluationContext& context) const noexcept { - const EligibilityFacts facts = eligibility_facts(live, context); + return evaluation_eligible(live, context, false); +} + +bool WorkingRequestCore::evaluation_eligible(const LiveRequest& live, + const EvaluationContext& context, + bool open_activation_grant) const noexcept { + const EligibilityFacts facts = eligibility_facts(live, context, open_activation_grant); return facts.ready_to_match; } @@ -2800,24 +2828,39 @@ void WorkingRequestCore::refresh_point_allowances(uint64_t point, bool WorkingRequestCore::refresh_allowance( const RequestHandle& target, const EvaluationContext& context, const TargetObservation& observation) { - return refresh_allowance(target, context, observation, nullptr); + return refresh_allowance(target, context, observation, nullptr, false); +} + +bool WorkingRequestCore::refresh_allowance( + const RequestHandle& target, const EvaluationContext& context, + const TargetObservation& observation, bool open_activation_grant) { + return refresh_allowance(target, context, observation, nullptr, open_activation_grant); } bool WorkingRequestCore::refresh_allowance( const RequestHandle& target, const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal) { - return refresh_allowance(target, context, observation, &next_timeline_ordinal); + return refresh_allowance(target, context, observation, &next_timeline_ordinal, false); +} + +bool WorkingRequestCore::refresh_allowance( + const RequestHandle& target, const EvaluationContext& context, + const TargetObservation& observation, uint64_t& next_timeline_ordinal, + bool open_activation_grant) { + return refresh_allowance(target, context, observation, &next_timeline_ordinal, + open_activation_grant); } bool WorkingRequestCore::refresh_allowance( const RequestHandle& target, const EvaluationContext& context, - const TargetObservation& observation, uint64_t* next_timeline_ordinal) { + const TargetObservation& observation, uint64_t* next_timeline_ordinal, + bool open_activation_grant) { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) return false; LiveRequest& live = live_[live_index]; if (std::holds_alternative(live.authority)) return false; - const EligibilityFacts facts = eligibility_facts(live, context); + const EligibilityFacts facts = eligibility_facts(live, context, open_activation_grant); if (!facts.birth_ok || !facts.driver_ok) return false; if (std::holds_alternative(live.authority)) { if (!context.cohort_side) return false; @@ -2925,6 +2968,15 @@ Preparation WorkingRequestCore::prepare_evaluation( const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal) { + return prepare_evaluation(target, context, observation, next_timeline_ordinal, false); +} + +Preparation WorkingRequestCore::prepare_evaluation( + const RequestHandle& target, + const EvaluationContext& context, + const TargetObservation& observation, + uint64_t& next_timeline_ordinal, + bool open_activation_grant) { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) { @@ -2934,7 +2986,7 @@ Preparation WorkingRequestCore::prepare_evaluation( if (std::holds_alternative(live.authority)) { return NoChange{NoChangeReason::StillWaiting}; } - const EligibilityFacts facts = eligibility_facts(live, context); + const EligibilityFacts facts = eligibility_facts(live, context, open_activation_grant); if (!facts.birth_ok || !facts.driver_ok) return NoChange{NoChangeReason::NotEligible}; if (std::holds_alternative(live.authority)) { @@ -3061,6 +3113,15 @@ Preparation WorkingRequestCore::apply_evaluation( const EvaluationContext& context, const TargetObservation& observation, uint64_t& next_timeline_ordinal) { + return apply_evaluation(target, context, observation, next_timeline_ordinal, false); +} + +Preparation WorkingRequestCore::apply_evaluation( + const RequestHandle& target, + const EvaluationContext& context, + const TargetObservation& observation, + uint64_t& next_timeline_ordinal, + bool open_activation_grant) { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) { @@ -3070,7 +3131,7 @@ Preparation WorkingRequestCore::apply_evaluation( if (std::holds_alternative(live.authority)) { return NoChange{NoChangeReason::StillWaiting}; } - const EligibilityFacts facts = eligibility_facts(live, context); + const EligibilityFacts facts = eligibility_facts(live, context, open_activation_grant); if (!facts.birth_ok || !facts.driver_ok) return NoChange{NoChangeReason::NotEligible}; if (std::holds_alternative(live.authority)) { @@ -3168,6 +3229,18 @@ Preparation WorkingRequestCore::prepare_trigger( uint64_t& next_timeline_ordinal, std::optional cohort_side, const ActivationGrid& activation_grid) { + return prepare_trigger(target, transition, driver_class, next_timeline_ordinal, + cohort_side, activation_grid, false); +} + +Preparation WorkingRequestCore::prepare_trigger( + const RequestHandle& target, + const TriggerTransition& transition, + DriverEligibilityClass driver_class, + uint64_t& next_timeline_ordinal, + std::optional cohort_side, + const ActivationGrid& activation_grid, + bool open_activation_grant) { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) { @@ -3179,7 +3252,7 @@ Preparation WorkingRequestCore::prepare_trigger( return NoChange{NoChangeReason::NotEligible}; } const MatchCursor& cursor = transition_cursor(transition); - if (!trigger_cursor_eligible(updated, cursor) + if (!trigger_cursor_eligible(updated, cursor, open_activation_grant) && !same_point_allowance(updated.allowance, cursor.point.ordinal)) { return NoChange{NoChangeReason::NotEligible}; } @@ -3360,6 +3433,18 @@ Preparation WorkingRequestCore::apply_trigger( uint64_t& next_timeline_ordinal, std::optional cohort_side, const ActivationGrid& activation_grid) { + return apply_trigger(target, transition, driver_class, next_timeline_ordinal, + cohort_side, activation_grid, false); +} + +Preparation WorkingRequestCore::apply_trigger( + const RequestHandle& target, + const TriggerTransition& transition, + DriverEligibilityClass driver_class, + uint64_t& next_timeline_ordinal, + std::optional cohort_side, + const ActivationGrid& activation_grid, + bool open_activation_grant) { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) { @@ -3371,7 +3456,7 @@ Preparation WorkingRequestCore::apply_trigger( return NoChange{NoChangeReason::NotEligible}; } const MatchCursor& cursor = transition_cursor(transition); - if (!trigger_cursor_eligible(live, cursor) + if (!trigger_cursor_eligible(live, cursor, open_activation_grant) && !same_point_allowance(live.allowance, cursor.point.ordinal)) { return NoChange{NoChangeReason::NotEligible}; } @@ -3969,6 +4054,14 @@ Preparation WorkingRequestCore::prepare_execution( const RequestHandle& target, const ExecutionProposal& proposal, uint64_t& next_timeline_ordinal) { + return prepare_execution(target, proposal, next_timeline_ordinal, false); +} + +Preparation WorkingRequestCore::prepare_execution( + const RequestHandle& target, + const ExecutionProposal& proposal, + uint64_t& next_timeline_ordinal, + bool open_activation_grant) { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) { @@ -3993,7 +4086,8 @@ Preparation WorkingRequestCore::prepare_execution( && std::holds_alternative(live.request().trigger) && std::holds_alternative(live.request().capacity) && proposal.cursor.point.path_phase == NativePathPhase::Open - && proposal.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound); + && proposal.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound) + || open_grant_birth_ok(live, proposal.cursor, open_activation_grant); if (!birth_ok) { return NoChange{NoChangeReason::NotEligible}; } @@ -4434,7 +4528,8 @@ InstallResult WorkingRequestCore::install_execution(PreparedExecution&& prepared Preparation WorkingRequestCore::execution_values( const RequestHandle& target, const ExecutionProposal& proposal, - uint64_t& next_timeline_ordinal) const { + uint64_t& next_timeline_ordinal, + bool open_activation_grant) const { require_identity(identity_); std::size_t live_index = 0; if (classify(target, &live_index) != TargetKind::Live) { @@ -4459,7 +4554,8 @@ Preparation WorkingRequestCore::execution_v && std::holds_alternative(live.request().trigger) && std::holds_alternative(live.request().capacity) && proposal.cursor.point.path_phase == NativePathPhase::Open - && proposal.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound); + && proposal.cursor.point.effective_time_ms >= live.birth().decision_time_lower_bound) + || open_grant_birth_ok(live, proposal.cursor, open_activation_grant); if (!birth_ok) { return NoChange{NoChangeReason::NotEligible}; } @@ -4792,7 +4888,16 @@ Preparation WorkingRequestCore::check_execution( const RequestHandle& target, const ExecutionProposal& proposal, uint64_t& next_timeline_ordinal) { - auto values = execution_values(target, proposal, next_timeline_ordinal); + return check_execution(target, proposal, next_timeline_ordinal, false); +} + +Preparation WorkingRequestCore::check_execution( + const RequestHandle& target, + const ExecutionProposal& proposal, + uint64_t& next_timeline_ordinal, + bool open_activation_grant) { + auto values = execution_values(target, proposal, next_timeline_ordinal, + open_activation_grant); if (const auto* none = std::get_if(&values)) return *none; if (const auto* error = std::get_if(&values)) return *error; // prepare_execution's plan: begin_plan's checks, then seal_plan for the @@ -4807,7 +4912,17 @@ InstallResult WorkingRequestCore::apply_execution( const ExecutionProposal& proposal, const CommittedExecutionFacts& facts, uint64_t& next_timeline_ordinal) { - auto computed = execution_values(target, proposal, next_timeline_ordinal); + return apply_execution(target, proposal, facts, next_timeline_ordinal, false); +} + +InstallResult WorkingRequestCore::apply_execution( + const RequestHandle& target, + const ExecutionProposal& proposal, + const CommittedExecutionFacts& facts, + uint64_t& next_timeline_ordinal, + bool open_activation_grant) { + auto computed = execution_values(target, proposal, next_timeline_ordinal, + open_activation_grant); auto* values = std::get_if(&computed); if (!values || epoch_ == std::numeric_limits::max()) { return InstallError::StalePreparation; diff --git a/src/native_selected_epoch.hpp b/src/native_selected_epoch.hpp new file mode 100644 index 000000000..84ba97939 --- /dev/null +++ b/src/native_selected_epoch.hpp @@ -0,0 +1,87 @@ +#pragma once + +#include +#include + +#include + +namespace pineforge { +inline namespace engine_script_run_v19 { + +// The numbers pf_execution_observation_v1 carries (execution_observer.h), named +// once for the lifecycle that writes them. The phase is the generation's own; +// the outcome is the LATEST attempt's, and a terminal write to it is owner-only +// (the attempt serial must equal NativeSelectedEpoch::owner_serial). +inline constexpr std::uint32_t kEpochIdle = 0; +inline constexpr std::uint32_t kEpochExecuting = 1; +inline constexpr std::uint32_t kEpochCapturing = 2; +inline constexpr std::uint32_t kEpochSealed = 3; +inline constexpr std::uint32_t kEpochResults = 4; +inline constexpr std::uint32_t kEpochFaultNone = 0; +inline constexpr std::uint32_t kEpochFaultExecution = 1; +inline constexpr std::uint32_t kEpochFaultAfterExecution = 2; +inline constexpr std::uint32_t kEpochOutcomeNotCompleted = 0; +inline constexpr std::uint32_t kEpochOutcomeRefused = 1; +inline constexpr std::uint32_t kEpochOutcomeFailed = 2; +inline constexpr std::uint32_t kEpochOutcomeAborted = 3; +inline constexpr std::uint32_t kEpochOutcomeResultsOpen = 4; +inline constexpr std::uint32_t kEpochOutcomeLive = 5; + +// What one native consumer remembers for the execution observer and the +// selected window (the two installed C headers above): the descriptors a caller +// registered or configured for the NEXT admitted generation, the copy the +// latest admitted generation took, and the snapshots the two C getters copy +// out. Report and control bookkeeping only: it is not a broker or trading-state +// input and nothing in it is hashed. It owns no memory (the observer's context +// and callback are borrowed from the caller until they are replaced or removed, +// or the consumer is destroyed) and holds no map, registry, global, thread or +// version switch. The consumer cannot be copied and the engine's slot drops it +// when the engine is copied, so this state goes with it; the provenance that +// must outlive a copy stays in the slot. +struct NativeSelectedEpoch { + // The registered observer: the caller's descriptor copied by value. + pf_execution_observer_v1 observer{}; + bool observer_registered = false; + // The selected window the next admitted generation will take (start_ms is + // T, end_ms is E), and the copy the latest admitted generation took. A + // setter writes only the pending one, so setting or clearing it never + // reopens or rewrites a generation that was already admitted. + pf_selected_window_config_v1 pending_window{}; + bool window_configured = false; + pf_selected_window_config_v1 admitted_window{}; + bool admitted_selected = false; + // Whether the admitted generation had an observer: distinct from window + // selection, since either can exist without the other. Removing the + // registration afterwards leaves it as it was. + bool phase_observed = false; + // A transient reentry barrier, held while a capture or an observer call is + // in flight. It is never the persisted phase, which is observation.phase. + bool dispatching = false; + // Set when a public entry, a command or a report read was refused because + // `dispatching` was held (the reentry barrier). It is a latch, not a phase: + // the terminal helper consumes it after each host return and rejects that + // generation with engine_invariant when no earlier cause exists, and every + // admission clears it. Mutable because commands_allowed() and the report + // read, which are const, latch it. + mutable bool dispatch_reentry_refused = false; + // The mint state: the serial of the public run and stream-begin entries, + // the count of admitted generations, and the serial that owns the latest + // admitted generation. All start at zero. + std::uint64_t attempt_counter = 0; + std::uint64_t generation_counter = 0; + std::uint64_t owner_serial = 0; + // The snapshots the two C getters copy out. Only struct_size and version + // are set at construction (sizeof, and 1); every other field starts zero. + pf_execution_observation_v1 observation{}; + pf_selected_window_counts_v1 counts{}; + + NativeSelectedEpoch() noexcept { + observation.struct_size = static_cast(sizeof observation); + observation.version = 1; + counts.struct_size = static_cast(sizeof counts); + counts.version = 1; + } +}; + +} // inline namespace engine_script_run_v19 +} // namespace pineforge diff --git a/src/run_failure.cpp b/src/run_failure.cpp index f8c27e53c..65ddca37f 100644 --- a/src/run_failure.cpp +++ b/src/run_failure.cpp @@ -58,14 +58,34 @@ bool text_kind(RunFailureArgKind kind) noexcept { case RunFailureArgKind::pine_source: case RunFailureArgKind::symbol: case RunFailureArgKind::timeframe: + case RunFailureArgKind::string: return true; case RunFailureArgKind::integer: case RunFailureArgKind::number: + case RunFailureArgKind::nullable_string: + case RunFailureArgKind::scalar: return false; } return false; } +// A scalar argument is text, an integer, a finite number, a boolean (held as 0 +// or 1) or a null. A non-finite number, a boolean outside 0 and 1 and any +// type value this build does not know are refused. +bool scalar_value_ok(const RunFailureArg& arg) noexcept { + switch (arg.type) { + case RunFailureArg::Type::text: + case RunFailureArg::Type::integer: + case RunFailureArg::Type::null_value: + return true; + case RunFailureArg::Type::number: + return std::isfinite(arg.number); + case RunFailureArg::Type::boolean: + return arg.integer == 0 || arg.integer == 1; + } + return false; +} + bool string_equals(const char* a, const std::string& b) noexcept { return b.compare(a) == 0; } @@ -173,11 +193,27 @@ RunFailureValue make_run_failure(RunFailureCode code, const RunFailureArgs& args if (arg.type != RunFailureArg::Type::integer) return invariant(); break; case RunFailureArgKind::number: - if (arg.type == RunFailureArg::Type::text) return invariant(); + // An integer or a finite number: never text, a boolean or a null. + if (arg.type != RunFailureArg::Type::integer + && arg.type != RunFailureArg::Type::number) { + return invariant(); + } if (arg.type == RunFailureArg::Type::number && !std::isfinite(arg.number)) { return invariant(); } break; + case RunFailureArgKind::string: + if (arg.type != RunFailureArg::Type::text) return invariant(); + break; + case RunFailureArgKind::nullable_string: + if (arg.type != RunFailureArg::Type::text + && arg.type != RunFailureArg::Type::null_value) { + return invariant(); + } + break; + case RunFailureArgKind::scalar: + if (!scalar_value_ok(arg)) return invariant(); + break; case RunFailureArgKind::vocab: { if (arg.type != RunFailureArg::Type::text) return invariant(); bool known = false; @@ -228,6 +264,16 @@ RunFailureValue make_run_failure(RunFailureCode code, const RunFailureArgs& args case RunFailureArg::Type::text: append_json_string(*json, arg->text); break; + case RunFailureArg::Type::boolean: + if (arg->integer != 0) { + *json += "true"; + } else { + *json += "false"; + } + break; + case RunFailureArg::Type::null_value: + *json += "null"; + break; } } json->push_back('}'); diff --git a/src/run_failure_registry.inc b/src/run_failure_registry.inc index cacf537a4..dbe903be4 100644 --- a/src/run_failure_registry.inc +++ b/src/run_failure_registry.inc @@ -28,6 +28,16 @@ static constexpr const char* kRegistryValues[] = { "columns", "value", "empty", + "nonfinite", + "invalid_unicode", + "numeric_overflow", + "sizing", + "margin", + "fee", + "slippage", + "relative_price", + "risk", + "settlement", "request.financial", "request.earnings", "request.dividends", @@ -115,6 +125,25 @@ static constexpr const char* kRegistryValues[] = { "invalid_arrays", "times_not_increasing", "restore_failed", + "os", + "architecture", + "pointer_bits", + "endianness", + "engine_library_sha256", + "harness_sha256", + "result_assembly", + "results_digest", + "serialization", + "deadline", + "exception", + "io", + "resource", + "nonfinite", + "invalid_unicode", + "column_schema", + "type_schema", + "report_shape", + "curve_point_budget", "request.security", "request.security_lower_tf", "finer_than_input", @@ -254,6 +283,9 @@ static constexpr const char* kRegistryValues[] = { "risk_day_basis", "risk_action", "quantity_tolerance", + "nonfinite", + "invalid_unicode", + "numeric_overflow", "instrument_feed_instrument", "instrument_feed_timeframe", "instrument_feed_bars", @@ -319,6 +351,33 @@ static constexpr const char* kRegistryValues[] = { "not_json", "not_object", "value_type", + "start", + "end", + "fed_start", + "run_input_bars", + "run_script_bars", + "native_input_bars", + "hpo_input_bars", + "hpo_script_bars", + "hpo_memory_bytes", + "symbol_feed_input_bars", + "symbol_feeds_total_input_bars", + "counts.window_input_bars", + "counts.window_script_bars", + "metering.billable_input_bars", + "report_policy", + "window_start_ms", + "window_end_ms", + "preroll_bars", + "fed_start_ms", + "feed_bytes_sha256", + "input_tf", + "script_tf", + "chart_tz", + "trade_start_ms", + "window", + "schema_version", + "objective", }; static constexpr RegistryArg kRegistryArgs[] = { @@ -326,100 +385,176 @@ static constexpr RegistryArg kRegistryArgs[] = { {"index", RunFailureArgKind::integer, true, 6, 0}, {"reason", RunFailureArgKind::vocab, false, 6, 12}, {"reason", RunFailureArgKind::vocab, false, 18, 4}, - {"call", RunFailureArgKind::pine_source, false, 22, 0}, - {"function", RunFailureArgKind::vocab, false, 22, 9}, - {"line", RunFailureArgKind::integer, false, 31, 0}, - {"timeframe", RunFailureArgKind::timeframe, true, 31, 0}, - {"call", RunFailureArgKind::pine_source, false, 31, 0}, - {"function", RunFailureArgKind::vocab, false, 31, 2}, - {"line", RunFailureArgKind::integer, false, 33, 0}, - {"symbol", RunFailureArgKind::symbol, true, 33, 0}, - {"max", RunFailureArgKind::integer, false, 33, 0}, - {"reason", RunFailureArgKind::vocab, false, 33, 2}, - {"reason", RunFailureArgKind::vocab, false, 35, 3}, - {"collection", RunFailureArgKind::vocab, true, 38, 2}, - {"method", RunFailureArgKind::vocab, true, 40, 12}, - {"reason", RunFailureArgKind::vocab, false, 52, 5}, - {"argument", RunFailureArgKind::identifier, true, 57, 0}, - {"function", RunFailureArgKind::identifier, true, 57, 0}, - {"rule", RunFailureArgKind::vocab, false, 57, 6}, - {"function", RunFailureArgKind::vocab, false, 63, 15}, - {"reason", RunFailureArgKind::vocab, false, 78, 10}, - {"object", RunFailureArgKind::vocab, false, 88, 10}, - {"limit", RunFailureArgKind::vocab, false, 98, 4}, - {"max", RunFailureArgKind::integer, true, 102, 0}, - {"reason", RunFailureArgKind::vocab, false, 102, 2}, - {"reason", RunFailureArgKind::vocab, false, 104, 5}, - {"api", RunFailureArgKind::vocab, false, 109, 2}, - {"input_tf", RunFailureArgKind::timeframe, true, 111, 0}, - {"reason", RunFailureArgKind::vocab, false, 111, 7}, - {"script_tf", RunFailureArgKind::timeframe, true, 118, 0}, - {"line", RunFailureArgKind::integer, true, 118, 0}, - {"reason", RunFailureArgKind::vocab, false, 118, 2}, - {"feature", RunFailureArgKind::vocab, false, 120, 10}, - {"input_tf", RunFailureArgKind::timeframe, true, 130, 0}, - {"option", RunFailureArgKind::vocab, false, 130, 11}, - {"script_tf", RunFailureArgKind::timeframe, true, 141, 0}, - {"option", RunFailureArgKind::vocab, false, 141, 20}, - {"reason", RunFailureArgKind::vocab, false, 161, 11}, - {"entrypoint", RunFailureArgKind::vocab, true, 172, 4}, - {"input", RunFailureArgKind::pine_source, true, 176, 0}, - {"reason", RunFailureArgKind::vocab, true, 176, 21}, - {"abi", RunFailureArgKind::integer, true, 197, 0}, - {"missing", RunFailureArgKind::vocab, true, 197, 13}, - {"reason", RunFailureArgKind::vocab, false, 210, 8}, - {"field", RunFailureArgKind::vocab, false, 218, 30}, - {"field", RunFailureArgKind::vocab, true, 248, 6}, - {"input_tf", RunFailureArgKind::timeframe, true, 254, 0}, - {"reason", RunFailureArgKind::vocab, false, 254, 41}, - {"script_tf", RunFailureArgKind::timeframe, true, 295, 0}, - {"field", RunFailureArgKind::vocab, false, 295, 14}, - {"reason", RunFailureArgKind::vocab, false, 309, 4}, + {"field", RunFailureArgKind::string, false, 22, 0}, + {"reason", RunFailureArgKind::vocab, false, 22, 3}, + {"stage", RunFailureArgKind::vocab, false, 25, 7}, + {"call", RunFailureArgKind::pine_source, false, 32, 0}, + {"function", RunFailureArgKind::vocab, false, 32, 9}, + {"line", RunFailureArgKind::integer, false, 41, 0}, + {"timeframe", RunFailureArgKind::timeframe, true, 41, 0}, + {"call", RunFailureArgKind::pine_source, false, 41, 0}, + {"function", RunFailureArgKind::vocab, false, 41, 2}, + {"line", RunFailureArgKind::integer, false, 43, 0}, + {"symbol", RunFailureArgKind::symbol, true, 43, 0}, + {"max", RunFailureArgKind::integer, false, 43, 0}, + {"reason", RunFailureArgKind::vocab, false, 43, 2}, + {"reason", RunFailureArgKind::vocab, false, 45, 3}, + {"collection", RunFailureArgKind::vocab, true, 48, 2}, + {"method", RunFailureArgKind::vocab, true, 50, 12}, + {"reason", RunFailureArgKind::vocab, false, 62, 5}, + {"argument", RunFailureArgKind::identifier, true, 67, 0}, + {"function", RunFailureArgKind::identifier, true, 67, 0}, + {"rule", RunFailureArgKind::vocab, false, 67, 6}, + {"function", RunFailureArgKind::vocab, false, 73, 15}, + {"reason", RunFailureArgKind::vocab, false, 88, 10}, + {"object", RunFailureArgKind::vocab, false, 98, 10}, + {"limit", RunFailureArgKind::vocab, false, 108, 4}, + {"max", RunFailureArgKind::integer, true, 112, 0}, + {"reason", RunFailureArgKind::vocab, false, 112, 2}, + {"reason", RunFailureArgKind::vocab, false, 114, 5}, + {"field", RunFailureArgKind::vocab, false, 119, 6}, + {"phase", RunFailureArgKind::vocab, false, 125, 3}, + {"reason", RunFailureArgKind::vocab, false, 128, 4}, + {"field", RunFailureArgKind::string, false, 132, 0}, + {"reason", RunFailureArgKind::vocab, false, 132, 4}, + {"option", RunFailureArgKind::vocab, false, 136, 2}, + {"value", RunFailureArgKind::scalar, false, 138, 0}, + {"shape", RunFailureArgKind::string, false, 138, 0}, + {"api", RunFailureArgKind::vocab, false, 138, 2}, + {"input_tf", RunFailureArgKind::timeframe, true, 140, 0}, + {"reason", RunFailureArgKind::vocab, false, 140, 7}, + {"script_tf", RunFailureArgKind::timeframe, true, 147, 0}, + {"line", RunFailureArgKind::integer, true, 147, 0}, + {"reason", RunFailureArgKind::vocab, false, 147, 2}, + {"requested", RunFailureArgKind::string, false, 149, 0}, + {"feature", RunFailureArgKind::vocab, false, 149, 10}, + {"input_tf", RunFailureArgKind::timeframe, true, 159, 0}, + {"option", RunFailureArgKind::vocab, false, 159, 11}, + {"script_tf", RunFailureArgKind::timeframe, true, 170, 0}, + {"option", RunFailureArgKind::vocab, false, 170, 20}, + {"reason", RunFailureArgKind::vocab, false, 190, 11}, + {"entrypoint", RunFailureArgKind::vocab, true, 201, 4}, + {"input", RunFailureArgKind::pine_source, true, 205, 0}, + {"reason", RunFailureArgKind::vocab, true, 205, 21}, + {"abi", RunFailureArgKind::integer, true, 226, 0}, + {"missing", RunFailureArgKind::vocab, true, 226, 13}, + {"reason", RunFailureArgKind::vocab, false, 239, 8}, + {"field", RunFailureArgKind::vocab, false, 247, 30}, + {"field", RunFailureArgKind::string, false, 277, 0}, + {"reason", RunFailureArgKind::vocab, false, 277, 3}, + {"field", RunFailureArgKind::vocab, true, 280, 6}, + {"input_tf", RunFailureArgKind::timeframe, true, 286, 0}, + {"reason", RunFailureArgKind::vocab, false, 286, 41}, + {"script_tf", RunFailureArgKind::timeframe, true, 327, 0}, + {"field", RunFailureArgKind::vocab, false, 327, 14}, + {"reason", RunFailureArgKind::vocab, false, 341, 4}, + {"bound", RunFailureArgKind::vocab, false, 345, 3}, + {"next_boundary_ms", RunFailureArgKind::integer, false, 348, 0}, + {"previous_boundary_ms", RunFailureArgKind::integer, false, 348, 0}, + {"value_ms", RunFailureArgKind::integer, false, 348, 0}, + {"session", RunFailureArgKind::nullable_string, false, 348, 0}, + {"timezone", RunFailureArgKind::string, false, 348, 0}, + {"actual_sha256", RunFailureArgKind::string, false, 348, 0}, + {"expected_sha256", RunFailureArgKind::string, false, 348, 0}, + {"feed_id", RunFailureArgKind::string, false, 348, 0}, + {"end_ms", RunFailureArgKind::integer, false, 348, 0}, + {"fed_start_ms", RunFailureArgKind::integer, false, 348, 0}, + {"time_ms", RunFailureArgKind::integer, false, 348, 0}, + {"actual", RunFailureArgKind::integer, false, 348, 0}, + {"cap_name", RunFailureArgKind::vocab, false, 348, 8}, + {"limit", RunFailureArgKind::integer, false, 356, 0}, + {"option", RunFailureArgKind::string, false, 356, 0}, + {"actual", RunFailureArgKind::scalar, false, 356, 0}, + {"expected", RunFailureArgKind::scalar, false, 356, 0}, + {"field", RunFailureArgKind::string, false, 356, 0}, + {"capability", RunFailureArgKind::string, false, 356, 0}, + {"requested", RunFailureArgKind::integer, true, 356, 0}, + {"timeframe", RunFailureArgKind::string, true, 356, 0}, + {"end_ms", RunFailureArgKind::integer, false, 356, 0}, + {"start_ms", RunFailureArgKind::integer, false, 356, 0}, + {"requested", RunFailureArgKind::integer, false, 356, 0}, + {"fill_ms", RunFailureArgKind::integer, false, 356, 0}, + {"start_ms", RunFailureArgKind::integer, false, 356, 0}, + {"position_size", RunFailureArgKind::number, false, 356, 0}, + {"resting_orders", RunFailureArgKind::integer, false, 356, 0}, + {"time_ms", RunFailureArgKind::integer, false, 356, 0}, + {"pruner", RunFailureArgKind::string, false, 356, 0}, + {"end_ms", RunFailureArgKind::integer, false, 356, 0}, + {"fed_start_ms", RunFailureArgKind::integer, false, 356, 0}, + {"start_ms", RunFailureArgKind::integer, false, 356, 0}, + {"actual", RunFailureArgKind::integer, false, 356, 0}, + {"expected", RunFailureArgKind::integer, false, 356, 0}, + {"field", RunFailureArgKind::vocab, false, 356, 3}, + {"option", RunFailureArgKind::vocab, false, 359, 13}, }; static constexpr RegistryCode kRegistryCodes[] = { {"", RunFailureClass::engine_fault, false, 0, 0}, {"chart_bars_rejected", RunFailureClass::input, false, 0, 3}, {"chart_bars_unreadable", RunFailureClass::input, false, 3, 1}, - {"engine_invariant", RunFailureClass::engine_fault, false, 4, 0}, - {"engine_unclassified_error", RunFailureClass::engine_fault, false, 4, 0}, - {"harness_internal_error", RunFailureClass::engine_fault, false, 4, 0}, - {"lot_grid_rejected", RunFailureClass::symbol_metadata, true, 4, 0}, - {"no_data_request", RunFailureClass::no_data, false, 4, 3}, - {"other_symbol_feed_missing", RunFailureClass::no_data, false, 7, 1}, - {"other_symbol_request", RunFailureClass::no_data, false, 8, 4}, - {"out_of_memory", RunFailureClass::resource, true, 12, 0}, - {"outputs_limit", RunFailureClass::strategy_limit, false, 12, 2}, - {"outputs_rejected", RunFailureClass::input, false, 14, 1}, - {"pine_array_error", RunFailureClass::strategy, false, 15, 3}, - {"pine_invalid_argument", RunFailureClass::strategy, false, 18, 3}, - {"pine_matrix_error", RunFailureClass::strategy, false, 21, 2}, - {"pine_na_reference", RunFailureClass::strategy, false, 23, 1}, - {"pine_runtime_limit", RunFailureClass::strategy_limit, false, 24, 2}, - {"pine_string_error", RunFailureClass::strategy, false, 26, 1}, - {"recalc_cap", RunFailureClass::strategy_limit, false, 27, 0}, - {"recorded_request_missing", RunFailureClass::no_data, false, 27, 0}, - {"recorded_request_refused", RunFailureClass::input, false, 27, 1}, - {"request_symbol_invalid", RunFailureClass::strategy, false, 28, 0}, - {"request_timeframe_invalid", RunFailureClass::strategy, false, 28, 1}, - {"request_timeframe_unsupported", RunFailureClass::unsupported, false, 29, 3}, - {"request_unsupported", RunFailureClass::unsupported, false, 32, 2}, - {"run_aborted", RunFailureClass::engine_fault, true, 34, 0}, - {"run_mode_unsupported", RunFailureClass::unsupported, false, 34, 1}, - {"run_options_rejected", RunFailureClass::input, false, 35, 3}, - {"run_request_invalid", RunFailureClass::input, false, 38, 1}, - {"security_feed_refused", RunFailureClass::input, false, 39, 1}, - {"setting_rejected", RunFailureClass::input, false, 40, 3}, - {"setting_unsupported", RunFailureClass::unsupported, false, 43, 0}, - {"strategy_create_failed", RunFailureClass::engine_fault, false, 43, 0}, - {"strategy_library_incompatible", RunFailureClass::engine_fault, false, 43, 3}, - {"strategy_runtime_error", RunFailureClass::strategy, false, 46, 0}, - {"strategy_settings_rejected", RunFailureClass::strategy, false, 46, 1}, - {"stream_input_rejected", RunFailureClass::input, false, 47, 0}, - {"symbol_feeds_refused", RunFailureClass::symbol_feeds, false, 47, 4}, - {"symbol_metadata_rejected", RunFailureClass::symbol_metadata, true, 51, 1}, - {"syminfo_unreadable", RunFailureClass::symbol_metadata, true, 52, 1}, + {"engine_command_value_invalid", RunFailureClass::engine_fault, false, 4, 3}, + {"engine_invariant", RunFailureClass::engine_fault, false, 7, 0}, + {"engine_unclassified_error", RunFailureClass::engine_fault, false, 7, 0}, + {"harness_internal_error", RunFailureClass::engine_fault, false, 7, 0}, + {"lot_grid_rejected", RunFailureClass::symbol_metadata, true, 7, 0}, + {"no_data_request", RunFailureClass::no_data, false, 7, 3}, + {"other_symbol_feed_missing", RunFailureClass::no_data, false, 10, 1}, + {"other_symbol_request", RunFailureClass::no_data, false, 11, 4}, + {"out_of_memory", RunFailureClass::resource, true, 15, 0}, + {"outputs_limit", RunFailureClass::strategy_limit, false, 15, 2}, + {"outputs_rejected", RunFailureClass::input, false, 17, 1}, + {"pine_array_error", RunFailureClass::strategy, false, 18, 3}, + {"pine_invalid_argument", RunFailureClass::strategy, false, 21, 3}, + {"pine_matrix_error", RunFailureClass::strategy, false, 24, 2}, + {"pine_na_reference", RunFailureClass::strategy, false, 26, 1}, + {"pine_runtime_limit", RunFailureClass::strategy_limit, false, 27, 2}, + {"pine_string_error", RunFailureClass::strategy, false, 29, 1}, + {"recalc_cap", RunFailureClass::strategy_limit, false, 30, 0}, + {"recorded_request_missing", RunFailureClass::no_data, false, 30, 0}, + {"recorded_request_refused", RunFailureClass::input, false, 30, 1}, + {"report_platform_unavailable", RunFailureClass::engine_fault, false, 31, 1}, + {"report_post_execution_failed", RunFailureClass::engine_fault, false, 32, 2}, + {"report_result_invalid", RunFailureClass::engine_fault, false, 34, 2}, + {"report_shape_request_invalid", RunFailureClass::input, false, 36, 2}, + {"report_shape_unsupported", RunFailureClass::unsupported, false, 38, 1}, + {"request_symbol_invalid", RunFailureClass::strategy, false, 39, 0}, + {"request_timeframe_invalid", RunFailureClass::strategy, false, 39, 1}, + {"request_timeframe_unsupported", RunFailureClass::unsupported, false, 40, 3}, + {"request_unsupported", RunFailureClass::unsupported, false, 43, 2}, + {"results_digest_unsupported", RunFailureClass::unsupported, false, 45, 1}, + {"run_aborted", RunFailureClass::engine_fault, true, 46, 0}, + {"run_mode_unsupported", RunFailureClass::unsupported, false, 46, 1}, + {"run_options_rejected", RunFailureClass::input, false, 47, 3}, + {"run_request_invalid", RunFailureClass::input, false, 50, 1}, + {"security_feed_refused", RunFailureClass::input, false, 51, 1}, + {"setting_rejected", RunFailureClass::input, false, 52, 3}, + {"setting_unsupported", RunFailureClass::unsupported, false, 55, 0}, + {"strategy_create_failed", RunFailureClass::engine_fault, false, 55, 0}, + {"strategy_library_incompatible", RunFailureClass::engine_fault, false, 55, 3}, + {"strategy_runtime_error", RunFailureClass::strategy, false, 58, 0}, + {"strategy_settings_rejected", RunFailureClass::strategy, false, 58, 1}, + {"strategy_value_invalid", RunFailureClass::strategy, false, 59, 2}, + {"stream_input_rejected", RunFailureClass::input, false, 61, 0}, + {"symbol_feeds_refused", RunFailureClass::symbol_feeds, false, 61, 4}, + {"symbol_metadata_rejected", RunFailureClass::symbol_metadata, true, 65, 1}, + {"syminfo_unreadable", RunFailureClass::symbol_metadata, true, 66, 1}, + {"window_boundary_unaligned", RunFailureClass::input, false, 67, 4}, + {"window_calendar_unsupported", RunFailureClass::unsupported, false, 71, 2}, + {"window_feed_identity_changed", RunFailureClass::input, false, 73, 3}, + {"window_feed_range_invalid", RunFailureClass::input, false, 76, 3}, + {"window_input_cap_exceeded", RunFailureClass::input, false, 79, 3}, + {"window_legacy_flag_conflict", RunFailureClass::input, false, 82, 1}, + {"window_lineage_mismatch", RunFailureClass::input, false, 83, 3}, + {"window_mode_unsupported", RunFailureClass::unsupported, false, 86, 3}, + {"window_no_observations", RunFailureClass::input, false, 89, 2}, + {"window_preroll_out_of_range", RunFailureClass::input, false, 91, 1}, + {"window_prestart_fill", RunFailureClass::engine_fault, false, 92, 2}, + {"window_prestart_state_violation", RunFailureClass::engine_fault, false, 94, 3}, + {"window_pruner_unsupported", RunFailureClass::unsupported, false, 97, 1}, + {"window_range_invalid", RunFailureClass::input, false, 98, 3}, + {"window_report_mismatch", RunFailureClass::engine_fault, false, 101, 3}, + {"window_request_invalid", RunFailureClass::input, false, 104, 1}, }; -static_assert(sizeof(kRegistryCodes) / sizeof(kRegistryCodes[0]) == 41, +static_assert(sizeof(kRegistryCodes) / sizeof(kRegistryCodes[0]) == 65, "the registry must have one row per RunFailureCode"); diff --git a/src/selected_report_view.cpp b/src/selected_report_view.cpp new file mode 100644 index 000000000..9dbff9747 --- /dev/null +++ b/src/selected_report_view.cpp @@ -0,0 +1,64 @@ +/* + * selected_report_view.cpp — the selected-window report's equity curve + * (selected_report_view.hpp): the window anchor followed by the script-bar + * observations, assembled without a callback or any engine state. + */ +#include "selected_report_view.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge { +namespace report { + +namespace { + +// A call the engine itself got wrong, coded the way the recorded outputs code +// a broken recorder contract (engine_invariant); never a strategy refusal. +[[noreturn]] void refuse_view(const char* text) { + throw coded(RunFailureCode::engine_invariant, {}, text); +} + +} // namespace + +SelectedEquityCurve build_selected_equity_curve( + int64_t window_start_ms, double initial_capital, + const pf_equity_point_t* observations, int64_t observation_count) { + static_assert(std::is_trivially_copyable::value, + "observations are copied byte for byte"); + if (observation_count < 0) + refuse_view("selected report view: the observation count is negative"); + if (observation_count > 0 && observations == nullptr) + refuse_view("selected report view: observations is null for a nonzero count"); + // M + 1 points must be countable by the int64_t `count` and their bytes must + // fit a size_t, whatever size_t is. Compared as M >= most_points, never + // M + 1 > most_points: M + 1 itself could overflow int64_t. + const uint64_t countable = static_cast(std::numeric_limits::max()); + const uint64_t sizable = static_cast( + std::numeric_limits::max() / sizeof(pf_equity_point_t)); + const uint64_t most_points = countable < sizable ? countable : sizable; + if (static_cast(observation_count) >= most_points) + refuse_view("selected report view: the observation count exceeds the largest curve"); + + // One array, from `new[]` because whoever takes it frees it with `delete[]` + // (BacktestEngine::free_report on ReportC::equity_curve). The empty `()` + // value-initializes every element; the anchor and the copy below write all of them. + const std::size_t observed = static_cast(observation_count); + std::unique_ptr points(new pf_equity_point_t[observed + 1]()); + points[0].time_ms = window_start_ms; + points[0].equity = initial_capital; + points[0].open_profit = 0.0; + if (observed > 0) + std::memcpy(points.get() + 1, observations, observed * sizeof(pf_equity_point_t)); + return SelectedEquityCurve{std::move(points), observation_count + 1}; +} + +} // namespace report +} // namespace pineforge diff --git a/src/selected_report_view.hpp b/src/selected_report_view.hpp new file mode 100644 index 000000000..75cd4f4bd --- /dev/null +++ b/src/selected_report_view.hpp @@ -0,0 +1,66 @@ +#pragma once + +// The selected-window report's equity curve: pure data construction, owned by +// the selected-window report boundary (wire contract v1.3). Internal to the +// kernel's report path and to the rows that hold it to that contract +// (tests/test_selected_report_view.cpp); not installed. +// Engine-generic: it reads no strategy, callback, timezone, calendar, source +// adapter or engine state, and a selected caller supplies everything. + +#include + +#include +#include + +namespace pineforge { +namespace report { + +// A built curve: one array of exactly `count` points, owned by `points`. The +// array comes from a single `new[]`, so whoever takes it frees it the way +// BacktestEngine::free_report frees ReportC::equity_curve -- `delete[]`. +// `count` is plain data and does not follow `points`: it keeps its value after +// `points` is moved from or released. +struct SelectedEquityCurve { + std::unique_ptr points; + int64_t count; +}; + +// The curve of a window that starts at `window_start_ms` (T) over +// `observation_count` (M) script-bar observations: exactly M + 1 points, so +// count = M + 1. Index 0 is the window anchor -- time_ms = T, +// equity = initial_capital, open_profit = 0 -- and indices 1..M copy the M +// observations byte for byte, so observation i sits at position i + 1. The +// anchor is a curve point, not a script bar: no bar index belongs to it, no +// synthetic bar is inserted, and an observation stamped T (the window's first +// bar opening at its start) stands beside the anchor with the same time. +// M = 0 returns the anchor alone, and only then may `observations` be null. +// +// `observations` is the already selected, chronologically ordered run of +// script-bar points from the engine's execution state, not a caller-sliced +// report; selecting it and ordering it are the caller's invariants. No point is +// filtered, reordered or recomputed, and no argument is judged beyond what the +// construction itself needs: any T and any initial_capital are taken as given. +// +// Giving the curve to a report: `out->equity_curve = curve.points.release();` +// and `out->equity_curve_len = curve.count;` cannot throw, and from there the +// report owns the array, as it owns any array it holds -- whoever frees the +// report frees it, with delete[]. Do it where the report takes its curve today, +// before the presentation hook and the statistics, which read (and may re-mark) +// the report's own copy; the statistics then take n = equity_curve_len and +// observation_count = equity_curve_len - 1. A refusal or allocation failure in +// the build happens before the release and leaves the report untouched, and +// nothing copies the curve. +// +// Refusals: a negative M, a null `observations` with M > 0, and an M whose +// M + 1 points cannot be counted by the int64_t `count` or sized in bytes by a +// size_t are a call the engine itself got wrong, all checked before anything is +// added, allocated or read; each throws coded carrying +// RunFailureCode::engine_invariant (the way the recorded outputs refuse a broken +// recorder contract). They are not strategy refusals. An allocation failure is +// std::bad_alloc, which the failure channel already classifies as out_of_memory. +SelectedEquityCurve build_selected_equity_curve( + int64_t window_start_ms, double initial_capital, + const pf_equity_point_t* observations, int64_t observation_count); + +} // namespace report +} // namespace pineforge diff --git a/src/selected_window_plan.cpp b/src/selected_window_plan.cpp new file mode 100644 index 000000000..9d3138d78 --- /dev/null +++ b/src/selected_window_plan.cpp @@ -0,0 +1,586 @@ +// The selected primary planner (include/pineforge/selected_window_plan.hpp +// holds the contract). This file states how the two passes mirror the native +// consumer. +// +// Pass 1 walks the rows once. Per row it checks order and range, resolves the +// row's input and script intervals with the native memo (one calendar, one +// pairing, no per-row calendar or engine), and replays the consumer's script +// bucket (BucketWalk) to count the groups that open before T, the window +// callbacks a batch run will execute, and whether a pre-T group is still +// pending at the end. Pass 2 runs only when some pre-roll groups are trimmed +// away while others are kept: it replays the same walk, from a fresh memo and +// so the same answers, up to the first row of the first kept group. Auxiliary +// storage is a fixed set of scalars, the parsed calendar and the memo's bounded +// tables; no row, bar or interval is copied or kept. +// +// The retained count (count_selected_retained_primary) is pass 1 alone over rows +// the caller has already retained: the same clock admission (admit_clock), the +// same boundary check and the same row walk (walk_rows), with no F boundary, no +// pre-roll request and no trimming. Its answer is the groups that opened before +// T plus the window groups the walk sealed, so for the rows a plan keeps it is +// that plan's fed_script_bars. + +#include + +#include + +#include "native_calendar_memo.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge { +inline namespace selected_window_plan_v1 { +namespace { + +namespace nc = native_calendar; + +constexpr std::int64_t kI53Max = (std::int64_t{1} << 53) - 1; +constexpr std::uint32_t kMaxPrerollBars = 5000; +// The first build's eight intraday primary tokens, exactly as a request spells +// them: no alias, no second token, no daily or weekly token. +constexpr const char* kPrimaryTokens[] = {"1", "3", "5", "15", "30", "60", "120", "240"}; +constexpr const char* kBoundOptions[] = {"start_ms", "end_ms", "fed_start_ms"}; + +bool primary_token(const std::string& token) { + for (const char* candidate : kPrimaryTokens) { + if (token == candidate) return true; + } + return false; +} + +bool in_i53(std::int64_t value) noexcept { return value >= -kI53Max && value <= kI53Max; } + +// The conservative clock identity: after the empty default only the exact spellings +// UTC, GMT, Etc/UTC and Etc/GMT are one token; every other zone string is its own +// token, compared exactly (no trim, no case folding, no alias inference). +std::string_view clock_token(const std::string& timezone) noexcept { + if (timezone.empty() || timezone == "UTC" || timezone == "GMT" || timezone == "Etc/UTC" + || timezone == "Etc/GMT") { + return "UTC"; + } + return timezone; +} + +SelectedPrimaryPlan refusal(SelectedPlanStatus status, const char* option, int bound = -1, + std::int64_t value_ms = 0, std::int64_t previous_boundary_ms = 0, + std::int64_t next_boundary_ms = 0) { + SelectedPrimaryPlan plan; + plan.status = status; + plan.option = option; + plan.bound = bound; + plan.value_ms = value_ms; + plan.previous_boundary_ms = previous_boundary_ms; + plan.next_boundary_ms = next_boundary_ms; + return plan; +} + +// The one calendar/input/script pairing the rows are grouped under. +struct Clock { + nc::SessionCalendar calendar; + nc::Timeframe input_tf; + nc::Timeframe script_tf; + bool passthrough = false; + bool tolerant = true; +}; + +// The clock identity, then the native session, timezone and timeframe grammar +// and the pairing. Both entries' requests (SelectedPlanRequest and +// SelectedRetainedRequest) carry the same six members this reads. Returns a +// refusal, or nothing with `clock` filled in. std::bad_alloc propagates; any +// other exception is contained as a named status, never a guessed value. +template +std::optional admit_clock(const Request& request, Clock& clock) { + if (clock_token(request.chart_timezone) != clock_token(request.engine_timezone)) { + return refusal(SelectedPlanStatus::CalendarUnsupported, "timezone"); + } + // The adapter's spec builder: a missing or empty syminfo timezone is UTC, a + // missing or empty session is 24x7. The session string is shared by the + // alignment and the aggregation clock. + const std::string zone_name = + request.engine_timezone.empty() ? std::string("UTC") : request.engine_timezone; + const std::string session_text = + request.session.empty() ? std::string("24x7") : request.session; + try { + if (!nc::timezone_accepted(zone_name)) { + return refusal(SelectedPlanStatus::CalendarUnsupported, "engine_timezone"); + } + std::optional calendar = nc::parse_session(session_text, zone_name); + if (!calendar) return refusal(SelectedPlanStatus::CalendarUnsupported, "session"); + std::optional input_tf = nc::parse_timeframe(request.input_tf); + if (!input_tf) return refusal(SelectedPlanStatus::RequestInvalid, "input_tf"); + std::optional script_tf = nc::parse_timeframe(request.script_tf); + if (!script_tf) return refusal(SelectedPlanStatus::InternalError, "script_tf"); + const nc::TimeframeCompatibility pairing = nc::compatibility(*input_tf, *script_tf); + const bool passthrough = pairing.pairing == nc::TimeframePairing::Passthrough; + if (!passthrough && !nc::pairing_aggregates(pairing)) { + return refusal(SelectedPlanStatus::CalendarUnsupported, "timeframes"); + } + clock.calendar = std::move(*calendar); + clock.input_tf = std::move(*input_tf); + clock.script_tf = std::move(*script_tf); + clock.passthrough = passthrough; + clock.tolerant = request.feed_tolerant; + return std::nullopt; + } catch (const std::bad_alloc&) { + throw; + } catch (...) { + return refusal(SelectedPlanStatus::CalendarUnsupported, "calendar"); + } +} + +// The consumer's timestamp_partition: no duration is known, every boundary is +// the row's own timestamp. +nc::NativeInterval raw_partition(std::int64_t timestamp) noexcept { + return nc::NativeInterval{timestamp, timestamp, timestamp, timestamp, timestamp}; +} + +// The consumer's grouping for one row: input_interval_at(timestamp), then +// script_interval_at at that input interval's open. FeedTolerant on a +// Passthrough pairing is the raw partition for both; otherwise a failed lookup +// falls back to the same raw partition under FeedTolerant and is a failure +// without it. +bool resolve_row(const Clock& clock, nc::SessionDayMemo& memo, std::int64_t timestamp, + nc::NativeInterval& input, nc::NativeInterval& script) { + if (clock.tolerant && clock.passthrough) { + input = raw_partition(timestamp); + script = raw_partition(input.open_ms); + return true; + } + std::optional held = + nc::interval_containing(clock.calendar, clock.input_tf, timestamp, memo); + if (!held && clock.tolerant) held = raw_partition(timestamp); + if (!held) return false; + input = *held; + held = nc::interval_containing(clock.calendar, clock.script_tf, input.open_ms, memo); + if (!held && clock.tolerant) held = raw_partition(input.open_ms); + if (!held) return false; + script = *held; + return true; +} + +// A boundary (bound 0 = T, 1 = E, 2 = F) is the nominal open of the pure native +// script interval that holds it, independent of the rows. Returns the refusal, +// or nothing when it is aligned. Nothing snaps. +std::optional check_boundary(const Clock& clock, nc::SessionDayMemo& memo, + int bound, std::int64_t at) { + const std::optional held = + nc::interval_containing(clock.calendar, clock.script_tf, at, memo); + if (!held) { + return refusal(SelectedPlanStatus::CalendarUnsupported, kBoundOptions[bound], bound, at); + } + if (held->open_ms != at) { + return refusal(SelectedPlanStatus::BoundaryUnaligned, kBoundOptions[bound], bound, at, + held->open_ms, held->next_period_open_ms); + } + return std::nullopt; +} + +// The consumer's script bucket, replayed without its OHLC aggregate: the same +// has-data / key / interval / last-accepted-input state that contribute_input, +// seal_stale_script, seal_script and pump_batch's closing rule read, so the +// groups it opens and the callbacks it seals are the ones a batch run +// delivers. A group's key (what a later row is compared with) and label (what +// the sealed bar is published under) are separate even though both are the +// script interval's open_ms here. "Before" means the group's first row lies +// before T. +class BucketWalk { +public: + explicit BucketWalk(std::int64_t start_ms) noexcept : start_ms_(start_ms) {} + + struct Step { + bool opened = false; // this row opened a group + bool straddled = false; // one group would hold rows on both sides of T + std::int64_t straddle_open_ms = 0; + std::int64_t straddle_next_ms = 0; + }; + + Step feed(std::int64_t timestamp, const nc::NativeInterval& input, + const nc::NativeInterval& script) noexcept { + Step step; + const std::int64_t script_key = script.open_ms; + // seal_stale_script: the first input of a later group seals a pending + // one (LazyComplete) before it contributes. + if (open_ && key_ != script_key) seal(); + if (!open_) { + // On aligned admitted clocks no script interval holds T strictly + // inside; one that does is not silently reassigned. + if (script.open_ms < start_ms_ && start_ms_ < script.next_period_open_ms) { + step.straddled = true; + step.straddle_open_ms = script.open_ms; + step.straddle_next_ms = script.next_period_open_ms; + return step; + } + open_ = true; + before_ = timestamp < start_ms_; + key_ = script_key; + label_ = script.open_ms; + eligible_open_ms_ = script.eligible_open_ms; + last_traded_close_ms_ = script.last_traded_close_ms; + next_period_open_ms_ = script.next_period_open_ms; + if (before_) { + if (opened_before_ == 0) { + first_key_ = key_; + first_label_ = label_; + } + ++opened_before_; + last_before_label_ = label_; + } + opened_key_ = key_; + opened_label_ = label_; + step.opened = true; + } else if (before_ && timestamp >= start_ms_) { + step.straddled = true; + step.straddle_open_ms = label_; + step.straddle_next_ms = next_period_open_ms_; + return step; + } + input_last_traded_close_ms_ = input.last_traded_close_ms; + // contribute_input: an input that reaches the script interval's next + // period open seals the group (Confirmed). + if (input.next_period_open_ms >= next_period_open_ms_) seal(); + return step; + } + + // pump_batch's closing rule (final_script_session_closed): the final input + // reaches the pending group's last traded close. Any other pending group is + // never sealed. + void finish() noexcept { + if (open_ && last_traded_close_ms_ > eligible_open_ms_ + && input_last_traded_close_ms_ >= last_traded_close_ms_) { + seal(); + } + } + + std::uint64_t opened_before() const noexcept { return opened_before_; } + std::uint64_t sealed_window() const noexcept { return sealed_window_; } + bool pending_before() const noexcept { return open_ && before_; } + std::int64_t first_key() const noexcept { return first_key_; } + std::int64_t first_label() const noexcept { return first_label_; } + std::int64_t last_before_label() const noexcept { return last_before_label_; } + std::int64_t opened_key() const noexcept { return opened_key_; } + std::int64_t opened_label() const noexcept { return opened_label_; } + +private: + void seal() noexcept { + if (!before_) ++sealed_window_; + open_ = false; + } + + std::int64_t start_ms_; + bool open_ = false; // the consumer's script_.has_data && !script_.sealed + bool before_ = false; // the pending group's first row lies before T + std::int64_t key_ = 0; + std::int64_t label_ = 0; + std::int64_t eligible_open_ms_ = 0; + std::int64_t last_traded_close_ms_ = 0; + std::int64_t next_period_open_ms_ = 0; + std::int64_t input_last_traded_close_ms_ = 0; // last_accepted_input_ + std::uint64_t opened_before_ = 0; + std::uint64_t sealed_window_ = 0; + std::int64_t first_key_ = 0; + std::int64_t first_label_ = 0; + std::int64_t last_before_label_ = 0; + std::int64_t opened_key_ = 0; + std::int64_t opened_label_ = 0; +}; + +// Pass 1, the one row walk the planner and the retained count share: each row's +// order and range, its input and script intervals, and the consumer's script +// bucket (BucketWalk). `floor_ms` is the planner's F (a row below it is +// refused); the retained count has none. Returns a refusal, or nothing with the +// walk finished (pump_batch's closing rule applied) and `first_window_row` the +// index of the first row at or after T, or `count` when there is none. +std::optional walk_rows(const Bar* rows, std::size_t count, + const Clock& clock, nc::SessionDayMemo& memo, + BucketWalk& walk, std::optional floor_ms, + std::int64_t start_ms, std::int64_t end_ms, + std::size_t& first_window_row) { + using Status = SelectedPlanStatus; + first_window_row = count; // count: no row at or after T + for (std::size_t i = 0; i < count; ++i) { + const std::int64_t at = rows[i].timestamp; + if (i != 0 && !(at > rows[i - 1].timestamp)) { + return refusal(Status::RowsUnordered, "rows", -1, at, rows[i - 1].timestamp); + } + if (floor_ms && at < *floor_ms) { + return refusal(Status::FeedRangeInvalid, "rows", 2, at, *floor_ms, end_ms); + } + if (at >= end_ms) { + return refusal(Status::FeedRangeInvalid, "rows", 1, at, floor_ms.value_or(0), end_ms); + } + nc::NativeInterval input; + nc::NativeInterval script; + if (!resolve_row(clock, memo, at, input, script)) { + return refusal(Status::CalendarUnsupported, "rows", -1, at); + } + const BucketWalk::Step step = walk.feed(at, input, script); + if (step.straddled) { + return refusal(Status::CalendarUnsupported, "start_ms", 0, start_ms, + step.straddle_open_ms, step.straddle_next_ms); + } + if (first_window_row == count && at >= start_ms) first_window_row = i; + } + walk.finish(); + return std::nullopt; +} + +SelectedPrimaryPlan plan_rows(const Bar* rows, std::size_t count, const SelectedPlanRequest& request, + const Clock& clock) { + using Status = SelectedPlanStatus; + const std::int64_t start_ms = request.start_ms; + const std::int64_t end_ms = request.end_ms; + const std::int64_t fed_start_ms = request.fed_start_ms; + + nc::SessionDayMemo memo; + + // T, E and F are each the nominal open of the pure native script interval + // that holds them, independent of the rows. Nothing snaps. + const std::int64_t boundaries[3] = {start_ms, end_ms, fed_start_ms}; + for (int bound = 0; bound < 3; ++bound) { + if (std::optional refused = + check_boundary(clock, memo, bound, boundaries[bound])) { + return std::move(*refused); + } + } + + // Pass 1. The memo starts fresh so pass 2 asks the same questions in the + // same order and gets the same answers. + memo.reset(); + BucketWalk walk(start_ms); + std::size_t first_window_row = count; // count: no row at or after T + if (std::optional refused = walk_rows( + rows, count, clock, memo, walk, fed_start_ms, start_ms, end_ms, first_window_row)) { + return std::move(*refused); + } + + const std::uint64_t available = walk.opened_before(); + const std::uint64_t used = std::min(request.preroll_bars, available); + const std::uint64_t trimmed = available - used; + const std::uint64_t window_script = walk.sealed_window(); + + // The first kept row: the first row of the first kept pre-T group, or with + // no kept group the first row at or after T (trim_start is then T itself). + std::size_t trim_index = first_window_row; + std::int64_t trim_key = start_ms; + std::int64_t trim_label = 0; + if (used != 0 && trimmed == 0) { + trim_index = 0; + trim_key = walk.first_key(); + trim_label = walk.first_label(); + } else if (used != 0) { + memo.reset(); + BucketWalk again(start_ms); + bool found = false; + for (std::size_t i = 0; i < count && !found; ++i) { + const std::int64_t at = rows[i].timestamp; + nc::NativeInterval input; + nc::NativeInterval script; + if (!resolve_row(clock, memo, at, input, script)) { + return refusal(Status::InternalError, "rows", -1, at); + } + const BucketWalk::Step step = again.feed(at, input, script); + if (step.straddled) return refusal(Status::InternalError, "rows", -1, at); + if (step.opened && at < start_ms && again.opened_before() == trimmed + 1) { + found = true; + trim_index = i; + trim_key = again.opened_key(); + trim_label = again.opened_label(); + } + } + if (!found) return refusal(Status::InternalError, "rows"); + } + + SelectedPrimaryPlan plan; + plan.status = Status::Ok; + plan.supplied_input_bars = count; + plan.supplied_script_bars = available + window_script; + plan.available_script_bars = available; + plan.used_script_bars = used; + plan.trimmed_script_bars = trimmed; + plan.trim_index = trim_index; + plan.fed_input_bars = count - trim_index; + plan.fed_script_bars = used + window_script; + plan.preroll_input_bars = first_window_row - trim_index; + plan.window_input_bars = count - first_window_row; + plan.window_script_bars = window_script; + plan.trim_start_ms = used != 0 ? trim_key : start_ms; + if (used != 0) { + plan.preroll_first_bar_ms = trim_label; + plan.preroll_last_bar_ms = walk.last_before_label(); + } + if (count != 0) { + plan.supplied_first_data_ms = rows[0].timestamp; + plan.supplied_last_data_ms = rows[count - 1].timestamp; + } + if (trim_index < count) { + plan.fed_first_data_ms = rows[trim_index].timestamp; + plan.fed_last_data_ms = rows[count - 1].timestamp; + } + if (first_window_row < count) { + plan.window_first_data_ms = rows[first_window_row].timestamp; + plan.window_last_data_ms = rows[count - 1].timestamp; + } + plan.shortfall = available < request.preroll_bars; + // A pre-T group nothing seals is completed once at T from its own retained + // aggregate: only when it is a kept group. + plan.complete_pending_preroll_at_horizon = used != 0 && walk.pending_before(); + return plan; +} + +SelectedRetainedCount retained_refusal(SelectedPlanStatus status, const char* option) { + SelectedRetainedCount refused; + refused.status = status; + refused.option = option; + return refused; +} + +// A refusal of the shared admission, boundary check or row walk, in the count's +// two-field form. +SelectedRetainedCount retained_refusal(const SelectedPrimaryPlan& refused_plan) { + SelectedRetainedCount refused; + refused.status = refused_plan.status; + refused.option = refused_plan.option; + return refused; +} + +// The retained count over an admitted clock: T and E each aligned, then the one +// row walk with no F boundary. Every group that opened before T is executed (the +// last pending one by the completion at the accepted T), and so is every window +// group the walk sealed; an unsealed window tail is neither. E bounds none of +// those groups: the walk only refuses a row at or after it. With `unbounded_end` +// E is neither checked nor a bound, so the answer is the one any accepted E gives. +SelectedRetainedCount count_rows(const Bar* rows, std::size_t count, + const SelectedRetainedRequest& request, const Clock& clock, + bool unbounded_end) { + nc::SessionDayMemo memo; + + const std::int64_t boundaries[2] = {request.start_ms, request.end_ms}; + for (int bound = 0; bound < (unbounded_end ? 1 : 2); ++bound) { + if (std::optional refused = + check_boundary(clock, memo, bound, boundaries[bound])) { + return retained_refusal(*refused); + } + } + + memo.reset(); + BucketWalk walk(request.start_ms); + std::size_t first_window_row = count; + const std::int64_t end_ms = + unbounded_end ? std::numeric_limits::max() : request.end_ms; + if (std::optional refused = + walk_rows(rows, count, clock, memo, walk, std::nullopt, request.start_ms, + end_ms, first_window_row)) { + return retained_refusal(*refused); + } + + SelectedRetainedCount counted; + counted.status = SelectedPlanStatus::Ok; + counted.script_bars = walk.opened_before() + walk.sealed_window(); + return counted; +} + +} // namespace + +SelectedPrimaryPlan plan_selected_primary(const Bar* rows, std::size_t count, + const SelectedPlanRequest& request) { + using Status = SelectedPlanStatus; + const std::int64_t start_ms = request.start_ms; + const std::int64_t end_ms = request.end_ms; + const std::int64_t fed_start_ms = request.fed_start_ms; + + if (!primary_token(request.script_tf)) return refusal(Status::RequestInvalid, "script_tf"); + if (request.input_tf.empty()) return refusal(Status::RequestInvalid, "input_tf"); + if (!in_i53(start_ms)) return refusal(Status::RequestInvalid, "start_ms", 0, start_ms); + if (!in_i53(end_ms)) return refusal(Status::RequestInvalid, "end_ms", 1, end_ms); + if (!in_i53(fed_start_ms)) { + return refusal(Status::RequestInvalid, "fed_start_ms", 2, fed_start_ms); + } + if (!(start_ms < end_ms)) { + return refusal(Status::RequestInvalid, "end_ms", 1, end_ms, start_ms); + } + if (!(fed_start_ms <= start_ms)) { + return refusal(Status::RequestInvalid, "fed_start_ms", 2, fed_start_ms, 0, start_ms); + } + if (request.preroll_bars > kMaxPrerollBars) { + return refusal(Status::RequestInvalid, "preroll_bars", -1, + static_cast(request.preroll_bars)); + } + if ((rows == nullptr && count != 0) + || static_cast(count) > static_cast(kI53Max)) { + return refusal(Status::RequestInvalid, "rows"); + } + + // The clock identity is the first step of admit_clock. + Clock clock; + if (std::optional refused = admit_clock(request, clock)) { + return std::move(*refused); + } + try { + return plan_rows(rows, count, request, clock); + } catch (const std::bad_alloc&) { + throw; + } catch (...) { + return refusal(Status::InternalError, "calendar"); + } +} + +namespace { + +// The two retained entries: the same request checks, clock admission and row walk, +// with E checked and bounding the rows unless `unbounded_end`. +SelectedRetainedCount count_retained(const Bar* rows, std::size_t count, + const SelectedRetainedRequest& request, bool unbounded_end) { + using Status = SelectedPlanStatus; + + if (!primary_token(request.script_tf)) { + return retained_refusal(Status::RequestInvalid, "script_tf"); + } + if (request.input_tf.empty()) return retained_refusal(Status::RequestInvalid, "input_tf"); + if (!in_i53(request.start_ms)) return retained_refusal(Status::RequestInvalid, "start_ms"); + if (!unbounded_end) { + if (!in_i53(request.end_ms)) return retained_refusal(Status::RequestInvalid, "end_ms"); + if (!(request.start_ms < request.end_ms)) { + return retained_refusal(Status::RequestInvalid, "end_ms"); + } + } + if ((rows == nullptr && count != 0) + || static_cast(count) > static_cast(kI53Max)) { + return retained_refusal(Status::RequestInvalid, "rows"); + } + + // The clock identity is the first step of admit_clock. + Clock clock; + if (std::optional refused = admit_clock(request, clock)) { + return retained_refusal(*refused); + } + try { + return count_rows(rows, count, request, clock, unbounded_end); + } catch (const std::bad_alloc&) { + throw; + } catch (...) { + return retained_refusal(Status::InternalError, "calendar"); + } +} + +} // namespace + +SelectedRetainedCount count_selected_retained_primary(const Bar* rows, std::size_t count, + const SelectedRetainedRequest& request) { + return count_retained(rows, count, request, /*unbounded_end=*/false); +} + +SelectedRetainedCount count_selected_executed_primary(const Bar* rows, std::size_t count, + const SelectedRetainedRequest& request) { + return count_retained(rows, count, request, /*unbounded_end=*/true); +} + +} // inline namespace selected_window_plan_v1 +} // namespace pineforge diff --git a/src/selected_window_plan_c.cpp b/src/selected_window_plan_c.cpp new file mode 100644 index 000000000..89464882f --- /dev/null +++ b/src/selected_window_plan_c.cpp @@ -0,0 +1,273 @@ +/* + * selected_window_plan_c.cpp - the C bridge to the selected primary planner. + * include/pineforge/selected_window_plan.h holds the contract and + * include/pineforge/selected_window_plan.hpp the planner's. + * + * The bridge only moves values across the boundary. It validates the two + * descriptors, copies the five request strings into a SelectedPlanRequest, reads + * the caller's pf_bar_t rows in place as pineforge::Bar (the c_abi.cpp + * convention; no row is copied), calls pineforge::plan_selected_primary once and + * maps the plan into a local, zeroed result that is committed to the caller with + * one whole assignment. It keeps no state, constructs no strategy or engine, + * calls no host or library callback and repeats no calendar, count or hash + * logic. It is compiled only into the standalone window-plan shared + * library, which links the kernel archive and nothing of the source layer: this + * file is in neither PINEFORGE_KERNEL_SOURCES nor libpineforge.a, so no existing + * artifact gains a symbol. + */ + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include + +/* ── Layout ─────────────────────────────────────────────────────── */ + +/* pf_bar_t is read in place as pineforge::Bar, the convention every feed setter + * in c_abi.cpp uses. The two are asserted layout-identical here as well, so this + * library does not lean on that file's asserts, and against the documented + * layout itself: five doubles, then the int64 timestamp. */ +static_assert(sizeof(pf_bar_t) == sizeof(pineforge::Bar), + "pf_bar_t / pineforge::Bar size mismatch"); +static_assert(alignof(pf_bar_t) == alignof(pineforge::Bar), + "pf_bar_t / pineforge::Bar alignment mismatch"); +static_assert(offsetof(pf_bar_t, open) == offsetof(pineforge::Bar, open), + "pf_bar_t::open offset mismatch"); +static_assert(offsetof(pf_bar_t, high) == offsetof(pineforge::Bar, high), + "pf_bar_t::high offset mismatch"); +static_assert(offsetof(pf_bar_t, low) == offsetof(pineforge::Bar, low), + "pf_bar_t::low offset mismatch"); +static_assert(offsetof(pf_bar_t, close) == offsetof(pineforge::Bar, close), + "pf_bar_t::close offset mismatch"); +static_assert(offsetof(pf_bar_t, volume) == offsetof(pineforge::Bar, volume), + "pf_bar_t::volume offset mismatch"); +static_assert(offsetof(pf_bar_t, timestamp) == offsetof(pineforge::Bar, timestamp), + "pf_bar_t::timestamp offset mismatch"); +static_assert(sizeof(double) == 8 && sizeof(std::int64_t) == 8, + "pf_bar_t is five 8-byte doubles then an 8-byte timestamp"); +static_assert(sizeof(pf_bar_t) == 48, "pf_bar_t size moved"); +static_assert(offsetof(pf_bar_t, open) == 0 && offsetof(pf_bar_t, high) == 8 + && offsetof(pf_bar_t, low) == 16 && offsetof(pf_bar_t, close) == 24 + && offsetof(pf_bar_t, volume) == 32 && offsetof(pf_bar_t, timestamp) == 40, + "pf_bar_t field offsets moved"); + +/* The frozen 64-bit layout of the two descriptors. The executed C, C++ and + * ctypes layout assertions are separate (tests/test_selected_window_plan_c.cpp). */ +#if UINTPTR_MAX == 0xFFFFFFFFFFFFFFFFu +static_assert(sizeof(pf_selected_plan_request_v1) == 80, + "pf_selected_plan_request_v1 size moved"); +static_assert(offsetof(pf_selected_plan_request_v1, input_tf) == 40, + "pf_selected_plan_request_v1::input_tf offset moved"); +static_assert(sizeof(pf_selected_plan_result_v1) == 248, + "pf_selected_plan_result_v1 size moved"); +static_assert(offsetof(pf_selected_plan_result_v1, present_mask) == 200, + "pf_selected_plan_result_v1::present_mask offset moved"); +static_assert(offsetof(pf_selected_plan_result_v1, option) == 216, + "pf_selected_plan_result_v1::option offset moved"); +#endif +static_assert(sizeof(pf_selected_plan_result_v1::option) == 32, + "pf_selected_plan_result_v1::option must be 32 bytes"); + +namespace { + +constexpr std::uint32_t kVersion = 1; +constexpr std::size_t kOptionCapacity = sizeof(pf_selected_plan_result_v1::option); + +/* ── Descriptors ────────────────────────────────────────────────── */ + +/* Only reads the caller's output header; nothing is written before the result + * is whole. */ +bool output_header_valid(const pf_selected_plan_result_v1* out) noexcept { + return out != nullptr + && out->struct_size == sizeof(pf_selected_plan_result_v1) + && out->version == kVersion; +} + +bool request_valid(const pf_selected_plan_request_v1* request) noexcept { + return request != nullptr + && request->struct_size == sizeof(pf_selected_plan_request_v1) + && request->version == kVersion + && request->feed_tolerant <= 1u + && request->input_tf != nullptr + && request->script_tf != nullptr + && request->chart_timezone != nullptr + && request->engine_timezone != nullptr + && request->session != nullptr; +} + +/* The count must be representable as a size_t (always true on LP64, where the + * round trip below is the identity). */ +bool row_count_of(std::uint64_t count, std::size_t& rows) noexcept { + rows = static_cast(count); + return static_cast(rows) == count; +} + +/* String conversion: each NUL-terminated string is copied into the C++ request. + * An allocation failure propagates as std::bad_alloc. */ +pineforge::SelectedPlanRequest planner_request_of(const pf_selected_plan_request_v1& request) { + pineforge::SelectedPlanRequest planner{}; + planner.start_ms = request.start_ms; + planner.end_ms = request.end_ms; + planner.fed_start_ms = request.fed_start_ms; + planner.preroll_bars = request.preroll_bars; + planner.input_tf = request.input_tf; + planner.script_tf = request.script_tf; + planner.chart_timezone = request.chart_timezone; + planner.engine_timezone = request.engine_timezone; + planner.session = request.session; + planner.feed_tolerant = request.feed_tolerant != 0u; + return planner; +} + +/* ── Result ─────────────────────────────────────────────────────── */ + +/* The C++ enumerators mapped explicitly to the frozen words (0 Ok, 1 + * RequestInvalid, 2 CalendarUnsupported, 3 BoundaryUnaligned, 4 + * FeedRangeInvalid, 5 RowsUnordered, 6 InternalError). The word is not inferred + * from the enumerator's value or width, and a new enumerator is a compile error + * here. A value outside the enumeration is a bridge failure. */ +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wswitch" +#endif +bool status_word_of(pineforge::SelectedPlanStatus status, std::int32_t& word) noexcept { + switch (status) { + case pineforge::SelectedPlanStatus::Ok: + word = 0; + return true; + case pineforge::SelectedPlanStatus::RequestInvalid: + word = 1; + return true; + case pineforge::SelectedPlanStatus::CalendarUnsupported: + word = 2; + return true; + case pineforge::SelectedPlanStatus::BoundaryUnaligned: + word = 3; + return true; + case pineforge::SelectedPlanStatus::FeedRangeInvalid: + word = 4; + return true; + case pineforge::SelectedPlanStatus::RowsUnordered: + word = 5; + return true; + case pineforge::SelectedPlanStatus::InternalError: + word = 6; + return true; + } + return false; +} +#if defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif + +/* The option is ASCII, NUL-terminated and zero-padded in 32 bytes. One that is + * longer than 31 characters, or holds a NUL or a non-ASCII byte, is a bridge + * failure and is never truncated into another error. */ +bool copy_option(const std::string& option, char (&target)[kOptionCapacity]) noexcept { + if (option.size() >= kOptionCapacity) return false; + for (const char ch : option) { + const unsigned char byte = static_cast(ch); + if (byte == 0u || byte >= 0x80u) return false; + } + std::memset(target, 0, kOptionCapacity); + std::memcpy(target, option.data(), option.size()); + return true; +} + +/* An optional timestamp: absent leaves the zeroed slot and its mask bit clear. */ +void put_timestamp(const std::optional& source, unsigned bit, std::int64_t& slot, + std::uint32_t& mask) noexcept { + if (!source.has_value()) return; + slot = *source; + mask |= std::uint32_t{1} << bit; +} + +/* Maps a plan into the zeroed local result. Returns false, leaving the caller to + * report a bridge failure, when the plan breaks a bridge invariant. */ +bool fill_result(const pineforge::SelectedPrimaryPlan& plan, + pf_selected_plan_result_v1& local) noexcept { + std::memset(&local, 0, sizeof local); + std::int32_t status = 0; + if (!status_word_of(plan.status, status)) return false; + if (plan.bound < -1 || plan.bound > 2) return false; + if (!copy_option(plan.option, local.option)) return false; + + local.struct_size = static_cast(sizeof local); + local.version = kVersion; + local.status = status; + local.bound = static_cast(plan.bound); + local.value_ms = plan.value_ms; + local.previous_boundary_ms = plan.previous_boundary_ms; + local.next_boundary_ms = plan.next_boundary_ms; + local.supplied_input_bars = plan.supplied_input_bars; + local.supplied_script_bars = plan.supplied_script_bars; + local.available_script_bars = plan.available_script_bars; + local.used_script_bars = plan.used_script_bars; + local.trimmed_script_bars = plan.trimmed_script_bars; + local.trim_index = plan.trim_index; + local.fed_input_bars = plan.fed_input_bars; + local.fed_script_bars = plan.fed_script_bars; + local.preroll_input_bars = plan.preroll_input_bars; + local.window_input_bars = plan.window_input_bars; + local.window_script_bars = plan.window_script_bars; + local.trim_start_ms = plan.trim_start_ms; + + /* Bits 0..7 are the eight optional timestamps in the result's field order. */ + std::uint32_t mask = 0; + put_timestamp(plan.preroll_first_bar_ms, 0, local.preroll_first_bar_ms, mask); + put_timestamp(plan.preroll_last_bar_ms, 1, local.preroll_last_bar_ms, mask); + put_timestamp(plan.supplied_first_data_ms, 2, local.supplied_first_data_ms, mask); + put_timestamp(plan.supplied_last_data_ms, 3, local.supplied_last_data_ms, mask); + put_timestamp(plan.fed_first_data_ms, 4, local.fed_first_data_ms, mask); + put_timestamp(plan.fed_last_data_ms, 5, local.fed_last_data_ms, mask); + put_timestamp(plan.window_first_data_ms, 6, local.window_first_data_ms, mask); + put_timestamp(plan.window_last_data_ms, 7, local.window_last_data_ms, mask); + local.present_mask = mask; + + local.shortfall = plan.shortfall ? 1u : 0u; + local.complete_pending_preroll_at_horizon = + plan.complete_pending_preroll_at_horizon ? 1u : 0u; + local.reserved = 0u; + return true; +} + +} // namespace + +/* ── Exports ────────────────────────────────────────────────────── */ + +PF_API uint32_t pf_selected_plan_version(void) { return kVersion; } + +PF_API int pf_plan_selected_primary_v1(const pf_selected_plan_request_v1* request, + const pf_bar_t* rows, uint64_t count, + pf_selected_plan_result_v1* out) { + /* Every refusal of the descriptors, and of the count, is -1 and happens + * before anything could be written. */ + if (!output_header_valid(out) || !request_valid(request)) return -1; + std::size_t row_count = 0; + if (!row_count_of(count, row_count)) return -1; + + try { + const pineforge::SelectedPlanRequest planner = planner_request_of(*request); + /* The rows are never dereferenced here: a NULL pointer with a positive + * count reaches the planner, which refuses it as RequestInvalid. */ + const auto* native = reinterpret_cast(rows); + const pineforge::SelectedPrimaryPlan plan = + pineforge::plan_selected_primary(native, row_count, planner); + + pf_selected_plan_result_v1 local; + if (!fill_result(plan, local)) return -3; + *out = local; /* the one write: a whole result or nothing */ + return 0; + } catch (const std::bad_alloc&) { + return -2; + } catch (...) { + return -3; + } +} diff --git a/src/source/pine_adapter.cpp b/src/source/pine_adapter.cpp index bbd2dd8ba..6809add26 100644 --- a/src/source/pine_adapter.cpp +++ b/src/source/pine_adapter.cpp @@ -1999,10 +1999,13 @@ void PineExecutionAdapter::suspend_coof_declined_reversal_at_open( void PineExecutionAdapter::hold_reversal_pair_brackets(const SourceId& from_entry) { const auto point = detail::callback_point(require_host()); - if (!point) return; + // The cause's ordinal is the kernel's own event number, the live frame's even + // where `point` is a retained one being replayed. + const auto live = detail::live_callback_point(require_host()); + if (!point || !live) return; const auto domain = config_.calc_on_order_fills ? exit_legs::Domain::FillRecalc : exit_legs::Domain::Ordinary; - const exit_legs::Frame cause{point->decision.coordinate.ordinal, + const exit_legs::Frame cause{live->decision.coordinate.ordinal, point->decision.coordinate.interval_index, domain, exit_legs::Phase::Observation}; // As in suspend_brackets_for_reversal: no erased lifecycle is read again. PINEFORGE_AUDIT_PLACEMENT_SCAN(placement_, @@ -2062,7 +2065,10 @@ void PineExecutionAdapter::retire_cancelled_exits( const SourceId* id, const std::vector>& cancelled) { const auto point = detail::callback_point(require_host()); - if (!point) return; + // The sweep's event ordinal below is the kernel's own, the live frame's even + // where `point` is a retained one being replayed (the bar stays `point`'s). + const auto live = detail::live_callback_point(require_host()); + if (!point || !live) return; const auto domain = config_.calc_on_order_fills ? exit_legs::Domain::FillRecalc : exit_legs::Domain::Ordinary; const auto retire_legs = [&](PlacementSnapshot& candidate, std::uint64_t event) { @@ -2095,7 +2101,7 @@ void PineExecutionAdapter::retire_cancelled_exits( for (auto row : placement_) { auto& candidate = row.second; if (id && candidate.source_id != *id) continue; - retire_legs(candidate, point->decision.coordinate.ordinal); + retire_legs(candidate, live->decision.coordinate.ordinal); } } @@ -3225,7 +3231,9 @@ bool PineExecutionAdapter::core_sizing_price_matches( const PineSizingSnapshot& sizing, bool is_long) const { const double tick = staged_.syminfo.mintick; if (!finite_positive(tick) || !finite_positive(sizing.price)) return false; - const auto point = detail::callback_point(require_host()); + // The core freezes at its own live frame, never at a retained one a replay + // is reading, so this prediction reads the live point. + const auto point = detail::live_callback_point(require_host()); if (!point || !finite_positive(point->price)) return false; // The core freezes the quotient at the FX of the acceptance coordinate (a // script calculation's coordinate is the next bar's open); the source @@ -21843,6 +21851,14 @@ void PineExecutionAdapter::order_open_marketable_limit_entries( void PineExecutionAdapter::apply_terminal_explicit_market_policy( const NativeDecisionContext& context) { + apply_terminal_explicit_market_policy(context, true); +} + +// With `execute_now` false the pass only submits: the reconciled entries stay +// working for the matcher, which fills them where a source replay of the last +// pre-T bar opens. The cancellations and re-submissions are the same either way. +void PineExecutionAdapter::apply_terminal_explicit_market_policy( + const NativeDecisionContext& context, bool execute_now) { if (!config_.process_orders_on_close || config_.pyramiding != 0 || !modeled_input()) { return; @@ -21988,8 +22004,10 @@ void PineExecutionAdapter::apply_terminal_explicit_market_policy( replacement_key); if (accepted) { simulated_sign = requested_sign; - (void)require_host().execute_current( - {*accepted, NativeCurrentPriceRule::NearestTick}); + if (execute_now) { + (void)require_host().execute_current( + {*accepted, NativeCurrentPriceRule::NearestTick}); + } } } } @@ -26228,6 +26246,32 @@ void PineExecutionAdapter::set_risk_max_intraday_loss(double value, bool percent if (percent) risk_.max_intraday_loss_percent = true; } void PineExecutionAdapter::set_risk_max_position_size(double value) noexcept { risk_.max_position_size = value; } +PineReplayRiskConfig PineExecutionAdapter::capture_replay_risk_config() const noexcept { + PineReplayRiskConfig snapshot; + snapshot.direction = risk_.direction; + snapshot.max_cons_loss_days = risk_.max_cons_loss_days; + snapshot.max_drawdown = risk_.max_drawdown; + snapshot.max_drawdown_percent = risk_.max_drawdown_percent; + snapshot.max_intraday_loss = risk_.max_intraday_loss; + snapshot.max_intraday_loss_percent = risk_.max_intraday_loss_percent; + snapshot.max_position_size = risk_.max_position_size; + snapshot.cap_attachment = cap.attachment(); + snapshot.cap_configuration = cap.configuration(); + snapshot.cap_declared = cap.declared(); + return snapshot; +} +void PineExecutionAdapter::restore_replay_risk_config( + const PineReplayRiskConfig& snapshot) noexcept { + risk_.direction = snapshot.direction; + risk_.max_cons_loss_days = snapshot.max_cons_loss_days; + risk_.max_drawdown = snapshot.max_drawdown; + risk_.max_drawdown_percent = snapshot.max_drawdown_percent; + risk_.max_intraday_loss = snapshot.max_intraday_loss; + risk_.max_intraday_loss_percent = snapshot.max_intraday_loss_percent; + risk_.max_position_size = snapshot.max_position_size; + cap.restore_replay_configuration(snapshot.cap_attachment, snapshot.cap_configuration, + snapshot.cap_declared); +} void PineExecutionAdapter::set_margin_call_enabled(bool enabled) noexcept { source_margin_call_enabled_ = enabled; } diff --git a/src/source/pine_host_reads.hpp b/src/source/pine_host_reads.hpp index 2c4dfbffb..df5012a75 100644 --- a/src/source/pine_host_reads.hpp +++ b/src/source/pine_host_reads.hpp @@ -19,6 +19,7 @@ // Not installed API. #include "../native_execution_consumer.hpp" +#include "pine_intent_state.hpp" #include #include @@ -66,11 +67,21 @@ inline NativePhysicalPosition run_position(const NativeStrategyHost& host) { } // current_execution_point(), by reference: null where it answers nullopt. -// Valid inside the callback that asked. -inline const NativeCurrentPointView* callback_point(const NativeStrategyHost& host) { +// Valid inside the callback that asked. The consumer's own frame, which the +// kernel acts on whether or not the source layer is replaying a retained one. +inline const NativeCurrentPointView* live_callback_point(const NativeStrategyHost& host) { return run_consumer(host).current_point(); } +// The point the source layer reads at a callback: the retained point of the +// row a selected run is replaying (selected_replay_point), else the live one. +// With nothing replaying, as in every run with the selected window off, it is +// live_callback_point's pointer itself. +inline const NativeCurrentPointView* callback_point(const NativeStrategyHost& host) { + if (const NativeCurrentPointView* replayed = selected_replay_point(host)) return replayed; + return live_callback_point(host); +} + // The source layer's state parked with the consumer for one run (R5 lane // PERF-P7's NativeHostCache: the adapter's lookup index derives from this, // pine_adapter.cpp). The consumer drops it at every run begin and never reads diff --git a/src/source/pine_intent_state.hpp b/src/source/pine_intent_state.hpp new file mode 100644 index 000000000..d07adcbea --- /dev/null +++ b/src/source/pine_intent_state.hpp @@ -0,0 +1,340 @@ +#pragma once + +// Selected-window capture: the source layer's record of the last pre-roll +// evaluation, parked with the consumer for one run. A selected run feeds the +// bars before the window's start T through the script body with the book +// inactive. The nine command doors and the six risk setters of PineStrategyHost +// append here instead of reaching the adapter (the risk setters still apply +// live as well), and a later step replays the rows at the window's first Open. +// This header is the representation, the two seam accessors +// (selected_intents, selected_replay_point) and the capture helpers the host's +// hooks share. It replays nothing and places nothing. +// +// The state is the consumer's mandatory NativeHostState slot, never the +// optional NativeHostCache: it is allocated outside the kernel, keyed by owner +// and run, and an allocation failure surfaces where it is adopted. Nothing here +// grows a class (no installed member, no vtable) and there is no process-global +// table. A run that is not selected never reaches any of it: every entry checks +// NativeExecutionConsumer::selected_window_admitted first, with a descriptor on +// the stack, so OFF allocates nothing and folds nothing. +// Not installed API. + +#include "../native_execution_consumer.hpp" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace pineforge::source::detail { + +// The nine command doors of PineStrategyHost (pine_strategy_commands.cpp), one +// payload each, over the door's own argument types. Strings are owned copies; +// nothing borrows the script's arguments. The two strategy_close overloads are +// distinct alternatives: only the second carries the callsite token. +struct IntentEntry { + std::string id; + bool is_long; + double limit_price; + double stop_price; + double qty; + std::string comment; + std::string oca_name; + int oca_type; + int qty_type; +}; +struct IntentClose { + std::string id; + std::string comment; + double qty; + double qty_percent; + bool immediately; +}; +struct IntentCloseToken { + std::string id; + std::string comment; + double qty; + double qty_percent; + bool immediately; + std::uint64_t callsite_token; +}; +struct IntentCloseAll {}; +struct IntentExit { + std::string id; + std::string from_entry; + double limit_price; + double stop_price; + double trail_points; + double trail_offset; + double trail_price; + double qty_percent; + std::string comment; + double qty; + std::string oca_name; + double profit_ticks; + double loss_ticks; +}; +struct IntentExitCancelBracket { + std::string exit_id; + std::string from_entry; + std::string comment; +}; +struct IntentCancel { + std::string id; +}; +struct IntentCancelAll {}; +struct IntentOrder { + std::string id; + bool is_long; + double qty; + double limit_price; + double stop_price; + std::string oca_name; + int oca_type; +}; + +// The six risk setter statements of PineStrategyHost (set_pine_risk_*), over +// their own argument types. Kept as ordered rows beside the commands because +// the generated order of statements is arbitrary and the percent flags are +// sticky in the setters. +struct IntentRiskDirection { + int direction; +}; +struct IntentRiskMaxConsLossDays { + int value; +}; +struct IntentRiskMaxDrawdown { + double value; + bool percent; +}; +struct IntentRiskMaxIntradayLoss { + double value; + bool percent; +}; +struct IntentRiskMaxIntradayFilledOrders { + int limit; +}; +struct IntentRiskMaxPositionSize { + double value; +}; + +// The variant index is the row's tag and is folded into the hash. +using PineIntentPayload = std::variant< + IntentEntry, IntentClose, IntentCloseToken, IntentCloseAll, IntentExit, + IntentExitCancelBracket, IntentCancel, IntentCancelAll, IntentOrder, + IntentRiskDirection, IntentRiskMaxConsLossDays, IntentRiskMaxDrawdown, + IntentRiskMaxIntradayLoss, IntentRiskMaxIntradayFilledOrders, + IntentRiskMaxPositionSize>; + +// One captured call: what the script passed, the configuration the host held +// at the call, and the live point the call ran at. All owned values. +struct PineIntentRow { + PineIntentPayload payload; + PineStrategyConfig config; + NativeCurrentPointView point; +}; + +// The state's kind tag (identity only, never read for its value). +inline const char kPineIntentStateKind = 0; + +// The last pre-roll evaluation of the selected run `run` served by `owner`. +// Written only while `capturing`, which spans exactly one evaluation of a bar +// labelled before T. `rows` is that evaluation's alone: every pre-roll +// evaluation resets it, even a script that calls nothing, and a suppressed +// pre-roll tail resets it to an empty last-present evaluation. `replay_point` +// is transient (the replay's own RAII sets and clears it); it is neither +// hashed nor adopted. +struct PineIntentState : NativeHostState { + PineIntentState(const NativeStrategyHost* owner_host, std::uint64_t native_run) noexcept + : NativeHostState(&kPineIntentStateKind), owner(owner_host), run(native_run) {} + + const NativeStrategyHost* owner = nullptr; + std::uint64_t run = 0; + bool has_evaluation = false; + bool capturing = false; + bool replayed = false; + std::int64_t label_ms = 0; + PineStrategyConfig start_config{}; + PineStrategyConfig end_config{}; + PineReplayRiskConfig start_risk{}; + PineReplayRiskConfig end_risk{}; + NativeCurrentPointView terminal_point{}; + std::vector rows; + const NativeCurrentPointView* replay_point = nullptr; +}; + +// A selected pre-roll call the capture cannot take is the engine's fault, never +// the script's: it fails the run. It never drops the intent and never replays +// it from a default point. +[[noreturn]] inline void intent_outside_bracket() { + throw coded(RunFailureCode::engine_invariant, {}, + "selected pre-roll command outside its capture bracket"); +} +[[noreturn]] inline void intent_without_point() { + throw coded(RunFailureCode::engine_invariant, {}, + "selected pre-roll capture without a live callback point"); +} +[[noreturn]] inline void intent_without_state() { + throw coded(RunFailureCode::engine_invariant, {}, + "selected pre-roll evaluation without its intent state"); +} +[[noreturn]] inline void intent_without_spec() { + throw coded(RunFailureCode::engine_invariant, {}, + "selected run begin without a running spec for its intent state"); +} + +// The consumer's live frame point: valid inside the callback that asked, and +// the point a row retains. Read from the consumer itself, never through the +// adapter's replay-aware read. +inline const NativeCurrentPointView* intent_live_point(const NativeStrategyHost& host) noexcept { + return NativeExecutionConsumer::bound(host).current_point(); +} + +// The intent state of the selected run `host` serves, or null: the run is not +// selected (OFF, or no consumer yet), is not running, parks no state, or parks +// one of another kind, owner or native run. A mismatch is never reused. +// Allocates nothing, so a const or hash read may ask. +inline PineIntentState* selected_intents(const NativeStrategyHost& host) noexcept { + pf_selected_window_config_v1 window{}; + if (!NativeExecutionConsumer::selected_window_admitted(host, &window)) return nullptr; + const NativeExecutionConsumer& consumer = NativeExecutionConsumer::bound(host); + if (!consumer.running()) return nullptr; + NativeHostState* const parked = consumer.host_state(); + if (parked == nullptr || parked->kind() != &kPineIntentStateKind) return nullptr; + PineIntentState* const state = static_cast(parked); + const NativeRunSpec* const spec = consumer.state_spec(); + if (spec == nullptr || state->owner != &host || state->run != spec->identity.run_number) + return nullptr; + return state; +} + +// The retained point of the row being replayed, or null: no selected state, or +// no replay open. The private seam the adapter's read path consumes. Allocates +// nothing. +inline const NativeCurrentPointView* selected_replay_point( + const NativeStrategyHost& host) noexcept { + const PineIntentState* const state = selected_intents(host); + return state != nullptr ? state->replay_point : nullptr; +} + +// Parks the state of a selected run, from on_native_run_begin: after the +// consumer's final pre-host reset, so it belongs to this run alone. Does nothing +// for a run that is not selected. The state is allocated before it is adopted +// (the adoption cannot fail), so an allocation failure propagates out of the +// begin hook and nothing is dropped silently. +inline void adopt_selected_intents(NativeStrategyHost& host) { + pf_selected_window_config_v1 window{}; + if (!NativeExecutionConsumer::selected_window_admitted(host, &window)) return; + NativeExecutionConsumer& consumer = NativeExecutionConsumer::bound(host); + const NativeRunSpec* const spec = consumer.state_spec(); + if (spec == nullptr) intent_without_spec(); + consumer.adopt_host_state( + std::make_unique(&host, spec->identity.run_number)); +} + +// One command door's pre-roll capture. False: the call is not a selected +// pre-roll call (OFF, or a label inside the window), no payload was built, and +// the door runs exactly as it did. True: the payload built by `make_payload`, +// this call's configuration and the live point were appended, and the door +// returns before its legacy trading-window gate and before the adapter. A +// selected pre-roll call outside a capture bracket, with no state or no live +// point, fails as engine_invariant. +template +bool capture_pre_roll_command(const NativeStrategyHost& host, std::int64_t label_ms, + const PineStrategyConfig& config, MakePayload&& make_payload) { + pf_selected_window_config_v1 window{}; + if (!NativeExecutionConsumer::selected_window_admitted(host, &window) + || label_ms >= window.start_ms) { + return false; + } + PineIntentState* const state = selected_intents(host); + if (state == nullptr || !state->capturing) intent_outside_bracket(); + const NativeCurrentPointView* const point = intent_live_point(host); + if (point == nullptr) intent_without_point(); + state->rows.push_back(PineIntentRow{make_payload(), config, *point}); + return true; +} + +// One risk setter statement's capture: a row while a pre-roll evaluation +// captures, nothing otherwise. The setter applies live either way. +template +void capture_risk_row(const NativeStrategyHost& host, const PineStrategyConfig& config, + Payload&& payload) { + PineIntentState* const state = selected_intents(host); + if (state == nullptr || !state->capturing) return; + const NativeCurrentPointView* const point = intent_live_point(host); + if (point == nullptr) intent_without_point(); + state->rows.push_back(PineIntentRow{std::forward(payload), config, *point}); +} + +// The recording target of a source evaluation labelled `label_ms`: the intent +// state and the live point it runs at. Both are null when the evaluation is not +// a selected pre-roll one (OFF, or a label inside the window). A selected +// pre-roll evaluation without its state or its point fails as engine_invariant. +struct PreRollEvaluation { + PineIntentState* state = nullptr; + const NativeCurrentPointView* point = nullptr; +}; +inline PreRollEvaluation selected_pre_roll_evaluation(const NativeStrategyHost& host, + std::int64_t label_ms) { + PreRollEvaluation found; + pf_selected_window_config_v1 window{}; + if (!NativeExecutionConsumer::selected_window_admitted(host, &window) + || label_ms >= window.start_ms) { + return found; + } + found.state = selected_intents(host); + if (found.state == nullptr) intent_without_state(); + found.point = intent_live_point(host); + if (found.point == nullptr) intent_without_point(); + return found; +} + +// Resets the buffer to one evaluation labelled `label_ms` that holds no rows: +// the bracket is closed, the configuration and risk configuration are the +// current ones at both ends, and the terminal point is the live one. A pre-roll +// evaluation opens its bracket on top of this; a suppressed tail leaves it as +// the empty last-present evaluation, so no earlier bar's signal outlives it. +inline void reset_intent_evaluation(PineIntentState& state, std::int64_t label_ms, + const PineStrategyConfig& config, + const PineReplayRiskConfig& risk, + const NativeCurrentPointView& point) noexcept { + state.rows.clear(); + state.has_evaluation = true; + state.capturing = false; + state.replayed = false; + state.label_ms = label_ms; + state.start_config = config; + state.end_config = config; + state.start_risk = risk; + state.end_risk = risk; + state.terminal_point = point; +} + +// Closes a capture bracket when its evaluation's scope ends, normally or by +// exception. Only a flag is written, so the destructor cannot throw. +class IntentCaptureScope { +public: + IntentCaptureScope(const NativeStrategyHost& host, bool opened) noexcept + : host_(&host), opened_(opened) {} + IntentCaptureScope(const IntentCaptureScope&) = delete; + IntentCaptureScope& operator=(const IntentCaptureScope&) = delete; + ~IntentCaptureScope() { + if (!opened_) return; + if (PineIntentState* const state = selected_intents(*host_)) state->capturing = false; + } + +private: + const NativeStrategyHost* host_; + bool opened_; +}; + +} // namespace pineforge::source::detail diff --git a/src/source/pine_scheduler_native.cpp b/src/source/pine_scheduler_native.cpp index 75c11ff73..32a9f4d6d 100644 --- a/src/source/pine_scheduler_native.cpp +++ b/src/source/pine_scheduler_native.cpp @@ -5,8 +5,11 @@ #include "pine_host_reads.hpp" #include +#include +#include #include +#include #include #include #include @@ -131,6 +134,45 @@ void PineScheduler::commit_coof_script_state(PineStrategyHost& host) { host.commit_script_state(); } +namespace { +// The script bars a SELECTED admitted batch run executes over its retained +// primary rows: the kernel's count_selected_executed_primary, which walks them as +// the native consumer does (the groups that open before T, the last pending one +// completed at the accepted T, and the window groups its seal rules execute; an +// unsealed window tail adds none). The request is the run's own: its window (E +// unchecked: it bounds no such group), its spec's timeframes, timezone and session, +// the host's retained chart timezone and its slot-label policy. A refusal of the +// count after admission, or a count the scheduler's int cannot hold, is the +// engine's fault and fails as engine_invariant; no count is guessed. +int selected_executed_source_bars(const PineStrategyHost& host, const NativeRunSpec* spec, + const pf_selected_window_config_v1& window, + const std::vector& retained) { + if (spec == nullptr) { + throw coded(RunFailureCode::engine_invariant, {}, + "selected run begin without a running spec for its " + "source tail count"); + } + SelectedRetainedRequest request; + request.start_ms = window.start_ms; + request.end_ms = window.end_ms; + request.input_tf = spec->input_tf; + request.script_tf = spec->script_tf; + request.chart_timezone = host.chart_timezone(); + request.engine_timezone = spec->timezone; + request.session = spec->session; + request.feed_tolerant = spec->slot_label_policy == NativeSlotLabelPolicy::FeedTolerant; + const SelectedRetainedCount counted = + count_selected_executed_primary(retained.data(), retained.size(), request); + if (counted.status != SelectedPlanStatus::Ok + || counted.script_bars > static_cast(std::numeric_limits::max())) { + throw coded(RunFailureCode::engine_invariant, {}, + "selected run source tail count refused or out of range " + "after admission"); + } + return static_cast(counted.script_bars); +} +} // namespace + void PineScheduler::run_begin(PineStrategyHost& host) { reset_language(); const bool static_eligible = !retained_.is_stream && !retained_.bar_magnifier @@ -157,6 +199,20 @@ void PineScheduler::run_begin(PineStrategyHost& host) { for (const auto complete : input_script_completes_) { expected_source_bars_ += complete != 0U ? 1 : 0; } + // A selected admitted batch run executes the script bars of the consumer's + // own bucket walk, which the aggregator's per-input hint above does not + // model: the last pending pre-T group is completed at the accepted T and an + // unsealed window tail is never executed. The tail expectations + // (terminal_source_bar, the probe tail) read the executed count. An OFF run + // and a stream never reach this and keep the hint; the per-input completion + // vectors are unchanged either way. + if (!retained_.is_stream && state.phase == NativeRunPhase::Batch) { + pf_selected_window_config_v1 window{}; + if (NativeExecutionConsumer::selected_window_admitted(host, &window)) { + expected_source_bars_ = + selected_executed_source_bars(host, state.spec, window, retained_.bars); + } + } host.stream_warmup_mode_ = retained_.is_stream; host.scheduler_prepare_script_run(retained_.bars, static_eligible, expected_source_bars_, !needs_aggregation); diff --git a/src/source/pine_state_hash.cpp b/src/source/pine_state_hash.cpp index 3304f7038..f21a63c8a 100644 --- a/src/source/pine_state_hash.cpp +++ b/src/source/pine_state_hash.cpp @@ -2,6 +2,7 @@ #include #include "../broker_state_hash_internal.hpp" +#include "pine_intent_state.hpp" #include #include @@ -894,6 +895,142 @@ void source::PineScheduler::hash_state(BrokerStateHashSink& f) const { f.b(deferred_boundary_input_.active); } +namespace { + +// The selected window's pre-roll capture (src/source/pine_intent_state.hpp), in +// its own domain and folded only while a selected run parks the state, so every +// run without one keeps the hash it had. Each payload folds its own argument +// fields in declaration order; the configuration and point folds follow the ones +// the adapter and the kernel already use. +constexpr char kSourceIntentDomain[] = "pineforge-source-intent/v1"; + +void hash_intent_config(BrokerStateHashSink& f, const source::PineStrategyConfig& config) { + f.b(config.process_orders_on_close); f.b(config.calc_on_order_fills); + f.d(config.initial_capital); f.i(config.default_qty_type); f.d(config.default_qty_value); + f.i(config.pyramiding); f.d(config.commission_value); f.i(config.commission_type); + f.i(config.slippage); f.d(config.margin_long); f.d(config.margin_short); + f.b(config.close_entries_rule_any); f.b(config.src_series_active); +} + +// The saved risk configuration alone: the seven setter fields with their +// sticky percent flags, and the cap's three configuration fields. +void hash_intent_risk(BrokerStateHashSink& f, const source::PineReplayRiskConfig& risk) { + f.i(risk.direction); f.i(risk.max_cons_loss_days); f.d(risk.max_drawdown); + f.b(risk.max_drawdown_percent); f.d(risk.max_intraday_loss); + f.b(risk.max_intraday_loss_percent); f.d(risk.max_position_size); + f.i(static_cast(risk.cap_attachment)); f.i(risk.cap_configuration.limit); + f.b(risk.cap_configuration.skip_noop_market); f.b(risk.cap_configuration.defer_pooc_close); + f.b(risk.cap_configuration.count_pooc_full_close); f.u(risk.cap_declared); +} + +void hash_intent_interval(BrokerStateHashSink& f, const native_calendar::NativeInterval& interval) { + f.i(interval.open_ms); f.i(interval.eligible_open_ms); f.i(interval.last_traded_close_ms); + f.i(interval.next_period_open_ms); f.i(interval.next_input_open_ms); +} + +// A retained point, every value field in declaration order. The four session-day +// facts are presentation to the kernel's own digests, but the replay reads the +// retained point back, so the retained value folds whole. +void hash_intent_point(BrokerStateHashSink& f, const NativeCurrentPointView& point) { + const NativeCoordinate& c = point.decision.coordinate; + f.u(c.ordinal); f.i(c.interval_index); f.i(c.input_interval_index); f.i(c.open_ms); + f.i(c.eligible_open_ms); f.i(c.last_traded_close_ms); f.i(c.next_period_open_ms); + f.i(c.next_input_open_ms); f.i(c.effective_time_ms); f.i(c.source_price_time_ms); + f.u(static_cast(c.provenance)); f.u(static_cast(c.path_phase)); + f.u(static_cast(c.completion)); + f.i(point.decision.decision_floor_ms); + hash_intent_interval(f, point.decision.input_interval); + hash_intent_interval(f, point.decision.script_interval); + f.i(point.decision.sub_index); f.i(point.decision.sub_count); + f.b(point.decision.is_terminal_sub_bar); + f.b(point.decision.in_session); f.b(point.decision.opens_session_day); + f.b(point.decision.closes_session_day); f.b(point.decision.closes_session_day_open_ended); + f.i(point.decision.sub_bar_open_ms); f.i(point.decision.script_bar_open_ms); + const NativeDriverStatistics& statistics = point.decision.driver_statistics; + f.b(statistics.intrabar_path_enabled); f.i(statistics.sub_bars_per_script_bar); + f.i(statistics.samples_per_sub_bar); f.u(statistics.sub_bars_processed); + f.u(statistics.sample_ticks_processed); + f.d(point.price); f.u(static_cast(point.quote_kind)); + f.u(point.quote_origin_ordinal); +} + +void hash_intent_payload(BrokerStateHashSink& f, const source::detail::IntentEntry& v) { + f.s(v.id); f.b(v.is_long); f.d(v.limit_price); f.d(v.stop_price); f.d(v.qty); + f.s(v.comment); f.s(v.oca_name); f.i(v.oca_type); f.i(v.qty_type); +} +void hash_intent_payload(BrokerStateHashSink& f, const source::detail::IntentClose& v) { + f.s(v.id); f.s(v.comment); f.d(v.qty); f.d(v.qty_percent); f.b(v.immediately); +} +void hash_intent_payload(BrokerStateHashSink& f, const source::detail::IntentCloseToken& v) { + f.s(v.id); f.s(v.comment); f.d(v.qty); f.d(v.qty_percent); f.b(v.immediately); + f.u(v.callsite_token); +} +void hash_intent_payload(BrokerStateHashSink&, const source::detail::IntentCloseAll&) {} +void hash_intent_payload(BrokerStateHashSink& f, const source::detail::IntentExit& v) { + f.s(v.id); f.s(v.from_entry); f.d(v.limit_price); f.d(v.stop_price); + f.d(v.trail_points); f.d(v.trail_offset); f.d(v.trail_price); f.d(v.qty_percent); + f.s(v.comment); f.d(v.qty); f.s(v.oca_name); f.d(v.profit_ticks); f.d(v.loss_ticks); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentExitCancelBracket& v) { + f.s(v.exit_id); f.s(v.from_entry); f.s(v.comment); +} +void hash_intent_payload(BrokerStateHashSink& f, const source::detail::IntentCancel& v) { + f.s(v.id); +} +void hash_intent_payload(BrokerStateHashSink&, const source::detail::IntentCancelAll&) {} +void hash_intent_payload(BrokerStateHashSink& f, const source::detail::IntentOrder& v) { + f.s(v.id); f.b(v.is_long); f.d(v.qty); f.d(v.limit_price); f.d(v.stop_price); + f.s(v.oca_name); f.i(v.oca_type); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentRiskDirection& v) { + f.i(v.direction); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentRiskMaxConsLossDays& v) { + f.i(v.value); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentRiskMaxDrawdown& v) { + f.d(v.value); f.b(v.percent); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentRiskMaxIntradayLoss& v) { + f.d(v.value); f.b(v.percent); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentRiskMaxIntradayFilledOrders& v) { + f.i(v.limit); +} +void hash_intent_payload(BrokerStateHashSink& f, + const source::detail::IntentRiskMaxPositionSize& v) { + f.d(v.value); +} + +void hash_selected_intents(BrokerStateHashSink& f, const source::detail::PineIntentState& state) { + f.s(kSourceIntentDomain); + // The native run follows the source identity normalization of a handle's run + // (hash_source_run_identity): it protects a stale state and is not a + // source-visible fact, so it is never a per-run nonce. The owner, the kind + // tag, the bracket flag and the replay pointer are addresses or transient and + // are not folded. + f.u(0); + f.b(state.has_evaluation); f.b(state.replayed); f.i(state.label_ms); + hash_intent_config(f, state.start_config); hash_intent_config(f, state.end_config); + hash_intent_risk(f, state.start_risk); hash_intent_risk(f, state.end_risk); + hash_intent_point(f, state.terminal_point); + f.u(state.rows.size()); + for (const source::detail::PineIntentRow& row : state.rows) { + f.u(row.payload.index()); + std::visit([&f](const auto& payload) { hash_intent_payload(f, payload); }, row.payload); + hash_intent_config(f, row.config); + hash_intent_point(f, row.point); + } +} + +} // namespace + void source::PineStrategyHost::hash_host_extension(BrokerStateHashSink& f) const { f.s(kSourceAdapterDomain); f.b(config_.process_orders_on_close); f.b(config_.calc_on_order_fills); @@ -1011,6 +1148,10 @@ void source::PineStrategyHost::hash_host_extension(BrokerStateHashSink& f) const f.i(site.handed); } } + // The selected window's pre-roll capture, only while a selected run parks it + // (selected_intents allocates nothing and is null for every other run). + if (const source::detail::PineIntentState* const intents = detail::selected_intents(*this)) + hash_selected_intents(f, *intents); adapter_.hash_state(f); scheduler_.hash_state(f); } diff --git a/src/source/pine_strategy_commands.cpp b/src/source/pine_strategy_commands.cpp index 290bd7eb8..222fc2f35 100644 --- a/src/source/pine_strategy_commands.cpp +++ b/src/source/pine_strategy_commands.cpp @@ -1,5 +1,7 @@ #include +#include "pine_intent_state.hpp" + #include namespace pineforge::source { @@ -27,6 +29,15 @@ void PineStrategyHost::strategy_entry(const std::string& id, bool is_long, const std::string& comment, const std::string& oca_name, int oca_type, int qty_type) { + // A selected window's pre-roll call is kept for the replay, ahead of the + // trading-window gate and the adapter (pine_intent_state.hpp); every door + // opens this way, the unguarded ones included. + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentEntry{id, is_long, limit_price, stop_price, qty, comment, + oca_name, oca_type, qty_type}; + })) { + return; + } if (!trading_window_active(current_bar_.timestamp, trade_start_time_, script_tf_seconds_)) return; adapter_.set_configuration(config_); @@ -36,6 +47,11 @@ void PineStrategyHost::strategy_entry(const std::string& id, bool is_long, void PineStrategyHost::strategy_close(const std::string& id, const std::string& comment, double qty, double qty_percent, bool immediately) { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentClose{id, comment, qty, qty_percent, immediately}; + })) { + return; + } adapter_.set_configuration(config_); adapter_.close(id, comment, qty, qty_percent, immediately); } @@ -43,6 +59,12 @@ void PineStrategyHost::strategy_close(const std::string& id, const std::string& void PineStrategyHost::strategy_close(const std::string& id, const std::string& comment, double qty, double qty_percent, bool immediately, std::uint64_t callsite_token) { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentCloseToken{id, comment, qty, qty_percent, immediately, + callsite_token}; + })) { + return; + } if (!trading_window_active(current_bar_.timestamp, trade_start_time_, script_tf_seconds_)) return; adapter_.set_configuration(config_); @@ -50,6 +72,11 @@ void PineStrategyHost::strategy_close(const std::string& id, const std::string& } void PineStrategyHost::strategy_close_all() { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [] { + return detail::IntentCloseAll{}; + })) { + return; + } if (!trading_window_active(current_bar_.timestamp, trade_start_time_, script_tf_seconds_)) return; adapter_.set_configuration(config_); @@ -63,6 +90,13 @@ void PineStrategyHost::strategy_exit(const std::string& id, const std::string& f const std::string& comment, double qty, const std::string& oca_name, double profit_ticks, double loss_ticks) { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentExit{id, from_entry, limit_price, stop_price, trail_points, + trail_offset, trail_price, qty_percent, comment, qty, + oca_name, profit_ticks, loss_ticks}; + })) { + return; + } if (!trading_window_active(current_bar_.timestamp, trade_start_time_, script_tf_seconds_)) return; adapter_.set_configuration(config_); @@ -74,6 +108,11 @@ void PineStrategyHost::strategy_exit(const std::string& id, const std::string& f void PineStrategyHost::strategy_exit_cancel_bracket(const std::string& exit_id, const std::string& from_entry, const std::string& comment) { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentExitCancelBracket{exit_id, from_entry, comment}; + })) { + return; + } if (!trading_window_active(current_bar_.timestamp, trade_start_time_, script_tf_seconds_)) return; adapter_.set_configuration(config_); @@ -81,11 +120,21 @@ void PineStrategyHost::strategy_exit_cancel_bracket(const std::string& exit_id, } void PineStrategyHost::strategy_cancel(const std::string& id) { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentCancel{id}; + })) { + return; + } adapter_.set_configuration(config_); adapter_.cancel(id); } void PineStrategyHost::strategy_cancel_all() { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [] { + return detail::IntentCancelAll{}; + })) { + return; + } adapter_.set_configuration(config_); adapter_.cancel_all(); } @@ -93,6 +142,12 @@ void PineStrategyHost::strategy_cancel_all() { void PineStrategyHost::strategy_order(const std::string& id, bool is_long, double qty, double limit_price, double stop_price, const std::string& oca_name, int oca_type) { + if (detail::capture_pre_roll_command(*this, current_bar_.timestamp, config_, [&] { + return detail::IntentOrder{id, is_long, qty, limit_price, stop_price, oca_name, + oca_type}; + })) { + return; + } if (!trading_window_active(current_bar_.timestamp, trade_start_time_, script_tf_seconds_)) return; adapter_.set_configuration(config_); diff --git a/src/source/pine_strategy_host.cpp b/src/source/pine_strategy_host.cpp index c66597de8..5c8b738aa 100644 --- a/src/source/pine_strategy_host.cpp +++ b/src/source/pine_strategy_host.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -8,6 +9,7 @@ #include "../timezone.hpp" #include "../native_execution_consumer.hpp" #include "pine_host_reads.hpp" +#include "pine_intent_state.hpp" #include "pine_quiet_bar.hpp" #include "pine_run_failure.hpp" @@ -378,6 +380,11 @@ void source::PineStrategyHost::on_native_run_begin() { // The spec facts the per-bar callbacks read, once for the run (R5 lane // D2-C, pine_host_reads.hpp). detail::take_run_facts(detail::run_consumer(*this), adapter_, adapter_.run_counter_); + // A selected window's pre-roll capture state is parked for this run alone + // (pine_intent_state.hpp). It sits outside the try below on purpose: an + // allocation failure fails the begin instead of becoming a script preparation + // failure, and nothing is dropped silently. + detail::adopt_selected_intents(*this); source_bar_index_ = -1; source_last_bar_index_ = -1; source_callback_count_ = 0; @@ -557,6 +564,10 @@ void source::PineStrategyHost::on_native_bar_open( ? static_cast(context.driver_statistics.sub_bars_processed) : 0; diag_magnifier_sample_ticks_processed_ = bar_magnifier_enabled_ ? static_cast(context.driver_statistics.sample_ticks_processed) : 0; + // A selected window's pre-roll rows are placed here, once, before the adapter + // reads the bar's open: it must see them as the prior bar's orders + // (selected_replay_at_open). Nothing happens for any other run. + selected_replay_at_open(bar, context); adapter_.on_bar_open(bar, context); scheduler_.bar_open(bar, context, *this); } @@ -851,28 +862,37 @@ void source::PineStrategyHost::set_syminfo_session(const std::string& session) { BacktestEngine::set_syminfo_session(session); } +// The six risk setters apply live as they always did. While a selected window's +// pre-roll evaluation captures they also append an ordered row, so the replay +// can set them again in the script's own order against the saved configuration. void source::PineStrategyHost::set_pine_risk_direction(int direction) { + detail::capture_risk_row(*this, config_, detail::IntentRiskDirection{direction}); adapter_.set_risk_direction(direction); } void source::PineStrategyHost::set_pine_risk_max_cons_loss_days(int value) { + detail::capture_risk_row(*this, config_, detail::IntentRiskMaxConsLossDays{value}); adapter_.set_risk_max_cons_loss_days(value); } void source::PineStrategyHost::set_pine_risk_max_drawdown(double value, bool percent) { + detail::capture_risk_row(*this, config_, detail::IntentRiskMaxDrawdown{value, percent}); adapter_.set_risk_max_drawdown(value, percent); } void source::PineStrategyHost::set_pine_risk_max_intraday_loss(double value, bool percent) { + detail::capture_risk_row(*this, config_, detail::IntentRiskMaxIntradayLoss{value, percent}); adapter_.set_risk_max_intraday_loss(value, percent); } void source::PineStrategyHost::set_pine_risk_max_intraday_filled_orders(int limit) { + detail::capture_risk_row(*this, config_, detail::IntentRiskMaxIntradayFilledOrders{limit}); if (limit > 0) adapter_.cap.adopt_script_count(); adapter_.cap = limit; } void source::PineStrategyHost::set_pine_risk_max_position_size(double value) { + detail::capture_risk_row(*this, config_, detail::IntentRiskMaxPositionSize{value}); adapter_.set_risk_max_position_size(value); } @@ -1111,12 +1131,18 @@ void source::PineStrategyHost::attach_pine_execution_adapter() { void source::PineStrategyHost::set_syminfo_metadata( const std::string& key, double value) { if (key == "report_terminal_quote_time_ms") { + if (NativeExecutionConsumer::selected_query_closed(*this)) { + throw SelectedWindowQueryAfterSeal(); + } report_quote_time_ms_ = std::isfinite(value) && value >= 0.0 && value <= 9007199254740991.0 && std::floor(value) == value ? static_cast(value) : -1; return; } if (key == "report_terminal_quote_close") { + if (NativeExecutionConsumer::selected_query_closed(*this)) { + throw SelectedWindowQueryAfterSeal(); + } report_quote_close_ = std::isfinite(value) && value > 0.0 ? value : std::numeric_limits::quiet_NaN(); return; @@ -1183,14 +1209,17 @@ void source::PineStrategyHost::set_syminfo_metadata( } bool source::PineStrategyHost::set_report_terminal_quote(std::int64_t time_ms, double close) { - if (time_ms < 0 || time_ms > 9007199254740991LL - || !std::isfinite(close) || close <= 0.0) return false; + if (NativeExecutionConsumer::selected_query_closed(*this) || time_ms < 0 + || time_ms > 9007199254740991LL || !std::isfinite(close) || close <= 0.0) return false; report_quote_time_ms_ = time_ms; report_quote_close_ = close; return true; } void source::PineStrategyHost::clear_report_terminal_quote() { + if (NativeExecutionConsumer::selected_query_closed(*this)) { + throw SelectedWindowQueryAfterSeal(); + } report_quote_time_ms_ = -1; report_quote_close_ = std::numeric_limits::quiet_NaN(); } @@ -2335,6 +2364,12 @@ void source::PineStrategyHost::scheduler_publish_source_bar( // A named-entry cancellation token has source-evaluation scope. Clear a // prior callback before publishing receipts and entering this body. adapter_.begin_source_evaluation(); + // A selected window's pre-roll evaluation records its commands and risk + // statements instead of placing them: the buffer is this evaluation's alone, + // reset here even if the body calls nothing, and the scope closes the bracket + // on every exit (pine_intent_state.hpp). + const bool capturing = selected_begin_evaluation(bar.timestamp); + const detail::IntentCaptureScope capture_scope(*this, capturing); // Publish terminal and group-adjustment receipts before the source body // reads its public pending projection at this decision boundary. if (same_bar_exit_group_possible(trades_)) { @@ -2372,11 +2407,26 @@ void source::PineStrategyHost::scheduler_publish_source_bar( // source policy view synchronized at the callback boundary; the generic // NativeRunSpec remains immutable for the run. adapter_.set_configuration(config_); + if (capturing) selected_finish_evaluation(); if (temporary_index) { --source_bar_index_; bar_index_ = previous_bar_index; barstate_islast_ = previous_barstate_islast; } + finish_adapter_evaluation(); + if (advance_source_index) { + scheduler_mark_report_point(bar.timestamp); + prev_bar_timestamp_ = bar.timestamp; + } +} + +// The drains that close one source evaluation once its body has run, in the +// order they always ran in. The replay of a selected window's pre-roll rows ends +// its evaluation with this same list (selected_replay_at_open). +void source::PineStrategyHost::finish_adapter_evaluation() { + // Quiet-bar gates (pine_quiet_bar.hpp), as at the publication. + using detail::QuietHook; + using detail::skip_quiet; if (!skip_quiet(QuietHook::PendingCloses, adapter_.close_batch_callsites_.empty())) adapter_.flush_pending_closes(); #if PINEFORGE_PINE_QUIET_BAR_GATES @@ -2404,10 +2454,6 @@ void source::PineStrategyHost::scheduler_publish_source_bar( adapter_.anchor_relative_exits(); } adapter_.order_same_point_entries(); - if (advance_source_index) { - scheduler_mark_report_point(bar.timestamp); - prev_bar_timestamp_ = bar.timestamp; - } } void source::PineStrategyHost::scheduler_publish_suppressed_tail(const Bar& bar) { @@ -2422,11 +2468,263 @@ void source::PineStrategyHost::scheduler_publish_suppressed_tail(const Bar& bar) NativeExecutionConsumer::bound(*this).pump_ambient(), chart_day_partition_.empty() ? nullptr : &chart_day_partition_); adapter_.begin_source_evaluation(); + // A selected pre-roll tail runs no body: it leaves an empty last-present + // evaluation, so no earlier bar's signal outlives it. + selected_clear_suppressed_evaluation(bar.timestamp); adapter_.observe_terminal_receipts(); scheduler_mark_report_point(bar.timestamp); prev_bar_timestamp_ = bar.timestamp; } +// The selected window's pre-roll capture hooks (pine_intent_state.hpp). +bool source::PineStrategyHost::selected_begin_evaluation(std::int64_t label_ms) { + const detail::PreRollEvaluation pre_roll = detail::selected_pre_roll_evaluation(*this, label_ms); + if (pre_roll.state == nullptr) return false; + detail::reset_intent_evaluation(*pre_roll.state, label_ms, config_, + adapter_.capture_replay_risk_config(), *pre_roll.point); + pre_roll.state->capturing = true; + return true; +} + +void source::PineStrategyHost::selected_finish_evaluation() noexcept { + detail::PineIntentState* const state = detail::selected_intents(*this); + if (state == nullptr || !state->capturing) return; + state->end_config = config_; + state->end_risk = adapter_.capture_replay_risk_config(); + if (const NativeCurrentPointView* const point = detail::intent_live_point(*this)) + state->terminal_point = *point; + state->capturing = false; +} + +void source::PineStrategyHost::selected_clear_suppressed_evaluation(std::int64_t label_ms) { + const detail::PreRollEvaluation pre_roll = detail::selected_pre_roll_evaluation(*this, label_ms); + if (pre_roll.state == nullptr) return; + detail::reset_intent_evaluation(*pre_roll.state, label_ms, config_, + adapter_.capture_replay_risk_config(), *pre_roll.point); +} + +namespace { + +// The selected window's replay (selected_replay_at_open) hands each retained +// row to the adapter's own door with exactly the arguments the script passed. +// The host's wrappers are never used: they would take the trading-window gate +// again and copy the live configuration over the row's. The nine command doors +// first, then the six risk setter statements. +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentEntry& row) { + adapter.entry(row.id, row.is_long, row.limit_price, row.stop_price, row.qty, row.comment, + row.oca_name, row.oca_type, row.qty_type); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentClose& row) { + adapter.close(row.id, row.comment, row.qty, row.qty_percent, row.immediately); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentCloseToken& row) { + adapter.close(row.id, row.comment, row.qty, row.qty_percent, row.immediately, + row.callsite_token); +} +void replay_intent(source::PineExecutionAdapter& adapter, const source::detail::IntentCloseAll&) { + adapter.close_all(); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentExit& row) { + adapter.exit(row.id, row.from_entry, row.limit_price, row.stop_price, row.trail_points, + row.trail_offset, row.trail_price, row.qty_percent, row.comment, row.qty, + row.oca_name, row.profit_ticks, row.loss_ticks); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentExitCancelBracket& row) { + adapter.exit_cancel_bracket(row.exit_id, row.from_entry, row.comment); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentCancel& row) { + adapter.cancel(row.id); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentCancelAll&) { + adapter.cancel_all(); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentOrder& row) { + adapter.order(row.id, row.is_long, row.qty, row.limit_price, row.stop_price, row.oca_name, + row.oca_type); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentRiskDirection& row) { + adapter.set_risk_direction(row.direction); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentRiskMaxConsLossDays& row) { + adapter.set_risk_max_cons_loss_days(row.value); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentRiskMaxDrawdown& row) { + adapter.set_risk_max_drawdown(row.value, row.percent); +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentRiskMaxIntradayLoss& row) { + adapter.set_risk_max_intraday_loss(row.value, row.percent); +} +// As the host setter: the script's own statement adopts the count switches first. +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentRiskMaxIntradayFilledOrders& row) { + if (row.limit > 0) adapter.cap.adopt_script_count(); + adapter.cap = row.limit; +} +void replay_intent(source::PineExecutionAdapter& adapter, + const source::detail::IntentRiskMaxPositionSize& row) { + adapter.set_risk_max_position_size(row.value); +} + +// A selected replay the engine cannot take, or that a native failure cut short, +// is the engine's fault and fails the run: it is never skipped quietly. After a +// native failure the consumer already holds the first cause and keeps it (its +// bar-open catch records only when nothing failed before). +[[noreturn]] void replay_invariant(const char* what) { + throw coded(RunFailureCode::engine_invariant, {}, what); +} + +// Ends a replay bracket on every exit, normal or by exception: the retained +// point pointer cleared, the open activation scope closed (the grants it +// recorded stay for the Open match that follows) and the adapter's +// configuration back to the host's. Two stores, a noexcept end and a plain +// copy: nothing here executes a command or allocates. +class ReplayBracket { +public: + ReplayBracket(source::detail::PineIntentState& state, NativeExecutionConsumer& consumer, + source::PineExecutionAdapter& adapter, + const source::PineStrategyConfig& config) noexcept + : state_(&state), consumer_(&consumer), adapter_(&adapter), config_(&config) {} + ReplayBracket(const ReplayBracket&) = delete; + ReplayBracket& operator=(const ReplayBracket&) = delete; + ~ReplayBracket() noexcept { + state_->replay_point = nullptr; + consumer_->end_open_activation(); + adapter_->set_configuration(*config_); + } + +private: + source::detail::PineIntentState* state_; + NativeExecutionConsumer* consumer_; + source::PineExecutionAdapter* adapter_; + const source::PineStrategyConfig* config_; +}; + +} // namespace + +// The selected window's replay (pine_intent_state.hpp): the last pre-roll +// evaluation's rows, placed once at the window's first Open, before the adapter +// reads that bar's open. They are placed as the evaluation placed them, in its +// own order, each under the configuration and the retained point it was called +// with, so the Open matcher that follows sees the prior bar's orders. Nothing of +// the script runs again; nothing is published, matched or executed here. +void source::PineStrategyHost::selected_replay_at_open(const Bar& bar, + const NativeDecisionContext&) { + pf_selected_window_config_v1 window{}; + if (!NativeExecutionConsumer::selected_window_admitted(*this, &window) + || bar.timestamp < window.start_ms) { + return; + } + detail::PineIntentState* const state = detail::selected_intents(*this); + if (state == nullptr) replay_invariant("selected replay without its intent state"); + if (state->replayed) return; + if (!state->has_evaluation) { + state->replayed = true; + return; + } + // The bar before the window left the book as it was: nothing held, nothing + // working or queued in the source layer, no recalculation open. + if (state->capturing) replay_invariant("selected replay inside a pre-roll capture bracket"); + if (state->label_ms >= window.start_ms) + replay_invariant("selected replay of an evaluation labelled inside the window"); + if (detail::run_position(*this).signed_units != 0.0) + replay_invariant("selected replay over a held position"); + if (adapter_.coof_recalc_active_) + replay_invariant("selected replay inside a calc_on_order_fills recalculation"); + if (!adapter_.live_handles_.empty() || !adapter_.delayed_market_orders_.empty() + || !adapter_.pending_same_bar_commands_.empty() || !adapter_.pending_entries_.empty() + || !adapter_.close_batch_callsites_.empty() || !adapter_.pending_bracket_legs_.empty() + || !adapter_.anchored_relative_legs_.empty() + || !adapter_.pending_relative_exits_.empty()) { + replay_invariant("selected replay over live requests or queued source commands"); + } + // Marked before the bracket opens: a failure inside it never retries. + state->replayed = true; + NativeExecutionConsumer& consumer = NativeExecutionConsumer::bound(*this); + if (!consumer.begin_open_activation(*this)) + replay_invariant("selected replay without an open activation scope"); + const ReplayBracket bracket(*state, consumer, adapter_, config_); + const auto stop_if_failed = [this] { + if (detail::run_kind(*this) == NativeLifecycleKind::Failed) + replay_invariant("selected replay stopped by a native failure"); + }; + using detail::QuietHook; + using detail::skip_quiet; + + // The evaluation again, from its start: the configuration it began under, + // then its receipts and the legs staged before its body, as + // scheduler_publish_source_bar takes them. + state->replay_point = &state->terminal_point; + adapter_.restore_replay_risk_config(state->start_risk); + adapter_.set_configuration(state->start_config); + adapter_.begin_source_evaluation(); + adapter_.observe_terminal_receipts(); + stop_if_failed(); + if (!skip_quiet(QuietHook::BracketLegs, adapter_.pending_bracket_legs_.empty())) { + adapter_.flush_pending_bracket_legs({}, /*post_calculation=*/false, + /*pre_script_drain=*/true); + } + stop_if_failed(); + + // Its statements, in the order the script made them. + for (detail::PineIntentRow& row : state->rows) { + state->replay_point = &row.point; + adapter_.set_configuration(row.config); + std::visit([this](const auto& payload) { replay_intent(adapter_, payload); }, + row.payload); + stop_if_failed(); + } + + // Its end: the configuration it ended under (the risk configuration alone, + // never a counter), then the drains every source evaluation ends with. + state->replay_point = &state->terminal_point; + adapter_.set_configuration(state->end_config); + adapter_.restore_replay_risk_config(state->end_risk); + finish_adapter_evaluation(); + stop_if_failed(); + + // The terminal explicit-market policy of a process_orders_on_close pass, + // which the close of that bar would have run: submit-only here, so the + // reconciled entries stay working for the open matcher. The caller's own + // quiet and count gate, as PineExecutionAdapter::on_bar_close asks it; the + // policy keeps its own filters. + const NativeDecisionContext& terminal = state->terminal_point.decision; + const bool bound = adapter_.host_ != nullptr; + const int bar_index = adapter_.projection_bar_index(terminal); + std::size_t explicit_entries = 0; + if (adapter_.config_.process_orders_on_close && adapter_.config_.pyramiding == 0 + && adapter_.modeled_input()) { + for (const auto& handle : adapter_.live_handles_) { + const auto found = adapter_.placement_.find(handle.incarnation); + if (found != adapter_.placement_.end() && found->second.opening + && found->second.family == PineOrderFamily::Entry + && found->second.projection_created_bar == bar_index + && std::isfinite(found->second.requested_qty) + && found->second.requested_qty > 0.0) { + ++explicit_entries; + } + } + } + if (!skip_quiet(QuietHook::TerminalExplicitMarket, bound && explicit_entries < 2)) + adapter_.apply_terminal_explicit_market_policy(terminal, false); + stop_if_failed(); + + // Spent only once it all succeeded. The retained configurations, the terminal + // point and the replayed flag stay as the hash reads them. + state->rows.clear(); +} + // The Pine report series has one point per SOURCE slot this host published, // which is not the kernel's per-calculation cadence: a calc_on_order_fills // re-entry marks the slot it opened at the fill, the ordinary close diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index f5a8a3f87..e537c6574 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1322,6 +1322,15 @@ list(APPEND TEST_SOURCES test_utc_month_key_arithmetic) # month, and compute_equity_stats bit for bit against the 6c081f5d walk. # Source-free, so it also runs in the kernel-only profile. list(APPEND TEST_SOURCES test_utc_month_memo) +# Pure report view: the selected window's equity curve (the window anchor, +# then the M script-bar observations copied byte for byte) as one owned new[] +# array that a report takes by release() and free_report frees, and the +# observation count compute_equity_stats divides exposure by -- hand +# arithmetic for 100 -> 110 -> 104.5, the first bar at T or after a gap, the +# undefined domains, and the ordinary overload bit for bit against the +# reference walk (tests/equity_stats_reference.hpp). Source-free, so it also +# runs in the kernel-only profile. +list(APPEND TEST_SOURCES test_selected_report_view) # R5 lane PERF-P5611 (P5): the adapter's chart-day key on a UTC chart by the # same arithmetic, swept against gmtime_r through the host. Source-bound. list(APPEND TEST_SOURCES test_chart_day_key_arithmetic) @@ -1841,6 +1850,25 @@ list(APPEND TEST_SOURCES test_l8a_adapter_policy_delta ) +# Selected-window component TUs. test_selected_report_view is registered above +# and is not repeated here. These are ordinary mandatory rows: their sources +# are integrated before configure, and a missing +# source fails configure rather than being skipped. Every TEST_SOURCES row links +# pineforge with the FP and NDEBUG flags; the kernel-only reach filter further +# down decides whether each one runs without the source layer. +list(APPEND TEST_SOURCES + test_selected_primary_plan + test_selected_window_plan_c + test_selected_window_native_counts + test_selected_retained_count + test_selected_window_report_integration + test_selected_window_routes + test_selected_production_routes + # The source of test_selected_quote_seal is supplied by a separate change + # and integrated with this registration before configure. + test_selected_quote_seal +) + find_package(Threads REQUIRED) find_package(Python3 COMPONENTS Interpreter REQUIRED) @@ -1947,6 +1975,28 @@ add_test( ${PROJECT_SOURCE_DIR}/scripts/test_aggregate_cpp_versions.py ) +# Selected-window Python unittest files, run as direct scripts under the +# configured Python3 with -B (no bytecode written into the tree). Unconditional: +# no source-layer or opt-in gate. +add_test(NAME test_run_phase_transport + COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/tests/test_run_phase_transport.py) +add_test(NAME test_run_execution_observer + COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/tests/test_run_execution_observer.py) +add_test(NAME test_request_feed_inventory_admission + COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/tests/test_request_feed_inventory_admission.py) +add_test(NAME test_selected_window_plan_python + COMMAND ${Python3_EXECUTABLE} -B ${PROJECT_SOURCE_DIR}/tests/test_selected_window_plan_python.py) + +# Selected-window files that use pytest: Python3 -B -m pytest on the literal +# path, from the repository root. No check that pytest is installed: the test +# environment must provide it, and a missing pytest fails these rows. +add_test(NAME test_run_json_selected_window + COMMAND ${Python3_EXECUTABLE} -B -m pytest ${PROJECT_SOURCE_DIR}/tests/test_run_json_selected_window.py + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) +add_test(NAME test_compile_inventory_pipeline + COMMAND ${Python3_EXECUTABLE} -B -m pytest ${PROJECT_SOURCE_DIR}/tests/test_compile_inventory_pipeline.py + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) + set(_pf_script_cpp_abi_flags) set(_pf_abi_receipt_mode --skip-if-receipt-missing) if(PINEFORGE_REQUIRE_ABI_RECEIPTS) @@ -2780,6 +2830,11 @@ foreach(test_name ${TEST_SOURCES}) endif() endforeach() +# The C bridge test also links the shared window-plan helper, whose bridge +# (src/selected_window_plan_c.cpp) is not in the kernel archive. The loop above +# supplies pineforge and the -ffp-contract=off and -UNDEBUG flags. +target_link_libraries(test_selected_window_plan_c PRIVATE pineforge_window_plan) + # R5 PERF-L4: the witness's two references. The source layer is compiled # twice more, with the kernel archive below it: once with every quiet-bar gate # compiled out (PINEFORGE_PINE_QUIET_BAR_GATES=0), once gated; both carry the diff --git a/tests/pine_input_domain_gates.json b/tests/pine_input_domain_gates.json index 6f8c6ddaa..b1dacaea8 100644 --- a/tests/pine_input_domain_gates.json +++ b/tests/pine_input_domain_gates.json @@ -349,7 +349,7 @@ "predicate": "Modeled atomic bar-point transaction, not observed prints", "old": " || config_.pyramiding != 0 || stream_mode_) {", "new": " || config_.pyramiding != 0 || !modeled_input()) {", - "window": "void PineExecutionAdapter::apply_terminal_explicit_market_policy(\n const NativeDecisionContext& context) {\n if (!config_.process_orders_on_close\n || config_.pyramiding != 0 || !modeled_input()) {\n return;\n }\n struct Candidate {" + "window": "void PineExecutionAdapter::apply_terminal_explicit_market_policy(\n const NativeDecisionContext& context, bool execute_now) {\n if (!config_.process_orders_on_close\n || config_.pyramiding != 0 || !modeled_input()) {\n return;\n }\n struct Candidate {" }, { "file": "src/source/pine_adapter.cpp", diff --git a/tests/test_compile_inventory_pipeline.py b/tests/test_compile_inventory_pipeline.py new file mode 100644 index 000000000..ef7cae96d --- /dev/null +++ b/tests/test_compile_inventory_pipeline.py @@ -0,0 +1,836 @@ +"""Tests of the trusted compile pipeline's request-feed inventory handling: +docker/entrypoint.sh on the selected-window path and docker/bind_compiled_inventory.py. + +The entrypoint is driven as a process (bash) inside a private tree +that holds a stub g++, a stub run_json.py and a FAKE pineforge_codegen package on +PYTHONPATH. The fake mimics, from the interface text only, what these rows need of +transpile, transpile_with_request_inventory and bind_request_feed_inventory; it is +not the pinned code, and it does not reproduce the pinned shape check, so these +rows cannot show the pinned binder agrees. + +Every stub appends to one ordered event log, so a row can assert what ran and what +did not: gxx (the compiler), helper (bind_compiled_inventory.py started), pinned:bind +(the pinned binder was called), run_json (the harness started), codegen:transpile and +codegen:transpile_with_request_inventory. The entrypoint's exit trap removes its work +dir, so a row sees that dir only through what the stubs copied while it existed. +Two more events come from fixtures: cp:fail (the stub cp refused the entrypoint's +snapshot copy) and race:link (a racing writer created the output the instant the +helper linked it). Scratch is pytest's tmp_path; this file calls no deletion. +""" +import ast +import hashlib +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +ENTRYPOINT = ROOT / "docker" / "entrypoint.sh" +HELPER = ROOT / "docker" / "bind_compiled_inventory.py" + +SCHEMA = "pineforge-request-feed-inventory/v1" +POLICY = "selected-window/v1" +PINE_TEXT = "//@version=6\nstrategy('pipeline fixture')\n" +CPP_BYTES = "// precompiled é\nint pf_precompiled = 1;\n".encode("utf-8") +SO_BYTES = b"fake shared object\n" + +SAME_TICKER_DAILY = [{"kind": "token", "timeframe": "D"}] +EMPTY = [] +MIXED = [{"kind": "token", "timeframe": "D"}, + {"kind": "input", "key": "HTF", "default": "240"}, + {"kind": "unknown"}] +ENTRY_SETS = [SAME_TICKER_DAILY, EMPTY, MIXED] +ENTRY_IDS = ["same-ticker-daily", "empty", "mixed-input-and-unknown"] + +SELECTED_ENV = { + "PINEFORGE_REPORT_POLICY": POLICY, + "PINEFORGE_WINDOW_START_MS": "60000", + "PINEFORGE_WINDOW_END_MS": "240000", + "PINEFORGE_PREROLL_BARS": "1", + "PINEFORGE_FED_START_MS": "0", +} +SELECTED_FLAGS = ["--report-policy", POLICY, "--window-start-ms", "60000", + "--window-end-ms", "240000", "--preroll-bars", "1", "--fed-start-ms", "0"] +BINDING_FAILED = "request-feed inventory binding failed" + +# --- The fakes and stubs the private tree is made of ----------------------------- + +FAKE_CODEGEN_INIT = r''' +import hashlib +import json +import os + + +def _event(name): + with open(os.path.join(os.environ["STUB_LOG"], "events"), "a") as handle: + handle.write(name + "\n") + + +def _cpp(source): + return "// généré\nint pf_fake_strategy = %d;\n" % len(source) + + +def transpile(pine_source, *, check_support=True, filename="", libraries=None): + _event("codegen:transpile") + return _cpp(pine_source) + + +def transpile_with_request_inventory(pine_source, **kwargs): + _event("codegen:transpile_with_request_inventory") + with open(os.path.join(os.environ["STUB_LOG"], "codegen.kwargs.json"), "w") as handle: + json.dump(sorted(kwargs), handle) + if os.environ.get("FAKE_CODEGEN_FAIL"): + from .errors import CompileError + raise CompileError("forced transpile failure") + cpp = _cpp(pine_source) + if os.environ.get("FAKE_CODEGEN_WRONG_SOURCE"): + digest = hashlib.sha256(b"not the generated source").hexdigest() + else: + digest = hashlib.sha256(cpp.encode("utf-8")).hexdigest() + return { + "cpp": cpp, + "request_feed_inventory": { + "schema": "pineforge-request-feed-inventory/v1", + "source_sha256": digest, + "artifact_sha256": None, + "primary_chart_timeframe": None, + "entries": json.loads(os.environ.get("FAKE_CODEGEN_ENTRIES", "[]")), + }, + } +''' + +FAKE_CODEGEN_ERRORS = r''' +class CompileError(Exception): + pass +''' + +FAKE_CODEGEN_BINDER = r''' +import hashlib +import os + + +class RequestFeedInventoryError(ValueError): + pass + + +def bind_request_feed_inventory(inventory, cpp_bytes, artifact_bytes): + with open(os.path.join(os.environ["STUB_LOG"], "events"), "a") as handle: + handle.write("pinned:bind\n") + keys = ("schema", "source_sha256", "artifact_sha256", "primary_chart_timeframe", "entries") + if not isinstance(inventory, dict) or set(inventory) != set(keys): + raise RequestFeedInventoryError( + "inventory is not an object with exactly the keys " + ", ".join(keys)) + if inventory["artifact_sha256"] is not None: + raise RequestFeedInventoryError( + "inventory artifact_sha256 is already set; only an unbound inventory can be bound") + if hashlib.sha256(cpp_bytes).hexdigest() != inventory["source_sha256"]: + raise RequestFeedInventoryError( + "inventory source_sha256 does not match the supplied C++ bytes") + return { + "schema": inventory["schema"], + "source_sha256": inventory["source_sha256"], + "artifact_sha256": hashlib.sha256(artifact_bytes).hexdigest(), + "primary_chart_timeframe": inventory["primary_chart_timeframe"], + "entries": [dict(entry) for entry in inventory["entries"]], + } +''' + +GXX_STUB = r'''#!/usr/bin/env bash +out=""; src=""; prev="" +for a in "$@"; do + [ "$prev" = "-o" ] && out="$a" + case "$a" in *.cpp) src="$a" ;; esac + prev="$a" +done +echo gxx >> "$STUB_LOG/events" +echo "$src" > "$STUB_LOG/gxx.source.path" +if [ -n "${STUB_GXX_MUTATE:-}" ]; then + printf 'the caller changed this file after the snapshot\n' > "$STUB_GXX_MUTATE" +fi +if [ -n "${STUB_GXX_FAIL:-}" ]; then + echo "stub g++: forced link failure" >&2 + exit 1 +fi +/bin/cp "$src" "$STUB_LOG/gxx.source.bin" +printf 'fake shared object\n' > "$out" +/bin/cp "$out" "$STUB_LOG/gxx.artifact.bin" +''' + +# Passes every cp through to the real one, except that with STUB_CP_FAIL set it refuses +# a copy whose destination is named strategy.cpp: the entrypoint's snapshot. +CP_STUB = r'''#!/usr/bin/env bash +if [ -n "${STUB_CP_FAIL:-}" ]; then + for last in "$@"; do :; done + case "$last" in + */strategy.cpp) + echo cp:fail >> "$STUB_LOG/events" + echo "stub cp: forced copy failure" >&2 + exit 1 + ;; + esac +fi +exec /bin/cp "$@" +''' + +# Runs the real helper after replacing os.link with a link that first creates the output, +# as a racing writer would, at the very moment the helper publishes. +RACE_SHIM = r''' +import os +import runpy +import sys + +racer = bytes.fromhex(os.environ["RACE_BYTES_HEX"]) +real_link = os.link + + +def racing_link(src, dst, *args, **kwargs): + with open(os.path.join(os.environ["STUB_LOG"], "events"), "a") as handle: + handle.write("race:link\n") + with open(dst, "wb") as handle: + handle.write(racer) + return real_link(src, dst, *args, **kwargs) + + +os.link = racing_link +real = os.environ["REAL_BIND_HELPER"] +sys.argv[0] = real +runpy.run_path(real, run_name="__main__") +''' + +RUN_JSON_STUB = r''' +import json +import os +import shutil +import sys + +log = os.environ["STUB_LOG"] +argv = sys.argv[1:] +with open(os.path.join(log, "events"), "a") as handle: + handle.write("run_json\n") +with open(os.path.join(log, "run_json.argv.json"), "w") as handle: + json.dump(argv, handle) +work = os.path.dirname(argv[argv.index("--so") + 1]) +with open(os.path.join(log, "run_json.work.json"), "w") as handle: + json.dump(sorted(os.listdir(work)), handle) +if "--request-feed-inventory" in argv: + shutil.copyfile(argv[argv.index("--request-feed-inventory") + 1], + os.path.join(log, "run_json.inventory.json")) +if "--generated-cpp" in argv: + shutil.copyfile(argv[argv.index("--generated-cpp") + 1], + os.path.join(log, "run_json.generated.bin")) +print('{"engine":"pineforge","marker":"run_json_stub"}') +''' + +BINDER_SHIM = r''' +import json +import os +import runpy +import sys + +log = os.environ["STUB_LOG"] +with open(os.path.join(log, "events"), "a") as handle: + handle.write("helper\n") +with open(os.path.join(log, "helper.argv.json"), "w") as handle: + json.dump(sys.argv[1:], handle) +real = os.environ["REAL_BIND_HELPER"] +sys.argv[0] = real +runpy.run_path(real, run_name="__main__") +''' + + +class Box: + """One test's private tree: a fake install prefix with the stubs, a fake + pineforge_codegen on PYTHONPATH (and an empty one that has no binder), a stub + g++ first on PATH, an input dir, a TMPDIR the entrypoint's work dir is made in, + a log dir the stubs write to and a work dir for the helper rows.""" + + def __init__(self, root): + self.root = root + self.bin = root / "bin" + self.prefix = root / "prefix" + self.indir = root / "in" + self.tmp = root / "tmp" + self.log = root / "log" + self.work = root / "work" + self.fakepkgs = root / "fakepkgs" + self.nopkgs = root / "nopkgs" + package = self.fakepkgs / "pineforge_codegen" + for directory in (self.bin, self.prefix / "bin", self.prefix / "lib", + self.prefix / "include", self.indir, self.tmp, self.log, + self.work, package, self.nopkgs / "pineforge_codegen"): + directory.mkdir(parents=True) + (package / "__init__.py").write_text(FAKE_CODEGEN_INIT, encoding="utf-8") + (package / "errors.py").write_text(FAKE_CODEGEN_ERRORS, encoding="utf-8") + (package / "request_feed_inventory.py").write_text(FAKE_CODEGEN_BINDER, + encoding="utf-8") + (self.nopkgs / "pineforge_codegen" / "__init__.py").write_text("", encoding="utf-8") + gxx = self.bin / "g++" + gxx.write_text(GXX_STUB, encoding="utf-8") + gxx.chmod(0o755) + cp = self.bin / "cp" + cp.write_text(CP_STUB, encoding="utf-8") + cp.chmod(0o755) + (self.prefix / "bin" / "run_json.py").write_text(RUN_JSON_STUB, encoding="utf-8") + (self.prefix / "bin" / "bind_compiled_inventory.py").write_text(BINDER_SHIM, + encoding="utf-8") + (self.prefix / "lib" / "libpineforge.a").write_bytes(b"") + (self.indir / "ohlcv.csv").write_text("timestamp,open,high,low,close,volume\n", + encoding="utf-8") + + def env(self, extra=None, pythonpath=None): + env = {key: value for key, value in os.environ.items() + if not key.startswith(("PINEFORGE_", "STUB_", "FAKE_", "REAL_"))} + search = str(self.fakepkgs if pythonpath is None else pythonpath) + if os.environ.get("PYTHONPATH"): + search += os.pathsep + os.environ["PYTHONPATH"] + env.update({ + "PATH": str(self.bin) + os.pathsep + os.environ.get("PATH", ""), + "PYTHONPATH": search, + "PYTHONUTF8": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "TMPDIR": str(self.tmp), + "STUB_LOG": str(self.log), + "REAL_BIND_HELPER": str(HELPER), + "PINEFORGE_PREFIX": str(self.prefix), + "PINEFORGE_IN_DIR": str(self.indir), + }) + env.update(extra or {}) + return env + + def events(self): + path = self.log / "events" + return path.read_text(encoding="utf-8").splitlines() if path.exists() else [] + + def logged(self, name): + return json.loads((self.log / name).read_text(encoding="utf-8")) + + +@pytest.fixture +def box(tmp_path): + return Box(tmp_path) + + +# --- Fixtures and drivers -------------------------------------------------------- + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def fake_cpp(pine_text): + return "// généré\nint pf_fake_strategy = %d;\n" % len(pine_text) + + +def source_inventory(cpp_bytes, entries, artifact=None, chart=None): + return {"schema": SCHEMA, "source_sha256": sha(cpp_bytes), "artifact_sha256": artifact, + "primary_chart_timeframe": chart, "entries": entries} + + +def bound_inventory(cpp_bytes, so_bytes, entries): + return source_inventory(cpp_bytes, entries, artifact=sha(so_bytes)) + + +def old_argv(transpiled): + """run_json.py's argv on the unchanged path, with the three paths masked.""" + return ["--so", "", "--ohlcv", "", "--inputs", "", "--overrides", "", + "--input-tf", "", "--script-tf", "", "--bar-magnifier", "", + "--magnifier-samples", "4", "--magnifier-dist", "endpoints", + "--generated-cpp", "", "--transpiled", transpiled] + + +def normalized(argv): + masked = list(argv) + for flag in ("--so", "--ohlcv", "--generated-cpp", "--request-feed-inventory"): + if flag in masked: + masked[masked.index(flag) + 1] = "" + return masked + + +def put_pine(box): + (box.indir / "strategy.pine").write_text(PINE_TEXT, encoding="utf-8") + + +def put_cpp(box): + (box.indir / "strategy.cpp").write_bytes(CPP_BYTES) + + +def put_supplied(box, inventory): + path = box.root / "supplied-inventory.json" + path.write_text(json.dumps(inventory), encoding="utf-8") + return path + + +def run_entrypoint(box, extra=None, xtrace=False, pythonpath=None): + command = ["bash", "-x", str(ENTRYPOINT)] if xtrace else ["bash", str(ENTRYPOINT)] + return subprocess.run(command, env=box.env(extra, pythonpath), stdin=subprocess.DEVNULL, + capture_output=True, encoding="utf-8", errors="replace", + timeout=120, check=False) + + +def run_helper(box, inventory, cpp, so, out, pythonpath=None): + command = [sys.executable, str(HELPER), "--inventory", str(inventory), "--cpp", str(cpp), + "--so", str(so), "--out", str(out)] + return subprocess.run(command, env=box.env(None, pythonpath), stdin=subprocess.DEVNULL, + capture_output=True, encoding="utf-8", errors="replace", + timeout=120, check=False) + + +def helper_files(box, inventory=None, entries=SAME_TICKER_DAILY): + cpp, so = box.work / "strategy.cpp", box.work / "strategy.so" + supplied = box.work / "supplied.json" + cpp.write_bytes(CPP_BYTES) + so.write_bytes(SO_BYTES) + document = source_inventory(CPP_BYTES, entries) if inventory is None else inventory + supplied.write_text(json.dumps(document), encoding="utf-8") + return supplied, cpp, so, box.work / "bound.json" + + +# --- The helper: bind_compiled_inventory.py -------------------------------------- + +@pytest.mark.parametrize("entries", ENTRY_SETS, ids=ENTRY_IDS) +def test_helper_binds_without_dropping_or_reducing_entries(box, entries): + supplied, cpp, so, out = helper_files(box, entries=entries) + before = supplied.read_bytes() + result = run_helper(box, supplied, cpp, so, out) + assert result.returncode == 0, result.stderr + bound = json.loads(out.read_text(encoding="utf-8")) + assert bound == bound_inventory(CPP_BYTES, SO_BYTES, entries) + assert bound["primary_chart_timeframe"] is None + assert supplied.read_bytes() == before + assert box.events() == ["pinned:bind"] + + +def test_helper_refuses_a_wrong_source_digest(box): + other = source_inventory(b"some other generated source", SAME_TICKER_DAILY) + supplied, cpp, so, out = helper_files(box, inventory=other) + before = supplied.read_bytes() + result = run_helper(box, supplied, cpp, so, out) + assert result.returncode == 14 + assert result.stderr.startswith("bind_compiled_inventory: ") + assert "source_sha256" in result.stderr + assert not out.exists() + assert supplied.read_bytes() == before + + +def test_helper_refuses_an_inventory_that_is_already_bound(box): + bound = bound_inventory(CPP_BYTES, SO_BYTES, SAME_TICKER_DAILY) + supplied, cpp, so, out = helper_files(box, inventory=bound) + result = run_helper(box, supplied, cpp, so, out) + assert result.returncode == 14 + assert "already set" in result.stderr + assert not out.exists() + + +BAD_INVENTORY_FILES = [ + (b"", 11), + (b'{"schema":', 11), + (b'{"schema": "\xff"}', 11), + (b"\xef\xbb\xbf{}", 11), + (b'{"schema": "a", "schema": "b"}', 11), + (b'{"primary_chart_timeframe": NaN}', 11), + (b"{} trailing", 11), + (b"[]", 14), +] +BAD_INVENTORY_IDS = ["empty", "truncated", "bad-utf8", "bom", "duplicate-key", "nan", + "trailing-garbage", "not-an-object"] + + +@pytest.mark.parametrize("data,code", BAD_INVENTORY_FILES, ids=BAD_INVENTORY_IDS) +def test_helper_refuses_a_malformed_inventory(box, data, code): + supplied, cpp, so, out = helper_files(box) + supplied.write_bytes(data) + result = run_helper(box, supplied, cpp, so, out) + assert result.returncode == code, result.stderr + assert result.stderr.startswith("bind_compiled_inventory: ") + assert not out.exists() + assert supplied.read_bytes() == data + + +def test_helper_refuses_an_oversize_inventory(box): + supplied, cpp, so, out = helper_files(box) + supplied.write_bytes(b" " * (8 * 1024 * 1024 + 1)) + result = run_helper(box, supplied, cpp, so, out) + assert result.returncode == 10 + assert "over" in result.stderr + assert not out.exists() + + +def test_helper_refuses_an_unreadable_inventory(box): + supplied, cpp, so, out = helper_files(box) + result = run_helper(box, box.work / "absent.json", cpp, so, out) + assert result.returncode == 10 + assert "cannot be read" in result.stderr + assert not out.exists() + + +@pytest.mark.parametrize("missing", ["cpp", "so"]) +def test_helper_refuses_an_unreadable_source_or_library(box, missing): + supplied, cpp, so, out = helper_files(box) + absent = box.work / "absent.bin" + result = run_helper(box, supplied, absent if missing == "cpp" else cpp, + absent if missing == "so" else so, out) + assert result.returncode == 12 + assert "cannot be read" in result.stderr + assert not out.exists() + assert box.events() == [] + + +def test_helper_never_replaces_a_file(box): + supplied, cpp, so, out = helper_files(box) + before = supplied.read_bytes() + same = run_helper(box, supplied, cpp, so, supplied) + assert same.returncode == 15 + assert "exists already" in same.stderr + assert supplied.read_bytes() == before + out.write_bytes(b"earlier output") + again = run_helper(box, supplied, cpp, so, out) + assert again.returncode == 15 + assert out.read_bytes() == b"earlier output" + assert box.events() == [] + + +def test_helper_refuses_an_output_directory_that_does_not_exist(box): + supplied, cpp, so, _ = helper_files(box) + result = run_helper(box, supplied, cpp, so, box.work / "absent-dir" / "bound.json") + assert result.returncode == 15 + assert "does not exist" in result.stderr + + +def test_helper_publishes_by_link_so_the_temporary_name_is_a_second_name(box): + supplied, cpp, so, out = helper_files(box) + result = run_helper(box, supplied, cpp, so, out) + assert result.returncode == 0, result.stderr + names = sorted(path.name for path in box.work.iterdir()) + temporaries = [name for name in names if name.startswith(".bound-inventory-")] + assert len(temporaries) == 1 + assert names == sorted(["bound.json", "strategy.cpp", "strategy.so", "supplied.json"] + + temporaries) + temporary = box.work / temporaries[0] + assert os.path.samefile(temporary, out) + assert out.stat().st_nlink == 2 + assert temporary.read_bytes() == out.read_bytes() + assert out.read_text(encoding="utf-8").endswith("\n") + + +def test_helper_refuses_and_keeps_the_original_bytes_when_out_appears_at_the_link(box): + supplied, cpp, so, out = helper_files(box) + before = supplied.read_bytes() + racer = b"the racing writer's original bytes\n" + shim = box.root / "race_helper.py" + shim.write_text(RACE_SHIM, encoding="utf-8") + command = [sys.executable, str(shim), "--inventory", str(supplied), "--cpp", str(cpp), + "--so", str(so), "--out", str(out)] + result = subprocess.run(command, env=box.env({"RACE_BYTES_HEX": racer.hex()}), + stdin=subprocess.DEVNULL, capture_output=True, encoding="utf-8", + errors="replace", timeout=120, check=False) + assert result.returncode == 15, result.stderr + assert result.stderr.startswith("bind_compiled_inventory: ") + assert "exists already" in result.stderr + assert box.events() == ["pinned:bind", "race:link"] + assert out.read_bytes() == racer + assert out.stat().st_nlink == 1 + assert supplied.read_bytes() == before + leftovers = [path for path in box.work.iterdir() if path.name.startswith(".bound-inventory-")] + assert len(leftovers) == 1 + assert not os.path.samefile(leftovers[0], out) + + +def test_helper_refuses_when_the_pinned_binder_is_missing(box): + supplied, cpp, so, out = helper_files(box) + result = run_helper(box, supplied, cpp, so, out, pythonpath=box.nopkgs) + assert result.returncode == 13 + assert "bind_request_feed_inventory" in result.stderr + assert not out.exists() + + +def test_helper_source_reimplements_nothing_and_loads_no_library(): + tree = ast.parse(HELPER.read_text(encoding="utf-8")) + imported = set() + pinned_names = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add((node.module or "").split(".")[0]) + if node.module == "pineforge_codegen.request_feed_inventory": + pinned_names.update(alias.name for alias in node.names) + assert not imported & {"ctypes", "cffi", "subprocess", "hashlib", "re", "importlib", + "runpy", "socket", "urllib", "http", "shutil"} + assert "bind_request_feed_inventory" in pinned_names + called = {node.func.attr for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute)} + assert not called & {"sha256", "sha1", "md5", "CDLL", "LoadLibrary", "dlopen", "system", + "popen", "unlink", "remove", "rmdir", "rmtree", "removedirs", + "rename", "renames", "replace"} + assert "link" in called + + +# --- The entrypoint --------------------------------------------------------------- + +def test_entrypoint_adds_no_deletion_beyond_its_one_exit_trap(): + text = ENTRYPOINT.read_text(encoding="utf-8") + assert re.findall(r"\brm\b", text) == ["rm"] + assert "trap 'rm -rf \"$WORK\"' EXIT" in text + + +@pytest.mark.parametrize("kind", ["pine", "cpp"]) +def test_off_branch_reads_no_inventory_and_keeps_the_old_argv(box, kind): + sentinel = "/nonexistent/off-branch-sentinel-inventory.json" + put_pine(box) if kind == "pine" else put_cpp(box) + extra = dict(SELECTED_ENV, PINEFORGE_REQUEST_FEED_INVENTORY=sentinel, STUB_CP_FAIL="1") + del extra["PINEFORGE_REPORT_POLICY"] + result = run_entrypoint(box, extra, xtrace=True) + assert result.returncode == 0, result.stderr + assert "run_json_stub" in result.stdout + assert box.events() == (["codegen:transpile", "gxx", "run_json"] if kind == "pine" + else ["gxx", "run_json"]) + argv = box.logged("run_json.argv.json") + assert normalized(argv) == old_argv("true" if kind == "pine" else "false") + assert not any("inventory" in name for name in box.logged("run_json.work.json")) + if kind == "cpp": + # The ordinary path still compiles and reports the caller's own file, with no + # snapshot (a stub cp that refuses the snapshot is on PATH and never reached). + caller = str(box.indir / "strategy.cpp") + assert argv[argv.index("--generated-cpp") + 1] == caller + assert (box.log / "gxx.source.path").read_text(encoding="utf-8").strip() == caller + assert box.logged("run_json.work.json") == ["strategy.so"] + assert sentinel not in result.stderr + assert not (box.log / "helper.argv.json").exists() + + +def test_transpile_only_keeps_the_old_output_and_reads_no_inventory(box): + sentinel = "/nonexistent/transpile-only-sentinel-inventory.json" + put_pine(box) + result = run_entrypoint(box, {"PINEFORGE_TRANSPILE_ONLY": "1", + "PINEFORGE_REPORT_POLICY": POLICY, + "PINEFORGE_REQUEST_FEED_INVENTORY": sentinel}, xtrace=True) + assert result.returncode == 0, result.stderr + assert result.stdout == fake_cpp(PINE_TEXT) + assert box.events() == ["codegen:transpile"] + assert sentinel not in result.stderr + + +@pytest.mark.parametrize("entries", ENTRY_SETS, ids=ENTRY_IDS) +def test_selected_pine_path_keeps_the_source_and_binds_after_linking(box, entries): + put_pine(box) + result = run_entrypoint(box, dict(SELECTED_ENV, FAKE_CODEGEN_ENTRIES=json.dumps(entries))) + assert result.returncode == 0, result.stderr + assert box.events() == ["codegen:transpile_with_request_inventory", "gxx", "helper", + "pinned:bind", "run_json"] + assert box.logged("codegen.kwargs.json") == ["filename"] + cpp = fake_cpp(PINE_TEXT).encode("utf-8") + assert (box.log / "gxx.source.bin").read_bytes() == cpp + artifact = (box.log / "gxx.artifact.bin").read_bytes() + assert normalized(box.logged("run_json.argv.json")) == ( + old_argv("true") + SELECTED_FLAGS + ["--request-feed-inventory", ""]) + assert box.logged("run_json.inventory.json") == bound_inventory(cpp, artifact, entries) + assert {"strategy.cpp", "strategy.so", "request_feed_inventory.source.json", + "request_feed_inventory.json"} <= set(box.logged("run_json.work.json")) + + +@pytest.mark.parametrize("entries", ENTRY_SETS, ids=ENTRY_IDS) +def test_selected_precompiled_cpp_binds_the_callers_inventory(box, entries): + put_cpp(box) + supplied = put_supplied(box, source_inventory(CPP_BYTES, entries)) + before = supplied.read_bytes() + result = run_entrypoint(box, dict(SELECTED_ENV, + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied))) + assert result.returncode == 0, result.stderr + assert box.events() == ["gxx", "helper", "pinned:bind", "run_json"] + assert (box.log / "gxx.source.bin").read_bytes() == CPP_BYTES + artifact = (box.log / "gxx.artifact.bin").read_bytes() + assert box.logged("run_json.inventory.json") == bound_inventory(CPP_BYTES, artifact, entries) + assert supplied.read_bytes() == before + helper_argv = box.logged("helper.argv.json") + assert helper_argv[helper_argv.index("--inventory") + 1] == str(supplied) + run_argv = box.logged("run_json.argv.json") + assert run_argv[run_argv.index("--request-feed-inventory") + 1] != str(supplied) + + +@pytest.mark.parametrize("supplied_env", [{}, {"PINEFORGE_REQUEST_FEED_INVENTORY": ""}], + ids=["unset", "empty"]) +def test_selected_precompiled_cpp_without_inventory_gets_none_never_an_empty_one(box, + supplied_env): + put_cpp(box) + result = run_entrypoint(box, dict(SELECTED_ENV, **supplied_env)) + assert result.returncode == 0, result.stderr + assert box.events() == ["gxx", "run_json"] + argv = box.logged("run_json.argv.json") + assert "--request-feed-inventory" not in argv + assert normalized(argv) == old_argv("false") + SELECTED_FLAGS + assert not any("inventory" in name for name in box.logged("run_json.work.json")) + + +def test_a_supplied_inventory_is_not_used_with_a_pine_input(box): + sentinel = "/nonexistent/pine-input-sentinel-inventory.json" + put_pine(box) + result = run_entrypoint(box, dict(SELECTED_ENV, PINEFORGE_REQUEST_FEED_INVENTORY=sentinel), + xtrace=True) + assert result.returncode == 0, result.stderr + assert box.events() == ["codegen:transpile_with_request_inventory", "gxx", "helper", + "pinned:bind", "run_json"] + helper_argv = box.logged("helper.argv.json") + assert helper_argv[helper_argv.index("--inventory") + 1].endswith( + "request_feed_inventory.source.json") + assert sentinel not in result.stderr + + +@pytest.mark.parametrize("kind", ["pine", "cpp"]) +def test_link_failure_stops_before_the_binder(box, kind): + if kind == "pine": + put_pine(box) + extra = dict(SELECTED_ENV, STUB_GXX_FAIL="1") + else: + put_cpp(box) + supplied = put_supplied(box, source_inventory(CPP_BYTES, SAME_TICKER_DAILY)) + extra = dict(SELECTED_ENV, STUB_GXX_FAIL="1", + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied)) + result = run_entrypoint(box, extra) + assert result.returncode == 3 + assert "compile failed" in result.stderr + assert box.events()[-1] == "gxx" + assert not (box.log / "helper.argv.json").exists() + assert not (box.log / "run_json.argv.json").exists() + assert result.stdout == "" + + +def test_binder_refusal_stops_before_run_json_with_no_fallback(box): + put_cpp(box) + supplied = put_supplied(box, source_inventory(b"some other generated source", + SAME_TICKER_DAILY)) + result = run_entrypoint(box, dict(SELECTED_ENV, + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied))) + assert result.returncode == 3 + assert box.events() == ["gxx", "helper", "pinned:bind"] + assert BINDING_FAILED in result.stderr + assert "source_sha256" in result.stderr + assert result.stdout == "" + assert not (box.log / "run_json.argv.json").exists() + + +def test_a_pine_inventory_whose_source_digest_is_wrong_stops_before_run_json(box): + put_pine(box) + result = run_entrypoint(box, dict(SELECTED_ENV, FAKE_CODEGEN_WRONG_SOURCE="1")) + assert result.returncode == 3 + assert box.events() == ["codegen:transpile_with_request_inventory", "gxx", "helper", + "pinned:bind"] + assert BINDING_FAILED in result.stderr + assert not (box.log / "run_json.argv.json").exists() + + +@pytest.mark.parametrize("content", [None, b"", b"{", b'{"schema": 1, "schema": 2}'], + ids=["absent", "empty", "truncated", "duplicate-key"]) +def test_an_unreadable_or_malformed_supplied_inventory_ends_the_run(box, content): + put_cpp(box) + supplied = box.root / "supplied-inventory.json" + if content is not None: + supplied.write_bytes(content) + result = run_entrypoint(box, dict(SELECTED_ENV, + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied))) + assert result.returncode == 3 + assert box.events() == ["gxx", "helper"] + assert BINDING_FAILED in result.stderr + assert not (box.log / "run_json.argv.json").exists() + + +@pytest.mark.parametrize("kind", ["pine", "cpp"]) +def test_an_unknown_policy_is_forwarded_raw_and_binds_nothing(box, kind): + put_pine(box) if kind == "pine" else put_cpp(box) + result = run_entrypoint(box, {"PINEFORGE_REPORT_POLICY": "selected-window/v2", + "PINEFORGE_WINDOW_START_MS": "1", + "PINEFORGE_REQUEST_FEED_INVENTORY": "/nonexistent/x.json", + "STUB_CP_FAIL": "1"}) + assert result.returncode == 0, result.stderr + assert box.events() == (["codegen:transpile", "gxx", "run_json"] if kind == "pine" + else ["gxx", "run_json"]) + argv = box.logged("run_json.argv.json") + assert normalized(argv) == old_argv("true" if kind == "pine" else "false") + [ + "--report-policy", "selected-window/v2", "--window-start-ms", "1"] + if kind == "cpp": + assert argv[argv.index("--generated-cpp") + 1] == str(box.indir / "strategy.cpp") + + +def test_selected_precompiled_cpp_is_frozen_once_and_every_consumer_uses_the_copy(box): + put_cpp(box) + caller = box.indir / "strategy.cpp" + supplied = put_supplied(box, source_inventory(CPP_BYTES, SAME_TICKER_DAILY)) + result = run_entrypoint(box, dict(SELECTED_ENV, + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied), + STUB_GXX_MUTATE=str(caller))) + assert result.returncode == 0, result.stderr + assert box.events() == ["gxx", "helper", "pinned:bind", "run_json"] + # Control: the caller's file really was changed after the snapshot was taken. + assert caller.read_bytes() != CPP_BYTES + # The compiler was given a private file under the entrypoint's work dir and saw the + # original bytes, not the changed ones. + compiled = (box.log / "gxx.source.path").read_text(encoding="utf-8").strip() + assert compiled != str(caller) + assert compiled.startswith(str(box.tmp) + os.sep) + assert os.path.basename(compiled) == "strategy.cpp" + assert (box.log / "gxx.source.bin").read_bytes() == CPP_BYTES + # The binder was handed that same file and bound the original bytes ... + helper_argv = box.logged("helper.argv.json") + assert helper_argv[helper_argv.index("--cpp") + 1] == compiled + artifact = (box.log / "gxx.artifact.bin").read_bytes() + assert box.logged("run_json.inventory.json") == bound_inventory(CPP_BYTES, artifact, + SAME_TICKER_DAILY) + # ... and run_json's provenance file is the same private file, still not transpiled. + run_argv = box.logged("run_json.argv.json") + assert run_argv[run_argv.index("--generated-cpp") + 1] == compiled + assert run_argv[run_argv.index("--transpiled") + 1] == "false" + assert (box.log / "run_json.generated.bin").read_bytes() == CPP_BYTES + + +def test_a_failed_snapshot_ends_the_selected_path_before_compile_and_run(box): + put_cpp(box) + supplied = put_supplied(box, source_inventory(CPP_BYTES, SAME_TICKER_DAILY)) + result = run_entrypoint(box, dict(SELECTED_ENV, + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied), + STUB_CP_FAIL="1")) + assert result.returncode == 3 + assert "cannot snapshot strategy.cpp into the work dir" in result.stderr + assert box.events() == ["cp:fail"] + assert result.stdout == "" + + +def test_a_source_digest_claimed_by_the_strategy_code_is_not_accepted(box): + built_for = b"the bytes the inventory was built for\n" + claiming = ("// source_sha256: " + sha(built_for) + "\n").encode("ascii") + CPP_BYTES + (box.indir / "strategy.cpp").write_bytes(claiming) + supplied = put_supplied(box, source_inventory(built_for, SAME_TICKER_DAILY)) + result = run_entrypoint(box, dict(SELECTED_ENV, + PINEFORGE_REQUEST_FEED_INVENTORY=str(supplied))) + assert result.returncode == 3 + assert box.events() == ["gxx", "helper", "pinned:bind"] + assert BINDING_FAILED in result.stderr + assert "source_sha256" in result.stderr + assert not (box.log / "run_json.argv.json").exists() + + +def test_window_flags_keep_raw_values_and_invent_no_default(box): + put_cpp(box) + result = run_entrypoint(box, {"PINEFORGE_REPORT_POLICY": POLICY, + "PINEFORGE_WINDOW_START_MS": " 12 ", + "PINEFORGE_WINDOW_END_MS": "abc"}) + assert result.returncode == 0, result.stderr + assert normalized(box.logged("run_json.argv.json")) == old_argv("false") + [ + "--report-policy", POLICY, "--window-start-ms", " 12 ", "--window-end-ms", "abc"] + + +def test_selected_transpile_failure_is_exit_5_with_no_ordinary_fallback(box): + put_pine(box) + result = run_entrypoint(box, dict(SELECTED_ENV, FAKE_CODEGEN_FAIL="1")) + assert result.returncode == 5 + assert "transpile error" in result.stderr + assert box.events() == ["codegen:transpile_with_request_inventory"] + + +def test_a_codegen_without_the_inventory_api_is_exit_5_with_no_ordinary_fallback(box): + put_pine(box) + result = run_entrypoint(box, SELECTED_ENV, pythonpath=box.nopkgs) + assert result.returncode == 5 + assert "no request-feed inventory API" in result.stderr + assert box.events() == [] diff --git a/tests/test_request_feed_inventory_admission.py b/tests/test_request_feed_inventory_admission.py new file mode 100644 index 000000000..be59f18e2 --- /dev/null +++ b/tests/test_request_feed_inventory_admission.py @@ -0,0 +1,776 @@ +"""Tests of docker/request_feed_inventory.py, the consumer half of the request-feed inventory. + +These are unit controls on a standalone API. Fixtures write temporary inventory and artifact bytes +and call the helper; every refusal is compared with the exact code and detail dict of the +interface, and each negative is one defect injected into an inventory that a positive control in +the same test resolves. They do not cover run_json's placement of the admission (after request +validation, before the first feed or library operation), a real wrapper, or the pinned +transpiler's inventories. The poison sentinels are process-wide entry points (a library load, a +feed reader, a child process, a module import) patched to record and fail, and open() is +recorded: they show the helper starts none of them and opens only the two files it may read. +""" +import ast +import builtins +import contextlib +import csv +import ctypes +import hashlib +import importlib +import importlib.util +import inspect +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "docker" / "request_feed_inventory.py" + + +def load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +inventory = load_module("request_feed_inventory", MODULE_PATH) +Error = inventory.InventoryAdmissionError + +# What the interface says, written out here and not read back from the module under test. +SCHEMA = "pineforge-request-feed-inventory/v1" +CODE = "window_mode_unsupported" +INVENTORY_CAPABILITY = "selected_window_request_feed_inventory" +CHART_CAPABILITY = "selected_window_chart_timeframe" +FEED_CAPABILITY = "selected_window_request_feed_timeframe" +INCOMPLETE = {"capability": INVENTORY_CAPABILITY} # and no timeframe member + +ARTIFACT_BYTES = b"not a library: hashed by the helper, never loaded\n" +ARTIFACT_SHA = hashlib.sha256(ARTIFACT_BYTES).hexdigest() +OTHER_SHA = hashlib.sha256(b"the bytes of another artifact\n").hexdigest() +SOURCE_SHA = hashlib.sha256(b"the generated C++ bytes\n").hexdigest() +PRIMARY_LIST = ["1", "3", "5", "15", "30", "60", "120", "240"] # the first build's eight +FEED_LIST = ["1", "D"] +DEFAULT = object() + +CANONICAL = {"1": "1", "15": "15", "240": "240", "99999": "99999", "100000": "100000", + "999999": "999999", "1000000": "1000000", "D": "D", "1D": "D", + "W": "W", "1W": "W", "M": "1M", "1M": "1M", "S": "1S", "1S": "1S", "2D": "2D", + "3W": "3W", "12M": "12M", "30S": "30S", "9999D": "9999D", "10000D": "10000D", + "99999W": "99999W", "100000M": "100000M", "123456S": "123456S"} +BAD_TOKENS = ("", " ", " 15", "15 ", "15\n", "\t1D", "0", "00", "015", "-1", "+1", "1.5", "1e2", + "4h", "1H", "d", "1d", "w", "m", "s", "DD", "D1", "1DD", "0D", + "01D", "１５", "٣", "15m", "1 D", "None", "null") + + +def policy(primary=DEFAULT, feeds=DEFAULT, preroll="supported"): + return {"primary_chart_timeframes": list(PRIMARY_LIST) if primary is DEFAULT else primary, + "preroll": preroll, + "request_feed_timeframes": list(FEED_LIST) if feeds is DEFAULT else feeds} + + +def token(timeframe): + return {"kind": "token", "timeframe": timeframe} + + +def input_ref(key, default): + return {"kind": "input", "key": key, "default": default} + + +def unknown(): + return {"kind": "unknown"} + + +def inv(entries=(), binding=None, artifact_sha=ARTIFACT_SHA): + """An inventory object as the pinned transpiler emits it once bound to its artifact.""" + return {"schema": SCHEMA, "source_sha256": SOURCE_SHA, "artifact_sha256": artifact_sha, + "primary_chart_timeframe": binding, "entries": list(entries)} + + +def raw(entries="[]", binding="null", artifact_sha=ARTIFACT_SHA): + """An inventory written by hand, for what json.dumps cannot make (a duplicate key).""" + return ('{"schema": "%s", "source_sha256": "%s", "artifact_sha256": "%s", ' + '"primary_chart_timeframe": %s, "entries": %s}' + % (SCHEMA, SOURCE_SHA, artifact_sha, binding, entries)) + + +class Poison: + """A stand-in for work the helper must never start: calling it records the call and fails.""" + + def __init__(self, name, calls): + self.name = name + self.calls = calls + + def __call__(self, *args, **kwargs): + self.calls.append(self.name) + raise AssertionError(f"poison sentinel called: {self.name}") + + +@contextlib.contextmanager +def watch(): + """Patch the process-wide entry points through which a helper could load a library, read a + feed, start a child process or import a module, to record and fail, and record every + open(). No strategy or optimizer exists at this level: the child process and the module + import are the ways the helper could start a constructor or a trial. Yields (calls, opened): + the sentinels called, and the (path, mode) of every open().""" + calls, opened = [], [] + real_open = builtins.open + + def recording_open(file, *args, **kwargs): + mode = args[0] if args else kwargs.get("mode", "r") + opened.append((os.fspath(file) if isinstance(file, (str, os.PathLike)) else file, mode)) + return real_open(file, *args, **kwargs) + + sentinels = [(ctypes, "CDLL", "library"), (ctypes, "PyDLL", "library"), + (ctypes.cdll, "LoadLibrary", "library"), (ctypes.pydll, "LoadLibrary", "library"), + (csv, "reader", "feed"), (csv, "DictReader", "feed"), + (subprocess, "Popen", "trial"), (os, "system", "trial"), + (importlib, "import_module", "constructor")] + with contextlib.ExitStack() as stack: + for target, name, label in sentinels: + stack.enter_context(mock.patch.object(target, name, Poison(f"{label}:{name}", calls))) + stack.enter_context(mock.patch.object(builtins, "open", recording_open)) + yield calls, opened + + +class Case(unittest.TestCase): + def setUp(self): + scratch = tempfile.TemporaryDirectory() + self.addCleanup(scratch.cleanup) + self.dir = Path(scratch.name) + self.inventory_path = self.dir / "inventory.json" + self.artifact_path = self.dir / "strategy.bin" + self.artifact_path.write_bytes(ARTIFACT_BYTES) + + def write(self, content): + """content is a dict (written as JSON), a str (UTF-8) or bytes: the inventory file.""" + if isinstance(content, dict): + content = json.dumps(content) + if isinstance(content, str): + content = content.encode("utf-8") + self.inventory_path.write_bytes(content) + + def resolve(self, content=None, inputs=DEFAULT, primary="15"): + if content is not None: + self.write(content) + return inventory.resolve_request_feed_inventory( + self.inventory_path, self.artifact_path, {} if inputs is DEFAULT else inputs, + primary_timeframe=primary) + + def admit(self, content=None, inputs=DEFAULT, primary="15", support=DEFAULT): + if content is not None: + self.write(content) + return inventory.admit_selected_request( + primary, self.inventory_path, self.artifact_path, + {} if inputs is DEFAULT else inputs, policy() if support is DEFAULT else support) + + def refused(self, detail, call, *args, **kwargs): + with self.assertRaises(Error) as caught: + call(*args, **kwargs) + self.assertEqual(caught.exception.code, CODE) + self.assertEqual(caught.exception.detail, detail) + return caught.exception + + def incomplete(self, call, *args, **kwargs): + return self.refused(INCOMPLETE, call, *args, **kwargs) + + +class ErrorAndSurfaceTests(unittest.TestCase): + def test_the_error_carries_the_code_and_a_copied_detail_dict(self): + source = {"capability": CHART_CAPABILITY, "timeframe": "W"} + error = Error("text", source) + self.assertEqual(error.code, CODE) + self.assertEqual(Error.code, CODE) + self.assertEqual(error.detail, source) + source["timeframe"] = "D" + self.assertEqual(error.detail["timeframe"], "W", "the detail is copied") + self.assertEqual(str(error), "text") + self.assertEqual(error.args, ("text",), "BaseException.args stays the exception's own") + self.assertTrue(issubclass(Error, Exception)) + self.assertFalse(issubclass(Error, (OSError, ValueError))) + + def test_public_surface_and_signatures(self): + self.assertEqual(sorted(inventory.__all__), + ["InventoryAdmissionError", "admit_selected_request", + "resolve_request_feed_inventory"]) + resolve = inspect.signature(inventory.resolve_request_feed_inventory) + self.assertEqual(list(resolve.parameters), ["inventory_path", "artifact_path", + "validated_inputs", "primary_timeframe"]) + self.assertIs(resolve.parameters["primary_timeframe"].kind, + inspect.Parameter.KEYWORD_ONLY) + admit = inspect.signature(inventory.admit_selected_request) + self.assertEqual(list(admit.parameters), ["primary_timeframe", "inventory_path", + "artifact_path", "validated_inputs", + "support_policy"]) + for parameter in admit.parameters.values(): + self.assertIs(parameter.kind, inspect.Parameter.POSITIONAL_OR_KEYWORD) + + +class SourceTests(unittest.TestCase): + """The helper reads no feed and loads no library, by construction as well as by the + sentinels below: its imports, its calls and the modes of its two open() calls.""" + + def setUp(self): + self.tree = ast.parse(MODULE_PATH.read_text(encoding="utf-8")) + + def test_imports_are_the_small_allowed_set(self): + imported = set() + for node in ast.walk(self.tree): + if isinstance(node, ast.Import): + imported.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add(node.module) + self.assertEqual(imported, {"__future__", "hashlib", "json", "os", "re", + "collections.abc", "typing"}) + + def test_no_evaluation_coercion_or_library_names(self): + called = {node.func.id for node in ast.walk(self.tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + self.assertFalse(called & {"eval", "exec", "compile", "__import__", "int", "float"}) + names = ({node.id for node in ast.walk(self.tree) if isinstance(node, ast.Name)} + | {node.attr for node in ast.walk(self.tree) if isinstance(node, ast.Attribute)}) + self.assertFalse(names & {"ctypes", "CDLL", "cdll", "dlopen", "subprocess", "Popen", + "csv", "importlib", "run_json", "system"}) + + def test_the_two_open_calls_are_binary_reads(self): + opens = [node for node in ast.walk(self.tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) + and node.func.id == "open"] + self.assertEqual(len(opens), 2) + for call in opens: + self.assertEqual(ast.literal_eval(call.args[1]), "rb") + + +class ResolveTests(Case): + def test_complete_empty_inventory_yields_the_empty_set(self): + for binding in (None, "15"): + with self.subTest(binding=binding): + result = self.resolve(inv([], binding)) + self.assertEqual(result, []) + self.assertIsInstance(result, list) + + def test_literal_tokens_sort_and_collapse(self): + entries = [token("D"), token("1"), token("1"), token("1D"), token("60")] + result = self.resolve(inv(entries)) + self.assertEqual(result, ["1", "60", "D"]) + self.assertEqual(self.resolve(inv(list(reversed(entries)))), result) + self.assertIsNot(self.resolve(), result, "a fresh list each call") + + def test_wire_normalization(self): + for spelled, canonical in CANONICAL.items(): + with self.subTest(spelled=spelled): + self.assertEqual(self.resolve(inv([token(spelled)])), [canonical]) + + def test_malformed_tokens_are_incomplete(self): + self.assertEqual(self.resolve(inv([token("15")])), ["15"]) # the control + for bad in BAD_TOKENS: + with self.subTest(bad=bad): + self.incomplete(self.resolve, inv([token(bad)])) + + def test_long_minutes_and_counts_are_tokens_of_any_length(self): + self.assertEqual(self.resolve(inv([token("100000"), token("1"), token("999999")])), + ["1", "100000", "999999"]) + self.assertEqual(self.resolve(inv([token("1000000")])), ["1000000"]) + self.assertEqual(self.resolve(inv([token("10000D"), token("99999W")])), + ["10000D", "99999W"]) + self.assertEqual(self.resolve(inv([token("10000M"), token("10000S")])), + ["10000M", "10000S"]) + self.assertEqual(self.resolve(inv([token("100000D"), token("1D")])), ["100000D", "D"]) + # digits are compared as spelled: a six-digit binding matches a six-digit primary only + self.assertEqual(self.resolve(inv([token("1")], binding="100000"), primary="100000"), + ["1"]) + self.incomplete(self.resolve, inv([token("1")], binding="100000"), primary="15") + + def test_leading_zero_and_whitespace_spellings_stay_malformed(self): + # the grammar has no leading zero and no whitespace at any length + self.assertEqual(self.resolve(inv([token("100000")])), ["100000"]) # the control + for bad in ("0100000", "00100000", "0999999", "010D", "0010000D", "01W", "001M", + "0001S", " 100000", "100000 ", "100000\n", "\t10000D", "10000D\r", + "10000D\f", " 1D", "1D "): + with self.subTest(bad=bad): + self.incomplete(self.resolve, inv([token(bad)])) + self.incomplete(self.resolve, inv([input_ref("tf", bad)])) + self.incomplete(self.resolve, inv([input_ref("tf", "1")]), inputs={"tf": bad}) + + def test_any_unknown_entry_is_incomplete(self): + self.assertEqual(self.resolve(inv([token("1"), token("D")])), ["1", "D"]) + for entries in ([unknown()], [unknown(), token("1")], [token("1"), unknown()], + [token("1"), unknown(), token("D")]): + with self.subTest(entries=entries): + self.incomplete(self.resolve, inv(entries)) + + +class InputTests(Case): + def test_the_declared_default_is_used_without_a_validated_value(self): + self.assertEqual(self.resolve(inv([input_ref("tf", "D")])), ["D"]) + self.assertEqual(self.resolve(inv([input_ref("tf", "1D")])), ["D"]) + # the key is exact: another key, another case and the empty key are not "tf" + others = {"TF": "W", "tf2": "W", "": "W", "t": "W"} + self.assertEqual(self.resolve(inv([input_ref("tf", "1")]), inputs=others), ["1"]) + + def test_a_validated_value_overrides_the_default(self): + self.assertEqual(self.resolve(inv([input_ref("tf", "W")]), inputs={"tf": "1"}), ["1"]) + self.assertEqual(self.resolve(inv([input_ref("tf", "W")]), inputs={"tf": "1W"}), ["W"]) + self.assertEqual(self.resolve(inv([input_ref("tf", "1")]), inputs={"tf": "D"}), ["D"]) + entries = [input_ref("a", "1"), input_ref("b", "D")] + self.assertEqual(self.resolve(inv(entries), inputs={"a": "5"}), ["5", "D"]) + + def test_the_empty_key_is_an_exact_key_with_its_own_override_and_default(self): + entry = [input_ref("", "D")] + self.assertEqual(self.resolve(inv(entry)), ["D"]) # the control: the default + self.assertEqual(self.resolve(inv(entry), inputs={}), ["D"]) + self.assertEqual(self.resolve(inv(entry), inputs={"tf": "1"}), ["D"]) # another key + self.assertEqual(self.resolve(inv(entry), inputs={"": "1"}), ["1"]) # the override + self.assertEqual(self.resolve(inv(entry), inputs={"": "100000"}), ["100000"]) + self.assertEqual(self.resolve(inv(entry), inputs={"": "1", " ": "W"}), ["1"]) + # a present override never consults the default, and an invalid one never falls back + self.assertEqual(self.resolve(inv([input_ref("", "")]), inputs={"": "1"}), ["1"]) + self.incomplete(self.resolve, inv([input_ref("", "")])) + self.incomplete(self.resolve, inv(entry), inputs={"": ""}) + self.incomplete(self.resolve, inv(entry), inputs={"": 15}) + self.incomplete(self.resolve, inv(entry), inputs={"": "4h"}) + # the key is exact: a space is another key, and "" is not "tf" + self.assertEqual(self.resolve(inv([input_ref(" ", "1")]), inputs={"": "W"}), ["1"]) + self.assertEqual(self.resolve(inv([input_ref(" ", "1")]), inputs={" ": "W"}), ["W"]) + + def test_an_invalid_or_nonstring_value_is_incomplete_and_never_falls_back(self): + entries = [input_ref("tf", "1")] + self.assertEqual(self.resolve(inv(entries), inputs={"tf": "5"}), ["5"]) # the control + for value in (15, 1.5, True, None, ["15"], {"a": "1"}, "", " 15", "15 ", "4h", "15\n", + "１５"): + with self.subTest(value=value): + self.incomplete(self.resolve, inv(entries), inputs={"tf": value}) + + def test_an_invalid_default_with_no_validated_value_is_incomplete(self): + self.assertEqual(self.resolve(inv([input_ref("tf", "1")])), ["1"]) # the control + for default in ("", " 1", "4h", "d", "0", "1.0"): + with self.subTest(default=default): + self.incomplete(self.resolve, inv([input_ref("tf", default)])) + self.incomplete(self.resolve, inv([input_ref("tf", default)]), + inputs={"other": "1"}) + + def test_validated_inputs_must_be_a_mapping(self): + self.assertEqual(self.resolve(inv([]), inputs={}), []) # the control + for inputs in (None, [], ["tf"], "tf", 5): + with self.subTest(inputs=inputs): + self.incomplete(self.resolve, inv([]), inputs=inputs) + + +class InventoryFileTests(Case): + def test_missing_and_unreadable_inventory_is_incomplete(self): + self.write(inv([token("1")])) + self.assertEqual(self.resolve(), ["1"]) # the control + absent = self.dir / "absent.json" + directory = self.dir / "a-directory" + directory.mkdir() + nul = str(self.dir) + "/in\0v.json" + for path in (absent, str(absent), directory, None, 3, True, b"/x", object(), nul): + with self.subTest(path=path): + self.incomplete(inventory.resolve_request_feed_inventory, path, + self.artifact_path, {}, primary_timeframe="15") + + def test_bytes_that_are_not_strict_json_are_incomplete(self): + good = json.dumps(inv([token("1")])) + self.assertEqual(self.resolve(good), ["1"]) # the control + cases = { + "empty": b"", "blank": b" \n", "not json": b"inventory", + "truncated": good[:-1].encode(), "trailing data": (good + " {}").encode(), + "bad utf-8": b'{"schema": "\xff"}', "bom": b"\xef\xbb\xbf" + good.encode(), + "utf-16": good.encode("utf-16"), "single quotes": good.replace('"', "'"), + "top-level array": "[]", "top-level null": "null", "top-level string": '"x"', + "empty object": "{}", "deep nesting": "[" * 100000 + "]" * 100000, + } + for name, content in cases.items(): + with self.subTest(name=name): + self.incomplete(self.resolve, content) + + def test_an_oversize_inventory_is_incomplete(self): + content = json.dumps(inv([token("1")])) + size = len(content.encode("utf-8")) + self.write(content) + with mock.patch.object(inventory, "_INVENTORY_MAX_BYTES", size): + self.assertEqual(self.resolve(), ["1"]) + with mock.patch.object(inventory, "_INVENTORY_MAX_BYTES", size - 1): + self.incomplete(self.resolve) + + def test_duplicate_keys_are_refused_even_when_the_values_agree(self): + self.assertEqual(self.resolve(raw('[{"kind": "token", "timeframe": "1"}]')), ["1"]) + cases = { + "top level, equal values": raw()[:-1] + ', "schema": "%s"}' % SCHEMA, + "top level, other digest last": raw()[:-1] + ', "artifact_sha256": "%s"}' % OTHER_SHA, + "entry, permitted value last": + raw('[{"kind": "token", "timeframe": "W", "timeframe": "1"}]'), + "entry, permitted value first": + raw('[{"kind": "token", "timeframe": "1", "timeframe": "W"}]'), + "input entry": raw('[{"kind": "input", "key": "a", "key": "b", "default": "1"}]'), + } + for name, content in cases.items(): + with self.subTest(name=name): + self.incomplete(self.resolve, content) + + def test_extra_missing_and_mistyped_members_are_incomplete(self): + good = inv([token("1")]) + self.assertEqual(self.resolve(good), ["1"]) # the control + mutations = {"extra member": dict(good, extra=1)} + for member in good: + mutations[f"without {member}"] = {k: v for k, v in good.items() if k != member} + mutations.update({ + "schema v2": dict(good, schema="pineforge-request-feed-inventory/v2"), + "schema empty": dict(good, schema=""), + "schema null": dict(good, schema=None), + "schema number": dict(good, schema=1), + "source digest null": dict(good, source_sha256=None), + "source digest number": dict(good, source_sha256=5), + "source digest short": dict(good, source_sha256="0" * 63), + "source digest long": dict(good, source_sha256="0" * 65), + "source digest not hex": dict(good, source_sha256="g" * 64), + "source digest upper case": dict(good, source_sha256="A" * 64), + "source digest padded": dict(good, source_sha256=" " + "0" * 63), + "artifact digest number": dict(good, artifact_sha256=5), + "artifact digest short": dict(good, artifact_sha256="0" * 63), + "artifact digest upper case": dict(good, artifact_sha256="A" * 64), + "binding number": dict(good, primary_chart_timeframe=15), + "binding empty": dict(good, primary_chart_timeframe=""), + "binding text": dict(good, primary_chart_timeframe="4h"), + "entries null": dict(good, entries=None), + "entries object": dict(good, entries={"0": token("1")}), + "entries string": dict(good, entries="[]"), + }) + entries = { + "entry null": None, "entry string": "token", "entry list": [token("1")], + "entry empty": {}, "kind absent": {"timeframe": "1"}, + "kind null": {"kind": None, "timeframe": "1"}, + "kind upper case": {"kind": "TOKEN", "timeframe": "1"}, + "kind other": {"kind": "other", "timeframe": "1"}, + "kind list": {"kind": ["token"], "timeframe": "1"}, + "token extra": dict(token("1"), note="x"), "token without timeframe": {"kind": "token"}, + "token timeframe number": {"kind": "token", "timeframe": 15}, + "token timeframe null": {"kind": "token", "timeframe": None}, + "token timeframe list": {"kind": "token", "timeframe": ["1"]}, + "input without default": {"kind": "input", "key": "tf"}, + "input without key": {"kind": "input", "default": "1"}, + "input extra": dict(input_ref("tf", "1"), note="x"), + "input key number": input_ref(5, "1"), + "input default null": input_ref("tf", None), + "input default number": input_ref("tf", 1), + "unknown extra": dict(unknown(), note="x"), + } + for name, entry in entries.items(): + mutations[name] = dict(good, entries=[token("1"), entry]) + for name, mutated in mutations.items(): + with self.subTest(name=name): + self.incomplete(self.resolve, mutated) + + +class ArtifactBindingTests(Case): + def test_unbound_wrong_and_changed_artifacts_are_incomplete(self): + self.assertEqual(self.resolve(inv([token("1")])), ["1"]) # the control + self.incomplete(self.resolve, inv([token("1")], artifact_sha=None)) + self.incomplete(self.resolve, inv([token("1")], artifact_sha=OTHER_SHA)) + self.write(inv([token("1")])) + self.artifact_path.write_bytes(ARTIFACT_BYTES + b"\0") # changed after it was bound + self.incomplete(self.resolve) + self.artifact_path.write_bytes(ARTIFACT_BYTES) + self.assertEqual(self.resolve(), ["1"]) + + def test_missing_and_unreadable_artifact_is_incomplete(self): + self.write(inv([token("1")])) + self.assertEqual(self.resolve(), ["1"]) # the control + nul = str(self.dir) + "/a\0b" + for path in (self.dir / "absent.bin", self.dir, None, 3, True, b"/x", object(), nul): + with self.subTest(path=path): + self.incomplete(inventory.resolve_request_feed_inventory, self.inventory_path, + path, {}, primary_timeframe="15") + + def test_the_artifact_is_hashed_in_chunks(self): + self.write(inv([token("1")])) + with mock.patch.object(inventory, "_READ_CHUNK", 7): + self.assertEqual(self.resolve(), ["1"]) + + +class ChartBindingTests(Case): + def test_a_recorded_binding_must_equal_the_requests_primary(self): + entries = [token("1")] + self.assertEqual(self.resolve(inv(entries, binding="15"), primary="15"), ["1"]) + self.assertEqual(self.resolve(inv(entries, binding=None), primary="15"), ["1"]) + for binding, primary in (("5", "15"), ("15", "5"), ("D", "W"), ("W", "D"), ("D", "15"), + ("15", "D")): + with self.subTest(binding=binding, primary=primary): + self.incomplete(self.resolve, inv(entries, binding=binding), primary=primary) + # entries that do not depend on the chart do not excuse a mismatch + self.incomplete(self.resolve, inv([], binding="5"), primary="15") + + def test_the_requests_primary_aliases_are_canonicalized_first(self): + for binding, primary in (("D", "1D"), ("D", "D"), ("W", "1W"), ("W", "W")): + with self.subTest(binding=binding, primary=primary): + self.assertEqual(self.resolve(inv([token("1")], binding=binding), + primary=primary), ["1"]) + self.incomplete(self.resolve, inv([token("1")], binding="D"), primary="1W") + + +class AdmitTests(Case): + def test_permitted_tokens_are_admitted_and_returned(self): # the same/other positives + entries = [token("1"), token("1"), token("D")] # same-ticker and other-symbol sites + self.assertEqual(self.admit(inv(entries)), ["1", "D"]) + self.assertEqual(self.admit(), self.resolve()) + self.assertEqual(self.admit(inv([input_ref("tf", "W")]), inputs={"tf": "1D"}), ["D"]) + self.assertEqual(self.admit(inv([]), support=policy(feeds=[])), []) + self.assertEqual(self.admit(inv([token("1")]), support=policy(preroll="zero_only")), + ["1"], "preroll is not this helper's to enforce") + + def test_a_weekly_token_is_excluded_with_the_exact_token(self): + weekly = {"capability": FEED_CAPABILITY, "timeframe": "W"} + for entries, inputs in (([token("W")], {}), ([token("1W")], {}), + ([token("1"), token("W")], {}), + ([input_ref("tf", "W")], {}), + ([input_ref("tf", "1")], {"tf": "1W"}), + ([input_ref("tf", "D")], {"tf": "W"})): + with self.subTest(entries=entries, inputs=inputs): + self.refused(weekly, self.admit, inv(entries), inputs=inputs) + + def test_long_tokens_outside_the_policy_name_their_exact_spelling(self): + # a token of any length that the policy does not list is the feed or the chart capability, + # never the inventory one + self.refused({"capability": FEED_CAPABILITY, "timeframe": "100000"}, self.admit, + inv([token("100000")])) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "999999"}, self.admit, + inv([token("1"), token("999999")])) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "10000D"}, self.admit, + inv([token("10000D")])) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "99999W"}, self.admit, + inv([token("1"), token("99999W")])) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "100000"}, self.admit, + inv([input_ref("tf", "100000")])) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "100000"}, self.admit, + inv([input_ref("tf", "1")]), inputs={"tf": "100000"}) + self.refused({"capability": CHART_CAPABILITY, "timeframe": "100000"}, self.admit, + inv([]), primary="100000") + self.refused({"capability": CHART_CAPABILITY, "timeframe": "10000D"}, self.admit, + inv([]), primary="10000D") + # the same spellings are admitted when the policy lists them + self.assertEqual(self.admit(inv([token("100000")]), + support=policy(feeds=["100000", "10000D"])), ["100000"]) + self.assertEqual(self.admit(inv([token("10000D")]), + support=policy(feeds=["10000D"])), ["10000D"]) + + def test_the_empty_key_reaches_the_policy_like_any_key(self): + self.assertEqual(self.admit(inv([input_ref("", "W")]), inputs={"": "1"}), ["1"]) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "W"}, self.admit, + inv([input_ref("", "1")]), inputs={"": "W"}) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "W"}, self.admit, + inv([input_ref("", "W")])) + + def test_other_excluded_tokens_are_named_canonically(self): + for spelled, canonical in (("5", "5"), ("2D", "2D"), ("M", "1M"), ("S", "1S"), + ("1440", "1440")): + with self.subTest(spelled=spelled): + self.refused({"capability": FEED_CAPABILITY, "timeframe": canonical}, + self.admit, inv([token(spelled)])) + + def test_the_first_excluded_token_in_sorted_order_is_named(self): + # the interface names one token; this helper names the first of the sorted set + self.refused({"capability": FEED_CAPABILITY, "timeframe": "1M"}, self.admit, + inv([token("W"), token("2D"), token("1M"), token("1")])) + + def test_an_incomplete_inventory_is_the_inventory_capability(self): + self.assertEqual(self.admit(inv([token("1")])), ["1"]) # the control + self.incomplete(self.admit, inv([token("1"), unknown()])) + self.incomplete(self.admit, inv([token("1")], artifact_sha=OTHER_SHA)) + self.incomplete(self.admit, inv([token("1")], artifact_sha=None)) + self.incomplete(self.admit, inv([token("1")], binding="5")) + self.incomplete(self.admit, b"") + self.incomplete(inventory.admit_selected_request, "15", self.dir / "absent", + self.artifact_path, {}, policy()) + # the inventory resolves before its tokens are checked: an unknown entry hides no token + self.incomplete(self.admit, inv([token("W"), unknown()])) + + def test_a_primary_outside_the_policy_names_the_canonical_primary(self): + good = inv([token("1")]) + for spelled, canonical in (("W", "W"), ("1W", "W"), ("D", "D"), ("1D", "D"), ("2", "2"), + ("1440", "1440"), ("M", "1M"), ("S", "1S"), ("2D", "2D")): + with self.subTest(primary=spelled): + self.refused({"capability": CHART_CAPABILITY, "timeframe": canonical}, + self.admit, good, primary=spelled) + for primary in PRIMARY_LIST: + with self.subTest(primary=primary): + self.assertEqual(self.admit(good, primary=primary), ["1"]) + + def test_primary_aliases_match_a_policy_that_lists_the_canonical_token(self): + support = policy(primary=["D", "W"], feeds=["1"]) + for primary in ("D", "1D", "W", "1W"): + with self.subTest(primary=primary): + self.assertEqual(self.admit(inv([token("1")]), primary=primary, + support=support), ["1"]) + self.refused({"capability": CHART_CAPABILITY, "timeframe": "15"}, self.admit, + inv([token("1")]), primary="15", support=support) + + def test_a_primary_outside_the_policy_reads_no_file(self): + absent = self.dir / "absent" + for spelled, canonical in (("W", "W"), ("1D", "D")): + with self.subTest(primary=spelled): + with watch() as (calls, opened): + self.refused({"capability": CHART_CAPABILITY, "timeframe": canonical}, + inventory.admit_selected_request, spelled, absent, absent, {}, + policy()) + self.assertEqual((calls, opened), ([], [])) + # the same call with a permitted primary does reach the inventory, and finds it missing + self.incomplete(inventory.admit_selected_request, "15", absent, absent, {}, policy()) + + def test_a_missing_or_invalid_policy_supports_nothing_and_reads_no_file(self): + # a policy that cannot be read exactly declares no support; no new + # capability may be added, so the refusal is the primary one + good = policy() + self.write(inv([])) + self.assertEqual(self.admit(support=good), []) # the control + bad = {"none": None, "empty": {}, "list": [], "word": "all", "number": 1, + "extra member": dict(good, extra=[]), "preroll empty": dict(good, preroll=""), + "preroll case": dict(good, preroll="SUPPORTED"), + "preroll null": dict(good, preroll=None), "preroll number": dict(good, preroll=1)} + for key in good: + bad[f"without {key}"] = {k: v for k, v in good.items() if k != key} + for member in ("primary_chart_timeframes", "request_feed_timeframes"): + for name, value in (("word case", "ALL"), ("other word", "none"), ("null", None), + ("number", 5), ("object", {"1": 1}), ("set", {"1"}), + ("alias in list", ["1D"]), ("space in list", ["D "]), + ("number in list", [1]), ("null in list", [None]), + ("word in list", ["all"]), ("text in list", ["4h"]), + ("empty string in list", [""]), ("nested list", [["1"]])): + bad[f"{member}: {name}"] = dict(good, **{member: value}) + for name, support in bad.items(): + with self.subTest(name=name): + with watch() as (calls, opened): + self.refused({"capability": CHART_CAPABILITY, "timeframe": "15"}, + self.admit, support=support) + self.assertEqual((calls, opened), ([], [])) + + def test_policy_lists_may_be_tuples_repeat_tokens_or_be_empty(self): + self.assertEqual(self.admit(inv([token("1")]), + support=policy(primary=("15", "15"), feeds=("1", "1"))), ["1"]) + self.refused({"capability": CHART_CAPABILITY, "timeframe": "15"}, self.admit, + inv([]), support=policy(primary=[])) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "1"}, self.admit, + inv([token("1")]), support=policy(feeds=[])) + + def test_all_for_primary_admits_any_primary_and_still_checks_feeds(self): + support = policy(primary="all") + self.assertEqual(self.admit(inv([token("1")]), primary="W", support=support), ["1"]) + self.assertEqual(self.admit(inv([token("1")]), primary="1D", support=support), ["1"]) + self.refused({"capability": FEED_CAPABILITY, "timeframe": "W"}, self.admit, + inv([token("W")]), primary="W", support=support) + + def test_all_for_request_feeds_does_not_read_the_inventory(self): + # under all the third fact does not enter the decision. The interface does + # not say what is returned then: None, never an empty set, for a set not resolved + absent = self.dir / "absent" + with watch() as (calls, opened): + listed = inventory.admit_selected_request("15", absent, absent, {}, + policy(feeds="all")) + both = inventory.admit_selected_request("W", absent, absent, {}, + policy(primary="all", feeds="all")) + self.assertEqual((listed, both, calls, opened), (None, None, [], [])) + with watch() as (calls, opened): + self.refused({"capability": CHART_CAPABILITY, "timeframe": "W"}, + inventory.admit_selected_request, "W", absent, absent, {}, + policy(feeds="all")) + self.assertEqual((calls, opened), ([], [])) + + def test_a_primary_that_is_not_a_timeframe_token_is_a_caller_error(self): + # the interface is silent: the request was validated before this call, so this is not + # a refusal of the request + self.write(inv([])) + for primary in (None, 15, "", " 15", "4h", "d", "0100000", "100000 ", "10000D\n"): + with self.subTest(primary=primary): + with self.assertRaises(ValueError): + self.admit(primary=primary) + with self.assertRaises(ValueError): + self.resolve(primary=primary) + + def test_the_arguments_are_not_modified(self): + inputs, support = {"tf": "1D"}, policy() + before = (json.dumps(inputs), json.dumps(support)) + self.admit(inv([input_ref("tf", "1")]), inputs=inputs, support=support) + self.assertEqual((json.dumps(inputs), json.dumps(support)), before) + + +class NoWorkTests(Case): + """Every pre-feed negative of the interface's first proof, watched: no library load, feed + read, child process or import, and no file opened but the inventory and the artifact. The + wrapper below only stands in for the later run_json placement (admit first; the work of the + run begins only when admit returns): it is not that integration and proves nothing about it.""" + + COUNTERS = {"feed_opens": 0, "library_loads": 0, "constructors": 0, "trials": 0} + + def run_wrapper(self, content, inputs=None): + self.write(content) + counters = dict.fromkeys(self.COUNTERS, 0) + outcome = None + with watch() as (calls, opened): + try: + outcome = inventory.admit_selected_request( + "15", self.inventory_path, self.artifact_path, + {} if inputs is None else inputs, policy()) + except Error as error: + outcome = error + else: + for name in counters: + counters[name] += 1 + return outcome, counters, calls, opened + + def test_every_pre_feed_negative_does_no_work(self): + allowed = {(str(self.inventory_path), "rb"), (str(self.artifact_path), "rb")} + negatives = { + "known weekly exclusion": (inv([token("W")]), None), + "unknown entry": (inv([unknown()]), None), + "artifact mismatch": (inv([token("1")], artifact_sha=OTHER_SHA), None), + "unbound": (inv([token("1")], artifact_sha=None), None), + "chart binding mismatch": (inv([token("1")], binding="5"), None), + "malformed inventory": (b"not json", None), + "input resolves to weekly": (inv([input_ref("tf", "1")]), {"tf": "W"}), + "input is not a string": (inv([input_ref("tf", "1")]), {"tf": 15}), + } + for name, (content, inputs) in negatives.items(): + with self.subTest(name=name): + outcome, counters, calls, opened = self.run_wrapper(content, inputs) + self.assertIsInstance(outcome, Error) + self.assertEqual(counters, self.COUNTERS) + self.assertEqual(calls, []) + self.assertLessEqual(set(opened), allowed) + + def test_a_missing_inventory_does_no_work(self): + absent = self.dir / "absent" + with watch() as (calls, opened): + with self.assertRaises(Error): + inventory.admit_selected_request("15", absent, absent, {}, policy()) + self.assertEqual(calls, []) + self.assertLessEqual(set(opened), {(str(absent), "rb")}) + + def test_a_positive_opens_only_the_two_files_and_starts_nothing(self): + outcome, counters, calls, opened = self.run_wrapper(inv([token("1"), token("D")])) + self.assertEqual(outcome, ["1", "D"]) + self.assertEqual(calls, []) + self.assertEqual(sorted(opened), sorted([(str(self.inventory_path), "rb"), + (str(self.artifact_path), "rb")])) + # the wrapper's own counters moved: they are live, and only admit's return releases work + self.assertEqual(counters, {name: 1 for name in self.COUNTERS}) + + def test_the_sentinels_fire_when_work_is_attempted(self): + # Each attempt runs inside watch(), where os.system and subprocess.Popen are the poison + # sentinels: they record and raise before any shell or process could start. + attempts = (lambda: ctypes.CDLL("x"), lambda: ctypes.cdll.LoadLibrary("x"), + lambda: csv.reader([]), lambda: subprocess.Popen(["x"]), + lambda: os.system("x"), lambda: importlib.import_module("x")) + with watch() as (calls, opened): + for attempt in attempts: + with self.assertRaises(AssertionError): + attempt() + open(self.artifact_path, "rb").close() + self.assertEqual(calls, ["library:CDLL", "library:LoadLibrary", "feed:reader", + "trial:Popen", "trial:system", "constructor:import_module"]) + self.assertEqual(opened, [(str(self.artifact_path), "rb")]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_execution_observer.py b/tests/test_run_execution_observer.py new file mode 100644 index 000000000..38f42bd14 --- /dev/null +++ b/tests/test_run_execution_observer.py @@ -0,0 +1,1147 @@ +"""Tests of docker/run_execution_observer.py, the Python half of the run-only execution +observer bridge, and of the two receipt properties added to docker/run_phase_transport.py. + +The native side is a stub. FakeNative stands in for the three exports of the strategy +library as Python functions and plays the engine's part around the callback: it copies the +descriptor's values, borrows the callback by its raw address only, presets a boundary and a +receipt the way the engine does, and calls the callback through the C function pointer. +What is real here is the ctypes layer (the structures, the callback prototype and +the call through libffi into the thunk), the RunPhaseWriter, and the AF_UNIX socketpair it +writes to. os.write is wrapped by a recording pass-through where a test counts writes, and +replaced by a fake only where a short write has to be made deterministic; the test that shows +the record is in the socket before the callback returns leaves os.write alone. +""" +import ast +import contextlib +import ctypes +import faulthandler +import gc +import importlib.util +import inspect +import json +import os +import socket +import sys +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +OBSERVER_PATH = ROOT / "docker" / "run_execution_observer.py" +TRANSPORT_PATH = ROOT / "docker" / "run_phase_transport.py" +CATALOG_PATH = ROOT / "docker" / "run_failure_codes.json" +REAL_WRITE = os.write + + +def load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +observer = load_module("run_execution_observer", OBSERVER_PATH) +transport = load_module("run_phase_transport", TRANSPORT_PATH) +RunPhaseWriter = transport.RunPhaseWriter +PhaseTransportError = transport.PhaseTransportError +ObserverBindingError = observer.ObserverBindingError +ExecutionObserverBinding = observer.ExecutionObserverBinding + + +def setUpModule(): + # A writer that blocked would hang the whole suite: dump the stacks and exit instead. + faulthandler.dump_traceback_later(300, exit=True) + + +def tearDownModule(): + faulthandler.cancel_dump_traceback_later() + + +# What the frozen C declarations say, written out here and not read from the module under test. +U32, U64, VOIDP = ctypes.c_uint32, ctypes.c_uint64, ctypes.c_void_p +POINTER_SIZE = ctypes.sizeof(ctypes.c_void_p) +BOUNDARY_LAYOUT = [("struct_size", U32, 0), ("version", U32, 4), + ("run_generation", U64, 8), ("attempt_serial", U64, 16)] +RECEIPT_LAYOUT = [("struct_size", U32, 0), ("version", U32, 4), ("run_generation", U64, 8), + ("frame_bytes", U32, 16), ("handed_bytes", U32, 20), + ("export_requested", U32, 24), ("reserved", U32, 28)] +OBSERVER_LAYOUT = [("struct_size", U32, 0), ("version", U32, 4), ("context", VOIDP, 8), + ("before_results", observer.BeforeResultsFn, 8 + POINTER_SIZE)] +OBSERVATION_LAYOUT = [("struct_size", U32, 0), ("version", U32, 4), + ("run_generation", U64, 8), ("attempt_serial", U64, 16), + ("attempt_generation", U64, 24), ("phase", U32, 32), + ("fault_stage", U32, 36), ("attempt_outcome", U32, 40), + ("boundary_delivered", U32, 44)] +EXPORTS = ("pf_execution_observer_version", "strategy_set_execution_observer_v1", + "strategy_execution_observation_v1") + +UINT32_MAX = 0xFFFFFFFF +GENERATION = 7 +SERIAL = 3 +STATE = 4242 # the pf_strategy_t the binding borrows +# The receipt as the engine presets it before the callback: +# size 32, version 1, the generation, frame_bytes UINT32_MAX and every other field 0. +SENTINEL = (32, 1, GENERATION, UINT32_MAX, 0, 0, 0) +PHASES = ("preflight", "execution", "result_assembly", "results_digest", "serialization", + "completed") + + +def kernel_boundary(generation=GENERATION, serial=SERIAL, size=24, version=1): + """The boundary as the engine presets it; the size is the C literal, not read back from + the mirror under test.""" + return observer.BoundaryC(size, version, generation, serial) + + +def kernel_receipt(generation=GENERATION, size=32, version=1): + return observer.ReceiptC(size, version, generation, UINT32_MAX, 0, 0, 0) + + +def receipt_fields(receipt): + return (receipt.struct_size, receipt.version, receipt.run_generation, receipt.frame_bytes, + receipt.handed_bytes, receipt.export_requested, receipt.reserved) + + +def boundary_fields(boundary): + return (boundary.struct_size, boundary.version, boundary.run_generation, + boundary.attempt_serial) + + +def by_reference(value): + """A struct passed to C by pointer; None is the NULL pointer.""" + return None if value is None else ctypes.byref(value) + + +def descriptor_count(): + """How many descriptors this process holds; None where /dev/fd cannot be listed.""" + gc.collect() + try: + return len(os.listdir("/dev/fd")) + except OSError: + return None + + +def prefix_write(cut): + """A fake os.write: hands the first cut(len(data)) bytes to the real descriptor and + reports exactly that count (0 hands nothing over and reports 0).""" + def write(fd, data): + count = cut(len(data)) + return REAL_WRITE(fd, data[:count]) if count else 0 + return write + + +class WriteSpy: + """Stands in for os.write. Calls on `fd` (on every descriptor when fd is None) are + recorded; `fake`, when given, decides what such a call does and returns; every other + call passes through.""" + + def __init__(self, fd, fake=None): + self.fd = fd + self.fake = fake + self.calls = [] + + def __call__(self, fd, data): + if self.fd is not None and fd != self.fd: + return REAL_WRITE(fd, data) + self.calls.append(bytes(data)) + if self.fake is not None: + return self.fake(fd, data) + return REAL_WRITE(fd, data) + + +@contextlib.contextmanager +def spying_on_os_write(fd=None, fake=None): + spy = WriteSpy(fd, fake) + with mock.patch.object(os, "write", spy): + yield spy + + +@contextlib.contextmanager +def unraisable_spy(): + """Records what sys.unraisablehook is called with: how ctypes reports an exception that + escaped a callback. A contained failure leaves it empty.""" + seen = [] + previous = sys.unraisablehook + sys.unraisablehook = seen.append + try: + yield seen + finally: + sys.unraisablehook = previous + + +class ScriptedClock: + """A clock that returns the given readings, one per call, and counts its calls.""" + + def __init__(self, *readings): + self.readings = list(readings) + self.reads = 0 + + def __call__(self): + reading = self.readings[self.reads] + self.reads += 1 + return reading + + +def drain(sock): + """Everything a nonblocking socket holds right now.""" + chunks = [] + while True: + try: + chunk = sock.recv(65536) + except BlockingIOError: + break + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks) + + +class Channel: + """A real AF_UNIX stream socketpair. `fd` is the writer's end, the descriptor the parent + would hand down; read() drains the far end as the parent would.""" + + def __init__(self, test): + self.near, self.far = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + test.addCleanup(self.near.close) + test.addCleanup(self.far.close) + self.far.setblocking(False) + self.fd = self.near.fileno() + test.assertGreaterEqual(self.fd, 3) + + def read(self): + return drain(self.far) + + def fill(self): + """Send filler until the near end takes no more, so that any further write finds + the socket full. Returns how many filler bytes the socket holds.""" + sent = 0 + for size in (65536, 4096, 256, 1): + while True: + try: + sent += self.near.send(b"x" * size, socket.MSG_DONTWAIT) + except BlockingIOError: + break + return sent + + +class Export: + """One export of the strategy library as ctypes shows it to the binding: a callable + that takes argtypes and restype. `at_call` is what both were when it was last called.""" + + def __init__(self, name, body): + self.__name__ = name + self.body = body + self.argtypes = None + self.restype = None + self.at_call = None + + def __call__(self, *args): + argtypes = None if self.argtypes is None else list(self.argtypes) + self.at_call = (argtypes, self.restype) + return self.body(*args) + + +class Registration: + """What the engine keeps after it accepted a descriptor: the values it copied and the raw + address of the callback, which it borrows. Nothing here references the Python callback, + so a call through it works only while the binding keeps the callback alive.""" + + def __init__(self, struct_size, version, context, address): + self.struct_size = struct_size + self.version = version + self.context = context + self.address = address + + def call(self, boundary, receipt): + """The engine's call, through the C function pointer. The structs are passed by + pointer; None is NULL.""" + function = observer.BeforeResultsFn(self.address) + return function(self.context, by_reference(boundary), by_reference(receipt)) + + +class FakeNative: + """The three exports of the strategy library as Python functions, and the engine's + memory of a registration. A UNIT control only: no native code runs.""" + + def __init__(self, version=1, register_status=0, remove_status=0, missing=()): + self.version = version + self.register_status = register_status + self.remove_status = remove_status + self.registered = None + self.set_calls = [] # (handle, removal) for every call of the setter + bodies = { + "pf_execution_observer_version": self._probe, + "strategy_set_execution_observer_v1": self._set_observer, + "strategy_execution_observation_v1": self._observation, + } + for name, body in bodies.items(): + if name not in missing: + setattr(self, name, Export(name, body)) + + def _probe(self): + return self.version + + def _set_observer(self, handle, pointer): + removal = pointer is None + self.set_calls.append((handle, removal)) + if removal: + if self.remove_status == 0: + self.registered = None + return self.remove_status + if self.register_status == 0: + descriptor = pointer.contents + self.registered = Registration( + descriptor.struct_size, descriptor.version, descriptor.context, + ctypes.cast(descriptor.before_results, ctypes.c_void_p).value) + return self.register_status + + def _observation(self, handle, pointer): + return -1 + + +class DrainingWriter(RunPhaseWriter): + """Drains the far end the instant advance("result_assembly") returns, which is inside the + observer callback and before it fills the receipt, and notes the receipt as it is then.""" + + def __init__(self, fd, far, holder): + super().__init__(fd) + self.far = far + self.holder = holder + self.seen = None + + def advance(self, phase): + super().advance(phase) + if phase == "result_assembly": + self.seen = (drain(self.far), receipt_fields(self.holder[0])) + + +class StubWriter: + """A writer double for the receipt contract alone, since the real writer cannot report + these values: advance succeeds and writes nothing, the reported handoff is the test's.""" + + def __init__(self, export_requested, handed): + self.phase = "execution" + self.export_requested = export_requested + self.last_handed_frame_bytes = handed + self.advanced = [] + + def advance(self, phase): + self.advanced.append(phase) + + +class ExplodingWriter: + """A writer double whose advance raises whatever it is given.""" + + phase = "execution" + export_requested = True + last_handed_frame_bytes = 0 + + def __init__(self, error): + self.error = error + self.calls = 0 + + def advance(self, phase): + self.calls += 1 + raise self.error + + +class Boom(BaseException): + """An exception that is not an Exception.""" + + +class BridgeCase(unittest.TestCase): + def channel(self): + return Channel(self) + + def writer_in_execution(self, fd=None, **kwargs): + """A writer as the caller leaves it before native work starts: execution entered.""" + writer = RunPhaseWriter(fd, **kwargs) + writer.advance("preflight") + writer.advance("execution") + return writer + + def attached(self, writer, native=None): + native = FakeNative() if native is None else native + binding = ExecutionObserverBinding(native, STATE, writer) + binding.attach() + return binding, native + + def records(self, data): + self.assertTrue(data == b"" or data.endswith(b"\n"), "the stream ends inside a record") + return [json.loads(line.decode("utf-8")) for line in data.split(b"\n") if line] + + def assert_refused(self, binding, status, receipt, before): + """The callback returned -1, kept an ObserverBindingError as the failure and left the + receipt as the engine preset it.""" + self.assertEqual(status, -1) + failure = binding.failure + self.assertIsInstance(failure, ObserverBindingError) + self.assertEqual(failure.code, "harness_internal_error") + self.assertEqual(failure.code_args, {}) + if receipt is not None: + self.assertEqual(receipt_fields(receipt), before) + with self.assertRaises(ObserverBindingError) as caught: + binding.raise_if_failed() + self.assertIs(caught.exception, failure) + + +class LayoutTests(unittest.TestCase): + def check_layout(self, structure, layout, size): + self.assertEqual(list(structure._fields_), + [(name, ctype) for name, ctype, _ in layout]) + for name, ctype, offset in layout: + member = getattr(structure, name) + self.assertEqual(member.offset, offset, name) + self.assertEqual(member.size, ctypes.sizeof(ctype), name) + self.assertEqual(ctypes.sizeof(structure), size) + self.assertFalse(hasattr(structure, "_pack_"), "natural alignment, no packing") + + def test_boundary_matches_pf_execution_boundary_v1(self): + self.check_layout(observer.BoundaryC, BOUNDARY_LAYOUT, 24) + + def test_receipt_matches_pf_boundary_receipt_v1(self): + self.check_layout(observer.ReceiptC, RECEIPT_LAYOUT, 32) + + def test_observer_matches_pf_execution_observer_v1(self): + self.check_layout(observer.ObserverC, OBSERVER_LAYOUT, 8 + 2 * POINTER_SIZE) + + def test_observation_matches_pf_execution_observation_v1(self): + self.check_layout(observer.ObservationC, OBSERVATION_LAYOUT, 48) + + def test_lp64_sizes_are_the_frozen_ones(self): + if POINTER_SIZE == 8: + self.assertEqual([ctypes.sizeof(structure) for structure in ( + observer.BoundaryC, observer.ReceiptC, observer.ObserverC, + observer.ObservationC)], [24, 32, 24, 48]) + + def test_callback_prototype_is_pf_before_results_fn_v1(self): + prototype = observer.BeforeResultsFn + self.assertIs(prototype._restype_, ctypes.c_int) + self.assertEqual(prototype._argtypes_, ( + ctypes.c_void_p, ctypes.POINTER(observer.BoundaryC), + ctypes.POINTER(observer.ReceiptC))) + self.assertIs(prototype, ctypes.CFUNCTYPE( + ctypes.c_int, ctypes.c_void_p, ctypes.POINTER(observer.BoundaryC), + ctypes.POINTER(observer.ReceiptC))) + + def test_version_and_limits_are_the_frozen_numbers(self): + self.assertEqual(observer.OBSERVER_VERSION, 1) + self.assertEqual(observer.MAX_FRAME_BYTES, 1024) + self.assertEqual(observer.MAX_FRAME_BYTES, transport.MAX_RECORD_BYTES) + self.assertEqual(observer.CAPABILITY, "execution_observer_v1") + + +class ErrorTests(unittest.TestCase): + def test_the_error_carries_code_message_and_a_code_args_dictionary(self): + error = ObserverBindingError("window_mode_unsupported", "text", + {"capability": "execution_observer_v1"}) + self.assertEqual(error.code, "window_mode_unsupported") + self.assertEqual(error.message, "text") + self.assertEqual(str(error), "text") + self.assertEqual(error.code_args, {"capability": "execution_observer_v1"}) + self.assertEqual(error.args, ("text",), "BaseException.args stays the exception's own") + plain = ObserverBindingError("out_of_memory", "text") + self.assertEqual(plain.code_args, {}) + + def test_the_arguments_are_copied(self): + source = {"capability": "execution_observer_v1"} + error = ObserverBindingError("window_mode_unsupported", "text", source) + source["other"] = 1 + self.assertEqual(error.code_args, {"capability": "execution_observer_v1"}) + + def test_it_is_not_an_oserror_and_its_codes_exist_in_the_catalogue(self): + self.assertFalse(issubclass(ObserverBindingError, OSError)) + catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8")) + # window_mode_unsupported is not in this tree's catalogue; no + # code is introduced here. + for code in ("harness_internal_error", "out_of_memory"): + self.assertIn(code, catalog["codes"]) + + +class AttachTests(BridgeCase): + def test_a_missing_export_is_unsupported_and_registers_nothing(self): + for name in EXPORTS: + with self.subTest(missing=name): + native = FakeNative(missing=(name,)) + binding = ExecutionObserverBinding(native, STATE, RunPhaseWriter(None)) + with self.assertRaises(ObserverBindingError) as caught: + binding.attach() + self.assertEqual(caught.exception.code, "window_mode_unsupported") + self.assertEqual(caught.exception.code_args, + {"capability": "execution_observer_v1"}) + self.assertEqual(native.set_calls, []) + self.assertIsNone(native.registered) + binding.detach() + self.assertEqual(native.set_calls, [], "nothing registered: no native call") + + def test_a_version_other_than_one_is_unsupported_and_registers_nothing(self): + for version in (0, 2, 7, UINT32_MAX): + with self.subTest(version=version): + native = FakeNative(version=version) + binding = ExecutionObserverBinding(native, STATE, RunPhaseWriter(None)) + with self.assertRaises(ObserverBindingError) as caught: + binding.attach() + self.assertEqual(caught.exception.code, "window_mode_unsupported") + self.assertEqual(caught.exception.code_args, + {"capability": "execution_observer_v1"}) + self.assertIn(str(version), str(caught.exception)) + self.assertEqual(native.set_calls, []) + self.assertIsNone(native.registered) + + def test_registration_statuses_map_to_their_codes(self): + expected = {-4: "out_of_memory", -1: "harness_internal_error", + -2: "harness_internal_error", -3: "harness_internal_error", + 1: "harness_internal_error", 7: "harness_internal_error", + -5: "harness_internal_error"} + for status, code in expected.items(): + with self.subTest(status=status): + native = FakeNative(register_status=status) + binding = ExecutionObserverBinding(native, STATE, RunPhaseWriter(None)) + with self.assertRaises(ObserverBindingError) as caught: + binding.attach() + self.assertEqual(caught.exception.code, code) + self.assertEqual(caught.exception.code_args, {}) + self.assertIn(str(status), str(caught.exception)) + self.assertEqual(native.set_calls, [(STATE, False)]) + self.assertIsNone(native.registered) + binding.detach() + self.assertEqual(native.set_calls, [(STATE, False)], + "refused: nothing to unregister, no native call") + + def test_it_binds_argtypes_and_restype_before_the_first_call(self): + binding, native = self.attached(RunPhaseWriter(None)) + probe = native.pf_execution_observer_version + setter = native.strategy_set_execution_observer_v1 + getter = native.strategy_execution_observation_v1 + self.assertEqual(probe.at_call, ([], ctypes.c_uint32)) + self.assertEqual(setter.at_call, ([ctypes.c_void_p, + ctypes.POINTER(observer.ObserverC)], ctypes.c_int)) + self.assertEqual(list(getter.argtypes), + [ctypes.c_void_p, ctypes.POINTER(observer.ObservationC)]) + self.assertIs(getter.restype, ctypes.c_int) + self.assertIsNone(getter.at_call, "the observation getter is bound, not called") + + def test_it_registers_one_descriptor_for_the_handle(self): + binding, native = self.attached(RunPhaseWriter(None)) + registration = native.registered + self.assertEqual(native.set_calls, [(STATE, False)]) + self.assertEqual(registration.struct_size, 8 + 2 * POINTER_SIZE) + self.assertEqual(registration.version, 1) + self.assertIsNone(registration.context, "context is NULL: the closure captures self") + self.assertTrue(registration.address) + + def test_a_repeated_attach_is_refused_locally(self): + binding, native = self.attached(RunPhaseWriter(None)) + with self.assertRaises(ObserverBindingError) as caught: + binding.attach() + self.assertEqual(caught.exception.code, "harness_internal_error") + self.assertEqual(caught.exception.code_args, {}) + self.assertIn("already", str(caught.exception)) + self.assertEqual(native.set_calls, [(STATE, False)], "the library is not touched again") + self.assertIsNotNone(native.registered) + + def test_a_binding_is_single_use_even_after_a_refusal_or_a_detach(self): + refused = FakeNative(register_status=-2) + binding = ExecutionObserverBinding(refused, STATE, RunPhaseWriter(None)) + with self.assertRaises(ObserverBindingError): + binding.attach() + with self.assertRaises(ObserverBindingError) as caught: + binding.attach() + self.assertIn("already", str(caught.exception)) + self.assertEqual(refused.set_calls, [(STATE, False)]) + done, native = self.attached(RunPhaseWriter(None)) + done.detach() + with self.assertRaises(ObserverBindingError): + done.attach() + self.assertEqual(native.set_calls, [(STATE, False), (STATE, True)]) + + def test_attach_never_advances_the_writer(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + writer.advance("preflight") + channel.read() + with spying_on_os_write() as spy: + self.attached(writer) + self.assertEqual(spy.calls, []) + self.assertEqual((writer.phase, writer.sequence), ("preflight", 1)) + self.assertEqual(channel.read(), b"") + + +class DeliveryTests(BridgeCase): + def test_the_whole_result_assembly_record_is_in_the_socket_before_the_callback_returns(self): + channel = self.channel() + holder = [] + writer = DrainingWriter(channel.fd, channel.far, holder) + writer.advance("preflight") + writer.advance("execution") + binding, native = self.attached(writer) + boundary, receipt = kernel_boundary(), kernel_receipt() + holder.append(receipt) + with unraisable_spy() as unraisable: + status = native.registered.call(boundary, receipt) # the real os.write, untouched + self.assertEqual(status, 0) + self.assertEqual(unraisable, []) + self.assertIsNone(binding.failure) + binding.raise_if_failed() + # What the far end held at the moment advance("result_assembly") returned, which was + # inside the callback and before it filled the receipt: nothing came after it. + data, receipt_then = writer.seen + self.assertEqual(channel.read(), b"", "nothing reached the socket after advance returned") + self.assertEqual(receipt_then, SENTINEL, "the receipt is filled only after the handoff") + records = self.records(data) + self.assertEqual([record["phase"] for record in records], + ["preflight", "execution", "result_assembly"]) + lines = data.split(b"\n") + self.assertEqual(lines[-1], b"", "the stream ends with a whole record") + frame = lines[-2] + b"\n" # the result_assembly record as it reached the socket + self.assertTrue(1 <= len(frame) <= 1024) + self.assertEqual(json.loads(frame.decode("utf-8")), records[-1]) + self.assertEqual(receipt_fields(receipt), + (32, 1, GENERATION, len(frame), len(frame), 1, 0)) + self.assertEqual((writer.phase, writer.sequence), ("result_assembly", 3)) + + def test_a_writer_without_a_descriptor_hands_over_nothing_and_the_receipt_says_zero(self): + writer = self.writer_in_execution(None) + binding, native = self.attached(writer) + receipt = kernel_receipt() + with unraisable_spy() as unraisable: + with spying_on_os_write() as spy: + status = native.registered.call(kernel_boundary(), receipt) + self.assertEqual(status, 0) + self.assertEqual(unraisable, []) + self.assertEqual(spy.calls, [], "no descriptor: no os.write at all") + self.assertEqual(receipt_fields(receipt), (32, 1, GENERATION, 0, 0, 0, 0)) + self.assertEqual((writer.phase, writer.sequence), ("result_assembly", 3)) + self.assertIsNone(binding.failure) + binding.raise_if_failed() + + def test_boundary_identity_and_receipt_header_are_never_written(self): + for generation, serial in ((1, 1), (GENERATION, SERIAL), (2 ** 64 - 1, 2 ** 63)): + with self.subTest(generation=generation, serial=serial): + writer = self.writer_in_execution(None) + binding, native = self.attached(writer) + boundary = kernel_boundary(generation=generation, serial=serial) + receipt = kernel_receipt(generation=generation) + status = native.registered.call(boundary, receipt) + self.assertEqual(status, 0) + self.assertEqual(boundary_fields(boundary), (24, 1, generation, serial)) + self.assertEqual(receipt_fields(receipt), (32, 1, generation, 0, 0, 0, 0)) + + def test_the_callback_reads_no_clock_and_encodes_and_writes_one_record(self): + channel = self.channel() + clock = ScriptedClock(0.0, 0.5, 1.25) + writer = self.writer_in_execution(channel.fd, clock=clock) + binding, native = self.attached(writer) + self.assertEqual(clock.reads, 2, "attach reads no clock") + with mock.patch.object(transport, "encode_record", + wraps=transport.encode_record) as encode: + with spying_on_os_write(channel.fd) as spy: + status = native.registered.call(kernel_boundary(), kernel_receipt()) + self.assertEqual(status, 0) + self.assertEqual(encode.call_count, 1, "one record encoded") + self.assertEqual(len(spy.calls), 1, "one record written") + self.assertEqual(clock.reads, 3, "the writer's one reading, nothing else") + records = self.records(channel.read()) + self.assertEqual([record["sequence"] for record in records], [1, 2, 3]) + self.assertEqual(records[-1]["elapsed_ms"], 1250) + + def test_a_second_callback_is_a_failure_and_writes_nothing(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + binding, native = self.attached(writer) + first = kernel_receipt() + self.assertEqual(native.registered.call(kernel_boundary(), first), 0) + filled = receipt_fields(first) + self.assertIsNone(binding.failure) + second = kernel_receipt() + with unraisable_spy() as unraisable: + with spying_on_os_write() as spy: + status = native.registered.call(kernel_boundary(), second) + self.assert_refused(binding, status, second, SENTINEL) + self.assertIn("again", str(binding.failure)) + self.assertEqual(unraisable, []) + self.assertEqual(spy.calls, [], "the record is never written twice") + self.assertEqual(receipt_fields(first), filled) + self.assertEqual(len(self.records(channel.read())), 3) + self.assertEqual((writer.phase, writer.sequence), ("result_assembly", 3)) + + +class RefusalTests(BridgeCase): + CASES = [ + ("null boundary", lambda: None, kernel_receipt), + ("null receipt", kernel_boundary, lambda: None), + ("both null", lambda: None, lambda: None), + ("boundary size 0", lambda: kernel_boundary(size=0), kernel_receipt), + ("boundary size 23", lambda: kernel_boundary(size=23), kernel_receipt), + ("boundary size 25", lambda: kernel_boundary(size=25), kernel_receipt), + ("boundary size 32", lambda: kernel_boundary(size=32), kernel_receipt), + ("boundary size UINT32_MAX", lambda: kernel_boundary(size=UINT32_MAX), kernel_receipt), + ("boundary version 0", lambda: kernel_boundary(version=0), kernel_receipt), + ("boundary version 2", lambda: kernel_boundary(version=2), kernel_receipt), + ("generation 0", lambda: kernel_boundary(generation=0), + lambda: kernel_receipt(generation=0)), + ("attempt serial 0", lambda: kernel_boundary(serial=0), kernel_receipt), + ("receipt size 0", kernel_boundary, lambda: kernel_receipt(size=0)), + ("receipt size 31", kernel_boundary, lambda: kernel_receipt(size=31)), + ("receipt size 33", kernel_boundary, lambda: kernel_receipt(size=33)), + ("receipt size 24", kernel_boundary, lambda: kernel_receipt(size=24)), + ("receipt version 0", kernel_boundary, lambda: kernel_receipt(version=0)), + ("receipt version 2", kernel_boundary, lambda: kernel_receipt(version=2)), + ("receipt generation 0", kernel_boundary, lambda: kernel_receipt(generation=0)), + ("receipt generation differs", kernel_boundary, + lambda: kernel_receipt(generation=GENERATION + 1)), + ] + + def test_malformed_arguments_are_refused_without_a_handoff_or_a_receipt(self): + for label, make_boundary, make_receipt in self.CASES: + with self.subTest(case=label): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + binding, native = self.attached(writer) + boundary, receipt = make_boundary(), make_receipt() + before = None if receipt is None else receipt_fields(receipt) + with unraisable_spy() as unraisable: + with spying_on_os_write() as spy: + status = native.registered.call(boundary, receipt) + self.assert_refused(binding, status, receipt, before) + self.assertEqual(unraisable, []) + self.assertEqual(spy.calls, [], "nothing is handed over for a malformed call") + self.assertEqual((writer.phase, writer.sequence), ("execution", 2)) + self.assertIsNone(writer.failure) + + def test_the_callback_needs_the_writer_in_execution(self): + prefixes = { + "no phase yet": (), + "preflight": ("preflight",), + "result_assembly": ("preflight", "execution", "result_assembly"), + "serialization": ("preflight", "execution", "result_assembly", "serialization"), + } + for label, prefix in prefixes.items(): + with self.subTest(phase=label): + writer = RunPhaseWriter(None) + for phase in prefix: + writer.advance(phase) + binding, native = self.attached(writer) + receipt = kernel_receipt() + status = native.registered.call(kernel_boundary(), receipt) + self.assert_refused(binding, status, receipt, SENTINEL) + self.assertEqual(writer.sequence, len(prefix), "the writer was not advanced") + self.assertEqual(writer.phase, prefix[-1] if prefix else None) + self.assertIsNone(writer.failure) + + def test_a_handoff_the_receipt_contract_cannot_hold_is_refused(self): + cases = [(True, 0), (True, -1), (True, 1025), (True, 2 ** 32), (False, 1), + (False, 1024), (False, -1), (True, 1.0), (True, True), (True, "5"), + (True, None), (1, 5), (0, 0), (None, 0), ("yes", 5)] + for exported, handed in cases: + with self.subTest(exported=exported, handed=handed): + writer = StubWriter(exported, handed) + binding, native = self.attached(writer) + receipt = kernel_receipt() + status = native.registered.call(kernel_boundary(), receipt) + self.assert_refused(binding, status, receipt, SENTINEL) + self.assertEqual(writer.advanced, ["result_assembly"], + "one advance, then the check") + + def test_the_handoffs_the_receipt_contract_holds_are_filled_as_reported(self): + cases = [(True, 1, (1, 1, 1, 0)), (True, 87, (87, 87, 1, 0)), + (True, 1024, (1024, 1024, 1, 0)), (False, 0, (0, 0, 0, 0))] + for exported, handed, filled in cases: + with self.subTest(exported=exported, handed=handed): + writer = StubWriter(exported, handed) + binding, native = self.attached(writer) + receipt = kernel_receipt() + status = native.registered.call(kernel_boundary(), receipt) + self.assertEqual(status, 0) + self.assertEqual(receipt_fields(receipt), (32, 1, GENERATION) + filled) + self.assertEqual(writer.advanced, ["result_assembly"]) + self.assertIsNone(binding.failure) + + +class ContainmentTests(BridgeCase): + def test_a_full_socket_returns_minus_one_and_keeps_the_first_failure(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + binding, native = self.attached(writer) + channel.fill() + receipt = kernel_receipt() + with unraisable_spy() as unraisable: + with spying_on_os_write(channel.fd) as spy: + status = native.registered.call(kernel_boundary(), receipt) + self.assertEqual(status, -1) + self.assertEqual(unraisable, []) + self.assertEqual(len(spy.calls), 1, "one write attempt, no retry") + failure = binding.failure + self.assertIsInstance(failure, PhaseTransportError) + self.assertEqual(failure.reason, "would_block") + self.assertEqual(failure.phase, "result_assembly") + self.assertEqual(receipt_fields(receipt), SENTINEL, "a failed handoff supplies no receipt") + self.assertEqual(writer.failure, "would_block") + self.assertEqual((writer.phase, writer.sequence), ("execution", 2)) + with self.assertRaises(PhaseTransportError) as caught: + binding.raise_if_failed() + self.assertIs(caught.exception, failure, "the retained exception itself, still typed") + self.assertEqual(caught.exception.code, "harness_internal_error") + again = kernel_receipt() + self.assertEqual(native.registered.call(kernel_boundary(), again), -1) + self.assertIs(binding.failure, failure, "a later refusal does not replace the first") + self.assertEqual(receipt_fields(again), SENTINEL) + + def test_a_short_write_returns_minus_one_with_the_typed_cause(self): + cuts = {"nothing taken": lambda size: 0, "one byte": lambda size: 1, + "half": lambda size: size // 2, "all but the LF": lambda size: size - 1} + for name, cut in cuts.items(): + with self.subTest(cut=name): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + binding, native = self.attached(writer) + receipt = kernel_receipt() + with unraisable_spy() as unraisable: + with spying_on_os_write(channel.fd, fake=prefix_write(cut)) as spy: + status = native.registered.call(kernel_boundary(), receipt) + self.assertEqual(status, -1) + self.assertEqual(unraisable, []) + self.assertEqual(len(spy.calls), 1, "one write, the rest is not retried") + failure = binding.failure + self.assertIsInstance(failure, PhaseTransportError) + self.assertEqual(failure.reason, "short_write") + self.assertEqual(receipt_fields(receipt), SENTINEL) + self.assertEqual(writer.failure, "short_write") + with self.assertRaises(PhaseTransportError) as caught: + binding.raise_if_failed() + self.assertIs(caught.exception, failure) + + def test_a_disconnected_parent_returns_minus_one_with_the_typed_cause(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + binding, native = self.attached(writer) + channel.far.close() + receipt = kernel_receipt() + status = native.registered.call(kernel_boundary(), receipt) + self.assertEqual(status, -1) + self.assertEqual(binding.failure.reason, "disconnected") + self.assertEqual(receipt_fields(receipt), SENTINEL) + + def test_no_exception_escapes_ctypes_whatever_the_writer_raises(self): + errors = [RuntimeError("boom"), MemoryError(), Boom(), KeyboardInterrupt(), + SystemExit(3)] + for error in errors: + with self.subTest(error=type(error).__name__): + writer = ExplodingWriter(error) + binding, native = self.attached(writer) + receipt = kernel_receipt() + with unraisable_spy() as unraisable: + status = native.registered.call(kernel_boundary(), receipt) + self.assertEqual(status, -1) + self.assertEqual(unraisable, [], "ctypes was handed no exception") + self.assertEqual(writer.calls, 1) + self.assertIs(binding.failure, error) + self.assertEqual(receipt_fields(receipt), SENTINEL) + with self.assertRaises(type(error)) as caught: + binding.raise_if_failed() + self.assertIs(caught.exception, error) + self.assertEqual(native.registered.call(kernel_boundary(), kernel_receipt()), -1) + self.assertIs(binding.failure, error, "the first failure is the one kept") + self.assertEqual(writer.calls, 1, "a second call does not reach the writer") + + def test_a_failure_does_not_stop_detach(self): + writer = ExplodingWriter(RuntimeError("boom")) + binding, native = self.attached(writer) + self.assertEqual(native.registered.call(kernel_boundary(), kernel_receipt()), -1) + binding.detach() + self.assertIsNone(native.registered) + self.assertIsInstance(binding.failure, RuntimeError, "detach keeps the failure") + with self.assertRaises(RuntimeError): + binding.raise_if_failed() + + +class DetachTests(BridgeCase): + def test_detach_unregisters_with_a_null_descriptor_and_lets_go_only_then(self): + writer = self.writer_in_execution(None) + binding, native = self.attached(writer) + self.assertEqual(native.set_calls, [(STATE, False)]) + binding.detach() + self.assertEqual(native.set_calls, [(STATE, False), (STATE, True)]) + self.assertIsNone(native.registered) + self.assertIsNone(binding._callback, "released after a successful removal") + self.assertIsNone(binding._descriptor) + binding.detach() + self.assertEqual(len(native.set_calls), 2, "a detached binding calls native code no more") + + def test_detach_without_a_registration_does_nothing(self): + native = FakeNative() + binding = ExecutionObserverBinding(native, STATE, RunPhaseWriter(None)) + binding.detach() + self.assertEqual(native.set_calls, []) + + def test_a_failed_detach_keeps_the_callback_alive_and_can_be_retried(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + native = FakeNative(remove_status=-3) + binding, _ = self.attached(writer, native) + registration = native.registered + with self.assertRaises(ObserverBindingError) as caught: + binding.detach() + self.assertEqual(caught.exception.code, "harness_internal_error") + self.assertEqual(caught.exception.code_args, {}) + self.assertIn("-3", str(caught.exception)) + self.assertIsNotNone(binding._callback, "ownership is retained") + self.assertIsNotNone(binding._descriptor) + self.assertIsNotNone(native.registered, "the engine still holds the registration") + # The engine still holds the callback, so it must still be callable through the raw + # address: nothing but the binding keeps it alive. (A freed callback would crash here.) + gc.collect() + receipt = kernel_receipt() + self.assertEqual(registration.call(kernel_boundary(), receipt), 0) + self.assertEqual(receipt.export_requested, 1) + native.remove_status = 0 + binding.detach() + self.assertIsNone(native.registered) + self.assertEqual(native.set_calls, [(STATE, False), (STATE, True), (STATE, True)]) + self.assertIsNone(binding._callback) + + def test_removal_statuses_map_like_registration_statuses(self): + for status, code in ((-4, "out_of_memory"), (-3, "harness_internal_error"), + (-1, "harness_internal_error"), (9, "harness_internal_error")): + with self.subTest(status=status): + native = FakeNative(remove_status=status) + binding, _ = self.attached(RunPhaseWriter(None), native) + with self.assertRaises(ObserverBindingError) as caught: + binding.detach() + self.assertEqual(caught.exception.code, code) + self.assertEqual(caught.exception.code_args, {}) + + def test_the_binding_keeps_the_registered_callback_alive(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + binding, native = self.attached(writer) + registration = native.registered # the engine's view: values and a raw address only + gc.collect() + gc.collect() + receipt = kernel_receipt() + # Called through the borrowed address, as the engine does; this works only while the + # binding holds the callback (a freed one would crash here, not fail cleanly). + self.assertEqual(registration.call(kernel_boundary(), receipt), 0) + self.assertEqual(receipt.export_requested, 1) + self.assertGreaterEqual(receipt.frame_bytes, 1) + + def test_nothing_here_closes_or_duplicates_the_phase_descriptor(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + before = descriptor_count() + binding, native = self.attached(writer) + self.assertEqual(native.registered.call(kernel_boundary(), kernel_receipt()), 0) + binding.raise_if_failed() + binding.detach() + del binding, native + gc.collect() + os.fstat(channel.fd) + self.assertEqual(channel.near.fileno(), channel.fd) + channel.read() + self.assertEqual(channel.near.send(b"!"), 1, "still the live socket") + self.assertEqual(channel.far.recv(1), b"!") + if before is not None: + self.assertEqual(descriptor_count(), before) + + +class WriterReceiptTests(BridgeCase): + def test_before_any_advance(self): + channel = self.channel() + exporting, silent = RunPhaseWriter(channel.fd), RunPhaseWriter(None) + self.assertIs(exporting.export_requested, True) + self.assertIs(silent.export_requested, False) + self.assertEqual(exporting.last_handed_frame_bytes, 0) + self.assertEqual(silent.last_handed_frame_bytes, 0) + + def test_every_successful_advance_reports_the_one_record_it_handed_over(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + for phase in PHASES: + with self.subTest(phase=phase): + with spying_on_os_write(channel.fd) as spy: + writer.advance(phase) + data = channel.read() + self.assertEqual(len(spy.calls), 1) + self.assertEqual(spy.calls[0], data) + self.assertEqual(data.count(b"\n"), 1) + self.assertEqual(writer.last_handed_frame_bytes, len(data)) + self.assertTrue(1 <= writer.last_handed_frame_bytes <= 1024) + self.assertIs(writer.export_requested, True) + + def test_a_writer_without_a_descriptor_stores_zero_after_every_advance(self): + writer = RunPhaseWriter(None) + with spying_on_os_write() as spy: + for phase in PHASES: + writer.advance(phase) + self.assertEqual(writer.last_handed_frame_bytes, 0, phase) + self.assertIs(writer.export_requested, False) + self.assertEqual(spy.calls, []) + + def test_a_refused_advance_leaves_what_the_writer_reports(self): + channel = self.channel() + writer = self.writer_in_execution(channel.fd) + handed = writer.last_handed_frame_bytes + self.assertGreater(handed, 0) + channel.read() + channel.fill() + with self.assertRaises(PhaseTransportError): + writer.advance("result_assembly") + self.assertEqual(writer.last_handed_frame_bytes, handed, "nothing was handed over") + other = self.channel() + short = self.writer_in_execution(other.fd) + handed = short.last_handed_frame_bytes + with spying_on_os_write(other.fd, fake=prefix_write(lambda size: size // 2)): + with self.assertRaises(PhaseTransportError): + short.advance("result_assembly") + self.assertEqual(short.last_handed_frame_bytes, handed) + misordered = RunPhaseWriter(None) + with self.assertRaises(PhaseTransportError): + misordered.advance("execution") + self.assertEqual(misordered.last_handed_frame_bytes, 0) + + def test_the_properties_are_read_only(self): + for name in ("export_requested", "last_handed_frame_bytes"): + with self.subTest(name=name): + member = getattr(RunPhaseWriter, name) + self.assertIsInstance(member, property) + self.assertIsNone(member.fset) + self.assertIsNone(member.fdel) + with self.assertRaises(AttributeError): + setattr(RunPhaseWriter(None), name, 1) + + def test_one_encode_one_clock_reading_and_one_write_per_advance(self): + channel = self.channel() + clock = ScriptedClock(0.0, 1.0, 2.0, 3.0, 4.0, 5.0) + writer = RunPhaseWriter(channel.fd, clock=clock) + for count, phase in enumerate(PHASES, start=1): + with self.subTest(phase=phase): + with mock.patch.object(transport, "encode_record", + wraps=transport.encode_record) as encode: + with spying_on_os_write(channel.fd) as spy: + writer.advance(phase) + self.assertEqual(encode.call_count, 1) + self.assertEqual(len(spy.calls), 1) + self.assertEqual(clock.reads, count) + silent = RunPhaseWriter(None, clock=ScriptedClock(0.0, 1.0)) + with mock.patch.object(transport, "encode_record", + wraps=transport.encode_record) as encode: + silent.advance("preflight") + silent.advance("execution") + self.assertEqual(encode.call_count, 0, "nothing is encoded when nothing is exported") + + +def called_names(root): + names = [] + for node in ast.walk(root): + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Name): + names.append(node.func.id) + elif isinstance(node.func, ast.Attribute): + names.append(node.func.attr) + return names + + +def find_class(tree, name): + for node in tree.body: + if isinstance(node, ast.ClassDef) and node.name == name: + return node + raise AssertionError("class %s not found" % name) + + +def find_method(class_node, name): + for item in class_node.body: + if isinstance(item, ast.FunctionDef) and item.name == name: + return item + raise AssertionError("method %s not found" % name) + + +class SourceContractTests(unittest.TestCase): + FORBIDDEN_CALLS = {"print", "sleep", "close", "write", "dup", "dup2", "fromfd", "fdopen", + "makefile", "send", "sendall", "recv", "setblocking", "settimeout", + "set_blocking", "monotonic", "perf_counter", "time", "encode_record", + "_hand_off", "_read_clock"} + FORBIDDEN_IMPORTS = {"os", "time", "socket", "select", "selectors", "threading", "sys", + "logging", "subprocess", "signal", "fcntl", "atexit", "weakref", "gc"} + + @classmethod + def setUpClass(cls): + cls.tree = ast.parse(OBSERVER_PATH.read_text(encoding="utf-8")) + cls.transport_tree = ast.parse(TRANSPORT_PATH.read_text(encoding="utf-8")) + + def test_the_bridge_advances_the_writer_exactly_once_and_only_to_result_assembly(self): + advances = [node for node in ast.walk(self.tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr == "advance"] + self.assertEqual(len(advances), 1, "one call site, one boundary") + self.assertEqual([ast.literal_eval(argument) for argument in advances[0].args], + ["result_assembly"]) + self.assertEqual(advances[0].keywords, []) + + def test_the_bridge_writes_no_bytes_reads_no_clock_and_has_no_destructor(self): + self.assertEqual(set(called_names(self.tree)) & self.FORBIDDEN_CALLS, set()) + imported = set() + for node in ast.walk(self.tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add((node.module or "").split(".")[0]) + self.assertEqual(imported & self.FORBIDDEN_IMPORTS, set()) + functions = {node.name for node in ast.walk(self.tree) + if isinstance(node, ast.FunctionDef)} + self.assertNotIn("__del__", functions) + + def test_the_callback_catches_every_base_exception_and_answers_minus_one_or_zero(self): + callback = find_method(find_class(self.tree, "ExecutionObserverBinding"), + "_before_results") + self.assertEqual([type(node).__name__ for node in callback.body], + ["Expr", "Try", "Return"], "docstring, the guarded call, the answer") + guarded = callback.body[1] + self.assertEqual([handler.type.id for handler in guarded.handlers], ["BaseException"]) + returns = sorted((node for node in ast.walk(callback) if isinstance(node, ast.Return)), + key=lambda node: node.lineno) + self.assertEqual([ast.literal_eval(node.value) for node in returns], [-1, 0]) + + def test_the_writer_enters_a_phase_with_one_clock_reading_one_encode_and_one_handoff(self): + enter = find_method(find_class(self.transport_tree, "RunPhaseWriter"), "_enter") + names = called_names(enter) + self.assertEqual(names.count("_read_clock"), 1) + self.assertEqual(names.count("encode_record"), 1) + self.assertEqual(names.count("_hand_off"), 1) + self.assertEqual(names.count("_record"), 1) + self.assertEqual(called_names(self.transport_tree).count("encode_record"), 1, + "no other call site encodes a record") + + +class ApiContractTests(unittest.TestCase): + def test_the_binding_has_exactly_the_fixed_api(self): + public = {name for name in dir(ExecutionObserverBinding) if not name.startswith("_")} + self.assertEqual(public, {"attach", "detach", "failure", "raise_if_failed"}) + self.assertEqual(list(inspect.signature(ExecutionObserverBinding.__init__).parameters), + ["self", "library", "state", "writer"]) + for method in (ExecutionObserverBinding.attach, ExecutionObserverBinding.detach, + ExecutionObserverBinding.raise_if_failed): + self.assertEqual(list(inspect.signature(method).parameters), ["self"]) + self.assertIsInstance(ExecutionObserverBinding.failure, property) + self.assertIsNone(ExecutionObserverBinding.failure.fset) + + def test_the_module_exports_what_it_names(self): + self.assertEqual(set(observer.__all__), { + "BeforeResultsFn", "BoundaryC", "CAPABILITY", "ExecutionObserverBinding", + "MAX_FRAME_BYTES", "OBSERVER_VERSION", "ObservationC", "ObserverBindingError", + "ObserverC", "ReceiptC"}) + for name in observer.__all__: + self.assertTrue(hasattr(observer, name), name) + + def test_the_writer_gained_exactly_two_public_members(self): + # tests/test_run_phase_transport.py still pins the earlier six-name set and needs the + # two names added; that file is outside the scope of this test and is not edited here. + public = {name for name in dir(RunPhaseWriter) if not name.startswith("_")} + self.assertEqual(public, {"advance", "failure", "phase", "phase_timing", "sequence", + "unsent", "export_requested", "last_handed_frame_bytes"}) + self.assertEqual(list(inspect.signature(RunPhaseWriter.__init__).parameters), + ["self", "fd", "clock"]) + self.assertEqual(list(inspect.signature(RunPhaseWriter.advance).parameters), + ["self", "phase"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_failure_codes.cpp b/tests/test_run_failure_codes.cpp index f034635d1..a0f1e2867 100644 --- a/tests/test_run_failure_codes.cpp +++ b/tests/test_run_failure_codes.cpp @@ -1281,17 +1281,542 @@ void check_code_args(RunFailureCode code, const std::string& name, const Json* a expect(spec.name + " outside its list", with({arg, "not_a_listed_value"}), invariant, "{}"); expect(spec.name + " as an integer", with({arg, std::int64_t{1}}), invariant, "{}"); + expect(spec.name + " as a boolean", with({arg, true}), invariant, "{}"); + expect(spec.name + " as a null", with({arg, nullptr}), invariant, "{}"); } else if (spec.kind == "integer") { expect(spec.name + " as text", with({arg, "7"}), invariant, "{}"); expect(spec.name + " as a number", with({arg, 7.5}), invariant, "{}"); + expect(spec.name + " as a boolean", with({arg, true}), invariant, "{}"); + expect(spec.name + " as a null", with({arg, nullptr}), invariant, "{}"); } else if (spec.kind == "number") { expect(spec.name + " as text", with({arg, "1.5"}), invariant, "{}"); expect(spec.name + " not finite", with({arg, std::numeric_limits::quiet_NaN()}), invariant, "{}"); + expect(spec.name + " infinite", + with({arg, std::numeric_limits::infinity()}), invariant, "{}"); + expect(spec.name + " as a boolean", with({arg, true}), invariant, "{}"); + expect(spec.name + " as a null", with({arg, nullptr}), invariant, "{}"); + } else if (spec.kind == "string") { + expect(spec.name + " as an integer", with({arg, std::int64_t{1}}), invariant, "{}"); + expect(spec.name + " as a boolean", with({arg, true}), invariant, "{}"); + expect(spec.name + " as a null", with({arg, nullptr}), invariant, "{}"); + } else if (spec.kind == "nullable_string") { + expect(spec.name + " as an integer", with({arg, std::int64_t{1}}), invariant, "{}"); + expect(spec.name + " as a boolean", with({arg, true}), invariant, "{}"); + const RunFailureValue got = pineforge::make_run_failure(code, with({arg, nullptr})); + CHECK_EQ(at + spec.name + " as a null accepted", code_name(got.code), name); + } else if (spec.kind == "scalar") { + const auto accepts = [&](const std::string& what, RunFailureArg replacement) { + const RunFailureValue got = + pineforge::make_run_failure(code, with(std::move(replacement))); + CHECK_EQ(at + spec.name + " as " + what + " accepted", code_name(got.code), name); + }; + accepts("an integer", {arg, std::int64_t{-3}}); + accepts("true", {arg, true}); + accepts("false", {arg, false}); + accepts("a null", {arg, nullptr}); + accepts("a finite number", {arg, 2.5}); + expect(spec.name + " as NaN", with({arg, std::numeric_limits::quiet_NaN()}), + invariant, "{}"); + expect(spec.name + " as infinity", + with({arg, std::numeric_limits::infinity()}), invariant, "{}"); } else { expect(spec.name + " as an integer", with({arg, std::int64_t{1}}), invariant, "{}"); + expect(spec.name + " as a boolean", with({arg, true}), invariant, "{}"); + expect(spec.name + " as a null", with({arg, nullptr}), invariant, "{}"); + } + } +} + +// --------------------------------------------------------------------------- +// The selected-window vocabulary: the generic string, nullable_string and +// scalar kinds, a boolean and a null written as JSON true/false/null (never as +// strings), and the old kinds and codes keeping their bytes beside them. The +// values here are written out, not read from the catalog, so a catalog edit +// that changes a documented class or argument is caught. + +void wv_expect(const std::string& what, RunFailureCode code, const RunFailureArgs& args, + RunFailureCode want_code, const std::string& want_args) { + const RunFailureValue got = pineforge::make_run_failure(code, args); + CHECK_EQ(what + ": code", code_name(got.code), code_name(want_code)); + CHECK_EQ(what + ": args", args_json(got), want_args); +} + +void window_vocabulary_rows() { + using Code = RunFailureCode; + const Code invariant = Code::engine_invariant; + const double nan = std::numeric_limits::quiet_NaN(); + const double inf = std::numeric_limits::infinity(); + + // The old kinds and types keep their values; the new ones are appended. + const auto number_of = [](auto value) { return std::to_string(static_cast(value)); }; + using Kind = pineforge::RunFailureArgKind; + using Type = RunFailureArg::Type; + CHECK_EQ("kind identifier", number_of(Kind::identifier), "0"); + CHECK_EQ("kind keyword", number_of(Kind::keyword), "1"); + CHECK_EQ("kind vocab", number_of(Kind::vocab), "2"); + CHECK_EQ("kind integer", number_of(Kind::integer), "3"); + CHECK_EQ("kind number", number_of(Kind::number), "4"); + CHECK_EQ("kind pine_source", number_of(Kind::pine_source), "5"); + CHECK_EQ("kind symbol", number_of(Kind::symbol), "6"); + CHECK_EQ("kind timeframe", number_of(Kind::timeframe), "7"); + CHECK_EQ("kind string", number_of(Kind::string), "8"); + CHECK_EQ("kind nullable_string", number_of(Kind::nullable_string), "9"); + CHECK_EQ("kind scalar", number_of(Kind::scalar), "10"); + CHECK_EQ("type text", number_of(Type::text), "0"); + CHECK_EQ("type integer", number_of(Type::integer), "1"); + CHECK_EQ("type number", number_of(Type::number), "2"); + CHECK_EQ("type boolean", number_of(Type::boolean), "3"); + CHECK_EQ("type null_value", number_of(Type::null_value), "4"); + + // Every documented row: its name, class and retryable=false. + struct DocumentedRow { + RunFailureCode code; + const char* name; + RunFailureClass cls; + }; + const DocumentedRow documented_rows[] = { + {Code::window_mode_unsupported, "window_mode_unsupported", RunFailureClass::unsupported}, + {Code::window_request_invalid, "window_request_invalid", RunFailureClass::input}, + {Code::window_range_invalid, "window_range_invalid", RunFailureClass::input}, + {Code::window_preroll_out_of_range, "window_preroll_out_of_range", RunFailureClass::input}, + {Code::window_legacy_flag_conflict, "window_legacy_flag_conflict", RunFailureClass::input}, + {Code::window_calendar_unsupported, "window_calendar_unsupported", + RunFailureClass::unsupported}, + {Code::window_boundary_unaligned, "window_boundary_unaligned", RunFailureClass::input}, + {Code::window_feed_range_invalid, "window_feed_range_invalid", RunFailureClass::input}, + {Code::window_input_cap_exceeded, "window_input_cap_exceeded", RunFailureClass::input}, + {Code::window_no_observations, "window_no_observations", RunFailureClass::input}, + {Code::window_prestart_state_violation, "window_prestart_state_violation", + RunFailureClass::engine_fault}, + {Code::window_prestart_fill, "window_prestart_fill", RunFailureClass::engine_fault}, + {Code::window_lineage_mismatch, "window_lineage_mismatch", RunFailureClass::input}, + {Code::window_feed_identity_changed, "window_feed_identity_changed", + RunFailureClass::input}, + {Code::window_report_mismatch, "window_report_mismatch", RunFailureClass::engine_fault}, + {Code::report_shape_unsupported, "report_shape_unsupported", RunFailureClass::unsupported}, + {Code::report_shape_request_invalid, "report_shape_request_invalid", + RunFailureClass::input}, + {Code::results_digest_unsupported, "results_digest_unsupported", + RunFailureClass::unsupported}, + {Code::report_platform_unavailable, "report_platform_unavailable", + RunFailureClass::engine_fault}, + {Code::report_result_invalid, "report_result_invalid", RunFailureClass::engine_fault}, + {Code::strategy_value_invalid, "strategy_value_invalid", RunFailureClass::strategy}, + }; + for (const DocumentedRow& row : documented_rows) { + CHECK_EQ(std::string("documented name ") + row.name, code_name(row.code), row.name); + CHECK_EQ(std::string("documented class of ") + row.name, + class_name(pineforge::run_failure_code_class(row.code)), class_name(row.cls)); + CHECK_EQ(std::string("documented retryable of ") + row.name, + pineforge::run_failure_code_retryable(row.code) ? "true" : "false", "false"); + } + + // window_mode_unsupported: a required capability; the canonical timeframe and + // the pre-roll length are optional. The registry does not tie an optional + // argument to a capability (the trusted wrappers do), so each branch of the + // contract is shown to be expressible and nothing more. + wv_expect("chart timeframe", Code::window_mode_unsupported, + {{"capability", "selected_window_chart_timeframe"}, {"timeframe", "D"}}, + Code::window_mode_unsupported, + R"({"capability":"selected_window_chart_timeframe","timeframe":"D"})"); + wv_expect("feed timeframe", Code::window_mode_unsupported, + {{"capability", "selected_window_request_feed_timeframe"}, {"timeframe", "W"}}, + Code::window_mode_unsupported, + R"({"capability":"selected_window_request_feed_timeframe","timeframe":"W"})"); + wv_expect("incomplete inventory carries no token", Code::window_mode_unsupported, + {{"capability", "selected_window_request_feed_inventory"}}, + Code::window_mode_unsupported, + R"({"capability":"selected_window_request_feed_inventory"})"); + wv_expect("pre-roll length", Code::window_mode_unsupported, + {{"capability", "selected_window_preroll"}, {"requested", std::int64_t{5000}}}, + Code::window_mode_unsupported, + R"({"capability":"selected_window_preroll","requested":5000})"); + wv_expect("capability missing", Code::window_mode_unsupported, + {{"timeframe", "D"}}, invariant, "{}"); + wv_expect("capability repeated", Code::window_mode_unsupported, + {{"capability", "a"}, {"capability", "b"}}, invariant, "{}"); + wv_expect("undeclared argument", Code::window_mode_unsupported, + {{"capability", "a"}, {"token", "D"}}, invariant, "{}"); + wv_expect("timeframe as an integer", Code::window_mode_unsupported, + {{"capability", "a"}, {"timeframe", std::int64_t{1}}}, invariant, "{}"); + wv_expect("timeframe as a null", Code::window_mode_unsupported, + {{"capability", "a"}, {"timeframe", nullptr}}, invariant, "{}"); + wv_expect("requested as text", Code::window_mode_unsupported, + {{"capability", "a"}, {"requested", "5000"}}, invariant, "{}"); + wv_expect("requested as a boolean", Code::window_mode_unsupported, + {{"capability", "a"}, {"requested", true}}, invariant, "{}"); + + // window_calendar_unsupported: session is a nullable string. + const char* null_pointer = nullptr; + wv_expect("session null", Code::window_calendar_unsupported, + {{"session", nullptr}, {"timezone", "Etc/UTC"}}, Code::window_calendar_unsupported, + R"({"session":null,"timezone":"Etc/UTC"})"); + wv_expect("session text", Code::window_calendar_unsupported, + {{"session", "0930-1600"}, {"timezone", "America/New_York"}}, + Code::window_calendar_unsupported, + R"({"session":"0930-1600","timezone":"America/New_York"})"); + wv_expect("a null const char* variable is still the empty text", + Code::window_calendar_unsupported, + {{"session", null_pointer}, {"timezone", "Etc/UTC"}}, + Code::window_calendar_unsupported, R"({"session":"","timezone":"Etc/UTC"})"); + wv_expect("session missing", Code::window_calendar_unsupported, + {{"timezone", "Etc/UTC"}}, invariant, "{}"); + wv_expect("session as an integer", Code::window_calendar_unsupported, + {{"session", std::int64_t{0}}, {"timezone", "Etc/UTC"}}, invariant, "{}"); + wv_expect("session as a boolean", Code::window_calendar_unsupported, + {{"session", false}, {"timezone", "Etc/UTC"}}, invariant, "{}"); + wv_expect("timezone as a null", Code::window_calendar_unsupported, + {{"session", "x"}, {"timezone", nullptr}}, invariant, "{}"); + + // scalar: text, an integer, a finite number, true, false or null; the JSON + // literals are never strings. + const std::string shape = R"({"option":"curve_point_budget","value":)"; + wv_expect("scalar true", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", true}}, + Code::report_shape_request_invalid, shape + "true}"); + wv_expect("scalar false", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", false}}, + Code::report_shape_request_invalid, shape + "false}"); + wv_expect("scalar null", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", nullptr}}, + Code::report_shape_request_invalid, shape + "null}"); + wv_expect("scalar integer", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", std::int64_t{2500}}}, + Code::report_shape_request_invalid, shape + "2500}"); + wv_expect("scalar number", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", 1.5}}, + Code::report_shape_request_invalid, shape + "1.5}"); + wv_expect("scalar text", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", "abc"}}, + Code::report_shape_request_invalid, shape + "\"abc\"}"); + wv_expect("scalar text that spells a literal stays text", Code::report_shape_request_invalid, + {{"option", "report_shape"}, {"value", "true"}}, Code::report_shape_request_invalid, + R"({"option":"report_shape","value":"true"})"); + wv_expect("scalar NaN", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", nan}}, invariant, "{}"); + wv_expect("scalar infinity", Code::report_shape_request_invalid, + {{"option", "curve_point_budget"}, {"value", inf}}, invariant, "{}"); + wv_expect("option outside its list", Code::report_shape_request_invalid, + {{"option", "report_policy"}, {"value", true}}, invariant, "{}"); + wv_expect("option as a boolean", Code::report_shape_request_invalid, + {{"option", true}, {"value", true}}, invariant, "{}"); + wv_expect("lineage scalars", Code::window_lineage_mismatch, + {{"actual", false}, {"expected", true}, {"field", "preroll_bars"}}, + Code::window_lineage_mismatch, + R"({"actual":false,"expected":true,"field":"preroll_bars"})"); + wv_expect("lineage field as a null", Code::window_lineage_mismatch, + {{"actual", 1}, {"expected", 2}, {"field", nullptr}}, invariant, "{}"); + + // integer and number reject the new types; number still takes an integer. + wv_expect("integer as text", Code::window_range_invalid, + {{"start_ms", "1"}, {"end_ms", 2}, {"fed_start_ms", 0}}, invariant, "{}"); + wv_expect("integer as a boolean", Code::window_range_invalid, + {{"start_ms", true}, {"end_ms", 2}, {"fed_start_ms", 0}}, invariant, "{}"); + wv_expect("integer as a null", Code::window_range_invalid, + {{"start_ms", nullptr}, {"end_ms", 2}, {"fed_start_ms", 0}}, invariant, "{}"); + wv_expect("integer as a number", Code::window_range_invalid, + {{"start_ms", 1.5}, {"end_ms", 2}, {"fed_start_ms", 0}}, invariant, "{}"); + wv_expect("range", Code::window_range_invalid, + {{"start_ms", std::int64_t{1}}, {"end_ms", std::int64_t{2}}, + {"fed_start_ms", std::int64_t{0}}}, + Code::window_range_invalid, R"({"end_ms":2,"fed_start_ms":0,"start_ms":1})"); + wv_expect("number as an integer", Code::window_prestart_state_violation, + {{"position_size", 3}, {"resting_orders", 0}, {"time_ms", 5}}, + Code::window_prestart_state_violation, + R"({"position_size":3,"resting_orders":0,"time_ms":5})"); + wv_expect("number as a double", Code::window_prestart_state_violation, + {{"position_size", 2.5}, {"resting_orders", 0}, {"time_ms", 5}}, + Code::window_prestart_state_violation, + R"({"position_size":2.5,"resting_orders":0,"time_ms":5})"); + wv_expect("number NaN", Code::window_prestart_state_violation, + {{"position_size", nan}, {"resting_orders", 0}, {"time_ms", 5}}, invariant, "{}"); + wv_expect("number infinity", Code::window_prestart_state_violation, + {{"position_size", -inf}, {"resting_orders", 0}, {"time_ms", 5}}, invariant, "{}"); + wv_expect("number as a boolean", Code::window_prestart_state_violation, + {{"position_size", true}, {"resting_orders", 0}, {"time_ms", 5}}, invariant, "{}"); + wv_expect("number as a null", Code::window_prestart_state_violation, + {{"position_size", nullptr}, {"resting_orders", 0}, {"time_ms", 5}}, invariant, "{}"); + wv_expect("number as text", Code::window_prestart_state_violation, + {{"position_size", "3"}, {"resting_orders", 0}, {"time_ms", 5}}, invariant, "{}"); + + // Dotted vocab values are exact; the digests are plain strings. + for (const char* field : {"counts.window_input_bars", "counts.window_script_bars", + "metering.billable_input_bars"}) { + wv_expect(std::string("report field ") + field, Code::window_report_mismatch, + {{"field", field}, {"expected", std::int64_t{10}}, {"actual", std::int64_t{9}}}, + Code::window_report_mismatch, + std::string(R"({"actual":9,"expected":10,"field":")") + field + "\"}"); + } + wv_expect("report field outside its list", Code::window_report_mismatch, + {{"field", "counts.window_other"}, {"expected", 1}, {"actual", 1}}, invariant, "{}"); + wv_expect("feed digests", Code::window_feed_identity_changed, + {{"feed_id", "primary"}, {"expected_sha256", std::string(64, 'a')}, + {"actual_sha256", std::string(64, 'b')}}, + Code::window_feed_identity_changed, + std::string(R"({"actual_sha256":")") + std::string(64, 'b') + + R"(","expected_sha256":")" + std::string(64, 'a') + + R"(","feed_id":"primary"})"); + wv_expect("feed digest as an integer", Code::window_feed_identity_changed, + {{"feed_id", "primary"}, {"expected_sha256", 1}, {"actual_sha256", "b"}}, + invariant, "{}"); + + // Old kinds and codes keep their bytes, and a string keeps the old escaping + // and replacement of invalid UTF-8. + wv_expect("old integer and vocab", Code::pine_runtime_limit, + {{"limit", "map_pairs"}, {"max", std::int64_t{100000}}}, Code::pine_runtime_limit, + R"({"limit":"map_pairs","max":100000})"); + wv_expect("old integer rejects a boolean", Code::pine_runtime_limit, + {{"limit", "map_pairs"}, {"max", true}}, invariant, "{}"); + wv_expect("old integer rejects a null", Code::pine_runtime_limit, + {{"limit", "map_pairs"}, {"max", nullptr}}, invariant, "{}"); + wv_expect("old identifier", Code::pine_invalid_argument, + {{"function", "ta.sma"}, {"rule", "must_be_positive"}}, Code::pine_invalid_argument, + R"({"function":"ta.sma","rule":"must_be_positive"})"); + wv_expect("old identifier rejects a boolean", Code::pine_invalid_argument, + {{"function", true}, {"rule", "must_be_positive"}}, invariant, "{}"); + wv_expect("old vocab rejects a null", Code::pine_invalid_argument, + {{"rule", nullptr}}, invariant, "{}"); + wv_expect("old optional and required", Code::chart_bars_rejected, + {{"reason", "not_finite"}, {"index", std::int64_t{3}}, {"field", "close"}}, + Code::chart_bars_rejected, R"({"field":"close","index":3,"reason":"not_finite"})"); + const std::string messy = std::string("a\"b\n") + "\xFF"; + wv_expect("string escaping and U+FFFD", Code::window_legacy_flag_conflict, + {{"option", messy}}, Code::window_legacy_flag_conflict, + std::string("{\"option\":\"a\\\"b\\u000a") + "\xEF\xBF\xBD" + "\"}"); +} + +// --------------------------------------------------------------------------- +// The three codes the documented contract adds to its base table: +// report_post_execution_failed, engine_command_value_invalid and +// window_pruner_unsupported. Each one's documented class, retryable +// flag, since, English and arguments are written out here and held to the +// registry and to the catalog file; every closed list is walked value by +// value, a value outside it is refused, and the bytes of the canonical +// arguments are pinned. This holds the catalog and the registry only: it says +// nothing about whether any source path raises these codes. + +struct ChainArg { + std::string name; + std::string kind; + std::vector values; // the closed list of a vocab arg +}; + +struct ChainCode { + RunFailureCode code; + std::string name; + std::string cls; + std::string english; + std::vector args; // in catalog order (sorted by name), all required +}; + +std::string join_values(const std::vector& items) { + std::string out; + for (const std::string& item : items) out += (out.empty() ? "" : ",") + item; + return out; +} + +void chain_vocabulary_rows() { + using Code = RunFailureCode; + const Code invariant = Code::engine_invariant; + const std::vector phases = {"result_assembly", "results_digest", + "serialization"}; + const std::vector post_reasons = {"deadline", "exception", "io", "resource"}; + const std::vector value_reasons = {"nonfinite", "invalid_unicode", + "numeric_overflow"}; + const std::vector stages = {"sizing", "margin", "fee", "slippage", + "relative_price", "risk", "settlement"}; + const std::vector chain = { + {Code::report_post_execution_failed, "report_post_execution_failed", "engine_fault", + "The engine could not finish producing the result after strategy execution completed.", + {{"phase", "vocab", phases}, {"reason", "vocab", post_reasons}}}, + {Code::engine_command_value_invalid, "engine_command_value_invalid", "engine_fault", + "The engine derived an invalid execution value.", + {{"field", "string", {}}, {"reason", "vocab", value_reasons}, + {"stage", "vocab", stages}}}, + {Code::window_pruner_unsupported, "window_pruner_unsupported", "unsupported", + "The selected window cannot be combined with this trial pruner.", + {{"pruner", "string", {}}}}, + }; + + // The registry: the name, the class and retryable=false. + for (const ChainCode& row : chain) { + CHECK_EQ("chain name " + row.name, code_name(row.code), row.name); + CHECK_EQ("chain class of " + row.name, + class_name(pineforge::run_failure_code_class(row.code)), row.cls); + CHECK_EQ("chain retryable of " + row.name, + pineforge::run_failure_code_retryable(row.code) ? "true" : "false", "false"); + } + + // The catalog file: the same class, retryable=false, since 1.5.0, the one + // fixed English and the arguments with their kinds and closed lists. + std::ifstream in(PINEFORGE_RUN_FAILURE_CATALOG, std::ios::binary); + CHECK_EQ("chain: catalog readable", in ? "open" : "missing", "open"); + if (in) { + const std::string text((std::istreambuf_iterator(in)), + std::istreambuf_iterator()); + Json catalog; + bool parsed = true; + try { + catalog = JsonReader(text).document(); + } catch (const std::exception& error) { + parsed = false; + CHECK_EQ("chain: catalog parses", error.what(), ""); + } + const Json* codes = parsed ? catalog.find("codes") : nullptr; + for (const ChainCode& row : chain) { + const std::string at = "chain catalog " + row.name + ": "; + const Json* entry = codes != nullptr ? codes->find(row.name) : nullptr; + CHECK_EQ(at + "present", entry != nullptr ? "present" : "absent", "present"); + if (entry == nullptr) continue; + const Json* cls = entry->find("class"); + const Json* retryable = entry->find("retryable"); + const Json* since = entry->find("since"); + const Json* english = entry->find("english"); + const Json* args = entry->find("args"); + CHECK_EQ(at + "class", cls ? cls->string : "", row.cls); + const bool never_retry = retryable != nullptr + && retryable->kind == Json::Kind::boolean + && !retryable->boolean; + CHECK_EQ(at + "retryable false", never_retry ? "false" : "not false", "false"); + CHECK_EQ(at + "since", since ? since->string : "", "1.5.0"); + CHECK_EQ(at + "one English template", + std::to_string(english ? english->array.size() : std::size_t{0}), "1"); + CHECK_EQ(at + "English", + english && !english->array.empty() ? english->array.front().string : "", + row.english); + const std::size_t arg_count = args ? args->object.size() : std::size_t{0}; + CHECK_EQ(at + "argument count", std::to_string(arg_count), + std::to_string(row.args.size())); + if (args == nullptr || arg_count != row.args.size()) continue; + for (std::size_t i = 0; i < row.args.size(); ++i) { + const ChainArg& want = row.args[i]; + const std::string& got_name = args->object[i].first; + const Json& arg = args->object[i].second; + CHECK_EQ(at + "argument " + std::to_string(i), got_name, want.name); + const Json* kind = arg.find("kind"); + const Json* values = arg.find("values"); + std::vector got_values; + if (values != nullptr) { + for (const Json& value : values->array) got_values.push_back(value.string); + } + CHECK_EQ(at + want.name + " kind", kind ? kind->string : "", want.kind); + CHECK_EQ(at + want.name + " is required", + arg.find("optional") == nullptr ? "required" : "optional", "required"); + CHECK_EQ(at + want.name + " closed list", join_values(got_values), + join_values(want.values)); + } + } + } + + // report_post_execution_failed: every phase with every reason is accepted, + // raised in the reverse of key order; the bytes are the canonical JSON. + for (const std::string& phase : phases) { + for (const std::string& reason : post_reasons) { + wv_expect("post-execution " + phase + "/" + reason, + Code::report_post_execution_failed, + {{"reason", reason}, {"phase", phase}}, Code::report_post_execution_failed, + R"({"phase":")" + phase + R"(","reason":")" + reason + R"("})"); + } + } + wv_expect("post-execution phase outside its list", Code::report_post_execution_failed, + {{"phase", "completed"}, {"reason", "deadline"}}, invariant, "{}"); + wv_expect("post-execution phase of the wire's logical phases but not of this list", + Code::report_post_execution_failed, {{"phase", "execution"}, {"reason", "io"}}, + invariant, "{}"); + wv_expect("post-execution phase in upper case", Code::report_post_execution_failed, + {{"phase", "Serialization"}, {"reason", "io"}}, invariant, "{}"); + wv_expect("post-execution reason outside its list", Code::report_post_execution_failed, + {{"phase", "serialization"}, {"reason", "timeout"}}, invariant, "{}"); + wv_expect("post-execution reason of the other code's list", Code::report_post_execution_failed, + {{"phase", "serialization"}, {"reason", "nonfinite"}}, invariant, "{}"); + wv_expect("post-execution phase as an integer", Code::report_post_execution_failed, + {{"phase", std::int64_t{1}}, {"reason", "io"}}, invariant, "{}"); + wv_expect("post-execution reason as a null", Code::report_post_execution_failed, + {{"phase", "serialization"}, {"reason", nullptr}}, invariant, "{}"); + wv_expect("post-execution reason as a boolean", Code::report_post_execution_failed, + {{"phase", "serialization"}, {"reason", true}}, invariant, "{}"); + wv_expect("post-execution without a phase", Code::report_post_execution_failed, + {{"reason", "io"}}, invariant, "{}"); + wv_expect("post-execution without a reason", Code::report_post_execution_failed, + {{"phase", "serialization"}}, invariant, "{}"); + wv_expect("post-execution without arguments", Code::report_post_execution_failed, {}, + invariant, "{}"); + wv_expect("post-execution phase repeated", Code::report_post_execution_failed, + {{"phase", "serialization"}, {"phase", "results_digest"}, {"reason", "io"}}, + invariant, "{}"); + wv_expect("post-execution undeclared argument", Code::report_post_execution_failed, + {{"phase", "serialization"}, {"reason", "io"}, {"detail", "x"}}, invariant, "{}"); + + // engine_command_value_invalid: every stage with every reason, and a free + // text field. + for (const std::string& stage : stages) { + for (const std::string& reason : value_reasons) { + wv_expect("command value " + stage + "/" + reason, Code::engine_command_value_invalid, + {{"stage", stage}, {"reason", reason}, {"field", "quantity"}}, + Code::engine_command_value_invalid, + R"({"field":"quantity","reason":")" + reason + R"(","stage":")" + stage + + R"("})"); } } + wv_expect("command value field is free text", Code::engine_command_value_invalid, + {{"field", "qty\"x"}, {"reason", "nonfinite"}, {"stage", "fee"}}, + Code::engine_command_value_invalid, + R"({"field":"qty\"x","reason":"nonfinite","stage":"fee"})"); + wv_expect("command value stage outside its list", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "nonfinite"}, {"stage", "settle"}}, invariant, + "{}"); + wv_expect("command value stage taken from the reason list", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "nonfinite"}, {"stage", "nonfinite"}}, invariant, + "{}"); + wv_expect("command value stage in upper case", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "nonfinite"}, {"stage", "Sizing"}}, invariant, + "{}"); + wv_expect("command value reason outside its list", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "overflow"}, {"stage", "sizing"}}, invariant, + "{}"); + wv_expect("command value reason of the post-execution list", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "deadline"}, {"stage", "sizing"}}, invariant, + "{}"); + wv_expect("command value field as an integer", Code::engine_command_value_invalid, + {{"field", std::int64_t{1}}, {"reason", "nonfinite"}, {"stage", "sizing"}}, + invariant, "{}"); + wv_expect("command value field as a null", Code::engine_command_value_invalid, + {{"field", nullptr}, {"reason", "nonfinite"}, {"stage", "sizing"}}, invariant, "{}"); + wv_expect("command value field as a boolean", Code::engine_command_value_invalid, + {{"field", false}, {"reason", "nonfinite"}, {"stage", "sizing"}}, invariant, "{}"); + wv_expect("command value without a stage", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "nonfinite"}}, invariant, "{}"); + wv_expect("command value without a reason", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"stage", "sizing"}}, invariant, "{}"); + wv_expect("command value without a field", Code::engine_command_value_invalid, + {{"reason", "nonfinite"}, {"stage", "sizing"}}, invariant, "{}"); + wv_expect("command value undeclared argument", Code::engine_command_value_invalid, + {{"field", "quantity"}, {"reason", "nonfinite"}, {"stage", "sizing"}, + {"phase", "serialization"}}, + invariant, "{}"); + + // window_pruner_unsupported: one string, the pruner as the caller sent it. + wv_expect("pruner median", Code::window_pruner_unsupported, {{"pruner", "median"}}, + Code::window_pruner_unsupported, R"({"pruner":"median"})"); + wv_expect("pruner halving", Code::window_pruner_unsupported, {{"pruner", "halving"}}, + Code::window_pruner_unsupported, R"({"pruner":"halving"})"); + wv_expect("pruner name as the caller sent it", Code::window_pruner_unsupported, + {{"pruner", "Hyper\"band"}}, Code::window_pruner_unsupported, + R"({"pruner":"Hyper\"band"})"); + wv_expect("pruner as an integer", Code::window_pruner_unsupported, + {{"pruner", std::int64_t{1}}}, invariant, "{}"); + wv_expect("pruner as a number", Code::window_pruner_unsupported, {{"pruner", 1.5}}, invariant, + "{}"); + wv_expect("pruner as a null", Code::window_pruner_unsupported, {{"pruner", nullptr}}, invariant, + "{}"); + wv_expect("pruner as a boolean", Code::window_pruner_unsupported, {{"pruner", true}}, invariant, + "{}"); + wv_expect("pruner missing", Code::window_pruner_unsupported, {}, invariant, "{}"); + wv_expect("pruner repeated", Code::window_pruner_unsupported, + {{"pruner", "median"}, {"pruner", "halving"}}, invariant, "{}"); + wv_expect("pruner undeclared argument", Code::window_pruner_unsupported, + {{"pruner", "median"}, {"capability", "x"}}, invariant, "{}"); } void registry_rows() { @@ -1376,6 +1901,8 @@ int main() { stream_refusal_rows(minute); run_spec_field_rows(); registry_rows(); + window_vocabulary_rows(); + chain_vocabulary_rows(); std::printf("test_run_failure_codes: %d checks, %d failed\n", g_checks, g_failures); return g_failures == 0 ? 0 : 1; } diff --git a/tests/test_run_json_selected_window.py b/tests/test_run_json_selected_window.py new file mode 100644 index 000000000..199038e9d --- /dev/null +++ b/tests/test_run_json_selected_window.py @@ -0,0 +1,2095 @@ +"""Tests of the single-run selected-window orchestration of docker/run_json.py and of +docker/selected_window_report.py. + +The orchestration is tested on fakes. The strategy library is a Python +stand-in (FakeLib) with the frozen export names, the planner is a stand-in object, and the +phase transport is a real AF_UNIX socketpair driven by the real RunPhaseWriter and the real +ExecutionObserverBinding (the fake engine calls the registered callback through its raw C +function pointer). The rows cover the order of work, what is admitted or refused before any feed, +library, planner or phase record exists, the trim / hash / R mapping, the comparison of native +counts with the plan, cleanup, and that an ordinary run is unchanged. The native engine itself +(the planner library, the selected-window ABI, the observer inside run_backtest_full, the report +of the selected generation and the socket boundaries) is not exercised here, and no fake result +stands in for it. + +Poison counters: the entries a refused request must never reach (load_bars, load_symbol_feeds, +load_strategy, create_strategy, the planner constructor, ctypes.CDLL, a phase advance) are +patched with mocks that count and fail; every refusal row asserts all counts are zero and +the positive row asserts that work begins. +""" +import base64 +import collections +import contextlib +import ctypes +import hashlib +import io +import json +import socket +import struct +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "docker")) + +import request_feed_inventory as rfi # noqa: E402 +import run_execution_observer as reo # noqa: E402 +import run_json # noqa: E402 +import run_phase_transport as rpt # noqa: E402 +import selected_window_plan as swp # noqa: E402 +import selected_window_report as swr # noqa: E402 + +MINUTE = 60_000 +F = 1_759_536_000_000 # 2025-10-04T00:00:00Z, minute aligned: the frozen feed start +T = F + 4 * MINUTE # window start +E = F + 10 * MINUTE # window end +N = 3 # requested pre-roll script bars +STATE = 7 +CAPITAL = 1_000_000.0 # the effective initial capital of a provenance without overrides +# The lookback record of a build with NO lookback producer, written out from the documented +# contract and never read back from the module under test: status "unknown", a null declared +# count and a null has_unbounded_state. Null means "not produced by this build": it is neither +# true nor false. +NOT_PRODUCED_LOOKBACK = {"declared_required_chart_bars": None, "status": "unknown", + "has_unbounded_state": None} +INVENTORY_CAPABILITY = {"capability": "selected_window_request_feed_inventory"} +INSIDE = [{"kind": "token", "timeframe": "1"}, {"kind": "token", "timeframe": "D"}] + + +# --- fixtures --------------------------------------------------------------- + +def csv_rows(count, start=F): + return [(start + i * MINUTE, 100.0 + i, 102.0 + i, 99.0 + i, 101.0 + i, 10.0 + i) + for i in range(count)] + + +def write_ohlcv(path, rows): + lines = ["timestamp,open,high,low,close,volume"] + lines += [f"{ts},{o},{h},{l},{c},{v}" for ts, o, h, l, c, v in rows] + path.write_text("\n".join(lines) + "\n") + + +def values_hash(rows): + """The existing primary hash domain, written out from its definition.""" + digest = hashlib.sha256() + digest.update(b"pineforge:ohlcv:barc-le:v1\0") + for ts, o, h, l, c, v in rows: + digest.update(struct.pack("<5dq", o, h, l, c, v, ts)) + return digest.hexdigest() + + +def full_plan(**over): + """The planner's plan for a 10 row, one minute feed: 4 rows before T, 6 in the window, + N=3 kept, one pre-T row cut. Every wire equality holds.""" + plan = { + "status": 0, "option": "", "bound": -1, "value_ms": 0, "previous_boundary_ms": 0, + "next_boundary_ms": 0, "supplied_input_bars": 10, "supplied_script_bars": 10, + "available_script_bars": 4, "used_script_bars": 3, "trimmed_script_bars": 1, + "trim_index": 1, "fed_input_bars": 9, "fed_script_bars": 9, "preroll_input_bars": 3, + "window_input_bars": 6, "window_script_bars": 6, "trim_start_ms": F + MINUTE, + "preroll_first_bar_ms": F + MINUTE, "preroll_last_bar_ms": F + 3 * MINUTE, + "supplied_first_data_ms": F, "supplied_last_data_ms": F + 9 * MINUTE, + "fed_first_data_ms": F + MINUTE, "fed_last_data_ms": F + 9 * MINUTE, + "window_first_data_ms": T, "window_last_data_ms": F + 9 * MINUTE, + "shortfall": False, "complete_pending_preroll_at_horizon": False} + plan.update(over) + return plan + + +def empty_window_plan(): + """A 4 row feed that ends before T: pre-roll only, no window row.""" + return full_plan( + supplied_input_bars=4, supplied_script_bars=4, available_script_bars=4, + used_script_bars=3, trimmed_script_bars=1, trim_index=1, fed_input_bars=3, + fed_script_bars=3, preroll_input_bars=3, window_input_bars=0, window_script_bars=0, + supplied_last_data_ms=F + 3 * MINUTE, fed_last_data_ms=F + 3 * MINUTE, + window_first_data_ms=None, window_last_data_ms=None, + complete_pending_preroll_at_horizon=True) + + +def request(): + return swr.WindowRequest(T, E, N, F, "1", "1", 60) + + +def expected_window(plan, hashes, *, applied=True): + """R written out field by field from the wire tables, not produced by the code under test.""" + window_script = plan["window_script_bars"] + return { + "wire_version": 1, "policy": "selected-window/v1", "applied": applied, + "window": {"start_ms": T, "end_ms": E, "first_data_ms": plan["window_first_data_ms"], + "last_data_ms": plan["window_last_data_ms"], + "coverage": "nonempty" if plan["window_input_bars"] else "empty"}, + "feed": {"requested_start_ms": F, "first_data_ms": plan["fed_first_data_ms"], + "last_data_ms": plan["fed_last_data_ms"], + "supplied_first_data_ms": plan["supplied_first_data_ms"], + "supplied_last_data_ms": plan["supplied_last_data_ms"], + "input_tf_seconds": 60, "script_tf": "1", + "source_bytes_sha256": hashes[0], "source_values_sha256": hashes[1], + "evaluated_source_values_sha256": hashes[2]}, + "calendar": {"timezone": "UTC", "session": None}, + "preroll": {"requested_script_bars": N, + "available_script_bars": plan["available_script_bars"], + "used_script_bars": plan["used_script_bars"], + "trimmed_script_bars": plan["trimmed_script_bars"], + "trim_start_ms": plan["trim_start_ms"], + "trimmed_input_bars": plan["trim_index"], + "first_bar_ms": plan["preroll_first_bar_ms"], + "last_bar_ms": plan["preroll_last_bar_ms"], "shortfall": False}, + "lookback": dict(NOT_PRODUCED_LOOKBACK), + "counts": {"supplied_input_bars": plan["supplied_input_bars"], + "supplied_script_bars": plan["supplied_script_bars"], + "fed_input_bars": plan["fed_input_bars"], + "fed_script_bars": plan["fed_script_bars"], + "preroll_input_bars": plan["preroll_input_bars"], + "window_input_bars": plan["window_input_bars"], + "window_script_bars": window_script, + "equity_points": window_script + 1 if applied else 0, + "anchor_points": 1 if applied else 0}, + "metering": {"billable_input_bars": plan["window_input_bars"], + "preroll_billable_input_bars": 0, + "basis": "window-primary-input-bars/v1"}, + "indices": {"fed_script_index_of_window_first": + plan["used_script_bars"] if window_script else None, + "supplied_script_index_of_window_first": + plan["available_script_bars"] if window_script else None}, + } + + +def selected_argv(so, ohlcv, inventory, *, script_tf="1", input_tf="1", extra=()): + argv = ["--so", str(so), "--ohlcv", str(ohlcv), "--input-tf", input_tf, + "--script-tf", script_tf, "--report-policy", "selected-window/v1", + "--window-start-ms", str(T), "--window-end-ms", str(E), + "--preroll-bars", str(N), "--fed-start-ms", str(F)] + if inventory is not None: + argv += ["--request-feed-inventory", str(inventory)] + return argv + list(extra) + + +def write_inventory(directory, entries, *, chart=None, artifact_bytes=b"not a library\n", + artifact_sha=None): + artifact = Path(directory) / "strategy.so" + artifact.write_bytes(artifact_bytes) + doc = {"schema": rfi.SCHEMA, "source_sha256": "0" * 64, + "artifact_sha256": artifact_sha or hashlib.sha256(artifact_bytes).hexdigest(), + "primary_chart_timeframe": chart, "entries": entries} + inventory = Path(directory) / "inventory.json" + inventory.write_text(json.dumps(doc)) + return artifact, inventory + + +UNRESOLVED_RESOLUTION = {"initial_capital": { + "status": "unresolved", "reason": "unsupported_default", "raw_default": "capital_expr()"}} + + +def unresolved_capital_normalizer(seen=None): + """A stand-in for run_json.normalize_release_provenance that leaves the initial capital the + way the real one leaves a strategy() default it cannot certify (its legacy branch): null in + provenance["strategy"], named in provenance["strategy_resolution"], nothing else changed. + `seen`, when given, receives a JSON snapshot of the provenance as the normalizer handed it + on, so a test can show that nothing after it wrote to the provenance (no capital guessed + from the native anchor, no key added). + + It is a stand-in: it does NOT stand for the real normalizer on a real unresolved source.""" + def normalize(provenance, cpp_text, receipt, checked): + provenance["strategy"]["initial_capital"] = None + provenance["strategy_resolution"] = json.loads(json.dumps(UNRESOLVED_RESOLUTION)) + if seen is not None: + seen["normalized"] = json.loads(json.dumps(provenance)) + return provenance + return normalize + + +class PoisonTripped(Exception): + pass + + +class Stdout: + """A stdout that records its writes and flushes in the same event list as the run.""" + + def __init__(self, events): + self.events = events + self.parts = [] + + def write(self, text): + self.events.append("stdout.write") + self.parts.append(text) + return len(text) + + def flush(self): + self.events.append("stdout.flush") + + def text(self): + return "".join(self.parts) + + +class PartialStdout(Stdout): + """A stdout that takes the first half of the document and then fails.""" + + def write(self, text): + self.events.append("stdout.write") + self.parts.append(text[:len(text) // 2]) + raise OSError(28, "No space left on device") + + +def run_main(argv, events=None, stdout_class=Stdout): + stdout = stdout_class(events if events is not None else []) + err = io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(err): + status = run_json.main(argv) + return status, stdout.text(), err.getvalue() + + +def last_line(text): + return json.loads(text.strip().splitlines()[-1]) + + +class FakeLib: + """A strategy library stand-in: the exports run_json calls, a native-like run that + calls the registered observer callback through its raw C function pointer and fills a + ReportC, and the selected-window and observer ABIs. Only unit-level behaviour.""" + + def __init__(self, plan, events=None, *, call_observer=True, counts=None, + observation=None, selected_abi=True, observer_abi=True, + anchor_equity=CAPITAL, raise_before_observer=None, engine_failure=None): + self.plan = plan + self.events = events if events is not None else [] + self.calls = [] + self.call_observer = call_observer + self.anchor_equity = anchor_equity + self.raise_before_observer = raise_before_observer + if engine_failure is not None: # (code, text): what run_failure reads after the run + code, text = engine_failure + self.strategy_get_last_error = lambda state: text.encode() + self.strategy_get_last_error_code = lambda state: code.encode() + self.strategy_last_run_status = lambda state: 1 + self.observer_descriptor = None + self.observer_rc = None + self.receipt = None + self.run_args = None + self.window_config = "unset" + self._curve = None + self.native_counts = { + "run_generation": 5, "attempt_serial": 9, "attempt_generation": 5, + "fed_input_bars": plan["fed_input_bars"], "fed_script_bars": plan["fed_script_bars"], + "preroll_input_bars": plan["preroll_input_bars"], + "preroll_script_bars": plan["used_script_bars"], + "window_input_bars": plan["window_input_bars"], + "window_script_bars": plan["window_script_bars"]} + self.native_counts.update(counts or {}) + self.observation = {"run_generation": 5, "attempt_serial": 9, "attempt_generation": 5} + self.observation.update(observation or {}) + lib = self + + def fn(name, result=0): + def call(*args): + lib.calls.append((name,) + args) + return result + return call + + self.strategy_create = fn("strategy_create", STATE) + self.strategy_free = fn("strategy_free") + self.report_free = fn("report_free") + self.strategy_set_input = fn("strategy_set_input") + self.strategy_set_override = fn("strategy_set_override") + self.run_backtest_full = self._run + if selected_abi: + def set_selected(state, pointer): + if pointer: + c = pointer.contents + lib.window_config = (c.struct_size, c.version, c.start_ms, c.end_ms) + lib.calls.append(("set_selected_window", c.start_ms, c.end_ms)) + else: + lib.window_config = None + lib.calls.append(("set_selected_window", None)) + return 0 + + def selected_counts(state, ref): + lib.calls.append(("selected_window_counts",)) + for name, value in lib.native_counts.items(): + setattr(ref._obj, name, value) + return 0 + + self.pf_selected_window_version = lambda: 1 + self.strategy_set_selected_window_v1 = set_selected + self.strategy_selected_window_counts_v1 = selected_counts + if observer_abi: + def set_observer(state, pointer): + lib.observer_descriptor = pointer.contents if pointer else None + lib.calls.append(("set_observer", "descriptor" if pointer else None)) + return 0 + + def observation_snapshot(state, ref): + for name, value in lib.observation.items(): + setattr(ref._obj, name, value) + return 0 + + self.pf_execution_observer_version = lambda: 1 + self.strategy_set_execution_observer_v1 = set_observer + self.strategy_execution_observation_v1 = observation_snapshot + + def _run(self, state, rows, count, input_tf, script_tf, magnifier, samples, dist, ref): + self.events.append("native.start") + self.calls.append(("run_backtest_full", state, count, input_tf, script_tf)) + self.run_args = (rows, count) + if self.raise_before_observer is not None: + raise self.raise_before_observer + if self.call_observer and self.observer_descriptor is not None: + boundary = reo.BoundaryC(ctypes.sizeof(reo.BoundaryC), 1, 5, 9) + receipt = reo.ReceiptC(ctypes.sizeof(reo.ReceiptC), 1, 5, 0xFFFFFFFF, 0, 0, 0) + self.observer_rc = self.observer_descriptor.before_results( + None, ctypes.byref(boundary), ctypes.byref(receipt)) + self.receipt = (receipt.frame_bytes, receipt.handed_bytes, receipt.export_requested) + if self.observer_rc != 0: + self.events.append("native.end") + return + report = ref._obj + plan = self.plan + points = plan["window_script_bars"] + 1 + self._curve = (run_json.EquityPointC * points)() + for i in range(points): + self._curve[i].time_ms = T if i == 0 else T + (i - 1) * MINUTE + self._curve[i].equity = self.anchor_equity if i == 0 else CAPITAL + i + self._curve[i].open_profit = 0.0 + report.equity_curve = ctypes.cast(self._curve, ctypes.POINTER(run_json.EquityPointC)) + report.equity_curve_len = points + report.input_bars_processed = plan["fed_input_bars"] + report.script_bars_processed = plan["fed_script_bars"] + report.input_tf_seconds = 60 + report.script_tf_seconds = 60 + report.script_tf_ratio = 1 + self.events.append("native.end") + + +class FakePlanner: + """Stands in for SelectedPrimaryPlanner: records its construction and calls, returns + the configured plan as a fresh dictionary.""" + constructed = [] + calls = [] + result = None + + def __init__(self, library_path): + FakePlanner.constructed.append(library_path) + + def plan(self, bars, count, **kwargs): + FakePlanner.calls.append((bars, count, kwargs)) + return dict(FakePlanner.result) + + @classmethod + def reset(cls, result): + cls.constructed, cls.calls, cls.result = [], [], result + + +def phase_records(parent): + parent.setblocking(False) + data = b"" + try: + while True: + chunk = parent.recv(65536) + if not chunk: + break + data += chunk + except BlockingIOError: + pass + return [json.loads(line) for line in data.split(b"\n") if line] + + +def recording_writer(events, fail_phase=None): + class Writer(rpt.RunPhaseWriter): + def advance(self, phase): + events.append("advance:" + phase) + if phase == fail_phase: + raise rpt.PhaseTransportError("injected", reason="write_failed", phase=phase) + super().advance(phase) + return Writer + + +class Counters: + """The entries a refused request must never reach, patched to count and fail. With + allow=("load_bars",) the first feed read is the expected, failing end of a positive row.""" + + ENTRIES = ((run_json, "load_bars"), (run_json, "load_symbol_feeds"), + (run_json, "load_strategy"), (run_json, "create_strategy"), + (swp, "SelectedPrimaryPlanner"), (ctypes, "CDLL")) + + def __init__(self): + self.stack = contextlib.ExitStack() + self.mocks = {} + + def __enter__(self): + for owner, name in self.ENTRIES: + mocked = mock.Mock(side_effect=PoisonTripped(name)) + self.stack.enter_context(mock.patch.object(owner, name, mocked)) + self.mocks[name] = mocked + original = rpt.RunPhaseWriter.advance + self.advance = mock.Mock(side_effect=lambda writer, phase: original(writer, phase)) + self.stack.enter_context(mock.patch.object( + rpt.RunPhaseWriter, "advance", lambda writer, phase: self.advance(writer, phase))) + self.reads = mock.Mock() + for name in ("_read_inventory_bytes", "_artifact_digest"): + real = getattr(rfi, name) + self.stack.enter_context(mock.patch.object( + rfi, name, lambda *a, _real=real, **k: (self.reads(), _real(*a, **k))[1])) + return self + + def __exit__(self, *exc): + self.stack.close() + return False + + def counts(self): + counts = {name: m.call_count for name, m in self.mocks.items()} + counts["advance"] = self.advance.call_count + counts["inventory_reads"] = self.reads.call_count + return counts + + +# --- request validation, unsupported options, planner refusals -------------- + +class ValidationTests(unittest.TestCase): + GOOD = dict(policy="selected-window/v1", window_start_ms=str(T), window_end_ms=str(E), + preroll_bars=str(N), fed_start_ms=str(F), trade_start_ms=None, + input_tf="1", script_tf="60") + + def refuse(self, code, args, **over): + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.validate_request(**{**self.GOOD, **over}) + self.assertEqual((caught.exception.code, caught.exception.code_args), (code, args)) + + def test_a_good_request_validates_with_the_canonical_primary(self): + got = swr.validate_request(**self.GOOD) + self.assertEqual(tuple(got), (T, E, N, F, "1", "60", 60)) + + def test_missing_or_wrong_policy_and_malformed_integers(self): + self.refuse("window_request_invalid", {"option": "report_policy"}, policy=None) + self.refuse("window_request_invalid", {"option": "report_policy"}, policy="selected-window/v2") + for option in ("window_start_ms", "window_end_ms", "preroll_bars", "fed_start_ms"): + for bad in (None, "", "1.5", "1e3", "01", "+5", " 5", "abc", str(2 ** 53)): + with self.subTest(option=option, bad=bad): + self.refuse("window_request_invalid", {"option": option}, **{option: bad}) + + def test_explicit_timeframes_are_required_and_must_be_tokens(self): + self.refuse("window_request_invalid", {"option": "input_tf"}, input_tf="") + self.refuse("window_request_invalid", {"option": "script_tf"}, script_tf="") + self.refuse("window_request_invalid", {"option": "script_tf"}, script_tf="4h") + self.refuse("window_request_invalid", {"option": "input_tf"}, input_tf="1H") + self.refuse("window_request_invalid", {"option": "input_tf"}, input_tf="1M") + + def test_legacy_flag_range_and_preroll_refusals(self): + self.refuse("window_legacy_flag_conflict", {"option": "trade_start_ms"}, trade_start_ms=5) + self.refuse("window_preroll_out_of_range", {"requested": 5001}, preroll_bars="5001") + self.refuse("window_preroll_out_of_range", {"requested": -1}, preroll_bars="-1") + self.refuse("window_range_invalid", {"start_ms": E, "end_ms": E, "fed_start_ms": F}, + window_start_ms=str(E)) + self.refuse("window_range_invalid", {"start_ms": T, "end_ms": E, "fed_start_ms": T + 1}, + fed_start_ms=str(T + 1)) + + def test_zero_and_the_maximum_preroll_are_accepted(self): + self.assertEqual(swr.validate_request(**{**self.GOOD, "preroll_bars": "0"}).preroll_bars, 0) + self.assertEqual(swr.validate_request(**{**self.GOOD, "preroll_bars": "5000"}).preroll_bars, 5000) + + def test_timeframe_seconds_is_a_unit_conversion_only(self): + self.assertEqual([swr.timeframe_seconds(t) for t in ("1", "60", "D", "W", "30S", "2D")], + [60, 3600, 86400, 604800, 30, 172800]) + self.assertIsNone(swr.timeframe_seconds("1M")) + self.assertIsNone(swr.timeframe_seconds("1H")) + + +class UnsupportedOptionTests(unittest.TestCase): + def refuse(self, code, args, **kw): + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.check_unsupported_options( + **{"selected": True, "report_shape": None, "curve_point_budget": None, + "results_digest": None, **kw}) + self.assertEqual((caught.exception.code, caught.exception.code_args), (code, args)) + + def test_full_shape_alone_is_accepted_for_a_selected_request(self): + swr.check_unsupported_options(selected=True, report_shape="full/v1", + curve_point_budget=None, results_digest=None) + swr.check_unsupported_options(selected=False, report_shape=None, + curve_point_budget=None, results_digest=None) + + def test_compact_budget_and_digest_fail_typed(self): + self.refuse("report_shape_unsupported", {"shape": "compact/v1"}, + report_shape="compact/v1") + self.refuse("report_shape_request_invalid", {"option": "report_shape", "value": "tiny"}, + report_shape="tiny") + self.refuse("report_shape_request_invalid", + {"option": "curve_point_budget", "value": "2500"}, curve_point_budget="2500") + self.refuse("results_digest_unsupported", {"requested": "sha256/v1"}, + results_digest="sha256/v1") + + def test_a_legacy_request_names_no_shape_in_this_build(self): + self.refuse("report_shape_unsupported", {"shape": "full/v1"}, selected=False, + report_shape="full/v1") + + +class PlanRefusalTests(unittest.TestCase): + def refusal(self, **over): + return swr.refusal_for_plan(full_plan(**over), request(), timezone="UTC", session=None) + + def check(self, error, code, args): + self.assertEqual((error.code, error.code_args), (code, args)) + + def test_ok_is_none(self): + self.assertIsNone(self.refusal()) + + def test_request_invalid_by_option(self): + self.check(self.refusal(status=1, option="script_tf"), "window_request_invalid", + {"option": "script_tf"}) + self.check(self.refusal(status=1, option="input_tf"), "window_request_invalid", + {"option": "input_tf"}) + self.check(self.refusal(status=1, option="fed_start_ms"), "window_range_invalid", + {"start_ms": T, "end_ms": E, "fed_start_ms": F}) + self.check(self.refusal(status=1, option="preroll_bars"), "window_preroll_out_of_range", + {"requested": N}) + self.assertEqual(self.refusal(status=1, option="rows").code, "harness_internal_error") + + def test_calendar_boundary_range_rows_and_internal(self): + self.check(self.refusal(status=2, option="timezone"), "window_calendar_unsupported", + {"timezone": "UTC", "session": None}) + self.check(self.refusal(status=3, bound=0, value_ms=T + 1, previous_boundary_ms=T, + next_boundary_ms=T + MINUTE), + "window_boundary_unaligned", + {"bound": "start", "value_ms": T + 1, "previous_boundary_ms": T, + "next_boundary_ms": T + MINUTE}) + self.check(self.refusal(status=3, bound=1, value_ms=5), "window_boundary_unaligned", + {"bound": "end", "value_ms": 5, "previous_boundary_ms": 0, "next_boundary_ms": 0}) + self.check(self.refusal(status=3, bound=2, value_ms=5), "window_boundary_unaligned", + {"bound": "fed_start", "value_ms": 5, "previous_boundary_ms": 0, + "next_boundary_ms": 0}) + self.assertEqual(self.refusal(status=3, bound=-1).code, "harness_internal_error") + self.check(self.refusal(status=4, value_ms=E + 1), "window_feed_range_invalid", + {"time_ms": E + 1, "fed_start_ms": F, "end_ms": E}) + self.check(self.refusal(status=5, value_ms=9, previous_boundary_ms=9), + "chart_bars_rejected", {"field": "timestamp", "reason": "not_increasing"}) + self.assertEqual(self.refusal(status=6, option="calendar").code, "engine_invariant") + self.assertEqual(self.refusal(status=99).code, "harness_internal_error") + + +# --- trim by pointer offset, hash, R, ABI, native counts -------------------- + +class TrimAndHashTests(unittest.TestCase): + def feed(self, count): + bars = (run_json.BarC * count)() + for i, (ts, o, h, l, c, v) in enumerate(csv_rows(count)): + bars[i].open, bars[i].high, bars[i].low = o, h, l + bars[i].close, bars[i].volume, bars[i].timestamp = c, v, ts + return bars + + def test_the_view_shares_the_allocation_and_starts_at_the_offset(self): + bars = self.feed(5) + view, count = swr.retained_rows(bars, 5, 2, 3, run_json.BarC) + self.assertEqual(count, 3) + self.assertEqual(ctypes.addressof(view), + ctypes.addressof(bars) + 2 * ctypes.sizeof(run_json.BarC)) + self.assertEqual(view[0].timestamp, bars[2].timestamp) + bars[3].close = 777.0 # a write to the original shows in the view: no copy + self.assertEqual(view[1].close, 777.0) + + def test_nothing_cut_returns_the_original_and_nothing_left_returns_count_zero(self): + bars = self.feed(4) + self.assertIs(swr.retained_rows(bars, 4, 0, 4, run_json.BarC)[0], bars) + self.assertEqual(swr.retained_rows(bars, 4, 4, 0, run_json.BarC), (bars, 0)) + + def test_a_plan_that_breaks_supplied_equals_trimmed_plus_fed_is_an_invariant(self): + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.retained_rows(self.feed(4), 4, 1, 4, run_json.BarC) + self.assertEqual(caught.exception.code, "engine_invariant") + + def test_evaluated_hash_restarts_the_existing_domain_over_the_retained_rows(self): + rows = csv_rows(6) + bars = self.feed(6) + hasher = run_json._new_source_feed_hasher() + swr.update_rows_hash(hasher, bars, 2, 4, run_json.BarC, run_json._SOURCE_FEED_RECORD) + self.assertEqual(hasher.hexdigest(), values_hash(rows[2:])) + whole = run_json._new_source_feed_hasher() + swr.update_rows_hash(whole, bars, 0, 6, run_json.BarC, run_json._SOURCE_FEED_RECORD) + self.assertEqual(whole.hexdigest(), values_hash(rows)) + + def test_the_row_by_row_fallback_gives_the_same_digest(self): + rows = csv_rows(5) + bars = self.feed(5) + hasher = run_json._new_source_feed_hasher() + with mock.patch.object(swr.sys, "byteorder", "big"): + swr.update_rows_hash(hasher, bars, 1, 4, run_json.BarC, run_json._SOURCE_FEED_RECORD) + self.assertEqual(hasher.hexdigest(), values_hash(rows[1:])) + + +class ReportWindowTests(unittest.TestCase): + HASHES = ("a" * 64, "b" * 64, "c" * 64) + + def build(self, plan, *, applied=True): + return swr.build_report_window( + request(), plan, applied=applied, + feed={"input_tf_seconds": 60, "source_bytes_sha256": self.HASHES[0], + "source_values_sha256": self.HASHES[1], + "evaluated_source_values_sha256": self.HASHES[2]}, + calendar=swr.calendar_record("", "")) + + def test_r_maps_every_field_of_the_wire_table(self): + self.assertEqual(self.build(full_plan()), expected_window(full_plan(), self.HASHES)) + + def test_validation_only_counts_are_plans_with_no_curve_points(self): + got = self.build(full_plan(), applied=False) + self.assertEqual(got, expected_window(full_plan(), self.HASHES, applied=False)) + self.assertEqual((got["counts"]["equity_points"], got["counts"]["anchor_points"]), (0, 0)) + + def test_an_empty_window_is_valid_with_null_extents_and_indices(self): + got = self.build(empty_window_plan()) + self.assertEqual(got, expected_window(empty_window_plan(), self.HASHES)) + self.assertEqual(got["window"]["coverage"], "empty") + self.assertEqual(got["counts"]["equity_points"], 1) + self.assertIsNone(got["indices"]["fed_script_index_of_window_first"]) + + def test_shortfall_follows_available_less_than_requested_only(self): + plan = full_plan(available_script_bars=2, used_script_bars=2, trimmed_script_bars=0, + supplied_script_bars=8, fed_script_bars=8, shortfall=True, + trim_index=0, supplied_input_bars=9, fed_input_bars=9, + preroll_input_bars=3) + got = self.build(plan) + self.assertTrue(got["preroll"]["shortfall"]) + self.assertEqual(got["preroll"]["requested_script_bars"], N) + + def test_the_counting_equalities_are_checked(self): + for broken in ({"fed_input_bars": 8}, {"trim_index": 2}, {"fed_script_bars": 10}, + {"used_script_bars": 2}, {"shortfall": True}, {"window_script_bars": 5}): + with self.subTest(broken=broken), self.assertRaises(swr.SelectedWindowError) as caught: + self.build(full_plan(**broken)) + self.assertEqual(caught.exception.code, "engine_invariant") + + def test_lookback_is_not_produced_by_this_build_and_states_nothing(self): + # This build has no lookback producer, so R says status "unknown", a null + # declared count and a null has_unbounded_state ("not produced by this build", neither + # true nor false). The real builder, every plan shape, a run and validation-only alike. + shortfall = full_plan(available_script_bars=2, used_script_bars=2, trimmed_script_bars=0, + supplied_script_bars=8, fed_script_bars=8, shortfall=True, + trim_index=0, supplied_input_bars=9, fed_input_bars=9, + preroll_input_bars=3) + for name, plan in (("full", full_plan()), ("empty window", empty_window_plan()), + ("shortfall", shortfall)): + for applied in (True, False): + with self.subTest(plan=name, applied=applied): + look = self.build(plan, applied=applied)["lookback"] + self.assertEqual(look, NOT_PRODUCED_LOOKBACK) + self.assertEqual(list(look), ["declared_required_chart_bars", "status", + "has_unbounded_state"]) + self.assertEqual(look["status"], "unknown") + self.assertIsNone(look["declared_required_chart_bars"]) + self.assertIsNone(look["has_unbounded_state"]) # not True and not False + + def test_every_record_gets_its_own_lookback_object(self): + first = self.build(full_plan())["lookback"] + second = self.build(full_plan())["lookback"] + self.assertEqual(first, second) + self.assertIsNot(first, second) + first["status"] = "changed after the fact" + self.assertEqual(second, NOT_PRODUCED_LOOKBACK) + self.assertEqual(self.build(full_plan())["lookback"], NOT_PRODUCED_LOOKBACK) + + def test_calendar_defaults_and_report_shape(self): + self.assertEqual(swr.calendar_record("", ""), {"timezone": "UTC", "session": None}) + self.assertEqual(swr.calendar_record("America/New_York", "24x7"), + {"timezone": "America/New_York", "session": None}) + self.assertEqual(swr.calendar_record("", "0930-1600"), + {"timezone": "UTC", "session": "0930-1600"}) + self.assertEqual(swr.report_shape_full(7, 0), { + "version": 1, "name": "full/v1", "curve_point_budget": None, + "canonical_equity_points": 7, "emitted_equity_points": 7, "canonical_trades": 0, + "emitted_trades": 0, "retained_point_indices": None, "curve_decimated": False}) + + # Capitals the provenance does not establish: null (how an unresolved strategy() default is + # recorded), text and values that are no finite number. _established_number reads none of + # them as a capital, so the anchor equity has nothing to be compared with. + UNESTABLISHED_CAPITALS = (None, "lots", "", " ", True, False, float("nan"), float("inf"), + float("-inf"), "inf", "nan", "1e999", 10 ** 400, [], {}, [CAPITAL]) + + @staticmethod + def selected_curve(anchor_equity=CAPITAL): + return [{"time_ms": T, "equity": anchor_equity, "open_profit": 0.0}, + {"time_ms": T, "equity": CAPITAL + 1.0, "open_profit": 0.5}] + + def refused_curve(self, curve, bars, capital, fragment): + """check_selected_curve refuses with engine_invariant, and with the message of the check + that fires (so a row proves the intended check and not another one).""" + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.check_selected_curve(curve, T, bars, capital) + self.assertEqual(caught.exception.code, "engine_invariant") + self.assertIn(fragment, caught.exception.message) + + def test_the_selected_curve_is_checked_not_rewritten(self): + curve = self.selected_curve() + before = json.loads(json.dumps(curve)) + swr.check_selected_curve(curve, T, 1, CAPITAL) + swr.check_selected_curve(curve, T, 1, "1000000") # text that spells the capital + swr.check_selected_curve(curve, T, 1, 1000000) # an integer capital + self.assertEqual(curve, before) # nothing added, dropped or rewritten + self.assertEqual(set(curve[0]), {"time_ms", "equity", "open_profit"}) + wrong_anchor = [{**curve[0], "equity": CAPITAL - 1.0}, curve[1]] + extra_field = [{**curve[0], "point_kind": "window_anchor"}, curve[1]] + for name, bad, bars, capital, fragment in ( + ("length", curve, 2, CAPITAL, "points, the plan says"), + ("open profit", curve[1:], 0, CAPITAL + 1.0, + "does not start with the window anchor"), + ("anchor time", [{**curve[0], "time_ms": T - 1}], 0, CAPITAL, + "does not start with the window anchor"), + ("a per-point field", extra_field, 1, CAPITAL, "does not have exactly"), + ("known capital, unequal anchor", wrong_anchor, 1, CAPITAL, + "is not the initial capital"), + ("known text capital, unequal anchor", wrong_anchor, 1, "1000000", + "is not the initial capital"), + ("known integer capital, unequal anchor", wrong_anchor, 1, 1000000, + "is not the initial capital")): + with self.subTest(name=name): + self.refused_curve(bad, bars, capital, fragment) + + def test_no_established_capital_skips_only_the_comparison_with_the_anchor(self): + # An unresolved initial-capital provenance is not evidence that the native anchor is + # wrong, and the selected path has no refusal ordinary mode lacks: a missing, + # unreadable, boolean or non-finite capital is never a refusal. + curve = self.selected_curve() + before = json.loads(json.dumps(curve)) + for capital in self.UNESTABLISHED_CAPITALS: + with self.subTest(capital=repr(capital)[:40]): + self.assertIsNone(swr._established_number(capital)) + self.assertIsNone(swr.check_selected_curve(curve, T, 1, capital)) + self.assertEqual(curve, before) # no capital was written back or guessed + # With nothing to compare, ANY finite anchor value completes, and the very anchor that a + # known capital refuses completes (so the equality alone was skipped). + for equity in (CAPITAL - 1.0, 250_000.0, 0.0, -0.0, -5.0, 3, 1e300): + with self.subTest(anchor=equity): + swr.check_selected_curve(self.selected_curve(equity), T, 1, None) + self.refused_curve(self.selected_curve(CAPITAL - 1.0), 1, CAPITAL, + "is not the initial capital") + # An established capital is compared even when it spells a number as text, and a capital + # the provenance establishes as zero or negative is a capital like any other. + swr.check_selected_curve(self.selected_curve(0.0), T, 1, "0") + swr.check_selected_curve(self.selected_curve(-5.0), T, 1, -5.0) + self.refused_curve(self.selected_curve(1.0), 1, "0", "is not the initial capital") + + def test_every_other_check_still_applies_without_an_established_capital(self): + curve = self.selected_curve() + extra_field = [{**curve[0], "point_kind": "window_anchor"}, curve[1]] + for capital in (None, "lots", True, float("inf")): + for name, bad, bars, fragment in ( + ("length", curve, 2, "points, the plan says"), + ("open profit", curve[1:], 0, "does not start with the window anchor"), + ("anchor time", [{**curve[0], "time_ms": T - 1}], 0, + "does not start with the window anchor"), + ("a per-point field", extra_field, 1, "does not have exactly"), + ("a missing field", [{"time_ms": T, "open_profit": 0.0}, curve[1]], 1, + "does not have exactly")): + with self.subTest(capital=repr(capital), name=name): + self.refused_curve(bad, bars, capital, fragment) + + def test_a_malformed_actual_anchor_is_refused_whatever_the_capital(self): + # The anchor the engine produced must be a finite number (the report writes a non-finite + # double as null): not null, not a bool, not text, not NaN or an infinity, and not an int + # beyond binary64. An unresolved capital never excuses it, and neither does an equal- + # looking value: True == 1.0 must not pass for a capital of 1. + malformed = (None, float("nan"), float("inf"), float("-inf"), True, False, "1000000", + 10 ** 400, [CAPITAL], {"equity": CAPITAL}) + for equity in malformed: + for capital in (CAPITAL, "1000000", None, "lots", 1, 0): + with self.subTest(anchor=repr(equity)[:30], capital=repr(capital)): + self.refused_curve(self.selected_curve(equity), 1, capital, + "not a finite number") + + def test_the_three_placements_the_version_and_the_hashed_bytes_are_checked(self): + window = expected_window(full_plan(), ("a" * 64, "b" * 64, "c" * 64)) + provenance = {"schema_version": 2, "runtime": {"report_window": window}} + raw = json.dumps(provenance, separators=(",", ":")).encode() + fingerprint = {"token": base64.b64encode(raw).decode(), + "digest": "sha256:" + hashlib.sha256(raw).hexdigest(), + "provenance": provenance, "version": 2} + + def report(**over): + doc = {"report_window": json.loads(json.dumps(window)), + "applied_runtime": {"report_window": json.loads(json.dumps(window))}, + "fingerprint": json.loads(json.dumps(fingerprint))} + doc.update(over) + return doc + + swr.check_report_placements(report(), window) # equal content in all three places + other = {**window, "applied": False} + broken = ( + ("missing fingerprint", report(fingerprint=None)), + ("report_window", report(report_window=other)), + ("applied_runtime", report(applied_runtime={"report_window": other})), + ("no applied_runtime", report(applied_runtime=None)), + ("provenance runtime", report(fingerprint={ + **fingerprint, "provenance": {**provenance, "runtime": {"report_window": other}}})), + ("version", report(fingerprint={**fingerprint, "version": 1})), + ("schema_version", report(fingerprint={ + **fingerprint, "provenance": {**provenance, "schema_version": 1}})), + ("digest", report(fingerprint={**fingerprint, "digest": "sha256:" + "0" * 64})), + ("token", report(fingerprint={**fingerprint, "token": "not base64 !"})), + ("hashed R", report(fingerprint=self.refingerprint( + {"schema_version": 2, "runtime": {"report_window": other}}, window))), + ("hashed schema", report(fingerprint=self.refingerprint( + {"schema_version": 1, "runtime": {"report_window": window}}, window))), + ) + for name, doc in broken: + with self.subTest(name=name), self.assertRaises(swr.SelectedWindowError) as caught: + swr.check_report_placements(doc, window) + self.assertEqual(caught.exception.code, "engine_invariant") + + @staticmethod + def refingerprint(hashed, visible_window): + """A fingerprint whose visible provenance is right (it holds `visible_window`) and whose + hashed bytes, with a matching digest, are `hashed`.""" + raw = json.dumps(hashed, separators=(",", ":")).encode() + return {"token": base64.b64encode(raw).decode(), + "digest": "sha256:" + hashlib.sha256(raw).hexdigest(), "version": 2, + "provenance": {"schema_version": 2, + "runtime": {"report_window": visible_window}}} + + @staticmethod + def window_with_lookback(lookback): + """R as the builder writes it, with its lookback member replaced by `lookback`.""" + record = expected_window(full_plan(), ("a" * 64, "b" * 64, "c" * 64)) + record["lookback"] = lookback + return record + + @staticmethod + def report_carrying(record): + """A report that carries `record` as R in all three places AND in the hashed bytes, with + a matching digest, fingerprint version 2 and schema_version 2: every equality row of + check_report_placements holds, so only a rule about R's own content can refuse it.""" + provenance = {"schema_version": 2, "runtime": {"report_window": record}} + raw = json.dumps(provenance, separators=(",", ":")).encode() + return {"report_window": json.loads(json.dumps(record)), + "applied_runtime": {"report_window": json.loads(json.dumps(record))}, + "fingerprint": {"token": base64.b64encode(raw).decode(), + "digest": "sha256:" + hashlib.sha256(raw).hexdigest(), + "provenance": json.loads(json.dumps(provenance)), + "version": 2}} + + def test_the_documented_null_triple_passes_the_report_checker(self): + record = self.window_with_lookback(dict(NOT_PRODUCED_LOOKBACK)) + self.assertIsNone( + swr.check_report_placements(self.report_carrying(record), record)) + + def test_a_null_has_unbounded_state_is_valid_only_beside_unknown_and_a_null_count(self): + # A null beside any other status or beside a number is an + # invalid report. Every row carries the malformed R in all three places and in the + # hashed bytes, so the equalities hold and only the pairing rule can be what fires; the + # message proves it was. + null = None + invalid = ( + ("null beside sufficient_declared", + {"declared_required_chart_bars": null, "status": "sufficient_declared", + "has_unbounded_state": null}), + ("null beside insufficient", + {"declared_required_chart_bars": null, "status": "insufficient", + "has_unbounded_state": null}), + ("null beside a number", + {"declared_required_chart_bars": 4800, "status": "unknown", + "has_unbounded_state": null}), + ("null beside zero", + {"declared_required_chart_bars": 0, "status": "unknown", + "has_unbounded_state": null}), + ("null beside another status AND a number", + {"declared_required_chart_bars": 4800, "status": "sufficient_declared", + "has_unbounded_state": null}), + ("null beside a status spelled in other case", + {"declared_required_chart_bars": null, "status": "UNKNOWN", + "has_unbounded_state": null}), + ("null beside a null status", + {"declared_required_chart_bars": null, "status": null, + "has_unbounded_state": null}), + ("null beside an absent count (null is not omission)", + {"status": "unknown", "has_unbounded_state": null}), + ("null beside an absent status", + {"declared_required_chart_bars": null, "has_unbounded_state": null}), + ) + for name, lookback in invalid: + record = self.window_with_lookback(lookback) + with self.subTest(name=name), self.assertRaises(swr.SelectedWindowError) as caught: + swr.check_report_placements(self.report_carrying(record), record) + self.assertEqual(caught.exception.code, "engine_invariant") + self.assertIn("null is valid only beside status 'unknown' and a null " + "declared_required_chart_bars", caught.exception.message) + + def test_the_report_checker_asks_nothing_more_of_lookback_than_the_null_pairing(self): + # A build WITH a lookback producer (this one is not) emits a boolean beside any status + # and count. The checker does not invent a schema for them. + producer_shapes = ( + {"declared_required_chart_bars": 4800, "status": "sufficient_declared", + "has_unbounded_state": False}, + {"declared_required_chart_bars": 120, "status": "insufficient", + "has_unbounded_state": True}, + ) + for lookback in producer_shapes: + record = self.window_with_lookback(lookback) + with self.subTest(lookback=lookback): + self.assertIsNone( + swr.check_report_placements(self.report_carrying(record), record)) + + +class AbiAndNativeCountsTests(unittest.TestCase): + def test_struct_sizes_are_the_frozen_ones(self): + self.assertEqual((ctypes.sizeof(swr.SelectedWindowConfigC), + ctypes.sizeof(swr.SelectedWindowCountsC)), (24, 80)) + + def test_a_missing_export_or_version_is_window_mode_unsupported(self): + lib = FakeLib(full_plan(), selected_abi=False) + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.SelectedWindowAbi(lib) + self.assertEqual((caught.exception.code, caught.exception.code_args), + ("window_mode_unsupported", {"capability": "selected_window_v1"})) + lib = FakeLib(full_plan()) + lib.pf_selected_window_version = lambda: 2 + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.SelectedWindowAbi(lib) + self.assertEqual(caught.exception.code, "window_mode_unsupported") + + def test_configure_sends_the_window_and_maps_statuses(self): + lib = FakeLib(full_plan()) + abi = swr.SelectedWindowAbi(lib) + abi.configure(STATE, T, E) + self.assertEqual(lib.window_config, (24, 1, T, E)) + for status, code in ((-4, "out_of_memory"), (-2, "window_mode_unsupported"), + (-1, "harness_internal_error"), (-3, "harness_internal_error")): + lib.strategy_set_selected_window_v1 = lambda state, pointer, _s=status: _s + with self.subTest(status=status), self.assertRaises(swr.SelectedWindowError) as caught: + swr.SelectedWindowAbi(lib).configure(STATE, T, E) + self.assertEqual(caught.exception.code, code) + + def test_counts_read_back_and_a_missing_generation_is_an_invariant(self): + lib = FakeLib(full_plan()) + abi = swr.SelectedWindowAbi(lib) + self.assertEqual(abi.counts(STATE), lib.native_counts) + lib.strategy_selected_window_counts_v1 = lambda state, ref: -2 + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.SelectedWindowAbi(lib).counts(STATE) + self.assertEqual(caught.exception.code, "engine_invariant") + + def test_native_counts_must_equal_the_plan_and_the_observed_attempt(self): + plan = full_plan() + native = FakeLib(plan).native_counts + observation = {"run_generation": 5, "attempt_serial": 9, "attempt_generation": 5} + swr.check_native_counts(plan, native, observation) + for name in ("fed_input_bars", "fed_script_bars", "preroll_input_bars", + "preroll_script_bars", "window_input_bars", "window_script_bars"): + with self.subTest(name=name), self.assertRaises(swr.SelectedWindowError) as caught: + swr.check_native_counts(plan, {**native, name: native[name] + 1}, observation) + self.assertEqual(caught.exception.code, "engine_invariant") + for name in ("run_generation", "attempt_serial", "attempt_generation"): + with self.subTest(name=name), self.assertRaises(swr.SelectedWindowError): + swr.check_native_counts(plan, native, {**observation, name: 99}) + with self.assertRaises(swr.SelectedWindowError): + swr.check_native_counts(plan, {**native, name: 0}, {**observation, name: 0}) + + def test_capability_record_when_unsupported_claims_nothing(self): + got = swr.capabilities_record(False)["capabilities"] + self.assertEqual((got["selected_window_wire_version"], got["report_policy"], + got["fingerprint_version"], got["report_shapes"]), (0, None, None, [])) + on = swr.capabilities_record(True)["capabilities"] + self.assertEqual((on["selected_window_wire_version"], on["report_policy"], + on["fingerprint_version"], on["report_shapes"], + on["curve_point_budget"], on["results_digest_version"], + on["results_digest_platform"]), (1, "selected-window/v1", 2, + ["full/v1"], None, None, None)) + + def test_the_supported_record_declares_the_parity_revision_alone_never_the_full_one(self): + # A parity build (a prerelease build, not full 1.3 coverage) emits + # selected_window_contract_revisions == ["1.3-parity"]; a full build carries ["1.3"]; + # never both. The literals are written out here, not read from the module. + supported = { + "engine": "pineforge", + "capabilities": { + "selected_window_wire_version": 1, + "selected_window_contract_revisions": ["1.3-parity"], + "report_policy": "selected-window/v1", + "fingerprint_version": 2, + "report_shapes": ["full/v1"], + "curve_point_budget": None, + "results_digest_version": None, + "results_digest_platform": None, + }, + } + unsupported = json.loads(json.dumps(supported)) + unsupported["capabilities"].update({ + "selected_window_wire_version": 0, "selected_window_contract_revisions": [], + "report_policy": None, "fingerprint_version": None, "report_shapes": []}) + # Whole documents: wire version 1, policy, fingerprint 2 and every other member are what + # they were, and the declaration added no member. + self.assertEqual(swr.capabilities_record(True), supported) + self.assertEqual(swr.capabilities_record(False), unsupported) + revisions = ( + swr.capabilities_record(True)["capabilities"]["selected_window_contract_revisions"]) + self.assertEqual(revisions, ["1.3-parity"]) + self.assertEqual(len(revisions), 1) # a singleton ... + self.assertNotIn("1.3", revisions) # ... that is never the full revision + self.assertEqual( + swr.capabilities_record(False)["capabilities"]["selected_window_contract_revisions"], + []) # unsupported lists nothing at all + self.assertEqual(swr.CONTRACT_REVISIONS, ("1.3-parity",)) # the one constant + + def test_the_declared_revisions_are_a_fresh_list_of_the_one_constant(self): + first = swr.capabilities_record(True)["capabilities"]["selected_window_contract_revisions"] + first.append("1.3") # a caller that edits its copy changes neither the constant ... + self.assertEqual(swr.CONTRACT_REVISIONS, ("1.3-parity",)) + again = swr.capabilities_record(True)["capabilities"]["selected_window_contract_revisions"] + self.assertEqual(again, ["1.3-parity"]) # ... nor the next record + self.assertIsNot(first, again) + + +# --- admission: nothing is touched before it passes -------------------------- + +class AdmissionTests(unittest.TestCase): + """Refusal rows through run_json.main: poisoned feed, library, planner, constructor + and phase entries. Every refusal leaves every counter at zero.""" + + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.path = Path(self.dir.name) + self.ohlcv = self.path / "feed.csv" + write_ohlcv(self.ohlcv, csv_rows(10)) + + def refused(self, argv, code, detail, *, reads=0): + with Counters() as poison: + status, out, _ = run_main(argv) + counts = poison.counts() + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), (code, detail)) + zero = {name: 0 for name in counts} + zero["inventory_reads"] = reads + self.assertEqual(counts, zero) + + def test_primary_daily_weekly_and_their_aliases_name_the_canonical_token(self): + artifact, inventory = write_inventory(self.path, INSIDE) + for spelling, token in (("D", "D"), ("1D", "D"), ("W", "W"), ("1W", "W")): + with self.subTest(primary=spelling): + self.refused(selected_argv(artifact, self.ohlcv, inventory, script_tf=spelling), + "window_mode_unsupported", + {"capability": "selected_window_chart_timeframe", "timeframe": token}) + + def test_a_known_excluded_request_feed_names_its_token(self): + artifact, inventory = write_inventory( + self.path, [{"kind": "token", "timeframe": "1"}, {"kind": "token", "timeframe": "W"}]) + self.refused(selected_argv(artifact, self.ohlcv, inventory), "window_mode_unsupported", + {"capability": "selected_window_request_feed_timeframe", "timeframe": "W"}, + reads=2) + + def test_an_incomplete_inventory_has_the_capability_only(self): + artifact, inventory = write_inventory(self.path, [{"kind": "unknown"}]) + self.refused(selected_argv(artifact, self.ohlcv, inventory), "window_mode_unsupported", + INVENTORY_CAPABILITY, reads=2) # the helper hashes the artifact first + # no inventory given at all: the helper is asked for a path of None and refuses it + self.refused(selected_argv(artifact, self.ohlcv, None), "window_mode_unsupported", + INVENTORY_CAPABILITY, reads=1) + # an inventory bound to another artifact + artifact, inventory = write_inventory(self.path, INSIDE, artifact_sha="f" * 64) + self.refused(selected_argv(artifact, self.ohlcv, inventory), "window_mode_unsupported", + INVENTORY_CAPABILITY, reads=2) + + def test_an_input_override_that_resolves_outside_the_list_names_that_token(self): + artifact, inventory = write_inventory( + self.path, [{"kind": "input", "key": "HTF", "default": "1"}]) + self.refused(selected_argv(artifact, self.ohlcv, inventory, + extra=["--inputs", json.dumps({"HTF": "1W"})]), + "window_mode_unsupported", + {"capability": "selected_window_request_feed_timeframe", "timeframe": "W"}, + reads=2) + + def test_an_inventory_inside_the_list_proceeds_to_work(self): + artifact, inventory = write_inventory(self.path, INSIDE) + with Counters() as poison: + status, out, _ = run_main(selected_argv(artifact, self.ohlcv, inventory)) + counts = poison.counts() + self.assertEqual(status, 1) # the poisoned feed read is the expected end of this row + self.assertEqual(last_line(out)["code"], "harness_internal_error") + self.assertEqual((counts["load_bars"], counts["advance"], counts["inventory_reads"]), + (1, 1, 2)) + for name in ("load_symbol_feeds", "load_strategy", "create_strategy", + "SelectedPrimaryPlanner", "CDLL"): + self.assertEqual(counts[name], 0) + + def test_an_ordinary_run_never_reads_the_inventory_or_the_policy_or_the_planner(self): + artifact, inventory = write_inventory( + self.path, [{"kind": "token", "timeframe": "W"}]) + argv = ["--so", str(artifact), "--ohlcv", str(self.ohlcv), "--input-tf", "1", + "--script-tf", "1", "--request-feed-inventory", str(inventory)] + with Counters() as poison, mock.patch.object(rfi, "admit_selected_request") as admit: + status, out, _ = run_main(argv) + counts = poison.counts() + self.assertEqual(status, 1) # the ordinary body reached its (poisoned) feed read + self.assertEqual(admit.call_count, 0) + self.assertEqual((counts["load_bars"], counts["advance"], counts["inventory_reads"], + counts["SelectedPrimaryPlanner"], counts["CDLL"]), (1, 0, 0, 0, 0)) + + def test_unsupported_options_and_bad_requests_fail_before_any_work(self): + artifact, inventory = write_inventory(self.path, INSIDE) + base = selected_argv(artifact, self.ohlcv, inventory) + for extra, code, detail in ( + (["--report-shape", "compact/v1"], "report_shape_unsupported", + {"shape": "compact/v1"}), + (["--curve-point-budget", "2500"], "report_shape_request_invalid", + {"option": "curve_point_budget", "value": "2500"}), + (["--results-digest", "sha256/v1"], "results_digest_unsupported", + {"requested": "sha256/v1"})): + with self.subTest(extra=extra): + self.refused(base + extra, code, detail) + self.refused(selected_argv(artifact, self.ohlcv, inventory, script_tf=""), + "window_request_invalid", {"option": "script_tf"}) + self.refused(base + ["--trade-start-ms", "5"], "window_legacy_flag_conflict", + {"option": "trade_start_ms"}) + self.refused([a for a in base if a != "--report-policy" and a != "selected-window/v1"], + "window_request_invalid", {"option": "report_policy"}) + self.refused(["--so", str(artifact), "--ohlcv", str(self.ohlcv), "--report-shape", + "full/v1"], "report_shape_unsupported", {"shape": "full/v1"}) + + +# --- the selected run, end to end on fakes ---------------------------------- + +class SelectedRunTests(unittest.TestCase): + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.path = Path(self.dir.name) + self.events = [] + self.parent, self.child = socket.socketpair() + self.addCleanup(self.parent.close) + self.addCleanup(self.child.close) + self.rows = csv_rows(10) + self.ohlcv = self.path / "feed.csv" + write_ohlcv(self.ohlcv, self.rows) + self.artifact, self.inventory = write_inventory(self.path, INSIDE) + + def fresh_phase_socket(self): + """A new phase socketpair and an empty event list, for the next row of a loop.""" + self.events.clear() + self.parent.close() + self.child.close() + self.parent, self.child = socket.socketpair() + + def run_selected(self, lib, plan, *, extra=(), fail_phase=None, rows=None, writer=True, + patches=(), stdout_class=Stdout): + if rows is not None: + write_ohlcv(self.ohlcv, rows) + FakePlanner.reset(plan) + self.captured = {} + real_load = run_json.load_bars + + def load(path): + result = real_load(path) + self.captured["bars"] = result[0] + return result + + argv = selected_argv(self.artifact, self.ohlcv, self.inventory, + extra=list(extra) + (["--run-phase-fd", str(self.child.fileno())] + if writer else [])) + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(run_json, "load_strategy", lambda p: lib)) + stack.enter_context(mock.patch.object(run_json, "load_bars", load)) + stack.enter_context(mock.patch.object(swp, "SelectedPrimaryPlanner", FakePlanner)) + stack.enter_context(mock.patch.object( + run_json, "normalize_release_provenance", lambda p, c, r, k: p)) + stack.enter_context(mock.patch.object( + rpt, "RunPhaseWriter", recording_writer(self.events, fail_phase))) + for patch in patches: + stack.enter_context(patch) + return run_main(argv, self.events, stdout_class) + + def test_a_selected_run_end_to_end(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + status, out, err = self.run_selected(lib, plan) + self.assertEqual((status, err), (0, "")) + report = json.loads(out) + # --- the feed: planned once, trimmed by pointer offset, retained rows fed + bars = self.captured["bars"] + planner_bars, planner_count, planner_kwargs = FakePlanner.calls[0] + self.assertIs(planner_bars, bars) + self.assertEqual(planner_count, 10) + self.assertEqual( + {k: planner_kwargs[k] for k in ("start_ms", "end_ms", "fed_start_ms", "preroll_bars", + "input_tf", "script_tf", "chart_timezone", + "engine_timezone", "session", "feed_tolerant")}, + {"start_ms": T, "end_ms": E, "fed_start_ms": F, "preroll_bars": N, "input_tf": "1", + "script_tf": "1", "chart_timezone": "", "engine_timezone": "", "session": "", + "feed_tolerant": True}) + self.assertEqual(len(FakePlanner.constructed), 1) + self.assertTrue(FakePlanner.constructed[0].endswith("/lib/libpineforge_window_plan.so")) + self.assertTrue(Path(FakePlanner.constructed[0]).is_absolute()) + rows_seen, count_seen = lib.run_args + self.assertEqual(count_seen, 9) + self.assertEqual(ctypes.addressof(rows_seen), + ctypes.addressof(bars) + ctypes.sizeof(run_json.BarC)) + # --- R, in the three places, identical, with the expected hashes + hashes = (hashlib.sha256(self.ohlcv.read_bytes()).hexdigest(), values_hash(self.rows), + values_hash(self.rows[1:])) + window = expected_window(plan, hashes) + self.assertEqual(report["report_window"], window) + self.assertEqual(report["applied_runtime"]["report_window"], window) + self.assertEqual(report["fingerprint"]["provenance"]["runtime"]["report_window"], window) + # --- the lookback triple of a build with no lookback producer, the same + # in all three copies: unknown, null, null; never true, never false + for copy in (report["report_window"], report["applied_runtime"]["report_window"], + report["fingerprint"]["provenance"]["runtime"]["report_window"]): + self.assertEqual(copy["lookback"], NOT_PRODUCED_LOOKBACK) + self.assertIsNone(copy["lookback"]["has_unbounded_state"]) + # --- fingerprint v2, report shape, timing, evaluated-fed legacy fields + self.assertEqual(report["fingerprint"]["version"], 2) + self.assertEqual(report["fingerprint"]["provenance"]["schema_version"], 2) + raw_token = base64.b64decode(report["fingerprint"]["token"]) + token = raw_token.decode("utf-8") + self.assertIn('"schema_version":2', token) + # --- the hashed bytes keep the same triple, and the digest is the hash of those bytes + hashed = json.loads(token) + self.assertEqual(hashed["runtime"]["report_window"]["lookback"], NOT_PRODUCED_LOOKBACK) + self.assertIn('"has_unbounded_state":null', token) + self.assertNotIn('"has_unbounded_state":true', token) + self.assertNotIn('"has_unbounded_state":false', token) + self.assertEqual(report["fingerprint"]["digest"], + "sha256:" + hashlib.sha256(raw_token).hexdigest()) + # --- the capability revision is a declaration of the probe only: no literal of it is in + # the report or the hashed fingerprint bytes (the phase records are checked below) + self.assertNotIn("1.3-parity", out) + self.assertNotIn("1.3-parity", token) + self.assertEqual(report["report_window"]["wire_version"], 1) + self.assertEqual(report["report_shape"], swr.report_shape_full(7, 0)) + timing = report["diagnostics"]["phase_timing"] + self.assertEqual((timing["version"], timing["results_digest_ms"]), (1, None)) + self.assertGreaterEqual(timing["execution_ms"], 0) + self.assertEqual((report["input"]["bars"], report["input"]["first_ts"], + report["input"]["last_ts"]), + (9, F + MINUTE, F + 9 * MINUTE)) + self.assertEqual(report["summary"]["bars_processed"], 9) + self.assertEqual(report["diagnostics"]["input_bars_processed"], 9) + # --- the native curve is serialized as it is: three fields, anchor first + self.assertEqual(len(report["equity_curve"]), 7) + self.assertTrue(all(set(p) == {"time_ms", "equity", "open_profit"} + for p in report["equity_curve"])) + self.assertEqual(report["equity_curve"][0]["time_ms"], T) + # --- order: selection configured before the run and cleared after, observer detached + names = [c[0] for c in lib.calls] + self.assertEqual(lib.window_config, None) + self.assertLess(names.index("set_selected_window"), names.index("run_backtest_full")) + self.assertLess(names.index("set_observer"), names.index("run_backtest_full")) + tail = names[names.index("run_backtest_full"):] + self.assertEqual(tail[-4:], ["set_observer", "set_selected_window", "report_free", + "strategy_free"]) + self.assertIsNone(lib.observer_descriptor) + self.assertEqual(lib.calls[names.index("set_selected_window")][1:], (T, E)) + # --- phases: the observer announced result_assembly INSIDE the native call; completed + # follows the flush; the receipt names the whole record handed over + records = phase_records(self.parent) + self.assertEqual([r["phase"] for r in records], + ["preflight", "execution", "result_assembly", "serialization", + "completed"]) + self.assertEqual([r["sequence"] for r in records], [1, 2, 3, 4, 5]) + self.assertEqual({r["version"] for r in records}, {1}) # phase version 1, unchanged + self.assertNotIn("1.3-parity", json.dumps(records)) # no revision literal in a phase + self.assertEqual(lib.observer_rc, 0) + self.assertEqual(lib.receipt[2], 1) + self.assertEqual(lib.receipt[0], lib.receipt[1]) + ev = self.events + self.assertLess(ev.index("advance:execution"), ev.index("native.start")) + self.assertLess(ev.index("native.start"), ev.index("advance:result_assembly")) + self.assertLess(ev.index("advance:result_assembly"), ev.index("native.end")) + self.assertLess(ev.index("native.end"), ev.index("advance:serialization")) + self.assertLess(ev.index("advance:serialization"), ev.index("stdout.write")) + self.assertLess(max(i for i, e in enumerate(ev) if e == "stdout.flush"), + ev.index("advance:completed")) + + def test_an_empty_window_is_a_valid_anchor_only_run(self): + plan = empty_window_plan() + lib = FakeLib(plan, self.events) + rows = csv_rows(4) + status, out, _ = self.run_selected(lib, plan, rows=rows) + self.assertEqual(status, 0) + report = json.loads(out) + hashes = (hashlib.sha256(self.ohlcv.read_bytes()).hexdigest(), values_hash(rows), + values_hash(rows[1:])) + self.assertEqual(report["report_window"], expected_window(plan, hashes)) + for copy in (report["report_window"], report["applied_runtime"]["report_window"], + report["fingerprint"]["provenance"]["runtime"]["report_window"]): + self.assertEqual(copy["lookback"], NOT_PRODUCED_LOOKBACK) # an empty window too + self.assertEqual(len(report["equity_curve"]), 1) + self.assertEqual((report["trades"], report["input"]["bars"]), ([], 3)) + + def test_validation_only_runs_nothing_and_reports_applied_false(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + loader = mock.Mock(side_effect=PoisonTripped("load_strategy")) + writer = mock.Mock(side_effect=PoisonTripped("RunPhaseWriter")) + status, out, _ = self.run_selected( + lib, plan, extra=["--validate-window-only"], writer=False, + patches=[mock.patch.object(run_json, "load_strategy", loader), + mock.patch.object(rpt, "RunPhaseWriter", writer)]) + self.assertEqual(status, 0) + doc = json.loads(out) + self.assertEqual(set(doc), {"engine", "validation_only", "report_window"}) + self.assertTrue(doc["validation_only"]) + self.assertFalse(doc["report_window"]["applied"]) + self.assertEqual((doc["report_window"]["counts"]["equity_points"], + doc["report_window"]["counts"]["anchor_points"]), (0, 0)) + self.assertEqual((loader.call_count, writer.call_count, lib.calls), (0, 0, [])) + # The validation-only R is the real builder's: the whole record, and in it the same + # lookback triple as a run, not a different or an invented one. + hashes = (hashlib.sha256(self.ohlcv.read_bytes()).hexdigest(), values_hash(self.rows), + values_hash(self.rows[1:])) + self.assertEqual(doc["report_window"], expected_window(plan, hashes, applied=False)) + self.assertEqual(doc["report_window"]["lookback"], NOT_PRODUCED_LOOKBACK) + self.assertIsNone(doc["report_window"]["lookback"]["has_unbounded_state"]) + + def test_a_planner_refusal_ends_the_run_before_any_strategy_library(self): + plan = full_plan(status=3, bound=0, value_ms=T + 1, previous_boundary_ms=T, + next_boundary_ms=T + MINUTE) + loader = mock.Mock(side_effect=PoisonTripped("load_strategy")) + status, out, _ = self.run_selected( + FakeLib(plan, self.events), plan, + patches=[mock.patch.object(run_json, "load_strategy", loader)]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]["bound"]), ("window_boundary_unaligned", "start")) + self.assertEqual(loader.call_count, 0) + self.assertNotIn("advance:execution", self.events) + + def test_a_missing_planner_library_is_window_mode_unsupported(self): + plan = full_plan() + + class Missing: + def __init__(self, path): + raise swp.PlanBridgeError("window_mode_unsupported", "no helper", + {"capability": "selected_window_planner_v1"}) + + status, out, _ = self.run_selected( + FakeLib(plan, self.events), plan, + patches=[mock.patch.object(swp, "SelectedPrimaryPlanner", Missing)]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), + ("window_mode_unsupported", {"capability": "selected_window_planner_v1"})) + + def test_the_callbacks_retained_failure_is_raised_before_any_report_access(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + spy = mock.Mock(side_effect=PoisonTripped("build_report_dict")) + status, out, _ = self.run_selected( + lib, plan, fail_phase="result_assembly", + patches=[mock.patch.object(run_json, "build_report_dict", spy)]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual(line["code"], "harness_internal_error") + self.assertEqual((lib.observer_rc, spy.call_count), (-1, 0)) + names = [c[0] for c in lib.calls] + self.assertEqual(names[-4:], ["set_observer", "set_selected_window", "report_free", + "strategy_free"]) + self.assertNotIn("advance:completed", self.events) + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution"]) + + def test_an_engine_that_never_announces_the_end_of_execution_is_a_harness_fault(self): + plan = full_plan() + lib = FakeLib(plan, self.events, call_observer=False) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + self.assertEqual(last_line(out)["code"], "harness_internal_error") + self.assertNotIn("advance:result_assembly", self.events) # never emulated afterwards + + def test_native_counts_that_differ_from_the_plan_are_an_engine_invariant(self): + plan = full_plan() + for counts, observation in (({"window_script_bars": 7}, {}), + ({}, {"attempt_serial": 10})): + with self.subTest(counts=counts, observation=observation): + self.events.clear() + self.parent.close() + self.parent, self.child = socket.socketpair() + lib = FakeLib(plan, self.events, counts=counts, observation=observation) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + self.assertEqual(last_line(out)["code"], "engine_invariant") + self.assertNotIn("report_window", out) + self.assertEqual([c[0] for c in lib.calls][-2:], ["report_free", "strategy_free"]) + + def test_a_native_curve_of_the_wrong_length_is_an_engine_invariant(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + lib.plan = {**plan, "window_script_bars": 5} # the fake engine curve has 6 points + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + self.assertEqual(last_line(out)["code"], "engine_invariant") + + def test_a_library_without_the_selected_window_abi_is_refused_typed(self): + plan = full_plan() + lib = FakeLib(plan, self.events, selected_abi=False) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), + ("window_mode_unsupported", {"capability": "selected_window_v1"})) + self.assertEqual([c[0] for c in lib.calls if c[0] == "strategy_create"], []) + + def test_a_library_without_the_observer_is_refused_typed_and_still_cleaned_up(self): + plan = full_plan() + lib = FakeLib(plan, self.events, observer_abi=False) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), + ("window_mode_unsupported", {"capability": "execution_observer_v1"})) + self.assertEqual([c[0] for c in lib.calls][-2:], ["report_free", "strategy_free"]) + self.assertNotIn("run_backtest_full", [c[0] for c in lib.calls]) + + def test_a_run_without_a_phase_descriptor_still_tracks_the_phases_internally(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + status, out, _ = self.run_selected(lib, plan, writer=False) + self.assertEqual(status, 0) + self.assertEqual(lib.receipt, (0, 0, 0)) # the receipt of a writer that exports nothing + self.assertIn("phase_timing", json.loads(out)["diagnostics"]) + + # --- fail-closed fingerprint, anchor equity, post-boundary classification -------------- + + def test_a_provenance_failure_never_yields_a_report_or_a_null_fingerprint(self): + plan = full_plan() + for error, reason in ((ValueError("bad provenance"), "exception"), + (MemoryError(), "resource"), (OSError(5, "I/O error"), "io")): + with self.subTest(error=type(error).__name__): + self.fresh_phase_socket() + lib = FakeLib(plan, self.events) + status, out, _ = self.run_selected(lib, plan, patches=[ + mock.patch.object(run_json, "build_fingerprint", + mock.Mock(side_effect=error))]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"], line["error"]), + ("report_post_execution_failed", + {"phase": "result_assembly", "reason": reason}, + run_json.POST_EXECUTION_TEXT)) + self.assertEqual(len(out.strip().splitlines()), 1) # no report, no 2nd document + self.assertNotIn("fingerprint", out) + self.assertNotIn("advance:serialization", self.events) + self.assertNotIn("advance:completed", self.events) + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution", "result_assembly"]) + self.assertEqual([c[0] for c in lib.calls][-2:], ["report_free", "strategy_free"]) + + def test_a_typed_fault_in_the_provenance_keeps_its_own_code(self): + plan = full_plan() + typed = swr.SelectedWindowError("engine_invariant", "injected typed fault") + lib = FakeLib(plan, self.events) + status, out, _ = self.run_selected(lib, plan, patches=[ + mock.patch.object(run_json, "normalize_release_provenance", + mock.Mock(side_effect=typed))]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["error"]), + ("engine_invariant", "injected typed fault")) + + def test_a_fingerprint_whose_digest_is_not_its_hash_is_refused_before_serialization(self): + plan = full_plan() + real = run_json.build_fingerprint + + def corrupted(provenance): + fingerprint = real(provenance) + fingerprint["digest"] = "sha256:" + "0" * 64 + return fingerprint + + lib = FakeLib(plan, self.events) + status, out, _ = self.run_selected(lib, plan, patches=[ + mock.patch.object(run_json, "build_fingerprint", corrupted)]) + self.assertEqual((status, last_line(out)["code"]), (1, "engine_invariant")) + self.assertNotIn("advance:serialization", self.events) + + def test_an_anchor_that_is_not_the_initial_capital_is_refused_before_serialization(self): + plan = full_plan() + lib = FakeLib(plan, self.events, anchor_equity=CAPITAL - 1.0) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual((status, last_line(out)["code"]), (1, "engine_invariant")) + self.assertIn("is not the initial capital", last_line(out)["error"]) + self.assertNotIn("report_window", out) + self.assertNotIn("advance:serialization", self.events) + + # --- an initial capital the provenance leaves unresolved ------------------------------ + # + # The selected path keeps an unresolved initial-capital provenance unresolved, as ordinary + # mode does, and skips only the comparison that cannot be made; there is no selected-only + # refusal. The native anchor derives from the engine's own initial capital, so an + # unresolved SOURCE provenance is not evidence that the anchor is wrong. + # + # The normalizer here is a stand-in and the engine a fake; ordinary mode against selected + # mode on the real engine (same source, same feed bytes, same inputs) is not covered here. + + def test_an_unresolved_initial_capital_completes_and_stays_unresolved(self): + plan = full_plan() + seen = {} + anchor = 250_000.0 # not the seed capital: a capital copied from the anchor would show + lib = FakeLib(plan, self.events, anchor_equity=anchor) + status, out, err = self.run_selected(lib, plan, patches=[ + mock.patch.object(run_json, "normalize_release_provenance", + unresolved_capital_normalizer(seen))]) + self.assertEqual((status, err), (0, "")) + report = json.loads(out) + # Completed, as a v2 report whose R is in all three places. + hashes = (hashlib.sha256(self.ohlcv.read_bytes()).hexdigest(), values_hash(self.rows), + values_hash(self.rows[1:])) + window = expected_window(plan, hashes) + self.assertEqual(report["report_window"], window) + self.assertEqual(report["applied_runtime"]["report_window"], window) + provenance = report["fingerprint"]["provenance"] + self.assertEqual(provenance["runtime"]["report_window"], window) + self.assertEqual((report["fingerprint"]["version"], provenance["schema_version"]), (2, 2)) + # The provenance still says "unresolved", in the report and in the hashed bytes. + self.assertIsNone(provenance["strategy"]["initial_capital"]) + self.assertEqual(provenance["strategy_resolution"], UNRESOLVED_RESOLUTION) + hashed = json.loads(base64.b64decode(report["fingerprint"]["token"]).decode("utf-8")) + self.assertIsNone(hashed["strategy"]["initial_capital"]) + self.assertEqual(hashed["strategy_resolution"], UNRESOLVED_RESOLUTION) + self.assertEqual(hashed["schema_version"], 2) + self.assertEqual(hashed["runtime"]["report_window"], window) + # Not mutated and not invented: the provenance is what the normalizer returned plus the + # version key run_json always adds, and the native anchor is untouched. + self.assertEqual(provenance, {**seen["normalized"], "schema_version": 2}) + self.assertIsNone(seen["normalized"]["strategy"]["initial_capital"]) + self.assertEqual(len(report["equity_curve"]), 7) + self.assertEqual(report["equity_curve"][0], + {"time_ms": T, "equity": anchor, "open_profit": 0.0}) + # Phases: serialization, and completed after the flush. + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution", "result_assembly", "serialization", + "completed"]) + + def test_ordinary_mode_records_an_unresolved_initial_capital_the_same_way(self): + # The representation the selected run keeps above is the ordinary run's: ordinary mode + # never refuses on it either. Same stand-in normalizer, so this shows no more than that + # both bodies hand the normalizer's result on unchanged. + lib = FakeLib(full_plan(fed_input_bars=10, fed_script_bars=10, window_script_bars=10)) + lib.run_backtest_full = lambda *a: OrdinaryAndCapabilityTests._fill(a[-1]._obj) + argv = ["--so", str(self.artifact), "--ohlcv", str(self.ohlcv), "--input-tf", "1", + "--script-tf", "1"] + with mock.patch.object(run_json, "load_strategy", lambda p: lib), \ + mock.patch.object(run_json, "normalize_release_provenance", + unresolved_capital_normalizer()): + status, out, _ = run_main(argv) + self.assertEqual(status, 0) + provenance = json.loads(out)["fingerprint"]["provenance"] + self.assertIsNone(provenance["strategy"]["initial_capital"]) + self.assertEqual(provenance["strategy_resolution"], UNRESOLVED_RESOLUTION) + self.assertNotIn("schema_version", provenance) + + def test_an_established_capital_is_still_compared_with_the_native_anchor(self): + plan = full_plan() + override = ["--overrides", json.dumps({"initial_capital": "250000"})] + for anchor, completes in ((250_000.0, True), (CAPITAL, False), (250_000.5, False)): + with self.subTest(anchor=anchor): + self.fresh_phase_socket() + lib = FakeLib(plan, self.events, anchor_equity=anchor) + status, out, _ = self.run_selected(lib, plan, extra=override) + if completes: + self.assertEqual(status, 0) + strategy = json.loads(out)["fingerprint"]["provenance"]["strategy"] + self.assertEqual(strategy["initial_capital"], "250000") # as given, unchanged + continue + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual(line["code"], "engine_invariant") + self.assertIn("is not the initial capital", line["error"]) + self.assertEqual(len(out.strip().splitlines()), 1) + self.assertNotIn("report_window", out) + self.assertNotIn("advance:serialization", self.events) + self.assertNotIn("advance:completed", self.events) + self.assertEqual([c[0] for c in lib.calls][-2:], ["report_free", "strategy_free"]) + + def test_a_malformed_actual_anchor_is_refused_even_when_the_capital_is_unresolved(self): + plan = full_plan() + # A non-finite native anchor reaches the report as null (build_report_dict writes every + # non-finite double that way); it must not be written out as a valid-looking number. + for anchor in (float("nan"), float("inf"), float("-inf")): + for unresolved in (False, True): + with self.subTest(anchor=anchor, unresolved=unresolved): + self.fresh_phase_socket() + lib = FakeLib(plan, self.events, anchor_equity=anchor) + patches = [mock.patch.object(run_json, "normalize_release_provenance", + unresolved_capital_normalizer()) + ] if unresolved else [] + status, out, _ = self.run_selected(lib, plan, patches=patches) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual(line["code"], "engine_invariant") + self.assertIn("not a finite number", line["error"]) + self.assertEqual(len(out.strip().splitlines()), 1) # no report, no 2nd line + self.assertNotIn("report_window", out) + self.assertNotIn("advance:serialization", self.events) + self.assertNotIn("advance:completed", self.events) + self.assertEqual([c[0] for c in lib.calls][-2:], + ["report_free", "strategy_free"]) + + def test_unclassified_failures_after_the_boundary_are_report_post_execution_failed(self): + plan = full_plan() + for error, reason in ((RuntimeError("boom"), "exception"), (MemoryError(), "resource"), + (OSError(28, "No space left on device"), "io")): + with self.subTest(error=type(error).__name__): + self.fresh_phase_socket() + lib = FakeLib(plan, self.events) + status, out, _ = self.run_selected(lib, plan, patches=[ + mock.patch.object(run_json, "build_report_dict", + mock.Mock(side_effect=error))]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), + ("report_post_execution_failed", + {"phase": "result_assembly", "reason": reason})) + self.assertEqual(len(out.strip().splitlines()), 1) + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution", "result_assembly"]) + + def test_a_failure_after_serialization_began_names_the_serialization_phase(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + status, out, _ = self.run_selected(lib, plan, patches=[ + mock.patch.object(swr, "report_shape_full", lambda points, trades: {"bad": {1, 2}})]) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), + ("report_post_execution_failed", + {"phase": "serialization", "reason": "exception"})) + self.assertEqual(len(out.strip().splitlines()), 1) # nothing of the report was written + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution", "result_assembly", "serialization"]) + self.assertNotIn("advance:completed", self.events) + + def test_the_same_error_before_the_boundary_is_not_a_post_execution_failure(self): + plan = full_plan() + lib = FakeLib(plan, self.events, raise_before_observer=RuntimeError("early")) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual(line["code"], "harness_internal_error") + self.assertIn("RuntimeError", line["error"]) + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution"]) + self.assertEqual([c[0] for c in lib.calls][-2:], ["report_free", "strategy_free"]) + + def test_engine_failures_are_classified_by_the_real_phase(self): + plan = full_plan() + after = {"phase": "result_assembly"} + cases = ( # (engine code, observer called, expected code, expected args) + ("out_of_memory", True, "report_post_execution_failed", {**after, "reason": "resource"}), + ("", True, "report_post_execution_failed", {**after, "reason": "exception"}), + ("engine_invariant", True, "engine_invariant", {}), + ("out_of_memory", False, "out_of_memory", {}), + ("strategy_runtime_error", False, "strategy_runtime_error", {}), + ("", False, "engine_unclassified_error", {}), + ) + for code, observed, expected_code, expected_args in cases: + with self.subTest(code=code, observer=observed): + self.fresh_phase_socket() + lib = FakeLib(plan, self.events, call_observer=observed, + engine_failure=(code, "the engine said so")) + status, out, _ = self.run_selected(lib, plan) + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), (expected_code, expected_args)) + self.assertEqual(len(out.strip().splitlines()), 1) + + def test_partial_stdout_emits_no_second_document_and_keeps_the_terminal_phase(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + status, out, err = self.run_selected(lib, plan, stdout_class=PartialStdout) + self.assertEqual(status, 1) + self.assertNotIn('"engine":"pineforge","error"', out) # no failure line follows + with self.assertRaises(ValueError): + json.loads(out) # what reached stdout is half a document, and only that + self.assertIn("report_post_execution_failed (phase serialization, reason io)", err) + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution", "result_assembly", "serialization"]) + self.assertNotIn("advance:completed", self.events) + + def test_a_completed_handoff_failure_is_a_harness_fault_and_never_a_success(self): + plan = full_plan() + lib = FakeLib(plan, self.events) + status, out, err = self.run_selected(lib, plan, fail_phase="completed") + self.assertEqual(status, 1) + report = json.loads(out) # one document only: the whole report, no failure line after it + self.assertTrue(report["report_window"]["applied"]) + self.assertIn("harness_internal_error", err) + self.assertEqual([r["phase"] for r in phase_records(self.parent)], + ["preflight", "execution", "result_assembly", "serialization"]) + + def test_a_managed_ordinary_run_is_classified_alike_and_a_plain_one_is_not(self): + plan = full_plan() + argv = ["--so", str(self.artifact), "--ohlcv", str(self.ohlcv), "--input-tf", "1", + "--script-tf", "1"] + for managed in (True, False): + with self.subTest(managed=managed): + self.fresh_phase_socket() + lib = FakeLib(plan, self.events) + extra = ["--run-phase-fd", str(self.child.fileno())] if managed else [] + with contextlib.ExitStack() as stack: + stack.enter_context(mock.patch.object(run_json, "load_strategy", + lambda p, _l=lib: _l)) + stack.enter_context(mock.patch.object( + rpt, "RunPhaseWriter", recording_writer(self.events))) + stack.enter_context(mock.patch.object( + run_json, "build_report_dict", + mock.Mock(side_effect=RuntimeError("boom")))) + status, out, _ = run_main(argv + extra, self.events) + self.assertEqual(status, 1) + line = last_line(out) + if managed: + self.assertEqual((line["code"], line["args"]), + ("report_post_execution_failed", + {"phase": "result_assembly", "reason": "exception"})) + else: + self.assertEqual(line["code"], "harness_internal_error") + + +class OrdinaryAndCapabilityTests(unittest.TestCase): + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.path = Path(self.dir.name) + + def test_the_ordinary_report_gains_no_selected_window_field(self): + ohlcv = self.path / "feed.csv" + write_ohlcv(ohlcv, csv_rows(10)) + plan = full_plan(fed_input_bars=10, fed_script_bars=10, window_script_bars=10) + lib = FakeLib(plan) + lib.run_backtest_full = lambda *a: self._fill(a[-1]._obj) + with mock.patch.object(run_json, "load_strategy", lambda p: lib), \ + mock.patch.object(run_json, "normalize_release_provenance", lambda p, c, r, k: p): + status, out, _ = run_main(["--so", "x.so", "--ohlcv", str(ohlcv), "--input-tf", "1", + "--script-tf", "1"]) + self.assertEqual(status, 0) + report = json.loads(out) + for key in ("report_window", "report_shape"): + self.assertNotIn(key, report) + self.assertNotIn("report_window", report["applied_runtime"]) + self.assertNotIn("phase_timing", report["diagnostics"]) + self.assertNotIn("version", report["fingerprint"]) + self.assertNotIn("schema_version", report["fingerprint"]["provenance"]) + self.assertEqual((report["input"]["bars"], report["input"]["first_ts"]), (10, F)) + # An ordinary report gains no R and so no lookback record of any kind (the lookback + # member belongs to R only): not in the report, not in the hashed bytes. + self.assertNotIn("lookback", out) + self.assertNotIn("has_unbounded_state", out) + self.assertNotIn(b"lookback", base64.b64decode(report["fingerprint"]["token"])) + + @staticmethod + def _fill(report): + report.input_tf_seconds = 60 + report.script_tf_seconds = 60 + report.script_tf_ratio = 1 + + def test_capabilities_probe_reports_a_supported_runtime_without_an_ohlcv(self): + lib = FakeLib(full_plan()) + FakePlanner.reset(None) + with mock.patch.object(run_json, "load_strategy", lambda p: lib), \ + mock.patch.object(swp, "SelectedPrimaryPlanner", FakePlanner): + status, out, _ = run_main(["--so", "x.so", "--capabilities-json"]) + self.assertEqual((status, json.loads(out)), (0, swr.capabilities_record(True))) + self.assertEqual(lib.calls, []) # nothing was constructed or run + # The probe's real stdout carries the parity declaration alone, not the full revision. + printed = json.loads(out)["capabilities"] + self.assertEqual(printed["selected_window_contract_revisions"], ["1.3-parity"]) + self.assertIn('"selected_window_contract_revisions":["1.3-parity"]', out) + self.assertNotIn('"1.3"', out) + self.assertEqual((printed["selected_window_wire_version"], printed["fingerprint_version"]), + (1, 2)) + + def test_capabilities_probe_reports_zero_when_any_piece_is_missing(self): + for lib in (FakeLib(full_plan(), selected_abi=False), FakeLib(full_plan(), observer_abi=False)): + with mock.patch.object(run_json, "load_strategy", lambda p, _l=lib: _l), \ + mock.patch.object(swp, "SelectedPrimaryPlanner", FakePlanner): + status, out, _ = run_main(["--so", "x.so", "--capabilities-json"]) + self.assertEqual((status, json.loads(out)), (0, swr.capabilities_record(False))) + # Not supported: no revision is listed at all, so neither "1.3-parity" nor "1.3". + self.assertEqual( + json.loads(out)["capabilities"]["selected_window_contract_revisions"], []) + self.assertNotIn("1.3", out) + + def test_the_trusted_planner_path_is_absolute_under_the_installation_prefix(self): + path = Path(run_json.trusted_planner_path()) + self.assertTrue(path.is_absolute()) + self.assertEqual((path.name, path.parent.name), ("libpineforge_window_plan.so", "lib")) + self.assertEqual(path.parent.parent, Path(run_json.__file__).resolve().parent.parent) + + +class NumericBoundTests(unittest.TestCase): + """Oversized decimal request numbers are typed request errors: the length is tested before + any int(), so Python's integer digit-limit ValueError (a harness fault) is unreachable. + Recognising a timeframe token stays unbounded; only the numeric conversion is bounded.""" + HUGE = "9" * 5000 # over the 4300 digits int() accepts by default + + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.path = Path(self.dir.name) + self.ohlcv = self.path / "feed.csv" + write_ohlcv(self.ohlcv, csv_rows(10)) + self.artifact, self.inventory = write_inventory(self.path, INSIDE) + + def argv_with(self, flag, value): + argv = selected_argv(self.artifact, self.ohlcv, self.inventory) + argv[argv.index(flag) + 1] = value + return argv + + def test_every_request_number_is_bounded_by_length_before_int(self): + for option in ("window_start_ms", "window_end_ms", "preroll_bars", "fed_start_ms"): + for text in (self.HUGE, "-" + self.HUGE, "9" * 17, "-" + "9" * 17): + with self.subTest(option=option, length=len(text)), \ + self.assertRaises(swr.SelectedWindowError) as caught: + swr.validate_request(**{**ValidationTests.GOOD, option: text}) + self.assertEqual((caught.exception.code, caught.exception.code_args), + ("window_request_invalid", {"option": option})) + + def test_the_largest_i53_numbers_are_still_accepted(self): + got = swr.validate_request(**{**ValidationTests.GOOD, "window_start_ms": str(2 ** 53 - 2), + "window_end_ms": str(2 ** 53 - 1), + "fed_start_ms": str(-(2 ** 53 - 1)), "preroll_bars": "0"}) + self.assertEqual((got.start_ms, got.end_ms, got.fed_start_ms), + (2 ** 53 - 2, 2 ** 53 - 1, -(2 ** 53 - 1))) + + def test_token_recognition_is_unbounded_and_its_conversion_is_bounded(self): + token = "1" + "0" * 5000 + self.assertEqual(swr.canonical_timeframe(token), token) # unchanged recognition + self.assertIsNone(swr.timeframe_seconds(token)) + self.assertIsNone(swr.timeframe_seconds("9999999999999999")) # over u53 seconds + self.assertEqual(swr.timeframe_seconds("100"), 6000) + with self.assertRaises(swr.SelectedWindowError) as caught: + swr.validate_request(**{**ValidationTests.GOOD, "input_tf": token}) + self.assertEqual((caught.exception.code, caught.exception.code_args), + ("window_request_invalid", {"option": "input_tf"})) + self.assertEqual( + swr.validate_request(**{**ValidationTests.GOOD, "script_tf": token}).script_tf, token) + + def test_oversized_inputs_stop_the_run_before_any_feed_library_or_phase_work(self): + for flag, option in (("--window-start-ms", "window_start_ms"), + ("--window-end-ms", "window_end_ms"), + ("--preroll-bars", "preroll_bars"), + ("--fed-start-ms", "fed_start_ms"), + ("--input-tf", "input_tf")): + value = "1" + "0" * 5000 if flag == "--input-tf" else self.HUGE + with self.subTest(flag=flag), Counters() as poison: + status, out, _ = run_main(self.argv_with(flag, value)) + counts = poison.counts() + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]), + ("window_request_invalid", {"option": option})) + self.assertEqual(set(counts.values()), {0}) + + def test_an_oversized_primary_token_is_refused_by_admission_not_by_python(self): + token = "1" + "0" * 5000 + with Counters() as poison: + status, out, _ = run_main(self.argv_with("--script-tf", token)) + counts = poison.counts() + self.assertEqual(status, 1) + line = last_line(out) + self.assertEqual((line["code"], line["args"]["capability"]), + ("window_mode_unsupported", "selected_window_chart_timeframe")) + self.assertTrue(line["args"]["timeframe"].startswith("1000")) + self.assertEqual(set(counts.values()), {0}) + + def test_an_oversized_phase_descriptor_is_a_harness_fault_without_int(self): + with Counters() as poison: + status, out, _ = run_main(selected_argv(self.artifact, self.ohlcv, self.inventory, + extra=["--run-phase-fd", self.HUGE])) + counts = poison.counts() + self.assertEqual((status, last_line(out)["code"]), (1, "harness_internal_error")) + self.assertEqual(set(counts.values()), {0}) + + +ORIGINAL_LONG = ( # the long options of the command line before selected-window mode + "--help", "--so", "--ohlcv", "--inputs", "--overrides", "--input-tf", "--script-tf", + "--bar-magnifier", "--magnifier-samples", "--magnifier-dist", "--generated-cpp", + "--transpiled", "--syminfo", "--trade-start-ms", "--chart-tz", + "--magnifier-volume-weighted", "--bench", "--warmup", "--repeats", "--symbol-feeds", + "--outputs") +NEW_LONG = ( + "--report-policy", "--window-start-ms", "--window-end-ms", "--preroll-bars", + "--fed-start-ms", "--capabilities-json", "--validate-window-only", "--run-phase-fd", + "--request-feed-inventory", "--report-shape", "--curve-point-budget", "--results-digest") + + +class AbbreviationTests(unittest.TestCase): + """argparse accepts a prefix that exactly one long option begins with. The new flags made + some formerly unique prefixes ambiguous (--r, --re, --rep for --repeats; --c for + --chart-tz; --w for --warmup); the original command lines must keep working.""" + + def canonical(self, *argv): + return run_json._legacy_abbreviations(list(argv), ORIGINAL_LONG, + ORIGINAL_LONG + NEW_LONG) + + def test_formerly_unique_prefixes_become_their_original_option(self): + for prefix, option in (("--r", "--repeats"), ("--re", "--repeats"), + ("--rep", "--repeats"), ("--repe", "--repeats"), + ("--c", "--chart-tz"), ("--ch", "--chart-tz"), + ("--w", "--warmup"), ("--wa", "--warmup"), + ("--h", "--help"), ("--g", "--generated-cpp")): + with self.subTest(prefix=prefix): + self.assertEqual(self.canonical(prefix, "2"), [option, "2"]) + self.assertEqual(self.canonical(prefix + "=2"), [option + "=2"]) # = form + + def test_nothing_else_is_touched(self): + untouched = ( + ["--repeats", "2"], ["--so", "x"], ["--chart-tz", "UTC"], # full options + ["--report-policy", "p"], ["--window-start-ms", "1"], # new full flags + ["--capabilities-json"], ["--curve-point-budget=5"], + ["--s", "1"], ["--i", "1"], ["--m", "1"], ["--o", "1"], ["--b", "1"], # ambiguous + ["--t", "1"], ["--input", "1"], ["--tra", "1"], ["--ma", "1"], # before, still + ["--report-p", "1"], ["--window-s", "1"], ["--va", "1"], # new only + ["--unknown", "1"], ["-r", "2"], # not ours + ["--inputs", '{"--r": 1}'], ["--inputs={\"--rep\": 1}"], # values + ["--overrides=--rep"], # after = + ["--", "--r"], # after -- + ) + for argv in untouched: + with self.subTest(argv=argv): + self.assertEqual(self.canonical(*argv), argv) + + def test_a_value_after_the_equals_sign_and_a_second_token_are_left_alone(self): + self.assertEqual(self.canonical("--rep=--re", "--c=--c"), ["--repeats=--re", "--chart-tz=--c"]) + self.assertEqual(self.canonical("--inputs", "--r x"), ["--inputs", "--r x"]) + + def test_no_argv_means_the_process_argv(self): + with mock.patch.object(sys, "argv", ["run_json.py", "--rep", "3"]): + self.assertEqual(run_json._legacy_abbreviations(None, ORIGINAL_LONG, + ORIGINAL_LONG + NEW_LONG), + ["--repeats", "3"]) + + def bench(self, *extra): + ohlcv = Path(self.dir.name) / "feed.csv" + write_ohlcv(ohlcv, csv_rows(10)) + lib = FakeLib(full_plan()) + runs = [] + + def runner(*args): + runs.append(args) + OrdinaryAndCapabilityTests._fill(args[-1]._obj) + + lib.run_backtest_full = runner + lib.strategy_set_chart_timezone = lambda state, tz: lib.calls.append(("chart_tz", tz)) + with mock.patch.object(run_json, "load_strategy", lambda p: lib), \ + mock.patch.object(run_json, "normalize_release_provenance", lambda p, c, r, k: p): + status, out, _ = run_main(["--so", "x.so", "--ohlcv", str(ohlcv), "--input-tf", "1", + "--script-tf", "1", "--bench", *extra]) + return status, json.loads(out), runs, lib + + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + + def test_original_abbreviated_command_lines_still_run_as_they_did(self): + for extra in (["--warmup", "0", "--repeats", "2"], ["--w", "0", "--rep", "2"], + ["--w=0", "--re=2"], ["--wa", "0", "--r", "2"]): + with self.subTest(extra=extra): + status, report, runs, _ = self.bench(*extra) + self.assertEqual(status, 0) + timing = report["diagnostics"]["timing"] + self.assertEqual((timing["warmup"], timing["repeats"]), (0, 2)) + self.assertEqual(len(runs), 3) # no warm-up, two timed runs, the body run + + def test_the_formerly_unique_chart_timezone_prefix_still_sets_the_timezone(self): + for extra in (["--c", "UTC"], ["--c=UTC"], ["--ch", "UTC"], ["--chart-tz", "UTC"]): + with self.subTest(extra=extra): + status, report, _, lib = self.bench("--warmup", "0", "--repeats", "1", *extra) + self.assertEqual(status, 0) + self.assertIn(("chart_tz", b"UTC"), lib.calls) + self.assertEqual(report["applied_runtime"]["chart_tz"], "UTC") + + def test_ambiguous_old_prefixes_are_still_refused_by_argparse(self): + for prefix in ("--s", "--i", "--m", "--b", "--t", "--o", "--input", "--tra"): + with self.subTest(prefix=prefix): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()), \ + self.assertRaises(SystemExit) as caught: + run_json.main(["--so", "x.so", "--ohlcv", "y.csv", prefix, "1"]) + self.assertEqual(caught.exception.code, 2) + line = last_line(out.getvalue()) + self.assertEqual(line["code"], "run_request_invalid") + self.assertIn("ambiguous", line["error"]) + + def test_the_new_full_flags_work_normally_beside_the_old_prefixes(self): + with Counters() as poison: + status, out, _ = run_main(["--so", "x.so", "--ohlcv", "y.csv", "--rep", "2", + "--results-digest", "sha256/v1"]) + counts = poison.counts() + self.assertEqual(status, 1) + self.assertEqual(last_line(out)["code"], "results_digest_unsupported") + self.assertEqual(set(counts.values()), {0}) + + +class SourceShapeTests(unittest.TestCase): + """Static reads of the orchestration: an ordinary run imports none of the new modules, and + run_json never opens the planner from a strategy path or a search.""" + + def test_new_siblings_are_imported_only_inside_functions(self): + import ast + tree = ast.parse((ROOT / "docker" / "run_json.py").read_text()) + new = {"selected_window_report", "selected_window_plan", "request_feed_inventory", + "run_phase_transport", "run_execution_observer"} + top_level = {alias.name for node in tree.body if isinstance(node, ast.Import) + for alias in node.names} + self.assertEqual(top_level & new, set()) + + def test_the_report_module_does_not_import_run_json(self): + import ast + tree = ast.parse((ROOT / "docker" / "selected_window_report.py").read_text()) + imported = {alias.name for node in ast.walk(tree) if isinstance(node, ast.Import) + for alias in node.names} + self.assertNotIn("run_json", imported) + + def test_the_lookback_triple_is_written_in_one_private_producer_the_builder_calls(self): + import ast + source = (ROOT / "docker" / "selected_window_report.py").read_text() + tree = ast.parse(source) + functions = {node.name: node for node in ast.walk(tree) + if isinstance(node, ast.FunctionDef)} + + def writes_the_member(node): + return (isinstance(node, ast.Dict) + and any(isinstance(key, ast.Constant) and key.value == "has_unbounded_state" + for key in node.keys)) + + literals = [node for node in ast.walk(tree) if writes_the_member(node)] + self.assertEqual(len(literals), 1) # the triple is written in exactly one place ... + producer = functions["_lookback_not_produced"] # ... a private function ... + self.assertIn(literals[0], list(ast.walk(producer))) + called = {node.func.id for node in ast.walk(functions["build_report_window"]) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name)} + self.assertIn("_lookback_not_produced", called) # ... which the builder calls + # No boolean is ever written for the member by this module: it has no producer. + self.assertNotIn('"has_unbounded_state": True', source) + self.assertNotIn('"has_unbounded_state": False', source) + + def test_the_contract_revision_is_one_constant_with_one_literal_read_in_one_place(self): + # The parity declaration ("1.3-parity") is the single switchable place + # for a future full build ("1.3"): one tuple, one string literal, one reader, and no + # revision literal in run_json (so none in a request, report, phase or fingerprint). + import ast + report_tree = ast.parse((ROOT / "docker" / "selected_window_report.py").read_text()) + + def string_constants(tree): + return [node.value for node in ast.walk(tree) + if isinstance(node, ast.Constant) and isinstance(node.value, str)] + + strings = string_constants(report_tree) + self.assertEqual(strings.count("1.3-parity"), 1) # written once ... + self.assertNotIn("1.3", strings) # ... and never the full revision + assignments = [node for node in report_tree.body if isinstance(node, ast.Assign) + and any(isinstance(target, ast.Name) and target.id == "CONTRACT_REVISIONS" + for target in node.targets)] + self.assertEqual(len(assignments), 1) + self.assertIsInstance(assignments[0].value, ast.Tuple) + self.assertEqual(string_constants(assignments[0].value), ["1.3-parity"]) + readers = [function.name for function in ast.walk(report_tree) + if isinstance(function, ast.FunctionDef) + and any(isinstance(name, ast.Name) and name.id == "CONTRACT_REVISIONS" + for name in ast.walk(function))] + self.assertEqual(readers, ["capabilities_record"]) # ... and read in one place + run_json_strings = set(string_constants( + ast.parse((ROOT / "docker" / "run_json.py").read_text()))) + self.assertEqual(run_json_strings & {"1.3", "1.3-parity"}, set()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_phase_transport.py b/tests/test_run_phase_transport.py new file mode 100644 index 000000000..e5c308620 --- /dev/null +++ b/tests/test_run_phase_transport.py @@ -0,0 +1,1034 @@ +"""Tests of docker/run_phase_transport.py, the run-only phase transport writer. + +Black box over real AF_UNIX socketpairs: the test reads the far end the way the +parent would and judges the writer by the bytes that arrive, the exceptions it +raises and the state of the borrowed descriptor. Two seams are injected and +nothing else: os.write, only to make a short write deterministic (the fake still +hands a real prefix to the socket), and the clock, only for deterministic timing +and for the type negatives of a reading. Everything else runs on the real +socket, and the cases that do not need a scripted clock use the real monotonic +clock. +""" +import ast +import contextlib +import errno +import faulthandler +import gc +import importlib.util +import inspect +import io +import json +import math +import os +import socket +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = ROOT / "docker" / "run_phase_transport.py" +CATALOG_PATH = ROOT / "docker" / "run_failure_codes.json" +REAL_WRITE = os.write + + +def load_transport(): + spec = importlib.util.spec_from_file_location("run_phase_transport", MODULE_PATH) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +transport = load_transport() +RunPhaseWriter = transport.RunPhaseWriter +PhaseTransportError = transport.PhaseTransportError + + +def setUpModule(): + # A writer that blocked would hang the whole suite: dump the stacks and exit instead. + faulthandler.dump_traceback_later(300, exit=True) + + +def tearDownModule(): + faulthandler.cancel_dump_traceback_later() + + +# What the documented contract says, written out here and not read from the module under test. +PHASES = ("preflight", "execution", "result_assembly", "results_digest", + "serialization", "completed") +WITH_DIGEST = PHASES +WITHOUT_DIGEST = tuple(phase for phase in PHASES if phase != "results_digest") +ENVELOPE_KEYS = ["record", "version", "sequence", "scope", "trial_id", "phase", "elapsed_ms"] +COMPLETED_KEYS = ENVELOPE_KEYS + ["execution_ms", "result_assembly_ms", + "results_digest_ms", "serialization_ms"] +PINNED_FIRST_RECORD = (b'{"record":"run_phase","version":1,"sequence":1,"scope":"run",' + b'"trial_id":null,"phase":"preflight","elapsed_ms":0}\n') + + +def reject_constant(name): + raise ValueError("non-finite JSON number " + name) + + +def is_number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def descriptor_count(): + """How many descriptors this process holds; None where /dev/fd cannot be listed.""" + gc.collect() + try: + return len(os.listdir("/dev/fd")) + except OSError: + return None + + +def prefix_write(cut): + """A fake os.write: hands the first cut(len(data)) bytes to the real descriptor + and reports exactly that count (0 hands nothing over and reports 0).""" + def write(fd, data): + count = cut(len(data)) + return REAL_WRITE(fd, data[:count]) if count else 0 + return write + + +class WriteSpy: + """Stands in for os.write. Calls on `fd` (on every descriptor when fd is None) + are recorded: the bytes handed over, and whether the descriptor was nonblocking + at that moment. `fake`, when given, decides what such a call does and returns; + every other call passes through.""" + + def __init__(self, fd, fake=None): + self.fd = fd + self.fake = fake + self.calls = [] + self.nonblocking = [] + + def __call__(self, fd, data): + if self.fd is not None and fd != self.fd: + return REAL_WRITE(fd, data) + self.calls.append(bytes(data)) + self.nonblocking.append(not os.get_blocking(fd)) + if self.fake is not None: + return self.fake(fd, data) + return REAL_WRITE(fd, data) + + +@contextlib.contextmanager +def spying_on_os_write(fd=None, fake=None): + spy = WriteSpy(fd, fake) + with mock.patch.object(os, "write", spy): + yield spy + + +class ScriptedClock: + """A clock that returns the given readings, one per call, and counts its calls.""" + + def __init__(self, *readings): + self.readings = list(readings) + self.reads = 0 + + def __call__(self): + reading = self.readings[self.reads] + self.reads += 1 + return reading + + +class Channel: + """A real AF_UNIX stream socketpair. `fd` is the writer's end, the descriptor the + parent would hand down; read() drains the far end as the parent would.""" + + def __init__(self, test): + self.near, self.far = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + test.addCleanup(self.near.close) + test.addCleanup(self.far.close) + self.far.setblocking(False) + self.fd = self.near.fileno() + test.assertGreaterEqual(self.fd, 3) + + def read(self): + chunks = [] + while True: + try: + chunk = self.far.recv(65536) + except BlockingIOError: + break + if not chunk: + break + chunks.append(chunk) + return b"".join(chunks) + + def fill(self): + """Send filler until the near end takes no more, so that any further write + finds the socket full. Returns how many filler bytes the socket holds.""" + sent = 0 + for size in (65536, 4096, 256, 1): + while True: + try: + sent += self.near.send(b"x" * size, socket.MSG_DONTWAIT) + except BlockingIOError: + break + return sent + + +class WriterCase(unittest.TestCase): + def channel(self): + return Channel(self) + + def decode(self, data): + """The records of a stream, after the receiver's own framing checks.""" + self.assertFalse(data.startswith(b"\xef\xbb\xbf"), "BOM") + if not data: + return [] + self.assertTrue(data.endswith(b"\n"), "the stream ends inside a record") + records = [] + for line in data[:-1].split(b"\n"): + self.assertLessEqual(len(line) + 1, 1024) + self.assertNotIn(b"\r", line) + self.assertNotIn(b" ", line, "records are compact") + record = json.loads(line.decode("utf-8"), parse_constant=reject_constant) + self.assertIsInstance(record, dict) + records.append(record) + return records + + def check_records(self, records, phases): + """The receiver's checks: vocabulary and order, exact members, sequence from 1 + without a gap, scope and trial_id, finite nonnegative nondecreasing times.""" + self.assertEqual([record["phase"] for record in records], list(phases)) + previous = 0 + for index, record in enumerate(records, start=1): + completed = record["phase"] == "completed" + self.assertEqual(list(record), COMPLETED_KEYS if completed else ENVELOPE_KEYS) + self.assertEqual(record["record"], "run_phase") + self.assertIs(type(record["version"]), int) + self.assertEqual(record["version"], 1) + self.assertIs(type(record["sequence"]), int) + self.assertEqual(record["sequence"], index) + self.assertEqual(record["scope"], "run") + self.assertIsNone(record["trial_id"]) + elapsed = record["elapsed_ms"] + self.assertTrue(is_number(elapsed)) + self.assertTrue(math.isfinite(elapsed)) + self.assertGreaterEqual(elapsed, previous) + previous = elapsed + if completed: + for name in COMPLETED_KEYS[len(ENVELOPE_KEYS):]: + value = record[name] + if name == "results_digest_ms" and "results_digest" not in phases: + self.assertIsNone(value) + continue + self.assertTrue(is_number(value), name) + self.assertTrue(math.isfinite(value), name) + self.assertGreaterEqual(value, 0, name) + if records: + self.assertEqual(records[0]["elapsed_ms"], 0) + + def assert_never_reused(self, writer): + """A failed writer refuses every phase and does not touch any descriptor.""" + with spying_on_os_write() as spy: + for phase in PHASES: + with self.subTest(refused=phase): + with self.assertRaises(PhaseTransportError) as caught: + writer.advance(phase) + self.assertEqual(caught.exception.reason, "writer_failed") + self.assertEqual(spy.calls, []) + + +class NormalRunTests(WriterCase): + def run_phases(self, phases): + channel = self.channel() + with spying_on_os_write(channel.fd) as spy: + writer = RunPhaseWriter(channel.fd) + self.assertEqual(spy.calls, [], "the constructor writes nothing") + for phase in phases: + writer.advance(phase) + return channel, writer, spy + + def test_full_run_with_the_digest_phase(self): + channel, writer, spy = self.run_phases(WITH_DIGEST) + data = channel.read() + self.check_records(self.decode(data), WITH_DIGEST) + self.assertIsNotNone(self.decode(data)[-1]["results_digest_ms"]) + self.assertEqual(len(spy.calls), 6, "exactly one os.write per boundary") + self.assertEqual(b"".join(spy.calls), data) + for call in spy.calls: + self.assertTrue(call.endswith(b"\n")) + self.assertEqual(call.count(b"\n"), 1, "one whole record per write") + self.assertEqual(spy.nonblocking, [True] * 6, "O_NONBLOCK is set before any write") + self.assertEqual((writer.phase, writer.sequence), ("completed", 6)) + self.assertIsNone(writer.failure) + self.assertEqual(writer.unsent, b"") + + def test_full_run_without_the_digest_phase(self): + channel, writer, spy = self.run_phases(WITHOUT_DIGEST) + records = self.decode(channel.read()) + self.check_records(records, WITHOUT_DIGEST) + self.assertIsNone(records[-1]["results_digest_ms"]) + self.assertEqual(len(spy.calls), 5) + self.assertEqual((writer.phase, writer.sequence), ("completed", 5)) + + def test_each_boundary_is_in_the_socket_before_advance_returns(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + for count, phase in enumerate(WITH_DIGEST, start=1): + writer.advance(phase) + records = self.decode(channel.read()) + self.assertEqual([record["phase"] for record in records], [phase]) + self.assertEqual(records[0]["sequence"], count) + self.assertEqual((writer.phase, writer.sequence), (phase, count)) + + def test_first_record_is_the_pinned_example_byte_for_byte(self): + channel = self.channel() + RunPhaseWriter(channel.fd).advance("preflight") + self.assertEqual(channel.read(), PINNED_FIRST_RECORD) + + def test_completed_durations_agree_with_the_records_that_bracket_them(self): + channel, _, _ = self.run_phases(WITH_DIGEST) + records = self.decode(channel.read()) + elapsed = {record["phase"]: record["elapsed_ms"] for record in records} + completed = records[-1] + self.assertAlmostEqual(completed["execution_ms"], + elapsed["result_assembly"] - elapsed["execution"], delta=0.001) + self.assertAlmostEqual(completed["result_assembly_ms"], + elapsed["results_digest"] - elapsed["result_assembly"], delta=0.001) + self.assertAlmostEqual(completed["results_digest_ms"], + elapsed["serialization"] - elapsed["results_digest"], delta=0.001) + self.assertAlmostEqual(completed["serialization_ms"], + elapsed["completed"] - elapsed["serialization"], delta=0.001) + + def test_serialization_ms_is_measured_at_the_completed_call_after_the_callers_flush(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + for phase in WITHOUT_DIGEST[:-1]: + writer.advance(phase) + time.sleep(0.02) # the caller serializes and flushes its report + writer.advance("completed") + completed = self.decode(channel.read())[-1] + self.assertGreaterEqual(completed["serialization_ms"], 15.0) + timing = writer.phase_timing() + self.assertNotIn("serialization_ms", timing) + self.assertEqual(timing["execution_ms"], completed["execution_ms"]) + self.assertEqual(timing["result_assembly_ms"], completed["result_assembly_ms"]) + + def test_writers_share_nothing(self): + first, second = self.channel(), self.channel() + a, b = RunPhaseWriter(first.fd), RunPhaseWriter(second.fd) + a.advance("preflight") + a.advance("execution") + self.assertIsNone(b.phase) + self.assertEqual(b.sequence, 0) + b.advance("preflight") + self.check_records(self.decode(first.read()), WITH_DIGEST[:2]) + self.check_records(self.decode(second.read()), WITH_DIGEST[:1]) + + def test_fd_none_tracks_phases_and_timing_and_exports_nothing(self): + clock = ScriptedClock(1.0, 1.5, 2.0, 2.5, 3.0) + with spying_on_os_write() as spy: + writer = RunPhaseWriter(None, clock=clock) + for phase in WITHOUT_DIGEST[:4]: + writer.advance(phase) + timing = writer.phase_timing() + writer.advance("completed") + self.assertEqual(spy.calls, [], "no os.write at all") + self.assertEqual((writer.phase, writer.sequence), ("completed", 5)) + self.assertEqual(timing, {"version": 1, "execution_ms": 500.0, + "result_assembly_ms": 500.0, "results_digest_ms": None}) + with self.assertRaises(PhaseTransportError) as caught: + RunPhaseWriter(None).advance("execution") + self.assertEqual(caught.exception.reason, "phase_order", + "the order rules bind an unexported run too") + + +class PhaseOrderTests(WriterCase): + LEGAL_PREFIXES = ( + (), + ("preflight",), + ("preflight", "execution"), + ("preflight", "execution", "result_assembly"), + ("preflight", "execution", "result_assembly", "results_digest"), + ("preflight", "execution", "result_assembly", "serialization"), + ("preflight", "execution", "result_assembly", "results_digest", "serialization"), + ("preflight", "execution", "result_assembly", "serialization", "completed"), + ("preflight", "execution", "result_assembly", "results_digest", "serialization", + "completed"), + ) + ALLOWED_AFTER = { + None: {"preflight"}, + "preflight": {"execution"}, + "execution": {"result_assembly"}, + "result_assembly": {"results_digest", "serialization"}, + "results_digest": {"serialization"}, + "serialization": {"completed"}, + "completed": set(), + } + + def test_only_the_pinned_order_is_accepted_from_every_state(self): + for prefix in self.LEGAL_PREFIXES: + last = prefix[-1] if prefix else None + for candidate in PHASES: + with self.subTest(after=last, next=candidate): + writer = RunPhaseWriter(None) + for phase in prefix: + writer.advance(phase) + if candidate in self.ALLOWED_AFTER[last]: + writer.advance(candidate) + self.assertEqual(writer.phase, candidate) + self.assertEqual(writer.sequence, len(prefix) + 1) + self.assertIsNone(writer.failure) + continue + with self.assertRaises(PhaseTransportError) as caught: + writer.advance(candidate) + self.assertEqual(caught.exception.reason, "phase_order") + self.assertEqual(caught.exception.phase, candidate) + self.assertEqual(writer.phase, last, "the phase was not entered") + self.assertEqual(writer.sequence, len(prefix)) + self.assertEqual(writer.failure, "phase_order") + + def test_unknown_phases_and_wrong_types_are_refused(self): + values = ["", "bogus", "Preflight", "preflight ", "completed\n", "result-assembly", + "trial", "heartbeat", None, 1, True, b"preflight", ["preflight"], + ("preflight",), object()] + for value in values: + for prefix in ((), ("preflight", "execution")): + with self.subTest(value=repr(value)[:30], after=len(prefix)): + writer = RunPhaseWriter(None) + for phase in prefix: + writer.advance(phase) + with self.assertRaises(PhaseTransportError) as caught: + writer.advance(value) + self.assertEqual(caught.exception.code, "harness_internal_error") + self.assertEqual(caught.exception.reason, "invalid_phase") + self.assertEqual(writer.sequence, len(prefix)) + + def test_a_refused_phase_writes_nothing_and_ends_the_writer(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + writer.advance("preflight") + with spying_on_os_write(channel.fd) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("serialization") # a gap: execution was skipped + self.assertEqual(caught.exception.reason, "phase_order") + self.assertEqual(spy.calls, []) + self.check_records(self.decode(channel.read()), ["preflight"]) + self.assert_never_reused(writer) + + def test_no_restart_and_no_seventh_record(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + for phase in WITH_DIGEST: + writer.advance(phase) + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("preflight") + self.assertEqual(caught.exception.reason, "phase_order") + self.check_records(self.decode(channel.read()), WITH_DIGEST) + self.assertEqual(writer.sequence, 6) + + +class DescriptorTests(WriterCase): + def assert_rejected(self, value, reason): + with self.assertRaises(PhaseTransportError) as caught: + RunPhaseWriter(value) + self.assertEqual(caught.exception.code, "harness_internal_error") + self.assertEqual(caught.exception.reason, reason) + + def test_descriptor_must_be_a_plain_int_from_3_up(self): + values = [True, False, 0, 1, 2, -1, 3.0, "3", b"3", [3], object(), + 2 ** 31, 2 ** 32 + 3, 2 ** 70] + for value in values: + with self.subTest(fd=repr(value)[:30]): + self.assert_rejected(value, "invalid_fd") + + def test_a_closed_descriptor_is_rejected(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + fd = sock.fileno() + sock.close() + self.assert_rejected(fd, "invalid_fd") + + def test_non_sockets_are_rejected_and_left_open_and_untouched(self): + read_end, write_end = os.pipe() + null = os.open(os.devnull, os.O_RDWR) + for fd in (read_end, write_end, null): + self.addCleanup(os.close, fd) + for fd in (read_end, write_end, null): + with self.subTest(fd=fd): + self.assert_rejected(fd, "invalid_fd") + os.fstat(fd) + self.assertTrue(os.get_blocking(fd), "a rejected descriptor is not changed") + + def test_wrong_family_is_rejected_and_left_open_and_untouched(self): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.addCleanup(sock.close) + self.assert_rejected(sock.fileno(), "invalid_family") + os.fstat(sock.fileno()) + self.assertTrue(os.get_blocking(sock.fileno())) + + def test_wrong_socket_type_is_rejected_and_left_open_and_untouched(self): + left, right = socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM) + self.addCleanup(left.close) + self.addCleanup(right.close) + self.assert_rejected(left.fileno(), "invalid_type") + os.fstat(left.fileno()) + self.assertTrue(os.get_blocking(left.fileno())) + + def test_a_stream_socket_is_borrowed_not_owned_and_made_nonblocking(self): + channel = self.channel() + self.assertTrue(os.get_blocking(channel.fd)) + before = descriptor_count() + writer = RunPhaseWriter(channel.fd) + self.assertFalse(os.get_blocking(channel.fd), "O_NONBLOCK is set at construction") + self.assertEqual(descriptor_count(), before, "validation left no extra descriptor") + os.fstat(channel.fd) + self.assertEqual(channel.near.fileno(), channel.fd) + self.assertIsNone(writer.phase) + + def test_the_descriptor_is_never_closed_or_duplicated_on_any_path(self): + # A scenario returns how many descriptors the test itself closed, None for none. + def complete(channel, writer): + for phase in WITH_DIGEST: + writer.advance(phase) + channel.read() + self.assertEqual(channel.near.send(b"!"), 1) # still the live socket + self.assertEqual(channel.far.recv(1), b"!") + + def misordered(channel, writer): + writer.advance("preflight") + with self.assertRaises(PhaseTransportError): + writer.advance("completed") + + def short_write(channel, writer): + with spying_on_os_write(channel.fd, fake=prefix_write(lambda size: size // 2)): + with self.assertRaises(PhaseTransportError): + writer.advance("preflight") + + def full_socket(channel, writer): + channel.fill() + with self.assertRaises(PhaseTransportError): + writer.advance("preflight") + + def disconnected(channel, writer): + channel.far.close() + with self.assertRaises(PhaseTransportError): + writer.advance("preflight") + return 1 # the far end's descriptor, which the test itself closed + + for scenario in (complete, misordered, short_write, full_socket, disconnected): + with self.subTest(scenario=scenario.__name__): + channel = self.channel() + before = descriptor_count() + writer = RunPhaseWriter(channel.fd) + closed = scenario(channel, writer) or 0 + del writer + gc.collect() + os.fstat(channel.fd) + if before is not None: + self.assertEqual(descriptor_count(), before - closed) + + def test_writable_access_mode_predicate(self): + writable = transport._is_writable_mode + self.assertFalse(writable(os.O_RDONLY)) + self.assertTrue(writable(os.O_WRONLY)) + self.assertTrue(writable(os.O_RDWR)) + self.assertFalse(writable(os.O_RDONLY | os.O_NONBLOCK)) + self.assertTrue(writable(os.O_RDWR | os.O_NONBLOCK | os.O_APPEND)) + self.assertTrue(writable(os.O_WRONLY | os.O_NONBLOCK)) + + +class BackpressureTests(WriterCase): + CUTS = { + "nothing taken": lambda size: 0, + "one byte": lambda size: 1, + "half": lambda size: size // 2, + "all but the LF": lambda size: size - 1, + } + + def test_full_socket_refuses_the_first_boundary(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + filler = channel.fill() + work = [] + with spying_on_os_write(channel.fd) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("preflight") + work.append("platform preflight") + error = caught.exception + self.assertEqual(error.code, "harness_internal_error") + self.assertEqual(error.reason, "would_block") + self.assertEqual(error.phase, "preflight") + self.assertIsInstance(error.__cause__, BlockingIOError) + self.assertEqual(work, [], "no work transition on a failed handoff") + self.assertEqual(len(spy.calls), 1, "one write, no retry") + self.assertIsNone(writer.phase) + self.assertEqual(writer.sequence, 0) + self.assertEqual(writer.failure, "would_block") + self.assertEqual(writer.unsent, PINNED_FIRST_RECORD) + self.assertEqual(channel.read(), b"x" * filler, "no byte of the record got in") + self.assert_never_reused(writer) + + def test_full_socket_in_the_middle_of_a_run_keeps_the_next_work_from_starting(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + work = [] + writer.advance("preflight") + work.append("platform preflight") + writer.advance("execution") + work.append("strategy execution") + filler = channel.fill() # the parent stops reading + with spying_on_os_write(channel.fd) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("result_assembly") + work.append("report assembly") + self.assertEqual(caught.exception.reason, "would_block") + self.assertEqual(caught.exception.phase, "result_assembly") + self.assertEqual(work, ["platform preflight", "strategy execution"]) + self.assertEqual(len(spy.calls), 1) + self.assertEqual((writer.phase, writer.sequence), ("execution", 2)) + failed = json.loads(writer.unsent) + self.assertEqual((failed["phase"], failed["sequence"]), ("result_assembly", 3)) + self.assertLessEqual(len(writer.unsent), 1024) + data = channel.read() + self.assertEqual(data[-filler:], b"x" * filler) + self.check_records(self.decode(data[:-filler]), WITH_DIGEST[:2]) + self.assert_never_reused(writer) + + def test_a_reader_that_catches_up_does_not_revive_a_failed_writer(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + channel.fill() + with self.assertRaises(PhaseTransportError): + writer.advance("preflight") + channel.read() # the socket is healthy again + self.assert_never_reused(writer) + self.assertEqual(channel.read(), b"", "no late record and no retried remainder") + + def test_refusal_is_immediate_and_not_a_wait(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + channel.fill() + started = time.monotonic() + with self.assertRaises(PhaseTransportError): + writer.advance("preflight") + self.assertLess(time.monotonic() - started, 1.0, + "run-only never waits for its reader, not even the optimizer's 2 s") + + def test_disconnect_before_the_first_boundary(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + channel.far.close() + with spying_on_os_write(channel.fd) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("preflight") + error = caught.exception + self.assertEqual(error.reason, "disconnected") + self.assertEqual(error.code, "harness_internal_error") + self.assertIsInstance(error.__cause__, (BrokenPipeError, ConnectionResetError)) + self.assertEqual(len(spy.calls), 1) + self.assertIsNone(writer.phase) + self.assertEqual(writer.failure, "disconnected") + os.fstat(channel.fd) + self.assert_never_reused(writer) + + def test_disconnect_in_the_middle_of_a_run(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + work = [] + writer.advance("preflight") + work.append("platform preflight") + writer.advance("execution") + work.append("strategy execution") + channel.far.close() + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("result_assembly") + work.append("report assembly") + self.assertEqual(caught.exception.reason, "disconnected") + self.assertEqual(work, ["platform preflight", "strategy execution"]) + self.assertEqual((writer.phase, writer.sequence), ("execution", 2)) + self.assert_never_reused(writer) + + def test_other_write_errors_are_typed_too(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + fd = channel.near.detach() # the socket object lets go; the descriptor is ours now + os.close(fd) # and it is gone behind the writer's back + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("preflight") + error = caught.exception + self.assertEqual(error.reason, "write_failed") + self.assertIsInstance(error.__cause__, OSError) + self.assertEqual(error.__cause__.errno, errno.EBADF) + self.assertIsNone(writer.phase) + self.assert_never_reused(writer) + + def test_short_write_retains_a_bounded_remainder_and_fails(self): + for name, cut in self.CUTS.items(): + with self.subTest(cut=name): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + writer.advance("preflight") + self.assertEqual(channel.read(), PINNED_FIRST_RECORD) + work = [] + with spying_on_os_write(channel.fd, fake=prefix_write(cut)) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("execution") + work.append("strategy execution") + error = caught.exception + self.assertEqual(error.reason, "short_write") + self.assertEqual(error.code, "harness_internal_error") + self.assertEqual(error.phase, "execution") + self.assertEqual(work, [], "no work transition on a failed handoff") + self.assertEqual(len(spy.calls), 1, "one write, the rest is not retried") + line = spy.calls[0] + taken = cut(len(line)) + sent = channel.read() + self.assertEqual(sent, line[:taken]) + self.assertEqual(writer.unsent, line[taken:]) + self.assertEqual(sent + writer.unsent, line) + self.assertGreater(len(writer.unsent), 0) + self.assertLessEqual(len(writer.unsent), 1024) + self.assertLessEqual(len(writer.unsent), 1048576) + self.assertFalse(sent.endswith(b"\n"), "a cut record never looks complete") + self.assertEqual((writer.phase, writer.sequence), ("preflight", 1)) + self.assertEqual(writer.failure, "short_write") + self.assert_never_reused(writer) + self.assertEqual(channel.read(), b"", "the remainder is never sent later") + + def test_a_cut_completed_record_leaves_the_report_timing_available(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + for phase in WITH_DIGEST[:-1]: + writer.advance(phase) + with spying_on_os_write(channel.fd, fake=prefix_write(lambda size: 1)): + with self.assertRaises(PhaseTransportError): + writer.advance("completed") + self.assertEqual((writer.phase, writer.sequence), ("serialization", 5)) + timing = writer.phase_timing() + self.assertEqual(list(timing), + ["version", "execution_ms", "result_assembly_ms", "results_digest_ms"]) + + def test_a_refused_serialization_boundary_leaves_no_timing(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd) + for phase in WITH_DIGEST[:4]: + writer.advance(phase) + channel.fill() + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("serialization") + self.assertEqual(caught.exception.reason, "would_block") + with self.assertRaises(PhaseTransportError) as unavailable: + writer.phase_timing() + self.assertEqual(unavailable.exception.reason, "timing_unavailable") + + +class ClockTests(WriterCase): + BAD_READINGS = [True, False, "1.0", None, b"1", [1.0], 1 + 0j, + float("nan"), float("inf"), float("-inf"), 10 ** 400] + + def advance_all(self, writer, phases): + for phase in phases: + writer.advance(phase) + + def test_elapsed_and_durations_are_exact_with_the_digest_phase(self): + channel = self.channel() + clock = ScriptedClock(10.0, 10.5, 12.0, 12.25, 13.0, 13.125) + writer = RunPhaseWriter(channel.fd, clock=clock) + self.assertEqual(clock.reads, 0, "the constructor does not read the clock") + for count, phase in enumerate(WITH_DIGEST[:4], start=1): + writer.advance(phase) + self.assertEqual(clock.reads, count, "one reading per advance") + with self.assertRaises(PhaseTransportError) as early: + writer.phase_timing() + self.assertEqual(early.exception.reason, "timing_unavailable") + self.assertIsNone(writer.failure, "asking too early does not end the writer") + writer.advance("serialization") + timing = writer.phase_timing() + self.assertEqual(list(timing), + ["version", "execution_ms", "result_assembly_ms", "results_digest_ms"]) + self.assertEqual(timing, {"version": 1, "execution_ms": 1500.0, + "result_assembly_ms": 250.0, "results_digest_ms": 750.0}) + writer.advance("completed") + self.assertEqual(clock.reads, 6) + records = self.decode(channel.read()) + self.check_records(records, WITH_DIGEST) + self.assertEqual([record["elapsed_ms"] for record in records], + [0, 500, 2000, 2250, 3000, 3125]) + completed = records[-1] + self.assertEqual(completed["execution_ms"], 1500) + self.assertEqual(completed["result_assembly_ms"], 250) + self.assertEqual(completed["results_digest_ms"], 750) + self.assertEqual(completed["serialization_ms"], 125) + self.assertEqual(writer.phase_timing(), timing) + + def test_elapsed_and_durations_are_exact_without_the_digest_phase(self): + channel = self.channel() + clock = ScriptedClock(100.0, 100.25, 101.0, 101.5, 102.0) + writer = RunPhaseWriter(channel.fd, clock=clock) + self.advance_all(writer, WITHOUT_DIGEST[:4]) + self.assertEqual(writer.phase_timing(), {"version": 1, "execution_ms": 750.0, + "result_assembly_ms": 500.0, + "results_digest_ms": None}) + writer.advance("completed") + records = self.decode(channel.read()) + self.check_records(records, WITHOUT_DIGEST) + self.assertEqual([record["elapsed_ms"] for record in records], + [0, 250, 1000, 1500, 2000]) + completed = records[-1] + self.assertEqual(completed["execution_ms"], 750) + self.assertEqual(completed["result_assembly_ms"], 500) + self.assertIsNone(completed["results_digest_ms"]) + self.assertEqual(completed["serialization_ms"], 500) + + def test_integer_and_flat_clocks_are_accepted(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd, clock=ScriptedClock(5, 6, 8, 9, 12)) + self.advance_all(writer, WITHOUT_DIGEST) + records = self.decode(channel.read()) + self.assertEqual([record["elapsed_ms"] for record in records], + [0, 1000, 3000, 4000, 7000]) + self.assertEqual(records[-1]["serialization_ms"], 3000) + flat = self.channel() + stopped = RunPhaseWriter(flat.fd, clock=ScriptedClock(7.0, 7.0, 7.0, 7.0, 7.0)) + self.advance_all(stopped, WITHOUT_DIGEST) + records = self.decode(flat.read()) + self.check_records(records, WITHOUT_DIGEST) + self.assertEqual([record["elapsed_ms"] for record in records], [0] * 5) + self.assertEqual(records[-1]["execution_ms"], 0) + + def test_a_bad_first_reading_refuses_preflight(self): + for bad in self.BAD_READINGS: + with self.subTest(reading=repr(bad)[:30]): + channel = self.channel() + writer = RunPhaseWriter(channel.fd, clock=ScriptedClock(bad)) + with spying_on_os_write(channel.fd) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("preflight") + self.assertEqual(caught.exception.reason, "clock_invalid") + self.assertEqual(caught.exception.code, "harness_internal_error") + self.assertEqual(spy.calls, []) + self.assertIsNone(writer.phase) + self.assertEqual(channel.read(), b"") + + def test_a_bad_later_reading_refuses_that_boundary_only(self): + for bad in self.BAD_READINGS: + with self.subTest(reading=repr(bad)[:30]): + channel = self.channel() + writer = RunPhaseWriter(channel.fd, clock=ScriptedClock(0.0, bad)) + writer.advance("preflight") + with spying_on_os_write(channel.fd) as spy: + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("execution") + self.assertEqual(caught.exception.reason, "clock_invalid") + self.assertEqual(spy.calls, [], "nothing is handed over for that boundary") + self.assertEqual((writer.phase, writer.sequence), ("preflight", 1)) + self.check_records(self.decode(channel.read()), ["preflight"]) + self.assert_never_reused(writer) + + def test_a_backward_clock_is_refused_at_any_boundary(self): + for readings, fails_at in (((5.0, 4.999), 1), ((1.0, 2.0, 3.0, 2.5), 3)): + with self.subTest(readings=readings): + channel = self.channel() + writer = RunPhaseWriter(channel.fd, clock=ScriptedClock(*readings)) + self.advance_all(writer, WITH_DIGEST[:fails_at]) + with self.assertRaises(PhaseTransportError) as caught: + writer.advance(WITH_DIGEST[fails_at]) + self.assertEqual(caught.exception.reason, "clock_invalid") + self.assertEqual(writer.sequence, fails_at) + self.check_records(self.decode(channel.read()), WITH_DIGEST[:fails_at]) + + def test_a_clock_that_raises_is_a_typed_failure(self): + def broken(): + raise RuntimeError("clock broke") + + channel = self.channel() + writer = RunPhaseWriter(channel.fd, clock=broken) + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("preflight") + self.assertEqual(caught.exception.reason, "clock_invalid") + self.assertIsInstance(caught.exception.__cause__, RuntimeError) + self.assertEqual(channel.read(), b"") + + def test_an_elapsed_span_that_is_not_finite_is_refused(self): + channel = self.channel() + writer = RunPhaseWriter(channel.fd, clock=ScriptedClock(-1e308, 1e308)) + writer.advance("preflight") + with self.assertRaises(PhaseTransportError) as caught: + writer.advance("execution") + self.assertEqual(caught.exception.reason, "clock_invalid") + self.check_records(self.decode(channel.read()), ["preflight"]) + + def test_the_default_clock_is_the_monotonic_clock(self): + parameter = inspect.signature(RunPhaseWriter.__init__).parameters["clock"] + self.assertIs(parameter.default, time.monotonic) + self.assertIs(parameter.kind, inspect.Parameter.KEYWORD_ONLY) + + +class FramingTests(WriterCase): + def test_long_float_digits_stay_inside_the_record_limit(self): + channel = self.channel() + clock = ScriptedClock(0.1, 0.30000000000000004, 1.0 / 3.0 + 1e6, + 2.0 ** 0.5 + 1e6, 1e6 + 1.0 / 7.0 + 5.0, 1e6 + 1e5 / 3.0 + 9.0) + writer = RunPhaseWriter(channel.fd, clock=clock) + self.advance_all(writer, WITH_DIGEST) + data = channel.read() + self.check_records(self.decode(data), WITH_DIGEST) + for line in data[:-1].split(b"\n"): + self.assertLessEqual(len(line) + 1, 1024) + + def advance_all(self, writer, phases): + for phase in phases: + writer.advance(phase) + + def test_encode_record_enforces_the_1024_byte_limit_including_the_lf(self): + exact = transport.encode_record({"p": "x" * 1015}) + self.assertEqual(len(exact), 1024) + self.assertTrue(exact.endswith(b"\n")) + with self.assertRaises(PhaseTransportError) as caught: + transport.encode_record({"p": "x" * 1016}) + self.assertEqual(caught.exception.reason, "record_oversize") + self.assertEqual(caught.exception.code, "harness_internal_error") + + def test_encode_record_refuses_values_json_cannot_hold(self): + for bad in (float("nan"), float("inf"), float("-inf"), object(), {1, 2}): + with self.subTest(value=repr(bad)[:30]): + with self.assertRaises(PhaseTransportError) as caught: + transport.encode_record({"elapsed_ms": bad}) + self.assertEqual(caught.exception.reason, "record_invalid") + + def test_encode_record_never_leaves_a_raw_newline_inside_a_record(self): + text = "a\nb\r\nc\u2028d\u00e9" + line = transport.encode_record({"p": text}) + self.assertEqual(line.count(b"\n"), 1) + self.assertTrue(line.endswith(b"\n")) + self.assertNotIn(b"\r", line) + self.assertFalse(line.startswith(b"\xef\xbb\xbf")) + self.assertEqual(json.loads(line.decode("utf-8"))["p"], text) + self.assertNotIn(b" ", transport.encode_record({"a": 1, "b": [1, 2]})) + + def test_limits_are_the_pinned_numbers(self): + self.assertEqual(transport.PHASES, PHASES) + self.assertEqual(transport.MAX_RECORD_BYTES, 1024) + self.assertEqual(transport.MAX_RECORDS, 6) + self.assertEqual(transport.MAX_UNSENT_BYTES, 1048576) + self.assertEqual(len(transport.PHASES), transport.MAX_RECORDS) + self.assertLessEqual(transport.MAX_RECORD_BYTES, transport.MAX_UNSENT_BYTES) + + +class StdoutIsNotAuthorityTests(WriterCase): + FORGED = ('{"record":"run_phase","version":1,"sequence":99,"scope":"run","trial_id":null,' + '"phase":"completed","elapsed_ms":0}') + + def test_nothing_is_printed_and_a_printed_record_is_not_the_transport(self): + channel = self.channel() + stdout, stderr = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + writer = RunPhaseWriter(channel.fd) + for phase in WITHOUT_DIGEST: + writer.advance(phase) + print(self.FORGED) # what a strategy can print + # failure paths print nothing either + full = self.channel() + failing = RunPhaseWriter(full.fd) + full.fill() + with self.assertRaises(PhaseTransportError): + failing.advance("preflight") + with self.assertRaises(PhaseTransportError): + RunPhaseWriter(None).advance("serialization") + with self.assertRaises(PhaseTransportError): + RunPhaseWriter(1) + self.assertEqual(stdout.getvalue(), (self.FORGED + "\n") * 5, + "only the forged lines: the writer printed nothing") + self.assertEqual(stderr.getvalue(), "") + records = self.decode(channel.read()) + self.check_records(records, WITHOUT_DIGEST) + self.assertNotIn(99, [record["sequence"] for record in records]) + + def test_a_forged_record_cannot_move_the_writer(self): + channel = self.channel() + stdout = io.StringIO() + writer = RunPhaseWriter(channel.fd) + with contextlib.redirect_stdout(stdout): + writer.advance("preflight") + print(self.FORGED) + self.assertEqual((writer.phase, writer.sequence), ("preflight", 1)) + writer.advance("execution") + self.check_records(self.decode(channel.read()), WITH_DIGEST[:2]) + + +class SourceContractTests(unittest.TestCase): + FORBIDDEN_CALLS = {"print", "sleep", "select", "poll", "epoll", "kqueue", "close", "dup", + "dup2", "fromfd", "fdopen", "makefile", "send", "sendall", "recv", + "setblocking", "settimeout", "wait", "join"} + FORBIDDEN_IMPORTS = {"select", "selectors", "asyncio", "threading", "sys", "logging", + "subprocess", "signal"} + + @classmethod + def setUpClass(cls): + cls.tree = ast.parse(MODULE_PATH.read_text(encoding="utf-8")) + + def test_exactly_one_os_write_and_no_loop_around_it(self): + parents = {} + for parent in ast.walk(self.tree): + for child in ast.iter_child_nodes(parent): + parents[child] = parent + writes = [node for node in ast.walk(self.tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) + and node.func.attr == "write" + and isinstance(node.func.value, ast.Name) and node.func.value.id == "os"] + self.assertEqual(len(writes), 1, "one call site, one write per boundary") + node = writes[0] + loops = (ast.For, ast.AsyncFor, ast.While, ast.ListComp, ast.SetComp, ast.DictComp, + ast.GeneratorExp) + while node in parents: + node = parents[node] + self.assertNotIsInstance(node, loops, "no retry or spin around the write") + + def test_no_poll_sleep_close_or_output_calls(self): + called = set() + for node in ast.walk(self.tree): + if isinstance(node, ast.Call): + if isinstance(node.func, ast.Name): + called.add(node.func.id) + elif isinstance(node.func, ast.Attribute): + called.add(node.func.attr) + self.assertEqual(called & self.FORBIDDEN_CALLS, set()) + + def test_no_forbidden_import_and_no_standard_stream_access(self): + imported = set() + for node in ast.walk(self.tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom): + imported.add((node.module or "").split(".")[0]) + self.assertEqual(imported & self.FORBIDDEN_IMPORTS, set()) + attributes = {node.attr for node in ast.walk(self.tree) if isinstance(node, ast.Attribute)} + self.assertEqual(attributes & {"stdout", "stderr", "stdin"}, set()) + + +class ApiContractTests(unittest.TestCase): + def test_constructor_and_methods_take_only_what_the_contract_names(self): + init = inspect.signature(RunPhaseWriter.__init__).parameters + self.assertEqual(list(init), ["self", "fd", "clock"]) + self.assertIs(init["clock"].kind, inspect.Parameter.KEYWORD_ONLY) + self.assertEqual(list(inspect.signature(RunPhaseWriter.advance).parameters), + ["self", "phase"]) + self.assertEqual(list(inspect.signature(RunPhaseWriter.phase_timing).parameters), + ["self"]) + + def test_public_surface_has_no_trial_wait_or_policy_member(self): + public = {name for name in dir(RunPhaseWriter) if not name.startswith("_")} + self.assertEqual(public, {"advance", "failure", "phase", "phase_timing", "sequence", + "unsent", "export_requested", "last_handed_frame_bytes"}) + self.assertEqual(set(transport.__all__), { + "MAX_RECORD_BYTES", "MAX_RECORDS", "MAX_UNSENT_BYTES", "PHASES", + "PhaseTransportError", "RunPhaseWriter", "encode_record"}) + + def test_the_error_is_a_harness_fault_that_an_oserror_handler_cannot_swallow(self): + error = PhaseTransportError("text", reason="short_write") + self.assertEqual(error.code, "harness_internal_error") + self.assertEqual(PhaseTransportError.code, "harness_internal_error") + self.assertFalse(issubclass(PhaseTransportError, OSError)) + catalog = json.loads(CATALOG_PATH.read_text(encoding="utf-8")) + entry = catalog["codes"]["harness_internal_error"] + self.assertEqual(entry["class"], "engine_fault") + self.assertIs(entry["retryable"], False) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_selected_primary_plan.cpp b/tests/test_selected_primary_plan.cpp new file mode 100644 index 000000000..a8bd74d65 --- /dev/null +++ b/tests/test_selected_primary_plan.cpp @@ -0,0 +1,802 @@ +// Selected primary planner fixtures (pineforge::plan_selected_primary). +// +// Every expected value below is derived by hand from the frozen planner +// semantics and the native consumer's grouping and seal rules. The native +// observed-counter equality and the OFF identity are covered by the consumer's +// own counters, never by this file. +// +// Black box through the public call only: rows in, plan out. The planner reads +// timestamps only, so the prices are inert. Clocks are UTC 24x7 except where a +// row says otherwise, so the expected groups are plain hour/minute arithmetic +// from kBase. Two rows need the host's tzdata (America/New_York) and say so. +// +// Group arithmetic used throughout (1m input unless stated): a group is a run +// of rows under one script interval; an input at the interval's last minute +// reaches the script's next period open and seals it; a later group's first +// row seals a pending one; a final window group neither rule seals is never +// executed and is not counted in window_script_bars, though its rows are in +// window_input_bars. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using pineforge::Bar; +using pineforge::SelectedPlanRequest; +using pineforge::SelectedPlanStatus; +using pineforge::SelectedPrimaryPlan; + +static_assert(std::is_same::value, + "the frozen call signature"); + +int failures = 0; +const char* scenario = "initialization"; + +#define CHECK(condition) do { \ + if (!(condition)) { \ + ++failures; \ + std::fprintf(stderr, "FAIL [%s] %s:%d %s\n", scenario, __FILE__, \ + __LINE__, #condition); \ + } \ +} while (0) + +constexpr std::int64_t kMin = 60'000; +constexpr std::int64_t kHour = 60 * kMin; +constexpr std::int64_t kBase = 1'704'067'200'000; // 2024-01-01T00:00:00Z, a Monday +constexpr std::int64_t kI53 = (std::int64_t{1} << 53) - 1; + +Bar row_at(std::int64_t timestamp) { return Bar{100.0, 101.0, 99.0, 100.5, 10.0, timestamp}; } + +void add_span(std::vector& rows, std::int64_t from, std::int64_t until, std::int64_t step) { + for (std::int64_t at = from; at < until; at += step) rows.push_back(row_at(at)); +} + +std::vector span(std::int64_t from, std::int64_t until, std::int64_t step) { + std::vector rows; + add_span(rows, from, until, step); + return rows; +} + +std::vector rows_at(std::initializer_list timestamps) { + std::vector rows; + for (const std::int64_t at : timestamps) rows.push_back(row_at(at)); + return rows; +} + +SelectedPlanRequest make_request(std::int64_t start, std::int64_t end, std::int64_t fed_start, + std::uint32_t preroll, const char* input_tf, + const char* script_tf) { + SelectedPlanRequest request{}; + request.start_ms = start; + request.end_ms = end; + request.fed_start_ms = fed_start; + request.preroll_bars = preroll; + request.input_tf = input_tf; + request.script_tf = script_tf; + request.chart_timezone = "UTC"; + request.engine_timezone = "UTC"; + request.session = ""; + return request; +} + +SelectedPrimaryPlan plan_of(const std::vector& rows, const SelectedPlanRequest& request) { + return pineforge::plan_selected_primary(rows.data(), rows.size(), request); +} + +struct Expect { + std::uint64_t supplied_input = 0, supplied_script = 0, available = 0, used = 0; + std::uint64_t trimmed = 0, trim_index = 0, fed_input = 0, fed_script = 0; + std::uint64_t preroll_input = 0, window_input = 0, window_script = 0; + std::int64_t trim_start = 0; + std::optional preroll_first, preroll_last, supplied_first, supplied_last; + std::optional fed_first, fed_last, window_first, window_last; + bool shortfall = false; + bool horizon = false; +}; + +void expect_ok(const SelectedPrimaryPlan& p, const Expect& x) { + CHECK(p.status == SelectedPlanStatus::Ok); + CHECK(p.option.empty()); + CHECK(p.bound == -1); + CHECK(p.supplied_input_bars == x.supplied_input); + CHECK(p.supplied_script_bars == x.supplied_script); + CHECK(p.available_script_bars == x.available); + CHECK(p.used_script_bars == x.used); + CHECK(p.trimmed_script_bars == x.trimmed); + CHECK(p.trim_index == x.trim_index); + CHECK(p.fed_input_bars == x.fed_input); + CHECK(p.fed_script_bars == x.fed_script); + CHECK(p.preroll_input_bars == x.preroll_input); + CHECK(p.window_input_bars == x.window_input); + CHECK(p.window_script_bars == x.window_script); + CHECK(p.trim_start_ms == x.trim_start); + CHECK(p.preroll_first_bar_ms == x.preroll_first); + CHECK(p.preroll_last_bar_ms == x.preroll_last); + CHECK(p.supplied_first_data_ms == x.supplied_first); + CHECK(p.supplied_last_data_ms == x.supplied_last); + CHECK(p.fed_first_data_ms == x.fed_first); + CHECK(p.fed_last_data_ms == x.fed_last); + CHECK(p.window_first_data_ms == x.window_first); + CHECK(p.window_last_data_ms == x.window_last); + CHECK(p.shortfall == x.shortfall); + CHECK(p.complete_pending_preroll_at_horizon == x.horizon); + // The frozen equalities, independent of the expected values above. + CHECK(p.supplied_input_bars == p.trim_index + p.fed_input_bars); + CHECK(p.supplied_script_bars == p.available_script_bars + p.window_script_bars); + CHECK(p.fed_input_bars == p.preroll_input_bars + p.window_input_bars); + CHECK(p.fed_script_bars == p.used_script_bars + p.window_script_bars); + CHECK(p.available_script_bars == p.used_script_bars + p.trimmed_script_bars); +} + +void expect_refused(const SelectedPrimaryPlan& p, SelectedPlanStatus status, const char* option, + int bound = -1, std::int64_t value = 0, std::int64_t previous = 0, + std::int64_t next = 0) { + CHECK(p.status == status); + CHECK(p.option == option); + CHECK(p.bound == bound); + CHECK(p.value_ms == value); + CHECK(p.previous_boundary_ms == previous); + CHECK(p.next_boundary_ms == next); + CHECK(p.supplied_input_bars == 0); + CHECK(p.fed_input_bars == 0); + CHECK(p.window_script_bars == 0); + CHECK(!p.shortfall); + CHECK(!p.complete_pending_preroll_at_horizon); + CHECK(!p.preroll_first_bar_ms.has_value()); + CHECK(!p.supplied_first_data_ms.has_value()); +} + +// Rows 00:00..05:59 every minute (six hourly groups), T = 03:00, E = 06:00, +// F = 00:00, N = 2: hours 0-2 are before T, hours 3-5 are the window. +std::vector full_six_hours() { return span(kBase, kBase + 6 * kHour, kMin); } + +SelectedPlanRequest full_six_hours_request(std::uint32_t preroll) { + return make_request(kBase + 3 * kHour, kBase + 6 * kHour, kBase, preroll, "1", "60"); +} + +Expect expect_full_six_hours_n2() { + Expect x; + x.supplied_input = 360; + x.supplied_script = 6; // 3 available + 3 window + x.available = 3; + x.used = 2; + x.trimmed = 1; + x.trim_index = 60; // the first row of hour 1 + x.fed_input = 300; + x.fed_script = 5; + x.preroll_input = 120; + x.window_input = 180; + x.window_script = 3; + x.trim_start = kBase + kHour; + x.preroll_first = kBase + kHour; + x.preroll_last = kBase + 2 * kHour; + x.supplied_first = kBase; + x.supplied_last = kBase + 6 * kHour - kMin; + x.fed_first = kBase + kHour; + x.fed_last = kBase + 6 * kHour - kMin; + x.window_first = kBase + 3 * kHour; + x.window_last = kBase + 6 * kHour - kMin; + return x; +} + +void fixture_full_1m_to_60m() { + scenario = "1m to 60m, full span, N=2"; + const SelectedPrimaryPlan p = plan_of(full_six_hours(), full_six_hours_request(2)); + expect_ok(p, expect_full_six_hours_n2()); +} + +void fixture_n_zero() { + scenario = "1m to 60m, N=0 keeps no pre-roll"; + const SelectedPrimaryPlan p = plan_of(full_six_hours(), full_six_hours_request(0)); + Expect x; + x.supplied_input = 360; + x.supplied_script = 6; + x.available = 3; + x.used = 0; + x.trimmed = 3; + x.trim_index = 180; // the first row at T + x.fed_input = 180; + x.fed_script = 3; + x.preroll_input = 0; + x.window_input = 180; + x.window_script = 3; + x.trim_start = kBase + 3 * kHour; // T itself + x.supplied_first = kBase; + x.supplied_last = kBase + 6 * kHour - kMin; + x.fed_first = kBase + 3 * kHour; + x.fed_last = kBase + 6 * kHour - kMin; + x.window_first = kBase + 3 * kHour; + x.window_last = kBase + 6 * kHour - kMin; + expect_ok(p, x); +} + +void fixture_n_above_available() { + scenario = "1m to 60m, N=10 above the 3 available groups"; + const SelectedPrimaryPlan p = plan_of(full_six_hours(), full_six_hours_request(10)); + Expect x; + x.supplied_input = 360; + x.supplied_script = 6; + x.available = 3; + x.used = 3; + x.trimmed = 0; + x.trim_index = 0; + x.fed_input = 360; + x.fed_script = 6; + x.preroll_input = 180; + x.window_input = 180; + x.window_script = 3; + x.trim_start = kBase; + x.preroll_first = kBase; + x.preroll_last = kBase + 2 * kHour; + x.supplied_first = kBase; + x.supplied_last = kBase + 6 * kHour - kMin; + x.fed_first = kBase; + x.fed_last = kBase + 6 * kHour - kMin; + x.window_first = kBase + 3 * kHour; + x.window_last = kBase + 6 * kHour - kMin; + x.shortfall = true; // available < N + expect_ok(p, x); +} + +void fixture_full_1m_to_240m() { + scenario = "1m to 240m, full span, N=3 above the 1 available group"; + const std::vector rows = span(kBase, kBase + 12 * kHour, kMin); // 720 rows + const SelectedPlanRequest request = + make_request(kBase + 4 * kHour, kBase + 12 * kHour, kBase, 3, "1", "240"); + Expect x; + x.supplied_input = 720; + x.supplied_script = 3; // [00,04) before T, [04,08) and [08,12) in the window + x.available = 1; + x.used = 1; + x.trimmed = 0; + x.trim_index = 0; + x.fed_input = 720; + x.fed_script = 3; + x.preroll_input = 240; + x.window_input = 480; + x.window_script = 2; + x.trim_start = kBase; + x.preroll_first = kBase; + x.preroll_last = kBase; + x.supplied_first = kBase; + x.supplied_last = kBase + 12 * kHour - kMin; + x.fed_first = kBase; + x.fed_last = kBase + 12 * kHour - kMin; + x.window_first = kBase + 4 * kHour; + x.window_last = kBase + 12 * kHour - kMin; + x.shortfall = true; + expect_ok(plan_of(rows, request), x); +} + +void fixture_sparse_1m_to_60m() { + scenario = "1m to 60m, sparse: a missing hour, partial hours, F above the first slot"; + // Hour 1 minutes 00-29 (before T, left pending, sealed by the next group), + // hour 2 absent (no group), hour 3 full, hour 4 minutes 10-12 (sealed by + // hour 5's first row, not by its own last minute), hour 5 full. + std::vector rows; + add_span(rows, kBase + kHour, kBase + kHour + 30 * kMin, kMin); + add_span(rows, kBase + 3 * kHour, kBase + 4 * kHour, kMin); + add_span(rows, kBase + 4 * kHour + 10 * kMin, kBase + 4 * kHour + 13 * kMin, kMin); + add_span(rows, kBase + 5 * kHour, kBase + 6 * kHour, kMin); + const SelectedPlanRequest request = + make_request(kBase + 3 * kHour, kBase + 6 * kHour, kBase + kHour, 2, "1", "60"); + Expect x; + x.supplied_input = 153; + x.supplied_script = 4; // 1 available + 3 window groups + x.available = 1; + x.used = 1; + x.trimmed = 0; + x.trim_index = 0; + x.fed_input = 153; + x.fed_script = 4; + x.preroll_input = 30; + x.window_input = 123; + x.window_script = 3; + x.trim_start = kBase + kHour; + x.preroll_first = kBase + kHour; + x.preroll_last = kBase + kHour; + x.supplied_first = kBase + kHour; + x.supplied_last = kBase + 6 * kHour - kMin; + x.fed_first = kBase + kHour; + x.fed_last = kBase + 6 * kHour - kMin; + x.window_first = kBase + 3 * kHour; + x.window_last = kBase + 6 * kHour - kMin; + x.shortfall = true; // 1 available < 2 + expect_ok(plan_of(rows, request), x); +} + +void fixture_sparse_1m_to_240m() { + scenario = "1m to 240m, sparse rows, last group sealed at 15:59, N=1 trims one group"; + // [00,04): 00:00 and 03:59 (sealed by its last minute); [04,08): 05:30 + // (pending, sealed by the next group); [08,12): 09:10 and 09:11 (sealed by + // 13:00); [12,16): 13:00 and 15:59 (sealed by its last minute). + const std::vector rows = rows_at({kBase, kBase + 3 * kHour + 59 * kMin, + kBase + 5 * kHour + 30 * kMin, + kBase + 9 * kHour + 10 * kMin, + kBase + 9 * kHour + 11 * kMin, kBase + 13 * kHour, + kBase + 15 * kHour + 59 * kMin}); + const SelectedPlanRequest request = + make_request(kBase + 8 * kHour, kBase + 16 * kHour, kBase, 1, "1", "240"); + Expect x; + x.supplied_input = 7; + x.supplied_script = 4; + x.available = 2; + x.used = 1; + x.trimmed = 1; + x.trim_index = 2; // the first row of [04,08) + x.fed_input = 5; + x.fed_script = 3; + x.preroll_input = 1; + x.window_input = 4; + x.window_script = 2; + x.trim_start = kBase + 4 * kHour; + x.preroll_first = kBase + 4 * kHour; + x.preroll_last = kBase + 4 * kHour; + x.supplied_first = kBase; + x.supplied_last = kBase + 15 * kHour + 59 * kMin; + x.fed_first = kBase + 5 * kHour + 30 * kMin; + x.fed_last = kBase + 15 * kHour + 59 * kMin; + x.window_first = kBase + 9 * kHour + 10 * kMin; + x.window_last = kBase + 15 * kHour + 59 * kMin; + expect_ok(plan_of(rows, request), x); +} + +void fixture_passthrough_raw() { + scenario = "input = script (5m), FeedTolerant raw partition and the canonical path"; + const std::vector rows = span(kBase, kBase + 60 * kMin, 5 * kMin); // 12 rows + for (const bool tolerant : {true, false}) { + SelectedPlanRequest request = + make_request(kBase + 30 * kMin, kBase + 60 * kMin, kBase, 4, "5", "5"); + request.feed_tolerant = tolerant; + Expect x; + x.supplied_input = 12; + x.supplied_script = 12; // every row is its own sealed bar + x.available = 6; + x.used = 4; + x.trimmed = 2; + x.trim_index = 2; + x.fed_input = 10; + x.fed_script = 10; + x.preroll_input = 4; + x.window_input = 6; + x.window_script = 6; + x.trim_start = kBase + 10 * kMin; + x.preroll_first = kBase + 10 * kMin; + x.preroll_last = kBase + 25 * kMin; + x.supplied_first = kBase; + x.supplied_last = kBase + 55 * kMin; + x.fed_first = kBase + 10 * kMin; + x.fed_last = kBase + 55 * kMin; + x.window_first = kBase + 30 * kMin; + x.window_last = kBase + 55 * kMin; + expect_ok(plan_of(rows, request), x); + } +} + +void fixture_off_grid_raw_labels() { + scenario = "FeedTolerant 5m passthrough on off-grid rows: the raw label is the row's own"; + const std::vector rows = + rows_at({kBase + 1 * kMin, kBase + 7 * kMin, kBase + 31 * kMin, kBase + 44 * kMin}); + const SelectedPlanRequest request = + make_request(kBase + 30 * kMin, kBase + 60 * kMin, kBase, 1, "5", "5"); + const SelectedPrimaryPlan p = plan_of(rows, request); + CHECK(p.status == SelectedPlanStatus::Ok); + CHECK(p.available_script_bars == 2); + CHECK(p.used_script_bars == 1); + CHECK(p.trim_index == 1); + CHECK(p.window_script_bars == 2); + CHECK(p.window_input_bars == 2); + CHECK(p.preroll_input_bars == 1); + CHECK(p.trim_start_ms == kBase + 7 * kMin); + CHECK(p.preroll_first_bar_ms == std::optional(kBase + 7 * kMin)); + CHECK(p.preroll_last_bar_ms == std::optional(kBase + 7 * kMin)); + CHECK(p.fed_first_data_ms == std::optional(kBase + 7 * kMin)); +} + +void fixture_empty_feed() { + scenario = "empty feed"; + const SelectedPlanRequest request = full_six_hours_request(3); + const SelectedPrimaryPlan p = pineforge::plan_selected_primary(nullptr, 0, request); + Expect x; + x.trim_start = kBase + 3 * kHour; // no kept group: T itself + x.shortfall = true; // requested 3, available 0 + expect_ok(p, x); + + const SelectedPrimaryPlan zero = + pineforge::plan_selected_primary(nullptr, 0, full_six_hours_request(0)); + Expect y; + y.trim_start = kBase + 3 * kHour; + expect_ok(zero, y); + + // A non-null array of count 0 is the same empty feed. + const std::vector none; + expect_ok(plan_of(none, request), x); +} + +void fixture_no_window_pending_preroll() { + scenario = "no window row: a pending pre-T group needs completion at the horizon"; + // Hour 1 full (sealed by its last minute), hour 2 minutes 00-29 (pending + // when the rows end; T = 03:00 is never reached). + const std::vector rows = span(kBase + kHour, kBase + 2 * kHour + 30 * kMin, kMin); + Expect x; + x.supplied_input = 90; + x.supplied_script = 2; + x.available = 2; + x.used = 2; + x.trimmed = 0; + x.trim_index = 0; + x.fed_input = 90; + x.fed_script = 2; + x.preroll_input = 90; + x.window_input = 0; + x.window_script = 0; + x.trim_start = kBase + kHour; + x.preroll_first = kBase + kHour; + x.preroll_last = kBase + 2 * kHour; + x.supplied_first = kBase + kHour; + x.supplied_last = kBase + 2 * kHour + 29 * kMin; + x.fed_first = kBase + kHour; + x.fed_last = kBase + 2 * kHour + 29 * kMin; + x.horizon = true; + expect_ok(plan_of(rows, full_six_hours_request(2)), x); + + // N=0 keeps no group, so there is nothing to complete. + const SelectedPrimaryPlan none = plan_of(rows, full_six_hours_request(0)); + CHECK(none.status == SelectedPlanStatus::Ok); + CHECK(none.used_script_bars == 0); + CHECK(none.trim_index == 90); + CHECK(none.fed_input_bars == 0); + CHECK(none.fed_script_bars == 0); + CHECK(!none.fed_first_data_ms.has_value()); + CHECK(!none.complete_pending_preroll_at_horizon); + + // A pre-T group its own last minute sealed is not pending. + const std::vector sealed = span(kBase + kHour, kBase + 3 * kHour, kMin); + const SelectedPrimaryPlan done = plan_of(sealed, full_six_hours_request(2)); + CHECK(done.status == SelectedPlanStatus::Ok); + CHECK(done.used_script_bars == 2); + CHECK(done.window_input_bars == 0); + CHECK(!done.complete_pending_preroll_at_horizon); +} + +void fixture_unsealed_window_tail() { + scenario = "the ordinary unsealed window tail is not an executed window callback"; + // Hours 2, 3 and 4 full, hour 5 minutes 00-29 only. T = 03:00, E = 06:00. + const std::vector rows = span(kBase + 2 * kHour, kBase + 5 * kHour + 30 * kMin, kMin); + const SelectedPlanRequest request = + make_request(kBase + 3 * kHour, kBase + 6 * kHour, kBase + 2 * kHour, 1, "1", "60"); + Expect x; + x.supplied_input = 210; + x.supplied_script = 3; // 1 available + 2 executed window bars + x.available = 1; + x.used = 1; + x.trimmed = 0; + x.trim_index = 0; + x.fed_input = 210; + x.fed_script = 3; + x.preroll_input = 60; + x.window_input = 150; // the tail's 30 rows still count as window rows + x.window_script = 2; // hour 5 never seals + x.trim_start = kBase + 2 * kHour; + x.preroll_first = kBase + 2 * kHour; + x.preroll_last = kBase + 2 * kHour; + x.supplied_first = kBase + 2 * kHour; + x.supplied_last = kBase + 5 * kHour + 29 * kMin; + x.fed_first = kBase + 2 * kHour; + x.fed_last = kBase + 5 * kHour + 29 * kMin; + x.window_first = kBase + 3 * kHour; + x.window_last = kBase + 5 * kHour + 29 * kMin; + expect_ok(plan_of(rows, request), x); +} + +void fixture_closing_seal_clipped_session() { + scenario = "session 0930-1600: the final input reaching the last traded close seals"; + const std::int64_t open = kBase + 9 * kHour + 30 * kMin; + // 09:30..15:59 every minute: groups open at 09:30, 10:30, ..., 15:30; the + // 15:30 group's nominal end is 16:30 but trading ends at 16:00. + std::vector rows = span(open, kBase + 16 * kHour, kMin); // 390 rows + SelectedPlanRequest request = + make_request(open + 5 * kHour, open + 24 * kHour, open, 1, "1", "60"); + request.session = "0930-1600"; + Expect x; + x.supplied_input = 390; + x.supplied_script = 7; // 5 available + 2 window bars + x.available = 5; + x.used = 1; + x.trimmed = 4; + x.trim_index = 240; + x.fed_input = 150; + x.fed_script = 3; + x.preroll_input = 60; + x.window_input = 90; + x.window_script = 2; // 14:30 by its last minute, 15:30 by the closing rule + x.trim_start = open + 4 * kHour; + x.preroll_first = open + 4 * kHour; + x.preroll_last = open + 4 * kHour; + x.supplied_first = open; + x.supplied_last = kBase + 16 * kHour - kMin; + x.fed_first = open + 4 * kHour; + x.fed_last = kBase + 16 * kHour - kMin; + x.window_first = open + 5 * kHour; + x.window_last = kBase + 16 * kHour - kMin; + expect_ok(plan_of(rows, request), x); + + // Without the 15:59 row the final input stops short of 16:00: unsealed. + rows.pop_back(); + Expect y = x; + y.supplied_input = 389; + y.supplied_script = 6; + y.fed_input = 149; + y.fed_script = 2; + y.window_input = 89; + y.window_script = 1; + y.supplied_last = kBase + 16 * kHour - 2 * kMin; + y.fed_last = kBase + 16 * kHour - 2 * kMin; + y.window_last = kBase + 16 * kHour - 2 * kMin; + expect_ok(plan_of(rows, request), y); +} + +void fixture_all_eight_primary_tokens() { + // Input 1m over 00:00..15:59, T = 08:00, E = 16:00, N = 2: every token's + // grid divides 480 minutes, so each hour/minute count is exact. + static const struct { + const char* token; + std::int64_t minutes; + } kTokens[] = {{"1", 1}, {"3", 3}, {"5", 5}, {"15", 15}, + {"30", 30}, {"60", 60}, {"120", 120}, {"240", 240}}; + const std::vector rows = span(kBase, kBase + 16 * kHour, kMin); // 960 rows + for (const bool tolerant : {true, false}) { + for (const auto& entry : kTokens) { + scenario = entry.token; + SelectedPlanRequest request = + make_request(kBase + 8 * kHour, kBase + 16 * kHour, kBase, 2, "1", entry.token); + request.feed_tolerant = tolerant; + const std::int64_t m = entry.minutes; + const std::uint64_t groups = static_cast(480 / m); + Expect x; + x.supplied_input = 960; + x.supplied_script = 2 * groups; + x.available = groups; + x.used = 2; + x.trimmed = groups - 2; + x.trim_index = static_cast(480 - 2 * m); + x.fed_input = static_cast(480 + 2 * m); + x.fed_script = 2 + groups; + x.preroll_input = static_cast(2 * m); + x.window_input = 480; + x.window_script = groups; + x.trim_start = kBase + (480 - 2 * m) * kMin; + x.preroll_first = x.trim_start; + x.preroll_last = kBase + (480 - m) * kMin; + x.supplied_first = kBase; + x.supplied_last = kBase + 16 * kHour - kMin; + x.fed_first = x.trim_start; + x.fed_last = x.supplied_last; + x.window_first = kBase + 8 * kHour; + x.window_last = x.supplied_last; + expect_ok(plan_of(rows, request), x); + } + } +} + +void fixture_utc_aliases() { + scenario = "UTC aliases admit pairwise"; + const char* const names[] = {"", "UTC", "GMT", "Etc/UTC", "Etc/GMT"}; + for (const char* chart : names) { + for (const char* engine : names) { + SelectedPlanRequest request = full_six_hours_request(2); + request.chart_timezone = chart; + request.engine_timezone = engine; + expect_ok(plan_of(full_six_hours(), request), expect_full_six_hours_n2()); + } + } +} + +void fixture_same_non_utc_zone() { + // Needs the host's tzdata for America/New_York. January is standard time + // all day, so the hourly grid stays on the UTC hours and the counts are + // the UTC fixture's. + scenario = "the same non-UTC zone admits (needs tzdata)"; + SelectedPlanRequest request = full_six_hours_request(2); + request.chart_timezone = "America/New_York"; + request.engine_timezone = "America/New_York"; + expect_ok(plan_of(full_six_hours(), request), expect_full_six_hours_n2()); +} + +void fixture_clock_refusals() { + scenario = "distinct clock identities refuse"; + struct Pair { + const char* chart; + const char* engine; + }; + // Refused at the identity comparison, before any tzdata is read. + static const Pair kMismatches[] = { + {"UTC", "America/New_York"}, {"America/New_York", "UTC"}, + {"US/Eastern", "America/New_York"}, {"Etc/UTC", "Asia/Tokyo"}, + {" UTC", "UTC"}, {"utc", "UTC"}, {"UTC+0", "UTC"}, {"Etc/GMT+0", "UTC"}}; + for (const Pair& pair : kMismatches) { + SelectedPlanRequest request = full_six_hours_request(2); + request.chart_timezone = pair.chart; + request.engine_timezone = pair.engine; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::CalendarUnsupported, + "timezone"); + } + + scenario = "equal clock tokens do not excuse invalid grammar"; + SelectedPlanRequest zone = full_six_hours_request(2); + zone.chart_timezone = "Not/AZone"; + zone.engine_timezone = "Not/AZone"; + expect_refused(plan_of(full_six_hours(), zone), SelectedPlanStatus::CalendarUnsupported, + "engine_timezone"); + + SelectedPlanRequest session = full_six_hours_request(2); + session.session = "garbage"; + expect_refused(plan_of(full_six_hours(), session), SelectedPlanStatus::CalendarUnsupported, + "session"); + session.session = "2500-0100"; + expect_refused(plan_of(full_six_hours(), session), SelectedPlanStatus::CalendarUnsupported, + "session"); +} + +void fixture_request_refusals() { + scenario = "primary token and input timeframe"; + const char* const bad_scripts[] = {"", "D", "1D", "W", "M", "2", "7", + "45", "1H", "5S", " 5", "05", "360", "15S"}; + for (const char* token : bad_scripts) { + SelectedPlanRequest request = full_six_hours_request(2); + request.script_tf = token; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, + "script_tf"); + } + const char* const bad_inputs[] = {"", "1H", "01"}; + for (const char* token : bad_inputs) { + SelectedPlanRequest request = full_six_hours_request(2); + request.input_tf = token; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, + "input_tf"); + } + // Valid grammar, a pairing the consumer does not group. + const char* const bad_pairs[] = {"D", "7", "120"}; + for (const char* token : bad_pairs) { + SelectedPlanRequest request = full_six_hours_request(2); + request.input_tf = token; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::CalendarUnsupported, + "timeframes"); + } + + scenario = "i53 bounds, order, pre-roll cap and the row array"; + const std::int64_t over = kI53 + 1; + SelectedPlanRequest request = full_six_hours_request(2); + request.start_ms = over; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, + "start_ms", 0, over); + request = full_six_hours_request(2); + request.end_ms = over; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, "end_ms", + 1, over); + request = full_six_hours_request(2); + request.fed_start_ms = -over; + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, + "fed_start_ms", 2, -over); + + request = full_six_hours_request(2); + request.end_ms = request.start_ms; // T == E + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, "end_ms", + 1, request.end_ms, request.start_ms); + request = full_six_hours_request(2); + request.end_ms = request.start_ms - kHour; // T > E + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, "end_ms", + 1, request.end_ms, request.start_ms); + request = full_six_hours_request(2); + request.fed_start_ms = request.start_ms + kHour; // F > T + expect_refused(plan_of(full_six_hours(), request), SelectedPlanStatus::RequestInvalid, + "fed_start_ms", 2, request.fed_start_ms, 0, request.start_ms); + + expect_refused(plan_of(full_six_hours(), full_six_hours_request(5001)), + SelectedPlanStatus::RequestInvalid, "preroll_bars", -1, 5001); + const SelectedPrimaryPlan cap = pineforge::plan_selected_primary( + nullptr, 0, full_six_hours_request(5000)); + CHECK(cap.status == SelectedPlanStatus::Ok); + CHECK(cap.shortfall); + + expect_refused(pineforge::plan_selected_primary(nullptr, 3, full_six_hours_request(2)), + SelectedPlanStatus::RequestInvalid, "rows"); +} + +void fixture_boundary_refusals() { + scenario = "T, E and F must each be the nominal open of the native script interval"; + const std::vector rows = full_six_hours(); + SelectedPlanRequest request = full_six_hours_request(2); + request.start_ms = kBase + 3 * kHour + kMin; + expect_refused(plan_of(rows, request), SelectedPlanStatus::BoundaryUnaligned, "start_ms", 0, + kBase + 3 * kHour + kMin, kBase + 3 * kHour, kBase + 4 * kHour); + + request = full_six_hours_request(2); + request.end_ms = kBase + 6 * kHour + 5 * kMin; + expect_refused(plan_of(rows, request), SelectedPlanStatus::BoundaryUnaligned, "end_ms", 1, + kBase + 6 * kHour + 5 * kMin, kBase + 6 * kHour, kBase + 7 * kHour); + + request = full_six_hours_request(2); + request.fed_start_ms = kBase + kHour + kMin; + expect_refused(plan_of(rows, request), SelectedPlanStatus::BoundaryUnaligned, "fed_start_ms", 2, + kBase + kHour + kMin, kBase + kHour, kBase + 2 * kHour); + + // T is checked before E and F. + request = full_six_hours_request(2); + request.start_ms = kBase + 3 * kHour + kMin; + request.end_ms = kBase + 6 * kHour + 5 * kMin; + const SelectedPrimaryPlan first = plan_of(rows, request); + CHECK(first.status == SelectedPlanStatus::BoundaryUnaligned); + CHECK(first.bound == 0); + + // 240m: 03:00 is on the hour grid but not on the 4-hour grid. + request = make_request(kBase + 3 * kHour, kBase + 8 * kHour, kBase, 1, "1", "240"); + expect_refused(plan_of(rows, request), SelectedPlanStatus::BoundaryUnaligned, "start_ms", 0, + kBase + 3 * kHour, kBase, kBase + 4 * kHour); +} + +void fixture_row_refusals() { + scenario = "rows must strictly increase and lie in F <= timestamp < E"; + const SelectedPlanRequest request = full_six_hours_request(2); + + expect_refused(plan_of(rows_at({kBase + 3 * kHour, kBase + 3 * kHour + 2 * kMin, + kBase + 3 * kHour + kMin}), + request), + SelectedPlanStatus::RowsUnordered, "rows", -1, kBase + 3 * kHour + kMin, + kBase + 3 * kHour + 2 * kMin); + expect_refused(plan_of(rows_at({kBase + kHour, kBase + kHour}), request), + SelectedPlanStatus::RowsUnordered, "rows", -1, kBase + kHour, kBase + kHour); + + expect_refused(plan_of(rows_at({kBase - kMin, kBase}), request), + SelectedPlanStatus::FeedRangeInvalid, "rows", 2, kBase - kMin, kBase, + kBase + 6 * kHour); + expect_refused(plan_of(rows_at({kBase + 5 * kHour, kBase + 6 * kHour}), request), + SelectedPlanStatus::FeedRangeInvalid, "rows", 1, kBase + 6 * kHour, kBase, + kBase + 6 * kHour); + // One row short of E is admitted. + const SelectedPrimaryPlan edge = + plan_of(rows_at({kBase, kBase + 6 * kHour - kMin}), request); + CHECK(edge.status == SelectedPlanStatus::Ok); + CHECK(edge.supplied_input_bars == 2); +} + +} // namespace + +int main() { + fixture_full_1m_to_60m(); + fixture_n_zero(); + fixture_n_above_available(); + fixture_full_1m_to_240m(); + fixture_sparse_1m_to_60m(); + fixture_sparse_1m_to_240m(); + fixture_passthrough_raw(); + fixture_off_grid_raw_labels(); + fixture_empty_feed(); + fixture_no_window_pending_preroll(); + fixture_unsealed_window_tail(); + fixture_closing_seal_clipped_session(); + fixture_all_eight_primary_tokens(); + fixture_utc_aliases(); + fixture_same_non_utc_zone(); + fixture_clock_refusals(); + fixture_request_refusals(); + fixture_boundary_refusals(); + fixture_row_refusals(); + if (failures != 0) { + std::fprintf(stderr, "test_selected_primary_plan: %d check(s) failed\n", failures); + return 1; + } + std::printf("test_selected_primary_plan: all checks passed\n"); + return 0; +} diff --git a/tests/test_selected_production_routes.cpp b/tests/test_selected_production_routes.cpp new file mode 100644 index 000000000..fcea9045d --- /dev/null +++ b/tests/test_selected_production_routes.cpp @@ -0,0 +1,1094 @@ +// Selected-window production-route controls (frozen observer ABI +// pineforge/execution_observer.h, frozen selected-window ABI pineforge/selected_window.h). +// Companion of tests/test_selected_window_routes.cpp, which is not reworked here. +// +// Two production entries that the route fixture did not reach, each driven from outside +// through its real public symbols, selected versus OFF, single read: +// +// 1. The native module entry. include/pineforge/native_module.hpp is the production macro: +// PINEFORGE_EXPORT_NATIVE_STRATEGY(RouteHost) below expands to the loadable module's +// real extern "C" exports (strategy_create, strategy_free, run_backtest, +// run_backtest_full, report_free), and run_backtest / run_backtest_full publish the +// report through native_module::publish_report (native_module.hpp:173-183), which is +// what this TU reads. Nothing is re-created locally: the TU calls the exports the macro +// defined, the only definitions of those names in this executable. +// 2. The C ABI stream family. The exported strategy_stream_begin, strategy_stream_push_bar, +// strategy_stream_end and strategy_stream_fill_report (src/c_abi.cpp:693-814) are called +// on a handle; no C++ stream_* member is used for the run. +// +// The host is a custom NativeStrategyHost (RouteHost) that notes every callback and its +// presentation into one test-owned event log (g_log); the checks read that log. The selected +// side registers an observer and a window through the exported strategy_set_execution_ +// observer_v1 / strategy_set_selected_window_v1; the OFF side registers neither. The window +// starts at the first row (N = 0), so the selected curve is the anchor plus the SAME M +// observations the OFF curve holds, and the comparison is anchor-aware: trades and scalars +// field by field, selected points 1..M against OFF points 0..M-1, the anchor at T with the +// initial capital, and the metrics the documented report semantics leave equal. The anchorless OFF +// curve is never asserted equal to the selected curve. +// +// Rows (stable labels, printed as "ROW