diff --git a/.github/workflows/ci-post.yml b/.github/workflows/ci-post.yml index 46f4de36f..f09472f1c 100644 --- a/.github/workflows/ci-post.yml +++ b/.github/workflows/ci-post.yml @@ -232,7 +232,78 @@ jobs: path: build-ci-preflight/ if-no-files-found: warn - # Every CI round fits fifteen minutes of wall time. Each configuration builds + # Full lab Release configuration, split by generated target names. Each shard + # captures ctest -N and independent CMake/link inventories before and after a + # real build. build-tests-result requires their exact union; smoke stays below. + build-tests: + needs: [authz] + if: ${{ needs.authz.result == 'success' }} + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout the exact workflow head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: Install dependencies + run: | + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev tzdata-legacy ccache ninja-build + - name: Exercise the broken registered header control + if: ${{ matrix.shard == 1 }} + run: python3 scripts/test_ci_build_tests.py WorkflowControl + - name: Build every Release test target + run: | + python3 scripts/ci_build_tests.py build --source . --build-dir build-tests-${{ matrix.shard }} --artifact-dir ci-build-tests --shard ${{ matrix.shard }}/8 --jobs "$(getconf _NPROCESSORS_ONLN)" + - name: Retain Release target inventories and build logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ci-release-build-${{ matrix.shard }} + path: ci-build-tests/ + if-no-files-found: error + + build-tests-result: + needs: [authz, build-tests] + if: ${{ needs.authz.result == 'success' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Require every Release build shard to succeed + env: + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + run: test "$BUILD_TESTS_RESULT" = success + - name: Checkout the exact workflow head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + - name: Collect every Release target inventory + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: ci-release-build-* + path: ci-build-tests/ + merge-multiple: true + - name: Require the complete Release build + run: | + python3 scripts/ci_build_tests.py union --artifact-dir ci-build-tests --shards 8 + + # Every CI round fits fifteen minutes of wall time. Each smoke configuration builds # the PR smoke target and runs the fast CTest label; the full profiles run in # the lab's verify on the exact head tree before every merge (docs/ci.md, # "Where the full suites run"). @@ -412,7 +483,7 @@ jobs: # maintainers, rather than this build context. build-gate: name: build - needs: [authz, changes, preflight, build-x86-gcc133, smoke, smoke-arm, sanitizers, corpus-parity-subset] + needs: [authz, changes, preflight, build-x86-gcc133, build-tests, build-tests-result, smoke, smoke-arm, sanitizers, corpus-parity-subset] if: always() # The aggregate runs inline Python and shell on the light pool class. runs-on: ubuntu-24.04 @@ -434,6 +505,8 @@ jobs: DOCS_ONLY: ${{ needs.changes.outputs.docs_only }} PREFLIGHT_RESULT: ${{ needs.preflight.result }} X86_GCC133_RESULT: ${{ needs.build-x86-gcc133.result }} + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + BUILD_TESTS_UNION_RESULT: ${{ needs.build-tests-result.result }} SMOKE_RESULT: ${{ needs.smoke.result }} ARM_SMOKE_RESULT: ${{ needs.smoke-arm.result }} SANITIZER_RESULT: ${{ needs.sanitizers.result }} @@ -441,6 +514,7 @@ jobs: run: | printf '%s\n' "authz: $AUTHZ_RESULT" "changes: $CHANGES_RESULT, docs only: ${DOCS_ONLY:-unanswered}" \ "preflight: $PREFLIGHT_RESULT" "native x86 GNU 13.3.0: $X86_GCC133_RESULT" \ + "Release test build: $BUILD_TESTS_RESULT" "Release target union: $BUILD_TESTS_UNION_RESULT" \ "amd64 smoke: $SMOKE_RESULT" "arm64 smoke: $ARM_SMOKE_RESULT" \ "sanitizers: $SANITIZER_RESULT" "corpus parity (subset): $PARITY_RESULT" # Every smoke leg and the subset parity must pass. Only a classified @@ -451,5 +525,7 @@ jobs: [[ "$AUTHZ_RESULT" == "success" && "$CHANGES_RESULT" == "success" ]] \ && [[ "$PREFLIGHT_RESULT" == "success" ]] \ && [[ "$X86_GCC133_RESULT" == "success" ]] \ + && [[ "$BUILD_TESTS_RESULT" == "success" ]] \ + && [[ "$BUILD_TESTS_UNION_RESULT" == "success" ]] \ && passed "$SMOKE_RESULT" && passed "$ARM_SMOKE_RESULT" \ && passed "$SANITIZER_RESULT" && passed "$PARITY_RESULT" diff --git a/.github/workflows/gke-pr.yml b/.github/workflows/gke-pr.yml index 78b6ab279..74de4bfdd 100644 --- a/.github/workflows/gke-pr.yml +++ b/.github/workflows/gke-pr.yml @@ -306,6 +306,74 @@ jobs: path: ci-native-x86-gcc133/ if-no-files-found: warn + # Compile the full lab Release population in parallel, including on docs-only + # and .github-only PRs. CTest execution stays in the existing proof jobs. + build-tests: + needs: [admit] + if: ${{ needs.admit.outputs.allowed == 'true' }} + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3, 4, 5, 6, 7, 8] + runs-on: ubuntu-24.04 + permissions: {contents: read} + steps: + - name: Checkout the admitted head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: ${{ github.repository }} + ref: ${{ needs.admit.outputs.head_sha }} + fetch-depth: 0 + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.12' + - name: Install dependencies + run: | + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y libeigen3-dev tzdata-legacy ccache ninja-build + - name: Exercise the broken registered header control + if: ${{ matrix.shard == 1 }} + run: python3 scripts/test_ci_build_tests.py WorkflowControl + - name: Build every Release test target + run: | + python3 scripts/ci_build_tests.py build --source . --build-dir build-tests-${{ matrix.shard }} --artifact-dir ci-build-tests --shard ${{ matrix.shard }}/8 --jobs "$(getconf _NPROCESSORS_ONLN)" + - name: Retain Release target inventories and build logs + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ci-release-build-${{ matrix.shard }} + path: ci-build-tests/ + if-no-files-found: error + + build-tests-result: + needs: [admit, build-tests] + if: ${{ needs.admit.outputs.allowed == 'true' }} + runs-on: ubuntu-24.04 + permissions: {contents: read} + steps: + - name: Require every Release build shard to succeed + env: + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + run: test "$BUILD_TESTS_RESULT" = success + - name: Checkout the admitted head + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + repository: ${{ github.repository }} + ref: ${{ needs.admit.outputs.head_sha }} + fetch-depth: 0 + persist-credentials: false + - name: Collect every Release target inventory + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: ci-release-build-* + path: ci-build-tests/ + merge-multiple: true + - name: Require the complete Release build + run: | + python3 scripts/ci_build_tests.py union --artifact-dir ci-build-tests --shards 8 + build: needs: [admit] if: ${{ needs.admit.outputs.allowed == 'true' && needs.admit.outputs.docs_only != 'true' && needs.admit.outputs.github_only != 'true' }} @@ -505,7 +573,7 @@ jobs: test "$PARITY_RESULT" = success result: - needs: [admit, preflight, build-x86-gcc133, legs] + needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs] if: always() runs-on: ubuntu-24.04 permissions: {contents: read} @@ -597,6 +665,8 @@ jobs: ALLOWED: ${{ needs.admit.outputs.allowed }} PREFLIGHT_RESULT: ${{ needs.preflight.result }} X86_RESULT: ${{ needs.build-x86-gcc133.result }} + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + BUILD_TESTS_UNION_RESULT: ${{ needs.build-tests-result.result }} LEGS_RESULT: ${{ needs.legs.result }} ADMIT_DOCS_ONLY: ${{ needs.admit.outputs.docs_only }} ADMIT_GITHUB_ONLY: ${{ needs.admit.outputs.github_only }} @@ -612,6 +682,8 @@ jobs: test "$ALLOWED" = true || fail "admission did not allow the head" test "$PREFLIGHT_RESULT" = success || fail "preflight did not pass" test "$X86_RESULT" = success || fail "native x86 GNU 13.3.0 build did not pass" + test "$BUILD_TESTS_RESULT" = success || fail "Release test build did not pass" + test "$BUILD_TESTS_UNION_RESULT" = success || fail "Release target union did not pass" test "$SECOND_DOCS_ONLY" = "$ADMIT_DOCS_ONLY" || fail "docs-only classification changed" test "$SECOND_GITHUB_ONLY" = "$ADMIT_GITHUB_ONLY" || fail "github-only classification changed" test "$SECOND_ARM_RELEVANT" = "$ADMIT_ARM_RELEVANT" || fail "arm relevance classification changed" diff --git a/CHANGELOG.md b/CHANGELOG.md index 87d4a9255..36818dd0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,12 @@ version number follows semantic versioning over the surfaces the ## Unreleased - Checkpoints for `ta.sma`, biased `ta.stdev`, `ta.bb`, `math.sum` and `ta.change` now copy contiguous owning window buffers and reuse them on restore, preserving the arithmetic and rewind rules (#809). +- The JSON harness and live runner now forward the symbol's quantity-step metadata for default `strategy.entry(..., qty=na)` sizing (#716 item 1). +- The same quantity-step metadata is now forwarded for explicit `strategy.entry(..., qty=...)` quantities (#716 item 2). + - Explicit `strategy.order()` quantities now floor executable units at declared decimal quantity steps while preserving the source quantity; the boundary control is not measured against TradingView (#716 item 2). +- Corrected the parity grading for the historical population reported in [#836](https://github.com/pineforge-4pass/pineforge-engine/issues/836) on 2026-11-11: counting unmatched trades net instead of gross overstated the excellent tier by 10 of 312 reference probes (311 instead of 301) and by 24 of 7,989 graded probes. The trade data is unchanged. +- The corpus grader counts every unmatched gated trade on either side as a gross mismatch. One engine-only trade and one TradingView-only trade count as two mismatches, so equal list sizes cannot satisfy the excellent count gate. The same gross count, divided by the larger gated trade count (zero for two empty pools), supplies the percentage for strong-tier grading and fragmented-FIFO eligibility; thresholds are unchanged (#836). - The prebuilt Linux archives need glibc 2.35 or newer (for example Ubuntu 22.04 or Debian 12 and later), and the release check now refuses an archive that needs a newer glibc. - Multi-day charts restart their trading-date grid each year; measured on 24x7 daily input for chart bars, last_bar_index, barstate.islast and the open-trade report row. Streams keep the previous chart path; selected windows on generated N-day chart partitions are refused. Time offsets, exchange calendars and intraday input are not measured against TradingView. - `str.match` no longer reports "no match" after a memory failure; the failure now reaches `out_of_memory`. @@ -34,6 +39,11 @@ version number follows semantic versioning over the surfaces the `path_order` modes outside 0, 1 and 2, now produce typed refusals instead of being ignored or reset to AUTO. An invalid `mintick` is refused as `symbol_feeds_refused` with reason `syminfo_mintick_invalid`; an invalid point value uses the existing symbol metadata error, `symbol_metadata_rejected` with field `point_value`. +- The JSON harness now honors explicit `qty_step`; both `qty_step` and + `mincontract` require positive finite JSON numbers. The explicit step takes + precedence while the minimum retains its own value, and minimum-only inputs + keep their previous behavior. `pineforge-live` also accepts + `--syminfo mincontract=...`. - `strategy_get_run_warnings` exposes one named warning per `request.security` call site whose requested timeframe is not a multiple of its evaluator's feed timeframe; values, trades and run acceptance are unchanged (#718). - The Docker report and `scripts/run_strategy.py` now show the engine's run warnings as `diagnostics.warnings` when a run has any, and are unchanged otherwise (#718). - A percent-of-equity default quantity above 100 percent is now capped at 100 percent of equity when `margin_long` and `margin_short` are both 0 (#805). diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c9618fef..0b3d50765 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -75,6 +75,7 @@ if(NOT Eigen3_FOUND) FetchContent_Declare( eigen URL https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.tar.gz + URL_HASH SHA256=8586084f71f9bde545ee7fa6d00288b264a2b7ac3607b974e54d13e7162c1c72 ) set(EIGEN_BUILD_DOC OFF CACHE BOOL "" FORCE) set(EIGEN_BUILD_TESTING OFF CACHE BOOL "" FORCE) diff --git a/README.md b/README.md index 70f668ce7..7c36fdf4f 100644 --- a/README.md +++ b/README.md @@ -304,6 +304,8 @@ the affected scripts and how to recognise it. **Measured 2026-11-12** on engine `6e27d0f2` with codegen-oss `126e055f` (baseline `pineforge-parity-baseline-20261112-release-tag-cd9c85a6`, snapshot `cd9c85a6`): **7,989 graded probes, 7,983 excellent + 6 strong**, 0 below strong and 0 engine errors across 18 market/timeframe lanes. The historical release 1.0.1 inventory holds 4,776,328 trades. +For the population reported in [#836](https://github.com/pineforge-4pass/pineforge-engine/issues/836) on **2026-11-11**, the historical gross-count regrade measured **7,959 excellent / 30 strong**: 24 probes moved from excellent to strong and none lower. That regrade and the dated scoreboard above describe their recorded pair and population; they are not a fresh measurement of this revision. The grader uses gross counting. Recorded release results remain historical measurements. + Release **1.4.0 (2026-10-08)** maps to **7,983 excellent / 6 strong** on 7,989 graded probes (baseline `pineforge-parity-baseline-20261007-engine-b3192bfc`, promoted 2026-10-07 UTC), with 0 below strong and 0 engine errors. These are recorded grading outcomes for that pair and population. Release **1.3.0 grades 7,982 excellent / 7 strong until the next release**, on 7,989 probes (baseline `pineforge-parity-baseline-20261006-engine-7a1f01c0`, 2026-10-06). A main scoreboard advance does not change release results. @@ -312,9 +314,11 @@ The quantities above render from the public [facts tokens](https://github.com/pi | Board | Test set | Result | |---|---|---| -| **Public** — [open corpus](https://github.com/pineforge-4pass/pineforge-corpus) | 312 reference strategies on BINANCE:ETHUSDT.P 15m, Apache-2.0, reproducible by anyone | this repository's sweep of the v1.0.1 library: **311 excellent + 1 declared anomaly**; the 309 of them in the measured population: **309 excellent** | +| **Public** — [open corpus](https://github.com/pineforge-4pass/pineforge-corpus) | 312 reference strategies on BINANCE:ETHUSDT.P 15m, Apache-2.0, reproducible by anyone | reference sweep **2026-11-16** ([#836](https://github.com/pineforge-4pass/pineforge-engine/issues/836), [#883](https://github.com/pineforge-4pass/pineforge-engine/issues/883)): **301 excellent + 9 strong + 1 moderate + 1 declared anomaly**; the 309 of them in the dated scoreboard above: **309 excellent** | | **Closed test** | 7,680 probes of 741 TradingView scripts across the 18 lanes: 680 community-shared scripts (7,179 probes), private under TradingView's Terms of Service, and 61 probe scripts the maintainers wrote (501 probes) | **7,674 excellent + 6 strong** | +The gross count percentage in [#836](https://github.com/pineforge-4pass/pineforge-engine/issues/836) exposes an unchanged trade mismatch in `validation/mtf-triple-tf-macd-hist-confluence-01`, which remains **Moderate** in the 2026-11-16 reference sweep. Aggregate reference verification accepts only that full id at Moderate under a dated floor exception that ends when [#883](https://github.com/pineforge-4pass/pineforge-engine/issues/883) closes. Every other strategy keeps the Strong floor; missing inputs or lower tiers still fail. This reference measurement is separate from the dated scoreboard metadata above; baseline promotion requires its own verification. + ### Lane by lane @@ -336,10 +340,19 @@ The hard lane combines 309 | Size | M | M | L | M (R5-11) | @@ -223,8 +223,8 @@ spans, including nullable Calmar. | f | Dead legacy stream path | **Done.** `stream_feed_input_bar` and `dispatch_source_stream_script_bar` are both gone — neither name is in `src/` or `include/` — and the virtual left with the epoch that carried it. `stream_state_hash` engine.hpp:2145 stays, and the declaration/definition pairing is still enforced by `scripts/test_native_source_guard.py` | NEUTRAL (removing the virtual changes the vtable → shared bump) | Delete both | O; F asked (its Q6) | | g | Legacy path resolver | **Done, and then deleted.** `src/engine_path_resolve.cpp` is 118 lines with five functions: `bar_path_uses_high_first`, both `first_touch_position` overloads, `fill_bar_path_points_ordered`, and `BacktestEngine::declare_opened_lot_entry_bar_mask`. Ten test files still mention `resolve_exit_path_fill` in historical comments; the resolver body and test-only oracle are gone. | NEUTRAL (pure move; verify with the link gate) | Keep the generic path helpers in the kernel TU; the old source resolver is not a live counterpart | F (single-source) | | | h | `pine_*`-named generic API | `timeframe_time` session_time.hpp:35-211 (live, e.g. `session_in_market`, which generated code reaches through `pine_session_ismarket` session_time.hpp:263), `str_utils.hpp`, `deterministic_random` math.hpp:9, `tz_util` timezone.hpp:8-30; the guard bans `pine_*` only inside `IDENTIFIER_ROOTS` (test_native_source_guard.py:35, test_native_source_guard.py:35-43) | NEUTRAL, M | Neutral aliases; `pine_*` kept as deprecated inlines for codegen. Large blast radius in generated code. **Status (L11 + N14): done** — session_time / str_utils / math / timezone in L11; N14 applied the same treatment to the two public matrix types: `PineMatrix` → `NumericMatrix`, `PineGenericMatrix` → `GenericMatrix`, old spellings kept as exact aliases (no version script pins either name) | O (single-source) | -| i | `pf_pending_order_v1_t` Pine-named fields | `tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135, `pine_exit_activation_owner_cycle_at_birth` pending_order_mirror.hpp:147-156, `pine_frozen_market_instruction_kind` pending_order_mirror.hpp:164-169; enumerated by `PF_PO_FIELD` pending_order_mirror.cpp:144-178; written only by `pine_birth_reach` pine_adapter.cpp:28243 | no change | Freeze (ABI). The neutral view arrives beside it (L7 / L13) | O (single-source) | -| j | Pine-shaped lot flags | **Mostly done.** `market_pyramid_add` moved to the adapter side table the row asked for — `market_pyramid_adds_` src/source/pine_state_hash.cpp:779, whose own comment records that it was `PyramidEntry::market_pyramid_add` — and `ordinary_market_open`, `pooc_terminal_market_entry`, `ordinary_stop_open`, `bracket_slot_shadowed` and `entry_path_position` are all deleted. What is left on the lot is the intrabar-fill excursion mask, `skip_entry_bar_high` / `skip_entry_bar_low` engine.hpp:169-170, hashed (`skip_entry_bar_high` engine_state_hash.cpp:80) and, since the detached-comment and mask cleanup, written by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the Pine host did until the adapter state consolidation left its excursions to the kernel's sampler) and the kernel derives the pair | HASH-VISIBLE | Move the live flag to an adapter side table keyed by `entry_incarnation` (the adapter already keeps `PlacementTable` pine_adapter.hpp:591-968) and fold it through `hash_source_extension`; delete the dead ones; update the hash-coverage waivers. Whether a kernel TU beyond the hash reads them is unverified (O) | F O S | +| i | `pf_pending_order_v1_t` Pine-named fields | `tv_carry_qty` pending_order_mirror.hpp:73, `pine_birth_reach` pending_order_mirror.hpp:135, `pine_exit_activation_owner_cycle_at_birth` pending_order_mirror.hpp:147-156, `pine_frozen_market_instruction_kind` pending_order_mirror.hpp:164-169; enumerated by `PF_PO_FIELD` pending_order_mirror.cpp:144-178; written only by `pine_birth_reach` pine_adapter.cpp:28449 | no change | Freeze (ABI). The neutral view arrives beside it (L7 / L13) | O (single-source) | +| j | Pine-shaped lot flags | **Mostly done.** `market_pyramid_add` moved to the adapter side table the row asked for — `market_pyramid_adds_` src/source/pine_state_hash.cpp:780, whose own comment records that it was `PyramidEntry::market_pyramid_add` — and `ordinary_market_open`, `pooc_terminal_market_entry`, `ordinary_stop_open`, `bracket_slot_shadowed` and `entry_path_position` are all deleted. What is left on the lot is the intrabar-fill excursion mask, `skip_entry_bar_high` / `skip_entry_bar_low` engine.hpp:169-170, hashed (`skip_entry_bar_high` engine_state_hash.cpp:80) and, since the detached-comment and mask cleanup, written by no host at all: a host that owns lot excursions declares where its fill sat (`declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the Pine host did until the adapter state consolidation left its excursions to the kernel's sampler) and the kernel derives the pair | HASH-VISIBLE | Move the live flag to an adapter side table keyed by `entry_incarnation` (the adapter already keeps `PlacementTable` pine_adapter.hpp:591-968) and fold it through `hash_source_extension`; delete the dead ones; update the hash-coverage waivers. Whether a kernel TU beyond the hash reads them is unverified (O) | F O S | | k | `margin_call_enabled_` | **Done.** The generic kernel model is enabled by `NativeRunSpec::margin` native_run_spec.hpp:672. TradingView-specific toggling remains `source::PineStrategyHost::set_margin_call_enabled` pine_strategy_host.hpp:582; the frozen C symbol remains source-owned. | HASH-VISIBLE (R5-8) | Keep the source setter in the adapter; the kernel toggle is the presence of `spec.margin` | F O | | l | Kernel decodes adapter strings | **Done.** `closed_trade_close_cause` engine_trade_accessors.cpp:166 branches on no string at all: it reads `Trade::open_at_end`, then the typed `execution::CloseCause` the closer recorded on the row, then `exit_from_bracket`. A kernel-originated liquidation or risk flatten states its own cause through the settling fill, so a bare host reads `3` / `4` where the retired string derivation answered `1`. The public C contract it implements is `strategy_closed_trade_close_cause` pineforge.h:1571 | HASH-VISIBLE (adds durable state) | Kernel `enum class CloseCause` recorded on the `Trade` at settlement (the kernel sets `MarginCall` when L4 liquidates; a source virtual covers adapter causes). C numbers unchanged. After L4 | O (single-source) | | m | `request.security` machinery | `register_security_eval` engine_security.cpp:45, `set_native_security_feed` engine_aux_security.cpp:82-134, `SecurityEvalState` engine.hpp:1457-1661 carry Pine semantics (lookahead, gaps, Heikin-Ashi, KI-55 range start) and are driven only by the source layer | HASH-VISIBLE / measured path | F planned to move both TUs and the state into the source host after native HTF. **R5-6 narrows this:** timeframe subscriptions reuse the feed store and routing, so those stay kernel; only the Pine-only evaluator semantics remain candidates. Do last. **Status (lane L12b): done** — the evaluator semantics are `source::PineSecurityEvalState` + `src/source/pine_security_eval.cpp`; the kernel keeps the registry, aggregator, feed store and one generic step | F S | @@ -339,11 +339,11 @@ Union of F §2.vi (9), O §2.vi (5 + 1) and S §2.6 (7), deduplicated. S had no | 4 | HTF "comes from the kernel feed `set_native_security_feed` / `prepare_native_security_feeds`" (ADR L84-89) | Not usable by a bare host: a partial boundary, not an absent kernel. `prepare_native_security_feeds` is protected with one caller; `configure_security_evaluators` has one caller; the public setter is inert pre-run and refused in-run | `prepare_native_security_feeds` engine.hpp:1987, `configure_security_evaluators` engine.hpp:1618, `set_native_security_feed` engine.hpp:2086, `guard_native_mutation` engine_aux_security.cpp:84 | F O S | | 5 | "`engine_aux_security.cpp` includes `source/pine_strategy_host.hpp`" (ADR L129-132) | False at `73817c1`: its includes are `engine_internal.hpp`, `ta.hpp` and std headers (*fresh* re-read; the run-failure and cache policies added `run_failure.hpp` and `cache_policy.hpp`, both kernel headers). No kernel TU includes a source header | `sha256:042df81acdce8114753e13712791fde4394928c83731b2c6457da111ce4f2a07` engine_aux_security.cpp:6-14 | F | | 6 | "Kernel TUs reference adapter types (`source::StrategyOverrides` …)" (ADR L129-131) | True only as a forward-declared opaque pointer the kernel never dereferences; the independence checker whitelists exactly that symbol | at `73817c1`; overtaken by `3f4fff5b` (§2.ii a), after which the rich begin carries `overrides_opaque` native_host.hpp:778 and the independence checker whitelists nothing | F O | -| 7 | `engine.hpp` hosts "the ten-significant-digit money rule, KI-62, the `strategy.close` batching + entry-id ledger, a TradingView margin-call toggle" (ADR L131-134) | Two of four survive as code. Money rule: a comment block; the arithmetic is in the adapter. KI-62: orphaned comments + one live, hashed lot flag. Close batching + id ledger: gone from the kernel (the ledger is in the adapter; the old names survive only in comments). Live: `margin_call_enabled_`, the KI-62 flag and — unmentioned by the ADR — string-sentinel decoding of adapter comments | the money-rule block was lines 927-961 of `engine.hpp` and the detached-comment and mask cleanup deleted it (lines 73-180 were and are live code), the arithmetic `source_money_round` pine_adapter.cpp:646 and its twin `tv_money_round` pine_policy_support.hpp:9-15; `skip_entry_bar_high` engine.hpp:169, `skip_entry_bar_low` engine_state_hash.cpp:80, `declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the ledger `close_logical_units_` pine_adapter.hpp:2871; `closed_trade_close_cause` engine_trade_accessors.cpp:166. **Overtaken:** the margin-call toggle and the string decoding are both gone — see ADR-0001, "Current state vs. target" | F O | +| 7 | `engine.hpp` hosts "the ten-significant-digit money rule, KI-62, the `strategy.close` batching + entry-id ledger, a TradingView margin-call toggle" (ADR L131-134) | Two of four survive as code. Money rule: a comment block; the arithmetic is in the adapter. KI-62: orphaned comments + one live, hashed lot flag. Close batching + id ledger: gone from the kernel (the ledger is in the adapter; the old names survive only in comments). Live: `margin_call_enabled_`, the KI-62 flag and — unmentioned by the ADR — string-sentinel decoding of adapter comments | the money-rule block was lines 927-961 of `engine.hpp` and the detached-comment and mask cleanup deleted it (lines 73-180 were and are live code), the arithmetic `source_money_round` pine_adapter.cpp:646 and its twin `tv_money_round` pine_policy_support.hpp:9-15; `skip_entry_bar_high` engine.hpp:169, `skip_entry_bar_low` engine_state_hash.cpp:80, `declare_opened_lot_entry_bar_mask` engine_path_resolve.cpp:95; the ledger `close_logical_units_` pine_adapter.hpp:2876; `closed_trade_close_cause` engine_trade_accessors.cpp:166. **Overtaken:** the margin-call toggle and the string decoding are both gone — see ADR-0001, "Current state vs. target" | F O | | 8 | "No kernel-level C API. `c_abi.cpp` exports only version/descriptor symbols" (ADR L136-138) | 105 runtime `PF_API` symbols: native configure, the 13-symbol stream family, pending-order views, broker-state hash, security feeds, FX curve, reports, scratch-table selection, cache policy, path/fd report sinks, the report-only T-summary query, per-run cache diagnostics, output-series limits and map-pair limits. The 105 compiled-strategy runtime symbols in `c_abi.cpp` are distinct from the native C host API in `native_c_host.cpp`: the latter exposes submit, replace, cancel and a callback table to a C host. | `strategy_closed_trade_entry_incarnation`, `strategy_initial_capital`, `strategy_configure_native_v1`, `strategy_stream_begin`, `strategy_stream_fill_report`, `strategy_pending_orders_len`, `strategy_position_cycle_seq`, `strategy_set_broker_state_hash_recording`, `strategy_broker_state_hash`, `strategy_set_aux_security_feed`, `strategy_set_native_security_feed`, `strategy_configure_native_fx_curve_v1`, `strategy_set_scratch_tables_v1`, `strategy_set_cache_policy_v1`, `strategy_report_sink_receipt_v1`, `strategy_set_report_sink_fd_v1`, `strategy_report_sink_summary_v1`, `pf_report_scratch_arena_stats`, `pf_report_cache_stats`, `pf_output_series_limit_set_checked`, `pf_output_series_limit_failure`, `pf_map_pairs_limit_set_checked` in `src/c_abi.cpp`; `EXPECTED_RUNTIME` in `scripts/check_c_abi_runtime.py` | F O S | | 9 | "No examples. `PINEFORGE_BUILD_EXAMPLES` is 'none yet'" (ADR L139) | At the audit snapshot the option was inert and two Pine-free hosts were available through the live runner. **Now:** `PINEFORGE_BUILD_EXAMPLES` builds 16 C++ and two C examples against the kernel, independently of the live runner | historical: `73817c1`; current option at `CMakeLists.txt:37`, C++ list `PINEFORGE_NATIVE_EXAMPLES` at `examples/native/CMakeLists.txt:16-33`, C list `PINEFORGE_NATIVE_C_EXAMPLES` at `examples/native/CMakeLists.txt:39-42` | F O S | | 10 | Follow-ups (1)-(4) (ADR L168-170) | Add native reporting, sizing, margin, HTF — and, per R5, calc-timing, price grid and order ergonomics. Without them "usable without Pine" is a build-system claim, not a feature claim | §3.5 | F | -| 11 | Build layout | One `pineforge` library always appends the source list; a kernel-only target does not exist yet; `compat/pine/market_admission.cpp` sits in the main list, a boundary bug. **Overtaken:** `PINEFORGE_KERNEL_SOURCES` CMakeLists.txt:118 and `add_library` CMakeLists.txt:185 build `PineForge::kernel`, and the compat unit is in `PINEFORGE_SOURCE_LAYER_SOURCES` CMakeLists.txt:91 | `add_library` CMakeLists.txt:185 | S (F, O agree in §2.i) | +| 11 | Build layout | One `pineforge` library always appends the source list; a kernel-only target does not exist yet; `compat/pine/market_admission.cpp` sits in the main list, a boundary bug. **Overtaken:** `PINEFORGE_KERNEL_SOURCES` CMakeLists.txt:119 and `add_library` CMakeLists.txt:186 build `PineForge::kernel`, and the compat unit is in `PINEFORGE_SOURCE_LAYER_SOURCES` CMakeLists.txt:92 | `add_library` CMakeLists.txt:186 | S (F, O agree in §2.i) | | 12 | Host constructors | `NativeStrategyHost` is a zero-argument C++ host with the callbacks and the request API; `PineStrategyHost` is the class that takes `CapAttachment` — do not attribute it to the native constructor | `NativeStrategyHost` native_host.hpp:865, `PineStrategyHost` pine_strategy_host.hpp:335 | S (single-source) | `docs/pages/native-engine.md` fixes, all applied (F unless noted). The page is cited by @@ -537,11 +537,11 @@ line numbers are re-anchored to this tree (the measurement ran on 683a82f); kern | Pine rule | adapter mechanism | kernel counterpart | class | measured divergence (test scenario) | ruling | |---|---|---|---|---|---| | `strategy.risk.allow_entry_in` | `risk_.direction`; `project()` already declares `spec.allowed_open_directions` (pine_adapter.cpp:3131); the forbidden opposite entry is reshaped to `CloseOpposite` in the adapter's own terms branch | per-opening cap `allowed_open_directions` | (i)+(ii), **already re-lowered** | none to measure: the cap is the kernel's, the close-only reshaping is the retained TV policy | done before N12; nothing to move | -| `strategy.risk.max_position_size` | precommit refuses a flat/same-side ENTRY at its fill when the LIVE book already holds ≥ the limit (`max_position_size` pine_adapter.cpp:17365; the retired legacy rule is `git show ab9714be:src/source/pine_risk.cpp`, lines 115 and 2531) | `max_abs_units`: refuses a fill whose RESULTING book would exceed it | (iv) | `PS`: two-unit entries against 3 — adapter admits the second (live 2 < 3, book 4) and refuses the third; kernel refuses the second (`MaxAbsUnits` at bars 1, 2), book 2 | retained: a pre-fill-book gate is a TV emulation fact, not a generic cap | -| `strategy.risk.max_drawdown` | After an applied fill, the running peak is seeded if absent; fills that close units update it and test the absolute or percent-of-peak threshold. A breach latches risk_.halted before an immediate flatten of any remaining position, including a reversal's new side, at that fill's resolved price. (`risk_.fill_peak_equity` pine_adapter.cpp:25465-25492; `max_drawdown_breached` pine_adapter.cpp:26893-26912). | `max_drawdown`: current drawdown vs percent-of-peak, at all three points; `BlockOpenings` refuses every opening | (ii) cadence + (iv) scope | `MG10-a` (**HISTORICAL**, measured on engine f5e929df, before the #804 change): bar 1 opens 600 under the entry and closes 100 under — kernel breaches at the open mark and refuses bar 2's add (book 100), adapter admits it (book 200). `MG10-b` (**HISTORICAL**, measured on engine f5e929df, before the #804 change): latched at bar 1's close, the adapter still reverses twice (`L→S` −600, `S→L`), the kernel refuses both reversals (`RiskLimit` at bars 2, 3) and holds. Corpus: `ta-closedtrades-risk-introspection-01` (20 % of peak) identical 1502/1502 rows with the kernel seeded — the rule never fires there | retained: the latched-reversal exemption cannot be composed on a kernel block; a close-only cadence knob alone would not close it | -| `strategy.risk.max_cons_loss_days` | `SourceDayLedger` (pine_adapter.hpp:1549-1560): +1 per losing TRADE on a new chart day, reset to 0 by any winning trade at its fill (`consecutive_loss_days` pine_adapter.cpp:26841-26843); gated immediately at precommit and latched at the close | `max_consecutive_loss_days`: a day's NET realized result settles the streak when the next day opens | (iv) | `MG11-a` (loss; win then loss; loss): adapter latches at day 3's first loss (4 rows), kernel's netted streak never reaches 2 (6 rows, no event). `MG11-b` (loss then smaller win, twice): adapter resets daily (5 rows), kernel blocks at day 3's open (4 rows, one event). No corpus probe declares the rule | retained: an order-dependent per-trade streak is not a generic day result | -| `strategy.risk.max_intraday_loss` | day-open equity at the day's first bar (`intraday_start_equity` pine_adapter.cpp:23471); checked at the bar open (`submit_intraday_loss_close(bar.open, context, true)` pine_adapter.cpp:23871), along the bar's PATH at its adverse extreme with a resting `Stop` flatten (`schedule_intraday_loss_path` pine_adapter.cpp:21867, `submit_intraday_loss_close` pine_adapter.cpp:21875), and at a closing fill with that fill's own P&L unbooked (`preclose_intraday_loss` pine_adapter.cpp:25767); epsilon `1e-9·max(1,|limit|)`; ticket `"Close Position (Max intraday Loss)"` → `CloseCause::RiskLimit`; the block refuses ALL placements for the chart day (`entry`/`order`/`close`/`close_all`/`exit`, `intraday_loss_orders_blocked` pine_adapter.cpp:7888, `:15428`, `:10268`, `:11196`, `:11727`) and `cancel_all()` withdraws the working book (`cancel_all()` pine_adapter.cpp:15392) | `max_intraday_loss` at the three points; `FlattenAndBlock` closes `AsPresented` under `"__kernel_risk__"` / `"Risk limit"`, blocks openings only, leaves working orders | (ii) ticket, day basis + (iv) path check, placement scope, unbooked fill, epsilon | `MG12-a`: day 2's first bar 100→85→95 — adapter closes at the extreme (85, −1500, cause 4) and refuses the day's later entry, kernel measures 0 at the open and 500 at the close, never breaches, ends with 300 units. `MG12-b`: breach at a bar open inside the day — same money on both sides (100 @ 88, −1200) under different tickets; the adapter withdraws a resting limit entry, the kernel leaves it working and it fills on day 3. Both sides mark the day's opening equity at its first bar: a gap there is not an intraday loss for either | retained: the path-extreme check is the rule's substance and the kernel has no path evaluation point; a risk ticket field and a `PathAdverseExtreme` check kind would be generic additions (mirroring `NativeMarginModel`) but would still leave the placement scope, the unbooked fill and the epsilon | -| `strategy.risk.max_intraday_filled_orders` | `compat::pine::IntradayCap` + `IntradayOrderBudget` (243 + 94 lines): charged slots per risk day, quota transfer, POOC deferral, noop-market skip, the close at the fill price or the bar's better extreme (`post_dispatch`), latch → all placements denied (`cap_placement_denied` pine_adapter.cpp:18350); ticket `"Close Position (Max number of filled orders in one day)"` → `CloseCause::FillCap`; day = `session_trading_day_index` for a `HHMM-HHMM` session, else the chart day (`chart_day_key` pine_adapter.cpp:18256, `mday·100+month` in the CHART timezone) | `max_fills_per_day`: applied fills counted as they settle, evaluated at the three points; `FlattenAndBlock` at the drain's cursor; `SessionDay` / `CalendarDayInTimezone` on the SPEC timezone | (i) count + (ii) ticket + (iv) close price, charged-slot budget, placement scope, chart-day key | `MG13`: the second fill reaches the cap on a bar that closes above its open — adapter closes at that bar's HIGH (103) under its ticket, kernel at the close (101) under its own. `MG14`: chart timezone `America/New_York` over a UTC symbol — bar 4 (04:00 UTC) is a new chart day for the adapter and the same UTC day for the kernel. Corpus, kernel seeded with the declared cap: `cap-risk-gates-allow-max-intraday-01` identical 1464/1464 (the third fill of each day flattens the book, so block and latch coincide); `cap-max-intraday-filled-orders-isolate-01` 3840 of 3916 rows differ from row 77; `cap-gatekeeper-intraday-risk-01` 312 of 604 differ from row 293; `composite-bracket-cap-range-pending-stop-01` 2370 of 2384 differ from row 17, two extra rows | retained: the count alone is expressible, but the kernel's block would refuse fills the transfer admits and the close price is TV's; a report-only action would give the adapter a count it cannot use (charged slots ≠ applied fills) | +| `strategy.risk.max_position_size` | precommit refuses a flat/same-side ENTRY at its fill when the LIVE book already holds ≥ the limit (`max_position_size` pine_adapter.cpp:17554; the retired legacy rule is `git show ab9714be:src/source/pine_risk.cpp`, lines 115 and 2531) | `max_abs_units`: refuses a fill whose RESULTING book would exceed it | (iv) | `PS`: two-unit entries against 3 — adapter admits the second (live 2 < 3, book 4) and refuses the third; kernel refuses the second (`MaxAbsUnits` at bars 1, 2), book 2 | retained: a pre-fill-book gate is a TV emulation fact, not a generic cap | +| `strategy.risk.max_drawdown` | After an applied fill, the running peak is seeded if absent; fills that close units update it and test the absolute or percent-of-peak threshold. A breach latches risk_.halted before an immediate flatten of any remaining position, including a reversal's new side, at that fill's resolved price. (`risk_.fill_peak_equity` pine_adapter.cpp:25657-25684; `max_drawdown_breached` pine_adapter.cpp:27098-27117). | `max_drawdown`: current drawdown vs percent-of-peak, at all three points; `BlockOpenings` refuses every opening | (ii) cadence + (iv) scope | `MG10-a` (**HISTORICAL**, measured on engine f5e929df, before the #804 change): bar 1 opens 600 under the entry and closes 100 under — kernel breaches at the open mark and refuses bar 2's add (book 100), adapter admits it (book 200). `MG10-b` (**HISTORICAL**, measured on engine f5e929df, before the #804 change): latched at bar 1's close, the adapter still reverses twice (`L→S` −600, `S→L`), the kernel refuses both reversals (`RiskLimit` at bars 2, 3) and holds. Corpus: `ta-closedtrades-risk-introspection-01` (20 % of peak) identical 1502/1502 rows with the kernel seeded — the rule never fires there | retained: the latched-reversal exemption cannot be composed on a kernel block; a close-only cadence knob alone would not close it | +| `strategy.risk.max_cons_loss_days` | `SourceDayLedger` (pine_adapter.hpp:1549-1560): +1 per losing TRADE on a new chart day, reset to 0 by any winning trade at its fill (`consecutive_loss_days` pine_adapter.cpp:27046-27048); gated immediately at precommit and latched at the close | `max_consecutive_loss_days`: a day's NET realized result settles the streak when the next day opens | (iv) | `MG11-a` (loss; win then loss; loss): adapter latches at day 3's first loss (4 rows), kernel's netted streak never reaches 2 (6 rows, no event). `MG11-b` (loss then smaller win, twice): adapter resets daily (5 rows), kernel blocks at day 3's open (4 rows, one event). No corpus probe declares the rule | retained: an order-dependent per-trade streak is not a generic day result | +| `strategy.risk.max_intraday_loss` | day-open equity at the day's first bar (`intraday_start_equity` pine_adapter.cpp:23661); checked at the bar open (`submit_intraday_loss_close(bar.open, context, true)` pine_adapter.cpp:24061), along the bar's PATH at its adverse extreme with a resting `Stop` flatten (`schedule_intraday_loss_path` pine_adapter.cpp:22057, `submit_intraday_loss_close` pine_adapter.cpp:22065), and at a closing fill with that fill's own P&L unbooked (`preclose_intraday_loss` pine_adapter.cpp:25959); epsilon `1e-9·max(1,|limit|)`; ticket `"Close Position (Max intraday Loss)"` → `CloseCause::RiskLimit`; the block refuses ALL placements for the chart day (`entry`/`order`/`close`/`close_all`/`exit`, `intraday_loss_orders_blocked` pine_adapter.cpp:7948, `:15617`, `:10328`, `:11256`, `:11787`) and `cancel_all()` withdraws the working book (`cancel_all()` pine_adapter.cpp:15581) | `max_intraday_loss` at the three points; `FlattenAndBlock` closes `AsPresented` under `"__kernel_risk__"` / `"Risk limit"`, blocks openings only, leaves working orders | (ii) ticket, day basis + (iv) path check, placement scope, unbooked fill, epsilon | `MG12-a`: day 2's first bar 100→85→95 — adapter closes at the extreme (85, −1500, cause 4) and refuses the day's later entry, kernel measures 0 at the open and 500 at the close, never breaches, ends with 300 units. `MG12-b`: breach at a bar open inside the day — same money on both sides (100 @ 88, −1200) under different tickets; the adapter withdraws a resting limit entry, the kernel leaves it working and it fills on day 3. Both sides mark the day's opening equity at its first bar: a gap there is not an intraday loss for either | retained: the path-extreme check is the rule's substance and the kernel has no path evaluation point; a risk ticket field and a `PathAdverseExtreme` check kind would be generic additions (mirroring `NativeMarginModel`) but would still leave the placement scope, the unbooked fill and the epsilon | +| `strategy.risk.max_intraday_filled_orders` | `compat::pine::IntradayCap` + `IntradayOrderBudget` (243 + 94 lines): charged slots per risk day, quota transfer, POOC deferral, noop-market skip, the close at the fill price or the bar's better extreme (`post_dispatch`), latch → all placements denied (`cap_placement_denied` pine_adapter.cpp:18540); ticket `"Close Position (Max number of filled orders in one day)"` → `CloseCause::FillCap`; day = `session_trading_day_index` for a `HHMM-HHMM` session, else the chart day (`chart_day_key` pine_adapter.cpp:18446, `mday·100+month` in the CHART timezone) | `max_fills_per_day`: applied fills counted as they settle, evaluated at the three points; `FlattenAndBlock` at the drain's cursor; `SessionDay` / `CalendarDayInTimezone` on the SPEC timezone | (i) count + (ii) ticket + (iv) close price, charged-slot budget, placement scope, chart-day key | `MG13`: the second fill reaches the cap on a bar that closes above its open — adapter closes at that bar's HIGH (103) under its ticket, kernel at the close (101) under its own. `MG14`: chart timezone `America/New_York` over a UTC symbol — bar 4 (04:00 UTC) is a new chart day for the adapter and the same UTC day for the kernel. Corpus, kernel seeded with the declared cap: `cap-risk-gates-allow-max-intraday-01` identical 1464/1464 (the third fill of each day flattens the book, so block and latch coincide); `cap-max-intraday-filled-orders-isolate-01` 3840 of 3916 rows differ from row 77; `cap-gatekeeper-intraday-risk-01` 312 of 604 differ from row 293; `composite-bracket-cap-range-pending-stop-01` 2370 of 2384 differ from row 17, two extra rows | retained: the count alone is expressible, but the kernel's block would refuse fills the transfer admits and the close price is TV's; a report-only action would give the adapter a count it cannot use (charged slots ≠ applied fills) | **Proposed generic additions (not implemented — no consumer today).** A risk ticket (`label` / `comment` on `NativeRiskLimits`, folded only when set, @@ -1101,9 +1101,9 @@ key, since replaced by the walk of the retained rows, `placement_` pine_state_hash.cpp:382) — one per accepted request, cleared only by `reset_for_run` pine_adapter.cpp:2653 — and reflected the whole admission journal (since replaced by a running digest of its events, `admission_journal` -pine_state_hash.cpp:848). `PineScheduler::hash_state` pine_state_hash.cpp:932 +pine_state_hash.cpp:849). `PineScheduler::hash_state` pine_state_hash.cpp:933 re-folded the consumed source prefix too (`consumed` -pine_state_hash.cpp:936-937). Measured with recording on and off, in two +pine_state_hash.cpp:937-938). Measured with recording on and off, in two shapes: - the audit's order-and-cancel probe (one `strategy_order` accepted and @@ -1146,10 +1146,10 @@ moving a single value, but not from inside lane F9's files: 1,339 rows each one callback old, an OCA re-price and a trail attach rows two callbacks old, and the row of an entry that filled at bar 1 is rewritten at bar 60, when its first exit is attached - (`has_full_entry_bracket` pine_adapter.cpp:12123). Full-table scans + (`has_full_entry_bracket` pine_adapter.cpp:12202). Full-table scans rewrite rows as well (`suspend_brackets_for_reversal` pine_adapter.cpp:2204, `revive_brackets_after_margin` pine_adapter.cpp:2453, - `preserved_by_close_all` pine_adapter.cpp:15461), and + `preserved_by_close_all` pine_adapter.cpp:15650), and `admission::Journal::retain` market_admission.hpp:244 drops events from the middle of the journal. So "a row stops changing once its request is done" is false, and a cache resting on it would move a value. @@ -1272,13 +1272,13 @@ retires every leg of each exit row it names and drops such a parked stop (`retire_cancelled_exits` pine_adapter.cpp:2362), the revival skips a withdrawn exit, and K1 holds a withdrawn leg only while its origin is live or opened -- the one reader left is `origin_leg_consumed`, and it asks about such origins alone (`sweep_retired_rows` -pine_adapter.cpp:3990). Both shapes now cost ×1.89 to ×2.12 per doubling up +pine_adapter.cpp:3994). Both shapes now cost ×1.89 to ×2.12 per doubling up to 64,000 bars and retain 3 rows. Row D was the kernel's half: the adapter enrolled every opening in its id's kernel roster (`cohort_add`) and never took one off, and `hash_cohorts` folds every member at every read. An origin whose request chain no longer works and that holds no open lot can never be bound again, so at every bar open that follows a command event the adapter takes -it off (`release_closed_cohort_origins` pine_adapter.cpp:4546); a lot opened +it off (`release_closed_cohort_origins` pine_adapter.cpp:4550); a lot opened by a replace successor keeps every older member of its roster while it is open, because its root is not a public fact. The flip then costs ×1.99, ×2.01, ×2.02 per doubling, and its rosters end with one member where they @@ -1648,10 +1648,10 @@ the rule gives 9.9 + 9.89). | # | Risk | Why it bites | Containment | Src | |---|---|---|---|---| | E1 | **Spec hashing.** `hash_spec` folded every field unconditionally (native_execution_consumer.cpp:257-360); every block added since folds only when set | Any new field, even defaulted, changes every continuation / broker-state hash, adapter runs included | Fold new blocks only when set, as a tagged extension (precedent `precommit_digest_` native_execution_consumer.hpp:2199); test that a default spec hashes to the pre-change constant | F | -| E2 | **Margin.** The TV margin call is built host-side from kernel primitives (a Stop-triggered `Reduce` bound to openings, or `execute_current`, which survives in `submit_margin_call_slice` pine_adapter.cpp:18829); its chronology lives at `schedule_margin_call_path` pine_adapter.cpp:21201 and its excursion sample differed by chronology (the Pine host's own excursion model, retired by the adapter state consolidation: the kernel's sampler books it) | A kernel model under an adapter run would liquidate twice or reorder settlement; a checkpoint that reduces mid-path changes lot state exactly where the adapter's chronology lives | Superseded by R5: the adapter declares `spec.margin` — maintenance-only, so no opening is decided twice (`AdmitWithHostMargin`, MG4) — and the kernel owns the mechanism (check points, resting liquidation, receipt) while the adapter's hooks answer TradingView's scheduling, money and slice (ADR-0001 "Margin"). What stays host-side is named there: the chronology exceptions, the pre-open slice, the 1x-long money call and the broker-open FX rollover, whose kernel `FxRoll` point measures elsewhere (MG9). Corrected by the margin and cancellation comparison | F O S | -| E3 | **Sizing.** TV sizes from the tick-rounded signal close, ten-significant-digit-rounded percent-of-equity money (`source_money_round` pine_adapter.cpp:3465, applied when the quantity grid is staged) and fee-reserved cash, frozen at signal, with fill-time exceptions in COOF paths (`default_sizing_shape` pine_adapter.cpp:3516, the COOF `at_fill` exception pine_adapter.cpp:9018) | One "generic" percent sizing serving both would break parity or import TV rounding into the kernel; the last ulp differs on lot-stepped instruments | Two paths on purpose: `Sized` (kernel, plain arithmetic, timing as a first-class field) vs `HostSized` + the adapter resolver. Never "simplify" the adapter onto the kernel basis. What the two paths share is the conversion: since the kernel/source split and the margin and cancellation comparison the division by price × point value × FX and the percentage fee reserve exist once, in the kernel (`native_sized_units`), for a default AND a typed (`qty_type` cash / percent) quantity. The adapter keeps what is measured TradingView: the money basis (signal-time equity, TradingView's margin equity at a typed fill, the hypothetical Flatten's balance for a typed reversal), the snapshot, the fill-time exceptions and both lot floors (`tests/test_adapter_sizing_relower.cpp` cases 8, 9, 11: kernel 0.0392 against the percent floor's 0.0391 on a 0.0001 lot), and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (case 12) | F O S | -| E4 | **Calc on fill.** Kernel suffix eligibility is continuous along the segment; TV refills only at O/H/L/C waypoints -- on a magnified path at each intrabar's open, twice, and its two extremes, never its close (the TradingView intrabar import) (`PineScheduler::recalculate` pine_scheduler_native.cpp:824, `coof_next_waypoint` pine_adapter.cpp:7553) | Changing `born_on_remaining_path` or the notification order shifts adapter fills | Keep fixed `born_on_remaining_path` native_execution_consumer.cpp:7048 and `drain_applied_notifications` (native_execution_consumer.cpp:7809); new behaviour only behind new opt-ins | F | -| E5 | **Open-bar lookahead.** The adapter reads the full bar in `on_bar_open` (margin path scheduling uses H/L, pine_adapter.cpp:23230) | The complete bar cannot be masked globally | Opt-in `OpenOnly` view (A.4); loud doc warning in L0 | F | +| E2 | **Margin.** The TV margin call is built host-side from kernel primitives (a Stop-triggered `Reduce` bound to openings, or `execute_current`, which survives in `submit_margin_call_slice` pine_adapter.cpp:19019); its chronology lives at `schedule_margin_call_path` pine_adapter.cpp:21391 and its excursion sample differed by chronology (the Pine host's own excursion model, retired by the adapter state consolidation: the kernel's sampler books it) | A kernel model under an adapter run would liquidate twice or reorder settlement; a checkpoint that reduces mid-path changes lot state exactly where the adapter's chronology lives | Superseded by R5: the adapter declares `spec.margin` — maintenance-only, so no opening is decided twice (`AdmitWithHostMargin`, MG4) — and the kernel owns the mechanism (check points, resting liquidation, receipt) while the adapter's hooks answer TradingView's scheduling, money and slice (ADR-0001 "Margin"). What stays host-side is named there: the chronology exceptions, the pre-open slice, the 1x-long money call and the broker-open FX rollover, whose kernel `FxRoll` point measures elsewhere (MG9). Corrected by the margin and cancellation comparison | F O S | +| E3 | **Sizing.** TV sizes from the tick-rounded signal close, ten-significant-digit-rounded percent-of-equity money (`source_money_round` pine_adapter.cpp:3465, applied when the quantity grid is staged) and fee-reserved cash, frozen at signal, with fill-time exceptions in COOF paths (`default_sizing_shape` pine_adapter.cpp:3516, the COOF `at_fill` exception pine_adapter.cpp:9078) | One "generic" percent sizing serving both would break parity or import TV rounding into the kernel; the last ulp differs on lot-stepped instruments | Two paths on purpose: `Sized` (kernel, plain arithmetic, timing as a first-class field) vs `HostSized` + the adapter resolver. Never "simplify" the adapter onto the kernel basis. What the two paths share is the conversion: since the kernel/source split and the margin and cancellation comparison the division by price × point value × FX and the percentage fee reserve exist once, in the kernel (`native_sized_units`), for a default AND a typed (`qty_type` cash / percent) quantity. The adapter keeps what is measured TradingView: the money basis (signal-time equity, TradingView's margin equity at a typed fill, the hypothetical Flatten's balance for a typed reversal), the snapshot, the fill-time exceptions and both lot floors (`tests/test_adapter_sizing_relower.cpp` cases 8, 9, 11: kernel 0.0392 against the percent floor's 0.0391 on a 0.0001 lot), and a staged FX series keeps `Sized` wherever the acceptance coordinate converts at the source's rate (case 12) | F O S | +| E4 | **Calc on fill.** Kernel suffix eligibility is continuous along the segment; TV refills only at O/H/L/C waypoints -- on a magnified path at each intrabar's open, twice, and its two extremes, never its close (the TradingView intrabar import) (`PineScheduler::recalculate` pine_scheduler_native.cpp:824, `coof_next_waypoint` pine_adapter.cpp:7613) | Changing `born_on_remaining_path` or the notification order shifts adapter fills | Keep fixed `born_on_remaining_path` native_execution_consumer.cpp:7048 and `drain_applied_notifications` (native_execution_consumer.cpp:7809); new behaviour only behind new opt-ins | F | +| E5 | **Open-bar lookahead.** The adapter reads the full bar in `on_bar_open` (margin path scheduling uses H/L, pine_adapter.cpp:23420) | The complete bar cannot be masked globally | Opt-in `OpenOnly` view (A.4); loud doc warning in L0 | F | | E6 | **Double equity recording.** The source host already records per bar (history: only its cadence remains, `scheduler_mark_report_point` pine_strategy_host.cpp:3498) | Kernel recording would duplicate points and shift drawdown | Report policy / ownership (A.1); the adapter declares `KernelRecordedAtHostMarks` (R1): one recorder, at the adapter's own marks | F O | | E7 | **Price grid.** TV does not merely quantize: it tests triggers against a quantized bar while booking the fill on the *unrounded* level, so the adapter shifts the threshold (`source_trigger_threshold` pine_adapter.cpp:472) and books elsewhere | A kernel grid that quantizes both sides is correct and not TV; moving rounding into the kernel would change both native results and the adapter | `NativePriceGrid::None` for the adapter permanently; never "unify". Before L8: a byte-level experiment that the default price spelling on decimal ticks (one-ULP representations) does not move (S) | O S | | E8 | **HTF.** The evaluator machinery carries TV publication semantics (lookahead, boundary deferral, early close, `syminfo.type` branching engine_security.cpp:67-73). The feed store is TV-calibrated too: "W" / "M" from native daily bars, native stamps as the period partition (`set_native_security_feed` `sha256:3ada9f7e0bde989376f15e4e27c1fa3e77494750f2568ee721bd2a649e7354ef` engine_aux_security.cpp:52-84, *fresh*). Two aggregators exist: script buckets use `native_calendar`, `request.security` uses `TimeframeAggregator`, whose completion rules are TV-pinned (timeframe.hpp:347-389) | Reuse risks changing the *order* of the Pine scheduler's pumps; a bare host inherits TV partition rules; native HTF may differ from Pine HTF on early-close sessions | Registration-gated pump; no change to `scheduler_feed_security_input` pine_strategy_host.cpp:2282; full corpus diff; supplied-bar provenance preserved, never silently replaced by chart aggregation (S); differences on early-close sessions are recorded, not "fixed" (F). The Pine scheduler keeps one more `TimeframeAggregator` pass, before the run (`PineScheduler::run_begin`, audit M26): the script-bar count it yields (`terminal_source_bar`, the probe-tail suppression) and the per-input completion flags it hands `on_native_input` are needed before the first input arrives, while the kernel seals script intervals only as inputs arrive and offers no pre-run interval query (`NativeExecutionConsumer::script_interval_at` is consumer-private). Retained by the margin and cancellation comparison: replacing it needs that kernel capability | F O S | @@ -1667,11 +1667,11 @@ the rule gives 9.9 + 9.89). | P1 | **Epoch bumps** move the frozen C++ ABI fixtures and the settlement ABI checker `CLAUDE.md` names; a neutral-refactor worker may not edit frozen headers | Bump lanes are not neutral-refactor lanes; the brief lists the header extension (§3.1 c) | F O S | | P2 | **Stale fence.** `CLAUDE.md` names "measured sources" `engine_strategy_commands.cpp`, `engine_fills.cpp`, `engine_risk.cpp`, `engine_market_admission.cpp`; none exists in `src/` at `73817c1` (only `engine_orders.cpp` and `engine_run.cpp` do) | Done: `CLAUDE.md` now names the post-R4-C measured sources (`src/native_execution_consumer.cpp`, `src/native_order.cpp`, `src/engine_execution.cpp`, `src/engine_orders.cpp`, `src/engine_run.cpp`, `src/engine_path_resolve.cpp`, `src/source/*`) | F | | P3 | **Corpus gate state.** S's audit found `scripts/run_corpus.sh` stopping at the corpus strategy compile: committed `generated.cpp` files no longer match the engine (8/8 drifted from the pinned codegen commit under `VERIFY=1 ONLY=validation/analyzer scripts/regen_corpus_cpp.sh`) | §3.1 a relies on that script. Re-pin the engine / codegen corpus tuple before L1. Not re-verified in this merge (no builds were run). Resolved since: the corpus was re-pinned (the submodule is at `b40aa8e`) and the sweep runs on `main` | S (single-source) | -| P4 | **Hash compatibility.** The source hash covers hundreds of adapter and scheduler fields (`PineExecutionAdapter::hash_state` pine_state_hash.cpp:315, `PineScheduler::hash_state` pine_state_hash.cpp:932); L4 / L9 add hashed state, L12 removes some | A hash mismatch is a diagnostic until the trade diff is known, never a reason to relax parity (S). Open: does the campaign gate compare broker-state hashes across releases, or only trades (O)? → Q3 | O S | +| P4 | **Hash compatibility.** The source hash covers hundreds of adapter and scheduler fields (`PineExecutionAdapter::hash_state` pine_state_hash.cpp:315, `PineScheduler::hash_state` pine_state_hash.cpp:933); L4 / L9 add hashed state, L12 removes some | A hash mismatch is a diagnostic until the trade diff is known, never a reason to relax parity (S). Open: does the campaign gate compare broker-state hashes across releases, or only trades (O)? → Q3 | O S | | P5 | **Test asymmetry:** 430 adapter test files vs 14 native when this plan was written (on this tree 486 `tests/*.cpp` name `source::PineStrategyHost`, 128 `NativeStrategyHost`) | native-only tests in every lane (§3.1 b) | O | | P6 | **Stream vs batch.** The stream refuses monthly input, mixed modes, in-session gaps, security feeds and source setters, and tick input under an FX curve (any non-empty curve until the FX stream audit) (`native-engine.md`, "Stream limitations (current)") | L5 and L6 must state which refusals are generic and which are deliberate source limits | S | | P7 | **C ABI variant growth** | §2.iii hardening rules | S | -| P8 | **Unverified in the inputs:** whether `execution::LifecycleEffects` (execution.hpp:136-139) is ever non-empty from the Pine path, else `PlacementSnapshot::legs` (pine_adapter.hpp:386) is the only live lifecycle (O); the `strategy.risk.allow_entry_in` setter chain beyond `set_pine_risk_direction` pine_strategy_host.cpp:1245 → `set_risk_direction` pine_adapter.cpp:27629 (O); what the ABI receipts pin (F); the probe corpus itself lives outside this tree (F) | verify inside the work item that touches each | F O | +| P8 | **Unverified in the inputs:** whether `execution::LifecycleEffects` (execution.hpp:136-139) is ever non-empty from the Pine path, else `PlacementSnapshot::legs` (pine_adapter.hpp:386) is the only live lifecycle (O); the `strategy.risk.allow_entry_in` setter chain beyond `set_pine_risk_direction` pine_strategy_host.cpp:1245 → `set_risk_direction` pine_adapter.cpp:27835 (O); what the ABI receipts pin (F); the probe corpus itself lives outside this tree (F) | verify inside the work item that touches each | F O | ### 4.3 Open design questions (supervisor level) @@ -1774,7 +1774,7 @@ virtual std::optional resolve_margin_call_units(const NativeMarginCallVi virtual void on_native_margin_call(const native_order::MarginCallEvent&) {} // O, S ``` -- Mechanism (F): a **kernel-originated request**. At each script-bar open and after each applied fill the kernel computes the liquidation level and rests a `Reduce` / `Flatten` with `Stop{level}` bound to the live openings — the shape the adapter builds host-side today (pine_adapter.cpp:21188). New hashed `RequestDefinition::origin { Host, KernelLiquidation, KernelRisk }`, new `CancelReason::Superseded` on re-price, `MarginCallEvent` joins `CommandEvent` (17 → 18, native_order.hpp:1401). O's alternative, a checkpoint on the excursion walk (`apply_excursion` native_execution_consumer.cpp:3726), is not chosen: a reduction mid-path is the entanglement O itself flags (E2). +- Mechanism (F): a **kernel-originated request**. At each script-bar open and after each applied fill the kernel computes the liquidation level and rests a `Reduce` / `Flatten` with `Stop{level}` bound to the live openings — the shape the adapter builds host-side today (pine_adapter.cpp:21378). New hashed `RequestDefinition::origin { Host, KernelLiquidation, KernelRisk }`, new `CancelReason::Superseded` on re-price, `MarginCallEvent` joins `CommandEvent` (17 → 18, native_order.hpp:1401). O's alternative, a checkpoint on the excursion walk (`apply_excursion` native_execution_consumer.cpp:3726), is not chosen: a reduction mid-path is the entanglement O itself flags (E2). - FX revaluation (MG9) falls out once `native_fx_curve` drives the mark (`native_fx_curve.cpp`). - TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:4050 per side; scheduling next to `invoke_bar_open_callback` / `drain_after_applied`), `native_order.*`, `engine_state_hash.cpp` + the coverage checker, `c_abi.cpp` + `pineforge.h` (`pf_native_run_spec_v2` or an extension struct; `CLAUDE.md` SOP). @@ -1797,7 +1797,7 @@ std::optional current_partial_bar() const; Callback chronology contract (the normative part of the lane; F P4.1 + O G8): 1. At one point: match and settle → `on_native_applied` per applied event, FIFO (rule already at `on_native_applied` native-engine.md:1729-1730) → with `BarCloseAndFills`, one recalculation at the fill cursor (reason `OrderFill`), driven from the existing notification drain (native_execution_consumer.cpp:8666-8686) under its re-entrancy guard, bounded by `max_recalculations_per_point`. S would run it *before* the drain; not chosen, the drain order is frozen (E4). -2. Requests born in any of these callbacks follow the existing birth rule: eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:7048). `born_on_remaining_path` (native_execution_consumer.cpp:7048) is not touched — the adapter's COOF deferral (`pending_coof_requests_`, element type `PendingCoofRequest` pine_adapter.hpp:2230) is calibrated against it. +2. Requests born in any of these callbacks follow the existing birth rule: eligible on the unconsumed rest of the bar (`born_on_remaining_path` native_execution_consumer.cpp:7048). `born_on_remaining_path` (native_execution_consumer.cpp:7048) is not touched — the adapter's COOF deferral (`pending_coof_requests_`, element type `PendingCoofRequest` pine_adapter.hpp:2233) is calibrated against it. 3. `EveryModeledPoint`: a calculation at each magnifier sample / observed print, in batch as well as stream (every delivery loop asks the per-point guard `recalculate_at_point` native_execution_consumer.hpp:1470-1475, whose body is `recalculate_at_modeled_point` native_execution_consumer.cpp:8504). `on_native_tick` stays the observation hook (S). 4. `current_partial_bar()` derives from the driver points already recorded (`record_driver` native_execution_consumer.cpp:3670); valid in bar-open, applied and tick callbacks. 5. The kernel never attempts language-state rollback; that stays Pine's (`snapshot_coof_script_state` pine_scheduler_native.cpp:100, `restore_coof_script_state` :118, `commit_coof_script_state` :135) (O). @@ -1884,7 +1884,7 @@ std::optional risk; // NativeRunSpec - Breach: `MatchRejectReason::RiskLimit` for openings (O; F: `RiskHalted`), an optional kernel-issued flatten through A.3's kernel-originated request (`origin = KernelRisk`, F), and a `NativeRiskEvent` in `native_events` (native_host.hpp:1371) (O) or a callback `on_native_risk_event` (S — a virtual, so only inside an epoch batch). The day key uses the existing calendar (`session_day_ordinal` native_calendar.hpp:416), not a chart clock. - S's `directions` / `max_units` / `max_position_units` are existing spec fields (MG1-MG3), not duplicated. -- Interim (F): a C++ host can build all four today from `native_marked_equity` + `on_native_applied`, which is what the adapter does (`update_risk_state` pine_adapter.cpp:18359; `on_applied` pine_adapter.cpp:25422; the `opening_margin_checkpoint` block pine_adapter.cpp:18191); a toolkit helper is an acceptable stand-in until L9. +- Interim (F): a C++ host can build all four today from `native_marked_equity` + `on_native_applied`, which is what the adapter does (`update_risk_state` pine_adapter.cpp:18549; `on_applied` pine_adapter.cpp:25614; the `opening_margin_checkpoint` block pine_adapter.cpp:18381); a toolkit helper is an acceptable stand-in until L9. - TUs: `native_run_spec.{hpp,cpp}`, `native_execution_consumer.cpp` (`admit_opening_inspect` native_execution_consumer.cpp:4050 and the per-point walk), `native_order.hpp`, the state hash. ### A.9 Single-source proposals not adopted diff --git a/docs/design/selected-window-wire-v1.3-revision-3.1.md b/docs/design/selected-window-wire-v1.3-revision-3.1.md index 172479237..5e0eb97c7 100644 --- a/docs/design/selected-window-wire-v1.3-revision-3.1.md +++ b/docs/design/selected-window-wire-v1.3-revision-3.1.md @@ -149,7 +149,7 @@ cases or to any other field; in particular `sharpe_bar` and `sortino_bar` have none on any span. If `cagr` took its compounded form, `diagnostics.annualization` is absent, whatever `calmar` is. -`_annualization_diagnostics` (`docker/run_json.py:4224`) derives the span from +`_annualization_diagnostics` (`docker/run_json.py:4235`) derives the span from the same first and last curve times. Under one day a finite `cagr` on a positive span is enough, since the kernel makes `cagr` null only in the existing domains. From one day on the writer repeats the kernel's arithmetic, `(pow(E/E0, diff --git a/docs/native-refactor-progress.md b/docs/native-refactor-progress.md index dfe91d316..9b165820c 100644 --- a/docs/native-refactor-progress.md +++ b/docs/native-refactor-progress.md @@ -173,7 +173,7 @@ them for the record only. "Linux aarch64" is the maintainers' Cortex-A725 host. | Item | What it is | Measured size | Deferred by | Prototype and evidence | Reopens when | | --- | --- | --- | --- | --- | --- | -| The quiet-bar protocol and one receipt read per bar | The prototype adds a per-bar bar-open request, a bulk fold of quiet driver points, the adapter's quiet-open predicate with lazy open bookkeeping, and a gate before the receipt poll. The optional-hook and quiet-point changes landed the hook declarations and the O(1) `has_command_after` (`src/native_execution_consumer.hpp:419`). The quiet-bar gates gated the read inside `observe_terminal_receipts` (`src/source/pine_adapter.cpp:7048`), which still runs about three times a bar | Quiet-bar measurements, Linux aarch64 (Cortex-A725), callgrind and GBench: about ×0.88 on the 14 probes (an estimate from a measured −13 to −14 % floor bound for the bar-open request alone); after PB the receipt poll still costs about 250 instructions a bar on the no-order floor. On the profiled tree the quiet-bar probe asks the receipt gate 40,996 times and each once-per-bar-open gate 13,248 times: 3.09 reads a bar, 82 % of them stopped at the gate (macOS arm64, AppleClang 17, Release) | the quiet-bar completion report of 2026-09-24 19:56 (after the handle-preserving reissue change); the maintainer decision of 2026-09-25 00:15 UTC+08:00: "The quiet-bar protocol waits for the structural performance prototype's verdict"; the deferred-work review §D | An unpublished prototype branch and its patches. The structural performance prototype's S1 (next row) prototyped the request and the lazy opening | A post-release performance epoch. Done when the receipt gate is asked as often as the once-per-bar gates | +| The quiet-bar protocol and one receipt read per bar | The prototype adds a per-bar bar-open request, a bulk fold of quiet driver points, the adapter's quiet-open predicate with lazy open bookkeeping, and a gate before the receipt poll. The optional-hook and quiet-point changes landed the hook declarations and the O(1) `has_command_after` (`src/native_execution_consumer.hpp:419`). The quiet-bar gates gated the read inside `observe_terminal_receipts` (`src/source/pine_adapter.cpp:7108`), which still runs about three times a bar | Quiet-bar measurements, Linux aarch64 (Cortex-A725), callgrind and GBench: about ×0.88 on the 14 probes (an estimate from a measured −13 to −14 % floor bound for the bar-open request alone); after PB the receipt poll still costs about 250 instructions a bar on the no-order floor. On the profiled tree the quiet-bar probe asks the receipt gate 40,996 times and each once-per-bar-open gate 13,248 times: 3.09 reads a bar, 82 % of them stopped at the gate (macOS arm64, AppleClang 17, Release) | the quiet-bar completion report of 2026-09-24 19:56 (after the handle-preserving reissue change); the maintainer decision of 2026-09-25 00:15 UTC+08:00: "The quiet-bar protocol waits for the structural performance prototype's verdict"; the deferred-work review §D | An unpublished prototype branch and its patches. The structural performance prototype's S1 (next row) prototyped the request and the lazy opening | A post-release performance epoch. Done when the receipt gate is asked as often as the once-per-bar gates | | S1a, kernel quiet-bar driving (structural performance plan) | A bar with nothing live, a flat book and a declined opening is driven in a quiet form: the kernel writes the bookkeeping its four price points would leave (ordinals, the last two driver marks, the floor, the point epochs) without walking them or calling the bar-open callback. Additive host cadence declarations; C hosts declare from their callback table; one C symbol under the PF_API checklist | Structural performance measurements, Linux aarch64 (Cortex-A725), GBench time and `perf stat` instructions, on the profiled tree `b33ea659`, S1a with S1b: no-order floor 0.764 / 0.660 in time (0.718 / 0.656 in instructions), 14 probes 0.958 / 0.921 (0.892 / 0.856), 100 slots 0.972 / 0.956 per slot. A bare kernel host that opts in: idle 0.545× (35 % fewer instructions a bar), market 0.81×, bracket 0.92×. Byte-identical: corpus 312/312, the 132-configuration battery with its hash columns | Ledger finding: "filed for a post-release epoch per the maintainer's cap" (2026-09-24); the decision (2026-09-25); the deferred-work review, §D: S1/S2 ("post-release epoch") | An unpublished prototype branch (five commits on that profiling tree) and its patches | The post-release epoch, with the maintainer decisions listed in the structural performance plan (the API under the existing ruling, the restated gate count) | | S1b, the Pine lazy opening | The Pine host records what its skipped opening and its uncalled input callback would have written, at the bar's first host callback. Member functions only, no layout change | Measured with S1a. A quiet bar then asks 15 gates, not 30, so the quiet-bar cost check `CHECK(asked >= 20 * o.boundary.size())` (`tests/test_adapter_quiet_bar.cpp:1103`) must be restated | As S1a | As S1a. The in-script hash witnesses caught the prototype's one miss (`awaiting_legacy_script_open_ms_`) | As S1a | | S1c, the held-position wake band | S1 for bars that hold a position with nothing live, 41 % of the median public slot's bars. The host declares a price band, and the kernel calls the opening only when the bar's range leaves it. The adapter derives the band from its margin model and declines when it cannot bound it | Structural performance measurements, a measurement-only upper bound without the band, which is not sound in general: 100 slots 0.776 / 0.716 per slot, 14 probes 0.830 / 0.761, cumulative against the request-core profiling baseline with S1, S2 and the initial quiet-bar protocol implementation | As S1a; the maintainer decision "Fund the price band" | An unpublished measurement-only script | As S1a. Size L, parity risk high: TradingView's margin money rules | @@ -183,7 +183,7 @@ them for the record only. "Linux aarch64" is the maintainers' Cortex-A725 host. | Compact journal records and a history ring buffer (request-core profiles) | Smaller command records and a ring buffer for the request core's history. Since the bounded-journal change, `WorkingRequestCore::history` (`include/pineforge/native_order.hpp:1885`) returns a `JournalView` over `journal_` (`include/pineforge/native_order.hpp:2651`) instead of exposing the former `history_` vector. Compact records and a ring remain the deferred request-core profiling proposal | Request-core profiles, Linux aarch64 (Cortex-A725), `perf` call graphs, inclusive shares: at most 3.68 % of bracket, 11.99 % of re-issue, 4.26 % of market and 3.42 % of the 14 probes (retiring the history alone: 1.64, 4.51, 2.53 and 1.00 %); only part is realisable, since records must still be written | The accepted request-core profiling report (2026-09-24), ruling 1: deferred to the structural performance plan; the deferred-work review, §D: "compact journal records" | The request-core profiling code, merged in #285; its profiles are unpublished | A `native_order` layout epoch | | A checked-values handoff for executions | `execution_values` runs twice, in `check_execution` (`src/native_order.cpp:7261`) and again in `apply_execution` (`src/native_order.cpp:7292`); handing the checked values across needs a public type, an API addition with no layout change | As measured above: 2.21 % of bracket for the two together, about 1.1 % recoverable | Deferral ruling 2: "a public checked-values handoff for executions (~1.1% bracket) -> DEFERRED" | As above | A maintainer decision on the API addition | | A chunked `trades_` store | Closed trades in chunks rather than one vector; `trades_` (`include/pineforge/engine.hpp:719`) is protected `BacktestEngine` state, so script ABI | As measured above: on bracket, vector growth 0.64 %, `Trade` moves 0.74 % and `record_close_trade` 0.43 % (at most about 1.4 %); at most 0.35 % on the 14 probes | Deferral ruling 4: "chunked trades_ store (ABI) -> DEFERRED" | As above | An `engine_script_run` epoch that changes the engine's protected layout | -| A zoned chart-day memo (finding 4 of the UTC memo study) | On a chart with a timezone, `libc_chart_day_key` (`src/source/pine_adapter.cpp:18229`) takes the process timezone lock through `ScopedTimezone` and calls `localtime_r` on every bar, because `on_bar_open` (`src/source/pine_adapter.cpp:23230`) reads `chart_day_key` (`src/source/pine_adapter.cpp:23468`) unconditionally. The non-UTC monthly Sharpe/Sortino walk, `month_key_local` (`src/engine_metrics.cpp:55`), pays one `localtime_r` per equity point. UTC charts use the UTC chart-day memo's memo, `civil_chart_day_key` (`src/source/pine_adapter.cpp:18213`) | Not measured on a zoned chart. The UTC path cost 134 instructions a bar before the UTC chart-day memo's memo and 68 after (the UTC chart-day memo, Linux aarch64, callgrind on the floor). The timezone lock profiling: `ScopedTimezone` copies the zone name, which allocates for names of 16 or more characters under libstdc++ | the deferred-work review §D: "the UTC chart-day memo finding 4 (the timezone `localtime_r` memo)", under the maintainer's cap. the UTC memo report: an exact zoned memo "would need the zone's transition table, which belongs in `timezone.cpp`" | None; the UTC memo's witness is `tests/test_chart_day_memo.cpp` | A change to `src/timezone.cpp` that builds an exact zone-transition memo (the timezone lock profiling's Sitka case shows why a shortcut is not exact), byte-identical, with the zoned keys of `test_chart_day_memo` still equal to the computed ones | +| A zoned chart-day memo (finding 4 of the UTC memo study) | On a chart with a timezone, `libc_chart_day_key` (`src/source/pine_adapter.cpp:18419`) takes the process timezone lock through `ScopedTimezone` and calls `localtime_r` on every bar, because `on_bar_open` (`src/source/pine_adapter.cpp:23420`) reads `chart_day_key` (`src/source/pine_adapter.cpp:23658`) unconditionally. The non-UTC monthly Sharpe/Sortino walk, `month_key_local` (`src/engine_metrics.cpp:55`), pays one `localtime_r` per equity point. UTC charts use the UTC chart-day memo's memo, `civil_chart_day_key` (`src/source/pine_adapter.cpp:18403`) | Not measured on a zoned chart. The UTC path cost 134 instructions a bar before the UTC chart-day memo's memo and 68 after (the UTC chart-day memo, Linux aarch64, callgrind on the floor). The timezone lock profiling: `ScopedTimezone` copies the zone name, which allocates for names of 16 or more characters under libstdc++ | the deferred-work review §D: "the UTC chart-day memo finding 4 (the timezone `localtime_r` memo)", under the maintainer's cap. the UTC memo report: an exact zoned memo "would need the zone's transition table, which belongs in `timezone.cpp`" | None; the UTC memo's witness is `tests/test_chart_day_memo.cpp` | A change to `src/timezone.cpp` that builds an exact zone-transition memo (the timezone lock profiling's Sitka case shows why a shortcut is not exact), byte-identical, with the zoned keys of `test_chart_day_memo` still equal to the computed ones | | Thread-local reads that remain per call | The per-pump state cache moved the three scopes' writes onto the pump's block, `pump_ambient` (`src/native_execution_consumer.hpp:641`). A reader still reaches the block through `tl_runtime_ambient` (`src/ta_extremes_volume.cpp:30`): `bar_context` (`src/ta_extremes_volume.cpp:83`), read by every `ExtremeRing::update` (`src/ta_extremes_volume.cpp:123`), which backs `ta.highest`, `ta.lowest`, `ta.highestbars` and `ta.lowestbars` and, through an embedded `Highest` and `Lowest`, `ta.stoch`, `ta.wpr` and `ta.range`; and `matching_day_partition` (`src/timeframe.cpp:687`), read by the default-anchored `ta.vwap` through `session_day_index`, by `crosses_boundary` on a DAY period (`timeframe.change("D")`) and by the session-period helpers. `decompose_ms_local` (`src/session_time.cpp:250`) reads two thread-locals per call on a zoned clock. The prepared order storage now caches one idle block per token type through `free_block` (`src/native_order.cpp:1001`), replacing the former `free_blocks` vector; prepared allocation and deallocation still reach that thread-local cache. `ema_na_warmup_flag` is read once per EMA, at its first compute | One thread-local access per reader call. The fourth review, macOS arm64, a dlopen'd tutorial-MACD probe, counted by `_tlv_get_addr` breakpoints and by descriptor patching: 1.000 a bar for one `ta::Highest` and for one default `ta::VWAP`, 1.999 for one `pine_hour` on a New York clock, 0.000 on the no-order floor and with orders. On Linux a TLS-descriptor call costs about 20 instructions: the per-pump state cache's 13 → 4 calls a bar was 260 → 80 instructions (Linux aarch64, GCC 13, callgrind). The accepted integration: 2.00 a bar on the zoned probe 064, 0.08 on probe 008 | None for the TA and day-partition readers: the integration acceptance (ledger, 2026-09-24 23:30 UTC) reads "dlopen TLS 13->0/bar", and the accepted integration's report names only the timezone and order-core residuals. This row is their record | Unpublished (the fourth review's scratch) | A byte-identical change that hands these readers the pump's block without a thread-local read. `ta::bar_context()` is script ABI, so its public declaration stays as it is | | "audit-off mode", "LTO", "data-oriented layout" | Named in "Maintainer decision 2026-09-25 00:20 UTC+08:00" among the post-release structural items: "S1 bulk quiet bars, S2 per-strategy specialization, audit-off mode, LTO+PGO, data-oriented layout". No engine or campaign document defines, designs or measures the three | None | The maintainer decision | None | Struck from this inventory until a design note defines one of them | @@ -210,7 +210,7 @@ or more, and the same run under Full holds 66,000 points in one block of **PERF-L4 reads the receipts about three times a bar, not once.** PERF-L4's scope asked it to "read the receipts ONCE per bar instead of three times". The implementation gated the read instead: `observe_terminal_receipts` -(`src/source/pine_adapter.cpp:7048`) returns at its receipt gate when no +(`src/source/pine_adapter.cpp:7108`) returns at its receipt gate when no command lies above its cursor. `every_gate_is_taken_both_ways` (`tests/test_adapter_quiet_bar.cpp:1110`) prints `gate ReceiptRead asked 40996 quiet 33552 live 7444`, against `asked 13248` @@ -241,7 +241,7 @@ Mac's 489 and 412 and 1003 of 1249, and no Mac output moved. - **PERF-L4's retired reader slots.** `event_high_water_reader_` and `terminal_receipt_high_water_reader_` - (`include/pineforge/source/pine_adapter.hpp:2661-2662`), with their type + (`include/pineforge/source/pine_adapter.hpp:2666-2667`), with their type `ReceiptHighWaterReader`, are storage nothing reads or writes since PERF-L4: they keep `PineExecutionAdapter`'s layout, which is part of `PineStrategyHost`'s and so of the generated script's ABI. They go at the next diff --git a/docs/pages/contributing-llm.md b/docs/pages/contributing-llm.md index 2da8fbdf2..f124a998e 100644 --- a/docs/pages/contributing-llm.md +++ b/docs/pages/contributing-llm.md @@ -265,7 +265,7 @@ decision, and removing one is a regression: **kernel** — the Pine-agnostic half of the engine: matching, fills, sizing, margin, settlement, indicators, calendars. Buildable alone as -`PineForge::kernel` CMakeLists.txt:185 with `PINEFORGE_BUILD_SOURCE_LAYER` +`PineForge::kernel` CMakeLists.txt:186 with `PINEFORGE_BUILD_SOURCE_LAYER` CMakeLists.txt:43 off. **adapter** — `src/source/` and `src/compat/pine/`: the TradingView-parity diff --git a/docs/pages/memory-management.md b/docs/pages/memory-management.md index 6b6a59ddf..720166657 100644 --- a/docs/pages/memory-management.md +++ b/docs/pages/memory-management.md @@ -69,7 +69,7 @@ read the records from the sink. See [Report schema](@ref report_schema). The default **unjailed** `run_json` route, with a sink-capable module, installs a temporary path sink (`_new_report_sink` -docker/run_json.py:6116-6156; `_ReportSink` docker/run_json.py:6740-6792). +docker/run_json.py:6166-6206; `_ReportSink` docker/run_json.py:6790-6842). The full equity curve is **not resident**: with disk scratch and no arena attached, its resident equity-curve buffers are **O(1) in bars**. Binary curve scratch on **disk** still costs **24 B per script bar** @@ -77,16 +77,16 @@ Binary curve scratch on **disk** still costs **24 B per script bar** **6 MiB at 262,144 bars** and **48 MiB at 2,097,152 bars**, calculated payload sizes excluding other scratch and output. The sink's `E` output and the default route's JSON artifact also grow with -the bar count: `_build_sink_report` docker/run_json.py:8109-8207 supplies lazy -record arrays, and `_prepare_stream_report` docker/run_json.py:8031-8053 +the bar count: `_build_sink_report` docker/run_json.py:8194-8291 supplies lazy +record arrays, and `_prepare_stream_report` docker/run_json.py:8051-8073 serializes them to a temporary artifact. The final artifact's aligned I/O -window is fixed at **1 MiB** (`report_window` docker/run_json.py:7654; -`_json_temporary` docker/run_json.py:8044-8045). +window is fixed at **1 MiB** (`report_window` docker/run_json.py:7665; +`_json_temporary` docker/run_json.py:8055-8056). Columns and spools select **1 MiB** or **64 KiB** windows from the per-run budget below; the sink reader keeps its **64 KiB** window -(`_BoundedJSONFile.window` docker/run_json.py:7362; -`_UncachedSinkFile` docker/run_json.py:7569-7571; -`_JSONColumn` docker/run_json.py:7681-7689). +(`_BoundedJSONFile.window` docker/run_json.py:7371; +`_UncachedSinkFile` docker/run_json.py:7580-7582; +`_JSONColumn` docker/run_json.py:7692-7700). Lazy serialization bounds buffers, not output bytes. Python artifact windows share the per-run default budget `_ARTIFACT_WINDOW_BUDGET_BYTES = 6 * 1024 * 1024` (**6 MiB**). @@ -137,7 +137,7 @@ There is no RAM fallback (the comment on `ScratchFile` include/pineforge/detail/scratch_file.hpp:173-179). A **jailed run without `--report-sink-fd` has no sink at all**: -`_new_report_sink` docker/run_json.py:6146-6167 returns `None`, so it retains +`_new_report_sink` docker/run_json.py:6166-6187 returns `None`, so it retains the native curve and owning FULL report copy described above. It also builds the whole report text in memory before writing stdout, and that text is the largest per-bar term of this route. It is held as UTF-8 bytes: about one byte @@ -190,13 +190,13 @@ bytes. The sink-off jailed route buffers the whole report text before writing to stdout, so a serialization error publishes no partial report -(`_prepare_report_json`, `docker/run_json.py:8065`; publication in -`_write_report`, `docker/run_json.py:4916`). Its UTF-8 byte buffer uses about +(`_prepare_report_json`, `docker/run_json.py:8076`; publication in +`_write_report`, `docker/run_json.py:4927`). Its UTF-8 byte buffer uses about one byte of resident text per output byte, plus bounded encoding and read-back buffers, on top of the input mapping and the engine's storage: this is `O(output)` memory for the report the caller requested. Passing `--report-sink-fd` selects the fd sink before the jailed no-sink fallback -(`_new_report_sink`, `docker/run_json.py:6155`), avoiding the full report-text +(`_new_report_sink`, `docker/run_json.py:6166`), avoiding the full report-text buffer; stdout then buffers only the bounded summary. ## The report-sink descriptor: what a caller passes and what it gets {#memory_sink_fd} @@ -217,19 +217,19 @@ passes every argument that is not `hpo` to `/opt/pineforge/bin/entrypoint.sh`, t decides it is `grep -c REPORT_SINK_FD /opt/pineforge/bin/entrypoint.sh` (0 on 1.7.0, 1 from 1.8.0), or the sha256 of that file against the release's `HARNESS-SHA256SUMS`. After a run, a descriptor that received no header line, with the trade and equity arrays on stdout, means the variable was ignored. -`run_json.py --capabilities-json` (`_capabilities_main` at `docker/run_json.py:4993`) carries no +`run_json.py --capabilities-json` (`_capabilities_main` at `docker/run_json.py:5004`) carries no report-sink key, so it is not a probe. **What the caller passes.** A descriptor number N, open for writing in the process that executes the entrypoint: a FIFO or an `AF_UNIX` stream socket. N is from 3 through `INT_MAX` and is not 5. A request that breaks any rule below fails at argument validation, before any bar is read, as one failure line on stdout with code `run_request_invalid` and exit status 2 (`_validate_sink_options` at -`docker/run_json.py:6059-6113`): +`docker/run_json.py:6120-6163`): | Refused | Why | | --- | --- | | N below 3 or above `INT_MAX` | outside the range | -| a value that is not an integer, an empty string included | the option is an integer option (`--report-sink-fd` at `docker/run_json.py:5475`), so the argument parser refuses it with the same code and exit status | +| a value that is not an integer, an empty string included | the option is an integer option (`--report-sink-fd` at `docker/run_json.py:5548`), so the argument parser refuses it with the same code and exit status | | N = 5 | the engine's scratch arena owns descriptor 5 (`PF_SCRATCH_ARENA_FD`) | | together with `--report-sink PATH` | the file sink and the descriptor sink are exclusive | | together with `--bench` or `--validate-window-only` | the descriptor carries one run's stream | @@ -247,7 +247,7 @@ number: in particular 6 is not reserved. A host that gives 6 to its own sink and it. **Route.** The descriptor sink is chosen before the jailed check, so a jailed run that is given the -descriptor uses it (`_new_report_sink` at `docker/run_json.py:6146-6167`). A strategy library without the five descriptor-sink +descriptor uses it (`_new_report_sink` at `docker/run_json.py:6166-6187`). A strategy library without the five descriptor-sink symbols fails with the typed `symbol_missing` reason (same lines). **What is written to the descriptor.** The `pf-report-sink/v1` stream described in [Report @@ -257,7 +257,7 @@ the `O`, `V`, `K`, `X` and `H` records, then one trailer line `{"trailer":"pf-report-sink","version":1,"counts":{...},"sha256":"..."}`. The sha256 covers the header and every data line with its LF, not the trailer. The writer and the header are byte-identical in v1.7.0 and at main. `run_json.py` closes the descriptor after the run through the `close` method of `_InheritedSinkFd` -(`docker/run_json.py:6159-6173`), so a reader sees EOF. +(`docker/run_json.py:6209-6223`), so a reader sees EOF. **What stdout carries.** Only the bounded summary document: about 9 KB in the measured runs (9,274 bytes at 2 M bars), with no trade or equity array. Do not use it to verify the stream: the stdout summary of a @@ -265,8 +265,8 @@ measured jailed descriptor run carried no sha256 of the stream, so a caller has the trailer against. The process entry reserves the caller's original stdout as a private report descriptor before argument parsing or -strategy loading (`_ReportOutput` at `docker/run_json.py:8529`; `_host_main` at -`docker/run_json.py:8734`). It redirects native fd 1 to fd 2, and writes the summary and every typed +strategy loading (`_ReportOutput` at `docker/run_json.py:8540`; `_host_main` at +`docker/run_json.py:8745`). It redirects native fd 1 to fd 2, and writes the summary and every typed failure through the saved descriptor. The native stream N stays on the descriptor supplied by the caller. Isolation holds only when the harness keeps stdout and stderr separate. If they name the same file (for example, with `2>&1`), the host writes one stderr line, @@ -274,15 +274,15 @@ Isolation holds only when the harness keeps stdout and stderr separate. If they publication through the saved descriptor. The host refuses a report descriptor that shares a number or open file with N, the phase descriptor, or arena fd 5, using `run_request_invalid` (`_validate_report_fd_separation` at -`docker/run_json.py:8696`). The library can write the saved descriptor because it runs in this process. +`docker/run_json.py:8707`). The library can write the saved descriptor because it runs in this process. The descriptor is non-inheritable, so exec'd children do not get it. Code that deliberately writes through that file description can still affect the report. This transport arrangement isolates ordinary native fd 1 output when the streams stay separate; it is not a security boundary. Imported library calls write reports and failures to the current Python stdout without reserving or -redirecting descriptors (`_report_output` at `docker/run_json.py:8651`). Each call to the library entry +redirecting descriptors (`_report_output` at `docker/run_json.py:8662`). Each call to the library entry gets its own publication guard, so repeated runs remain independent (`_LibraryReportRun` at -`docker/run_json.py:8508`). +`docker/run_json.py:8519`). **When the report is complete.** The verified trailer is the completion marker, and EOF alone is not. Read every record as provisional until the trailer's counts and sha256 equal what the reader counted and @@ -300,14 +300,14 @@ trailer, 7 internal error, 8 `sink_exhausted` (the stream exceeds `--max-bytes`; `sink_exhausted` at `docker/pineforge_sink_finalize.py:115-123`). Its output is provisional until exit 0, and a consumer must discard everything it received on exit 3 to 8 (`PROVISIONAL` at `docker/pineforge_sink_finalize.py:24-26`). The failure catalog maps exit 8 to `report_sink_failed` -with `reason` `sink_exhausted` (`sink_exhausted` at `docker/run_failure_codes.json:1174`). A sink failure carries code `report_sink_failed` and a `reason` -from the closed list of the catalog (`report_sink_failed` at `docker/run_failure_codes.json:1114`): +with `reason` `sink_exhausted` (`sink_exhausted` at `docker/run_failure_codes.json:1175`). A sink failure carries code `report_sink_failed` and a `reason` +from the closed list of the catalog (`report_sink_failed` at `docker/run_failure_codes.json:1115`): `arena_exhausted`, `arena_io`, `arena_backing`, `scratch_io`, `sink_io`, `sink_peer_closed` and `sink_exhausted`. Only some reasons carry sizes: `arena_exhausted` carries `high_water_bytes`, `arena_bytes`, `claimed_bytes`, `store` and `needed_bytes_lower_bound`, `sink_exhausted` carries `sink_bytes` and `cap_bytes`, and every other reason carries `reason` alone, a closed reader included. A write error while `run_json.py` prepares the report in scratch is raised as `RunFailure` with -`scratch_io` (`docker/run_json.py:8056-8057`); it is not a write to the descriptor. `sink_peer_closed` is also raised when the descriptor setup itself +`scratch_io` (`docker/run_json.py:8067-8068`); it is not a write to the descriptor. `sink_peer_closed` is also raised when the descriptor setup itself finds a closed peer (`EPIPE`, `ECONNRESET` or `ENOTCONN`, with the message "report sink fd setup failed"; `RunFailure` at `docker/run_json.py:2199-2202`). The descriptor then holds records without a verified trailer, and a caller treats it as absent. The engine writer loops a short write to completion, @@ -333,9 +333,9 @@ to EOF got exit status 0 and a complete verified trailer. | `PF_SCRATCH_WINDOW_BYTES` | environment | 32 MiB | Total cap on the arena bytes mapped at one time, rounded down to whole pages. Mapped pages count toward the resident size while mapped | | `pf_scratch_tables_v1` through `strategy_set_scratch_tables_v1` | descriptor of eleven maxima | disk tables; with `bounded_memory`, the policy values of [Scratch tables](@ref scratch_tables) | Sizes the fixed slab of the historical tables: `K_e`, `B_e`, `K_c`, `B_c`, `U`, `B_U`, `V`, `B_V`, `R` and the unused q pair. Exceeding a maximum fails the run with `scratch_capacity_exceeded` | | A report sink (`strategy_set_report_sink_v1`, `strategy_set_report_sink_fd_v1`) | setter | none | Streams the trades and the equity curve to the sink instead of keeping them in the report. See [Report schema](@ref report_schema) | -| `--spill-dir DIR` | `docker/run_json.py` flag (`docker/run_json.py:5486`), available only when `run_json.py` is invoked directly: the release entrypoint forwards no command-line flags | the system temporary directory, then `/var/tmp` if disk backing is required | The writable directory for engine scratch and unlinked report temporaries when no arena is attached. The jailed profile refuses it | -| `--profile jailed` or `PINEFORGE_RUN_JSON_PROFILE=jailed` | `docker/run_json.py` flag or environment (`docker/run_json.py:5524,5546`); the release entrypoint forwards the environment form only | `unjailed` | Selects the bounded-memory table backend and the inherited arena for the Python host. It is the scratch profile of the engine run, not a read-only container: the entrypoint itself still compiles in `${TMPDIR:-/tmp}`. Every executed success report carries `diagnostics.run_profile` (`"jailed"` or `"unjailed"`) and `diagnostics.scratch_arena_jailed` (true only when the jailed profile ran and the arena admission succeeded; `_add_scratch_arena_diagnostics` at `docker/run_json.py:6035-6040`); the marker records the checked profile and the admission, not an independent check of operating-system confinement | -| `PINEFORGE_REPORT_SINK_FD` | environment, read by the release entrypoint (`PINEFORGE_REPORT_SINK_FD` at `docker/entrypoint.sh:875`) | unset | When set, the entrypoint passes `--report-sink-fd ` to `run_json.py`, so the report records stream to that inherited descriptor. `run_json.py` validates the value: a descriptor number from 3 through `INT_MAX`, not 5, not combined with `--report-sink`, not in a benchmark run (`PINEFORGE_BENCH`), not in a validate-only run and not the same number as `--run-phase-fd` (`_validate_sink_options` at `docker/run_json.py:6059-6113`); a refusal is a `run_request_invalid` failure with exit status 2. Unset, nothing changes | +| `--spill-dir DIR` | `docker/run_json.py` flag (`docker/run_json.py:5559`), available only when `run_json.py` is invoked directly: the release entrypoint forwards no command-line flags | the system temporary directory, then `/var/tmp` if disk backing is required | The writable directory for engine scratch and unlinked report temporaries when no arena is attached. The jailed profile refuses it | +| `--profile jailed` or `PINEFORGE_RUN_JSON_PROFILE=jailed` | `docker/run_json.py` flag or environment (`docker/run_json.py:5597,5474`); the release entrypoint forwards the environment form only | `unjailed` | Selects the bounded-memory table backend and the inherited arena for the Python host. It is the scratch profile of the engine run, not a read-only container: the entrypoint itself still compiles in `${TMPDIR:-/tmp}`. Every executed success report carries `diagnostics.run_profile` (`"jailed"` or `"unjailed"`) and `diagnostics.scratch_arena_jailed` (true only when the jailed profile ran and the arena admission succeeded; `_add_scratch_arena_diagnostics` at `docker/run_json.py:6046-6051`); the marker records the checked profile and the admission, not an independent check of operating-system confinement | +| `PINEFORGE_REPORT_SINK_FD` | environment, read by the release entrypoint (`PINEFORGE_REPORT_SINK_FD` at `docker/entrypoint.sh:875`) | unset | When set, the entrypoint passes `--report-sink-fd ` to `run_json.py`, so the report records stream to that inherited descriptor. `run_json.py` validates the value: a descriptor number from 3 through `INT_MAX`, not 5, not combined with `--report-sink`, not in a benchmark run (`PINEFORGE_BENCH`), not in a validate-only run and not the same number as `--run-phase-fd` (`_validate_sink_options` at `docker/run_json.py:6120-6163`); a refusal is a `run_request_invalid` failure with exit status 2. Unset, nothing changes | | `pf_cache_policy_v1` through `strategy_set_cache_policy_v1` | optional bytes per supported cache kind, per handle | internal defaults (next section) | Chooses optional resident cache sizes. Omitted kinds use defaults; explicit zero bypasses caching. Nonzero ledger bytes must convert to 1–1,048,576 decoded entries; other kinds bypass below one line. A cold read uses backing. Settings freeze at run admission; an idle replacement affects the next run. A standalone C++ ledger also accepts an optional row budget through its constructor or `TradeLedger::set_cache_rows`, including zero to bypass. An omitted ledger setting keeps the engine's provisional default byte cap of 67,108,864 (64 MiB, unmeasured), which holds 215,070 ledger entries with or without a report sink (`CacheDefaults::apply_ledger_cache`, `include/pineforge/cache_policy.hpp:119-124`); run begin applies it (`apply_ledger_cache`, `src/native_execution_consumer.cpp:3335`) | | Calendar projection and recorded-tape pages | C++ overloads | 256 KiB per calendar column and per recorded tape | Selects optional resident payload and complete shared read-through-cache metadata, allocated lazily; zero, sub-page sizes and optional allocation failures read from the provider through bypass. Production projections up to one transfer page stay in memory; larger projections use the report-sink scratch directory or its default. Each active read has a separate fixed transfer page. Logical rows and raw-array ownership are unchanged; see [Feed projection readers](../feed-projections.md) | | Resident event and row limits of the command journal and order book | implementation constants | fixed constants (next section) | The optional cache policy does not resize these fixed workspaces or cap retained history. The journal's resident-event threshold and the book's resident-row limit remain fixed | diff --git a/docs/pages/native-engine.md b/docs/pages/native-engine.md index e31230279..03ada92ea 100644 --- a/docs/pages/native-engine.md +++ b/docs/pages/native-engine.md @@ -644,7 +644,7 @@ segment, and discrete points, keep the ordinary birth gate above. `on_native_bar_open` fires at the modeled opening, before that point's matching pass (`native_execution_consumer.cpp:8757`). **Lookahead warning:** the `Bar` it receives is the *complete* script bar — the consumer has already set -`engine.current_bar_ = open_view` (`native_execution_consumer.cpp:8610`), the +`engine.current_bar_ = open_view` (`native_execution_consumer.cpp:8749`), the complete bar unless the spec asks for `NativeOpenBarView::OpenOnly` — so its high, low and close are the finished bar's, not what is known at the open. A host that must decide on open-only information reads @@ -1496,7 +1496,7 @@ adapter answers `margin_check_allowed` with TradingView's scheduling — which includes the post-exit re-size: when a priced bracket leg of the script bar fills, the slice resting at that bar's adverse extreme was sized on the pre-exit book, and the legacy broker cancelled and re-scheduled it there -(`margin_check_allowed` `pine_adapter.cpp:18665-18698`), so the adapter admits +(`margin_check_allowed` `pine_adapter.cpp:18855-18888`), so the adapter admits the kernel's own point for that driver point while (and only while) a slice rests — `resolve_margin_requirement` with its ten-significant-digit money and @@ -4551,10 +4551,10 @@ to `source::PineStrategyHost` (`pine_strategy_host.hpp:335-338`), and the cap type itself lives in the adapter (`IntradayCap` `intraday_cap.hpp:84`). Nor is there a build-level one. The two source sets are disjoint: -`PINEFORGE_SOURCE_LAYER_SOURCES` (`CMakeLists.txt:91`) holds all six +`PINEFORGE_SOURCE_LAYER_SOURCES` (`CMakeLists.txt:92`) holds all six `src/compat/pine/` units and all fourteen `src/source/` ones, and -`PINEFORGE_KERNEL_SOURCES` (`CMakeLists.txt:118`) holds the kernel's own -forty-five, which are what `add_library` (`CMakeLists.txt:185`) compiles into +`PINEFORGE_KERNEL_SOURCES` (`CMakeLists.txt:119`) holds the kernel's own +forty-five, which are what `add_library` (`CMakeLists.txt:186`) compiles into `pineforge_kernel`. `libpineforge.a` still carries both sets when `PINEFORGE_BUILD_SOURCE_LAYER` is ON, which is the default; the kernel archive exists either way. The installed-header closure is clean too, which the diff --git a/docs/pages/pine-to-native.md b/docs/pages/pine-to-native.md index 8bb85b5e0..dc7f09c9d 100644 --- a/docs/pages/pine-to-native.md +++ b/docs/pages/pine-to-native.md @@ -1025,7 +1025,7 @@ cmake -S . -B build-kernel -DPINEFORGE_BUILD_SOURCE_LAYER=OFF -DPINEFORGE_BUILD_ ``` `PINEFORGE_BUILD_SOURCE_LAYER` CMakeLists.txt:43 OFF leaves `libpineforge.a` -holding exactly the objects of `PineForge::kernel` CMakeLists.txt:185, and +holding exactly the objects of `PineForge::kernel` CMakeLists.txt:186, and every example already links that target, so a source-layer symbol reaching an example is a link error in the default build too. diff --git a/docs/pages/public-contract.md b/docs/pages/public-contract.md index 1530c643e..fe3ea81bc 100644 --- a/docs/pages/public-contract.md +++ b/docs/pages/public-contract.md @@ -192,8 +192,8 @@ The selected-window path is **PUBLIC for walk-forward use** through the A walk-forward driver such as the HPO native layer may submit successive non-overlapping windows over one chart, for example `[T0, T1)` followed by `[T1, T2)`. Each window is one independent request. The CLI creates a fresh -strategy state for its request in `_make_state` (`docker/run_json.py:5105`) -inside `_selected_run` (`docker/run_json.py:5010`). +strategy state for its request in `_make_state` (`docker/run_json.py:5116`) +inside `_selected_run` (`docker/run_json.py:5021`). ### Public request and capability surface @@ -201,18 +201,18 @@ These are the selected-window options declared by the CLI: | Option | Contract | Declaration | | --- | --- | --- | -| `--report-policy selected-window/v1` | Opt into the selected-window report policy. | `add_argument` (`docker/run_json.py:5525`) | -| `--window-start-ms` | Start T in calendar-aligned Unix milliseconds. | `add_argument` (`docker/run_json.py:5518`) | -| `--window-end-ms` | End E in calendar-aligned Unix milliseconds; T < E. | `add_argument` (`docker/run_json.py:5529`) | -| `--preroll-bars` | Requested warm-up N in script bars, from 0 through 5000. | `add_argument` (`docker/run_json.py:5531`) | -| `--fed-start-ms` | Frozen start F of the supplied primary feed; F <= T. | `add_argument` (`docker/run_json.py:5533`) | -| `--capabilities-json` | With `--so LIB`, report runtime support without reading OHLCV or running a strategy. | `add_argument` (`docker/run_json.py:5512`) | -| `--validate-window-only` | Validate and plan the request; return `report_window` with `applied: false`, without executing a strategy. | `add_argument` (`docker/run_json.py:5559`) | -| `--run-phase-fd` | Inherited AF_UNIX SOCK_STREAM descriptor N >= 3 for phase transport v1. | `add_argument` (`docker/run_json.py:5533`) | -| `--request-feed-inventory` | Inventory bound to the strategy library, used for selected-window admission. | `add_argument` (`docker/run_json.py:5535`) | -| `--report-shape` | This build offers `full/v1` for selected-window requests. | `add_argument` (`docker/run_json.py:5525`) | -| `--curve-point-budget` | Refused by this build, which does not offer a compact shape. | `add_argument` (`docker/run_json.py:5579`) | -| `--results-digest` | Refused by this build, which does not offer a results digest. | `add_argument` (`docker/run_json.py:5559`) | +| `--report-policy selected-window/v1` | Opt into the selected-window report policy. | `add_argument` (`docker/run_json.py:5565`) | +| `--window-start-ms` | Start T in calendar-aligned Unix milliseconds. | `add_argument` (`docker/run_json.py:5568`) | +| `--window-end-ms` | End E in calendar-aligned Unix milliseconds; T < E. | `add_argument` (`docker/run_json.py:5570`) | +| `--preroll-bars` | Requested warm-up N in script bars, from 0 through 5000. | `add_argument` (`docker/run_json.py:5572`) | +| `--fed-start-ms` | Frozen start F of the supplied primary feed; F <= T. | `add_argument` (`docker/run_json.py:5574`) | +| `--capabilities-json` | With `--so LIB`, report runtime support without reading OHLCV or running a strategy. | `add_argument` (`docker/run_json.py:5576`) | +| `--validate-window-only` | Validate and plan the request; return `report_window` with `applied: false`, without executing a strategy. | `add_argument` (`docker/run_json.py:5579`) | +| `--run-phase-fd` | Inherited AF_UNIX SOCK_STREAM descriptor N >= 3 for phase transport v1. | `add_argument` (`docker/run_json.py:5582`) | +| `--request-feed-inventory` | Inventory bound to the strategy library, used for selected-window admission. | `add_argument` (`docker/run_json.py:5585`) | +| `--report-shape` | This build offers `full/v1` for selected-window requests. | `add_argument` (`docker/run_json.py:5588`) | +| `--curve-point-budget` | Refused by this build, which does not offer a compact shape. | `add_argument` (`docker/run_json.py:5590`) | +| `--results-digest` | Refused by this build, which does not offer a results digest. | `add_argument` (`docker/run_json.py:5593`) | An execution request supplies the policy, all four window arguments and explicit `--input-tf` and `--script-tf` values, as checked by @@ -234,8 +234,8 @@ the inventory setting reaches both binder and runner. Before relying on the CLI path, probe the library with `docker/run_json.py --so LIB --capabilities-json`. The probe is -`_capabilities_main` (`docker/run_json.py:4993`); its support decision is -`_runtime_supports_selected_window` (`docker/run_json.py:4964`), which checks +`_capabilities_main` (`docker/run_json.py:5004`); its support decision is +`_runtime_supports_selected_window` (`docker/run_json.py:4975`), which checks the selected-window ABI, execution observer, phase transport and planner. The capability record is built by `capabilities_record` (`docker/selected_window_report.py:756`). @@ -247,7 +247,7 @@ understands. This tree declares `["1.3"]` on a supported runtime and `[]` otherwise; see the [capability table](../../docker/README.md#selected-window-contract-revision) for revision history. The revision is declared in the probe, not in each report. The probe also sets `selected_window_bars_bin_version` -(`docker/run_json.py:5006`) to 0: selected-window requests +(`docker/run_json.py:5017`) to 0: selected-window requests require a CSV chart. General binary-feed support does not imply selected-window binary-chart support. @@ -300,7 +300,7 @@ does not establish execution success. Request or execution failures have a nonzero harness status and the typed `code` / `args` failure line described in @ref run_failure_codes. If stdout cannot take a whole report, the harness returns 1 with the reason on stderr and no second JSON document, as enforced by -`_write_report` (`docker/run_json.py:4916`). +`_write_report` (`docker/run_json.py:4927`). **Functions and flags are append-only.** No published function or flag above is removed or renamed in 1.x; new ones are added. The C layout rules above diff --git a/docs/pages/run-failure-codes.md b/docs/pages/run-failure-codes.md index 26ac70062..dc856a24e 100644 --- a/docs/pages/run-failure-codes.md +++ b/docs/pages/run-failure-codes.md @@ -210,7 +210,7 @@ other `strategy_create_failed` lines omit it. | `engine_invariant` | `engine_fault` | no | `reason` (vocab, 1 values, optional) | An engine invariant failed: a PineForge bug. | | `engine_unclassified_error` | `engine_fault` | no | none | An error no coded site reported: a PineForge bug. | | `harness_internal_error` | `engine_fault` | no | none | The run harness failed: a PineForge bug. | -| `lot_grid_rejected` | `symbol_metadata` | yes | none | The symbol's lot size (syminfo.mincontract, the quantity grid) is not a value the engine accepts. | +| `lot_grid_rejected` | `symbol_metadata` | yes | none | The symbol's quantity step (syminfo.qty_step) or minimum contract size (syminfo.mincontract) is not a value the engine accepts. | | `no_data_request` | `no_data` | no | `call` (pine_source), `function` (vocab, 9 values), `line` (integer) | The script read a request whose data PineForge does not pin for this run. | | `other_symbol_feed_missing` | `no_data` | no | `timeframe` (timeframe, optional) | No feed is installed for another symbol's request at its timeframe. | | `other_symbol_request` | `no_data` | no | `call` (pine_source), `function` (vocab, 2 values), `line` (integer), `symbol` (symbol, optional) | The script read another symbol's request and no bars of that symbol were installed. | diff --git a/include/pineforge/source/pine_adapter.hpp b/include/pineforge/source/pine_adapter.hpp index b178be1a0..22587b2d5 100644 --- a/include/pineforge/source/pine_adapter.hpp +++ b/include/pineforge/source/pine_adapter.hpp @@ -1766,7 +1766,8 @@ class PineExecutionAdapter { double qty = std::numeric_limits::quiet_NaN(), const std::string& oca_name = {}, double profit_ticks = std::numeric_limits::quiet_NaN(), - double loss_ticks = std::numeric_limits::quiet_NaN()); + double loss_ticks = std::numeric_limits::quiet_NaN(), + std::uint64_t reservation_sequence = 0); void exit_cancel_bracket(const SourceId& exit_id, const SourceId& from_entry, const std::string& comment = {}); void cancel(const SourceId& id); @@ -2162,6 +2163,8 @@ class PineExecutionAdapter { struct PendingRelativeExit { SourceId exit_id; SourceId from_entry; + // Queue position of the source call, before its levels can be lowered. + std::uint64_t reservation_sequence = 0; double trail_points = std::numeric_limits::quiet_NaN(); double trail_offset = std::numeric_limits::quiet_NaN(); double trail_price = std::numeric_limits::quiet_NaN(); @@ -2318,7 +2321,9 @@ class PineExecutionAdapter { double& qty_percent, double live_basis, double percentage_basis, - double& reserved_qty) const; + double& reserved_qty, + std::uint64_t& reservation_sequence, + std::uint64_t origin = 0) const; void reconcile_deferred_exit_reservations(const SourceId& from_entry, double live_basis); void reconcile_exit_reservations_after_margin(); diff --git a/runner/README.md b/runner/README.md index 6886be92d..876825e57 100644 --- a/runner/README.md +++ b/runner/README.md @@ -721,9 +721,14 @@ chart-input ledger above. Keep symbol metadata consistent with the corresponding backtest. `--syminfo` supports `type`, `currency`, `basecurrency`, `description`, `volumetype`, -`mintick`, `pointvalue`, `qty_step`, `margin_long` and `margin_short`. Omitted -values retain the engine defaults; a symbol name alone does not download -instrument metadata. `--session` defaults to `24x7`, `--timezone` to `UTC`; +`mintick`, `pointvalue`, `qty_step`, `mincontract`, `margin_long` and `margin_short`. +`qty_step` and `mincontract` must be finite positive numbers. The runner forwards +both through the existing numeric metadata setter: `mincontract` supplies the +quantity grid when `qty_step` is absent; an explicit `qty_step` wins in either +CLI order. For example, `--syminfo mincontract=0.00001` supplies a grid without +also declaring `qty_step`. Omitted values retain the engine defaults; a symbol +name alone does not download instrument metadata. `--session` defaults to `24x7`, +`--timezone` to `UTC`; `--chart-timezone` is independent and defaults to the engine UTC chart path. Repeat `--input TITLE=VALUE` and `--override KEY=VALUE` for strategy settings. Those setters belong to the compiled strategy; it must validate unsupported diff --git a/runner/main.cpp b/runner/main.cpp index 36a9a21f7..6386e7180 100644 --- a/runner/main.cpp +++ b/runner/main.cpp @@ -711,7 +711,8 @@ class Strategy { throw std::runtime_error("strategy rejected string metadata"); } else { double v = parse_json(value).real(); - if (key == "mintick" || key == "pointvalue" || key == "qty_step") { + if (key == "mintick" || key == "pointvalue" || key == "qty_step" || + key == "mincontract") { if (v <= 0) throw std::runtime_error( "symbol tick/point/quantity steps must be positive"); diff --git a/scripts/check_corpus_parity.sh b/scripts/check_corpus_parity.sh index 246177643..cfc417752 100755 --- a/scripts/check_corpus_parity.sh +++ b/scripts/check_corpus_parity.sh @@ -87,10 +87,15 @@ DIFF_LINES="${DIFF_LINES:-20}" SUBSET_FILE="${SUBSET_FILE:-scripts/corpus_parity_subset.txt}" PY="${PYTHON:-python3}" -# The parity headline this engine is pinned to. Moving it is a deliberate act: -# it means the corpus gitlink moved, and the commit that moves it carries the -# evidence. scripts/verify_corpus.py prints this line verbatim. -EXPECTED_VERIFY="${EXPECTED_VERIFY:-Verified 312 strategies — excellent=311, strong=0, moderate=0, weak=0, minimal=0, anomaly=1, engine_only=0, missing=0}" +# The parity headline this engine is pinned to; changes require measured evidence. +# expectation corrected: excellent=311, strong=0 -> excellent=301, strong=10, +# because the excellent-only rule requires zero gross unmatched TV + engine +# trades; all 312 pinned strategy trade outputs still match the baseline hashes. +# expectation corrected: excellent=301, strong=10, moderate=0 -> +# excellent=301, strong=9, moderate=1, because the gross count percentage exposes +# an unchanged trade mismatch (#836/#883; measured 2026-11-16). +# scripts/verify_corpus.py prints this line verbatim. +EXPECTED_VERIFY="${EXPECTED_VERIFY:-Verified 312 strategies — excellent=301, strong=9, moderate=1, weak=0, minimal=0, anomaly=1, engine_only=0, missing=0}" log() { printf '\033[1;34m[corpus-parity]\033[0m %s\n' "$*"; } warn() { printf '\033[1;33m[corpus-parity]\033[0m %s\n' "$*" >&2; } diff --git a/scripts/ci_build_tests.py b/scripts/ci_build_tests.py new file mode 100644 index 000000000..69799bc7f --- /dev/null +++ b/scripts/ci_build_tests.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Build the configured lab Release target population, without running CTest.""" +from __future__ import annotations + +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re +import subprocess +import sys +import time + +import ci_verify + + +def json_object(raw): + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('duplicate JSON key: ' + key) + result[key] = value + return result + + value = json.loads(raw, object_pairs_hook=unique) + if not isinstance(value, dict): + raise ValueError('inventory must be a JSON object') + return value + + +def names(values, *, empty=False): + if (not isinstance(values, list) or (not values and not empty) + or any(not isinstance(value, str) or not value or value.strip() != value + or any(char in value for char in '\r\n\0') for value in values) + or len(values) != len(set(values)) or values != sorted(values)): + raise ValueError('inventory names must be sorted, unique, nonempty single lines') + return values + + +def registered_names(raw): + text = raw.decode('utf-8') + registry = ci_verify.ctest_registry(raw) + rows = list(ci_verify.CTEST_LIST_NAME.finditer(text)) + if (len(ci_verify.CTEST_LIST_COUNT.findall(text)) != 1 + or [int(row['number']) for row in rows] != list(range(1, len(rows) + 1)) + or len(re.findall(r'(?m)^\s*Test\s+#', text)) != len(rows)): + raise ValueError('malformed CTest discovery rows or summary') + names(sorted(registry)) + return registry + + +def relative_artifact(raw, build): + if not isinstance(raw, str) or not raw or any(char in raw for char in '\r\n\0'): + raise ValueError('malformed target artifact') + path = Path(raw) + path = (path if path.is_absolute() else build / path).resolve() + if build not in path.parents: + raise ValueError('target artifact escapes the configured build: ' + raw) + return path.relative_to(build).as_posix() + + +def release_targets(build): + """CMake file-api inventory, independent of CTest and generated link parsing.""" + reply = build / '.cmake/api/v1/reply' + indexes = list(reply.glob('index-*.json')) + if len(indexes) != 1: + raise ValueError('expected exactly one readable CMake file-api index') + + def read_reply(filename): + if not isinstance(filename, str) or Path(filename).name != filename: + raise ValueError('malformed CMake reply filename') + return json_object((reply / filename).read_bytes()) + + index = json_object(indexes[0].read_bytes()) + model_ref = index['reply']['codemodel-v2'] + if model_ref['kind'] != 'codemodel' or model_ref['version']['major'] != 2: + raise ValueError('unsupported CMake codemodel') + model = read_reply(model_ref['jsonFile']) + configurations = model['configurations'] + if not isinstance(configurations, list) or len(configurations) != 1: + raise ValueError('expected the single configured Release inventory') + configuration = configurations[0] + if configuration['name'] != 'Release': + raise ValueError('target inventory is not Release') + refs = configuration['targets'] + if not isinstance(refs, list) or not refs: + raise ValueError('empty CMake target inventory') + seen_ids, seen_names, targets, outputs = set(), set(), {}, set() + for ref in refs: + target_id, name = ref['id'], ref['name'] + if (not isinstance(target_id, str) or not target_id + or not isinstance(name, str) or not re.fullmatch(r'[A-Za-z0-9_.+-]+', name) + or target_id in seen_ids or name in seen_names): + raise ValueError('duplicate or malformed CMake target row') + seen_ids.add(target_id) + seen_names.add(name) + target = read_reply(ref['jsonFile']) + if target['id'] != target_id or target['name'] != name: + raise ValueError('CMake target reference differs from its reply') + if target['type'] in {'UTILITY', 'INTERFACE_LIBRARY'}: + continue + if target['type'] not in {'EXECUTABLE', 'STATIC_LIBRARY', 'SHARED_LIBRARY', + 'MODULE_LIBRARY', 'OBJECT_LIBRARY'}: + raise ValueError('unsupported compiled target type: ' + target['type']) + artifacts = target.get('artifacts') + if not isinstance(artifacts, list) or not artifacts: + raise ValueError('compiled target has no artifacts: ' + name) + paths = [relative_artifact(artifact['path'], build) for artifact in artifacts] + if len(paths) != len(set(paths)) or outputs.intersection(paths): + raise ValueError('duplicate target artifact') + outputs.update(paths) + targets[name] = sorted(paths) + names(sorted(targets)) + return dict(sorted(targets.items())) + + +def validate_plan(plan): + if not isinstance(plan, dict): + raise ValueError('missing build plan') + registry = names(plan['registeredNames']) + targets = plan['targets'] + if not isinstance(targets, dict): + raise ValueError('malformed Release target inventory') + names(sorted(targets)) + all_outputs = [] + for target, paths in targets.items(): + if not re.fullmatch(r'[A-Za-z0-9_.+-]+', target): + raise ValueError('malformed build target') + for path in names(paths): + if Path(path).is_absolute() or '..' in Path(path).parts: + raise ValueError('malformed build output') + all_outputs.append(path) + if len(all_outputs) != len(set(all_outputs)): + raise ValueError('duplicate build output') + mapping = plan['targetMap'] + if not isinstance(mapping, dict) or sorted(mapping) != registry: + raise ValueError('CTest and mapped row populations differ') + for row_targets in mapping.values(): + if not set(names(row_targets, empty=True)) <= set(targets): + raise ValueError('registered row has an unknown target producer') + if plan['rowsWithoutBuildTargets'] != sorted(name for name in registry if not mapping[name]): + raise ValueError('a row without a compiled producer was counted as built') + + +def check_union(records, count): + if (type(count) is not int or count < 1 or not isinstance(records, list) + or len(records) != count): + raise ValueError('missing or extra build shard') + plan = records[0]['plan'] + validate_plan(plan) + indexes, built = [], [] + for record in records: + if (record['plan'] != plan or type(record['index']) is not int + or type(record['count']) is not int or record['count'] != count): + raise ValueError('build shards disagree on the Release population') + index = record['index'] + indexes.append(index) + selected = names(record['builtTargets']) + if selected != sorted(plan['targets'])[index - 1::count]: + raise ValueError('built target population differs from assigned targets') + expected_outputs = sorted(path for target in selected for path in plan['targets'][target]) + outputs = record['outputs'] + if (not isinstance(outputs, dict) or sorted(outputs) != expected_outputs + or any(not isinstance(value, str) or not re.fullmatch(r'[0-9a-f]{64}', value) + for value in outputs.values())): + raise ValueError('missing, extra or unreadable built output') + built.extend(selected) + if sorted(indexes) != list(range(1, count + 1)): + raise ValueError('duplicate or missing shard index') + if Counter(built) != Counter(plan['targets'].keys()): + raise ValueError('Release target inventory and actual built target union differ') + return plan + + +def command(argv, log, *, cwd): + # No shell, pipeline or successful later command can mask the build status. + print('+ ' + repr(argv), flush=True) + with log.open('wb') as output: + result = subprocess.run(argv, cwd=cwd, stdout=output, stderr=subprocess.STDOUT) + raw = log.read_bytes() + print(raw.decode('utf-8', errors='replace'), end='', flush=True) + result.check_returncode() + return raw + + +def build_shard(args): + source, build = args.source.resolve(), args.build_dir.resolve() + index, count = map(int, args.shard.split('/')) + if not 1 <= index <= count <= 8 or args.jobs < 1: + raise ValueError('require 1 <= shard index <= count <= 8 and positive jobs') + artifacts = args.artifact_dir.resolve() / f'shard-{index}' + artifacts.mkdir(parents=True, exist_ok=True) + receipt = artifacts / 'built.json' + # An interrupted/failed second invocation must never leave a successful + # prior invocation's record available to the union. + receipt.write_text('{}\n') + started = time.monotonic() + query = build / '.cmake/api/v1/query/codemodel-v2' + query.parent.mkdir(parents=True, exist_ok=True) + query.touch() + cfg = ci_verify.VerifyConfig( + profile=ci_verify.PROFILE['release'], source=source, build_dir=build, + jobs=args.jobs, generator='Ninja', curl_dir=None, + ccache_path=shutil_ccache(), require_websocket=False, runner=None) + command(ci_verify.cmake_configure_argv(cfg), artifacts / 'configure.log', cwd=source) + + def discover(label): + raw = command(['ctest', '--test-dir', str(build), '-N'], + artifacts / f'ctest-{label}.txt', cwd=source) + registry = registered_names(raw) + document = command(['ctest', '--test-dir', str(build), '-N', '--show-only=json-v1'], + artifacts / f'ctest-{label}.json', cwd=source) + parsed = json_object(document) + # ctest -N omits executable paths before they exist. The generated + # add_test commands supply those paths; they must name exactly the same rows. + if set(ci_verify.ctest_configure_commands(build)) != set(registry): + raise ValueError('generated CTest names differ from ctest -N') + inventory = ci_verify.ctest_build_inventory( + json.dumps(parsed).encode(), build, registry, registry) + targets = release_targets(build) + # Compare file-api products with independent generated link producers. + link_outputs = inventory['configuredTargetOutputs'] + for path, target in link_outputs.items(): + if target not in targets or path not in targets[target]: + raise ValueError('generated link inventory differs from CMake target inventory') + for target, paths in targets.items(): + for path in paths: + if not path.endswith(('.o', '.obj')) and link_outputs.get(path) != target: + raise ValueError('CMake target has no generated link producer: ' + target) + plan = { + 'registeredNames': sorted(registry), + 'targetMap': inventory['targetMap'], + 'rowsWithoutBuildTargets': sorted(name for name in registry + if not inventory['targetMap'][name]), + 'targets': targets, + } + validate_plan(plan) + (artifacts / f'inventory-{label}.json').write_text(json.dumps(plan, indent=2) + '\n') + return plan + + plan = discover('before') + selected = sorted(plan['targets'])[index - 1::count] + if not selected: + raise ValueError('build shard has no targets; reduce the shard count') + command(['cmake', '--build', str(build), '--parallel', str(args.jobs), + '--target', *selected], artifacts / 'build.log', cwd=source) + outputs = {} + for target in selected: + for path in plan['targets'][target]: + product = build / path + if not product.is_file() or product.stat().st_size == 0: + raise ValueError('build did not produce ' + path) + outputs[path] = hashlib.sha256(product.read_bytes()).hexdigest() + if discover('after') != plan: + raise ValueError('CTest or Release build target population changed during compilation') + record = {'plan': plan, 'index': index, 'count': count, + 'builtTargets': selected, 'outputs': outputs, + 'elapsedSeconds': time.monotonic() - started} + receipt.write_text(json.dumps(record, indent=2) + '\n') + print(f'Built {len(selected)} assigned Release targets; CTest execution is separate.', flush=True) + + +def shutil_ccache(): + import shutil + path = shutil.which('ccache') + if path is None: + raise ValueError('the existing CI compiler launcher ccache is required') + return path + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest='command', required=True) + build = subparsers.add_parser('build') + build.add_argument('--source', type=Path, default=Path('.')) + build.add_argument('--build-dir', type=Path, required=True) + build.add_argument('--artifact-dir', type=Path, required=True) + build.add_argument('--jobs', type=int, required=True) + build.add_argument('--shard', required=True) + union = subparsers.add_parser('union') + union.add_argument('--artifact-dir', type=Path, required=True) + union.add_argument('--shards', type=int, required=True) + args = parser.parse_args(argv) + try: + if args.command == 'build': + build_shard(args) + else: + paths = sorted(args.artifact_dir.glob('shard-*/built.json')) + plan = check_union([json_object(path.read_bytes()) for path in paths], args.shards) + print(f'Release build union: {len(plan["targets"])} exact targets; ' + f'{len(plan["registeredNames"])} mapped CTest rows; ' + f'{len(plan["rowsWithoutBuildTargets"])} rows have no compiled producer ' + 'and are not counted as built.') + except (OSError, ValueError, KeyError, TypeError, subprocess.CalledProcessError) as error: + print('Release test build FAILED: ' + str(error), file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/ci_preflight.py b/scripts/ci_preflight.py index eac23b118..6aee8630d 100644 --- a/scripts/ci_preflight.py +++ b/scripts/ci_preflight.py @@ -129,10 +129,12 @@ GKE_JOB_LABELS = { 'gke-pr.yml': {'preflight': HOSTED_X86_RUNNER, 'build': HOSTED_X86_RUNNER, NATIVE_X86_JOB: HOSTED_X86_RUNNER, + 'build-tests': HOSTED_X86_RUNNER, 'build-tests-result': HOSTED_X86_RUNNER, 'build-debug': HOSTED_X86_RUNNER, 'corpus-parity-subset': HOSTED_X86_RUNNER}, 'ci-post.yml': {'preflight': HOSTED_X86_RUNNER, NATIVE_X86_JOB: HOSTED_X86_RUNNER, + 'build-tests': HOSTED_X86_RUNNER, 'build-tests-result': HOSTED_X86_RUNNER, 'smoke': HOSTED_X86_RUNNER, 'smoke-arm': HOSTED_ARM_RUNNER, 'sanitizers': HOSTED_X86_RUNNER}, 'corpus-parity.yml': {'corpus-parity': HOSTED_X86_RUNNER, @@ -275,6 +277,7 @@ CORES = '"$(getconf _NPROCESSORS_ONLN)"' VERIFY, PARITY = 'ci_verify.py', 'check_corpus_parity.sh' NATIVE_X86_COMMAND = 'ci_native_x86_release.py' +RELEASE_BUILD_COMMAND = 'ci_build_tests.py' NATIVE_X86_BUILD_DIR = 'build-x86-gcc133' NATIVE_X86_ARTIFACT_DIR = 'ci-native-x86-gcc133' NATIVE_X86_UPLOAD = 'actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02' @@ -294,10 +297,12 @@ 'ci.yml': {'authz': None}, 'gke-pr.yml': {'admit': None, 'preflight': None, 'build': None, NATIVE_X86_JOB: NATIVE_X86_COMMAND, + 'build-tests': RELEASE_BUILD_COMMAND, 'build-tests-result': None, 'build-debug': None, 'corpus-parity-subset': PARITY, 'legs': None, 'result': None}, 'ci-post.yml': {'authz': None, 'changes': None, 'preflight': None, NATIVE_X86_JOB: NATIVE_X86_COMMAND, + 'build-tests': RELEASE_BUILD_COMMAND, 'build-tests-result': None, 'smoke': None, 'smoke-arm': None, 'sanitizers': None, 'build-gate': None}, 'native-live.yml': {'authz': None, 'native-live': VERIFY, 'native-live-shards': VERIFY, @@ -376,7 +381,9 @@ def _release_split_requested(jobs: dict[str, str]) -> bool: ' fail-fast: false\n' ' matrix:\n' ' build_type: [Release, Debug]\n') -POST_ALWAYS_JOBS = {NATIVE_X86_JOB: 'X86_GCC133_RESULT'} +POST_ALWAYS_JOBS = {NATIVE_X86_JOB: 'X86_GCC133_RESULT', + 'build-tests': 'BUILD_TESTS_RESULT', + 'build-tests-result': 'BUILD_TESTS_UNION_RESULT'} POST_PROOF_JOBS = {**PROOF_JOBS, **POST_ALWAYS_JOBS} PR_PROOF_JOBS = {'build': 'BUILD_RESULT', 'build-debug': 'DEBUG_RESULT', 'corpus-parity-subset': 'PARITY_RESULT'} @@ -498,6 +505,8 @@ def _release_split_requested(jobs: dict[str, str]) -> bool: ALLOWED: ${{ needs.admit.outputs.allowed }} PREFLIGHT_RESULT: ${{ needs.preflight.result }} X86_RESULT: ${{ needs.build-x86-gcc133.result }} + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + BUILD_TESTS_UNION_RESULT: ${{ needs.build-tests-result.result }} LEGS_RESULT: ${{ needs.legs.result }} ADMIT_DOCS_ONLY: ${{ needs.admit.outputs.docs_only }} ADMIT_GITHUB_ONLY: ${{ needs.admit.outputs.github_only }} @@ -511,6 +520,8 @@ def _release_split_requested(jobs: dict[str, str]) -> bool: test "$ALLOWED" = true || fail "admission did not allow the head" test "$PREFLIGHT_RESULT" = success || fail "preflight did not pass" test "$X86_RESULT" = success || fail "native x86 GNU 13.3.0 build did not pass" + test "$BUILD_TESTS_RESULT" = success || fail "Release test build did not pass" + test "$BUILD_TESTS_UNION_RESULT" = success || fail "Release target union did not pass" test "$SECOND_DOCS_ONLY" = "$ADMIT_DOCS_ONLY" || fail "docs-only classification changed" test "$SECOND_GITHUB_ONLY" = "$ADMIT_GITHUB_ONLY" || fail "github-only classification changed" test "$CODE_EXTENSION" != true || { @@ -553,6 +564,8 @@ def _release_split_requested(jobs: dict[str, str]) -> bool: ALLOWED: ${{ needs.admit.outputs.allowed }} PREFLIGHT_RESULT: ${{ needs.preflight.result }} X86_RESULT: ${{ needs.build-x86-gcc133.result }} + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + BUILD_TESTS_UNION_RESULT: ${{ needs.build-tests-result.result }} LEGS_RESULT: ${{ needs.legs.result }} ADMIT_DOCS_ONLY: ${{ needs.admit.outputs.docs_only }} ADMIT_GITHUB_ONLY: ${{ needs.admit.outputs.github_only }} @@ -568,6 +581,8 @@ def _release_split_requested(jobs: dict[str, str]) -> bool: test "$ALLOWED" = true || fail "admission did not allow the head" test "$PREFLIGHT_RESULT" = success || fail "preflight did not pass" test "$X86_RESULT" = success || fail "native x86 GNU 13.3.0 build did not pass" + test "$BUILD_TESTS_RESULT" = success || fail "Release test build did not pass" + test "$BUILD_TESTS_UNION_RESULT" = success || fail "Release target union did not pass" test "$SECOND_DOCS_ONLY" = "$ADMIT_DOCS_ONLY" || fail "docs-only classification changed" test "$SECOND_GITHUB_ONLY" = "$ADMIT_GITHUB_ONLY" || fail "github-only classification changed" test "$SECOND_ARM_RELEVANT" = "$ADMIT_ARM_RELEVANT" || fail "arm relevance classification changed" @@ -1046,6 +1061,8 @@ def field(obj, *keys): DOCS_ONLY: ${{ needs.changes.outputs.docs_only }} PREFLIGHT_RESULT: ${{ needs.preflight.result }} X86_GCC133_RESULT: ${{ needs.build-x86-gcc133.result }} + BUILD_TESTS_RESULT: ${{ needs.build-tests.result }} + BUILD_TESTS_UNION_RESULT: ${{ needs.build-tests-result.result }} SMOKE_RESULT: ${{ needs.smoke.result }} ARM_SMOKE_RESULT: ${{ needs.smoke-arm.result }} SANITIZER_RESULT: ${{ needs.sanitizers.result }} @@ -1053,6 +1070,7 @@ def field(obj, *keys): run: | printf '%s\n' "authz: $AUTHZ_RESULT" "changes: $CHANGES_RESULT, docs only: ${DOCS_ONLY:-unanswered}" \ "preflight: $PREFLIGHT_RESULT" "native x86 GNU 13.3.0: $X86_GCC133_RESULT" \ + "Release test build: $BUILD_TESTS_RESULT" "Release target union: $BUILD_TESTS_UNION_RESULT" \ "amd64 smoke: $SMOKE_RESULT" "arm64 smoke: $ARM_SMOKE_RESULT" \ "sanitizers: $SANITIZER_RESULT" "corpus parity (subset): $PARITY_RESULT" # Every smoke leg and the subset parity must pass. Only a classified @@ -1063,6 +1081,8 @@ def field(obj, *keys): [[ "$AUTHZ_RESULT" == "success" && "$CHANGES_RESULT" == "success" ]] \ && [[ "$PREFLIGHT_RESULT" == "success" ]] \ && [[ "$X86_GCC133_RESULT" == "success" ]] \ + && [[ "$BUILD_TESTS_RESULT" == "success" ]] \ + && [[ "$BUILD_TESTS_UNION_RESULT" == "success" ]] \ && passed "$SMOKE_RESULT" && passed "$ARM_SMOKE_RESULT" \ && passed "$SANITIZER_RESULT" && passed "$PARITY_RESULT" ''' @@ -1081,6 +1101,7 @@ def field(obj, *keys): _JOBS_FLAG = re.compile(r'(? list proofs = PR_ARM_PROOF_JOBS if arm_variant else PR_PROOF_JOBS legs_step = GKE_ARM_LEGS_STEP if arm_variant else GKE_LEGS_STEP result_template = GKE_ARM_RESULT_STEP if arm_variant else GKE_RESULT_STEP - required = {'admit', 'preflight', *proofs, NATIVE_X86_JOB, 'legs', 'result'} - result_needs = ['admit', 'preflight', NATIVE_X86_JOB, 'legs'] - result_dependency = NATIVE_X86_JOB + required = {'admit', 'preflight', *proofs, NATIVE_X86_JOB, + 'build-tests', 'build-tests-result', 'legs', 'result'} + result_needs = ['admit', 'preflight', NATIVE_X86_JOB, + 'build-tests', 'build-tests-result', 'legs'] + result_dependency = NATIVE_X86_JOB + ', build-tests, build-tests-result' if corpus_variant: required.add('corpus-gitlink-result') result_needs.insert(-1, 'corpus-gitlink-result') @@ -1949,6 +1972,7 @@ def gke_admit_workflow_findings(workflow: str, name: str = 'gke-pr.yml') -> list if _job_block(body, 'strategy') or _job_values(body, 'strategy'): findings.append(f'{name} {job} must remain one fixed proof job') findings += native_x86_job_findings(jobs.get(NATIVE_X86_JOB, ''), name, pr=True) + findings += release_build_workflow_findings(jobs, pr=True) preflight = steps.get('preflight', []) preflight_run = _step_named(preflight, ' - name: Run the class-specific preflight') if (_step_values(preflight_run, 'run') != [ @@ -2335,23 +2359,25 @@ def _sized(text: str, command: str | None) -> bool: """Whether a job's work runs at the runner's core count. Counted over the whole job, in order, so no line, block or chaining form - matters: every ci_verify call carries one core-count --jobs before the next + matters: every verifier or Release build call carries one core-count --jobs before the next call, no other --jobs spelling appears, and every JOBS and parity-script mention is part of the one parity command. """ marks = sorted([(match.start(), 'C') for match in _VERIFY_CALL.finditer(text)] + [(match.start(), 'N') for match in _NATIVE_X86_CALL.finditer(text)] + + [(match.start(), 'B') for match in _RELEASE_BUILD_CALL.finditer(text)] + [(match.start(), 'F' if text.startswith( (f'--jobs {CORES}', f'--jobs={CORES}'), match.start()) else 'U') for match in _JOBS_FLAG.finditer(text)]) calls = ''.join(kind for _, kind in marks) parity = text.count(PARITY_COMMAND) - allowed_calls = ({VERIFY: r'(?:CF)*', NATIVE_X86_COMMAND: r'(?:NF)*'} + allowed_calls = ({VERIFY: r'(?:CF)*', NATIVE_X86_COMMAND: r'(?:NF)*', + RELEASE_BUILD_COMMAND: r'(?:BF)*'} .get(command, r'')) return (re.fullmatch(allowed_calls, calls) is not None and len(re.findall(r'\bJOBS\s*[:=]', text)) == parity and text.count(PARITY) == parity - and (command not in (VERIFY, NATIVE_X86_COMMAND) or bool(calls)) + and (command not in (VERIFY, NATIVE_X86_COMMAND, RELEASE_BUILD_COMMAND) or bool(calls)) and (command != PARITY or parity > 0)) @@ -5259,6 +5285,104 @@ def native_workflow_findings(native: str, curl_action: str | None = None) -> lis return findings +def release_build_workflow_findings(jobs: dict[str, str], *, pr: bool = False) -> list[str]: + """Require the real Release build, complete shard union and compiler control.""" + findings = [] + build = jobs.get('build-tests', '') + union = jobs.get('build-tests-result', '') + admission = 'admit' if pr else 'authz' + condition = NATIVE_X86_PR_IF if pr else AUTHZ_SUCCESS + permissions = '{contents: read}' if pr else 'contents: read' + checkout_name = 'Checkout the admitted head' if pr else 'Checkout the exact workflow head' + checkout_ref = '${{ needs.admit.outputs.head_sha }}' if pr else '${{ github.sha }}' + expected_names = { + 'build-tests': [checkout_name, 'Set up Python', + 'Install dependencies', 'Exercise the broken registered header control', + 'Build every Release test target', + 'Retain Release target inventories and build logs'], + 'build-tests-result': ['Require every Release build shard to succeed', + checkout_name, + 'Collect every Release target inventory', + 'Require the complete Release build'], + } + for name, body in (('build-tests', build), ('build-tests-result', union)): + steps = _steps(body) + if (not body or _job_values(body, 'if') != [condition] + or _needs(body) != ([admission] if name == 'build-tests' + else [admission, 'build-tests']) + or _job_values(body, 'runs-on') != [HOSTED_X86_RUNNER] + or _job_values(body, 'permissions') != [permissions] + or [_step_values(step, 'name') for step in steps] + != [[value] for value in expected_names[name]] + or any(_job_values(body, key) for key in ( + 'continue-on-error', 'defaults', 'env', 'container', 'services', 'uses')) + or any(_step_values(step, key) for step in steps for key in ( + 'continue-on-error', 'shell', 'working-directory'))): + findings.append(f'{name} must retain its exact admitted Release build steps') + checkout = _step_named(steps, ' - name: ' + checkout_name) + if (_step_values(checkout, 'uses') != [ + 'actions/checkout@11d5960a326750d5838078e36cf38b85af677262'] + or _step_values(checkout, 'if') + or _step_block(checkout, 'with') != [ + ' with:', ' repository: ${{ github.repository }}', + ' ref: ' + checkout_ref, ' fetch-depth: 0', + ' persist-credentials: false']): + findings.append(f'{name} must check out the exact workflow head') + if _job_block(build, 'strategy') != [ + ' strategy:', ' fail-fast: false', ' matrix:', + ' shard: [1, 2, 3, 4, 5, 6, 7, 8]']: + findings.append('build-tests must build every one of its eight parallel partitions') + if _job_block(union, 'strategy'): + findings.append('build-tests-result must be a single complete union') + steps = _steps(build) + compile_step = _step_named(steps, ' - name: Build every Release test target') + expected = ('| python3 scripts/ci_build_tests.py build --source . ' + '--build-dir build-tests-${{ matrix.shard }} --artifact-dir ci-build-tests ' + '--shard ${{ matrix.shard }}/8 --jobs ' + CORES) + if (_step_values(compile_step, 'run') != [expected] + or _step_values(compile_step, 'if') or _step_block(compile_step, 'env')): + findings.append('build-tests must run the actual full Release target build') + control = _step_named(steps, ' - name: Exercise the broken registered header control') + if (_step_values(control, 'run') != [ + 'python3 scripts/test_ci_build_tests.py WorkflowControl'] + or _step_values(control, 'if') != ['${{ matrix.shard == 1 }}']): + findings.append('build-tests must execute its real broken-header workflow control') + install = _step_named(steps, ' - name: Install dependencies') + python = _step_named(steps, ' - name: Set up Python') + if (_step_values(install, 'run') != [SMOKE_INSTALL_RUN + ' ninja-build'] + or _step_values(install, 'if') + or _step_values(python, 'if') + or _step_values(python, 'uses') != [ + 'actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065'] + or _step_block(python, 'with') != [ + ' with:', " python-version: '3.12'"]): + findings.append('build-tests must preserve the hosted Python and compiler setup') + upload = _step_named(steps, ' - name: Retain Release target inventories and build logs') + if (_step_values(upload, 'if') != ['always()'] + or _step_values(upload, 'uses') != [NATIVE_X86_UPLOAD] + or _step_block(upload, 'with') != [ + ' with:', ' name: ci-release-build-${{ matrix.shard }}', + ' path: ci-build-tests/', ' if-no-files-found: error']): + findings.append('build-tests must retain every shard inventory and fail on absent artifacts') + steps = _steps(union) + status = _step_named(steps, ' - name: Require every Release build shard to succeed') + download = _step_named(steps, ' - name: Collect every Release target inventory') + merge = _step_named(steps, ' - name: Require the complete Release build') + if (_step_values(status, 'run') != ['test "$BUILD_TESTS_RESULT" = success'] + or _step_block(status, 'env') != [ + ' env:', ' BUILD_TESTS_RESULT: ${{ needs.build-tests.result }}'] + or _step_values(download, 'uses') != [ + 'actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093'] + or _step_block(download, 'with') != [ + ' with:', ' pattern: ci-release-build-*', + ' path: ci-build-tests/', ' merge-multiple: true'] + or _step_values(merge, 'run') != [ + '| python3 scripts/ci_build_tests.py union --artifact-dir ci-build-tests --shards 8'] + or any(_step_values(step, 'if') for step in steps)): + findings.append('build-tests-result must require successful builds and their exact inventory union') + return findings + + def post_smoke_workflow_findings(jobs: dict[str, str]) -> list[str]: """ci-post's smoke legs: the PR smoke target and the whole fast label, cold, on the named machine.""" findings = [] @@ -5489,6 +5613,10 @@ def pipeline_workflow_findings(workflow: str, name: str, *, pr: bool) -> list[st findings.append('preflight must keep only checkout, linters, fast checks and diagnostics') for job in proofs.keys(): body = jobs.get(job, '') + if not pr and job in ('build-tests', 'build-tests-result'): + # This second job follows compilation to compare the actual outputs. + # Both are mandatory even for documentation-only events. + continue always_job = not pr and job in POST_ALWAYS_JOBS expected_needs = ['authz'] if always_job else ['authz', 'changes'] expected_if = AUTHZ_ALWAYS if always_job else ( @@ -5528,6 +5656,7 @@ def pipeline_workflow_findings(workflow: str, name: str, *, pr: bool) -> list[st findings.append('build-gate must keep the exact result policy: success or a classified docs-only skip') if not pr: findings += native_x86_job_findings(jobs.get(NATIVE_X86_JOB, ''), name, pr=False) + findings += release_build_workflow_findings(jobs) findings += post_smoke_workflow_findings(jobs) for job in (('build',) if pr else ()): build = jobs.get(job, '') @@ -6049,6 +6178,8 @@ def check_commands(source: Path, output: Path | None = None, parallel=True), ('native-x86-release-tests', [sys.executable, str(source / 'scripts/test_ci_native_x86_release.py')]), + ('release-build-population-tests', + [sys.executable, str(source / 'scripts/test_ci_build_tests.py'), 'PopulationContract']), ('kernel-residuals-tests', [sys.executable, str(source / 'scripts/test_check_kernel_residuals.py')]), ('corpus-parity-identity-tests', @@ -6244,7 +6375,8 @@ def run_stage(item: tuple[int, tuple]) -> None: stage['seconds'] = seconds if log: print(f'ci_preflight: {name}: output ({stage["log"]})', flush=True) - print(log.decode('utf-8', 'replace'), end='', flush=True) + print(log.decode('utf-8', 'replace'), + end='' if log.endswith(b'\n') else '\n', flush=True) if (code == 77 and not advisory and not private_fixtures_overlay() and PRIVATE_FIXTURE_SKIP_LITERAL.encode() in log): # A source guard whose private fixture input the tree lacks exits @@ -6261,6 +6393,9 @@ def run_stage(item: tuple[int, tuple]) -> None: stage.update(status='reported', exitCode=code) else: stage.update(status='passed' if code == 0 else 'failed', exitCode=code) + if stage['status'] == 'failed': + print(f'ci_preflight: FAIL {name}: exit {code}; ' + f'log: {output / stage["log"]}', flush=True) record() parallel = [] @@ -6281,6 +6416,9 @@ def run_stage(item: tuple[int, tuple]) -> None: summary.update(status='passed' if passed else 'failed', exitCode=0 if passed else 1, seconds=round(time.monotonic() - run_t0, 3)) record() + failed = [stage['name'] for stage in summary['stages'] if stage['status'] == 'failed'] + print(f'ci_preflight: summary: {summary["status"]}; {len(failed)} failed binding stage(s)' + + (': ' + ', '.join(failed) if failed else ''), flush=True) return summary['exitCode'] diff --git a/scripts/fixtures/ci_build_tests/CMakeLists.txt b/scripts/fixtures/ci_build_tests/CMakeLists.txt new file mode 100644 index 000000000..8432227f2 --- /dev/null +++ b/scripts/fixtures/ci_build_tests/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 3.16) +project(RegisteredBuildControl LANGUAGES C CXX) +enable_testing() +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/bin") +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/lib") +add_library(pineforge STATIC support.cpp) +add_library(pineforge_kernel STATIC support.cpp) +add_library(control_object OBJECT support.cpp) +add_executable(control_smoke smoke.cpp) +add_test(NAME control_smoke COMMAND control_smoke) +set_tests_properties(control_smoke PROPERTIES LABELS fast) +add_custom_target(pineforge_pr_smoke DEPENDS control_smoke) + +# This ordinary Release target is registered but outside the smoke closure. +add_executable(control_deferred deferred.cpp) +add_test(NAME control_registered COMMAND control_deferred) +add_test(NAME control_alias COMMAND control_deferred) +if(PINEFORGE_BUILD_EXAMPLES) + add_executable(control_example smoke.cpp) + add_test(NAME control_example COMMAND control_example) +endif() +if(PINEFORGE_BUILD_TUTORIAL) + add_library(control_tutorial MODULE support.cpp) +endif() diff --git a/scripts/fixtures/ci_build_tests/deferred.cpp b/scripts/fixtures/ci_build_tests/deferred.cpp new file mode 100644 index 000000000..47c75f6d0 --- /dev/null +++ b/scripts/fixtures/ci_build_tests/deferred.cpp @@ -0,0 +1,2 @@ +#include "fixture.hpp" +int main() { return control_fixture_value() != 42; } diff --git a/scripts/fixtures/ci_build_tests/fixture.hpp b/scripts/fixtures/ci_build_tests/fixture.hpp new file mode 100644 index 000000000..d991c7bbe --- /dev/null +++ b/scripts/fixtures/ci_build_tests/fixture.hpp @@ -0,0 +1,2 @@ +#pragma once +inline int control_fixture_value() { return 6 * 7; } diff --git a/scripts/fixtures/ci_build_tests/smoke.cpp b/scripts/fixtures/ci_build_tests/smoke.cpp new file mode 100644 index 000000000..eb89945aa --- /dev/null +++ b/scripts/fixtures/ci_build_tests/smoke.cpp @@ -0,0 +1,4 @@ +int main() { + volatile int input = 6; + return input * 7 != 42; +} diff --git a/scripts/fixtures/ci_build_tests/support.cpp b/scripts/fixtures/ci_build_tests/support.cpp new file mode 100644 index 000000000..7fb1edf85 --- /dev/null +++ b/scripts/fixtures/ci_build_tests/support.cpp @@ -0,0 +1 @@ +int control_support() { return 6 * 7; } diff --git a/scripts/test_ci_build_tests.py b/scripts/test_ci_build_tests.py new file mode 100644 index 000000000..2e1514602 --- /dev/null +++ b/scripts/test_ci_build_tests.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""Compile controls for the actual hosted Release build decision; stdlib only.""" +from __future__ import annotations + +import copy +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +import ci_preflight + +ROOT = Path(__file__).resolve().parents[1] +BASE = '5cf08f5c16d6477a91dea8a18c331cf4a6c57496' +FIXTURE = ROOT / 'scripts/fixtures/ci_build_tests' + + +def step_run(workflow, job, name): + step = ci_preflight._step_named(ci_preflight._steps( + ci_preflight._jobs(workflow)[job]), ' - name: ' + name) + lines = step.splitlines() + indexes = [index for index, line in enumerate(lines) if line == ' run: |'] + if len(indexes) != 1: + raise AssertionError('control requires the real multiline workflow command') + command = [] + for line in lines[indexes[0] + 1:]: + if line.strip() and not line.startswith(' ' * 10): + break + command.append(line[10:]) + return '\n'.join(command) + '\n' + + +def mutate_workflow(workflow, job, step, before, after): + """Change exactly one occurrence in its explicit owning job and step.""" + headers = list(ci_preflight._JOB_HEADER.finditer(workflow)) + selected = [index for index, header in enumerate(headers) if header[1] == job] + if len(selected) != 1: + raise AssertionError('mutation requires one owning job: ' + job) + index, = selected + start = headers[index].end() + end = headers[index + 1].start() if index + 1 < len(headers) else len(workflow) + body = workflow[start:end] + scope = body + if step is not None: + matches = [block for block in ci_preflight._steps(body) + if ci_preflight._step_values(block, 'name') == [step]] + if len(matches) != 1 or body.count(matches[0]) != 1: + raise AssertionError('mutation requires one owning step: ' + step) + scope, = matches + if not before or scope.count(before) != 1 or before == after: + raise AssertionError('mutation requires one changing occurrence in ' + job) + offset = body.index(scope) if step is not None else 0 + offset += scope.index(before) + start += offset + return workflow[:start] + after + workflow[start + len(before):] + + +class WorkflowControl(unittest.TestCase): + def test_broken_registered_header_changes_the_workflow_verdict(self): + # The baseline is Git's exact old workflow, not a reimplementation of + # smoke selection. Fetch-depth 0 in the hosted job retains this object. + old = subprocess.run( + ['git', 'show', BASE + ':.github/workflows/ci-post.yml'], + cwd=ROOT, check=True, capture_output=True, text=True).stdout + new = (ROOT / '.github/workflows/gke-pr.yml').read_text() + old_run = step_run(old, 'smoke', 'Verify ${{ matrix.build_type }} smoke') + new_run = step_run(new, 'build-tests', 'Build every Release test target') + strategy = ci_preflight._job_block( + ci_preflight._jobs(new)['build-tests'], 'strategy') + shard_line, = [line for line in strategy if line.strip().startswith('shard:')] + shards = json.loads(shard_line.split(':', 1)[1]) + self.assertTrue(shards) + self.assertEqual(shards, list(range(1, len(shards) + 1))) + + with tempfile.TemporaryDirectory(prefix='registered-build-control-') as tmp: + source = Path(tmp) + shutil.copytree(FIXTURE, source, dirs_exist_ok=True) + # Keep this fixture scalable with the workflow's parallelism, without + # admitting an empty build shard or pinning a test population total. + with (source / 'CMakeLists.txt').open('a') as cmake: + for shard in shards: + cmake.write(f'\nadd_library(control_aux_{shard} STATIC support.cpp)\n') + scripts = source / 'scripts' + scripts.mkdir() + for name in ('ci_build_tests.py', 'ci_verify.py', + 'prepare_settlement_cpp_abi_base.py'): + shutil.copyfile(ROOT / 'scripts' / name, scripts / name) + environment = dict(os.environ, BUILD_TYPE='Release', PYTHONDONTWRITEBYTECODE='1') + + def execute(command, label): + result = subprocess.run( + ['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', command], + cwd=source, env=environment, capture_output=True, text=True) + # Leave the compiler's actual first diagnostic in the hosted log. + print(f'\nCONTROL {label}: exit {result.returncode}\n' + + result.stdout + result.stderr, flush=True) + return result + + header = source / 'fixture.hpp' + original = header.read_bytes() + header.write_text('#pragma once\ninline int control_fixture_value() {\n' + ' return absent_fixture_type::value;\n}\n') + broken = header.read_bytes() + admitted = execute(old_run, 'old workflow, broken registered header') + self.assertEqual(admitted.returncode, 0, admitted.stdout + admitted.stderr) + self.assertFalse((source / 'build/bin/control_deferred').exists()) + rejected = [] + for shard in shards: + command = new_run.replace('${{ matrix.shard }}', str(shard)) + result = execute(command, f'new workflow shard {shard}, same broken header') + self.assertEqual(header.read_bytes(), broken) + rejected.append(result) + failures = [result for result in rejected if result.returncode != 0] + self.assertTrue(failures, 'all new workflow shards admitted the broken tree') + self.assertTrue(any('absent_fixture_type' in result.stdout + result.stderr + for result in failures), + 'failure must be the injected compile error') + merge_run = step_run(new, 'build-tests-result', 'Require the complete Release build') + result = execute(merge_run, 'broken workflow union') + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) + + # Repair the same actual input. Every workflow shard must now build, + # and the real union decision must accept its exact inventories. + header.write_bytes(original) + for shard in shards: + result = execute(new_run.replace('${{ matrix.shard }}', str(shard)), + f'new workflow shard {shard}, repaired header') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + result = execute(merge_run, 'repaired workflow union') + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +class PopulationContract(unittest.TestCase): + def test_union_refuses_each_kind_of_population_loss_or_invention(self): + import ci_build_tests as build_tests + + plan = { + 'registeredNames': ['alias', 'row', 'script'], + 'targetMap': {'alias': ['fixture'], 'row': ['fixture'], 'script': []}, + 'rowsWithoutBuildTargets': ['script'], + 'targets': {'fixture': ['bin/fixture'], 'support': ['lib/support.a']}, + } + records = [ + {'plan': plan, 'index': 1, 'count': 2, 'builtTargets': ['fixture'], + 'outputs': {'bin/fixture': 'a' * 64}}, + {'plan': plan, 'index': 2, 'count': 2, 'builtTargets': ['support'], + 'outputs': {'lib/support.a': 'b' * 64}}, + ] + build_tests.check_union(records, 2) + mutations = [ + records[:-1], + records + [records[0]], + [dict(records[0], builtTargets=[]), records[1]], + [dict(records[0], builtTargets=['fixture', 'invented']), records[1]], + [dict(records[0], builtTargets=['fixture', 'fixture']), records[1]], + [dict(records[0], outputs={}), records[1]], + [dict(records[0], outputs={'bin/fixture': 'unreadable'}), records[1]], + [dict(records[0], count=3), records[1]], + ] + for mutation in mutations: + with self.subTest(mutation=mutation), self.assertRaises((ValueError, TypeError)): + build_tests.check_union(mutation, 2) + for change in ('missing', 'extra', 'duplicate', 'unmapped', 'fake-built-script'): + mutation = copy.deepcopy(records) + changed = mutation[0]['plan'] + if change == 'missing': + del changed['targetMap']['row'] + elif change == 'extra': + changed['targetMap']['extra'] = ['fixture'] + elif change == 'duplicate': + changed['registeredNames'].append('row') + elif change == 'unmapped': + changed['targetMap']['row'] = ['no_producer'] + else: + changed['rowsWithoutBuildTargets'] = [] + with self.subTest(change=change), self.assertRaises(ValueError): + build_tests.check_union(mutation, 2) + + def test_discovery_rejects_changed_or_unreadable_rows(self): + import ci_build_tests as build_tests + + raw = b'Test project build\n Test #1: one\n Test #2: two\n\nTotal Tests: 2\n' + self.assertEqual(build_tests.registered_names(raw), ('one', 'two')) + for bad in ( + raw.replace(b'#2', b'#1'), raw.replace(b'two', b'one'), + raw.replace(b'#2: two', b'#2 two'), raw.replace(b'Tests: 2', b'Tests: 3'), + raw + b'Total Tests: 2\n', raw.replace(b'#2', b'#3'), b'\xff'): + with self.subTest(raw=bad), self.assertRaises(ValueError): + build_tests.registered_names(bad) + for bad in ('{"tests":[],"tests":[]}', '{', 'null'): + with self.subTest(json=bad), self.assertRaises(ValueError): + build_tests.json_object(bad) + + def test_cmake_inventory_refuses_duplicate_missing_or_unreadable_producers(self): + import ci_build_tests as build_tests + + with tempfile.TemporaryDirectory() as tmp: + build = Path(tmp).resolve() + reply = build / '.cmake/api/v1/reply' + reply.mkdir(parents=True) + index = {'reply': {'codemodel-v2': { + 'kind': 'codemodel', 'version': {'major': 2}, 'jsonFile': 'model.json'}}} + ref = {'id': 'fixture::id', 'name': 'fixture', 'jsonFile': 'target.json'} + model = {'configurations': [{'name': 'Release', 'targets': [ref]}]} + target = {'id': 'fixture::id', 'name': 'fixture', 'type': 'EXECUTABLE', + 'artifacts': [{'path': 'bin/fixture'}]} + index_file = reply / 'index-control.json' + model_file = reply / 'model.json' + target_file = reply / 'target.json' + index_file.write_text(json.dumps(index)) + model_file.write_text(json.dumps(model)) + target_file.write_text(json.dumps(target)) + self.assertEqual(build_tests.release_targets(build), {'fixture': ['bin/fixture']}) + for change in ('duplicate-row', 'no-rows', 'different-name', 'no-artifact', + 'duplicate-artifact', 'escaping-artifact', 'unreadable'): + changed_model, changed_target = copy.deepcopy(model), copy.deepcopy(target) + if change == 'duplicate-row': + changed_model['configurations'][0]['targets'].append(ref) + elif change == 'no-rows': + changed_model['configurations'][0]['targets'] = [] + elif change == 'different-name': + changed_target['name'] = 'invented' + elif change == 'no-artifact': + changed_target['artifacts'] = [] + elif change == 'duplicate-artifact': + changed_target['artifacts'] *= 2 + elif change == 'escaping-artifact': + changed_target['artifacts'] = [{'path': '../outside'}] + model_file.write_text(json.dumps(changed_model)) + target_file.write_text('{' if change == 'unreadable' else json.dumps(changed_target)) + with self.subTest(change=change), self.assertRaises(ValueError): + build_tests.release_targets(build) + + def test_workflow_requires_real_build_union_and_header_control(self): + mutations = ( + ('build-tests', 'Build every Release test target', + '--shard ${{ matrix.shard }}/8', '--shard 1/8'), + ('build-tests', 'Build every Release test target', + 'python3 scripts/ci_build_tests.py build', 'echo build'), + ('build-tests-result', 'Require the complete Release build', + 'python3 scripts/ci_build_tests.py union', 'echo union'), + ('build-tests', 'Exercise the broken registered header control', + 'python3 scripts/test_ci_build_tests.py WorkflowControl', 'echo control'), + ('build-tests', 'Build every Release test target', + '--jobs "$(getconf _NPROCESSORS_ONLN)"', '--jobs 1'), + ('build-tests', 'Retain Release target inventories and build logs', + 'if-no-files-found: error', 'if-no-files-found: warn'), + ('build-tests-result', None, + 'needs: [authz, build-tests]', 'needs: [authz]'), + ('build-tests', None, + 'shard: [1, 2, 3, 4, 5, 6, 7, 8]', 'shard: [1, 2, 3, 4, 5, 6, 7]'), + ('build-tests', 'Build every Release test target', + '--source .', '--source scripts/fixtures/ci_build_tests'), + ('build-tests-result', 'Require the complete Release build', + '--shards 8', '--shards 7'), + ('build-tests-result', 'Require every Release build shard to succeed', + 'test "$BUILD_TESTS_RESULT" = success', 'true'), + ('build-tests-result', 'Require every Release build shard to succeed', + 'BUILD_TESTS_RESULT: ${{ needs.build-tests.result }}', + 'BUILD_TESTS_RESULT: success'), + ('build-tests', 'Build every Release test target', + ' run: |', ' if: false\n run: |'), + ('build-tests', 'Set up Python', + ' with:', ' if: false\n with:'), + ) + for name, pr in (('ci-post.yml', False), ('gke-pr.yml', True)): + workflow = (ROOT / '.github/workflows' / name).read_text() + self.assertEqual(ci_preflight.release_build_workflow_findings( + ci_preflight._jobs(workflow), pr=pr), []) + if pr: + self.assertEqual(ci_preflight.gke_admit_workflow_findings(workflow), []) + for job, step, before, after in mutations: + if pr and step is None: + before, after = before.replace('authz', 'admit'), after.replace('authz', 'admit') + with self.subTest(workflow=name, job=job, mutation=after): + self.assertIn(before, workflow) + changed = mutate_workflow(workflow, job, step, before, after) + self.assertTrue(ci_preflight.release_build_workflow_findings( + ci_preflight._jobs(changed), pr=pr), (before, after)) + if pr: + self.assertTrue(ci_preflight.gke_admit_workflow_findings(changed), + (before, after)) + jobs = ci_preflight._jobs(workflow) + for job in ('build-tests', 'build-tests-result'): + with self.subTest(workflow=name, missing=job): + self.assertTrue(ci_preflight.release_build_workflow_findings( + {key: body for key, body in jobs.items() if key != job}, pr=pr)) + before = ci_preflight._job_values(jobs[job], 'if')[0] + classifier = 'admit' if pr else 'changes' + conditions = ['${{ false }}'] + [ + before[:-3] + f" && needs.{classifier}.outputs.{field} != 'true' " + '}}' + for field in (('docs_only', 'github_only') if pr else ('docs_only',))] + for after in conditions: + with self.subTest(workflow=name, job=job, condition=after): + changed = mutate_workflow(workflow, job, None, before, after) + self.assertTrue(ci_preflight.release_build_workflow_findings( + ci_preflight._jobs(changed), pr=pr)) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_ci_corpus_variant.py b/scripts/test_ci_corpus_variant.py index 1fb18f713..1cefaadac 100644 --- a/scripts/test_ci_corpus_variant.py +++ b/scripts/test_ci_corpus_variant.py @@ -145,8 +145,8 @@ def without_corpus(self, workflow: str) -> str: workflow = workflow.replace(CORPUS_JOB, '', 1) workflow = self.replace_job( workflow, 'result', - ' needs: [admit, preflight, build-x86-gcc133, corpus-gitlink-result, legs]\n', - ' needs: [admit, preflight, build-x86-gcc133, legs]\n') + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, corpus-gitlink-result, legs]\n', + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs]\n') for addition in ( ' CORPUS_RESULT: ${{ needs.corpus-gitlink-result.result }}\n', ' test "$CORPUS_RESULT" = success || fail "corpus validation did not pass"\n'): @@ -161,8 +161,8 @@ def with_corpus(self, workflow: str) -> str: self.assertEqual(changed.replace(CORPUS_JOB, '', 1), workflow) changed = self.replace_job( changed, 'result', - ' needs: [admit, preflight, build-x86-gcc133, legs]\n', - ' needs: [admit, preflight, build-x86-gcc133, corpus-gitlink-result, legs]\n') + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs]\n', + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, corpus-gitlink-result, legs]\n') # Literal result hunk; these anchors are independent of checker templates. for anchor, addition in ( (' X86_RESULT: ${{ needs.build-x86-gcc133.result }}\n', @@ -240,11 +240,11 @@ def test_corpus_selector_only_selects_its_workflow_and_defined_job(self) -> None def test_job_present_with_old_result_needs_is_refused(self) -> None: changed = self.replace_job( self.workflow, 'result', - ' needs: [admit, preflight, build-x86-gcc133, corpus-gitlink-result, legs]\n', - ' needs: [admit, preflight, build-x86-gcc133, legs]\n') + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, corpus-gitlink-result, legs]\n', + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs]\n') self.assert_refused( changed, 'gke-pr.yml result must need admit, preflight, build-x86-gcc133, ' - 'corpus-gitlink-result and legs') + 'build-tests, build-tests-result, corpus-gitlink-result and legs') self.assertIn( 'gke-pr.yml result must require admission, preflight, native x86, ' 'corpus validation and consistent proof results', @@ -263,10 +263,11 @@ def test_job_present_with_old_result_template_is_refused(self) -> None: def test_job_absent_with_corpus_result_needs_is_refused(self) -> None: changed = self.replace_job( self.original, 'result', - ' needs: [admit, preflight, build-x86-gcc133, legs]\n', - ' needs: [admit, preflight, build-x86-gcc133, corpus-gitlink-result, legs]\n') + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs]\n', + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, corpus-gitlink-result, legs]\n') self.assert_refused( - changed, 'gke-pr.yml result must need admit, preflight, build-x86-gcc133 and legs', + changed, 'gke-pr.yml result must need admit, preflight, build-x86-gcc133, ' + 'build-tests, build-tests-result and legs', baseline=self.original) def test_job_absent_with_corpus_result_environment_is_refused(self) -> None: diff --git a/scripts/test_ci_preflight.py b/scripts/test_ci_preflight.py index 4748ac96e..047ae9173 100644 --- a/scripts/test_ci_preflight.py +++ b/scripts/test_ci_preflight.py @@ -103,9 +103,11 @@ class ParallelCommand(tuple): EXPECTED_GKE_JOBS = { 'gke-pr.yml': {'preflight': HOSTED_X86, 'build': HOSTED_X86, 'build-debug': HOSTED_X86, NATIVE_X86_JOB: HOSTED_X86, + 'build-tests': HOSTED_X86, 'build-tests-result': HOSTED_X86, 'build-arm': HOSTED_ARM, 'corpus-parity-subset': HOSTED_X86}, 'ci-post.yml': {'preflight': HOSTED_X86, NATIVE_X86_JOB: HOSTED_X86, + 'build-tests': HOSTED_X86, 'build-tests-result': HOSTED_X86, 'smoke': HOSTED_X86, 'smoke-arm': HOSTED_ARM, 'sanitizers': HOSTED_X86}, 'corpus-parity.yml': {'corpus-parity': HOSTED_X86, 'corpus-parity-subset': HOSTED_X86}, 'native-live.yml': {'native-live': NATIVE_RUNNER, 'native-live-shards': HOSTED_X86}, @@ -166,11 +168,13 @@ class ParallelCommand(tuple): 'and hosted ARM matrix') CI_JOB_IDS = ('authz',) GKE_PR_JOB_IDS = ('admit', 'preflight', NATIVE_X86_JOB, 'build', 'build-debug', 'build-arm', + 'build-tests', 'build-tests-result', 'corpus-parity-subset', 'legs', 'result') GKE_PROOF_IF = ("${{ needs.admit.outputs.allowed == 'true' && " "needs.admit.outputs.docs_only != 'true' && " "needs.admit.outputs.github_only != 'true' }}") POST_CI_JOB_IDS = ('authz', 'changes', 'preflight', NATIVE_X86_JOB, 'smoke', 'smoke-arm', + 'build-tests', 'build-tests-result', 'sanitizers', 'corpus-parity-subset', 'build-gate') NATIVE_HOSTED_RUNNERS = {'authz': HOSTED_X86, 'native-live': NATIVE_RUNNER, 'native-live-shards': HOSTED_X86, @@ -179,7 +183,7 @@ class ParallelCommand(tuple): 'ci.yml': ('authz',), 'gke-pr.yml': GKE_PR_JOB_IDS, 'ci-post.yml': ('authz', 'changes', 'preflight', 'smoke', 'smoke-arm', NATIVE_X86_JOB, - 'sanitizers', 'build-gate'), + 'build-tests', 'build-tests-result', 'sanitizers', 'build-gate'), 'native-live.yml': ('authz', 'native-live', 'native-live-shards', 'native-live-native', 'native-live-gate'), 'corpus-parity.yml': ('authz', 'corpus-parity', 'corpus-parity-subset'), 'docs.yml': ('authz', 'build'), @@ -2997,6 +3001,8 @@ def test_real_gke_jobs_allow_original_sudo_apt_steps(self): 'build': 'libeigen3-dev tzdata-legacy ccache', 'build-debug': 'libeigen3-dev tzdata-legacy ccache', NATIVE_X86_JOB: None, + 'build-tests': 'libeigen3-dev tzdata-legacy ccache ninja-build', + 'build-tests-result': None, # expectation corrected: 'libeigen3-dev tzdata-legacy ccache strace' -> # 'libeigen3-dev tzdata-legacy ccache', because the arm64 leg runs the fast # Debug smoke, which installs only the smoke dependencies. @@ -3010,6 +3016,8 @@ def test_real_gke_jobs_allow_original_sudo_apt_steps(self): 'ci-post.yml': { 'preflight': 'shellcheck', NATIVE_X86_JOB: None, + 'build-tests': 'libeigen3-dev tzdata-legacy ccache ninja-build', + 'build-tests-result': None, 'smoke': 'libeigen3-dev tzdata-legacy ccache', 'smoke-arm': 'libeigen3-dev tzdata-legacy ccache', 'sanitizers': 'libeigen3-dev tzdata-legacy ccache', @@ -5185,7 +5193,8 @@ def test_complete_stage_inventory_and_tier_counts(self): 'source-guard-cancellation', 'source-guard-cr150-source-bounds', 'cr150-source-gate-tests', 'rng-draw-order-tests', 'rng-draw-order', 'verifier-tests', 'preflight-tests', 'unittest-shards-tests', 'verifier-paths-tests', - 'native-c-surface-tests', 'native-x86-release-tests', 'kernel-residuals-tests', + 'native-c-surface-tests', 'native-x86-release-tests', 'release-build-population-tests', + 'kernel-residuals-tests', 'corpus-parity-identity-tests', 'corpus-parity-subset-cover-tests', 'corpus-sha-validation-tests', 'corpus-gitlink-result-tests', 'corpus-file-digest-tests', @@ -5205,11 +5214,16 @@ def test_complete_stage_inventory_and_tier_counts(self): {entry[0]: entry[1] for entry in check_commands(ROOT, env={})}[ 'native-x86-release-tests'], [sys.executable, str(ROOT / 'scripts/test_ci_native_x86_release.py')]) + self.assertEqual( + dict(check_commands(ROOT, env={}))['release-build-population-tests'], + [sys.executable, str(ROOT / 'scripts/test_ci_build_tests.py'), 'PopulationContract']) with mock.patch.object(ci_preflight.sys, 'platform', 'linux'): self.assertEqual([entry[0] for entry in check_commands(ROOT, env={})], expected) # main() prepends actionlint-version to all/fast/github-only. - self.assertEqual(len(expected) + 1, 71) - self.assertEqual(len(check_commands(ROOT, tier='fast')), 66) + # expectation corrected: all/fast 71/66 -> 72/67, because the + # Release population contract is a new fast preflight stage. + self.assertEqual(len(expected) + 1, 72) + self.assertEqual(len(check_commands(ROOT, tier='fast')), 67) self.assertEqual(len(check_commands(ROOT, tier='slow')), 4) self.assertEqual(len(check_commands(ROOT, tier='github-only')), 7) for platform in ('darwin', 'win32'): @@ -5218,7 +5232,9 @@ def test_complete_stage_inventory_and_tier_counts(self): if name not in ('prebuilt-bookworm-tests', 'prebuilt-glibc-tests')] self.assertEqual([entry[0] for entry in check_commands(ROOT, env={})], expected_platform) - self.assertEqual(len(check_commands(ROOT, tier='fast')), 64) + # expectation corrected: 64 -> 65, because the same new + # population contract also runs on these source-check hosts. + self.assertEqual(len(check_commands(ROOT, tier='fast')), 65) class PrebuiltGlibcRegistration(unittest.TestCase): @@ -5433,7 +5449,7 @@ def test_light_job_metadata_cannot_replace_the_hosted_pin(self): ci_preflight.runner_findings({**workflows, name: changed})) def gcp_sites(self): - """Keep the 18 admitted sites, including the synthetic isolated release form.""" + """Keep admitted sites, including the synthetic isolated release form.""" workflows = self.workflow_sources(legacy_release=True) routes = ci_preflight.workflow_event_routes(workflows) for name, workflow in workflows.items(): @@ -7783,7 +7799,11 @@ def test_gcp_sites_reject_cross_class_routes(self): # expectation corrected: 21 -> 18, because ci-post's preflight-slow, build, build-debug, # build-arm, sanitizers and kernel-only rows became the smoke, smoke-arm and sanitizers # smoke jobs (every CI round fits 15 minutes; the full profiles run in the lab's verify) - self.assertEqual(len(sites), 18) + # expectation corrected: 18 -> 20, because the Release target build + # and its inventory union each add a pinned hosted verification job. + # expectation corrected: 20 -> 22, because PRs also require the + # Release target build and its inventory union on hosted runners. + self.assertEqual(len(sites), 22) self.assertCountEqual([(name, job) for name, job, *_ in sites], [(name, job) for name, jobs in FLEET_JOB_IDS.items() for job in jobs if job not in LIGHT_JOB_IDS.get(name, ()) @@ -8105,6 +8125,7 @@ def test_build_aggregate_requires_matrix_success_on_pr_and_full_events(self): 'CHANGES_RESULT': 'success', 'DOCS_ONLY': 'false', 'PREFLIGHT_RESULT': 'success', 'X86_GCC133_RESULT': 'success', + 'BUILD_TESTS_RESULT': 'success', 'BUILD_TESTS_UNION_RESULT': 'success', **{variable: 'success' for variable in PROOF_JOBS.values()}} pr_script = job_script(self.gke(), 'legs', 'Require every proof leg') for variable in PR_ARM_PROOF_JOBS.values(): @@ -8121,6 +8142,8 @@ def test_build_aggregate_requires_matrix_success_on_pr_and_full_events(self): script = job_script(self.ci_sources()[6], 'build-gate', 'Require every verification lane to have passed') matrices = [('X86_GCC133_RESULT', 'native x86 GNU 13.3.0')] + matrices += [('BUILD_TESTS_RESULT', 'Release test build'), + ('BUILD_TESTS_UNION_RESULT', 'Release target union')] matrices.append(('SMOKE_RESULT', 'amd64 smoke')) if event != 'pull_request': matrices.append(('ARM_SMOKE_RESULT', 'arm64 smoke')) @@ -8145,6 +8168,7 @@ def test_the_build_aggregate_passes_a_docs_only_skip_and_nothing_else(self): script.write_text(job_script(self.ci_sources()[6], 'build-gate', 'Require every verification lane to have passed')) env = {'PATH': os.environ.get('PATH', '/usr/bin:/bin'), 'AUTHZ_RESULT': 'success', + 'BUILD_TESTS_RESULT': 'success', 'BUILD_TESTS_UNION_RESULT': 'success', 'GITHUB_EVENT_NAME': 'push'} lanes = list(PROOF_JOBS.values()) vectors = set() @@ -8264,6 +8288,7 @@ def transport(argv, **kwargs): 'PATH_FILE': str(root / 'paths'), 'GH_TOKEN': 'fixture-token', 'ADMIT_RESULT': 'success', 'ALLOWED': 'true', 'PREFLIGHT_RESULT': 'success', 'X86_RESULT': 'success', + 'BUILD_TESTS_RESULT': 'success', 'BUILD_TESTS_UNION_RESULT': 'success', 'LEGS_RESULT': 'skipped', **(results or {})} requests = [] @@ -8369,7 +8394,8 @@ def test_github_only_aggregate_defers_only_full_event_lanes(self): self.assertEqual(set(ci_preflight._needs(pr_gate)), {'admit', NATIVE_X86_JOB, *PR_ARM_PROOF_JOBS}) self.assertEqual(ci_preflight._needs(pr_jobs['result']), - ['admit', 'preflight', NATIVE_X86_JOB, 'legs']) + ['admit', 'preflight', NATIVE_X86_JOB, + 'build-tests', 'build-tests-result', 'legs']) self.assertIn('ARM_RESULT: ${{ needs.build-arm.result }}', pr_gate) self.assertIn('ARM_SMOKE_RESULT: ${{ needs.smoke-arm.result }}', post_gate) rows = [{'filename': '.github/workflows/ci.yml', 'status': 'modified'}] @@ -8520,6 +8546,7 @@ def test_build_aggregate_excuses_only_pr_deferrals_and_classified_docs_skips(sel for class_name in ('docs', 'github', 'code'): base = {'ADMIT_RESULT': 'success', 'ALLOWED': 'true', 'PREFLIGHT_RESULT': 'success', 'X86_RESULT': 'success', + 'BUILD_TESTS_RESULT': 'success', 'BUILD_TESTS_UNION_RESULT': 'success', 'LEGS_RESULT': 'success' if class_name == 'code' else 'skipped', 'ADMIT_DOCS_ONLY': 'true' if class_name == 'docs' else 'false', 'SECOND_DOCS_ONLY': 'true' if class_name == 'docs' else 'false', @@ -8529,10 +8556,13 @@ def test_build_aggregate_excuses_only_pr_deferrals_and_classified_docs_skips(sel 'SECOND_ARM_RELEVANT': 'true' if class_name == 'code' else 'false', 'CODE_EXTENSION': 'true' if class_name == 'code' else 'false'} for field in base: - for outcome in dict.fromkeys((base[field], 'success', 'skipped', 'failure', 'cancelled', '')): + for outcome in dict.fromkeys((base[field], 'success', 'skipped', 'failure', + 'cancelled', 'malformed', '')): values = {**base, field: outcome} expected = (values['ADMIT_RESULT'] == values['PREFLIGHT_RESULT'] == 'success' and values['X86_RESULT'] == 'success' + and values['BUILD_TESTS_RESULT'] == 'success' + and values['BUILD_TESTS_UNION_RESULT'] == 'success' and values['ALLOWED'] == 'true' and values['ADMIT_DOCS_ONLY'] == values['SECOND_DOCS_ONLY'] and values['ADMIT_GITHUB_ONLY'] == values['SECOND_GITHUB_ONLY'] @@ -8550,13 +8580,22 @@ def test_build_aggregate_excuses_only_pr_deferrals_and_classified_docs_skips(sel env={'PATH': os.environ.get('PATH', '/usr/bin:/bin'), **values}, capture_output=True, text=True) self.assertEqual(result.returncode == 0, expected, result.stdout + result.stderr) + for field in ('BUILD_TESTS_RESULT', 'BUILD_TESTS_UNION_RESULT'): + values = {key: value for key, value in base.items() if key != field} + with self.subTest(class_name=class_name, missing=field): + result = subprocess.run( + ['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', pr_script], + env={'PATH': os.environ.get('PATH', '/usr/bin:/bin'), **values}, + capture_output=True, text=True) + self.assertNotEqual(result.returncode, 0, result.stdout + result.stderr) for event in ('push', 'schedule', 'workflow_dispatch'): script = job_script(self.ci_sources()[6], 'build-gate', 'Require every verification lane to have passed') # Every CI round fits 15 minutes: the slow preflight tier, the leg # aggregates, kernel-only, native-live and the full parity sweep left # ci-post for the lab's verify; the smoke legs and the subset parity remain. - required = {'AUTHZ_RESULT', 'CHANGES_RESULT', 'PREFLIGHT_RESULT', 'X86_GCC133_RESULT'} + required = {'AUTHZ_RESULT', 'CHANGES_RESULT', 'PREFLIGHT_RESULT', 'X86_GCC133_RESULT', + 'BUILD_TESTS_RESULT', 'BUILD_TESTS_UNION_RESULT'} proof = {'SMOKE_RESULT', 'ARM_SMOKE_RESULT', 'SANITIZER_RESULT', 'PARITY_RESULT'} for docs_only in ('false', 'true'): base = {name: 'success' for name in required | proof} @@ -8600,6 +8639,10 @@ def test_build_gate_refuses_widened_deferrals_and_lost_heavy_dependencies(self): 'build-gate must require corpus-parity-subset success or a docs-only skip'), ('&& [[ "$X86_GCC133_RESULT" == "success" ]]', '&& true', 'build-gate must require build-x86-gcc133 success on every event'), + ('&& [[ "$BUILD_TESTS_RESULT" == "success" ]]', '&& true', + 'build-gate must require build-tests success on every event'), + ('&& [[ "$BUILD_TESTS_UNION_RESULT" == "success" ]]', '&& true', + 'build-gate must require build-tests-result success on every event'), (' SMOKE_RESULT: ${{ needs.smoke.result }}\n', '', 'build-gate must read the result of smoke'), ('PARITY_RESULT: ${{ needs.corpus-parity-subset.result }}', @@ -9386,24 +9429,79 @@ def test_the_preflight_stage_fails_bad_spot_labels_or_paid_runners(self): self.assertEqual(code, 1, output) self.assertIn(finding, output.splitlines()) - def run_preflight(self, commands): + def run_preflight(self, commands, *, console=None): directory = tempfile.TemporaryDirectory() self.addCleanup(directory.cleanup) root = Path(directory.name) - with contextlib.redirect_stdout(io.StringIO()): + console = console if console is not None else io.StringIO() + with contextlib.redirect_stdout(console), contextlib.redirect_stderr(console): code = run_checks(commands, root / 'logs', source=root) return code, json.loads((root / 'logs/preflight-summary.json').read_text()), root / 'logs' + def assert_named_failures(self, console, failures, *, nonfailures=()): + """Require per-stage exit summaries and a separate final ordered listing.""" + lines = console.splitlines() + failure_lines = [] + for name, exit_code in failures: + matches = [index for index, line in enumerate(lines) + if re.search(r'\bFAIL ' + re.escape(name) + r'\b', line)] + self.assertTrue(matches, f'missing FAIL {name}\n{console}') + index = matches[0] + self.assertRegex( + lines[index], + r'(?i)\b(?:exit(?:[ _-]?(?:code|status))?|returncode|status|rc)\s*[:=]?\s*' + + str(exit_code) + r'\b', console) + self.assertEqual(re.findall(r'\bFAIL [\w-]+', lines[index]), + [f'FAIL {name}'], console) + failure_lines.append(index) + tail = lines[max(failure_lines) + 1:] + summary_start = next((index for index, line in enumerate(tail) + if re.search(r'\b(?:fail(?:ed|ures)?|summary)\b', + line, re.IGNORECASE)), None) + self.assertIsNotNone(summary_start, f'missing final failure summary\n{console}') + final_summary = '\n'.join(tail[summary_start:]) + names = [name for name, _ in failures] + name_pattern = r'(? list: """A stage that prints the pinned skip literal and exits 77.""" @@ -9600,21 +9713,27 @@ def test_a_guard_skip_is_a_skip_only_without_the_overlay(self): # Each half states the variable itself: the job's own environment (the # overlay jobs export PINEFORGE_PRIVATE_FIXTURES) must not decide which # world this control asserts. + skipped_console = io.StringIO() with mock.patch.dict(os.environ) as own_env: own_env.pop('PINEFORGE_PRIVATE_FIXTURES', None) - code, summary, logs = self.run_preflight([('guard', self.guard_witness())]) + code, summary, logs = self.run_preflight([('guard', self.guard_witness())], + console=skipped_console) self.assertEqual(code, 0) self.assertEqual(summary['stages'][0]['status'], 'skipped') self.assertEqual(summary['stages'][0]['exitCode'], 77) # The stage's log keeps the guard's own output, literal included. self.assertIn('SKIP private fixtures not present:', (logs / 'guard.log').read_text()) + self.assertNotRegex(skipped_console.getvalue(), r'\bFAIL\b') + failed_console = io.StringIO() with mock.patch.dict(os.environ, {'PINEFORGE_PRIVATE_FIXTURES': '1'}): - code, summary, _ = self.run_preflight([('guard', self.guard_witness())]) + code, summary, _ = self.run_preflight([('guard', self.guard_witness())], + console=failed_console) self.assertEqual(code, 1) self.assertEqual(summary['status'], 'failed') self.assertEqual(summary['stages'][0]['status'], 'failed') self.assertEqual(summary['stages'][0]['exitCode'], 77) + self.assert_named_failures(failed_console.getvalue(), [('guard', 77)]) def test_a_bare_77_is_a_failure_in_both_worlds(self): # The literal is required: 77 alone is never a skip. @@ -9622,9 +9741,16 @@ def test_a_bare_77_is_a_failure_in_both_worlds(self): for flag in ('', '1'): with self.subTest(overlay=flag), \ mock.patch.dict(os.environ, {'PINEFORGE_PRIVATE_FIXTURES': flag}): - code, summary, _ = self.run_preflight([('guard', bare)]) + console = io.StringIO() + code, summary, logs = self.run_preflight([('guard', bare)], console=console) self.assertEqual(code, 1) + self.assertEqual(summary['status'], 'failed') self.assertEqual(summary['stages'][0]['status'], 'failed') + self.assertEqual(summary['stages'][0]['exitCode'], 77) + self.assertEqual(summary['stages'][0]['argv'], bare) + self.assertEqual(summary['stages'][0]['log'], 'guard.log') + self.assertEqual((logs / 'guard.log').read_bytes(), b'') + self.assert_named_failures(console.getvalue(), [('guard', 77)]) def test_the_skip_literal_has_one_home(self): # R2-F7: the branch reads ci_verify's literal, not a second copy. @@ -9637,19 +9763,25 @@ def test_empty_check_plan_cannot_pass(self): code, summary, _ = self.run_preflight([]) self.assertEqual(code, 1) self.assertEqual(summary['status'], 'failed') + self.assertEqual(summary['exitCode'], 1) + self.assertEqual(summary['stages'], []) def test_advisory_failure_is_reported_without_failing_the_run(self): # R5 L14-A: the documentation guards report today's drift until the # page-rewriting lanes land. Reported, logged, and not binding. + console = io.StringIO() code, summary, logs = self.run_preflight([ ('docs', [sys.executable, '-c', 'import sys; print("drift"); sys.exit(1)'], True), ('next', [sys.executable, '-c', 'print("next check ran")']), - ]) + ], console=console) self.assertEqual(code, 0) self.assertEqual(summary['status'], 'passed') self.assertEqual(summary['stages'][0]['status'], 'reported') self.assertEqual(summary['stages'][0]['exitCode'], 1) self.assertIn('drift', (logs / 'docs.log').read_text()) + self.assertIn('drift', console.getvalue()) + self.assertIn('next check ran', console.getvalue()) + self.assertNotRegex(console.getvalue(), r'\bFAIL\b') def test_every_documentation_stage_is_binding(self): # R5 int12: the page lanes landed, so the guards and their self-tests @@ -10057,10 +10189,13 @@ def test_no_stage_is_advisory_today(self): self.assertFalse(len(entry) > 2 and entry[2], entry[0]) def test_success_is_explicitly_only_preflight(self): - code, summary, _ = self.run_preflight([('ok', [sys.executable, '-c', 'pass'])]) + console = io.StringIO() + code, summary, _ = self.run_preflight([('ok', [sys.executable, '-c', 'pass'])], + console=console) self.assertEqual(code, 0) self.assertEqual(summary['status'], 'passed') self.assertIn('full verification still required', summary['scope']) + self.assertNotRegex(console.getvalue(), r'\bFAIL\b') def test_summary_and_stages_record_started_at_and_seconds_while_preserving_existing_keys(self): iso_utc_second = re.compile(r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$') @@ -11824,6 +11959,7 @@ def arm_result(self, relevant, outcome, *, legs_script=None, second=None, 'Require an independently consistent successful result'), {'ADMIT_RESULT': 'success', 'ALLOWED': 'true', 'PREFLIGHT_RESULT': 'success', 'X86_RESULT': native, + 'BUILD_TESTS_RESULT': 'success', 'BUILD_TESTS_UNION_RESULT': 'success', 'LEGS_RESULT': ('skipped' if classified else 'success' if proof.returncode == 0 else 'failure'), 'ADMIT_DOCS_ONLY': docs_only, 'ADMIT_GITHUB_ONLY': github_only, @@ -12663,7 +12799,7 @@ def test_result_and_checkout_mutations_are_red(self) -> None: in_job(original, 'result', ' if: always()\n', ' if: success()\n'), 'gke-pr.yml result must run always') cases = ( - (' needs: [admit, preflight, build-x86-gcc133, legs]\n', + (' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs]\n', ' needs: [admit, preflight]\n', 'gke-pr.yml result must require admission, preflight, native x86 and consistent proof results'), (' ref: ${{ needs.admit.outputs.head_sha }}\n', @@ -12890,12 +13026,35 @@ def legacy_gke_arm_smoke(legacy): The pinned fixture still carries #701's full Debug profile arm64 job, which the contract refuses since every CI round fits 15 minutes (owner, 2026-11-13). Swap - in the reviewed arm64 smoke job; the fixture bytes and their hash stay unchanged. + in the reviewed arm64 smoke job and mandatory Release build jobs; the fixture + bytes and their hash stay unchanged. """ old = GkeArmAdmissionVariantTests.LEGACY_ARM_JOB if legacy.count(old) != 1: raise AssertionError('the legacy gke-pr.yml must hold the full-profile arm64 job once') - return legacy.replace(old, GkeArmAdmissionVariantTests.ARM_JOB, 1) + legacy = legacy.replace(old, GkeArmAdmissionVariantTests.ARM_JOB, 1) + workflow = ROOT / '.github/workflows/gke-pr.yml' + announce_compare_fixture_paths(workflow) + jobs = _jobs(workflow.read_text(encoding='utf-8')) + for job in ('build-tests', 'build-tests-result'): + if job in _jobs(legacy) or not jobs.get(job): + raise AssertionError('legacy fixture requires the real Release build job: ' + job) + legacy = legacy.rstrip('\n') + '\n\n ' + job + ':' + jobs[job] + legacy = in_job( + legacy, 'result', + ' needs: [admit, preflight, build-x86-gcc133, legs]\n', + ' needs: [admit, preflight, build-x86-gcc133, build-tests, build-tests-result, legs]\n') + for anchor, addition in ( + (' X86_RESULT: ${{ needs.build-x86-gcc133.result }}\n', + ' BUILD_TESTS_RESULT: ${{ needs.build-tests.result }}\n' + ' BUILD_TESTS_UNION_RESULT: ${{ needs.build-tests-result.result }}\n'), + (' test "$X86_RESULT" = success || fail "native x86 GNU 13.3.0 build did not pass"\n', + ' test "$BUILD_TESTS_RESULT" = success || fail "Release test build did not pass"\n' + ' test "$BUILD_TESTS_UNION_RESULT" = success || fail "Release target union did not pass"\n')): + legacy = in_step( + legacy, 'result', 'Require an independently consistent successful result', + anchor, anchor + addition) + return legacy class SplitHostedReleaseAdmission(unittest.TestCase): diff --git a/scripts/test_cr150_source_gate.py b/scripts/test_cr150_source_gate.py index 18b4a262a..0062e1ccb 100644 --- a/scripts/test_cr150_source_gate.py +++ b/scripts/test_cr150_source_gate.py @@ -1252,7 +1252,9 @@ def test_real_tree_scans_files_and_sites_and_is_not_a_pass_by_construction(self) # deque groups and adds four current groups; every new group is OPEN. # The combined tree adds two #858 cache-id site identities inside # existing U0307/U0871 OPEN groups. No new group: the union stays1616. - expected_open = 1356 + 76 + 184 + 2 - 6 + 4 + # expectation corrected: 1616 -> 1618, because per-origin exit sizing + # adds origin_shares/U1860 and reserved_by_origin/U1861, both OPEN. + expected_open = 1356 + 76 + 184 + 2 - 6 + 4 + 2 self.assertEqual(c['OPEN'], expected_open) # Every matched entry is exactly one of PASS(bounded) / OPEN. self.assertEqual(c['PASS(bounded)'] + c['OPEN'], c['entries_matched']) diff --git a/scripts/test_verify_corpus_count_rule.py b/scripts/test_verify_corpus_count_rule.py index e58c6b7d1..6a299fe3e 100644 --- a/scripts/test_verify_corpus_count_rule.py +++ b/scripts/test_verify_corpus_count_rule.py @@ -15,11 +15,17 @@ import sys import tempfile import unittest +from contextlib import redirect_stdout from datetime import datetime, timedelta, timezone +from io import StringIO from pathlib import Path +from unittest.mock import patch -from verify_corpus import MATCH_WINDOW_SECONDS, analyze_strategy -from test_verify_corpus_trail_profile import TRADE_COLUMNS, TRAIL_STRATEGY +import verify_corpus +from verify_corpus import MATCH_WINDOW_SECONDS, analyze_strategy, main +from test_verify_corpus_trail_profile import ( + STRICT_STRATEGY, TRADE_COLUMNS, TRAIL_STRATEGY, +) def _checked_path(path: Path) -> Path: @@ -44,6 +50,23 @@ def _checked_path(path: Path) -> Path: ClosedTrade = tuple[datetime, datetime, str] +REFERENCE_STRATEGY_ID = "validation/mtf-triple-tf-macd-hist-confluence-01" + + +def _reference_floor_tapes(replacements=(10,)): + start = datetime(2025, 1, 1, tzinfo=timezone.utc) + tv_trades = [ + (start + timedelta(days=index), + start + timedelta(days=index, hours=1), f"entry-{index + 1}") + for index in range(24) + ] + engine_trades = [ + (entry + timedelta(hours=12), exit_time + timedelta(hours=12), + identity + "-replacement") + if index in replacements else (entry, exit_time, identity) + for index, (entry, exit_time, identity) in enumerate(tv_trades) + ] + return tv_trades, engine_trades def _write_trades( @@ -79,7 +102,10 @@ def _write_trades( def _analyze_tapes( run_root: Path, name: str, - tv_trades: list[ClosedTrade], engine_trades: list[ClosedTrade], + tv_trades: list, engine_trades: list, + *, write_trades=_write_trades, + strategy_source=TRAIL_STRATEGY, aggregate=False, + omit_tv=False, omit_engine=False, ): strategy = _checked_path(run_root / name) files = { @@ -93,21 +119,42 @@ def _analyze_tapes( print(f"Count-rule fixture files to follow ({name}):", flush=True) for path in files.values(): print(str(path), flush=True) + for filename in ("verify_corpus.py", "parity_paths.py"): + print(str(_checked_path(Path(__file__).with_name(filename))), flush=True) # The real analyzer probes these OHLCV candidates. They are deliberately # absent in the fresh fixture directory; no trim implementation is replaced. - strategy.mkdir() + strategy.mkdir(parents=True) with files["strategy.pine"].open("x", encoding="utf-8") as handle: - handle.write(TRAIL_STRATEGY) + handle.write(strategy_source) with files["inputs.json"].open("x", encoding="utf-8") as handle: json.dump({"tv_trades_csv_tz": "utc_plus_8"}, handle) - _write_trades(files["tv_trades.csv"], tv_trades, tv_timezone=True) - _write_trades(files["engine_trades.csv"], engine_trades, tv_timezone=False) + if not omit_tv: + write_trades(files["tv_trades.csv"], tv_trades, tv_timezone=True) + if not omit_engine: + write_trades(files["engine_trades.csv"], engine_trades, tv_timezone=False) result = analyze_strategy(strategy) row = result.report_row() + cli_output = StringIO() + corpus_root = run_root if aggregate else strategy + cli_argv = [ + "verify_corpus.py", "--all" if aggregate else str(strategy), + "--corpus-root", str(corpus_root), + "--derived-workspace", str(corpus_root), + "--harness-dir", str(corpus_root), + ] + with patch.object(sys, "argv", cli_argv), redirect_stdout(cli_output): + cli_exit = main() + cli = { + "execution": "in_process_main", + "argv": cli_argv, + "exit": cli_exit, + "stdout": cli_output.getvalue(), + } # Retain actual observations before assertions, including on a failure. observations = { "report_row": row, + "cli": cli, "result_fields": { field: getattr(result, field) for field in ( @@ -128,6 +175,10 @@ def _analyze_tapes( "tv_signal": tv.entry_signal, "engine_identity": eng.entry_identity, "tv_exit_time": tv.exit_time, "engine_exit_time": eng.exit_time, + "tv_entry_price": tv.entry_price, "engine_entry_price": eng.entry_price, + "tv_exit_price": tv.exit_price, "engine_exit_price": eng.exit_price, + "tv_qty": tv.qty, "engine_qty": eng.qty, + "tv_pnl": tv.pnl, "engine_pnl": eng.pnl, } for tv, eng in result.matched ], @@ -136,7 +187,7 @@ def _analyze_tapes( json.dump(observations, handle, indent=2, sort_keys=True) handle.write("\n") print(json.dumps(row, sort_keys=True), flush=True) - return result, row + return result, row, cli class ExactCountBehaviorTests(unittest.TestCase): @@ -156,28 +207,35 @@ def setUp(self): ))) print(f"Count-rule retained run directory: {self.run_root}", flush=True) - def _assert_other_gates(self, result, row, shared, engine_count): + def _assert_other_gates( + self, result, row, shared, engine_count, *, tv_count=None + ): + shared_count = len(shared) + if tv_count is None: + tv_count = shared_count self.assertTrue(result.tv_exists) self.assertTrue(result.eng_exists) self.assertFalse(result.no_aligned_trades) self.assertEqual(result.profile, "production") self.assertEqual( - (result.tv_raw_count, result.eng_raw_count), (200, engine_count) + (result.tv_raw_count, result.eng_raw_count), (tv_count, engine_count) ) - self.assertEqual((result.tv_count, result.eng_count), (200, engine_count)) + self.assertEqual((result.tv_count, result.eng_count), (tv_count, engine_count)) self.assertEqual( - (result.tv_gate_count, result.eng_gate_count), (200, engine_count) + (result.tv_gate_count, result.eng_gate_count), (tv_count, engine_count) ) self.assertEqual((result.matched_count, result.gating_matched_count), - (200, 200)) - self.assertEqual(len(result.matched), 200) + (shared_count, shared_count)) + self.assertEqual(len(result.matched), shared_count) self.assertEqual((result.trim_bars, result.warmup_bars), (0, 0)) self.assertIsNone(result.bounds) self.assertEqual(result.open_mark_pairs, 0) - self.assertEqual(result.coverage_tv_count, 200) - self.assertEqual(result.coverage, 1.0) - self.assertEqual(result.unmatched_total, 0) - self.assertEqual(result.unmatched_in_window, engine_count - 200) + self.assertEqual(result.coverage_tv_count, tv_count) + self.assertEqual(result.coverage, shared_count / tv_count) + self.assertEqual(result.unmatched_total, tv_count - shared_count) + self.assertEqual( + result.unmatched_in_window, max(tv_count, engine_count) - shared_count + ) for field in ( "entry_ok", "exit_ok", "pnl_ok", "coverage_ok", "distinct_entry_identity_ok", @@ -185,14 +243,15 @@ def _assert_other_gates(self, result, row, shared, engine_count): self.assertIs(getattr(result, field), True, field) self.assertEqual(result.distinct_entry_mismatches, 0) self.assertEqual( - row["metric_observations"], {"entry": 200, "exit": 200, "pnl": 200} + row["metric_observations"], + {"entry": shared_count, "exit": shared_count, "pnl": shared_count}, ) policy = row["grading_policy"] self.assertEqual(policy["count_rule"], "exact") self.assertEqual(policy["profile"], result.profile) self.assertGreater(policy["thresholds"]["count"], 0.0) for metric in ("entry", "exit", "pnl"): - self.assertEqual(len(getattr(result, metric + "_deltas")), 200) + self.assertEqual(len(getattr(result, metric + "_deltas")), shared_count) self.assertEqual(getattr(result, metric + "_p90"), 0.0) self.assertEqual(getattr(result, metric + "_p100"), 0.0) self.assertEqual(row[metric + "_p90"], getattr(result, metric + "_p90")) @@ -200,7 +259,13 @@ def _assert_other_gates(self, result, row, shared, engine_count): self.assertEqual(result.qty_p100, 0.0) self.assertEqual((row["tv"], row["eng"], row["matched"]), (result.tv_count, result.eng_count, result.matched_count)) - self.assertEqual(row["count_delta"], result.count_delta) + # Count every unmatched trade on both sides, including substitutions + # that leave the two pool sizes equal. + gross_count_delta = (tv_count + engine_count - 2 * shared_count) / max( + tv_count, engine_count, 1 + ) + self.assertEqual(result.count_delta, gross_count_delta) + self.assertEqual(row["count_delta"], gross_count_delta) self.assertEqual(row["count_abs_delta"], result.count_abs_delta) self.assertIs(row["distinct_entry_identity_ok"], True) self.assertEqual(row["distinct_entry_mismatches"], 0) @@ -247,15 +312,22 @@ def test_exact_rule_rejects_one_interior_extra_within_nominal_fraction(self): self.assertEqual(len({entry for entry, _, _ in engine_extra}), 201) run_root = self.run_root - control, control_row = _analyze_tapes(run_root, "equal", shared, shared) + control, control_row, control_cli = _analyze_tapes( + run_root, "equal", shared, shared + ) self._assert_other_gates(control, control_row, shared, 200) self.assertEqual(control.count_abs_delta, 0) self.assertEqual(control.count_delta, 0.0) self.assertIs(control.count_ok, True) self.assertEqual(control.tier, "excellent") + self.assertEqual(control_cli["exit"], 0) + self.assertRegex( + control_cli["stdout"], r"Count delta:\s+0\.0000%\s+\(OK;" + ) + self.assertIn("-> excellent", control_cli["stdout"]) # A failed equal-count control stops this method before unequal analysis. - unequal, unequal_row = _analyze_tapes( + unequal, unequal_row, unequal_cli = _analyze_tapes( run_root, "interior-extra", shared, engine_extra ) self._assert_other_gates(unequal, unequal_row, shared, 201) @@ -269,6 +341,244 @@ def test_exact_rule_rejects_one_interior_extra_within_nominal_fraction(self): # A nominal-fraction count gate must fail this real analyzer assertion. self.assertIs(unequal.count_ok, False) self.assertEqual(unequal.tier, "strong") + self.assertEqual(unequal_cli["exit"], 0) + self.assertIn("abs=1", unequal_cli["stdout"]) + self.assertIn("-> strong", unequal_cli["stdout"]) + + def test_balanced_unmatched_trades_do_not_cancel(self): + start = datetime(2025, 1, 1, tzinfo=timezone.utc) + shared = [ + (start + timedelta(days=index), + start + timedelta(days=index, hours=1), f"entry-{index + 1}") + for index in range(200) + ] + engine_entry = shared[49][0] + timedelta(hours=12) + tv_entry = shared[149][0] + timedelta(hours=12) + for entry in (engine_entry, tv_entry): + for shared_entry, _, _ in shared: + self.assertGreater( + abs((entry - shared_entry).total_seconds()), + MATCH_WINDOW_SECONDS, + ) + self.assertGreater( + (tv_entry - engine_entry).total_seconds(), MATCH_WINDOW_SECONDS + ) + engine_only = ( + engine_entry, engine_entry + timedelta(hours=1), "engine-only" + ) + tv_only = (tv_entry, tv_entry + timedelta(hours=1), "tv-only") + tv_trades = sorted([*shared, tv_only]) + engine_trades = sorted([*shared, engine_only]) + for trades in (tv_trades, engine_trades): + self.assertEqual((trades[0], trades[-1]), (shared[0], shared[-1])) + self.assertEqual(len({entry for entry, _, _ in trades}), 201) + + # Same row counts as the regression, but all 201 entries really match. + # Moving the engine-only input to the TV-only entry must restore Excellent. + control, control_row, control_cli = _analyze_tapes( + self.run_root, "balanced-matched-control", tv_trades, tv_trades + ) + self._assert_other_gates(control, control_row, tv_trades, 201) + self.assertEqual(control.count_abs_delta, 0) + self.assertEqual(control.count_delta, 0.0) + self.assertIs(control.count_ok, True) + self.assertEqual(control.tier, "excellent") + self.assertEqual(control_cli["exit"], 0) + self.assertRegex( + control_cli["stdout"], r"Count delta:\s+0\.0000%\s+\(OK;" + ) + self.assertIn("-> excellent", control_cli["stdout"]) + + balanced, balanced_row, balanced_cli = _analyze_tapes( + self.run_root, "balanced-unmatched", tv_trades, engine_trades + ) + self._assert_other_gates( + balanced, balanced_row, shared, 201, tv_count=201 + ) + # expectation corrected: 0 -> 2, because each unmatched TV/engine + # entry contributes one to the canonical absolute mismatch count. + # This assertion must fail on both the net grader and a grader that + # keeps net counting here while exposing a separate gross field. + self.assertEqual( + balanced.count_abs_delta, 2, + "count_abs_delta must count one unmatched TV trade plus one " + "unmatched engine trade", + ) + self.assertEqual(balanced_row["count_abs_delta"], 2) + self.assertIs( + balanced.count_ok, False, + f"balanced unmatched trades cancelled: " + f"delta={balanced.count_delta}, tier={balanced.tier}", + ) + self.assertEqual(balanced.gross_unmatched_count, 2) + self.assertEqual(balanced_row["gross_unmatched_count"], 2) + # expectation corrected: 0 -> 2 / 201, because the percentage uses + # the same two gross unmatched trades as the absolute count. + self.assertEqual(balanced.count_delta, 2 / 201) + self.assertLess( + balanced.count_delta, + balanced_row["grading_policy"]["thresholds"]["count"], + ) + self.assertEqual(balanced.tier, "strong") + self.assertNotEqual(balanced.tier, "excellent") + self.assertEqual(balanced_cli["exit"], 0) + self.assertRegex( + balanced_cli["stdout"], r"Count delta:\s+0\.9950%\s+\(X;" + ) + self.assertIn("abs=2", balanced_cli["stdout"]) + self.assertIn("net=0", balanced_cli["stdout"]) + self.assertIn("-> strong", balanced_cli["stdout"]) + self.assertEqual(balanced.net_count_abs_delta, 0) + self.assertEqual(balanced_row["net_count_abs_delta"], 0) + self.assertEqual( + balanced_row["grading_policy"]["count_basis"], "unmatched_both_sides" + ) + self.assertEqual(control.gross_unmatched_count, 0) + self.assertEqual(control_row["gross_unmatched_count"], 0) + self.assertEqual(control.net_count_abs_delta, 0) + self.assertIn("abs=0", control_cli["stdout"]) + + def test_strong_rejects_balanced_unmatched_at_gross_limit(self): + start = datetime(2025, 1, 1, tzinfo=timezone.utc) + tv_trades = [ + (start + timedelta(days=index), + start + timedelta(days=index, hours=1), f"entry-{index + 1}") + for index in range(100) + ] + results = [] + for substitutions, expected_label in ((2, "strong"), (3, "moderate")): + moved = set((20, 40, 60)[:substitutions]) + engine_trades = [ + (entry + timedelta(hours=12), + exit_time + timedelta(hours=12), identity + "-replacement") + if index in moved else (entry, exit_time, identity) + for index, (entry, exit_time, identity) in enumerate(tv_trades) + ] + self.assertEqual(engine_trades[0], tv_trades[0]) + self.assertEqual(engine_trades[-1], tv_trades[-1]) + result, row, cli = _analyze_tapes( + self.run_root, f"strong-{substitutions}", tv_trades, engine_trades + ) + self.assertEqual((result.tv_gate_count, result.eng_gate_count), (100, 100)) + self.assertEqual(result.gating_matched_count, 100 - substitutions) + self.assertEqual(result.count_abs_delta, 2 * substitutions) + self.assertEqual(result.net_count_abs_delta, 0) + self.assertEqual(result.coverage, (100 - substitutions) / 100) + self.assertGreaterEqual(result.coverage, 0.95) + self.assertFalse(result.coverage_ok) # Excellent still requires 99%. + self.assertTrue(result.distinct_entry_identity_ok) + for metric in ("entry", "exit", "pnl"): + self.assertTrue(getattr(result, metric + "_ok")) + self.assertEqual(getattr(result, metric + "_p90"), 0.0) + # On the previous grader this is the first failing assertion: + # the 6% gross case incorrectly remains strong at 0% net. + self.assertEqual( + result.label, expected_label, + "strong must reject gross unmatched count at the 6% limit", + ) + self.assertEqual(cli["exit"], 0 if expected_label == "strong" else 1) + results.append((result, row, substitutions)) + + for result, row, substitutions in results: + self.assertEqual(result.count_delta, 2 * substitutions / 100) + self.assertEqual(row["count_delta"], result.count_delta) + + def _assert_reference_moderate(self, result, row): + self.assertEqual(result.label, "moderate") + self.assertEqual(row["tier"], "moderate") + self.assertEqual(result.profile, "strict") + self.assertEqual((result.tv_raw_count, result.eng_raw_count), (24, 24)) + self.assertEqual((result.tv_gate_count, result.eng_gate_count), (24, 24)) + self.assertEqual(result.gating_matched_count, 23) + self.assertEqual(result.count_abs_delta, 2) + self.assertEqual(result.net_count_abs_delta, 0) + self.assertEqual(result.count_delta, 2 / 24) + self.assertEqual(result.coverage, 23 / 24) + for metric in ("entry", "exit", "pnl"): + self.assertEqual(getattr(result, metric + "_p90"), 0.0) + + def test_dated_reference_floor_accepts_measured_moderate(self): + """#836/#883: aggregate acceptance preserves the measured Moderate tier.""" + result, row, cli = _analyze_tapes( + self.run_root, REFERENCE_STRATEGY_ID, *_reference_floor_tapes(), + strategy_source=STRICT_STRATEGY, aggregate=True, + ) + self._assert_reference_moderate(result, row) + self.assertIn("-> moderate", cli["stdout"]) + self.assertIn( + "Verified 1 strategies — excellent=0, strong=0, moderate=1, " + "weak=0, minimal=0, anomaly=0, engine_only=0, missing=0", + cli["stdout"], + ) + self.assertEqual( + cli["exit"], 0, + "dated reference floor must accept the measured Moderate state (#836/#883)", + ) + self.assertNotIn("Below strong tier:", cli["stdout"]) + + def test_reference_floor_rejects_other_moderate_ids(self): + """#836/#883: every other full id retains the Strong aggregate floor.""" + basename = REFERENCE_STRATEGY_ID.split("/", 1)[1] + for strategy_id in ( + "validation/other-moderate", f"validation/{basename}-variant", + f"validation/prefix-{basename}", f"basic/{basename}", + f"community/{basename}", + ): + with self.subTest(strategy_id=strategy_id): + result, row, cli = _analyze_tapes( + self.run_root / strategy_id.replace("/", "-"), + strategy_id, *_reference_floor_tapes(), + strategy_source=STRICT_STRATEGY, aggregate=True, + ) + self._assert_reference_moderate(result, row) + self.assertEqual(cli["exit"], 1) + self.assertIn("moderate=1", cli["stdout"]) + self.assertIn(f"Below strong tier:\n {strategy_id}", cli["stdout"]) + + def test_reference_floor_rejects_moderate_when_exception_removed(self): + """#836/#883: removing only the floor policy restores aggregate exit 1.""" + with patch.dict(verify_corpus.REFERENCE_FLOOR_EXCEPTIONS, clear=True): + result, row, cli = _analyze_tapes( + self.run_root, REFERENCE_STRATEGY_ID, *_reference_floor_tapes(), + strategy_source=STRICT_STRATEGY, aggregate=True, + ) + self._assert_reference_moderate(result, row) + self.assertEqual(cli["exit"], 1) + self.assertIn("moderate=1", cli["stdout"]) + self.assertIn(f"Below strong tier:\n {REFERENCE_STRATEGY_ID}", cli["stdout"]) + + def test_reference_floor_rejects_missing_or_lower_tiers(self): + """#836/#883: the exact id still fails with missing input or a lower tier.""" + for name, replacements, omit_tv, omit_engine, expected_tier in ( + ("weak", (3, 7, 12, 18), False, False, "weak"), + ("minimal", tuple(range(24)), False, False, "minimal"), + ("missing-tv", (10,), True, False, "missing"), + ("missing-engine", (10,), False, True, "missing"), + ): + with self.subTest(name=name): + result, row, cli = _analyze_tapes( + self.run_root / name, REFERENCE_STRATEGY_ID, + *_reference_floor_tapes(replacements), + strategy_source=STRICT_STRATEGY, aggregate=True, + omit_tv=omit_tv, omit_engine=omit_engine, + ) + self.assertEqual(result.label, expected_tier) + self.assertEqual(row["tier"], expected_tier) + self.assertEqual(cli["exit"], 1) + self.assertIn(f"{expected_tier}=1", cli["stdout"]) + self.assertIn( + f"Below strong tier:\n {REFERENCE_STRATEGY_ID}", cli["stdout"] + ) + + def test_reference_floor_does_not_change_single_strategy_exit(self): + """#836/#883: the dated allowance applies only to aggregate verification.""" + result, row, cli = _analyze_tapes( + self.run_root, REFERENCE_STRATEGY_ID, *_reference_floor_tapes(), + strategy_source=STRICT_STRATEGY, + ) + self._assert_reference_moderate(result, row) + self.assertEqual(cli["exit"], 1) + self.assertIn("-> moderate", cli["stdout"]) if __name__ == "__main__": diff --git a/scripts/test_verify_corpus_metrics.py b/scripts/test_verify_corpus_metrics.py index 4db326154..79c229b91 100644 --- a/scripts/test_verify_corpus_metrics.py +++ b/scripts/test_verify_corpus_metrics.py @@ -3,6 +3,8 @@ from __future__ import annotations +import argparse +import csv import json import math import sys @@ -15,6 +17,8 @@ from pathlib import Path from unittest.mock import patch +from test_verify_corpus_count_rule import _analyze_tapes, _checked_path +from test_verify_corpus_trail_profile import TRADE_COLUMNS from verify_corpus import ( STRICT_ENTRY_DELTA, TradePair, @@ -60,6 +64,27 @@ def trade( ) +def _write_trade_pairs(path: Path, rows: list[TradePair], *, tv_timezone: bool) -> None: + """Keep raw partial-close quantities and their actual broker events.""" + offset = timedelta(hours=8 if tv_timezone else 0) + with _checked_path(path).open("x", encoding="utf-8", newline="") as handle: + writer = csv.writer(handle) + writer.writerow([*TRADE_COLUMNS, "Engine entry ID"]) + for row in rows: + entry = datetime.fromtimestamp(row.entry_time, timezone.utc) + offset + exit_time = datetime.fromtimestamp(row.exit_time, timezone.utc) + offset + writer.writerow([ + row.trade_num, "Entry " + row.direction, + entry.strftime("%Y-%m-%d %H:%M"), row.entry_signal, + row.entry_price, row.qty, 0.0, 0.0, row.entry_identity, + ]) + writer.writerow([ + row.trade_num, "Exit " + row.direction, + exit_time.strftime("%Y-%m-%d %H:%M"), "", + row.exit_price, row.qty, row.pnl, 0.0, "", + ]) + + class FragmentConsolidationIdentityTests(unittest.TestCase): def test_distinct_tv_entry_signals_preserve_same_tick_trades(self) -> None: rows = [ @@ -632,6 +657,9 @@ def test_identity_collision_outside_declared_interior_is_excluded( self.assertIsNotNone(result.bounds) self.assertEqual(result.tv_gate_count, 0) self.assertEqual(result.eng_gate_count, 0) + self.assertEqual(result.count_abs_delta, 0) + self.assertEqual(result.net_count_abs_delta, 0) + self.assertTrue(result.count_ok) self.assertTrue(result.distinct_entry_identity_ok) self.assertEqual(result.distinct_entry_mismatches, 0) self.assertEqual(result.label, "excellent") @@ -668,12 +696,20 @@ def test_identity_gate_blocks_false_excellent_direction_substitution( self.assertEqual(result.tv_count, 2) self.assertEqual(result.eng_count, 2) - self.assertEqual(result.count_abs_delta, 0) + # expectation corrected: 0 -> 2, because the unmatched long and short + # are separate mismatches despite equal gated list sizes. + self.assertEqual(result.count_abs_delta, 2) + self.assertEqual(result.net_count_abs_delta, 0) + self.assertFalse(result.count_ok) self.assertEqual(result.unmatched_in_window, 1) self.assertTrue(result.coverage_ok) self.assertFalse(result.distinct_entry_identity_ok) self.assertEqual(result.distinct_entry_mismatches, 1) - self.assertEqual(result.label, "strong") + # expectation corrected: 0 -> 1 and strong -> weak, because both + # unmatched trades count against the two-trade compared population. + self.assertEqual(result.count_delta, 1.0) + self.assertEqual(result.report_row()["count_delta"], 1.0) + self.assertEqual(result.label, "weak") PRIVATE_FIXTURE_SKIP_LITERAL = "SKIP private fixtures not present: " @@ -743,6 +779,88 @@ def test_refill_cascade_tape_certifies_every_physical_entry(self) -> None: class FragmentedFifoEligibilityTests(unittest.TestCase): + retain_artifacts: Path | None = None + + def test_gross_count_limit_blocks_fifo_metric_replacement(self) -> None: + if self.retain_artifacts is None: + temporary = tempfile.TemporaryDirectory(prefix="verify-corpus-fifo-") + self.addCleanup(temporary.cleanup) + run_root = _checked_path(Path(temporary.name)) + else: + run_root = _checked_path(Path(tempfile.mkdtemp( + prefix="verify-corpus-fifo-", dir=_checked_path(self.retain_artifacts) + ))) + start = int(datetime(2025, 1, 1, tzinfo=timezone.utc).timestamp()) + results = [] + for substitutions in (2, 3): + tv_rows = [] + engine_rows = [] + moved = set((10, 20, 30)[:substitutions]) + for group in range(50): + base = start + group * 2 * 86400 + for entry_index in range(2): + entry_time = base + entry_index * 12 * 3600 + engine_entry_time = entry_time + ( + 6 * 3600 if group in moved and entry_index == 0 else 0 + ) + # Same exit schedule, different allocation across the two + # entry fills: TV consolidates to 115/125, engine to 125/115. + for exit_index, exit_price in enumerate((110.0, 130.0)): + tv_qty = 0.75 if entry_index == exit_index else 0.25 + engine_qty = 1.0 - tv_qty + exit_time = base + (24 + exit_index * 12) * 3600 + number = len(tv_rows) + 1 + tv_rows.append(trade( + number, entry_time, 100.0, exit_time, exit_price, + qty=tv_qty, pnl=(exit_price - 100.0) * tv_qty, + entry_signal=f"entry-{group}-{entry_index}", + )) + engine_rows.append(trade( + number, engine_entry_time, 100.0, exit_time, exit_price, + qty=engine_qty, pnl=(exit_price - 100.0) * engine_qty, + entry_identity=f"engine-{group}-{entry_index}", + )) + self.assertTrue(has_cross_entry_fifo_allocation(tv_rows)) + self.assertTrue(has_cross_entry_fifo_allocation(engine_rows)) + schedule_exit, schedule_pnl = schedule_exit_metrics(tv_rows, engine_rows) + self.assertEqual(schedule_exit, [0.0] * 100) + self.assertEqual(schedule_pnl, [0.0]) + + result, row, cli = _analyze_tapes( + run_root, f"fifo-{substitutions}", tv_rows, engine_rows, + write_trades=_write_trade_pairs, + ) + self.assertEqual((result.tv_gate_count, result.eng_gate_count), (100, 100)) + self.assertEqual(result.gating_matched_count, 100 - substitutions) + self.assertEqual(result.count_abs_delta, 2 * substitutions) + self.assertEqual(result.net_count_abs_delta, 0) + self.assertEqual(result.entry_p90, 0.0) + self.assertEqual(result.coverage, (100 - substitutions) / 100) + self.assertGreaterEqual(result.coverage, 0.95) + self.assertFalse(result.coverage_ok) # Excellent still requires 99%. + self.assertTrue(result.distinct_entry_identity_ok) + if substitutions == 2: + self.assertEqual((result.exit_p90, result.pnl_p90), (0.0, 0.0)) + self.assertEqual(row["metric_observations"]["exit"], 100) + self.assertEqual(result.label, "strong") + self.assertEqual(cli["exit"], 0) + else: + # On the previous grader this is the first failing assertion: + # net zero wrongly enables the perfect schedule at 6% gross. + self.assertAlmostEqual( + result.exit_p90, 0.08, + msg="gross count at the 6% limit must block FIFO replacement", + ) + self.assertAlmostEqual(result.pnl_p90, 2 / 3) + self.assertEqual(row["metric_observations"]["exit"], 97) + self.assertEqual(row["metric_observations"]["pnl"], 97) + self.assertEqual(result.label, "moderate") + self.assertEqual(cli["exit"], 1) + results.append((result, substitutions)) + + for result, substitutions in results: + self.assertEqual(result.count_delta, 2 * substitutions / 100) + def test_true_fifo_requires_split_entry_and_cross_entry_exit(self) -> None: rows = [ trade(1, 100, 10.0, 200, 12.0, qty=0.5), @@ -926,11 +1044,25 @@ def test_zero_alignment_honors_declared_tiers_and_precedence(self) -> None: result = analyze_strategy(self.strategy(name, meta)) self.assertTrue(result.no_aligned_trades) self.assertEqual(result.label, expected) + self.assertEqual((result.tv_count, result.eng_count), (1, 1)) + self.assertEqual(result.count_abs_delta, 2) + self.assertEqual(result.net_count_abs_delta, 0) + self.assertFalse(result.count_ok) + self.assertEqual(result.report_row()["count_abs_delta"], 2) + # expectation corrected: 0 -> 2, because two unmatched trades + # are normalized by the larger one-trade pool, even unaligned. + self.assertEqual(result.count_delta, 2.0) + self.assertEqual(result.report_row()["count_delta"], 2.0) output = StringIO() with redirect_stdout(output): _print_verification(result) self.assertIn(f"-> {expected}", output.getvalue()) + self.assertRegex( + output.getvalue(), + r"Count delta:\s+200\.0000%\s+\(X; abs=2; net=0\)", + ) + self.assertIn("abs=2; net=0", output.getvalue()) def test_both_empty_excellent_is_not_masked_by_stale_declarations(self) -> None: strategy = self.strategy( @@ -947,6 +1079,9 @@ def test_both_empty_excellent_is_not_masked_by_stale_declarations(self) -> None: self.assertTrue(result.no_aligned_trades) self.assertEqual(result.label, "excellent") + self.assertEqual(result.count_abs_delta, 0) + self.assertEqual(result.count_delta, 0.0) + self.assertTrue(result.count_ok) def test_single_sided_empty_results_honor_valid_declarations(self) -> None: anomaly = analyze_strategy(self.strategy( @@ -966,6 +1101,11 @@ def test_single_sided_empty_results_honor_valid_declarations(self) -> None: (engine_only.tv_raw_count, engine_only.eng_raw_count), (1, 0) ) self.assertEqual(engine_only.label, "engine_only") + for result in (anomaly, engine_only): + self.assertEqual(result.count_abs_delta, 1) + self.assertEqual(result.net_count_abs_delta, 1) + self.assertEqual(result.count_delta, 1.0) + self.assertFalse(result.count_ok) def test_matched_excellent_preserves_malformed_override_failure(self) -> None: strategy = self.strategy( @@ -1039,6 +1179,7 @@ def test_single_strategy_cli_exit_matches_declared_zero_alignment(self) -> None: self.assertEqual(rc, expected_rc) self.assertIn(f"-> {expected_label}", output.getvalue()) + self.assertIn("abs=2; net=0", output.getvalue()) # TradingView's current trade-list header, verbatim. Only the header is real: @@ -1250,7 +1391,11 @@ def test_the_percent_never_reaches_a_gate_or_the_tier(self) -> None: if __name__ == "__main__": + parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) + parser.add_argument("--retain-artifacts", type=Path, metavar="DIR") + options, unittest_args = parser.parse_known_args() + FragmentedFifoEligibilityTests.retain_artifacts = options.retain_artifacts if not _coof_cascade_tape.exists(): print(PRIVATE_FIXTURE_SKIP_LITERAL + "tests/fixtures/coof_cascade_identity/w7-coof-cascade-identity") - unittest.main() + unittest.main(argv=[sys.argv[0], *unittest_args]) diff --git a/scripts/test_verify_corpus_open_mark.py b/scripts/test_verify_corpus_open_mark.py index 858324f41..8eecde2b2 100644 --- a/scripts/test_verify_corpus_open_mark.py +++ b/scripts/test_verify_corpus_open_mark.py @@ -105,6 +105,7 @@ def test_marked_lot_pairs_with_the_engines_mark(self) -> None: self.assertEqual((r.tv_gate_count, r.eng_gate_count, r.gating_matched_count), (2, 2, 2)) self.assertTrue(r.count_ok) self.assertEqual(r.count_abs_delta, 0) + self.assertEqual(r.net_count_abs_delta, 0) self.assertTrue(r.entry_ok) self.assertTrue(r.exit_ok) # the mark's 0.029% is not gated self.assertEqual(r.exit_p90, 0.0) @@ -215,6 +216,7 @@ def test_engine_mark_without_a_tv_open_row_is_an_ordinary_engine_row(self) -> No self.assertEqual(r.matched_count, 2) self.assertEqual((r.tv_gate_count, r.eng_gate_count), (2, 3)) self.assertEqual(r.count_abs_delta, 1) + self.assertEqual(r.net_count_abs_delta, 1) self.assertFalse(r.count_ok) def test_a_tape_of_one_open_lot_is_one_matched_trade(self) -> None: @@ -230,6 +232,8 @@ def test_a_tape_of_one_open_lot_is_one_matched_trade(self) -> None: self.assertFalse(r.no_aligned_trades) self.assertEqual(r.open_mark_pairs, 1) self.assertEqual(r.matched_count, 1) + self.assertEqual(r.count_abs_delta, 0) + self.assertTrue(r.count_ok) self.assertEqual(r.coverage, 1.0) self.assertEqual(r.label, "excellent") diff --git a/scripts/verify_corpus.py b/scripts/verify_corpus.py index 0da45cffc..8098ca7e5 100755 --- a/scripts/verify_corpus.py +++ b/scripts/verify_corpus.py @@ -326,8 +326,10 @@ class VerificationResult: gating_matched_count: int = 0 tv_gate_count: int = 0 eng_gate_count: int = 0 - count_delta: float = 0.0 - count_abs_delta: int = 0 + count_delta: float = 0.0 # Gross unmatched count / larger gated pool size. + count_abs_delta: int = 0 # Unmatched TV trades plus unmatched engine trades. + net_count_abs_delta: int = 0 # Report-only absolute difference of pool sizes. + gross_unmatched_count: int = 0 # Alias of the canonical count_abs_delta. entry_p90: float = 0.0 exit_p90: float = 0.0 pnl_p90: float = 0.0 @@ -398,6 +400,8 @@ def report_row(self) -> dict: "matched": self.matched_count, "count_delta": self.count_delta, "count_abs_delta": self.count_abs_delta, + "net_count_abs_delta": self.net_count_abs_delta, + "gross_unmatched_count": self.gross_unmatched_count, "entry_p90": self.entry_p90, "exit_p90": self.exit_p90, "pnl_p90": self.pnl_p90, @@ -1222,7 +1226,9 @@ def analyze_strategy(strategy_dir: Path) -> VerificationResult: matched = align_by_time(tv_cmp, eng_cmp) if not matched and not mark_pairs: - label = "excellent" if len(tv_cmp) == 0 and len(eng_cmp) == 0 else "minimal" + count_abs_delta = len(tv_cmp) + len(eng_cmp) + count_ok = count_abs_delta == 0 + label = "excellent" if count_ok else "minimal" # The historical both-empty Excellent branch returned before reading # override metadata. Keep that exact behavior while fixing declared # divergence for the non-excellent zero-alignment branch. @@ -1241,9 +1247,11 @@ def analyze_strategy(strategy_dir: Path) -> VerificationResult: eng_count=len(eng_cmp), tv_raw_count=len(tv), eng_raw_count=len(eng), - count_delta=relative_max(len(tv_cmp), len(eng_cmp)), - count_abs_delta=abs(len(tv_cmp) - len(eng_cmp)), - count_ok=len(tv_cmp) == len(eng_cmp), + count_delta=count_abs_delta / max(len(tv_cmp), len(eng_cmp), 1), + count_abs_delta=count_abs_delta, + net_count_abs_delta=abs(len(tv_cmp) - len(eng_cmp)), + gross_unmatched_count=count_abs_delta, + count_ok=count_ok, ) # Resolve parity profile (strict vs production) per probe. @@ -1259,6 +1267,7 @@ def analyze_strategy(strategy_dir: Path) -> VerificationResult: "scope": "per_probe", "comparison": "<", "count_rule": "exact", + "count_basis": "unmatched_both_sides", } grade_percentile = grading_policy["percentile"] @@ -1292,8 +1301,19 @@ def _split_interior(pairs: list[tuple[TradePair, TradePair]]) -> list[tuple[Trad eng_gate_n = len(eng_gate) + n_gate_marks gating_matched_n = len(gating_matched) + n_gate_marks - count_abs_delta = abs(tv_gate_n - eng_gate_n) - count_delta = relative_max(tv_gate_n, eng_gate_n) + net_count_abs_delta = abs(tv_gate_n - eng_gate_n) + # Each unmatched gated trade counts, even when the two list sizes agree. + # Count only pairs with both rows in the gated pools: metric fallback can + # include matches outside the interior when no interior pair exists. + tv_gate_ids = {id(t) for t in tv_gate} + eng_gate_ids = {id(e) for e in eng_gate} + count_matched_n = n_gate_marks + sum( + id(t) in tv_gate_ids and id(e) in eng_gate_ids for t, e in matched + ) + unmatched_tv = tv_gate_n - count_matched_n + unmatched_engine = eng_gate_n - count_matched_n + count_abs_delta = unmatched_tv + unmatched_engine + count_delta = count_abs_delta / max(tv_gate_n, eng_gate_n, 1) # A mark pair's entry is a fill on both sides, so the identity gate # sees it with the others: the engine's range-end row must carry an # entry incarnation at a TV-proven multi-entry key exactly as any @@ -1355,10 +1375,11 @@ def _split_interior(pairs: list[tuple[TradePair, TradePair]]) -> list[tuple[Trad exit_p90 = percentile(exit_deltas, grade_percentile) pnl_p90 = percentile(pnl_deltas, grade_percentile) if pnl_deltas else 0.0 - # Fragmented FIFO grids can have exact entry/count parity and exact - # position-level close events while entry-grouped consolidation smears exit - # prices across different FIFO lot boundaries. Under a strict guard, score - # exit/PnL on the raw exit schedule instead of the consolidated deal blend. + # Fragmented FIFO grids can have exact entry and position-level close + # events while entry-grouped consolidation smears exit prices across + # different FIFO lot boundaries. Below the same gross-count percentage + # limit as strong grading, score exit/PnL on the raw exit schedule instead + # of the consolidated deal blend. tv_sched_rows, eng_sched_rows = schedule_rows_for_gate( tv_raw_all, eng_raw_all, tv_gate, eng_gate, gating_matched ) @@ -1412,9 +1433,10 @@ def _split_interior(pairs: list[tuple[TradePair, TradePair]]) -> list[tuple[Trad entry_ok = entry_p90 < thresh["entry"] # Count mismatch is a primary reproduction signal. Percent thresholds can # hide dozens of missing/extra trades in large strategies, so excellent - # requires exact gated TV/engine count parity. Any non-zero absolute count - # mismatch is an engine/codegen gap unless separately proven as a TV/export - # anomaly. + # requires zero unmatched gated trades on either side. An engine-only and + # a TV-only trade are two mismatches, even if their net count is zero. + # Any mismatch is an engine/codegen gap unless separately proven as a + # TV/export anomaly. count_ok = count_abs_delta == 0 # HARD GATES ONLY. The former heuristic escape hatches (tiny_exit_pnl_noise, # strong_exit_pnl_coupling, pnl_validated_exit_noise, and the mae_* waivers) @@ -1532,6 +1554,8 @@ def _is_dropped_open(t): eng_gate_count=eng_gate_n, count_delta=count_delta, count_abs_delta=count_abs_delta, + net_count_abs_delta=net_count_abs_delta, + gross_unmatched_count=count_abs_delta, entry_p90=entry_p90, exit_p90=exit_p90, pnl_p90=pnl_p90, @@ -1577,14 +1601,19 @@ def _print_verification(result: VerificationResult, *, show_diffs: int = 0) -> N f" MISSING (tv: {result.tv_exists}, engine: {result.eng_exists})" ) return + check = lambda b: "OK" if b else "X" + count_line = ( + f" Count delta: {result.count_delta * 100:8.4f}% " + f"({check(result.count_ok)}; abs={result.count_abs_delta}; " + f"net={result.net_count_abs_delta})\n" + ) if result.no_aligned_trades: print( f"{result.rel}: TV={result.tv_raw_count} engine={result.eng_raw_count} " - f"matched=0 (no aligned trades)\n -> {result.label}" + f"matched=0 (no aligned trades)\n{count_line} -> {result.label}" ) return - check = lambda b: "OK" if b else "X" match_pct = 100.0 * result.matched_count / max(result.tv_raw_count, 1) interior_line = "" if result.bounds is not None: @@ -1600,7 +1629,7 @@ def _print_verification(result: VerificationResult, *, show_diffs: int = 0) -> N f" Engine trades: {result.eng_count} (raw {result.eng_raw_count})\n" f" Matched: {result.matched_count} ({match_pct:.1f}% of TV)\n" f"{interior_line}" - f" Count delta: {result.count_delta * 100:8.4f}% ({check(result.count_ok)}; abs={result.count_abs_delta})\n" + f"{count_line}" f" Entry-price p90 delta: {result.entry_p90 * 100:8.4f}% ({check(result.entry_ok)})\n" f" Exit-price p90 delta: {result.exit_p90 * 100:8.4f}% ({check(result.exit_ok)})\n" f" PnL p90 delta: {result.pnl_p90 * 100:8.4f}% ({check(result.pnl_ok)})\n" @@ -1649,6 +1678,14 @@ def verify_one(strategy_dir: Path, *, verbose: bool = True, show_diffs: int = 0) return result.label +# 2026-11-16 (#836/#883): allow only this measured Moderate state through the +# aggregate reference floor; its grade and trades remain unchanged. The exception +# ends when #883 closes: https://github.com/pineforge-4pass/pineforge-engine/issues/883. +REFERENCE_FLOOR_EXCEPTIONS = { + "validation/mtf-triple-tf-macd-hist-confluence-01": "moderate", +} + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("strategy_dir", nargs="?", help="Single strategy folder, e.g. corpus/basic/greedy") @@ -1723,8 +1760,12 @@ def main() -> int: counts[label] = counts.get(label, 0) + 1 # `anomaly` and `engine_only` are documented success outcomes # (declared via inputs.json::expected_tier) — not failures. - if label not in {"excellent", "strong", "anomaly", "engine_only"}: - n_fail.append(f"{cat}/{strat.name}") + strategy_id = f"{cat}/{strat.name}" + if ( + label not in {"excellent", "strong", "anomaly", "engine_only"} + and label != REFERENCE_FLOOR_EXCEPTIONS.get(strategy_id) + ): + n_fail.append(strategy_id) if n_total == 0: print( f"error: refused corpus root {corpus_root}: " diff --git a/src/source/pine_adapter.cpp b/src/source/pine_adapter.cpp index d08b38d9f..4b7af657e 100644 --- a/src/source/pine_adapter.cpp +++ b/src/source/pine_adapter.cpp @@ -3810,11 +3810,15 @@ void PineExecutionAdapter::remember(const native_order::RequestHandle& handle, // A re-issue made once its entry exists does not continue a void call's // chain: its place in the exit queue is its own (exit_queue_rank). if (snapshot.projection_predecessor == 0) { - snapshot.chain_origin_sequence = snapshot.source_sequence; + // A deferred exit already holds its source queue slot. + if (snapshot.chain_origin_sequence == 0) + snapshot.chain_origin_sequence = snapshot.source_sequence; } else if (placement_.contains(snapshot.projection_predecessor)) { const auto& predecessor = placement_.at(snapshot.projection_predecessor); snapshot.chain_origin_sequence = predecessor.void_issue && !snapshot.void_issue - ? snapshot.source_sequence : predecessor.chain_origin_sequence; + ? (snapshot.chain_origin_sequence != 0 + ? snapshot.chain_origin_sequence : snapshot.source_sequence) + : predecessor.chain_origin_sequence; } else if (snapshot.chain_origin_sequence == 0) { snapshot.chain_origin_sequence = snapshot.source_sequence; } @@ -5767,7 +5771,8 @@ double PineExecutionAdapter::quantize_percent_exit_units( bool PineExecutionAdapter::compute_exit_reservation( const SourceId& exit_id, const SourceId& from_entry, double requested_qty, double& qty_percent, double live_basis, - double percentage_basis, double& reserved_qty) const { + double percentage_basis, double& reserved_qty, + std::uint64_t& reservation_sequence, std::uint64_t origin) const { qty_percent = std::isfinite(qty_percent) ? std::clamp(qty_percent, 0.0, 100.0) : 100.0; reserved_qty = kNaN; @@ -5783,8 +5788,8 @@ bool PineExecutionAdapter::compute_exit_reservation( std::uint64_t family = 0; double units = kNaN; double percent = 100.0; - bool explicit_units = false; - std::vector origins; + double legacy_units = 0.0; + std::vector> origin_shares; // The family's place in the exit queue (exit_queue_rank); a // strategy.close reservation has none and always holds its units. std::uint64_t rank = std::numeric_limits::max(); @@ -5798,6 +5803,8 @@ bool PineExecutionAdapter::compute_exit_reservation( const bool exit = queued_exit || snapshot.family == PineOrderFamily::Close; // A call TradingView voided reserves nothing (void_issue). if (!exit || snapshot.from_entry != from_entry || snapshot.void_issue) return; + if (origin != 0 && snapshot.bracket_origin.incarnation != 0 + && snapshot.bracket_origin.incarnation != origin) return; // ab9714be pine_orders.cpp:599-602: close reservations only apply to positions matching active position_cycle_seq_ if (snapshot.family == PineOrderFamily::Close && snapshot.placement_cycle != 0 @@ -5817,8 +5824,13 @@ bool PineExecutionAdapter::compute_exit_reservation( // src/source/pine_fills.cpp:7094 both reserve a sibling leg's // share as ``live_pos * (oqp / 100.0)``, percent divided first. : live_basis * (percent / 100.0)); - const bool explicit_units = std::isfinite(snapshot.requested_qty); - const auto origin = snapshot.bracket_origin.incarnation; + const auto bound_origin = snapshot.bracket_origin.incarnation; + // A parent-fill stamp alone does not make a row an independent + // share. Legacy percentages/full exits project the whole family's + // reservation on each leg; explicit units and per-entry quantities + // actually own one share for each bound opening. + const bool origin_share = std::isfinite(snapshot.requested_qty) + || snapshot.per_entry_origin != 0; const auto rank = queued_exit ? exit_queue_rank(placement_, snapshot) : std::numeric_limits::max(); if (row == reservations.end()) { @@ -5826,21 +5838,24 @@ bool PineExecutionAdapter::compute_exit_reservation( next.family = family; next.units = units; next.percent = percent; - next.explicit_units = explicit_units; - if (explicit_units) next.origins.push_back(origin); + if (origin_share) next.origin_shares.emplace_back(bound_origin, units); + else next.legacy_units = units; next.rank = rank; next.queued_exit = queued_exit; reservations.push_back(std::move(next)); } else { - if (explicit_units && row->explicit_units - && std::find(row->origins.begin(), row->origins.end(), origin) - == row->origins.end()) { - row->units += units; - row->origins.push_back(origin); + if (origin_share) { + const auto share = std::find_if(row->origin_shares.begin(), row->origin_shares.end(), + [&](const auto& value) { return value.first == bound_origin; }); + if (share == row->origin_shares.end()) + row->origin_shares.emplace_back(bound_origin, units); + else share->second = std::max(share->second, units); } else { - row->units = std::max(row->units, units); + row->legacy_units = std::max(row->legacy_units, units); } - row->explicit_units = row->explicit_units || explicit_units; + double shares = 0.0; + for (const auto& share : row->origin_shares) shares += share.second; + row->units = std::max(row->legacy_units, shares); row->percent = std::max(row->percent, percent); row->rank = std::min(row->rank, rank); row->queued_exit = row->queued_exit && queued_exit; @@ -5853,15 +5868,36 @@ bool PineExecutionAdapter::compute_exit_reservation( for (const auto& pending : pending_bracket_legs_) observe(pending.snapshot); for (const auto& pending : pending_coof_requests_) observe(pending.snapshot); for (const auto& delayed : delayed_market_orders_) observe(delayed.snapshot); + // An unresolved level still owns a place and a share in the source queue. + // Its definition survives the transient shadow projection and becomes the + // same reservation when the parent's fill supplies the level. + for (const auto& pending : pending_relative_exits_) { + if (pending.from_entry != from_entry || pending.void_global) continue; + const auto family = key_for(pending.exit_id, pending.from_entry); + if (std::any_of(reservations.begin(), reservations.end(), + [&](const Reservation& row) { return row.family == family; })) continue; + PlacementSnapshot snapshot; + snapshot.family = PineOrderFamily::ExitLimit; + snapshot.source_id = pending.exit_id; + snapshot.from_entry = pending.from_entry; + snapshot.requested_qty = pending.qty; + snapshot.qty_percent = pending.qty_percent; + snapshot.chain_origin_sequence = pending.reservation_sequence; + observe(snapshot); + } const auto this_family = key_for(exit_id, from_entry); // A re-issued exit keeps its place in the queue (exit_queue_rank): an exit // created after it holds nothing ahead of it, however it was re-issued // since. A new exit is at the back and yields to every one already there. - std::uint64_t this_rank = std::numeric_limits::max(); + std::uint64_t this_rank = reservation_sequence != 0 + ? reservation_sequence : std::numeric_limits::max(); for (const auto& reservation : reservations) { - if (reservation.family == this_family) this_rank = reservation.rank; + if (reservation.family == this_family) + this_rank = std::min(this_rank, reservation.rank); } + if (this_rank != std::numeric_limits::max()) + reservation_sequence = this_rank; double already_reserved = 0.0; double preserved_reserved = kNaN; bool other_full_exit = false; @@ -5909,6 +5945,7 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( struct Family { std::uint64_t key = 0; + std::uint64_t origin = 0; std::uint64_t command_sequence = 0; std::uint64_t rank = std::numeric_limits::max(); double percent = 100.0; @@ -5929,11 +5966,13 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( || snapshot.family == PineOrderFamily::ExitTrail; if (!exit || snapshot.from_entry != from_entry || snapshot.void_issue) continue; const auto key = key_for(snapshot.source_id, snapshot.from_entry); + const auto origin = from_entry.empty() ? 0 : snapshot.per_entry_origin; auto family = std::find_if(families.begin(), families.end(), - [&](const Family& value) { return value.key == key; }); + [&](const Family& value) { return value.key == key && value.origin == origin; }); if (family == families.end()) { Family next; next.key = key; + next.origin = origin; next.command_sequence = snapshot.command_sequence; next.rank = exit_queue_rank(placement_, snapshot); next.percent = std::isfinite(snapshot.qty_percent) @@ -5968,11 +6007,13 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( || snapshot.family == PineOrderFamily::ExitTrail; if (!exit || snapshot.from_entry != from_entry || snapshot.void_issue) continue; const auto key = key_for(snapshot.source_id, snapshot.from_entry); + const auto origin = from_entry.empty() ? 0 : snapshot.per_entry_origin; auto family = std::find_if(families.begin(), families.end(), - [&](const Family& value) { return value.key == key; }); + [&](const Family& value) { return value.key == key && value.origin == origin; }); if (family == families.end()) { Family next; next.key = key; + next.origin = origin; next.command_sequence = snapshot.command_sequence; next.rank = exit_queue_rank(placement_, snapshot); next.percent = std::isfinite(snapshot.qty_percent) @@ -6007,11 +6048,13 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( || snapshot.family == PineOrderFamily::ExitTrail; if (!exit || snapshot.from_entry != from_entry || snapshot.void_issue) continue; const auto key = key_for(snapshot.source_id, snapshot.from_entry); + const auto origin = from_entry.empty() ? 0 : snapshot.per_entry_origin; auto family = std::find_if(families.begin(), families.end(), - [&](const Family& value) { return value.key == key; }); + [&](const Family& value) { return value.key == key && value.origin == origin; }); if (family == families.end()) { Family next; next.key = key; + next.origin = origin; next.command_sequence = snapshot.command_sequence; next.rank = exit_queue_rank(placement_, snapshot); next.percent = std::isfinite(snapshot.qty_percent) @@ -6048,9 +6091,22 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( }); double reserved = 0.0; + std::map reserved_by_origin; std::vector cancel; for (const auto& family : families) { - const double available = std::max(0.0, live_basis - reserved); + double basis = live_basis; + if (family.origin != 0) { + const auto cohort = cohorts_by_id_.find(from_entry); + if (cohort == cohorts_by_id_.end()) continue; + const auto live = cohort->second.live_units_by_origin.find(family.origin); + // A later pending entry has not supplied its sizing basis yet. + if (live == cohort->second.live_units_by_origin.end()) continue; + basis = live->second; + } + const double available = family.origin != 0 + ? std::max(0.0, std::min(live_basis - reserved, + basis - reserved_by_origin[family.origin])) + : std::max(0.0, live_basis - reserved); double units = 0.0; if (std::isfinite(family.explicit_requested)) { // An explicit strategy.exit quantity belongs to each bound entry @@ -6070,7 +6126,7 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( // that never flattens (ab9714be pine_fills.cpp:1618 spells the // same whole-position coverage as `qty - kQtyEpsilon`). double requested = family.percent >= 100.0 - internal::kFullPercentEps - ? live_basis : live_basis * family.percent / 100.0; + ? basis : basis * family.percent / 100.0; if (family.percent < 100.0 - internal::kFullPercentEps) { requested = quantize_percent_exit_units(requested, available); } @@ -6088,11 +6144,12 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( } continue; } - const double normalized_percent = units / live_basis * 100.0; + const double normalized_percent = units / basis * 100.0; for (const auto& handle : family.handles) { const auto found = placement_.find(handle.incarnation); if (found == placement_.end()) continue; found->second.projection_remaining_qty = units; + if (family.origin != 0) found->second.per_entry_units = units; found->second.qty_percent = normalized_percent; found->second.fixed_exit_reservation = std::isfinite(family.explicit_requested) @@ -6104,6 +6161,7 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( if (index >= pending_bracket_legs_.size()) continue; auto& snapshot = pending_bracket_legs_[index].snapshot; snapshot.projection_remaining_qty = units; + if (family.origin != 0) snapshot.per_entry_units = units; snapshot.qty_percent = normalized_percent; snapshot.fixed_exit_reservation = std::isfinite(family.explicit_requested) @@ -6115,6 +6173,7 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( if (index >= delayed_market_orders_.size()) continue; auto& snapshot = delayed_market_orders_[index].snapshot; snapshot.projection_remaining_qty = units; + if (family.origin != 0) snapshot.per_entry_units = units; snapshot.qty_percent = normalized_percent; snapshot.fixed_exit_reservation = std::isfinite(family.explicit_requested) @@ -6129,6 +6188,7 @@ void PineExecutionAdapter::reconcile_deferred_exit_reservations( reserved += std::isfinite(family.explicit_requested) ? std::min(available, units * static_cast(multiplicity)) : units; + if (family.origin != 0) reserved_by_origin[family.origin] += units; } for (const auto& handle : cancel) { const auto result = require_host().cancel(handle); @@ -11722,7 +11782,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en double trail_offset, double trail_price, double qty_percent, const std::string& comment, double qty, const std::string& oca_name, double profit_ticks, - double loss_ticks) { + double loss_ticks, std::uint64_t reservation_sequence) { const double requested_qty_percent = qty_percent; if (intraday_loss_orders_blocked()) return; if (const auto point = detail::callback_point(require_host()); @@ -11741,6 +11801,10 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en const double floored = explicit_entry_quantity_floor(qty, staged_.quantity_grid); if (floored > 0.0 || !(qty_percent < 100.0)) qty = floored > 0.0 ? floored : kNaN; } + const bool percent_by_origin = config_.pyramiding > 1 + && !from_entry.empty() && std::isnan(qty) + && finite_positive(qty_percent) + && qty_percent < 100.0 - internal::kFullPercentEps; if (!from_entry.empty()) { const auto token = named_entry_cancel_tokens_.find(from_entry); if (token != named_entry_cancel_tokens_.end()) { @@ -11842,6 +11906,15 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en "Pine source command sequence exhausted"); } const std::uint64_t command_sequence = ++source_command_sequence_; + const auto capture_reservation_sequence = [&] { + if (reservation_sequence != 0) return; + if (source_sequence_ >= static_cast( + std::numeric_limits::max())) { + throw coded(RunFailureCode::engine_invariant, {}, + "Pine exit queue sequence exhausted"); + } + reservation_sequence = ++source_sequence_; + }; // Trail point and offset operands are source tick counts, whereas the // generic native Trail carries prices. Preserve the source operands in // the placement projection and lower only the executable request here. @@ -11869,12 +11942,13 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en reissue_percent = std::min(qty, live_exit_units) / live_exit_units * 100.0; const bool partial_reissue = reissue_percent < 100.0 - internal::kFullPercentEps; if (const auto consumed = consumed_partial_exit_cycles_.lookup(partial_exit_key); - consumed && partial_reissue + consumed && partial_reissue && !percent_by_origin && live_exit_units > internal::kQtyEpsilon && *consumed == current_position_cycle_) { return; } - if (partial_reissue && exit_tombstoned(exit_id, from_entry)) return; + if (partial_reissue && !percent_by_origin && exit_tombstoned(exit_id, from_entry)) + return; // A held global exit with relative legs over two or more entries covers // them entry by entry (per_entry_exit_call). if (from_entry.empty() @@ -12052,8 +12126,12 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en return value.exit_id == exit_id && value.from_entry == from_entry; }); if (existing == pending_relative_exits_.end()) { + capture_reservation_sequence(); + pending.reservation_sequence = reservation_sequence; pending_relative_exits_.push_back(std::move(pending)); } else { + reservation_sequence = existing->reservation_sequence; + pending.reservation_sequence = reservation_sequence; const auto same = [](double left, double right) { return same_double_bits(left, right) || (std::isnan(left) && std::isnan(right)); }; @@ -12099,7 +12177,8 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en shadow.requested_qty = qty; shadow.qty_percent = qty_percent; shadow.command_sequence = command_sequence; - shadow.source_sequence = ++source_sequence_; + shadow.source_sequence = reservation_sequence; + shadow.chain_origin_sequence = reservation_sequence; shadow.projection_position_side = static_cast(PositionSide::FLAT); shadow.exit_levels = {limit_price, stop_price, source_trail_points, source_trail_offset, source_trail_price, @@ -12277,6 +12356,11 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en if (finite_positive(units)) pending_parent_units += units; }; for (const auto& handle : live_handles_) { + // Relative exits materialize inside their parent's applied callback, + // before that opening handle retires. Its units already belong to + // the physical book and source cohort, so adding them as pending + // again doubles a same-bar percentage's basis. + if (handle == materializing_parent_) continue; const auto parent = placement_.find(handle.incarnation); if (parent != placement_.end()) observe_pending_parent(parent->second); } @@ -12366,7 +12450,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en ? book_basis + completed_entry_units + pending_parent_units : live_reservation_basis; const bool reservation_ok = compute_exit_reservation(exit_id, from_entry, qty, qty_percent, - live_reservation_basis, percentage_basis, reserved_exit_qty); + live_reservation_basis, percentage_basis, reserved_exit_qty, reservation_sequence); if (!reservation_ok) { // clear_existing_exit_order ran before sizing on the legacy route: // a zero-capacity reissue removes its predecessor as well as refusing @@ -12375,6 +12459,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en exit_cancel_bracket(exit_id, from_entry, comment); return; } + capture_reservation_sequence(); const auto source_point = detail::callback_point(require_host()); const OrderBirth exit_birth = capture_order_birth(); const auto exit_birth_reach = compat::pine::select_historical_birth_reach( @@ -12675,6 +12760,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en snapshot.pending_bound_exit = pending_bound_exit; snapshot.is_long = false; snapshot.command_sequence = command_sequence; + snapshot.chain_origin_sequence = reservation_sequence; snapshot.bracket_origin = std::move(bracket_origin); snapshot.exit_levels = {limit_price, stop_price, source_trail_points, source_trail_offset, source_trail_price, @@ -12699,6 +12785,76 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en source_point->decision.script_bar_open_ms; snapshot.placement_sub_open_ms = source_point->decision.sub_bar_open_ms; } + if (percent_by_origin) { + // Percentages, like explicit units, belong to each source + // entry instance. Never divide a cohort reservation equally: + // entries of one id can have different sizes. + const auto origin = snapshot.bracket_origin.incarnation; + const auto cohort = cohorts_by_id_.find(from_entry); + double basis = origin == 0 ? live_reservation_basis : 0.0; + if (origin != 0 && cohort != cohorts_by_id_.end()) { + const auto live = cohort->second.live_units_by_origin.find(origin); + if (live != cohort->second.live_units_by_origin.end()) basis = live->second; + if (origin_is_pending(snapshot.bracket_origin)) { + const auto parent = placement_.find(origin); + if (parent != placement_.end()) { + if (parent->second.projection_over_pyramiding) return; + basis = finite_positive(parent->second.requested_qty) + ? parent->second.requested_qty + : parent->second.frozen_market_own_units; + } + } + } + double completed = 0.0; + if (completed_entry_units > 0.0 && basis > 0.0 + && cohort != cohorts_by_id_.end() + && cohort->second.live_units_by_origin.count(origin) != 0) { + const auto& host = require_host(); + const auto* pine = pine_view_of(host_); + completed = detail::replay_position_tail( + host.closed_trade_count(), pine ? pine->trades_.base() : 0, + [&](std::size_t index) -> Trade { + return host.get_trade(static_cast(index)); + }, position_tail_spill_.get(), position_open_script_bar_, + [&](bool is_long, std::uint64_t incarnation, bool margin_call, + const auto&) { + return incarnation == origin + && is_long == (physical.signed_units > 0.0) + && (ancestry_sized_target || margin_call); + }, from_entry, position_tail_summary_->work, + PINEFORGE_POSITION_TAIL_DIAGNOSTICS{}); + } + // A same-bar close owns held units before these exits. + // Pending parents supply their own capacity, including an + // unsized parent whose share resolves only when it opens. + const bool pending_origin = origin_is_pending(snapshot.bracket_origin) + || (origin == 0 && entry_order_pending(from_entry)); + const double own_basis = ancestry_sized_target ? basis : basis + completed; + const double own_live_basis = pending_origin ? own_basis + : std::min(own_basis, std::max( + 0.0, live_reservation_basis - pending_parent_units)); + if (!pending_origin && !(own_live_basis > internal::kQtyEpsilon)) return; + const double own_percentage_basis = ancestry_sized_target + ? basis + completed : own_live_basis; + double percent = requested_qty_percent; + double units = kNaN; + auto rank = reservation_sequence; + if (!compute_exit_reservation(exit_id, from_entry, kNaN, percent, + own_live_basis, own_percentage_basis, units, rank, origin)) return; + reservation_sequence = rank; + snapshot.chain_origin_sequence = rank; + snapshot.qty_percent = percent; + snapshot.projection_remaining_qty = units; + snapshot.fixed_exit_reservation = true; + snapshot.per_entry_origin = origin; + snapshot.per_entry_units = units; + if (origin != 0 && config_.close_entries_rule_any + && cohort != cohorts_by_id_.end() + && cohort->second.live_units_by_origin.count(origin) != 0) { + request.owner = native_order::BindOpening{ + snapshot.bracket_origin, cohort->second.cycle}; + } + } initialize_l4c_policy(snapshot, {}); if (finite_positive(coof_limit_waypoint_price)) snapshot.forced_execution_price = coof_limit_waypoint_price; @@ -12737,7 +12893,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en } } const double source_position = std::abs(detail::run_position(require_host()).signed_units); - if (std::isfinite(reserved_exit_qty)) { + if (!percent_by_origin && std::isfinite(reserved_exit_qty)) { snapshot.projection_remaining_qty = reserved_exit_qty; const bool foreign_pending_opening = std::any_of( live_handles_.begin(), live_handles_.end(), [&](const auto& handle) { @@ -12761,7 +12917,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en || (source_position > 0.0 && reserved_exit_qty < source_position - 1e-9) || foreign_pending_opening)); - } else if (!binds_pending_reversal_entry && host_sized + } else if (!percent_by_origin && !binds_pending_reversal_entry && host_sized && !std::isfinite(snapshot.requested_qty) && source_position > 0.0) { const double percent = std::isfinite(snapshot.qty_percent) @@ -13090,7 +13246,7 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en } }; - if (dynamic) { + if (dynamic && !percent_by_origin) { native_order::RequestHandle pending_origin{}; bool defer_until_parent = false; if (!from_entry.empty()) { @@ -13171,7 +13327,8 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en + std::to_string(static_cast(family)) + "\x1f" + origin_key; const auto group_name = oca_name.empty() ? exit_id + "\x1f" + from_entry + "\x1f" + origin_key - : oca_name + (origin.incarnation != 0 ? "\x1f" + origin_key : ""); + : (percent_by_origin ? oca_name + : oca_name + (origin.incarnation != 0 ? "\x1f" + origin_key : "")); const bool has_live_leg = live_by_source_key_.find(checked_live_key(replacement_key)) != live_by_source_key_.end(); const bool origin_opened = found != cohorts_by_id_.end() @@ -13196,14 +13353,15 @@ void PineExecutionAdapter::exit(const SourceId& exit_id, const SourceId& from_en && (!origin_opened || consumed_origin_leg())) { continue; } - native_order::Owner owner = !void_issue && !config_.close_entries_rule_any - && origin_opened - ? native_order::Owner{native_order::Independent{}} + native_order::Owner owner = !void_issue && origin_opened + ? (config_.close_entries_rule_any + ? native_order::Owner{native_order::BindOpening{origin, found->second.cycle}} + : native_order::Owner{native_order::Independent{}}) : native_order::Owner{native_order::BindCohort{cohort}}; // ab9714be pine_orders.cpp:483-488: cancel_oca_group scopes cancellation strictly to matching oca_name group submit_one(std::move(owner), true, replacement_key, group_for(group_name, 1, static_cast(family)), - !has_live_leg, origin); + !has_live_leg && !(percent_by_origin && materializing_relative_), origin); } }; const bool exit_is_buy = !parent_long; @@ -13827,8 +13985,12 @@ void PineExecutionAdapter::flush_pending_bracket_legs( // exactly as exit() binds a leg issued after a live pending parent: // it waits for the parent's opening instead of pre-arming on the // signal bar. - if (leg.snapshot.bracket_origin.incarnation == 0 && config_.pyramiding == 2 - && std::isfinite(leg.snapshot.requested_qty) + const bool named_percent_leg = !leg.snapshot.from_entry.empty() + && !std::isfinite(leg.snapshot.requested_qty) + && leg.snapshot.fixed_exit_reservation + && leg.snapshot.qty_percent < 100.0 - internal::kFullPercentEps; + if (leg.snapshot.bracket_origin.incarnation == 0 + && (std::isfinite(leg.snapshot.requested_qty) || named_percent_leg) && !leg.snapshot.from_entry.empty()) { const auto cohort = cohorts_by_id_.find(leg.snapshot.from_entry); if (cohort != cohorts_by_id_.end() && cohort->second.opened.empty()) { @@ -13848,12 +14010,17 @@ void PineExecutionAdapter::flush_pending_bracket_legs( const std::string family_tag = std::to_string( static_cast(leg.snapshot.family)); leg.snapshot.bracket_origin = origin; + if (named_percent_leg) { + leg.snapshot.per_entry_origin = origin.incarnation; + leg.snapshot.per_entry_units = leg.snapshot.projection_remaining_qty; + } leg.replacement_key = leg.snapshot.source_id + "\x1f" + leg.snapshot.from_entry + "\x1f" + family_tag + "\x1f" + origin_key; const auto group_name = leg.snapshot.oca_name.empty() ? leg.snapshot.source_id + "\x1f" + leg.snapshot.from_entry + "\x1f" + origin_key - : leg.snapshot.oca_name + "\x1f" + origin_key; + : (named_percent_leg ? leg.snapshot.oca_name + : leg.snapshot.oca_name + "\x1f" + origin_key); leg.request.group = group_for(group_name, 1, static_cast(leg.snapshot.family)); break; @@ -13926,6 +14093,19 @@ void PineExecutionAdapter::flush_pending_bracket_legs( pending_write(pending_bracket_legs_).push_back(std::move(leg)); continue; } + if ((leg.snapshot.per_entry_origin != 0 + || (config_.close_entries_rule_any && std::isfinite(leg.snapshot.requested_qty))) + && leg.snapshot.bracket_origin.incarnation != 0 && !leg.snapshot.from_entry.empty()) { + const auto cohort = cohorts_by_id_.find(leg.snapshot.from_entry); + if (cohort != cohorts_by_id_.end() + && cohort->second.live_units_by_origin.count( + leg.snapshot.bracket_origin.incarnation) != 0) { + leg.request.owner = config_.close_entries_rule_any + ? native_order::Owner{native_order::BindOpening{ + leg.snapshot.bracket_origin, cohort->second.cycle}} + : native_order::Owner{native_order::Independent{}}; + } + } bool execute_after_calculation = false; // ab9714be src/source/pine_fills.cpp:7695-7728 evaluates a priced exit on // the bar its parent entry fills, and a leg this route parked until @@ -14109,6 +14289,14 @@ void PineExecutionAdapter::materialize_pending_bracket_legs( && !config_.close_entries_rule_any) { leg.request.owner = owner_for_close(leg.snapshot.from_entry, false); } + if ((leg.snapshot.per_entry_origin != 0 + || (config_.close_entries_rule_any && std::isfinite(leg.snapshot.requested_qty))) + && leg.snapshot.bracket_origin.incarnation != 0 && !leg.snapshot.from_entry.empty()) { + leg.request.owner = config_.close_entries_rule_any + ? native_order::Owner{native_order::BindOpening{ + leg.snapshot.bracket_origin, event.cycle_after}} + : native_order::Owner{native_order::Independent{}}; + } const auto accepted = submit_or_replace(std::move(leg.request), std::move(leg.snapshot), false, leg.replacement_key); if (accepted) bracket_family(leg.family_key).push_back(*accepted); @@ -14809,7 +14997,8 @@ void PineExecutionAdapter::materialize_relative_exits( exit(value.exit_id, value.from_entry, limit, stop, value.trail_points, offset, value.trail_price, value.qty_percent, value.comment, value.qty, value.oca_name, value.from_entry.empty() ? value.profit_ticks : kNaN, - value.from_entry.empty() ? value.loss_ticks : kNaN); + value.from_entry.empty() ? value.loss_ticks : kNaN, + value.reservation_sequence); } catch (...) { materializing_relative_ = false; materializing_parent_ = {}; @@ -16510,7 +16699,7 @@ native_order::ExecutionTerms PineExecutionAdapter::resolve_source_terms( } // One entry's bracket of a per-entry global exit closes that entry's // units, the oldest lots first (global_exit_per_entry_levels). - if (PF_RARE(source.per_entry_origin != 0)) { + if (PF_RARE(source.per_entry_origin != 0 && source.from_entry.empty())) { result.grid_policy = native_order::ExecutionGridPolicy::ExplicitUnits; result.units = std::min(source.per_entry_units, facts.scope_exposure_units); return result; @@ -17853,7 +18042,8 @@ NativePrecommitVerdict PineExecutionAdapter::validate_precommit(const NativePrec // candidate preserves the same observable trade roster. if (!view.account.would_open && !(view.inspected_closed_units > 0.0)) return NativePrecommitVerdict::Refuse; - if (PF_RARE(((std::isfinite(source.requested_qty) && source.requested_qty > 0.0) + if (PF_RARE(source.bracket_origin.incarnation == 0 + && ((std::isfinite(source.requested_qty) && source.requested_qty > 0.0) || (std::isfinite(source.qty_percent) && source.qty_percent < 100.0 - 1e-9)) && exit_tombstoned(source.source_id, source.from_entry))) { return NativePrecommitVerdict::Refuse; @@ -24245,6 +24435,7 @@ detail::ExitScratchRows PineExecutionAdapter::per_e for (const auto& handle : live_handles_) { const auto row = placement_.find(handle.incarnation); if (row != placement_.end() && row->second.per_entry_origin != 0 + && row->second.from_entry.empty() && row->second.source_id == exit_id && (origin == 0 || row->second.per_entry_origin == origin)) { legs.push_back(handle); @@ -24581,7 +24772,8 @@ void PineExecutionAdapter::per_entry_exits_after_fill( const std::optional& placement, const native_order::ExecutionAppliedEvent& event) { if (per_entry_exits_.empty()) return; - if (placement && placement->per_entry_origin != 0 && event.closed_units > 0.0) { + if (placement && placement->from_entry.empty() + && placement->per_entry_origin != 0 && event.closed_units > 0.0) { for (auto& exit : per_entry_exits_) { if (exit.exit_id != placement->source_id) continue; for (auto& child : exit.children) @@ -25859,6 +26051,18 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& // ab9714be pine_orders.cpp:330-337: execute_partial_exit_qty recognizes explicit requested quantity reductions && ((std::isfinite(placement_snapshot->requested_qty) && placement_snapshot->requested_qty > 0.0) || (std::isfinite(placement_snapshot->qty_percent) && placement_snapshot->qty_percent < 100.0 - 1e-9))) { + if (!placement_snapshot->from_entry.empty() + && placement_snapshot->bracket_origin.incarnation != 0) { + auto& family = bracket_family(key_for( + placement_snapshot->source_id, placement_snapshot->from_entry)); + // One priced fill consumes this bracket's share for its bound + // origin, even if a later call changes which price leg it uses. + for (const auto leg : {PineOrderFamily::ExitLimit, PineOrderFamily::ExitStop, + PineOrderFamily::ExitTrail}) + family.record_consumed(static_cast(leg), + placement_snapshot->bracket_origin.incarnation); + invalidate_retirement(); + } // The exit tombstones (exit_child_tombstones): a partial strategy.exit // child that filled is never revived for the entry incarnation it // filled under -- the newest opening of its from_entry then -- by a @@ -25869,6 +26073,7 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& // for the unreserved two shares). An exit armed for a later opening // of the id fills as before (test_exit_bracket_pending_entry_leg). if (detail::margin_rule_switches().exit_child_tombstones + && placement_snapshot->bracket_origin.incarnation == 0 && !placement_snapshot->from_entry.empty() && pinned_lot_grid()) { const std::uint64_t origin = entry_incarnation_of(placement_snapshot->from_entry); if (origin != 0) @@ -27344,6 +27549,7 @@ void PineExecutionAdapter::on_applied(const native_order::ExecutionAppliedEvent& || placement_snapshot->family == PineOrderFamily::ExitStop || placement_snapshot->family == PineOrderFamily::ExitTrail) && !placement_snapshot->from_entry.empty() + && placement_snapshot->per_entry_origin == 0 && detail::run_position(require_host()).signed_units != 0.0 && current_position_cycle_ > 0) { auto* pine = pine_view_of(&require_host()); diff --git a/src/source/pine_state_hash.cpp b/src/source/pine_state_hash.cpp index 8afab2723..601308edf 100644 --- a/src/source/pine_state_hash.cpp +++ b/src/source/pine_state_hash.cpp @@ -464,6 +464,7 @@ void source::PineExecutionAdapter::hash_state(BrokerStateHashSink& f) const { f.s(exit.exit_id); f.s(exit.from_entry); f.d(exit.trail_points); f.d(exit.trail_offset); f.d(exit.trail_price); f.d(exit.qty_percent); f.s(exit.comment); f.d(exit.qty); f.s(exit.oca_name); f.d(exit.profit_ticks); f.d(exit.loss_ticks); + f.u(exit.reservation_sequence); // Folded only when set, like PlacementSnapshot::void_global_issue. if (exit.void_global) f.u(0x766f6964676cULL); } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 996677b7a..504900f35 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -404,6 +404,7 @@ set(TEST_SOURCES test_declined_reversal_close_leg test_bracket_lifecycle_declined_reversal test_exit_bracket_pending_entry_leg + test_issue810_partial_exits test_dual_entry_placement_sizing test_fams_same_bar_market_tx test_default_flat_market_gross_admission @@ -8549,6 +8550,19 @@ if(PINEFORGE_BUILD_SOURCE_LAYER AND UNIX) endif() # --- lane run-json-run-warnings end --- +# --- lane entry-quantity-callers begin --- +# Real source-host modules for the production JSON/LIVE caller tests. +if(PINEFORGE_BUILD_SOURCE_LAYER AND UNIX AND NOT PINEFORGE_ENABLE_SANITIZERS AND NOT PINEFORGE_ENABLE_COVERAGE) + add_library(entry_quantity_percent MODULE entry_quantity_strategy.cpp) + target_link_libraries(entry_quantity_percent PRIVATE pineforge) + target_compile_definitions(entry_quantity_percent PRIVATE PINEFORGE_ENTRY_PERCENT=1) + target_compile_options(entry_quantity_percent PRIVATE -ffp-contract=off) + add_library(entry_quantity_explicit MODULE entry_quantity_strategy.cpp) + target_link_libraries(entry_quantity_explicit PRIVATE pineforge) + target_compile_options(entry_quantity_explicit PRIVATE -ffp-contract=off) +endif() +# --- lane entry-quantity-callers end --- + foreach(_pf_fast_test IN LISTS PINEFORGE_PR_FAST_TESTS) if(PINEFORGE_BUILD_SOURCE_LAYER AND (NOT TEST ${_pf_fast_test} OR NOT TARGET ${_pf_fast_test})) message(FATAL_ERROR "PR fast row or executable is missing: ${_pf_fast_test}") diff --git a/tests/entry_quantity_strategy.cpp b/tests/entry_quantity_strategy.cpp new file mode 100644 index 000000000..544ecb089 --- /dev/null +++ b/tests/entry_quantity_strategy.cpp @@ -0,0 +1,101 @@ +// The two entry patterns from issue #716, exposed through the ordinary module +// ABI. Only strategy lifecycle/inputs live here; metadata, sizing, funding, +// streaming and reporting all come from the linked engine. +#include +#include + +#include +#include +#include +#include + +namespace { + +class EntryQuantityStrategy final : public pineforge::source::PineStrategyHost { +public: + EntryQuantityStrategy() { + pineforge::source::PineStrategyConfig config; + config.initial_capital = 100000.0; +#ifdef PINEFORGE_ENTRY_PERCENT + config.default_qty_type = static_cast(pineforge::QtyType::PERCENT_OF_EQUITY); + config.default_qty_value = 100.0; +#endif + configure_pine_strategy(config); + set_syminfo_string("currency", "USDT"); + set_syminfo_string("basecurrency", "BTC"); + } + + void on_source_bar(const pineforge::Bar&) override { + if (pine_bar_index() != 1) return; +#ifdef PINEFORGE_ENTRY_PERCENT + const double quantity = std::numeric_limits::quiet_NaN(); +#else + const double quantity = 0.123456789; +#endif + // The optional input is the issue's explicitly sized item-1 control. + strategy_entry("L", true, std::numeric_limits::quiet_NaN(), + std::numeric_limits::quiet_NaN(), + get_input_double("Quantity", quantity)); + } +}; + +EntryQuantityStrategy& host(pf_strategy_t state) { + return *static_cast( + static_cast(state)); +} + +} // namespace + +extern "C" { + +PF_API pf_strategy_t strategy_create(const char*) { + try { + return static_cast(new EntryQuantityStrategy); + } catch (...) { + return nullptr; + } +} + +PF_API void strategy_free(pf_strategy_t state) { + delete static_cast(state); +} + +PF_API void strategy_set_input(pf_strategy_t state, const char* key, const char* value) { + host(state).set_input(key, value); +} + +PF_API void strategy_set_override(pf_strategy_t state, const char* key, const char* value) { + if (std::string(key) != "initial_capital") + throw std::invalid_argument("unsupported fixture override"); + std::size_t consumed = 0; + const double capital = std::stod(value, &consumed); + if (consumed != std::string(value).size() || !std::isfinite(capital) || capital <= 0.0) + throw std::invalid_argument("invalid fixture initial_capital"); + pineforge::source::StrategyOverrides overrides; + overrides.initial_capital = capital; + host(state).set_strategy_override(overrides); +} + +PF_API void run_backtest_full(pf_strategy_t state, pineforge::Bar* bars, int count, + const char* input_tf, const char* script_tf, + int magnifier, int samples, int distribution, + pineforge::ReportC* out) { + host(state).run(bars, count, input_tf, script_tf, magnifier != 0, samples, + static_cast(distribution)); + host(state).fill_report(out); +} + +PF_API void run_backtest(pf_strategy_t state, pineforge::Bar* bars, int count, + pineforge::ReportC* out) { + run_backtest_full(state, bars, count, "60", "60", 0, 4, PF_MAGNIFIER_ENDPOINTS, out); +} + +PF_API void report_free(pineforge::ReportC* out) { + pineforge::BacktestEngine::free_report(out); +} + +// Retain the archive object providing the production metadata and stream ABI, +// as runner/examples/strategy.cpp does. +PF_API int entry_quantity_fixture_abi_version() { return pf_abi_version(); } + +} // extern "C" diff --git a/tests/fixtures/entry_quantity/item1.tv_trades.csv b/tests/fixtures/entry_quantity/item1.tv_trades.csv new file mode 100644 index 000000000..bc594f8c7 --- /dev/null +++ b/tests/fixtures/entry_quantity/item1.tv_trades.csv @@ -0,0 +1,3 @@ +Trade number,Type,Date and time,Signal,Price USDT,Size (qty),Size (value),Net PnL USDT,Return %,Commission USDT,Favorable excursion USDT,Favorable excursion %,Adverse excursion USDT,Adverse excursion %,Cumulative PnL USDT,Cumulative PnL %,Duration (bars) +1,Exit long,2024-03-01 08:00,,61501.11,2.34667,99999.9863119,44322.824,44.32,0,50186.895,50.19,-9524.124,-9.52,44322.824,44.32,1438 +1,Entry long,2024-01-01 10:00,L,42613.57,2.34667,99999.9863119,44322.824,44.32,0,50186.895,50.19,-9524.124,-9.52,44322.824,44.32,1438 diff --git a/tests/fixtures/entry_quantity/item2.tv_trades.csv b/tests/fixtures/entry_quantity/item2.tv_trades.csv new file mode 100644 index 000000000..6090138b7 --- /dev/null +++ b/tests/fixtures/entry_quantity/item2.tv_trades.csv @@ -0,0 +1,3 @@ +Trade number,Type,Date and time,Signal,Price USDT,Size (qty),Size (value),Net PnL USDT,Return %,Commission USDT,Favorable excursion USDT,Favorable excursion %,Adverse excursion USDT,Adverse excursion %,Cumulative PnL USDT,Cumulative PnL %,Duration (bars) +1,Exit long,2024-03-01 08:00,,61501.11,0.12345,5260.6452165,2331.6667,44.32,0,2640.1548,50.19,-501.03046,-9.52,2331.6667,2.33,1438 +1,Entry long,2024-01-01 10:00,L,42613.57,0.12345,5260.6452165,2331.6667,44.32,0,2640.1548,50.19,-501.03046,-9.52,2331.6667,2.33,1438 diff --git a/tests/lanes.d/entry-quantity-callers.cmake b/tests/lanes.d/entry-quantity-callers.cmake new file mode 100644 index 000000000..73ef2a5a2 --- /dev/null +++ b/tests/lanes.d/entry-quantity-callers.cmake @@ -0,0 +1,17 @@ +if(PINEFORGE_BUILD_SOURCE_LAYER AND UNIX AND NOT PINEFORGE_ENABLE_SANITIZERS AND NOT PINEFORGE_ENABLE_COVERAGE) + add_test(NAME test_entry_quantity_json + COMMAND ${CMAKE_COMMAND} -E env + "PINEFORGE_ENTRY_PERCENT_LIBRARY=$" + "PINEFORGE_ENTRY_EXPLICIT_LIBRARY=$" + ${Python3_EXECUTABLE} -B -m pytest -q -p no:cacheprovider + ${PROJECT_SOURCE_DIR}/tests/test_entry_quantity_callers.py::test_json_entry_quantity) + if(PINEFORGE_BUILD_LIVE_RUNNER) + add_test(NAME test_entry_quantity_live + COMMAND ${CMAKE_COMMAND} -E env + "PINEFORGE_ENTRY_PERCENT_LIBRARY=$" + "PINEFORGE_ENTRY_EXPLICIT_LIBRARY=$" + "PINEFORGE_ENTRY_LIVE_RUNNER=$" + ${Python3_EXECUTABLE} -B -m pytest -q -p no:cacheprovider + ${PROJECT_SOURCE_DIR}/tests/test_entry_quantity_callers.py::test_live_entry_quantity) + endif() +endif() diff --git a/tests/test_entry_quantity_callers.py b/tests/test_entry_quantity_callers.py new file mode 100644 index 000000000..7ae865214 --- /dev/null +++ b/tests/test_entry_quantity_callers.py @@ -0,0 +1,234 @@ +"""Issue #716 entry patterns through the production JSON and LIVE callers. + +The modules are hand-written translations of the two literal entry patterns, +not codegen or parser tests. They link the real PineStrategyHost/runtime and +provide no metadata setter, quantity calculator, stream implementation or +report accessor. CTest must supply both CMake-built modules. + +The retained TradingView exports are BINANCE:BTCUSDT, 60 minutes, +2024-01-01..2024-03-01, exported 2026-11-14 at 14:50:24.088Z (item 1) +and 14:50:24.961Z (item 2). Their quantities/prices and quote-currency PnL +are independent value oracles. Their display timezone is unknown: timestamps +below are checked against the input feed, never an inferred TV timezone. +Funding/precedence/gridless controls use Decimal arithmetic, not TV authority. +""" +from dataclasses import dataclass +import csv +from decimal import Decimal, ROUND_FLOOR, ROUND_HALF_UP +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +FEED = ROOT / "tests/fixtures/qty_step_mincontract/btc60/feed/bars.csv" +TAPES = ROOT / "tests/fixtures/entry_quantity" +STEP = Decimal("0.00001") +EXPLICIT = "0.123456789" + + +@dataclass(frozen=True) +class Case: + pattern: str + metadata: dict + grid: Decimal | None + capital: str = "100000" + explicit_control: bool = False + tv_item: str | None = None + + +DECLARATIONS = ( + ("step-only", {"qty_step": 0.00001}), + ("minimum-only", {"mincontract": 0.00001}), + ("explicit-first", {"qty_step": 0.00001, "mincontract": 0.001}), + ("minimum-first", {"mincontract": 0.001, "qty_step": 0.00001}), +) +CASES = [ + pytest.param(Case(pattern, metadata, STEP, + tv_item=item if name in ("step-only", "minimum-only") else None), + id=f"{item}-{name}") + for pattern, item in (("percent", "item1"), ("explicit", "item2")) + for name, metadata in DECLARATIONS +] + [ + pytest.param(Case("percent", {}, None), id="item1-gridless-declined"), + pytest.param(Case("explicit", {}, None), id="item2-gridless-raw"), + pytest.param(Case("percent", {"qty_step": 0.00001}, STEP, capital="42613.57"), + id="item1-affordable-control"), + pytest.param(Case("percent", {"qty_step": 0.00001}, STEP, capital="42613.56"), + id="item1-unaffordable-control"), + pytest.param(Case("percent", {"qty_step": 0.00001}, STEP, explicit_control=True), + id="item1-explicit-control"), +] + + +def required_file(variable): + value = os.environ.get(variable) + assert value, f"CTest must supply {variable}" + path = Path(value) + assert path.is_file(), f"required CMake-built fixture is missing: {path}" + return path + + +@pytest.fixture(scope="module") +def modules(): + return { + "percent": required_file("PINEFORGE_ENTRY_PERCENT_LIBRARY"), + "explicit": required_file("PINEFORGE_ENTRY_EXPLICIT_LIBRARY"), + } + + +def bars_for(case): + with FEED.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + assert len(rows) == 1441 + assert rows[1]["close"] == "42613.56" + assert rows[2]["open"] == "42613.57" + assert rows[-1]["close"] == "61501.11" + assert [int(row["timestamp"]) for row in rows] == [ + 1704067200000 + index * 3600000 for index in range(1441) + ] + # Retain the whole measured history for literal-pattern/TV rows. + # Controls need only the signal, fill and one later observation. + return rows if case.tv_item else rows[:4] + + +def write_csv(path, rows): + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=rows[0].keys()) + writer.writeheader() + writer.writerows(rows) + + +def checked_json(command): + result = subprocess.run(list(map(str, command)), cwd=ROOT, + capture_output=True, text=True) + observed = f"command={command!r}\nstdout={result.stdout}\nstderr={result.stderr}" + # A LIVE refusal of mincontract is a caller admission failure, not a + # quantity observation. Missing modules/import/build failures are setup + # failures and must never be reported as the intended quantity RED. + assert result.returncode == 0, observed + document = json.loads(result.stdout) + assert "error" not in document, observed + return document + + +def quantity_contract(case, rows): + capital = Decimal(case.capital) + explicit = case.pattern == "explicit" or case.explicit_control + raw = Decimal(EXPLICIT) if explicit else capital / Decimal(rows[1]["close"]) + quantity = raw if case.grid is None else ( + (raw / case.grid).to_integral_value(rounding=ROUND_FLOOR) * case.grid + ) + affordable = quantity > 0 and quantity * Decimal(rows[2]["open"]) <= capital + return quantity, affordable + + +def assert_values(case, rows, trades, history): + quantity, affordable = quantity_contract(case, rows) + assert history == (len(rows), len(rows)) + # This is the intended item-1 RED when JSON drops qty_step: the unfloored + # default quantity costs more than the available capital at the next open. + assert len(trades) == (1 if affordable else 0), (case, quantity, trades) + if not affordable: + return + trade = trades[0] + actual = Decimal(str(trade["qty"])) + tolerance = max(Decimal("1e-12"), abs(quantity) * Decimal("1e-12")) + # The intended item-2 RED is the retained extra digits, not a runner error. + assert abs(actual - quantity) <= tolerance, (case, quantity, trade) + assert Decimal(str(trade["entry_price"])) == Decimal(rows[2]["open"]) + assert trade["entry_time"] == int(rows[2]["timestamp"]) + assert trade["entry_bar_index"] == 2 + assert trade["exit_time"] == int(rows[-1]["timestamp"]) + assert Decimal(str(trade["exit_price"])) == Decimal(rows[-1]["close"]) + assert trade["open_at_end"] in (True, 1) + assert trade["commission"] == 0 + expected_pnl = quantity * (Decimal(rows[-1]["close"]) - Decimal(rows[2]["open"])) + cent = Decimal("0.01") + assert Decimal(str(trade["pnl"])).quantize(cent, rounding=ROUND_HALF_UP) == ( + expected_pnl.quantize(cent, rounding=ROUND_HALF_UP) + ) + if case.tv_item: + with (TAPES / f"{case.tv_item}.tv_trades.csv").open( + newline="", encoding="utf-8-sig") as handle: + tape = list(csv.DictReader(handle)) + entries = [row for row in tape if row["Type"] == "Entry long"] + exits = [row for row in tape if row["Type"] == "Exit long"] + assert len(entries) == len(exits) == 1 + entry, exit_row = entries[0], exits[0] + assert quantity == Decimal(entry["Size (qty)"]) + assert abs(actual - Decimal(entry["Size (qty)"])) <= tolerance + assert Decimal(str(trade["entry_price"])) == Decimal(entry["Price USDT"]) + assert Decimal(str(trade["exit_price"])) == Decimal(exit_row["Price USDT"]) + assert Decimal(str(trade["pnl"])).quantize(cent, rounding=ROUND_HALF_UP) == ( + Decimal(entry["Net PnL USDT"]).quantize(cent, rounding=ROUND_HALF_UP) + ) + + +@pytest.mark.parametrize("case", CASES) +def test_json_entry_quantity(modules, tmp_path, case): + rows = bars_for(case) + bars, metadata = tmp_path / "bars.csv", tmp_path / "syminfo.json" + write_csv(bars, rows) + metadata.write_text(json.dumps({"syminfo": { + "mintick": 0.01, "pointvalue": 1, "timezone": "UTC", "session": "24x7", + **case.metadata, + }}), encoding="utf-8") + document = checked_json([ + sys.executable, "-B", ROOT / "docker/run_json.py", + "--so", modules[case.pattern], "--ohlcv", bars, + "--input-tf", "60", "--script-tf", "60", "--bar-magnifier", "false", + "--syminfo", metadata, "--overrides", json.dumps({"initial_capital": case.capital}), + "--inputs", json.dumps({"Quantity": EXPLICIT} if case.explicit_control else {}), + ]) + assert_values(case, rows, document["trades"], ( + document["diagnostics"]["input_bars_processed"], + document["diagnostics"]["script_bars_processed"], + )) + assert document["equity_curve"][0]["equity"] == float(case.capital) + for trade in document["trades"]: + assert trade["side"] == "long" and trade["entry_incarnation"] > 0 + + +@pytest.mark.parametrize("case", CASES) +def test_live_entry_quantity(modules, tmp_path, case): + runner = required_file("PINEFORGE_ENTRY_LIVE_RUNNER") + rows = bars_for(case) + warmup, feed, ledger = (tmp_path / name for name in + ("warmup.csv", "feed.jsonl", "ledger.sqlite")) + # The signal is in the live leg, not historical warmup. + write_csv(warmup, rows[:1]) + with feed.open("w", encoding="utf-8") as handle: + for row in rows[1:]: + bar = dict(zip(("o", "h", "l", "c", "v"), + (float(row[key]) for key in + ("open", "high", "low", "close", "volume")))) + bar["ts_open"] = int(row["timestamp"]) + handle.write(json.dumps({"type": "bar", "bar": bar}) + "\n") + command = [ + runner, "run", "--strategy", modules[case.pattern], + "--warmup", warmup, "--feed", feed, "--ledger", ledger, + "--mode", "bars", "--input-tf", "60", "--script-tf", "60", + "--symbol", "BINANCE:BTCUSDT", "--timezone", "UTC", + "--chart-timezone", "UTC", "--session", "24x7", + "--override", f"initial_capital={case.capital}", + "--syminfo", "mintick=0.01", "--syminfo", "pointvalue=1", + ] + for key, value in case.metadata.items(): + command += ["--syminfo", f"{key}={value}"] + if case.explicit_control: + command += ["--input", f"Quantity={EXPLICIT}"] + checked_json(command) + snapshot = checked_json([runner, "report", "--ledger", ledger]) + report = snapshot["report"] + assert_values(case, rows, report["trades"], ( + report["input_bars_processed"], report["script_bars_processed"], + )) + assert report["equity_curve"][0]["equity"] == float(case.capital) + for trade in report["trades"]: + assert trade["is_long"] == 1 + assert snapshot["closed_trades"] == [] diff --git a/tests/test_issue810_partial_exits.cpp b/tests/test_issue810_partial_exits.cpp new file mode 100644 index 000000000..7e8bf38b9 --- /dev/null +++ b/tests/test_issue810_partial_exits.cpp @@ -0,0 +1,383 @@ +/* + * Issue 810: partial strategy.exit reservations must survive the parent fill, + * later re-issues, and same-id pyramiding. + * + * The quantities and control substitutions come from the reporter's public + * issue. The bars below isolate the reservation lifecycles with real source + * commands and fills; they are not a replacement TradingView tape. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +using namespace pineforge; + +namespace { + +constexpr double kNaN = std::numeric_limits::quiet_NaN(); +constexpr double kEntryQty = 0.02; +constexpr double kTolerance = 1e-12; + +int tests_passed = 0; +int tests_failed = 0; + +#define CHECK(expr) \ + do { \ + if (!(expr)) { \ + std::printf(" FAIL %s:%d %s\n", __FILE__, __LINE__, #expr); \ + ++tests_failed; \ + } else { \ + ++tests_passed; \ + } \ + } while (0) + +bool near(double left, double right) { + return std::fabs(left - right) <= kTolerance; +} + +Bar bar(int index, double open, double high, double low, double close) { + Bar value; + value.open = open; + value.high = high; + value.low = low; + value.close = close; + value.volume = 1000.0; + value.timestamp = 1'000'000 + static_cast(index) * 60'000; + return value; +} + +enum class Case { + ARelative, + APriceControl, + BExplicitQty, + BPercentControl, + CPercent, + CFullControl, +}; + +bool is_a(Case value) { + return value == Case::ARelative || value == Case::APriceControl; +} + +bool is_b(Case value) { + return value == Case::BExplicitQty || value == Case::BPercentControl; +} + +bool is_c(Case value) { + return value == Case::CPercent || value == Case::CFullControl; +} + +class Issue810Probe final : public source::PineStrategyHost { +public: + struct Opening { + std::uint64_t origin; + std::string id; + int bar; + double units; + }; + + explicit Issue810Probe(Case test_case) : test_case_(test_case) { + attach_pine_execution_adapter(); + source::PineStrategyConfig config; + config.initial_capital = 10'000.0; + config.default_qty_type = static_cast(QtyType::FIXED); + config.default_qty_value = kEntryQty; + config.pyramiding = is_c(test_case) ? 2 : 1; + config.process_orders_on_close = is_a(test_case); + config.commission_value = 0.0; + config.slippage = 0; + config.margin_long = 0.0; + config.margin_short = 0.0; + config.calc_on_order_fills = false; + configure_pine_strategy(config); + set_syminfo_session("24x7"); + set_syminfo_timezone("Etc/UTC"); + set_syminfo_mintick(0.01); + set_syminfo_metadata("qty_step", 0.00001); + } + + double signed_position() const { + return signed_position_size(); + } + + std::vector openings; + std::vector pending_tp_quantities; + bool pending_observation_ok = true; + +protected: + void on_native_applied_event(const native_order::ExecutionAppliedEvent& event, + const NativeDecisionContext& context) override { + if (std::abs(event.opened_units) > 0.0) { + openings.push_back({ + event.handle().incarnation, + event.request().label, + static_cast(context.coordinate.interval_index), + std::abs(event.opened_units), + }); + } + } + + void on_source_bar(const Bar&) override { + if (is_a(test_case_)) { + if (bar_index_ != 0) return; + strategy_entry("Long", true, kNaN, kNaN, kEntryQty); + if (test_case_ == Case::ARelative) { + // Issue A: reporter's 50% tick exit is placed before the + // same-bar entry fills. + strategy_exit("Half", "Long", kNaN, kNaN, kNaN, kNaN, kNaN, + 50.0, "", kNaN, "", 50'000.0, kNaN); + } else { + // One-change control for A: the same target is absolute. + strategy_exit("Half", "Long", 44415.71, kNaN, kNaN, kNaN, kNaN, + 50.0); + } + strategy_exit("Rest", "Long", kNaN, 42915.71, kNaN, kNaN, kNaN, + 100.0); + return; + } + + if (is_b(test_case_)) { + if (bar_index_ == 0) { + strategy_entry("Long", true, kNaN, kNaN, kEntryQty); + return; + } + if (signed_position() <= 0.0) return; + if (test_case_ == Case::BExplicitQty) { + // Issue B: the explicit 0.01 partial is reissued on every + // live bar, before the full sibling. + strategy_exit("Half", "Long", 101.0, kNaN, kNaN, kNaN, kNaN, + 100.0, "", 0.01); + } else { + // One-change control for B: the same partial is percentage + // sized instead of explicitly sized. + strategy_exit("Half", "Long", 101.0, kNaN, kNaN, kNaN, kNaN, + 50.0); + } + strategy_exit("Rest", "Long", 102.0, kNaN, kNaN, kNaN, kNaN, + 100.0); + return; + } + + if (bar_index_ == 0 || bar_index_ == 2) + strategy_entry("Long", true, kNaN, kNaN, kEntryQty); + + if (test_case_ == Case::CPercent && bar_index_ == 4) { + for (int index = 0; index < pending_order_count(); ++index) { + pf_pending_order_v1_t row{}; + if (observe_pending_copy_v1(index, &row) != 0) { + pending_observation_ok = false; + } else if (std::string(row.id) == "TP") { + pending_tp_quantities.push_back(row.qty); + } + } + std::sort(pending_tp_quantities.begin(), pending_tp_quantities.end()); + } + + // Issue C reissues the one named exit as the second same-ID origin is + // added. The control omits qty_percent and is therefore full-sized. + strategy_exit("TP", "Long", 101.0, kNaN, kNaN, kNaN, kNaN, + test_case_ == Case::CPercent ? 50.0 : 100.0); + } + +private: + Case test_case_; +}; + +std::vector bars_for(Case test_case) { + if (is_a(test_case)) { + return { + bar(0, 43915.71, 43915.71, 43915.71, 43915.71), + bar(1, 43915.71, 44415.71, 43800.0, 44400.0), + bar(2, 44400.0, 44410.0, 42915.71, 43000.0), + bar(3, 43000.0, 43010.0, 42950.0, 43000.0), + }; + } + if (is_b(test_case)) { + return { + bar(0, 100.0, 100.0, 100.0, 100.0), + bar(1, 100.0, 100.5, 99.0, 100.0), + bar(2, 100.0, 100.5, 99.0, 100.0), + bar(3, 100.0, 100.5, 99.0, 100.0), + bar(4, 100.0, 101.0, 99.0, 100.0), + bar(5, 100.0, 102.0, 99.0, 100.0), + bar(6, 100.0, 101.0, 99.0, 100.0), + }; + } + return { + bar(0, 100.0, 100.0, 100.0, 100.0), + bar(1, 100.0, 100.0, 100.0, 100.0), + bar(2, 100.0, 100.0, 100.0, 100.0), + bar(3, 100.0, 100.0, 100.0, 100.0), + bar(4, 100.0, 100.0, 100.0, 100.0), + bar(5, 100.0, 101.0, 99.0, 100.0), + bar(6, 100.0, 100.0, 100.0, 100.0), + }; +} + +void check_issue_a(Case test_case) { + std::printf("case A %s\n", test_case == Case::ARelative ? "reported" : "price-control"); + Issue810Probe probe(test_case); + const auto bars = bars_for(test_case); + probe.run(bars.data(), static_cast(bars.size())); + CHECK(probe.last_error().empty()); + CHECK(probe.openings.size() == 1); + if (probe.openings.size() == 1) { + CHECK(probe.openings[0].id == "Long"); + CHECK(probe.openings[0].bar == 0); + CHECK(near(probe.openings[0].units, kEntryQty)); + } + CHECK(probe.trade_count() == 2); + int half_rows = 0; + int rest_rows = 0; + double closed = 0.0; + for (int index = 0; index < probe.trade_count(); ++index) { + const Trade& trade = probe.get_trade(index); + CHECK(trade.entry_id == "Long"); + CHECK(trade.entry_bar_index == 0); + CHECK(near(trade.qty, 0.01)); + closed += trade.qty; + if (trade.exit_id == "Half") { + ++half_rows; + CHECK(trade.exit_bar_index == 1); + CHECK(near(trade.exit_price, 44415.71)); + } else if (trade.exit_id == "Rest") { + ++rest_rows; + CHECK(trade.exit_bar_index == 2); + CHECK(near(trade.exit_price, 42915.71)); + } else { + CHECK(false); + } + } + CHECK(half_rows == 1); + CHECK(rest_rows == 1); + CHECK(near(closed, kEntryQty)); + CHECK(near(probe.signed_position(), 0.0)); +} + +void check_issue_b(Case test_case) { + std::printf("case B %s\n", test_case == Case::BExplicitQty ? "reported" : "percent-control"); + Issue810Probe probe(test_case); + const auto bars = bars_for(test_case); + probe.run(bars.data(), static_cast(bars.size())); + CHECK(probe.last_error().empty()); + CHECK(probe.openings.size() == 1); + if (probe.openings.size() == 1) { + CHECK(probe.openings[0].id == "Long"); + CHECK(probe.openings[0].bar == 1); + CHECK(near(probe.openings[0].units, kEntryQty)); + } + CHECK(probe.trade_count() == 2); + int half_rows = 0; + int rest_rows = 0; + double closed = 0.0; + for (int index = 0; index < probe.trade_count(); ++index) { + const Trade& trade = probe.get_trade(index); + CHECK(trade.entry_id == "Long"); + CHECK(trade.entry_bar_index == 1); + CHECK(near(trade.qty, 0.01)); + closed += trade.qty; + if (trade.exit_id == "Half") { + ++half_rows; + CHECK(trade.exit_bar_index == 4); + CHECK(near(trade.exit_price, 101.0)); + } else if (trade.exit_id == "Rest") { + ++rest_rows; + CHECK(trade.exit_bar_index == 5); + CHECK(near(trade.exit_price, 102.0)); + } else { + CHECK(false); + } + } + CHECK(half_rows == 1); + CHECK(rest_rows == 1); + CHECK(near(closed, kEntryQty)); + CHECK(near(probe.signed_position(), 0.0)); +} + +void check_issue_c(Case test_case) { + std::printf("case C %s\n", test_case == Case::CPercent ? "reported" : "full-control"); + Issue810Probe probe(test_case); + const auto bars = bars_for(test_case); + probe.run(bars.data(), static_cast(bars.size())); + CHECK(probe.last_error().empty()); + CHECK(probe.openings.size() == 2); + if (probe.openings.size() == 2) { + CHECK(probe.openings[0].id == "Long"); + CHECK(probe.openings[1].id == "Long"); + CHECK(probe.openings[0].bar == 1); + CHECK(probe.openings[1].bar == 3); + CHECK(near(probe.openings[0].units, kEntryQty)); + CHECK(near(probe.openings[1].units, kEntryQty)); + CHECK(probe.openings[0].origin != probe.openings[1].origin); + } + + const bool partial = test_case == Case::CPercent; + if (partial) { + CHECK(probe.pending_observation_ok); + CHECK(probe.pending_tp_quantities.size() == 2); + if (probe.pending_tp_quantities.size() == 2) { + CHECK(near(probe.pending_tp_quantities[0], 0.01)); + CHECK(near(probe.pending_tp_quantities[1], 0.01)); + } + } + + CHECK(probe.trade_count() > 0); + double closed = 0.0; + double from_first = 0.0; + double from_second = 0.0; + for (int index = 0; index < probe.trade_count(); ++index) { + const Trade& trade = probe.get_trade(index); + CHECK(trade.entry_id == "Long"); + CHECK(trade.exit_id == "TP"); + CHECK(trade.exit_bar_index == 5); + CHECK(near(trade.exit_price, 101.0)); + if (trade.entry_bar_index == 1) { + from_first += trade.qty; + } else if (trade.entry_bar_index == 3) { + from_second += trade.qty; + } else { + CHECK(false); + } + closed += trade.qty; + } + CHECK(near(closed, partial ? 0.02 : 0.04)); + if (partial) { + // Under the reporter's default FIFO rule, both 0.01 reservations + // book against the first lot, while 0.02 remains open. + CHECK(near(from_first, 0.02)); + CHECK(near(from_second, 0.0)); + CHECK(near(probe.signed_position(), 0.02)); + } else { + CHECK(near(from_first, 0.02)); + CHECK(near(from_second, 0.02)); + CHECK(near(probe.signed_position(), 0.0)); + } +} + +void test_reporter_shapes_and_controls() { + std::printf("test_reporter_shapes_and_controls\n"); + check_issue_a(Case::ARelative); + check_issue_a(Case::APriceControl); + check_issue_b(Case::BExplicitQty); + check_issue_b(Case::BPercentControl); + check_issue_c(Case::CPercent); + check_issue_c(Case::CFullControl); +} + +} // namespace + +int main() { + test_reporter_shapes_and_controls(); + std::printf("\n%d passed, %d failed\n", tests_passed, tests_failed); + return tests_failed == 0 ? 0 : 1; +} diff --git a/tests/test_run_json_syminfo_values.py b/tests/test_run_json_syminfo_values.py index bdee73d31..6ed466625 100644 --- a/tests/test_run_json_syminfo_values.py +++ b/tests/test_run_json_syminfo_values.py @@ -142,3 +142,199 @@ def test_syminfo_positive_and_absent_values_keep_their_application( assert library.calls.count("run_backtest_full") == 1, ( f"expected exactly one run_backtest_full call; {observed}") assert stderr == "", f"expected empty stderr for valid metadata; {observed}" + + +class QuantityLibrary(SymbolLibrary): + """Records caller dispatch, not native sizing or TradingView parity.""" + + def __init__(self, failure=None): + super().__init__() + + def metadata(state, key, value): + self.applied.append((key.decode("utf-8"), value)) + if failure is not None: + raise failure + + name = "strategy_set_syminfo_metadata" + setattr(self, name, self.export(name, metadata)) + + +QTY_GRID_VALID_CASES = ( + pytest.param({"qty_step": 0.00001}, [("qty_step", 0.00001)], id="step-only-fine"), + pytest.param({"qty_step": 0.25}, [("qty_step", 0.25)], id="step-only-coarse"), + pytest.param({"mincontract": 0.01}, + [("qty_step", 0.01), ("mincontract", 0.01)], id="minimum-only-control"), + pytest.param({"qty_step": 0.25, "mincontract": 0.01}, + [("qty_step", 0.25), ("mincontract", 0.01)], id="explicit-first"), + pytest.param({"mincontract": 0.01, "qty_step": 0.25}, + [("qty_step", 0.25), ("mincontract", 0.01)], id="minimum-first"), + pytest.param({"mincontract": 1, "qty_step": 0.001}, + [("qty_step", 0.001), ("mincontract", 1.0)], id="conflict-finer-step"), + pytest.param({"qty_step": 0.01, "mincontract": 0.01}, + [("qty_step", 0.01), ("mincontract", 0.01)], id="same-value-control"), + pytest.param({}, [], id="absent"), + pytest.param({"qty_step": None}, [], id="null-step"), + pytest.param({"mincontract": None}, [], id="null-minimum"), + pytest.param({"qty_step": None, "mincontract": None}, [], id="both-null"), + pytest.param({"qty_step": None, "mincontract": 0.01}, + [("qty_step", 0.01), ("mincontract", 0.01)], id="null-step-fallback"), + pytest.param({"mincontract": None, "qty_step": 0.25}, + [("qty_step", 0.25)], id="null-minimum-explicit"), + pytest.param({"mintick": 0.5, "qty_step": 0.125, "pointvalue": 2}, + [("qty_step", 0.125), ("mintick", 0.5), ("pointvalue", 2.0)], + id="step-with-other-metadata"), +) + +QTY_GRID_INVALID_CASES = ( + pytest.param("0", id="zero"), + pytest.param("-0.01", id="negative"), + pytest.param("true", id="true"), + pytest.param("false", id="false"), + pytest.param("NaN", id="nan"), + pytest.param("Infinity", id="infinity"), + pytest.param("-Infinity", id="negative-infinity"), + pytest.param("1e9999", id="float-overflow"), + pytest.param("1" + "0" * 400, id="integer-overflow"), + pytest.param("1e-9999", id="underflow"), + pytest.param('"0.25"', id="numeric-text"), + pytest.param('"invalid"', id="text"), + pytest.param("[]", id="array"), + pytest.param("{}", id="object"), +) + +# Exact focused collection: 28 + 28 + 28 + 6 + 4 + 2 = 96 caller rows. +QTY_GRID_CASE_COUNTS = { + "test_syminfo_qty_grid_valid_dispatch": 28, + "test_syminfo_qty_grid_invalid_step_is_atomic": 28, + "test_syminfo_qty_grid_invalid_minimum_is_atomic": 28, + "test_syminfo_qty_grid_later_invalid_field_is_atomic": 6, + "test_syminfo_qty_grid_missing_setter_is_typed": 4, + "test_syminfo_qty_grid_setter_failure_is_preserved": 2, +} + + +def _qty_grid_invoke(tmp_path, monkeypatch, record_property, metadata, wrapped, library=None): + text = metadata if isinstance(metadata, str) else json.dumps(metadata) + if wrapped: + text = '{"syminfo":' + text + "}" + path = tmp_path / "quantity-syminfo.json" + path.write_text(text, encoding="utf-8") + library = QuantityLibrary() if library is None else library + status, document, stderr, _ = harness.invoke( + monkeypatch, "--ohlcv", harness.csv_file(tmp_path), "--syminfo", path, library=library) + # Retain actual main() status, setter arguments and serialized output even + # when the next assertion fails. This is a Python caller library stand-in. + record_property("q716_json_call", json.dumps({ + "scope": "JSON caller dispatch/admission", + "input": text, "main_status": status, "document": document, + "stderr": stderr, "setter_calls": library.applied, "calls": library.calls, + }, sort_keys=True)) + return status, document, stderr, library + + +def _qty_grid_refusal(result, code, args): + status, document, stderr, library = result + observed = _observed_run(status, document, stderr, library) + assert status == 1, observed + assert set(document) == {"engine", "error", "code", "args"}, observed + assert document["engine"] == "pineforge", observed + assert (document["code"], document["args"]) == (code, args), observed + assert "run_backtest_full" not in library.calls, observed + assert library.calls[-1:] == ["strategy_free"], observed + assert stderr == "", observed + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["flat", "wrapped"]) +@pytest.mark.parametrize("metadata,expected", QTY_GRID_VALID_CASES) +def test_syminfo_qty_grid_valid_dispatch( + tmp_path, monkeypatch, record_property, wrapped, metadata, expected): + result = _qty_grid_invoke(tmp_path, monkeypatch, record_property, metadata, wrapped) + status, document, stderr, library = result + observed = _observed_run(*result) + assert status == 0 and "summary" in document and "error" not in document, observed + assert library.applied == expected, observed + assert all(type(value) is float for _, value in library.applied), observed + grid = {key: value for key, value in expected if key in ("qty_step", "mincontract")} + assert document["applied_runtime"].get("syminfo", {}) == grid, observed + assert library.calls.count("run_backtest_full") == 1, observed + setter_indices = [i for i, name in enumerate(library.calls) + if name.startswith("strategy_set_syminfo_")] + assert all(i < library.calls.index("run_backtest_full") for i in setter_indices), observed + assert stderr == "", observed + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["flat", "wrapped"]) +@pytest.mark.parametrize("raw", QTY_GRID_INVALID_CASES) +def test_syminfo_qty_grid_invalid_step_is_atomic( + tmp_path, monkeypatch, record_property, wrapped, raw): + # A valid minimum and other valid metadata must not be installed before an + # invalid explicit step is discovered. Base main ignores that step instead. + text = json.dumps({"mintick": 0.5, "pointvalue": 2, "mincontract": 0.01})[:-1] + text += ',"qty_step":' + raw + "}" + result = _qty_grid_invoke(tmp_path, monkeypatch, record_property, text, wrapped) + _qty_grid_refusal(result, "lot_grid_rejected", {}) + _, document, _, library = result + assert library.applied == [], _observed_run(*result) + assert document["error"] == ( + "syminfo.qty_step must be a positive finite number, got " + + json.dumps(json.loads(raw))[:80]), _observed_run(*result) + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["flat", "wrapped"]) +@pytest.mark.parametrize("raw", QTY_GRID_INVALID_CASES) +def test_syminfo_qty_grid_invalid_minimum_is_atomic( + tmp_path, monkeypatch, record_property, wrapped, raw): + text = json.dumps({"qty_step": 0.25, "mintick": 0.5, "pointvalue": 2})[:-1] + text += ',"mincontract":' + raw + "}" + result = _qty_grid_invoke(tmp_path, monkeypatch, record_property, text, wrapped) + _qty_grid_refusal(result, "lot_grid_rejected", {}) + _, document, _, library = result + assert library.applied == [], _observed_run(*result) + assert document["error"] == ( + "syminfo.mincontract must be a positive finite number, got " + + json.dumps(json.loads(raw))[:80]), _observed_run(*result) + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["flat", "wrapped"]) +@pytest.mark.parametrize("key,value,code,args", [ + pytest.param("mintick", -0.5, "symbol_feeds_refused", + {"reason": "syminfo_mintick_invalid"}, id="tick"), + pytest.param("pointvalue", -2, "symbol_metadata_rejected", + {"field": "point_value"}, id="point-value"), + pytest.param("series_origin_ms", False, "syminfo_unreadable", + {"reason": "value_type"}, id="origin"), +]) +def test_syminfo_qty_grid_later_invalid_field_is_atomic( + tmp_path, monkeypatch, record_property, wrapped, key, value, code, args): + metadata = {"qty_step": 0.25, "mincontract": 0.01, key: value} + result = _qty_grid_invoke(tmp_path, monkeypatch, record_property, metadata, wrapped) + _qty_grid_refusal(result, code, args) + assert result[3].applied == [], _observed_run(*result) + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["flat", "wrapped"]) +@pytest.mark.parametrize("metadata", [ + pytest.param({"qty_step": 0.25}, id="step-only"), + pytest.param({"qty_step": 0.25, "mincontract": 0.01}, id="both"), +]) +def test_syminfo_qty_grid_missing_setter_is_typed( + tmp_path, monkeypatch, record_property, wrapped, metadata): + library = QuantityLibrary() + del library.strategy_set_syminfo_metadata + result = _qty_grid_invoke( + tmp_path, monkeypatch, record_property, metadata, wrapped, library=library) + _qty_grid_refusal(result, "strategy_library_incompatible", + {"reason": "setter_missing", "missing": "strategy_set_syminfo_metadata"}) + assert library.applied == [], _observed_run(*result) + + +@pytest.mark.parametrize("wrapped", [False, True], ids=["flat", "wrapped"]) +def test_syminfo_qty_grid_setter_failure_is_preserved( + tmp_path, monkeypatch, record_property, wrapped): + failure = harness.run_json.RunFailure("metadata setter refused qty_step", "lot_grid_rejected") + library = QuantityLibrary(failure=failure) + result = _qty_grid_invoke(tmp_path, monkeypatch, record_property, + {"qty_step": 0.125}, wrapped, library=library) + _qty_grid_refusal(result, "lot_grid_rejected", {}) + assert result[1]["error"] == str(failure), _observed_run(*result) + assert library.applied == [("qty_step", 0.125)], _observed_run(*result) diff --git a/tests/test_syminfo_mincontract_qty_step.cpp b/tests/test_syminfo_mincontract_qty_step.cpp index 0a87e84fb..0a68e3a06 100644 --- a/tests/test_syminfo_mincontract_qty_step.cpp +++ b/tests/test_syminfo_mincontract_qty_step.cpp @@ -10,6 +10,7 @@ #include #include +#include #include #include