diff --git a/.dockerignore b/.dockerignore index 26bf9e4..e814444 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,9 @@ venv data logs target +# Historical certification artifacts stay in Git/source-mounted test runs, not +# in serving images where package scanners treat old SBOMs as live dependencies. +upgrade/evidence/ *.parquet *.csv *.gz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e26a7e..bd299ad 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,6 +24,7 @@ jobs: uses: bufbuild/buf-setup-action@a47c93e0b1648d5651a065437926377d060baa99 # v1.50.0 with: version: "1.50.0" + github_token: ${{ github.token }} - name: Check schema format and compatibility working-directory: contracts @@ -76,7 +77,11 @@ jobs: --output "${archive}" echo "${CARGO_DENY_SHA256} ${archive}" | sha256sum --check --strict tar --no-same-owner -xzf "${archive}" - "./${directory}/cargo-deny" check + docker run --rm \ + -v "$PWD:/workspace:ro" \ + -v "$PWD/${directory}:/tools/cargo-deny:ro" \ + -w /workspace rust:1.82-slim \ + sh -c 'apt-get update -qq && apt-get install -y --no-install-recommends git >/dev/null && /tools/cargo-deny/cargo-deny check' sdk-python310: runs-on: ubuntu-latest @@ -98,6 +103,10 @@ jobs: unit-tests: runs-on: ubuntu-latest + env: + # The mutable compatibility tag remains for legacy test helpers. Security + # evidence and the release rehearsal use this SHA-bound tag instead. + QDL_CI_IMAGE: data-layer-ci:${{ github.sha }} steps: - name: Checkout uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 @@ -114,11 +123,14 @@ jobs: run: sudo scripts/prepare_nonroot_runtime.sh - name: Build data_layer image - run: docker compose -f docker-compose.yml -f docker-compose.ci.yml build data_layer + run: | + docker compose -f docker-compose.yml -f docker-compose.ci.yml build data_layer + docker image tag data-layer:v0.1.0 "${QDL_CI_IMAGE}" + docker run --rm "${QDL_CI_IMAGE}" python -c 'from importlib.metadata import version; from packaging.version import Version; assert Version(version("anyio")) >= Version("4.14.2")' - name: Audit final runtime dependencies run: | - docker run --rm data-layer:v0.1.0 sh -c ' + docker run --rm "${QDL_CI_IMAGE}" sh -c ' test ! -e /opt/venv/bin/poetry test "$(head -n 1 /opt/venv/bin/uvicorn)" = "#!/opt/venv/bin/python" uvicorn --version @@ -205,7 +217,7 @@ jobs: - name: Scan final runtime image for critical and high vulnerabilities uses: aquasecurity/trivy-action@a9c7b0f06e461e9d4b4d1711f154ee024b8d7ab8 # v0.36.0 with: - image-ref: data-layer:v0.1.0 + image-ref: ${{ env.QDL_CI_IMAGE }} format: table exit-code: "1" ignore-unfixed: true @@ -228,7 +240,7 @@ jobs: -out "${evidence_dir}/private.pem" >/dev/null 2>&1 openssl pkey -in "${evidence_dir}/private.pem" -pubout \ -out "${evidence_dir}/public.pem" - image_id="$(docker image inspect data-layer:v0.1.0 --format '{{.Id}}')" + image_id="$(docker image inspect "${QDL_CI_IMAGE}" --format '{{.Id}}')" python -m scripts.phase6_release_bundle \ --repo . --output-dir "${evidence_dir}/bundle" \ --release "qdl-ci-${GITHUB_SHA}" --git-sha "${GITHUB_SHA}" \ diff --git a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md index 6212f54..508ac52 100644 --- a/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md +++ b/DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md @@ -43689,3 +43689,5388 @@ projector posts every canonical batch to both stream gateways and the non-holder answers 409 - 6,080 rejected mTLS POSTs per projector per 30 minutes, ~36,000 an hour across three. The projector could remember the lease holder and re-probe on failure. It is write-path code and was not taken at a release gate. + +### R1.32 - Execution MARK/INDEX live-view correction (`SOURCE_TESTED / RUNTIME_PRECHECK_FAILED`, 2026-09-19) + +**Approved goal.** Remove the execution-grade `MARK_INDEX_PRICE` dependency on +venue REST and the expensive spool-query path without weakening the existing +`2,000 ms` consumer bound. The stream gateway already receives the canonical +latest-state event. This slice makes that authoritative, fenced live state +available to the V2 query role through a private authenticated read path. + +**Evidence and decision.** The completed real-provider measurement establishes +that the currently subscribed ten Binance/OKX MARK/INDEX bindings can be read +from the stream gateway with p99 `1693.8 ms`, with `0/239` samples over the +execution threshold. The spool path measured p99 `2318 ms`; direct OKX REST +owns `2.1-2.9 s` before local processing. Therefore neither a TTL adjustment +nor a retry/narrower provider poll is an acceptable execution fix. The chosen +path is a bounded in-memory latest execution view in the existing stream +gateway; it does not add a service, a symbol worker, a public endpoint, or a +consumer-manifest revision. + +**Approved source scope.** + +1. The stream role retains only the newest verified canonical + `MARK_INDEX_PRICE` event per exact instrument identity, fence epoch and + generation. It is bounded to declared execution bindings and is cleared or + rejected on lease loss/generation mismatch. +2. A private mTLS plus existing internal-signature read endpoint exposes one + typed view to query replicas. It never returns a stale, incomplete, + gap-open, non-authoritative, identity-mismatched, or passive-gateway value. +3. The V2 query reference-batch path uses that view **only** for execution + `MARK_INDEX_PRICE` requests. Alpha/research reference products, history, + durable replay and spool projection retain their current contracts. +4. A missing/invalid live view is a typed fail-closed result; it must not + silently call venue REST for an execution request. Existing consumer + V1-fallback policy remains outside this service and unchanged. + +**Invariants.** Price/unit/decimal values stay canonical; mark and index remain +a paired result from one exact instrument; provider/source-role lineage is +preserved; source-event and provider-confirmation freshness semantics are not +relaxed; no cross-venue or cross-symbol substitution is permitted. The four VN +bindings intentionally remain `V1_PRIMARY`; six inactive Spot catalog entries +are excluded from the V2 active-demand execution gate and are not silently +promoted by this work. + +**Source gates.** Unit/contract tests cover identity, units/decimals, bounded +replacement, lease fence, generation reset, stale/gap/incomplete rejection, +active/passive behavior, MARK/INDEX pairing, no REST invocation for an eligible +execution read, and unchanged alpha/research reference behavior. Targeted +integration tests cover private signature/TLS client behavior and query-result +lineage. Only provider-captured/replayed bytes are valid outside deterministic +tests; no generated market value is used as evidence. + +**Runtime gate and rollback decision boundary.** Source work may build/test in +this feature worktree only. A later separately approved packet may build one +immutable Python image and rolling-recreate exactly `stream_v2_active`, +`stream_v2_passive`, `query_v2_1`, and `query_v2_2`, retaining their current +runtime mounts and an exact per-role rollback image. It must not touch V1, +Rust/ingestors, projectors, bar edge, Kafka topology/offsets, Redis, SQLite, +Trading System, alpha or order paths. Acceptance is a bounded real-provider +measurement through a real V2 consumer identity: all ten active MARK/INDEX +bindings, p99 below `2,000 ms`, zero stale/gap/identity rejection, no added +venue REST calls, and no consumer/order mutation. Until that packet passes, +this item is source-tested only. + +**Source slice completed (2026-09-19).** Added the bounded +`ExecutionMarkIndexLiveView` to the existing leased stream process and a +private signed read edge. The view retains one canonical paired event per +declared identity, rejects an older provider generation, stays blocked after a +gap until a newer generation arrives, and is cleared with the gateway fence. +The query role now selects this reader only for +`INTERNAL_EXECUTION`/execution-grade current `MARK_INDEX_PRICE`; it cannot +fall through to a venue REST adapter. Alpha/research reference reads retain the +existing `ReferenceBatch` path. + +**Source evidence.** `python3 -m py_compile` passed for all changed modules. +The host intentionally has no `pytest`; the existing immutable +`qdl-v2-python:2.0.20-95d9595` image ran the isolated, read-only, +network-disabled `python -m unittest -v tests.test_execution_mark_index_live_view +tests.test_phase113_reference_v2 tests.test_mark_index_paired_lineage`: **26 +passed, 0 failed**. Coverage includes Binance USD-M and OKX Swap exact identity +and cross-venue rejection, paired decimals/lineage, stale/gap/generation/fence +rejection, private signature validation, no execution REST fallback, unchanged +alpha REST behavior, and existing MARK/INDEX freshness regressions. The suite +also caught and fixed one local refactor regression: the pre-existing HTTPS URL +validator still required `urlsplit` after HMAC helper extraction. No runtime +role, provider connection, Kafka/Redis/SQLite state, consumer, alpha, order +path, image or cache was changed by this source evidence. + +**Expanded source evidence.** The same immutable image then ran the broader +read-only, network-disabled regression selection: +`tests.test_execution_mark_index_live_view`, `tests.test_phase113_reference_v2`, +`tests.test_mark_index_paired_lineage`, `tests.test_phase104_reference_batch`, +`tests.test_phase104_v2_query_stream_integration`, +`tests.test_phase105_consumer_acceptance`, +`tests.test_phase115c_five_liquid_handoff`, and +`tests.test_reference_l2_consumer_acceptance`: **93 passed, 0 failed** in +`14.473 s`. It exercises strict provider identity, paired mark/index decimals, +execution freshness, five-symbol two-venue scope, L2 isolation, entitlement +scope, and the unchanged reference/warmup contracts. The invocation remained +`--rm`, `--network none`, `--read-only`, with only a temporary `/tmp` filesystem; +it created no image, container, provider, or runtime data artifact. + +**Latency-path correction before rollout (2026-09-19).** Read-only inspection +of the live implementation found that the first source implementation called +`ExecutionMarkIndexLiveView.remember(...)` *after* +`DurableStreamGateway.publish_many(...)`. That gateway deliberately fsyncs the +secondary SQLite spool before normal fan-out. It was correct for identity and +fail-closed behavior, but it could not remove the measured spool tail and is +therefore not a valid latency repair. The source slice is reopened before any +runtime action. The corrected boundary is: a stable projector has already read +the canonical event from Kafka `read_committed`; after the active stream +gateway validates signed canonical/raw lineage and its current lease, it may +offer the bounded execution latest-state view **before** the secondary spool +projection completes. A later spool success promotes only its optional spool +watermark; any append failure withdraws that exact event, and any lease fence +clears every view. Query still fails closed on missing, stale, gapped or fenced +state and never falls back to venue REST. This does not add a producer, broker, +service, public API or another cache. The previous `93/93` source suite is +re-run after this correction; the old commit is not eligible for rollout. + +**Corrected-source evidence (2026-09-19).** The correction was recompiled with +`python3 -m py_compile` for every changed production and test module. The same +immutable `qdl-v2-python:2.0.20-95d9595` image then ran the complete isolated, +network-disabled, read-only regression selection listed above: **96 passed, 0 +failed** in `14.026 s`. The added integration-ordering case deliberately holds +`publish_many` after signed canonical validation and proves that a typed, +read-committed MARK/INDEX view is available before the secondary SQLite spool; +after release it is promoted to `SPOOL_CONFIRMED`. Unit cases prove exact-event +withdrawal through the actual `503` backpressure path, promotion without replacement, lease fencing, +identity isolation, and no venue REST fallback. + +**Hot-path measurement (source-only).** A 2,000-request in-process ASGI/HMAC +read of the corrected private endpoint measured `p50 0.5958 ms`, `p95 0.7651 +ms`, `p99 0.9307 ms`, and `max 1.8597 ms`. This is deliberately not presented +as a real-provider or consumer result: it excludes mTLS/process scheduling, +network and provider cadence. The current deployed runtime still uses the old +image and its read-only 900-second MARK/INDEX report retains the secondary +spool path, with `venue -> durable` p99 between `1,443.4 ms` and `3,759.0 ms` +across the ten active bindings. A real consumer latency certificate requires +the bounded rollout and acceptance gate already defined above. + +**Read-only deployed baseline audit (2026-09-19).** The current runtime still +uses `sha256:9039236e7a8e570f2364b470b33386ab702bc1dde5ae9d5e7d90a4dda531e8f0` +for query/stream and is intentionally unchanged. A bounded 30-minute scan of +the two ingestors, three Rust cores, two query replicas, two stream roles, bar +edge and three projectors found no matching `ERROR`, `FATAL`, `Traceback` or +`WARN` records in the most recent 1,200 lines per role. All listed +health-checked roles remained healthy. Rust core 2's cumulative +`quarantines=576` and `duplicates=140` were unchanged while it processed +287,924 further records; `scope_quarantines=0`. The passive stream emitted +three normal subscription-progress records with bounded coalescing/aged reads +but `overflowed=false`; this is not a MARK/INDEX rejection, is not changed by +this scope, and remains observable during the later acceptance window. + +**Current status.** `SOURCE_TESTED / RUNTIME_PENDING`. No service, image, +runtime configuration, provider connection, Kafka/Redis/SQLite state, +consumer, alpha, order path, or test artifact was changed by this source +slice. The only next decision is the bounded four-role rollout described +above; it is deliberately not implied by source tests or a health response. + +**Runtime acceptance authorised (2026-09-19, `RUNTIME_PREPARING`).** The +approved packet is deliberately narrow: build one immutable Python candidate +from this source revision, then rolling-recreate only `stream_v2_passive`, +`stream_v2_active`, `query_v2_1`, and `query_v2_2`, serially and with their +existing `/runtime`, TLS and state mounts. The current reader rollback image is +`sha256:9039236e7a8e570f2364b470b33386ab702bc1dde5ae9d5e7d90a4dda531e8f0`; +rollback recreates only an affected reader role against that exact image and +its unchanged runtime mount. This packet must not recreate or configure V1, +Rust core, ingestors, projectors, bar edge, Kafka topology or offsets, Redis, +SQLite, Trading System, alpha, or an order path. Normal real-provider reads +continue as before; the probe itself performs V2 query reads only. + +**Consumer-observed latency gate.** Before the runtime packet is built, this +branch adds a small reusable read-only harness rather than inferring consumer +latency from a server log. It loads the registered +`trading-system.paper.stable` manifest, selects exactly its ten execution +`MARK_INDEX_PRICE` requirements, and calls the public `AsyncDataLayerClient` +over the same mTLS/JWT REST query route that `market_data_service` uses. Each +sample records (a) `consumer_call_to_usable_ms`, timed around the completed SDK +request and model validation, and (b) `provider_confirmation_to_usable_ms`, +calculated from the typed provider-confirmation timestamp to that completed +consumer receipt. It rejects a missing/non-OK result, cross-identity response, +V1/direct-provider fallback, stale/gap/fenced view, invalid delivery stage or +unbounded source lineage. The real-provider acceptance runs one bounded 300 s +window below the registered quota, records p50/p95/p99/max per exact binding +and aggregate, and requires every binding to have valid receipts, zero typed +failure/fallback and aggregate provider-confirmation-to-usable p99 below +`2,000 ms`. It creates only a disposable `--rm` client container and a bounded +secret-free evidence JSON under the external runtime state directory. + +**Decision boundary and hygiene.** Passing this packet is a runtime +certificate for this MARK/INDEX correction only, not a release, manifest +promotion or broader consumer cutover. The active candidate image and the +named reader rollback image are retained until the release decision; no other +test image, container, cache, volume, network or source state is retained. + +**Acceptance-harness source evidence (2026-09-19).** Added +`scripts/measure_execution_mark_index_consumer_latency.py`. It loads the +canonical `ConsumerManifestLoader` representation rather than re-parsing SDK +enums, so the measured request has the same feed, grade, freshness, gap and +source-policy values enforced by query. Its focused regression rejects a +non-internal/provider lineage and confirms the stable paper manifest resolves +to exactly ten execution MARK/INDEX requests. An initial test exposed a probe +only enum-boundary error (`qdl_sdk.Grade` is not the internal +`ConsumerGrade`); it was corrected before any runtime action. The immutable +existing image ran the new probe regression plus the complete MARK/INDEX, +reference, consumer, L2 and five-liquid selection with `--rm`, `--network +none`, source read-only and temporary `/tmp`: **98 passed, 0 failed** in +`14.352 s`. No image was built and no runtime resource was changed by this +evidence. + +**Bounded runtime precheck (`RUNTIME_PRECHECK_FAILED`, 2026-09-19).** The +approved four-role candidate packet was applied exactly once: only +`stream_v2_active`, `stream_v2_passive`, `query_v2_1`, and `query_v2_2` now +run `qdl-v2-python:2.0.21-rc.1-25dce61` +(`sha256:ee0154e88e9a26213d7354532edc01e441f3a5754cb2b64c8f672d8593189414`, +source `25dce612cc11bf246320c36c7360ec128072033f`). All four became healthy. +The named rollback remains +`sha256:9039236e7a8e570f2364b470b33386ab702bc1dde5ae9d5e7d90a4dda531e8f0`. +No V1, Rust core, ingestor, projector, BAR edge, Kafka offset/topology, Redis, +SQLite, Trading System, alpha or order-path role was recreated or configured. + +The actual `trading-system.paper.stable` mTLS/JWT identity then called the +public V2 SDK `reference_batch` route for all ten registered execution +MARK/INDEX requirements for 60 seconds. This was a diagnostic precheck with +`require_all=False` solely to retain typed per-binding evidence; it is not a +substitute for the required strict 300-second C2 acceptance. It completed 21 +batches / 210 exact binding reads, made no V1 or direct-venue request, and +created no order or consumer state. Consumer call-to-usable time was `p50 +1998.718 ms`, `p95 2029.260 ms`, `p99 2054.484 ms`, `max 2054.484 ms`. +`96/210` exact results were `DATA_STALE`; per-binding provider-confirmation to +consumer-receipt maxima reached `3179.391 ms`, so the `2,000 ms` execution +gate fails honestly. The data was real provider-derived live state, not test +or generated market data. + +The route itself is correct. Every returned observation carried +`execution_view=STABLE_STREAM_GATEWAY`, the internal V2 live-view lineage and +the expected exact identity; no secondary spool or venue REST route appeared. +Direct read-only timing from *each* query replica showed that the container +named `stream_v2_active` is not the current Redis lease holder and rejects the +ten reads quickly (`1.7-4.7 ms` after initial TLS), while the current holder +named `stream_v2_passive` returns the same ten exact live records in +`2.1-11.2 ms` after handshake. The labels are deployment names, not authority +claims; the two-URL fallback behaves correctly and is not the two-second tail. + +**Root cause and decision boundary.** `MarketDataService.reference_data_batch` +uses `BoundedWarmupExecutor` for all reference products. The new local +`INTERNAL_STREAM` lane has no explicit policy, so it inherits the external +provider default: max concurrency `4`, token bucket `5 requests/s`, burst `5`, +and retry policy `4`. A ten-item execution batch therefore queues behind a +provider rate budget even though all reads are bounded, authenticated local +gateway calls. That artificial queue consumes the full freshness margin and +makes otherwise-current data stale at consumer receipt. This is an in-scope +blocking defect, not an SLA relaxation or provider-quality failure. + +The only valid next source slice is a named bounded `INTERNAL_STREAM` policy: +retain finite local concurrency, disable external-provider rate pacing, and +perform one attempt because the reader already tries its declared current +lease-holder URLs and a missing/stale/gapped/fenced view must remain typed +fail-closed. Required regressions: ten-item execution batch has no artificial +rate queue; local transport failure/stale/gap remain terminal typed outcomes; +external Binance/OKX/DNSE budgets remain unchanged; and both query replicas +preserve active/passive lease fallback. It requires a new immutable **query +reader only** image and rolling recreate of only `query_v2_1` and `query_v2_2`, +with the currently active `25dce61` candidate retained as exact rollback. Then +and only then rerun one strict 300-second C2 test with `require_all=True`. +No further runtime action or release is permitted from this precheck. + +### R1.33 - Internal execution-read admission and freshness correction (`SOURCE_COMPLETE / RUNTIME_ROLLED / C2_BLOCKED`, 2026-09-19) + +**Goal.** Remove the self-inflicted external-provider queue from the V2 +execution MARK/INDEX read path while keeping the existing `2,000 ms` +freshness, canonical lineage, lease/gap fencing and typed fail-closed behavior. +This is a narrow reader-path correction; it is not a Kafka, provider quota or +freshness-SLA relaxation. + +**Approved scope.** + +1. Declare `INTERNAL_STREAM` as an explicit bounded local policy: finite + concurrency aligned with the existing reader connection pool, no token-rate + pacing, one logical attempt, and a short local circuit cooldown rather than + the 30-second external-provider default. The reader may still try its + declared active/passive gateway URLs within that one deadline. +2. Bind execution-reader singleflight identity to the full caller policy + (`source_policy_id` and `max_freshness_ms`) so one consumer cannot receive a + result admitted under another consumer's conditions. +3. Carry the catalog's `freshness_basis` through the private live-view response + and use it during query admission. This is required because the stream role + is the only authority that knows which basis admitted its current record. + Provider confirmation can govern recency only for bindings that explicitly + declare it; source event time remains immutable lineage and is never + rewritten as fresh. +4. Bound the complete active/passive read by the execution request's remaining + deadline, rather than allowing two independent per-URL two-second waits. + A timeout, stale view, gap, fence or source-policy mismatch remains a typed + failure with no venue REST retry. +5. Revalidate every returned current snapshot against one response-time clock + before assembly. Execution-grade reference batches already cannot contain + history or a different reference product by contract; this gate prevents a + valid-at-fetch snapshot becoming an `OK` stale response during batch work. + +**Invariants and exclusions.** Binance, OKX and DNSE external provider budgets, +retry behavior and circuit policy remain byte-for-byte unchanged. V1, public +schemas, Rust, stream-gateway ownership, Kafka, projectors, Redis, SQLite, +catalog bindings, consumer manifests and all order paths are outside this +source slice. No synthetic market value is used outside deterministic tests. + +**Required source gates.** Regressions must prove a ten-item internal batch has +no token wait and remains concurrency-bounded; external venue policy values +are unchanged; policy-distinct concurrent requests never singleflight together; +provider-confirmation and source-event freshness evaluate correctly; transport +deadline covers active/passive failover as one logical read; stale/gap/fence +remain terminal; and shared response-time validation cannot return stale data. + +**Runtime decision boundary.** After source gates pass, build one immutable +reader image. Because the private response must carry stream-authoritative +freshness basis, a later bounded packet may rolling-recreate exactly +`stream_v2_active`, `stream_v2_passive`, `query_v2_1` and `query_v2_2`, +retaining the currently deployed `25dce61` candidate as exact rollback. It +then runs the existing strict C2 300-second `require_all=True` test over all +ten active execution MARK/INDEX bindings. No rollout is implied by this source +task. + +**Approved runtime packet (2026-09-19).** Owner approved immutable reader +image `qdl-v2-python:2.0.21-292fb97@sha256:531554f99f4cc03e5768c6bdc2172d58528311aa1e1df601d74a651c07992c25` +from source `292fb97e3ca52a59750b247f0f38ac89a43ce36b`, with serial rolling +recreate of exactly `query_v2_1`, `query_v2_2`, `stream_v2_active`, and +`stream_v2_passive`. It preserves their existing runtime, TLS and stable-state +mounts and leaves V1, Rust, ingestors, projectors, bar edge, Kafka, Redis, +SQLite, Trading System, alpha and all order paths untouched. Exact rollback is +`qdl-v2-python:2.0.21-rc.1-25dce61@sha256:ee0154e88e9a26213d7354532edc01e441f3a5754cb2b64c8f672d8593189414` +on only those four roles. Runtime evidence is held outside Git at +`/home/bobby/.local/state/qdl-v2/mark-index-r133-292fb97-20260919T151548Z`; +it records no secret value. Exit requires the four reader health/image/mount +checks plus one strict 300-second C2 run with `require_all=True` over the ten +execution MARK/INDEX bindings and a measured consumer-call-to-usable p99 at or +below `2,000 ms`. This packet is still a narrow correction, not a broad release +or manifest promotion. + +**Completed source slice (2026-09-19).** `BoundedWarmupExecutor` now declares +`INTERNAL_STREAM` with concurrency `4`, no token-rate pacing, one attempt and +a `1,000 ms` local circuit cooldown. This retains bounded failure pressure +without turning a recovered lease-holder into a 30-second stale-data outage. +The execution reader applies one total deadline across both declared gateway +URLs, while still allowing a fast fenced/lease-miss response to fail over to +the active holder. Query singleflight identity now includes source policy, +freshness bound and deadline. Current execution reads inherit the smaller of +their request deadline and freshness bound, so a nominal `20,000 ms` reference +deadline cannot hold a `2,000 ms` execution price read open. + +The stream private response now sends its actual catalog freshness basis in a +private response header. It is additive for an older query reader; a new query +reader treats an absent header conservatively as `SOURCE_EVENT`. The new reader +preserves that basis and provider-confirmation timestamp in typed observation +lineage. Query validates provider confirmation only when the active stream +explicitly declared it; otherwise it validates immutable source-event time. +The response assembly uses one clock after all bounded refresh work completes. + +**Source evidence.** `python3 -m py_compile` passed for every changed +production and test module, and `git diff --check` passed. The immutable +existing `qdl-v2-python:2.0.21-rc.1-25dce61` image ran network-disabled, +read-only tests with a temporary `/tmp` filesystem and `--rm` cleanup: + +1. `tests.test_execution_mark_index_live_view`, + `tests.test_phase10_universal_warmup`, and + `tests.test_phase113_reference_v2`: **76 passed, 0 failed** after the final + stale-at-response regression. +2. The final broader query/reference/consumer selection including + `tests.test_phase104_reference_batch`, + `tests.test_phase104_v2_query_stream_integration`, + `tests.test_phase105_consumer_acceptance`, + `tests.test_phase115c_five_liquid_handoff`, and + `tests.test_reference_l2_consumer_acceptance`: **143 passed, 0 failed**. + +The new regressions cover ten-item internal admission with no token wait, +bounded concurrency, one retryable failure attempt, unchanged Binance/OKX/DNSE +policy values, the explicit `1,000 ms` local circuit cooldown, active/passive +total-deadline behavior, policy-distinct singleflight, explicit +provider-confirmation versus source-event freshness, stale-at-response +rejection, existing gap/fence rejection and no REST fallback. +`ruff` is not installed in the retained immutable runtime image; syntax, +whitespace and the repository's relevant deterministic suites passed. No image, +container, service, Kafka/Redis/SQLite state, provider call or market data was +created by source verification. + +**Pre-roll status.** `SOURCE_COMPLETE / RUNTIME_PENDING`. The required rollout +packet was four reader roles rather than query-only because the stream must +emit the authoritative freshness-basis header. That is an additive private +protocol change with query-first rolling compatibility; the approved packet +below was subsequently applied. + +**Runtime result (2026-09-19, approved four-reader packet).** Compose preflight +used the existing full runtime chain plus the final candidate overlay. It then +serially recreated only `query_v2_1`, `query_v2_2`, `stream_v2_passive`, and +`stream_v2_active`. All four are healthy on +`qdl-v2-python:2.0.21-292fb97@sha256:531554f99f4cc03e5768c6bdc2172d58528311aa1e1df601d74a651c07992c25`; +their stable-state, TLS and host runtime mounts are unchanged. V1, Rust, +ingestors, projectors, bar edge, Kafka, Redis, SQLite, Trading System, alpha +and order paths were not recreated or configured. The exact four-role rollback +image remains +`qdl-v2-python:2.0.21-rc.1-25dce61@sha256:ee0154e88e9a26213d7354532edc01e441f3a5754cb2b64c8f672d8593189414`. + +The strict C2 harness correctly failed closed before its 300-second duration: +`require_all=True` received typed `DATA_STALE` results and terminates on the +first non-OK batch by design. Successful real consumer reads prove the R1.33 +admission correction itself: no direct venue REST or V1 fallback was attempted; +query replica 1 recorded `242.881 ms` consumer-call-to-usable and provider +confirmation-to-usable p99 `1,251.462 ms` for ten bindings, while replica 2 +recorded call p99 `209.934 ms` and provider-confirmation-to-usable p99 +`1,978.744 ms` across five complete batches. These are diagnostic samples, not +a C2 certificate, because later batches returned typed stale results. + +**Fail-closed diagnosis and decision boundary.** The remaining failure is not +the former external-provider limiter or its `1,000 ms` local cooldown: C2 saw +`DATA_STALE`, not queue timeout or circuit rejection. Five OKX SWAP logical +MARK/INDEX products pair `mark-price` with `index-tickers`; the Rust core +correctly preserves each component's immutable receipt lineage and publishes +the pair with the older component confirmation. OKX documents quiet-channel +cadence of up to 10 seconds for unchanged mark price and up to 60 seconds for +unchanged index tickers. A hard two-second event-confirmation rule for this +paired product therefore rejects a healthy, unchanged provider state. This is +a contract-semantic defect, not a condition that can be fixed by relaxing the +SLA, retrying C2, or rewriting a timestamp. + +The required follow-up is a separately approved, provider-neutral source +slice: retain both component timestamps unchanged, add component-aware +session/subscription liveness to the live-view admission, and allow a quiet +component only when its exact session/generation is live, no gap/fence is open, +and the bounded venue-declared quiet cadence is still valid. It must fail +closed on disconnect, generation change, missing subscription state or expired +quiet cadence, and expose the distinction in typed lineage/status. Required +tests are quiet-connected, actual stale/disconnect, reconnect/generation, +gap/resync and identity isolation for Binance and OKX; only then may a new +Rust/reader rollout packet and one replacement 300-second C2 be proposed. +No semantic source or runtime change for that follow-up has been made here. + +**Operational observation.** The runtime remains correct but carries a +19-file historical Compose override chain. This rollout used that exact chain +and a final four-role overlay, so it did not alter topology or mounts. The +chain is an auditability/operability debt, not the C2 root cause; it is outside +this narrow packet and must be consolidated in a separately approved cleanup +slice. Candidate, named rollback image and bounded external evidence directory +are retained; no cleanup is performed while the packet is unresolved. + +### R1.34 - Component-aware quiet-channel MARK/INDEX admission (`SOURCE_COMPLETE / RUNTIME_ROLLED / C2 PASS / CLOSED`, 2026-09-19) + +**Goal.** Correct the execution MARK/INDEX read contract for provider channels +which legitimately repeat an unchanged value slowly, without rewriting source +timestamps, accepting a stale/disconnected source, changing the global +`2,000 ms` request latency target, or introducing a provider REST fallback. +The bounded latest view remains a read-committed, fenced projection of the +Rust canonical event. It is not a new cache or a second authority. + +**Approved scope and semantic contract.** + +1. Propagate the existing governed `event_recency_policy` and + `max_session_liveness_ms` from `DataRequirement` through the additive V2 + reference SDK/API/domain request into the private live-view request. A + quiet read is possible only for an explicit `OBSERVE` requirement with a + declared provider-session SLA; existing callers retain blocking event-age + behavior. +2. Add a signed, generated, provider-neutral component cadence policy to the + existing MARK/INDEX acquisition contract. It declares bounded quiet windows + for each physical component, rather than hard-coding a symbol in Python: + Binance's combined `BOTH` frame and OKX's distinct `MARK`/`INDEX` frames + are both represented by the same component model. The Rust core continues + to own parsing, same-generation pairing, canonical identity and immutable + component receipts; it must not invent a current timestamp. +3. The active stream live-view admits a quiet pair only after it verifies all + of: exact canonical instrument/policy/revision; current gateway epoch; no + gap/resync fence; valid paired component lineage; every component still + within its signed quiet window; and an exact source session ID, connection + generation and config revision whose bounded liveness record is `LIVE`. + Missing, malformed, ambiguous, disconnected, clock-skewed, generation- or + config-mismatched session evidence, a component whose cadence expires, or a + gap/resync remains typed fail-closed. The original source event time and + per-component receipt timestamps remain observable lineage. +4. The private protocol remains rolling-safe: the stream accepts legacy + strict requests and the additive quiet-aware request; new result evidence + is carried in additive headers/labels so an older query reader remains + conservative. Public `/v2` schema changes are additive and require + regenerated OpenAPI/SDK evidence. + +**Explicit exclusions.** No V1 change or fallback, external Binance/OKX/DNSE +quota/circuit change, REST polling bypass, Kafka/topic/offset mutation, Redis +flush, SQLite deletion, new service/container/symbol worker, Trading System, +alpha or order-path change is permitted. `max_freshness_ms=2,000` remains the +bounded consumer request/usable-read budget; component value age is reported +separately and may only be admitted under the declared quiet contract. + +**Required source gates.** Python/Rust/API/SDK regressions cover both Binance +and OKX: fresh pair; quiet-but-live pair; expired mark; expired index; +disconnected heartbeat; stopped heartbeat; malformed/duplicate session record; +generation/config mismatch; gap/resync; reconnect; delayed old generation; +component and venue isolation; unchanged original timestamps; exact +manifest/entitlement propagation; legacy strict request; OpenAPI/SDK +compatibility; and no direct venue REST. Rust configuration tests prove that a +component policy is complete and bounded for every generated MARK/INDEX +binding. Source tests run in the immutable builder image with network disabled. + +**Runtime boundary and exit.** After source gates, build one immutable Rust +core image and one immutable Python reader image, seal one runtime revision, +and rolling-recreate only `rust_core`, `rust_core_2`, `rust_core_3`, +`stream_v2_active`, `stream_v2_passive`, `query_v2_1`, and `query_v2_2`. +Current exact image/config pairs become named rollback coordinates; V1, +ingestors, projectors, BAR edge, Kafka topology/offsets, Redis, SQLite, +Trading System, alpha and the order path stay untouched. A replacement strict +300-second C2 must demonstrate all ten Binance USD-M/OKX Swap MARK/INDEX +bindings, typed quiet/session/component evidence, no gap/fallback/direct REST, +and consumer-call-to-usable p99 at or below 2,000 ms. It reports component +value age separately rather than mislabelling an unchanged provider value as a +new event. Only that receipt can remove the R1.33 C2 block. + +**Rollback and decision boundary.** A source/test failure stops before image +build. A runtime/C2 failure rolls only those seven roles back to the recorded +pre-R1.34 images and runtime directory; it never retries by weakening a +cadence or freshness rule. Test images/cache are inventoried after the source +slice; only active and named rollback images are retained while the runtime +packet remains unresolved. Other read-only findings are recorded separately; +they are not changed by this task without scope approval. + +**Source gate (`PASS / IMMUTABLE IMAGES AND SEALED RUNTIME PENDING`, +2026-09-19).** The source contract now carries `event_recency_policy` and +`max_session_liveness_ms` end-to-end as an additive V2 reference request. +`OBSERVE` is permitted only for an execution `MARK_INDEX_PRICE` snapshot with +an explicit session SLA; legacy requests remain `BLOCK`. The generated +acquisition contract supplies complete bounded component cadence for every +newly generated execution binding (Binance USD-M `BOTH=5,000 ms`; OKX Swap +`MARK=15,000 ms`, `INDEX=70,000 ms`). Rust rejects an out-of-bounds policy and +refuses to re-materialize a pair when any retained component expires; it never +re-dates the provider event. The stream live view additionally requires exact +session ID/generation/config, `LIVE` liveness, current gateway epoch and no +gap/resync fence, while query rechecks the same evidence immediately before +returning it to a consumer. The private reader rejects missing session +provenance and validates headers against immutable pair lineage. + +**Tests actually run.** All ran with `--network none`, read-only source and +disposable tmpfs target/state unless stated otherwise: + +- `python -m unittest -v tests.test_execution_mark_index_live_view + tests.test_phase113_reference_v2 tests.test_production_catalog`: `37/37` + pass. Covers both venues, strict legacy behavior, quiet-but-live, expired + component, disconnect/stopped heartbeat, session generation mismatch, + gap fence, exact identity, session-provenance corruption, no REST fallback + and generated cadence completeness. +- `python -m unittest -v tests.test_fund_phase5_api + tests.test_fund_phase5_contracts tests.test_fund_phase5_consumer + tests.test_fund_phase5_stream_sdk tests.test_fund_phase5_e2e`: `45/45` + pass. `contracts/v2/openapi.snapshot.json` was regenerated and the + additive public SDK/API schema is frozen by the contract test. +- `cargo fmt --all -- --check`, `cargo clippy -p qdl-realtime-core --lib + --locked --offline -- -D warnings`, and `cargo test -p qdl-realtime-core + --lib --locked --offline`: format/clippy pass; `42` pass, `1` explicitly + skipped Redis integration test because no isolated Redis URL was supplied. + The disposable builder `qdl-rust-builder:r134-test` existed only to provide + pinned dependency cache for the offline Rust test; it has not started a + runtime role and will be removed after the final immutable image is built. +- A network-isolated parse of `consumers/stable/trading-system-paper.yaml` + confirms exactly `10` execution `MARK_INDEX_PRICE` requirements, each with + `event_recency_policy=OBSERVE`, `max_session_liveness_ms=45,000`, and the + independent quality `stale_policy=BLOCK`. This proves that quiet-channel + admission is a narrow event-recency entitlement, not a relaxation of + quality/freshness blocking. + +No provider request, V1 fallback, Kafka/Redis/SQLite mutation, service +recreate, Trading System/alpha/order action or source-runtime mount change +occurred during this source gate. The active R129 runtime has the ten logical +MARK/INDEX bindings but predates the signed component cadence. The next +permitted action is therefore exactly one new sealed runtime revision from +this source, followed by the approved seven-role rolling packet and one strict +300-second C2 receipt. R1.34 remains `SOURCE_COMPLETE / RUNTIME_PENDING`. + +**Pre-roll validation correction (`IN PROGRESS`, 2026-09-19).** A direct +`StableSourceCatalog` import exposed a real Python import cycle through a +type-only MARK/INDEX annotation; the annotation is being made type-check-only +and a clean-process regression is required. A separate candidate preflight +also tried to parse the legacy acquisition file mounted under `/runtime`. +That file is not an active reader input: the standard query/stream roles load +the signed source and acquisition contracts from their immutable image under +`/app/config/v2`, while `/runtime` supplies authority/core state. The correct +response is to keep the acquisition schema strict and remove the attempted +optional-URL parser relaxation, not to weaken a stable contract. This +correction is source-only; it changes no demand, provider URL, runtime mount, +service, image or data-plane state. Exit requires the direct-import regression +plus the complete existing R1.34 source suite before one final Python reader +image is built. The previously built pre-correction Python candidate is +test-only and will not be deployed. + +**Correction result (`PASS`, 2026-09-19).** The type-only import now uses +`TYPE_CHECKING`, and a fresh Python process imports `StableSourceCatalog` +without a cycle. The strict acquisition parser was retained unchanged: the +legacy `/runtime/stable-acquisition-bindings.yaml` is not read by standard +query/stream roles, whose environment explicitly resolves both source and +acquisition contracts from immutable `/app/config/v2`. A network-isolated, +read-only image run completed `83/83` tests across the direct-import, +MARK/INDEX live-view, reference, catalog, V2 API/contract/consumer/SDK and +end-to-end suites. The only output was an existing TestClient deprecation and +an expected gRPC task diagnostic; neither failed a test. The final reader +image must be rebuilt from this corrected source; the Rust binary remains the +already-tested R1.34 artifact because no Rust source changed after its +successful format/clippy/test gate. + +**Sealed-pair preflight (`IN PROGRESS`, 2026-09-19).** The new stream image +parses an acquisition contract at startup, while the currently deployed +pre-R1.34 stream does not. Read-only inspection found that the active reader +pins a `216`-binding source catalog under `/runtime` but leaves the acquisition +path unset. A new image would otherwise pair that runtime catalog with the +different image-default contract. The narrow packet must therefore explicitly +pin both `QDL_STABLE_SOURCE_BINDINGS` and +`QDL_STABLE_ACQUISITION_BINDINGS` to one sealed runtime pair. Its legacy +acquisition document will be normalized only for schema-required transport +metadata using the existing venue rules (Binance routed `public`/`market`, OKX +public/business, and explicit null for non-WebSocket modes), then parsed +against the same 216 source IDs. The transformation is required to preserve +strict validation; it must prove no binding identity, mode, provider kind, +channel, demand, L2/MARK_INDEX semantic field or topology changes other than +the approved component cadence. It creates no provider connection by itself. +Only after this parse/mapping gate passes may the approved seven-role packet +run. + +**Sealed-pair result (`PASS / ROLLING READY`, 2026-09-19).** The final +non-root reader image is `qdl-v2-python:2.0.22-58998ae`, image ID +`sha256:1ad34175322f…`, labelled with source revision +`58998ae1f4e84161c1c6dcf681ce92f4247faa63`; its network-isolated, +no-source-mount regression remains `83/83` pass. The already-tested Rust core +artifact is `qdl-v2-rust:2.0.20-f1c9e1d`, image ID +`sha256:389753b37c4f…`. The sealed R1.34 runtime pair parses under that final +reader as `216` source bindings plus `216` acquisition bindings; it preserves +ten logical MARK/INDEX contracts and maps `45` physical component entries +across the three core JSON files. The normalizer changed only `249` required +transport metadata fields and the ten approved cadence maps; it proves all +other semantic fields are byte-equivalent after restoration. Compose render +shows exactly the approved three core and four reader roles on the new images, +with only their existing TLS/state mounts and the sealed `/runtime` bind. It +explicitly pins both reader paths to the same runtime pair. + +**Read-only runtime check.** A 30-minute V2 warning/error scan found no +matching warning, error, panic or OOM record across the three cores, two +ingestors, three projectors, BAR edge, two streams and two queries; all were +`running`, had `restart=0`, and were not OOM-killed. The separate V1 fallback +service continues to log Binance Futures kline first-frame timeouts and a few +publisher-queue-full reconnects, while DNSE is stale outside its session. V1 +is explicitly preserved by this packet, so this is recorded as an existing +V1 operational finding, not altered or used to weaken V2 acceptance. The only +next action is the owner-approved seven-role rolling recreate followed by the +strict C2 300-second receipt; on failure rollback is limited to those same +roles and recorded image/runtime coordinates. + +**Rolling checkpoint 1/2 (`RUST CORE PASS`, 2026-09-19).** `rust_core`, +`rust_core_2` and `rust_core_3` were recreated sequentially to +`sha256:389753b37c4f…` with only their individual sealed core JSON bind mounts. +After each recreation, and again after all three, every core was `running`, +`restart=0`, `OOM=false`, and emitted no bounded warning/error/panic record. +Kafka, Redis, SQLite, V1, both ingestors, all projectors, BAR edge, reader +roles, Trading System, alpha and order paths were not recreated or changed in +this checkpoint. Reader rolling and strict C2 remain pending. + +**Rolling checkpoint 2/2 (`READER PASS / C2 PENDING`, 2026-09-19).** +`stream_v2_active`, `stream_v2_passive`, `query_v2_1` and `query_v2_2` were +then recreated one at a time to `sha256:1ad34175322f…`. Every reader is +`running`, healthy, `restart=0` and `OOM=false`, with no bounded startup +warning/error record. A process-local read through each new stream resolves +and parses the sealed pair as `216/216`; both queries resolve the sealed +216-binding catalog. No other role was recreated. The next action is exactly +one existing strict C2 no-order acceptance for 300 seconds with +`require_all=True`; it must report all ten execution MARK/INDEX bindings, +consumer-call-to-usable latency, no V1/direct-provider fallback and no order +or consumer-state mutation. + +**C2 identity preflight (`FAIL-CLOSED / ADDITIVE KEYRING REPAIR REQUIRED`, +2026-09-19).** The first disposable Trading-System C2 launcher attempts were +stopped before a provider or data-plane request by launcher-only Docker +argument/permission defects; their containers used `--rm` and left no runtime +state. The first actual V2 TLS request then proved the historical +`stable-trading-system` leaf expired on `2026-08-22`; replacing only its +client-side server CA with the current query CA removed the old trust error but +correctly did not make an expired identity acceptable. The preserved recovery +extension contains a successor `stable-trading-system` client certificate +valid through `2026-12-03`, and `openssl verify` confirms it is already trusted +by the active `query/client-ca-bundle.crt`. A no-order preflight with that +successor reached the V2 API and failed closed as `untrusted workload token key +or algorithm`: read-only query inspection proves the active public JWT keyring +has only the five `v1` IDs, while the prepared additive recovery packet +`c42e1160...4400adde0` defines the matching +`stable-trading-system-rs256-v2` key and subject. + +This is a reader rollout configuration omission, not a MARK/INDEX, provider, +Rust, latency or quality-SLA failure. The exact repair is to layer the existing +public-only recovery keyring onto the existing R1.34 Compose environment and +serially recreate only `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive` on their current images and sealed runtime. Before the +replacement C2, query inspection must prove the additive key IDs/subjects +without printing any key value. Rollback is the current five-key environment +and recreation of those same four reader roles only. V1, Rust cores, +ingestors, projectors, BAR edge, Kafka, Redis, SQLite, TLS files, Trading +System, alpha and order paths remain excluded. The replacement acceptance is +still exactly two no-order 300-second `require_all=True` ten-binding probes, +one per query replica, with the current query CA and successor workload +identity copied only to container tmpfs. + +**C2 quiet-channel acceptance metric correction (`SOURCE COMPLETE / C2 PASS`, +2026-09-19).** The approved additive keyring repair was applied to +only `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive`; read-only process inspection confirms the successor +Trading-System key ID and subject are present without recording any key +material. A disposable authenticated preflight then completed five exact +ten-binding V2 batches with no V1 fallback, direct-provider call, order, or +consumer-state mutation. Consumer-call-to-usable measured `p50 74.967 ms`, +`p95/p99 280.999 ms`. Its old acceptance predicate nevertheless rejected a +quiet component because it incorrectly treated immutable +`provider_confirmation_ns` as the delivery-latency SLA (`p99 2115.892 ms`). + +That predicate conflicts with this R1.34 contract: an admitted +`COMPONENT_SESSION_LIVE` response retains original component/provider receipt +timestamps by design, while query has already fail-closed on exact session, +generation, config revision, gap/fence and signed per-component cadence. The +approved source-only correction is limited to the disposable C2 harness and +its deterministic tests. It will retain and report immutable provider and +component ages as lineage diagnostics, independently validate the returned +quiet/session evidence at consumer receipt, and gate only +consumer-call-to-usable `p99 <= 2,000 ms` plus complete typed live evidence. +It may not alter source timestamps, component cadence, requirement freshness, +manifest, provider adapters, Rust core, query/stream runtime, V1, Kafka, +Redis, SQLite, Trading System, alpha or the order path. Required tests cover +strict and quiet-live responses; missing/malformed session evidence; expired +session/cadence; and preservation of original provider confirmation lineage. +After the source gate, one disposable no-order 300-second C2 probe per query +replica will run against the already-rolled R1.34 runtime. The probe image is +ephemeral and `--rm`; there is no additional service rollout. Any failure +remains fail-closed and leaves the current seven-role runtime untouched. + +**C2 quiet-channel source gate (`PASS / CLIENT BUILT`, 2026-09-19).** The +acceptance harness now emits schema +`qdl.execution-mark-index-consumer-latency.v2`. It retains +`provider_confirmation_to_usable_ms` and the two component ages as immutable +lineage diagnostics, separately records session-liveness-to-usable, and gates +only the actual SDK `consumer_call_to_usable_ms` p99 together with complete +per-binding quiet/session evidence. It does not change a source timestamp, +quiet cadence, requirement, manifest, endpoint, runtime role or provider +adapter. Deterministic regressions prove a quiet connected response with an +unchanged 60-second provider receipt remains admissible only within its +component cadence; strict mode remains typed; disconnected, expired-session, +expired-component, missing/malformed evidence and direct-provider lineage all +fail closed; and a 70-second immutable provider-age cannot override a +281-millisecond consumer-call gate. The complete relevant isolated suite ran +inside the final immutable client image with network disabled: + +`python -m unittest -v tests.test_execution_mark_index_consumer_latency tests.test_execution_mark_index_live_view tests.test_phase113_reference_v2 tests.test_production_catalog`: +`44/44 PASS`. + +`git diff --check` and `python3 -m py_compile` for the changed harness/test +also passed. No runtime, durable store, provider connection, order, consumer +state or V1 path was modified by this source gate. The subsequent C2 probes +used that image; no reader/core recreate was needed because the deployed +R1.34 reader/runtime already contained the quiet contract. + +**Replacement C2 acceptance and closure (`PASS`, 2026-09-19).** Two exact +no-order, `require_all=True`, 300-second consumer probes ran from the +immutable client image built from `5ac0200b734902d43b5873e2a07cd3c5d4d7a404`, +one targeted at each existing V2 query replica. Each completed `151` batches +and `1,510` exact MARK/INDEX reads across all ten Binance USD-M/OKX Swap +execution bindings; every binding recorded at least `151` samples against the +required `149`. Both receipts use the successor Trading-System identity and +the additive keyring repair, report `errors=[]`, `gate_passed=true`, +`v1_fallback_attempted=false`, and `direct_provider_request_attempted=false`. +They are retained outside Git under +`/home/bobby/.local/state/qdl-v2/mark-index-r134-58998ae-r3-20260919T172500Z/c2-execution-mark-index-r134-20260919T174000Z/query_v2_{1,2}-r5/`. + +| Metric | `query_v2_1` | `query_v2_2` | +| --- | ---: | ---: | +| Consumer call -> usable p50 / p95 / p99 | 71.386 / 169.013 / **285.985 ms** | 71.991 / 149.056 / **266.952 ms** | +| Session-liveness -> usable p99 | 1,462.775 ms | 1,957.237 ms | +| Immutable provider-confirmation age p99 (lineage only) | 3,120.321 ms | 3,334.456 ms | +| Component age p99 (lineage/cadence only) | 2,923.991 ms | 3,157.486 ms | + +The two delivery stages and both recency modes were observed. The component +ages remain under their signed cadence; they are intentionally not relabelled +as consumer response time. All seven R1.34 runtime roles remain healthy, +`restart=0`, `OOM=false`: reader roles use +`qdl-v2-python:2.0.22-58998ae@sha256:1ad34175322f…`, Rust core roles use +`qdl-v2-rust:2.0.20-f1c9e1d@sha256:389753b37c4…`, and sealed runtime remains +`mark-index-r134-58998ae-r3-20260919T172500Z`. No post-source reader/core +recreate was needed for the harness-only correction. V1, Kafka, Redis, +SQLite, ingestors, projectors, BAR edge, Trading System, alpha and order paths +were untouched. + +**Scoped hygiene.** The disposable C2 container image +`qdl-v2-python:2.0.22-5ac0200@sha256:0e5f67fc443a…` had no container users and +was removed after C2. Docker image storage returned from `19.85 GB` to +`19.14 GB`; all C2 containers used `--rm` and none remain. BuildKit cache is +`13.51 GB` with `2.587 GB` reclaimable after the build; it includes pre-existing +shared cache, so no broad cache prune was performed without a separately scoped +cleanup approval. Active and named rollback runtime images remain retained. + +**Bounded observations, not R1.34 blockers.** No V2 reader/core +warning/error/panic/OOM record appeared during the acceptance window. V1 logged +one Binance Futures kline first-frame timeout followed by its existing +`303 s` reconnect backoff; DNSE reported stale outside its trading session. +The historical `19`-override Compose chain remains an auditability debt. These +are pre-existing operational items outside this MARK/INDEX correction; no +quality rule was weakened and no unrelated source/runtime change was made. + +### R1.35 - Universal quality convergence, endpoint certification and release closure (`IN_PROGRESS / B2.1 CANDIDATE ROLLED BACK`, 2026-09-19) + +**Why this exists.** R1.34 correctly certified the narrow execution +`MARK_INDEX_PRICE` live-view route, but it is not a certificate for every V2 +product or consumer route. A read-only spool scan subsequently classified +`185/216` catalog bindings as live, `21` as over their raw event-age budget and +`10` as no-event. The latter ten are deliberate inventory entries: four +DNSE/VN bindings remain `V1_PRIMARY`, and six Spot bindings have neither an +active ingestor nor consumer entitlement. The other twenty-one cannot be +accepted or dismissed from raw spool age alone: `MARK_INDEX_PRICE`, `TRADE` +and `BOOK_DELTA` may use the explicit quiet/session contract. `QUOTE` is +`STRICT_EVENT` by default; whether native update-on-change BBO may use a +separately signed session-aware contract is the exact R1.35-B1 question, not +an assumed exception. Four quote rows were near or beyond their raw-event +threshold in that sample: Binance USD-M BNB and OKX Swap ETH/BNB/DOGE. + +This closure implements and proves one coherent answer. It does not weaken an +SLA, backdate provider timestamps, turn an absent event into zero, silently +route through V1/venue REST, or make an unrequested catalog entry an execution +product. It follows the fund-grade guide sections 3.2-3.8, 4, 9, 13-19, +24-29 and 36-43, especially the distinction between durable event log, +latest-state projection, per-feed quality semantics, consumer-visible +freshness, replay and release evidence. + +**Program invariants.** + +1. The typed quality answer must be semantically identical at Rust core, + Python V2 query/SDK and audit/report boundaries. Cross-language parity is + mandatory; a second raw-age heuristic is not an authority. +2. `QUOTE`, price-bearing `BOOK_SNAPSHOT`, and final BAR reads remain + execution-strict: exact identity, complete data, no open gap and declared + finality must all hold. A `QUOTE` may use session-aware admission only when + its *signed source binding* declares the native provider lane as + `ON_CHANGE` BBO and its exact consumer requirement elects `OBSERVE`; this + is a delivery contract, not an SLA relaxation. `STRICT_EVENT` remains the + default for every other quote source. An `ON_CHANGE` quote is blocked on a + disconnected/expired session, generation or config mismatch, open gap, + missing immutable last-event lineage, or any source/consumer-policy + disagreement. `BOOK_SNAPSHOT` and final BAR never use quiet admission. +3. `TRADE`, `BOOK_DELTA` and execution `MARK_INDEX_PRICE` may be admitted only + when their manifest explicitly selects `OBSERVE` and the typed answer proves + a live provider session, exact session/generation/config revision, no + gap/resync fence and an unexpired per-component cadence. Their immutable + source/provider timestamps remain diagnostic lineage, never forged current + data. +4. `V1_PRIMARY`, `CATALOG_DARK`, no-active-ingestor and out-of-session states + are explicit expected classifications. They cannot be counted as V2-ready or + hidden as generic failures. +5. All real acceptance reads use the public V2 SDK with workload mTLS/JWT, + declared entitlement and signed cursor. They make no direct venue request, + submit no order and mutate no alpha, Trading System or consumer state. +6. V1, Kafka topology/offsets, Redis persistence, SQLite state, active alpha + containers and the order path remain unchanged unless a later approved + packet names an exact role, digest, runtime revision, rollback and blast + radius. + +#### R1.35 Closure Delivery Map - Three technical phases plus hygiene/release (`B/C PASS / D PENDING`, 2026-09-21) + +This is the authoritative compact execution map for the remaining R1.35 +closure. It refines, rather than replaces, the detailed B/C/D sections below +and follows the fund-grade guide sections 3.2-3.8, 4.1-4.3, 9.4-9.10, +13.1-13.7, 16, 17, 18, 19, 24 and 25. No step silently promotes a route, +changes a source timestamp, creates a per-symbol service/container, or treats +a passing process health check as data acceptance. + +| Delivery phase | Status | Outcome required before the next phase | +| --- | --- | --- | +| **1. R1.35-B1 - Source delivery semantics and quality authority** | `PASS / source-only` | One Rust/Python/query decision for `STRICT_EVENT` versus signed `ON_CHANGE` BBO delivery, with exact manifest authorization and no false live result. | +| **2. R1.35-B2 - Bounded reader rollout and strict quote C2** | `PASS / runtime-certified` | A sealed reader bundle/image proved all ten current Binance USD-M/OKX Swap execution BBO routes through both query replicas for 300 seconds without stale false rejects or hidden fallback. | +| **3. R1.35-C - Full endpoint, binding and consumer certification** | `PASS / runtime-certified` | Every currently active, entitled V2 product has per-binding consumer evidence, typed status parity and bounded latency/resource evidence. | +| **R1.35-D - Hygiene, provenance and immutable release** | `PENDING / requires C exit and explicit cleanup/release approvals` | Source, runtime, artifact, rollback, Git lineage and published release are one auditable state; only disposable artifacts are removed. | + +**Closure decision record (2026-09-20).** This is still exactly a three-phase +technical closure plus a hygiene/release phase; the findings below do not +create a fourth runtime architecture track. + +1. The audit's `21` stale rows must first be interpreted through the shared + typed contract. `MARK_INDEX_PRICE`, `TRADE` and `BOOK_DELTA` rows declared + `QUIET_SESSION` are evaluated by session/component liveness, generation, + gap and cadence -- never by raw last-event age alone. The scanner, public + SDK status and acceptance verifier must return the same classification. + They may not hide a true disconnect, gap, generation change, expired + cadence or incomplete view. +2. Strict-feed evidence remains independently non-negotiable. Before B2 can + pass, a read-only two-query-replica status matrix must explicitly classify + each active execution `QUOTE`; the currently named strict candidates are + Binance USD-M `BNBUSDT` and OKX Swap `ETH-USDT-SWAP`, `BNB-USDT-SWAP` and + `DOGE-USDT-SWAP`. The matrix seals the actual binding IDs, routing revision, + state/reason, event age, session age, gap/watermark and consumer eligibility + at test time. A genuine strict stale result is an in-scope B2 defect: fix + its shared source/projection lineage and rerun the matrix; neither relabel + it quiet nor widen its SLA. +3. `lucid_sinoussi`, `youthful_shamir` and `qdl-admit-1d` are non-canonical + test containers, not evidence of a serving-role fault. They remain outside + B/C data acceptance and are handled only by the exact D cleanup inventory, + retention and post-removal health gate. No broad Docker prune is implied. + +**Invariant for all three phases.** A measurement must distinguish provider +event age, session liveness, host receipt, durable projection, and +consumer-call-to-usable latency. A quiet-channel result is never used to +upgrade a strict route; a healthy process is never accepted as proof of data +quality. No phase adds a per-symbol service, container, image or timer, and no +phase changes V1, Kafka/Redis/SQLite durability, provider quotas, public V2 +schemas, Trading System, alpha or order authority outside an explicitly +approved reader rollout packet. + +##### R1.35 closure charter - three technical phases plus hygiene/release (`PLANNED / NO NEW RUNTIME AUTHORITY`, 2026-09-20) + +This charter is the single execution order for the remaining release closure. +It reconciles the older `R1.35-A/B/C/D` records below without opening another +architecture program: **A** maps to the shared quality/auditor source contract, +**B** to the strict-QUOTE reader correction and C2 proof, **C** to the complete +public endpoint/consumer certificate, and **D** to cleanup and immutable +publication. Detailed design remains governed by +`upgrade/quant-data-layer-fund-grade-upgrade-architecture.md` sections +3.2-3.8, 4.1-4.3, 9.4-9.10, 13.1-13.7, 16.1-16.4, 17.1-17.7, 18.1-18.7, +19.1-19.5 and 24-25. + +**Common invariants and exclusions.** Every phase uses the sealed, +manifest-derived active inventory rather than an assumed BTC-only universe. +`EXPECTED_V1_PRIMARY`, `EXPECTED_DARK`, `OUT_OF_SESSION` and no-active-ingestor +rows are explicit exclusions with a named reason; they are neither V2-ready nor +generic test failures. Real acceptance reads only via public V2 SDK with the +declared workload identity, mTLS/JWT and signed cursor. It makes no direct +venue request and performs no order, signal, sizing, Trading System, alpha or +broker mutation. The work may not change provider timestamps, widen a freshness +budget, reinterpret a strict route as quiet, create a per-symbol worker/service +or use V1/REST as an unrecorded execution substitute. + +###### A. Typed quality semantics and stale-inventory convergence (`PASS / SOURCE-CONTRACT; runtime proof belongs to B`) + +**Goal.** Give Rust core, Query, SDK and the read-only auditor one +provider-neutral quality decision so raw event age is not mistakenly treated as +the liveness test for an explicitly quiet feed, while strict market data remains +strict. This phase covers the observed `MARK_INDEX_PRICE`, `TRADE` and +`BOOK_DELTA` stale rows and prevents their scanner-only false positives from +being confused with real execution degradation. + +**Required implementation and tests.** The shared decision must preserve exact +instrument/feed/venue identity, immutable source/receipt timestamps, event age, +session liveness, component cadence, generation/config revision, watermark, +completeness and gap/resync state. It must classify `STRICT_EVENT`, +`QUIET_SESSION`, `FINAL_SCHEDULED`, expected V1/dark and out-of-session states +once and identically across Rust/Python/SDK/auditor. Golden and contract suites +must cover quiet-but-connected, stopped heartbeat, disconnect, changed +generation/configuration, cadence expiry, duplicate, out-of-order, open gap, +resync, final/non-final BAR, cross-symbol/venue mix and missing lineage. A +quiet result must never satisfy `QUOTE`, `BOOK_SNAPSHOT` or final-BAR strict +requirements. + +**A exit gate.** The inventory count and every expected classification are +sealed; Rust/Python/Query/SDK/auditor parity has zero disagreement; all source +format, Clippy, Rust/Python golden, contract/SDK and generated-contract drift +tests pass. This is source-only: its rollback is the preceding source commit; +it authorizes neither an image build nor a role recreate. A discovered strict +route remains a B defect, never an A exception or an SLA change. + +###### B. Strict QUOTE lineage repair and bounded reader acceptance (`IN PROGRESS / REQUIRES A EXIT`) + +**Goal.** Resolve actual strict `QUOTE` health for every active Binance USD-M +and OKX Swap execution BBO binding, beginning with Binance `BNBUSDT` and OKX +`ETH-USDT-SWAP`, `BNB-USDT-SWAP`, `DOGE-USDT-SWAP`. The result must distinguish +a quiet but live native BBO lane from a genuinely stale, disconnected, gapped or +mis-materialized view. `ON_CHANGE` admission is permitted only when the signed +source binding and exact consumer requirement authorize it; all other quotes +remain `STRICT_EVENT`. + +**Required evidence and repair discipline.** First collect a bounded, +payload-free typed-status matrix from both V2 query replicas for the full +manifest-derived execution quote set and control feeds (`TRADE`, `BOOK_DELTA`, +`BOOK_SNAPSHOT`, `MARK_INDEX_PRICE`). Record state/reason, eligibility, raw +event age, session/component age, generation/config revision, gap, completeness, +watermark, source lineage and consumer-call-to-usable timing. If a strict route +fails, repair only the proven shared provider admission, canonical +materialization, latest-view or Query projection boundary. Do not hard-code a +symbol, add a timer, poll venue REST for execution or weaken the 2-second +quote policy. Deterministic/replay tests cover connected quiet delivery, +disconnect, expired heartbeat, generation swap, duplicate, gap/resync, +cross-identity mixing and replica parity. + +**B runtime packet and test gate.** After source gates pass, seal one immutable +reader image, exact catalog/acquisition/manifest/routing hashes, TLS/identity +revision and per-role rollback pair. Recreate only the roles proven affected, +normally `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive`, one at a time. Preserve V1, Kafka topology/offsets, Redis, +SQLite, Rust cores, ingestors, projectors, Trading System, alpha and order path. +Run exactly one public-SDK `require_all=true`, no-order C2 window for at least +300 seconds. It must test both replicas, signed cursor/reconnect and the full +quote set, while observing the control feeds. Any unexpected strict rejection, +V1/direct-provider fallback, gap, duplicate, cross-mix, restart/OOM or resource +breach stops the packet and rolls back only its named roles/image/config pair. + +**B exit gate.** Every sealed strict quote is complete, identity-correct, +gap-free and inside its signed bound on both replicas; quiet routes carry their +typed liveness proof without rewritten source timestamps; no fallback is hidden; +and the C2 receipt contains per-binding consumer latency plus resource evidence. +No C work or release is allowed from a source-only pass or a partial C2. + +###### C. Complete endpoint, binding and consumer latency certification (`PENDING / REQUIRES B EXIT`) + +**Goal.** Certify the data products actually consumable today, not merely the +new MARK/INDEX or BBO path. The certificate is manifest-derived and reports +coverage and exclusions separately, so it cannot overclaim a broad universe or +unbound reference/L2 capability. + +**Coverage and test matrix.** Through both query replicas and public SDK, +exercise every active entitled route for catalog/identity resolution, snapshot +and pagination; `TRADE`, `QUOTE`, `MARK_INDEX_PRICE`, `BOOK_SNAPSHOT`, +`BOOK_DELTA`; final BAR/history/warmup/batch warmup; entitled reference batch +(funding, OI, long/short, taker flow, mark/index, metadata and native/continuous +basis); and stream/replay/signed-cursor/reconnect. Verify decimal/unit/timezone, +venue/instrument isolation, finality/revision, ordering, book depth/sequence/ +checksum, missing-value behavior, maxlen `700/2500/5000/10000`, duplicate/gap/ +resync, active/passive handoff, latest-view versus durable replay parity, +allowed V1 fallback versus `BLOCKED`, and zero direct venue connections by the +consumer. + +**Required measured evidence.** One bounded report must enumerate every active +binding, endpoint family and replica with sample count, typed outcome/error and +p50/p95/p99/max for: venue event to host receipt; receipt to Kafka; Kafka to +canonical; canonical to durable spool and latest view; SDK request start to +usable response; stream event to consumer receipt; final close to usable BAR; +and signed cursor/reconnect completion. Provider event age, session age, +durable-projection delay and consumer-call-to-usable latency are separate +columns. The scanner must use the A quality decision: quiet `MARK_INDEX_PRICE`, +`TRADE` and `BOOK_DELTA` are not false-positive stale, but strict `QUOTE`, +`BOOK_SNAPSHOT` and final BAR failures remain visible. Record queue/lag, +reconnect, dropped-message, CPU/RSS, disk/I/O, restart and OOM counters for the +same window. + +**C exit gate.** All active entitled products pass their declared policy on both +replicas, with no unexplained loss, duplication, gap, cross-identity mix, strict +stale execution data, resource breach or unapproved fallback. Every excluded +V1/dark/VN route remains named in the certificate. A defect in an active +endpoint is fixed and retested inside C; it is not deferred as release debt. + +###### D. Scoped hygiene, provenance reconciliation and immutable release (`PENDING / REQUIRES C EXIT AND RELEASE APPROVAL`) + +**Goal.** Publish a release whose Git commit, manifest/catalog/API/Proto, +immutable image digest, standard runtime role set, rollback coordinates and +consumer latency certificate are one auditable object, while removing only +disposable test artifacts. + +**Required cleanup and release sequence.** + +1. Seal bounded evidence only: route inventory, hashes/digests, test command + result/count, latency/resource aggregates, rollback map and no-order proof. + Exclude provider payloads, credentials, JWTs, cursors, caches and unbounded + logs from Git and release artefacts. +2. Inventory canonical containers/images, explicit rollback images, stopped + test containers, BuildKit cache, disk and inode usage. Prove + `lucid_sinoussi`, `youthful_shamir` and `qdl-admit-1d` are not mounted or + referenced by a standard service before a separately scoped removal. Retain + active production images and one named rollback image per changed role; + remove only unreferenced R1.35 test/client images and matching unused build + cache. No broad prune, volume/network deletion, Kafka offset reset, Redis + flush or SQLite deletion is part of D. +3. Run `git diff --check`, verify the user's Git identity, inspect staged scope, + commit coherent tested slices, push the feature branch and require green CI + on a PR into `dev`. After approved merge, rebuild and attest from the exact + `dev` SHA, roll only the already-approved role/digest set and repeat affected + C2/latency checks. A new binary or image invalidates inherited runtime + evidence. +4. Only after the `dev` certificate and V1 rollback drill pass, merge `dev` to + `main`, tag the exact main SHA with the next semantic version, build/attest + the tag image, publish release notes/certificate and synchronize canonical + local `dev` and `main` from remote. Verify feature containment, then remove + the merged feature worktree and local branch. Record pre/post disk and inode + values plus post-cleanup standard-service health. + +**D final release gate.** A/B/C are all `PASS`; CI and release provenance are +green; runtime uses the attested tag digest; the public endpoint/latency matrix +matches the sealed inventory; V1 rollback is proven; cleanup retained only the +declared active/rollback artefacts; and no in-scope quality, binding, runtime, +cleanup or provenance gap is carried as technical debt. Until then the result is +`NOT CERTIFIED`, regardless of a green health endpoint. + +##### Phase 1 - R1.35-B1: Source delivery semantics and quality authority (`PASS / SOURCE ONLY`, 2026-09-19) + +**Goal.** Repair the real false-positive class without weakening genuine +staleness: native Binance/OKX BBO lanes are update-on-change, so raw +last-event age alone cannot diagnose a live unchanged best bid/offer. The +canonical quality answer must distinguish this explicitly from an +event-driven quote, a quiet/disconnected provider session, or a stale/gapped +data plane. + +**Allowed implementation.** Add one provider-neutral, catalog-bound delivery +semantic (`STRICT_EVENT` default; `ON_CHANGE` only for documented native BBO +source lanes). Carry it through the canonical Rust quality contract, Python +evaluator, stable catalog parser/compiler, Query/SDK status/snapshot +projection and shared golden corpus. Only the exact Trading System execution +quote routes may select `OBSERVE`, and only after the source binding states +`ON_CHANGE`. The route remains eligible only when all of the following hold: + +1. immutable last-event/source/receipt lineage exists and remains exposed; +2. provider session is `LIVE` and its independently persisted liveness age is + within the route's signed bound; +3. source generation, configuration revision, instrument/feed identity and + authority match exactly; +4. completeness is true and no sequence, resync or watermark fence is open; +5. the consuming manifest's policy, freshness/session budget and route + entitlement match the signed source declaration. + +The change may touch the shared Rust/Python quality contract, catalog compiler +and parser, exact consumer manifest/routing material, Query/SDK projection and +their tests. It may not add a symbol timer, vendor REST execution fallback, +new consumer identity/service/topology, rewritten timestamps, an external +provider rate-limit change, or a broad consumer entitlement. + +**Required source tests.** + +- Rust/Python golden parity for strict fresh/stale, `ON_CHANGE` quiet-but-live, + heartbeat stopped, disconnect, generation/config change, source authority + mismatch, gap/resync, duplicate/out-of-order, missing lineage, incomplete + snapshot and cross-venue/symbol identity mix. +- A strict event-driven `QUOTE` with an old event remains rejected even if a + different lane is live; an `ON_CHANGE` declaration on a non-BBO source is + rejected by catalog compilation; a consumer cannot self-elect `OBSERVE` + unless the exact signed binding permits it. +- Query status and snapshot cannot disagree: both preserve raw event age and + return the same typed state/reason/eligibility. A second generic raw-age + predicate must not downgrade a source-authorized `ON_CHANGE` quote, nor + upgrade an unauthorized one. +- Consumer-manifest/parser/routing tests for the ten declared Binance USD-M / + OKX Swap BTC/ETH/SOL/DOGE/BNB execution quote routes; all other routes retain + their current policy until independently declared. +- Format, Clippy with warnings denied, Rust unit/golden, Python unit/contract/ + SDK, catalog/manifest compile, generated-contract drift, `compileall` and + `git diff --check`. + +**Exit gate.** Both language implementations and public Query/SDK return an +identical, fail-closed answer for every case above. No generic quote becomes +quiet, no timestamp is overwritten, no event/gap/identity fence is weakened, +and the exact manifest/catalog hashes are reproducible. Any source failure is +fixed inside B1; it is not carried as technical debt or papered over by an SLA +increase. + +**Rollback / boundary.** This is source-only. Its rollback is the preceding +Git commit. No image build, bundle seal, role recreate, offset/cache mutation +or consumer rollout is authorized by B1. + +**Implementation start (2026-09-19).** The approved source slice is now in +progress. It is limited to the shared Rust/Python evaluator, catalog and +acquisition validation, the exact Trading System paper manifest/routing +declaration, Query/SDK-preserving eligibility projection, shared golden and +regression tests. No runtime, provider, container, image, Kafka, Redis, +SQLite, V1, Trading System, alpha or order-path action has occurred. + +**Completed source slice and decision.** The source now declares +`delivery_semantics=ON_CHANGE` only for the ten currently certified native BBO +bindings: Binance USD-M and OKX Swap BTC/ETH/SOL/DOGE/BNB. Spot remains +`STRICT_EVENT`, and DNSE remains outside this V2 route. The declaration is +accepted only when its acquisition entry is a Rust-native documented BBO lane +(`@bookTicker` for Binance USD-M or `bbo-tbt` for OKX); invalid feed, mode, +provider kind or channel fails catalog/acquisition loading. Rust and Python +share the same evaluator and golden corpus. The stable source preserves the +immutable raw event age and `LAST_EVENT_STALE`, adds bounded +`DELIVERY_ON_CHANGE` provenance, and may make only the exact +`ON_CHANGE`/`OBSERVE`/live-session/no-gap/no-mismatch view execution-eligible. +Query carries that already-pinned source answer instead of reapplying a raw +age predicate, while still requiring entitlement, authority, policy, complete +coverage, live session and liveness bound. No REST fallback, timestamp rewrite, +symbol worker, public V2 field or provider quota change was introduced. + +**Sealed source contract.** `trading-system-paper.yaml` is manifest revision +`10`, with exactly the ten quote requirements at `OBSERVE` and a `2,000 ms` +session-liveness bound. Its canonical manifest SHA-256 is +`5ca9aff1960883f59827e3a34ed709f4c30cf5db96c4e1253dd6f1116fb20cdf`. +`stable-v2-release-routing.yaml` is revision `19`; it binds the unchanged +catalog revision `8` and the changed catalog-file SHA-256 +`1aacf39153a6fd6346309c6705c24f2daa13ee5a083993b1304cceaa9d3450b2`. +The catalog revision intentionally did not change because it is raw-event +provenance, whereas this source-only routing/quality policy change is sealed +by the catalog file digest and route revision. A first implementation used the +raw YAML digest for the consumer manifest; the loader correctly rejected it, +and the routing reference was corrected to the manifest's canonical digest +above before acceptance. + +**Tests and evidence (`PASS`).** All commands ran in an isolated read-only +container/image or the local source-only interpreter; no provider socket, +runtime role, persistent test namespace or data plane was created. + +1. `python3 -m py_compile` over every changed Python module/test and structural + JSON/YAML validation: pass. +2. `docker run --rm --network none --read-only ... qdl-v2-python:2.0.25-a23fcbe + python -m unittest` for the B1 golden, catalog/acquisition, stable source, + Query/SDK, release-evidence, routing and catalog-regeneration set: `25` + passed. It proves strict quote rejection; quiet-but-connected BBO; heartbeat + expiry; disconnect; configuration/generation mismatch; open gap; route + entitlement; both venue channel shapes; Spot exclusion; and canonical + manifest/routing containment. +3. The wider isolated compatibility suite for stable edge/deployment, release + routing/observations, universal release, execution mark/index, stale-reason + and Trading System scope completed with exit status zero. Its intentional + backpressure/recovery injections remained test-only and produced no runtime + mutation. +4. `cargo test -p qdl-core`: `49 passed, 0 failed`; the Rust golden matches the + Python fixture. `cargo clippy -p qdl-core --all-targets -- -D warnings` and + `cargo fmt --all -- --check`: pass. +5. Read-only `python -m compileall -q qdl tests`, `git diff --check`, and + generated-contract scope inspection: pass; no Proto/generated SDK file + drift exists because this is an internal quality/evidence extension rather + than a public wire-schema change. + +**Exit / debt / cleanup.** B1 exit is met: generic quotes remain strict; raw +lineage remains observable; all quiet-session and identity/continuity fences +fail closed; Rust, Query and release evidence agree; and the sealed exact +route set is ten, not a broad universe. There is no in-scope B1 technical debt. +`R1.35-B2` is a deliberately separate real-runtime acceptance gate, not debt. +Every test invocation used `docker run --rm` with read-only source and +temporary storage; it left no container, image, volume, cache, provider data +or runtime mutation to clean. The sealed source commit is `6be8d54` +(`fix(quality): authorize native BBO on-change delivery`). + +##### Phase 2 - R1.35-B2: Bounded reader rollout and strict quote C2 (`FAIL-CLOSED / ROLLED BACK / REQUIRES CORRECTION`) + +**Goal.** Prove the B1 contract against real Binance USD-M and OKX Swap data, +from an external authenticated consumer through both existing V2 query +replicas. The test must show that the same `ON_CHANGE` source contract resolves +the observed BBO false-positive without accepting a stale, disconnected or +gapped route. + +**Runtime scope.** First seal the exact source commit, catalog/acquisition +revision, consumer-manifest revision, identity/key revision and immutable +reader image digest. The expected bounded packet is only +`query_v2_1`, `query_v2_2`, `stream_v2_active` and `stream_v2_passive`, rolled +one role at a time with their current mounts/TLS/config and one recorded +rollback image/config pair. The final packet must name actual digests and +runtime directory before execution. It does not recreate projectors, Rust +cores, ingestors, V1, Kafka, Redis, SQLite, Trading System, alpha or an order +path; it does not reset offsets, flush caches or delete state. + +**Acceptance and measurements.** Run exactly one public-SDK, no-order C2 for +at least 300 seconds with `require_all=true`, using workload mTLS/JWT and the +ten entitled execution BBO routes: Binance USD-M and OKX Swap +BTC/ETH/SOL/DOGE/BNB. Include declared controls for TRADE, BOOK_DELTA, +BOOK_SNAPSHOT and MARK_INDEX_PRICE so the quote fix cannot mask a regression. +For every binding and both replicas, record bounded evidence only: + +1. typed state, reason, execution eligibility, raw event age, session + liveness, generation/config revision, gap/completeness and watermark; +2. consumer request start -> SDK-usable response p50/p95/p99/max and sample + count, separately from venue event age and host durable timing; +3. request/status/snapshot/stream latency, direct-provider and V1-fallback + counts, reconnect/cursor results and any typed error reason; and +4. Kafka lag, queue depth, projector/query/stream CPU/RSS, restart/OOM, + connection/reconnect and dropped-message counters. + +**Negative proof.** Isolated deterministic/replay fixtures must prove +quiet-but-connected, heartbeat expiry, disconnect, generation transition, +duplicate, open gap/resync and wrong-identity outcomes. The shared serving +stream is never intentionally severed to manufacture a fault. Any unexpected +strict rejection, silent fallback, cross-mix, duplicate, gap, restart/OOM or +resource breach fails B2 and triggers rollback only of the named reader roles. + +**Exit gate.** All ten BBO routes and declared controls pass on both replicas; +no percentile hides a failure; raw event lineage remains unchanged; V1/direct +provider fallback is zero unless an exact non-execution policy explicitly +authorizes it; no order, alpha signal/sizing, Trading System or broker state +mutation occurred. The evidence contains a reproducible consumer-call-to-usable +latency matrix and exact runtime/image/bundle provenance. + +**Rollback / boundary.** Stop at the recorded active role/image/config pair if +any acceptance invariant fails. Do not retry for luck or widen a freshness +budget. Only B2 pass permits Phase 3. + +**B2 packet preflight (2026-09-19; no runtime mutation).** `R1.35-B1` source +is sealed at `6be8d54`; the reader candidate will be built from that code as +one immutable Python image. Read-only container inspection found two distinct +active rollback coordinates: `query_v2_1` and `query_v2_2` use +`sha256:1ad34175322f2f8eec34b3e772e3935d999248e40432ecf5852f832df1dfa88d` +(`58998ae`), while `stream_v2_active` and `stream_v2_passive` use +`sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06` +(`40a1155`). The packet therefore records rollback per role rather than +pretending one historical Python image covers all readers. A new private, +reader-only runtime directory will copy the currently mounted non-secret +runtime files byte-for-byte, preserve authority/core/ingestor/acquisition +lineage, and replace only the signed source catalog plus release-routing +copies required by manifest revision `10`. Core and ingestor roles retain the +current runtime directory; no shared runtime inode is edited. Before any +recreate, the packet must prove those preservation hashes, candidate image +digest, source/catalog/manifest/route hash chain, compose render and exact +four-role rollback map. This preparation is not a rollout and does not grant +Phase 3. + +**B2 provenance correction (2026-09-20; no runtime mutation).** The first +candidate preflight correctly failed closed before any role recreate: +`StableAcquisitionPlan` rejected the copied source catalog because it had 206 +bindings while the active declared acquisition plan had 216. The ten active +only bindings are exactly the Binance USD-M and OKX Swap `MARK_INDEX_PRICE` +routes for BTC, ETH, SOL, DOGE and BNB. The same active-only set is absent from +the checked-in acquisition plan and authority-promotion scope; the active +crypto-demand declaration also contains the two approved alpha consumers that +the checked-in declaration lacks. This is source/runtime provenance drift, not +a provider, latency, quality or reader defect. The failed candidate never +mounted a serving role and remains non-authoritative evidence. + +**Corrected B2 source boundary.** Before rebuilding one replacement reader +candidate, recover the active non-secret catalog, acquisition, promotion and +crypto-demand declarations into source as the exact baseline; retain their +deployed revisions and all existing binding behavior; then apply only B1's +signed `ON_CHANGE` semantics to the exact ten native BBO `QUOTE` bindings. +Regenerate release routing from that recovered source with a new revision and +the already sealed manifest revision `10`. A no-network equivalence probe must +prove that the recovered source differs from the active baseline only in the +declared B1 BBO delivery semantics and routing provenance. No runtime role, +image, mount, offset, cache, topic, V1 consumer, Trading System, alpha or +order path is changed by this correction. The replacement image and packet +remain B2 gates; Phase 3 is still prohibited until strict C2 passes. + +**B2 provenance-recovery source result (2026-09-20; PASS / source only).** +Recovered the active non-secret `stable-source-bindings`, +`stable-acquisition-bindings`, `stable-authority-promotion-scope` and +`stable-crypto-demand` declarations into source. The source now records the +serving 216 catalog/acquisition bindings, 206 promoted Binance/OKX bindings, +and three active demand consumers (`trading-system.paper.stable` 190 routes, +`alpha.binance.paper.stable` 75, `alpha.okx.paper.stable` 75). The source +catalog remains deployed revision `9`; acquisition remains `17`; promotion +scope remains `8`; demand remains `6`. The only semantic delta versus the +active catalog is `delivery_semantics: ON_CHANGE` on the exact ten certified +native BBO `QUOTE` bindings. Release routing is revision `20`, binds catalog +SHA-256 `2072202c76683788cf1d59905038787e4db194209df8266b5da44b6487f4947e`, +the recovered demand SHA-256 +`9ace71cd3e4ed6e224151e31699cfc3fe57ba2a6c17664c832da07eb1c6aa30f`, and +the already sealed Trading System manifest revision `10`. + +Added a regression that requires every one of the ten Binance/OKX execution +`MARK_INDEX_PRICE` bindings to remain present in catalog, acquisition, +promotion and Trading System demand; it also asserts the two approved alpha +demand declarations remain present. An isolated, read-only, no-network source +container ran 47 affected deployment/consumer/quality/observation tests with +no failure or skip. Expected fixture error lines for missing CLI scope, +incomplete BAR catch-up and bounded DNSE queue fencing were asserted by those +tests. A separate read-only active-runtime equivalence probe passed: all three +recovered declarations were byte-identical to the active runtime; all 216 +catalog records matched after removing only the ten declared B1 fields; and +the sole route delta was the stated revision/hash/manifest provenance. Loader +validation passed (`catalog=216`, `acquisition=216`, `scope=206`, +`route_revision=20`). No image, container, runtime directory, provider call, +Kafka/Redis/SQLite state, V1, consumer, alpha or order path changed. The +previous unused B1 candidate image and the failed B2 packet are retained only +until a replacement candidate passes, then enter the exact R1.35-D cleanup +inventory; no broad prune is authorized. + +**B2 replacement packet preflight (2026-09-20; PASS / not rolled).** Built +the one replacement Python reader image from source commit +`2ce8f1da2767c9cca8c0a0291019492c164876b5`: +`qdl-v2-python:2.0.26-2ce8f1d@sha256:c8177013b6fce1d9cf32c08acf847a3101745f9803965f54f571306ddbd4d4f3`. +Its non-root, read-only, no-network packaged-config smoke passed with +`catalog=216`, `acquisition=216`, `scope=206`, `route=20`. The sealed packet +is `/home/bobby/.local/state/qdl-v2/r135-b2-onchange-2ce8f1d-20260920T001000Z`; +its `packet.json` records no secret values and an exact per-role rollback map. +It copies the active runtime as baseline, retains byte-identical +authority/core/ingestor/acquisition/promotion/demand files, and changes only +the B1 catalog plus stable/C2 routing files. Candidate scope is exactly +`query_v2_1`, `query_v2_2`, `stream_v2_active`, `stream_v2_passive`. +Rollback is exact: Query roles restore +`sha256:1ad34175322f2f8eec34b3e772e3935d999248e40432ecf5852f832df1dfa88d` +and stream roles restore +`sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06`, +all against the current active runtime directory. Candidate and rollback +Compose renders both passed with `config --quiet`; no role has been recreated +and no data-plane state has changed. The next action is the declared +four-role rolling B2 acceptance packet, followed by exactly one strict C2; +Phase 3 remains blocked until that C2 result. + +**B2 replacement rollout and C2 result (2026-09-20; FAIL-CLOSED / rolled +back).** The sealed replacement packet rolled exactly four reader roles, one +at a time: `query_v2_1`, `query_v2_2`, `stream_v2_active`, then +`stream_v2_passive`. Each reached candidate image +`sha256:c8177013b6fce1d9cf32c08acf847a3101745f9803965f54f571306ddbd4d4f3`, +healthy with `restart=0`; no projector, Rust core, ingestor, V1, Kafka +topology/offset, Redis, SQLite, Trading System, alpha or order-path object was +changed. The exact packet and non-secret evidence are under +`/home/bobby/.local/state/qdl-v2/r135-b2-onchange-2ce8f1d-20260920T001000Z/`. + +The sole strict public-SDK C2 used the Trading System workload identity with +mTLS/JWT, public V2 Query pair, both stream targets, the real sealed +catalog/acquisition/release routing, and only manifest-authorized V1 fallback +readback. Its bootstrap verified UID `10001`, empty effective/inheritable/ +ambient capabilities and `NoNewPrivs=1`. It started the opening proof but +failed before the 300-second observation window at reference validation: +`ValueError: reference response exceeds its governed freshness bound`. +The receipt is intentionally empty because opening did not complete; bounded +stderr SHA-256 is +`800bf648d08cf2c397129ffc416d4d55b17edb88299cece0e9dc58c00ba7906a`. +There were zero order actions, no provider connection by the consumer client, +no cursor retained and no alpha/Trading-System mutation. This is a real B2 +gate failure, not a retry or an SLA relaxation opportunity. + +As required by this phase, the helper then restored only those four roles to +their exact pre-packet coordinates: both Query roles are healthy at +`sha256:1ad34175322f2f8eec34b3e772e3935d999248e40432ecf5852f832df1dfa88d`; +both stream roles are healthy at +`sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06`; +each has `restart=0`. The failed candidate is not serving authority. Two +post-rollback generic status probes were discarded as harness-invalid because +they targeted the rollback runtime with manifest revision `10` while that +runtime correctly requires its prior revision; they are not data-quality +evidence. Temporary staged identity material, invalid diagnostic output and +client bytecode were removed exactly after the bounded C2 evidence was sealed: +the C2 directory fell from `53,688` to `7,846` bytes, and no `qdl-r135` test +container remained. B2 remains blocked pending a narrow typed diagnosis and +fix for the MARK/INDEX reference freshness path; Phase 3 and release remain +prohibited. + +**B2 corrective source scope (2026-09-20; approved by the existing B2 +boundary).** The failed C2 exposed verifier drift, not a basis to relax a +freshness SLA: `V2QueryService` already admits an execution +`MARK_INDEX_PRICE` live-view result under the signed quiet-session/component +contract, while `reference_quality()` re-applies generic raw +`source_event_time` freshness to that same returned result. The correction is +limited to the shared C2/reference acceptance verifier: recognize only the +existing internal stable-stream execution live-view lineage; preserve raw +event/confirmation timestamps as evidence; validate live session state, +checked-at age, generation/gap-fenced response, both component receipt ages +and their signed cadence; then report those ages separately instead of using +the raw event age as the acceptance SLA. Generic MARK/INDEX, strict execution +MARK/INDEX, disconnected/expired session, malformed/missing component evidence +and expired component cadence must remain fail-closed. Add product-bound C2 +failure evidence so a future reference failure names its exact manifest +identity without storing a payload. Run focused Python unit/golden acceptance +tests plus the affected no-network SDK/quality suite; only then build one new +immutable reader image and request a new exact four-role B2 packet. No runtime +role, source catalog/manifest/SLA, Rust/provider adapter, V1, Kafka, Redis, +SQLite, Trading System, alpha or order path changes in this source slice. + +**B2 source-regression correction (2026-09-20; source-only, before the next +reader image).** The recovered acquisition declaration in `2ce8f1d` correctly +restored the active MARK/INDEX set but inadvertently changed the five already +admitted Binance USD-M final `BAR 1m` bindings (`BTC/ETH/SOL/DOGE/BNB`) from +the R1.28 `RUST_NATIVE` `/market` kline contract back to `PYTHON_REST`. +`tests/test_phaseb_stable_deployment.py` and the R1.28 generator contract +exposed the contradiction. This is a source provenance defect, not a decision +to withdraw the native final-BAR route: restore exactly those five acquisition +records to `RUST_NATIVE`, `binance_usdm_bar`, `{symbol}@kline_1m` and their +routed public/market WebSocket endpoints, leaving every other Binance interval +on REST and retaining acquisition revision `17`, the revision which originally +admitted this exact five-binding move. Add the current `bindings` property to +the minimal projector catalog test double so it models the real catalog API +introduced by watermark prewarm; it must not weaken the production projector's +catalog requirement. Run the R1.28 native-BAR scope suite and the affected +deployment/projector suites. This source correction creates no runtime change: +the later C source/runtime reconciliation must explicitly prove the deployed +ingestor/core/bar-edge bundle has the same five native owners before a release +certificate can cover final BAR. + +**B2 corrective implementation and source evidence (2026-09-20; source-only +PASS / runtime still unchanged).** Implemented the one pure shared +`qdl.data_quality.execution_mark_index` evidence validator and made the Query +service and C2 reference verifier use it, so an exact signed execution +MARK/INDEX live view is accepted only from the internal stable-stream lineage +with current session, checked-at, generation/gap, source/confirmation and both +component-cadence fences. Raw timestamps remain visible; generic/strict +MARK/INDEX, absent/zero/future fences, expired component cadence and malformed +lineage continue to fail closed. The C2 identity wrapper now emits a bounded +product/replica error without persisting a response payload. The five R1.28 +Binance USD-M `BAR 1m` acquisition declarations were restored exactly as +specified above. Regression assertions now distinguish the ten sealed +execution BBO `ON_CHANGE + OBSERVE` routes from all alpha/generic strict quote +routes, inventory the ten MARK/INDEX bindings explicitly, remove a shared +demand key across every consumer when testing release rejection, and pin the +two intentionally namespaced public stale-policy schemas rather than a stale +schema count. + +Actual no-network, read-only, disposable-container evidence: + +1. `tests.test_phase105_consumer_acceptance` plus + `tests.test_phaseb_stable_edge`: `67` passed, `1` pre-existing isolated + Redis skip. +2. Phase105/R1.35 release, fallback, handoff, identity, native-basis and + final-BAR matrix: `91` passed. +3. Projector/WAL/stale/SDK/reference-L2/quality matrix: `214` passed, `1` + same pre-existing isolated Redis skip. +4. `git diff --check` passed before and after the source changes. Expected + injected backpressure, invalid-argument, stale-BAR, DNSE queue and poisoned + checkpoint logs in those suites were asserted negative paths, not runtime + events. + +The correction was then sealed into exactly one immutable reader candidate: +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`, +whose OCI revision label is +`335792a582c2a1c8a130c47980ffe6992a22f2a3`. The build changes no running +container, runtime/config bundle, Kafka, Redis, SQLite, V1, consumer, alpha or +provider state. This closes the B2 *source correction and candidate-build* +slice only. B2 remains `PENDING` until a bounded four-reader packet completes +and its single 300-second real-provider C2 evidence passes; Phase C and release +remain blocked until then. + +**B2 replacement packet v2 preflight (2026-09-20; `PREPARED_NOT_ROLLED`).** +The one sealed packet is +`/home/bobby/.local/state/qdl-v2/r135-b2-335792a-20260920T015057Z/packet.json` +(`sha256=f68e56a9c046b4f33594b2390105d3c0eaec3c87a6d8d5b572af92bf13fca2fe`). +It pins source `335792a582c2a1c8a130c47980ffe6992a22f2a3` and exactly one +candidate reader image +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`. +It copies the active runtime into a private directory and changes only +`stable-source-bindings.yaml`, `stable-acquisition-bindings.yaml`, +`stable-v2-release-routing.yaml`, and C2's private route projection. Authority, +all three core files, both ingestor files, promotion scope and crypto demand +are byte-identical to the currently mounted runtime. The exact candidate scope +is `query_v2_1`, `query_v2_2`, `stream_v2_active`, `stream_v2_passive`; +rollback restores Query to +`sha256:1ad34175322f2f8eec34b3e772e3935d999248e40432ecf5852f832df1dfa88d` +and Stream to +`sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06`. +Candidate and rollback Compose renders pass. An immutable, network-disabled, +read-only bundle load passed `catalog=216`, `acquisition=216`, `route=20` and +all five R1.28 native Binance final `BAR 1m` bindings. The C2 route is mounted +at `/app/qdl-runtime`, not `/runtime`, because the release-plan loader fences +all artifact references beneath `/app`; this binds C2 to the exact private +runtime bundle without weakening its checksum/path validation. Baseline four +reader roles are all `healthy`, `restart=0`, `OOMKilled=false`. No role has +been recreated by this packet yet. Its only permitted next action is serial +four-reader candidate recreation followed by one `require_all=true`, 300-second +no-order C2 using `executor_network`; a nonzero result restores exactly the +same four roles and stops B2. + +**B2 first C2 attempt and rollback (2026-09-20; launcher evidence only, +`NOT AN ACCEPTANCE RECEIPT`).** The four readers were serially recreated to the +candidate and each reached `healthy`, `restart=0`, `OOMKilled=false`, then the +disposable client reached the unprivileged boundary (`uid=10001`, empty +effective/permitted/inheritable/ambient capabilities, `NoNewPrivs=1`). It +stopped before the 300-second observation with +`workload token verification failed`; the compact stderr hash is +`368126f737990b7ad894c0eb30c5ceeda543d639293f9e1bad66ebc9c5a2fa0f` and +there is no acceptance payload. Read-only public-key fingerprints identified +the packet error: its historical identity extension signed +`stable-trading-system-rs256-v1` with `92ba5cb8...e83433`, while both Query +keyrings trust the canonical current Trading System key +`e922a8da...4a0683`. The canonical bundle's private/public pair matches that +trusted fingerprint exactly. This is identity-selection/provenance error in +the disposable packet, not a reader, provider, quote-quality or data-plane +failure. The packet immediately restored exactly the four named readers to +their prior images/runtime mounts; each is healthy, `restart=0`, +`OOMKilled=false`. No V1/Rust/ingestor/projector/Kafka/Redis/SQLite/Trading +System/alpha/order object changed, no provider credential was mounted, and no +order/signal/sizing mutation occurred. + +The replacement launcher now mounts only the canonical matching identity. A +network-disabled bootstrap proof with that same root-only bootstrap boundary +has passed; its child is UID `10001`, capability-empty and +`NoNewPrivs=1`. It made no endpoint request. This permits one replacement B2 +rolling/C2 packet on the unchanged image/runtime candidate; it is not a +luck-based repeat because the authenticated identity presented to the server is +different and now fingerprint-bound to the server's configured public key. + +**B2 replacement rollout and C2 exit (2026-09-20; `PASS / R1.35-C UNBLOCKED`).** +The unchanged sealed candidate was again rolled serially to exactly +`query_v2_1`, `query_v2_2`, `stream_v2_active`, `stream_v2_passive`; all four +are currently `healthy`, `restart=0`, `OOMKilled=false`, mounted at the private +packet runtime and pinned to `sha256:8c53d37f...01668c5c`. The one replacement +authenticated C2 completed with receipt SHA-256 +`f8c86c0ae25c8a1e8c7a86d95a3b2d5dbfe2fdb2adaa386a3954cdc362649516` and +status `PASS_V2_DATA_PLANE_ONLY`: `60/60` opening and closing products, +`BINANCE=30`, `OKX=30`, and exactly ten each of `TRADE`, `QUOTE`, +`MARK_INDEX_PRICE`, `BOOK_SNAPSHOT`, `BOOK_DELTA`, and final `BAR`. It observed +for `300.1s` (`300s` requested), recorded `provider_connections=0`, +`order_actions=0`, no fallback detail, and removed its signed-cursor directory. +The feed delivery receipt reports `50` durable and `10` legitimate on-demand +views; all 60 retain closing V2 reads. Capture resource context was `18` +millicores and `245800960` RSS bytes. The disposable client self-removed, and +post-C2 bounded reader logs contain no fatal/TLS/catalog/lineage/OOM record. +The prior rejected identity receipt and network-none launcher evidence remain +as compact packet provenance, not release evidence. V1, Rust cores, ingestors, +bar edge, projectors, Kafka topology/offsets, Redis, SQLite, Trading System, +alpha and the order path remain unchanged. This closes R1.35-B; only now may +R1.35-C begin. + +##### Phase 3 - R1.35-C: Full endpoint, binding and consumer certification (`PASS / RUNTIME-CERTIFIED`, 2026-09-21) + +**Goal.** Turn a successful BBO correction into a release certificate for the +whole currently active V2 consumer surface, rather than extrapolating from +BTC/ETH or a single endpoint. The sealed inventory distinguishes active +execution/alpha/monitoring demand from `V1_PRIMARY`, `CATALOG_DARK` and +out-of-session inventory; exclusions are reported, never silently counted as +coverage. + +**Coverage matrix.** On both query replicas and through the public SDK, +exercise every sealed active entitlement for catalog/resolve/pagination; +TRADE, QUOTE, MARK_INDEX_PRICE, BOOK_SNAPSHOT and BOOK_DELTA; final +BAR/history/warmup and batch warmup; reference batch (funding, OI, long-short, +taker flow, mark/index, metadata and native/continuous basis only where +entitled); and authenticated stream/Replay/signed cursor/reconnect. Validate +per-feed semantics, decimal/unit/timezone, instrument/venue identity, +finality/revision, ordering, depth/sequence/checksum, no cross-symbol data and +the declared `maxlen` limits (`700`, `2,500`, `5,000`, `10,000`) wherever the +consumer contract supports them. + +**Latency and health evidence.** Generate one bounded per-binding report, not +just a new-endpoint sample. For each endpoint/route/replica, report sample +count, p50/p95/p99/max, typed outcome and resource context for: venue event -> +host receipt; receipt -> Kafka; Kafka -> canonical; canonical -> durable spool +and latest view; request start -> SDK usable; stream event -> consumer receipt; +final close -> usable BAR; and cursor/reconnect completion. Event age, +session-liveness age, durable-projection delay and consumer-call latency remain +separate columns. The scanner must apply the shared typed semantics so +TRADE/BOOK_DELTA/MARK_INDEX quiet-session rows are not false-positive stale; +genuinely strict QUOTE, BOOK_SNAPSHOT and final BAR failures remain visible. + +**Required tests.** Run contract/golden/SDK/API/Proto/OpenAPI drift, +provider-conformance, parser, compatibility, migration-idempotency, +cross-language parity, bounded real-provider no-order acceptance, replay, +cursor expiry, reconnect, slow consumer, duplicate/gap/resync, active/passive +handoff, latest-view/durable parity and bounded capacity tests. No full +universe websocket subscription is inferred from warmup/read coverage; every +actual stream route must be named in the sealed manifest. `V2_PRIMARY`, +allowed V1 fallback and `BLOCKED` behavior are tested independently. + +**Exit gate.** Every sealed active binding succeeds under its declared policy +on both replicas; all exposed endpoint families have consumer-side evidence; +the scanner classification is typed and reproducible; resource limits remain +bounded; and there is zero unexplained loss, duplicate, gap, stale strict +execution data, cross-identity mix, restart/OOM or unapproved fallback. If a +new in-scope endpoint defect appears, fix and retest it in C before release. +DNSE/VN `V1_PRIMARY` and explicit dark Spot rows remain stated release +exclusions, not technical debt hidden in the certificate. + +**C kickoff and sealed inventory (2026-09-20; no runtime mutation).** B2's +single Trading System receipt remains valid evidence for its exact `60` +V2-primary product routes only; it is not being extrapolated into this phase. +The sealed routing revision `20` has `303` consumer product routes: `299` +V2-primary routes (`235` distinct requirement keys) across +`monitoring.multivenue.stable` (`4`), `alpha.binance.paper.stable` (`125`), +`alpha.okx.paper.stable` (`110`) and `trading-system.paper.stable` (`60`). +The remaining four routes are explicit V1-primary exclusions: one monitoring +VN BAR, two alpha-VN routes and one Trading System VN TRADE. The V2 feed +inventory is BAR `150`, TRADE `24`, QUOTE `20`, MARK_INDEX_PRICE `20`, +BOOK_SNAPSHOT `20`, BOOK_DELTA `20`, FUNDING_RATE `10`, OPEN_INTEREST `10`, +CONTRACT_METADATA `10`, BASIS `5`, LONG_SHORT_RATIO `5` and TAKER_FLOW `5`. + +Reader preflight found a concrete authentication coverage gap before any broad +certificate run: the active query/stream client trust bundle is present and the +current Trading System and Alpha-Binance private identities match the active +JWT keyring, but the only retained Monitoring/Alpha-OKX external private +identities do not match their active keyring public-key fingerprints. Running +a full C2 with them would only create a false negative unrelated to market +data. C will generate one additive, short-lived external identity extension +for exactly Monitoring and Alpha-OKX, append only its client CA and replace +only those two public-key entries in the query/stream reader keyring, then +recreate only the four existing reader roles with the same `335792a` image and +the exact current runtime bundle. The packet will retain the present trust/key +files and reader environment as rollback. It will not change V1, Rust core, +ingestors, projectors, bar edge, Kafka, Redis, SQLite, consumer manifests, +consumer routes, provider credentials, alpha, Trading System or the order +path. The next C evidence must then cover all `299` V2-primary routes under +their own identities on both replicas. + +**C identity packet preflight (2026-09-20; `PREPARED_NOT_ROLLED`).** The +sealed state-only packet is +`/home/bobby/.local/state/qdl-v2/r135-c-identity-20260920T023708Z/`. It uses +the active reader image +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c` +and the current B2 runtime directory unchanged. It generates an external CA +and exactly Monitoring/Alpha-OKX client/JWT identities; its generator deletes +the external CA private key before return. The preflight records only hashes: +trust `441af369...e874 -> 6fbfb41e...6dd1`, rollout environment +`66b5fce5...b99a -> 2333afeb...acc5`, and replacement public-key fingerprints +for `stable-monitoring-rs256-v1` and `stable-alpha-okx-rs256-v1`. Existing +Trading System, Alpha-Binance and Reference-L2 key entries are byte-preserved. +The exact rollback retains the old environment and query/stream client trust +bundles in the same protected packet. + +The generated `roll-readers.sh` and `rollback-readers.sh` passed `bash -n`. +An exact Compose render over the currently deployed 22-file configuration +chain proves that, for `query_v2_1`, `query_v2_2`, `stream_v2_active` and +`stream_v2_passive`, the candidate changes only +`QDL_DATA_JWT_KEYS_JSON`; image, command, user, runtime mount, state/TLS +volumes, network, healthcheck and CPU/RAM limits are identical. No provider, +runtime, container, trust volume or consumer request has yet been changed. +The next bounded action is a serial recreation of exactly those four readers; +any health/OOM failure invokes that packet's exact reader/trust rollback before +the all-identity C2 starts. + +**C identity rollout result (2026-09-20; `PASS`).** The packet atomically +replaced only `query/client-ca-bundle.crt` and +`stream/client-ca-bundle.crt`, then serially recreated exactly +`stream_v2_passive`, `query_v2_1`, `query_v2_2` and `stream_v2_active`. +All four are `healthy`, `restart=0`, `OOMKilled=false`, retain the B2 runtime +mount and retain image `sha256:8c53d37f...01668c5c`. Both live trust files now +hash to `6fbfb41e...6dd1`; all four reader keyrings report the new Monitoring +and Alpha-OKX public-key fingerprints `8e38d04e...f698` and +`8769b86a...171c`. The public-key changes are identical across query and +stream replicas. V1, Rust cores, ingestors, projectors, bar edge, Kafka, +Redis, SQLite, consumer manifests/routes, Trading System, alpha and order +paths were not recreated or changed. The exact old trust/env files remain in +the packet's rollback directory. Full all-identity C2 is now unblocked. + +**C full-C2 preflight (2026-09-20; `PREPARED_NOT_RUN`).** The same protected +packet now contains one disposable `qdl-r135-c-full-c2` launcher. It mounts +the unchanged B2 runtime, the existing Trading System/Alpha-Binance private +identities and the new Monitoring/Alpha-OKX extension separately, copies them +only into a `tmpfs` input directory as UID `10001` with empty capabilities and +`NoNewPrivs=1`, and self-removes on exit. It has no Docker socket, provider +credential, order path or mutable runtime mount. The source C2 parser passed +offline in the exact immutable reader image; launcher scripts pass `sh -n` / +`bash -n`. An offline scope build from the mounted runtime resolves exactly +`299` V2-primary products: `234` durable and `65` on-demand, with the four +consumer counts and feed inventory recorded above, scope digest +`66e158b4...3d87`. The one real run will use both query replicas and both +stream targets, run a 300-second observation, exercise only declared V1 +fallback products, require `BLOCKED` products to remain blocked, and retain +only payload-free receipt hashes/metrics. + +**C first full-C2 result and bounded root-cause decision (2026-09-20; +`FAIL_CLOSED / SOURCE REPAIR REQUIRED`).** The first all-identity run began +from the sealed `299`-route scope and stopped at its opening reference sweep; +it made no provider-direct request, order, alpha signal/sizing, broker or +runtime-state mutation. Its compact failure was Trading System paper, +`OKX/SWAP/DOGE-USDT-SWAP/MARK_INDEX_PRICE`, on the secondary query route with +typed `SOURCE_UNAVAILABLE`. This is not treated as a certificate or retried as +luck. Read-only spool inspection proved that the exact canonical DOGE +MARK/INDEX partition contained real committed records throughout the failure +window. A read-only internal HMAC probe from a current Query replica then +tested all ten declared Binance USD-M/OKX Swap MARK/INDEX bindings against +both stream roles: the passive role returned the expected fenced `409`, while +the active lease holder returned `200` for all ten, including DOGE. No payload, +credential or provider response was retained. + +The root cause is a bounded lifecycle hole: `ExecutionMarkIndexLiveView` is +memory-only and was populated only by a *new* canonical POST. A freshly +started or newly promoted active stream gateway can therefore be healthy yet +have an empty execution view until each on-change provider emits again, even +though the same exact canonical record is already durably committed. This +explains the early C2 `NOT_READY` classification and why a later direct probe +was healthy; it is not a per-symbol or venue exception. The approved repair is +limited to the existing stream role: on every successfully acquired gateway +lease, hydrate at most one latest already-committed canonical MARK/INDEX event +per declared execution binding into the fenced in-memory view. The hydrate +path reuses the existing exact binding, canonical parser, identity/provenance, +gap, generation, metadata and session-quality checks; it performs no provider +call, replay/publication, spool write or timestamp rewrite. A malformed, +gap-open, identity-mismatched or unavailable durable record leaves the route +fail-closed. On loss of the lease the existing fence still clears the view. + +Source gates before any reader rollout: deterministic startup/promotion +hydration tests; empty/stale/gap/identity fencing tests; same-epoch ordering +against a concurrent fresh canonical event; active/passive lease callback +failure tests; existing Rust/Python execution-MARK quality and SDK tests. The +only permitted runtime packet after those gates is a serial stream/query +reader rollout using a newly built immutable Python image with the present +`335792a` reader image retained as rollback. V1, Rust, ingestors, bar edge, +projectors, Kafka topology/offsets, Redis, SQLite durable contents, Trading +System, alpha and order path remain excluded. One fresh full `299`-route, +four-identity, no-order `300s` C2 follows; it must not inherit this failed +opening run. + +**C hydration source slice (2026-09-20; `IMPLEMENTED / SOURCE GATES PASS`).** +`ActivePassiveGatewayLease` now has one reusable `on_acquired` initialization +hook. A callback failure releases the just-acquired lease, invokes the existing +fence cleanup and leaves the role standby with a bounded diagnostic; it cannot +serve a partially initialized active gateway. The stable stream wires that hook +to `ExecutionMarkIndexLiveView.hydrate_from_spool`: at lease acquisition it +reads at most one latest canonical durable event from each exact declared +MARK/INDEX partition and submits it through the existing `remember` validator. +The normal newer-live-event ordering, metadata/source-policy identity, gap and +generation fences remain authoritative, while the loss-of-lease callback still +clears the whole view. No public endpoint, SDK schema, provider adapter, +timestamp, event identity or data-plane write path changed. + +The source-only, no-network/read-only test evidence is green: `131/131` +focused Python tests across execution MARK/INDEX, paired lineage, consumer +latency, Phase-10.5 identity/C2/fallback, Reference/L2 and stable-deployment +modules; targeted hydration/lease tests are included in that count. The Rust +provider-neutral core/realtime suite is also green (`91` passed; one explicit +isolated-Redis test ignored because no isolated Redis URL was supplied). A +Python compile check using a tmpfs pycache and `git diff --check` pass. The +tests cover durable exact-tail hydration, empty/gap/identity fail-closed +behavior, preserving a newer live event over an older tail, lease activation +failure/release/fence, existing quiet-session component behavior and existing +consumer C2/SDK routes. No runtime role, provider connection, durable state, +order action or cleanup occurred for this source slice. The next action is a +single immutable Python image build from its committed SHA, then only the +existing query/stream roles may be serially rolled under the prepared rollback +packet. + +##### R1.35-D: Hygiene, provenance and immutable release (`PENDING / REQUIRES PHASE 3 EXIT`) + +**Goal.** Close the release without leaving test containers, images, cache, +worktrees or ambiguous source/runtime provenance behind. + +**Cleanup and release sequence.** + +1. Preserve bounded acceptance evidence and seal the source/catalog/manifest/ + API/Proto/image/runtime digests, rollback pair and exact tested route + inventory in this plan and the release certificate. Never retain provider + payloads, credentials, cursors or unbounded logs. +2. Before cleanup, inventory canonical service containers/images, exact + rollback images, stopped test containers, BuildKit cache, disk and inode + usage. Confirm that the three known test containers (`lucid_sinoussi`, + `youthful_shamir`, `qdl-admit-1d`) are not canonical services and have no + referenced mounts/volumes. A separate bounded cleanup approval remains + required before stopping/removing them. +3. Retain only active role images plus one explicitly named rollback image per + changed role. Remove only unreferenced R1.35 client/test images and matching + unreferenced build cache; no broad prune, volume/network removal, Kafka + offset reset, Redis flush, SQLite deletion or data retention change. Record + pre/post disk and inode metrics plus post-cleanup service health. +4. Run `git diff --check`, verify the user's Git identity, inspect staged + scope, complete all tests/plan evidence, commit each coherent tested source + slice, push the feature branch and obtain green CI on a PR to `dev`. +5. After approved `dev` merge, build and attest an immutable release candidate + from the exact `dev` SHA, perform only the formally approved role/digest + handoff and repeat affected C2/latency acceptance. A changed source or image + invalidates inherited runtime evidence. +6. After the `dev` certificate and rollback drill pass, merge `dev -> main`, + tag the exact main commit with the next semantic version, build/attest the + tag image, publish release notes/certificate and synchronize canonical local + `dev`/`main` from remote. Remove this merged feature worktree and local + branch only after target-branch containment is verified. + +**Final release gate.** R1.35-A/B1/B2/C are `PASS`; cleanup has an exact +retention and disk record; source, tag, image, runtime configuration and +certificate identify the same release lineage; V1 rollback is available and +proven; and the public endpoint/latency matrix agrees with the sealed consumer +inventory. No in-scope quality, binding, runtime, cleanup or provenance gap is +carried into release as technical debt. + +**Planning record (2026-09-19).** This delivery map is documentation-only: it +adds no source/runtime/data change, no image, no container and no acceptance +claim. `git diff --check` passed for the plan edit. The first permitted +implementation action is Phase 1/B1 source work; Phase 2/B2 runtime and all +cleanup/release actions remain separately gated as written above. + +#### R1.35-A - Quality semantics convergence and audit parity (`PASS / SOURCE-CONTRACT`, 2026-09-19) + +**Execution record.** Approved scope is limited to a provider-neutral, pure +quality evaluator; its Rust/Python shared golden corpus; stable Query/SDK +mapping; and the read-only auditor. No catalog or manifest entitlement, image, +runtime role, Kafka offset/topology, Redis, SQLite, V1, Trading System, alpha +or order-path mutation is permitted in this slice. The source-only rollback is +the pre-slice commit `614b3df`. The next decision boundary is the complete +Rust/Python/API/SDK test matrix below; a real strict `QUOTE` defect discovered +there becomes R1.35-B evidence rather than an SLA/configuration change. + +**Goal.** Replace the divergent raw spool-age report with a provider-neutral, +typed quality evaluator whose result is exactly the consumer-facing decision. +The evaluator must distinguish strict event recency from quiet session +liveness, scheduled finality and expected unserved inventory without reducing +any execution quality rule. + +**Implementation scope.** + +1. Define one serializable `BindingQualityDecision` contract containing exact + binding identity, feed semantics, expected availability class, state, + event-recency state/age, provider-session state/liveness, component cadence, + generation/config revision, watermark, gap/completeness/execution eligibility + and reason flags. Preserve raw/source/provider/commit timestamps as separate + named fields. +2. Make the Rust provider/core quality path authoritative for the fields it + owns (session, generation, sequence/gap, component receipt and watermark). + Python query, SDK validation and the read-only auditor must consume the same + contract/rules or pass a shared golden corpus; duplicate independent + classification logic is prohibited. +3. Classify every sealed catalog binding exactly once into + `STRICT_EVENT`, `QUIET_SESSION`, `FINAL_SCHEDULED`, `EXPECTED_V1_PRIMARY`, + `EXPECTED_DARK`, `OUT_OF_SESSION`, or a typed failure state. The current + 216-binding inventory is a baseline only; the phase seals the actual + catalog/config revision it tests and records any count change explicitly. +4. Change the liveness report to report `last durable append age` separately + from per-record `ingest_to_durable_latency`; it must never label either as + consumer call latency. It may read spool state only read-only and must not + scan inside a constrained serving role. + +**Required tests and evidence.** + +- Deterministic Rust/Python golden parity for strict fresh/stale, quiet-live, + quiet session stopped, component cadence expiry, generation/config mismatch, + duplicate, open gap, resync, final/not-final BAR, expected V1/dark and + out-of-session classification. +- Contract/API/SDK source tests proving `FeedStatusResponse` and snapshot + admission cannot disagree for the same binding and policy. The deployed + two-query-replica matrix is deliberately executed by R1.35-B: an A + source-only slice must not present the still-running older reader image as + proof for newly introduced evaluator source. +- Regression proving a quiet response cannot satisfy a strict `QUOTE` or + `BOOK_SNAPSHOT` request, and a strict stale result cannot be relabelled + `LIVE` by the auditor. +- `cargo fmt`, strict Clippy, Rust unit/golden suite, Python unit/contract/SDK + suite, generated-contract drift gate and `git diff --check`. + +**Exit gate.** Every binding has one deterministic expected classification; +Rust/Python evaluator, stable Query mapping, SDK contract and auditor agree on +identity/state/reason fields in the shared source/golden suite; no quiet +false-positive or strict false-negative remains; all source tests pass. The +required deployed two-replica matrix is a non-transferable R1.35-B gate and +cannot be inherited from an older reader image. This phase has no runtime +rollout. A failed parity case is an in-scope defect, not technical debt. + +**Rollback and decision boundary.** Source-only rollback is the prior commit. +Do not alter a manifest SLA, source binding or runtime role to make this phase +pass. If the matrix reveals a real strict-feed defect, carry the exact evidence +into R1.35-B. + +**Completion record (2026-09-19).** Implemented one pure, +provider-neutral `BindingQualityDecision` evaluator in Python and Rust, bound +through a shared fourteen-case golden corpus. The stable spool Query backend, +MARK/INDEX bounded live view and Query freshness predicate now use that one +policy; the SDK remains public-contract compatible and proves the same typed +quality surface through its existing transport tests. The read-only auditor now +uses the evaluator and reports three distinct timestamps: event age, +ingest-to-durable latency and last-durable-append age. It carries the true +spool logical offset as watermark rather than inventing a zero value. + +**Tests actually run.** A disposable, network-disabled Python container ran +`tests.test_r135_quality_convergence`, `tests.test_dlv2_r1_stale_reason`, +`tests.test_execution_mark_index_live_view`, +`tests.test_mark_index_paired_lineage`, `tests.test_qdl_sdk_feed_status`, +`tests.test_qdl_sdk_stream_projection` and `tests.test_phaseb_stable_edge`: +`106 passed`, `1 skipped` (`isolated Redis is not configured`). The Rust +builder ran `cargo fmt --all -- --check`, strict +`cargo clippy -p qdl-core --all-targets -- -D warnings`, and the shared golden +test successfully. Buf format/lint/two frozen breaking baselines/generation +drift passed. `git diff --check` and Python `compileall` passed. No generated +contract output drifted. + +**Read-only inventory evidence.** The new auditor ran outside serving roles +with the V2 state/runtime mounted read-only against sealed +`mark-index-r134-58998ae-r3-20260919T172500Z`: `216` catalog bindings, +`206 ACTIVE`, `6 EXPECTED_DARK`, `4 EXPECTED_V1_PRIMARY`; `191 LIVE`, `15 +STALE`, `10 DISABLED`. The disabled rows are the explicit six dark Spot and +four V1-primary VN entries. The fifteen active failures are five paired +MARK/INDEX rows and all ten Binance USD-M/OKX Swap execution `QUOTE` rows for +BTC/ETH/SOL/DOGE/BNB, each with a live provider session but stale event age. +This is real B diagnostic evidence, not an A failure or an SLA change. + +**Runtime/cleanup.** No runtime image, role, Kafka offset/topology, Redis, +SQLite, V1, consumer, alpha or order path changed. The read-only test +containers used `--rm`; no new image or persistent test artifact was created. +The three pre-existing leaked test containers remain intentionally untouched +until the separately scoped R1.35-D cleanup gate. + +#### R1.35-B - Strict quote root-cause repair and bounded runtime proof (`IN_PROGRESS / R1.35-A SOURCE EXIT`, 2026-09-19) + +**Execution record.** R1.35-A source/golden exit is sealed at `bcb9540`. +The only known live defect admitted into this scope is strict execution quote +event age for the ten Binance USD-M/OKX Swap BTC/ETH/SOL/DOGE/BNB bindings; +the same audit also observed five paired MARK/INDEX component-cadence failures. +This B slice first obtains typed, read-only evidence from both readers and the +actual consumer path, then changes only the shared layer proven responsible. +No SLA relaxation, per-symbol timer, REST execution substitute, direct +provider fallback, or broad/unsourced consumer-manifest change is allowed. An +exact signed-source and exact-entitlement policy update is permitted only if +the B1 contract and golden gates below prove it is necessary. Any runtime +packet is deferred until source tests identify its exact role/image/config +scope and a rollback pair. + +**Current diagnostic slice (source-only).** Add one bounded public-SDK probe +that records at most ten typed failures per binding/reader: observation time, +declared freshness budget, typed quality/session/gap/completeness/eligibility +facts and exception class only. It must not retain prices, book levels, raw +payloads, credentials, cursor state or direct-provider output. This lets the +same real two-reader matrix distinguish an intermittent projection/query tail +from a venue/session failure before any serving role is changed. The probe is +not a new product route and has no runtime rollout or cleanup side effect. + +**Selected repair boundary (source, before runtime packet).** The two-reader +probe and read-only timestamp trace rule out a Binance/OKX session outage and +consumer/API-call latency: all ten sessions remained `LIVE`, complete and +gap-free while upstream accepted events continued. During the failing window +(`2026-09-19T19:59:18Z` through `20:00:11Z`), the shared projector pipeline +delivered canonical records to SQLite up to `10.75`--`23.3 s` after its own +accepted timestamp. Projector replicas 2 and 3 both recorded 12--26 s +canonical age spikes; their source sends are presently configured as one +1,000-record transaction that holds all selected partition locks through the +SQLite append, compatibility projection and checkpoint sequence. The exact +individual slow substage is not yet emitted, so this slice must add bounded +stage spans rather than infer a venue fault from one aggregate age. + +The repair keeps the efficient bounded Kafka fetch but introduces a +provider-neutral **commit micro-batch** cap below the fetch cap. Each chunk +preserves FIFO within every partition and performs durable append, projection +and checkpoint before the next chunk; it limits the lock footprint, SQLite +write, Redis pipeline and checkpoint work that one projector turn may hold. +It is not a symbol-specific throttle, does not change source timestamp, +manifest, cursor semantics, provider quota or V1 policy, and does not create a +second latest-state authority. Per-turn spans must separately report broker +poll, canonical lookup/lineage, durable append, compatibility projection and +checkpoint time. The runtime candidate will use a `512` record fetch cap and +`128` record commit cap only after source/unit/golden tests pass. If the +instrumented evidence shows a different shared stage is responsible, repair +that same shared stage rather than adding a cache bypass. + +**Test-harness boundary discovered during B.** A clean isolated import of the +projector exposed an existing package cycle: `mark_index_lineage` imported the +public `qdl.query` re-export, which imports Query service and then the +MARK/INDEX live reader back into lineage. This is a narrow module-boundary +defect that can hide or skip a direct projector regression depending on import +order; it has no intended runtime behavior. B may make the lower-level lineage +verifier compare the binding's already-serialized contract feed value without +initializing the public Query package, and add a clean-process regression. No +public contract, role or runtime configuration changes because of this repair. + +**Source implementation and test record (2026-09-19; runtime unchanged).** +Implemented the bounded public-SDK quality probe as +`scripts/measure_binding_quality.py`; it has a fixed evidence cap and records +only typed status/latency facts. Its real read-only two-replica run showed +that the failure is shared-pipeline behavior: all ten quote sessions were +`LIVE`, complete and gap-free, while one reader window saw repeated strict +event-age rejection across the five Binance and five OKX bindings. SQLite +timestamp correlation retained no payloads and showed continuous source +acceptance but `10.75`--`23.3 s` accepted-to-durable tail during the failure. +Projector-2/3 span logs independently showed `12`--`26 s` canonical age +spikes in that same window. + +The source repair makes routine SQLite maintenance use only nonblocking +`PASSIVE` checkpointing; a bounded `TRUNCATE` remains exclusively in the +physical-capacity fail-closed path. `StableProjectorEngine` now fetches a +bounded batch but commits it in an independently validated FIFO micro-batch; +the stable Compose candidate declares fetch `512` / commit `128` for every +projector. It adds bounded aggregate spans for broker poll, canonical +lookup/lineage, durable append, compatibility projection and checkpoint, so +the next real window identifies any remaining shared stage rather than +guessing. A clean-process import test also fixes the lineage verifier's +dependency on the eager public Query package initializer; this does not alter +the serialized feed contract. + +**Source gates actually run.** A disposable `--network none`, read-only +Python container executed the spool WAL, projector poll, stable-edge, +R1.35-quality, stale-reason, SDK feed-status and SDK stream suites: `102 +passed`, `1 skipped` (the pre-existing isolated-Redis case). The probe's +strict-versus-quiet semantics regression was then added; the final combined +source suite passed `104`, with the same `1` isolated-Redis skip. The +clean-process projector import/batch suite is included in that final run. The +new quality-probe CLI loaded successfully in the same isolated image. `git +diff --check` and Python `compileall` passed before the final test run. No +image, provider, runtime +role, Kafka offset/topology, Redis, SQLite, V1, Trading System, alpha or order +path changed. Source commit and the explicitly bounded projector rollout are +the next B boundary; the real-provider matrix remains mandatory and cannot be +inherited from this source-only evidence. + +**Goal.** Make every active execution `QUOTE` binding execution-strict under +its declared provider delivery semantics for Binance USD-M and OKX Swap, +including the initial BNB/OKX ETH/BNB/DOGE findings, without symbol-specific +workarounds, forged timestamps or a relaxed quality policy. + +**Diagnostic before change.** Run a read-only typed-status and timestamp matrix +for BTC/ETH/SOL/DOGE/BNB across Binance USD-M and OKX Swap, through both query +replicas. For every quote, record provider receipt, Kafka publish/ack, +canonicalization, core watermark, stream latest-view receipt, durable append, +query response and consumer usable time. Include controls for TRADE, +BOOK_DELTA, BOOK_SNAPSHOT and MARK/INDEX so a quote fix cannot regress their +different semantics. + +**Allowed repair paths, selected only from evidence.** + +1. Provider/session fault: repair the shared Rust venue admission, subscription + lifecycle, reconnect or resubscribe path. Do not poll REST as an execution + substitute. +2. Canonical/core fault: repair the shared Rust normalization/materialization + or sequence/gap path, preserving deterministic event identity and replay. +3. Projection/query fault: extend the existing bounded execution latest-view + only to exact declared `QUOTE` bindings, with the same identity, session, + generation, gap and watermark fences as MARK/INDEX. Durable spool remains + authoritative for replay/history; the hot view is never a new source of + truth. + +The chosen repair must be provider-neutral and cover all declared Binance/OKX +quote identities, not hard-code a failing symbol. External venue quotas, +fallback policy and V1 behavior remain unchanged. + +**Required tests and evidence.** + +- Unit/golden coverage of missing frame; strict-event quote rejection; + source-authorized `ON_CHANGE` BBO quiet-but-connected admission; disconnect, + stale provider session, reconnect, generation change, duplicate, gap/resync, + identity cross-mix and two-replica parity. +- Real-provider matrix for all ten execution quotes across both replicas, + sampled long enough to cover multiple source updates; every usable strict + result must be complete, gap-free and inside its declared effective freshness + bound. No percentile may hide a strict over-SLA execution response. +- Bounded consumer-call-to-usable latency report for each binding and replica, + with p50/p95/p99/max/sample count/error classification and all timestamp + stages kept distinct. +- Capacity checks for queue depth, Kafka lag, projector span, CPU/RAM, + restart/OOM and connection/reconnect counters. Test failure/recovery using + isolated/replay fixtures; do not disrupt the shared production stream merely + to manufacture a fault. + +**Exit gate.** Zero unexpected quote stale/reject results under each sealed +delivery semantic in the real-provider matrix; typed query and auditor parity +holds; no direct-provider/V1 fallback is observed; no unexplained +gap/duplicate/cross-identity state exists; resource bounds hold. A runtime +packet may recreate only roles proved affected by the selected repair, one at a +time, with exact image digest, runtime revision and previous digest/config +rollback recorded before execution. + +**Rollback and decision boundary.** Roll back only the named role(s) to the +recorded active digest/runtime pair. Do not reset Kafka offsets, flush Redis, +delete SQLite or remove V1. If quote data is truly unavailable from the venue, +the binding remains fail-closed and the release cannot pass. + +**Approved bounded projector packet (prepared 2026-09-19; not yet applied).** +The source gate is sealed at `7e53a12` and was built locally as +`qdl-v2-python:2.0.23-7e53a12@sha256:970a1ce4f9dfef31001a1b0c9239b6ef5fc009f81edc67beb7ab097a3fe8f511`. +The packet may recreate **only**, one at a time, `projector_v2`, +`projector_v2_3`, then `projector_v2_2`. It preserves the currently deployed +Compose chain, sealed runtime mount +`/home/bobby/.local/state/qdl-v2/mark-index-compat-a366d0e-20260905T185000Z/runtime`, +TLS/state volumes, `stable-projector-v1` consumer group, and every existing +topic/offset. Its sole effective deltas are the candidate image and +`QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS=512` plus +`QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS=128`. It is recorded outside Git at +`/home/bobby/.local/state/qdl-v2/r135-b-projector-20260919T203500Z/`: +`candidate.override.yml` SHA-256 `5e9900a6...466cb35`, +`rollback.override.yml` SHA-256 `cfc64a7d...1944ce`, and owner-only +`roll-r135-b.sh` SHA-256 `bae70066...4be0db`. The script enumerates the exact +active Compose chain and accepts one named projector only; both overrides +passed isolated YAML validation and the script passed `bash -n`. + +The rollback is per named projector through that same script in `rollback` +mode, restoring the exact active image +`sha256:9039236e7a8e570f2364b470b33386ab702bc1dde5ae9d5e7d90a4dda531e8f0` +and its `1000`-record batch setting. The rollout allows normal existing +market-data writes while a restarted member catches up. It does **not** touch +V1, Kafka brokers/topics/offsets, Redis, SQLite deletion or reset, Rust core, +ingestors, BAR edge, query/stream readers, Trading System, alpha containers, +broker credentials, or any order path. After each single recreate, require +health, no restart/OOM, bounded lag/catch-up and continuous remaining +projectors before proceeding. After all three, run the two-reader public-SDK, +real-provider strict-QUOTE matrix for at least 300 seconds, with all ten +Binance USD-M/OKX Swap BTC/ETH/SOL/DOGE/BNB bindings and the declared +TRADE/BOOK/MARK-INDEX controls. It records consumer-call-to-usable latency, +typed quality, resource/restart evidence and no direct-provider/V1 fallback; +any strict failure stops the packet and rolls back only the affected +projector(s). + +**Candidate result and corrected B2 repair boundary (2026-09-19).** The +prepared `7e53a12` packet was applied only to the three named projectors, +one at a time, then rolled back exactly to the recorded pre-packet digest +after the real two-reader 300-second matrix found `24` strict rejections on +query replica 1 and `25` on replica 2 (one additional snapshot transport +error). Every observed binding session was `LIVE`, complete and gap-free; +there was no direct provider or V1 fallback. The successful reads had +low request latency, so neither provider availability nor Query admission is +the selected fault. + +A bounded read-only canonical window over `2026-09-19T20:59:30Z` through +`21:01:30Z` then established the missing shared cause. It contained `70` +final BAR records, including `35` in the 10-second `21:00:00Z` bucket and +`25` in the following bucket, alongside continuing quote/trade/book/reference +traffic. For every BAR, `_ready_batch()` rebuilt a local watermark map and +called `_latest_bar_close_ns()`, which read and decoded up to `10,000` +retained records for that one BAR partition. Thus a normal aligned final-BAR +burst repeatedly performed expensive tail scans on the same SQLite database +which has to accept strict quote materialization. The new spans reported +multi-second `canonical_lookup` tails at the same boundary. This is a real +shared projector defect, not a provider latency, and the prior micro-batch +repair alone is insufficient. + +**B2 approved source scope before the next runtime packet.** Add one additive, +rebuildable `final_bar_watermarks` SQLite table owned by the existing canonical +spool. The active stream gateway derives a validated final/revised BAR close +watermark only from the already catalog-validated canonical envelope and stores +the max close atomically in the same SQLite transaction as its immutable +canonical append. Projectors consult that O(1) durable watermark before +selecting the latest projection. A pre-existing spool with no row is handled +by one bounded legacy tail scan per exact BAR partition, immediately seedable +with an atomic max; it is never repeated once the row exists. Equal close +revisions remain projectable; older backfills remain durable history but do +not regress latest state. No provider timestamp, payload, source policy, +SLA, Kafka ordering, Redis authority or public contract changes. + +**B2 required proof and rollback.** Add deterministic regression for atomic +max/upsert, malformed internal watermark rejection, duplicate/late/revised +BAR behavior, clean restart/rebalance lookup, one-time legacy hydration, +cross-partition isolation, and a mixed BAR/strict-QUOTE batch proving no +repeated tail scan after hydration. Run the existing no-network projector, +spool, quality and SDK suites, then build one new immutable Python image. +The subsequent packet must roll only the two existing stream roles and three +existing projector roles, one at a time, with exact active digest/config +rollback recorded before execution; V1, Kafka topology/offsets, Redis flush, +SQLite deletion, Rust, ingestors, query, Trading System, alpha and order path +remain excluded. A fresh two-replica 300-second public-SDK matrix is required; +any strict failure rolls back only those named roles. The prior failed +candidate is retained only as an explicit rollback/audit artifact until B2 +ends; it is not runtime authority. + +**B2 source implementation and proof (2026-09-19; runtime unchanged).** +Implemented an additive `final_bar_watermarks(stream, partition_key)` table in +the existing canonical SQLite spool. A final/revised BAR watermark is private +durable metadata, derived only after catalog validation at the active stream +ingress; the protobuf, source timestamp, public API and consumer contract are +unchanged. The table max is updated in the same append transaction as the +immutable event, including idempotent duplicate replay. A lower late backfill +can never regress either `close_time_ns` or its effective update time. + +`StableProjectorEngine` now reads that exact partition watermark in O(1). A +cache predating the additive table performs one retained-tail scan only while a +SQLite `BEGIN IMMEDIATE` lock is held, then atomically seeds the max; another +replica/restart observes the row and never rescans. The projector also retains +an in-memory per-batch max so an older BAR arriving after a newer BAR in the +same fetched batch cannot overwrite latest state before the durable append is +committed. Equal-close revisions remain eligible; old final BARs stay in +durable history but do not project latest. Non-final historical BAR handling +retains the prior isolated selection path. + +Source verification used only disposable Docker containers with `--network +none`, read-only source and tmpfs bytecode/cache. The focused spool/projector +suite passed `70 tests`, `1 skipped` (the pre-existing isolated-Redis case). +The complete affected no-network matrix passed `110 tests`, `1` same +pre-existing skip: R1.35 quality probe, spool WAL/watermark, projector batch +poll, stable edge/recovery, quality convergence, stale-reason, SDK feed-status +and SDK stream projection. `compileall` passed with `PYTHONPYCACHEPREFIX` on +tmpfs, and `git diff --check` passed. These tests cover atomic max/upsert, +malformed metadata rollback, duplicate hydration, cross-partition isolation, +restart/rebalance reuse, final revision, late/out-of-order in one batch, +mixed strict quote and HTTP ingress derivation. No image was built and no +runtime role, V1, Kafka topology/offset, Redis, SQLite data, Rust core, +ingestor, Query, Trading System, alpha or order path changed in this source +slice. B remains `IN_PROGRESS` until the exact five-role B2 image packet and +fresh two-reader real-provider matrix pass; that is an acceptance gate, not +technical debt. + +**B2 bounded runtime packet prepared (2026-09-19; not yet applied).** Source +commit `40a1155` was built as immutable +`qdl-v2-python:2.0.24-40a1155@sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06`. +The owner-only packet is outside Git at +`/home/bobby/.local/state/qdl-v2/r135-b2-final-bar-watermark-20260919T213906Z/`: +candidate override SHA-256 `442089b1...e7e036`, rollback override SHA-256 +`4c022fa3...7f5446`, and bounded roll helper SHA-256 `ac31f23e...93ea25`. +Both Compose renders and `bash -n` passed. The packet preserves the current +canonical Compose chain, sealed runtime mounts, TLS, state volume, +`stable-projector-v1` group and all existing Kafka/Redis/SQLite identities. + +The only allowed recreation sequence is `stream_v2_passive`, +`stream_v2_active`, `projector_v2`, `projector_v2_3`, then `projector_v2_2`, +one role at a time. Candidate changes only those five image references plus +the already-selected projector fetch/commit bounds `512/128`. Before the +packet all five are healthy with restart count zero: streams use exact image +`sha256:1ad34175322f2f8eec34b3e772e3935d999248e40432ecf5852f832df1dfa88d`; +projectors use exact image +`sha256:9039236e7a8e570f2364b470b33386ab702bc1dde5ae9d5e7d90a4dda531e8f0` +and `1000/null` batch settings. Per-role rollback recreates only the named +role with precisely those image/config values. After each role, require health, +no restart/OOM, bounded catch-up and a live remaining peer before continuing. +Any failure stops the packet and rolls back only changed named roles; it never +resets offsets, flushes Redis, deletes SQLite, touches V1, Rust, ingestors, +Query, Trading System, alpha or any order path. Normal existing market-data +writes during stream/projector catch-up are expected and are not a data-plane +reset. + +After all five are stable, the sole acceptance is a fresh two-query-replica, +public-SDK real-provider matrix for at least 300 seconds over the ten strict +Binance USD-M/OKX Swap BTC/ETH/SOL/DOGE/BNB `QUOTE` bindings, with declared +TRADE/BOOK/MARK-INDEX controls. Evidence must include typed state/eligibility, +session/gap/watermark parity, direct-provider/V1 fallback count, consumer-call +to-usable p50/p95/p99/max and resource/restart/lag observations. Any strict +failure is a B failure and invokes the exact five-role rollback; C cannot +start from a partially accepted packet. + +**B2 first role attempt, rollback and packet correction (2026-09-19).** The +first candidate recreation intentionally stopped at `stream_v2_passive`; it +never progressed to active stream or any projector. Startup failed closed with +`stable acquisition and source catalog binding sets differ`, and the role +restarted nine times. The first rollback used the same incomplete base chain +and therefore also could not load the required source/acquisition pair. The +active stream remained healthy at restart count zero throughout; no V1, Kafka, +Redis, SQLite, Query, Rust, ingestor, Trading System, alpha or order path was +touched. + +Root cause was packet provenance, not final-BAR source behavior: the active +runtime's `mark-index-r134-58998ae-r3` override mounts the matched +`stable-source-bindings.yaml` and `stable-acquisition-bindings.yaml`, but the +new packet had omitted that last active override. The helper now includes it; +both candidate and rollback full-chain renders pass. A second, exact rollback +of only `stream_v2_passive` restored image `sha256:1ad341...fa88d`, health and +restart count `0`, while active stayed healthy on that same digest. The B2 +helper digest above is the corrected one. This incident is retained as bounded +rollout evidence and makes the full current config-chain requirement explicit; +it does not broaden B scope or certify the candidate. + +**B2 runtime result and cold-cache correction boundary (2026-09-19; B remains +`IN_PROGRESS`).** The corrected five-role packet completed in its exact order: +`stream_v2_passive`, `stream_v2_active`, `projector_v2`, `projector_v2_3`, +then `projector_v2_2`. Every named role is healthy, `restart=0`, +`OOM=false`, on `sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06`. +V1, Kafka topology/offsets, Redis, SQLite reset/deletion, Rust, ingestors, +Query, Trading System, alpha and order paths remain unchanged. + +The first public-SDK preflight proved mTLS/JWT, the named Query route and the +ten declared quote bindings are reachable, but correctly failed strict quality +(`48` typed stale outcomes over eight rounds). There was no API error, gap, +incomplete record, session loss, direct-provider call or V1 fallback. At the +first aligned final-BAR boundary after the additive table appeared, the old +cache had no `final_bar_watermarks` rows. Each of the `70` final BAR partitions +therefore performed its permitted one-time retained-tail hydration inside the +live projector path. Projector spans recorded canonical lookup up to `10.91 s`, +and the resulting canonical age reached `27.03 s`; quote session liveness +remained live while strict event freshness properly failed. The durable table +then reached exactly `70` rows, proving the diagnosis. This is a migration +cold-start latency defect, not provider latency or an SLA/configuration issue. + +**B2.1 approved in-scope correction before C2.** Before a projector begins +Kafka polling, it must pre-hydrate the catalog's declared final-BAR partitions +through the existing bounded, atomic spool primitive. The first projector may +do the finite legacy work before accepting a live batch; concurrent/restarted +projectors observe the same persisted rows. The live `_ready_batch()` path must +therefore never initiate a multi-partition cold-cache hydration burst. There is +no new service, table, public API, provider call, timestamp alteration, manifest +change, fallback, or topology. Required proof: a pre-existing cache with many +BAR partitions is hydrated before poll; the first mixed final-BAR/strict-QUOTE +batch performs no retained-tail lookup; restart/rebalance reuse is O(1); +empty-cache and malformed/late/revised BAR behavior stay fail-closed and +correct. Run the affected no-network projector/spool/quality/SDK matrix, build +one immutable Python image, roll only the three existing projectors with exact +rollback, and then repeat the two-reader 300-second public-SDK matrix. A strict +failure still blocks B and invokes only that bounded rollback. + +**B2.1 source implementation and proof (2026-09-19; no B2.1 runtime role +changed).** `StableProjectorEngine` now derives the exact final-BAR partition +set from the sealed catalog and atomically hydrates missing rows before it is +registered ready or polls Kafka. `run_once()` and direct `accept_many()` share +the same idempotent guard, while the supervisor performs preparation before +publishing its broker to readiness. Thus a normal live batch can only read an +already persisted O(1) watermark; a migration scan is finite startup work, not +a quote-path side effect. The spool remains the sole durable authority, and an +empty partition remains valid until its first final BAR rather than being +invented. + +The regression converts the prior lazy-hydration proof into the required +pre-poll behavior: a pre-table historical BAR is seeded once during prepare; +the first newer/revised/late BAR plus a quote batch makes zero retained-tail +reads; a restarted engine reuses the shared row; and the supervisor records +`prepare` before it advertises a broker generation ready. Isolated no-network +source gates passed: `tests.test_phaseb_stable_edge` `58 passed, 1 skipped` +(the pre-existing isolated-Redis case); the remaining R1.35 quality/spool/ +batch-poll/stale/SDK modules `51 passed`; and stable deployment contract +coverage `28 passed`. `compileall` and `git diff --check` passed. The expected +argument-validation and injected-backpressure/recovery log lines in those +tests are fixture assertions, not runtime warnings. No image, stream, +projector, V1, Kafka topology/offset, Redis, SQLite data, Query, Trading +System, alpha or order path changed in this source slice. The next allowed +action is one immutable Python build followed by a three-projector-only packet +with the current B2 image as exact rollback, then a fresh two-reader C2. + +**B2.1 immutable build and bounded runtime packet (READY / NOT YET +EXECUTED, 2026-09-19).** The source commit is +`a23fcbe35ab586ba64bd045092c694eaca93f457`; its immutable Python image is +`sha256:73a3e677c56c196c063fa425dd1ccb200c3523f0573692abf09cb8febb66a13e` +(`qdl-v2-python:2.0.25-a23fcbe`, OCI revision `a23fcbe35ab586ba64bd045092c694eaca93f457`). +The only candidate roles are the existing `projector_v2`, `projector_v2_3` +and `projector_v2_2`, recreated one at a time with their established +`512/128` batch/commit bounds. Exact rollback for each named role is the +currently active B2 image +`sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06` +(`2.0.24-40a1155`) using exactly the same runtime mounts and `512/128` +bounds. The sealed external packet directory is +`/home/bobby/.local/state/qdl-v2/r135-b21-prewarm-20260919T221719Z`; its +candidate override SHA-256 is +`1a26f5788bd074d20bf6d2c8be401429bf3177dc2ce2837c1537c99c9389fc0d`, +rollback override SHA-256 is +`ad5e3ed1ca4bc5058fb2dc605c77f3e60900b39f4c74df8efce93ffbb510a35d`, +and exact-role helper SHA-256 is +`bdcae1706aad90c8ea55c61f15fa559c7b62ea333658eee06e93383a41c18771`. +Both full Compose chains rendered successfully and the helper passed +`bash -n`. + +The packet is deliberately projector-only: it does not recreate stream, +Query, Rust, ingestor, V1, Trading System or alpha roles, and it does not +reset Kafka offsets/topology, flush Redis, delete SQLite, change a manifest, +or touch an order path. After each named role, require its exact candidate +digest, health, `restart=0`, no OOM, bounded catch-up and continued healthy +peer service. Any failure stops immediately and uses the same helper in +`rollback` mode for only the changed named role. A successful packet must log +one finite watermark-prewarm summary, preserve/complete durable watermark +coverage for every declared final-BAR partition, and then run the fresh two-reader, public-SDK, +no-order strict C2 for `300` seconds before B can close. + +**B2.1 first-role result (2026-09-19; rollout continues).** `projector_v2` +alone was recreated at `2026-09-19T22:22:56Z`; it reached `healthy`, +`restart=0`, `OOM=false` on the exact candidate digest. Before broker +readiness it logged `partitions=144 seeded=70 empty=4`. Read-only durable +inspection reconciled the result: the sealed catalog declares `144` final-BAR +partitions, of which the cache already held `70`, the prewarm atomically added +the remaining `70` crypto rows, and `4` declared VN partitions genuinely have +no retained history while the market is closed. The shared watermark table now +contains exactly `140` durable rows; no empty row or synthetic BAR was +invented. The initial one-time preparation finished before polling; subsequent +projector spans dropped from the prior cold path to canonical lookup max +`10.7 ms` and canonical age max `534.6 ms` in the observed post-start sample. +Existing stream `409`/peer `200` responses are the established active/passive +idempotent fan-out behavior, not a delivery error. The remaining two roles may +only reuse these durable rows and must prove the same health/no-restart/OOM +gate before C2. + +**B2.1 packet outcome and strict rollback (`FAIL-CLOSED / CORRECTION +REQUIRED`, 2026-09-19).** All three named projectors reached the candidate +digest with `healthy`, `restart=0` and no OOM. The two public-SDK, no-order +strict-QUOTE probes then completed their full 300 seconds: each made `150` +rounds over the ten declared execution quotes and returned `1,496` usable +reads, but each rejected the same four observations of the OKX Swap BNB quote +binding `f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3`. Query-1 request/status and +snapshot p99 were `12.759 ms`/`12.566 ms`; query-2 were `44.832 ms`/`45.541 +ms`. There were no request errors, V1/direct-provider fallback, gap, identity +cross-mix, restart or OOM. The immutable bounded evidence is under +`/home/bobby/.local/state/qdl-v2/r135-b21-prewarm-20260919T221719Z/evidence/`. + +Read-only canonical timing establishes that this is not a B2.1 durable-cache +regression: the two rejected provider events bracket an approximately +`4,099.97 ms` upstream accepted-time gap, while their durable delivery commits +took `481.6 ms` and `648.6 ms`; the same source lane then resumed normal +sub-second quote delivery. Both typed responses remained `LIVE`, complete and +gap-free at the session/generation/config fences. The selected provider BBO +channels are update-on-change, so an unchanged best bid/offer can correctly +have an old *last event* while its one shared WebSocket session remains live. +Raw event age must remain immutable evidence; treating it as a connection +failure is an incorrect generic quality contract. + +The B2.1 packet declared a strict-matrix rollback rule. Before any new source +correction, the exact helper rolls only `projector_v2`, `projector_v2_3` and +`projector_v2_2` back to `2.0.24-40a1155` with their unchanged mounts and +`512/128` bounds. It does not reset offsets, flush Redis, delete SQLite, or +touch V1, Query, stream, Rust, ingestors, Trading System, alpha or any order +path. Disposable C2 client containers are removed after their already +persisted bounded evidence is copied out. + +**Rollback execution confirmation.** The declared helper was run for exactly +those three projectors; each returned to +`sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06` +(`2.0.24-40a1155`) with unchanged mounts, `healthy`, `restart=0` and no OOM. +The two disposable C2 client containers were removed after bounded evidence +persisted. This was a role-only rollback: no Kafka offset/topology, Redis, +SQLite, V1, reader/stream, Rust, ingestor, Trading System, alpha, broker or +order-path object changed. + +**In-scope source-contract correction after rollback.** The next repair is +not an SLA relaxation or a per-symbol exception. The catalog must explicitly +declare an `ON_CHANGE` delivery semantic only for documented native BBO quote +lanes; all other QUOTE sources remain `STRICT_EVENT`. A consumer may select +quiet observation only where its exact manifest route and the signed source +binding both permit it. Such a quote remains execution-eligible only while +the original event lineage is retained *and* its exact source session is +`LIVE`, generation/config match, no sequence gap is open, and independently +written transport liveness is inside that route's declared bound. A +disconnected, heartbeat-expired, generation-changed, gap-open or generic quote +remains fail-closed. Python and Rust must share this decision through golden +tests; the query edge may not rewrite an event timestamp or infer the source +property from a symbol name. The correction may update the catalog compiler, +binding parser, shared evaluators, exact quote entitlements and their tests; +it cannot add a service, provider REST fallback, relaxed external quota, new +consumer identity or runtime topology. A fresh sealed bundle and a query/ +stream-only packet will be prepared only after source parity passes. + +#### R1.35-C - Full endpoint, binding and consumer release certification (`PASS / RUNTIME-CERTIFIED`, 2026-09-21) + +**Goal.** Produce one reproducible, consumer-side certificate for every active +V2 binding and public V2 transport that the current release actually exposes. +This is the release gate, not a process-up check and not a BTC-only smoke. + +**Sealed acceptance inventory.** At start, seal source/acquisition/catalog, +consumer manifests, identity/key revisions, API/Proto/OpenAPI digests and +runtime image/config digest. The matrix must cover every active binding for: + +| Product family | Required consumer proof | +| --- | --- | +| Instrument/catalog | list, resolve, pagination/cursor and revision identity on both query replicas | +| Latest feeds | typed `feed_status` plus `snapshot` for TRADE, QUOTE, MARK/INDEX, BOOK_SNAPSHOT and BOOK_DELTA, each only under its declared strict/quiet policy | +| Bars | final-only snapshot/warmup/history for every declared interval; preserve OHLC, decimal/unit, ordering, bar revision/finality and no interior gap | +| Warmup | one exact requirement per active binding plus bounded batch coverage; maxlen boundaries `700`, `2,500`, `5,000` and `10,000` are tested where the consumer contract declares/supports them | +| Reference | `reference:batch` for every entitled funding, OI, long/short, taker flow, mark/index, metadata and native/continuous basis requirement; uncovered capability-only products remain explicitly non-execution, not fabricated as active demand | +| Stream/replay | authenticated Subscribe, signed cursor replay, reconnect and resume on every active stream product with per-partition identity, ordering/watermark, no duplicate and no gap proof | +| Quality/control | readiness, feed status and gap views agree with the SDK outcome; expected V1/dark/out-of-session rows remain explicitly excluded | + +**Latency method.** Every measurement is made from an authenticated consumer +outside the Data Layer role and records, per endpoint/binding/replica, sample +count, p50/p95/p99/max, typed errors, fallback/direct-provider attempts and +resource context. The certificate separately reports: + +1. venue timestamp -> host receipt; +2. host receipt -> Kafka acknowledgement; +3. Kafka -> canonical/core; +4. canonical -> hot view and canonical -> durable spool; +5. consumer request start -> SDK-usable response; +6. stream event -> consumer receipt; +7. final bar close -> SDK-usable final BAR; and +8. reconnect/cursor-replay completion time. + +No aggregate average may replace a per-binding result, and immutable event age +may not be reported as API-call latency. The reporting artifact stores bounded +metrics, hashes and typed state only, never secrets, prices, book levels or raw +credentials. + +**Required test gates.** + +- Full Rust/Python contract, generated schema/SDK, unit, golden, parser, + provider-conformance, compatibility and migration-idempotency suites. +- Real-provider no-order C2 of at least 300 seconds using the sealed Trading + System and representative alpha workload identities, `require_all=true`. + It must prove V2-primary; allowed V1 fallback is drilled only where the + manifest permits it, and `BLOCKED` products must stay blocked. +- Fault/recovery tests for connection loss, resubscribe, durable replay, + slow consumer/cursor expiry, duplicate/gap/resync and active/passive query or + stream handoff. Real production connections are not intentionally severed; + failure injection uses an isolated runtime or durable captured provider + bytes with explicit provenance. +- Cross-venue, decimal/unit, instrument identity, final BAR, reference lineage, + L2 sequence/depth/checksum and resource/capacity assertions. Zero order, + alpha signal/sizing or broker-state mutation is permitted. + +**Exit gate.** Every expected active binding succeeds under its declared +semantic policy on both query replicas; all endpoint families above have +consumer-side evidence; V1 fallback behavior is policy-correct; no silent +loss, unexplained gap, duplicate, cross-venue mix, stale strict execution data, +runtime restart/OOM or unbounded resource growth remains. The certificate must +state limits honestly: DNSE/VN remains V1-primary until its market-hours +certificate, and dark Spot/catalog entries are not V2 execution coverage. + +**C2 replacement source and bounded runtime decision (2026-09-20; prepared, +not executed).** The first full C2 stopped fail-closed before its observation +window when the secondary Query returned `SOURCE_UNAVAILABLE` for the entitled +`OKX/SWAP/DOGE-USDT-SWAP/MARK_INDEX_PRICE` product. Read-only durable evidence +showed a current canonical row and a direct authenticated active-stream probe +served all ten declared Binance/OKX mark/index bindings. The defect is therefore +the active-stream lease/promotion lifecycle: an empty in-memory execution view +can exist until a later provider update even though the exact durable canonical +row is available. It is not a DOGE exception, provider fallback, stale-policy +relaxation, timestamp rewrite or manifest change. + +Source commit `1a9da358c853c224bdba517f791d309b774bdee1` hydrates only the +exact declared MARK_INDEX bindings from the existing canonical spool when a +stream newly acquires its lease. Hydration reuses normal identity/generation/ +gap/watermark validation, never publishes, calls a provider, rewrites source +time, or writes spool state. Callback failure fences/releases the lease +fail-closed. Source proof: `131` affected Python tests passed, Rust core/realtime +tests passed `91` with one pre-existing optional Redis integration skip, +`compileall` and `git diff --check` passed. The immutable candidate is +`qdl-v2-python:2.0.26-1a9da35@sha256:c062ded23350de63661e6f93dc1c2ace7caa6d913a47ee7e18654050e8828b72`. + +The prepared replacement packet may recreate only, serially, +`stream_v2_passive`, `query_v2_1`, `query_v2_2`, and `stream_v2_active`. It +retains the exact existing sealed runtime +`/home/bobby/.local/state/qdl-v2/r135-b2-335792a-20260920T015057Z/runtime`, +identity/trust extension, TLS and state volumes, Kafka topology/offsets, Redis, +SQLite, V1, Rust cores, ingestors, bar edge, projectors, Trading System, alpha +and every order path. The exact rollback for each changed reader is +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c` +with the same runtime/config mounts. After each recreate require `healthy`, +`restart=0`, `OOM=false`, a healthy unchanged peer, and an authenticated +ten-binding mark/index active/passive preflight. Then run one new isolated, +no-order, `require_all=true` C2 for the sealed `303` routes for `300` seconds. +Any preflight/C2 failure stops the packet and rolls back only reader roles +already changed; it cannot be relabelled C pass or trigger an unplanned repair. + +**C2 replacement execution (`FAIL-CLOSED / ROLLBACK REQUIRED`, 2026-09-20).** +The first invocation of the disposable client was rejected before it reached +any Data Layer endpoint because the copied harness had an unreadable +drop-privilege script and a malformed local heredoc. The harness-only defect +was corrected in the disposable packet; no route, provider, consumer or +runtime service result was inherited from that invocation. The replacement +client then ran with the candidate reader image and the four readers stayed +`healthy`, `restart=0`, `OOM=false`. It stopped during the opening exhaustive +product read, before the 300-second observation, at the independent product +`alpha.okx.paper.stable / OKX.SWAP.PERPETUAL.DOGE-USDT / BAR / 12h` because the +SDK correctly returned `required feed has an unresolved sequence gap`. + +This is a genuine full-C2 failure, not a MARK/INDEX hydration regression, +timeout, quota error, fallback, direct-provider request or order mutation. The +acceptance cancelled its remaining active products and emitted no success +certificate. The evidence is bounded at +`/home/bobby/.local/state/qdl-v2/r135-c-hydrate-1a9da35-20260920T033124Z/c2-full/evidence/`. +Per the sealed rollback policy, the four reader roles must return to +`sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c` +before any read-only investigation of the exact BAR sequence/generation state. +R1.35-C remains `IN_PROGRESS`; no release, merge or cleanup is permitted from +this result. + +**Rollback confirmation.** The packet restored exactly `stream_v2_passive`, +`query_v2_1`, `query_v2_2` and `stream_v2_active` to the recorded +`sha256:8c53d...168c5c` image. Each is `healthy`, `restart=0`, `OOM=false` and +again mounts the unchanged sealed runtime directory. No other named service was +recreated and no durable store, offset, credential, manifest, consumer, alpha +or order state was modified. + +**C2 BAR-history investigation and bounded repair decision (`IN_PROGRESS`, +2026-09-20).** The first read-only scanner used a `10,000` *logical-offset* +tail. That is not the Query path: Query reads the bounded `12,064` physical +tail, orders BARs by market open, and then selects the consumer's market-time +window. Its apparent `55` affected bindings therefore included harmless +late-backfill history omitted by the diagnostic's shorter logical tail. That +reading is withdrawn, not used as a repair scope. + +The corrected disposable verifier used the exact shared physical capacity +(`12,064`) and scanned all `144` active BAR bindings / `893,360` retained rows. +It found no sequence flags and exactly `25` real recent continuity gaps, all on +the five OKX Swap symbols: `1h` (`25` missing final bars), `2h` (`15`), `4h` +(`10`), `6h` (`5`) and `12h` (`5`). The `12h` gap is the one that stopped C2 at +`OKX/SWAP/DOGE-USDT`; it is neither a MARK/INDEX hydration regression nor a +false sequence-flag interpretation. The scanner changed no canonical, spool, +provider, consumer or runtime state. + +The only approved in-scope correction is the existing +`scripts/repair_stable_final_bar_history.py` path, invoked serially per +affected binding with a bounded `700`-BAR provider window: first record its +provider-backed dry-run's exact missing count, then publish only the matching +real final envelopes through normal V2 Kafka/canonical/projector flow. The +active writer already uses the shared `10,000 + 2,064` physical capacity. The +repair's own convergence check protects the requested market-time window; a +physical append tail is not itself a public warmup contract. The script +validates the count before write and never fabricates a candle, rewrites a +timestamp, resets checkpoint/offset, flushes Redis, deletes SQLite or recreates +a serving role. V1, Rust, ingestors, projectors, readers, Trading System, alpha +and order paths remain unchanged. If a provider cannot supply the exact bounded +history or an expected window cannot converge, repair and C2 stop fail-closed; +no SLA, warmup, gap policy or manifest route may be relaxed. + +**Repair/C2 exit and rollback.** After each real-provider repair, verify the +same provider-confirmed market-time window has zero remaining opens; then run a +read-only semantic scan matching Query's physical-read / market-time-tail logic +for each declared `700`, `2,500`, `5,000` and `10,000` BAR boundary. Every +consumer-visible requested window must be gap-free before the already-built +reader candidate is retried. The only later reader mutation remains the +previously sealed four-reader candidate packet with its exact old-image +rollback. A repair failure is recovered by leaving existing durable data intact +and recording the provider limitation; no broad rollback or deletion is +allowed. + +**OKX bounded repair dry-run (`PASS / APPLY COUNT-FENCED`, 2026-09-20).** The +existing repair client ran serially for all `25` affected OKX bindings with +`rows=700` and the exact per-binding count fence derived from the corrected +physical verifier. Every invocation returned `DRY_RUN`; all `25` provider +windows were complete and the aggregate missing count was exactly `60` final +BARs (`1h=25`, `2h=15`, `4h=10`, `6h=5`, `12h=5`). There were zero production +mutations and no stderr. The active bar edge remained `running`, `restart=0`, +`OOMKilled=false`. Evidence is bounded at +`/home/bobby/.local/state/qdl-v2/r135-c-hydrate-1a9da35-20260920T033124Z/c2-full/evidence/okx-bar-repair-dry-run.jsonl`. + +The next permitted action is one serial apply over precisely these same `25` +bindings, each with its same count fence, `180s` convergence limit and normal +Kafka -> Rust canonical -> existing projector/cache path. A count change, +generation fence, provider failure or non-converged binding stops the serial +run immediately and blocks C2; it does not widen scope or retry blindly. + +**OKX bounded repair and semantic-window exit (`PASS / C2 RETRY PERMITTED`, +2026-09-20).** The serial apply completed all `25/25` count-fenced bindings. +It published exactly `60` provider-confirmed final BARs, every invocation +reported `CONVERGED`, and every `remaining_rows` value is zero; stderr is empty. +The post-repair disposable verifier then executed Query's actual physical-read, +market-time-sort and bounded-tail semantics for all `144` BAR bindings at +`700`, `2,500`, `5,000` and `10,000` rows. It found `0` duplicate opens, +sequence flags or interior continuity failures across every consumer-visible +window. This deliberately replaces the withdrawn logical-tail inventory, not +the public no-gap rule. The active bar edge remained `running`, `restart=0`, +`OOMKilled=false`, about `135 MiB / 512 MiB`; host free disk remained `166 GB`. +Evidence: `okx-bar-repair-apply.jsonl`, +`bar-semantic-window-scan.json` and their bounded stderr files under the C2 +evidence directory above. + +The BAR continuity blocker is closed. The next permitted R1.35-C action is the +already-sealed, serial four-reader candidate retry followed by exactly one +`require_all=true`, no-order C2 observation of `300` seconds. It must produce +a new certificate; this repair result does not inherit success from the earlier +failed C2. + +**C2 retry quote-semantic finding and bounded correction (`IN_PROGRESS`, +2026-09-20).** The approved retry ran the four candidate reader roles and then +stopped fail-closed during its exhaustive opening read at +`trading-system.paper.stable / OKX.SWAP.PERPETUAL.BNB-USDT / QUOTE`: the +acceptance validator reported `V2 receipt exceeds the governed freshness +bound`. The packet immediately restored exactly the same four readers to +`sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`; +each is healthy, restart `0`, and not OOM-killed. No V1, durable store, offset, +identity, consumer, alpha, Trading System or order-path object changed. + +Read-only evidence rules out a provider disconnect: all five OKX native BBO +quote bindings had a `LIVE` source session (about `960 ms`), complete retained +lineage and no gap, while their last *on-change* BBO frame was about `3.3 s` +old. That is valid documented BBO behavior when bid/ask is unchanged. The +sealed catalog already declares those bindings `delivery_semantics: ON_CHANGE`; +the failure is shared evidence logic: the offline auditor omitted that field +when invoking the quality reducer, and the C2 receipt validator did not admit a +declared, execution-eligible, session-live `QUOTE` through the same quiet +semantic already implemented by Query. It is not a BNB exception, a freshness +SLA relaxation, a provider fallback, a timestamp rewrite, or a market-data +repair. + +**Approved source-only repair scope and exit.** Propagate the declared +`ON_CHANGE` semantic into the audit reducer and extend C2's quiet receipt +predicate only for an execution-eligible `QUOTE` carrying the authoritative +`DELIVERY_ON_CHANGE` receipt flag. Add Python/Rust/golden-compatible regression +coverage for on-change quiet/live, generic strict quote, session loss, +heartbeat expiry, generation/config mismatch and gap-open rejection. The exact +source gate is all affected quality, Query, audit and C2 tests passing with no +semantic drift. A fresh immutable reader image and a new exact four-reader +packet require a separately recorded digest/config rollback before any runtime +retry; the prior `c062...` approval does not authorize a changed binary. + +**Source correction and verification (`PASS / NOT ROLLED`, 2026-09-20).** The +audit now passes the catalog-declared `delivery_semantics` into the same +provider-neutral reducer used by Rust and Query. The C2 validator now admits a +stale `QUOTE` only when its governed requirement is `OBSERVE`, its Query-issued +receipt carries `DELIVERY_ON_CHANGE`, and that receipt is already +`execution_eligible`; the existing state, session-liveness, completeness, +gap, lineage, durable-primary and replica checks remain mandatory. The new +regression uses the exact `trading-system.paper.stable` OKX BNB BBO product and +rejects absent on-change authority, non-eligible price, disconnected/expired +session and an open gap. It does not alter provider timestamps, source quotas, +manifest freshness values or fallback policy. + +Source-only evidence, all with read-only source/no network and no runtime data +mutation: focused quality/C2 tests `25 passed`; full affected +quality/query/audit/C2/stable-edge/deployment/session matrix `123 passed, 1 +pre-existing isolated-Redis skip`; Rust `qdl-core` +`rust_matches_shared_binding_quality_golden_corpus` exited `0`. The named +`qdl-r135-source-matrix` test container and all `--rm` Rust test containers +were removed after their result. `git diff --check` passes. The active runtime +remains the exact four-reader rollback image +`sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`; +no V1, Kafka, Redis, SQLite, core, ingestor, Trading System, alpha or order +path object changed. A new immutable reader image, an exact four-role packet +and a fresh `require_all=true` C2 observation remain the next decision +boundary; this source result alone is not a certificate. + +**Immutable-reader build preflight (`APPROVED SOURCE / NO RUNTIME MUTATION`, +2026-09-20).** Build exactly one candidate from the sealed current feature +tree as `qdl-v2-python:2.0.26-`, with its full OCI revision and +release string recorded after the immutable build. The Docker context excludes +Git, `data`, logs and caches. The sole retained reader rollback coordinate is +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`. +This build may create an image/cache only; it does not authorize Compose, +runtime/bundle/TLS changes, or a reader recreate. Its digest and image-local +tests must be recorded before a new exact four-role packet is requested. + +**Immutable-reader build and image-local gate (`PASS / NOT ROLLED`, +2026-09-20).** The sealed code revision +`2af2cdd394514387923bee89eaf6bdd841bee4d8` built as +`qdl-v2-python:2.0.26-2af2cdd39451@sha256:4f29daaa89da74e916739cf367592f94c6f06c50829dc2a8b72bde7cb98542a9`. +Its OCI revision and release labels are respectively that full SHA and +`2.0.26-2af2cdd39451`; the runtime user is `qdl:qdl` (`10001:10001`). With no +source mount or network, the same affected quality/query/audit/C2/stable-edge/ +deployment/session matrix passed `123`, with the one pre-existing isolated +Redis skip. The exact `qdl-r135-image-matrix` test container was removed after +exit `0`. This image is a candidate only, not an active role image; the +documentation commit that records this evidence is intentionally a descendant +of the built source SHA and does not alter the candidate's code provenance. + +**Next packet boundary.** The only permitted runtime change is a serial +recreate of `query_v2_2`, `query_v2_1`, `stream_v2_active` and +`stream_v2_passive` to this exact candidate digest, retaining their existing +runtime/TLS/state mounts and all other Compose configuration. The exact +rollback for every one of those roles is +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`. +No C2 retry, release, cleanup, V1/Rust/Kafka/Redis/SQLite change, or consumer/ +order-path mutation is authorized by this source/image evidence alone. + +**Reader packet preflight (`PASS / NOT ROLLED`, 2026-09-20).** The bounded +packet is stored outside Git at +`/home/bobby/.local/state/qdl-v2/r135-c-on-change-2af2cdd-20260920T051916Z/`. +It contains only a payload-free packet descriptor, the four-image candidate +override, the exact four-image rollback override and executable operator-only +preflight/roll/rollback scripts. It names no secret values. Script syntax and +packet schema passed; the preflight rendered the complete existing Compose +chain without mutation, verified each reader at the recorded rollback digest +and runtime mount, and observed exactly one cooperative stream lease: +`stream_v2_active=STANDBY`, `stream_v2_passive=READY`. The rollout script rolls +the standby before the current leader and recomputes the lease between steps; +its rollback guard invokes rollback only for roles actually recreated. This +packet has not run `up`, did not restart a role, and does not authorize the +following `300`-second C2 itself. + +**C2 runner preflight (`PASS / NOT RUN`, 2026-09-20).** The same operator-only +packet now contains one exact no-order C2 client launcher. Its packet descriptor +SHA-256 is `46690e2724af60431ea50daced7edfe5b385f879f098045e44b6a51fcfa6dc64`. +The C2 bootstrap and acceptance command are byte-identical to the prior +reviewed runner; only the candidate image/digest, packet path, evidence +namespace and disposable client container name differ. Static shell checks +passed for packet scripts, and C2 preserves four identities, both query +replicas, both stream endpoints, V1 fallback policy, signed cursor/reconnect, +`require_all=true`, the full `303`-route scope and an observation of exactly +`300` seconds. It mounts identities/runtime read-only, drops privileges before +the client, creates no order/signal/sizing mutation, and removes its client +container on exit. It has not been executed. + +**Runtime approval (`APPROVED`, 2026-09-20).** The operator explicitly +approved the packet above: serially recreate only `query_v2_2`, `query_v2_1`, +the observed stream standby and then the observed stream leader to candidate +`sha256:4f29daaa89da74e916739cf367592f94c6f06c50829dc2a8b72bde7cb98542a9`; +retain runtime/TLS/state mounts and current Compose selectors; rollback only +changed roles to `sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`. +The same approval permits exactly one isolated no-order `303`-route C2 for +`300` seconds. V1, Rust, ingestors, bar edge, projectors, Kafka topology/ +offsets, Redis, SQLite, Trading System, alpha and the order path remain +excluded. + +**Packet orchestration interruption (`FAIL-CLOSED / NO C2`, 2026-09-20).** The +first approved invocation recreated `query_v2_2`, `query_v2_1` and the initially +observed standby `stream_v2_active`, then stopped before C2 with +`stream_v2_passive` still on rollback image. No C2 client, provider call, +order/signal/sizing action, V1, durable store, offset, manifest or consumer +mutation occurred. The packet, not the reader binary or data plane, was wrong: +after the new standby acquired the cooperative lease it recomputed "leader" by +current status and could select that same role again instead of the remaining +old-image peer. The in-scope recovery is to rollback exactly those three changed +roles, then correct only the operator packet to capture two distinct roles +before either recreate, wait boundedly for one `READY` plus one `STANDBY` after +each role, and explicitly rollback every changed role on any fence failure. +This preserves the approved four-role/image/mount boundary; it changes no +source image, contract, topology, data or consumer policy. A fresh serial +preflight and the one approved C2 are still required after the baseline is +restored. + +**Rollback and packet correction (`PASS / RETRY READY`, 2026-09-20).** The +rollback restored exactly `query_v2_2`, `query_v2_1`, `stream_v2_active` and +the subsequently observed changed `stream_v2_passive` to +`sha256:8c53d37f...01668c5c`. All four are `healthy`, `restart=0`, +`OOMKilled=false`, retain the sealed runtime mount, and the stream pair is one +`READY` plus one `STANDBY`. The corrected external operator runner is +SHA-256 `63da98e540ce0f097b4e6d8ebc10c2775d6cbf41b012663bcab2818221c0ad4f`; +it captures two distinct stream roles before either recreate, waits up to +30 seconds for lease convergence after each step, verifies the remaining peer +is still on rollback image before its recreate, and explicitly rolls every +changed role back on an error. `bash -n` passed for preflight, rollout and +rollback scripts. This is packet-only safety work; it neither changes the +candidate binary nor supplies C2 evidence. The next permitted action remains +one fresh serial four-reader rollout, then the one approved C2. + +**Hardened retry timing (`FAIL-CLOSED / BASELINE RESTORED`, 2026-09-20).** The +first hardened retry reached the initially observed standby +`stream_v2_passive` after the two Query roles, but its `30`-second lease wait +expired before that new replica advertised the stable pair. It never recreated +the remaining `stream_v2_active` and never ran C2. Exact rollback restored the +three changed roles (`query_v2_2`, `query_v2_1`, `stream_v2_passive`); all four +readers are again healthy on `sha256:8c53d37f...01668c5c`, restart `0`, +OOM false, with one `READY` and one `STANDBY`. The deployed cooperative lease +is TTL `15s`, renew `5s`; therefore the packet-only convergence bound is +increased to `60s` (four TTLs), without changing that runtime configuration, +the candidate binary, service set, mount, source policy or accepted rollback. +The next retry remains the same four distinct roles and the same single C2; +this failed fence supplies no acceptance evidence. + +**Operator transport correction (`IN_PROGRESS / SAME PACKET`, 2026-09-20).** +The execution shell used for the packet terminates a foreground orchestration +process at its short tool window and does not preserve detached host children; +the trace stopped during a normal `starting` health poll and the detached +wrapper left no exit file. This is not a reader, lease, image or data-plane +failure. The all-rollback baseline is again confirmed. The same approved packet +will therefore be executed as four individual Compose recreates with the same +override chain, candidate digest, role order, mount checks and rollback digest; +after every recreate a separate bounded health/lease observation is recorded. +No extra role, image, service, topology or C2 invocation is introduced. The +final C2 will likewise use a disposable Docker-contained client so its +300-second observation is not subject to the host shell window. + +**Four-reader rollout (`PASS / C2 READY`, 2026-09-20).** From the restored +rollback baseline, the exact approved roles were recreated serially through the +sealed Compose chain: `query_v2_2`, `query_v2_1`, then observed standby +`stream_v2_passive`, then its retained peer `stream_v2_active`. After each +step, the recreated role reached `healthy`, `restart=0`, `OOMKilled=false` and +the unchanged sealed `/runtime` mount before the next step. The final state is +all four roles at candidate +`sha256:4f29daaa89da74e916739cf367592f94c6f06c50829dc2a8b72bde7cb98542a9`, +with one stream `READY` and one `STANDBY`. No other role was recreated; V1, +Rust, ingestors, bar edge, projectors, Kafka, Redis, SQLite, Trading System, +alpha and order path remain untouched. C2 has not run. The next and only +remaining acceptance action in this packet is the approved disposable, +no-order, `303`-route, `require_all=true` observation for `300` seconds. + +**C2 bootstrap permission fence (`FAIL-CLOSED / NO ROUTE EXECUTED`, 2026-09-20).** +The disposable client exited `2` before the acceptance process began. Its +root bootstrap correctly dropped to UID/GID `10001`, but the mounted +`/run-c2-full.sh` packet script was not readable by that non-root identity; +the original runner reported `cannot open /run-c2-full.sh: Permission denied`. +There is no acceptance receipt, provider request, order/signal/sizing action, +or runtime mutation from this attempt. The in-scope packet repair is only to +make that reviewed, secret-free shell script readable (`0644`) while identities +remain copied only inside the client tmpfs and all secret mounts remain +read-only/non-printed. The four candidate readers remain healthy. A new C2 +client after that bootstrap repair is the first actual `300`-second observation; +the rejected bootstrap is not counted as a C2 retry. + +**C2 opening-budget failure (`FAIL-CLOSED / ROLLBACK REQUIRED`, 2026-09-20).** +The first actual client ran, retained the privilege-drop proof, then exited `1` +before `C2_OPENING_PASS`; `acceptance.json` is empty and the bounded stderr +shows `asyncio.TimeoutError` at the sealed `900s` opening deadline. The +cancelled tail was `alpha.okx.paper.stable` `TRADE`/`QUOTE`, specifically while +the C2-local `_C2ConsumerRequestPacer` waited for its next permitted request. +This is not a provider, Query, stream, source-quality, manifest, identity or +order-path rejection: the probe deliberately serializes every real request per +consumer at 75 percent of its manifest quota and its fixed opening deadline is +too small for the complete V2-primary proof. No receipt, fallback success, +provider mutation, order/signal/sizing action or release evidence was emitted. +The `--rm` client removed itself; its only persisted artifacts are bounded +stderr, exit code and privilege proof in this packet. + +Per the sealed failure policy, rollback exactly the four reader roles to +`sha256:8c53d37f...01668c5c` before any retry. The only admitted correction is +an offline, scope-derived C2 opening budget computed from the sealed product +count, request shape and each real consumer's declared quota at the existing +75-percent pacer fraction. It may increase the acceptance *deadline* only up +to its computed operation budget, never change quota, concurrency, product +scope, quality SLA, source, route, identity, fallback policy or runtime +topology. An explicit operator timeout can only be larger than that calculated +minimum; a smaller value is rejected before any request. A new client after the +calculation and rollback is a fresh acceptance attempt, not inherited evidence. + +**Scope-derived C2 deadline (`IN-SCOPE PACKET CORRECTION`, 2026-09-20).** The +sealed manifests contain `5` monitoring, `61` Trading System, `125` alpha +Binance and `110` alpha OKX requirements. The release has `303` routes: +`299 V2_PRIMARY` products and `4 V1_PRIMARY` products. The four C2 consumer +manifests contain `301` routes, comprising all `299` V2-primary products plus +two explicitly excluded V1-primary VN routes; the remaining two V1-primary +routes belong to the out-of-scope `alpha.vn.paper.stable` consumer. This is +recorded in C2 evidence rather than silently treating `303` as a V2 read set. +Monitoring/alpha Binance/alpha OKX each declare +`180 rpm`, so their existing C2 pacers remain `135 rpm` (75 percent); Trading +System remains `1,125 rpm` from its `1,500 rpm` declaration. The failed run +proved the alpha OKX lane can consume its legitimate bounded retry/stream +handoff budget beyond `900s` without any typed product failure. The replacement +keeps the same four identities, all `299` V2-primary products, rate fraction, concurrency +`4`, Query/stream targets and `300s` observation, and changes only the +opening deadline from a fixed default to the sealed operation budget. This is a +capacity-correct acceptance deadline, not an API/runtime quota change or a +reduced test. It remains fail-closed: a requirement-level stale/gap/identity +failure still fails C2 rather than being converted into a longer deadline. + +**C2 operation-budget correction (`IN_PROGRESS / SOURCE-ONLY`, 2026-09-20).** +The preceding fixed `900s` value and its direct `1,800s` replacement are +operator containment values, not venue facts or execution-quality SLOs. The +failed run exposed that the C2 harness has to distinguish certification +capacity from the market-data product it certifies. Before any further reader +rollout, C2 will compile a payload-free, deterministic opening-operation plan +from the sealed acceptance scope and consumer manifest: exact nominal Query and +stream-open operations per identity, allowed V1 fallback-return operations, +reference-batch operations, declared per-identity quota and the explicit +two-session cursor handoff. The plan will derive the smallest safe opening +deadline from the actual SDK path and emit it in evidence. An optional +operator-supplied timeout may only be larger; there is no hidden fixed cap. +It must reject a scope that cannot fit rather than silently lengthening, +reducing coverage, lowering quota safety, +or changing a data freshness policy. + +This correction is restricted to the C2 source/test harness. It must not alter +Binance/OKX/DNSE quotas, source bindings, `max_freshness_ms`, quiet/session +semantics, retry behavior of deployed SDK consumers, runtime images, roles, +Kafka, Redis, SQLite, V1 or any consumer/order path. Required source gates are +deterministic nominal-plan, batch/reference, fallback, two-session handoff, +multi-identity, exact V2/V1 route-accounting and under-budget regressions; the existing C2 correctness +matrix must remain green. Only after those gates pass may one immutable +client-only image and the same four-role reader packet be prepared. The replacement C2 +will retain all `299` V2-primary products and its `300s` observation; a full pass remains +the sole R1.35-C exit evidence. + +**C2 operation-budget source exit (`PASS / RUNTIME PACKET NEXT`, 2026-09-20).** +The C2 harness now compiles its opening budget from the actual sealed SDK +operation graph and records only aggregate operation counts, quotas and typed +timing boundaries. The exact full-release reconciliation is: `303` global +release routes = `299 V2_PRIMARY + 4 V1_PRIMARY`; the selected C2 identities +own `301` routes = `299 V2_PRIMARY + 2 V1_PRIMARY`, while the other two V1 +routes belong to `alpha.vn.paper.stable` and remain outside the crypto C2 by +declared policy. The scope builder proves the C2 product identity set equals +every selected V2-primary route; no V1-primary route can be silently read as +V2 evidence. + +For the current sealed scope, the derived opening deadline is exactly `935s`: +the slowest declared identity has a `260s` 75-percent-quota pacing floor, the +single shared Rust-native BASIS lane has at most `600s` of already-contractual +typed deferral, and the longest declared response tail is `75s`. These values +are not quote, bar, book or execution freshness values. A caller may supply a +larger explicit containment timeout, but the harness no longer has an arbitrary +default cap; a lower value is rejected before opening a request. Retryable +strict-data failures remain product failures under their own existing +requirements, rather than being converted into extra certification capacity. + +The isolated non-root, read-only, no-network source suite passed +`28/28` direct C2 tests and `147/147` affected acceptance/quality/reference/ +fallback/route/stable-edge tests, with `1` pre-existing isolated-Redis skip. +The full-scope compiler independently emitted `1,482` nominal opening +operations and the count/timing evidence above. `git diff --check` and source +parse checks pass. All test containers used `--rm`; no persistent test +container, provider connection, role, image, Kafka/Redis/SQLite/V1, Trading +System, alpha or order-path state changed. + +**Narrow runtime decision.** This source slice changes only the disposable C2 +client harness. It does not justify rebuilding or changing the already-tested +reader binary `qdl-v2-python:2.0.26-2af2cdd39451@sha256:4f29daaa...98542a9`. +After this commit, build one immutable *client-only* image from the committed +source for the exact `--rm` acceptance launcher; retain it only through +R1.35-D. Then serially recreate the same four reader roles to the existing +`2af2...` candidate, with the recorded `335792a...` rollback, and run one +full `299`-product C2 for `300s`. No other role, topology or consumer changes +are permitted. + +**Full-scope C2 result and bounded pre-C2 correction (`FAIL_TYPED_STATUS / ROLLED BACK / SOURCE ONLY NEXT`, 2026-09-20).** The one permitted full C2 started +with the derived `935s` opening budget, completed opening for all `299` +V2-primary products in `905.117s`, then completed the required `300.100s` +observation. Its secondary closing BAR batch returned a generic +`DataLayerError`; the payload-free status representative was +`binance-usdm-dogeusdt-bar-12h`, which was itself `LIVE`, complete, gap-free +and execution-eligible with a valid interval-scaled age. The generic SDK +exception discarded the server's per-item batch problem, so that receipt is +not sufficient to attribute the fault to a product or to relax any SLA. The +packet therefore restored exactly `query_v2_1`, `query_v2_2`, +`stream_v2_active` and `stream_v2_passive` to +`qdl-v2-python:2.0.26-335792a@sha256:8c53d37f...01668c5c`; all four are +healthy with zero restarts/OOM, and stream dependency lease is `READY/STANDBY`. +V1, Rust, ingestors, BAR edge, projectors, Kafka, Redis, SQLite, Trading +System, alpha and the order path were not changed. + +One bounded, read-only diagnostic then reissued the exact alpha-Binance final +BAR closing matrix through both rollback query replicas: `70` products in two +batches per replica, `partial=false`, zero per-item problems and no payload +recorded. That validates the interval-scaled final-BAR contract on the known +rollback coordinate but does **not** make the failed candidate C2 pass. + +Before any candidate retry, this phase now adds one source-only +**pre-C2 read-plane matrix**: it must batch-read every selected V2-primary +product through both replicas using the same sealed identities and requirements, +record only per-item identity/problem/quality hashes, and fail if any result is +partial, stale, gapped, incomplete, non-final, non-authoritative or +cross-replica inconsistent. It opens no stream, executes no fallback drill, +creates no provider/order action and stays below each manifest identity's +declared safe request budget. The C2 closing path must preserve bounded +per-item failure codes rather than collapsing a partial batch to a generic +`DataLayerError`. Unit/contract tests cover successful long-interval BAR, +partial item diagnostics, scope/accounting and quota-bound preflight. Only +after this matrix passes on the rolled candidate may exactly one new full C2 +run; it remains the sole R1.35-C release exit and no SLA, freshness policy, +manifest or product scope is loosened. + +**Approved threshold semantics and accelerated certification gate (`IN_PROGRESS / SOURCE ONLY`, 2026-09-20).** This corrective slice prevents C2 from +being used as a debugger while preserving its release authority. It makes the +following four clocks explicit and forbids substituting one for another: + +1. **Acceptance operation budget.** C2 opening time is a manifest-derived + orchestration budget: exact selected products, identity request quota, + batch shape, bounded concurrency and a fixed observation tail. It is not a + provider SLA, consumer freshness limit, retry budget or serving timeout. + A fixed `900/1800s` cap is prohibited; the runner must reject a requested + concurrency that exceeds the signed identity quota and publish the derived + operation plan. +2. **Provider recovery budget.** External Binance/OKX/DNSE work alone may use + provider-shared token pacing/concurrency, documented `Retry-After`/ + rate-limit signals and bounded jitter. Circuit state is keyed by + `provider + declared route/generation`, so a successful route resets only + its own failure state and one broken route cannot cool down unrelated + products. `INTERNAL_STREAM` is a local authenticated canonical read: it + has finite concurrency, one attempt, no external token bucket and only a + short route-local failure circuit. It must never inherit an external venue + cooldown or retry ladder. +3. **Binding usability budget.** `max_freshness_ms` is immutable-event age + only when a route is `STRICT_EVENT`; `event_recency_policy=OBSERVE` shifts + a quiet channel to its independently signed provider-session and component + cadence bounds. Sequence/gap, generation/config identity, completeness and + execution eligibility remain independent fences. This covers quiet + `TRADE`, `BOOK_DELTA`, native on-change `QUOTE` and paired + `MARK_INDEX_PRICE` without accepting a disconnected or gapped channel. +4. **Scheduled final-BAR budget.** A BAR's interval-scaled + `max_freshness_ms` is a continuity/dropout horizon, not close-to-usable + latency. Finality is proved separately from native final flags or repeated + provider settlement; close-to-usable latency is measured separately in the + certificate. A late final BAR may never be relabelled fresh merely because + its dropout horizon is large. + +The source change is limited to the existing C2/quality acceptance tooling, +the shared bounded Query warmup executor that implements the local +`INTERNAL_STREAM` boundary, and their tests: validate every selected +requirement belongs to the correct semantic class; emit its semantic class and +declared budgets in payload-free evidence; retain exact per-item batch +problems; reset local failure state by route/generation; and run a fast exact +read-plane matrix before any full C2. It may not change a consumer manifest, +runtime authority, provider adapter, C2 product inventory or public schema in +order to pass. The current BAR continuity values and all provider budgets are +measured/reported first; a configuration change is permitted only if a +real-provider measurement proves that its declared contract is wrong. + +**Fast gate order and exit.** (a) source unit/contract/golden tests cover +threshold class, strict/quiet/session/gap/generation and recovery reset; (b) +the read-plane matrix invokes every selected V2 product through both Query +replicas with the sealed consumer identity and requirements, retaining only +identity, typed error code and quality hash; (c) only the affected feed-class +protocol matrix exercises stream/cursor/reconnect/duplicate/gap/resync; then, +and only then, one full all-identity C2 observes the real consumer plane for +`300s`. The fast gates do not certify a release and may not invoke V1, +provider-direct reads, orders, signals or sizing. A C2 failure must leave a +typed product-level receipt and route the repair back through (a)-(c), not a +second blind C2. This slice exits only when the thresholds are classified and +measured, all fast gates are green on the candidate image, and the next C2 is +the single final release gate. Its rollback is source-only until that later +approved four-reader packet; the currently restored `335792a` readers remain +unchanged. + +**Threshold/source checkpoint (`PASS / CANDIDATE IMAGE AND REAL READ-PLANE PENDING`, 2026-09-20).** + +- The C2 harness now emits one payload-free timing profile per governed + product. `FINAL_SCHEDULED` BAR records its continuity horizon separately + from a measured close-to-usable result; `QUIET_SESSION` requires the + declared session/gap/generation fences; `STRICT_EVENT_WITH_SESSION` uses + both event and declared numeric session bounds; a research-only + `STRICT_EVENT` with no numeric session SLA remains event/gap fenced and is + never upgraded to quiet execution; and `REFERENCE_SNAPSHOT` records + provider-observation freshness, identity, lineage and coverage rather than + inventing a stream session. +- The sealed 299-product C2 scope compiles fully with these exact semantic + counts: `FINAL_SCHEDULED=150`, `QUIET_SESSION=60`, + `STRICT_EVENT_WITH_SESSION=10`, `STRICT_EVENT=4`, + `BOOK_BASELINE=20`, `REFERENCE_SNAPSHOT=10`, and + `REFERENCE_CADENCE=45`. Its 20 MARK/INDEX requests split correctly into 10 + execution `OBSERVE` live-view products with a declared session SLA and 10 + alpha `BLOCK` reference snapshots without one. This caught and corrected a + pre-runtime overconstraint that incorrectly required session liveness for + every MARK/INDEX request; no manifest threshold was relaxed or changed. +- `BoundedWarmupExecutor` keeps external provider semaphore/token limits + shared, but keys only circuit/recovery state by `provider + declared + route/generation`. `INTERNAL_STREAM` stays one-attempt, finite-concurrency, + no-token-rate-limit. A route success clears only its own circuit state. + Partial closing batches now preserve each failed product identity, typed + problem code and compact quality SHA instead of collapsing to a generic + `DataLayerError`. +- **Tests actually run, source-only:** `python3 -m py_compile` and + `git diff --check` passed; isolated, read-only, no-network Python suites + passed `86/86` (`test_phase10_universal_warmup`, + `test_phase105_identity_acceptance`, `test_r135_quality_convergence`) and + `72/72` targeted protocol cases + (`test_phase103_consumer_receipt_harness`, + `test_qdl_sdk_stream_projection`, `test_r135_quality_convergence`). The + protocol set covers quiet/live, disconnect/reconnect, generation, + duplicate/gap and stream projection. The existing Rust/Python shared + quality golden passed `1/1` in `qdl-core` using a no-network, read-only + builder with an executable tmpfs target. All test containers used `--rm`; + no provider, role, Kafka, Redis, SQLite, V1, consumer, alpha or order state + changed. +- **Decision boundary:** this is not a certificate and did not measure live + latency. The next permitted runtime sequence remains narrowly bounded: + build one immutable candidate image from the committed source, roll only + the named Query/Stream reader roles with an explicit rollback digest, run + the both-replica fast read-plane matrix, then the single final 300-second + C2. A failed matrix yields the per-product typed receipt and blocks C2. + +**Candidate image admission (`PASS / READER ROLLOUT NOT STARTED`, 2026-09-20).** + +- Built exactly one retained candidate from committed source + `e4fc2418a6e61da578f2cd4691e17918a84e74ce`: + `qdl-v2-python:2.0.26-e4fc241@sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7`. + OCI `revision` and `version` labels match that source/release coordinate; + the runtime user is `qdl:qdl` (`10001:10001`). +- The immutable image, with no source mount, `--network none`, read-only root + filesystem and tmpfs-only test state, passed the focused C2 timing/identity + suite `17/17`. Source-only full regressions remain the proof for the wider + matrix above. The candidate is retained solely for this R1.35-C packet; + active readers remain + `qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c` + as rollback. The brief pre-amend `c5574bc` candidate was never referenced + by a container and was removed by exact image ID before this checkpoint. +- No reader was recreated and no provider, Kafka, Redis, SQLite, V1, + Trading System, alpha or order path was touched. After exact cleanup, + Docker inventory was `27GB` images / `13.06GB` reclaimable and BuildKit + cache `20GB` / `4.067GB` reclaimable. Candidate cleanup is intentionally + deferred until the single C2 result is resolved. + +**R1.35-C bounded reader packet (`PREPARED / RUNTIME NOT STARTED`, 2026-09-20).** + +- **Scope:** serially recreate only `query_v2_2`, `query_v2_1`, + `stream_v2_active`, and `stream_v2_passive` in the existing + `qdl_v2_stable_candidate` Compose project. Each moves to + `qdl-v2-python:2.0.26-e4fc241@sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7`. + The current image for all four roles, + `qdl-v2-python:2.0.26-335792a@sha256:8c53d37f6e9d5dd8efddcd61948f57e1e56ad55e4fbf245eabf90e9f01668c5c`, + is the exact named rollback for all four. +- **Invariant:** preserve the active Compose configuration chain and sealed + identity environment + `/home/bobby/.local/state/qdl-v2/r135-c-identity-20260920T023708Z/identity-rollout.env`; + do not change volumes, runtime directory, TLS, Kafka topology/offsets, + Redis, SQLite, V1, Rust core, ingestors, bar edge, projectors, Trading + System, alpha, consumer manifests, or any order path. The rollout may + create only normal reader process restarts and no provider/order action. +- **Serial health/rollback:** recreate in the listed order; after each role, + verify its exact digest, `healthy` state, no restart/OOM indication and the + unchanged companion reader. Any failure immediately recreates only the + changed role at the named rollback digest and blocks all later steps. +- **Fast gates after all four are healthy:** run the source-defined + `--read-plane-preflight` over every selected V2 route through both Query + replicas, then retain the already-passed targeted quiet/live, + disconnect/reconnect, generation, duplicate/gap and projection regression + evidence. The real preflight is no-stream, no-fallback, no-provider-direct, + no-order, no-signal and no-sizing. It records per-product typed problem code + and quality hash. Any failed item blocks C2 and causes one precise repair, + not a retry. +- **Final gate:** only a passed both-replica preflight permits exactly one C2 + full-scope 300-second real-consumer acceptance. Its result, consumer-call + latency and resource observations determine `R1.35-C`; neither health nor + source tests alone can certify it. Disposable client/cursor state is removed + afterward. Candidate and named rollback images remain until that outcome is + reconciled; no broad Docker cleanup is allowed during the packet. + +**Fast-gate live launcher finding (`FAIL_PRE_C2 / SOURCE REPAIR REQUIRED`, 2026-09-20).** + +- The four named reader roles rolled serially to the candidate and each proved + the exact `f248...6ad7` digest, `healthy`, restart `0`, OOM `false`, preserved + `/runtime`, and the stream pair remained `READY/STANDBY`. No other role or + durable dependency changed. +- The disposable no-order preflight then stopped before its first product + receipt: the acceptance helper called public + `AsyncDataLayerClient.warmup_batch(..., require_all=False)` for a batch that + contains execution-grade requirements. The SDK correctly rejected that with + `ValueError: execution-grade warmup batch must require all items`. + This is an acceptance-tool violation of the existing public contract, not a + provider, binding, freshness or replica result. It consumed no C2 window, + opened no stream and performed no fallback, provider-direct or order action. +- Required narrow repair: all execution-grade batches must use + `require_all=True`. If one fails, the tool may perform bounded per-item V2 + reads under the same identity solely to identify the exact typed error and + quality hash; all success-path batches stay batched and fast. It must never + weaken the SDK guard, use a partial execution response as usable data, or + rerun C2. Regression must prove public SDK compatibility, fast all-pass + batching, failure-only individual diagnostics, no payload evidence and typed + receipt. The current candidate is not certified; it is retained only until + the corrected immutable candidate is built and the same four-reader packet + is re-run. The named `335792a` rollback remains valid. + +**Strict execution-batch repair (`PASS / CLIENT-ONLY IMAGE PENDING`, 2026-09-20).** + +- The acceptance helper now uses public strict `warmup_batch(..., + require_all=True)` for every durable/execution batch. It preserves the SDK's + no-partial execution invariant. On a `PARTIAL_RESULT` only, it bisects the + failed strict batch and calls public `warmup` only for the failing leaf/leaves + to retain their server code, retryability, detail hash and matching status + quality hash. A transport/non-partial failure stays batch-scoped with its + code/detail hash and status matrix; it is never mislabelled as a product + failure. A leaf which becomes readable during diagnosis is explicitly marked + `BATCH_FAILURE_NOT_REPRODUCED` and still blocks C2. +- `C2ClosingBatchError` now retains bounded batch transport code/retryability/ + detail hash in addition to per-item evidence. No market payload is retained. + Regressions prove strict all-pass batching, no `require_all=False` execution + call, bisection of only the failed leaf, typed leaf diagnostic, status quality + hash and pre-existing quiet/session/gap/replica checks. Source-only, + read-only/no-network Python regression passed for the affected timing, + warmup, quality, receipt and stream-projection suites; `py_compile` and + `git diff --check` also passed. +- This repair changes only the disposable C2 client harness. The four reader + roles already on `f248...6ad7` remain healthy and need no second recreate. + Build one immutable client image from the repaired commit, replace only the + disposable `--rm` client image in this packet, then run the exact two-replica + read-plane preflight. The active reader rollback stays `335792a`; no durable + system component is changed by this source repair. + +**Strict client image admission (`PASS / REAL READ-PLANE NEXT`, 2026-09-20).** + +- Built one immutable client-only candidate from committed + `37c5acbfe38533900abf7fac55e2cdacbf542981`: + `qdl-v2-python:2.0.26-37c5acb@sha256:e77572ca7d3daf57b6400fcd5b0795ca7eee8eaf1fae9a7596ae5e1c4023f33f`. + OCI revision/version labels match; it runs as `qdl:qdl`. +- With no source mount, no network, read-only root and tmpfs-only test state, + the image passed focused timing/identity/quality regressions `38/38`. + This image is bound only to the disposable matrix/C2 launcher. The active + reader roles remain the separately attested + `qdl-v2-python:2.0.26-e4fc241@sha256:f248...6ad7`; they were not recreated + for this client-only repair. +- Next permitted action: one all-scope, both-query-replica read-plane + preflight using this client image. A typed failure blocks C2. A pass permits + exactly one 300-second C2 with this same client image and the unchanged + reader digest. + +**L2 fast-matrix closure (`IN_PROGRESS / PRE-C2 / NO RUNTIME MUTATION`, 2026-09-20).** + +- The corrected client image ran the real both-replica preflight. It did not + reach C2. The bounded receipt identified one strict `PARTIAL_RESULT` in the + `trading-system.paper.stable` secondary `BOOK_DELTA` batch: the current + `OKX.SWAP.PERPETUAL.ETH-USDT` row reported + `SOURCE_SESSION_UNAVAILABLE`/`SOURCE_SESSION_UNKNOWN`, `gap_open=false`. + The isolated leaf read recovered before diagnosis, so this is neither a + proven provider outage nor evidence that a partial response was used. +- A control-plane inspection shows both Query replicas share the same + `stable_state` liveness volume. An OKX lane had rolled to a newer source + session/generation while the durable latest delta still named the preceding + session. The existing fail-closed result is correct: a pre-reconnect book + may not inherit liveness from a new session. What remains to prove is that + the shared materializer promptly publishes a verified current-generation + snapshot/delta pair rather than leaving a replica intermittently stuck. +- **Bounded source scope:** extend the existing execution-L2 fast matrix from + ten `BOOK_SNAPSHOT` rows to the exact manifest-derived ten physical + snapshot/delta pairs, through both Query replicas. It records only product + identity, feed, source identity, generation/sequence verification, compact + session/gap/quality state and hashes; it never retains levels, prices, + credentials or cursors. The matrix must reject cross-book identity, an + unknown/disconnected/stale session, old generation, duplicate/gap/resync, + incomplete depth or replica disagreement. `BOOK_SNAPSHOT` is the + price-bearing execution view and must be execution eligible; `BOOK_DELTA` + is continuity/sequence evidence and may be quiet while its session remains + live, verified and gap-free. It must not change the declared + provider refresh cadence, loosen a liveness/freshness policy, add topology, + open a stream, invoke V1/provider-direct reads, or issue an order. +- **Fast gates:** (1) deterministic Python/Rust-facing tests cover + quiet/live, disconnect, reconnect generation change, duplicate/gap/resync, + source-pair identity and both-replica parity for all ten logical books; + (2) one bounded real read-only stability window samples the twenty logical + L2 rows through both replicas several times after the current session + generation has settled; (3) rerun the all-scope read-plane preflight once. + A typed failure carries product identity, code, flags and quality hash and + routes to the owning layer. Only all three green gates permit the one final + C2 300-second consumer certificate. +- **Exit/rollback:** no runtime image or durable-state mutation is authorized + by this source diagnostic. If current-generation materialization does not + settle, C2 remains blocked and the repair targets the shared Rust/core + lineage or liveness projection with a separate bounded packet; retrying C2 + or relaxing a session SLA is forbidden. + +**L2 fast-matrix evidence checkpoint (`PASS / ALL-SCOPE PREFLIGHT STILL REQUIRED`, 2026-09-20).** + +- The matrix now derives all ten manifest-backed physical books as twenty + typed products (`BOOK_SNAPSHOT` plus `BOOK_DELTA` for each source), rejects + a missing/mismatched pair or generation, and retains only compact control + facts. Its source regressions plus adjacent identity/SDK projection coverage + passed `49/49`; the targeted L2 protocol suite passed `81/81`; the Rust + `qdl-realtime-core` L2 gap/resync/materialization suite passed `8/8` in an + isolated, network-disabled, one-job, `1.5 GiB`-capped disposable builder. +- A real no-stream/no-fallback/no-order matrix used the sealed Trading System + V2 identity and both active Query replicas. It passed `3/3` two-second + rounds for all ten physical pairs (`20` products, two replicas), including + the formerly failing OKX ETH `BOOK_DELTA`: both replicas reported a verified + current generation, `LIVE` session and no gap. The corresponding + `BOOK_SNAPSHOT` was execution-eligible; delta is recorded as continuity + evidence rather than a price-bearing execution view. The bounded receipt is + retained only under the existing R1.35 packet evidence namespace; it + contains no levels, prices, credentials or cursors. +- This proves the observed transition settled correctly; it does not erase + the earlier typed failure or certify C2. Remaining source gate: rerun the + compact diagnostic, build one immutable disposable client image from that + commit, then run the all-scope two-replica read-plane preflight once. The + affected C2/quality/receipt suite passed `161/161` in the same immutable, + network-disabled test environment. A preflight pass alone unlocks the + single final C2 300-second run. + +**L2 quiet-feed semantics and fast-gate regression (`PASS / PRE-C2`, 2026-09-20).** + +- The fast matrix now enforces the declared product semantics rather than one + generic event-age rule: `BOOK_SNAPSHOT` must be a depth-`>=100`, verified, + gap-free and execution-eligible price view; `BOOK_DELTA` must be a verified, + same-generation, gap-free, non-reset continuity view with a `LIVE` bounded + provider session. A quiet delta may have `event_recency_state=STALE` and + remain continuity-ready; a disconnected, unknown or generation-mismatched + session remains fail-closed. This does not weaken execution-price policy. +- The compact pre-C2 matrix now reads both feed halves for every manifest + execution-L2 source through both Query replicas over several rounds. It + rejects a missing pair, identity cross-mix, generation disagreement, + duplicate/gap/resync, session loss and replica disagreement before C2. It + retains only product identity, typed code/flags, compact quality facts and + a quality hash; it never opens a consumer stream, calls a provider directly, + falls back to V1, retains book payload or emits an order. +- In the disposable `37c5acb` client image with source mounted read-only, + network disabled, tmpfs-only bytecode state and no runtime mutation: + `py_compile` plus the matrix/identity/SDK projection/L2 protocol/C2 + quality/receipt/warmup suite passed `199/199`. This is the fast diagnostic + gate replacing speculative C2 retries. The next source action is to commit + this matrix-only slice, bind one immutable disposable client image to it, + then run one all-scope preflight. Only a green preflight permits the single + final `C2 300s` certificate. + +**Paired-L2 client artifact admission (`PASS / REAL PREFLIGHT NEXT`, 2026-09-20).** + +- Committed matrix-only source `1f7e0a523dd7c7f2b493aa0bb7ab45da769e0bdf` + as `test(certification): add paired l2 fast matrix`, using the configured + `BobbyAxerol ` identity. `git diff --check` was + clean before the commit. +- Built one new disposable client image only: + `qdl-v2-python:2.0.26-1f7e0a5@sha256:33f06873d6d2fa24f06f5d512dbf60bc4d8c57c7b26fbf421d9c22fce289a7d2`. + OCI revision is the full committed SHA, version is `2.0.26-1f7e0a5`, and it + runs as non-root `qdl:qdl`. Its packaged, no-source-mount, network-disabled, + read-only-root fast-gate command exited successfully for the same `199` test + cases; all bytecode state was tmpfs-only. +- No reader, stream, provider, Kafka, Redis, SQLite, V1, Trading System, + alpha or order component was recreated or changed. The only next mutation is + packet-local: bind this disposable image to the existing read-only + preflight/C2 launcher. First run the all-scope two-replica preflight; a + typed failure returns to the relevant fast gate, while a pass permits exactly + one final C2 observation. + +**Strict local-BAR batch-shape closure (`IN_PROGRESS / PRE-C2 / SOURCE-ONLY DIAGNOSTIC`, 2026-09-20).** + +- **Trigger and exact finding.** The real all-scope preflight stopped before + C2 on one `alpha.okx.paper.stable` secondary-Query strict `BAR` batch at the + manifest maximum `50` items: public V2 returned retryable `PARTIAL_RESULT`, + while all 50 compact status rows were `LIVE` and subsequent strict leaf reads + did not reproduce an item failure. `FIELD_MISSING` records the documented + absence of a native OKX candle trade-count and is preserved as provenance; it + is not assumed to be the batch failure cause. No C2 observation, stream, + fallback, provider-direct request, order, signal or sizing action occurred. +- **Approved scope and invariant.** Add one manifest-derived, V2-only + batch-shape matrix for the exact affected consumer/feed/batch partition. It + must issue strict `require_all=True` reads through both Query replicas at + bounded shapes `1`, `8`, `16`, `32`, and the signed manifest maximum, using + the existing sealed identity and the same closing requirements. Evidence may + retain only product/batch identity hashes, replica, shape, typed error code, + retryability, compact quality hashes and latency percentiles. It may not + lower the manifest batch limit, substitute isolated successes for a failed + maximum batch, relax BAR finality/freshness, call V1/provider-direct, open a + stream, alter a consumer manifest or change provider quotas. +- **Decision boundary.** If the manifest-maximum shape fails, diagnose whether + the common `LOCAL_CANONICAL_CACHE` executor has bounded-capacity/deadline or + scheduling loss. Repair only that shared local lane, preserving finite + concurrency, strict all-or-nothing semantics, route-local circuit state and + every external Binance/OKX/DNSE policy. Add deterministic saturation, + deadline, fairness and no-external-token regression. If all shapes pass, the + receipt identifies a preflight orchestration collision and the repair stays + in the harness scheduler; it may not reclassify the failure as a provider + success. A source repair that changes Query runtime requires a separately + attested image and rolling only `query_v2_1`/`query_v2_2` with the current + reader image as rollback; a client-only repair changes no reader role. +- **Exit sequence.** Source/contract tests, real batch-shape matrix and the + already scoped affected-feed protocol matrix must pass before exactly one + rerun of the all-scope both-replica preflight. Only that green preflight + authorizes the single final C2 `300s` certificate. No blind C2 retry is + allowed. The resulting release report must include endpoint inventory, + per-feed consumer-call-to-usable latency, quality/freshness/session gates, + resource observations, rollback coordinate and remaining explicit product + exclusions before release approval. +- **Source implementation checkpoint (`PASS / REAL MATRIX NEXT`, 2026-09-20).** + The existing C2 client harness now has `--batch-shape-matrix`: it derives + the current `alpha.okx.paper.stable` manifest-maximum `BAR` partition rather + than naming symbols, tests strict boundary shapes `1/8/16/32/max` through + both replicas, and runs one collocated largest-BAR read per governed + identity to distinguish isolated local-cache capacity from cross-consumer + scheduler contention. Every read reuses the public V2 SDK, + `require_all=True`, the closing final-BAR validator, sealed identity and + quota pacer. The receipt contains only batch identity/quality-content hashes, + latency percentiles and typed failure context; it creates no cursor, + stream, fallback or provider connection. Deterministic regressions cover the + exact maximum partition, boundary windows, strict typed failure receipt and + four-identity collocation. `python3 -m py_compile`, `git diff --check`, and + the read-only/network-disabled `tests.test_phase105_identity_acceptance` + + `tests.test_phase10_universal_warmup` suite passed `86/86`. No runtime role, + Kafka, Redis, SQLite, V1, consumer, alpha or order state changed. + +- **Harness applicability correction (`PASS / SOURCE-ONLY / REAL MATRIX RETRY`, + 2026-09-20).** The first client launch reached the new collocation phase and + exposed a harness assumption, not a Query/data result: one governed identity + has no durable `BAR` entitlement, so an unconditional “one BAR lane per + identity” raised before a receipt could be emitted. The matrix now executes + only identities with an entitled durable BAR lane and records every other + identity as `NOT_APPLICABLE_NO_DURABLE_BAR` with `read_actions=0`; it neither + substitutes a feed nor weakens the affected `alpha.okx.paper.stable` exact + `50`-BAR test. Read-only, network-disabled + `tests.test_phase105_identity_acceptance` passed `37/37`, including the + no-BAR collocation regression. The discarded named bootstrap diagnostic was + removed; no Query/Stream role, Kafka, Redis, SQLite, V1, consumer, alpha, + order or provider path changed. The actual matrix has not yet produced a + data receipt and C2 remains unconsumed. +- **Saturation-threshold evidence (`PASS / SOURCE-ONLY / REAL MATRIX RETRY`, + 2026-09-20).** Collocation is now a deterministic ladder rather than one + opaque fan-out: it runs the affected entitled consumer alone, then with two + and finally with every entitled durable-BAR lane. Each successful wave keeps + compact batch/quality hashes and latency percentiles; a later failure retains + completed lower-lane waves and the exact failed parallel-lane count. This + distinguishes a true local canonical-cache capacity/fairness threshold from + an isolated batch defect without changing any manifest quota, deadline, + freshness rule, external-provider limiter or `INTERNAL_STREAM` policy. + Source-only `tests.test_phase105_identity_acceptance` passed `37/37` after + the ladder regression; the full runtime matrix remains the next gate. +- **Real batch-ladder receipt (`FAIL_TYPED_STATUS / PRE-C2`, 2026-09-20).** + The V2-only matrix reached the actual capacity boundary without a provider, + stream, fallback, order or data-plane mutation. The affected exact `50`-BAR + partition passed isolated with primary/secondary p95 `11,486.286ms` and + `12,162.344ms`; the one-lane collocation repeated that pass at + `11,493.378ms` and `12,152.250ms`. At two concurrent entitled `50`-BAR + lanes, primary returned strict retryable `PARTIAL_RESULT` for + `alpha.okx.paper.stable`; the SDK's all-or-nothing boundary intentionally + hides item payloads, while bounded status reads still reported all 50 rows + `LIVE` with the documented `FIELD_MISSING` provenance flag. This proves a + local batch concurrency/response-path issue, not provider freshness, but + does not yet identify whether an internal item deadline, executor queue or + serialization step produced the partial result. C2 remains unconsumed. +- **Next narrow diagnostic.** Preserve strict `require_all=True` and capture + only the server response's per-item `status`, canonical problem code, + retryability and identity hash inside the existing paced test transport + before the public SDK converts the partial response into `DataLayerError`. + Add source regressions for success/partial sanitization and then rerun only + the same batch ladder. No production endpoint, provider policy, local-cache + concurrency, deadline, manifest, reader role or runtime state changes until + that receipt identifies the failing internal boundary. +- **Strict-response evidence instrumentation (`PASS / SOURCE-ONLY / MATRIX + RETRY`, 2026-09-20).** The paced query transport now snapshots only response + shape before the public SDK raises on strict partial: aggregate counts plus + ordinal `status`, canonical problem code and retryability for failing items. + It explicitly excludes response data, instrument strings and problem detail; + the closing error maps a failing ordinal back to the already-authorized + compact product identity only when needed. The source regression passed + `38/38`, proving successful and partial responses retain no market payload, + and the next real matrix will use the same strict `require_all=True` call. +- **Capacity root cause and approved local-lane repair (`IN_PROGRESS / + SOURCE-ONLY`, 2026-09-20).** The retained real receipt from client image + `68115d5` identifies the failing boundary without exposing market payloads: + on the primary Query replica, the second concurrent `50`-BAR lane returned + `12` successes and `38` retryable `DEPENDENCY_UNAVAILABLE` results at + ordinals `12..49`. Each affected binding remained independently `LIVE`; the + local stable backend declares every warmup local, so no Binance/OKX provider + request, provider quota, stream, fallback or data-plane write participated. + The common executor starts its fixed item deadline at enqueue time, before + waiting for the finite eight-slot `LOCAL_CANONICAL_CACHE` semaphore. This is + a local admission/deadline bug, not stale data or a provider failure. +- **Repair scope/invariants.** For `LOCAL_CANONICAL_CACHE` only, retain finite + execution concurrency and add bounded pending admission. Start the existing + per-item read/retry deadline only after a local worker owns a cache slot, so + valid work is not relabelled as a dependency outage merely because another + manifest-legal batch was ahead of it. A full local pending queue must return + a typed retryable capacity result rather than grow unboundedly. The existing + external-provider end-to-end deadline, token buckets, retry/cooldown and + circuit semantics remain byte-for-byte behaviorally unchanged; so do + `INTERNAL_STREAM`, public schemas, manifest limits, BAR finality/freshness, + V1 and all runtime/data-plane authority. +- **Required source gates before any reader rollout.** Deterministic tests must + prove two collocated maximum local batches complete without an internal queue + deadline loss, work execution itself still fails at its declared deadline, + queue saturation fails typed/bounded, cancellation leaves no admission leak, + route singleflight and per-route circuits remain scoped, and Binance/OKX/DNSE + provider token/retry behavior is unchanged. Then run the existing source + suite and only the affected `QUOTE`, `TRADE`, `MARK_INDEX`, `BOOK_DELTA` and + final-BAR protocol matrices. A source pass authorizes one immutable Query + image and a bounded serial recreate of only `query_v2_1` and `query_v2_2` + with the current `sha256:f2489160...e957b6ad7` reader image as rollback. The + real strict batch ladder, all-scope two-replica fast preflight, and one final + C2 `300s` remain subsequent gates; none is consumed by this source repair. +- **Local admission repair result (`PASS / SOURCE-ONLY / QUERY ROLLOUT NEXT`, + 2026-09-20).** `BoundedWarmupExecutor` now has an explicit local-only policy + for finite pending admission (`128` outstanding reads) and starts a local + read's declared deadline after it owns one of the existing eight worker + permits. A true admitted read still times out typed; a full queue fails typed + without unbounded task growth; cancellation drains its reservation; and + route singleflight/circuits remain unchanged. All other lanes retain their + prior end-to-end deadline semantics, including external token/retry behavior + and the `INTERNAL_STREAM` one-attempt policy. The V2 Query integration + regression proves two collocated local batches retain each item read budget; + no query response can be silently promoted from partial to success. +- **Tests actually run.** Disposable, read-only, `--network none` containers + using `qdl-v2-python:2.0.26-68115d5` passed `21/21` focused executor/Query + cases, `93/93` across `tests.test_phase10_universal_warmup` and + `tests.test_phase105_identity_acceptance`, and `72/72` targeted receipt, + SDK stream and R1.35 quality protocol cases. `py_compile` for the changed + modules passed. The first Query-level timing test used a `100ms` artificial + read budget and was too close to debug-runner thread startup; it was replaced + by a deterministically wider `150ms` work / `400ms` admitted-read relation that + still fails under the old enqueue-deadline behavior but does not encode any + product SLA. `git diff --check` passed before this journal update. No image, + reader role, runtime directory, provider request, Kafka/Redis/SQLite state, + V1, consumer, alpha, signal, sizing or order state changed. +- **Next decision boundary.** Build one immutable Python Query candidate from + the committed source. If its image-level source suite passes, serially + recreate only `query_v2_1` and `query_v2_2` under the already-approved + bounded packet, retaining `sha256:f2489160...e957b6ad7` as exact rollback. + Then rerun only the strict local-BAR ladder, the two-replica all-scope fast + preflight and, only if both pass, one final C2 `300s`. + +- **Immutable candidate gate (`PASS / QUERY-ONLY ROLLOUT NEXT`, 2026-09-20).** + Built `qdl-v2-python:2.0.26-48e26be` from the sealed source commit + `48e26be5a00bb9c2ee080effa810bcb16770fcb5`; its immutable image digest is + `sha256:f201fab0b0055e32cca4b4256db881cbc91a7fc952cecb370d89e83f7dd0730d`. + In packaged-image, network-disabled, read-only-root, non-root containers + with tmpfs-only bytecode state, the local admission/consumer acceptance + suite passed `93/93` and the affected receipt/SDK projection/R1.35 quality + protocol suite passed `72/72`. No source mount, runtime role, provider, + durable state, V1, Trading System, alpha or order path participated. The + only authorized runtime mutation is now a serial recreate of + `query_v2_1`, then `query_v2_2`, using the current reader image + `sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7` + as the exact rollback coordinate. + +- **First bounded rollout and strict-ladder result (`FAIL-CLOSED / ROLLED BACK / SOURCE REPAIR REQUIRED`, 2026-09-20).** + The candidate was serially applied only to `query_v2_1` and `query_v2_2` + under packet + `/home/bobby/.local/state/qdl-v2/r135-local-admission-48e26be-20260920T131024Z`. + Both became healthy with `restart=0` and `OOMKilled=false`; the unchanged + Stream pair remained `READY/STANDBY`. The real V2-only ladder then proved + every isolated `1/8/16/32/50` BAR shape for the affected consumer through + both replicas. The exact isolated `50`-BAR request measured primary + `10,370.625ms`, secondary `10,819.703ms`. At the first two-consumer legal + collocation wave, the secondary Query request for + `alpha.binance.paper.stable` ended in `ReadTimeout`; its compact post-failure + status probe saw `47` `LIVE` rows and `3` probe timeouts. No partial data was + accepted, no provider/direct/fallback/stream/order action occurred, and no + C2 was consumed. This is a real local Query throughput defect because the + public SDK default request timeout is `10s`; it is not repaired by raising + a harness timeout. The packet rolled exactly the two Query roles back to + `sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7`, + both healthy with zero restart/OOM; every other role and durable component + remained untouched. +- **Next in-scope source slice: fair local batch admission.** The current + global eight-slot local executor queues all 50 tasks from the first HTTP + batch before a second legal batch gets an admission turn. Add a + `LOCAL_CANONICAL_CACHE`-only per-batch gate below the existing global bound, + so collocated batches share finite worker ownership instead of one batch + monopolizing the queue. Batch-gate waiting must stay outside a local item + execution deadline; global pending capacity, cancellation cleanup, + per-route singleflight and strict all-or-nothing batch semantics remain + intact. External Binance/OKX/DNSE and `INTERNAL_STREAM` policies must remain + byte-for-byte behaviorally unchanged. Required source gates: deterministic + two-batch fairness/order, no deadline loss while waiting, true admitted-work + deadline, finite queue rejection, cancellation leak, route isolation and + unchanged external token/retry coverage. Only a new immutable Query image + after those gates may repeat the same two-role rollout and one strict ladder. +- **Fair local batch admission source result (`PASS / IMAGE BUILD NEXT`, 2026-09-20).** + `ProviderBudgetPolicy` now supports an explicit request-local + `max_batch_concurrency`; `LOCAL_CANONICAL_CACHE` uses four batch permits + beneath its unchanged eight global worker permits and bounded `128` pending + admission. Pending capacity is reserved before the batch gate, so a flood of + request-local waiters cannot bypass the global bound; the local read deadline + still starts only after the global worker permit, not while waiting at either + queue. External policies, `INTERNAL_STREAM`, global singleflight, route + circuits, strict batch response handling and public schemas are unchanged. + Deterministic regressions prove a second legal local batch receives a global + worker before the first batch can consume both, batch-gate waiting preserves + each item's admitted execution deadline, queue state drains, and invalid + batch-vs-global policy is rejected. Read-only, network-disabled containers + passed `24/24` focused executor/query tests and `168/168` affected warmup, + identity acceptance, receipt, SDK projection and R1.35 quality tests. No + runtime role or durable/data-plane state changed in this source gate. The + next permitted mutation is one new immutable Query image, then the same + two-role packet rollout and exact strict batch ladder; C2 remains unconsumed. +- **Fair-admission immutable candidate (`PASS / QUERY-ONLY ROLLOUT NEXT`, 2026-09-20).** + Built `qdl-v2-python:2.0.26-e905747` from code commit + `e9057475774aed4da1e263333c7f5fcf22a5a629`; immutable digest + `sha256:5541a3348ac0d4af83de8710b7ee68ce7826fe9fc417295dee45d70937983536`. + A packaged-image, network-disabled, read-only-root, non-root run with only + tmpfs scratch state passed `168/168` across warmup, identity acceptance, + receipt, SDK stream projection and R1.35 quality tests. The active Query + readers were independently re-inventoried before the packet: both remain + healthy at `sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7` + (`qdl-v2-python:2.0.26-e4fc241`), which is the exact rollback coordinate. + No runtime role, durable state, provider, V1, Trading System, alpha or order + path changed in this gate. The next permitted mutation is a serial recreate + of only `query_v2_1`, then `query_v2_2`, followed by one strict batch ladder; + all-scope preflight and C2 remain blocked on its result. +- **Query rollout and C2 identity-harness repair (`IN_PROGRESS / BATCH NOT YET EXECUTED`, 2026-09-20).** + The bounded packet serially recreated only `query_v2_1` and `query_v2_2` + onto the `e905747` candidate; both are healthy with `restart=0`, + `OOMKilled=false` and the unchanged runtime mount. The Stream pair remained + `READY/STANDBY`; V1, Rust, ingestors, projectors, Kafka, Redis, SQLite, + Trading System, alpha and order paths were untouched. Its first batch + invocation stopped before issuing any Query request because the legacy C2 + bootstrap silently lost root-owned alpha-OKX identity files under Docker + user-namespace permissions. The packet now contains an ephemeral, + read-only, UID-`10001` identity/provenance copy with `0700` directories and + `0600` files; source identities were not changed or printed. Bootstrap was + made fail-closed by materializing tar archives before extraction rather than + relying on a masking shell pipeline. An isolated no-network probe proved all + 22 required identity/provenance files extract for the unprivileged client. + This is a harness correction, not a batch retry: no matrix request, stream, + fallback, provider-direct call, order or durable-data mutation happened in + the failed attempt. The permitted next action is exactly one strict batch + ladder using this corrected packet; preflight and C2 remain unconsumed. +- **Strict batch materialization finding and bounded repair (`IN_PROGRESS / PRE-C2`, 2026-09-20).** + With identity extraction corrected, the real ladder executed rather than + stopping in the harness. Isolated strict shapes `1/8/16/32/50` passed both + Query replicas; the maximum `50` took about `11.0s` primary and `11.4s` + secondary. The two-lane collocation then correctly failed the public SDK's + `10s` request boundary with `ReadTimeout`, while the first legal lane was + complete. The fair-admission candidate was rolled back immediately and only + `query_v2_1`/`query_v2_2` returned to + `sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7`; + both are healthy, zero-restart and non-OOM. No C2 was consumed and no + provider/direct, stream, V1 fallback, order or durable-data mutation occurred. + + The source inspection identifies the remaining local bottleneck precisely: + each cache-backed BAR item takes the shared SQLite spool lock independently + and reparses the same bounded protobuf rows in record selection, lineage + validation, gap detection and projection. Fair worker admission prevents a + queue-attribution error but cannot make `50 x 700` repeated local + materializations fit the public request contract. The approved P0 repair is + a bounded local-batch snapshot path only: read declared cache tails as one + consistent local snapshot, parse each selected envelope once, and reuse the + same filtering, lineage, gap, finality, coverage, quality, cursor and strict + all-or-nothing result functions as single reads. One admitted request may + share its immutable batch result; it must not create a cross-consumer cache, + weaken finite admission, alter provider quotas, call a venue, lower the + manifest maximum, raise the SDK timeout, alter public schemas or bypass V1 + policy. Required source gates are single-vs-batch parity for normal, + late-backfill, missing/gap and failure cases; one-snapshot/no-cross-mix + regression; bounded concurrent-request/cancellation behavior; and unchanged + external/`INTERNAL_STREAM` policy tests. Only then may a new Query image + repeat the one strict ladder; green ladder then permits one all-scope + preflight and one final C2. + +- **Strict batch materialization source slice (`PASS / IMAGE BUILD NEXT`, + 2026-09-20).** `SQLiteDurableSpool.read_tails()` now reads at most the public + `100` requested physical tails through one bounded SQL snapshot; duplicate + physical requests take the largest declared tail once and callers retain + their own logical cap. `StableSpoolQueryBackend.history_many()` decodes each + selected canonical envelope once, then calls the same private history + builder used by single reads for exact feed/interval selection, lineage, + late-backfill market order, gap/coverage, finality, quality, cursor and + watermark semantics. `RoutedQueryBackend` exposes this only when every route + is already authoritative-local; a pass-through-eligible recovery route + fails closed rather than being silently downgraded. `V2QueryService` starts + the shared snapshot only after a local executor item is admitted, shares it + only inside that request, preserves per-item typed failures, and cancels the + request task on outer cancellation. No cross-consumer result cache, external + provider call, provider/`INTERNAL_STREAM` policy change, timeout increase, + schema/manifest change or durable write was introduced. + + Tests actually run in disposable `--network none`, read-only, non-root + containers using the existing `qdl-v2-python:2.0.26-e4fc241` image: + `51/51` focused transport/stable-query/executor/routed tests passed, + including the public fifty-partition SQL shape, bounded snapshot isolation, + single-vs-batch parity, late-backfill, missing/gap, per-item typed failure, + request-local sharing and cancellation admission drain. A separate `109/109` + affected identity/readiness/mark-index/query-stream/pass-through matrix also + passed with network disabled, for `160/160` completed source cases across the + two bounded matrices. + `python3 -m compileall` for every changed production/test module and + `git diff --check` passed. A subsequent unbounded `unittest discover` was + intentionally stopped after it entered unrelated integration/gRPC cases + outside this source-only gate; it is not counted as evidence and its two + verified read-only test containers (`jovial_stonebraker`, + `amazing_dubinsky`) were removed. No V2 runtime role, V1, Kafka, Redis, + SQLite, provider, Trading System, alpha or order path changed. + + The next permitted action is one immutable Python Query image from this + source slice, followed by a serial recreate of only `query_v2_1` and + `query_v2_2`. The strict `1/8/16/32/50` ladder remains unconsumed for this + source revision; only a green ladder permits the one all-scope two-replica + fast preflight and then the one final C2 `300s`. + +- **Strict batch materialization immutable candidate (`PASS / QUERY-ONLY + ROLLOUT NEXT`, 2026-09-20).** Committed source + `1d81becb0739159089ba94400b6c32d569be03b4` built as exactly one retained + candidate: `qdl-v2-python:2.0.26-1d81bec` at + `sha256:05ee5702cae9fa8aa07fc48a289a5f711c0aa8401f64ca08c150e8cc8e5de60e`. + OCI revision is the full committed SHA, OCI version is `2.0.26-1d81bec`, + and the configured runtime user is `qdl:qdl`. The packaged artifact, with + no source mount, `--network none`, a read-only root, tmpfs-only test state + and UID/GID `10001`, passed the exact selected `160/160` source matrix. + The isolated containers removed themselves after the runs; no candidate + container, source mount, provider request, runtime role, V1, Kafka, Redis, + SQLite, Trading System, alpha or order path changed. The only retained + reader rollback remains active `qdl-v2-python:2.0.26-e4fc241` at + `sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7`. + + The next permitted mutation is the already-approved serial replacement of + only `query_v2_1`, then `query_v2_2`, with that exact rollback. It must run + the strict `1/8/16/32/50` both-replica BAR ladder first. Only a green ladder + permits one all-scope fast preflight and then exactly one C2 `300s`; neither + acceptance has been consumed by this image gate. + +- **Strict batch candidate runtime finding (`IN_PROGRESS / SOURCE REPAIR`, + 2026-09-21).** The Query-only candidate was rolled only after its exact + active Compose chain and rollback digest were checked. Both readers reached + `1d81bec` healthy with zero restart/OOM; V1 and every other role remained + unchanged. The first strict ladder reached a real local `BAR` batch of + shape `8` on the primary replica and returned eight typed + `INTERNAL_ERROR` items, which the public strict contract correctly surfaced + as `PARTIAL_RESULT`. Individual typed status for the same eight OKX DOGE + bars was `LIVE`, so this is neither a provider/staleness diagnosis nor an + allowed reason to weaken strictness. The client bootstrap initially had a + read-mode error and was corrected before the real request; that failed + bootstrap made no Query call and did not consume a ladder result. + + A disposable no-network, read-only reproduction of the exact manifest + requirements reached the same shared `history_many()` branch and recorded + `sqlite3.OperationalError: database or disk is full` per item. Host disk and + inode headroom are healthy (`142 GiB` free; `5%` inode use); SQLite reports + default temp-store behavior over the multi-gigabyte canonical cache. The + current `ROW_NUMBER` CTE ranks the event table before it can retain each + requested tail, creating an unbounded planner-temp shape. The approved + in-scope repair replaces only that transport implementation with one locked + read transaction containing at most `100` indexed partition-tail scans. + It retains one SQLite-consistent snapshot, exact chronological ordering, + physical-tail deduplication, per-route logical caps and every query-quality + semantic, while avoiding a full-table window sort. It does not change + provider/`INTERNAL_STREAM` policy, timeout, public API, manifest, durable + state or any service topology. The current candidate is not certified; + all-scope preflight and C2 remain unconsumed. + + **Repair evidence (`PASS / NEXT: QUERY-ONLY CANDIDATE`, 2026-09-21).** + `read_tails()` now opens one SQLite read transaction and performs at most + one primary-key ordered tail lookup for each deduplicated physical + partition. The focused transport/order/snapshot suite passed `3/3`; the + selected source regression matrix passed `161/161` in a no-network, + read-only-root container with tmpfs-only test state. `git diff --check` and + Python compilation of the changed transport/test files passed. A second + disposable no-network, read-only reproduction mounted the live canonical + cache and exact sealed runtime bindings; all eight formerly failing OKX + DOGE BAR requirements (`12h`, `15m`, `1d`, `1h`, `1m`, `1w`, `2d`, `2h`) + returned `HistoryResult` without an SQLite error. This is real-cache + transport evidence only: it did not call a provider, mutate the cache, + invoke Query, change runtime roles, consume preflight/C2, or perform an + order action. The next permitted action is one newly attested Python image, + serial rolling replacement of only the two Query readers, the strict + `1/8/16/32/50` ladder, then the already-defined preflight/C2 sequence. + + **Candidate ladder diagnostic (`IN_PROGRESS / SECOND SOURCE REPAIR`, + 2026-09-21).** Candidate `9c205e5` was built with immutable digest + `sha256:3c8d245e7ecd51d3d4a87b2ddaa741e832a7436167ba542c0899caf27272f961` + and rolled serially to only `query_v2_1` and `query_v2_2`; both reached + healthy, non-OOM state on the unchanged runtime mount. The first real ladder + no longer produced SQLite `INTERNAL_ERROR`, but its primary `16`-BAR window + returned `ConnectError` immediately after both Query processes restarted. + The receipt is retained under the scoped runtime packet and reports zero + order actions; it is not a C2 run. Docker's live cgroup counters show no + OOM kill, so the exact simultaneous process restart is not attributed to a + provider, a stale route, or this source patch without evidence. + + Independently, code inspection found a boundedness defect that the 50-route + gate is designed to expose: `history_many()` still retained raw rows and + decoded protobufs for every physical BAR partition before materializing any + result. With the governed physical window of `12,064` records, a maximum + batch can create an unnecessary whole-batch memory peak. The next in-scope + source repair preserves the one SQLite read snapshot but visits one indexed + partition tail at a time and immediately materializes its logical results. + It changes neither public batch shape, data quality/finality, provider + policy, quota, timeout nor fallback behavior. New regressions must prove + one-snapshot semantics, physical-tail deduplication, per-item parity and + bounded traversal before one fresh candidate/ladder attempt. Preflight and + C2 remain unconsumed. + + **Bounded traversal evidence (`PASS / FINAL SOURCE REGRESSION NEXT`, + 2026-09-21).** The transport now exposes a private callback traversal under + one deferred SQLite read snapshot; `history_many()` parses and materializes + one deduplicated physical tail before advancing to the next. It keeps the + full `12,064` BAR physical window, including authentic late-backfill + headroom, rather than using an incomplete header-derived shortcut: live + cache inspection showed only `1,951/12,064` rows in one retained legacy + partition carry the newer final-BAR header. The targeted suite passed `5/5` + for snapshot, ordering, 50-partition transport bound, per-item parity, late + gap/missing behavior and sequential tail visitation. + + A disposable no-network/read-only run against the real canonical cache and + the exact sealed `alpha.okx.paper.stable` BAR partition passed all + `16/16` and `50/50` histories under the same `512 MiB` memory limit as a + Query reader: shape `16` took `3,563.012 ms` at `228,052 KiB` peak RSS, and + shape `50` took `10,534.067 ms` at `229,032 KiB`. It made no provider call, + cache write, runtime mutation or order action. The current runtime readers + remain the prior candidate until this final source slice is committed, + rebuilt and rolled narrowly; the failed transport attempt remains retained + as typed evidence and has not been overwritten. + + The complete selected transport/query/readiness/identity regression matrix + passed `162/162` in the same no-network, read-only-root test environment; + syntax compilation and `git diff --check` passed before commit. + + **Whole-local-batch admission finding (`FAIL_TYPED_STATUS / PRE-C2`, + 2026-09-21).** The newly rolled `331d3ff` readers are healthy, non-OOM and + zero-restart. Its strict ladder proved all nine isolated `1/8/16/32/50` + windows through both replicas, including the original 50-item OKX BAR + partition. The deterministic two-consumer collocation then failed on the + secondary reader with `ReadTimeout`: `42/50` compact status reads remained + `LIVE` and the remaining eight timed out while the cancelled request's + in-process SQLite/protobuf work was still draining. No provider/direct, + fallback, stream, order or durable-data action occurred. This is therefore + not a stale-provider conclusion and does not consume all-scope preflight or + C2. + + The exact capacity shape is a legal heavy warmup (`50` requirements, each + with a declared maximum of `10,000` BAR rows) on one `1 CPU / 512 MiB` Query + reader. The current per-item local executor admits the first items from two + whole batches at once; each then starts a shared batch materialization while + the single SQLite spool connection serializes its full physical-tail scan. + The correction is confined to the local canonical-cache batch boundary: + admit one whole local batch per reader, bound its pending queue, preserve the + lease until its thread-backed materialization drains on caller cancellation, + and begin item execution only after that admission. It must retain exact + batch payload/quality/finality semantics and fail closed with a typed local + capacity result when the queue is full. It must not alter any Binance/OKX/DNSE + provider quota, `INTERNAL_STREAM` policy, public endpoint/schema, manifest, + cache, topology or non-local request behavior. Required regressions cover + serial admission, bounded rejection, cancellation/non-leak, post-success + fairness, and existing per-item parity. Only after source/image proof may + the two Query readers repeat the one strict ladder. + + **Whole-local-batch admission source gate (`PASS / CANDIDATE BUILD + PENDING`, 2026-09-21).** `V2QueryService` now treats a fully local + `history_many()` batch as one canonical-cache materialization unit: exactly + one active batch may decode retained SQLite/protobuf BAR tails per Query + reader and exactly one additional batch may wait. A saturated lane yields a + per-item retryable `RATE_LIMITED` result before provider or executor work; + it does not borrow or modify the Binance, OKX, DNSE or `INTERNAL_STREAM` + budgets. `asyncio.shield` retains the bounded lease when an HTTP caller + disconnects while a thread-backed SQLite read drains, and the detached task + retrieves its terminal result to avoid an orphan-task warning. Existing + mixed local/provider and non-local paths are unchanged. + + The no-network/read-only regression suite explicitly proves: a collocated + local batch cannot start a second `history_many()` sweep before the first + finishes; queue exhaustion is typed and performs no second cache read; + cancellation retains then drains the admission lease; the next legal batch + succeeds; and an item-level local cache absence remains typed. The focused + `SingleWarmupExecutionTests` passed `8/8`; the complete warmup module passed + `62/62`; and the selected Query/route/readiness/API/SDK/identity matrix + passed `188/188` using the Data Layer image entrypoint with network disabled, + a read-only repository mount and a temporary bytecode cache. Syntax + compilation and `git diff --check` passed. The source gate does not consume + the all-scope preflight or C2 and performs no runtime, provider, durable-data + or order mutation. The next permitted step is one immutable Python candidate + build, followed by a narrow two-Query-reader packet with the currently active + `331d3ff` image as rollback. + + **Candidate packet fence repair (`FAIL-CLOSED / RESTORED`, 2026-09-21).** + The first narrow candidate attempt recreated only `query_v2_1`; it reached + `healthy`, restart `0` and non-OOM on the unchanged runtime mount, but the + packet's candidate compose-chain hash had been computed from an obsolete + label serialization. The assertion failed before `query_v2_2`, the strict + ladder, all-scope preflight or C2 could start. A second packet defect was + exposed by that fail-closed path: Bash `ERR` did not propagate through the + helper function, and the rollback assertion checked Docker health before its + normal startup window elapsed. The reader was immediately restored to the + exact active `331d3ff` image and pre-existing compose chain; both readers + then verified `healthy`, restart `0`, non-OOM, with the stream pair unchanged + as `READY/STANDBY`. + + The runtime-only packet now fences its observed candidate chain, explicitly + invokes rollback on every failed reader/stream assertion, and waits a bounded + health window before accepting or restoring a reader. It changes no source + contract or data-plane behavior. The repaired packet has passed `bash -n` + and config-only validation; its one permitted retry remains two Query readers + only, with the exact `331d3ff` image/runtime chain as rollback. No acceptance + gate has been consumed by this packet repair. + + **Collocated local response-assembly correction (`IN PROGRESS / SOURCE + ONLY`, 2026-09-21).** The retried candidate made the intended whole-batch + SQLite admission observable, but the real two-lane ladder still returned a + typed `ReadTimeout` on secondary: every isolated `1/8/16/32/50` shape and the + first 50-BAR lane passed, while the second legal 50-BAR lane crossed the + public `30s` client boundary. This is not a provider, provenance or + freshness failure. The current lease protects `history_many()` only; it is + released before the first batch completes the CPU-heavy per-item history + validation and response assembly. The second request can therefore begin a + new 50-tail decode while the first is still consuming the single reader CPU. + + The approved P0 source correction is deliberately narrower than a cache or + scheduler redesign: retain the existing one-active/one-waiting + `LOCAL_CANONICAL_CACHE` lease across one fully-local batch's immutable + materialization **and** bounded per-item warmup execution. Mixed and + non-local batches remain on their existing paths. Cancellation must retain + the lease until the full in-process batch drains; a full pending lane remains + per-item typed `RATE_LIMITED`. No timeout increase, manifest change, + provider/`INTERNAL_STREAM` limiter change, external call, public schema, + durable write or topology change is permitted. Required source gates are + lease coverage through response assembly, two legal collocated maximum + batches, queue rejection/recovery, cancellation drain, normal/error parity + and unchanged external policies. Only a new immutable Query image after + those gates may re-run the already-authorized two-reader ladder; the + all-scope preflight and one final C2 remain unconsumed. + + **Collocated response-assembly source gate (`PASS / IMAGE BUILD NEXT`, + 2026-09-21).** `V2QueryService` now holds the same bounded local admission + lease from one fully-local `history_many()` snapshot through the existing + bounded per-item executor and response construction. A second legal local + batch therefore waits without starting another SQLite/protobuf sweep while + the first still owns the reader CPU; a third remains per-item retryable + `RATE_LIMITED`. The work is still subject to the unchanged item executor, + route singleflight, circuit and content/finality/quality validators. Mixed + local/provider and non-local batches retain their previous path exactly. + An outer caller cancellation leaves the shielded request-local task owning + its lease until response assembly drains, then releases it without orphan + task warnings. + + **Tests actually run:** an immutable existing Data Layer image with + `--network none`, read-only source/root and tmpfs-only scratch passed + `SingleWarmupExecutionTests 10/10`, the full + `tests.test_phase10_universal_warmup 64/64`, and the targeted + Query/route/identity/consumer/SDK/L2/quality matrix `154/154`. The new + behavioral regressions specifically hold the first batch in response + assembly, prove the second cannot begin `history_many()`, and prove + cancellation in that phase retains then drains the lease. `py_compile` and + `git diff --check` pass. No image was built, no role/runtime/provider/durable + component was changed, and no preflight or C2 attempt was consumed by this + source slice. The next permitted operation is one immutable Query candidate + build from its committed SHA, then an exact two-reader rollout with the + current `3beb1e9` candidate image as rollback. + + **REST response-boundary finding (`IN PROGRESS / PRE-C2`, 2026-09-21).** + Commit `5761cc9` correctly extended the fully-local admission through the + service's bounded executor and `BatchQueryResult` construction. Its + immutable Query-only candidate + `qdl-v2-python:2.0.26-5761cc9` + (`sha256:5490db55234902966c06caaca803396f0a7a09d1638b9c0f74310462a95a81fb`) + then rolled only `query_v2_1` and `query_v2_2`; both readers are healthy, + restart `0`, OOM `false`. The exact real strict ladder still failed only for + the second lane of two collocated 50-item OKX final-BAR reads on secondary + Query with client `ReadTimeout`: every isolated shape `1/8/16/32/50` and the + first 50-item lane passed, while neither all-scope preflight nor C2 was + started. It made no provider-direct, fallback, stream, order or durable + write. + + The remaining critical boundary is now identified precisely: the REST router + receives `BatchQueryResult` after the service lease has been released, then + builds fifty nested `WarmupResponse`/`BatchItemResponse` models and FastAPI + serializes that public response. A second request can overlap that CPU-heavy + response conversion even though the cache snapshot and executor are + serialized. The next approved source correction remains P0-only: expose an + internal completion hook that retains the existing fully-local canonical + cache lease until the router has built and JSON-serialized the unchanged + `BatchResponse`. It applies only to a fully-local `history_many` batch; + mixed/non-local requests, public path/schema, status/partial semantics, + cursor/quality/finality, provider policies, `INTERNAL_STREAM`, timeout, + manifest and every durable/topology component remain unchanged. Cancellation + must keep the detached completion holding the finite lease until it drains. + + Required gates before another image are: exact public JSON/SDK decode parity; + two collocated full local REST batches proving the second cannot begin cache, + executor or response serialization early; partial/error parity; cancellation + drain; capacity/recovery; and unchanged mixed/external behavior. The existing + focused module, targeted route/SDK/quality matrix, `py_compile` and + `git diff --check` must pass in an immutable read-only/no-network image. + Only then may one new Query image roll exactly the same two readers with + `5761cc9` as rollback and repeat the strict ladder. All-scope preflight and + the single final C2 remain blocked until that ladder passes. Cleanup, push, + merge and release remain explicitly deferred. + + **REST response-boundary source gate (`PASS / IMAGE BUILD NEXT`, + 2026-09-21).** `V2QueryService.warmup_batch_async()` remains source/API + compatible and returns the same `BatchQueryResult`. Its new internal + `warmup_batch_completed_async()` runs a router-supplied completion under the + already-bounded local lease only for a fully-local batch. The V2 router now + binds signed cursors, builds the existing validated `BatchResponse`, and + creates its `JSONResponse` inside that completion. Thus FastAPI cannot make + a second response-model/JSON pass after the local lease is released. Existing + focused service doubles retain their old method through an explicit fallback; + production `V2QueryService` always takes the bounded path. The contract, + JSON aliases/null behavior, typed partial/error results, cursors, quality, + finality and all non-local lanes are unchanged. + + **Tests actually run.** In an immutable existing Data Layer image with + `--network none`, read-only source/root and tmpfs-only scratch: + `SingleWarmupExecutionTests` plus `Phase5ApiTests` passed `23/23`; the + complete universal-warmup/API/routed-query/identity/consumer/SDK/L2/quality + matrix passed `167/167` in `27.282s`; and `python -B -m py_compile` passed + for all four changed Python modules. New regressions prove a second local + batch cannot start history work while the first is inside an async HTTP + completion, cancellation retains then drains that completion lease, and the + HTTP response validates back to the exact public `BatchResponse` contract. + `git diff --check` passes. This is source-only evidence: no new image, + reader recreation, provider call, durable mutation, all-scope preflight or + C2 has occurred. The sole next mutation after commit is one immutable Query + candidate build, followed by a two-reader packet with `5761cc9` as rollback. + + **Candidate packet lineage correction (`PASS / NO RUNTIME MUTATION`, + 2026-09-21).** The packaged candidate + `qdl-v2-python:2.0.26-e8394ff` + (`sha256:0919ef66877d6d622d4f3d69ca6d4cf6b959e929e2f3d92b9befef1d0ec50e92`) + passed the same immutable-image matrix `167/167`. Its first two-reader packet + exited before any recreate because a mechanically copied assertion still + expected the predecessor `3beb1e9` Compose chain. Read-only inspection proved + both current readers remained on `5761cc9` (`5490db...a81fb`), healthy, + restart `0`, OOM `false`, with the exact current chain + `049314...191ebe16`; the unaffected active/passive stream pair remained + healthy. No role, provider, durable store, consumer or order path changed. + The packet now asserts that actual current chain and its newline-inclusive + candidate chain `e71e00...29f0de4`, while retaining `5761cc9` as rollback. + This is a rollout metadata correction only; source/image/evidence and C2 + status are unchanged. The next permitted action is the same exact two-reader + serial packet, then the one strict batch ladder. + + **Candidate Compose-chain repair (`IN PROGRESS / PACKET ONLY`, + 2026-09-21).** The corrected first assertion permitted the two-reader roll to + `e8394ff`; both readers became healthy with restart `0` and OOM `false`. + Before the strict client started, its candidate assertion failed closed: + the copied packet's `compose_current` list had omitted the live `5761cc9` + image override, so the newly recreated readers had the intended new image + but an incomplete Compose provenance chain `bdd3c2...517d99`. This changed + no runtime behavior beyond the approved two-reader recreation and made no + provider/client/data/order request; the strict ladder, all-scope preflight + and C2 remain unconsumed. The same packet now restores that `5761cc9` + override in both candidate and rollback composition, recognizes only this + one bounded interim chain as its apply start, and will recreate the same two + readers once more into the exact candidate chain `e71e00...29f0de4`. Its + rollback chain remains `5761cc9`/`049314...191ebe16`. No source, image, + topology, data-plane, V1, stream, consumer or order-path change is involved. + + **P0 strict 50-BAR capacity diagnosis (`FAIL-CLOSED / SOURCE REPAIR`, + 2026-09-21).** The repaired `e8394ff` Query readers are both healthy on + `sha256:0919ef66877d6d622d4f3d69ca6d4cf6b959e929e2f3d92b9befef1d0ec50e92`, + `restart=0`, `OOMKilled=false`; the original collocated HTTP response timeout + is no longer the observed failure. The strict read-only ladder instead + returned one typed `DATA_STALE/LAST_EVENT_STALE` for + `OKX/SWAP/DOGE-USDT-SWAP/BAR/1m`, while its eight sibling DOGE BAR routes were + `LIVE`. No provider-direct request, fallback, stream subscription, order + action, Kafka offset mutation, Redis/SQLite mutation, all-scope preflight or + C2 was performed by that ladder. + + Read-only Kafka and cache inspection identified a real projection-capacity + defect rather than a provider or manifest defect. `stable-projector-v1` has + six canonical Kafka partitions but only three generic projector consumers, + pairing partitions `0/1`, `2/3` and `4/5`. At observation, lag was + `93/50/13079/532985/234488/627784` respectively. A 15.088-second direct + read-only Kafka sample measured `17,328` real canonical events (`1,148.5/s`): + `0:1248`, `1:1457`, `2:1915`, `3:6685`, `4:2313`, `5:3710`. The hot records + are real demanded Binance/OKX `TRADE`, `QUOTE`, `BOOK_DELTA` and + `MARK_INDEX_PRICE` bindings. In particular, BTC `*-stable-001` and the + BTC/ETH dated-book bindings are referenced by active consumer manifests; they + are not stale test debris and must not be silently removed to make a gate + pass. The shared cache remains a rebuildable projection behind Kafka's + durable authority; an isolated benchmark measured `25.5k-29.3k events/s` for + its local SQLite append path, so changing SQLite durability or deleting + events is neither justified nor approved. + + **Approved source scope and invariants.** Keep one provider-neutral + `stable-projector-v1` consumer group, the existing six Kafka partitions, + unchanged source timestamps/event IDs, all active bindings, `FULL` SQLite + durability, bounded `8 MiB` fetches and the existing active/passive Stream + writer lease. Correct capacity by making projector ownership one generic + replica per existing canonical partition and using the already bounded + `512`-record commit turn. This is not a per-symbol service, a Kafka topology + change, a new provider path, a quality/SLA relaxation or a replay/reset. The + source gate must add regressions for six-way generic ownership, FIFO/no + cross-partition starvation, bounded pending bytes/records and unchanged + downstream-before-checkpoint ordering; it must retain all prior query, + external-provider and `INTERNAL_STREAM` policies. + + **Decision boundary.** Source/config tests and an immutable image may proceed + without runtime mutation. Only if they pass may a separately enumerated + packet roll the existing three projector roles and add the three generic + projector replicas, preserving Kafka offsets/topology, Redis, SQLite, V1, + Rust cores, ingestors, Query/Stream readers, Trading System, alpha and order + path. It must prove a bounded live-lag window before retrying the strict + `1/8/16/32/50` BAR ladder. A green ladder alone permits one all-scope + two-replica preflight; only that green preflight permits the one final C2 + `300s` run. Cleanup, push, merge and release remain blocked. + + **P0 projector-capacity source/config gate (`PASS / RUNTIME PACKET NEXT`, + 2026-09-21).** The stable Compose now has six identical generic + `stable-projector-v1` members, one for each existing `md.canonical.v2` + partition, with unique instance/client/audit identities and no public ports. + It retains the same shared durable cache, active/passive Stream endpoints, + `2048` pending-record and `32 MiB` pending-byte ceilings, `512` fetched + records and `8 MiB` fetched bytes. Each complete bounded fetch is now one + `512`-record commit turn. This amortizes the existing durable SQLite + transaction under its existing `FULL` durability; it neither raises an + external-provider/`INTERNAL_STREAM` quota nor changes event IDs, source + timestamps, topic partitions, consumer group, offsets, quality semantics, + V1, Rust, ingest, Query/Stream contract or binding set. + + The new six-partition regression proves a full two-turn selection visits each + partition before returning to any of them, preserves each partition's + `0,1,...` FIFO prefix and does not mutate queued values. The Compose contract + now requires exactly six projector services for Kafka's six partitions, + one consumer group, six unique clients/audit paths and the same bounded + `512/8 MiB/2048/32 MiB` policy for every member. The pre-existing commit + ordering tests continue to prove downstream durable append/projection before + Kafka checkpoint. In the immutable `qdl-v2-python:2.0.26-e8394ff` image with + network disabled, source mounted read-only and tmpfs-only scratch, + `StableComposeAndBundleTests.test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route`, + `StableProjectorRecoveryTests` and `StableRuntimeBoundaryTests` passed + `36/36` in `12.373s`. `docker compose ... config --no-interpolate --quiet`, + test-module syntax compilation with bytecode redirected to tmpfs and + `git diff --check` passed. No runtime role, image, provider, Kafka offset, + Redis, SQLite, V1, Trading System, alpha or order path changed. + + **Remaining bounded runtime proof.** Build one immutable candidate from the + committed source, then serially recreate `projector_v2`, `projector_v2_2` + and `projector_v2_3` into this exact configuration and add only + `projector_v2_4`, `projector_v2_5` and `projector_v2_6`. Rollback stops the + three new members and recreates the original three with their exact old image, + runtime mount and `128`-record commit setting. It must not reset/reseek + Kafka, flush Redis, delete SQLite, alter V1/Rust/ingestor/Query/Stream, + change a binding or use provider replay. The live proof is a bounded + per-partition lag/age window, followed in order by the strict + `1/8/16/32/50` BAR ladder, all-scope two-replica preflight and one C2 + `300s`; a failed earlier gate consumes neither later gate. + + **R1.35-C projector-capacity runtime packet (`READY / APPROVED`, + 2026-09-21).** Candidate source `987b1a2` was built as immutable + `sha256:d724764b17dc9f21e681c2ebeac4fb48588d45010b0f1ff3a00b6cfa408f2a3b` + (`qdl-v2-python:2.0.26-987b1a2`). Packet + `/home/bobby/.local/state/qdl-v2/r135-projector-capacity-987b1a2-20260921T103546Z` + uses the active sealed runtime chain and changes only six generic members of + `stable-projector-v1`: serially replace the three current members and add + members `4..6`. Its exact rollback is image + `sha256:1329c9d7692b207c1aecd3cd562c0ba4b35638160e167132bb06fcba687ebe06` + plus the previous `128`-record commit window on only members `1..3`, after + stopping only `4..6`. It has no Kafka topology/offset reset, Redis flush, + SQLite deletion, V1, Rust, ingestor, Query/Stream, Trading System, alpha or + order-path scope. Render must pass before serial rollout; a live lag/age + receipt is required before any BAR ladder, all-scope preflight or C2. + + **Replica-health correction (`IN PROGRESS / SAME RUNTIME SCOPE`, + 2026-09-21).** The first addition of generic member `4` stopped before any + old member was recreated: it was processing canonical events but had no + Docker healthcheck because the historical liveness override named only + members `1..3`. This is a Compose-contract omission, not a projector crash, + Kafka loss or provider fault. The canonical Compose contract now assigns + every generic member a unique `QDL_STABLE_HEARTBEAT_PATH` and the same + bounded `30s` loop-turn check; the regression asserts all six paths and + healthcheck parameters. The packet must re-render and recreate member `4` + with this canonical health contract before continuing `5`, `6`, then `1..3`. + The source gate passed `48/48` targeted recovery, boundary, Compose and + heartbeat tests in the immutable `e8394ff` image with network disabled, + read-only source and tmpfs-only scratch; Compose render and `git diff --check` + also passed. Only member `4` was started before this correction; it processed + real canonical events, but no existing projector was recreated and no later + BAR/preflight/C2 gate was consumed. The stated rollback and all no-mutation + invariants remain unchanged. + + **Projector-capacity runtime result (`PASS / STRICT READ-PLANE RECHECK`, + 2026-09-21).** The approved packet was rendered against its exact sealed + Compose chain, then added members `4..6` and serially recreated the original + three only. All six `stable-projector-v1` members now run the immutable + `987b1a2` image `sha256:d724764b17dc9f21e681c2ebeac4fb48588d45010b0f1ff3a00b6cfa408f2a3b`, + report their individual heartbeat healthchecks, have `restart=0` and no OOM + kill. The unchanged Kafka group converged in three explicit observations: + total/per-partition lag was `273/74`, `112/28`, then `350/146`, within the + declared `<=500` total and `<=250` per-partition gate. Projector working + sets remained below `162 MiB` of their `768 MiB` caps during catch-up. This + proves generic six-way ownership and bounded durable catch-up; it does not + alter a topic, offset, Redis key, SQLite file, V1, Rust, ingestor, reader, + Trading System, alpha or order path. + + **Strict BAR ladder result after capacity (`FAIL-CLOSED / QUERY SOURCE + REPAIR`, 2026-09-21).** The capacity fix removed the prior projected-cache + lag. The read-only strict `1/8/16/32/50` matrix then passed every isolated + shape through both Query replicas. A legal two-lane `50`-BAR collocation + still timed out on the second lane: every batch alone completed in about + `13s`, while one Query reader deliberately serializes fully-local batches + and the second caller crosses its public deadline. The receipt is + `/home/bobby/.local/state/qdl-v2/r135-projector-capacity-987b1a2-20260921T103546Z/evidence/strict-bar-batch`; + it records `FAIL_TYPED_STATUS`, zero order actions, no provider-direct + request, no stream/fallback action and no durable mutation. Kafka group lag + was already within the capacity gate during this check, so this is not a + provider, DOGE, Kafka or freshness finding. All-scope preflight and the one + final C2 remain unconsumed. + + **P0 final-BAR local-read diagnosis and approved narrow repair (`IN + PROGRESS / SOURCE ONLY`, 2026-09-21).** A disposable `--network none`, + read-only, `1 CPU/512 MiB` profile mounted the live cache read-only and used + only materialized OKX BAR bindings. It measured the actual current + `StableSpoolQueryBackend.history_many()` path for one legal `50`-BAR, + two-row batch at `10,188.713 ms` with `50/50` successful results. The + immutable durable header `qdl.final_bar_close_time_ns` and the existing + `final_bar_watermarks` table identify the current one/two final BARs without + parsing every retained tail: the same profile found the exact final rows in + `587.442 ms` across the 50 physical tails. This is a measurement, not a + runtime fast path yet. Some long-history tails did not contain an immediately + preceding exact close, so a correct implementation must **fall back to the + existing full one-snapshot scan** for any absent, duplicate, revision-ambiguous, + non-continuous-calendar or otherwise non-exact tail. + + The approved source scope is a private, provider-neutral SQLite transport + helper plus the local `history_many()` planner: only a fully-local, + row-bounded, continuous-calendar final-BAR request for one or two rows may + use the durable header lookup. It must run all fast and fallback tails under + one SQLite read transaction; validate exact binding/close/open identity, + finality, unique market-time rows and normal history/quality/gap semantics; + and use the existing full tail materialization for every non-eligible or + uncertain request. No provider call, public schema/SDK change, timeout + increase, manifest/SLA change, cache migration, topology change or new + service is permitted. Required source evidence is fast-vs-full parity for + normal, late-backfill, missing, duplicate/revision and gap cases; one shared + snapshot for hybrid fast/fallback batches; unchanged `2500/5000/10000` warmup + behavior; cancellation/admission bounds; and a two legal maximum-batch + regression completing under the unchanged public deadline. Only one newly + attested Query image and a two-reader-only rollout may repeat the strict + ladder after those gates pass. + + **P0 exact final-BAR source repair (`PASS / QUERY-ONLY IMAGE NEXT`, + 2026-09-21).** `SQLiteDurableSpool.visit_final_bar_windows()` now reads a + one/two-row, watermark-derived window and lets the stable query layer accept + it only after exact binding, interval, final lifecycle, unique market-close + and canonical open/close duration validation. Every missing predecessor, + duplicate/revised close, gap, invalid optimization watermark, non-continuous + calendar, time-range or `2500/5000/10000` request takes the unchanged + retained-tail path. Fast and fallback physical tails remain inside one + deferred SQLite read transaction; no provider request, cache/schema/index + change, public endpoint/SDK/manifest change, timeout increase, durable write + or runtime mutation occurred. + + The isolated `qdl-v2-python:2.0.26-e8394ff` source-mount suite ran with + `--network none`, read-only root/source and tmpfs scratch: transport, + stable edge and universal warmup passed `162`, with `1` pre-existing isolated + Redis skip. Focused stable-query coverage was `18/18`, including normal and + late-backfill parity, missing/gap/revision fallback, corrupt watermark + fallback, hybrid single-snapshot behavior and unchanged large warmups. + `py_compile` and `git diff --check` passed. A disposable read-only profile + against the live durable cache, under `1 CPU/512 MiB`, returned `50/50` + two-row histories in `720.491 ms` for OKX and `820.615 ms` for Binance; + both had zero errors. The 11/50 long-interval incomplete windows were + deliberately full-tail fallback, not a partial promotion. Bounded evidence: + `/home/bobby/.local/state/qdl-v2/r135-projector-capacity-987b1a2-20260921T103546Z/evidence/final-bar-fastpath-source-profile-20260921T123235Z.json`. + + **Next permitted action.** Commit this source/journal slice, build exactly + one immutable Query image, then serially recreate only `query_v2_1` and + `query_v2_2` with the current `e8394ff` reader image as rollback. Run the + strict two-lane BAR ladder, then the all-scope two-replica preflight. The + single C2 `300s` remains unconsumed until both pass. + + **Strict C2 revision-policy discrepancy (`FAIL-CLOSED / NARROW SOURCE + REPAIR`, 2026-09-21).** The immutable `b7d4f1d` Query image + `sha256:d1abb81d6deb8401c6bba22dfa4bd91264ad068b5df9fa64d469e752af4a912c` + was serially applied only to `query_v2_1` and `query_v2_2`; both retained the + exact runtime mount, became `healthy`, have `restart=0`, and have no OOM. + The strict ladder then correctly failed before preflight/C2: its C2 closing + requirement preserves the declared `EMIT_REVISIONS` BAR policy, while the + new exact-window planner admitted only `LATEST`. It therefore took the + retained-tail path and the second two-lane 50-BAR read crossed the public + deadline after the first completed in about `13s`. This is an in-scope + source-policy omission, not a provider, Kafka, cache, data-quality or + manifest failure; no provider-direct, fallback, stream, order or durable + write occurred. + + The sole follow-up repair remains private and provider-neutral: admit + `EMIT_REVISIONS` only when the header-indexed rows already prove the exact + same unique final-close window that the retained-tail selector would return. + Any duplicate/revised close, missing predecessor, gap, invalid watermark or + ambiguity continues to fall back inside the same SQLite snapshot. Required + regressions are normal `EMIT_REVISIONS` fast-path parity and duplicate/revised + `EMIT_REVISIONS` full-tail parity. No public contract, manifest/SLA, timeout, + provider policy, cache schema, topology or service count may change. A new + Query image and the same two-reader-only packet are required before retrying + the still-unconsumed ladder/preflight/C2 sequence. + + **EMIT_REVISIONS exact-window source gate (`PASS / REBUILD QUERY ONLY`, + 2026-09-21).** The planner now admits both declared revision policies only + after the existing exact unique-final-close proof. A normal two-row + `EMIT_REVISIONS` request takes the header-indexed path and is byte-for-byte + equal to retained-tail history; a duplicate revised close deterministically + takes the retained-tail path and remains equal to it. The focused stable + query class passed `20/20`. The isolated, network-disabled, read-only + affected suite passed `164` with `1` existing Redis-dependent skip. This + source-only repair made no runtime, provider, durable-data, consumer or + order mutation. The rolled `b7d4f1d` readers remain healthy but are not a + certificate candidate for this follow-up source; build one replacement + Query image from the committed repair, update the same two-reader packet's + candidate/rollback chain, then repeat the strict ladder exactly once. + + **Three-consumer local-admission finding and approved narrow repair (`IN + PROGRESS / SOURCE ONLY`, 2026-09-21).** The replacement `3aa7048` Query + image was serially applied only to `query_v2_1` and `query_v2_2`; both are + healthy, restart-free and non-OOM. The read-only strict ladder now proves + the `EMIT_REVISIONS` exact-final path itself: the first 50-BAR OKX lane + completed on both replicas in about `0.9s/1.5s`, and the two-lane + Binance/OKX step completed under its unchanged `30s` deadline. The third + declared consumer lane (`trading-system.paper.stable`, ten BAR requests) + was instead returned as typed `RATE_LIMITED` before a cache read, provider + call, stream/fallback action or durable mutation. The direct cause is the + private `_LocalBatchAdmission(max_active=1, max_pending=2)` guard: it + accepts one active and one queued whole-local batch, while the actual C2 + consumer graph has three bounded local batch lanes. + + The only permitted follow-up is a provider-neutral Query admission repair. + It keeps exactly one expensive SQLite/history-materialization lane active, + bounds the global resident batch count at three (one active plus two queued), + and bounds each queued admission wait by the smallest declared batch work + deadline. A fourth collocated lane or an expired queued request remains a + typed local-capacity rejection; cancellation must drain its pending count. + It must not raise external provider concurrency/rate limits, relax any + freshness/finality policy, change public API/SDK/manifest/cache schema, + create a worker/service, or start C2. Required tests are FIFO three-lane + admission, fourth-lane rejection, admission-wait expiry, cancellation + cleanup, and preservation of one-active history work. After source gates, + build exactly one Query image, roll only the same two Query readers with + `3aa7048` as rollback, then repeat the strict ladder once. + + **Three-consumer local-admission source gate (`PASS / BUILD QUERY ONLY`, + 2026-09-21).** The shared local gate now retains one active SQLite/history + lane, admits two finite FIFO waiters, and rejects a fourth lane before it + reads cache data. Each queued lane uses the minimum declared work deadline + as its admission-wait bound; once admitted, the existing local executor + retains the unchanged per-item execution deadline. A timed-out waiter is a + typed local `RATE_LIMITED` result, and a cancelled waiter drains without + leaking a permit. The local lane still never applies venue token pacing, + provider retry or external circuit policy. + + A network-disabled, read-only, non-root container ran the focused + `SingleWarmupExecutionTests` `15/15`; this includes the three-lane FIFO, + fourth-lane typed rejection, wait-expiry and queued-cancellation regressions. + The complete affected source matrix (`test_phase10_universal_warmup`, + `test_phase105_identity_acceptance`, `test_phaseb_stable_edge`) contains + `178` cases and completed with exit `0` in the same container discipline. + `py_compile` and `git diff --check` passed. No runtime role, provider, + durable state, V1, consumer, alpha or order path changed. The next permitted + action is one immutable Query image built from this source, a serial + two-reader-only rollout with `3aa7048` as rollback, and one repeat of the + strict ladder; all-scope preflight and C2 remain unconsumed. + + **Packet-chain defect detected before acceptance (`FAIL-CLOSED / RESTORE + KNOWN READER PAIR`, 2026-09-21).** The first `feba690` packet application + did not reach the strict ladder. Its copied rollback script omitted the + active `3aa7048` image override from the Compose *rollback* file chain. + When its candidate assertion failed, the error trap restored + `query_v2_1` to the older terminal `b7d4f1d` override while + `query_v2_2` remained on `3aa7048`; both containers were healthy, but the + pair was no longer homogeneous. This is an operator-packet provenance bug, + not a Query, provider, Kafka, cache, data-quality or consumer finding. No + C2, preflight, provider-direct request, fallback, stream, order or durable + mutation occurred. + + The immediate bounded correction is to restore the reader pair's known + `3aa7048` image and exact recorded Compose chain. `query_v2_2` already has + that coordinate and therefore is inspected but not needlessly recreated; + only the drifted `query_v2_1` is recreated serially. All other roles remain + untouched. The replacement candidate packet must derive both its candidate + and rollback chains explicitly from the restored reader labels, validate + those chains before apply, and retain the current image as its only rollback. + No further candidate rollout or ladder may run until that repair is verified. + + **Corrected Query rollout and strict BAR ladder (`PASS / ALL-SCOPE + PREFLIGHT NEXT`, 2026-09-21).** The pair was first restored to homogeneous + `3aa7048`, then the corrected packet preserved that override in both + candidate and rollback chains and serially recreated only `query_v2_1` and + `query_v2_2` to immutable `feba690` + `sha256:d36cf8c25048a2e330c4580bb9f1a2681cda84a282fdee260b958cf7c1fcba3b`. + Both are `healthy`, `restart=0`, `OOMKilled=false`, with matching config + provenance; adjacent V2 roles were inspected and remained running. Packet + evidence is bounded under + `/home/bobby/.local/state/qdl-v2/r135-local-admission-feba690-20260921T134700Z/`. + + Its read-only strict `1/8/16/32/50` BAR matrix passed through both replicas, + including three concurrent declared consumer lanes. Isolated 50-BAR latency + was `1059.029ms` primary and `1734.505ms` secondary. The three-lane wave + returned all `50 + 50 + 10` BAR reads: primary `2304.655ms`, `1030.282ms`, + `1426.253ms`; secondary `2062.450ms`, `2947.207ms`, `795.035ms`. + The earlier third-lane `RATE_LIMITED` is closed. The receipt reports + `PASS_STRICT_BAR_BATCH_SHAPE`, `order_actions=0`, `provider_connections=0` + and cleaned cursor state. This is real durable V2 data but not C2, and it + does not certify the release. The next allowed gate is the one all-scope + two-replica read-plane preflight; C2 remains unconsumed. + + **All-scope MARK/INDEX diagnostic (`IN_PROGRESS / PRE-C2 / NO POLICY + RELAXATION`, 2026-09-21).** The first all-scope public-SDK preflight stopped + before C2 at the strict execution requirement + `trading-system.paper.stable/OKX/SWAP/PERPETUAL/DOGE-USDT-SWAP/MARK_INDEX_PRICE` + on secondary Query. Its leaf diagnosis was typed `SOURCE_UNAVAILABLE`; the + active stream gateway returned `COMPONENT_STALE` and the passive gateway + returned its expected writer-fence response. There was no consumer direct + provider call, V1 fallback, stream subscription, order action, Kafka offset, + Redis or SQLite mutation from that diagnosis. + + Read-only evidence narrows the event without inventing a new SLA: all five + OKX session files were `LIVE`; the sampled DOGE pair had an old provider + confirmation (`476.881s`) which reached the durable cache only `7.393s` + before the read. A bounded primary-key inspection of the next 4,096 canonical + DOGE pairs then showed normal recovery: MARK maximum inter-update gap + `283ms`, INDEX `1,997ms`, pair maximum `282ms`, and `0` pair records over the + existing `15s` MARK / `70s` INDEX component cadence. The sample therefore + proves a transient materialization/component incident recovered; it does not + prove that changing the cadence, hiding `COMPONENT_STALE`, polling venue REST + or hard-coding DOGE would be correct. Rust's existing component fence remains + the authority. The required next gate is the all-scope two-replica fast + preflight again; a typed failure returns to the exact source/projection + boundary, while a pass alone authorizes the single final C2 300-second run. + + **Canonical hot-partition receipt (`IN_PROGRESS / PRE-C2 / DRAIN BEFORE + RETRY`, 2026-09-21).** The subsequent all-scope recheck did not expose a + provider outage or an invalid MARK/INDEX contract. Read-only real-provider + probes received all five demanded symbols from both venues: Binance USD-M + `BTCUSDT/ETHUSDT/SOLUSDT/DOGEUSDT/BNBUSDT` emitted 25--26 mark frames per + symbol in 25 seconds, and OKX Swap emitted 77--144 INDEX and 124--126 MARK + frames per corresponding `*-USDT-SWAP` product in the same bounded window. + The corrected OKX probe used a valid WebSocket request identifier; no + provider fallback, order, stream subscription, cache, Redis, SQLite or Kafka + mutation was made by either probe. + + A read-only exact-ten cache/query matrix then found current durable receipts + for Binance `BTC/ETH/SOL/BNB` and OKX `BTC/DOGE/BNB`, while Binance `DOGE` and + OKX `ETH/SOL` could still surface old provider receipts through the active + gateway. Kafka group evidence identifies the common cause: consumer group + `stable-projector-v1` owns all six existing `md.canonical.v2` partitions, but + partition `3` is assigned to `stable-projector-4` and had lag `78,183`, later + `60,552`; the other five partitions were only `71--340`. Projector-4 has no + restart/OOM/error and is draining in order: its observed canonical age fell + from about `470s` to `181s`. Its bounded spans show broker polling and + checkpointing are low-cost; durable append plus compatibility projection are + the limiting turn. This is a live catch-up after the approved six-way + capacity rollout, not a reason to relax freshness or introduce a venue REST + shortcut. + + **Invariant and next gate.** Keep the current six Kafka partitions, offsets, + shared SQLite/Redis state, V1, Rust cores, ingestors, readers, Trading System, + alpha and order path unchanged. First obtain two bounded zero/near-zero lag + samples after the active backlog drains, then run the exact-ten MARK/INDEX + matrix and the already-authorized all-scope two-replica preflight. Only a + green preflight consumes the final C2 `300s` gate. If partition `3` fails to + reach or maintain the declared live-lag window, profile and repair its + bounded append/projection turn with FIFO/checkpoint regressions before any + C2 retry; do not add a service, alter Kafka topology, reset offsets, delete + cache state or lower data-quality policy to force a pass. + + **Exact-ten recovery and all-scope preflight (`MARK/INDEX PASS; PREFLIGHT + FAIL-CLOSED AT BOOK_DELTA`, 2026-09-21).** Two later read-only group samples + showed the partition-3 catch-up reached steady state (`422`, then `393`) + alongside partitions `53--285`. A disposable, read-only Query-identity + probe then read the exact ten execution MARK/INDEX products through the + active V2 gateway: all `10/10` returned one complete `OK` observation; the + active Stream returned `200` and the passive writer correctly returned + `409` fenced for every product. The client was removed and its temporary + host environment file was deleted. This closes the MARK/INDEX backlog + symptom without a policy or provider change. + + The one permitted all-scope no-stream/no-fallback preflight was then run + through both Query replicas using the four governed identities and real + manifest scope. It stopped before C2 at a strict eight-item + `trading-system.paper.stable` `BOOK_DELTA` batch on the secondary replica: + all eight results were typed `RATE_LIMITED`/`PARTIAL_RESULT`. Its bounded + leaf receipt records no provider payload, no order actions, no cursor + persistence and no fallback. Several leaves also retained correctly + fail-closed `LAST_EVENT_STALE` evidence while their provider sessions were + `LIVE`; this is not permission to treat the batch as usable. The disposable + client exited and was removed; readers, streams, six projectors, Kafka + offsets/topology, Redis, SQLite, V1, Trading System, alpha and order path + remain unchanged. C2 remains unconsumed. + + **Narrow diagnostic boundary.** Determine whether the batch entered an + external-provider limiter despite declared durable delivery, or whether the + bounded local cache/history executor emitted a false `RATE_LIMITED` under + the exact `BOOK_DELTA` shape. Required evidence is the exact secondary + Query logs/config path, the per-item local/cache state and a source + regression that preserves external Binance/OKX/DNSE limits and all existing + fail-closed freshness/gap behavior. Do not retry preflight/C2, relax the + BOOK_DELTA SLA, call venue REST or change topology until that boundary is + proven. + + **Four-consumer local admission correction (`IN PROGRESS / SOURCE-ONLY`, + 2026-09-21).** Source inspection now establishes the boundary without a + provider hypothesis: `BOOK_DELTA` uses the fully local canonical-cache + history path, while read-plane preflight starts the four governed consumer + coroutines concurrently. The default `_LocalBatchAdmission` admits only + three resident whole batches (one active plus two FIFO waiters), so the + fourth legal consumer batch is returned as typed `RATE_LIMITED` before any + cache or provider work. This exactly explains the all-eight-item secondary + `PARTIAL_RESULT`; provider labels in the receipt are binding provenance, not + a venue request. + + The approved repair is deliberately bounded: admit the four manifest-declared + local lanes (one active plus three FIFO waiters), retain one active SQLite + materialization, and reject a fifth lane before cache work. Add an exact + four-consumer FIFO/overflow regression plus existing cancellation and + deadline coverage. Do not alter Binance/OKX/DNSE or `INTERNAL_STREAM` + budgets, freshness/gap/session semantics, public contract, manifest, Kafka, + Redis, SQLite, topology or any order path. Source tests must pass before one + Query-only candidate image, a two-reader rolling packet, one all-scope + preflight, and only then the single final C2 `300s` certificate. + + **Four-consumer admission source result (`PASS / QUERY-ONLY BUILD NEXT`, + 2026-09-21).** The default bound now retains exactly one active + materialization plus three finite FIFO waiters. The regression uses the four + actual governed consumer identities, proves all four drain in FIFO order + without overlapping `history_many()`, and proves a fifth lane is typed + `RATE_LIMITED` before cache work. Existing deadline-expiry and cancellation + regressions continue to prove no pending/permit leak. `git diff --check` and + `py_compile` passed. In the existing non-root, read-only-root, network-disabled + Query image, focused `SingleWarmupExecutionTests` passed `15/15`; the affected + source matrix `test_phase10_universal_warmup`, + `test_phase105_identity_acceptance`, and `test_phaseb_stable_edge` passed + `178/178` with one existing Redis-dependent skip. No provider, runtime role, + Kafka, Redis, SQLite, V1, consumer, alpha or order action participated. + + The next permitted action is one immutable Query reader image from this + source, then a serial two-reader rollout retaining the active `feba690` + digest as rollback. Only the exact all-scope no-stream/no-fallback preflight + may follow a healthy reader pair; C2 remains unconsumed until that preflight + passes. + + **Four-lane packet provenance failure and recovery (`FAIL-CLOSED / REPAIRED + BEFORE RETRY`, 2026-09-21).** Candidate image + `qdl-v2-python:2.0.26-a7a16ac` + `sha256:fb351505dc205dcac48e16148b50424b064e817305053ab2dbb01850017f9984` + built from `a7a16ac` and passed the same immutable-image `178/178` source + matrix. Its first two-reader packet stopped after the first reader before + any acceptance gate: the copied template omitted the active additive + `feba690` override from its *base* Compose list, even though the live + container label correctly included it. Candidate-chain assertion therefore + failed, and the old rollback helper silently tolerated a failed restore, + leaving `query_v2_1` at `3aa7048` while `query_v2_2` remained at `feba690`. + No provider, data-plane, C2, cursor, fallback or order action occurred. + + Recovery recreated only `query_v2_1` from the exact healthy + `query_v2_2` Compose-chain label, returning both readers to `feba690`, + `healthy`, `restart=0`, `OOMKilled=false` and the same read-only runtime + mount. The replacement packet adds the current `feba690` override to the + base chain, records only redacted container summaries, and treats a rollback + failure as terminal rather than swallowing it. Its syntax, JSON and + read-only Compose/provenance verification passed. The candidate was not + applied after that correction yet. The next permitted mutation remains a + serial recreate of only the two Query readers; all named durable state and + adjacent roles remain unchanged. + + **Four-lane reader rollout and all-scope preflight (`FAIL-CLOSED / DIAGNOSIS + NEXT`, 2026-09-21).** The repaired packet serially recreated only + `query_v2_1` and `query_v2_2` into the immutable `a7a16ac` candidate + `sha256:fb351505dc205dcac48e16148b50424b064e817305053ab2dbb01850017f9984`. + Both are `healthy`, `restart=0`, `OOMKilled=false`, retain the same + read-only runtime revision and retain `feba690` + `sha256:d36cf8c25048a2e330c4580bb9f1a2681cda84a282fdee260b958cf7c1fcba3b` + as their exact rollback image. No other role, durable state, V1 route, + provider connection, consumer, alpha or order path changed. + + The one permitted all-scope, no-stream/no-fallback/no-order preflight then + ran through an ephemeral non-root, read-only client and removed that client + at exit. It correctly cleared the prior local-admission `RATE_LIMITED` + result, but stopped fail-closed at `trading-system.paper.stable` on + `OKX.SWAP.PERPETUAL.BTC-USDT` `BOOK_DELTA`: `DATA_STALE` with + `LAST_EVENT_STALE`, `SOURCE_SESSION_UNAVAILABLE` and + `SOURCE_SESSION_UNKNOWN` (provider liveness absent). The adjacent + `OKX BNB BOOK_DELTA` observation was session `LIVE` with an older on-change + event, so it must not be conflated with the BTC session failure. This is a + typed quality/projection finding, not a quota, manifest, fallback or + provider-direct failure. `C2` remains unconsumed. The next action is a + bounded read-only ten-book/two-replica status matrix, followed by a narrow + shared projection or session-lineage repair only if that matrix proves one; + no SLA relaxation or acceptance retry is permitted first. + + **Execution-L2 typed recovery matrix (`PASS / ALL-SCOPE PREFLIGHT NEXT`, + 2026-09-21).** The bounded no-stream status matrix read all five execution + symbols on Binance USD-M and OKX Swap as both `BOOK_SNAPSHOT` and + `BOOK_DELTA`, through both Query replicas, for three consecutive rounds. + All `10` physical books / `20` products were `LIVE`, complete, gap-free, + sequence-verified and replica-parity matched; every delta had a matching + `LIVE` provider session and every snapshot met depth `100`. The matrix used + an ephemeral non-root/read-only client, made `0` provider connections and + `0` order actions, and removed its cursor directory. The earlier BTC/OKX + `UNKNOWN` therefore was a real, fail-closed transient during session/event + recovery, not evidence for a source mapping defect or a reason to relax the + policy. Evidence: + `/home/bobby/.local/state/qdl-v2/r135-four-lane-a7a16ac-20260921T163500Z/evidence/l2-status/`. + The single next gate is the existing all-scope two-replica read-plane + preflight; C2 remains unconsumed. + + **All-scope preflight after L2 recovery (`FAIL-CLOSED / LOCAL BATCH + DIAGNOSIS NEXT`, 2026-09-21).** The one permitted all-scope preflight was + run exactly once after the L2 matrix, through the same ephemeral non-root + and read-only client. It made `0` order actions and stopped before C2. + `monitoring.multivenue.stable` received typed `PARTIAL_RESULT` for its + four-item durable `TRADE` batch: Binance BTC, Binance ETH, OKX BTC and OKX + ETH were all returned as `RATE_LIMITED`. The receipt records individual + identity, consumer, source-policy and typed status without payloads; the + three inspected non-timeout feed statuses were independently `LIVE`, + complete and execution-eligible. This is therefore not a provider outage, + a stale-data policy failure, a manifest entitlement error or an L2 defect; + it is an unresolved local Query batch-admission/scheduling boundary. + Evidence: + `/home/bobby/.local/state/qdl-v2/r135-four-lane-a7a16ac-20260921T163500Z/evidence/read-plane/`. + Do not rerun preflight or C2. The permitted next action is source inspection + and a targeted batch-shape/concurrency regression that reproduces this + exact four-item durable request, then the smallest policy correction if + proven. + + **Five-slot local admission correction (`APPROVED SCOPE / SOURCE NEXT`, + 2026-09-21).** Inspection proves the `RATE_LIMITED` response came from the + fully-local batch admission, not the HTTP request boundary or Redis identity + quota: the endpoint returned a typed four-item partial response rather than + one HTTP `429`. The previous `max_pending=4` includes its active batch, so it + only works if the reader is idle when the four governed consumers begin. + One normal in-flight cache request plus monitoring, Trading System, + alpha-Binance and alpha-OKX creates five legitimate, finite whole-batch + lanes and rejects the fourth governed request. The approved minimal repair + is `max_pending=5`, preserving one active SQLite materialization, FIFO + serialization, per-request deadlines and an explicit sixth-lane rejection. + Add an exact regression for incumbent + four declared lanes and sixth-lane + overflow; do not change external provider quotas, `INTERNAL_STREAM`, HTTP + request bounds, manifests, routes, topology or data state. + + **Five-slot local admission source result (`PASS / QUERY-ONLY IMAGE NEXT`, + 2026-09-21).** Query now admits exactly five finite local-cache batch slots: + one incumbent reader batch plus the four named stable consumer lanes. The + regression holds the incumbent materialization, admits monitoring, Trading + System, alpha-Binance and alpha-OKX, proves exactly one active + `history_many()` materialization, rejects a sixth lane as typed + `RATE_LIMITED`, then drains all five without a pending or permit leak. + `py_compile` and `git diff --check` passed. In the immutable predecessor + Query image with the source mounted read-only, non-root and `--network none`, + the exact regression passed `1/1`; the focused suite passed `124` with one + existing Redis-dependent skip; the complete affected + `test_phase10_universal_warmup`, `test_phase105_identity_acceptance` and + `test_phaseb_stable_edge` matrix passed `178/178` with that same one + existing skip. No runtime role/image/provider/durable state/consumer/alpha + or order path changed. The next permitted mutation is one immutable + Query-only image followed by a serial two-reader packet retaining `a7a16ac` + as rollback; all-scope preflight and C2 remain unconsumed. + + **Five-slot immutable image gate (`PASS / TWO-READER PACKET NEXT`, + 2026-09-21).** Exactly one candidate image was built from committed source + `ed164a91dde453f5bbe57bdd2fb6ca88b089b06a`: + `qdl-v2-python:2.0.26-ed164a9` + `sha256:3b06543620dda25ab70db566d808841e137dfa55e54bd5bc366a372cf665c6ee`. + Its OCI revision and release labels match that source and it runs as + `qdl:qdl`. With no source mount, `--network none`, read-only root, tmpfs + `/tmp` and non-root UID/GID, the same complete affected matrix passed + `178/178` with one existing Redis-dependent skip in `38.214s`. The expected + `a7a16ac` Query image + `sha256:fb351505dc205dcac48e16148b50424b064e817305053ab2dbb01850017f9984` + remains the exact rollback coordinate. Next is a provenance-checked serial + recreate of only `query_v2_1` then `query_v2_2`; no other role, state, + topology, V1 route, consumer, alpha or order path is in scope. + + **All-scope preflight after five-slot rollout (`FAIL-CLOSED / DECLARED QUOTE + SEMANTIC CORRECTION NEXT`, 2026-09-21).** Both Query readers reached + `ed164a9` healthy with `restart=0` and no OOM, then the single all-scope + no-order preflight stopped before C2 with `0` order actions. The typed leaf + receipt identifies `alpha.binance.paper.stable` `QUOTE` rows BTC, ETH and + BNB as `DATA_STALE`: their immutable BBO events were `5.7-6.3s` old but all + three source sessions were `LIVE` under `1s`, complete and gap-free. This is + neither a provider/session failure nor an SLA relaxation opportunity. + + The source inventory proves the same compiler defect affects all ten alpha + `QUOTE` routes (five Binance USD-M and five OKX Swap): the stable catalog + correctly declares native BBO `delivery_semantics: ON_CHANGE`, but + `phase533_materialize_alpha_runtime_entitlements.py` applies observed + recency only to `TRADE`. It thereby emits strict-event alpha quote + requirements despite the declared source contract. The in-scope correction + derives `event_recency_policy: OBSERVE` only when the exact QUOTE binding + declares `ON_CHANGE`; generic strict quotes remain strict. It must render + the two alpha manifests, their sealed route hashes/revisions and primary + routing revision through the existing materializer, then prove all declared + on-change alpha quotes have a session SLA/OBSERVE policy while a synthetic + strict quote cannot acquire that policy. No provider quota, freshness limit, + event timestamp, route fallback, topology, durable state or order path may + change. A new image/two-reader packet and one fresh preflight are required + before the still-unconsumed C2. + + **Materializer union precondition (`IN_PROGRESS / SOURCE-ONLY`, + 2026-09-21).** Rendering the two corrected alpha manifests exposed a second + compiler defect before any generated artifact or runtime state changed. The + production demand declares `330` realtime rows but only `180` physical + identities; `150` identities appear exactly twice, once for the Trading + System paper consumer and once for the matching alpha consumer. Compact + comparison found `0` conflicting duplicate payloads. That is the expected + shared-feed union, not an ambiguous consumer request. The current alpha + materializer incorrectly rejects it before generation. The bounded repair is + to canonicalize identical rows by the existing physical identity and to + continue fail-closed if two rows with that identity differ in any declared + field. Required source proof: identical duplicate union, conflicting + duplicate rejection, all ten declared ON_CHANGE quote requirements render + `OBSERVE`, synthetic strict QUOTE remains blocking, and the pre-existing + manifest/release-route idempotency proof. No runtime, image, manifest file, + provider, quota, freshness, route, durable state, topology or C2 action is + permitted until the source gate passes. + + The first generated-diff inspection also caught a preservation hazard before + any candidate was built: replacing managed realtime requirements would have + removed the existing `OBSERVE` policy from all ten `BOOK_DELTA` requirements. + `BOOK_DELTA` is already a declared quiet-session feed in the shared quality + decision and its prior alpha policy is intentional; losing it would turn a + quiet but gap-free verified book into a false stale result. The same bounded + compiler patch must therefore preserve `OBSERVE` for `TRADE` and + `BOOK_DELTA`, derive it for QUOTE only when the exact binding declares + `ON_CHANGE`, and prove all three cases in the materializer regression. This + is preservation of the current contract, not a freshness/SLA relaxation or + a source-catalog/routing change. + + **Declared QUOTE compiler source gate (`PASS / COMMIT AND QUERY-ONLY BUILD + NEXT`, 2026-09-21).** The compiler now unions only byte-equivalent shared + physical demand rows, rejects a conflicting row under the same identity, + preserves `OBSERVE` for the existing TRADE/BOOK_DELTA quiet-session contracts, + and derives it for QUOTE only from a catalog-declared `ON_CHANGE` binding. + The final generator materialized only its four governed artifacts: + `alpha-binance-paper` manifest `12`, `alpha-okx-paper` manifest `11`, stable + release route `22`, and primary route `6`; the sealed manifest SHA values in + the route were regenerated by the generator. A subsequent non-networked, + non-root read-only dry-run reported `manifest_changed=false` and + `release_route_changed=false`. + + Source evidence was run inside the existing immutable `ed164a9` image with + the worktree mounted read-only, `--network none`, read-only root and tmpfs: + `py_compile` passed; focused materializer tests passed `7/7`; the final + routing/identity/fallback/release matrix passed `117/117` in `58.398s`; and + the targeted QUOTE/TRADE/BOOK_DELTA/MARK_INDEX/final-BAR/SDK/query quality + matrix passed `177/177` with one existing Redis-dependent skip in `41.861s`. + The test fixtures deliberately emit provider/backpressure failure messages; + neither suite performed provider I/O, stream/fallback action, order action or + durable mutation. `git diff --check` passed. No new image, runtime role, + provider quota, freshness limit, route policy, V1 path, Kafka/Redis/SQLite + state or C2 action has occurred in this source gate. The only next mutation + is one immutable Query image from the committed source, a serial + two-reader-only packet retaining `ed164a9` as rollback, then one all-scope + fast preflight; C2 remains unconsumed until that preflight passes. + + **Lossless L2 transport correction (`APPROVED SCOPE / SOURCE-ONLY`, + 2026-09-21).** The subsequent `137633b` Query-only rollout was healthy on + both readers, but its one all-scope preflight stopped before C2 at an + eight-item `BOOK_DELTA` `warmup:batch` with typed `RATE_LIMITED`. This is a + local `history_many()` scheduling outcome: BOOK snapshot and delta share one + physical SQLite partition, so a generic historical batch must scan bounded + physical tails before it can select each logical feed. It is not an OKX or + Binance request, provider quota, stale-data waiver, manifest entitlement, or + L2 sequence failure. + + More importantly, the generic closing/preflight transport was wrong for this + product class. `BOOK_DELTA` is declared lossless and is useful only after a + verified snapshot plus signed stream/replay cursor; a latest-history batch is + not its execution handoff. The existing bounded L2 matrix already proved the + correct no-stream read plane for all ten physical execution books: typed + `feed_status` plus one exact `snapshot` for both logical products through + both Query replicas, three consecutive rounds, with complete, gap-free, + sequence-verified depth-100 views and replica parity. C2 opening separately + proves signed cursor/reconnect/replay for each stream product. + + The narrow repair changes the acceptance harness only: split lossless L2 + products from generic closing `warmup:batch`; for each replica, read the + sealed SDK `feed_status` and `snapshot`, retain the same exact + identity/freshness/gap/quality/replica checks, and record a transport class + of `L2_STATUS_SNAPSHOT`. BAR and other history products retain existing + batch behavior. No Query/Stream/Rust runtime, provider limiter, freshness + threshold, catalog/manifest/route, V1 policy, Kafka, Redis, SQLite or order + path changes in this slice. Required source gates are: all L2 products never + enter `warmup_batch`; each remains cardinality-complete across both replicas; + snapshot/delta typed failures preserve product identity; generic BAR batch + behavior is unchanged; and the existing fast L2 status matrix remains the + independent sequence/generation oracle. If the corrected preflight cannot + establish a current snapshot or C2 opening cannot establish lossless replay, + it fails closed and a data-plane repair must be proposed separately; no + deadline/SLA relaxation is allowed. C2 remains unconsumed until all source + and all-scope preflight gates pass. + + **Source implementation and focused gate (`PASS / NO RUNTIME MUTATION`, + 2026-09-21).** The C2 closing harness now dispatches only `BOOK_SNAPSHOT` + and `BOOK_DELTA` through `L2_STATUS_SNAPSHOT`: each exact logical demand + reads typed status and a bounded snapshot on both replicas, verifies + identity, `LIVE`, complete, gap-free, source policy, declared session bound + and replica content parity, then records only hashes and compact quality + evidence. The generic strict history batch explicitly rejects lossless L2, + while C2 opening remains responsible for signed delta cursor/reconnect/replay. + New regressions prove both logical L2 feeds never reach `warmup_batch`, a + typed stale status fails before a snapshot/payload can be used, and the CLI + emits that failure as compact `FAIL_TYPED_STATUS` evidence rather than a + traceback. `py_compile` passed. In the existing immutable + `qdl-v2-python:2.0.26-137633b` image with a read-only source mount, non-root + UID, `--network none`, tmpfs-only test state and `--rm`, the three targeted + regressions passed `3/3`; the affected C2/L2/SDK matrix passed `73/73` in + `4.760s`. The expected gRPC shutdown GOAWAY was emitted by an isolated test + transport after its calls completed; no provider, durable state, runtime + role, image, container, order or consumer mutation occurred. The bounded + wider matrix then passed `213`, skipped `1` pre-existing isolated-Redis case + (`214` total) in `39.016s`, with `RuntimeWarning` promoted to an error. Its + deliberately injected projector backpressure/recovery and gRPC shutdown logs + were observed only inside the disposable source-test process. Next: inspect + and commit this source-only slice, then run one all-scope fast preflight + against the already healthy two Query readers. C2 remains unconsumed. + + **All-scope read-plane preflight (`PASS / C2 FINAL GATE NEXT`, 2026-09-21).** + Committed source `e72905e` was mounted read-only over only the client harness + in an ephemeral `--rm` launcher; the two running readers remained on + `qdl-v2-python:2.0.26-137633b@sha256:0a69fbf0c883cad27a433ea529555269ab7386706de6e1c635fa5fef2107a545`, + healthy with `restart=0`, `OOMKilled=false` and the unchanged runtime mount + `r135-b2-335792a-20260920T015057Z/runtime`. The real, authenticated, + two-replica preflight returned `PASS_READ_PLANE_PREFLIGHT` for all `299` + active products, `exit_code=0`, `provider_connections=0`, `order_actions=0` + and `cursor_directory_removed=true`. Evidence is bounded under + `/home/bobby/.local/state/qdl-v2/r135-lossless-l2-e72905e-20260921T181741Z/evidence/read-plane/`. + It opened no stream, invoked no fallback/provider-direct call, recreated no + role and changed no V1/Kafka/Redis/SQLite/market-data/order state. The one + remaining R1.35-C action is exactly one C2 300-second no-order acceptance + using this committed harness and the same reader runtime; failure remains + fail-closed and does not permit a retry without a distinct typed cause. + + **C2 launcher provenance abort (`NO DATA-PLANE ACTION`, 2026-09-21).** The + first foreground launcher exited before authentication, Query, Stream, + fallback or observation because its historical command selected + `/app/qdl-runtime/stable-v2-release-routing.yaml`, whose manifest binding no + longer matches the active image source. Its bounded stderr ended at + `stable release consumer manifest/demand binding differs`; `acceptance.json` + was empty, the `--rm` client was absent afterward, and no order/provider/data + action occurred. This is a launcher provenance mismatch, not a second C2 + result or a market-data failure. The passed all-scope preflight already used + the matching active-image route + `/app/config/v2/stable-v2-release-routing.yaml`; the one replacement C2 + launcher must use exactly that route while retaining every other identity, + runtime, timeout, no-order and cleanup invariant. No role/image/runtime + change is needed. + + **C2 identity provenance abort (`NO ACCEPTANCE RESULT / NO DATA-PLANE + ACTION`, 2026-09-21).** The corrected route launcher derived its real + manifest budget (`935s`, not a literal `1800s`) and began opening the scope, + but the historical `r135-c-on-change` identity composition supplied an old + `alpha-binance` workload signing material. Stream correctly rejected it with + `UNAUTHENTICATED: workload token is not bound to the active consumer manifest + revision`; sibling opening tasks were cancelled, `acceptance.json` remained + empty, and no observation/certificate was produced. This is not a product, + freshness, L2, fallback or provider failure. The already-passed all-scope + preflight used the current `r135-local-batch-fair` identity/acceptance-input + bundle instead. Before the one replacement C2, the launcher must reuse that + exact bundle and run a bounded current-identity Stream authentication probe; + only if all four governed identities are accepted may the single 300-second + acceptance consume its observation window. No service, runtime, route, + manifest, V1, Kafka, Redis, SQLite or order-path mutation is required. + + **Stream revision skew (`P0 RUNTIME CORRECTION REQUIRED`, 2026-09-21).** + Read-only provenance inspection established the actual cause of the Stream + rejection: both Query roles run + `qdl-v2-python:2.0.26-137633b@sha256:0a69fbf0...2107a545`, whose embedded + `alpha.binance.paper.stable` route is manifest revision `12`; both Stream + roles remain on `qdl-v2-python:2.0.26-e4fc241@sha256:f2489160...957b6ad7`, + whose embedded route is revision `10`. They share the same runtime mount, + but public data-plane authorization is image/config-derived, so accepting a + revision-12 token in Query cannot make the revision-10 Stream accept it. + This is an incomplete reader rollout and must be repaired before certification. + + **Approved bounded correction.** Serially recreate only + `stream_v2_active`, then `stream_v2_passive`, using the exact healthy Query + digest `sha256:0a69fbf0c883cad27a433ea529555269ab7386706de6e1c635fa5fef2107a545`, + retaining the same Compose base, runtime/TLS/state mounts, ports, network and + environment. Verify each role is healthy, `restart=0`, `OOMKilled=false` and + advertises revision `12` before moving to the next. Rollback is each role + alone to `sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7` + with its existing configuration. Do not change Query, Rust, ingestors, + projectors, V1, Kafka offsets/topology, Redis, SQLite, Trading System, alpha + or order path. The replacement client uses the same current identity bundle + that passed preflight; this correction is prerequisite to the single valid + C2 result, not a new certification phase. + + **Serial Stream convergence (`PASS / VALID C2 NEXT`, 2026-09-21).** The + guarded packet first ran Compose `config --quiet` using the sealed existing + env-file, then recreated only `stream_v2_active` followed by + `stream_v2_passive`. Both now run + `sha256:0a69fbf0c883cad27a433ea529555269ab7386706de6e1c635fa5fef2107a545`, + remain `healthy`, `restart=0`, `OOMKilled=false`, retain the exact + `r135-b2-335792a-20260920T015057Z/runtime` mount, and expose + `alpha.binance.paper.stable` revision `12`. Before/after provenance is under + `.../r135-lossless-l2-e72905e-20260921T181741Z/evidence/stream-roll/`. + Compose warned about three existing orphan projector containers but the packet + did not pass `--remove-orphans` and did not touch them. The image-only + override is retained at `/tmp/qdl-r135-e72905e-stream-revision.override.yml` + until R1.35-D replaces it with a durable release-config path; it must not be + removed while the two active Stream roles still reference it. Next: use the + current `r135-local-batch-fair` identity/acceptance-input bootstrap and run + the single valid C2 300-second no-order acceptance. No further preflight or + retry is authorized before that result. + + **Final all-scope C2 (`PASS / R1.35-C EXIT`, 2026-09-21).** The one valid + foreground, authenticated, no-order acceptance used the exact current + identity/bootstrap bundle, the active-image route + `/app/config/v2/stable-v2-release-routing.yaml`, public V2 Query/Stream SDK + paths and the committed `e72905e` lossless-L2 harness mounted read-only. It + returned `PASS_V2_DATA_PLANE_ONLY` for all `299` active V2 products: `234` + durable and `65` on-demand, with `157` Binance USD-M and `142` OKX Swap + products. Opening covered `299/299` products in `903.722s` inside its + manifest-derived `935s` budget; observation was `300.089s` for the declared + `300s`; closing read `299/299` completed in `30.111s`; total elapsed was + `1233.925s`, including `34.165s` of manifest quota-window pacing. + + The receipt records `order_actions=0`, `provider_connections=0`, no direct + provider/fallback action, no alpha signal/sizing or broker mutation, + `cursor_directory_removed=true`, `secret_values_recorded=false` and + `test_provenance=false`. Its bounded fallback drill passed all seven + manifest-permitted `TRADE` routes through `V2_PRIMARY -> V1_FALLBACK -> + V2_PRIMARY`, with zero order/provider action and cursor cleanup; no + `BLOCKED` route was downgraded. Closing used `BATCH_V2_PRIMARY` for `194` + history/latest products and `L2_STATUS_SNAPSHOT` for all `40` lossless L2 + products. The remaining `65` on-demand rows correctly have no closing + durable-read transport. + + Consumer-call-to-SDK-usable latency is retained per binding/replica in the + receipt (hash `6cbbbac45c7f8f9e321677499dc698addd5cce62f74201f261a1dce68a5b53aa`). + The compact per-feed values below are `p50/p95/p99/max` milliseconds for + Query primary then secondary under the 299-product concurrent C2 workload; + they are request latency, not venue event age or final-bar close latency: + + | Feed | n | Primary ms | Secondary ms | + | --- | ---: | --- | --- | + | `BAR` | 150 | 1830.597 / 3179.390 / 3489.862 / 3812.450 | 1245.510 / 2023.475 / 2192.539 / 2301.537 | + | `BASIS` | 5 | 886.705 / 888.447 / 888.447 / 893.538 | 891.183 / 893.159 / 893.159 / 1334.722 | + | `BOOK_DELTA` | 20 | 883.495 / 1339.858 / 1339.858 / 4021.283 | 445.496 / 1514.821 / 1514.821 / 1538.067 | + | `BOOK_SNAPSHOT` | 20 | 821.371 / 1484.512 / 1484.512 / 1569.164 | 433.237 / 1580.740 / 1580.740 / 1846.667 | + | `CONTRACT_METADATA` | 10 | 1499.290 / 1681.382 / 1681.382 / 1681.382 | 1502.754 / 1811.357 / 1811.357 / 1811.357 | + | `FUNDING_RATE` | 10 | 1499.290 / 1681.382 / 1681.382 / 1681.382 | 1502.754 / 1811.357 / 1811.357 / 1811.357 | + | `LONG_SHORT_RATIO` | 5 | 942.118 / 1499.290 / 1499.290 / 1499.290 | 937.146 / 1502.754 / 1502.754 / 1502.754 | + | `MARK_INDEX_PRICE` | 20 | 295.827 / 1681.382 / 1681.382 / 1681.382 | 223.773 / 1811.357 / 1811.357 / 1811.357 | + | `OPEN_INTEREST` | 10 | 1499.290 / 1681.382 / 1681.382 / 1681.382 | 1502.754 / 1811.357 / 1811.357 / 1811.357 | + | `QUOTE` | 20 | 675.120 / 1632.278 / 1632.278 / 1714.665 | 617.633 / 1779.428 / 1779.428 / 1781.658 | + | `TAKER_FLOW` | 5 | 942.118 / 1499.290 / 1499.290 / 1499.290 | 937.146 / 1502.754 / 1502.754 / 1502.754 | + | `TRADE` | 24 | 1170.872 / 1681.358 / 1737.660 / 1793.812 | 889.734 / 1777.627 / 1778.117 / 1778.972 | + + At completion the C2 client captured `94` millicores and `261300224` bytes + RSS. The two Query and two Stream readers all run + `qdl-v2-python:2.0.26-137633b@sha256:0a69fbf0c883cad27a433ea529555269ab7386706de6e1c635fa5fef2107a545`, + are `healthy`, restart `0`, not OOM-killed and retain the sealed + `r135-b2-335792a-20260920T015057Z/runtime` mount. Bounded evidence is under + `/home/bobby/.local/state/qdl-v2/r135-lossless-l2-e72905e-20260921T181741Z/evidence/c2-valid/`. + The two launcher/provenance aborts above remain explicitly non-acceptance + provenance; the receipt here is the only C2 certificate. R1.35-C is closed: + no further data correctness matrix or C2 retry is required. The sole + remaining release work is R1.35-D: durable replacement of the temporary + Stream override, scoped test-artifact inventory/cleanup, source-image + reconciliation, CI/PR and an explicitly approved release packet. + +#### R1.35-D - Hygiene, source reconciliation and immutable stable release (`PENDING / REQUIRES R1.35-C EXIT`) + +**Goal.** Make source, runtime and published release refer to one auditable +revision, and remove only disposable test artifacts without touching market +data durability or active consumers. + +**Scoped test-container cleanup packet (`APPROVED / PRE-MUTATION`, 2026-09-21).** +The post-C2 inventory found exactly two leaked disposable source-test +containers: `lucid_sinoussi` and `youthful_shamir`. Both are `restart=no`, +read-only, `network=none`, mount only +`/home/bobby/.worktrees/data-layer-mark-index-live-view:/src`, and have been +stuck running their isolated Python unittest commands for about two days. They +are not Compose services, carry no volume/state mount, and are not in the +canonical V2 service set. `qdl-admit-1d` is already absent. Pre-cleanup Docker +inventory is `67` images / `41.34GB` (`27.4GB` reclaimable), `59` containers, +`16` volumes / `86.84GB`, and BuildKit `35.08GB` (`5.529GB` reclaimable). + +The approved mutation is only `docker rm -f lucid_sinoussi youthful_shamir`. +It stops/removes neither image nor volume/network; in particular the old +`qdl-v2-python:2.0.20-95d9595` image remains retained because active +`binance_bar_edge` still references it. Retain active reader +`sha256:0a69fbf0...2107a545`, Query rollback `sha256:3b065436...665c6ee` and +Stream rollback `sha256:f2489160...957b6ad7`. No `system prune`, image removal, +BuildKit prune, Compose down, V1/Kafka/Redis/SQLite/runtime configuration or +serving-role action is authorized by this packet. After removal, re-read the +four readers and disk inventory before deciding any further cleanup. + +**Scoped test-container cleanup result (`PASS`, 2026-09-21).** Exactly the two +approved containers were stopped and removed. A post-action lookup returns no +matching container. All four reader roles remain `healthy`, restart `0` and +not OOM-killed on `sha256:0a69fbf0...2107a545`; no serving role was recreated. +Docker container inventory changed only from `59` to `57`; images remained +`67` / `41.34GB` with `27.4GB` reclaimable, and BuildKit remained `35.08GB` +with `5.529GB` reclaimable. The tiny test container writable layers were not a +meaningful disk source; no image, cache, volume, network, V1 or runtime state +was removed. Further cleanup is intentionally deferred until active-image and +override provenance is normalized, so no named rollback artifact is mistaken +for disposable cache. + +**Durable Stream override reconciliation packet (`APPROVED / PRE-MUTATION`, +2026-09-21).** Both active Stream roles are healthy on the same `137633b` +digest as Query, but Docker labels show their last image override at temporary +`/tmp/qdl-r135-e72905e-stream-revision.override.yml`. Its content is only the +two exact Stream image pins and its SHA-256 is +`01e341ecb94a9b037500d49f0c76d22fcda5f771d28194a5abb4a4fb911c3711`; it holds +no environment value, credential, volume or topology change. Leaving an active +runtime dependent on `/tmp` violates release provenance even though the data +plane is currently healthy. + +The bounded repair copies that exact file, mode `0644`, to the external durable +release-state path +`/home/bobby/.local/state/qdl-v2/releases/r1.35-137633b/stream-reader-image.override.yml`. +It then resolves the existing Compose chain with only that final filename +substituted and requires `docker compose config --quiet` success plus equality +of the two resolved Stream service definitions apart from compose-label source +path. Serially recreate only `stream_v2_active`, then `stream_v2_passive`, +using the existing sealed env-file, all existing durable override files and the +same active image `sha256:0a69fbf0...2107a545`. Verify per role: exact digest, +same mounts/ports/TLS/runtime, `healthy`, restart `0`, not OOM-killed and +revision `12`; verify the untouched peer before moving on. A failed check +recreates only the changed Stream role with the retained temporary override and +same `0a69...` image, then blocks release. This is a path/provenance repair, +not a binary or behavior change, so it does not consume another C2. + +Do not alter Query, Rust core, ingestors, bar edge, projectors, V1, Kafka +topology/offsets, Redis, SQLite, Trading System, alpha, identities or order +path. Retain the temporary file until both roles pass; only then remove it and +write the durable file hash/config-hash into the release ledger. No broad +historical override consolidation is in this packet: existing state paths are +already durable and their full ordered chain remains recorded in Docker labels +and the final release provenance inventory. + +**Durable Stream override reconciliation result (`PASS`, 2026-09-21).** The +durable file was copied byte-for-byte with the expected SHA-256 +`01e341ecb94a9b037500d49f0c76d22fcda5f771d28194a5abb4a4fb911c3711`. A +read-only Compose resolution using the existing 24-file chain and only the +final filename substitution passed `config --quiet`; the normalized two-Service +definition hash was equal before/after: +`83ce54046e48575504a6a8bd6c565f7c58d04ce61b37a4b9cf139bf4512062cc`. +Exactly `stream_v2_active` then `stream_v2_passive` were serially recreated. +Both retain the same sealed env-file, mounts, TLS/runtime directory and +`sha256:0a69fbf0...2107a545` image, are `healthy`, restart `0` and not +OOM-killed; their Docker labels now name the durable release-state file and no +container label names the former `/tmp` path. The temporary file was then +removed. The unchanged Query pair also remains `healthy`, restart `0` and not +OOM-killed. Compose reported existing projector orphans but `--remove-orphans` +was never used and none was touched. This does not alter the C2 certificate or +require another C2 because the resolved service definitions were byte-identical +and only the external source filename changed. + +**Component-source reconciliation and release-artifact scope (`APPROVED / +SOURCE-ONLY`, 2026-09-21).** The proposed stable coordinate is `v2.0.26`, +matching the immutable V2 reader/projector image family already certified in +this closure. This is intentionally a component-attested release, not a +pointless full-image rebuild: active Query/Stream image source `137633b` is an +ancestor of the candidate branch, and `137633b..HEAD` changes only the +lossless-L2 acceptance harness, its tests and this journal. It contains no +`qdl/`, `qdl_sdk/`, Rust, config, contract, consumer or Compose serving-code +change. The active Rust core remains `f1c9e1d`; its later Rust delta is only +the pure `qdl-core::quality` golden/parity module. `qdl-realtime-core` and +provider admission source have no post-image delta and do not call that module +at runtime. The current Python projector source is `987b1a2` and its own +entrypoint/projector implementation has no later serving delta; the Binance +bar-edge entrypoint/path remains unchanged after `95d9595`. + +The source-only artifact slice adds a bounded `v2.0.26` release certificate, +notes and scope summary containing only release coordinate, component image +digests/source revisions, C2 receipt hash/counts/latency semantics, declared +V1 exclusions and rollback references. It must not copy runtime evidence, +credentials, raw market data, book levels, external state or mutable logs into +Git. Validate JSON/Markdown structure and the release workflow's required +`status=PASS` and notes paths before committing. No image build, tag, push, +merge, runtime mutation, cleanup or broker/consumer action belongs to this +source-only slice; CI and remote release await its committed SHA. + +**Component-attested release artifact result (`PASS / CI NEXT`, 2026-09-21).** +Added `upgrade/evidence/releases/v2.0.26/certificate.json`, +`scope-evidence.json` and `RELEASE_NOTES.md`. The machine certificate records +only the exact `299`-product C2 receipt hash, component image/source digests, +two named reader rollback digests, bounded resource/fallback facts and declared +DNSE/Spot boundaries. Both JSON documents passed `python3 -m json.tool`; the +same local condition enforced by the release workflow passed: certificate +`status=PASS` and release notes are present. Component-diff checks confirmed +the source reconciliation stated above. No new image was built, no tag/push/ +merge occurred, and no role, data-plane, credential, state, V1, broker or +consumer object changed in this source-only slice. The next gate is CI on the +committed feature SHA, followed by the normal feature -> `dev` release PR. + +**CI diagnostic and narrow correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-21).** PR `#20` for `fix/mark-index-live-view` reached the normal +feature -> `dev` CI gate. `sdk-python310` passed; `contract-tests` stopped at +the pinned Rust 1.82 Clippy step before contract publication, and `unit-tests` +stopped in its runtime dependency audit before any unit suite ran. Reproducing +the exact contract command locally isolates the first issue to a test-only +assertion in `rust/qdl-realtime-core/src/lib.rs`: `ProcessBatch` stores +`Option<&'static str>`, so calling `as_deref()` is a no-op that Clippy rejects +under `-D warnings`. The bounded correction replaces only that no-op assertion +expression, then re-runs the pinned format/Clippy/test command. It changes no +provider, contract, runtime, image, manifest, C2 result or data-plane state. +The dependency-audit failure is investigated separately with the CI's exact +runtime image and must be resolved or explicitly pinned before CI/release may +continue; no later unit/contract gate is treated as passed until then. + +The CI audit identifies `anyio 4.13.0` only, with `CVE-2026-63374` and +`CVE-2026-64847`, both fixed by `4.14.2`. `fastapi 0.136.1` and +`starlette 1.6.0` already constrain AnyIO to the compatible `<5` line, so the +approved source-only security correction is a lockfile refresh from `4.13.0` +to a fixed supported release using the Dockerfile-pinned Poetry `2.3.4` +environment. The resolver selected `anyio 4.15.1` and its compatible +transitive `typing-extensions 4.16.0`; no direct application constraint +changed. It must not change any runtime role, image or live state. The exact +CI dependency audit and relevant Python/Rust tests must pass before this +correction is committed. + +The first CI rerun also exposed a workflow-only infrastructure failure before +any contract command: `bufbuild/buf-setup-action` resolved its pinned Buf +binary anonymously and GitHub returned API rate-limit exhaustion. The narrow +correction passes the ephemeral built-in `${{ github.token }}` through the +action's supported `github_token` input. It does not add a repository secret, +relax Buf format/lint/breaking/generation/Rust gates, alter contracts or change +runtime behavior. The workflow's normal token scope is read-only for this +download operation. + +**Local CI correction evidence (`PASS / REMOTE CI NEXT`, 2026-09-21).** A +Dockerfile-equivalent temporary image built from the corrected lockfile and +passed the exact runtime dependency audit: Poetry/venv has no Poetry binary, +the Uvicorn shebang remains `/opt/venv/bin/python`, `msgpack >= 1.2.1`, +`setuptools >= 78.1.1`, `anyio >= 4.14.2`, and `pip-audit` returns no known +vulnerability. The pinned Rust 1.82 builder passed format, `clippy -D +warnings`, and the locked workspace test command after the one assertion +correction. An isolated `qdl-r135-ci` invocation of CI's full unittest command +ran to automatic `--rm` completion; the local Compose client did not retain an +exit receipt after auto-removal, so it is explicitly not counted as a local +full-suite PASS. The authoritative full suite remains the pending GitHub CI +rerun. Its still-running dependency containers are isolated under the exact +`qdl-r135-ci` project and await explicit cleanup authorization. + +Because this is a runtime lockfile change, the previously certified +`sha256:0a69fbf0...2107a545` reader remains a valid pre-security-fix rollback +and functional baseline, but cannot be the final release image. Once remote CI +is green, build one immutable reader image from the corrected commit, roll only +`query_v2_1`, `query_v2_2`, `stream_v2_active`, and `stream_v2_passive` with +the current image as rollback, run the affected no-order acceptance, and only +then tag/publish `v2.0.26`. This does not reopen data correctness discovery or +alter Rust/ingestors/projectors/V1/Kafka/Redis/SQLite/Trading System/alpha. + +**Full-suite source-contract reconciliation (`IN PROGRESS / SOURCE-ONLY`, +2026-09-21).** The first remote CI rerun after the audit and Buf-token fixes +passed `contract-tests` and `sdk-python310`, then ran the full unit suite and +reported `10` failures plus `11` errors. This is a release-blocking source +consistency finding, not a new provider, C2, freshness, routing or runtime +failure. Inspection narrows it to three shared declaration/tooling mismatches: + +1. The stable compiler legitimately emits two physical OKX `MARK_INDEX` + components (`MARK` and `INDEX`) under one logical `source_id`. Rust + `RealtimeCoreConfig::validate` admits this exact pair while rejecting a + duplicate component or ordinary/mark-index source collision. Three Python + runtime-refresh/convergence scripts still require every `source_id` to be + globally unique, so their own current compiler output fails their safety + checks. +2. The native-ingestor compiler already emits `MARK_INDEX` rows and active + ingestor runtime contains them, but the refresh utility's stale allow-list + rejects that feed before it can validate an otherwise unchanged config. +3. Historical tests still assert a six-slice/Spot inventory and a literal + `130` materialization count. The sealed current demand is intentionally + derivative-only for five Binance USD-M plus five OKX Swap symbols; the + retained broader catalog is capability/compatibility inventory, not an + instruction to activate Spot. + +Approved implementation is limited to: one shared Python representation of +the Rust core binding identity/duplicate rule, use of that rule by the three +offline refresh/convergence tools, the native-ingestor feed projection, and +tests derived from the sealed demand rather than historic literals. It must +not alter catalog/demand/runtime files, public endpoints, provider adapters, +Rust serving behavior, images, roles, quotas, V1, Kafka, Redis, SQLite, +consumers, orders or credentials. Required exit gates are focused semantic +regressions (including MARK+INDEX accepted; repeated component and +ordinary/collision rejected), the affected deployment/catalog/demand suites, +then the full CI suite. A later immutable reader image and affected C2 remain +required because the already committed dependency-lock security correction +changes the released Python artifact; this source slice itself does not roll +anything. + +**Source-contract correction and focused matrix (`PASS / FULL CI NEXT`, +2026-09-21).** Added the small shared Python core-binding identity validator +used only by offline runtime-refresh/convergence tooling. It preserves Rust's +physical-key uniqueness, ordinary/mark-index source exclusion and exact +`BOTH` versus `MARK`+`INDEX` component-completeness rule. The three rollout +tools now use that identity instead of global `source_id` uniqueness; native +ingestor identity remains logical for ordinary feeds and physical for the +paired `MARK_INDEX` feed, so a changed TRADE/QUOTE channel is still semantic +drift rather than a hidden new subscription. The Phase 10 read-only admission +fixture now validates a Binance paired mark/index response and both real OKX +mark/index endpoint shapes under one logical requirement. Catalog and demand +regressions now compare the active sealed derivative scope with the retained +capability inventory instead of treating inactive Spot rows or a historic +`130` count as activation. + +In the existing CI image with read-only source mount, `--network none`, +non-root UID, tmpfs `/tmp` and automatic removal, the focused source matrix +passed `64/64`: core identity `4`, Rust refresh `5`, L2 refresh `7`, native +ingestor refresh `5`, primary convergence `3`, catalog/demand `17`, native +BAR materialization `10`, and Phase 10 universal/provider-admission `13`. +Host Python lacks the repository's runtime dependencies and was used only for +syntax checks; it is explicitly not evidence. No image, runtime role, +provider connection, Kafka/Redis/SQLite state, V1, consumer, alpha or order +path changed. Next is one fresh isolated CI image build followed by the full +CI unittest suite; only a green full suite permits feature-branch commit/CI +publication and the later four-reader release packet. + +**CI scanner image-provenance correction (`IMPLEMENTED / REMOTE CI NEXT`, +2026-09-21).** Remote run `35652804355` on source `dd7cc6d` completed every +unit, contract, SDK, migration, replay, performance and release-manifest gate +successfully. Its sole failure was the final Trivy image scan, which reported +`anyio 4.13.0` even though the exact PR merge-tree lockfile, Docker build log, +and a fresh local image inspection all prove `anyio 4.15.1`; the builder log +explicitly installs `anyio 4.15.1`. An exact local Trivy `0.70.0` reproduction +identified the real cause: the final image contains archived historical SBOMs +under `/app/upgrade/evidence/`, and two of those records still describe the +old `anyio 4.13.0` release. Runtime package metadata contains only +`anyio-4.15.1.dist-info`. This is neither a runtime CVE waiver nor a mutable +tag explanation. + +The narrow repair excludes `upgrade/evidence/` from the runtime Docker build +context; source-mounted tests and offline certification scripts retain those +Git-tracked records, while serving images no longer carry obsolete test/release +metadata. The image produced immediately after CI build also receives a unique +`data-layer-ci:${GITHUB_SHA}` local tag, verifies the fixed AnyIO floor from +that exact image, and makes both Trivy and the ephemeral SBOM/release rehearsal +use that SHA-bound tag. The mutable compatibility tag remains available for +legacy test scripts only. No vulnerability is ignored, no dependency is +downgraded, and no provider/runtime/catalog/consumer/data-plane behavior +changes. The required evidence is a rebuilt local scan with no high/critical +finding, then one green GitHub rerun where the image audit, Trivy, +secret/misconfiguration scan, and release rehearsal all complete from the same +SHA-bound image. Only then may R1.35-D progress to the separately approved +four-reader image packet. + +**CI scanner image-provenance source check (`PASS / REMOTE CI NEXT`, +2026-09-21).** The workflow tag/audit edit parsed with PyYAML, +`git diff --check` passed, and the exact runtime assertion returned +`anyio=4.15.1`; the independent Trivy reproduction then correctly prevented a +false closure by showing the stale SBOM files were still in the image. The +`.dockerignore` change is deliberately limited to historical evidence. + +**CI scanner rebuilt-image test (`PASS / REMOTE CI NEXT`, 2026-09-21).** A +disposable `qdl-r135-scan:ci-evidence-closure` image was built from this exact +source context after the evidence exclusion. It proved `/app/upgrade/evidence` +absent, runtime `anyio=4.15.1`, and Trivy `0.70.0` reported `0` +high/critical findings (the language-metadata target count fell from four to +two). The exact test image and its temporary Trivy binary/cache directory were +removed automatically; post-check found neither. No serving image, container, +role, provider, Kafka/Redis/SQLite object, V1, consumer, alpha or order path +changed. The clean GitHub runner remains the release authority. + +**Pinned Rust advisory-policy runner correction (`IN PROGRESS / SOURCE-ONLY`, +2026-09-21).** The subsequent remote run `35655156356` passed schema format, +breaking checks, generated Python/Rust contracts, Rust format/Clippy and every +locked Rust test. Its `cargo-deny` step did not reach license/source/advisory +policy: GitHub-host Rustup attempted to re-install the `1.82` Clippy component +and failed on an existing `bin/cargo-clippy` conflict. This is host-toolchain +drift, not a Rust dependency finding. The narrow correction runs the already +checksum-verified static `cargo-deny` binary in the same pinned +`rust:1.82-slim` container used for the preceding Rust contract gate, mounting +the workspace and extracted binary read-only and installing only `git` in that +ephemeral runner because the reviewed policy explicitly sets +`git-fetch-with-cli=true`. It preserves the actual `cargo-deny check` policy +and fails closed on any advisory/license/source/bans result. No Rust +dependency, lockfile, provider, runtime or data-plane code changes. +Required exit: the exact containerized check passes locally, then the clean +GitHub runner passes the complete contract job. + +**Pinned Rust advisory-policy runner test (`PASS / REMOTE CI NEXT`, +2026-09-21).** The checksum-verified `cargo-deny 0.20.2` command ran in the +ephemeral pinned Rust `1.82` container with a read-only workspace and passed +all four policy sections: `advisories`, `bans`, `licenses`, and `sources`. +The lockfile reports three duplicate-version warnings that the existing policy +intentionally classifies as warnings; no exception or ignore was added. The +temporary downloaded binary directory was removed on exit. No image, source +dependency, runtime role or data-plane state changed in this check. + +**Runtime-image versus CI-fixture separation (`IN PROGRESS / SOURCE-ONLY`, +2026-09-21).** The same run showed `19` unit-test errors after the serving +image correctly excluded `upgrade/evidence/`: legacy frozen-evidence contract +tests deliberately read those Git-tracked fixtures from `/app/upgrade/evidence`. +The failure proves the final image is leaner, not that a product contract is +invalid. The repair keeps evidence excluded from all serving image contexts and +adds one read-only bind of exactly `./upgrade/evidence` to the CI +`test_runner`; it does not mount source over application code, add the +directory to Query/Stream roles, or change any runtime Compose service. The +exit gate is resolved Compose mount identity plus targeted frozen-evidence +contract tests against an evidence-free disposable image; full CI then remains +the authority. + +**Runtime-image versus CI-fixture separation test (`PASS / REMOTE CI NEXT`, +2026-09-21).** Compose `config --quiet` passed and resolves `test_runner` with +exactly one bind: this worktree's `upgrade/evidence` to +`/app/upgrade/evidence`, read-only. A disposable image built after the +evidence exclusion proved the directory absent before the mount; with only that +mount, all previously affected frozen-evidence modules passed `55/55` in +`1.367s`. The exact test image was removed after the run. No source mount +overlaid `/app`, no service started, and no serving role, provider, durable +state, V1, consumer, alpha or order path changed. + +**CI fixture-contract assertion repair (`APPROVED / SOURCE-ONLY`, +2026-09-21).** Remote run `35656395049` executed `1793` tests and failed only +`ReleaseBundleTests.test_runtime_image_is_non_root_and_trivy_waiver_is_narrow`: +the legacy test counts raw `volumes: !reset []` YAML tokens and still expects +four, while the deliberate `test_runner` fixture uses Compose `!override` to +replace its inherited volume list with exactly the frozen evidence mount. The +approved narrow repair changes that assertion to verify the intended Compose +semantics: the three non-runner test services retain reset volumes and +`test_runner` has the one read-only `./upgrade/evidence` fixture mount. It does +not weaken image isolation, re-add evidence to serving images, change a runtime +role, or consume C2. Exit: the focused test and resolved Compose contract pass +locally, then one full GitHub CI run is green. + +**CI fixture-contract assertion result (`PASS / REMOTE CI NEXT`, +2026-09-21).** The test now asserts the actual Compose contract rather than a +legacy raw-token count: exactly three non-runner CI services reset inherited +volumes, while `test_runner` uses `!override` and exposes only +`./upgrade/evidence:/app/upgrade/evidence:ro`. `docker compose -f +docker-compose.yml -f docker-compose.ci.yml config --quiet` passed; the +disposable, `--network none`, read-only, non-root image test +`python -m unittest tests.test_fund_phase6_release` passed `3/3` in `0.135s`. +It created no Compose service, provider session, durable data, runtime change +or order action. The full GitHub unit suite remains the release authority. + +**Full source CI result (`PASS / RELEASE-IMAGE NEXT`, 2026-09-21).** GitHub +Actions run `35657755130` for source `580ce7d6ff05dae2f5accb6a5e3ba8102911554e` +completed successfully. `unit-tests`, `contract-tests`, and `sdk-python310` +are all green; the unit job completed all declared checks, including the +`1793`-test unit suite, V2 contract/SDK and shadow-consumer gates, Phase 6/7 +gates, isolated PostgreSQL/Redis recovery, bounded replica/load regressions, +immutable Rust replay build/provider fixtures, Trivy, secret scan and immutable +SBOM/release-manifest rehearsal. This proves the source/CI boundary only. It +does not alter the active reader image, consume a second C2, or certify a new +runtime image. Next is one immutable reader image built from the exact merged +`dev` source lineage, followed by the already-scoped four-reader rollout and +one affected no-order acceptance. + +**Required closure sequence.** + +1. Record each R1.35 phase result, exact commands, test counts, evidence paths, + runtime mutation and cleanup in this journal. A failed or skipped required + gate blocks the release; it is not relabelled technical debt. +2. Inspect and stop/remove only the exact leaked test containers + `lucid_sinoussi`, `youthful_shamir` and `qdl-admit-1d` after confirming their + commands, mounts, no-restart policy and absence from the canonical Compose + service set. Do not remove their images until no remaining container uses + them. This is a scoped operational cleanup requiring its own execution + approval; it must not touch V1, Kafka, Redis, SQLite, volumes, networks or + serving roles. +3. Inventory images and BuildKit cache before/after. Retain active image + digests and one named rollback digest per changed role; remove only + unreferenced R1.35 client/test images and unreferenced build cache. Record + disk/inode measurements and post-cleanup runtime health. No broad prune or + volume deletion is part of this phase. +4. Verify worktree and staged scope, `git diff --check`, user Git identity, + plan/evidence completeness and all CI gates. Commit each coherent tested + source slice with the user identity. +5. Push the feature branch, open/green PR to `dev`, merge only after required + CI passes, and build an immutable release candidate from the exact `dev` + SHA. Roll out only the formally approved role/digest packet and repeat the + affected C2 matrix from that candidate. If the source/image SHA changes, + repeat affected acceptance rather than inheriting evidence. +6. Only after `dev` certificate, source/runtime reconciliation and rollback + drill pass: merge `dev -> main`, tag the exact main commit with the next + semantic release version, build/attest the immutable tag image, verify the + tag resolves to the certified tree/digest, publish the release certificate + and release notes, and synchronize canonical local `main`/`dev` from their + remotes. Never claim a phase-worktree image name as the stable product name. + +**Final release gate.** The new V2 release is certified only when R1.35-A/B/C +are all `PASS`, cleanup has exact evidence, canonical `dev` and `main` contain +the same certified source lineage, the running role digest/config revision is +recorded, V1 rollback is proven available, and the public release tag, +certificate and consumer-facing endpoint/latency matrix agree. Trading System +work may consume the certified V2 contract afterward, but its independent OMS, +risk and broker certification remains a separate program. + +**Status and debt.** `PENDING`. No R1.35 phase may close with an in-scope +quality, latency, binding, test-cleanup or provenance gap. Intentional product +exclusions are release-boundary declarations, not hidden technical debt. diff --git a/config/v2/stable-acquisition-bindings.yaml b/config/v2/stable-acquisition-bindings.yaml index 9d7ac64..0bf9dfc 100644 --- a/config/v2/stable-acquisition-bindings.yaml +++ b/config/v2/stable-acquisition-bindings.yaml @@ -13,8 +13,8 @@ bindings: sequence_policy: NONE websocket_url: wss://stream.binance.com:9443/ws business_websocket_url: null - market_websocket_url: wss://stream.binance.com:9443/ws enabled: false + market_websocket_url: wss://stream.binance.com:9443/ws - binding_id: binance-spot-btcusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -23,8 +23,8 @@ bindings: sequence_policy: MONOTONIC websocket_url: wss://stream.binance.com:9443/ws business_websocket_url: null - market_websocket_url: wss://stream.binance.com:9443/ws enabled: false + market_websocket_url: wss://stream.binance.com:9443/ws - binding_id: binance-spot-btcusdt-trade mode: RUST_NATIVE runtime: BINANCE @@ -33,8 +33,8 @@ bindings: sequence_policy: MONOTONIC websocket_url: wss://stream.binance.com:9443/ws business_websocket_url: null - market_websocket_url: wss://stream.binance.com:9443/ws enabled: false + market_websocket_url: wss://stream.binance.com:9443/ws - binding_id: binance-usdm-bnbusdt-bar-12h mode: PYTHON_REST runtime: BINANCE @@ -169,13 +169,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-bnbusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -184,13 +184,27 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws +- binding_id: binance-usdm-bnbusdt-mark_index_price + mode: RUST_NATIVE + runtime: BINANCE + provider_kind: binance_usdm_mark_index + native_channel: bnbusdt@markPrice@1s + sequence_policy: NONE + websocket_url: wss://fstream.binance.com/public/ws + business_websocket_url: null + mark_index: + provider_protocol: BINANCE_MARK_PRICE + index_native_symbol: null + component_quiet_after_ms: + BOTH: 5000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-bnbusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -217,12 +231,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-btcusdt-260925-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -231,12 +245,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-btcusdt-261225-book_delta mode: RUST_NATIVE runtime: BINANCE @@ -245,12 +259,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-btcusdt-261225-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -259,12 +273,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-btcusdt-bar-12h mode: PYTHON_REST runtime: BINANCE @@ -399,13 +413,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-btcusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -414,13 +428,27 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws +- binding_id: binance-usdm-btcusdt-mark_index_price + mode: RUST_NATIVE + runtime: BINANCE + provider_kind: binance_usdm_mark_index + native_channel: btcusdt@markPrice@1s + sequence_policy: NONE + websocket_url: wss://fstream.binance.com/public/ws + business_websocket_url: null + mark_index: + provider_protocol: BINANCE_MARK_PRICE + index_native_symbol: null + component_quiet_after_ms: + BOTH: 5000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-btcusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -573,13 +601,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-dogeusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -588,13 +616,27 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws +- binding_id: binance-usdm-dogeusdt-mark_index_price + mode: RUST_NATIVE + runtime: BINANCE + provider_kind: binance_usdm_mark_index + native_channel: dogeusdt@markPrice@1s + sequence_policy: NONE + websocket_url: wss://fstream.binance.com/public/ws + business_websocket_url: null + mark_index: + provider_protocol: BINANCE_MARK_PRICE + index_native_symbol: null + component_quiet_after_ms: + BOTH: 5000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-dogeusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -621,12 +663,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-ethusdt-260925-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -635,12 +677,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-ethusdt-261225-book_delta mode: RUST_NATIVE runtime: BINANCE @@ -649,12 +691,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-ethusdt-261225-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -663,12 +705,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-ethusdt-bar-12h mode: PYTHON_REST runtime: BINANCE @@ -803,13 +845,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-ethusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -818,13 +860,27 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws +- binding_id: binance-usdm-ethusdt-mark_index_price + mode: RUST_NATIVE + runtime: BINANCE + provider_kind: binance_usdm_mark_index + native_channel: ethusdt@markPrice@1s + sequence_policy: NONE + websocket_url: wss://fstream.binance.com/public/ws + business_websocket_url: null + mark_index: + provider_protocol: BINANCE_MARK_PRICE + index_native_symbol: null + component_quiet_after_ms: + BOTH: 5000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-ethusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -977,13 +1033,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-solusdt-book_snapshot mode: RUST_NATIVE runtime: BINANCE @@ -992,13 +1048,27 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://fstream.binance.com/public/ws business_websocket_url: null - market_websocket_url: wss://fstream.binance.com/market/ws l2: provider_protocol: BINANCE_DIFF_DEPTH depth_per_side: 100 rest_snapshot_url: https://fapi.binance.com/fapi/v1/depth snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: wss://fstream.binance.com/market/ws +- binding_id: binance-usdm-solusdt-mark_index_price + mode: RUST_NATIVE + runtime: BINANCE + provider_kind: binance_usdm_mark_index + native_channel: solusdt@markPrice@1s + sequence_policy: NONE + websocket_url: wss://fstream.binance.com/public/ws + business_websocket_url: null + mark_index: + provider_protocol: BINANCE_MARK_PRICE + index_native_symbol: null + component_quiet_after_ms: + BOTH: 5000 + market_websocket_url: wss://fstream.binance.com/market/ws - binding_id: binance-usdm-solusdt-quote mode: RUST_NATIVE runtime: BINANCE @@ -1061,12 +1131,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-btc-usd-261225-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1075,12 +1145,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-btc-usd-270326-book_delta mode: RUST_NATIVE runtime: OKX @@ -1089,12 +1159,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-btc-usd-270326-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1103,12 +1173,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-eth-usd-261225-book_delta mode: RUST_NATIVE runtime: OKX @@ -1117,12 +1187,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-eth-usd-261225-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1131,12 +1201,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-eth-usd-270326-book_delta mode: RUST_NATIVE runtime: OKX @@ -1145,12 +1215,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-futures-eth-usd-270326-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1159,12 +1229,12 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 + market_websocket_url: null - binding_id: okx-spot-btcusdt-bar-1m mode: RUST_NATIVE runtime: OKX @@ -1173,8 +1243,8 @@ bindings: sequence_policy: NONE websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null enabled: false + market_websocket_url: null - binding_id: okx-spot-btcusdt-quote mode: RUST_NATIVE runtime: OKX @@ -1183,8 +1253,8 @@ bindings: sequence_policy: NONE websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null enabled: false + market_websocket_url: null - binding_id: okx-spot-btcusdt-trade mode: RUST_NATIVE runtime: OKX @@ -1193,8 +1263,8 @@ bindings: sequence_policy: MONOTONIC websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null enabled: false + market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-bar-12h mode: RUST_NATIVE runtime: OKX @@ -1329,13 +1399,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1344,13 +1414,28 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null +- binding_id: okx-swap-bnb-usdt-swap-mark_index_price + mode: RUST_NATIVE + runtime: OKX + provider_kind: okx_mark_index + native_channel: mark-price + sequence_policy: NONE + websocket_url: wss://ws.okx.com:8443/ws/v5/public + business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + mark_index: + provider_protocol: OKX_MARK_INDEX + index_native_symbol: BNB-USDT + component_quiet_after_ms: + MARK: 15000 + INDEX: 70000 + market_websocket_url: null - binding_id: okx-swap-bnb-usdt-swap-quote mode: RUST_NATIVE runtime: OKX @@ -1494,13 +1579,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null - binding_id: okx-swap-btc-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1509,13 +1594,28 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null +- binding_id: okx-swap-btc-usdt-swap-mark_index_price + mode: RUST_NATIVE + runtime: OKX + provider_kind: okx_mark_index + native_channel: mark-price + sequence_policy: NONE + websocket_url: wss://ws.okx.com:8443/ws/v5/public + business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + mark_index: + provider_protocol: OKX_MARK_INDEX + index_native_symbol: BTC-USDT + component_quiet_after_ms: + MARK: 15000 + INDEX: 70000 + market_websocket_url: null - binding_id: okx-swap-btcusdt-bar-1m mode: RUST_NATIVE runtime: OKX @@ -1677,13 +1777,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1692,13 +1792,28 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null +- binding_id: okx-swap-doge-usdt-swap-mark_index_price + mode: RUST_NATIVE + runtime: OKX + provider_kind: okx_mark_index + native_channel: mark-price + sequence_policy: NONE + websocket_url: wss://ws.okx.com:8443/ws/v5/public + business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + mark_index: + provider_protocol: OKX_MARK_INDEX + index_native_symbol: DOGE-USDT + component_quiet_after_ms: + MARK: 15000 + INDEX: 70000 + market_websocket_url: null - binding_id: okx-swap-doge-usdt-swap-quote mode: RUST_NATIVE runtime: OKX @@ -1851,13 +1966,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -1866,13 +1981,28 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null +- binding_id: okx-swap-eth-usdt-swap-mark_index_price + mode: RUST_NATIVE + runtime: OKX + provider_kind: okx_mark_index + native_channel: mark-price + sequence_policy: NONE + websocket_url: wss://ws.okx.com:8443/ws/v5/public + business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + mark_index: + provider_protocol: OKX_MARK_INDEX + index_native_symbol: ETH-USDT + component_quiet_after_ms: + MARK: 15000 + INDEX: 70000 + market_websocket_url: null - binding_id: okx-swap-eth-usdt-swap-quote mode: RUST_NATIVE runtime: OKX @@ -2025,13 +2155,13 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-book_snapshot mode: RUST_NATIVE runtime: OKX @@ -2040,13 +2170,28 @@ bindings: sequence_policy: CONTIGUOUS websocket_url: wss://ws.okx.com:8443/ws/v5/public business_websocket_url: wss://ws.okx.com:8443/ws/v5/business - market_websocket_url: null l2: provider_protocol: OKX_PUBLIC_BOOKS depth_per_side: 100 rest_snapshot_url: null snapshot_refresh_seconds: 30 materialized_snapshot_interval_ms: 1000 + market_websocket_url: null +- binding_id: okx-swap-sol-usdt-swap-mark_index_price + mode: RUST_NATIVE + runtime: OKX + provider_kind: okx_mark_index + native_channel: mark-price + sequence_policy: NONE + websocket_url: wss://ws.okx.com:8443/ws/v5/public + business_websocket_url: wss://ws.okx.com:8443/ws/v5/business + mark_index: + provider_protocol: OKX_MARK_INDEX + index_native_symbol: SOL-USDT + component_quiet_after_ms: + MARK: 15000 + INDEX: 70000 + market_websocket_url: null - binding_id: okx-swap-sol-usdt-swap-quote mode: RUST_NATIVE runtime: OKX diff --git a/config/v2/stable-authority-promotion-scope.yaml b/config/v2/stable-authority-promotion-scope.yaml index 218366c..202be88 100644 --- a/config/v2/stable-authority-promotion-scope.yaml +++ b/config/v2/stable-authority-promotion-scope.yaml @@ -1,5 +1,5 @@ schema: qdl.v2.authority-promotion-scope.v1 -revision: 7 +revision: 8 binding_ids: - binance-usdm-bnbusdt-bar-12h - binance-usdm-bnbusdt-bar-15m @@ -17,6 +17,7 @@ binding_ids: - binance-usdm-bnbusdt-bar-8h - binance-usdm-bnbusdt-book_delta - binance-usdm-bnbusdt-book_snapshot +- binance-usdm-bnbusdt-mark_index_price - binance-usdm-bnbusdt-quote - binance-usdm-bnbusdt-trade - binance-usdm-btcusdt-260925-book_delta @@ -39,6 +40,7 @@ binding_ids: - binance-usdm-btcusdt-bar-8h - binance-usdm-btcusdt-book_delta - binance-usdm-btcusdt-book_snapshot +- binance-usdm-btcusdt-mark_index_price - binance-usdm-btcusdt-quote - binance-usdm-btcusdt-trade - binance-usdm-dogeusdt-bar-12h @@ -57,6 +59,7 @@ binding_ids: - binance-usdm-dogeusdt-bar-8h - binance-usdm-dogeusdt-book_delta - binance-usdm-dogeusdt-book_snapshot +- binance-usdm-dogeusdt-mark_index_price - binance-usdm-dogeusdt-quote - binance-usdm-dogeusdt-trade - binance-usdm-ethusdt-260925-book_delta @@ -79,6 +82,7 @@ binding_ids: - binance-usdm-ethusdt-bar-8h - binance-usdm-ethusdt-book_delta - binance-usdm-ethusdt-book_snapshot +- binance-usdm-ethusdt-mark_index_price - binance-usdm-ethusdt-quote - binance-usdm-ethusdt-trade - binance-usdm-solusdt-bar-12h @@ -97,6 +101,7 @@ binding_ids: - binance-usdm-solusdt-bar-8h - binance-usdm-solusdt-book_delta - binance-usdm-solusdt-book_snapshot +- binance-usdm-solusdt-mark_index_price - binance-usdm-solusdt-quote - binance-usdm-solusdt-trade - okx-futures-btc-usd-261225-book_delta @@ -123,6 +128,7 @@ binding_ids: - okx-swap-bnb-usdt-swap-bar-6h - okx-swap-bnb-usdt-swap-book_delta - okx-swap-bnb-usdt-swap-book_snapshot +- okx-swap-bnb-usdt-swap-mark_index_price - okx-swap-bnb-usdt-swap-quote - okx-swap-bnb-usdt-swap-trade - okx-swap-btc-usdt-swap-bar-12h @@ -140,6 +146,7 @@ binding_ids: - okx-swap-btc-usdt-swap-bar-6h - okx-swap-btc-usdt-swap-book_delta - okx-swap-btc-usdt-swap-book_snapshot +- okx-swap-btc-usdt-swap-mark_index_price - okx-swap-btcusdt-bar-1m - okx-swap-btcusdt-quote - okx-swap-btcusdt-trade @@ -159,6 +166,7 @@ binding_ids: - okx-swap-doge-usdt-swap-bar-6h - okx-swap-doge-usdt-swap-book_delta - okx-swap-doge-usdt-swap-book_snapshot +- okx-swap-doge-usdt-swap-mark_index_price - okx-swap-doge-usdt-swap-quote - okx-swap-doge-usdt-swap-trade - okx-swap-eth-usdt-swap-bar-12h @@ -177,6 +185,7 @@ binding_ids: - okx-swap-eth-usdt-swap-bar-6h - okx-swap-eth-usdt-swap-book_delta - okx-swap-eth-usdt-swap-book_snapshot +- okx-swap-eth-usdt-swap-mark_index_price - okx-swap-eth-usdt-swap-quote - okx-swap-eth-usdt-swap-trade - okx-swap-sol-usdt-swap-bar-12h @@ -195,5 +204,6 @@ binding_ids: - okx-swap-sol-usdt-swap-bar-6h - okx-swap-sol-usdt-swap-book_delta - okx-swap-sol-usdt-swap-book_snapshot +- okx-swap-sol-usdt-swap-mark_index_price - okx-swap-sol-usdt-swap-quote - okx-swap-sol-usdt-swap-trade diff --git a/config/v2/stable-crypto-demand.yaml b/config/v2/stable-crypto-demand.yaml index a5d590e..60b5fee 100644 --- a/config/v2/stable-crypto-demand.yaml +++ b/config/v2/stable-crypto-demand.yaml @@ -4,41 +4,6 @@ consumers: - consumer_id: trading-system.paper.stable consumer_grade: EXECUTION requirements: - - venue: BINANCE - market: SPOT - product_type: SPOT - native_symbol: BTCUSDT - feed: BAR - interval: 1m - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: SPOT - product_type: SPOT - native_symbol: BTCUSDT - feed: QUOTE - interval: null - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: SPOT - product_type: SPOT - native_symbol: BTCUSDT - feed: TRADE - interval: null - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: BTCUSDT - feed: BAR - interval: 1m - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: BTCUSDT - feed: QUOTE - interval: null - source_policy_id: crypto_primary_v2 - venue: BINANCE market: USDM product_type: PERPETUAL @@ -46,20 +11,6 @@ consumers: feed: TRADE interval: null source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: ETHUSDT - feed: BAR - interval: 1m - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: ETHUSDT - feed: QUOTE - interval: null - source_policy_id: crypto_primary_v2 - venue: BINANCE market: USDM product_type: PERPETUAL @@ -70,93 +21,44 @@ consumers: - venue: BINANCE market: USDM product_type: PERPETUAL - native_symbol: SOLUSDT - feed: BAR - interval: 1m - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: BTCUSDT feed: QUOTE interval: null source_policy_id: crypto_primary_v2 - venue: BINANCE market: USDM product_type: PERPETUAL - native_symbol: SOLUSDT - feed: TRADE - interval: null - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: DOGEUSDT - feed: BAR - interval: 1m - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: ETHUSDT feed: QUOTE interval: null source_policy_id: crypto_primary_v2 - venue: BINANCE market: USDM product_type: PERPETUAL - native_symbol: DOGEUSDT - feed: TRADE - interval: null - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: BTCUSDT feed: BAR interval: 1m source_policy_id: crypto_primary_v2 - venue: BINANCE market: USDM product_type: PERPETUAL - native_symbol: BNBUSDT - feed: QUOTE - interval: null - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: BNBUSDT - feed: TRADE - interval: null - source_policy_id: crypto_primary_v2 - - venue: OKX - market: SPOT - product_type: SPOT - native_symbol: BTC-USDT + native_symbol: ETHUSDT feed: BAR interval: 1m source_policy_id: crypto_primary_v2 - venue: OKX - market: SPOT - product_type: SPOT - native_symbol: BTC-USDT - feed: QUOTE - interval: null - source_policy_id: crypto_primary_v2 - - venue: OKX - market: SPOT - product_type: SPOT - native_symbol: BTC-USDT + market: SWAP + product_type: PERPETUAL + native_symbol: BTC-USDT-SWAP feed: TRADE interval: null source_policy_id: crypto_primary_v2 - venue: OKX market: SWAP product_type: PERPETUAL - native_symbol: BTC-USDT-SWAP - feed: BAR - interval: 1m + native_symbol: ETH-USDT-SWAP + feed: TRADE + interval: null source_policy_id: crypto_primary_v2 - venue: OKX market: SWAP @@ -168,14 +70,14 @@ consumers: - venue: OKX market: SWAP product_type: PERPETUAL - native_symbol: BTC-USDT-SWAP - feed: TRADE + native_symbol: ETH-USDT-SWAP + feed: QUOTE interval: null source_policy_id: crypto_primary_v2 - venue: OKX market: SWAP product_type: PERPETUAL - native_symbol: ETH-USDT-SWAP + native_symbol: BTC-USDT-SWAP feed: BAR interval: 1m source_policy_id: crypto_primary_v2 @@ -183,16 +85,19 @@ consumers: market: SWAP product_type: PERPETUAL native_symbol: ETH-USDT-SWAP - feed: QUOTE - interval: null + feed: BAR + interval: 1m source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: ETH-USDT-SWAP - feed: TRADE + native_symbol: BTCUSDT + feed: MARK_INDEX_PRICE interval: null - source_policy_id: crypto_primary_v2 + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: null - venue: BINANCE market: USDM product_type: PERPETUAL @@ -213,6 +118,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: null - venue: BINANCE market: USDM product_type: PERPETUAL @@ -233,6 +148,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BTC-USDT-SWAP + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: BTC-USDT - venue: OKX market: SWAP product_type: PERPETUAL @@ -253,6 +178,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: ETH-USDT-SWAP + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: ETH-USDT - venue: OKX market: SWAP product_type: PERPETUAL @@ -273,6 +208,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: null - venue: BINANCE market: USDM product_type: PERPETUAL @@ -293,6 +238,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: null - venue: BINANCE market: USDM product_type: PERPETUAL @@ -313,6 +268,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: null - venue: BINANCE market: USDM product_type: PERPETUAL @@ -333,6 +298,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: SOL-USDT - venue: OKX market: SWAP product_type: PERPETUAL @@ -353,6 +328,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: DOGE-USDT - venue: OKX market: SWAP product_type: PERPETUAL @@ -373,6 +358,16 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: MARK_INDEX_PRICE + interval: null + source_policy_id: crypto_liquid_v2 + max_freshness_ms: 2000 + require_live: true + index_native_symbol: BNB-USDT - venue: OKX market: SWAP product_type: PERPETUAL @@ -393,69 +388,132 @@ consumers: depth_per_side: 100 max_freshness_ms: 2000 require_live: true - - venue: OKX - market: SWAP - product_type: PERPETUAL - native_symbol: SOL-USDT-SWAP - feed: BAR - interval: 1m - source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: SOL-USDT-SWAP - feed: QUOTE + native_symbol: SOLUSDT + feed: TRADE interval: null source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: SOL-USDT-SWAP - feed: TRADE + native_symbol: SOLUSDT + feed: QUOTE interval: null source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: DOGE-USDT-SWAP + native_symbol: SOLUSDT feed: BAR interval: 1m source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: DOGE-USDT-SWAP - feed: QUOTE + native_symbol: DOGEUSDT + feed: TRADE interval: null source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: DOGE-USDT-SWAP - feed: TRADE + native_symbol: DOGEUSDT + feed: QUOTE interval: null source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: BNB-USDT-SWAP + native_symbol: DOGEUSDT feed: BAR interval: 1m source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP + - venue: BINANCE + market: USDM product_type: PERPETUAL - native_symbol: BNB-USDT-SWAP - feed: QUOTE + native_symbol: BNBUSDT + feed: TRADE interval: null source_policy_id: crypto_primary_v2 - - venue: OKX + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: QUOTE + interval: null + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: QUOTE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: QUOTE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: OKX market: SWAP product_type: PERPETUAL native_symbol: BNB-USDT-SWAP feed: TRADE interval: null source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: QUOTE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 - venue: BINANCE market: USDM product_type: PERPETUAL @@ -858,238 +916,1315 @@ consumers: - venue: BINANCE market: USDM product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: SOLUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 2d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 2d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: DOGE-USDT-SWAP + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 2d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BNB-USDT-SWAP + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 +- consumer_id: alpha.binance.paper.stable + consumer_grade: ALPHA + requirements: + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 15m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BTCUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: ETHUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: SOLUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: DOGEUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 3m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 5m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 30m + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 2h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 4h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 6h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 8h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 12h + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 3d + source_policy_id: crypto_primary_v2 + - venue: BINANCE + market: USDM + product_type: PERPETUAL + native_symbol: BNBUSDT + feed: BAR + interval: 1w + source_policy_id: crypto_primary_v2 +- consumer_id: alpha.okx.paper.stable + consumer_grade: ALPHA + requirements: + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BTC-USDT-SWAP + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: ETH-USDT-SWAP + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BTC-USDT-SWAP + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: ETH-USDT-SWAP + feed: BAR + interval: 1m + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: BTC-USDT-SWAP + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: ETH-USDT-SWAP + feed: BAR + interval: 1h + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP + feed: TRADE + interval: null + source_policy_id: crypto_primary_v2 + - venue: OKX + market: SWAP + product_type: PERPETUAL + native_symbol: SOL-USDT-SWAP feed: BAR - interval: 2h + interval: 1m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: SOL-USDT-SWAP feed: BAR - interval: 4h + interval: 1h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT - feed: BAR - interval: 6h + native_symbol: DOGE-USDT-SWAP + feed: TRADE + interval: null source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: DOGE-USDT-SWAP feed: BAR - interval: 8h + interval: 1m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: DOGE-USDT-SWAP feed: BAR - interval: 12h + interval: 1h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT - feed: BAR - interval: 1d + native_symbol: BNB-USDT-SWAP + feed: TRADE + interval: null source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: BNB-USDT-SWAP feed: BAR - interval: 3d + interval: 1m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: SOLUSDT + native_symbol: BNB-USDT-SWAP feed: BAR - interval: 1w + interval: 1h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 3m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 5m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 15m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 30m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: DOGEUSDT - feed: BAR - interval: 1h - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 2h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 4h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 6h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR - interval: 8h + interval: 12h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR - interval: 12h + interval: 1d source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR - interval: 1d + interval: 2d source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 3d source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: DOGEUSDT + native_symbol: BTC-USDT-SWAP feed: BAR interval: 1w source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 3m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 5m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 15m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 30m source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM - product_type: PERPETUAL - native_symbol: BNBUSDT - feed: BAR - interval: 1h - source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 2h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 4h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 6h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR - interval: 8h + interval: 12h source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR - interval: 12h + interval: 1d source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR - interval: 1d + interval: 2d source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 3d source_policy_id: crypto_primary_v2 - - venue: BINANCE - market: USDM + - venue: OKX + market: SWAP product_type: PERPETUAL - native_symbol: BNBUSDT + native_symbol: ETH-USDT-SWAP feed: BAR interval: 1w source_policy_id: crypto_primary_v2 @@ -1121,13 +2256,6 @@ consumers: feed: BAR interval: 30m source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP - product_type: PERPETUAL - native_symbol: SOL-USDT-SWAP - feed: BAR - interval: 1h - source_policy_id: crypto_primary_v2 - venue: OKX market: SWAP product_type: PERPETUAL @@ -1212,13 +2340,6 @@ consumers: feed: BAR interval: 30m source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP - product_type: PERPETUAL - native_symbol: DOGE-USDT-SWAP - feed: BAR - interval: 1h - source_policy_id: crypto_primary_v2 - venue: OKX market: SWAP product_type: PERPETUAL @@ -1303,13 +2424,6 @@ consumers: feed: BAR interval: 30m source_policy_id: crypto_primary_v2 - - venue: OKX - market: SWAP - product_type: PERPETUAL - native_symbol: BNB-USDT-SWAP - feed: BAR - interval: 1h - source_policy_id: crypto_primary_v2 - venue: OKX market: SWAP product_type: PERPETUAL diff --git a/config/v2/stable-primary-consumer-routing.yaml b/config/v2/stable-primary-consumer-routing.yaml index 3e8705a..5dc03b0 100644 --- a/config/v2/stable-primary-consumer-routing.yaml +++ b/config/v2/stable-primary-consumer-routing.yaml @@ -1,5 +1,5 @@ schema: qdl.v2.shared-primary-consumer-route.v1 -revision: 4 +revision: 6 contract_version: 2.0.0 target_route: V2_PRIMARY rollback_route: V1 diff --git a/config/v2/stable-source-bindings.yaml b/config/v2/stable-source-bindings.yaml index 2beebde..7a8639c 100644 --- a/config/v2/stable-source-bindings.yaml +++ b/config/v2/stable-source-bindings.yaml @@ -1,6 +1,6 @@ schema: qdl.v2.stable-source-bindings.v1 canonical_stream: md.canonical.v2 -catalog_revision: 8 +catalog_revision: 9 source_policy_revision: 1 authority_revision: 1 instruments: @@ -802,6 +802,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: binance-usdm-bnbusdt-mark_index_price + instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f + feed: MARK_INDEX_PRICE + interval: null + source: + provider: BINANCE_DIRECT + source_id: binance-usdm-bnbusdt-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: binance-usdm/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: binance-usdm-bnbusdt-quote instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: QUOTE @@ -818,6 +836,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: binance-usdm-bnbusdt-trade instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f @@ -1176,6 +1195,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: binance-usdm-btcusdt-mark_index_price + instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 + feed: MARK_INDEX_PRICE + interval: null + source: + provider: BINANCE_DIRECT + source_id: binance-usdm-btcusdt-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: binance-usdm/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: binance-usdm-btcusdt-quote instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: QUOTE @@ -1192,6 +1229,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: binance-usdm-btcusdt-trade instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 @@ -1482,6 +1520,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: binance-usdm-dogeusdt-mark_index_price + instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a + feed: MARK_INDEX_PRICE + interval: null + source: + provider: BINANCE_DIRECT + source_id: binance-usdm-dogeusdt-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: binance-usdm/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: binance-usdm-dogeusdt-quote instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: QUOTE @@ -1498,6 +1554,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: binance-usdm-dogeusdt-trade instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a @@ -1856,6 +1913,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: binance-usdm-ethusdt-mark_index_price + instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 + feed: MARK_INDEX_PRICE + interval: null + source: + provider: BINANCE_DIRECT + source_id: binance-usdm-ethusdt-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: binance-usdm/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: binance-usdm-ethusdt-quote instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: QUOTE @@ -1872,6 +1947,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: binance-usdm-ethusdt-trade instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 @@ -2162,6 +2238,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: binance-usdm-solusdt-mark_index_price + instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 + feed: MARK_INDEX_PRICE + interval: null + source: + provider: BINANCE_DIRECT + source_id: binance-usdm-solusdt-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: binance-usdm/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: binance-usdm-solusdt-quote instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: QUOTE @@ -2178,6 +2272,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: binance-usdm-solusdt-trade instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 @@ -2723,6 +2818,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: okx-swap-bnb-usdt-swap-mark_index_price + instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 + feed: MARK_INDEX_PRICE + interval: null + source: + provider: OKX_DIRECT + source_id: okx-swap-bnb-usdt-swap-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: okx-v5/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: okx-swap-bnb-usdt-swap-quote instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: QUOTE @@ -2739,6 +2852,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: okx-swap-bnb-usdt-swap-trade instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 @@ -3012,6 +3126,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: okx-swap-btc-usdt-swap-mark_index_price + instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f + feed: MARK_INDEX_PRICE + interval: null + source: + provider: OKX_DIRECT + source_id: okx-swap-btc-usdt-swap-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: okx-v5/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: okx-swap-btcusdt-bar-1m instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR @@ -3045,6 +3177,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: okx-swap-btcusdt-trade instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f @@ -3335,6 +3468,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: okx-swap-doge-usdt-swap-mark_index_price + instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 + feed: MARK_INDEX_PRICE + interval: null + source: + provider: OKX_DIRECT + source_id: okx-swap-doge-usdt-swap-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: okx-v5/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: okx-swap-doge-usdt-swap-quote instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: QUOTE @@ -3351,6 +3502,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: okx-swap-doge-usdt-swap-trade instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 @@ -3641,6 +3793,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: okx-swap-eth-usdt-swap-mark_index_price + instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 + feed: MARK_INDEX_PRICE + interval: null + source: + provider: OKX_DIRECT + source_id: okx-swap-eth-usdt-swap-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: okx-v5/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: okx-swap-eth-usdt-swap-quote instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: QUOTE @@ -3657,6 +3827,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: okx-swap-eth-usdt-swap-trade instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 @@ -3947,6 +4118,24 @@ bindings: require_final_bar: false continuous_calendar: true v1_compatibility: NONE +- binding_id: okx-swap-sol-usdt-swap-mark_index_price + instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de + feed: MARK_INDEX_PRICE + interval: null + source: + provider: OKX_DIRECT + source_id: okx-swap-sol-usdt-swap-mark_index_price-primary-v2 + source_role: PRIMARY + source_policy_id: crypto_liquid_v2 + authoritative: true + adapter_version: okx-v5/2.0.0 + normalizer_version: qdl-rust-core/2.0.0 + quality: + stale_after_ms: 2000 + require_final_bar: false + continuous_calendar: true + freshness_basis: PROVIDER_CONFIRMATION + v1_compatibility: NONE - binding_id: okx-swap-sol-usdt-swap-quote instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: QUOTE @@ -3963,6 +4152,7 @@ bindings: stale_after_ms: 5000 require_final_bar: false continuous_calendar: true + delivery_semantics: ON_CHANGE v1_compatibility: NONE - binding_id: okx-swap-sol-usdt-swap-trade instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de diff --git a/config/v2/stable-v2-release-routing.yaml b/config/v2/stable-v2-release-routing.yaml index 55cee9f..220f5ce 100644 --- a/config/v2/stable-v2-release-routing.yaml +++ b/config/v2/stable-v2-release-routing.yaml @@ -1,13 +1,13 @@ schema: qdl.v2.stable-release-routing.v1 -revision: 18 +revision: 22 contract_version: 2.0.0 source_catalog: path: /app/config/v2/stable-source-bindings.yaml - sha256: c2fe0fe5326856ffb504fc4c2251ac77de9bc315e743248c543b394d8df18d3b - revision: 8 + sha256: 2072202c76683788cf1d59905038787e4db194209df8266b5da44b6487f4947e + revision: 9 crypto_demand: path: /app/config/v2/stable-crypto-demand.yaml - sha256: 44abc4a3965a0ae2e4532a282caa29d4b11f4aa25d392c292972f68195ecd64f + sha256: 9ace71cd3e4ed6e224151e31699cfc3fe57ba2a6c17664c832da07eb1c6aa30f revision: 6 capability_matrix: path: /app/config/v2/stable-capabilities.yaml @@ -53,8 +53,8 @@ consumers: reason: V1_OKX_TRADE_EQUIVALENCE_UNPROVEN - consumer_id: alpha.binance.paper.stable manifest: /app/consumers/stable/alpha-binance-paper.yaml - manifest_revision: 10 - manifest_sha256: 2a1d98de3900d360a50e095293964d67aba7a5085a0ae74251e37e86160bcbd3 + manifest_revision: 12 + manifest_sha256: ed44dc306d62379bfacd8032ad2016da14617aaf44ec8fb6eb82183600dbaaea demand_revision: 6 products: - requirement_key: 8aedd349-6999-5874-b0dd-34c6451c0b3a:TRADE::crypto_primary_v2 @@ -559,8 +559,8 @@ consumers: reason: V1_REFERENCE_EQUIVALENCE_UNPROVEN - consumer_id: alpha.okx.paper.stable manifest: /app/consumers/stable/alpha-okx-paper.yaml - manifest_revision: 9 - manifest_sha256: 5b30563bb7aa7345f5b2c7f030cfad2f10263a36a02f98f0a9847d2aef23a671 + manifest_revision: 11 + manifest_sha256: 3b39c485e1043a9dd648121e9f62cdd94f4f801565e021ace13d59f56c23bc4e demand_revision: 6 products: - requirement_key: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7:TRADE::crypto_primary_v2 @@ -1019,8 +1019,8 @@ consumers: reason: VN_REAL_PROVIDER_GATE_UNEXERCISED - consumer_id: trading-system.paper.stable manifest: /app/consumers/stable/trading-system-paper.yaml - manifest_revision: 9 - manifest_sha256: b026df8d4b1e615899c2ba3b86923a4da4dc8d213a5758e58846be7e308a2811 + manifest_revision: 10 + manifest_sha256: 5ca9aff1960883f59827e3a34ed709f4c30cf5db96c4e1253dd6f1116fb20cdf demand_revision: 6 products: - requirement_key: a953e16e-7138-5562-b5e8-c337a44d0b65:TRADE::crypto_primary_v2 diff --git a/consumers/stable/alpha-binance-paper.yaml b/consumers/stable/alpha-binance-paper.yaml index 860ee01..5ad65df 100644 --- a/consumers/stable/alpha-binance-paper.yaml +++ b/consumers/stable/alpha-binance-paper.yaml @@ -5,7 +5,7 @@ metadata: owner: alpha-platform subject: spiffe://qdl/paper/alpha-binance-stable environment: paper - revision: 10 + revision: 12 spec: sdk_major: 2 rollback_contract: V1 @@ -54,6 +54,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: BAR @@ -258,13 +259,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: TRADE @@ -294,6 +295,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: a953e16e-7138-5562-b5e8-c337a44d0b65 feed: BAR @@ -498,13 +500,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: TRADE @@ -534,6 +536,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: b2d78145-b541-58e2-941b-06c44c65e45f feed: BAR @@ -738,13 +741,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: TRADE @@ -774,6 +777,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: e4ce7249-a4e8-5073-beb8-dab908596c34 feed: BAR @@ -978,13 +982,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: TRADE @@ -1014,6 +1018,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: ee93fabf-68df-5b50-8924-51bf25a5a757 feed: BAR @@ -1218,13 +1223,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: 8aedd349-6999-5874-b0dd-34c6451c0b3a feed: BASIS diff --git a/consumers/stable/alpha-okx-paper.yaml b/consumers/stable/alpha-okx-paper.yaml index cd72c62..42af206 100644 --- a/consumers/stable/alpha-okx-paper.yaml +++ b/consumers/stable/alpha-okx-paper.yaml @@ -5,7 +5,7 @@ metadata: owner: alpha-platform subject: spiffe://qdl/paper/alpha-okx-stable environment: paper - revision: 9 + revision: 11 spec: sdk_major: 2 rollback_contract: V1 @@ -54,6 +54,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: BAR @@ -258,13 +259,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: TRADE @@ -294,6 +295,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: a6884fb3-1fa0-53e0-9621-d01ba5f9a2de feed: BAR @@ -498,13 +500,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: TRADE @@ -534,6 +536,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: e49b54ae-c23d-5351-9e64-47934aac28f8 feed: BAR @@ -738,13 +741,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: TRADE @@ -774,6 +777,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: f2e37e2b-1386-5a32-9b79-0fd39ec7a5a3 feed: BAR @@ -978,13 +982,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: TRADE @@ -1014,6 +1018,7 @@ spec: gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: fb26214c-7b9b-5961-95b2-55154755af0f feed: BAR @@ -1218,13 +1223,13 @@ spec: interval: null warmup_limit: 0 max_freshness_ms: 2000 - event_recency_policy: OBSERVE require_full_coverage: true require_final_bars: false stale_policy: BLOCK gap_policy: BLOCK recovery: SNAPSHOT_AND_REPLAY bar_revision_policy: LATEST + event_recency_policy: OBSERVE max_session_liveness_ms: 45000 - instrument_uid: 6c7c9256-2905-5c75-a149-fa0ac36bbbc7 feed: CONTRACT_METADATA diff --git a/consumers/stable/trading-system-paper.yaml b/consumers/stable/trading-system-paper.yaml index 8cbaa69..44d0648 100644 --- a/consumers/stable/trading-system-paper.yaml +++ b/consumers/stable/trading-system-paper.yaml @@ -5,7 +5,7 @@ metadata: owner: trading-platform subject: spiffe://qdl/paper/trading-system-stable environment: paper - revision: 9 + revision: 10 spec: sdk_major: 2 rollback_contract: V1 @@ -60,7 +60,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -73,7 +74,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -140,7 +142,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -153,7 +156,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -206,7 +210,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -246,7 +251,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -286,7 +292,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -326,7 +333,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -366,7 +374,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -406,7 +415,8 @@ spec: source_policy_id: crypto_primary_v2 warmup_limit: 0 max_freshness_ms: 2000 - max_session_liveness_ms: 45000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 2000 require_full_coverage: true require_final_bars: true stale_policy: BLOCK @@ -435,6 +445,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -473,6 +485,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -511,6 +525,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -549,6 +565,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -587,6 +605,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -625,6 +645,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -663,6 +685,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -701,6 +725,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -739,6 +765,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK @@ -777,6 +805,8 @@ spec: source_policy_id: crypto_liquid_v2 warmup_limit: 0 max_freshness_ms: 2000 + event_recency_policy: OBSERVE + max_session_liveness_ms: 45000 require_full_coverage: true require_final_bars: false stale_policy: BLOCK diff --git a/contracts/golden/quality/binding-quality-decision-v1.json b/contracts/golden/quality/binding-quality-decision-v1.json new file mode 100644 index 0000000..c033506 --- /dev/null +++ b/contracts/golden/quality/binding-quality-decision-v1.json @@ -0,0 +1,90 @@ +{ + "schema": "qdl.binding-quality-decision.v1", + "cases": [ + { + "name": "strict_quote_fresh", + "input": {"binding_id":"binance-btc-quote","instrument_uid":"btc","feed":"QUOTE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":25,"event_limit_ms":2000,"event_recency_policy":"BLOCK","session_state":"LIVE","session_liveness_ms":25,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":7,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"STRICT_EVENT","availability":"ACTIVE","state":"LIVE","event_recency_state":"LIVE","complete":true,"execution_eligible":true,"reason_codes":[]} + }, + { + "name": "strict_quote_stale", + "input": {"binding_id":"okx-bnb-quote","instrument_uid":"bnb","feed":"QUOTE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":2001,"event_limit_ms":2000,"event_recency_policy":"BLOCK","session_state":"LIVE","session_liveness_ms":20,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":8,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"STRICT_EVENT","availability":"ACTIVE","state":"STALE","event_recency_state":"STALE","complete":true,"execution_eligible":false,"reason_codes":["LAST_EVENT_STALE"]} + }, + { + "name": "on_change_quote_live_session", + "input": {"binding_id":"okx-bnb-quote","instrument_uid":"bnb","feed":"QUOTE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":4001,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":1999,"session_limit_ms":2000,"delivery_semantics":"ON_CHANGE","components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":81,"allow_quiet_execution":true,"flags":["DELIVERY_ON_CHANGE"]}, + "expected": {"semantics":"QUIET_SESSION","delivery_semantics":"ON_CHANGE","availability":"ACTIVE","state":"LIVE","event_recency_state":"STALE","complete":true,"execution_eligible":true,"reason_codes":["DELIVERY_ON_CHANGE","LAST_EVENT_STALE"]} + }, + { + "name": "on_change_quote_heartbeat_expired", + "input": {"binding_id":"okx-bnb-quote","instrument_uid":"bnb","feed":"QUOTE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":4001,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":2001,"session_limit_ms":2000,"delivery_semantics":"ON_CHANGE","components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":82,"allow_quiet_execution":true,"flags":["DELIVERY_ON_CHANGE"]}, + "expected": {"semantics":"QUIET_SESSION","delivery_semantics":"ON_CHANGE","availability":"ACTIVE","state":"STALE","event_recency_state":"STALE","complete":true,"execution_eligible":false,"reason_codes":["DELIVERY_ON_CHANGE","LAST_EVENT_STALE","SOURCE_SESSION_HEARTBEAT_EXPIRED"]} + }, + { + "name": "on_change_quote_gap", + "input": {"binding_id":"okx-bnb-quote","instrument_uid":"bnb","feed":"QUOTE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":4001,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":20,"session_limit_ms":2000,"delivery_semantics":"ON_CHANGE","components":[],"generation_matches":true,"config_matches":true,"gap_open":true,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":83,"allow_quiet_execution":true,"flags":["DELIVERY_ON_CHANGE"]}, + "expected": {"semantics":"QUIET_SESSION","delivery_semantics":"ON_CHANGE","availability":"ACTIVE","state":"GAPPED","event_recency_state":"STALE","complete":false,"execution_eligible":false,"reason_codes":["DELIVERY_ON_CHANGE","LAST_EVENT_STALE","OPEN_SEQUENCE_GAP"]} + }, + { + "name": "quiet_trade_connected", + "input": {"binding_id":"okx-doge-trade","instrument_uid":"doge","feed":"TRADE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":18000,"event_limit_ms":3000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":100,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":9,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"ACTIVE","state":"LIVE","event_recency_state":"STALE","complete":true,"execution_eligible":false,"reason_codes":["LAST_EVENT_STALE"]} + }, + { + "name": "quiet_mark_index_components_live", + "input": {"binding_id":"binance-sol-mark-index","instrument_uid":"sol","feed":"MARK_INDEX_PRICE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":4500,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":50,"session_limit_ms":45000,"components":[{"name":"MARK","receipt_age_ms":300,"quiet_after_ms":15000},{"name":"INDEX","receipt_age_ms":1400,"quiet_after_ms":15000}],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":10,"allow_quiet_execution":true,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"ACTIVE","state":"LIVE","event_recency_state":"STALE","complete":true,"execution_eligible":true,"reason_codes":["LAST_EVENT_STALE"]} + }, + { + "name": "quiet_session_stopped", + "input": {"binding_id":"okx-eth-trade","instrument_uid":"eth","feed":"TRADE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":18000,"event_limit_ms":3000,"event_recency_policy":"OBSERVE","session_state":"DISCONNECTED","session_liveness_ms":100,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":11,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"ACTIVE","state":"STALE","event_recency_state":"STALE","complete":true,"execution_eligible":false,"reason_codes":["LAST_EVENT_STALE","SOURCE_SESSION_DISCONNECTED"]} + }, + { + "name": "quiet_component_expired", + "input": {"binding_id":"okx-bnb-mark-index","instrument_uid":"bnb","feed":"MARK_INDEX_PRICE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":4500,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":50,"session_limit_ms":45000,"components":[{"name":"MARK","receipt_age_ms":15001,"quiet_after_ms":15000},{"name":"INDEX","receipt_age_ms":100,"quiet_after_ms":15000}],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":12,"allow_quiet_execution":true,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"ACTIVE","state":"STALE","event_recency_state":"STALE","complete":true,"execution_eligible":false,"reason_codes":["LAST_EVENT_STALE","COMPONENT_MARK_STALE"]} + }, + { + "name": "generation_mismatch", + "input": {"binding_id":"binance-eth-book-delta","instrument_uid":"eth","feed":"BOOK_DELTA","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":100,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":10,"session_limit_ms":45000,"components":[],"generation_matches":false,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":13,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"ACTIVE","state":"STALE","event_recency_state":"LIVE","complete":true,"execution_eligible":false,"reason_codes":["GENERATION_MISMATCH"]} + }, + { + "name": "config_mismatch", + "input": {"binding_id":"okx-eth-book-delta","instrument_uid":"eth","feed":"BOOK_DELTA","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":100,"event_limit_ms":2000,"event_recency_policy":"OBSERVE","session_state":"LIVE","session_liveness_ms":10,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":false,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":14,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"ACTIVE","state":"STALE","event_recency_state":"LIVE","complete":true,"execution_eligible":false,"reason_codes":["CONFIG_REVISION_MISMATCH"]} + }, + { + "name": "book_gap", + "input": {"binding_id":"binance-btc-book-snapshot","instrument_uid":"btc","feed":"BOOK_SNAPSHOT","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":100,"event_limit_ms":2000,"event_recency_policy":"BLOCK","session_state":"LIVE","session_liveness_ms":10,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":true,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":15,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"STRICT_EVENT","availability":"ACTIVE","state":"GAPPED","event_recency_state":"LIVE","complete":false,"execution_eligible":false,"reason_codes":["OPEN_SEQUENCE_GAP"]} + }, + { + "name": "book_resync", + "input": {"binding_id":"okx-sol-book-snapshot","instrument_uid":"sol","feed":"BOOK_SNAPSHOT","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":100,"event_limit_ms":2000,"event_recency_policy":"BLOCK","session_state":"LIVE","session_liveness_ms":10,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":false,"final_bar":true,"require_final_bar":false,"watermark_offset":16,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"STRICT_EVENT","availability":"ACTIVE","state":"SYNCING","event_recency_state":"LIVE","complete":false,"execution_eligible":false,"reason_codes":["BOOK_SEQUENCE_UNVERIFIED"]} + }, + { + "name": "bar_not_final", + "input": {"binding_id":"binance-doge-bar-1m","instrument_uid":"doge","feed":"BAR","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":true,"event_age_ms":100,"event_limit_ms":180000,"event_recency_policy":"BLOCK","session_state":"NOT_APPLICABLE","session_liveness_ms":null,"session_limit_ms":null,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":false,"require_final_bar":true,"watermark_offset":17,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"FINAL_SCHEDULED","availability":"ACTIVE","state":"SYNCING","event_recency_state":"LIVE","complete":false,"execution_eligible":false,"reason_codes":["BAR_NOT_FINAL"]} + }, + { + "name": "expected_v1_primary", + "input": {"binding_id":"dnse-vn30-trade","instrument_uid":"vn30","feed":"TRADE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"PYTHON_VENDOR_SDK","market_open":true,"event_present":false,"event_age_ms":null,"event_limit_ms":15000,"event_recency_policy":"OBSERVE","session_state":"UNKNOWN","session_liveness_ms":null,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":0,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"QUIET_SESSION","availability":"EXPECTED_V1_PRIMARY","state":"DISABLED","event_recency_state":"NOT_APPLICABLE","complete":false,"execution_eligible":false,"reason_codes":["SOURCE_SESSION_UNKNOWN","EXPECTED_V1_PRIMARY"]} + }, + { + "name": "expected_dark", + "input": {"binding_id":"binance-spot-btc-quote","instrument_uid":"spotbtc","feed":"QUOTE","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":false,"acquisition_mode":"RUST_NATIVE","market_open":true,"event_present":false,"event_age_ms":null,"event_limit_ms":5000,"event_recency_policy":"BLOCK","session_state":"UNKNOWN","session_liveness_ms":null,"session_limit_ms":45000,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":false,"watermark_offset":0,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"STRICT_EVENT","availability":"EXPECTED_DARK","state":"DISABLED","event_recency_state":"NOT_APPLICABLE","complete":false,"execution_eligible":false,"reason_codes":["SOURCE_SESSION_UNKNOWN","EXPECTED_DARK"]} + }, + { + "name": "out_of_session", + "input": {"binding_id":"vn-bar","instrument_uid":"vn","feed":"BAR","source_role":"PRIMARY","authoritative":true,"acquisition_enabled":true,"acquisition_mode":"RUST_NATIVE","market_open":false,"event_present":false,"event_age_ms":null,"event_limit_ms":180000,"event_recency_policy":"BLOCK","session_state":"NOT_APPLICABLE","session_liveness_ms":null,"session_limit_ms":null,"components":[],"generation_matches":true,"config_matches":true,"gap_open":false,"book_verified":true,"final_bar":true,"require_final_bar":true,"watermark_offset":0,"allow_quiet_execution":false,"flags":[]}, + "expected": {"semantics":"FINAL_SCHEDULED","availability":"OUT_OF_SESSION","state":"MARKET_CLOSED","event_recency_state":"NOT_APPLICABLE","complete":false,"execution_eligible":false,"reason_codes":["OUT_OF_SESSION"]} + } + ] +} diff --git a/contracts/v2/openapi.snapshot.json b/contracts/v2/openapi.snapshot.json index 26585b6..1309e95 100644 --- a/contracts/v2/openapi.snapshot.json +++ b/contracts/v2/openapi.snapshot.json @@ -2162,6 +2162,16 @@ ], "title": "End Time Ns" }, + "event_recency_policy": { + "anyOf": [ + { + "$ref": "#/components/schemas/qdl_sdk__models__StalePolicy" + }, + { + "type": "null" + } + ] + }, "instrument_uid": { "maxLength": 200, "minLength": 1, @@ -2221,6 +2231,19 @@ "title": "Max Pages", "type": "integer" }, + "max_session_liveness_ms": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "maximum": 86400000.0, + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Max Session Liveness Ms" + }, "page_size": { "anyOf": [ { @@ -2283,7 +2306,7 @@ "event_recency_policy": { "anyOf": [ { - "$ref": "#/components/schemas/StalePolicy" + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy" }, { "type": "null" @@ -2362,7 +2385,7 @@ "type": "string" }, "stale_policy": { - "$ref": "#/components/schemas/StalePolicy", + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy", "default": "BLOCK" }, "warmup": { @@ -2501,16 +2524,6 @@ "title": "SourceView", "type": "object" }, - "StalePolicy": { - "enum": [ - "UNSPECIFIED", - "BLOCK", - "PAUSE", - "OBSERVE" - ], - "title": "StalePolicy", - "type": "string" - }, "SystemReadinessSummary": { "additionalProperties": false, "properties": { @@ -2902,6 +2915,25 @@ ], "title": "WarmupTimeRange", "type": "object" + }, + "qdl__query__contracts__StalePolicy": { + "enum": [ + "UNSPECIFIED", + "BLOCK", + "PAUSE", + "OBSERVE" + ], + "title": "StalePolicy", + "type": "string" + }, + "qdl_sdk__models__StalePolicy": { + "enum": [ + "BLOCK", + "PAUSE", + "OBSERVE" + ], + "title": "StalePolicy", + "type": "string" } }, "securitySchemes": { @@ -3029,7 +3061,7 @@ "schema": { "anyOf": [ { - "$ref": "#/components/schemas/StalePolicy" + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy" }, { "type": "null" @@ -3081,7 +3113,7 @@ "name": "stale_policy", "required": false, "schema": { - "$ref": "#/components/schemas/StalePolicy", + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy", "default": "BLOCK" } }, @@ -3553,7 +3585,7 @@ "schema": { "anyOf": [ { - "$ref": "#/components/schemas/StalePolicy" + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy" }, { "type": "null" @@ -3605,7 +3637,7 @@ "name": "stale_policy", "required": false, "schema": { - "$ref": "#/components/schemas/StalePolicy", + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy", "default": "BLOCK" } }, @@ -3760,7 +3792,7 @@ "schema": { "anyOf": [ { - "$ref": "#/components/schemas/StalePolicy" + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy" }, { "type": "null" @@ -3812,7 +3844,7 @@ "name": "stale_policy", "required": false, "schema": { - "$ref": "#/components/schemas/StalePolicy", + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy", "default": "BLOCK" } }, @@ -4052,7 +4084,7 @@ "schema": { "anyOf": [ { - "$ref": "#/components/schemas/StalePolicy" + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy" }, { "type": "null" @@ -4104,7 +4136,7 @@ "name": "stale_policy", "required": false, "schema": { - "$ref": "#/components/schemas/StalePolicy", + "$ref": "#/components/schemas/qdl__query__contracts__StalePolicy", "default": "BLOCK" } }, diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 9cbb8ae..c6596a0 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -14,7 +14,10 @@ services: test_runner: container_name: !reset null - volumes: !reset [] + # Evidence remains outside serving images. Contract tests receive only the + # frozen fixture directory and never a writable application-source mount. + volumes: !override + - ./upgrade/evidence:/app/upgrade/evidence:ro data_source_checker: container_name: !reset null diff --git a/docker-compose.v2-stable.yml b/docker-compose.v2-stable.yml index 55b3752..7bed9e5 100644 --- a/docker-compose.v2-stable.yml +++ b/docker-compose.v2-stable.yml @@ -95,6 +95,7 @@ x-stable-env: &stable-env QDL_STABLE_STATE_DIR: /var/lib/qdl-stable/runtime QDL_STABLE_DURABLE_STATE_DIR: /var/lib/qdl-stable/shared QDL_STABLE_SOURCE_BINDINGS: /app/config/v2/stable-source-bindings.yaml + QDL_STABLE_ACQUISITION_BINDINGS: /app/config/v2/stable-acquisition-bindings.yaml QDL_STABLE_CONSUMER_MANIFESTS: /app/consumers/stable/monitoring-multivenue.yaml:/app/consumers/stable/alpha-binance-paper.yaml:/app/consumers/stable/alpha-okx-paper.yaml:/app/consumers/stable/alpha-vn-paper.yaml:/app/consumers/stable/trading-system-paper.yaml:/app/consumers/stable/reference-l2-stable.yaml QDL_STABLE_INTERNAL_INGEST_SECRET: ${QDL_STABLE_INTERNAL_INGEST_SECRET:?set QDL_STABLE_INTERNAL_INGEST_SECRET} QDL_STABLE_REDIS_URL: redis://stable_redis:6379/0 @@ -321,6 +322,10 @@ services: QDL_STABLE_TLS_CLIENT_CA_FILE: /stable-certs/query/client-ca-bundle.crt QDL_STABLE_TLS_CERT_FILE: /stable-certs/query/server.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/query/server.key + # The read path is private mTLS plus the stable HMAC boundary. Query + # tries the current holder first and can use the passive peer only after + # it acquires the existing stream lease; it never calls venue REST. + QDL_STABLE_EXECUTION_MARK_INDEX_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' depends_on: {stable_redis: {condition: service_healthy}, stable_state_init: {condition: service_completed_successfully}, stable_tls_init: {condition: service_completed_successfully}} healthcheck: {test: [CMD, python, -c, "import ssl,urllib.request; c=ssl.create_default_context(cafile='/stable-certs/query/ca.crt'); c.load_cert_chain('/stable-certs/query/server.crt','/stable-certs/query/server.key'); urllib.request.urlopen('https://localhost:8200/health/ready',context=c,timeout=2)"], interval: 5s, timeout: 3s, retries: 20} @@ -347,6 +352,7 @@ services: QDL_STABLE_TLS_CLIENT_CA_FILE: /stable-certs/query/client-ca-bundle.crt QDL_STABLE_TLS_CERT_FILE: /stable-certs/query/server.crt QDL_STABLE_TLS_KEY_FILE: /stable-certs/query/server.key + QDL_STABLE_EXECUTION_MARK_INDEX_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' depends_on: {stable_redis: {condition: service_healthy}, stable_state_init: {condition: service_completed_successfully}, stable_tls_init: {condition: service_completed_successfully}} healthcheck: {test: [CMD, python, -c, "import ssl,urllib.request; c=ssl.create_default_context(cafile='/stable-certs/query/ca.crt'); c.load_cert_chain('/stable-certs/query/server.crt','/stable-certs/query/server.key'); urllib.request.urlopen('https://localhost:8200/health/ready',context=c,timeout=2)"], interval: 5s, timeout: 3s, retries: 20} @@ -427,6 +433,7 @@ services: QDL_STABLE_HTTP_PORT: "8230" QDL_STABLE_GRPC_PORT: "8231" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-projector-1-audit.jsonl + QDL_STABLE_HEARTBEAT_PATH: /var/lib/qdl-stable/runtime/heartbeat/projector-1.json QDL_STABLE_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 QDL_STABLE_KAFKA_CLIENT_ID: stable-projector-1 QDL_STABLE_KAFKA_CANONICAL_TOPIC: md.canonical.v2 @@ -439,12 +446,22 @@ services: # cache catches up; every durable event remains Kafka-authoritative. QDL_STABLE_MAX_PENDING_RECORDS: "2048" QDL_STABLE_MAX_PENDING_BYTES: "33554432" - QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "1000" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" + # One shared consumer owns one of the six existing canonical partitions. + # A complete bounded fetch is committed in one turn so the single durable + # SQLite transaction is amortized without exceeding the 512/8 MiB bounds. + QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS: "512" volumes: - stable_state:/var/lib/qdl-stable - stable_tls:/stable-certs:ro - ${QDL_STABLE_RUNTIME_DIR:?set QDL_STABLE_RUNTIME_DIR}:/runtime:ro + healthcheck: + test: [CMD-SHELL, "find /var/lib/qdl-stable/runtime/heartbeat/projector-1.json -newermt '-30 seconds' 2>/dev/null | grep -q ."] + interval: 20s + timeout: 5s + retries: 3 + start_period: 60s depends_on: stable_tls_init: {condition: service_completed_successfully} stable_state_init: {condition: service_completed_successfully} @@ -470,6 +487,7 @@ services: QDL_STABLE_HTTP_PORT: "8230" QDL_STABLE_GRPC_PORT: "8231" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-projector-2-audit.jsonl + QDL_STABLE_HEARTBEAT_PATH: /var/lib/qdl-stable/runtime/heartbeat/projector-2.json QDL_STABLE_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 QDL_STABLE_KAFKA_CLIENT_ID: stable-projector-2 QDL_STABLE_KAFKA_CANONICAL_TOPIC: md.canonical.v2 @@ -480,12 +498,19 @@ services: QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' QDL_STABLE_MAX_PENDING_RECORDS: "2048" QDL_STABLE_MAX_PENDING_BYTES: "33554432" - QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "1000" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" + QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS: "512" volumes: - stable_state:/var/lib/qdl-stable - stable_tls:/stable-certs:ro - ${QDL_STABLE_RUNTIME_DIR:?set QDL_STABLE_RUNTIME_DIR}:/runtime:ro + healthcheck: + test: [CMD-SHELL, "find /var/lib/qdl-stable/runtime/heartbeat/projector-2.json -newermt '-30 seconds' 2>/dev/null | grep -q ."] + interval: 20s + timeout: 5s + retries: 3 + start_period: 60s depends_on: stable_tls_init: {condition: service_completed_successfully} stable_state_init: {condition: service_completed_successfully} @@ -511,6 +536,7 @@ services: QDL_STABLE_HTTP_PORT: "8230" QDL_STABLE_GRPC_PORT: "8231" QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-projector-3-audit.jsonl + QDL_STABLE_HEARTBEAT_PATH: /var/lib/qdl-stable/runtime/heartbeat/projector-3.json QDL_STABLE_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 QDL_STABLE_KAFKA_CLIENT_ID: stable-projector-3 QDL_STABLE_KAFKA_CANONICAL_TOPIC: md.canonical.v2 @@ -521,12 +547,160 @@ services: QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' QDL_STABLE_MAX_PENDING_RECORDS: "2048" QDL_STABLE_MAX_PENDING_BYTES: "33554432" - QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "1000" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" + QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS: "512" volumes: - stable_state:/var/lib/qdl-stable - stable_tls:/stable-certs:ro - ${QDL_STABLE_RUNTIME_DIR:?set QDL_STABLE_RUNTIME_DIR}:/runtime:ro + healthcheck: + test: [CMD-SHELL, "find /var/lib/qdl-stable/runtime/heartbeat/projector-3.json -newermt '-30 seconds' 2>/dev/null | grep -q ."] + interval: 20s + timeout: 5s + retries: 3 + start_period: 60s + depends_on: + stable_tls_init: {condition: service_completed_successfully} + stable_state_init: {condition: service_completed_successfully} + kafka1: {condition: service_healthy} + kafka2: {condition: service_healthy} + kafka3: {condition: service_healthy} + stable_redis: {condition: service_healthy} + stream_v2_active: {condition: service_started} + stream_v2_passive: {condition: service_started} + + # Kafka assigns one of its six existing canonical partitions to each generic + # projector in this single consumer group. These are shared capacity replicas, + # not symbol-, interval-, venue- or alpha-specific services. + projector_v2_4: + <<: *python + cpus: 1.00 + mem_limit: 768m + command: [python, -m, app.entrypoints.projector_v2_stable] + environment: + <<: *stable-env + QDL_STABLE_INSTANCE_ID: stable-projector-4 + QDL_STABLE_CONSUMER_GROUP: stable-projector-v1 + QDL_STABLE_HTTP_PORT: "8230" + QDL_STABLE_GRPC_PORT: "8231" + QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-projector-4-audit.jsonl + QDL_STABLE_HEARTBEAT_PATH: /var/lib/qdl-stable/runtime/heartbeat/projector-4.json + QDL_STABLE_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 + QDL_STABLE_KAFKA_CLIENT_ID: stable-projector-4 + QDL_STABLE_KAFKA_CANONICAL_TOPIC: md.canonical.v2 + QDL_STABLE_KAFKA_CERT_ROOT: /stable-certs/projector + QDL_STABLE_TLS_CA_FILE: /stable-certs/projector/ca.crt + QDL_STABLE_TLS_CERT_FILE: /stable-certs/projector/client.crt + QDL_STABLE_TLS_KEY_FILE: /stable-certs/projector/client.key + QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' + QDL_STABLE_MAX_PENDING_RECORDS: "2048" + QDL_STABLE_MAX_PENDING_BYTES: "33554432" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" + QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" + QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS: "512" + volumes: + - stable_state:/var/lib/qdl-stable + - stable_tls:/stable-certs:ro + - ${QDL_STABLE_RUNTIME_DIR:?set QDL_STABLE_RUNTIME_DIR}:/runtime:ro + healthcheck: + test: [CMD-SHELL, "find /var/lib/qdl-stable/runtime/heartbeat/projector-4.json -newermt '-30 seconds' 2>/dev/null | grep -q ."] + interval: 20s + timeout: 5s + retries: 3 + start_period: 60s + depends_on: + stable_tls_init: {condition: service_completed_successfully} + stable_state_init: {condition: service_completed_successfully} + kafka1: {condition: service_healthy} + kafka2: {condition: service_healthy} + kafka3: {condition: service_healthy} + stable_redis: {condition: service_healthy} + stream_v2_active: {condition: service_started} + stream_v2_passive: {condition: service_started} + + projector_v2_5: + <<: *python + cpus: 1.00 + mem_limit: 768m + command: [python, -m, app.entrypoints.projector_v2_stable] + environment: + <<: *stable-env + QDL_STABLE_INSTANCE_ID: stable-projector-5 + QDL_STABLE_CONSUMER_GROUP: stable-projector-v1 + QDL_STABLE_HTTP_PORT: "8230" + QDL_STABLE_GRPC_PORT: "8231" + QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-projector-5-audit.jsonl + QDL_STABLE_HEARTBEAT_PATH: /var/lib/qdl-stable/runtime/heartbeat/projector-5.json + QDL_STABLE_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 + QDL_STABLE_KAFKA_CLIENT_ID: stable-projector-5 + QDL_STABLE_KAFKA_CANONICAL_TOPIC: md.canonical.v2 + QDL_STABLE_KAFKA_CERT_ROOT: /stable-certs/projector + QDL_STABLE_TLS_CA_FILE: /stable-certs/projector/ca.crt + QDL_STABLE_TLS_CERT_FILE: /stable-certs/projector/client.crt + QDL_STABLE_TLS_KEY_FILE: /stable-certs/projector/client.key + QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' + QDL_STABLE_MAX_PENDING_RECORDS: "2048" + QDL_STABLE_MAX_PENDING_BYTES: "33554432" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" + QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" + QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS: "512" + volumes: + - stable_state:/var/lib/qdl-stable + - stable_tls:/stable-certs:ro + - ${QDL_STABLE_RUNTIME_DIR:?set QDL_STABLE_RUNTIME_DIR}:/runtime:ro + healthcheck: + test: [CMD-SHELL, "find /var/lib/qdl-stable/runtime/heartbeat/projector-5.json -newermt '-30 seconds' 2>/dev/null | grep -q ."] + interval: 20s + timeout: 5s + retries: 3 + start_period: 60s + depends_on: + stable_tls_init: {condition: service_completed_successfully} + stable_state_init: {condition: service_completed_successfully} + kafka1: {condition: service_healthy} + kafka2: {condition: service_healthy} + kafka3: {condition: service_healthy} + stable_redis: {condition: service_healthy} + stream_v2_active: {condition: service_started} + stream_v2_passive: {condition: service_started} + + projector_v2_6: + <<: *python + cpus: 1.00 + mem_limit: 768m + command: [python, -m, app.entrypoints.projector_v2_stable] + environment: + <<: *stable-env + QDL_STABLE_INSTANCE_ID: stable-projector-6 + QDL_STABLE_CONSUMER_GROUP: stable-projector-v1 + QDL_STABLE_HTTP_PORT: "8230" + QDL_STABLE_GRPC_PORT: "8231" + QDL_STABLE_AUDIT_PATH: /var/lib/qdl-stable/runtime/stable-projector-6-audit.jsonl + QDL_STABLE_HEARTBEAT_PATH: /var/lib/qdl-stable/runtime/heartbeat/projector-6.json + QDL_STABLE_KAFKA_BOOTSTRAP_SERVERS: kafka1:9092,kafka2:9092,kafka3:9092 + QDL_STABLE_KAFKA_CLIENT_ID: stable-projector-6 + QDL_STABLE_KAFKA_CANONICAL_TOPIC: md.canonical.v2 + QDL_STABLE_KAFKA_CERT_ROOT: /stable-certs/projector + QDL_STABLE_TLS_CA_FILE: /stable-certs/projector/ca.crt + QDL_STABLE_TLS_CERT_FILE: /stable-certs/projector/client.crt + QDL_STABLE_TLS_KEY_FILE: /stable-certs/projector/client.key + QDL_STABLE_STREAM_INGEST_URLS_JSON: '["https://stream_v2_active:8200","https://stream_v2_passive:8200"]' + QDL_STABLE_MAX_PENDING_RECORDS: "2048" + QDL_STABLE_MAX_PENDING_BYTES: "33554432" + QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS: "512" + QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES: "8388608" + QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS: "512" + volumes: + - stable_state:/var/lib/qdl-stable + - stable_tls:/stable-certs:ro + - ${QDL_STABLE_RUNTIME_DIR:?set QDL_STABLE_RUNTIME_DIR}:/runtime:ro + healthcheck: + test: [CMD-SHELL, "find /var/lib/qdl-stable/runtime/heartbeat/projector-6.json -newermt '-30 seconds' 2>/dev/null | grep -q ."] + interval: 20s + timeout: 5s + retries: 3 + start_period: 60s depends_on: stable_tls_init: {condition: service_completed_successfully} stable_state_init: {condition: service_completed_successfully} diff --git a/poetry.lock b/poetry.lock index 57f1670..74221ad 100644 --- a/poetry.lock +++ b/poetry.lock @@ -26,20 +26,20 @@ files = [ [[package]] name = "anyio" -version = "4.13.0" +version = "4.15.1" description = "High-level concurrency and networking framework on top of asyncio or Trio" optional = false python-versions = ">=3.10" groups = ["main"] files = [ - {file = "anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708"}, - {file = "anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc"}, + {file = "anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101"}, + {file = "anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94"}, ] [package.dependencies] exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} idna = ">=2.8" -typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} +typing_extensions = {version = ">=4.16.0", markers = "python_version < \"3.15\""} [package.extras] trio = ["trio (>=0.32.0)"] @@ -2475,14 +2475,14 @@ files = [ [[package]] name = "typing-extensions" -version = "4.15.0" +version = "4.16.0" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548"}, - {file = "typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466"}, + {file = "typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8"}, + {file = "typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5"}, ] [[package]] diff --git a/qdl/api_v2/router.py b/qdl/api_v2/router.py index 3323158..86a7303 100644 --- a/qdl/api_v2/router.py +++ b/qdl/api_v2/router.py @@ -155,6 +155,12 @@ def _reference_requirement(model) -> ReferenceDataRequirement: basis_series=DomainBasisSeries(model.basis_series.value), basis_contract_type=model.basis_contract_type, max_freshness_ms=model.max_freshness_ms, + event_recency_policy=( + StalePolicy(model.event_recency_policy.value) + if model.event_recency_policy is not None + else None + ), + max_session_liveness_ms=model.max_session_liveness_ms, require_full_coverage=model.require_full_coverage, deadline_ms=model.deadline_ms, ) @@ -397,6 +403,50 @@ def _warmup(result) -> WarmupResponse: ) +def _warmup_batch_response( + request: Request, + access: DataPlaneAccess, + result, + requirements: tuple[DataRequirement, ...], +) -> BatchResponse: + """Build the public batch contract before a local batch lease is released.""" + + items = [] + for item, requirement in zip(result.results, requirements, strict=True): + problem = None + if item.problem is not None: + problem = _problem(QueryServiceError( + item.problem, + request_id=result.request_id, + instrument_uid=item.instrument_uid, + )) + warmup_data = None + if item.result is not None: + bound = type(item.result)( + item.result.request_id, + _bind_history_cursor( + request, + access, + requirement, + item.result.history, + ), + ) + warmup_data = _warmup(bound) + items.append(BatchItemResponse( + instrument_uid=item.instrument_uid, + status=item.status, + data=warmup_data, + problem=problem, + )) + return BatchResponse( + request_id=result.request_id, + partial=result.partial, + success_count=result.success_count, + error_count=result.error_count, + results=items, + ) + + def _reference_data(result) -> dict: """Serialize provider-authentic reference data without float coercion.""" @@ -791,41 +841,19 @@ async def warmup_batch( requirements, require_all=body.require_all, ) - result = await service.warmup_batch_async(batch, purpose=purpose) - items = [] - for item, requirement in zip(result.results, requirements, strict=True): - problem = None - if item.problem is not None: - problem = _problem(QueryServiceError( - item.problem, - request_id=result.request_id, - instrument_uid=item.instrument_uid, - )) - warmup_data = None - if item.result is not None: - bound = type(item.result)( - item.result.request_id, - _bind_history_cursor( - request, - access, - requirement, - item.result.history, - ), - ) - warmup_data = _warmup(bound) - items.append(BatchItemResponse( - instrument_uid=item.instrument_uid, - status=item.status, - data=warmup_data, - problem=problem, - )) - return BatchResponse( - request_id=result.request_id, - partial=result.partial, - success_count=result.success_count, - error_count=result.error_count, - results=items, - ) + + async def render(result): + response = _warmup_batch_response(request, access, result, requirements) + # Returning an already-rendered Response prevents FastAPI from doing a + # second Pydantic walk after the fully-local service lease is released. + return JSONResponse(content=response.model_dump(mode="json", by_alias=True)) + + complete = getattr(service, "warmup_batch_completed_async", None) + if callable(complete): + return await complete(batch, purpose=purpose, completion=render) + # Focused service doubles predating the internal completion hook retain the + # old public service protocol. Production V2QueryService always uses it. + return await render(await service.warmup_batch_async(batch, purpose=purpose)) @router.post("/market-data/reference:batch", response_model=ReferenceBatchResponse) diff --git a/qdl/certification/phase103_consumer_acceptance.py b/qdl/certification/phase103_consumer_acceptance.py index 98766b7..e52e52c 100644 --- a/qdl/certification/phase103_consumer_acceptance.py +++ b/qdl/certification/phase103_consumer_acceptance.py @@ -593,10 +593,24 @@ def validate_product_view( # provider session. The latter is admitted only through an explicit # session SLA. BOOK_DELTA remains non-price continuity/replay evidence: # callers must use a fresh BOOK_SNAPSHOT/QUOTE/MARK read to choose a price. - observed_quiet_continuity = ( - product.feed in {FeedType.TRADE, FeedType.BOOK_DELTA} + # Native BBO QUOTE is the narrow price-bearing exception: a signed source + # binding may explicitly declare it ON_CHANGE, and Query must have already + # made that exact receipt execution eligible. A generic stale QUOTE cannot + # self-upgrade through this path. + quiet_on_change_quote = ( + product.feed is FeedType.QUOTE and requirement.effective_event_recency_policy is StalePolicy.OBSERVE - and view.quality.event_recency_state == "STALE" + and "DELIVERY_ON_CHANGE" in view.quality.flags + and view.quality.execution_eligible + ) + observed_quiet_continuity = ( + ( + product.feed in {FeedType.TRADE, FeedType.BOOK_DELTA} + and requirement.effective_event_recency_policy is StalePolicy.OBSERVE + ) + or quiet_on_change_quote + ) and ( + view.quality.event_recency_state == "STALE" and view.quality.provider_session_state == "LIVE" ) # The session is the authority for an observed quiet continuity channel. diff --git a/qdl/certification/phase105_release.py b/qdl/certification/phase105_release.py index 0f51842..e7299f8 100644 --- a/qdl/certification/phase105_release.py +++ b/qdl/certification/phase105_release.py @@ -46,6 +46,7 @@ "v2_quality_state", "v2_session_state", "v2_session_liveness_ms", "v2_complete", "v2_execution_eligible", }) +_OPTIONAL_DELIVERY_SEMANTICS_FIELD = "v2_delivery_semantics" _RUNTIME_SCHEMA = "qdl.phase105c.runtime-handoff-evidence.v1" _ACCEPTANCE_SCHEMA = "qdl.phase105.v2-identity-acceptance.v1" _FALLBACK_SCHEMA = "qdl.phase105.v1-fallback-return.v1" @@ -147,7 +148,13 @@ def parse_release_observations(raw: object) -> tuple[ReleaseRouteObservation, .. values: list[ReleaseRouteObservation] = [] for index, item in enumerate(raw): value = _mapping(item, f"observation[{index}]") - if set(value) not in (_OBSERVATION_FIELDS, _OBSERVATION_FIELDS | _SESSION_OBSERVATION_FIELDS): + if set(value) not in ( + _OBSERVATION_FIELDS, + _OBSERVATION_FIELDS | _SESSION_OBSERVATION_FIELDS, + _OBSERVATION_FIELDS | _SESSION_OBSERVATION_FIELDS | { + _OPTIONAL_DELIVERY_SEMANTICS_FIELD + }, + ): raise ValueError("Phase 10.5-D observation fields differ from public contract") if not all( isinstance(value[field], str) and value[field] diff --git a/qdl/certification/phase105_release_observations.py b/qdl/certification/phase105_release_observations.py index 538af9d..9ceed58 100644 --- a/qdl/certification/phase105_release_observations.py +++ b/qdl/certification/phase105_release_observations.py @@ -31,6 +31,7 @@ "state", "provider_session_state", "provider_session_liveness_ms", "complete", "execution_eligible", }) +_DELIVERY_SEMANTICS_FIELD = "delivery_semantics" _CAPTURE_FIELDS = frozenset({"captured_at_ms", "cpu_millicores", "rss_bytes"}) _BUNDLE_FIELDS = frozenset({ "schema", @@ -90,9 +91,12 @@ def _require_positive_int(value: object, field: str) -> int: def _quality(value: object, field: str) -> dict[str, object]: - if not isinstance(value, Mapping) or set(value) not in ( - _QUALITY_FIELDS, _QUALITY_FIELDS | _SESSION_QUALITY_FIELDS - ): + allowed_shapes = ( + _QUALITY_FIELDS, + _QUALITY_FIELDS | _SESSION_QUALITY_FIELDS, + _QUALITY_FIELDS | _SESSION_QUALITY_FIELDS | {_DELIVERY_SEMANTICS_FIELD}, + ) + if not isinstance(value, Mapping) or set(value) not in allowed_shapes: raise ValueError(f"Phase 10.5 B3 {field} quality fields are invalid") gap_open = value.get("gap_open") if not isinstance(gap_open, bool): @@ -116,6 +120,10 @@ def _quality(value: object, field: str) -> dict[str, object]: v1_source_age_ms=None, v1_receive_age_ms=None, consumer_lag=0, cpu_millicores=0, rss_bytes=0, **_session_observation_fields(result, result), ) + if _DELIVERY_SEMANTICS_FIELD in value: + if value[_DELIVERY_SEMANTICS_FIELD] not in {"STRICT_EVENT", "ON_CHANGE"}: + raise ValueError(f"Phase 10.5 B3 {field}.delivery_semantics is invalid") + result[_DELIVERY_SEMANTICS_FIELD] = value[_DELIVERY_SEMANTICS_FIELD] return result @@ -126,12 +134,20 @@ def _session_observation_fields(primary, secondary) -> dict[str, object]: sessions = (primary["provider_session_state"], secondary["provider_session_state"]) ages = (primary["provider_session_liveness_ms"], secondary["provider_session_liveness_ms"]) # Either replica can block readiness; a healthy peer never hides a fault. + delivery_semantics = None + if ( + primary.get(_DELIVERY_SEMANTICS_FIELD) + == secondary.get(_DELIVERY_SEMANTICS_FIELD) + and primary.get(_DELIVERY_SEMANTICS_FIELD) in {"STRICT_EVENT", "ON_CHANGE"} + ): + delivery_semantics = primary[_DELIVERY_SEMANTICS_FIELD] return { "v2_quality_state": "LIVE" if states == ("LIVE", "LIVE") else next(s for s in states if s != "LIVE"), "v2_session_state": sessions[0] if sessions[0] == sessions[1] else "UNKNOWN", "v2_session_liveness_ms": max(ages) if all(a is not None for a in ages) else None, "v2_complete": primary["complete"] and secondary["complete"], "v2_execution_eligible": primary["execution_eligible"] and secondary["execution_eligible"], + "v2_delivery_semantics": delivery_semantics, } @@ -179,6 +195,14 @@ def compact_view_quality(view: object, *, observed_at_ns: int | None = None) -> } if all(hasattr(quality, key) for key in _SESSION_QUALITY_FIELDS): result.update({key: getattr(quality, key) for key in _SESSION_QUALITY_FIELDS}) + flags = getattr(quality, "flags", ()) + if not isinstance(flags, (tuple, list)) or any( + not isinstance(flag, str) for flag in flags + ): + raise ValueError("Phase 10.5 B3 view.quality flags are invalid") + result[_DELIVERY_SEMANTICS_FIELD] = ( + "ON_CHANGE" if "DELIVERY_ON_CHANGE" in flags else "STRICT_EVENT" + ) return _quality(result, "view") diff --git a/qdl/certification/reference_l2_acceptance.py b/qdl/certification/reference_l2_acceptance.py index 55e9dd0..261a934 100644 --- a/qdl/certification/reference_l2_acceptance.py +++ b/qdl/certification/reference_l2_acceptance.py @@ -19,7 +19,10 @@ _decimal_value, ) from qdl.consumer.manifest import ConsumerManifest, ConsumerManifestLoader -from qdl.query import ConsumerGrade, DataRequirement, FeedType, RecoveryPolicy +from qdl.data_quality.execution_mark_index import ( + validate_quiet_execution_mark_index_evidence, +) +from qdl.query import ConsumerGrade, DataRequirement, FeedType, RecoveryPolicy, StalePolicy from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan from qdl_sdk import Grade @@ -51,6 +54,13 @@ _STRICT_MARK_FRESHNESS_MS = 2_000 _ACCEPTANCE_TRANSPORT_MARGIN_SECONDS = 15.0 _ACCEPTANCE_TRANSPORT_MAX_SECONDS = 90.0 +_EXECUTION_MARK_INDEX_LIVE_ENDPOINT = ( + "qdl://stable-stream/internal/v2/execution/mark-index/latest" +) +_EXECUTION_MARK_INDEX_DELIVERY_STAGES = frozenset({ + "CANONICAL_READ_COMMITTED", + "SPOOL_CONFIRMED", +}) @dataclass(frozen=True, slots=True) @@ -239,6 +249,12 @@ def reference_request_for_requirement( "consumer_grade": Grade(requirement.consumer_grade.value), "source_policy_id": requirement.source_policy_id, "max_freshness_ms": requirement.max_freshness_ms, + "event_recency_policy": ( + requirement.event_recency_policy.value + if requirement.event_recency_policy is not None + else None + ), + "max_session_liveness_ms": requirement.max_session_liveness_ms, "require_full_coverage": requirement.require_full_coverage, "deadline_ms": 60_000, } @@ -483,6 +499,21 @@ def reference_quality( raise ValueError("reference response timing is in the future") source_age_ms = (observed_at_ns - newest_observed_ns) // _MILLISECOND_NS receive_age_ms = (observed_at_ns - received_at_ns) // _MILLISECOND_NS + quiet_execution_evidence = _quiet_execution_mark_index_evidence( + product, + item, + observed_at_ns=observed_at_ns, + ) + if quiet_execution_evidence is not None: + return { + "source_age_ms": int(source_age_ms), + "receive_age_ms": int(receive_age_ms), + "session_liveness_ms": quiet_execution_evidence.session_liveness_ms, + "session_checked_age_ms": quiet_execution_evidence.session_checked_age_ms, + "component_mark_age_ms": quiet_execution_evidence.component_mark_age_ms, + "component_index_age_ms": quiet_execution_evidence.component_index_age_ms, + "gap_open": False, + } if source_age_ms > (request.max_freshness_ms or 86_400_000): raise ValueError("reference response exceeds its governed freshness bound") return { @@ -492,6 +523,75 @@ def reference_quality( } +def _quiet_execution_mark_index_evidence( + product, + item, + *, + observed_at_ns: int, +): + """Recognize only the existing internal execution live-view response.""" + + requirement = product.requirement + request = product.sdk_requirement + if not ( + requirement.feed is FeedType.MARK_INDEX_PRICE + and requirement.consumer_grade is ConsumerGrade.EXECUTION + and requirement.effective_event_recency_policy is StalePolicy.OBSERVE + and request.product is ReferenceProduct.MARK_INDEX_PRICE + and request.consumer_grade is Grade.EXECUTION + and request.event_recency_policy is not None + and request.event_recency_policy.value == StalePolicy.OBSERVE.value + and request.start_time_ns is None + and request.end_time_ns is None + and request.max_session_liveness_ms is not None + and request.max_session_liveness_ms == requirement.max_session_liveness_ms + ): + return None + data = item.data + if ( + data is None + or len(data.observations) != 1 + or len(data.lineage) != 1 + or data.lineage[0].provider_endpoint != _EXECUTION_MARK_INDEX_LIVE_ENDPOINT + or data.lineage[0].source_role != "REFERENCE" + or data.coverage.terminal_reason != "LIVE_EXECUTION_VIEW" + ): + raise ValueError("quiet execution MARK/INDEX live-view lineage is invalid") + observation = data.observations[0] + labels = observation.labels + try: + source_event_time_ns = int(labels["source_event_time_ns"]) + provider_confirmation_ns = int(labels["provider_confirmation_ns"]) + connection_generation = int(labels["connection_generation"]) + gateway_lease_epoch = int(labels["gateway_lease_epoch"]) + spool_watermark = labels["spool_watermark_offset"] + spool_watermark_offset = ( + None if spool_watermark == "PENDING" else int(spool_watermark) + ) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("quiet execution MARK/INDEX provenance labels are malformed") from error + if ( + labels.get("execution_view") != "STABLE_STREAM_GATEWAY" + or labels.get("freshness_basis") not in {"SOURCE_EVENT", "PROVIDER_CONFIRMATION"} + or labels.get("delivery_stage") not in _EXECUTION_MARK_INDEX_DELIVERY_STAGES + or source_event_time_ns != observation.observed_at_ns + or provider_confirmation_ns != data.received_at_ns + or source_event_time_ns <= 0 + or provider_confirmation_ns <= 0 + or source_event_time_ns > provider_confirmation_ns + or provider_confirmation_ns > observed_at_ns + or connection_generation < 1 + or gateway_lease_epoch < 1 + or (spool_watermark_offset is not None and spool_watermark_offset < 0) + ): + raise ValueError("quiet execution MARK/INDEX provenance is invalid") + return validate_quiet_execution_mark_index_evidence( + labels, + at_ns=observed_at_ns, + max_session_liveness_ms=request.max_session_liveness_ms, + ) + + def validate_reference_batch( products: tuple[ReferenceAcceptanceProduct, ...], response: ReferenceBatchResponse, diff --git a/qdl/consumer/release.py b/qdl/consumer/release.py index 8c6d90a..0299c8a 100644 --- a/qdl/consumer/release.py +++ b/qdl/consumer/release.py @@ -579,6 +579,10 @@ class ReleaseRouteObservation: v2_session_liveness_ms: int | None = None v2_complete: bool | None = None v2_execution_eligible: bool | None = None + # This is release-evidence metadata derived from the existing public + # quality flags, not a new consumer-facing data-plane field. Older signed + # evidence remains parseable but cannot certify a quiet BBO route. + v2_delivery_semantics: str | None = None def __post_init__(self) -> None: if ( @@ -616,6 +620,10 @@ def __post_init__(self) -> None: )) ): raise ValueError("release route typed session evidence is invalid") + if self.v2_delivery_semantics is not None and self.v2_delivery_semantics not in { + "STRICT_EVENT", "ON_CHANGE" + }: + raise ValueError("release route delivery semantics are invalid") def public_record(self) -> dict[str, object]: result = { @@ -640,6 +648,8 @@ def public_record(self) -> dict[str, object]: "v2_complete": self.v2_complete, "v2_execution_eligible": self.v2_execution_eligible, }) + if self.v2_delivery_semantics is not None: + result["v2_delivery_semantics"] = self.v2_delivery_semantics return result @@ -665,10 +675,22 @@ def v2_observation_is_current(requirement, observed: ReleaseRouteObservation) -> maximum = requirement.max_freshness_ms if maximum is None or all(age <= maximum for age in ages): return True - return ( + if ( requirement.feed.value in {"TRADE", "BOOK_DELTA"} and requirement.effective_event_recency_policy.value == "OBSERVE" - and session_live and observed.v2_execution_eligible is False + and session_live + and observed.v2_execution_eligible is False + ): + return True + # An old BBO event can be current only when its sealed source declaration + # proves native update-on-change delivery and Query has independently made + # it execution eligible. A generic observed QUOTE cannot self-upgrade. + return ( + requirement.feed.value == "QUOTE" + and requirement.effective_event_recency_policy.value == "OBSERVE" + and session_live + and observed.v2_execution_eligible is True + and observed.v2_delivery_semantics == "ON_CHANGE" ) diff --git a/qdl/data_quality/__init__.py b/qdl/data_quality/__init__.py index 4dc50c6..0c682ee 100644 --- a/qdl/data_quality/__init__.py +++ b/qdl/data_quality/__init__.py @@ -9,6 +9,21 @@ ValidationLevel, ) from qdl.data_quality.calendar import CalendarAssessment, assess_bar_availability +from qdl.data_quality.binding_decision import ( + AvailabilityClass, + BindingQualityDecision, + BindingQualityInput, + ComponentEvidence, + FeedSemantics, + availability_for, + evaluate_binding_quality, + freshness_verdict, + semantics_for, +) +from qdl.data_quality.execution_mark_index import ( + QuietExecutionMarkIndexEvidence, + validate_quiet_execution_mark_index_evidence, +) from qdl.data_quality.source_authority import ( AuthorityAction, SourceAuthorityController, @@ -19,6 +34,12 @@ __all__ = [ "CalendarAssessment", + "AvailabilityClass", + "BindingQualityDecision", + "BindingQualityInput", + "ComponentEvidence", + "FeedSemantics", + "QuietExecutionMarkIndexEvidence", "AuthorityAction", "FeedKey", "FeedQualityLedger", @@ -31,4 +52,9 @@ "SourceRole", "ValidationLevel", "assess_bar_availability", + "availability_for", + "evaluate_binding_quality", + "freshness_verdict", + "semantics_for", + "validate_quiet_execution_mark_index_evidence", ] diff --git a/qdl/data_quality/binding_decision.py b/qdl/data_quality/binding_decision.py new file mode 100644 index 0000000..e63968c --- /dev/null +++ b/qdl/data_quality/binding_decision.py @@ -0,0 +1,343 @@ +"""One bounded, provider-neutral quality decision for one declared binding. + +The canonical event is immutable evidence; its age is not a synonym for a +provider session being alive. This module makes that distinction explicit so +the stable query edge and offline audit cannot accidentally apply different +rules to the same binding. Rust carries the same pure decision through the +shared golden corpus in ``contracts/golden/quality``. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from qdl._compat import StrEnum + + +class FeedSemantics(StrEnum): + STRICT_EVENT = "STRICT_EVENT" + QUIET_SESSION = "QUIET_SESSION" + FINAL_SCHEDULED = "FINAL_SCHEDULED" + + +class AvailabilityClass(StrEnum): + ACTIVE = "ACTIVE" + EXPECTED_V1_PRIMARY = "EXPECTED_V1_PRIMARY" + EXPECTED_DARK = "EXPECTED_DARK" + OUT_OF_SESSION = "OUT_OF_SESSION" + + +@dataclass(frozen=True, slots=True) +class ComponentEvidence: + """Receipt age and explicit quiet cadence for one paired component.""" + + name: str + receipt_age_ms: int + quiet_after_ms: int + + def __post_init__(self) -> None: + if ( + not self.name.strip() + or self.receipt_age_ms < 0 + or self.quiet_after_ms < 1 + ): + raise ValueError("component quality evidence is invalid") + + +@dataclass(frozen=True, slots=True) +class BindingQualityInput: + """Typed facts used to judge a single binding without I/O or mutation.""" + + binding_id: str + instrument_uid: str + feed: str + source_role: str + authoritative: bool + acquisition_enabled: bool + acquisition_mode: str + market_open: bool + event_present: bool + event_age_ms: int | None + event_limit_ms: int + event_recency_policy: str + session_state: str + session_liveness_ms: int | None + session_limit_ms: int | None + # Provider delivery behavior is a source contract, not a caller choice. + # Native BBO can legitimately be quiet while the best bid/offer is + # unchanged; every other lane remains strict by default. + delivery_semantics: str = "STRICT_EVENT" + components: tuple[ComponentEvidence, ...] = () + generation_matches: bool = True + config_matches: bool = True + gap_open: bool = False + book_verified: bool = True + final_bar: bool = True + require_final_bar: bool = False + watermark_offset: int = 0 + allow_quiet_execution: bool = False + flags: tuple[str, ...] = () + + def __post_init__(self) -> None: + if ( + not self.binding_id.strip() + or not self.instrument_uid.strip() + or not self.feed.strip() + or self.source_role not in {"PRIMARY", "SECONDARY", "REFERENCE", "BACKFILL"} + or not self.acquisition_mode.strip() + or self.event_limit_ms < 1 + or self.event_recency_policy not in {"BLOCK", "PAUSE", "OBSERVE"} + or self.delivery_semantics not in {"STRICT_EVENT", "ON_CHANGE"} + or self.session_state not in { + "LIVE", "STALE", "DISCONNECTED", "UNKNOWN", "NOT_APPLICABLE" + } + or self.watermark_offset < 0 + ): + raise ValueError("binding quality input is invalid") + if self.event_present != (self.event_age_ms is not None): + raise ValueError("event presence and age must agree") + if self.event_age_ms is not None and self.event_age_ms < 0: + raise ValueError("event age cannot be negative") + if self.session_liveness_ms is not None and self.session_liveness_ms < 0: + raise ValueError("session liveness cannot be negative") + if self.session_limit_ms is not None and self.session_limit_ms < 1: + raise ValueError("session limit must be positive") + + +@dataclass(frozen=True, slots=True) +class BindingQualityDecision: + """Serializable policy answer derived only from :class:`BindingQualityInput`.""" + + binding_id: str + instrument_uid: str + feed: str + semantics: FeedSemantics + delivery_semantics: str + availability: AvailabilityClass + state: str + event_recency_state: str + provider_session_state: str + provider_session_liveness_ms: int | None + complete: bool + execution_eligible: bool + watermark_offset: int + reason_codes: tuple[str, ...] + + def as_mapping(self) -> dict[str, object]: + """Stable bounded audit shape; it contains no raw provider payload.""" + + return { + "binding_id": self.binding_id, + "instrument_uid": self.instrument_uid, + "feed": self.feed, + "semantics": self.semantics.value, + "delivery_semantics": self.delivery_semantics, + "availability": self.availability.value, + "state": self.state, + "event_recency_state": self.event_recency_state, + "provider_session_state": self.provider_session_state, + "provider_session_liveness_ms": self.provider_session_liveness_ms, + "complete": self.complete, + "execution_eligible": self.execution_eligible, + "watermark_offset": self.watermark_offset, + "reason_codes": list(self.reason_codes), + } + + +def semantics_for( + *, + feed: str, + event_recency_policy: str, + require_final_bar: bool, + delivery_semantics: str = "STRICT_EVENT", +) -> FeedSemantics: + normalized_feed = feed.upper() + if normalized_feed == "BAR" or require_final_bar: + return FeedSemantics.FINAL_SCHEDULED + if ( + event_recency_policy == "OBSERVE" + and ( + normalized_feed in {"TRADE", "BOOK_DELTA", "MARK_INDEX_PRICE"} + or ( + normalized_feed == "QUOTE" + and delivery_semantics == "ON_CHANGE" + ) + ) + ): + return FeedSemantics.QUIET_SESSION + return FeedSemantics.STRICT_EVENT + + +def availability_for(value: BindingQualityInput) -> AvailabilityClass: + """Classify declared non-serving inventory before looking at event age.""" + + # VN uses the vendor-edge / V1 compatibility train until its independent + # market-hours certificate. It is intentionally not a broken V2 feed. + if value.acquisition_mode == "PYTHON_VENDOR_SDK": + return AvailabilityClass.EXPECTED_V1_PRIMARY + if not value.acquisition_enabled: + return AvailabilityClass.EXPECTED_DARK + if not value.market_open: + return AvailabilityClass.OUT_OF_SESSION + return AvailabilityClass.ACTIVE + + +def _append_once(target: list[str], *values: str) -> None: + for value in values: + if value and value not in target: + target.append(value) + + +def evaluate_binding_quality(value: BindingQualityInput) -> BindingQualityDecision: + """Evaluate strict, quiet and scheduled feeds without weakening policy. + + A quiet channel is only *observable* after all of its session/generation/ + cadence fences pass. ``allow_quiet_execution`` is deliberately explicit: + the durable generic query path cannot make a quiet price executable merely + because its session is alive. + """ + + semantics = semantics_for( + feed=value.feed, + event_recency_policy=value.event_recency_policy, + require_final_bar=value.require_final_bar, + delivery_semantics=value.delivery_semantics, + ) + availability = availability_for(value) + reasons = list(value.flags) + event_state = "NOT_APPLICABLE" + if value.event_present: + assert value.event_age_ms is not None + event_state = "STALE" if value.event_age_ms > value.event_limit_ms else "LIVE" + if event_state == "STALE": + _append_once(reasons, "LAST_EVENT_STALE") + session_state = value.session_state + session_ok = session_state == "NOT_APPLICABLE" + if value.session_limit_ms is not None: + session_ok = ( + session_state == "LIVE" + and value.session_liveness_ms is not None + and value.session_liveness_ms <= value.session_limit_ms + ) + elif session_state in {"STALE", "DISCONNECTED", "UNKNOWN"}: + session_ok = False + + component_ok = True + for component in value.components: + if component.receipt_age_ms > component.quiet_after_ms: + component_ok = False + _append_once(reasons, f"COMPONENT_{component.name.upper()}_STALE") + if not value.generation_matches: + _append_once(reasons, "GENERATION_MISMATCH") + if not value.config_matches: + _append_once(reasons, "CONFIG_REVISION_MISMATCH") + if value.gap_open: + _append_once(reasons, "OPEN_SEQUENCE_GAP") + if not value.book_verified: + _append_once(reasons, "BOOK_SEQUENCE_UNVERIFIED") + if value.require_final_bar and not value.final_bar: + _append_once(reasons, "BAR_NOT_FINAL") + if not session_ok: + if session_state in {"STALE", "DISCONNECTED", "UNKNOWN"}: + _append_once(reasons, f"SOURCE_SESSION_{session_state}") + else: + _append_once(reasons, "SOURCE_SESSION_HEARTBEAT_EXPIRED") + + if availability is AvailabilityClass.EXPECTED_V1_PRIMARY: + _append_once(reasons, "EXPECTED_V1_PRIMARY") + state = "DISABLED" + elif availability is AvailabilityClass.EXPECTED_DARK: + _append_once(reasons, "EXPECTED_DARK") + state = "DISABLED" + elif availability is AvailabilityClass.OUT_OF_SESSION: + _append_once(reasons, "OUT_OF_SESSION") + state = "MARKET_CLOSED" + elif not value.event_present: + _append_once(reasons, "NO_DURABLE_EVENT") + state = "NOT_READY" + elif value.gap_open: + state = "GAPPED" + elif not value.book_verified or (value.require_final_bar and not value.final_bar): + state = "SYNCING" + elif not value.generation_matches or not value.config_matches or not session_ok: + state = "STALE" + elif not component_ok: + state = "STALE" + elif semantics is not FeedSemantics.QUIET_SESSION and event_state == "STALE": + state = "STALE" + else: + state = "LIVE" + + complete = ( + value.event_present + and not value.gap_open + and value.book_verified + and (not value.require_final_bar or value.final_bar) + ) + event_ok = event_state in {"LIVE", "NOT_APPLICABLE"} + quiet_execution_ok = ( + semantics is FeedSemantics.QUIET_SESSION + and value.allow_quiet_execution + and session_ok + and component_ok + ) + execution_eligible = ( + availability is AvailabilityClass.ACTIVE + and value.authoritative + and value.source_role == "PRIMARY" + and state == "LIVE" + and complete + and (event_ok or quiet_execution_ok) + ) + return BindingQualityDecision( + binding_id=value.binding_id, + instrument_uid=value.instrument_uid, + feed=value.feed.upper(), + semantics=semantics, + delivery_semantics=value.delivery_semantics, + availability=availability, + state=state, + event_recency_state=event_state, + provider_session_state=session_state, + provider_session_liveness_ms=value.session_liveness_ms, + complete=complete, + execution_eligible=execution_eligible, + watermark_offset=value.watermark_offset, + reason_codes=tuple(reasons), + ) + + +def freshness_verdict( + *, + state: str, + freshness_ms: int, + event_recency_policy: str, + max_freshness_ms: int | None, + provider_session_state: str, + provider_session_liveness_ms: int | None, + max_session_liveness_ms: int | None, +) -> tuple[bool, str | None]: + """Shared query-admission predicate for already-materialized quality.""" + + if state == "MARKET_CLOSED": + return True, None + # Gap/completeness are evaluated by the caller's declared gap policy after + # this freshness verdict. Collapsing them into DATA_STALE would lose the + # actionable OPEN_SEQUENCE_GAP error contract. + if state in {"STALE", "OFFLINE", "UNAVAILABLE"}: + return False, "EVENT_AGE" + if provider_session_state in {"STALE", "DISCONNECTED", "UNKNOWN"}: + return False, "SESSION_STATE" + if max_session_liveness_ms is not None and not ( + provider_session_state == "LIVE" + and provider_session_liveness_ms is not None + and provider_session_liveness_ms <= max_session_liveness_ms + ): + return False, "SESSION_LIVENESS" + if ( + max_freshness_ms is not None + and freshness_ms > max_freshness_ms + and event_recency_policy in {"BLOCK", "PAUSE"} + ): + return False, "EVENT_AGE" + return True, None diff --git a/qdl/data_quality/execution_mark_index.py b/qdl/data_quality/execution_mark_index.py new file mode 100644 index 0000000..b65f7b4 --- /dev/null +++ b/qdl/data_quality/execution_mark_index.py @@ -0,0 +1,93 @@ +"""Pure validation for the explicit execution MARK/INDEX quiet contract.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Mapping + + +_VALID_RECENCY_MODES = frozenset({ + "STRICT_EVENT_SESSION_LIVE", + "COMPONENT_SESSION_LIVE", +}) +_MIN_COMPONENT_CADENCE_MS = 250 +_MAX_COMPONENT_CADENCE_MS = 120_000 + + +@dataclass(frozen=True, slots=True) +class QuietExecutionMarkIndexEvidence: + """Bounded session/component ages derived from immutable response labels.""" + + session_liveness_ms: int + session_checked_age_ms: int + component_mark_age_ms: int + component_index_age_ms: int + component_mark_quiet_after_ms: int + component_index_quiet_after_ms: int + + +def validate_quiet_execution_mark_index_evidence( + labels: Mapping[str, str], + *, + at_ns: int, + max_session_liveness_ms: int | None, +) -> QuietExecutionMarkIndexEvidence: + """Validate the signed live-view proof without rewriting event lineage. + + Only an execution MARK/INDEX request already selected by its caller reaches + this helper. It therefore validates the current session and paired-component + cadence that make an old, unchanged source event observable. Missing or + malformed evidence always fails closed. + """ + + if max_session_liveness_ms is None or max_session_liveness_ms < 1: + raise ValueError("quiet execution MARK/INDEX contract is incomplete") + if ( + labels.get("event_recency_policy") != "OBSERVE" + or labels.get("recency_mode") not in _VALID_RECENCY_MODES + or labels.get("provider_session_state") != "LIVE" + ): + raise ValueError("quiet execution MARK/INDEX session evidence is not live") + try: + session_liveness_ms = int(labels["provider_session_liveness_ms"]) + session_checked_at_ns = int(labels["provider_session_checked_at_ns"]) + component_values = tuple( + ( + name, + int(labels[f"component_{name.lower()}_received_at_ns"]), + int(labels[f"component_{name.lower()}_quiet_after_ms"]), + ) + for name in ("MARK", "INDEX") + ) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("quiet execution MARK/INDEX evidence is malformed") from error + if ( + session_liveness_ms < 0 + or session_checked_at_ns <= 0 + or session_checked_at_ns > at_ns + ): + raise ValueError("quiet execution MARK/INDEX session clock is invalid") + session_checked_age_ms = (at_ns - session_checked_at_ns) // 1_000_000 + if session_liveness_ms + session_checked_age_ms > max_session_liveness_ms: + raise ValueError("quiet execution MARK/INDEX provider session exceeded its SLA") + + component_ages: dict[str, tuple[int, int]] = {} + for name, receipt_ns, cadence_ms in component_values: + if ( + receipt_ns <= 0 + or not _MIN_COMPONENT_CADENCE_MS <= cadence_ms <= _MAX_COMPONENT_CADENCE_MS + or receipt_ns > at_ns + ): + raise ValueError("quiet execution MARK/INDEX component evidence is invalid") + age_ms = (at_ns - receipt_ns) // 1_000_000 + if age_ms > cadence_ms: + raise ValueError("quiet execution MARK/INDEX component exceeded its cadence") + component_ages[name] = (age_ms, cadence_ms) + return QuietExecutionMarkIndexEvidence( + session_liveness_ms=session_liveness_ms, + session_checked_age_ms=int(session_checked_age_ms), + component_mark_age_ms=component_ages["MARK"][0], + component_index_age_ms=component_ages["INDEX"][0], + component_mark_quiet_after_ms=component_ages["MARK"][1], + component_index_quiet_after_ms=component_ages["INDEX"][1], + ) diff --git a/qdl/query/reference.py b/qdl/query/reference.py index 8d33504..44944ed 100644 --- a/qdl/query/reference.py +++ b/qdl/query/reference.py @@ -22,6 +22,7 @@ DataRequirement, FeedType, RecoveryPolicy, + StalePolicy, ) from qdl.reference.contracts import ( BasisSeries, @@ -85,6 +86,8 @@ class ReferenceDataRequirement: basis_series: BasisSeries = BasisSeries.NATIVE basis_contract_type: str | None = None max_freshness_ms: int | None = None + event_recency_policy: StalePolicy | None = None + max_session_liveness_ms: int | None = None require_full_coverage: bool = True deadline_ms: int = 20_000 @@ -115,6 +118,32 @@ def __post_init__(self) -> None: raise ValueError("reference interval cannot be blank") if self.max_freshness_ms is not None and self.max_freshness_ms <= 0: raise ValueError("reference max_freshness_ms must be positive") + if ( + self.max_session_liveness_ms is not None + and self.max_session_liveness_ms <= 0 + ): + raise ValueError("reference max_session_liveness_ms must be positive") + if self.event_recency_policy is not None and not isinstance( + self.event_recency_policy, StalePolicy + ): + raise TypeError("reference event_recency_policy must use StalePolicy") + if ( + self.event_recency_policy is StalePolicy.UNSPECIFIED + or ( + self.event_recency_policy is StalePolicy.OBSERVE + and self.max_session_liveness_ms is None + ) + ): + raise ValueError( + "observed reference event recency requires a provider session SLA" + ) + if ( + self.event_recency_policy is StalePolicy.OBSERVE + and not execution_mark_snapshot + ): + raise ValueError( + "observed reference event recency only applies to execution MARK_INDEX_PRICE" + ) if not 100 <= self.deadline_ms <= 120_000: raise ValueError("reference deadline_ms must be between 100 and 120000") self._validate_shape() @@ -136,6 +165,8 @@ def data_requirement(self) -> DataRequirement: # provider pagination limit outside the access-control boundary. warmup_limit=self.limit, max_freshness_ms=self.max_freshness_ms, + event_recency_policy=self.event_recency_policy, + max_session_liveness_ms=self.max_session_liveness_ms, require_full_coverage=self.require_full_coverage, require_final_bars=False, recovery=RecoveryPolicy.FRESH_SNAPSHOT, @@ -220,6 +251,10 @@ def _validate_shape(self) -> None: elif self.basis_contract_type is not None or self.basis_series is not BasisSeries.NATIVE: raise ValueError("basis selector fields only apply to BASIS") + @property + def effective_event_recency_policy(self) -> StalePolicy: + return self.event_recency_policy or StalePolicy.BLOCK + @dataclass(frozen=True, slots=True) class ReferenceBatchRequirement: diff --git a/qdl/query/service.py b/qdl/query/service.py index 8901ef1..3fa03a8 100644 --- a/qdl/query/service.py +++ b/qdl/query/service.py @@ -5,9 +5,13 @@ import uuid import time from dataclasses import dataclass, replace -from typing import Callable +from typing import Awaitable, Callable, TypeVar from qdl.adapters.intervals import canonical_interval_ms +from qdl.data_quality.binding_decision import freshness_verdict +from qdl.data_quality.execution_mark_index import ( + validate_quiet_execution_mark_index_evidence, +) from qdl.domain.calendar import trading_calendar_for_id from qdl.domain.instrument import InstrumentRecord from qdl.query.contracts import ( @@ -42,6 +46,7 @@ ) from qdl.warmup.executor import BoundedWarmupExecutor, RetryableWarmupError from qdl.reference.batch import ReferenceBatch +from qdl.reference.execution_live import ExecutionMarkIndexReader from qdl.reference.contracts import ( ReferenceBatchResult, ReferenceProduct, @@ -50,6 +55,14 @@ ) +_EXECUTION_MARK_INDEX_LIVE_ENDPOINT = ( + "qdl://stable-stream/internal/v2/execution/mark-index/latest" +) + + +_BatchCompletion = TypeVar("_BatchCompletion") + + def _freshness_verdict(requirement, quality) -> tuple[bool, str | None]: """Split the freshness verdict into its three independent causes. @@ -60,26 +73,15 @@ def _freshness_verdict(requirement, quality) -> tuple[bool, str | None]: rule is exactly the one this replaced; only the reason is new. """ - if quality.state == "MARKET_CLOSED": - return True, None - if quality.state in {"STALE", "OFFLINE", "UNAVAILABLE"}: - return False, STALE_REASON_EVENT_AGE - if quality.provider_session_state in {"STALE", "DISCONNECTED", "UNKNOWN"}: - return False, STALE_REASON_SESSION_STATE - if requirement.max_session_liveness_ms is not None and not ( - quality.provider_session_state == "LIVE" - and quality.provider_session_liveness_ms is not None - and quality.provider_session_liveness_ms <= requirement.max_session_liveness_ms - ): - return False, STALE_REASON_SESSION_LIVENESS - if ( - requirement.max_freshness_ms is not None - and quality.freshness_ms > requirement.max_freshness_ms - and requirement.effective_event_recency_policy - in {StalePolicy.BLOCK, StalePolicy.PAUSE} - ): - return False, STALE_REASON_EVENT_AGE - return True, None + return freshness_verdict( + state=quality.state, + freshness_ms=quality.freshness_ms, + event_recency_policy=requirement.effective_event_recency_policy.value, + max_freshness_ms=requirement.max_freshness_ms, + provider_session_state=quality.provider_session_state, + provider_session_liveness_ms=quality.provider_session_liveness_ms, + max_session_liveness_ms=requirement.max_session_liveness_ms, + ) class QueryServiceError(RuntimeError): @@ -177,6 +179,82 @@ class ReadinessResult: results: tuple[ReadinessItemResult, ...] +class _LocalBatchAdmissionRejected(RuntimeError): + """A bounded canonical-cache batch lane cannot accept more queued work.""" + + +class _LocalBatchAdmission: + """Serialize expensive local history snapshots without touching venue policy. + + A local batch can decode up to fifty independent retained BAR tails. It is + one cache/materialization unit, not fifty independent provider calls, so + per-item executor permits cannot provide a meaningful fairness boundary. + The pending count includes the active batch and is intentionally small: + one incumbent batch plus the four declared stable consumer lanes fits in + the reader while preventing abandoned client work from piling up. A queued + lane still has a finite admission wait, derived from the request's + declared work deadline; it is never provider pacing. + """ + + def __init__(self, *, max_active: int = 1, max_pending: int = 5) -> None: + if max_active != 1: + raise ValueError("local canonical batch admission currently requires one active lane") + if max_pending < max_active: + raise ValueError("local canonical batch pending bound must include active work") + self._gate = asyncio.Semaphore(max_active) + self._lock = asyncio.Lock() + self._max_pending = max_pending + self._pending = 0 + self._active = 0 + self._admitted = 0 + self._rejected = 0 + self._queue_wait_timeouts = 0 + + async def run(self, work, *, wait_timeout_ms: int | None = None): + if wait_timeout_ms is not None and wait_timeout_ms < 1: + raise ValueError("local canonical batch admission wait must be positive") + async with self._lock: + if self._pending >= self._max_pending: + self._rejected += 1 + raise _LocalBatchAdmissionRejected( + "local canonical-cache batch admission is at capacity" + ) + self._pending += 1 + acquired = False + try: + try: + if wait_timeout_ms is None: + await self._gate.acquire() + else: + await asyncio.wait_for( + self._gate.acquire(), timeout=wait_timeout_ms / 1_000 + ) + except asyncio.TimeoutError as error: + self._queue_wait_timeouts += 1 + raise _LocalBatchAdmissionRejected( + "local canonical-cache batch admission wait exceeded the declared deadline" + ) from error + acquired = True + self._active += 1 + self._admitted += 1 + return await work() + finally: + if acquired: + self._active -= 1 + self._gate.release() + async with self._lock: + self._pending -= 1 + + def stats(self) -> dict[str, int]: + return { + "active": self._active, + "pending": self._pending, + "admitted": self._admitted, + "rejected": self._rejected, + "queue_wait_timeouts": self._queue_wait_timeouts, + } + + class V2QueryService: """Provider-neutral policy boundary shared by REST, gRPC and SDK.""" @@ -190,9 +268,16 @@ def __init__( warmup_executor: BoundedWarmupExecutor | None = None, reference_batch: ReferenceBatch | None = None, reference_source_id: Callable[[InstrumentRecord], str] | None = None, + execution_mark_index_reader: ExecutionMarkIndexReader | None = None, ) -> None: if reference_batch is not None and reference_source_id is None: raise ValueError("reference batch requires an explicit source-id resolver") + if execution_mark_index_reader is not None and ( + reference_batch is None or reference_source_id is None + ): + raise ValueError( + "execution MARK/INDEX live reader requires the reference policy boundary" + ) self.instruments = instruments self.backend = backend self.entitlements = entitlements @@ -200,6 +285,8 @@ def __init__( self.warmup_executor = warmup_executor or BoundedWarmupExecutor() self.reference_batch = reference_batch self._reference_source_id = reference_source_id + self.execution_mark_index_reader = execution_mark_index_reader + self._local_batch_admission = _LocalBatchAdmission() self.last_batch_evidence: dict[str, object] = {} self.last_reference_batch_evidence: dict[str, object] = {} @@ -265,6 +352,29 @@ def warmup( request_id=request_id, instrument_uid=requirement.instrument_uid, ) from error + return self._warmup_from_history( + requirement, + history, + purpose=purpose, + request_id=request_id, + ) + + def _warmup_from_history( + self, + requirement: DataRequirement, + history: HistoryResult | None, + *, + purpose: AccessPurpose, + request_id: str, + ) -> WarmupResult: + """Apply the public warmup contract to an already-read history view. + + Both one-item and bounded local batch reads deliberately reach this + exact method. Batch materialization may change SQLite read shape, but + it cannot change readiness, coverage, quality, cursor or execution + eligibility semantics. + """ + if history is None or not history.items: self._raise_not_ready(requirement, request_id) quality = history.items[-1].quality @@ -316,6 +426,31 @@ def warmup_batch( ) return BatchQueryResult(request_id, tuple(results)) + def _local_batch_admission_for(self) -> _LocalBatchAdmission: + """Lazily preserve compatibility for focused service test doubles.""" + + admission = getattr(self, "_local_batch_admission", None) + if admission is None: + admission = _LocalBatchAdmission() + self._local_batch_admission = admission + return admission + + @staticmethod + def _consume_detached_local_batch(task: asyncio.Task) -> None: + """Drain a shielded local batch after its HTTP caller went away.""" + + try: + task.result() + except asyncio.CancelledError: + # The original caller has already observed cancellation. The task + # owns its admission lease until its bounded local work finishes. + pass + except Exception: + # The original caller has already observed cancellation. The task + # owns its admission lease until its bounded local work finishes; + # retrieving the terminal result prevents an orphan warning. + pass + async def warmup_async( self, requirement: DataRequirement, @@ -352,6 +487,33 @@ async def warmup_batch_async( request_id: str | None = None, ) -> BatchQueryResult: """Execute a batch concurrently without hiding item-level failures.""" + + async def return_result(result: BatchQueryResult) -> BatchQueryResult: + return result + + return await self.warmup_batch_completed_async( + batch, + purpose=purpose, + completion=return_result, + request_id=request_id, + ) + + async def warmup_batch_completed_async( + self, + batch: BatchRequirement, + *, + purpose: AccessPurpose, + completion: Callable[[BatchQueryResult], Awaitable[_BatchCompletion]], + request_id: str | None = None, + ) -> _BatchCompletion: + """Run an internal response completion under a fully-local batch lease. + + The public batch method retains its ``BatchQueryResult`` contract. The + REST router alone supplies a completion that binds cursors and renders + its public JSON before this finite canonical-cache lane is released. + That prevents a second large local HTTP response from overlapping the + first response's CPU-heavy conversion after the cache read is done. + """ request_id = request_id or self.request_id() executor_before = self.warmup_executor.stats() backend_before = self._warmup_backend_stats() @@ -365,8 +527,53 @@ def provider(requirement: DataRequirement) -> str: except KeyError: return "UNKNOWN" + local_requirements = tuple( + requirement + for requirement in batch.requirements + if provider(requirement) == "LOCAL_CANONICAL_CACHE" + ) + history_many = getattr(self.backend, "history_many", None) + fully_local_batch = bool( + local_requirements + and len(local_requirements) == len(batch.requirements) + and callable(history_many) + ) + local_history_lock = asyncio.Lock() + local_histories_task = None + prefetched_local_histories = None + + async def local_history(requirement: DataRequirement): + """Share one immutable local snapshot after an item is admitted.""" + + nonlocal local_histories_task + if prefetched_local_histories is not None: + return prefetched_local_histories[requirement] + if fully_local_batch: + raise RuntimeError("fully-local batch reached item work before materialization") + async with local_history_lock: + if local_histories_task is None: + local_histories_task = asyncio.create_task( + asyncio.to_thread(history_many, local_requirements) + ) + task = local_histories_task + histories = await asyncio.shield(task) + return histories[requirement] + async def work(requirement: DataRequirement) -> WarmupResult: try: + if ( + provider(requirement) == "LOCAL_CANONICAL_CACHE" + and callable(history_many) + ): + history = await local_history(requirement) + if isinstance(history, Exception): + raise history + return self._warmup_from_history( + requirement, + history, + purpose=purpose, + request_id=request_id, + ) return await asyncio.to_thread( self.warmup, requirement, @@ -389,89 +596,167 @@ def deadline(requirement: DataRequirement) -> int: specification = requirement.warmup_specification return specification.deadline_ms if specification else 20_000 - executions = await self.warmup_executor.execute( - batch.requirements, - work=work, - identity=lambda requirement: requirement, - provider=provider, - deadline_ms=deadline, - ) - results = [] - for execution in executions: - requirement = execution.item - if execution.ok: - assert execution.value is not None + async def execute_items(): + return await self.warmup_executor.execute( + batch.requirements, + work=work, + identity=lambda requirement: requirement, + provider=provider, + deadline_ms=deadline, + ) + + # One whole-local batch has one deterministic admission budget. The + # constituent work may have different declared deadlines, so a queue + # cannot wait longer than the most restrictive consumer item. Its + # execution deadline remains owned by the local warmup executor after + # this gate admits the batch. + local_admission_wait_ms = min( + deadline(requirement) for requirement in local_requirements + ) if local_requirements else None + + def assemble_batch(executions) -> BatchQueryResult: + results = [] + for execution in executions: + requirement = execution.item + if execution.ok: + assert execution.value is not None + results.append( + BatchItemResult( + requirement.instrument_uid, + "OK", + result=execution.value, + ) + ) + continue + error = execution.error + if isinstance(error, RetryableWarmupError) and isinstance( + error.cause, QueryServiceError + ): + problem = error.cause.problem + elif isinstance(error, QueryServiceError): + problem = error.problem + elif isinstance(error, RetryableWarmupError): + problem = QueryProblem( + CanonicalErrorCode.DEPENDENCY_UNAVAILABLE, + str(error), + True, + error.retry_after_ms, + ) + else: + problem = QueryProblem( + CanonicalErrorCode.INTERNAL_ERROR, + "warmup batch item failed inside the bounded executor", + False, + ) results.append( BatchItemResult( requirement.instrument_uid, - "OK", - result=execution.value, + problem.code.value, + problem=problem, ) ) - continue - error = execution.error - if isinstance(error, RetryableWarmupError) and isinstance( - error.cause, QueryServiceError - ): - problem = error.cause.problem - elif isinstance(error, QueryServiceError): - problem = error.problem - elif isinstance(error, RetryableWarmupError): - problem = QueryProblem( - CanonicalErrorCode.DEPENDENCY_UNAVAILABLE, - str(error), - True, - error.retry_after_ms, - ) - else: - problem = QueryProblem( - CanonicalErrorCode.INTERNAL_ERROR, - "warmup batch item failed inside the bounded executor", - False, - ) - results.append( - BatchItemResult( - requirement.instrument_uid, - problem.code.value, - problem=problem, + elapsed = sorted(execution.elapsed_ms for execution in executions) + percentile = lambda fraction: ( + elapsed[min(len(elapsed) - 1, max(0, math.ceil(len(elapsed) * fraction) - 1))] + if elapsed + else 0.0 + ) + executor_after = self.warmup_executor.stats() + backend_after = self._warmup_backend_stats() + executor_delta = { + f"executor_{key}": executor_after.get(key, 0) - executor_before.get(key, 0) + for key in executor_after + } + backend_delta = { + key: backend_after.get(key, 0) - backend_before.get(key, 0) + for key in backend_after + if key != "cache_entries" + } + cache_lookups = backend_delta.get("cache_hits", 0) + backend_delta.get( + "cache_misses", 0 + ) + self.last_batch_evidence = { + "request_id": request_id, + "item_count": len(executions), + "success_count": sum(item.problem is None for item in results), + "error_count": sum(item.problem is not None for item in results), + "p50_ms": percentile(0.50), + "p95_ms": percentile(0.95), + "cache_hit_rate": ( + backend_delta.get("cache_hits", 0) / cache_lookups + if cache_lookups + else 0.0 + ), + "local_batch_snapshot": bool( + callable(history_many) and local_requirements + ), + "local_batch_items": len(local_requirements), + "local_batch_admission": ( + self._local_batch_admission_for().stats() + if local_requirements + else None + ), + **executor_delta, + **backend_delta, + } + return BatchQueryResult(request_id, tuple(results)) + + async def complete(executions) -> _BatchCompletion: + return await completion(assemble_batch(executions)) + + if fully_local_batch: + admission = self._local_batch_admission_for() + + async def execute_whole_local_batch() -> _BatchCompletion: + nonlocal prefetched_local_histories + try: + prefetched_local_histories = await asyncio.to_thread( + history_many, local_requirements + ) + except Exception as error: + prefetched_local_histories = { + requirement: error for requirement in local_requirements + } + return await complete(await execute_items()) + + local_batch_task = asyncio.create_task( + admission.run( + execute_whole_local_batch, + wait_timeout_ms=local_admission_wait_ms, ) ) - elapsed = sorted(execution.elapsed_ms for execution in executions) - percentile = lambda fraction: ( - elapsed[min(len(elapsed) - 1, max(0, math.ceil(len(elapsed) * fraction) - 1))] - if elapsed - else 0.0 - ) - executor_after = self.warmup_executor.stats() - backend_after = self._warmup_backend_stats() - executor_delta = { - f"executor_{key}": executor_after.get(key, 0) - executor_before.get(key, 0) - for key in executor_after - } - backend_delta = { - key: backend_after.get(key, 0) - backend_before.get(key, 0) - for key in backend_after - if key != "cache_entries" - } - cache_lookups = backend_delta.get("cache_hits", 0) + backend_delta.get( - "cache_misses", 0 - ) - self.last_batch_evidence = { - "request_id": request_id, - "item_count": len(executions), - "success_count": sum(item.problem is None for item in results), - "error_count": sum(item.problem is not None for item in results), - "p50_ms": percentile(0.50), - "p95_ms": percentile(0.95), - "cache_hit_rate": ( - backend_delta.get("cache_hits", 0) / cache_lookups - if cache_lookups - else 0.0 - ), - **executor_delta, - **backend_delta, - } - return BatchQueryResult(request_id, tuple(results)) + try: + return await asyncio.shield(local_batch_task) + except asyncio.CancelledError: + # The caller may disappear while a SQLite tail read, item work + # or HTTP response serialization holds this reader's finite + # CPU budget. Keep the request-local lease until it drains. + local_batch_task.add_done_callback(self._consume_detached_local_batch) + raise + except _LocalBatchAdmissionRejected as error: + # A bounded queued lane must fail typed before it can create + # cache work or borrow an external-provider budget. + prefetched_local_histories = { + requirement: QueryServiceError( + QueryProblem( + CanonicalErrorCode.RATE_LIMITED, + str(error), + True, + ), + request_id=request_id, + instrument_uid=requirement.instrument_uid, + ) + for requirement in local_requirements + } + return await complete(await execute_items()) + + try: + executions = await execute_items() + finally: + if local_histories_task is not None and not local_histories_task.done(): + local_histories_task.cancel() + await asyncio.gather(local_histories_task, return_exceptions=True) + return await complete(executions) def _warmup_backend_stats(self) -> dict[str, int]: stats = getattr(self.backend, "warmup_stats", None) @@ -509,6 +794,11 @@ async def reference_data_batch_async( executor_before = self.warmup_executor.stats() reference_before = self.reference_batch.stats() + execution_live_before = ( + self.execution_mark_index_reader.stats() + if self.execution_mark_index_reader is not None + else {} + ) results: list[ReferenceBatchItemResult | None] = [None] * len(batch.requirements) admitted: list[tuple[int, ReferenceDataRequirement, ReferenceRequest]] = [] for index, requirement in enumerate(batch.requirements): @@ -564,8 +854,23 @@ async def work( *, bypass_cache: bool = False, ) -> ReferenceBatchResult: - _index, _requirement, request = candidate - result = await self.reference_batch.fetch_one(request, bypass_cache=bypass_cache) + _index, requirement, request = candidate + if self._uses_execution_mark_index_live_reader(requirement, request, purpose): + # This is deliberately not a provider retry/cache path. The + # active stream gateway either has one verified current view or + # query returns its typed fail-closed reason to the consumer. + result = await self.execution_mark_index_reader.fetch( + request, + max_freshness_ms=requirement.max_freshness_ms or 0, + source_policy_id=requirement.source_policy_id, + event_recency_policy=requirement.effective_event_recency_policy, + max_session_liveness_ms=requirement.max_session_liveness_ms, + deadline_ms=self._reference_deadline_ms(candidate, purpose), + ) + else: + result = await self.reference_batch.fetch_one( + request, bypass_cache=bypass_cache + ) # Rust provider admission deliberately communicates bounded # pressure through a typed retry delay. Keep Rust as the only # admission authority and let the shared executor honor that @@ -583,9 +888,9 @@ async def work( executions = await self.warmup_executor.execute( admitted, work=work, - identity=lambda candidate: candidate[2].cache_key, - provider=lambda candidate: candidate[2].instrument.identity.venue, - deadline_ms=lambda candidate: candidate[1].deadline_ms, + identity=lambda candidate: self._reference_batch_identity(candidate, purpose), + provider=lambda candidate: self._reference_provider_lane(candidate, purpose), + deadline_ms=lambda candidate: self._reference_deadline_ms(candidate, purpose), ) # Revalidate after every bounded initial task has returned. A current @@ -595,11 +900,17 @@ async def work( # provider MARK/INDEX row gets the same one bounded, cache-bypassing # re-read; any still-stale result remains fail-closed. refresh_candidates = [] + initial_validation_ns = self._clock_ns() for execution in executions: if execution.error is not None or execution.value is None: continue _index, requirement, request = execution.item - problem = self._reference_problem(requirement, request, execution.value) + problem = self._reference_problem( + requirement, + request, + execution.value, + at_ns=initial_validation_ns, + ) if ( problem is not None and problem.code is CanonicalErrorCode.DATA_STALE @@ -610,26 +921,18 @@ async def work( ) ): refresh_candidates.append(execution.item) - refresh_by_index = {candidate[0]: candidate for candidate in refresh_candidates} + refreshed = await self.warmup_executor.execute( + refresh_candidates, + work=lambda candidate: work(candidate, bypass_cache=True), + identity=lambda candidate: self._reference_batch_identity(candidate, purpose), + provider=lambda candidate: self._reference_provider_lane(candidate, purpose), + deadline_ms=lambda candidate: self._reference_deadline_ms(candidate, purpose), + ) if refresh_candidates else () + refresh_by_index = {execution.item[0]: execution for execution in refreshed} + assembly_validation_ns = self._clock_ns() for initial_execution in executions: - execution = initial_execution - refresh_candidate = refresh_by_index.get(initial_execution.item[0]) - if refresh_candidate is not None: - # A bounded batch refresh can itself make an early MARK/INDEX - # result stale before response assembly. Re-read and validate - # this exact already-admitted item at its assembly turn instead. - # It is still one cache-bypass recovery through the same - # provider lane; a second stale result remains fail-closed. - execution = ( - await self.warmup_executor.execute( - (refresh_candidate,), - work=lambda candidate: work(candidate, bypass_cache=True), - identity=lambda candidate: candidate[2].cache_key, - provider=lambda candidate: candidate[2].instrument.identity.venue, - deadline_ms=lambda candidate: candidate[1].deadline_ms, - ) - )[0] + execution = refresh_by_index.get(initial_execution.item[0], initial_execution) index, requirement, request = execution.item if execution.error is not None: retry_after_ms = getattr(execution.error, "retry_after_ms", None) @@ -646,7 +949,12 @@ async def work( continue assert execution.value is not None result = execution.value - problem = self._reference_problem(requirement, request, result) + problem = self._reference_problem( + requirement, + request, + result, + at_ns=assembly_validation_ns, + ) results[index] = ReferenceBatchItemResult( requirement, result.status.value if problem is None else problem.code.value, @@ -659,6 +967,11 @@ async def work( raise RuntimeError("reference batch lost a result during bounded scheduling") executor_after = self.warmup_executor.stats() reference_after = self.reference_batch.stats() + execution_live_after = ( + self.execution_mark_index_reader.stats() + if self.execution_mark_index_reader is not None + else {} + ) self.last_reference_batch_evidence = { "request_id": request_id, "item_count": len(resolved), @@ -673,9 +986,72 @@ async def work( for key in reference_after if key != "cache_entries" and key != "inflight" }, + **{ + f"execution_live_{key}": execution_live_after.get(key, 0) + - execution_live_before.get(key, 0) + for key in execution_live_after + }, } return ReferenceBatchQueryResult(request_id, resolved) + def _uses_execution_mark_index_live_reader( + self, + requirement: ReferenceDataRequirement, + request: ReferenceRequest, + purpose: AccessPurpose, + ) -> bool: + return ( + self.execution_mark_index_reader is not None + and purpose is AccessPurpose.INTERNAL_EXECUTION + and requirement.consumer_grade is ConsumerGrade.EXECUTION + and requirement.max_freshness_ms is not None + and request.product is ReferenceProduct.MARK_INDEX_PRICE + and not request.is_history + ) + + def _reference_provider_lane( + self, + candidate: tuple[int, ReferenceDataRequirement, ReferenceRequest], + purpose: AccessPurpose, + ) -> str: + if self._uses_execution_mark_index_live_reader( + candidate[1], candidate[2], purpose + ): + return "INTERNAL_STREAM" + return candidate[2].instrument.identity.venue + + def _reference_batch_identity( + self, + candidate: tuple[int, ReferenceDataRequirement, ReferenceRequest], + purpose: AccessPurpose, + ) -> tuple[object, ...]: + _index, requirement, request = candidate + if self._uses_execution_mark_index_live_reader(requirement, request, purpose): + # A stream view is authorized and freshness-bound by these exact + # caller values. Sharing a singleflight task across a different + # policy or deadline would make the result's admission ambiguous. + return ( + *request.cache_key, + "INTERNAL_EXECUTION", + requirement.source_policy_id, + requirement.max_freshness_ms, + requirement.effective_event_recency_policy.value, + requirement.max_session_liveness_ms, + requirement.deadline_ms, + ) + return request.cache_key + + def _reference_deadline_ms( + self, + candidate: tuple[int, ReferenceDataRequirement, ReferenceRequest], + purpose: AccessPurpose, + ) -> int: + _index, requirement, request = candidate + if self._uses_execution_mark_index_live_reader(requirement, request, purpose): + assert requirement.max_freshness_ms is not None + return min(requirement.deadline_ms, requirement.max_freshness_ms) + return requirement.deadline_ms + def _reference_snapshot_requires_refresh( self, requirement: ReferenceDataRequirement, @@ -693,6 +1069,8 @@ def _reference_snapshot_requires_refresh( relaxed and a second stale observation remains terminal. """ + if self._is_execution_mark_index_live_result(result): + return False if self._reference_snapshot_was_current_at_receipt( requirement, request, @@ -732,9 +1110,35 @@ def _reference_snapshot_was_current_at_receipt( return source_age_at_receipt_ms <= freshness_ms @staticmethod - def _reference_freshness_timestamp(result: ReferenceBatchResult) -> int: + def _is_execution_mark_index_live_result(result: ReferenceBatchResult) -> bool: + return bool(result.lineage) and all( + item.provider_endpoint == _EXECUTION_MARK_INDEX_LIVE_ENDPOINT + for item in result.lineage + ) + + @classmethod + def _reference_freshness_timestamp(cls, result: ReferenceBatchResult) -> int: # A snapshot pair is only as current as its oldest component. History # instead measures how recently the series was updated, not its start. + if ( + result.request.product is ReferenceProduct.MARK_INDEX_PRICE + and not result.request.is_history + and cls._is_execution_mark_index_live_result(result) + ): + labels = tuple(dict(item.labels) for item in result.observations) + bases = {item.get("freshness_basis", "SOURCE_EVENT") for item in labels} + if bases == {"PROVIDER_CONFIRMATION"}: + try: + confirmations = tuple( + int(item["provider_confirmation_ns"]) for item in labels + ) + except (KeyError, TypeError, ValueError): + return 0 + return min(confirmations, default=0) if all( + value > 0 for value in confirmations + ) else 0 + if bases != {"SOURCE_EVENT"}: + return 0 select = ( min if result.request.product is ReferenceProduct.MARK_INDEX_PRICE @@ -751,6 +1155,8 @@ def _reference_problem( requirement: ReferenceDataRequirement, request: ReferenceRequest, result: ReferenceBatchResult, + *, + at_ns: int | None = None, ) -> QueryProblem | None: if result.request != request: return QueryProblem( @@ -779,9 +1185,20 @@ def _reference_problem( "reference provider history did not cover the requested complete window", False, ) + if self._uses_quiet_execution_mark_index_contract( + requirement, request, result + ): + return self._quiet_execution_mark_index_problem( + requirement, + result, + at_ns=self._clock_ns() if at_ns is None else at_ns, + ) if requirement.max_freshness_ms is not None: observed_ns = self._reference_freshness_timestamp(result) - freshness_ms = max(0, (self._clock_ns() - observed_ns) // 1_000_000) + freshness_ms = max( + 0, + ((self._clock_ns() if at_ns is None else at_ns) - observed_ns) // 1_000_000, + ) if freshness_ms > requirement.max_freshness_ms: return QueryProblem( CanonicalErrorCode.DATA_STALE, @@ -801,6 +1218,24 @@ def _reference_problem( result.error_detail or "reference product is unavailable at this provider", False, ) + if result.error_code == "LIVE_VIEW_STALE": + return QueryProblem( + CanonicalErrorCode.DATA_STALE, + result.error_detail or "execution MARK/INDEX live view is stale", + True, + ) + if result.error_code == "LIVE_VIEW_GAPPED": + return QueryProblem( + CanonicalErrorCode.DATA_NOT_READY, + result.error_detail or "execution MARK/INDEX live view has an open gap", + True, + ) + if result.error_code == "LIVE_VIEW_IDENTITY": + return QueryProblem( + CanonicalErrorCode.CONFLICT, + result.error_detail or "execution MARK/INDEX live view identity differs", + False, + ) return QueryProblem( CanonicalErrorCode.SOURCE_UNAVAILABLE, result.error_detail or "reference provider request failed", @@ -808,6 +1243,61 @@ def _reference_problem( result.retry_after_ms, ) + @classmethod + def _uses_quiet_execution_mark_index_contract( + cls, + requirement: ReferenceDataRequirement, + request: ReferenceRequest, + result: ReferenceBatchResult, + ) -> bool: + """Identify the one explicit exception to normal event-age admission. + + A quiet provider component is not a fresh market event. The exception + is therefore deliberately limited to the execution MARK/INDEX live + view, whose stream gateway has already verified the paired lineage and + current provider session. Every other reference product retains the + normal immutable event-age check below. + """ + + return ( + requirement.effective_event_recency_policy is StalePolicy.OBSERVE + and requirement.consumer_grade is ConsumerGrade.EXECUTION + and request.product is ReferenceProduct.MARK_INDEX_PRICE + and not request.is_history + and cls._is_execution_mark_index_live_result(result) + ) + + @staticmethod + def _quiet_execution_mark_index_problem( + requirement: ReferenceDataRequirement, + result: ReferenceBatchResult, + *, + at_ns: int, + ) -> QueryProblem | None: + """Recheck stream evidence at query assembly without altering lineage. + + The private reader proves headers match the canonical envelope. Query + still has to account for time spent in its own bounded executor before + handing the result to the consumer. This keeps session and component + cadences fail-closed at the outer admission boundary as well. + """ + + if len(result.observations) != 1: + return QueryProblem( + CanonicalErrorCode.DATA_STALE, + "quiet execution MARK/INDEX contract is incomplete", + True, + ) + try: + validate_quiet_execution_mark_index_evidence( + dict(result.observations[0].labels), + at_ns=at_ns, + max_session_liveness_ms=requirement.max_session_liveness_ms, + ) + except ValueError as error: + return QueryProblem(CanonicalErrorCode.DATA_STALE, str(error), True) + return None + def status(self, requirement: DataRequirement) -> QualityMetadata: request_id = self.request_id() try: @@ -942,14 +1432,37 @@ def _with_execution_eligibility( at_ns=self._clock_ns(), ) quality = item.quality + # A source-authorized native BBO can be unchanged for longer than the + # consumer's raw-event window. Its immutable source timestamp remains + # stale for audit, but the stable source evaluator may authorize the + # exact ON_CHANGE/OBSERVE binding from verified session/fence facts. + # No caller can opt into this through a request: the source catalog + # emits DELIVERY_ON_CHANGE only for the validated native BBO lanes. + on_change_quote = ( + requirement.feed is FeedType.QUOTE + and "DELIVERY_ON_CHANGE" in quality.flags + ) + event_recency_eligible = ( + quality.event_recency_state != "STALE" or on_change_quote + ) + freshness_eligible = ( + requirement.max_freshness_ms is None + or quality.freshness_ms <= requirement.max_freshness_ms + or on_change_quote + ) eligible = ( - execution_entitlement.allowed + # The source backend owns feed delivery semantics. In particular, + # it is the only layer allowed to make a quiet native BBO eligible + # after its signed ON_CHANGE binding and all session/fence checks. + # Do not reapply a raw-age predicate here and undo that decision. + quality.execution_eligible + and execution_entitlement.allowed and item.source.authoritative and quality.policy_id == requirement.source_policy_id and quality.state == "LIVE" and quality.complete and not quality.gap_open - and quality.event_recency_state != "STALE" + and event_recency_eligible and quality.provider_session_state not in {"STALE", "DISCONNECTED", "UNKNOWN"} and ( @@ -961,10 +1474,7 @@ def _with_execution_eligibility( <= requirement.max_session_liveness_ms ) ) - and ( - requirement.max_freshness_ms is None - or quality.freshness_ms <= requirement.max_freshness_ms - ) + and freshness_eligible ) return replace( item, diff --git a/qdl/reference/execution_live.py b/qdl/reference/execution_live.py new file mode 100644 index 0000000..2525e47 --- /dev/null +++ b/qdl/reference/execution_live.py @@ -0,0 +1,502 @@ +"""Private stable-stream reader for execution-grade MARK/INDEX snapshots. + +The public V2 reference contract remains unchanged. This client replaces only +the provider REST call for an execution request after the stable stream gateway +has already admitted the canonical paired event under its writer lease. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import ssl +import time +from dataclasses import dataclass, field +from typing import Protocol + +import httpx +from google.protobuf.message import DecodeError + +from qdl.domain.capabilities import FeedCapability +from qdl.reference.batch import CapabilityResolver, default_capability_resolver +from qdl.reference.contracts import ( + MarkIndexKind, + ReferenceBatchResult, + ReferenceCoverage, + ReferenceField, + ReferenceLineage, + ReferenceObservation, + ReferenceProduct, + ReferenceRequest, + ReferenceStatus, + decimal_field, + product_feed_name, +) +from qdl.runtime.internal_auth import is_stable_internal_url, stable_hmac_signature +from qdl.runtime.mark_index_lineage import paired_mark_index_lineage +from qdl.marketdata.v2 import market_data_pb2 +from qdl.query.contracts import StalePolicy + + +_LEGACY_REQUEST_SCHEMA = "qdl.v2.execution-mark-index-read.v1" +_REQUEST_SCHEMA = "qdl.v2.execution-mark-index-read.v2" +_RESPONSE_SCHEMA = "qdl.v2.execution-mark-index-view.v2" +_ENDPOINT = "/internal/v2/execution/mark-index/latest" +_FRESHNESS_BASIS_HEADER = "X-QDL-Execution-Freshness-Basis" +_RECENCY_MODE_HEADER = "X-QDL-Execution-Recency-Mode" +_SESSION_STATE_HEADER = "X-QDL-Execution-Session-State" +_SESSION_LIVENESS_HEADER = "X-QDL-Execution-Session-Liveness-Ms" +_SESSION_CHECKED_AT_HEADER = "X-QDL-Execution-Session-Checked-At-Ns" +_COMPONENT_RECEIPTS_HEADER = "X-QDL-Execution-Component-Receipts-Ns" +_COMPONENT_CADENCE_HEADER = "X-QDL-Execution-Component-Quiet-After-Ms" + + +class ExecutionMarkIndexReader(Protocol): + async def fetch( + self, + request: ReferenceRequest, + *, + max_freshness_ms: int, + source_policy_id: str, + event_recency_policy: StalePolicy = StalePolicy.BLOCK, + max_session_liveness_ms: int | None = None, + deadline_ms: int | None = None, + ) -> ReferenceBatchResult: ... + + def stats(self) -> dict[str, int]: ... + + +@dataclass(slots=True) +class HttpExecutionMarkIndexReader: + """Read the current active gateway view, never venue REST, for execution.""" + + urls: tuple[str, ...] + secret: bytes + capability_resolver: CapabilityResolver = default_capability_resolver + timeout_seconds: float = 2.0 + client: httpx.AsyncClient | None = None + ssl_context: ssl.SSLContext | None = None + _owns_client: bool = field(init=False) + _calls: int = field(init=False, default=0) + _successes: int = field(init=False, default=0) + _failures: int = field(init=False, default=0) + + def __post_init__(self) -> None: + if ( + not self.urls + or any(not is_stable_internal_url(value) for value in self.urls) + or len(self.secret) < 32 + or not 0.1 <= self.timeout_seconds <= 10.0 + ): + raise ValueError("execution MARK/INDEX live reader configuration is invalid") + if self.client is not None and self.ssl_context is not None: + raise ValueError("execution MARK/INDEX client and TLS context are mutually exclusive") + self._owns_client = self.client is None + if self.client is None: + self.client = httpx.AsyncClient( + follow_redirects=False, + limits=httpx.Limits(max_connections=4, max_keepalive_connections=2), + timeout=self.timeout_seconds, + verify=self.ssl_context or True, + ) + + async def fetch( + self, + request: ReferenceRequest, + *, + max_freshness_ms: int, + source_policy_id: str, + event_recency_policy: StalePolicy = StalePolicy.BLOCK, + max_session_liveness_ms: int | None = None, + deadline_ms: int | None = None, + ) -> ReferenceBatchResult: + """Fetch one exact current snapshot without an external-provider fallback.""" + + self._calls += 1 + if deadline_ms is not None and deadline_ms < 1: + raise ValueError("execution MARK/INDEX read deadline must be positive") + capability = self._capability(request) + if ( + request.product is not ReferenceProduct.MARK_INDEX_PRICE + or request.is_history + or max_freshness_ms <= 0 + or not source_policy_id.strip() + or not isinstance(event_recency_policy, StalePolicy) + or ( + event_recency_policy is StalePolicy.OBSERVE + and (max_session_liveness_ms is None or max_session_liveness_ms <= 0) + ) + ): + self._failures += 1 + return self._failure( + request, capability, "LIVE_VIEW_INVALID_REQUEST", + "execution live view accepts only a current MARK_INDEX_PRICE request", + ) + quiet_request = event_recency_policy is StalePolicy.OBSERVE + payload = { + "schema": _REQUEST_SCHEMA if quiet_request else _LEGACY_REQUEST_SCHEMA, + "instrument_uid": request.instrument.instrument_uid, + "instrument_revision": request.instrument.metadata_revision, + "source_policy_id": source_policy_id, + "max_freshness_ms": max_freshness_ms, + } + if quiet_request: + assert max_session_liveness_ms is not None + payload.update({ + "event_recency_policy": event_recency_policy.value, + "max_session_liveness_ms": max_session_liveness_ms, + }) + body = json.dumps( + payload, + sort_keys=True, + separators=(",", ":"), + ).encode() + errors: list[str] = [] + deadline_at = ( + time.monotonic() + deadline_ms / 1_000 if deadline_ms is not None else None + ) + assert self.client is not None + for url in self.urls: + timeout_seconds = self.timeout_seconds + if deadline_at is not None: + timeout_seconds = min(timeout_seconds, deadline_at - time.monotonic()) + if timeout_seconds <= 0: + errors.append("DEADLINE") + break + try: + response = await asyncio.wait_for( + self.client.post( + f"{url.rstrip('/')}{_ENDPOINT}", + content=body, + headers={ + "Content-Type": "application/json", + "X-QDL-Stable-Signature": stable_hmac_signature(self.secret, body), + }, + timeout=timeout_seconds, + ), + timeout=timeout_seconds, + ) + except (asyncio.TimeoutError, httpx.TransportError): + errors.append("TIMEOUT" if deadline_at is not None else "TRANSPORT") + continue + if response.status_code == 409: + errors.append(self._bounded_reason(response)) + continue + if response.status_code != 200: + errors.append(f"HTTP_{response.status_code}") + continue + try: + result = self._result_from_response( + request, + capability, + response, + freshness_basis=response.headers.get( + _FRESHNESS_BASIS_HEADER, "SOURCE_EVENT" + ).strip().upper(), + event_recency_policy=event_recency_policy, + max_session_liveness_ms=max_session_liveness_ms, + ) + except (DecodeError, ValueError, TypeError, KeyError): + errors.append("PROTOCOL") + continue + self._successes += 1 + return result + self._failures += 1 + code = self._failure_code(errors) + return self._failure( + request, + capability, + code, + "execution MARK/INDEX live view did not return a current active record", + ) + + def stats(self) -> dict[str, int]: + return { + "calls": self._calls, + "successes": self._successes, + "failures": self._failures, + } + + async def close(self) -> None: + if self._owns_client and self.client is not None: + await self.client.aclose() + + def _capability(self, request: ReferenceRequest) -> FeedCapability: + profile = self.capability_resolver(request.instrument) + capability = profile.capability(product_feed_name(request.product)) + if not capability.enabled: + raise ValueError("execution MARK/INDEX live capability is unavailable") + return capability + + @staticmethod + def _bounded_reason(response: httpx.Response) -> str: + try: + detail = response.json().get("detail", "") + except (ValueError, AttributeError): + return "UNAVAILABLE" + normalized = str(detail).upper() + if normalized.endswith(":STALE"): + return "STALE" + if normalized.endswith(":GAP_OR_RESYNC"): + return "GAPPED" + if normalized.endswith(":IDENTITY_MISMATCH"): + return "IDENTITY_MISMATCH" + if normalized.endswith(":SOURCE_POLICY_MISMATCH"): + return "SOURCE_POLICY_MISMATCH" + if "FENCED" in normalized: + return "FENCED" + return "NOT_READY" + + @staticmethod + def _failure_code(errors: list[str]) -> str: + if "STALE" in errors: + return "LIVE_VIEW_STALE" + if "GAPPED" in errors: + return "LIVE_VIEW_GAPPED" + if "IDENTITY_MISMATCH" in errors or "SOURCE_POLICY_MISMATCH" in errors: + return "LIVE_VIEW_IDENTITY" + if "PROTOCOL" in errors: + return "LIVE_VIEW_PROTOCOL" + return "LIVE_VIEW_UNAVAILABLE" + + def _result_from_response( + self, + request: ReferenceRequest, + capability: FeedCapability, + response: httpx.Response, + *, + freshness_basis: str, + event_recency_policy: StalePolicy, + max_session_liveness_ms: int | None, + ) -> ReferenceBatchResult: + payload = response.json() + if ( + set(payload) != { + "schema", "lease_epoch", "spool_watermark_offset", + "delivery_stage", "canonical", + } + or payload["schema"] != _RESPONSE_SCHEMA + or int(payload["lease_epoch"]) < 1 + or payload["delivery_stage"] not in { + "CANONICAL_READ_COMMITTED", "SPOOL_CONFIRMED", + } + or freshness_basis not in {"SOURCE_EVENT", "PROVIDER_CONFIRMATION"} + ): + raise ValueError("execution MARK/INDEX live view response is invalid") + spool_watermark_offset = payload["spool_watermark_offset"] + if spool_watermark_offset is not None and int(spool_watermark_offset) < 0: + raise ValueError("execution MARK/INDEX spool watermark is invalid") + canonical = base64.b64decode(str(payload["canonical"]), validate=True) + envelope = market_data_pb2.EventEnvelope.FromString(canonical) + if ( + envelope.WhichOneof("payload") != "mark_index_price" + or envelope.instrument_uid != request.instrument.instrument_uid + or int(envelope.instrument_revision) != request.instrument.metadata_revision + or envelope.instrument_id != request.instrument.instrument_id + or envelope.venue != request.instrument.identity.venue + or envelope.market != request.instrument.identity.market + or envelope.native_symbol != request.instrument.native_symbol + or envelope.source_event_time_ns <= 0 + or envelope.received_at_ns < envelope.source_event_time_ns + or not envelope.source_session_id + or envelope.connection_generation < 1 + or envelope.config_revision < 1 + ): + raise ValueError("execution MARK/INDEX live view identity/provenance mismatch") + fields = self._fields(request, envelope) + observed_at_ns = int(envelope.source_event_time_ns) + labels = [ + ("native_symbol", request.instrument.native_symbol), + ("execution_view", "STABLE_STREAM_GATEWAY"), + ("freshness_basis", freshness_basis), + ("source_event_time_ns", str(observed_at_ns)), + ("provider_confirmation_ns", str(int(envelope.received_at_ns))), + ("connection_generation", str(int(envelope.connection_generation))), + ("gateway_lease_epoch", str(int(payload["lease_epoch"]))), + ("delivery_stage", str(payload["delivery_stage"])), + ( + "spool_watermark_offset", + "PENDING" if spool_watermark_offset is None + else str(int(spool_watermark_offset)), + ), + ] + if event_recency_policy is StalePolicy.OBSERVE: + labels.extend(self._quiet_labels( + envelope, + response, + max_session_liveness_ms=max_session_liveness_ms, + )) + observation = ReferenceObservation( + instrument_uid=request.instrument.instrument_uid, + instrument_revision=request.instrument.metadata_revision, + product=ReferenceProduct.MARK_INDEX_PRICE, + observed_at_ns=observed_at_ns, + fields=fields, + labels=tuple(labels), + ) + lineage = ReferenceLineage( + provider=self.capability_resolver(request.instrument).provider, + provider_endpoint="qdl://stable-stream/internal/v2/execution/mark-index/latest", + source_role="REFERENCE", + adapter_version=( + "qdl-execution-mark-index-live/1+" + envelope.adapter_version + ), + capability_name="mark_index_price", + capability_constraint=capability.constraint, + ) + observed_ms = observed_at_ns // 1_000_000 + return ReferenceBatchResult( + request=request, + status=ReferenceStatus.OK, + capability=capability, + lineage=(lineage,), + coverage=ReferenceCoverage( + requested_start_ms=None, + requested_end_ms=None, + observed_min_ms=observed_ms, + observed_max_ms=observed_ms, + complete_left=True, + complete_right=True, + truncated=False, + terminal_reason="LIVE_EXECUTION_VIEW", + ), + received_at_ns=int(envelope.received_at_ns), + observations=(observation,), + ) + + @staticmethod + def _quiet_labels( + envelope: market_data_pb2.EventEnvelope, + response: httpx.Response, + *, + max_session_liveness_ms: int | None, + ) -> tuple[tuple[str, str], ...]: + if max_session_liveness_ms is None: + raise ValueError("quiet execution MARK/INDEX request has no session SLA") + recency_mode = response.headers.get(_RECENCY_MODE_HEADER, "").strip() + if recency_mode not in { + "STRICT_EVENT_SESSION_LIVE", + "COMPONENT_SESSION_LIVE", + }: + raise ValueError("quiet execution MARK/INDEX response mode is invalid") + session_state = response.headers.get(_SESSION_STATE_HEADER, "").strip() + if session_state != "LIVE": + raise ValueError("quiet execution MARK/INDEX session state is invalid") + try: + session_liveness_ms = int(response.headers[_SESSION_LIVENESS_HEADER]) + session_checked_at_ns = int(response.headers[_SESSION_CHECKED_AT_HEADER]) + except (KeyError, TypeError, ValueError) as error: + raise ValueError("quiet execution MARK/INDEX session evidence is invalid") from error + if ( + not 0 <= session_liveness_ms <= max_session_liveness_ms + or session_checked_at_ns <= 0 + ): + raise ValueError("quiet execution MARK/INDEX session evidence exceeds policy") + receipts = _component_header_values( + response.headers.get(_COMPONENT_RECEIPTS_HEADER, "") + ) + cadence = _component_header_values( + response.headers.get(_COMPONENT_CADENCE_HEADER, "") + ) + lineage = paired_mark_index_lineage(envelope) + if receipts != { + "MARK": lineage.mark_received_at_ns, + "INDEX": lineage.index_received_at_ns, + }: + raise ValueError("quiet execution MARK/INDEX receipt lineage differs") + if any(not 250 <= value <= 120_000 for value in cadence.values()): + raise ValueError("quiet execution MARK/INDEX cadence is invalid") + return ( + ("event_recency_policy", StalePolicy.OBSERVE.value), + ("recency_mode", recency_mode), + ("provider_session_state", session_state), + ("provider_session_liveness_ms", str(session_liveness_ms)), + ("provider_session_checked_at_ns", str(session_checked_at_ns)), + ("component_mark_received_at_ns", str(receipts["MARK"])), + ("component_index_received_at_ns", str(receipts["INDEX"])), + ("component_mark_quiet_after_ms", str(cadence["MARK"])), + ("component_index_quiet_after_ms", str(cadence["INDEX"])), + ) + + @staticmethod + def _fields( + request: ReferenceRequest, + envelope: market_data_pb2.EventEnvelope, + ) -> tuple[ReferenceField, ...]: + values: list[ReferenceField] = [] + pairs = ( + ("mark_price", envelope.mark_index_price.mark_price), + ("index_price", envelope.mark_index_price.index_price), + ) + required = { + MarkIndexKind.MARK: {"mark_price"}, + MarkIndexKind.INDEX: {"index_price"}, + MarkIndexKind.BOTH: {"mark_price", "index_price"}, + }[request.mark_index_kind] + for name, value in pairs: + if name not in required: + continue + field = decimal_field(name, value.source_text, "QUOTE_PRICE") + if field is None or field.value.as_decimal() <= 0: + raise ValueError("execution MARK/INDEX live view price is invalid") + values.append(field) + if {item.name for item in values} != required: + raise ValueError("execution MARK/INDEX live view fields are incomplete") + return tuple(values) + + @staticmethod + def _failure( + request: ReferenceRequest, + capability: FeedCapability, + code: str, + detail: str, + ) -> ReferenceBatchResult: + return ReferenceBatchResult( + request=request, + status=ReferenceStatus.ERROR, + capability=capability, + lineage=(ReferenceLineage( + provider=request.instrument.identity.venue.upper() + "_DIRECT", + provider_endpoint="qdl://stable-stream/internal/v2/execution/mark-index/latest", + source_role="REFERENCE", + adapter_version="qdl-execution-mark-index-live/1", + capability_name="mark_index_price", + capability_constraint=capability.constraint, + ),), + coverage=ReferenceCoverage( + requested_start_ms=None, + requested_end_ms=None, + observed_min_ms=None, + observed_max_ms=None, + complete_left=False, + complete_right=False, + truncated=False, + terminal_reason=code, + ), + received_at_ns=time.time_ns(), + error_code=code, + error_detail=detail, + ) + + +def _component_header_values(value: str) -> dict[str, int]: + """Parse one bounded private header without accepting partial components.""" + + result: dict[str, int] = {} + for item in value.split(","): + name, separator, raw = item.partition("=") + if not separator or name in result: + raise ValueError("quiet execution MARK/INDEX component header is invalid") + try: + parsed = int(raw) + except ValueError as error: + raise ValueError( + "quiet execution MARK/INDEX component header is invalid" + ) from error + if name not in {"MARK", "INDEX"} or parsed <= 0: + raise ValueError("quiet execution MARK/INDEX component header is invalid") + result[name] = parsed + if set(result) != {"MARK", "INDEX"}: + raise ValueError("quiet execution MARK/INDEX component header is incomplete") + return result diff --git a/qdl/runtime/core_binding_identity.py b/qdl/runtime/core_binding_identity.py new file mode 100644 index 0000000..baa9297 --- /dev/null +++ b/qdl/runtime/core_binding_identity.py @@ -0,0 +1,140 @@ +"""Python-side validation for the Rust realtime-core binding identity rule. + +The stable compiler may project one logical OKX MARK_INDEX requirement into +two physical provider channels. They deliberately share the logical +``source_id`` but differ by MARK versus INDEX component. Offline rollout +tools must accept that valid pair while retaining the Rust core's fail-closed +duplicate and completeness rules. +""" + +from __future__ import annotations + +from typing import Any, Mapping + + +CoreBindingIdentity = tuple[str, ...] +_MARK_INDEX_COMPONENTS = frozenset({"MARK", "INDEX", "BOTH"}) + + +def _string(value: Mapping[str, Any], name: str, *, field: str) -> str: + result = value.get(name) + if not isinstance(result, str) or not result.strip(): + raise ValueError(f"{field} has an invalid {name}") + return result + + +def _physical_identity(value: Mapping[str, Any], *, field: str) -> tuple[str, ...]: + symbol = value.get("physical_native_symbol", value.get("native_symbol")) + channel = value.get("physical_native_channel", value.get("native_channel")) + if not isinstance(symbol, str) or not symbol.strip(): + raise ValueError(f"{field} has an invalid physical native symbol") + if not isinstance(channel, str) or not channel.strip(): + raise ValueError(f"{field} has an invalid physical native channel") + return ( + _string(value, "provider", field=field), + _string(value, "venue", field=field), + _string(value, "market", field=field), + _string(value, "product_type", field=field), + symbol, + channel, + ) + + +def core_binding_identity(value: Mapping[str, Any], *, field: str) -> CoreBindingIdentity: + """Return the semantic identity used for offline core-config comparison.""" + source_id = _string(value, "source_id", field=field) + mark_index = value.get("mark_index") + if mark_index is None: + return ("ORDINARY", source_id) + if not isinstance(mark_index, Mapping): + raise ValueError(f"{field} has an invalid mark_index contract") + component = mark_index.get("component") + if not isinstance(component, str) or component not in _MARK_INDEX_COMPONENTS: + raise ValueError(f"{field} has an invalid mark_index component") + return ("MARK_INDEX", _string(value, "instrument_uid", field=field), source_id, component) + + +def native_ingestor_binding_identity( + value: Mapping[str, Any], *, field: str +) -> tuple[str, ...]: + """Return a physical native subscription identity. + + Ordinary TRADE/QUOTE/BOOK sources have one physical subscription, so their + logical ID is sufficient to expose a changed channel as semantic drift. + An OKX MARK_INDEX source intentionally has two physical subscriptions; + include symbol and channel only for that paired feed so neither component + is collapsed or confused with a duplicate retry declaration. + """ + feed = _string(value, "feed", field=field) + subscription_id = _string(value, "subscription_id", field=field) + if feed != "MARK_INDEX": + return (feed, subscription_id) + return ( + feed, + subscription_id, + _string(value, "native_symbol", field=field), + _string(value, "native_channel", field=field), + ) + + +def format_core_binding_identity(identity: CoreBindingIdentity) -> str: + if identity[0] == "ORDINARY": + return identity[1] + return f"{identity[2]}:{identity[3]}@{identity[1]}" + + +def core_binding_map( + bindings: object, + *, + field: str, +) -> dict[CoreBindingIdentity, dict[str, Any]]: + """Validate and index bindings with the same source rule as Rust. + + This mirrors ``RealtimeCoreConfig::validate`` at the declaration layer: + physical channels must be unique; ordinary and MARK_INDEX source IDs may + not collide; each MARK_INDEX target must be exactly ``BOTH`` or + ``MARK`` plus ``INDEX``. + """ + if not isinstance(bindings, list) or not bindings: + raise ValueError(f"{field} bindings are invalid") + + result: dict[CoreBindingIdentity, dict[str, Any]] = {} + physical: set[tuple[str, ...]] = set() + ordinary_source_ids: set[str] = set() + mark_index_source_ids: set[str] = set() + mark_index_components: dict[tuple[str, str], set[str]] = {} + + for raw in bindings: + if not isinstance(raw, dict): + raise ValueError(f"{field} has a non-object binding") + item = dict(raw) + physical_key = _physical_identity(item, field=field) + if physical_key in physical: + raise ValueError(f"{field} has a duplicate physical binding") + physical.add(physical_key) + + identity = core_binding_identity(item, field=field) + source_id = identity[2] if identity[0] == "MARK_INDEX" else identity[1] + if identity[0] == "MARK_INDEX": + if source_id in ordinary_source_ids: + raise ValueError(f"{field} mixes ordinary and mark_index source_id {source_id}") + target = (identity[1], source_id) + components = mark_index_components.setdefault(target, set()) + if identity[3] in components: + raise ValueError(f"{field} has a duplicate mark_index component") + components.add(identity[3]) + mark_index_source_ids.add(source_id) + else: + if source_id in mark_index_source_ids or source_id in ordinary_source_ids: + raise ValueError(f"{field} has a duplicate ordinary source_id") + ordinary_source_ids.add(source_id) + if identity in result: + raise ValueError(f"{field} has a duplicate semantic binding") + result[identity] = item + + for target, components in mark_index_components.items(): + if components not in ({"BOTH"}, {"MARK", "INDEX"}): + raise ValueError( + f"{field} has an incomplete mark_index component pair: {target[0]}/{target[1]}" + ) + return result diff --git a/qdl/runtime/execution_mark_index.py b/qdl/runtime/execution_mark_index.py new file mode 100644 index 0000000..beb6af5 --- /dev/null +++ b/qdl/runtime/execution_mark_index.py @@ -0,0 +1,687 @@ +"""Fenced latest execution view for canonical MARK/INDEX events. + +This is intentionally a tiny in-process view owned by the existing active +stream gateway. It is not a second cache, replay source, provider adapter, or +public API: a query replica can only read the newest already-validated canonical +event from the current lease holder. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hmac +import json +import time +from dataclasses import dataclass +from typing import Mapping + +from fastapi import FastAPI, Header, HTTPException, Request, Response + +from qdl.common.v1 import common_pb2 +from qdl.data_quality.binding_decision import ( + BindingQualityInput, + ComponentEvidence, + evaluate_binding_quality, +) +from qdl.marketdata.v2 import market_data_pb2 +from qdl.query.contracts import FeedType, StalePolicy +from qdl.runtime.internal_auth import stable_hmac_signature +from qdl.runtime.lease import GatewayFenced +from qdl.runtime.mark_index_lineage import paired_mark_index_lineage +from qdl.runtime.stable_catalog import StableSourceBinding, StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan +from qdl.runtime.session_liveness import ( + ProviderSessionStatus, + StableSessionLivenessReader, +) +from qdl.stream import DurableStreamGateway +from qdl.transport import SQLiteDurableSpool, StoredEvent + + +_LEGACY_REQUEST_SCHEMA = "qdl.v2.execution-mark-index-read.v1" +_REQUEST_SCHEMA = "qdl.v2.execution-mark-index-read.v2" +_RESPONSE_SCHEMA = "qdl.v2.execution-mark-index-view.v2" +_DELIVERY_CANONICAL_READ_COMMITTED = "CANONICAL_READ_COMMITTED" +_DELIVERY_SPOOL_CONFIRMED = "SPOOL_CONFIRMED" +_FRESHNESS_BASIS_HEADER = "X-QDL-Execution-Freshness-Basis" +_RECENCY_MODE_HEADER = "X-QDL-Execution-Recency-Mode" +_SESSION_STATE_HEADER = "X-QDL-Execution-Session-State" +_SESSION_LIVENESS_HEADER = "X-QDL-Execution-Session-Liveness-Ms" +_SESSION_CHECKED_AT_HEADER = "X-QDL-Execution-Session-Checked-At-Ns" +_COMPONENT_RECEIPTS_HEADER = "X-QDL-Execution-Component-Receipts-Ns" +_COMPONENT_CADENCE_HEADER = "X-QDL-Execution-Component-Quiet-After-Ms" +_GAP_FLAGS = frozenset({ + common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE, + common_pb2.QUALITY_FLAG_OUT_OF_ORDER, + common_pb2.QUALITY_FLAG_RESYNC_REQUIRED, +}) + + +@dataclass(frozen=True, slots=True) +class ExecutionMarkIndexQuietPolicy: + """Signed component cadence for one logical MARK/INDEX product.""" + + component_quiet_after_ms: tuple[tuple[str, int], ...] + + def __post_init__(self) -> None: + values = dict(self.component_quiet_after_ms) + if ( + set(values) != {"MARK", "INDEX"} + or any(not 250 <= value <= 120_000 for value in values.values()) + ): + raise ValueError("execution MARK/INDEX quiet policy is incomplete") + + @classmethod + def from_acquisition(cls, acquisition) -> "ExecutionMarkIndexQuietPolicy | None": + mark_index = acquisition.mark_index + if mark_index is None or not mark_index.component_quiet_after_ms: + return None + values = dict(mark_index.component_quiet_after_ms) + if "BOTH" in values: + values = {"MARK": values["BOTH"], "INDEX": values["BOTH"]} + return cls(tuple(sorted((str(name), int(value)) for name, value in values.items()))) + + +@dataclass(frozen=True, slots=True) +class ExecutionMarkIndexRecord: + """One already-admitted canonical MARK/INDEX event and its stream fence.""" + + canonical: bytes + event_id: bytes + instrument_uid: str + instrument_revision: int + source_policy_id: str + freshness_basis: str + stale_after_ms: int + source_event_time_ns: int + received_at_ns: int + venue: str + market: str + source_session_id: str + config_revision: int + connection_generation: int + partition_sequence: int + spool_watermark_offset: int | None + delivery_stage: str + gateway_epoch: int + + +@dataclass(frozen=True, slots=True) +class ExecutionMarkIndexRead: + """A bounded read result; non-available values are never serialized.""" + + record: ExecutionMarkIndexRecord | None + reason: str | None = None + recency_mode: str | None = None + session: ProviderSessionStatus | None = None + session_checked_at_ns: int | None = None + component_receipts_ns: tuple[tuple[str, int], ...] = () + component_quiet_after_ms: tuple[tuple[str, int], ...] = () + + +class ExecutionMarkIndexLiveView: + """Bounded latest-state view keyed by exact canonical instrument identity.""" + + def __init__( + self, + allowed_instrument_uids: frozenset[str], + *, + quiet_policies: Mapping[str, ExecutionMarkIndexQuietPolicy] | None = None, + session_liveness_reader: StableSessionLivenessReader | None = None, + bindings: Mapping[str, StableSourceBinding] | None = None, + ) -> None: + if not allowed_instrument_uids: + raise ValueError("execution MARK/INDEX view requires allowed bindings") + policies = dict(quiet_policies or {}) + if not set(policies).issubset(allowed_instrument_uids): + raise ValueError("execution MARK/INDEX quiet policy is outside allowed bindings") + declared_bindings = dict(bindings or {}) + if declared_bindings and set(declared_bindings) != set(allowed_instrument_uids): + raise ValueError("execution MARK/INDEX hydration bindings are incomplete") + if any( + binding.instrument.instrument_uid != instrument_uid + or binding.feed is not FeedType.MARK_INDEX_PRICE + or not binding.authoritative + or binding.source_role != "PRIMARY" + for instrument_uid, binding in declared_bindings.items() + ): + raise ValueError("execution MARK/INDEX hydration binding is invalid") + self._allowed_instrument_uids = allowed_instrument_uids + self._quiet_policies = policies + self._session_liveness_reader = session_liveness_reader + self._bindings = declared_bindings + self._records: dict[str, ExecutionMarkIndexRecord] = {} + self._invalid: dict[str, tuple[int, int, str]] = {} + self._lock = asyncio.Lock() + + @classmethod + def from_catalog( + cls, + catalog: StableSourceCatalog, + *, + acquisition: StableAcquisitionPlan | None = None, + session_liveness_reader: StableSessionLivenessReader | None = None, + ) -> "ExecutionMarkIndexLiveView": + bindings = tuple( + binding + for binding in catalog.bindings + if ( + binding.feed is FeedType.MARK_INDEX_PRICE + and binding.authoritative + and binding.source_role == "PRIMARY" + ) + ) + bindings_by_uid: dict[str, StableSourceBinding] = {} + for binding in bindings: + existing = bindings_by_uid.setdefault(binding.instrument.instrument_uid, binding) + if existing != binding: + raise ValueError("execution MARK/INDEX binding is ambiguous") + allowed = frozenset(bindings_by_uid) + quiet_policies: dict[str, ExecutionMarkIndexQuietPolicy] = {} + if acquisition is not None: + acquisitions = {item.binding_id: item for item in acquisition.bindings} + for binding in bindings: + try: + candidate = acquisitions[binding.binding_id] + except KeyError as error: + raise ValueError( + "execution MARK/INDEX acquisition binding is unavailable" + ) from error + policy = ExecutionMarkIndexQuietPolicy.from_acquisition(candidate) + if policy is not None: + existing = quiet_policies.setdefault( + binding.instrument.instrument_uid, policy + ) + if existing != policy: + raise ValueError( + "execution MARK/INDEX quiet policy differs for one instrument" + ) + return cls( + allowed, + quiet_policies=quiet_policies, + session_liveness_reader=session_liveness_reader, + bindings=bindings_by_uid, + ) + + async def hydrate_from_spool( + self, + *, + spool: SQLiteDurableSpool, + canonical_stream: str, + gateway_epoch: int, + ) -> int: + """Restore one bounded durable latest record per declared binding. + + A lease acquisition must not wait for a new update-on-change provider + frame when the exact canonical event is already durably committed. The + restored record still travels through ``remember`` and therefore keeps + all normal identity, gap, generation and later session/freshness gates. + This method never publishes, replays, rewrites timestamps or contacts a + provider. + """ + + if not canonical_stream.strip() or gateway_epoch < 1: + raise ValueError("execution MARK/INDEX hydration scope is invalid") + restored = 0 + for instrument_uid, binding in sorted(self._bindings.items()): + records = await asyncio.to_thread( + spool.read_tail, + stream=canonical_stream, + partition_key=binding.partition_key, + limit=1, + ) + if not records: + continue + stored = records[-1] + if ( + stored.event.stream != canonical_stream + or stored.cursor.stream != canonical_stream + or stored.event.partition_key != binding.partition_key + or stored.cursor.partition_key != binding.partition_key + ): + raise ValueError("execution MARK/INDEX hydration spool identity differs") + try: + envelope = market_data_pb2.EventEnvelope.FromString(stored.event.payload) + except DecodeError as error: + raise ValueError("execution MARK/INDEX hydration canonical payload is invalid") from error + await self.remember( + binding=binding, + envelope=envelope, + stored=stored, + gateway_epoch=gateway_epoch, + ) + restored += 1 + return restored + + async def remember( + self, + *, + binding: StableSourceBinding, + envelope: market_data_pb2.EventEnvelope, + stored: StoredEvent | None, + gateway_epoch: int, + ) -> None: + """Offer one verified canonical event in the current gateway generation. + + The stable projector only posts records it read from Kafka + ``read_committed``. A pre-spool view may therefore be served as a + short-lived execution latest-state read; the secondary SQLite spool + remains the replay source and promotes the same record once confirmed. + """ + + if binding.feed is not FeedType.MARK_INDEX_PRICE: + return + if not binding.authoritative or binding.source_role != "PRIMARY": + raise ValueError("execution MARK/INDEX binding is not authoritative primary") + if envelope.WhichOneof("payload") != "mark_index_price": + raise ValueError("execution MARK/INDEX view received a different payload") + uid = envelope.instrument_uid + if uid not in self._allowed_instrument_uids: + raise ValueError("execution MARK/INDEX event is outside declared bindings") + if ( + envelope.instrument_id != binding.instrument.instrument_id + or int(envelope.instrument_revision) != binding.instrument.metadata_revision + or envelope.source_id != binding.source_id + or envelope.source_event_time_ns <= 0 + or envelope.received_at_ns <= 0 + or not envelope.source_session_id + or envelope.connection_generation < 1 + or envelope.config_revision < 1 + or gateway_epoch < 1 + ): + raise ValueError("execution MARK/INDEX event identity/provenance is invalid") + canonical = envelope.SerializeToString(deterministic=True) + if stored is not None and ( + stored.event.event_id != bytes(envelope.event_id) + or stored.event.payload != canonical + ): + raise ValueError("execution MARK/INDEX spool confirmation differs from canonical") + record = ExecutionMarkIndexRecord( + canonical=canonical, + event_id=bytes(envelope.event_id), + instrument_uid=uid, + instrument_revision=int(envelope.instrument_revision), + source_policy_id=binding.source_policy_id, + freshness_basis=binding.freshness_basis, + stale_after_ms=binding.stale_after_ms, + source_event_time_ns=int(envelope.source_event_time_ns), + received_at_ns=int(envelope.received_at_ns), + venue=str(envelope.venue), + market=str(envelope.market), + source_session_id=str(envelope.source_session_id), + config_revision=int(envelope.config_revision), + connection_generation=int(envelope.connection_generation), + partition_sequence=int(envelope.partition_sequence), + spool_watermark_offset=(stored.cursor.offset if stored is not None else None), + delivery_stage=( + _DELIVERY_SPOOL_CONFIRMED + if stored is not None + else _DELIVERY_CANONICAL_READ_COMMITTED + ), + gateway_epoch=gateway_epoch, + ) + has_gap = bool(_GAP_FLAGS.intersection(envelope.quality_flags)) + async with self._lock: + current = self._records.get(uid) + if has_gap: + self._records.pop(uid, None) + self._invalid[uid] = ( + gateway_epoch, + record.connection_generation, + "GAP_OR_RESYNC", + ) + return + invalid = self._invalid.get(uid) + if invalid is not None: + invalid_epoch, invalid_generation, _reason = invalid + if gateway_epoch < invalid_epoch: + return + # A gap remains blocking until the provider has established a + # newer connection generation. A later frame from the same + # uncertain session is not proof that the missing range was + # recovered. + if ( + gateway_epoch == invalid_epoch + and record.connection_generation <= invalid_generation + ): + return + self._invalid.pop(uid, None) + if current is not None: + if current.gateway_epoch > gateway_epoch: + return + if current.gateway_epoch == gateway_epoch: + # Never let a delayed frame from an older provider session + # overwrite the confirmed current generation. + if record.connection_generation < current.connection_generation: + return + if record.connection_generation == current.connection_generation: + current_order = ( + current.received_at_ns, + current.partition_sequence, + ) + incoming_order = ( + record.received_at_ns, + record.partition_sequence, + ) + if current_order > incoming_order: + return + if current_order == incoming_order: + # A spool confirmation may only promote the exact + # event already visible from the read-committed + # canonical plane. It may not replace an equally + # ordered, different event. + if current.event_id != record.event_id: + return + if ( + current.spool_watermark_offset is not None + or record.spool_watermark_offset is None + ): + return + self._records[uid] = record + + async def withdraw( + self, + *, + instrument_uid: str, + event_id: bytes, + gateway_epoch: int, + ) -> None: + """Withdraw only an unconfirmed event when its spool append fails.""" + + if not instrument_uid or not event_id or gateway_epoch < 1: + raise ValueError("execution MARK/INDEX withdrawal identity is invalid") + async with self._lock: + current = self._records.get(instrument_uid) + if ( + current is not None + and current.gateway_epoch == gateway_epoch + and current.event_id == event_id + and current.spool_watermark_offset is None + ): + self._records.pop(instrument_uid, None) + + async def read( + self, + *, + instrument_uid: str, + instrument_revision: int, + source_policy_id: str, + max_freshness_ms: int, + gateway_epoch: int, + event_recency_policy: StalePolicy = StalePolicy.BLOCK, + max_session_liveness_ms: int | None = None, + now_ns: int | None = None, + ) -> ExecutionMarkIndexRead: + """Return only a view valid for this exact execution request.""" + + if ( + not instrument_uid + or instrument_revision < 1 + or max_freshness_ms <= 0 + or not isinstance(event_recency_policy, StalePolicy) + ): + raise ValueError("execution MARK/INDEX read identity/freshness is invalid") + if ( + event_recency_policy is StalePolicy.UNSPECIFIED + or ( + event_recency_policy is StalePolicy.OBSERVE + and (max_session_liveness_ms is None or max_session_liveness_ms <= 0) + ) + ): + raise ValueError("execution MARK/INDEX quiet policy is invalid") + now_ns = time.time_ns() if now_ns is None else now_ns + async with self._lock: + invalid = self._invalid.get(instrument_uid) + if invalid is not None and invalid[0] == gateway_epoch: + return ExecutionMarkIndexRead(None, invalid[2]) + record = self._records.get(instrument_uid) + if record is None: + return ExecutionMarkIndexRead(None, "NOT_READY") + if record.gateway_epoch != gateway_epoch: + return ExecutionMarkIndexRead(None, "FENCED") + if record.instrument_revision != instrument_revision: + return ExecutionMarkIndexRead(None, "IDENTITY_MISMATCH") + if record.source_policy_id != source_policy_id: + return ExecutionMarkIndexRead(None, "SOURCE_POLICY_MISMATCH") + freshness_anchor_ns = ( + record.received_at_ns + if record.freshness_basis == "PROVIDER_CONFIRMATION" + else record.source_event_time_ns + ) + bound_ms = min(max_freshness_ms, record.stale_after_ms) + event_age_ms = max(0, (now_ns - freshness_anchor_ns) // 1_000_000) + if event_recency_policy is not StalePolicy.OBSERVE: + decision = evaluate_binding_quality(BindingQualityInput( + binding_id=f"execution-mark-index:{instrument_uid}", + instrument_uid=instrument_uid, + feed=FeedType.MARK_INDEX_PRICE.value, + source_role="PRIMARY", + authoritative=True, + acquisition_enabled=True, + acquisition_mode="RUST_NATIVE", + market_open=True, + event_present=True, + event_age_ms=event_age_ms, + event_limit_ms=bound_ms, + event_recency_policy=event_recency_policy.value, + session_state="NOT_APPLICABLE", + session_liveness_ms=None, + session_limit_ms=None, + watermark_offset=record.spool_watermark_offset or 0, + )) + if decision.state != "LIVE": + return ExecutionMarkIndexRead(None, "STALE") + return ExecutionMarkIndexRead(record, recency_mode="STRICT_EVENT") + + policy = self._quiet_policies.get(instrument_uid) + if policy is None or self._session_liveness_reader is None: + return ExecutionMarkIndexRead(None, "QUIET_POLICY_UNAVAILABLE") + try: + envelope = market_data_pb2.EventEnvelope.FromString(record.canonical) + lineage = paired_mark_index_lineage(envelope) + except (TypeError, ValueError): + return ExecutionMarkIndexRead(None, "LINEAGE_INVALID") + component_receipts = ( + ("MARK", lineage.mark_received_at_ns), + ("INDEX", lineage.index_received_at_ns), + ) + component_cadence = policy.component_quiet_after_ms + cadence_by_component = dict(component_cadence) + session = self._session_liveness_reader.status( + venue=record.venue, + market=record.market, + source_session_id=record.source_session_id, + connection_generation=record.connection_generation, + config_revision=record.config_revision, + now_ns=now_ns, + ) + assert max_session_liveness_ms is not None + decision = evaluate_binding_quality(BindingQualityInput( + binding_id=f"execution-mark-index:{instrument_uid}", + instrument_uid=instrument_uid, + feed=FeedType.MARK_INDEX_PRICE.value, + source_role="PRIMARY", + authoritative=True, + acquisition_enabled=True, + acquisition_mode="RUST_NATIVE", + market_open=True, + event_present=True, + event_age_ms=event_age_ms, + event_limit_ms=bound_ms, + event_recency_policy=event_recency_policy.value, + session_state=session.state, + session_liveness_ms=session.liveness_ms, + session_limit_ms=max_session_liveness_ms, + components=tuple( + ComponentEvidence( + component, + max(0, (now_ns - receipt_ns) // 1_000_000), + cadence_by_component[component], + ) + for component, receipt_ns in component_receipts + ), + generation_matches="SOURCE_SESSION_AMBIGUOUS" not in session.flags, + config_matches="SOURCE_SESSION_CONFIG_MISMATCH" not in session.flags, + watermark_offset=record.spool_watermark_offset or 0, + allow_quiet_execution=True, + flags=session.flags, + )) + if decision.state != "LIVE": + if any(value.startswith("COMPONENT_") for value in decision.reason_codes): + return ExecutionMarkIndexRead(None, "COMPONENT_STALE") + if session.state != "LIVE": + return ExecutionMarkIndexRead(None, "SESSION_STATE") + if ( + session.liveness_ms is None + or session.liveness_ms > max_session_liveness_ms + ): + return ExecutionMarkIndexRead(None, "SESSION_LIVENESS") + return ExecutionMarkIndexRead(None, "STALE") + return ExecutionMarkIndexRead( + record, + recency_mode=( + "STRICT_EVENT_SESSION_LIVE" + if decision.event_recency_state == "LIVE" + else "COMPONENT_SESSION_LIVE" + ), + session=session, + session_checked_at_ns=now_ns, + component_receipts_ns=component_receipts, + component_quiet_after_ms=component_cadence, + ) + + async def fence_all(self) -> None: + """A passive/reacquired gateway may never retain old writer state.""" + + async with self._lock: + self._records.clear() + self._invalid.clear() + + async def size(self) -> int: + async with self._lock: + return len(self._records) + + +def install_execution_mark_index_read( + app: FastAPI, + *, + gateway: DurableStreamGateway, + view: ExecutionMarkIndexLiveView, + secret: bytes, +) -> None: + """Install the private read edge used only by stable query replicas.""" + + if len(secret) < 32: + raise ValueError("stable internal read secret must contain at least 256 bits") + + @app.post("/internal/v2/execution/mark-index/latest", include_in_schema=False) + async def latest_mark_index( + request: Request, + response: Response, + signature: str | None = Header(None, alias="X-QDL-Stable-Signature"), + ): + body = await request.body() + if not signature or not hmac.compare_digest( + signature, stable_hmac_signature(secret, body) + ): + raise HTTPException(status_code=401, detail="invalid stable read signature") + try: + payload = json.loads(body) + legacy_fields = { + "schema", "instrument_uid", "instrument_revision", + "source_policy_id", "max_freshness_ms", + } + quiet_fields = legacy_fields | { + "event_recency_policy", "max_session_liveness_ms", + } + if ( + not isinstance(payload, dict) + or payload.get("schema") not in { + _LEGACY_REQUEST_SCHEMA, + _REQUEST_SCHEMA, + } + or ( + payload["schema"] == _LEGACY_REQUEST_SCHEMA + and set(payload) != legacy_fields + ) + or ( + payload["schema"] == _REQUEST_SCHEMA + and set(payload) != quiet_fields + ) + ): + raise ValueError("execution MARK/INDEX read schema is invalid") + instrument_uid = str(payload["instrument_uid"]) + instrument_revision = int(payload["instrument_revision"]) + source_policy_id = str(payload["source_policy_id"]) + max_freshness_ms = int(payload["max_freshness_ms"]) + if payload["schema"] == _REQUEST_SCHEMA: + event_recency_policy = StalePolicy( + str(payload["event_recency_policy"]).upper() + ) + max_session_liveness_ms = int(payload["max_session_liveness_ms"]) + if event_recency_policy is not StalePolicy.OBSERVE: + raise ValueError("quiet execution MARK/INDEX policy is invalid") + else: + event_recency_policy = StalePolicy.BLOCK + max_session_liveness_ms = None + if not instrument_uid or not source_policy_id or not 1 <= max_freshness_ms <= 300_000: + raise ValueError("execution MARK/INDEX read fields are invalid") + except (TypeError, ValueError, json.JSONDecodeError) as error: + raise HTTPException(status_code=400, detail=str(error)) from error + try: + epoch = gateway.assert_active() + assert epoch is not None + result = await view.read( + instrument_uid=instrument_uid, + instrument_revision=instrument_revision, + source_policy_id=source_policy_id, + max_freshness_ms=max_freshness_ms, + gateway_epoch=epoch, + event_recency_policy=event_recency_policy, + max_session_liveness_ms=max_session_liveness_ms, + ) + gateway.assert_active(epoch) + except GatewayFenced as error: + raise HTTPException(status_code=409, detail="execution MARK/INDEX gateway fenced") from error + if result.record is None: + raise HTTPException( + status_code=409, + detail=f"execution MARK/INDEX live view unavailable:{result.reason}", + ) + record = result.record + # A header keeps the private JSON response additive-compatible with a + # rolling reader deployment while carrying the stream-authoritative + # basis that governed this exact record's live admission. + response.headers[_FRESHNESS_BASIS_HEADER] = record.freshness_basis + if result.recency_mode is not None: + response.headers[_RECENCY_MODE_HEADER] = result.recency_mode + if result.session is not None and result.session_checked_at_ns is not None: + response.headers[_SESSION_STATE_HEADER] = result.session.state + response.headers[_SESSION_LIVENESS_HEADER] = str( + result.session.liveness_ms + ) + response.headers[_SESSION_CHECKED_AT_HEADER] = str( + result.session_checked_at_ns + ) + response.headers[_COMPONENT_RECEIPTS_HEADER] = _component_header( + result.component_receipts_ns + ) + response.headers[_COMPONENT_CADENCE_HEADER] = _component_header( + result.component_quiet_after_ms + ) + return { + "schema": _RESPONSE_SCHEMA, + "lease_epoch": record.gateway_epoch, + "spool_watermark_offset": record.spool_watermark_offset, + "delivery_stage": record.delivery_stage, + "canonical": base64.b64encode(record.canonical).decode("ascii"), + } + + +def _component_header(values: tuple[tuple[str, int], ...]) -> str: + """Encode fixed component evidence into one additive private header.""" + + if set(name for name, _value in values) != {"MARK", "INDEX"}: + raise ValueError("execution MARK/INDEX component header is incomplete") + return ",".join(f"{name}={value}" for name, value in sorted(values)) diff --git a/qdl/runtime/final_bar_watermark.py b/qdl/runtime/final_bar_watermark.py new file mode 100644 index 0000000..772edd0 --- /dev/null +++ b/qdl/runtime/final_bar_watermark.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from qdl.marketdata.v2 import market_data_pb2 +from qdl.transport import FINAL_BAR_CLOSE_TIME_NS_HEADER + + +def final_bar_close_time_ns( + envelope: market_data_pb2.EventEnvelope, +) -> int | None: + """Return the durable latest-state watermark for an admitted final BAR.""" + + if envelope.WhichOneof("payload") != "bar": + return None + bar = envelope.bar + if ( + not bar.is_final + or bar.lifecycle + not in { + market_data_pb2.BAR_LIFECYCLE_FINAL, + market_data_pb2.BAR_LIFECYCLE_REVISED, + } + ): + return None + close_time_ns = int(bar.close_time_ns) + if close_time_ns <= 0: + raise ValueError("final BAR close time is invalid") + return close_time_ns + + +def final_bar_watermark_headers( + envelope: market_data_pb2.EventEnvelope, +) -> dict[str, str]: + """Return private durable metadata; the protobuf payload remains unchanged.""" + + close_time_ns = final_bar_close_time_ns(envelope) + return ( + {FINAL_BAR_CLOSE_TIME_NS_HEADER: str(close_time_ns)} + if close_time_ns is not None + else {} + ) diff --git a/qdl/runtime/internal_auth.py b/qdl/runtime/internal_auth.py new file mode 100644 index 0000000..f84bc76 --- /dev/null +++ b/qdl/runtime/internal_auth.py @@ -0,0 +1,37 @@ +"""Small shared authentication helpers for stable-internal HTTP edges. + +The stable projector and query roles already share an internal HMAC secret and +workload mTLS. Keeping the signature exact in one module prevents a new +private read endpoint from drifting from the existing canonical-ingest boundary. +""" + +from __future__ import annotations + +import hashlib +import hmac +import ipaddress +from urllib.parse import urlsplit + + +def stable_hmac_signature(secret: bytes, body: bytes) -> str: + """Return the stable-internal signature for one exact request body.""" + + return "sha256=" + hmac.new(secret, body, hashlib.sha256).hexdigest() + + +def is_stable_internal_url(value: str) -> bool: + """Allow only the fixed internal stream-gateway address space.""" + + parsed = urlsplit(value) + if parsed.scheme not in {"http", "https"} or not parsed.hostname: + return False + try: + return ipaddress.ip_address(parsed.hostname).is_loopback + except ValueError: + return parsed.hostname in { + "localhost", + "stream_v2", + "stream_v2_active", + "stream_v2_passive", + "qdl-stable-stream", + } or parsed.hostname.endswith(".internal") diff --git a/qdl/runtime/lease.py b/qdl/runtime/lease.py index 347f5ba..cce9f6b 100644 --- a/qdl/runtime/lease.py +++ b/qdl/runtime/lease.py @@ -193,6 +193,7 @@ def __init__( ttl_seconds: int = 15, renew_interval_seconds: float = 5.0, on_fenced: Callable[[], Awaitable[None]] | None = None, + on_acquired: Callable[[GatewayLease], Awaitable[None]] | None = None, clock_ns=time.time_ns, ) -> None: if not shard_id.strip() or not owner_id.strip(): @@ -207,6 +208,7 @@ def __init__( self.ttl_seconds = ttl_seconds self.renew_interval_seconds = renew_interval_seconds self.on_fenced = on_fenced + self.on_acquired = on_acquired self._clock_ns = clock_ns self.lease: GatewayLease | None = None self._task: asyncio.Task | None = None @@ -233,12 +235,31 @@ async def acquire_once(self) -> bool: lease = await self.store.acquire( self.shard_id, self.owner_id, self.ttl_seconds ) - self.last_error = None except Exception as error: self.last_error = f"{type(error).__name__}: {error}" lease = None + if lease is None: + self.lease = None + return False + previous = self.lease self.lease = lease - return lease is not None + if self.on_acquired is not None and previous != lease: + try: + await self.on_acquired(lease) + except Exception as error: + self.lease = None + self.last_error = f"activation {type(error).__name__}: {error}" + try: + await self.store.release(lease) + except Exception as release_error: + self.last_error += ( + f"; release {type(release_error).__name__}: {release_error}" + ) + if self.on_fenced is not None: + await self.on_fenced() + return False + self.last_error = None + return True async def _lose_lease(self) -> None: had_lease = self.lease is not None diff --git a/qdl/runtime/mark_index_lineage.py b/qdl/runtime/mark_index_lineage.py index 3546121..f1c6a32 100644 --- a/qdl/runtime/mark_index_lineage.py +++ b/qdl/runtime/mark_index_lineage.py @@ -2,12 +2,14 @@ import hashlib from dataclasses import dataclass +from typing import TYPE_CHECKING from qdl.marketdata.v2 import market_data_pb2 from qdl.provider.v1 import raw_provider_pb2 -from qdl.query import FeedType from qdl.raw.envelope import validate_raw_envelope -from qdl.runtime.stable_catalog import StableSourceBinding + +if TYPE_CHECKING: + from qdl.runtime.stable_catalog import StableSourceBinding DERIVED_MARK_INDEX_COMPONENT_V1 = "DERIVED_MARK_INDEX_COMPONENT_V1" @@ -20,6 +22,10 @@ class DerivedMarkIndexLineage: mark_capture_id: bytes index_capture_id: bytes + mark_source_event_time_ms: int + index_source_event_time_ms: int + mark_received_at_ns: int + index_received_at_ns: int def paired_mark_index_lineage( @@ -65,6 +71,10 @@ def paired_mark_index_lineage( return DerivedMarkIndexLineage( mark_capture_id=mark_capture_id, index_capture_id=index_capture_id, + mark_source_event_time_ms=source_times[0], + index_source_event_time_ms=source_times[1], + mark_received_at_ns=source_times[2], + index_received_at_ns=source_times[3], ) @@ -107,7 +117,12 @@ def validate_derived_mark_index_component( """Validate one physical raw component of a Rust-derived mark/index pair.""" if ( - binding.feed is not FeedType.MARK_INDEX_PRICE + # Keep this lower-level lineage verifier independent of qdl.query's + # public package initializer. The binding already carries the stable + # contract enum's wire value; importing that package here creates a + # cycle through the execution live-reader during a clean projector + # import. + getattr(binding.feed, "value", binding.feed) != "MARK_INDEX_PRICE" or binding.v1_compatibility != "NONE" ): raise ValueError("derived MARK_INDEX lineage is not permitted for binding") diff --git a/qdl/runtime/production_catalog.py b/qdl/runtime/production_catalog.py index 32d3bb9..ee46f0d 100644 --- a/qdl/runtime/production_catalog.py +++ b/qdl/runtime/production_catalog.py @@ -60,6 +60,7 @@ "SPOT": "wss://stream.binance.com:9443/ws", } _BOOK_FEEDS = frozenset({FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA}) +_ON_CHANGE_QUOTE_MARKETS = frozenset({("BINANCE", "USDM"), ("OKX", "SWAP")}) _SUPPORTED_FEEDS = { FeedType.TRADE, FeedType.QUOTE, @@ -72,6 +73,14 @@ "SPOT": "https://api.binance.com/api/v3/depth", } _OKX_PUBLIC_WS = "wss://ws.okx.com:8443/ws/v5/public" +# Provider-declared quiet cadence is compiled into the signed acquisition +# bundle, never inferred by a reader at runtime. The margin is deliberately +# finite: it covers normal provider cadence plus transport jitter, while the +# independent session-liveness bound still detects a dead connection. +_MARK_INDEX_COMPONENT_QUIET_AFTER_MS = { + ("BINANCE", "USDM"): {"BOTH": 5_000}, + ("OKX", "SWAP"): {"MARK": 15_000, "INDEX": 70_000}, +} @dataclass(frozen=True, slots=True, order=True) @@ -662,6 +671,14 @@ def _source_binding( if item.feed is FeedType.MARK_INDEX_PRICE else {} ), + **( + {"delivery_semantics": "ON_CHANGE"} + if ( + item.feed is FeedType.QUOTE + and (item.venue, item.market) in _ON_CHANGE_QUOTE_MARKETS + ) + else {} + ), }, "v1_compatibility": compatibility, } @@ -804,6 +821,14 @@ def _acquisition(binding_id: str, item: ProductionDemand) -> dict[str, Any]: "snapshot_refresh_seconds": 30, } if item.feed is FeedType.MARK_INDEX_PRICE: + try: + component_quiet_after_ms = _MARK_INDEX_COMPONENT_QUIET_AFTER_MS[ + (item.venue, item.market) + ] + except KeyError as error: + raise ValueError( + "MARK_INDEX provider quiet cadence is not certified" + ) from error result["mark_index"] = { "provider_protocol": ( "BINANCE_MARK_PRICE" @@ -811,6 +836,7 @@ def _acquisition(binding_id: str, item: ProductionDemand) -> dict[str, Any]: else "OKX_MARK_INDEX" ), "index_native_symbol": item.index_native_symbol, + "component_quiet_after_ms": dict(component_quiet_after_ms), } return result diff --git a/qdl/runtime/routed_query.py b/qdl/runtime/routed_query.py index f6837fe..3985531 100644 --- a/qdl/runtime/routed_query.py +++ b/qdl/runtime/routed_query.py @@ -112,6 +112,21 @@ def history(self, requirement: DataRequirement) -> HistoryResult | None: return None raise QueryBackendError(error.problem) from error + def history_many( + self, + requirements: tuple[DataRequirement, ...], + ) -> dict[DataRequirement, HistoryResult | None | Exception]: + """Expose the spool's one-snapshot batch path only for local routes. + + A pass-through-eligible requirement must retain its ordinary per-item + provider path. This guard keeps local batch acceleration from silently + turning a declared recovery policy into a cache-only answer. + """ + + if not all(self.warmup_is_local(requirement) for requirement in requirements): + raise ValueError("batch history acceleration requires local authoritative routes") + return self.spool.history_many(requirements) + def latest(self, requirement: DataRequirement) -> MarketDataItem | None: if not self.routes_to_pass_through(requirement): item = self.spool.latest(requirement) diff --git a/qdl/runtime/stable.py b/qdl/runtime/stable.py index 8c95d03..347dd51 100644 --- a/qdl/runtime/stable.py +++ b/qdl/runtime/stable.py @@ -23,7 +23,12 @@ from qdl.projection.stable import RedisStableProjectionTarget, StableCompatibilityProjector from qdl.replay import GapFreeHandoff, SignedHandoffCursorCodec from qdl.runtime.bounds import BoundedRequestMiddleware, RequestBounds -from qdl.runtime.lease import ActivePassiveGatewayLease, RedisGatewayLeaseStore +from qdl.runtime.execution_mark_index import ( + ExecutionMarkIndexLiveView, + install_execution_mark_index_read, +) +from qdl.runtime.internal_auth import is_stable_internal_url +from qdl.runtime.lease import ActivePassiveGatewayLease, GatewayLease, RedisGatewayLeaseStore from qdl.runtime.readiness import ( CallableReadinessProbe, ComponentReadiness, @@ -32,7 +37,11 @@ ) from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_capacity import STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW -from qdl.runtime.stable_deployment import validate_shared_authority_record +from qdl.runtime.stable_deployment import ( + StableAcquisitionPlan, + validate_shared_authority_record, +) +from qdl.runtime.session_liveness import StableSessionLivenessReader from qdl.runtime.stable_ingest import ( StableHttpCanonicalSink, install_stable_canonical_ingest, @@ -46,6 +55,7 @@ StableGrpcSnapshotLoader, build_stable_query_stack, ) +from qdl.reference.execution_live import HttpExecutionMarkIndexReader from qdl.security import ( AuditChain, DataPlaneIdentityService, @@ -161,6 +171,7 @@ class StableRuntimeConfig: audit_path: Path manifest_paths: tuple[Path, ...] source_bindings_path: Path + acquisition_bindings_path: Path tls_ca_path: Path tls_certificate_path: Path tls_private_key_path: Path @@ -189,10 +200,12 @@ class StableRuntimeConfig: kafka_canonical_topic: str | None = None kafka_cert_root: Path | None = None stream_ingest_urls: tuple[str, ...] = () + execution_mark_index_urls: tuple[str, ...] = () max_pending_records: int = 10_000 max_pending_bytes: int = 256 * 1024 * 1024 projector_max_batch_records: int = 128 projector_max_batch_bytes: int = 8 * 1024 * 1024 + projector_max_commit_records: int = 128 # DL-V2 R1.11. How long a drain waits for more records before it commits to # the batch it has. With a backlog this is irrelevant, the batch fills at # once; at the head it is everything, because a 10 ms window collects only a @@ -236,6 +249,8 @@ def __post_init__(self) -> None: raise ValueError("stable authority revision and consumer manifests are required") if not self.source_bindings_path.is_file(): raise ValueError("stable source binding catalog is unavailable") + if self.role == "stream_v2" and not self.acquisition_bindings_path.is_file(): + raise ValueError("stable acquisition binding plan is unavailable") missing_tls = [ path for path in ( self.tls_ca_path, @@ -270,6 +285,10 @@ def __post_init__(self) -> None: if self.role == "projector_v2": if not 1 <= self.projector_max_batch_records <= 1000: raise ValueError("stable projector batch bound must be 1..1000") + if not 1 <= self.projector_max_commit_records <= self.projector_max_batch_records: + raise ValueError( + "stable projector commit batch bound must fit inside fetch batch" + ) if not 0 < self.projector_batch_wait_seconds <= 1: raise ValueError("stable projector batch wait must be within 0..1 seconds") if self.max_pending_records < self.projector_max_batch_records: @@ -292,6 +311,12 @@ def __post_init__(self) -> None: self.kafka_canonical_topic, self.kafka_cert_root, )) or not self.stream_ingest_urls: raise ValueError("stable projector Kafka/stream dependencies are required") + if any(not is_stable_internal_url(value) for value in self.execution_mark_index_urls): + raise ValueError("stable execution MARK/INDEX URLs must be private stream endpoints") + if self.execution_mark_index_urls and not self.reference_data_enabled: + raise ValueError( + "stable execution MARK/INDEX reader requires reference data to be enabled" + ) @property def session_liveness_dir(self) -> Path: @@ -319,6 +344,13 @@ def from_environment( urls_raw = json.loads(env.get("QDL_STABLE_STREAM_INGEST_URLS_JSON", "[]")) if not isinstance(urls_raw, list): raise ValueError("QDL_STABLE_STREAM_INGEST_URLS_JSON must be an array") + execution_mark_index_urls_raw = json.loads( + env.get("QDL_STABLE_EXECUTION_MARK_INDEX_URLS_JSON", "[]") + ) + if not isinstance(execution_mark_index_urls_raw, list): + raise ValueError( + "QDL_STABLE_EXECUTION_MARK_INDEX_URLS_JSON must be an array" + ) instance_id = env.get("QDL_STABLE_INSTANCE_ID", f"stable-{role}-local") return cls( role=role, @@ -337,6 +369,10 @@ def from_environment( )), manifest_paths=manifests, source_bindings_path=Path(env["QDL_STABLE_SOURCE_BINDINGS"]), + acquisition_bindings_path=Path(env.get( + "QDL_STABLE_ACQUISITION_BINDINGS", + "/app/config/v2/stable-acquisition-bindings.yaml", + )), tls_ca_path=Path(env["QDL_STABLE_TLS_CA_FILE"]), tls_certificate_path=Path(env["QDL_STABLE_TLS_CERT_FILE"]), tls_private_key_path=Path(env["QDL_STABLE_TLS_KEY_FILE"]), @@ -370,6 +406,9 @@ def from_environment( kafka_canonical_topic=env.get("QDL_STABLE_KAFKA_CANONICAL_TOPIC"), kafka_cert_root=Path(cert_root_raw) if cert_root_raw else None, stream_ingest_urls=tuple(str(value) for value in urls_raw), + execution_mark_index_urls=tuple( + str(value) for value in execution_mark_index_urls_raw + ), max_pending_records=int(env.get("QDL_STABLE_MAX_PENDING_RECORDS", "10000")), max_pending_bytes=int(env.get("QDL_STABLE_MAX_PENDING_BYTES", "268435456")), projector_batch_wait_seconds=float( @@ -381,6 +420,9 @@ def from_environment( projector_max_batch_bytes=int( env.get("QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES", "8388608") ), + projector_max_commit_records=int( + env.get("QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS", "128") + ), pass_through_enabled=_env_flag( env, "QDL_STABLE_PASS_THROUGH_ENABLED", default=False ), @@ -608,6 +650,15 @@ def create_stable_query_app(config: StableRuntimeConfig | None = None) -> FastAP catalog = StableSourceCatalog.load(config.source_bindings_path) spool = build_stable_spool(config, catalog) handoff = build_stable_handoff(config, spool) + execution_mark_index_reader = ( + HttpExecutionMarkIndexReader( + config.execution_mark_index_urls, + config.internal_ingest_secret, + ssl_context=stable_client_ssl_context(config), + ) + if config.execution_mark_index_urls + else None + ) service, _backend, issuer = build_stable_query_stack( spool=spool, catalog=catalog, schema_digest=config.schema_digest, handoff=handoff, cursor_ttl_seconds=config.cursor_ttl_seconds, @@ -616,6 +667,7 @@ def create_stable_query_app(config: StableRuntimeConfig | None = None) -> FastAP provider_admission_url=config.provider_admission_url, provider_admission_secret=config.internal_ingest_secret, session_liveness_root=str(config.session_liveness_dir), + execution_mark_index_reader=execution_mark_index_reader, ) readiness = stable_readiness( config, manifests, spool, quota=identity.quota, @@ -633,10 +685,13 @@ def create_stable_query_app(config: StableRuntimeConfig | None = None) -> FastAP app.state.runtime_manifest = config.public_manifest() app.state.stable_spool = spool app.state.stable_audit = AuditChain(config.audit_path) + app.state.execution_mark_index_reader = execution_mark_index_reader install_stable_health(app, readiness, config.public_manifest()) @app.on_event("shutdown") async def close_stable_query(): + if execution_mark_index_reader is not None: + await execution_mark_index_reader.close() await asyncio.to_thread(spool.close) await asyncio.to_thread(identity.quota.close) @@ -649,6 +704,7 @@ class StableStreamRuntime: redis: AsyncRedis spool: SQLiteDurableSpool gateway: DurableStreamGateway + execution_mark_index_view: ExecutionMarkIndexLiveView lease: ActivePassiveGatewayLease grpc_server: grpc.aio.Server health_app: FastAPI @@ -681,6 +737,9 @@ def create_stable_stream_runtime( manifests = load_stable_manifests(config) identity = build_stable_identity(config, manifests) catalog = StableSourceCatalog.load(config.source_bindings_path) + acquisition = StableAcquisitionPlan.load( + config.acquisition_bindings_path, catalog=catalog + ) spool = build_stable_spool(config, catalog) handoff = build_stable_handoff(config, spool) async_redis = AsyncRedis.from_url(config.redis_url, decode_responses=True) @@ -696,7 +755,32 @@ def create_stable_stream_runtime( max_replay_events=config.max_replay_events, cursor_ttl_seconds=config.cursor_ttl_seconds, authority=lease, ) - lease.on_fenced = gateway.fence_all + execution_mark_index_view = ExecutionMarkIndexLiveView.from_catalog( + catalog, + acquisition=acquisition, + session_liveness_reader=StableSessionLivenessReader( + config.session_liveness_dir + ), + ) + + async def fence_gateway_execution_view() -> None: + await gateway.fence_all() + await execution_mark_index_view.fence_all() + + async def hydrate_gateway_execution_view(lease_record: GatewayLease) -> None: + restored = await execution_mark_index_view.hydrate_from_spool( + spool=spool, + canonical_stream=catalog.canonical_stream, + gateway_epoch=lease_record.epoch, + ) + logger.info( + "execution MARK/INDEX view hydrated epoch=%s bindings=%s", + lease_record.epoch, + restored, + ) + + lease.on_fenced = fence_gateway_execution_view + lease.on_acquired = hydrate_gateway_execution_view query_service, backend, issuer = build_stable_query_stack( spool=spool, catalog=catalog, schema_digest=config.schema_digest, handoff=handoff, cursor_ttl_seconds=config.cursor_ttl_seconds, @@ -734,9 +818,15 @@ def create_stable_stream_runtime( install_stable_canonical_ingest( app, gateway=gateway, catalog=catalog, spool=spool, secret=config.internal_ingest_secret, + execution_mark_index_view=execution_mark_index_view, + ) + install_execution_mark_index_read( + app, gateway=gateway, view=execution_mark_index_view, + secret=config.internal_ingest_secret, ) return StableStreamRuntime( - config, async_redis, spool, gateway, lease, grpc_server, app, identity.quota + config, async_redis, spool, gateway, execution_mark_index_view, + lease, grpc_server, app, identity.quota ) @@ -849,6 +939,7 @@ def broker_factory(): max_pending_bytes=config.max_pending_bytes, max_batch_records=config.projector_max_batch_records, max_batch_bytes=config.projector_max_batch_bytes, + max_commit_records=config.projector_max_commit_records, batch_wait_seconds=config.projector_batch_wait_seconds, ) diff --git a/qdl/runtime/stable_catalog.py b/qdl/runtime/stable_catalog.py index d274c47..b79c54d 100644 --- a/qdl/runtime/stable_catalog.py +++ b/qdl/runtime/stable_catalog.py @@ -79,6 +79,10 @@ class StableSourceBinding: # mark/index feed may instead use its authenticated receipt confirmation # for execution freshness when the provider repeats an unchanged value. freshness_basis: str = "SOURCE_EVENT" + # Native BBO may be emitted only when a best bid/offer changes. This source + # declaration is validated again against its physical acquisition binding; + # a consumer manifest cannot opt into it by itself. + delivery_semantics: str = "STRICT_EVENT" # Explicitly signed historical revisions are replay-only lineage. They # never change the current instrument metadata returned to consumers. historical_metadata_revisions: tuple[int, ...] = () @@ -121,6 +125,13 @@ def __post_init__(self) -> None: raise ValueError("stable source freshness bound must be positive") if self.freshness_basis not in {"SOURCE_EVENT", "PROVIDER_CONFIRMATION"}: raise ValueError("stable source freshness basis is invalid") + if self.delivery_semantics not in {"STRICT_EVENT", "ON_CHANGE"}: + raise ValueError("stable source delivery semantics is invalid") + if ( + self.delivery_semantics == "ON_CHANGE" + and self.feed is not FeedType.QUOTE + ): + raise ValueError("on-change delivery is reserved for native BBO QUOTE") if ( self.freshness_basis == "PROVIDER_CONFIRMATION" and self.feed is not FeedType.MARK_INDEX_PRICE @@ -403,7 +414,7 @@ def _binding( raise ValueError("stable source lineage fields are incomplete or unknown") required_quality = {"stale_after_ms", "require_final_bar", "continuous_calendar"} if not required_quality <= set(quality) or set(quality) - required_quality - { - "freshness_basis" + "freshness_basis", "delivery_semantics" }: raise ValueError("stable source quality fields are incomplete or unknown") interval = raw["interval"] @@ -425,6 +436,7 @@ def _binding( v1_compatibility=str(raw["v1_compatibility"]).upper(), canonical_stream=canonical_stream, freshness_basis=str(quality.get("freshness_basis", "SOURCE_EVENT")).upper(), + delivery_semantics=str(quality.get("delivery_semantics", "STRICT_EVENT")).upper(), historical_metadata_revisions=history_by_uid[instrument.instrument_uid], ) diff --git a/qdl/runtime/stable_deployment.py b/qdl/runtime/stable_deployment.py index 1424aa3..ecd02ce 100644 --- a/qdl/runtime/stable_deployment.py +++ b/qdl/runtime/stable_deployment.py @@ -154,6 +154,29 @@ class StableMarkIndexAcquisition: provider_protocol: str index_native_symbol: str | None + component_quiet_after_ms: tuple[tuple[str, int], ...] = () + + def __post_init__(self) -> None: + names = tuple(name for name, _value in self.component_quiet_after_ms) + if len(names) != len(set(names)): + raise ValueError("MARK_INDEX component cadence contains duplicates") + if any( + name not in {"BOTH", "MARK", "INDEX"} + or isinstance(value, bool) + or not isinstance(value, int) + or not 250 <= value <= 120_000 + for name, value in self.component_quiet_after_ms + ): + raise ValueError("MARK_INDEX component cadence is invalid") + + def quiet_after_ms_for(self, component: str) -> int | None: + """Return the signed quiet window for one physical component. + + An absent value deliberately preserves strict legacy behavior. New + generated bindings must carry a complete provider-specific mapping. + """ + + return dict(self.component_quiet_after_ms).get(component) def validate( self, @@ -177,6 +200,8 @@ def validate( or self.index_native_symbol is not None ): raise ValueError("Binance MARK_INDEX acquisition differs from provider protocol") + if self.component_quiet_after_ms and set(dict(self.component_quiet_after_ms)) != {"BOTH"}: + raise ValueError("Binance MARK_INDEX component cadence is incomplete") return if self.provider_protocol == "OKX_MARK_INDEX": index_symbol = (self.index_native_symbol or "").strip().upper() @@ -189,6 +214,8 @@ def validate( or index_symbol == source.instrument.native_symbol ): raise ValueError("OKX MARK_INDEX acquisition differs from provider protocol") + if self.component_quiet_after_ms and set(dict(self.component_quiet_after_ms)) != {"MARK", "INDEX"}: + raise ValueError("OKX MARK_INDEX component cadence is incomplete") return raise ValueError("MARK_INDEX provider protocol is not certified") @@ -237,6 +264,7 @@ class StablePhysicalEntry: physical_native_channel: str provider_kind: str mark_index_component: str | None = None + mark_index_quiet_after_ms: int | None = None def __iter__(self): """Keep the former private `(source, acquisition)` projection usable. @@ -385,6 +413,26 @@ def validate(self, source: StableSourceBinding) -> None: ) if self.provider_kind not in allowed: raise ValueError("stable acquisition provider kind differs from catalog feed") + if source.delivery_semantics == "ON_CHANGE": + if ( + source.feed is not FeedType.QUOTE + or self.mode != "RUST_NATIVE" + or self.provider_kind not in { + "binance_usdm_bbo", "binance_spot_bbo", "okx_bbo", + } + ): + raise ValueError( + "on-change delivery requires a Rust-native documented BBO lane" + ) + if ( + self.runtime == "BINANCE" + and not self.native_channel.endswith("@bookTicker") + ) or ( + self.runtime == "OKX" and self.native_channel != "bbo-tbt" + ): + raise ValueError( + "on-change delivery channel differs from the documented BBO lane" + ) if self.l2 is not None and self.mark_index is not None: raise ValueError("stable acquisition cannot combine L2 and MARK_INDEX") if self.l2 is not None: @@ -572,9 +620,11 @@ def load( mark_index_raw = value.get("mark_index") if mark_index_raw is not None: required_mark_index = {"provider_protocol", "index_native_symbol"} + optional_mark_index = {"component_quiet_after_ms"} if ( not isinstance(mark_index_raw, dict) - or set(mark_index_raw) != required_mark_index + or not required_mark_index <= set(mark_index_raw) + or set(mark_index_raw) - required_mark_index - optional_mark_index or ( mark_index_raw["index_native_symbol"] is not None and not isinstance(mark_index_raw["index_native_symbol"], str) @@ -583,6 +633,27 @@ def load( raise ValueError( "stable MARK_INDEX acquisition fields are incomplete or unknown" ) + cadence_raw = mark_index_raw.get("component_quiet_after_ms") + if cadence_raw is None: + cadence = () + elif ( + not isinstance(cadence_raw, dict) + or not cadence_raw + or any( + not isinstance(name, str) + or isinstance(value, bool) + or not isinstance(value, int) + for name, value in cadence_raw.items() + ) + ): + raise ValueError("stable MARK_INDEX component cadence is invalid") + else: + cadence = tuple( + sorted( + (str(name).upper(), int(value)) + for name, value in cadence_raw.items() + ) + ) mark_index = StableMarkIndexAcquisition( provider_protocol=str(mark_index_raw["provider_protocol"]).upper(), index_native_symbol=( @@ -590,6 +661,7 @@ def load( if mark_index_raw["index_native_symbol"] is not None else None ), + component_quiet_after_ms=cadence, ) else: mark_index = None @@ -736,6 +808,9 @@ def _physical_entries( physical_native_channel=physical_channel, provider_kind=provider_kind, mark_index_component=component, + mark_index_quiet_after_ms=( + acquisition.mark_index.quiet_after_ms_for(component) + ), )) return tuple(result) @@ -794,6 +869,10 @@ def core_config( item["physical_native_symbol"] = entry.physical_native_symbol item["physical_native_channel"] = entry.physical_native_channel item["mark_index"] = {"component": entry.mark_index_component} + if entry.mark_index_quiet_after_ms is not None: + item["mark_index"]["quiet_after_ms"] = ( + entry.mark_index_quiet_after_ms + ) bindings.append(item) return { "core": { diff --git a/qdl/runtime/stable_ingest.py b/qdl/runtime/stable_ingest.py index 2212b75..4333882 100644 --- a/qdl/runtime/stable_ingest.py +++ b/qdl/runtime/stable_ingest.py @@ -2,9 +2,7 @@ import asyncio import base64 -import hashlib import hmac -import ipaddress import json import logging import re @@ -25,8 +23,11 @@ validate_derived_mark_index_component, validate_single_raw_lineage, ) +from qdl.runtime.execution_mark_index import ExecutionMarkIndexLiveView +from qdl.runtime.final_bar_watermark import final_bar_watermark_headers +from qdl.runtime.internal_auth import is_stable_internal_url, stable_hmac_signature from qdl.runtime.lease import GatewayFenced -from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_catalog import StableSourceBinding, StableSourceCatalog from qdl.stream import DurableStreamGateway from qdl.transport import BackpressureRequired, DurableEvent, SQLiteDurableSpool, StoredEvent @@ -38,26 +39,6 @@ logger = logging.getLogger(__name__) -def _signature(secret: bytes, body: bytes) -> str: - return "sha256=" + hmac.new(secret, body, hashlib.sha256).hexdigest() - - -def _internal_url(value: str) -> bool: - parsed = urlsplit(value) - if parsed.scheme not in {"http", "https"} or not parsed.hostname: - return False - try: - return ipaddress.ip_address(parsed.hostname).is_loopback - except ValueError: - return parsed.hostname in { - "localhost", - "stream_v2", - "stream_v2_active", - "stream_v2_passive", - "qdl-stable-stream", - } or parsed.hostname.endswith(".internal") - - def _bounded_rejection_detail(response: httpx.Response) -> str: """Return a bounded, payload-safe stable-ingest validation reason.""" @@ -87,6 +68,7 @@ def install_stable_canonical_ingest( catalog: StableSourceCatalog, spool: SQLiteDurableSpool, secret: bytes, + execution_mark_index_view: ExecutionMarkIndexLiveView | None = None, ) -> None: if len(secret) < 32: raise ValueError("stable internal ingest secret must contain at least 256 bits") @@ -97,7 +79,9 @@ async def ingest( signature: str | None = Header(None, alias="X-QDL-Stable-Signature"), ): body = await request.body() - if not signature or not hmac.compare_digest(signature, _signature(secret, body)): + if not signature or not hmac.compare_digest( + signature, stable_hmac_signature(secret, body) + ): raise HTTPException(status_code=401, detail="invalid stable ingest signature") try: payload = json.loads(body) @@ -200,6 +184,7 @@ async def ingest( headers={ "raw_stream": raw_stream, "raw_event_id": raw_event_id.hex(), + **final_bar_watermark_headers(envelope), **( {"raw_lineage_kind": raw_lineage_kind} if raw_lineage_kind is not None @@ -208,13 +193,41 @@ async def ingest( }, ))) + pre_spool_view_records: list[tuple[StableSourceBinding, market_data_pb2.EventEnvelope]] = [] + if execution_mark_index_view is not None: + # The signed projector is the read-committed canonical Kafka + # consumer. Offer the strictly validated MARK/INDEX event before + # the secondary SQLite projection so an execution query does not + # inherit that projection's batch tail. Any append failure below + # withdraws exactly this unconfirmed record. + for binding, envelope, _event in prepared: + await execution_mark_index_view.remember( + binding=binding, + envelope=envelope, + stored=None, + gateway_epoch=lease_epoch, + ) + pre_spool_view_records.append((binding, envelope)) + + async def withdraw_pre_spool_view_records() -> None: + if execution_mark_index_view is None: + return + for _binding, envelope in pre_spool_view_records: + await execution_mark_index_view.withdraw( + instrument_uid=envelope.instrument_uid, + event_id=bytes(envelope.event_id), + gateway_epoch=lease_epoch, + ) + try: stored_values = await gateway.publish_many( [event for _binding, _envelope, event in prepared] ) except GatewayFenced as error: + await withdraw_pre_spool_view_records() raise HTTPException(status_code=409, detail="stable gateway was fenced") from error except BackpressureRequired as error: + await withdraw_pre_spool_view_records() logger.warning( "stable canonical ingest backpressure reason=%s", _bounded_rejection_text(str(error)), @@ -223,6 +236,9 @@ async def ingest( status_code=503, detail="stable canonical cache capacity temporarily unavailable", ) from error + except BaseException: + await withdraw_pre_spool_view_records() + raise duplicate_ids = [ event.event_id for (_binding, _envelope, event), stored in zip( @@ -251,6 +267,13 @@ async def ingest( stored = stored or duplicates.get(event.event_id) if stored is None: raise HTTPException(status_code=503, detail="stable cache ACK is unavailable") + if execution_mark_index_view is not None: + await execution_mark_index_view.remember( + binding=binding, + envelope=envelope, + stored=stored, + gateway_epoch=lease_epoch, + ) results.append({ "event_id": envelope.event_id.hex(), "partition_key": binding.partition_key, @@ -278,7 +301,7 @@ class StableHttpCanonicalSink: def __post_init__(self) -> None: if ( not self.urls - or any(not _internal_url(value) for value in self.urls) + or any(not is_stable_internal_url(value) for value in self.urls) or len(self.secret) < 32 or self.timeout_seconds <= 0 or self.max_request_bytes <= 0 @@ -413,7 +436,9 @@ async def _publish_chunk( content=body, headers={ "Content-Type": "application/json", - "X-QDL-Stable-Signature": _signature(self.secret, body), + "X-QDL-Stable-Signature": stable_hmac_signature( + self.secret, body + ), }, ) if response.status_code in {409, 503}: diff --git a/qdl/runtime/stable_projector.py b/qdl/runtime/stable_projector.py index ae57412..7bcc1c0 100644 --- a/qdl/runtime/stable_projector.py +++ b/qdl/runtime/stable_projector.py @@ -23,6 +23,10 @@ ) from qdl.raw.envelope import validate_raw_envelope from qdl.runtime.heartbeat import write_heartbeat +from qdl.runtime.final_bar_watermark import ( + final_bar_close_time_ns, + final_bar_watermark_headers, +) from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.stream import DurableStreamGateway from qdl.transport import ( @@ -163,6 +167,7 @@ def __init__( max_pending_bytes: int = 256 * 1024 * 1024, max_batch_records: int = 128, max_batch_bytes: int | None = None, + max_commit_records: int | None = None, batch_wait_seconds: float = 0.025, ) -> None: if ( @@ -173,7 +178,13 @@ def __init__( raise ValueError("stable projector topics are invalid") if max_pending_records <= 0 or max_pending_bytes <= 0: raise ValueError("stable projector pending bounds must be positive") - if not 1 <= max_batch_records <= 1000 or not 0 < batch_wait_seconds <= 1: + if max_commit_records is None: + max_commit_records = max_batch_records + if ( + not 1 <= max_batch_records <= 1000 + or not 1 <= max_commit_records <= max_batch_records + or not 0 < batch_wait_seconds <= 1 + ): raise ValueError("stable projector batch policy is invalid") if max_batch_bytes is None: max_batch_bytes = min(8 * 1024 * 1024, max_pending_bytes) @@ -187,6 +198,10 @@ def __init__( # them, so each line describes its own interval. self._canonical_age_span = _SpanSummary() self._append_span = _SpanSummary() + self._poll_span = _SpanSummary() + self._lookup_span = _SpanSummary() + self._projection_span = _SpanSummary() + self._checkpoint_span = _SpanSummary() self._spans_reported_at_ns = time.time_ns() self.canonical_topic = canonical_topic self.heartbeat_path = os.environ.get("QDL_STABLE_HEARTBEAT_PATH") or None @@ -198,6 +213,7 @@ def __init__( self.max_pending_bytes = max_pending_bytes self.max_batch_records = max_batch_records self.max_batch_bytes = max_batch_bytes + self.max_commit_records = max_commit_records self.batch_wait_seconds = batch_wait_seconds poll_headroom = min(max_batch_records, max(1, max_pending_records // 4)) self._canonical_pause_high_records = max( @@ -216,6 +232,66 @@ def __init__( self._canonical_committed = 0 self._duplicate_projections = 0 self._deferred_records: deque[KafkaProjectorRecord] = deque() + self._final_bar_watermarks_prepared = False + + def _final_bar_partition_keys(self) -> tuple[str, ...]: + """Declared final-BAR partitions whose legacy history needs one seed. + + A cache created before the additive watermark table has valid BAR + history but no O(1) latest-close row. Hydrating that history before + the first Kafka poll keeps an aligned final-BAR burst out of the live + quote materialization path. + """ + + return tuple(sorted({ + binding.partition_key + for binding in self.catalog.bindings + if binding.require_final_bar and binding.feed.value == "BAR" + })) + + def _prewarm_final_bar_watermarks(self) -> tuple[int, int, int]: + """Seed missing final-BAR watermarks before this generation polls. + + The spool owns the atomic read/seed race. Another live stream or + projector can write a row while this loop is running; that case reads + the durable row rather than scanning the retained tail a second time. + """ + + partitions = self._final_bar_partition_keys() + seeded = 0 + empty = 0 + for partition_key in partitions: + if self.spool.final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=partition_key, + ) is not None: + continue + value = self.spool.hydrate_final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=partition_key, + legacy_lookup=lambda key=partition_key: self._latest_bar_close_ns(key), + ) + if value is None: + empty += 1 + else: + seeded += 1 + return len(partitions), seeded, empty + + async def prepare_for_polling(self) -> None: + """Finish the bounded legacy cache migration before accepting live work.""" + + if self._final_bar_watermarks_prepared: + return + partitions, seeded, empty = await asyncio.to_thread( + self._prewarm_final_bar_watermarks + ) + self._final_bar_watermarks_prepared = True + logger.info( + "stable projector final BAR watermark prewarm partitions=%s seeded=%s empty=%s", + partitions, + seeded, + empty, + ) async def accept(self, record: KafkaProjectorRecord) -> None: await self.accept_many((record,)) @@ -226,6 +302,7 @@ async def accept_many( values = tuple(records) if not values: return + await self.prepare_for_polling() start = 0 while start < len(values): epoch = values[start].assignment_epoch @@ -285,6 +362,7 @@ async def run_once(self, timeout_seconds: float = 1.0) -> bool: if self.heartbeat_path is not None: write_heartbeat(self.heartbeat_path, role="stable_projector", detail=self.canonical_topic) + await self.prepare_for_polling() records: list[KafkaProjectorRecord] = [] batch_bytes = 0 while self._deferred_records and len(records) < self.max_batch_records: @@ -294,12 +372,14 @@ async def run_once(self, timeout_seconds: float = 1.0) -> bool: records.append(self._deferred_records.popleft()) batch_bytes += len(item.payload) if not records: + poll_started_ns = time.time_ns() fetched = await poll_projector_records( self.broker, max_records=self.max_batch_records, timeout_seconds=timeout_seconds, batch_wait_seconds=self.batch_wait_seconds, ) + self._poll_span.observe(time.time_ns() - poll_started_ns) if not fetched: # Another projector replica may have persisted the correlated raw # envelope into the shared cache. Retry bounded local partitions so @@ -326,10 +406,20 @@ def _report_spans(self) -> None: return self._spans_reported_at_ns = now_ns logger.info( - "qdl_stable_projector_spans canonical_age_ms=%s durable_append_ms=%s", - self._canonical_age_span.report(), + "qdl_stable_projector_spans broker_poll_ms=%s canonical_lookup_ms=%s " + "durable_append_ms=%s compatibility_projection_ms=%s " + "checkpoint_ms=%s canonical_age_ms=%s", + self._poll_span.report(), + self._lookup_span.report(), self._append_span.report(), + self._projection_span.report(), + self._checkpoint_span.report(), + self._canonical_age_span.report(), ) + self._poll_span.reset() + self._lookup_span.reset() + self._projection_span.reset() + self._checkpoint_span.reset() self._canonical_age_span.reset() self._append_span.reset() @@ -354,121 +444,145 @@ def _raw_event(self, record: KafkaProjectorRecord) -> tuple[bytes, DurableEvent] async def _drain_ready(self) -> None: while True: + lookup_started_ns = time.time_ns() ready = await self._ready_batch() + self._lookup_span.observe(time.time_ns() - lookup_started_ns) if not ready: return - fresh = tuple( - item - for item in ready - if not item.already_durable + for start in range(0, len(ready), self.max_commit_records): + await self._commit_ready_batch( + ready[start:start + self.max_commit_records] + ) + + async def _commit_ready_batch( + self, ready: tuple[_ReadyCanonical, ...] + ) -> None: + """Commit one bounded FIFO slice after a larger Kafka fetch. + + Polling a large Kafka batch keeps the consumer efficient. The durable + append, Redis compatibility projection and checkpoint have different + contention properties, however, so they must not hold every selected + partition in one unbounded turn. Each slice retains the existing + downstream-before-checkpoint order and can replay idempotently. + """ + + fresh = tuple( + item + for item in ready + if not item.already_durable + and not item.semantic_duplicate + and item.terminal_reason is None + ) + append_started_ns = time.time_ns() + fresh_stored = ( + await self.sink.publish_many([item.event for item in fresh]) + if fresh + else () + ) + if fresh: + self._append_span.observe(time.time_ns() - append_started_ns) + for item in fresh: + # How old the canonical record already was when this projector + # began its durable handoff. `accepted_at_ns` is the broker's + # own stamp, never a replacement for provider lineage. + self._canonical_age_span.observe( + append_started_ns - item.record.accepted_at_ns + ) + stored_iterator = iter(fresh_stored) + resolved: list[tuple[_ReadyCanonical, StoredEvent]] = [] + for item in ready: + stored = ( + item.existing + if item.already_durable + or item.semantic_duplicate + or item.terminal_reason is not None + else next(stored_iterator) + ) + if stored is None: + raise RuntimeError("stable retained canonical record has no cache record") + resolved.append((item, stored)) + + projected = tuple( + (item, stored) + for item, stored in resolved + if ( + item.project_latest and not item.semantic_duplicate and item.terminal_reason is None ) - append_started_ns = time.time_ns() - fresh_stored = ( - await self.sink.publish_many([item.event for item in fresh]) - if fresh - else () + ) + projection_started_ns = time.time_ns() + projections = [ + self.projector.build( + stored, + item.raw_envelope, + derived_mark_index_component=item.derived_mark_index_component, ) - if fresh: - self._append_span.observe(time.time_ns() - append_started_ns) - for item in fresh: - # How old the canonical record already was when this - # projector read it: the core's produce, Kafka, and this - # consume. `accepted_at_ns` is the broker's own stamp. - self._canonical_age_span.observe( - append_started_ns - item.record.accepted_at_ns - ) - self._report_spans() - stored_iterator = iter(fresh_stored) - resolved: list[tuple[_ReadyCanonical, StoredEvent]] = [] - for item in ready: - stored = ( - item.existing - if item.already_durable - or item.semantic_duplicate - or item.terminal_reason is not None - else next(stored_iterator) - ) - if stored is None: - raise RuntimeError("stable retained canonical record has no cache record") - resolved.append((item, stored)) - - projected = tuple( - (item, stored) - for item, stored in resolved - if ( - item.project_latest - and not item.semantic_duplicate - and item.terminal_reason is None - ) + for item, stored in projected + ] + applied = ( + await asyncio.to_thread(self.target.apply_many, projections) + if projections + else () + ) + if projected: + self._projection_span.observe(time.time_ns() - projection_started_ns) + if len(applied) != len(projected): + raise RuntimeError( + "stable projection target returned an invalid result count" ) - projections = [ - self.projector.build( - stored, - item.raw_envelope, - derived_mark_index_component=item.derived_mark_index_component, - ) - for item, stored in projected - ] - applied = ( - await asyncio.to_thread(self.target.apply_many, projections) - if projections - else () + terminalized = tuple( + item for item in ready if item.terminal_reason is not None + ) + if terminalized: + await asyncio.to_thread( + self._quarantine_terminal_recovery_overlaps, terminalized ) - if len(applied) != len(projected): - raise RuntimeError( - "stable projection target returned an invalid result count" - ) - terminalized = tuple( - item for item in ready if item.terminal_reason is not None + logger.warning( + "terminalized stale BAR recovery overlaps count=%s reason=%s", + len(terminalized), + terminalized[0].terminal_reason, ) - if terminalized: - await asyncio.to_thread( - self._quarantine_terminal_recovery_overlaps, terminalized - ) - logger.warning( - "terminalized stale BAR recovery overlaps count=%s reason=%s", - len(terminalized), - terminalized[0].terminal_reason, + applied_by_event = { + item.record.event_id: was_applied + for (item, _stored), was_applied in zip( + projected, applied, strict=True + ) + } + checkpoint_started_ns = time.time_ns() + await asyncio.to_thread( + self._checkpoint_records, [item.record for item in ready] + ) + self._checkpoint_span.observe(time.time_ns() - checkpoint_started_ns) + for item in ready: + if ( + item.already_durable + or item.semantic_duplicate + or ( + item.record.event_id in applied_by_event + and not applied_by_event[item.record.event_id] ) - applied_by_event = { - item.record.event_id: was_applied - for (item, _stored), was_applied in zip( - projected, applied, strict=True + ): + self._duplicate_projections += 1 + self._canonical_committed += 1 + queue = self._queues[item.partition] + current = queue.popleft() + if current.offset != item.record.offset: + raise RuntimeError( + "stable canonical queue order changed during batch" ) - } - await asyncio.to_thread( - self._checkpoint_records, [item.record for item in ready] - ) - for item in ready: - if ( - item.already_durable - or item.semantic_duplicate - or ( - item.record.event_id in applied_by_event - and not applied_by_event[item.record.event_id] - ) - ): - self._duplicate_projections += 1 - self._canonical_committed += 1 - queue = self._queues[item.partition] - current = queue.popleft() - if current.offset != item.record.offset: - raise RuntimeError( - "stable canonical queue order changed during batch" - ) - self._pending_records -= 1 - self._pending_bytes -= len(item.record.payload) - capture_id = bytes(item.envelope.raw_capture_id) - waiting = self._waiting.get(capture_id) - if waiting is not None: - waiting.discard(item.partition) - if not waiting: - self._waiting.pop(capture_id, None) - if not queue: - self._queues.pop(item.partition, None) - self._update_canonical_backpressure() + self._pending_records -= 1 + self._pending_bytes -= len(item.record.payload) + capture_id = bytes(item.envelope.raw_capture_id) + waiting = self._waiting.get(capture_id) + if waiting is not None: + waiting.discard(item.partition) + if not waiting: + self._waiting.pop(capture_id, None) + if not queue: + self._queues.pop(item.partition, None) + self._update_canonical_backpressure() + self._report_spans() @staticmethod def _verified_payload_hash( @@ -721,6 +835,26 @@ def _latest_bar_close_ns(self, partition_key: str) -> int | None: closes.append(int(envelope.bar.close_time_ns)) return max(closes) if closes else None + def _durable_bar_high_watermark(self, partition_key: str) -> int | None: + """Return one exact BAR partition watermark without repeated tail scans. + + Old caches predate the additive table. Their first BAR lookup is the + only bounded scan, immediately persisted as an atomic max; all later + projector turns and restarts use the O(1) durable value. + """ + + current = self.spool.final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=partition_key, + ) + if current is not None: + return current + return self.spool.hydrate_final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=partition_key, + legacy_lookup=lambda: self._latest_bar_close_ns(partition_key), + ) + async def _ready_batch(self) -> tuple[_ReadyCanonical, ...]: candidates = [] for partition, record in self._round_robin_candidates( @@ -765,7 +899,8 @@ async def _ready_batch(self) -> tuple[_ReadyCanonical, ...]: and record.raw_provider_envelope is None ) raw_by_id = await asyncio.to_thread(self._find_raw_many, fallback_ids) - bar_high_watermarks: dict[str, int | None] = {} + final_bar_high_watermarks: dict[str, int | None] = {} + legacy_bar_high_watermarks: dict[str, int | None] = {} ready = [] blocked_partitions = set() for partition, record, envelope, capture_id in candidates: @@ -848,16 +983,32 @@ async def _ready_batch(self) -> tuple[_ReadyCanonical, ...]: project_latest = True if envelope.WhichOneof("payload") == "bar": - if record.key not in bar_high_watermarks: - bar_high_watermarks[record.key] = await asyncio.to_thread( - self._latest_bar_close_ns, record.key - ) - current = bar_high_watermarks[record.key] - close_ns = int(envelope.bar.close_time_ns) + close_ns = final_bar_close_time_ns(envelope) + if close_ns is not None: + if record.key not in final_bar_high_watermarks: + final_bar_high_watermarks[record.key] = await asyncio.to_thread( + self._durable_bar_high_watermark, record.key + ) + current = final_bar_high_watermarks[record.key] + else: + # A catalog may still retain a non-final historical BAR. + # Preserve its legacy selection semantics without letting + # it pollute the final-BAR durable watermark. + if record.key not in legacy_bar_high_watermarks: + legacy_bar_high_watermarks[record.key] = await asyncio.to_thread( + self._latest_bar_close_ns, record.key + ) + current = legacy_bar_high_watermarks[record.key] + close_ns = int(envelope.bar.close_time_ns) project_latest = current is None or close_ns >= current - bar_high_watermarks[record.key] = ( - close_ns if current is None else max(current, close_ns) - ) + if close_ns is not None: + final_bar_high_watermarks[record.key] = ( + close_ns if current is None else max(current, close_ns) + ) + else: + legacy_bar_high_watermarks[record.key] = ( + close_ns if current is None else max(current, close_ns) + ) ready.append(_ReadyCanonical( partition=partition, record=record, @@ -874,6 +1025,7 @@ async def _ready_batch(self) -> tuple[_ReadyCanonical, ...]: headers={ "raw_stream": raw_stream, "raw_event_id": raw_event_id.hex(), + **final_bar_watermark_headers(envelope), "raw_provider_envelope": base64.b64encode( raw_envelope ).decode("ascii"), @@ -1068,6 +1220,9 @@ async def supervise_stable_projector( broker = None try: broker, engine = broker_factory() + prepare = getattr(engine, "prepare_for_polling", None) + if callable(prepare): + await prepare() on_broker(broker) while not should_stop(): if await engine.run_once(timeout_seconds=1.0): diff --git a/qdl/runtime/stable_source.py b/qdl/runtime/stable_source.py index 62d9b90..c1c7c39 100644 --- a/qdl/runtime/stable_source.py +++ b/qdl/runtime/stable_source.py @@ -2,13 +2,17 @@ import hashlib import time -from dataclasses import replace +from dataclasses import dataclass, replace from qdl.adapters.intervals import ( canonical_interval_ms, latest_closed_boundary_ms, ) from qdl.common.v1 import common_pb2 +from qdl.data_quality.binding_decision import ( + BindingQualityInput, + evaluate_binding_quality, +) from qdl.domain.calendar import trading_calendar_for_id from qdl.domain.decimal import CanonicalDecimal from qdl.domain.quantity import quantity_unit_name @@ -45,7 +49,9 @@ ) from qdl.runtime.session_liveness import StableSessionLivenessReader from qdl.stream import GrpcSnapshot +from qdl.reference.execution_live import ExecutionMarkIndexReader from qdl.transport import Cursor, SQLiteDurableSpool, StoredEvent +from qdl.transport.sqlite_spool import FinalBarTailWindow def _decimal_text(value) -> str: @@ -160,6 +166,14 @@ def bar_item_fields( } +@dataclass(frozen=True) +class _ParsedStoredEvent: + """One immutable durable row with its envelope decoded exactly once.""" + + stored: StoredEvent + envelope: market_data_pb2.EventEnvelope + + class StableSpoolQueryBackend: """Provider-neutral stable query view over a Kafka-rebuildable SQLite cache.""" @@ -219,27 +233,324 @@ def latest(self, requirement: DataRequirement) -> MarketDataItem | None: def history(self, requirement: DataRequirement) -> HistoryResult | None: requested, start_ns, end_ns, expected_opens = self._requested_window(requirement) binding = self.catalog.binding_for(requirement) - # A provider history repair can be appended after newer live BARs. Read - # the bounded retained BAR window before selecting the market-time tail; - # selecting logical append offsets first can manufacture a false gap. - read_limit = ( - STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW - if start_ns is not None or binding.feed is FeedType.BAR - else requested + all_records = self._records( + requirement, + limit=self._history_read_limit(binding, requested, start_ns), + ) + return self._history_from_records( + requirement, + binding, + all_records, + requested=requested, + start_ns=start_ns, + end_ns=end_ns, + expected_opens=expected_opens, ) - all_records = self._records(requirement, limit=read_limit) + + def history_many( + self, + requirements: tuple[DataRequirement, ...], + ) -> dict[DataRequirement, HistoryResult | None | Exception]: + """Materialize one bounded local-cache batch from one SQLite snapshot. + + The normal single-read history builder remains the authority for + selection, lineage, coverage, quality, cursor and finality semantics. + This method only shares the physical tail read and protobuf decode for + one already-admitted query batch. It neither caches across consumers + nor reaches a provider. + """ + + if len(requirements) > 100: + raise ValueError("stable history batch exceeds the public request bound") + plans: list[tuple[ + DataRequirement, + StableSourceBinding, + int, + int | None, + int | None, + tuple[int, ...] | None, + int, + int, + ]] = [] + tail_requests: list[tuple[str, str, int]] = [] + results: dict[DataRequirement, HistoryResult | None | Exception] = {} + for requirement in requirements: + try: + requested, start_ns, end_ns, expected_opens = self._requested_window( + requirement + ) + binding = self.catalog.binding_for(requirement) + read_limit = self._history_read_limit(binding, requested, start_ns) + physical_limit = self._physical_read_limit(binding, read_limit) + except Exception as error: + results[requirement] = error + continue + plans.append(( + requirement, + binding, + requested, + start_ns, + end_ns, + expected_opens, + read_limit, + physical_limit, + )) + tail_requests.append(( + binding.canonical_stream, + binding.partition_key, + physical_limit, + )) + if not plans: + return results + + plans_by_physical_tail: dict[ + tuple[str, str], + list[tuple[ + DataRequirement, + StableSourceBinding, + int, + int | None, + int | None, + tuple[int, ...] | None, + int, + int, + ]], + ] = {} + for plan in plans: + binding = plan[1] + plans_by_physical_tail.setdefault( + (binding.canonical_stream, binding.partition_key), [] + ).append(plan) + + final_windows: dict[tuple[str, str], FinalBarTailWindow] = {} + watermark_reader = getattr(self.spool, "final_bar_watermark", None) + final_window_visitor = getattr(self.spool, "visit_final_bar_windows", None) + if callable(watermark_reader) and callable(final_window_visitor): + for key, physical_plans in plans_by_physical_tail.items(): + if len(physical_plans) != 1: + continue + ( + requirement, + binding, + requested, + start_ns, + end_ns, + expected_opens, + _read_limit, + _physical_limit, + ) = physical_plans[0] + window = self._final_bar_window( + requirement=requirement, + binding=binding, + requested=requested, + start_ns=start_ns, + end_ns=end_ns, + expected_opens=expected_opens, + ) + # Keep the existing visitor for caches without a durable final + # watermark. This preserves the full-tail semantics and test + # doubles used by historical/replay coverage. + if window is not None: + try: + watermark_present = watermark_reader( + stream=key[0], partition_key=key[1] + ) is not None + except Exception: + # A corrupt/missing optimization watermark cannot make + # historical reads fail. The retained tail remains the + # authority for that partition. + watermark_present = False + if watermark_present: + final_windows[key] = window + + def materialize_tail( + key: tuple[str, str], rows: tuple[StoredEvent, ...], + ) -> None: + physical_plans = plans_by_physical_tail[key] + try: + parsed_tail = self._parse_records(rows) + except Exception as error: + for requirement, *_rest in physical_plans: + results[requirement] = error + return + for ( + requirement, + binding, + requested, + start_ns, + end_ns, + expected_opens, + read_limit, + physical_limit, + ) in physical_plans: + try: + # The physical tail can be larger because another declared + # logical feed shares it. Slice before filtering so each + # route retains exactly the same bounded semantics as + # ``history``. The next physical tail is not retained + # while this one is decoded/materialized. + all_records = self._select_records( + binding, + parsed_tail[-physical_limit:], + limit=read_limit, + ) + results[requirement] = self._history_from_records( + requirement, + binding, + all_records, + requested=requested, + start_ns=start_ns, + end_ns=end_ns, + expected_opens=expected_opens, + ) + except Exception as error: + results[requirement] = error + + def materialize_final_bar_tail( + key: tuple[str, str], + rows: tuple[StoredEvent, ...], + expected_closes: tuple[int, ...], + ) -> bool: + """Accept an exact final window only when it is fully unambiguous.""" + + physical_plans = plans_by_physical_tail[key] + if not expected_closes or len(physical_plans) != 1: + materialize_tail(key, rows) + return True + ( + requirement, + binding, + requested, + start_ns, + end_ns, + expected_opens, + read_limit, + physical_limit, + ) = physical_plans[0] + try: + parsed_tail = self._parse_records(rows) + all_records = self._select_records( + binding, + parsed_tail[-physical_limit:], + limit=read_limit, + ) + if not self._exact_final_bar_window( + binding=binding, + records=all_records, + requested=requested, + expected_closes=expected_closes, + ): + return False + history = self._history_from_records( + requirement, + binding, + all_records, + requested=requested, + start_ns=start_ns, + end_ns=end_ns, + expected_opens=expected_opens, + ) + if history is None or history.coverage is not CoverageStatus.FULL: + return False + except Exception: + # This is an optimization boundary. The existing retained-tail + # materializer remains authoritative for any uncertainty. + return False + results[requirement] = history + return True + try: + if final_windows: + final_window_visitor( + requests=tail_requests, + windows=final_windows, + visit=materialize_final_bar_tail, + ) + else: + self.spool.visit_tails(requests=tail_requests, visit=materialize_tail) + except Exception as error: + for requirement, *_rest in plans: + results[requirement] = error + return results + return results + + @staticmethod + def _final_bar_window( + *, + requirement: DataRequirement, + binding: StableSourceBinding, + requested: int, + start_ns: int | None, + end_ns: int | None, + expected_opens: tuple[int, ...] | None, + ) -> FinalBarTailWindow | None: + """Return the only row-window shape safe for exact final lookup.""" + + if ( + binding.feed is not FeedType.BAR + or not binding.require_final_bar + or not binding.continuous_calendar + or not requirement.require_final_bars + or requested not in {1, 2} + or start_ns is not None + or end_ns is not None + or expected_opens is not None + ): + return None + # Both LATEST and EMIT_REVISIONS retain the normal history semantics. + # The exact reader accepts only one unambiguous row per expected close; + # revised/duplicate closes are rejected by _exact_final_bar_window and + # return to the retained-tail authority inside the same transaction. + return FinalBarTailWindow( + interval_ns=_interval_ns(binding.interval or ""), + rows=requested, + ) + + @staticmethod + def _exact_final_bar_window( + *, + binding: StableSourceBinding, + records: tuple[_ParsedStoredEvent, ...], + requested: int, + expected_closes: tuple[int, ...], + ) -> bool: + """Validate header-indexed BAR rows before bypassing the retained tail.""" + + if len(records) != requested or len(expected_closes) != requested: + return False + interval_ns = _interval_ns(binding.interval or "") + observed_closes = tuple(int(item.envelope.bar.close_time_ns) for item in records) + if observed_closes != expected_closes or len(set(observed_closes)) != requested: + return False + return all( + item.envelope.bar.is_final + and item.envelope.bar.lifecycle in { + market_data_pb2.BAR_LIFECYCLE_FINAL, + market_data_pb2.BAR_LIFECYCLE_REVISED, + } + and int(item.envelope.bar.close_time_ns) + == int(item.envelope.bar.open_time_ns) + interval_ns - 1_000_000 + for item in records + ) + + def _history_from_records( + self, + requirement: DataRequirement, + binding: StableSourceBinding, + all_records: tuple[_ParsedStoredEvent, ...], + *, + requested: int, + start_ns: int | None, + end_ns: int | None, + expected_opens: tuple[int, ...] | None, + ) -> HistoryResult | None: if not all_records: return None records = all_records if start_ns is not None: records = tuple( - stored - for stored in all_records - if start_ns - <= market_data_pb2.EventEnvelope.FromString( - stored.event.payload - ).bar.open_time_ns - < end_ns + parsed + for parsed in all_records + if start_ns <= parsed.envelope.bar.open_time_ns < end_ns ) records = records[-requested:] if not records: @@ -251,10 +562,10 @@ def history(self, requirement: DataRequirement) -> HistoryResult | None: # provider backfill may legitimately append older final bars after live # ones, so the handoff cursor must fence the greatest durable offset # while the returned BAR window stays ordered by open time. - last = max(records, key=lambda item: item.cursor.offset) + last = max(records, key=lambda item: item.stored.cursor.offset) snapshot_hash = hashlib.sha256( - f"{last.cursor.stream}|{last.cursor.partition_key}|{last.cursor.offset}|" - f"{last.event.event_id.hex()}".encode() + f"{last.stored.cursor.stream}|{last.stored.cursor.partition_key}|" + f"{last.stored.cursor.offset}|{last.stored.event.event_id.hex()}".encode() ).hexdigest() exact_boundary = True if start_ns is not None and items: @@ -289,7 +600,7 @@ def history(self, requirement: DataRequirement) -> HistoryResult | None: coverage=CoverageStatus.FULL if full else CoverageStatus.PARTIAL, snapshot_id=f"qdl-v2-{snapshot_hash[:32]}", stream_cursor="CONSUMER_CURSOR_PENDING", - watermark_offset=last.cursor.offset, + watermark_offset=last.stored.cursor.offset, data_as_of_ns=( int(items[-1].payload["close_time_ns"]) if binding.feed is FeedType.BAR @@ -304,7 +615,7 @@ def feed_status(self, requirement: DataRequirement) -> QualityMetadata | None: def open_gaps(self) -> tuple[GapRecord, ...]: gaps = [] for binding in self.catalog.bindings: - records = tuple(self.spool.read_tail( + records = self._parse_records(self.spool.read_tail( stream=binding.canonical_stream, partition_key=binding.partition_key, limit=( @@ -321,38 +632,28 @@ def stored_events(self, requirement: DataRequirement) -> tuple[StoredEvent, ...] binding = self.catalog.binding_for(requirement) rows = self._records( requirement, - limit=( - STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW - if start_ns is not None or binding.feed is FeedType.BAR - else requested - ), + limit=self._history_read_limit(binding, requested, start_ns), ) if start_ns is None: selected = rows[-requested:] else: selected = tuple( - stored - for stored in rows - if start_ns - <= market_data_pb2.EventEnvelope.FromString( - stored.event.payload - ).bar.open_time_ns - < end_ns + parsed + for parsed in rows + if start_ns <= parsed.envelope.bar.open_time_ns < end_ns ) self._validate_records(binding, selected) - return selected + return tuple(item.stored for item in selected) def _validate_records( self, binding: StableSourceBinding, - records: tuple[StoredEvent, ...], + records: tuple[_ParsedStoredEvent, ...], ) -> None: """Fail closed on lineage mismatch within the returned data window.""" - for stored in records: - resolved = self.catalog.binding_for_envelope( - market_data_pb2.EventEnvelope.FromString(stored.event.payload) - ) + for parsed in records: + resolved = self.catalog.binding_for_envelope(parsed.envelope) if resolved.binding_id != binding.binding_id: raise ValueError("canonical event resolves to a different stable binding") @@ -402,56 +703,88 @@ def _requested_window( raise ValueError("stable spool time range exceeds bounded query rows") return rows, start_ns, end_ns, expected_opens - def _records( - self, requirement: DataRequirement, *, limit: int - ) -> tuple[StoredEvent, ...]: - binding = self.catalog.binding_for(requirement) + @staticmethod + def _history_read_limit( + binding: StableSourceBinding, + requested: int, + start_ns: int | None, + ) -> int: + # A provider history repair can be appended after newer live BARs. Read + # the bounded retained BAR window before selecting the market-time tail; + # selecting logical append offsets first can manufacture a false gap. + return ( + STABLE_SPOOL_PHYSICAL_PARTITION_WINDOW + if start_ns is not None or binding.feed is FeedType.BAR + else requested + ) + + @staticmethod + def _physical_read_limit(binding: StableSourceBinding, limit: int) -> int: # BOOK_SNAPSHOT and BOOK_DELTA deliberately share one physical - # partition for replay ordering. A one-row ``latest`` read can - # therefore land on a delta and incorrectly report that the most - # recent verified snapshot does not exist. Scan a bounded physical - # tail before applying the public logical-feed filter. The runtime - # refreshes Binance anchors at most every 30 seconds and this cap is - # explicit; it is not an unbounded recovery scan. - physical_limit = limit + # partition for replay ordering. A one-row ``latest`` read can land on + # the other logical feed, so bounded physical headroom is required + # before the public logical-feed filter. if binding.feed in {FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA}: - physical_limit = min( + return min( STABLE_SPOOL_PUBLIC_PARTITION_WINDOW, max(limit, limit * 512), ) - rows = self.spool.read_tail( + return limit + + @staticmethod + def _parse_records( + rows: tuple[StoredEvent, ...] | list[StoredEvent], + ) -> tuple[_ParsedStoredEvent, ...]: + return tuple( + _ParsedStoredEvent( + stored=row, + envelope=market_data_pb2.EventEnvelope.FromString(row.event.payload), + ) + for row in rows + ) + + def _records( + self, requirement: DataRequirement, *, limit: int + ) -> tuple[_ParsedStoredEvent, ...]: + binding = self.catalog.binding_for(requirement) + rows = self._parse_records(self.spool.read_tail( stream=binding.canonical_stream, partition_key=binding.partition_key, - limit=physical_limit, - ) - selected = [] - for row in rows: - envelope = market_data_pb2.EventEnvelope.FromString(row.event.payload) + limit=self._physical_read_limit(binding, limit), + )) + return self._select_records(binding, rows, limit=limit) + + @staticmethod + def _select_records( + binding: StableSourceBinding, + rows: tuple[_ParsedStoredEvent, ...], + *, + limit: int, + ) -> tuple[_ParsedStoredEvent, ...]: + selected = [ + parsed + for parsed in rows # BOOK_SNAPSHOT and BOOK_DELTA intentionally share a durable - # partition. Keep the public logical feed exact at the query - # boundary so a snapshot read can never return a delta (or vice - # versa) merely because both belong to the same physical book. + # partition. Keep the public logical feed exact at the query + # boundary so a snapshot can never return a delta (or vice versa). if ( - envelope.WhichOneof("payload") == binding.feed.value.lower() - and canonical_payload_interval(envelope) == binding.interval - ): - selected.append(row) + parsed.envelope.WhichOneof("payload") == binding.feed.value.lower() + and canonical_payload_interval(parsed.envelope) == binding.interval + ) + ] if binding.feed is FeedType.BAR: - selected.sort(key=lambda item: ( - market_data_pb2.EventEnvelope.FromString( - item.event.payload - ).bar.open_time_ns, - item.cursor.offset, + selected.sort(key=lambda parsed: ( + parsed.envelope.bar.open_time_ns, + parsed.stored.cursor.offset, )) - # ``read_tail`` is chronological. Keep only the requested logical - # tail after filtering the shared physical book partition so callers - # retain the same bounded/latest semantics as every other feed. + # ``read_tail`` is chronological. Keep only the requested logical tail + # after the exact feed filter, preserving the public bounded contract. return tuple(selected[-limit:]) def _items( self, requirement: DataRequirement, - records: tuple[StoredEvent, ...], + records: tuple[_ParsedStoredEvent, ...], *, gap_open: bool | None = None, ) -> tuple[MarketDataItem, ...]: @@ -465,11 +798,11 @@ def _items( self._item( requirement, binding, - stored, - market_data_pb2.EventEnvelope.FromString(stored.event.payload), + parsed.stored, + parsed.envelope, effective_gap, ) - for stored in records + for parsed in records ) def _quality( @@ -479,6 +812,7 @@ def _quality( envelope: market_data_pb2.EventEnvelope, *, gap_open: bool, + watermark_offset: int, ) -> QualityMetadata: source_observed_ns = ( envelope.bar.close_time_ns @@ -507,11 +841,9 @@ def _quality( if requirement.max_freshness_ms is None else min(binding.stale_after_ms, requirement.max_freshness_ms) ) - event_stale = freshness_ms > event_limit_ms source_value_age_ms = max( 0, (self._clock_ns() - source_observed_ns) // 1_000_000 ) - event_recency_state = "STALE" if event_stale else "LIVE" session_state = "NOT_APPLICABLE" session_liveness_ms = None session_flags: tuple[str, ...] = () @@ -547,71 +879,90 @@ def _quality( ) if book_unverified: flags = flags + ("BOOK_SEQUENCE_UNVERIFIED",) - if market_closed: - state = "MARKET_CLOSED" - elif gap_open: - state = "GAPPED" - elif book_unverified: - state = "SYNCING" - elif session_state in {"STALE", "DISCONNECTED", "UNKNOWN"}: - state = "STALE" - elif ( - event_stale - and requirement.effective_event_recency_policy - in {StalePolicy.BLOCK, StalePolicy.PAUSE} - ): - state = "STALE" - else: - state = "LIVE" - complete = not gap_open and not book_unverified - execution_eligible = ( - binding.authoritative - and binding.source_role == "PRIMARY" - and state == "LIVE" - and complete - and event_recency_state != "STALE" - and session_state in {"LIVE", "NOT_APPLICABLE"} + now_ns = self._clock_ns() + decision = evaluate_binding_quality( + BindingQualityInput( + binding_id=binding.binding_id, + instrument_uid=binding.instrument.instrument_uid, + feed=binding.feed.value, + source_role=binding.source_role, + authoritative=binding.authoritative, + # This backend only materializes a record after it has passed + # the active acquisition lane. Expected dark/V1 inventory is + # classified by the offline auditor before it reads a record. + acquisition_enabled=True, + acquisition_mode="RUST_NATIVE", + market_open=not market_closed, + event_present=True, + event_age_ms=int(freshness_ms), + event_limit_ms=int(event_limit_ms), + event_recency_policy=requirement.effective_event_recency_policy.value, + session_state=session_state, + session_liveness_ms=session_liveness_ms, + session_limit_ms=requirement.max_session_liveness_ms, + delivery_semantics=binding.delivery_semantics, + generation_matches="SOURCE_SESSION_AMBIGUOUS" not in session_flags, + config_matches="SOURCE_SESSION_CONFIG_MISMATCH" not in session_flags, + gap_open=gap_open, + book_verified=not book_unverified, + final_bar=( + bool(envelope.bar.is_final) + if envelope.WhichOneof("payload") == "bar" + else True + ), + require_final_bar=binding.require_final_bar, + watermark_offset=watermark_offset, + allow_quiet_execution=( + binding.delivery_semantics == "ON_CHANGE" + and requirement.effective_event_recency_policy is StalePolicy.OBSERVE + ), + flags=( + flags + + session_flags + + ( + ("DELIVERY_ON_CHANGE",) + if binding.delivery_semantics == "ON_CHANGE" + else () + ) + + ( + ("FRESHNESS_BASIS_PROVIDER_CONFIRMATION",) + if binding.freshness_basis == "PROVIDER_CONFIRMATION" + else () + ) + + ( + ("SOURCE_VALUE_TIMESTAMP_OLD",) + if ( + binding.freshness_basis == "PROVIDER_CONFIRMATION" + and source_value_age_ms > event_limit_ms + ) + else () + ) + + (("MARKET_CLOSED",) if market_closed else ()) + ), + ) ) return QualityMetadata( - state=state, + state=decision.state, freshness_ms=int(freshness_ms), gap_open=gap_open, - complete=complete, - execution_eligible=execution_eligible, + complete=decision.complete, + execution_eligible=decision.execution_eligible, policy_id=binding.source_policy_id, - flags=( - flags - + session_flags - + ( - ("FRESHNESS_BASIS_PROVIDER_CONFIRMATION",) - if binding.freshness_basis == "PROVIDER_CONFIRMATION" - else () - ) - + ( - ("SOURCE_VALUE_TIMESTAMP_OLD",) - if ( - binding.freshness_basis == "PROVIDER_CONFIRMATION" - and source_value_age_ms > event_limit_ms - ) - else () - ) - + (("LAST_EVENT_STALE",) if event_stale else ()) - + (("MARKET_CLOSED",) if market_closed else ()) - ), - event_recency_state=event_recency_state, - provider_session_state=session_state, - provider_session_liveness_ms=session_liveness_ms, + flags=decision.reason_codes, + event_recency_state=decision.event_recency_state, + provider_session_state=decision.provider_session_state, + provider_session_liveness_ms=decision.provider_session_liveness_ms, ) def _gaps( self, binding: StableSourceBinding, - records: tuple[StoredEvent, ...], + records: tuple[_ParsedStoredEvent, ...], ) -> tuple[GapRecord, ...]: detected_at_ns = self._clock_ns() result = [] - for stored in records: - envelope = market_data_pb2.EventEnvelope.FromString(stored.event.payload) + for parsed in records: + envelope = parsed.envelope if common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE in envelope.quality_flags: result.append(self._gap( binding, @@ -621,10 +972,7 @@ def _gaps( )) if binding.feed is not FeedType.BAR: return tuple(result) - opens = sorted({ - market_data_pb2.EventEnvelope.FromString(item.event.payload).bar.open_time_ns - for item in records - }) + opens = sorted({item.envelope.bar.open_time_ns for item in records}) if not opens: return tuple(result) step = _interval_ns(binding.interval or "") @@ -678,7 +1026,13 @@ def _item( gap_open: bool, ) -> MarketDataItem: payload_name = envelope.WhichOneof("payload") - quality = self._quality(requirement, binding, envelope, gap_open=gap_open) + quality = self._quality( + requirement, + binding, + envelope, + gap_open=gap_open, + watermark_offset=stored.cursor.offset, + ) source_role = common_pb2.SourceRole.Name(envelope.source_role).removeprefix( "SOURCE_ROLE_" ) @@ -1084,6 +1438,7 @@ def build_stable_query_stack( provider_admission_url: str | None = None, provider_admission_secret: bytes | None = None, session_liveness_root: str | None = None, + execution_mark_index_reader: ExecutionMarkIndexReader | None = None, ) -> tuple[V2QueryService, StableSpoolQueryBackend, StableConsumerCursorIssuer]: """Build the query stack, optionally including the pass-through product. @@ -1138,6 +1493,7 @@ def build_stable_query_stack( entitlements=entitlements, reference_batch=reference_batch, reference_source_id=reference_source_id, + execution_mark_index_reader=execution_mark_index_reader, ) issuer = StableConsumerCursorIssuer( handoff, catalog, ttl_seconds=cursor_ttl_seconds diff --git a/qdl/transport/__init__.py b/qdl/transport/__init__.py index 5d77b86..85cd642 100644 --- a/qdl/transport/__init__.py +++ b/qdl/transport/__init__.py @@ -11,6 +11,7 @@ EventIdCollision, EventSink, EventSource, + FINAL_BAR_CLOSE_TIME_NS_HEADER, RetryClass, RetryDecision, PayloadCorruption, @@ -37,6 +38,7 @@ "EventIdCollision", "EventSink", "EventSource", + "FINAL_BAR_CLOSE_TIME_NS_HEADER", "PublisherState", "PayloadCorruption", "RetryClass", diff --git a/qdl/transport/contracts.py b/qdl/transport/contracts.py index ad11fd4..4040115 100644 --- a/qdl/transport/contracts.py +++ b/qdl/transport/contracts.py @@ -7,6 +7,11 @@ from typing import Mapping, Protocol, runtime_checkable +# Internal-only durable metadata. The stable stream ingress derives this only +# after it has validated a canonical final/revised BAR against the catalog. +FINAL_BAR_CLOSE_TIME_NS_HEADER = "qdl.final_bar_close_time_ns" + + class StreamName(str, Enum): RAW = "md.raw.v1" CANONICAL = "md.canonical.v2" diff --git a/qdl/transport/sqlite_spool.py b/qdl/transport/sqlite_spool.py index 6b312b9..9b635c3 100644 --- a/qdl/transport/sqlite_spool.py +++ b/qdl/transport/sqlite_spool.py @@ -9,6 +9,7 @@ import uuid from dataclasses import dataclass from pathlib import Path +from typing import Callable from qdl.transport.contracts import ( AppendResult, @@ -18,6 +19,7 @@ CursorExpired, DurableEvent, EventIdCollision, + FINAL_BAR_CLOSE_TIME_NS_HEADER, PayloadCorruption, StoredEvent, ) @@ -106,6 +108,25 @@ class SpoolReadiness: payload_bytes: int +@dataclass(frozen=True) +class FinalBarTailWindow: + """One exact, bounded final-BAR lookup inside a spool read snapshot. + + This is deliberately transport-private. Callers must validate the + returned canonical rows and request the normal retained tail when the + exact window is incomplete or ambiguous. + """ + + interval_ns: int + rows: int + + def __post_init__(self) -> None: + if self.interval_ns <= 0: + raise ValueError("final BAR interval must be positive") + if self.rows not in {1, 2}: + raise ValueError("final BAR exact lookup supports one or two rows") + + class SQLiteDurableSpool: """Bounded, fsync-backed migration bridge with portable logical cursors. @@ -210,6 +231,14 @@ def _migrate(self) -> None: cache_id TEXT NOT NULL, created_at_ns INTEGER NOT NULL ); + + CREATE TABLE IF NOT EXISTS final_bar_watermarks ( + stream TEXT NOT NULL, + partition_key TEXT NOT NULL, + close_time_ns INTEGER NOT NULL, + updated_at_ns INTEGER NOT NULL, + PRIMARY KEY (stream, partition_key) + ); """ ) self._ensure_usage_state() @@ -270,6 +299,153 @@ def cache_id(self) -> str: def append(self, event: DurableEvent) -> AppendResult: return self.append_many([event])[0] + @staticmethod + def _final_bar_close_time_ns(event: DurableEvent) -> int | None: + value = event.headers.get(FINAL_BAR_CLOSE_TIME_NS_HEADER) + if value is None: + return None + if ( + not isinstance(value, str) + or not value.isascii() + or not value.isdecimal() + ): + raise ValueError("final BAR watermark header is invalid") + close_time_ns = int(value) + if not 0 < close_time_ns < 2**63: + raise ValueError("final BAR watermark header is invalid") + return close_time_ns + + def _upsert_final_bar_watermark_locked( + self, + *, + stream: str, + partition_key: str, + close_time_ns: int, + updated_at_ns: int, + ) -> int: + self._connection.execute( + """ + INSERT INTO final_bar_watermarks( + stream, partition_key, close_time_ns, updated_at_ns + ) VALUES (?, ?, ?, ?) + ON CONFLICT(stream, partition_key) DO UPDATE SET + close_time_ns = MAX( + final_bar_watermarks.close_time_ns, + excluded.close_time_ns + ), + updated_at_ns = CASE + WHEN excluded.close_time_ns >= final_bar_watermarks.close_time_ns + THEN excluded.updated_at_ns + ELSE final_bar_watermarks.updated_at_ns + END + """, + (stream, partition_key, close_time_ns, updated_at_ns), + ) + row = self._connection.execute( + """ + SELECT close_time_ns FROM final_bar_watermarks + WHERE stream = ? AND partition_key = ? + """, + (stream, partition_key), + ).fetchone() + if row is None or int(row["close_time_ns"]) <= 0: + raise PayloadCorruption("final BAR watermark is unavailable") + return int(row["close_time_ns"]) + + def final_bar_watermark( + self, *, stream: str, partition_key: str + ) -> int | None: + if not stream.strip() or not partition_key.strip(): + raise ValueError("final BAR watermark identity is incomplete") + with self._lock: + row = self._connection.execute( + """ + SELECT close_time_ns FROM final_bar_watermarks + WHERE stream = ? AND partition_key = ? + """, + (stream, partition_key), + ).fetchone() + if row is None: + return None + value = int(row["close_time_ns"]) + if value <= 0: + raise PayloadCorruption("final BAR watermark is invalid") + return value + + def seed_final_bar_watermark( + self, *, stream: str, partition_key: str, close_time_ns: int + ) -> int: + if not stream.strip() or not partition_key.strip() or not 0 < close_time_ns < 2**63: + raise ValueError("final BAR watermark is invalid") + with self._lock: + self._connection.execute("BEGIN IMMEDIATE") + try: + value = self._upsert_final_bar_watermark_locked( + stream=stream, + partition_key=partition_key, + close_time_ns=close_time_ns, + updated_at_ns=self._clock_ns(), + ) + self._connection.execute("COMMIT") + return value + except BaseException: + if self._connection.in_transaction: + self._connection.execute("ROLLBACK") + raise + + def hydrate_final_bar_watermark( + self, + *, + stream: str, + partition_key: str, + legacy_lookup: Callable[[], int | None], + ) -> int | None: + """Seed one legacy BAR partition exactly once across spool processes. + + The first process holding SQLite's write lock checks whether an active + stream owner has already supplied the watermark. Only if it is still + absent does it perform the bounded retained-tail lookup. This keeps a + restart/rebalance from multiplying legacy scans at an aligned BAR + boundary, while preserving the durable max fence. + """ + + if not stream.strip() or not partition_key.strip(): + raise ValueError("final BAR watermark identity is incomplete") + with self._lock: + self._connection.execute("BEGIN IMMEDIATE") + try: + row = self._connection.execute( + """ + SELECT close_time_ns FROM final_bar_watermarks + WHERE stream = ? AND partition_key = ? + """, + (stream, partition_key), + ).fetchone() + if row is not None: + value = int(row["close_time_ns"]) + if value <= 0: + raise PayloadCorruption("final BAR watermark is invalid") + self._connection.execute("COMMIT") + return value + close_time_ns = legacy_lookup() + if close_time_ns is None: + self._connection.execute("COMMIT") + return None + if not 0 < close_time_ns < 2**63: + raise PayloadCorruption("legacy final BAR watermark is invalid") + value = self._upsert_final_bar_watermark_locked( + stream=stream, + partition_key=partition_key, + close_time_ns=close_time_ns, + updated_at_ns=self._clock_ns(), + ) + self._connection.execute("COMMIT") + return value + except BaseException: + if self._connection.in_transaction: + self._connection.execute("ROLLBACK") + raise + def append_many(self, events: list[DurableEvent]) -> list[AppendResult]: if not events: return [] @@ -292,6 +468,7 @@ def append_many(self, events: list[DurableEvent]) -> list[AppendResult]: added_payload_bytes = 0 results = [] for event in events: + final_bar_close_time_ns = self._final_bar_close_time_ns(event) digest = hashlib.sha256(event.payload).hexdigest() headers_json = json.dumps( dict(event.headers), sort_keys=True, separators=(",", ":") @@ -311,6 +488,13 @@ def append_many(self, events: list[DurableEvent]) -> list[AppendResult]: raise EventIdCollision( "event ID maps to different immutable content" ) + if final_bar_close_time_ns is not None: + self._upsert_final_bar_watermark_locked( + stream=event.stream, + partition_key=event.partition_key, + close_time_ns=final_bar_close_time_ns, + updated_at_ns=self._clock_ns(), + ) results.append( AppendResult( cursor=Cursor( @@ -382,6 +566,13 @@ def append_many(self, events: list[DurableEvent]) -> list[AppendResult]: headers_json, ), ) + if final_bar_close_time_ns is not None: + self._upsert_final_bar_watermark_locked( + stream=event.stream, + partition_key=event.partition_key, + close_time_ns=final_bar_close_time_ns, + updated_at_ns=committed_at_ns, + ) added_records += 1 added_payload_bytes += len(event.payload) results.append( @@ -407,18 +598,15 @@ def append_many(self, events: list[DurableEvent]) -> list[AppendResult]: }) self._connection.execute("COMMIT") if maintenance_ran: - # PASSIVE never blocks readers or discards a committed event. - # It gives SQLite a bounded opportunity to recycle the WAL - # after retention work before the physical cache bound becomes - # a false backpressure signal. PASSIVE recycles the WAL but - # never shrinks the file, so a WAL that has already outgrown - # its declared journal_size_limit is reclaimed instead: that - # file, not the retained rows, is what reached the physical - # bound and froze every writer on 2026-09-17. - if self._wal_bytes() > JOURNAL_SIZE_LIMIT_BYTES: - self._checkpoint_wal_truncate_locked() - else: - self._checkpoint_wal_passive_locked() + # Routine retention must never turn a hot append into a + # blocking TRUNCATE checkpoint. Under a shared read-heavy + # cache, TRUNCATE can wait for another reader's snapshot + # until SQLite's busy timeout and make otherwise fresh + # market events stale. PASSIVE is nonblocking; the + # physical-capacity path below retains the bounded + # TRUNCATE reclaim/fail-closed policy when disk headroom + # actually requires it. + self._checkpoint_wal_passive_locked() return results except BaseException: if self._connection.in_transaction: @@ -487,6 +675,204 @@ def read_tail( ).fetchall() return [self._stored_event(row) for row in reversed(rows)] + def read_tails( + self, + *, + requests: tuple[tuple[str, str, int], ...] | list[tuple[str, str, int]], + ) -> dict[tuple[str, str], tuple[StoredEvent, ...]]: + """Read bounded tails for up to one public query batch in one snapshot. + + This is an internal local-cache primitive, not a replay API. A single + SQL statement gives every selected partition one SQLite-consistent + view, avoids one lock acquisition per item, and preserves the logical + ordering returned by :meth:`read_tail`. Multiple callers may request + the same physical partition with different limits; the largest bounded + tail is read once and callers select their own logical windows above + this transport boundary. + """ + + grouped: dict[tuple[str, str], list[StoredEvent]] = {} + + def collect(key: tuple[str, str], rows: tuple[StoredEvent, ...]) -> None: + grouped[key] = list(rows) + + self.visit_tails(requests=requests, visit=collect) + return {key: tuple(value) for key, value in grouped.items()} + + def visit_tails( + self, + *, + requests: tuple[tuple[str, str, int], ...] | list[tuple[str, str, int]], + visit: Callable[[tuple[str, str], tuple[StoredEvent, ...]], None], + ) -> None: + """Visit each deduplicated indexed tail under one read snapshot. + + The callback runs before the next physical partition is read. This is + intentionally separate from :meth:`read_tails`: large callers can + materialize a result and release a physical tail before reading the + next one, while preserving the same SQLite snapshot for every route in + the public batch. + """ + + normalized = self._normalized_tail_requests(requests) + if not normalized: + return + with self._lock: + # A deferred read transaction keeps one consistent generation + # without blocking WAL writers. Do not use a window-function CTE + # here: on a large spool SQLite may sort the whole events table + # into temp storage before applying each tail cap. + self._connection.execute("BEGIN") + try: + for (stream, partition_key), limit in normalized.items(): + rows = self._connection.execute( + """ + SELECT * FROM events + WHERE stream = ? AND partition_key = ? + ORDER BY logical_offset DESC LIMIT ? + """, + (stream, partition_key, limit), + ).fetchall() + visit( + (stream, partition_key), + tuple(self._stored_event(row) for row in reversed(rows)), + ) + except Exception: + self._connection.execute("ROLLBACK") + raise + else: + self._connection.execute("COMMIT") + + def visit_final_bar_windows( + self, + *, + requests: tuple[tuple[str, str, int], ...] | list[tuple[str, str, int]], + windows: dict[tuple[str, str], FinalBarTailWindow], + visit: Callable[[tuple[str, str], tuple[StoredEvent, ...], tuple[int, ...]], bool], + ) -> None: + """Visit exact final-BAR windows with retained-tail fallback in one snapshot. + + A final watermark identifies the current one/two closed BARs without + decoding the full retained partition. The visitor is the semantic + authority: it returns ``True`` only after it accepts the exact rows. + Missing, duplicate, revised or otherwise ambiguous rows therefore use + the existing physical-tail path under the same SQLite read transaction. + """ + + normalized = self._normalized_tail_requests(requests) + if not normalized: + return + unknown = set(windows) - set(normalized) + if unknown: + raise ValueError("final BAR window is not part of the tail request") + with self._lock: + self._connection.execute("BEGIN") + try: + for key, limit in normalized.items(): + window = windows.get(key) + if window is not None: + exact = self._final_bar_window_rows_locked( + stream=key[0], + partition_key=key[1], + window=window, + ) + if exact is not None: + rows, expected_closes = exact + if visit(key, rows, expected_closes): + continue + rows = self._read_tail_rows_locked( + stream=key[0], partition_key=key[1], limit=limit + ) + visit(key, rows, ()) + except Exception: + self._connection.execute("ROLLBACK") + raise + else: + self._connection.execute("COMMIT") + + def _final_bar_window_rows_locked( + self, + *, + stream: str, + partition_key: str, + window: FinalBarTailWindow, + ) -> tuple[tuple[StoredEvent, ...], tuple[int, ...]] | None: + watermark = self._connection.execute( + """ + SELECT close_time_ns FROM final_bar_watermarks + WHERE stream = ? AND partition_key = ? + """, + (stream, partition_key), + ).fetchone() + if watermark is None: + return None + latest_close_ns = int(watermark["close_time_ns"]) + if latest_close_ns <= 0: + raise PayloadCorruption("final BAR watermark is invalid") + expected_closes = tuple( + latest_close_ns - window.interval_ns * offset + for offset in range(window.rows - 1, -1, -1) + ) + if any(value <= 0 for value in expected_closes): + raise PayloadCorruption("final BAR watermark window is invalid") + placeholders = ",".join("?" for _ in expected_closes) + try: + rows = self._connection.execute( + f""" + SELECT * FROM events + WHERE stream = ? AND partition_key = ? + AND CAST(json_extract( + headers_json, '$."qdl.final_bar_close_time_ns"' + ) AS INTEGER) IN ({placeholders}) + ORDER BY logical_offset ASC + """, + (stream, partition_key, *expected_closes), + ).fetchall() + except sqlite3.OperationalError as error: + # JSON extraction is an optimization only. Older SQLite builds or + # malformed legacy metadata must retain the authoritative tail path. + if "json" in str(error).lower(): + return None + raise + return ( + tuple(self._stored_event(row) for row in rows), + expected_closes, + ) + + def _read_tail_rows_locked( + self, + *, + stream: str, + partition_key: str, + limit: int, + ) -> tuple[StoredEvent, ...]: + rows = self._connection.execute( + """ + SELECT * FROM events + WHERE stream = ? AND partition_key = ? + ORDER BY logical_offset DESC LIMIT ? + """, + (stream, partition_key, limit), + ).fetchall() + return tuple(self._stored_event(row) for row in reversed(rows)) + + def _normalized_tail_requests( + self, + requests: tuple[tuple[str, str, int], ...] | list[tuple[str, str, int]], + ) -> dict[tuple[str, str], int]: + max_tail_rows = max(10_000, self.config.max_partition_records) + normalized: dict[tuple[str, str], int] = {} + for stream, partition_key, limit in requests: + if not stream.strip() or not partition_key.strip(): + raise ValueError("tail stream and partition_key are required") + if limit <= 0 or limit > max_tail_rows: + raise ValueError(f"limit must be between 1 and {max_tail_rows}") + key = (stream, partition_key) + normalized[key] = max(normalized.get(key, 0), int(limit)) + if len(normalized) > 100: + raise ValueError("batch tail read exceeds the public request bound") + return normalized + def find_event(self, *, stream: str, event_id: bytes) -> StoredEvent | None: with self._lock: row = self._connection.execute( diff --git a/qdl/warmup/executor.py b/qdl/warmup/executor.py index e28fb7d..2f374b4 100644 --- a/qdl/warmup/executor.py +++ b/qdl/warmup/executor.py @@ -32,6 +32,17 @@ class ProviderBudgetPolicy: max_attempts: int = 4 circuit_failures: int = 5 circuit_cooldown_ms: int = 30_000 + # Local durable-cache reads may wait for bounded CPU/SQLite admission before + # they begin useful work. Keep that queue finite, then measure the existing + # read deadline from admission rather than falsely calling queued work a + # provider outage. External providers intentionally retain end-to-end + # deadlines because their queue/pacing time is part of the venue boundary. + max_pending: int | None = None + deadline_starts_after_admission: bool = False + # A request-local gate stops one legal large local-cache batch from + # monopolising every global worker before a collocated batch gets a turn. + # It is intentionally absent from external provider lanes. + max_batch_concurrency: int | None = None def __post_init__(self) -> None: if not 1 <= self.max_concurrency <= 64: @@ -42,6 +53,13 @@ def __post_init__(self) -> None: raise ValueError("provider attempts must be between 1 and 10") if self.circuit_failures < 1 or self.circuit_cooldown_ms < 1: raise ValueError("provider circuit policy values must be positive") + if self.max_pending is not None and not self.max_concurrency <= self.max_pending <= 8_192: + raise ValueError("provider pending admission must be between concurrency and 8192") + if ( + self.max_batch_concurrency is not None + and not 1 <= self.max_batch_concurrency <= self.max_concurrency + ): + raise ValueError("provider batch concurrency must be between 1 and concurrency") @dataclass(frozen=True, slots=True) @@ -78,6 +96,28 @@ class BoundedWarmupExecutor(Generic[T, R]): max_concurrency=8, requests_per_second=None, max_attempts=1, + # Two manifest-legal maximum batches contain one hundred local + # reads. 128 bounds resident queue work while leaving headroom for + # one collocated consumer; excess work returns typed backpressure. + max_pending=128, + deadline_starts_after_admission=True, + # Reserve four of the eight global permits for a second legal + # local batch. This is a request-local fairness gate, not a venue + # quota or a new data-plane queue. + max_batch_concurrency=4, + ), + # A stable-stream read is already canonical, authenticated and local to + # the V2 data plane. It still needs finite admission, but external + # venue token pacing and multi-attempt provider retry would only age an + # otherwise valid execution snapshot before the consumer can use it. + "INTERNAL_STREAM": ProviderBudgetPolicy( + max_concurrency=4, + requests_per_second=None, + max_attempts=1, + # A brief circuit still prevents a failed local reader from being + # hammered, but a venue-sized 30-second cooldown would mask a + # recovered lease holder and needlessly make execution stale. + circuit_cooldown_ms=1_000, ), "BINANCE": ProviderBudgetPolicy( max_concurrency=8, @@ -139,12 +179,19 @@ def __init__( self._inflight_lock = asyncio.Lock() self._rate_locks: dict[str, asyncio.Lock] = {} self._tokens: dict[str, _TokenState] = {} - self._circuit: dict[str, tuple[int, float]] = {} + self._pending: dict[str, int] = {} + self._pending_lock = asyncio.Lock() + # Quota/concurrency are intentionally provider-wide, but one failing + # route must not cool down a different instrument/feed or a newly + # materialized generation. ``key`` passed below already carries the + # normalized provider and caller-declared route identity. + self._circuit: dict[Hashable, tuple[int, float]] = {} self._circuit_lock = asyncio.Lock() self.source_calls = 0 self.singleflight_hits = 0 self.retry_count = 0 self.circuit_rejections = 0 + self.admission_rejections = 0 async def execute( self, @@ -157,14 +204,23 @@ async def execute( ) -> tuple[WarmupExecution[T, R], ...]: values = tuple(items) tasks = [] + batch_gates: dict[str, asyncio.Semaphore] = {} for item in values: provider_key = provider(item).upper() + policy = self.provider_policies.get(provider_key, self.default_policy) + batch_gate = None + if policy.max_batch_concurrency is not None: + batch_gate = batch_gates.setdefault( + provider_key, + asyncio.Semaphore(policy.max_batch_concurrency), + ) tasks.append(asyncio.create_task(self._one( item, work=work, key=(provider_key, identity(item)), provider_key=provider_key, deadline_ms=deadline_ms(item), + batch_gate=batch_gate, ))) tasks = tuple(tasks) try: @@ -183,8 +239,10 @@ async def _one( key: Hashable, provider_key: str, deadline_ms: int, + batch_gate: asyncio.Semaphore | None, ) -> WarmupExecution[T, R]: started = self._clock() + policy = self.provider_policies.get(provider_key, self.default_policy) shared = False attempts = 0 try: @@ -196,7 +254,10 @@ async def _one( item, work, provider_key, + circuit_key=key, + deadline_ms=deadline_ms, deadline_at=started + deadline_ms / 1000, + batch_gate=batch_gate, ) ) inflight = _Inflight(task=task, waiters=1) @@ -207,9 +268,12 @@ async def _one( self.singleflight_hits += 1 task = inflight.task try: - value, attempts = await asyncio.wait_for( - asyncio.shield(task), timeout=deadline_ms / 1000 - ) + if policy.deadline_starts_after_admission: + value, attempts = await asyncio.shield(task) + else: + value, attempts = await asyncio.wait_for( + asyncio.shield(task), timeout=deadline_ms / 1000 + ) return WarmupExecution( item, value, None, attempts, shared, (self._clock() - started) * 1000, @@ -268,47 +332,103 @@ async def _run_with_policy( work: Callable[[T], Awaitable[R]], provider: str, *, + circuit_key: Hashable, + deadline_ms: int, deadline_at: float | None = None, + batch_gate: asyncio.Semaphore | None = None, ) -> tuple[R, int]: policy = self.provider_policies.get(provider, self.default_policy) semaphore = self._semaphores.setdefault( provider, asyncio.Semaphore(policy.max_concurrency) ) - last_error: BaseException | None = None - for attempt in range(1, policy.max_attempts + 1): - await self._require_closed_circuit(provider) - try: - async with semaphore: - await self._acquire_provider_token(provider, policy) - self.source_calls += 1 - value = await work(item) - await self._record_success(provider) - return value, attempt - except RetryableWarmupError as error: - error.warmup_attempts = attempt - last_error = error - open_until = await self._record_failure(provider, policy) - if open_until: - break - if attempt == policy.max_attempts: - break - provider_delay = (error.retry_after_ms or 0) / 1000 - exponential = min(4.0, 0.1 * (2 ** (attempt - 1))) - delay = max(provider_delay, exponential) + self._random() * 0.05 - if deadline_at is not None and self._clock() + delay >= deadline_at: - raise RetryableWarmupError( - "provider retry delay exceeds the remaining bounded deadline", - retry_after_ms=error.retry_after_ms, - cause=error, - ) from error - self.retry_count += 1 - await self._sleep(delay) - assert last_error is not None - raise last_error + reserved = await self._reserve_pending(provider, policy) + batch_admitted = False + execution_deadline_at: float | None = None + try: + if batch_gate is not None: + await batch_gate.acquire() + batch_admitted = True + last_error: BaseException | None = None + for attempt in range(1, policy.max_attempts + 1): + await self._require_closed_circuit(provider, circuit_key) + try: + async with semaphore: + if ( + policy.deadline_starts_after_admission + and execution_deadline_at is None + ): + execution_deadline_at = self._clock() + deadline_ms / 1000 + await self._acquire_provider_token(provider, policy) + self.source_calls += 1 + if execution_deadline_at is None: + value = await work(item) + else: + remaining = execution_deadline_at - self._clock() + if remaining <= 0: + raise RetryableWarmupError( + f"warmup execution deadline exceeded after {deadline_ms}ms" + ) + try: + value = await asyncio.wait_for(work(item), timeout=remaining) + except asyncio.TimeoutError as error: + raise RetryableWarmupError( + f"warmup execution deadline exceeded after {deadline_ms}ms", + cause=error, + ) from error + await self._record_success(circuit_key) + return value, attempt + except RetryableWarmupError as error: + error.warmup_attempts = attempt + last_error = error + open_until = await self._record_failure(circuit_key, policy) + if open_until: + break + if attempt == policy.max_attempts: + break + provider_delay = (error.retry_after_ms or 0) / 1000 + exponential = min(4.0, 0.1 * (2 ** (attempt - 1))) + delay = max(provider_delay, exponential) + self._random() * 0.05 + active_deadline_at = execution_deadline_at or deadline_at + if active_deadline_at is not None and self._clock() + delay >= active_deadline_at: + raise RetryableWarmupError( + "provider retry delay exceeds the remaining bounded deadline", + retry_after_ms=error.retry_after_ms, + cause=error, + ) from error + self.retry_count += 1 + await self._sleep(delay) + assert last_error is not None + raise last_error + finally: + if batch_admitted: + batch_gate.release() + if reserved: + await self._release_pending(provider) - async def _require_closed_circuit(self, provider: str) -> None: + async def _reserve_pending(self, provider: str, policy: ProviderBudgetPolicy) -> bool: + if policy.max_pending is None: + return False + async with self._pending_lock: + pending = self._pending.get(provider, 0) + if pending >= policy.max_pending: + self.admission_rejections += 1 + raise RetryableWarmupError( + f"bounded admission capacity is exhausted for {provider}" + ) + self._pending[provider] = pending + 1 + return True + + async def _release_pending(self, provider: str) -> None: + async with self._pending_lock: + pending = self._pending.get(provider, 0) + if pending <= 1: + self._pending.pop(provider, None) + else: + self._pending[provider] = pending - 1 + + async def _require_closed_circuit(self, provider: str, circuit_key: Hashable) -> None: async with self._circuit_lock: - _, open_until = self._circuit.get(provider, (0, 0.0)) + _, open_until = self._circuit.get(circuit_key, (0, 0.0)) now = self._clock() if open_until <= now: return @@ -319,24 +439,24 @@ async def _require_closed_circuit(self, provider: str) -> None: retry_after_ms=retry_after_ms, ) - async def _record_success(self, provider: str) -> None: + async def _record_success(self, circuit_key: Hashable) -> None: async with self._circuit_lock: - self._circuit[provider] = (0, 0.0) + self._circuit[circuit_key] = (0, 0.0) async def _record_failure( self, - provider: str, + circuit_key: Hashable, policy: ProviderBudgetPolicy, ) -> float: async with self._circuit_lock: - failures, open_until = self._circuit.get(provider, (0, 0.0)) + failures, open_until = self._circuit.get(circuit_key, (0, 0.0)) if open_until > self._clock(): return open_until failures += 1 open_until = 0.0 if failures >= policy.circuit_failures: open_until = self._clock() + policy.circuit_cooldown_ms / 1000 - self._circuit[provider] = (failures, open_until) + self._circuit[circuit_key] = (failures, open_until) return open_until async def _acquire_provider_token( @@ -376,4 +496,5 @@ def stats(self) -> dict[str, int]: "singleflight_hits": self.singleflight_hits, "retry_count": self.retry_count, "circuit_rejections": self.circuit_rejections, + "admission_rejections": self.admission_rejections, } diff --git a/qdl_sdk/reference.py b/qdl_sdk/reference.py index 855c8d4..4f61e06 100644 --- a/qdl_sdk/reference.py +++ b/qdl_sdk/reference.py @@ -13,7 +13,13 @@ from pydantic import Field, model_validator -from qdl_sdk.models import ClosedModel, DecimalValue, Grade, ProblemDetails +from qdl_sdk.models import ( + ClosedModel, + DecimalValue, + Grade, + ProblemDetails, + StalePolicy, +) # Reference products may be deliberately less frequent than price data. The @@ -74,6 +80,12 @@ class ReferenceRequirement(ClosedModel): gt=0, le=MAX_REFERENCE_FRESHNESS_MS, ) + event_recency_policy: StalePolicy | None = None + max_session_liveness_ms: int | None = Field( + default=None, + gt=0, + le=86_400_000, + ) require_full_coverage: bool = True deadline_ms: int = Field(default=20_000, ge=100, le=120_000) @@ -96,6 +108,17 @@ def valid_shape(self): raise ValueError("reference time range must increase") if self.interval is not None and not self.interval.strip(): raise ValueError("reference interval cannot be blank") + if ( + self.event_recency_policy is StalePolicy.OBSERVE + and self.max_session_liveness_ms is None + ): + raise ValueError( + "observed reference event recency requires a provider session SLA" + ) + if self.event_recency_policy is StalePolicy.OBSERVE and not execution_mark_snapshot: + raise ValueError( + "observed reference event recency only applies to execution MARK_INDEX_PRICE" + ) if self.page_size is not None and self.page_size > self.limit: raise ValueError("reference page_size cannot exceed limit") historical = self.start_time_ns is not None diff --git a/rust/qdl-core/src/lib.rs b/rust/qdl-core/src/lib.rs index e61cd2e..8413751 100644 --- a/rust/qdl-core/src/lib.rs +++ b/rust/qdl-core/src/lib.rs @@ -11,6 +11,7 @@ pub mod l2_book; pub mod okx; pub mod okx_simulator; pub mod provider_admission; +pub mod quality; pub mod rate_limit; pub mod supervisor; pub mod telemetry; diff --git a/rust/qdl-core/src/quality.rs b/rust/qdl-core/src/quality.rs new file mode 100644 index 0000000..790402e --- /dev/null +++ b/rust/qdl-core/src/quality.rs @@ -0,0 +1,318 @@ +//! Pure binding-quality policy shared with Python through golden fixtures. +//! +//! Rust owns raw session, generation, gap, component-receipt and watermark +//! facts. Query and audit may map those facts to their public surfaces, but +//! this evaluator pins the policy vocabulary so neither layer can silently +//! invent a second freshness rule. + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum DeliverySemantics { + StrictEvent, + OnChange, +} + +impl Default for DeliverySemantics { + fn default() -> Self { + Self::StrictEvent + } +} + +impl DeliverySemantics { + fn as_str(self) -> &'static str { + match self { + Self::StrictEvent => "STRICT_EVENT", + Self::OnChange => "ON_CHANGE", + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct ComponentEvidence { + pub name: String, + pub receipt_age_ms: u64, + pub quiet_after_ms: u64, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct BindingQualityInput { + pub binding_id: String, + pub instrument_uid: String, + pub feed: String, + pub source_role: String, + pub authoritative: bool, + pub acquisition_enabled: bool, + pub acquisition_mode: String, + pub market_open: bool, + pub event_present: bool, + pub event_age_ms: Option, + pub event_limit_ms: u64, + pub event_recency_policy: String, + pub session_state: String, + pub session_liveness_ms: Option, + pub session_limit_ms: Option, + #[serde(default)] + pub delivery_semantics: DeliverySemantics, + pub components: Vec, + pub generation_matches: bool, + pub config_matches: bool, + pub gap_open: bool, + pub book_verified: bool, + pub final_bar: bool, + pub require_final_bar: bool, + pub watermark_offset: u64, + pub allow_quiet_execution: bool, + pub flags: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub struct BindingQualityDecision { + pub binding_id: String, + pub instrument_uid: String, + pub feed: String, + pub semantics: String, + pub delivery_semantics: String, + pub availability: String, + pub state: String, + pub event_recency_state: String, + pub provider_session_state: String, + pub provider_session_liveness_ms: Option, + pub complete: bool, + pub execution_eligible: bool, + pub watermark_offset: u64, + pub reason_codes: Vec, +} + +fn add_reason(reasons: &mut Vec, value: impl Into) { + let value = value.into(); + if !value.is_empty() && !reasons.contains(&value) { + reasons.push(value); + } +} + +fn semantics(input: &BindingQualityInput) -> &'static str { + let feed = input.feed.to_ascii_uppercase(); + if feed == "BAR" || input.require_final_bar { + "FINAL_SCHEDULED" + } else if input.event_recency_policy == "OBSERVE" + && (matches!(feed.as_str(), "TRADE" | "BOOK_DELTA" | "MARK_INDEX_PRICE") + || (feed == "QUOTE" && input.delivery_semantics == DeliverySemantics::OnChange)) + { + "QUIET_SESSION" + } else { + "STRICT_EVENT" + } +} + +fn availability(input: &BindingQualityInput) -> &'static str { + if input.acquisition_mode == "PYTHON_VENDOR_SDK" { + "EXPECTED_V1_PRIMARY" + } else if !input.acquisition_enabled { + "EXPECTED_DARK" + } else if !input.market_open { + "OUT_OF_SESSION" + } else { + "ACTIVE" + } +} + +/// Evaluate a declared binding strictly from facts supplied by the owning +/// runtime. It performs no I/O, retry, fallback or timestamp rewriting. +pub fn evaluate_binding_quality(input: &BindingQualityInput) -> BindingQualityDecision { + let semantics = semantics(input).to_owned(); + let availability = availability(input).to_owned(); + let mut reasons = input.flags.clone(); + let event_recency_state = match input.event_age_ms { + Some(age) if age > input.event_limit_ms => { + add_reason(&mut reasons, "LAST_EVENT_STALE"); + "STALE" + } + Some(_) => "LIVE", + None => "NOT_APPLICABLE", + } + .to_owned(); + + let session_ok = match input.session_limit_ms { + Some(limit) => { + input.session_state == "LIVE" + && input.session_liveness_ms.is_some_and(|age| age <= limit) + } + None => !matches!( + input.session_state.as_str(), + "STALE" | "DISCONNECTED" | "UNKNOWN" + ), + }; + let mut component_ok = true; + for component in &input.components { + if component.receipt_age_ms > component.quiet_after_ms { + component_ok = false; + add_reason( + &mut reasons, + format!("COMPONENT_{}_STALE", component.name.to_uppercase()), + ); + } + } + if !input.generation_matches { + add_reason(&mut reasons, "GENERATION_MISMATCH"); + } + if !input.config_matches { + add_reason(&mut reasons, "CONFIG_REVISION_MISMATCH"); + } + if input.gap_open { + add_reason(&mut reasons, "OPEN_SEQUENCE_GAP"); + } + if !input.book_verified { + add_reason(&mut reasons, "BOOK_SEQUENCE_UNVERIFIED"); + } + if input.require_final_bar && !input.final_bar { + add_reason(&mut reasons, "BAR_NOT_FINAL"); + } + if !session_ok { + match input.session_state.as_str() { + "STALE" | "DISCONNECTED" | "UNKNOWN" => add_reason( + &mut reasons, + format!("SOURCE_SESSION_{}", input.session_state), + ), + _ => add_reason(&mut reasons, "SOURCE_SESSION_HEARTBEAT_EXPIRED"), + } + } + + let state = if availability == "EXPECTED_V1_PRIMARY" { + add_reason(&mut reasons, "EXPECTED_V1_PRIMARY"); + "DISABLED" + } else if availability == "EXPECTED_DARK" { + add_reason(&mut reasons, "EXPECTED_DARK"); + "DISABLED" + } else if availability == "OUT_OF_SESSION" { + add_reason(&mut reasons, "OUT_OF_SESSION"); + "MARKET_CLOSED" + } else if !input.event_present { + add_reason(&mut reasons, "NO_DURABLE_EVENT"); + "NOT_READY" + } else if input.gap_open { + "GAPPED" + } else if !input.book_verified || (input.require_final_bar && !input.final_bar) { + "SYNCING" + } else if !input.generation_matches + || !input.config_matches + || !session_ok + || !component_ok + || (semantics != "QUIET_SESSION" && event_recency_state == "STALE") + { + "STALE" + } else { + "LIVE" + } + .to_owned(); + + let complete = input.event_present + && !input.gap_open + && input.book_verified + && (!input.require_final_bar || input.final_bar); + let event_ok = matches!(event_recency_state.as_str(), "LIVE" | "NOT_APPLICABLE"); + let quiet_execution_ok = + semantics == "QUIET_SESSION" && input.allow_quiet_execution && session_ok && component_ok; + let execution_eligible = availability == "ACTIVE" + && input.authoritative + && input.source_role == "PRIMARY" + && state == "LIVE" + && complete + && (event_ok || quiet_execution_ok); + + BindingQualityDecision { + binding_id: input.binding_id.clone(), + instrument_uid: input.instrument_uid.clone(), + feed: input.feed.clone(), + semantics, + delivery_semantics: input.delivery_semantics.as_str().to_owned(), + availability, + state, + event_recency_state, + provider_session_state: input.session_state.clone(), + provider_session_liveness_ms: input.session_liveness_ms, + complete, + execution_eligible, + watermark_offset: input.watermark_offset, + reason_codes: reasons, + } +} + +#[cfg(test)] +mod tests { + use super::{evaluate_binding_quality, BindingQualityInput}; + use serde::Deserialize; + + #[derive(Deserialize)] + struct Fixture { + schema: String, + cases: Vec, + } + + #[derive(Deserialize)] + struct Case { + name: String, + input: BindingQualityInput, + expected: Expected, + } + + #[derive(Deserialize)] + struct Expected { + semantics: String, + #[serde(default = "default_delivery_semantics")] + delivery_semantics: String, + availability: String, + state: String, + event_recency_state: String, + complete: bool, + execution_eligible: bool, + reason_codes: Vec, + } + + fn default_delivery_semantics() -> String { + "STRICT_EVENT".to_owned() + } + + #[test] + fn rust_matches_shared_binding_quality_golden_corpus() { + let fixture: Fixture = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../contracts/golden/quality/binding-quality-decision-v1.json" + ))) + .expect("quality golden JSON must parse"); + assert_eq!(fixture.schema, "qdl.binding-quality-decision.v1"); + assert!(!fixture.cases.is_empty()); + for case in fixture.cases { + let actual = evaluate_binding_quality(&case.input); + assert_eq!(actual.semantics, case.expected.semantics, "{}", case.name); + assert_eq!( + actual.delivery_semantics, case.expected.delivery_semantics, + "{}", + case.name + ); + assert_eq!( + actual.availability, case.expected.availability, + "{}", + case.name + ); + assert_eq!(actual.state, case.expected.state, "{}", case.name); + assert_eq!( + actual.event_recency_state, case.expected.event_recency_state, + "{}", + case.name + ); + assert_eq!(actual.complete, case.expected.complete, "{}", case.name); + assert_eq!( + actual.execution_eligible, case.expected.execution_eligible, + "{}", + case.name + ); + assert_eq!( + actual.reason_codes, case.expected.reason_codes, + "{}", + case.name + ); + } + } +} diff --git a/rust/qdl-realtime-core/src/lib.rs b/rust/qdl-realtime-core/src/lib.rs index 9fb043d..176e31d 100644 --- a/rust/qdl-realtime-core/src/lib.rs +++ b/rust/qdl-realtime-core/src/lib.rs @@ -48,6 +48,8 @@ pub enum MarkIndexComponent { #[serde(deny_unknown_fields)] pub struct MarkIndexBinding { pub component: MarkIndexComponent, + #[serde(default)] + pub quiet_after_ms: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] @@ -252,6 +254,14 @@ impl CoreBinding { )) } } + if mark_index + .quiet_after_ms + .is_some_and(|value| !(250..=120_000).contains(&value)) + { + return Err(CoreError::Configuration( + "mark/index component quiet cadence is outside the bounded contract".into(), + )); + } } else if self.physical_native_symbol.is_some() { return Err(CoreError::Configuration( "physical identity is reserved for a mark/index component binding".into(), @@ -379,6 +389,7 @@ struct MarkIndexComponentState { received_at_ns: i64, raw_capture_id: Vec, raw_frame_sha256: Vec, + quiet_after_ms: Option, } #[derive(Clone, Debug)] @@ -819,6 +830,7 @@ impl RealtimeCore { received_at_ns: raw.received_at_ns, raw_capture_id: raw.capture_id.clone(), raw_frame_sha256: raw.raw_frame_sha256.clone(), + quiet_after_ms: contract.quiet_after_ms, }; let update = |slot: &mut Option, value: (String, i64)| { update_mark_index_component(slot, candidate(value)) @@ -877,6 +889,18 @@ impl RealtimeCore { filtered_outcome: None, }; }; + if mark_index_component_expired(mark, processing_at_ns) + || mark_index_component_expired(index, processing_at_ns) + { + self.mark_index_pairs.insert(target_key, state); + return ProcessBatch { + canonical: vec![], + quarantines: vec![], + duplicates: 0, + filtered: 1, + filtered_outcome: Some("MARK_INDEX_COMPONENT_EXPIRED"), + }; + } let oldest_confirmation_ns = mark.received_at_ns.min(index.received_at_ns); let newest_confirmation_ns = mark.received_at_ns.max(index.received_at_ns); let source_event_time_ms = mark.source_event_time_ms.min(index.source_event_time_ms); @@ -1510,6 +1534,18 @@ fn update_mark_index_component( Ok(true) } +fn mark_index_component_expired( + component: &MarkIndexComponentState, + processing_at_ns: i64, +) -> bool { + let Some(quiet_after_ms) = component.quiet_after_ms else { + return false; + }; + let quiet_after_ns = quiet_after_ms.saturating_mul(1_000_000); + let quiet_after_ns = i64::try_from(quiet_after_ns).unwrap_or(i64::MAX); + processing_at_ns.saturating_sub(component.received_at_ns) > quiet_after_ns +} + fn pair_raw_identity( mark: &MarkIndexComponentState, index: &MarkIndexComponentState, @@ -1809,6 +1845,7 @@ mod tests { ); result.mark_index = Some(MarkIndexBinding { component: MarkIndexComponent::Both, + quiet_after_ms: None, }); result } @@ -1833,6 +1870,7 @@ mod tests { mark.physical_native_channel = Some("mark-price".into()); mark.mark_index = Some(MarkIndexBinding { component: MarkIndexComponent::Mark, + quiet_after_ms: None, }); let mut index = mark.clone(); index.provider_kind = "okx_index_price".into(); @@ -1840,6 +1878,7 @@ mod tests { index.physical_native_channel = Some("index-tickers".into()); index.mark_index = Some(MarkIndexBinding { component: MarkIndexComponent::Index, + quiet_after_ms: None, }); (mark, index) } @@ -1960,6 +1999,72 @@ mod tests { assert_eq!(evidence.reason, QuarantineReason::StaleGeneration as i32); } + #[test] + fn okx_mark_index_quiet_component_expiry_blocks_re_materialization() { + let (mut mark, mut index) = okx_mark_index_bindings("DOGE-USDT-SWAP", "DOGE-USDT"); + mark.mark_index.as_mut().unwrap().quiet_after_ms = Some(250); + index.mark_index.as_mut().unwrap().quiet_after_ms = Some(500); + let mut core = core_many(vec![mark.clone(), index.clone()], true); + let mark_frame = br#"{"arg":{"channel":"mark-price","instId":"DOGE-USDT-SWAP"},"data":[{"instId":"DOGE-USDT-SWAP","markPx":"0.14525","ts":"1786352400000"}]}"#; + let index_frame = br#"{"arg":{"channel":"index-tickers","instId":"DOGE-USDT"},"data":[{"instId":"DOGE-USDT","idxPx":"0.14510","ts":"1786352400100"}]}"#; + let origin_ns = 1_786_352_400_000_000_000; + + assert!(core + .process( + raw_with_receipt(&mark, mark_frame, 5, origin_ns), + origin_ns + 100, + ) + .unwrap() + .canonical + .is_empty()); + assert_eq!( + core.process( + raw_with_receipt(&index, index_frame, 5, origin_ns + 100_000_000), + origin_ns + 100_000_100, + ) + .unwrap() + .canonical + .len(), + 1 + ); + + // A fresh MARK cannot re-date the quiet INDEX component. The retained + // pair remains available only to the configured index cadence, and is + // never emitted as a new canonical MARK/INDEX observation after that. + let expired = core + .process( + raw_with_receipt(&mark, mark_frame, 5, origin_ns + 601_000_000), + origin_ns + 601_000_100, + ) + .unwrap(); + assert!(expired.canonical.is_empty()); + assert_eq!(expired.filtered, 1); + assert_eq!( + expired.filtered_outcome, + Some("MARK_INDEX_COMPONENT_EXPIRED") + ); + } + + #[test] + fn mark_index_quiet_cadence_must_remain_bounded() { + let mut binding = binance_mark_index_binding("DOGEUSDT"); + binding.mark_index.as_mut().unwrap().quiet_after_ms = Some(249); + let result = RealtimeCore::new(RealtimeCoreConfig { + canonical_stream: "qdl.test.canonical.v2".into(), + quarantine_stream: "qdl.test.quarantine.v1".into(), + allow_test_provenance: true, + dedup_capacity: 16, + bindings: vec![binding], + }); + let error = match result { + Err(error) => error, + Ok(_) => panic!("out-of-bounds MARK/INDEX quiet cadence was accepted"), + }; + assert!( + matches!(error, CoreError::Configuration(message) if message.contains("quiet cadence")) + ); + } + #[test] fn mark_index_conflicting_same_timestamp_is_quarantined() { let binding = binance_mark_index_binding("DOGEUSDT"); diff --git a/scripts/converge_v2_primary_runtime.py b/scripts/converge_v2_primary_runtime.py index 983035b..51675b9 100644 --- a/scripts/converge_v2_primary_runtime.py +++ b/scripts/converge_v2_primary_runtime.py @@ -38,6 +38,11 @@ validate_shared_authority_record, write_stable_runtime_bundle, ) +from qdl.runtime.core_binding_identity import ( + core_binding_map, + format_core_binding_identity, + native_ingestor_binding_identity, +) CONFIRM = "CONVERGE_QDL_V2_PRIMARY_RUNTIME" @@ -108,21 +113,20 @@ def _without(value: Mapping[str, Any], *fields: str) -> dict[str, Any]: return result -def _binding_map( +def _ingestor_binding_map( bindings: object, *, - key_field: str, field: str, -) -> dict[str, dict[str, Any]]: +) -> dict[tuple[str, ...], dict[str, Any]]: if not isinstance(bindings, list) or not bindings: raise ValueError(f"{field} bindings are invalid") - result: dict[str, dict[str, Any]] = {} + result: dict[tuple[str, ...], dict[str, Any]] = {} for item in bindings: if not isinstance(item, dict): raise ValueError(f"{field} has a non-object binding") - key = item.get(key_field) - if not isinstance(key, str) or not key or key in result: - raise ValueError(f"{field} has an invalid/duplicate {key_field}") + key = native_ingestor_binding_identity(item, field=field) + if key in result: + raise ValueError(f"{field} has a duplicate physical subscription") result[key] = dict(item) return result @@ -180,38 +184,43 @@ def _validate_core( ): raise ValueError(f"{file_name} has an unsupported dedup transition") - active_bindings = _binding_map( - active_core.get("bindings"), key_field="source_id", field=f"active {file_name}" + active_bindings = core_binding_map( + active_core.get("bindings"), field=f"active {file_name}" ) - expected_bindings = _binding_map( - expected_core.get("bindings"), key_field="source_id", field=f"expected {file_name}" + expected_bindings = core_binding_map( + expected_core.get("bindings"), field=f"expected {file_name}" + ) + unknown = sorted( + format_core_binding_identity(identity) + for identity in active_bindings.keys() - expected_bindings.keys() ) - unknown = sorted(active_bindings.keys() - expected_bindings.keys()) if unknown: raise ValueError(f"{file_name} has bindings absent from canonical catalog: {unknown}") drift = sorted( - source_id - for source_id, binding in active_bindings.items() - if not _lineage_equal(binding, expected_bindings[source_id]) + format_core_binding_identity(identity) + for identity, binding in active_bindings.items() + if not _lineage_equal(binding, expected_bindings[identity]) ) if drift: raise ValueError(f"{file_name} has retained binding semantic drift: {drift}") added = sorted(expected_bindings.keys() - active_bindings.keys()) - missing_liquid_books = sorted( - _FIVE_LIQUID_PERPETUAL_BOOK_IDS - expected_bindings.keys() - ) + expected_source_ids = {str(binding["source_id"]) for binding in expected_bindings.values()} + missing_liquid_books = sorted(_FIVE_LIQUID_PERPETUAL_BOOK_IDS - expected_source_ids) if missing_liquid_books: raise ValueError(f"{file_name} lacks five-liquid perpetual L2 scope: {missing_liquid_books}") return { "before_binding_count": len(active_bindings), "after_binding_count": len(expected_bindings), "added_binding_count": len(added), - "added_five_liquid_book_source_ids": sorted( - _FIVE_LIQUID_PERPETUAL_BOOK_IDS & set(added) - ), + "added_five_liquid_book_source_ids": sorted({ + str(expected_bindings[identity]["source_id"]) + for identity in added + if str(expected_bindings[identity]["source_id"]) + in _FIVE_LIQUID_PERPETUAL_BOOK_IDS + }), "retained_lineage_update_count": sum( - active_bindings[source_id] != expected_bindings[source_id] - for source_id in active_bindings + active_bindings[identity] != expected_bindings[identity] + for identity in active_bindings ), "dedup_capacity": {"before": active_dedup, "after": expected_dedup}, } @@ -243,11 +252,11 @@ def _validate_ingestor( ): raise ValueError(f"{file_name} lacks bounded session-liveness configuration") - active_bindings = _binding_map( - active.get("bindings"), key_field="subscription_id", field=f"active {file_name}" + active_bindings = _ingestor_binding_map( + active.get("bindings"), field=f"active {file_name}" ) - expected_bindings = _binding_map( - expected.get("bindings"), key_field="subscription_id", field=f"expected {file_name}" + expected_bindings = _ingestor_binding_map( + expected.get("bindings"), field=f"expected {file_name}" ) unknown = sorted(active_bindings.keys() - expected_bindings.keys()) if unknown: @@ -273,7 +282,11 @@ def _validate_ingestor( "before_binding_count": len(active_bindings), "after_binding_count": len(expected_bindings), "added_binding_count": len(added), - "added_book_subscription_ids": sorted(book_ids & set(added)), + "added_book_subscription_ids": sorted({ + str(expected_bindings[identity]["subscription_id"]) + for identity in added + if expected_bindings[identity].get("feed") == "BOOK" + }), "retained_lineage_update_count": sum( active_bindings[subscription_id] != expected_bindings[subscription_id] for subscription_id in active_bindings diff --git a/scripts/measure_binding_quality.py b/scripts/measure_binding_quality.py new file mode 100644 index 0000000..bf4b551 --- /dev/null +++ b/scripts/measure_binding_quality.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +"""Bounded, read-only consumer-quality matrix for declared V2 bindings. + +This probe measures what an authenticated consumer actually sees. It never +prints market payloads, contacts a venue directly, writes a cursor, or submits +an order. ``feed_status`` records typed quality; an optional snapshot read +confirms whether that same quality is usable by the public SDK. + +The default rate is deliberately below the declared 1,500 request/minute +Trading-System consumer quota when status and snapshot are both enabled. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import statistics +import time +from collections import Counter, defaultdict + +from measure_consumer_request_latency import requirements, transports +from qdl_sdk.client import AsyncDataLayerClient + + +_MAX_FLAGS_PER_VIOLATION = 16 + + +def _percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return round(ordered[min(len(ordered) - 1, int(len(ordered) * fraction))], 3) + + +def _summary(values: list[float]) -> dict[str, float | int | None]: + return { + "n": len(values), + "p50_ms": round(statistics.median(values), 3) if values else None, + "p95_ms": _percentile(values, 0.95), + "p99_ms": _percentile(values, 0.99), + "max_ms": round(max(values), 3) if values else None, + } + + +def _quality_is_usable(requirement, quality) -> bool: + """Apply the declared strict/quiet policy without inventing a raw-age rule.""" + + event_freshness_required = ( + requirement.effective_event_recency_policy.value != "OBSERVE" + ) + return bool( + quality.state == "LIVE" + and quality.complete + and quality.execution_eligible + and not quality.gap_open + and ( + not event_freshness_required + or requirement.max_freshness_ms is None + or quality.freshness_ms <= requirement.max_freshness_ms + ) + ) + + +async def _run( + *, + duration_seconds: float, + period_seconds: float, + include_snapshot: bool, + max_violation_evidence: int, +) -> dict[str, object]: + query, stream = transports() + client = AsyncDataLayerClient( + query_transport=query, + stream_transport=stream, + consumer_id=os.environ.get( + "QDL_CONSUMER_ID", "trading-system.paper.stable" + ), + ) + per_binding: dict[str, dict[str, object]] = {} + selected = requirements() + started = time.monotonic() + rounds = 0 + try: + while time.monotonic() - started < duration_seconds: + cycle_started = time.monotonic() + for label, requirement in selected: + key = f"{requirement.instrument_uid}:{requirement.feed.value}:{requirement.interval or '-'}" + row = per_binding.setdefault(key, { + "instrument_uid": requirement.instrument_uid, + "feed": requirement.feed.value, + "interval": requirement.interval, + "status_latency_ms": [], + "snapshot_latency_ms": [], + "quality_states": Counter(), + "event_recency_states": Counter(), + "session_states": Counter(), + "snapshot_outcomes": Counter(), + "errors": Counter(), + "strict_violation_count": 0, + "violations": [], + }) + started_call = time.perf_counter() + try: + status = await client.feed_status(requirement) + row["status_latency_ms"].append( + (time.perf_counter() - started_call) * 1000.0 + ) + quality = status.quality + row["quality_states"][quality.state] += 1 + row["event_recency_states"][quality.event_recency_state] += 1 + row["session_states"][quality.provider_session_state] += 1 + strict_ok = _quality_is_usable(requirement, quality) + if not strict_ok: + row["strict_violation_count"] += 1 + violations = row["violations"] + assert isinstance(violations, list) + if len(violations) < max_violation_evidence: + violations.append({ + "observed_at_ns": time.time_ns(), + "max_freshness_ms": requirement.max_freshness_ms, + "quality_state": quality.state, + "freshness_ms": quality.freshness_ms, + "event_recency_state": quality.event_recency_state, + "provider_session_state": quality.provider_session_state, + "provider_session_liveness_ms": ( + quality.provider_session_liveness_ms + ), + "gap_open": quality.gap_open, + "complete": quality.complete, + "execution_eligible": quality.execution_eligible, + "flags": sorted(quality.flags)[:_MAX_FLAGS_PER_VIOLATION], + }) + if include_snapshot and strict_ok: + started_snapshot = time.perf_counter() + try: + await client.snapshot(requirement) + except Exception as error: # noqa: BLE001 - typed output below + row["snapshot_outcomes"]["REJECTED"] += 1 + row["errors"][type(error).__name__] += 1 + else: + row["snapshot_latency_ms"].append( + (time.perf_counter() - started_snapshot) * 1000.0 + ) + row["snapshot_outcomes"]["USABLE"] += 1 + elif include_snapshot: + row["snapshot_outcomes"]["NOT_ATTEMPTED_STALE"] += 1 + except Exception as error: # noqa: BLE001 - typed output below + row["errors"][type(error).__name__] += 1 + row["snapshot_outcomes"]["STATUS_REJECTED"] += 1 + row["strict_violation_count"] += 1 + violations = row["violations"] + assert isinstance(violations, list) + if len(violations) < max_violation_evidence: + violations.append({ + "observed_at_ns": time.time_ns(), + "error_class": type(error).__name__, + }) + rounds += 1 + remaining = period_seconds - (time.monotonic() - cycle_started) + if remaining > 0: + await asyncio.sleep(remaining) + finally: + await client.close() + + rows = [] + for key, row in sorted(per_binding.items()): + rows.append({ + "binding": key, + "instrument_uid": row["instrument_uid"], + "feed": row["feed"], + "interval": row["interval"], + "status_latency": _summary(row["status_latency_ms"]), + "snapshot_latency": _summary(row["snapshot_latency_ms"]), + "quality_states": dict(sorted(row["quality_states"].items())), + "event_recency_states": dict(sorted(row["event_recency_states"].items())), + "session_states": dict(sorted(row["session_states"].items())), + "snapshot_outcomes": dict(sorted(row["snapshot_outcomes"].items())), + "errors": dict(sorted(row["errors"].items())), + "strict_violation_count": row["strict_violation_count"], + "violations": row["violations"], + }) + return { + "schema": "qdl.binding-quality-consumer-matrix.v1", + "duration_seconds": duration_seconds, + "period_seconds": period_seconds, + "rounds": rounds, + "rows": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--duration-seconds", type=float, default=60.0) + parser.add_argument("--period-seconds", type=float, default=2.0) + parser.add_argument("--without-snapshot", action="store_true") + parser.add_argument("--max-violation-evidence", type=int, default=10) + args = parser.parse_args() + if ( + args.duration_seconds <= 0 + or args.period_seconds < 1.0 + or not 1 <= args.max_violation_evidence <= 100 + ): + raise SystemExit("duration must be positive and period must be at least one second") + payload = asyncio.run(_run( + duration_seconds=args.duration_seconds, + period_seconds=args.period_seconds, + include_snapshot=not args.without_snapshot, + max_violation_evidence=args.max_violation_evidence, + )) + print(json.dumps(payload, sort_keys=True, separators=(",", ":"))) + return 0 if all( + not row["strict_violation_count"] and not row["errors"] + for row in payload["rows"] + ) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/measure_execution_mark_index_consumer_latency.py b/scripts/measure_execution_mark_index_consumer_latency.py new file mode 100644 index 0000000..9ac7b39 --- /dev/null +++ b/scripts/measure_execution_mark_index_consumer_latency.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 +"""Measure execution MARK/INDEX latency from a real V2 consumer's boundary. + +The probe is read-only. It uses the registered trading-system paper workload +identity and the public ``AsyncDataLayerClient`` exactly as a consumer does. +The primary metric is therefore not a server-side timestamp: it is the time +between initiating ``reference_batch`` and receiving the typed, SDK-validated +result which a consumer can use. + +It also records immutable provider/component lineage age separately. For a +quiet MARK/INDEX component this is deliberately *not* a delivery-latency +metric: the source timestamp remains unchanged while the governed session, +generation, gap fence and component cadence establish whether the result is +usable. The gate measures consumer-call-to-usable latency and validates that +typed quiet-session evidence independently. + +Run it from a disposable container on the stable internal network with the +existing ``trading-system`` identity mounted read-only. It never calls V1, +opens a venue connection, or sends an order. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import statistics +import sys +import time +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any, Iterable + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from qdl.certification.reference_l2_acceptance import ( # noqa: E402 + reference_request_for_requirement, +) +from qdl.consumer.manifest import ConsumerManifestLoader # noqa: E402 +from qdl.query import ConsumerGrade, FeedType # noqa: E402 +from qdl_sdk import AsyncDataLayerClient # noqa: E402 +from qdl_sdk.reference import ReferenceProduct, ReferenceRequirement # noqa: E402 +from scripts.measure_consumer_request_latency import transports # noqa: E402 + + +_LIVE_ENDPOINT = "qdl://stable-stream/internal/v2/execution/mark-index/latest" +_LIVE_VIEW = "STABLE_STREAM_GATEWAY" +_VALID_STAGES = frozenset({"CANONICAL_READ_COMMITTED", "SPOOL_CONFIRMED"}) +_VALID_RECENCY_MODES = frozenset({ + "STRICT_EVENT_SESSION_LIVE", + "COMPONENT_SESSION_LIVE", +}) +_DEFAULT_MANIFEST = ROOT / "consumers/stable/trading-system-paper.yaml" + + +def _percentile(values: Iterable[float], quantile: float) -> float: + ordered = sorted(values) + if not ordered: + raise ValueError("cannot calculate percentile for an empty sample") + index = min(len(ordered) - 1, max(0, math.ceil(len(ordered) * quantile) - 1)) + return ordered[index] + + +def _summary(samples: Iterable[float]) -> dict[str, float | int]: + values = list(samples) + if not values: + return {"n": 0} + return { + "n": len(values), + "min_ms": round(min(values), 3), + "p50_ms": round(statistics.median(values), 3), + "p95_ms": round(_percentile(values, 0.95), 3), + "p99_ms": round(_percentile(values, 0.99), 3), + "max_ms": round(max(values), 3), + } + + +def execution_mark_index_requirements(manifest: Path) -> tuple[ReferenceRequirement, ...]: + """Return the exact ten execution MARK/INDEX requirements of the manifest.""" + + document = ConsumerManifestLoader.load(manifest) + selected = [ + item + for item in document.requirements + if item.feed is FeedType.MARK_INDEX_PRICE + ] + if len(selected) != 10: + raise ValueError( + "execution MARK/INDEX consumer latency probe requires exactly 10 manifest bindings" + ) + if any(item.consumer_grade is not ConsumerGrade.EXECUTION for item in selected): + raise ValueError("execution MARK/INDEX manifest contains a non-execution requirement") + if len({item.instrument_uid for item in selected}) != len(selected): + raise ValueError("execution MARK/INDEX manifest contains duplicate instruments") + mapped = tuple( + reference_request_for_requirement(item, now_ns=time.time_ns()) + for item in selected + ) + if any( + item.product is not ReferenceProduct.MARK_INDEX_PRICE + or item.consumer_grade.value != ConsumerGrade.EXECUTION.value + or item.max_freshness_ms is None + or getattr(item.event_recency_policy, "value", item.event_recency_policy) + != "OBSERVE" + or item.max_session_liveness_ms is None + for item in mapped + ): + raise ValueError("execution MARK/INDEX mapping lost its governed reference policy") + return mapped + + +def validate_live_response( + requirements: tuple[ReferenceRequirement, ...], + response: Any, + *, + usable_at_ns: int, +) -> dict[str, dict[str, Any]]: + """Validate an SDK response and return only bounded per-binding evidence. + + This is intentionally stricter than a successful HTTP response. It proves + the live gateway route was selected and rejects a direct provider result, + a partial response, a missing price pair, or an identity mix-up. + """ + + if bool(getattr(response, "partial", True)): + raise ValueError("execution MARK/INDEX response is partial") + results = tuple(getattr(response, "results", ())) + if len(results) != len(requirements): + raise ValueError("execution MARK/INDEX response cardinality differs from request") + expected = {item.instrument_uid: item for item in requirements} + values: dict[str, dict[str, Any]] = {} + for item in results: + instrument_uid = str(getattr(item, "instrument_uid", "")) + requirement = expected.get(instrument_uid) + data = getattr(item, "data", None) + if ( + requirement is None + or getattr(item, "product", None) is not ReferenceProduct.MARK_INDEX_PRICE + or getattr(item, "status", None) != "OK" + or getattr(item, "problem", None) is not None + or data is None + or getattr(data, "status", None) != "OK" + or getattr(data, "instrument_uid", None) != instrument_uid + or getattr(data, "product", None) is not ReferenceProduct.MARK_INDEX_PRICE + ): + raise ValueError("execution MARK/INDEX result is non-OK or identity-mismatched") + observations = tuple(getattr(data, "observations", ())) + if len(observations) != 1: + raise ValueError("execution MARK/INDEX result must contain one current observation") + observation = observations[0] + if ( + getattr(observation, "instrument_uid", None) != instrument_uid + or getattr(observation, "product", None) is not ReferenceProduct.MARK_INDEX_PRICE + ): + raise ValueError("execution MARK/INDEX observation identity differs from result") + fields = {str(field.name) for field in getattr(observation, "fields", ())} + if fields != {"mark_price", "index_price"}: + raise ValueError("execution MARK/INDEX observation is not a complete price pair") + labels = dict(getattr(observation, "labels", {})) + received_at_ns = int(getattr(data, "received_at_ns", 0)) + provider_confirmation_ns = int(labels.get("provider_confirmation_ns", "0")) + source_event_time_ns = int(labels.get("source_event_time_ns", "0")) + stage = labels.get("delivery_stage") + if ( + labels.get("execution_view") != _LIVE_VIEW + or provider_confirmation_ns <= 0 + or provider_confirmation_ns != received_at_ns + or source_event_time_ns <= 0 + or source_event_time_ns > provider_confirmation_ns + or stage not in _VALID_STAGES + or received_at_ns > usable_at_ns + ): + raise ValueError("execution MARK/INDEX live-view lineage or freshness is invalid") + lineage = tuple(getattr(data, "lineage", ())) + if not lineage or any( + getattr(entry, "provider_endpoint", None) != _LIVE_ENDPOINT + for entry in lineage + ): + raise ValueError("execution MARK/INDEX response did not use the internal live reader") + values[instrument_uid] = _quiet_session_evidence( + requirement, + labels, + provider_confirmation_ns=provider_confirmation_ns, + usable_at_ns=usable_at_ns, + ) + values[instrument_uid]["delivery_stage"] = stage + if set(values) != set(expected): + raise ValueError("execution MARK/INDEX response did not cover every requested binding") + return values + + +def _quiet_session_evidence( + requirement: ReferenceRequirement, + labels: dict[str, str], + *, + provider_confirmation_ns: int, + usable_at_ns: int, +) -> dict[str, Any]: + """Validate the explicit quiet-channel contract at the SDK boundary. + + Provider timestamps are immutable lineage. A quiet component is admitted + only when the stream/query path proves a current provider session and the + exact component receipt remains inside its signed cadence. This mirrors + the fail-closed query check without treating an unchanged component as a + newly delivered market event. + """ + + policy = getattr( + requirement.event_recency_policy, + "value", + requirement.event_recency_policy, + ) + if policy != "OBSERVE" or requirement.max_session_liveness_ms is None: + raise ValueError("execution MARK/INDEX requirement lacks quiet-session policy") + if ( + labels.get("event_recency_policy") != "OBSERVE" + or labels.get("recency_mode") not in _VALID_RECENCY_MODES + or labels.get("provider_session_state") != "LIVE" + ): + raise ValueError("execution MARK/INDEX quiet-session evidence is not live") + try: + session_liveness_ms = int(labels["provider_session_liveness_ms"]) + session_checked_at_ns = int(labels["provider_session_checked_at_ns"]) + components = { + name: ( + int(labels[f"component_{name.lower()}_received_at_ns"]), + int(labels[f"component_{name.lower()}_quiet_after_ms"]), + ) + for name in ("MARK", "INDEX") + } + except (KeyError, TypeError, ValueError) as error: + raise ValueError("execution MARK/INDEX quiet-session evidence is malformed") from error + if ( + session_liveness_ms < 0 + or session_checked_at_ns <= 0 + or session_checked_at_ns > usable_at_ns + ): + raise ValueError("execution MARK/INDEX quiet-session clock is invalid") + session_age_ms = session_liveness_ms + ( + usable_at_ns - session_checked_at_ns + ) / 1_000_000 + if session_age_ms > requirement.max_session_liveness_ms: + raise ValueError("execution MARK/INDEX provider session exceeded its SLA") + component_ages: dict[str, float] = {} + for name, (receipt_ns, quiet_after_ms) in components.items(): + if ( + receipt_ns <= 0 + or receipt_ns > usable_at_ns + or not 250 <= quiet_after_ms <= 120_000 + ): + raise ValueError("execution MARK/INDEX component evidence is invalid") + age_ms = (usable_at_ns - receipt_ns) / 1_000_000 + if age_ms > quiet_after_ms: + raise ValueError("execution MARK/INDEX component exceeded its quiet cadence") + component_ages[name] = age_ms + return { + # This is an immutable lineage diagnostic, never the quiet-channel SLA. + "provider_confirmation_to_usable_ms": ( + usable_at_ns - provider_confirmation_ns + ) / 1_000_000, + "provider_session_liveness_to_usable_ms": session_age_ms, + "component_mark_age_to_usable_ms": component_ages["MARK"], + "component_index_age_to_usable_ms": component_ages["INDEX"], + "recency_mode": labels["recency_mode"], + } + + +async def collect( + client: AsyncDataLayerClient, + requirements: tuple[ReferenceRequirement, ...], + *, + duration_seconds: float, + cadence_seconds: float, +) -> dict[str, Any]: + if duration_seconds <= 0 or cadence_seconds <= 0: + raise ValueError("duration and cadence must be positive") + deadline = time.monotonic() + duration_seconds + next_call = time.monotonic() + calls_ms: list[float] = [] + provider_age_by_uid: dict[str, list[float]] = defaultdict(list) + session_age_by_uid: dict[str, list[float]] = defaultdict(list) + mark_age_by_uid: dict[str, list[float]] = defaultdict(list) + index_age_by_uid: dict[str, list[float]] = defaultdict(list) + stages: Counter[str] = Counter() + recency_modes: Counter[str] = Counter() + errors: list[str] = [] + batches = 0 + while time.monotonic() < deadline: + delay = next_call - time.monotonic() + if delay > 0: + await asyncio.sleep(delay) + started = time.perf_counter() + try: + response = await client.reference_batch(requirements, require_all=True) + usable_at_ns = time.time_ns() + values = validate_live_response( + requirements, response, usable_at_ns=usable_at_ns + ) + except Exception as error: # noqa: BLE001 - bounded evidence, no retry disguise + errors.append(f"{type(error).__name__}: {error}"[:240]) + break + calls_ms.append((time.perf_counter() - started) * 1_000) + batches += 1 + for uid, value in values.items(): + provider_age_by_uid[uid].append( + value["provider_confirmation_to_usable_ms"] + ) + session_age_by_uid[uid].append( + value["provider_session_liveness_to_usable_ms"] + ) + mark_age_by_uid[uid].append(value["component_mark_age_to_usable_ms"]) + index_age_by_uid[uid].append(value["component_index_age_to_usable_ms"]) + stages[str(value["delivery_stage"])] += 1 + recency_modes[str(value["recency_mode"])] += 1 + next_call += cadence_seconds + per_binding = { + uid: { + "provider_confirmation_to_usable_ms": _summary( + provider_age_by_uid[uid] + ), + "provider_session_liveness_to_usable_ms": _summary( + session_age_by_uid[uid] + ), + "component_mark_age_to_usable_ms": _summary(mark_age_by_uid[uid]), + "component_index_age_to_usable_ms": _summary(index_age_by_uid[uid]), + } + for uid in sorted(provider_age_by_uid) + } + all_provider_age = [ + value for values in provider_age_by_uid.values() for value in values + ] + all_session_age = [ + value for values in session_age_by_uid.values() for value in values + ] + all_component_age = [ + value + for values in (*mark_age_by_uid.values(), *index_age_by_uid.values()) + for value in values + ] + return { + "batches": batches, + "consumer_call_to_usable_ms": _summary(calls_ms), + "provider_confirmation_to_usable_ms": _summary(all_provider_age), + "provider_session_liveness_to_usable_ms": _summary(all_session_age), + "component_age_to_usable_ms": _summary(all_component_age), + "per_binding": per_binding, + "delivery_stages": dict(sorted(stages.items())), + "recency_modes": dict(sorted(recency_modes.items())), + "errors": errors, + } + + +def _write_evidence(path: Path, result: dict[str, Any]) -> None: + target = path.expanduser().resolve() + if target.exists(): + raise ValueError("consumer latency evidence path already exists") + target.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + target.write_text(json.dumps(result, sort_keys=True, indent=2) + "\n", encoding="utf-8") + target.chmod(0o600) + + +async def run(args: argparse.Namespace) -> dict[str, Any]: + requirements = execution_mark_index_requirements(args.manifest) + query, stream = transports() + client = AsyncDataLayerClient( + query_transport=query, + stream_transport=stream, + consumer_id=os.environ.get("QDL_CONSUMER_ID", "trading-system.paper.stable"), + ) + started_ns = time.time_ns() + try: + result = await collect( + client, + requirements, + duration_seconds=args.duration_seconds, + cadence_seconds=args.cadence_seconds, + ) + finally: + await client.close() + minimum_samples, gate_passed = _acceptance_gate( + result, + requirement_count=len(requirements), + duration_seconds=args.duration_seconds, + cadence_seconds=args.cadence_seconds, + max_consumer_call_p99_ms=args.max_p99_ms, + ) + evidence = { + "schema": "qdl.execution-mark-index-consumer-latency.v2", + "started_at_ns": started_ns, + "finished_at_ns": time.time_ns(), + "consumer_id": os.environ.get("QDL_CONSUMER_ID", "trading-system.paper.stable"), + "route": "V2_REFERENCE_BATCH_INTERNAL_EXECUTION", + "v1_fallback_attempted": False, + "direct_provider_request_attempted": False, + "requirement_count": len(requirements), + "minimum_samples_per_binding": minimum_samples, + "max_consumer_call_to_usable_p99_ms": args.max_p99_ms, + "gate_passed": gate_passed, + **result, + } + if args.output: + _write_evidence(args.output, evidence) + return evidence + + +def _acceptance_gate( + result: dict[str, Any], + *, + requirement_count: int, + duration_seconds: float, + cadence_seconds: float, + max_consumer_call_p99_ms: float, +) -> tuple[int, bool]: + """Apply the C2 gate without ever substituting immutable lineage for latency.""" + + minimum_samples = math.floor(duration_seconds / cadence_seconds) - 1 + aggregate = result["consumer_call_to_usable_ms"] + per_binding_complete = all( + data["provider_session_liveness_to_usable_ms"].get("n", 0) + >= minimum_samples + and data["component_mark_age_to_usable_ms"].get("n", 0) >= minimum_samples + and data["component_index_age_to_usable_ms"].get("n", 0) >= minimum_samples + for data in result["per_binding"].values() + ) + return ( + minimum_samples, + ( + not result["errors"] + and len(result["per_binding"]) == requirement_count + and per_binding_complete + and aggregate.get("n", 0) >= minimum_samples + and float(aggregate.get("p99_ms", float("inf"))) + < max_consumer_call_p99_ms + ), + ) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, default=_DEFAULT_MANIFEST) + parser.add_argument("--duration-seconds", type=float, default=300.0) + parser.add_argument("--cadence-seconds", type=float, default=2.0) + parser.add_argument("--max-p99-ms", type=float, default=2_000.0) + parser.add_argument("--output", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + result = asyncio.run(run(args)) + print(json.dumps(result, sort_keys=True, indent=2)) + return 0 if result["gate_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/phase105_consumer_v2_identity_acceptance.py b/scripts/phase105_consumer_v2_identity_acceptance.py index 13c9413..d78e70d 100644 --- a/scripts/phase105_consumer_v2_identity_acceptance.py +++ b/scripts/phase105_consumer_v2_identity_acceptance.py @@ -10,8 +10,10 @@ import argparse import asyncio +from collections import Counter import hashlib import json +from math import ceil import resource import shutil import sys @@ -19,7 +21,7 @@ import time from dataclasses import dataclass, replace from pathlib import Path -from typing import Callable +from typing import Callable, Mapping from urllib.parse import urlsplit import httpx @@ -60,7 +62,9 @@ validate_v1_provenance, validate_v1_runtime_binding, ) +from qdl.adapters.intervals import canonical_interval_ms from qdl.consumer import StableReleaseRoutePlan, requirement_key +from qdl.query import ConsumerGrade, FeedType, StalePolicy from qdl.certification.phase105_release_observations import compact_view_quality from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan @@ -96,8 +100,17 @@ _MAX_REFERENCE_BATCH_CONCURRENCY = 4 _C2_REQUEST_QUOTA_FRACTION = 0.75 _C2_QUOTA_WINDOW_MARGIN_SECONDS = 0.05 -_C2_OPENING_TIMEOUT_SECONDS = 900.0 _C2_CLOSING_REVALIDATION_MAX_SECONDS = 120.0 +_STRICT_BAR_BATCH_SHAPES = (1, 8, 16, 32) +_LOSSLESS_L2_FEEDS = frozenset({FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA}) + + +def _evidence_sha256(value: Mapping[str, object]) -> str: + """Hash already payload-free evidence without retaining its value twice.""" + + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() @dataclass(frozen=True, slots=True) @@ -108,6 +121,19 @@ class IdentityFiles: jwt_key_id: str +class C2OpeningCapacityError(RuntimeError): + """Payload-free evidence when C2 exceeds its declared opening protocol.""" + + def __init__(self, code: str, details: Mapping[str, object]) -> None: + super().__init__(f"Phase 10.5 C2 opening capacity failure: {code}") + self.evidence = { + "schema": "qdl.phase105.c2-opening-capacity-failure.v1", + "code": code, + "details": dict(details), + "payload_recorded": False, + } + + class C2ProductAcceptanceError(RuntimeError): """One compact, payload-free product failure for an operator C2 receipt.""" @@ -123,6 +149,32 @@ def __init__(self, product: AcceptanceProduct, error: C2StatusEvidenceError) -> "replica": error.replica or "unknown", "error_code": error.code, "typed_status": error.status_evidence, + "quality_sha256": _evidence_sha256(error.status_evidence), + "payload_recorded": False, + } + + +class C2ReferenceProductError(RuntimeError): + """One compact, payload-free reference failure for an operator C2 receipt.""" + + def __init__( + self, + product: ReferenceAcceptanceProduct, + *, + replica: str, + error: ValueError, + ) -> None: + super().__init__( + "Phase 10.5 V2 reference receipt failed " + f"consumer={product.consumer_id} instrument={product.instrument_id} " + f"feed={product.requirement.feed.value} replica={replica}" + ) + self.evidence = { + "schema": "qdl.phase105.c2-reference-product-failure.v1", + "product": product.evidence(), + "replica": replica, + "error_type": type(error).__name__, + "reason": str(error)[:240], "payload_recorded": False, } @@ -138,6 +190,8 @@ def __init__( products: tuple[AcceptanceProduct, ...], error: Exception, status_observations: list[dict[str, object]], + batch_item_problems: list[dict[str, object]] | None = None, + transport_batch_summary: Mapping[str, object] | None = None, ) -> None: if not products or any(item.consumer_id != consumer_id for item in products): raise ValueError("Phase 10.5 closing batch failure has an invalid consumer scope") @@ -152,6 +206,22 @@ def __init__( "Phase 10.5 V2 closing batch failed " f"consumer={consumer_id} replica={replica} size={len(products)}" ) + summary = dict(transport_batch_summary or {}) + raw_outcomes = summary.get("problem_outcomes") + server_problem_outcomes = [] + if isinstance(raw_outcomes, list): + for outcome in raw_outcomes: + if not isinstance(outcome, Mapping): + continue + index = outcome.get("index") + if not isinstance(index, int) or not 0 <= index < len(products): + continue + server_problem_outcomes.append({ + **products[index].evidence(), + "status": str(outcome.get("status") or "UNKNOWN"), + "problem_code": str(outcome.get("problem_code") or "UNKNOWN"), + "retryable": bool(outcome.get("retryable", False)), + }) self.evidence = { "schema": "qdl.phase105.c2-closing-batch-failure.v1", "consumer_id": consumer_id, @@ -159,7 +229,94 @@ def __init__( "batch_size": len(products), "batch_identity_sha256": digest, "transport_error": type(error).__name__, + "transport_error_code": getattr(error, "code", None), + "transport_retryable": bool(getattr(error, "retryable", False)), + "transport_detail_sha256": hashlib.sha256( + str(getattr(error, "detail", error)).encode() + ).hexdigest(), "typed_status": status_observations, + "batch_item_problems": list(batch_item_problems or ()), + "server_batch_summary": summary or None, + "server_problem_outcomes": server_problem_outcomes, + "payload_recorded": False, + } + + +class C2ClosingL2Error(RuntimeError): + """Compact typed evidence for one lossless-L2 closing read failure.""" + + def __init__( + self, + *, + product: AcceptanceProduct, + replica: str, + operation: str, + error: Exception, + status_evidence: Mapping[str, object] | None, + ) -> None: + if operation not in {"FEED_STATUS", "SNAPSHOT", "REPLICA_PARITY"}: + raise ValueError("Phase 10.5 L2 closing failure operation is invalid") + super().__init__( + "Phase 10.5 V2 lossless L2 closing read failed " + f"consumer={product.consumer_id} replica={replica} " + f"feed={product.feed.value} operation={operation}" + ) + self.evidence = { + "schema": "qdl.phase105.c2-closing-l2-failure.v1", + "product": product.evidence(), + "replica": replica, + "operation": operation, + "transport_error": type(error).__name__, + "transport_error_code": getattr(error, "code", None), + "transport_retryable": bool(getattr(error, "retryable", False)), + "transport_detail_sha256": hashlib.sha256( + str(getattr(error, "detail", error)).encode() + ).hexdigest(), + "typed_status": dict(status_evidence) if status_evidence is not None else None, + "payload_recorded": False, + } + + +class C2BatchShapeError(RuntimeError): + """Add batch-shape context to one bounded, payload-free V2 failure.""" + + def __init__( + self, + error: Exception, + *, + stage: str, + batch_shape: int, + window_index: int, + products: tuple[AcceptanceProduct, ...], + ) -> None: + if batch_shape < 1 or window_index < 0 or not products: + raise ValueError("Phase 10.5 batch-shape failure context is invalid") + if isinstance(error, C2ClosingBatchError): + source = dict(error.evidence) + else: + source = { + "consumer_id": products[0].consumer_id, + "replica": "both", + "batch_size": len(products), + "batch_identity_sha256": _batch_identity_sha256(products), + "transport_error": type(error).__name__, + "transport_error_code": getattr(error, "code", None), + "transport_retryable": bool(getattr(error, "retryable", False)), + "transport_detail_sha256": hashlib.sha256(str(error).encode()).hexdigest(), + "typed_status": [], + "batch_item_problems": [], + "payload_recorded": False, + } + super().__init__( + "Phase 10.5 strict BAR batch-shape matrix failed " + f"stage={stage} shape={batch_shape} window={window_index}" + ) + self.evidence = { + **source, + "schema": "qdl.phase105.strict-bar-batch-shape-failure.v1", + "stage": stage, + "batch_shape": batch_shape, + "window_index": window_index, "payload_recorded": False, } @@ -194,6 +351,7 @@ def __init__( self._clock = clock self._sleep = sleep self._lock = asyncio.Lock() + self._operation_counts = Counter() self._next_at: float | None = None self._request_count = 0 self._wait_seconds = 0.0 @@ -213,14 +371,18 @@ async def wait_for_clean_window(self) -> float: self._window_wait_seconds += wait_seconds return wait_seconds - async def acquire(self) -> None: + async def acquire(self, operation: str = "UNSPECIFIED") -> None: """Reserve one real REST request without borrowing quota from a peer.""" + operation = str(operation).strip() + if not operation: + raise ValueError("C2 operation name is required") async with self._lock: now = self._clock() target = now if self._next_at is None else max(now, self._next_at) self._next_at = target + self._seconds_per_request self._request_count += 1 + self._operation_counts[operation] += 1 wait_seconds = max(0.0, target - now) self._wait_seconds += wait_seconds sleeper = asyncio.sleep if self._sleep is None else self._sleep @@ -234,25 +396,67 @@ def evidence(self) -> dict[str, object]: "c2_request_count": self._request_count, "c2_pacing_wait_seconds": round(self._wait_seconds, 3), "c2_clean_window_wait_seconds": round(self._window_wait_seconds, 3), + "c2_operation_counts": dict(sorted(self._operation_counts.items())), } +def _compact_strict_batch_response(payload: object) -> dict[str, object] | None: + """Keep only response-shape diagnostics before SDK enforces strict failure.""" + + if not isinstance(payload, Mapping): + return None + raw_results = payload.get("results") + if not isinstance(raw_results, list): + return None + problems = [] + for index, item in enumerate(raw_results): + if not isinstance(item, Mapping): + continue + problem = item.get("problem") + code = problem.get("code") if isinstance(problem, Mapping) else None + if code is None and str(item.get("status") or "OK") == "OK": + continue + problems.append({ + "index": index, + "status": str(item.get("status") or "UNKNOWN"), + "problem_code": str(code or "UNKNOWN"), + "retryable": bool(problem.get("retryable", False)) if isinstance(problem, Mapping) else False, + }) + return { + "partial": bool(payload.get("partial", False)), + "success_count": int(payload.get("success_count", 0)), + "error_count": int(payload.get("error_count", 0)), + "result_count": len(raw_results), + "problem_outcomes": problems, + "payload_recorded": False, + } + + class _PacedQueryTransport: """Acceptance-only adapter that charges every C2 REST call to one pacer.""" def __init__(self, delegate, pacer: _C2ConsumerRequestPacer) -> None: self._delegate = delegate self._pacer = pacer + self._last_warmup_batch_summary: dict[str, object] | None = None async def _call(self, name: str, *args, **kwargs): - await self._pacer.acquire() + operation = "REFERENCE_BATCH" if name == "reference_batch" else "QUERY_READ" + await self._pacer.acquire(operation) return await getattr(self._delegate, name)(*args, **kwargs) async def warmup(self, *args, **kwargs): return await self._call("warmup", *args, **kwargs) async def warmup_batch(self, *args, **kwargs): - return await self._call("warmup_batch", *args, **kwargs) + response = await self._call("warmup_batch", *args, **kwargs) + self._last_warmup_batch_summary = _compact_strict_batch_response(response) + return response + + def last_warmup_batch_summary(self) -> dict[str, object] | None: + if self._last_warmup_batch_summary is None: + return None + return json.loads(json.dumps(self._last_warmup_batch_summary, sort_keys=True)) async def reference_batch(self, *args, **kwargs): return await self._call("reference_batch", *args, **kwargs) @@ -284,7 +488,7 @@ async def subscribe(self, *args, **kwargs): # `subscribe` is an async iterator. Reserving at iterator start covers # both the initial stream and every SDK reconnect without changing the # public stream contract. - await self._pacer.acquire() + await self._pacer.acquire("STREAM_SUBSCRIBE") async for item in self._delegate.subscribe(*args, **kwargs): yield item @@ -478,6 +682,137 @@ def _route_summary(release: StableReleaseRoutePlan, products: tuple[AcceptancePr return summary +def _timing_policy(product) -> dict[str, object]: + """Classify a sealed requirement without changing its declared threshold. + + The product requirement remains the source of truth. This only prevents + acceptance evidence from describing a continuity/dropout horizon as an + execution reaction SLA, or a quiet session as a broken provider because no + event was emitted. + """ + + requirement = product.requirement + feed = requirement.feed.value + event_policy = requirement.effective_event_recency_policy.value + freshness_ms = requirement.max_freshness_ms + session_ms = requirement.max_session_liveness_ms + common: dict[str, object] = { + "feed": feed, + "event_recency_policy": event_policy, + "declared_max_freshness_ms": freshness_ms, + "declared_max_session_liveness_ms": session_ms, + } + if feed == "BAR": + interval = requirement.interval + if not interval or not requirement.require_final_bars: + raise ValueError("Phase 10.5 final BAR timing requires interval and finality") + interval_ms = canonical_interval_ms(interval) + if freshness_ms is None or freshness_ms < interval_ms: + raise ValueError("Phase 10.5 final BAR continuity horizon is below its interval") + return { + **common, + "semantic_class": "FINAL_SCHEDULED", + "finality_required": True, + "interval_ms": interval_ms, + "freshness_role": "CONTINUITY_DROPOUT_HORIZON", + "close_to_usable_sla_ms": None, + "required_fences": ["FINALITY", "GAP", "GENERATION", "COMPLETENESS"], + } + if feed == "MARK_INDEX_PRICE": + # MARK/INDEX has two deliberately distinct V2 read contracts. The + # execution OBSERVE form is the current gateway live view, whose + # provider session/component evidence makes a quiet market usable. + # All other forms are bounded reference snapshots; they must prove + # provider-observation freshness and lineage, but cannot invent a + # stream session that the product did not declare. + execution_live_view = ( + requirement.consumer_grade is ConsumerGrade.EXECUTION + and requirement.effective_event_recency_policy is StalePolicy.OBSERVE + ) + if execution_live_view: + if session_ms is None: + raise ValueError( + "Phase 10.5 execution MARK_INDEX_PRICE timing requires a session-liveness bound" + ) + return { + **common, + "semantic_class": "QUIET_SESSION", + "freshness_role": "EVENT_RECENCY_OBSERVED_NOT_ADMITTED_ALONE", + "required_fences": [ + "SESSION", "COMPONENT_CADENCE", "GAP", "GENERATION", "COMPLETENESS", + ], + } + if freshness_ms is None: + raise ValueError( + "Phase 10.5 reference MARK_INDEX_PRICE timing requires a provider-observation bound" + ) + return { + **common, + "semantic_class": "REFERENCE_SNAPSHOT", + "freshness_role": "PROVIDER_OBSERVATION_AGE", + "required_fences": ["IDENTITY", "LINEAGE", "COVERAGE", "FRESHNESS"], + } + if feed in {"TRADE", "BOOK_DELTA"}: + if event_policy == "OBSERVE": + if session_ms is None: + raise ValueError( + f"Phase 10.5 observed {feed} timing requires a session-liveness bound" + ) + return { + **common, + "semantic_class": "QUIET_SESSION", + "freshness_role": "EVENT_RECENCY_OBSERVED_NOT_ADMITTED_ALONE", + "required_fences": ["SESSION", "GAP", "GENERATION", "COMPLETENESS"], + } + if freshness_ms is None: + raise ValueError(f"Phase 10.5 strict {feed} timing requires an event-age bound") + if session_ms is None: + # Some research-only strict routes deliberately do not declare a + # numeric session SLA. They remain event-age/gap fenced and cannot + # become a quiet execution route merely because a provider has not + # emitted a trade during this probe. + return { + **common, + "semantic_class": "STRICT_EVENT", + "freshness_role": "EVENT_AGE", + "session_contract": "NOT_DECLARED_STRICT_EVENT", + "required_fences": ["EVENT_AGE", "GAP", "GENERATION", "COMPLETENESS"], + } + return { + **common, + "semantic_class": "STRICT_EVENT_WITH_SESSION", + "freshness_role": "EVENT_AGE_AND_SESSION", + "required_fences": ["EVENT_AGE", "SESSION", "GAP", "GENERATION", "COMPLETENESS"], + } + if feed == "QUOTE": + if freshness_ms is None or session_ms is None: + raise ValueError("Phase 10.5 QUOTE timing requires event-age and session bounds") + return { + **common, + "semantic_class": ( + "QUIET_SESSION" if event_policy == "OBSERVE" else "STRICT_EVENT_WITH_SESSION" + ), + "freshness_role": "EVENT_AGE_OR_PROVIDER_ON_CHANGE_WITH_SESSION", + "delivery_semantics": "ASSERT_FROM_TYPED_QUALITY", + "required_fences": ["SESSION", "GAP", "GENERATION", "COMPLETENESS"], + } + if feed == "BOOK_SNAPSHOT": + if freshness_ms is None: + raise ValueError("Phase 10.5 BOOK_SNAPSHOT timing requires a baseline-age bound") + return { + **common, + "semantic_class": "BOOK_BASELINE", + "freshness_role": "SNAPSHOT_BASELINE_NOT_DELTA_EXECUTION_AGE", + "required_fences": ["GAP", "GENERATION", "COMPLETENESS", "BOOK_VERIFICATION"], + } + return { + **common, + "semantic_class": "REFERENCE_CADENCE", + "freshness_role": "PUBLISHED_VALUE_CADENCE", + "required_fences": ["IDENTITY", "LINEAGE", "COVERAGE"], + } + + def _reference_product( product: AcceptanceProduct, *, @@ -504,6 +839,213 @@ def _reference_product( ) +def _build_c2_opening_operation_plan( + products: tuple[AcceptanceProduct, ...], + release: StableReleaseRoutePlan, + probes, + consumer_ids: tuple[str, ...], + *, + generic_timeout_seconds: float, + reference_now_ns: int | None = None, +) -> dict[str, object]: + """Compile the C2 opening budget from the sealed SDK operation graph. + + The calculation deliberately follows the same helpers C2 later invokes. + It does not inspect provider payloads or change a manifest. Every durable + product gets two initial Query reads plus an explicit two-session + warmup/cursor handoff; provider pass-through gets only the two Query reads; + reference batches and the one documented native-BASIS deferral are counted + through their existing batching helper. + """ + + if not products or not consumer_ids: + raise ValueError("C2 opening operation plan requires products and consumers") + if generic_timeout_seconds <= 0: + raise ValueError("C2 opening operation plan requires a positive timeout") + selected = frozenset(consumer_ids) + if len(selected) != len(consumer_ids): + raise ValueError("C2 opening operation plan duplicates a consumer") + if {item.consumer_id for item in products} != selected: + raise ValueError("C2 opening operation plan consumer scope is incomplete") + routes = {item.consumer_id: item for item in release.consumers} + if not selected <= set(routes): + raise ValueError("C2 opening operation plan lacks a release consumer") + probe_counts = Counter(item.consumer_id for item in probes) + if not set(probe_counts) <= selected: + raise ValueError("C2 opening operation plan has an out-of-scope fallback probe") + now_ns = time.time_ns() if reference_now_ns is None else reference_now_ns + plans: dict[str, dict[str, object]] = {} + for consumer_id in consumer_ids: + consumer_products = tuple( + item for item in products if item.consumer_id == consumer_id + ) + on_demand = tuple( + item for item in consumer_products + if item.delivery is DeliveryClass.ON_DEMAND + ) + streamed = tuple( + item for item in consumer_products + if item.delivery is not DeliveryClass.ON_DEMAND + ) + durable = tuple( + item for item in streamed if item.delivery is DeliveryClass.DURABLE + ) + if any( + item.delivery not in { + DeliveryClass.DURABLE, + DeliveryClass.PROVIDER_PASS_THROUGH, + DeliveryClass.ON_DEMAND, + } + for item in consumer_products + ): + raise ValueError("C2 opening operation plan has an unknown delivery class") + references = tuple( + _reference_product(item, now_ns=now_ns) for item in on_demand + ) + reference_batches = reference_acceptance_batches(references) + native_basis_batches = tuple( + batch for batch in reference_batches + if len(batch) == 1 and is_rust_admitted_native_basis(batch[0]) + ) + if any( + is_rust_admitted_native_basis(item) + for batch in reference_batches + if len(batch) != 1 + for item in batch + ): + raise AssertionError("C2 native BASIS batch lost its singleton boundary") + reference_tail_timeout = ( + _reference_transport_timeout_seconds( + references, generic_timeout_seconds=generic_timeout_seconds + ) + if references + else generic_timeout_seconds + ) + native_basis_deferral_seconds = sum( + 2.0 * batch[0].sdk_requirement.deadline_ms / 1_000 + for batch in native_basis_batches + ) + manifest = routes[consumer_id].manifest + safe_rpm = max( + 1, + int( + manifest.quotas.requests_per_minute * _C2_REQUEST_QUOTA_FRACTION + ), + ) + budget = { + # Every stream-capable product reads both Query replicas first; + # durable products then do one warmup/snapshot per handoff side. + "QUERY_READ": ( + 2 * len(streamed) + + 2 * len(durable) + # V2 -> V1 -> V2 reads both replicas before and after fallback. + + 4 * probe_counts[consumer_id] + ), + # Both Query replicas read every declared reference batch. Each + # native-BASIS replica may use exactly one typed deferral retry. + "REFERENCE_BATCH": ( + 2 * len(reference_batches) + 2 * len(native_basis_batches) + ), + "STREAM_SUBSCRIBE": 2 * len(durable), + } + budget = {name: count for name, count in budget.items() if count} + total = sum(budget.values()) + plans[consumer_id] = { + "requests_per_minute": manifest.quotas.requests_per_minute, + "safe_requests_per_minute": safe_rpm, + "max_streams": manifest.quotas.max_streams, + "opening_operation_budget": budget, + "opening_total_operations": total, + "opening_pacing_floor_seconds": round( + max(0, total - 1) * 60.0 / safe_rpm, 3 + ), + "native_basis_deferral_seconds": round(native_basis_deferral_seconds, 3), + "tail_timeout_seconds": round( + max(generic_timeout_seconds, reference_tail_timeout), 3 + ), + } + global_route_products = tuple( + product + for consumer in release.consumers + for product in consumer.products + ) + selected_route_products = tuple( + product + for consumer_id in consumer_ids + for product in routes[consumer_id].products + ) + selected_v2_identities = { + (consumer_id, product.requirement_key) + for consumer_id in consumer_ids + for product in routes[consumer_id].products + if product.route == "V2_PRIMARY" + } + actual_v2_identities = { + (product.consumer_id, requirement_key(product.requirement)) + for product in products + } + if actual_v2_identities != selected_v2_identities: + raise ValueError("C2 opening operation plan differs from V2 primary routes") + pacing_floor = max(float(item["opening_pacing_floor_seconds"]) for item in plans.values()) + native_basis_deferral = sum( + float(item["native_basis_deferral_seconds"]) for item in plans.values() + ) + tail_timeout = max(float(item["tail_timeout_seconds"]) for item in plans.values()) + return { + "schema": "qdl.phase105.c2-opening-operation-plan.v1", + "global_release_route_count": len(global_route_products), + "global_v2_primary_product_count": sum( + product.route == "V2_PRIMARY" for product in global_route_products + ), + "global_v1_primary_route_count": sum( + product.route == "V1_PRIMARY" for product in global_route_products + ), + "selected_release_route_count": len(selected_route_products), + "selected_v2_primary_product_count": len(selected_v2_identities), + "selected_v1_primary_excluded_count": sum( + product.route == "V1_PRIMARY" for product in selected_route_products + ), + "product_count": len(products), + "total_operations": sum(int(item["opening_total_operations"]) for item in plans.values()), + "pacing_floor_seconds": round(pacing_floor, 3), + "native_basis_deferral_seconds": round(native_basis_deferral, 3), + "tail_timeout_seconds": round(tail_timeout, 3), + # The last quota-admitted call still owns its declared typed timeout. + "minimum_deadline_seconds": float( + ceil(pacing_floor + native_basis_deferral + tail_timeout) + ), + "consumers": plans, + } + + +def _effective_c2_opening_timeout_seconds( + operation_plan: Mapping[str, object], + requested_seconds: float | None, +) -> float: + """Use the exact derived budget unless an operator declares a larger one.""" + + minimum = float(operation_plan["minimum_deadline_seconds"]) + if requested_seconds is None: + return minimum + if requested_seconds < 1.0: + raise C2OpeningCapacityError( + "OPENING_TIMEOUT_NOT_POSITIVE", + { + "requested_seconds": requested_seconds, + }, + ) + if requested_seconds < minimum: + raise C2OpeningCapacityError( + "OPENING_TIMEOUT_BELOW_DERIVED_MINIMUM", + { + "requested_seconds": requested_seconds, + "minimum_deadline_seconds": minimum, + "operation_plan": dict(operation_plan), + }, + ) + return requested_seconds + + async def _certify_references( products: tuple[AcceptanceProduct, ...], *, @@ -550,11 +1092,21 @@ async def read_replica(client, *, label: str): ) latency_ms = (time.perf_counter() - started) * 1_000 observed_at_ns = time.time_ns() - hashes = tuple( - reference_evidence(item, result, observed_at_ns=observed_at_ns) - for item, result in zip(batch, response.results, strict=True) - ) - for item, result, content_hash in zip(batch, response.results, hashes, strict=True): + values_for_batch = [] + for item, result in zip(batch, response.results, strict=True): + try: + content_hash = reference_evidence( + item, result, observed_at_ns=observed_at_ns, + ) + quality = reference_quality( + item, result, observed_at_ns=observed_at_ns, + ) + except ValueError as error: + raise C2ReferenceProductError( + item, replica=label, error=error, + ) from error + values_for_batch.append((item, content_hash, quality)) + for item, content_hash, quality in values_for_batch: if item.identity in values: raise AssertionError("Phase 10.5 reference batch duplicated a product") values[item.identity] = ( @@ -562,7 +1114,7 @@ async def read_replica(client, *, label: str): latency_ms, attempts, deferred_ms, - reference_quality(item, result, observed_at_ns=observed_at_ns), + quality, ) finally: await client.close() @@ -606,6 +1158,11 @@ async def read_replica(client, *, label: str): "primary": primary_values[product.identity][4], "secondary": secondary_values[product.identity][4], }, + "quality_sha256": { + "primary": _evidence_sha256(primary_values[product.identity][4]), + "secondary": _evidence_sha256(secondary_values[product.identity][4]), + }, + "timing_policy": _timing_policy(product), } for product in reference_products ] @@ -773,15 +1330,75 @@ def _closing_requirement(product: AcceptanceProduct): ) -def _closing_status_representatives( +async def _closing_batch_problem_evidence( + client, products: tuple[AcceptanceProduct, ...], -) -> tuple[AcceptanceProduct, ...]: - """Keep transport-failure evidence bounded to one identity per feed.""" + *, + error: Exception, + status_observations: list[dict[str, object]], +) -> list[dict[str, object]]: + """Locate typed item failures without weakening execution batch semantics. + + A public execution-grade batch must use ``require_all=True``. The SDK then + deliberately raises a generic ``PARTIAL_RESULT`` rather than returning a + partially usable response. Only after that fail-closed batch result do we + bisect it with the same strict batch API, then issue a single public + ``warmup`` for each failing leaf to retain its server typed code. The all- + pass path stays batched; this diagnostic never turns a partial response into + usable execution data or records its payload. + """ - by_feed: dict[str, AcceptanceProduct] = {} - for product in products: - by_feed.setdefault(product.feed.value, product) - return tuple(by_feed[feed] for feed in sorted(by_feed)) + if not isinstance(error, DataLayerError) or error.code != "PARTIAL_RESULT": + return [] + quality_by_identity = { + tuple(item["product_identity"]): item.get("quality_sha256") + for item in status_observations + if isinstance(item.get("product_identity"), list) + } + + async def failed_leaves( + group: tuple[AcceptanceProduct, ...], + ) -> tuple[AcceptanceProduct, ...]: + if len(group) == 1: + return group + midpoint = len(group) // 2 + children = (group[:midpoint], group[midpoint:]) + leaves: list[AcceptanceProduct] = [] + for child in children: + try: + await client.warmup_batch( + tuple(_closing_requirement(product) for product in child), + require_all=True, + ) + except (httpx.HTTPError, TimeoutError, DataLayerError, ValueError): + leaves.extend(await failed_leaves(child)) + return tuple(leaves) + + leaves = await failed_leaves(products) + evidence: list[dict[str, object]] = [] + for product in leaves: + try: + await client.warmup(_closing_requirement(product)) + except DataLayerError as leaf_error: + code = leaf_error.code + retryable = leaf_error.retryable + detail = leaf_error.detail + except (httpx.HTTPError, TimeoutError, ValueError) as leaf_error: + code = f"DIAGNOSTIC_{type(leaf_error).__name__.upper()}" + retryable = False + detail = str(leaf_error) + else: + code = "BATCH_FAILURE_NOT_REPRODUCED" + retryable = True + detail = "strict batch failure was not reproduced by its isolated V2 read" + evidence.append({ + **product.evidence(), + "problem_code": code, + "retryable": retryable, + "problem_detail_sha256": hashlib.sha256(detail.encode()).hexdigest(), + "quality_sha256": quality_by_identity.get(product.identity), + }) + return evidence async def _closing_failure_status_observations( @@ -794,7 +1411,7 @@ async def _closing_failure_status_observations( timeout = min(5.0, timeout_seconds) observations: list[dict[str, object]] = [] - for product in _closing_status_representatives(products): + for product in products: try: status = await asyncio.wait_for( client.feed_status(_closing_requirement(product)), @@ -803,17 +1420,22 @@ async def _closing_failure_status_observations( except Exception as error: # Diagnostic must not hide the primary failure. observations.append({ **product.evidence(), + "product_identity": list(product.identity), "status_transport_error": type(error).__name__, + "quality_sha256": None, }) else: + quality = compact_feed_status(status) observations.append({ **product.evidence(), - "quality": compact_feed_status(status), + "product_identity": list(product.identity), + "quality": quality, + "quality_sha256": _evidence_sha256(quality), }) return observations -async def _closing_batch_revalidation( +async def _closing_history_batch_revalidation( products: tuple[AcceptanceProduct, ...], *, identity, @@ -825,7 +1447,7 @@ async def _closing_batch_revalidation( max_batch_items: int, client_factory, ) -> list[dict[str, object]]: - """Re-read every durable/pass-through product through both V2 replicas. + """Re-read non-L2 durable/pass-through products through both V2 replicas. C2's opening proof already establishes signed cursor/reconnect per product. Closing needs a strict current view for every route, not a second identical @@ -835,6 +1457,8 @@ async def _closing_batch_revalidation( if not products: return [] + if any(_is_lossless_l2_product(product) for product in products): + raise ValueError("Phase 10.5 historical closing batch contains lossless L2") if not 1 <= max_batch_items <= 100: raise ValueError("Phase 10.5 C2 batch size exceeds the V2 contract") @@ -853,22 +1477,42 @@ async def read_replica(base_url: str, *, label: str): started = time.perf_counter() try: response = await client.warmup_batch(requirements, require_all=True) - except (httpx.HTTPError, TimeoutError, DataLayerError) as error: + except (httpx.HTTPError, TimeoutError, DataLayerError, ValueError) as error: + summary_getter = getattr( + getattr(client, "query_transport", None), + "last_warmup_batch_summary", + None, + ) + transport_batch_summary = ( + summary_getter() if callable(summary_getter) else None + ) status_observations = await _closing_failure_status_observations( client, batch, timeout_seconds=timeout_seconds, ) + batch_item_problems = await _closing_batch_problem_evidence( + client, + batch, + error=error, + status_observations=status_observations, + ) raise C2ClosingBatchError( consumer_id=batch[0].consumer_id, replica=label, products=batch, error=error, status_observations=status_observations, + batch_item_problems=batch_item_problems, + transport_batch_summary=transport_batch_summary, ) from error latency_ms = (time.perf_counter() - started) * 1_000 - if response.partial or len(response.results) != len(batch): + if len(response.results) != len(batch): raise AssertionError("Phase 10.5 closing V2 batch cardinality differs") + if response.partial: + raise AssertionError( + "Phase 10.5 public strict warmup batch returned partial" + ) observed_at_ns = time.time_ns() for product, item in zip(batch, response.results, strict=True): if item.data is None or not item.data.data: @@ -930,6 +1574,11 @@ async def read_replica(base_url: str, *, label: str): "primary": primary["quality"], "secondary": secondary["quality"], }, + "quality_sha256": { + "primary": _evidence_sha256(primary["quality"]), + "secondary": _evidence_sha256(secondary["quality"]), + }, + "timing_policy": _timing_policy(product), "closing_read": "BATCH_V2_PRIMARY", } if bar_alignment is not None: @@ -938,6 +1587,229 @@ async def read_replica(base_url: str, *, label: str): return evidence +def _is_lossless_l2_product(product: AcceptanceProduct) -> bool: + """Keep L2 on snapshot/status plus stream-replay, never history batching.""" + + feed = getattr(product.feed, "value", product.feed) + return str(feed) in {item.value for item in _LOSSLESS_L2_FEEDS} + + +def _validate_l2_status( + product: AcceptanceProduct, + status_evidence: Mapping[str, object], +) -> None: + """Require the same exact identity and quality fences before snapshot use.""" + + expected_feed = str(getattr(product.feed, "value", product.feed)) + if ( + status_evidence.get("instrument_uid") != product.instrument_uid + or status_evidence.get("feed") != expected_feed + ): + raise ValueError("Phase 10.5 L2 status identity differs from demand") + quality = status_evidence.get("quality") + if not isinstance(quality, Mapping): + raise ValueError("Phase 10.5 L2 status quality is unavailable") + if ( + quality.get("state") != "LIVE" + or quality.get("complete") is not True + or quality.get("gap_open") is not False + or quality.get("policy_id") != product.source_policy_id + ): + raise ValueError("Phase 10.5 L2 status is not live, complete and gap-free") + requirement = product.requirement + if requirement.max_session_liveness_ms is not None and ( + quality.get("provider_session_state") != "LIVE" + or not isinstance(quality.get("provider_session_liveness_ms"), int) + or int(quality["provider_session_liveness_ms"]) + > requirement.max_session_liveness_ms + ): + raise ValueError("Phase 10.5 L2 status provider session differs from demand") + + +async def _closing_l2_revalidation( + products: tuple[AcceptanceProduct, ...], + *, + identity, + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + client_factory, +) -> list[dict[str, object]]: + """Read lossless L2 via status plus snapshot, not generic history batches. + + A snapshot is the bounded bootstrap state; `BOOK_DELTA` continuity and + cursor replay are proved in the C2 opening stream path. This read-only + closing/preflight routine validates both public Query replicas without + opening an extra stream or treating one latest delta as a warmup history. + """ + + if not products: + return [] + if any(not _is_lossless_l2_product(product) for product in products): + raise ValueError("Phase 10.5 L2 closing read contains a non-L2 product") + + async def read_replica(base_url: str, *, label: str): + client = client_factory( + identity, + base_url=base_url, + grpc_target=grpc_target, + cursor_path=state_dir / f"closing-l2-{label}.json", + timeout_seconds=timeout_seconds, + ) + values: dict[tuple[str, str, str, str, str], dict[str, object]] = {} + try: + for product in products: + requirement = _closing_requirement(product) + status_evidence: dict[str, object] | None = None + try: + status = await client.feed_status(requirement) + status_evidence = compact_feed_status(status) + _validate_l2_status(product, status_evidence) + except (httpx.HTTPError, TimeoutError, DataLayerError, ValueError) as error: + raise C2ClosingL2Error( + product=product, + replica=label, + operation="FEED_STATUS", + error=error, + status_evidence=status_evidence, + ) from error + started = time.perf_counter() + try: + response = await client.snapshot(requirement) + view = response.data + validate_product_view(product, view) + except (httpx.HTTPError, TimeoutError, DataLayerError, ValueError) as error: + raise C2ClosingL2Error( + product=product, + replica=label, + operation="SNAPSHOT", + error=error, + status_evidence=status_evidence, + ) from error + if product.identity in values: + raise AssertionError("Phase 10.5 L2 closing read duplicated a product") + values[product.identity] = { + "view": view, + "status": status_evidence, + "latency_ms": (time.perf_counter() - started) * 1_000, + "quality": compact_view_quality(view, observed_at_ns=time.time_ns()), + } + finally: + await client.close() + if len(values) != len(products): + raise AssertionError(f"Phase 10.5 {label} L2 closing read lost a product") + return values + + primary_values, secondary_values = await asyncio.gather( + read_replica(primary_url, label="primary"), + read_replica(secondary_url, label="secondary"), + ) + evidence: list[dict[str, object]] = [] + for product in products: + primary = primary_values[product.identity] + secondary = secondary_values[product.identity] + try: + primary_hash, secondary_hash = validate_replica_views( + product, primary["view"], secondary["view"] + ) + except ValueError as error: + raise C2ClosingL2Error( + product=product, + replica="both", + operation="REPLICA_PARITY", + error=error, + status_evidence={ + "primary": primary["status"], + "secondary": secondary["status"], + }, + ) from error + evidence.append({ + **product.evidence(), + "primary_content_sha256": primary_hash, + "secondary_content_sha256": secondary_hash, + "primary_latency_ms": round(float(primary["latency_ms"]), 3), + "secondary_latency_ms": round(float(secondary["latency_ms"]), 3), + "release_quality": { + "primary": primary["quality"], + "secondary": secondary["quality"], + }, + "status_quality": { + "primary": primary["status"], + "secondary": secondary["status"], + }, + "quality_sha256": { + "primary": _evidence_sha256(primary["quality"]), + "secondary": _evidence_sha256(secondary["quality"]), + }, + "timing_policy": _timing_policy(product), + "closing_read": "L2_STATUS_SNAPSHOT", + }) + return evidence + + +async def _closing_batch_revalidation( + products: tuple[AcceptanceProduct, ...], + *, + identity, + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + max_batch_items: int, + client_factory, +) -> list[dict[str, object]]: + """Dispatch every closing product through its declared domain transport.""" + + if not products: + return [] + history_products = tuple( + product for product in products if not _is_lossless_l2_product(product) + ) + l2_products = tuple( + product for product in products if _is_lossless_l2_product(product) + ) + history_task = asyncio.create_task(_closing_history_batch_revalidation( + history_products, + identity=identity, + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / "history", + timeout_seconds=timeout_seconds, + max_batch_items=max_batch_items, + client_factory=client_factory, + )) + l2_task = asyncio.create_task(_closing_l2_revalidation( + l2_products, + identity=identity, + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / "l2", + timeout_seconds=timeout_seconds, + client_factory=client_factory, + )) + history_evidence, l2_evidence = await _gather_or_cancel((history_task, l2_task)) + by_identity = { + tuple(item["product_identity"]) + if isinstance(item.get("product_identity"), list) + else ( + str(item["consumer_id"]), + str(item["instrument_uid"]), + str(item["feed"]), + str(item.get("interval") or ""), + str(item["source_policy_id"]), + ): item + for item in (*history_evidence, *l2_evidence) + } + if len(by_identity) != len(products): + raise AssertionError("Phase 10.5 closing read lost or duplicated a product") + return [by_identity[product.identity] for product in products] + + async def _reference_batch_for_c2( client, batch: tuple[ReferenceAcceptanceProduct, ...], @@ -1021,6 +1893,442 @@ async def _closing_revalidate_consumer( return [*stream_results, *reference_results] +def _batch_identity_sha256(products: tuple[AcceptanceProduct, ...]) -> str: + """Fingerprint an exact batch without persisting product data or payloads.""" + + return hashlib.sha256( + json.dumps( + [item.identity for item in products], + sort_keys=True, + separators=(",", ":"), + ).encode() + ).hexdigest() + + +def _manifest_maximum_bar_batch( + products: tuple[AcceptanceProduct, ...], + *, + max_batch_items: int, +) -> tuple[AcceptanceProduct, ...]: + """Return the exact first manifest-maximum BAR partition.""" + + if not 1 <= max_batch_items <= 100: + raise ValueError("Phase 10.5 batch-shape maximum exceeds the V2 contract") + bar_batches = tuple( + batch + for batch in _closing_batches(products, max_batch_items) + if batch and batch[0].feed is FeedType.BAR + ) + exact = next((batch for batch in bar_batches if len(batch) == max_batch_items), None) + if exact is None: + raise ValueError("Phase 10.5 batch-shape matrix lacks a manifest-maximum BAR partition") + return exact + + +def _largest_bar_batch( + products: tuple[AcceptanceProduct, ...], + *, + max_batch_items: int, +) -> tuple[AcceptanceProduct, ...]: + """Return the largest declared BAR batch for a collocated consumer lane.""" + + if not 1 <= max_batch_items <= 100: + raise ValueError("Phase 10.5 collocation batch maximum exceeds the V2 contract") + batches = tuple( + batch + for batch in _closing_batches(products, max_batch_items) + if batch and batch[0].feed is FeedType.BAR + ) + if not batches: + raise ValueError("Phase 10.5 collocation has no BAR partition") + return max(batches, key=len) + + +def _strict_bar_batch_windows( + products: tuple[AcceptanceProduct, ...], + *, + max_batch_items: int, +) -> tuple[tuple[int, int, tuple[AcceptanceProduct, ...]], ...]: + """Select boundary windows plus the exact maximum BAR partition. + + The existing failed receipt already bisected every member of the selected + maximum batch. This matrix diagnoses batch cardinality and queue position, + so smaller shapes test deterministic first/last windows while the maximum + shape always exercises every item of the exact manifest partition. + """ + + exact = _manifest_maximum_bar_batch( + products, max_batch_items=max_batch_items + ) + windows: list[tuple[int, int, tuple[AcceptanceProduct, ...]]] = [] + for shape in (*_STRICT_BAR_BATCH_SHAPES, max_batch_items): + if shape > len(exact): + continue + candidates = (exact[:shape],) if shape == len(exact) else (exact[:shape], exact[-shape:]) + seen: set[str] = set() + for candidate in candidates: + digest = _batch_identity_sha256(candidate) + if digest in seen: + continue + seen.add(digest) + windows.append((shape, len(seen) - 1, candidate)) + if not windows or windows[-1][0] != max_batch_items: + raise AssertionError("Phase 10.5 batch-shape matrix omitted the maximum BAR batch") + return tuple(windows) + + +def _batch_shape_observation( + products: tuple[AcceptanceProduct, ...], + observations: list[dict[str, object]], + *, + batch_shape: int, + window_index: int, +) -> dict[str, object]: + """Reduce a validated dual-replica batch to bounded diagnostic evidence.""" + + expected = {item.identity for item in products} + actual = { + ( + str(item.get("consumer_id")), + str(item.get("instrument_uid")), + str(item.get("feed")), + str(item.get("interval") or ""), + str(item.get("source_policy_id")), + ) + for item in observations + } + if actual != expected or len(observations) != len(expected): + raise AssertionError("Phase 10.5 batch-shape result differs from its exact manifest partition") + primary_latency = sorted(float(item["primary_latency_ms"]) for item in observations) + secondary_latency = sorted(float(item["secondary_latency_ms"]) for item in observations) + quality = [ + { + "identity": item.identity, + "primary": observation["quality_sha256"]["primary"], + "secondary": observation["quality_sha256"]["secondary"], + "primary_content": observation["primary_content_sha256"], + "secondary_content": observation["secondary_content_sha256"], + } + for item, observation in zip(products, observations, strict=True) + ] + return { + "batch_shape": batch_shape, + "window_index": window_index, + "batch_size": len(products), + "batch_identity_sha256": _batch_identity_sha256(products), + "primary_latency_ms": { + "p50": round(primary_latency[max(0, ceil(len(primary_latency) * 0.50) - 1)], 3), + "p95": round(primary_latency[max(0, ceil(len(primary_latency) * 0.95) - 1)], 3), + }, + "secondary_latency_ms": { + "p50": round(secondary_latency[max(0, ceil(len(secondary_latency) * 0.50) - 1)], 3), + "p95": round(secondary_latency[max(0, ceil(len(secondary_latency) * 0.95) - 1)], 3), + }, + "quality_content_sha256": hashlib.sha256( + json.dumps(quality, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest(), + "payload_recorded": False, + } + + +async def _strict_bar_batch_shape_matrix( + products: tuple[AcceptanceProduct, ...], + *, + identity, + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + max_batch_items: int, + client_factory, + revalidate=None, +) -> list[dict[str, object]]: + """Exercise exact strict BAR shapes without stream/fallback/provider reads.""" + + runner = _closing_batch_revalidation if revalidate is None else revalidate + evidence: list[dict[str, object]] = [] + for shape, window_index, window in _strict_bar_batch_windows( + products, max_batch_items=max_batch_items + ): + try: + observations = await runner( + window, + identity=identity, + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / f"shape-{shape}-{window_index}", + timeout_seconds=timeout_seconds, + max_batch_items=shape, + client_factory=client_factory, + ) + except Exception as error: + raise C2BatchShapeError( + error, + stage="ISOLATED", + batch_shape=shape, + window_index=window_index, + products=window, + ) from error + evidence.append(_batch_shape_observation( + window, + observations, + batch_shape=shape, + window_index=window_index, + )) + return evidence + + +async def _strict_bar_collocation_matrix( + scope, + release: StableReleaseRoutePlan, + *, + consumer_ids: tuple[str, ...], + identities: Mapping[str, object], + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + client_factories: Mapping[str, Callable], + preferred_consumer_id: str, + revalidate=None, +) -> dict[str, object]: + """Measure deterministic local-lane saturation without hiding its threshold.""" + + runner = _closing_batch_revalidation if revalidate is None else revalidate + routes = {item.consumer_id: item for item in release.consumers} + selected: list[tuple[str, int, tuple[AcceptanceProduct, ...]]] = [] + not_applicable: list[dict[str, object]] = [] + for consumer_id in consumer_ids: + route = routes.get(consumer_id) + if route is None: + raise ValueError("Phase 10.5 collocation route is unavailable") + stream_products = tuple( + item for item in scope.products + if item.consumer_id == consumer_id and item.delivery is not DeliveryClass.ON_DEMAND + ) + if not any(item.feed is FeedType.BAR for item in stream_products): + not_applicable.append({ + "consumer_id": consumer_id, + "status": "NOT_APPLICABLE_NO_DURABLE_BAR", + "read_actions": 0, + "payload_recorded": False, + }) + continue + batch = _largest_bar_batch( + stream_products, + max_batch_items=route.manifest.quotas.max_batch_items, + ) + selected.append((consumer_id, len(batch), batch)) + + selected.sort(key=lambda item: (item[0] != preferred_consumer_id, item[0])) + if selected and selected[0][0] != preferred_consumer_id: + raise ValueError("Phase 10.5 preferred BAR consumer is not entitled") + + async def run_one(consumer_id: str, shape: int, batch: tuple[AcceptanceProduct, ...]): + try: + observations = await runner( + batch, + identity=identities[consumer_id], + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / consumer_id.replace(".", "-"), + timeout_seconds=timeout_seconds, + max_batch_items=shape, + client_factory=client_factories[consumer_id], + ) + except Exception as error: + raise C2BatchShapeError( + error, + stage="COLLOCATED", + batch_shape=shape, + window_index=0, + products=batch, + ) from error + return _batch_shape_observation( + batch, + observations, + batch_shape=shape, + window_index=0, + ) + + waves: list[dict[str, object]] = [] + for parallel_lanes in range(1, len(selected) + 1): + lane = tuple(selected[:parallel_lanes]) + try: + executed = list(await _gather_or_cancel(tuple( + asyncio.create_task(run_one(consumer_id, shape, batch)) + for consumer_id, shape, batch in lane + ))) + except C2BatchShapeError as error: + error.evidence["completed_collocation_waves"] = waves + error.evidence["failed_parallel_lanes"] = parallel_lanes + raise + waves.append({ + "parallel_lanes": parallel_lanes, + "consumer_ids": [consumer_id for consumer_id, _shape, _batch in lane], + "observations": executed, + "payload_recorded": False, + }) + return { + "waves": waves, + "not_applicable": not_applicable, + "payload_recorded": False, + } + + +async def _read_plane_preflight( + scope, + release: StableReleaseRoutePlan, + *, + consumer_ids: tuple[str, ...], + identities: Mapping[str, object], + primary_url: str, + secondary_url: str, + grpc_target: str, + state_dir: Path, + timeout_seconds: float, + deadline_monotonic: float, + concurrency: int, + client_factories: Mapping[str, Callable], +) -> list[dict[str, object]]: + """Read every V2 route twice without opening a stream or fallback path. + + This is deliberately a release-candidate debugger, not a substitute for + C2: it exercises the same sealed SDK requirements and both query replicas, + but omits cursor/reconnect, fallback and the 300-second observation. It + catches materialization/quality/replica faults before a full C2 spends its + manifest-derived opening window. + """ + + release_consumers = {item.consumer_id: item for item in release.consumers} + reference_semaphore = asyncio.Semaphore(_reference_batch_concurrency(concurrency)) + native_basis_semaphore = asyncio.Semaphore(1) + + async def revalidate_consumer(consumer_id: str) -> list[dict[str, object]]: + products = tuple(item for item in scope.products if item.consumer_id == consumer_id) + route = release_consumers.get(consumer_id) + if not products or route is None: + raise ValueError("Phase 10.5 pre-C2 consumer route is unavailable") + return await _closing_revalidate_consumer( + consumer_id, + products, + identity=identities[consumer_id], + primary_url=primary_url, + secondary_url=secondary_url, + grpc_target=grpc_target, + state_dir=state_dir / consumer_id.replace(".", "-"), + timeout_seconds=timeout_seconds, + deadline_monotonic=deadline_monotonic, + max_batch_items=route.manifest.quotas.max_batch_items, + reference_semaphore=reference_semaphore, + native_basis_semaphore=native_basis_semaphore, + client_factory=client_factories[consumer_id], + ) + + groups = await _gather_or_cancel(tuple( + asyncio.create_task(revalidate_consumer(consumer_id)) + for consumer_id in consumer_ids + )) + return [item for group in groups for item in group] + + +def _read_plane_preflight_receipt( + *, + scope, + release: StableReleaseRoutePlan, + consumer_ids: tuple[str, ...], + observations: list[dict[str, object]], + authority_revision: object, + elapsed_seconds: float, + quota_window_wait_seconds: float, + pacers: Mapping[str, _C2ConsumerRequestPacer], +) -> dict[str, object]: + """Return compact proof that the fast matrix covered the exact route set.""" + + expected = { + (item.consumer_id, item.instrument_uid, item.feed.value, item.interval or "", + item.source_policy_id) + for item in scope.products + } + actual = { + ( + str(item.get("consumer_id")), + str(item.get("instrument_uid")), + str(item.get("feed")), + str(item.get("interval") or ""), + str(item.get("source_policy_id")), + ) + for item in observations + } + if actual != expected or len(observations) != len(expected): + raise AssertionError("Phase 10.5 pre-C2 read-plane scope differs from release routes") + timing_classes = Counter() + for item in observations: + timing_policy = item.get("timing_policy") + if not isinstance(timing_policy, Mapping): + raise AssertionError("Phase 10.5 pre-C2 timing policy is missing") + semantic_class = timing_policy.get("semantic_class") + if not isinstance(semantic_class, str) or not semantic_class: + raise AssertionError("Phase 10.5 pre-C2 timing semantic class is invalid") + timing_classes[semantic_class] += 1 + feed_counts = Counter(str(item[2]) for item in actual) + consumer_counts = Counter(str(item[0]) for item in actual) + primary_latency = sorted( + float(item["primary_latency_ms"]) + for item in observations + if isinstance(item.get("primary_latency_ms"), (int, float)) + ) + secondary_latency = sorted( + float(item["secondary_latency_ms"]) + for item in observations + if isinstance(item.get("secondary_latency_ms"), (int, float)) + ) + + def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + index = min(len(values) - 1, max(0, ceil(len(values) * fraction) - 1)) + return round(values[index], 3) + + return { + "schema": "qdl.phase105.v2-read-plane-preflight.v1", + "status": "PASS_READ_PLANE_PREFLIGHT", + "mode": "READ_PLANE_ONLY_NO_STREAM_NO_FALLBACK", + "release_route_plan_sha256": release.digest, + "authority_revision": authority_revision, + "scope_sha256": scope.sha256, + "product_count": len(observations), + "consumer_counts": dict(sorted(consumer_counts.items())), + "feed_counts": dict(sorted(feed_counts.items())), + "timing_class_counts": dict(sorted(timing_classes.items())), + "replica_read_count": 2, + "primary_batch_latency_ms": { + "p50": percentile(primary_latency, 0.50), + "p95": percentile(primary_latency, 0.95), + "p99": percentile(primary_latency, 0.99), + }, + "secondary_batch_latency_ms": { + "p50": percentile(secondary_latency, 0.50), + "p95": percentile(secondary_latency, 0.95), + "p99": percentile(secondary_latency, 0.99), + }, + "quota_window_wait_seconds": round(quota_window_wait_seconds, 3), + "quota_budget": { + consumer_id: pacer.evidence() + for consumer_id, pacer in sorted(pacers.items()) + }, + "provider_connections": 0, + "order_actions": 0, + "cursor_directory_removed": True, + "payload_recorded": False, + "elapsed_seconds": round(elapsed_seconds, 3), + } + + async def _run_consumer_groups( consumer_ids: tuple[str, ...], run_group, @@ -1051,6 +2359,11 @@ async def run(args: argparse.Namespace) -> dict[str, object]: authority = _authority(args.authority_record) consumer_ids = _consumer_ids(args) scope, release = _scope(args, consumer_ids) + timing_profiles = { + product.identity: _timing_policy(product) for product in scope.products + } + if len(timing_profiles) != len(scope.products): + raise AssertionError("Phase 10.5 timing policy duplicated a release product") files = _identity_files_for_consumers(args, consumer_ids) v1_base_url = _v1_base_url(args.v1_base_url) grpc_target = _c2_grpc_targets(args.grpc_target) @@ -1070,6 +2383,30 @@ async def run(args: argparse.Namespace) -> dict[str, object]: products=scope.products, consumer_ids=consumer_ids, ) + opening_operation_plan = _build_c2_opening_operation_plan( + scope.products, + release, + probes, + consumer_ids, + generic_timeout_seconds=args.timeout_seconds, + ) + opening_consumer_plans = opening_operation_plan["consumers"] + if not isinstance(opening_consumer_plans, dict): + raise AssertionError("C2 opening operation plan has invalid consumers") + if any( + args.concurrency > int(item["max_streams"]) + for item in opening_consumer_plans.values() + ): + raise C2OpeningCapacityError( + "OPENING_CONCURRENCY_EXCEEDS_MANIFEST_STREAM_QUOTA", + { + "requested_concurrency": args.concurrency, + "operation_plan": opening_operation_plan, + }, + ) + opening_timeout_seconds = _effective_c2_opening_timeout_seconds( + opening_operation_plan, args.opening_timeout_seconds + ) products_by_identity = { (item.consumer_id, requirement_key(item.requirement)): item for item in scope.products } @@ -1106,9 +2443,127 @@ async def run(args: argparse.Namespace) -> dict[str, object]: for consumer_id, pacer in pacers.items() } release_consumers = {item.consumer_id: item for item in release.consumers} + + if args.batch_shape_matrix: + consumer_id = args.batch_shape_consumer_id + if consumer_id not in consumer_ids: + raise ValueError("Phase 10.5 batch-shape consumer is outside the selected scope") + route = release_consumers.get(consumer_id) + if route is None: + raise ValueError("Phase 10.5 batch-shape consumer route is unavailable") + selected_products = tuple( + item for item in scope.products + if item.consumer_id == consumer_id and item.delivery is not DeliveryClass.ON_DEMAND + ) + if not selected_products: + raise ValueError("Phase 10.5 batch-shape consumer has no durable products") + quota_window_wait_seconds = await _wait_for_clean_quota_windows(pacers) + matrix_started = time.monotonic() + try: + isolated = await _strict_bar_batch_shape_matrix( + selected_products, + identity=identities[consumer_id], + primary_url=args.primary_url, + secondary_url=args.secondary_url, + grpc_target=grpc_target, + state_dir=temporary / "strict-bar-batch" / "isolated", + timeout_seconds=args.timeout_seconds, + max_batch_items=route.manifest.quotas.max_batch_items, + client_factory=client_factories[consumer_id], + ) + try: + collocated = await _strict_bar_collocation_matrix( + scope, + release, + consumer_ids=consumer_ids, + identities=identities, + primary_url=args.primary_url, + secondary_url=args.secondary_url, + grpc_target=grpc_target, + state_dir=temporary / "strict-bar-batch" / "collocated", + timeout_seconds=args.timeout_seconds, + client_factories=client_factories, + preferred_consumer_id=consumer_id, + ) + except C2BatchShapeError as error: + error.evidence["isolated"] = isolated + raise + finally: + shutil.rmtree(temporary, ignore_errors=True) + exact_max = next( + item for item in isolated + if item["batch_shape"] == route.manifest.quotas.max_batch_items + ) + return { + "schema": "qdl.phase105.strict-bar-batch-shape.v1", + "status": "PASS_STRICT_BAR_BATCH_SHAPE", + "mode": "READ_PLANE_ONLY_NO_STREAM_NO_FALLBACK", + "release_route_plan_sha256": release.digest, + "authority_revision": authority.get("revision"), + "scope_sha256": scope.sha256, + "consumer_id": consumer_id, + "manifest_max_batch_items": route.manifest.quotas.max_batch_items, + "exact_maximum_batch_identity_sha256": exact_max["batch_identity_sha256"], + "isolated": isolated, + "collocated": collocated, + "quota_window_wait_seconds": round(quota_window_wait_seconds, 3), + "quota_budget": { + item_consumer_id: pacer.evidence() + for item_consumer_id, pacer in sorted(pacers.items()) + }, + "provider_connections": 0, + "order_actions": 0, + "cursor_directory_removed": True, + "payload_recorded": False, + "elapsed_seconds": round(time.monotonic() - matrix_started, 3), + } + + if args.read_plane_preflight: + # Use the same per-identity 75% quota guard as C2, but only for the + # batched current read plane. There is no cursor, stream, fallback or + # 300-second observation in this diagnostic mode. + quota_window_wait_seconds = await _wait_for_clean_quota_windows(pacers) + preflight_started = time.monotonic() + try: + observations = await asyncio.wait_for( + _read_plane_preflight( + scope, + release, + consumer_ids=consumer_ids, + identities=identities, + primary_url=args.primary_url, + secondary_url=args.secondary_url, + grpc_target=grpc_target, + state_dir=temporary / "read-plane-preflight", + timeout_seconds=args.timeout_seconds, + deadline_monotonic=preflight_started + opening_timeout_seconds, + concurrency=args.concurrency, + client_factories=client_factories, + ), + timeout=opening_timeout_seconds, + ) + finally: + shutil.rmtree(temporary, ignore_errors=True) + return _read_plane_preflight_receipt( + scope=scope, + release=release, + consumer_ids=consumer_ids, + observations=observations, + authority_revision=authority.get("revision"), + elapsed_seconds=time.monotonic() - preflight_started, + quota_window_wait_seconds=quota_window_wait_seconds, + pacers=pacers, + ) + quota_window_wait_seconds = await _wait_for_clean_quota_windows(pacers) started = time.monotonic() - opening_deadline = started + args.opening_timeout_seconds + opening_deadline = started + opening_timeout_seconds + print(json.dumps({ + "stage": "C2_OPENING_OPERATION_PLAN", + "operator_timeout_seconds": args.opening_timeout_seconds, + "effective_timeout_seconds": opening_timeout_seconds, + "plan": opening_operation_plan, + }, sort_keys=True, separators=(",", ":")), file=sys.stderr, flush=True) async def certify(product: AcceptanceProduct) -> dict[str, object]: async with product_semaphore: @@ -1121,11 +2576,13 @@ async def certify(product: AcceptanceProduct) -> dict[str, object]: grpc_target=grpc_target, state_dir=temporary, timeout_seconds=args.timeout_seconds, - stream_open_timeout_seconds=args.opening_timeout_seconds, + stream_open_timeout_seconds=opening_timeout_seconds, client_factory=client_factories[product.consumer_id], ) except C2StatusEvidenceError as error: raise C2ProductAcceptanceError(product, error) from error + except C2OpeningCapacityError: + raise except asyncio.CancelledError: print(json.dumps({"stage": "C2_ACTIVE_PRODUCT_CANCELLED", "identity": product.identity}), file=sys.stderr, flush=True) @@ -1242,9 +2699,22 @@ async def revalidate_consumer(consumer_id: str) -> list[dict[str, object]]: # full proof is complete. Closing rechecks every route with batch V2 # reads; it deliberately does not create a second stream storm. opening_started = time.monotonic() - initial_results, initial_fallback_details = await asyncio.wait_for( - certify_ordered(), timeout=args.opening_timeout_seconds - ) + try: + initial_results, initial_fallback_details = await asyncio.wait_for( + certify_ordered(), timeout=opening_timeout_seconds + ) + except TimeoutError as error: + raise C2OpeningCapacityError( + "OPENING_DEADLINE_EXCEEDED", + { + "effective_timeout_seconds": opening_timeout_seconds, + "operation_plan": opening_operation_plan, + "quota_budget": { + consumer_id: pacer.evidence() + for consumer_id, pacer in sorted(pacers.items()) + }, + }, + ) from error opening_seconds = time.monotonic() - opening_started print(json.dumps({"stage": "C2_OPENING_PASS", "products": len(initial_results), "seconds": round(opening_seconds, 3)}), file=sys.stderr, flush=True) @@ -1268,7 +2738,7 @@ async def revalidate_consumer(consumer_id: str) -> list[dict[str, object]]: max_rss = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss rss_bytes = int(max_rss) * 1024 if elapsed_seconds > ( - args.opening_timeout_seconds + opening_timeout_seconds + args.observation_seconds + args.closing_timeout_seconds ): @@ -1290,6 +2760,7 @@ async def revalidate_consumer(consumer_id: str) -> list[dict[str, object]]: if set(initial_by_identity) != set(closing_by_identity) or len(initial_by_identity) != len(scope.products): raise AssertionError("Phase 10.5 identity scope changed during the observation window") for identity_key, item in initial_by_identity.items(): + item["timing_policy"] = timing_profiles[identity_key] item["closing_v2_read"] = closing_by_identity[identity_key] route_summary = _route_summary(release, scope.products) return { @@ -1324,6 +2795,9 @@ async def revalidate_consumer(consumer_id: str) -> list[dict[str, object]]: "observation_seconds_requested": args.observation_seconds, "observation_seconds_actual": round(observation_seconds, 3), "opening_product_count": len(initial_results), + "opening_operation_plan": opening_operation_plan, + "opening_timeout_seconds_operator": args.opening_timeout_seconds, + "opening_timeout_seconds_effective": opening_timeout_seconds, "closing_product_count": len(closing_results), "opening_seconds_actual": round(opening_seconds, 3), "closing_seconds_actual": round(closing_seconds, 3), @@ -1373,8 +2847,33 @@ def parser() -> argparse.ArgumentParser: value.add_argument("--concurrency", type=int, default=4) value.add_argument("--observation-seconds", type=float, default=300.0) value.add_argument( - "--opening-timeout-seconds", type=float, default=_C2_OPENING_TIMEOUT_SECONDS, - help="Bound for the full quota-paced opening proof before observation starts.", + "--read-plane-preflight", + action="store_true", + help=( + "Batch-read every selected V2 route through both replicas before a " + "full C2; it never opens streams, drills V1 fallback or observes data." + ), + ) + value.add_argument( + "--batch-shape-matrix", + action="store_true", + help=( + "Run the bounded strict local-BAR batch-shape/collocation matrix " + "before the full all-scope read-plane preflight." + ), + ) + value.add_argument( + "--batch-shape-consumer-id", + choices=tuple(IDENTITY_PREFIXES), + default="alpha.okx.paper.stable", + help="Governed consumer whose manifest-maximum BAR batch is diagnosed.", + ) + value.add_argument( + "--opening-timeout-seconds", type=float, + help=( + "Optional operator timeout at or above the manifest-derived opening " + "budget. When omitted, C2 uses that exact derived deadline." + ), ) value.add_argument( "--closing-timeout-seconds", type=float, @@ -1392,13 +2891,30 @@ def main() -> int: raise SystemExit("--concurrency must be between 1 and 8") if not 30.0 <= args.observation_seconds <= 300.0: raise SystemExit("--observation-seconds must be between 30 and 300") - if not 60.0 <= args.opening_timeout_seconds <= 1_800.0: - raise SystemExit("--opening-timeout-seconds must be between 60 and 1800") + if args.batch_shape_matrix and args.read_plane_preflight: + raise SystemExit("--batch-shape-matrix and --read-plane-preflight are exclusive") + if args.opening_timeout_seconds is not None and args.opening_timeout_seconds < 1.0: + raise SystemExit("--opening-timeout-seconds must be positive") if not 30.0 <= args.closing_timeout_seconds <= 300.0: raise SystemExit("--closing-timeout-seconds must be between 30 and 300") try: result = asyncio.run(run(args)) - except (C2ProductAcceptanceError, C2ClosingBatchError) as error: + except C2OpeningCapacityError as error: + print(json.dumps({ + "schema": "qdl.phase105.v2-identity-acceptance.v1", + "status": "FAIL_OPENING_CAPACITY", + "failure": error.evidence, + "order_actions": 0, + "payload_recorded": False, + }, sort_keys=True, separators=(",", ":"))) + return 1 + except ( + C2ProductAcceptanceError, + C2ReferenceProductError, + C2ClosingBatchError, + C2ClosingL2Error, + C2BatchShapeError, + ) as error: print(json.dumps({ "schema": "qdl.phase105.v2-identity-acceptance.v1", "status": "FAIL_TYPED_STATUS", diff --git a/scripts/phase105_execution_l2_status_matrix.py b/scripts/phase105_execution_l2_status_matrix.py index 6ffc2c5..ff8016e 100644 --- a/scripts/phase105_execution_l2_status_matrix.py +++ b/scripts/phase105_execution_l2_status_matrix.py @@ -55,7 +55,7 @@ def execution_book_products( execution_demand: Path, trading_manifest: Path, ) -> tuple[AcceptanceProduct, ...]: - """Join the Trading System manifest to all derived execution L2 sources.""" + """Join the Trading System manifest to every execution L2 source pair.""" plan = execution_l2_materialization_plan( demand_path=execution_demand, @@ -68,7 +68,10 @@ def execution_book_products( acquisition=acquisition, expected_consumer_ids=frozenset({EXPECTED_CONSUMER_ID}), schema="qdl.phase105.consumer-acceptance-scope.v1", - requirement_filter=lambda item: item.feed is FeedType.BOOK_SNAPSHOT, + requirement_filter=lambda item: item.feed in { + FeedType.BOOK_SNAPSHOT, + FeedType.BOOK_DELTA, + }, ) source_by_binding = {item.binding_id: item.source_id for item in catalog.bindings} products = tuple( @@ -77,28 +80,37 @@ def execution_book_products( if item.binding_id is not None and source_by_binding.get(item.binding_id) in plan.source_ids ) - actual_ids = { - source_by_binding[item.binding_id] - for item in products - if item.binding_id is not None - } - if actual_ids != set(plan.source_ids) or len(products) != len(plan.source_ids): + by_source: dict[str, set[FeedType]] = {} + for item in products: + assert item.binding_id is not None + source_id = source_by_binding[item.binding_id] + by_source.setdefault(source_id, set()).add(item.feed) + if set(by_source) != set(plan.source_ids) or any( + feeds != {FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA} + for feeds in by_source.values() + ): raise ValueError("Trading System execution L2 matrix differs from the declared demand") - return tuple(sorted(products, key=lambda item: (item.venue, item.native_symbol))) + if len(products) != len(plan.source_ids) * 2: + raise ValueError("Trading System execution L2 matrix has an incomplete source pair") + return tuple(sorted( + products, + key=lambda item: (item.venue, item.native_symbol, item.feed.value), + )) -def compact_book_snapshot(view: object) -> dict[str, object]: - """Keep readiness evidence, never book levels or price/quantity payloads.""" +def compact_book_view(view: object) -> dict[str, object]: + """Keep L2 readiness evidence, never levels, prices or quantities.""" payload = getattr(view, "payload", None) source = getattr(view, "source", None) quality = getattr(view, "quality", None) + feed = getattr(payload, "feed", None) + feed_value = getattr(feed, "value", feed) fields = { "source_id": getattr(source, "source_id", None), + "feed": feed_value, "book_generation": getattr(payload, "book_generation", None), "sequence_verified": getattr(payload, "sequence_verified", None), - "native_sequence": getattr(payload, "native_sequence", None), - "depth": getattr(payload, "depth", None), "revision": getattr(view, "revision", None), "watermark_offset": getattr(view, "watermark_offset", None), "received_at_ns": getattr(view, "received_at_ns", None), @@ -113,10 +125,7 @@ def compact_book_snapshot(view: object) -> dict[str, object]: or not isinstance(fields["book_generation"], int) or fields["book_generation"] < 0 or not isinstance(fields["sequence_verified"], bool) - or not isinstance(fields["native_sequence"], str) - or not fields["native_sequence"] - or not isinstance(fields["depth"], int) - or fields["depth"] < 1 + or feed_value not in {"BOOK_SNAPSHOT", "BOOK_DELTA"} or not isinstance(fields["revision"], int) or fields["revision"] < 0 or not isinstance(fields["watermark_offset"], int) @@ -129,32 +138,84 @@ def compact_book_snapshot(view: object) -> dict[str, object]: or not isinstance(fields["complete"], bool) or not isinstance(fields["execution_eligible"], bool) ): - raise ValueError("execution L2 snapshot evidence has invalid typed fields") - return {**fields, "payload_recorded": False} + raise ValueError("execution L2 view evidence has invalid typed fields") + if feed_value == "BOOK_SNAPSHOT": + depth = getattr(payload, "depth", None) + native_sequence = getattr(payload, "native_sequence", None) + if ( + not isinstance(depth, int) + or depth < 1 + or not isinstance(native_sequence, str) + or not native_sequence + ): + raise ValueError("execution L2 snapshot evidence is incomplete") + return { + **fields, + "depth": depth, + "sequence_present": True, + "payload_recorded": False, + } + sequence_fields = ( + getattr(payload, "native_sequence_start", None), + getattr(payload, "native_sequence_end", None), + getattr(payload, "snapshot_sequence", None), + ) + reset = getattr(payload, "reset", None) + if ( + not all(isinstance(value, str) and value for value in sequence_fields) + or not isinstance(reset, bool) + ): + raise ValueError("execution L2 delta evidence is incomplete") + return { + **fields, + "sequence_present": True, + "reset": reset, + "payload_recorded": False, + } def ready_book_row(row: Mapping[str, object]) -> bool: """Return true only for a fully verified execution-grade compact row.""" status = row.get("typed_status") - snapshot = row.get("snapshot") - if not isinstance(status, Mapping) or not isinstance(snapshot, Mapping): + view = row.get("view") + if not isinstance(status, Mapping) or not isinstance(view, Mapping): return False quality = status.get("quality") - return bool( + flags = status.get("flags") + if not isinstance(flags, list) or any( + not isinstance(flag, str) for flag in flags + ): + return False + session_unhealthy = any(flag.startswith("SOURCE_SESSION_") for flag in flags) + common_ready = bool( isinstance(quality, Mapping) and quality.get("state") == "LIVE" and quality.get("complete") is True and quality.get("gap_open") is False - and quality.get("execution_eligible") is True - and snapshot.get("sequence_verified") is True - and isinstance(snapshot.get("book_generation"), int) - and int(snapshot["book_generation"]) >= 1 - and isinstance(snapshot.get("depth"), int) - and int(snapshot["depth"]) >= 100 - and snapshot.get("complete") is True - and snapshot.get("gap_open") is False - and snapshot.get("execution_eligible") is True + and not session_unhealthy + and view.get("sequence_verified") is True + and isinstance(view.get("book_generation"), int) + and int(view["book_generation"]) >= 1 + and view.get("complete") is True + and view.get("gap_open") is False + and view.get("sequence_present") is True + ) + if not common_ready: + return False + if row.get("feed") == "BOOK_SNAPSHOT": + return bool( + quality.get("execution_eligible") is True + and view.get("execution_eligible") is True + and isinstance(view.get("depth"), int) + and int(view["depth"]) >= 100 + ) + return bool( + row.get("feed") == "BOOK_DELTA" + and view.get("reset") is False + and quality.get("provider_session_state") == "LIVE" + and isinstance(quality.get("provider_session_liveness_ms"), int) + and quality["provider_session_liveness_ms"] >= 0 ) @@ -176,6 +237,27 @@ def replica_parity(primary: Mapping[str, object], secondary: Mapping[str, object return ready_book_row(primary) and ready_book_row(secondary) +def source_pair_ready(rows: tuple[Mapping[str, object], ...]) -> bool: + """A physical L2 source is usable only as one verified snapshot/delta pair.""" + + if len(rows) != 2 or {row.get("feed") for row in rows} != { + "BOOK_SNAPSHOT", "BOOK_DELTA" + }: + return False + source_ids = {row.get("source_id") for row in rows} + generations = { + row.get("view", {}).get("book_generation") + for row in rows + if isinstance(row.get("view"), Mapping) + } + return ( + len(source_ids) == 1 + and None not in source_ids + and len(generations) == 1 + and all(ready_book_row(row) for row in rows) + ) + + async def _read_one( product: AcceptanceProduct, *, @@ -198,8 +280,8 @@ async def _read_one( result: dict[str, object] = { "typed_status": None, "status_error": None, - "snapshot": None, - "snapshot_error": None, + "view": None, + "view_error": None, } try: try: @@ -225,13 +307,13 @@ async def _read_one( response = await client.snapshot(requirement) view = response.data validate_product_view(product, view) - snapshot = compact_book_snapshot(view) - source_id = str(snapshot["source_id"]) - result["snapshot"] = snapshot + compact_view = compact_book_view(view) + source_id = str(compact_view["source_id"]) + result["view"] = compact_view except DataLayerError as error: - result["snapshot_error"] = {"code": error.code, "detail": error.detail} + result["view_error"] = {"code": error.code, "detail": error.detail} except ValueError as error: - result["snapshot_error"] = {"code": "INVALID_VIEW", "detail": str(error)} + result["view_error"] = {"code": "INVALID_VIEW", "detail": str(error)} finally: await client.close() return { @@ -249,7 +331,7 @@ async def _read_one( } -async def run(args: argparse.Namespace) -> dict[str, object]: +async def _run_round(args: argparse.Namespace) -> dict[str, object]: catalog = StableSourceCatalog.load(args.catalog) acquisition = StableAcquisitionPlan.load(args.acquisition, catalog=catalog) products = execution_book_products( @@ -299,23 +381,48 @@ async def run(args: argparse.Namespace) -> dict[str, object]: "venue": product.venue, "market": product.market, "native_symbol": product.native_symbol, + "feed": product.feed.value, "source_policy_id": product.source_policy_id, "primary": primary, "secondary": secondary, "replica_parity": replica_parity(primary, secondary), }) + by_replica_source: dict[str, dict[str, list[Mapping[str, object]]]] = { + "primary": {}, "secondary": {}, + } + for row in rows: + for label in ("primary", "secondary"): + replica = row[label] + assert isinstance(replica, Mapping) + source_id = replica.get("source_id") + if isinstance(source_id, str) and source_id: + by_replica_source[label].setdefault(source_id, []).append(replica) + source_pair_results = { + label: { + source_id: source_pair_ready(tuple(source_rows)) + for source_id, source_rows in sorted(by_source.items()) + } + for label, by_source in by_replica_source.items() + } ready = all( row["replica_parity"] and ready_book_row(row["primary"]) and ready_book_row(row["secondary"]) for row in rows + ) and all( + all(result.values()) for result in source_pair_results.values() + ) and all( + len(by_source) == len(products) // 2 + for by_source in by_replica_source.values() ) return { "schema": "qdl.phase105.execution-l2-status-matrix.v1", "status": "PASS" if ready else "FAIL", "consumer_id": EXPECTED_CONSUMER_ID, - "book_count": len(rows), + "book_count": len(rows) // 2, + "book_product_count": len(rows), "replica_count": 2, + "source_pair_results": source_pair_results, "rows": rows, "elapsed_seconds": round(time.monotonic() - started, 3), "provider_connections": 0, @@ -325,6 +432,44 @@ async def run(args: argparse.Namespace) -> dict[str, object]: } +async def run(args: argparse.Namespace) -> dict[str, object]: + """Require a short stable read window, not one lucky post-reconnect read.""" + + rounds: list[dict[str, object]] = [] + for index in range(args.rounds): + round_result = await _run_round(args) + rounds.append(round_result) + if index + 1 < args.rounds: + await asyncio.sleep(args.period_seconds) + final = dict(rounds[-1]) + final["status"] = "PASS" if all( + item["status"] == "PASS" for item in rounds + ) else "FAIL" + final["round_count"] = args.rounds + final["ready_round_count"] = sum( + item["status"] == "PASS" for item in rounds + ) + final["period_seconds"] = args.period_seconds + final["round_summaries"] = [ + { + "round": index + 1, + "status": item["status"], + "elapsed_seconds": item["elapsed_seconds"], + "failed_product_identities": [ + { + "venue": row["venue"], + "native_symbol": row["native_symbol"], + "feed": row["primary"].get("feed"), + } + for row in item["rows"] + if not row["replica_parity"] + ], + } + for index, item in enumerate(rounds) + ] + return final + + def parser() -> argparse.ArgumentParser: value = argparse.ArgumentParser(description=__doc__) value.add_argument("--catalog", type=Path, default=DEFAULT_CATALOG) @@ -342,6 +487,8 @@ def parser() -> argparse.ArgumentParser: value.add_argument("--issuer", default="https://identity.qdl.stable.internal") value.add_argument("--audience", default="qdl-v2-stable") value.add_argument("--timeout-seconds", type=float, default=15.0) + value.add_argument("--rounds", type=int, default=3) + value.add_argument("--period-seconds", type=float, default=2.0) return value @@ -349,6 +496,10 @@ def main() -> int: args = parser().parse_args() if not 5.0 <= args.timeout_seconds <= 60.0: raise SystemExit("--timeout-seconds must be between 5 and 60") + if not 1 <= args.rounds <= 10: + raise SystemExit("--rounds must be between 1 and 10") + if not 0.5 <= args.period_seconds <= 15.0: + raise SystemExit("--period-seconds must be between 0.5 and 15") result = asyncio.run(run(args)) print(json.dumps(result, sort_keys=True, separators=(",", ":"))) return 0 if result["status"] == "PASS" else 1 diff --git a/scripts/phase10_real_provider_admission.py b/scripts/phase10_real_provider_admission.py index 5005536..bda8961 100755 --- a/scripts/phase10_real_provider_admission.py +++ b/scripts/phase10_real_provider_admission.py @@ -74,7 +74,11 @@ def _timestamp_ms(value: Any, field: str) -> int: return result -def _load_slices(path: Path) -> tuple[DemandSlice, ...]: +def _load_slices( + path: Path, + *, + allowed_feeds: frozenset[str] | None = None, +) -> tuple[DemandSlice, ...]: raw = yaml.safe_load(path.read_bytes()) if not isinstance(raw, Mapping) or raw.get("schema") != "qdl.v2.production-demand.v1": raise ProviderAdmissionError("only qdl.v2.production-demand.v1 demand is supported") @@ -107,8 +111,15 @@ def _load_slices(path: Path) -> tuple[DemandSlice, ...]: raise ProviderAdmissionError(f"requirement is missing {error.args[0]}") from error if item.venue not in {"BINANCE", "OKX"}: continue + # A bounded caller may deliberately certify only one product class + # (for example BAR warmup) from the universal demand manifest. It + # must filter that declared class before this provider-admission + # reader rejects feeds it does not own. + if allowed_feeds is not None and item.feed not in allowed_feeds: + continue if item.feed not in { "TRADE", "QUOTE", "BAR", "BOOK_SNAPSHOT", "BOOK_DELTA", + "MARK_INDEX_PRICE", }: raise ProviderAdmissionError(f"unsupported Phase 10.1 feed: {item.feed}") slices[item.key] = item @@ -133,6 +144,10 @@ def _endpoint(slice_: DemandSlice) -> tuple[str, dict[str, str]]: # remains a Rust WebSocket/core responsibility, but both public # products must prove the same venue-owned depth source exists. return f"{base}/depth", {"symbol": symbol, "limit": "100"} + if slice_.feed == "MARK_INDEX_PRICE": + if slice_.market != "USDM": + raise ProviderAdmissionError("Binance MARK_INDEX requires USD-M") + return f"{base}/premiumIndex", {"symbol": symbol} return f"{base}/klines", {"symbol": symbol, "interval": slice_.interval or "", "limit": "3"} if slice_.venue == "OKX": if slice_.market not in {"SWAP", "SPOT"}: @@ -144,10 +159,41 @@ def _endpoint(slice_: DemandSlice) -> tuple[str, dict[str, str]]: return f"{base}/books", {"instId": symbol, "sz": "1"} if slice_.feed in {"BOOK_SNAPSHOT", "BOOK_DELTA"}: return f"{base}/books", {"instId": symbol, "sz": "100"} + if slice_.feed == "MARK_INDEX_PRICE": + # A single logical execution reference is paired from OKX's + # separate mark and index endpoints; `_mark_index_endpoints` + # carries the two bounded reads below. + raise ProviderAdmissionError("OKX MARK_INDEX requires paired endpoints") return f"{base}/candles", {"instId": symbol, "bar": slice_.interval or "", "limit": "3"} raise ProviderAdmissionError(f"unsupported venue: {slice_.venue}") +def _okx_index_symbol(native_symbol: str) -> str: + suffix = "-SWAP" + if not native_symbol.endswith(suffix): + raise ProviderAdmissionError("OKX MARK_INDEX requires a SWAP native symbol") + return native_symbol[: -len(suffix)] + + +def _mark_index_endpoints(slice_: DemandSlice) -> tuple[tuple[str, dict[str, str]], ...]: + if slice_.feed != "MARK_INDEX_PRICE": + raise ProviderAdmissionError("paired endpoints require MARK_INDEX_PRICE") + if slice_.venue == "BINANCE": + return (_endpoint(slice_),) + if slice_.venue == "OKX" and slice_.market == "SWAP": + return ( + ( + "https://www.okx.com/api/v5/public/mark-price", + {"instType": "SWAP", "instId": slice_.native_symbol}, + ), + ( + "https://www.okx.com/api/v5/market/index-tickers", + {"instId": _okx_index_symbol(slice_.native_symbol)}, + ), + ) + raise ProviderAdmissionError("MARK_INDEX provider pairing is unsupported for this slice") + + def _first_mapping(value: Any, field: str) -> Mapping[str, Any]: if not isinstance(value, list) or not value or not isinstance(value[0], Mapping): raise ProviderAdmissionError(f"{field} must contain one object") @@ -235,6 +281,32 @@ def _validate_okx(slice_: DemandSlice, payload: Any, received_ms: int) -> int: return timestamp +def _validate_mark_index( + slice_: DemandSlice, + payloads: tuple[Any, ...], + received_ms: int, +) -> int: + if slice_.venue == "BINANCE": + if len(payloads) != 1 or not isinstance(payloads[0], Mapping): + raise ProviderAdmissionError("binance mark/index response is invalid") + payload = payloads[0] + _positive_decimal(payload.get("markPrice"), "binance mark price") + _positive_decimal(payload.get("indexPrice"), "binance index price") + return _timestamp_ms(payload.get("time"), "binance mark/index time") + if slice_.venue == "OKX": + if len(payloads) != 2: + raise ProviderAdmissionError("OKX mark/index response pair is incomplete") + mark = _first_mapping(_okx_data(payloads[0], "okx mark"), "okx mark") + index = _first_mapping(_okx_data(payloads[1], "okx index"), "okx index") + _positive_decimal(mark.get("markPx"), "okx mark price") + _positive_decimal(index.get("idxPx"), "okx index price") + return min( + _timestamp_ms(mark.get("ts"), "okx mark time"), + _timestamp_ms(index.get("ts"), "okx index time"), + ) + raise ProviderAdmissionError("MARK_INDEX provider pairing is unsupported for this venue") + + def _validate(slice_: DemandSlice, payload: Any, received_ms: int) -> int: if slice_.venue == "BINANCE": return _validate_binance(slice_, payload, received_ms) @@ -252,17 +324,25 @@ def run( slices = _load_slices(demand_path) results: list[dict[str, Any]] = [] for slice_ in slices: - url, params = _endpoint(slice_) received_ms = int(time.time() * 1_000) - response = get( - url, - params=params, - timeout=timeout_seconds, - headers={"User-Agent": "qdl-phase10-read-only-admission/1.0"}, - ) - response.raise_for_status() - payload = response.json() - provider_time_ms = _validate(slice_, payload, received_ms) + if slice_.feed == "MARK_INDEX_PRICE": + endpoints = _mark_index_endpoints(slice_) + else: + endpoints = (_endpoint(slice_),) + payloads = [] + for url, params in endpoints: + response = get( + url, + params=params, + timeout=timeout_seconds, + headers={"User-Agent": "qdl-phase10-read-only-admission/1.0"}, + ) + response.raise_for_status() + payloads.append(response.json()) + if slice_.feed == "MARK_INDEX_PRICE": + provider_time_ms = _validate_mark_index(slice_, tuple(payloads), received_ms) + else: + provider_time_ms = _validate(slice_, payloads[0], received_ms) results.append( { "slice": slice_.key, @@ -270,7 +350,7 @@ def run( "provider_time_ms": provider_time_ms, "received_at_ms": received_ms, "payload_sha256": hashlib.sha256( - json.dumps(payload, sort_keys=True, separators=(",", ":")).encode() + json.dumps(payloads, sort_keys=True, separators=(",", ":")).encode() ).hexdigest(), } ) diff --git a/scripts/phase10_warmup_admission.py b/scripts/phase10_warmup_admission.py index 7d6b646..804ca72 100644 --- a/scripts/phase10_warmup_admission.py +++ b/scripts/phase10_warmup_admission.py @@ -50,7 +50,7 @@ class _AdmissionWork: def _bar_slices(path: Path) -> tuple[DemandSlice, ...]: - values = tuple(item for item in _load_slices(path) if item.feed == "BAR") + values = _load_slices(path, allowed_feeds=frozenset({"BAR"})) if not values or len(values) > MAX_BAR_SLICES: raise WarmupAdmissionError("bounded demanded BAR slice count is invalid") return values diff --git a/scripts/phase533_materialize_alpha_runtime_entitlements.py b/scripts/phase533_materialize_alpha_runtime_entitlements.py index 1d321a6..aa57a18 100644 --- a/scripts/phase533_materialize_alpha_runtime_entitlements.py +++ b/scripts/phase533_materialize_alpha_runtime_entitlements.py @@ -164,7 +164,7 @@ def _demand_rows( str(instrument["product_type"]).upper(), str(instrument["native_symbol"]).upper(), ) - values: list[Mapping[str, Any]] = [] + values: dict[tuple[str, str, str, str, str, str | None, str], Mapping[str, Any]] = {} for consumer in demand.get("consumers", []): if not isinstance(consumer, Mapping): continue @@ -178,8 +178,22 @@ def _demand_rows( str(row.get("native_symbol", "")).upper(), ) if identity == expected: - values.append(row) - return values + key = _identity(row) + incumbent = values.get(key) + if incumbent is None: + values[key] = row + elif dict(incumbent) != dict(row): + raise ValueError( + "stable crypto demand has conflicting shared runtime identity: " + f"{key}" + ) + return [ + values[key] + for key in sorted( + values, + key=lambda identity: tuple("" if value is None else value for value in identity), + ) + ] def _manifest_requirement( @@ -210,7 +224,10 @@ def _manifest_requirement( "recovery": "SNAPSHOT_AND_REPLAY", "bar_revision_policy": "EMIT_REVISIONS" if feed == "BAR" else "LATEST", } - if feed == "TRADE": + if feed in {"TRADE", "BOOK_DELTA"} or ( + feed == "QUOTE" + and str(quality.get("delivery_semantics", "STRICT_EVENT")).upper() == "ON_CHANGE" + ): result["event_recency_policy"] = "OBSERVE" if feed in {"TRADE", "QUOTE", "BOOK_SNAPSHOT", "BOOK_DELTA"}: result["max_session_liveness_ms"] = 45_000 @@ -229,9 +246,6 @@ def _realtime_templates( for uid in _target_uids(manifest, target=target, instruments=instruments): instrument = instruments[uid] source_rows = _demand_rows(demand, instrument=instrument) - keys = {_identity(row) for row in source_rows} - if len(keys) != len(source_rows): - raise ValueError("stable crypto demand has duplicate alpha runtime identities") expected = { "TRADE": 1, "QUOTE": 1, diff --git a/scripts/refresh_v2_l2_core_runtime.py b/scripts/refresh_v2_l2_core_runtime.py index 4d9207d..a57a58d 100644 --- a/scripts/refresh_v2_l2_core_runtime.py +++ b/scripts/refresh_v2_l2_core_runtime.py @@ -37,6 +37,10 @@ ExecutionL2MaterializationPlan, execution_l2_materialization_plan, ) +from qdl.runtime.core_binding_identity import ( + core_binding_map, + format_core_binding_identity, +) CONFIRM = "REFRESH_QDL_V2_L2_CORE_RUNTIME" @@ -69,17 +73,7 @@ def _bindings(value: Mapping[str, Any], *, field: str) -> tuple[dict[str, Any], bindings = core.get("bindings") if isinstance(core, Mapping) else None if not isinstance(bindings, list) or not bindings: raise ValueError(f"{field} lacks core bindings") - result: list[dict[str, Any]] = [] - source_ids: set[str] = set() - for item in bindings: - if not isinstance(item, dict): - raise ValueError(f"{field} has a non-object binding") - source_id = item.get("source_id") - if not isinstance(source_id, str) or not source_id or source_id in source_ids: - raise ValueError(f"{field} has an invalid/duplicate source_id") - source_ids.add(source_id) - result.append(item) - return tuple(result) + return tuple(core_binding_map(bindings, field=field).values()) def _without_catalog_revision(value: Mapping[str, Any]) -> dict[str, Any]: @@ -130,33 +124,39 @@ def _validate_and_render( raise ValueError(f"{file_name} changes non-binding core configuration") active_bindings = _bindings(active, field=f"active {file_name}") expected_bindings = _bindings(expected, field=f"expected {file_name}") - active_by_id = {str(item["source_id"]): item for item in active_bindings} - expected_by_id = {str(item["source_id"]): item for item in expected_bindings} - unknown = sorted(active_by_id.keys() - expected_by_id.keys()) + active_by_id = core_binding_map(list(active_bindings), field=f"active {file_name}") + expected_by_id = core_binding_map(list(expected_bindings), field=f"expected {file_name}") + unknown = sorted( + format_core_binding_identity(identity) + for identity in active_by_id.keys() - expected_by_id.keys() + ) if unknown: raise ValueError(f"{file_name} contains bindings absent from current catalog: {unknown}") - for source_id, current in active_by_id.items(): - generated = expected_by_id[source_id] + for identity, current in active_by_id.items(): + generated = expected_by_id[identity] if _without_catalog_revision(current) != _without_catalog_revision(generated): - raise ValueError(f"{file_name} has semantic drift for {source_id}") + raise ValueError( + f"{file_name} has semantic drift for " + f"{format_core_binding_identity(identity)}" + ) declared_book_source_ids = frozenset(execution_l2.source_ids) added_ids = expected_by_id.keys() - active_by_id.keys() - if added_ids and not added_ids.issubset(declared_book_source_ids): + additions = [ + expected_by_id[identity] + for identity in expected_by_id + if identity in added_ids + ] + added_source_ids = {str(item["source_id"]) for item in additions} + if additions and not added_source_ids.issubset(declared_book_source_ids): raise ValueError( f"{file_name} additive BOOK scope differs from the execution demand: " - f"{sorted(added_ids)}" + f"{sorted(added_source_ids)}" ) - if not declared_book_source_ids.issubset(expected_by_id): + expected_source_ids = {str(item["source_id"]) for item in expected_bindings} + if not declared_book_source_ids.issubset(expected_source_ids): raise ValueError(f"{file_name} generated BOOK scope is incomplete") - if not declared_book_source_ids.issubset(expected_by_id.keys() | active_by_id.keys()): - raise ValueError(f"{file_name} active/generated BOOK scope is incomplete") - additions = [ - expected_by_id[str(item["source_id"])] - for item in expected_bindings - if str(item["source_id"]) in added_ids - ] for item in additions: source_id = str(item["source_id"]) if ( @@ -169,10 +169,10 @@ def _validate_and_render( raise ValueError(f"{file_name} has an invalid declared L2 addition: {source_id}") revision_updates = [ - source_id - for source_id, current in active_by_id.items() + identity + for identity, current in active_by_id.items() if current.get("instrument_catalog_revision") - != expected_by_id[source_id].get("instrument_catalog_revision") + != expected_by_id[identity].get("instrument_catalog_revision") ] if not revision_updates: raise ValueError(f"{file_name} has no catalog revision lineage update") @@ -183,7 +183,7 @@ def _validate_and_render( raise ValueError(f"{file_name} lacks mutable core configuration") # Use the generated binding set, rather than preserving stale active # metadata, because Rust verifies raw.instrument_catalog_revision exactly. - core["bindings"] = copy.deepcopy(expected_bindings) + core["bindings"] = [copy.deepcopy(item) for item in expected_bindings] return result, { "file": file_name, "before_binding_count": len(active_bindings), @@ -196,7 +196,7 @@ def _validate_and_render( int(item["instrument_catalog_revision"]) for item in expected_bindings }), "declared_book_source_ids": sorted(declared_book_source_ids), - "added_book_source_ids": sorted(added_ids), + "added_book_source_ids": sorted(added_source_ids), "added_book_symbols": sorted(str(item["native_symbol"]) for item in additions), } diff --git a/scripts/refresh_v2_native_ingestor_runtime.py b/scripts/refresh_v2_native_ingestor_runtime.py index 0b4850c..6d69bb1 100644 --- a/scripts/refresh_v2_native_ingestor_runtime.py +++ b/scripts/refresh_v2_native_ingestor_runtime.py @@ -3,7 +3,7 @@ The stable BAR edge owns every final BAR bootstrap and recurring provider poll. This compiler intentionally refreshes only the physical realtime inputs of the -two existing shared Rust ingestors: TRADE, QUOTE and coalesced BOOK. It never +two existing shared Rust ingestors: TRADE, QUOTE, MARK_INDEX and coalesced BOOK. It never turns catalog BAR rows into native subscriptions, changes authority, creates a symbol worker, or edits a Compose environment file. """ @@ -29,6 +29,7 @@ StableAcquisitionPlan, validate_shared_authority_record, ) +from qdl.runtime.core_binding_identity import native_ingestor_binding_identity CONFIRM = "REFRESH_QDL_V2_NATIVE_INGESTOR_RUNTIME" @@ -37,7 +38,7 @@ "binance-usdm": "ingestor-binance-usdm.json", "okx-swap": "ingestor-okx-swap.json", } -REALTIME_FEEDS = frozenset({"TRADE", "QUOTE", "BOOK"}) +REALTIME_FEEDS = frozenset({"TRADE", "QUOTE", "MARK_INDEX", "BOOK"}) def _sha256(value: bytes) -> str: @@ -61,24 +62,23 @@ def _read_json(path: Path, *, field: str) -> dict[str, Any]: return value -def _binding_key(binding: Mapping[str, Any]) -> tuple[str, str]: +def _binding_key(binding: Mapping[str, Any]) -> tuple[str, ...]: feed = binding.get("feed") - subscription = binding.get("subscription_id") if not isinstance(feed, str) or feed not in REALTIME_FEEDS: raise ValueError("native ingestor binding feed is invalid") - if not isinstance(subscription, str) or not subscription: - raise ValueError("native ingestor binding subscription_id is invalid") - return feed, subscription + # OKX MARK_INDEX is a paired physical input under one logical source ID; + # the native channel is part of the physical subscription identity. + return native_ingestor_binding_identity(binding, field="native ingestor binding") def _binding_map( bindings: object, *, field: str, -) -> dict[tuple[str, str], dict[str, Any]]: +) -> dict[tuple[str, ...], dict[str, Any]]: if not isinstance(bindings, list) or not bindings: raise ValueError(f"{field} bindings are invalid") - result: dict[tuple[str, str], dict[str, Any]] = {} + result: dict[tuple[str, ...], dict[str, Any]] = {} for item in bindings: if not isinstance(item, dict): raise ValueError(f"{field} binding is not an object") diff --git a/scripts/refresh_v2_rust_core_runtime.py b/scripts/refresh_v2_rust_core_runtime.py index 8d6e14f..2d407cc 100644 --- a/scripts/refresh_v2_rust_core_runtime.py +++ b/scripts/refresh_v2_rust_core_runtime.py @@ -39,6 +39,7 @@ ExecutionL2MaterializationPlan, execution_l2_materialization_plan, ) +from qdl.runtime.core_binding_identity import core_binding_map CONFIRM = "REFRESH_QDL_V2_RUST_CORE_RUNTIME" @@ -87,17 +88,10 @@ def _bindings(value: Mapping[str, Any], *, field: str) -> tuple[dict[str, Any], bindings = core.get("bindings") if isinstance(core, Mapping) else None if not isinstance(bindings, list) or not bindings: raise ValueError(f"{field} lacks core bindings") - result: list[dict[str, Any]] = [] - source_ids: set[str] = set() - for item in bindings: - if not isinstance(item, dict): - raise ValueError(f"{field} contains a non-object binding") - source_id = item.get("source_id") - if not isinstance(source_id, str) or not source_id or source_id in source_ids: - raise ValueError(f"{field} contains an invalid/duplicate source_id") - source_ids.add(source_id) - result.append(item) - return tuple(result) + # Keep emitted order for the bounded cadence comparison below, but validate + # the same MARK+INDEX source identity rule as the Rust core first. + core_binding_map(bindings, field=field) + return tuple(dict(item) for item in bindings) def _without_bindings_and_dedup_capacity(value: Mapping[str, Any]) -> tuple[dict[str, Any], int]: @@ -135,9 +129,9 @@ def _validate_only_materialized_snapshot_interval( raise ValueError(f"{file_name} has an invalid bounded dedup transition") before = _bindings(active, field=f"active {file_name}") after = _bindings(expected, field=f"expected {file_name}") - before_ids = [str(item["source_id"]) for item in before] - after_ids = [str(item["source_id"]) for item in after] - if before_ids != after_ids: + before_identities = list(core_binding_map(list(before), field=f"active {file_name}")) + after_identities = list(core_binding_map(list(after), field=f"expected {file_name}")) + if before_identities != after_identities: raise ValueError(f"{file_name} changes binding order or membership") l2_sources: list[str] = [] diff --git a/scripts/report_binding_liveness.py b/scripts/report_binding_liveness.py index 330a4a1..9b002e5 100644 --- a/scripts/report_binding_liveness.py +++ b/scripts/report_binding_liveness.py @@ -1,149 +1,317 @@ #!/usr/bin/env python3 -"""Which source bindings are actually live, judged by each binding's own budget. - -Read-only. It opens the durable spool immutably and compares, per binding, the -age of the newest stored event against that binding's `quality.stale_after_ms` -from the runtime catalog. - -Why the budget comes from the catalog. A fixed threshold answers the wrong -question: a 1d bar that is five hours old is healthy - the bar has not closed -and the next one is not due - while a 1m bar five minutes old is a gap. Earlier -reports in this program used one 180 s threshold for every binding and called -long-interval bars "stale", which is a unit error, not a finding. The catalog -already sets the budget per binding (1m -> 180 s, 15m -> 45 min, 1d -> 3 d, -1w -> 21 d, MARK_INDEX_PRICE -> 2 s), so that is what this compares against. - -Why the age is read from the envelope and not from the spool row. The spool's -`committed_at_ns` is when the durable append landed, which trails the event and -is not what any gate compares. `qdl/runtime/stable_source.py:483-492` measures a -bar from `bar.close_time_ns`, a PROVIDER_CONFIRMATION binding - every -MARK_INDEX_PRICE binding - from `received_at_ns`, and everything else from -`source_event_time_ns`. Judging mark/index by the venue's own `ts` is precisely -the mistake that basis exists to avoid: OKX `index-tickers` returns a `ts` -roughly a second older than the row it is attached to, so a source-time rule -reports ten healthy bindings as stale. This mirrors the runtime rule instead. - -Run it in a throwaway container with the state volume mounted read-only, not -inside a data layer role: a full scan of `events` inside a role with a 512 MiB -limit is what restarted query_v2_1 during an earlier probe. - - docker run --rm \ - -v qdl_v2_stable_candidate_stable_state:/st:ro \ - -v /home/bobby/data_layer:/src:ro \ - -v :/runtime:ro \ - -e PYTHONPATH=/src --entrypoint python qdl-v2-python: \ - -B /src/scripts/report_binding_liveness.py +"""Read-only typed quality inventory for the sealed V2 binding catalog. + +This auditor feeds durable facts, acquisition state and session evidence into +the same ``BindingQualityDecision`` policy used by the stable query edge. It +keeps event age, ingest-to-durable latency and last-durable-append age separate; +none of them is consumer-call-to-usable latency. """ from __future__ import annotations import argparse +import json import sqlite3 import sys import time +from collections import Counter from pathlib import Path +from typing import Any ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) -from qdl.marketdata.v2 import market_data_pb2 # noqa: E402 -from qdl.runtime.stable_catalog import StableSourceCatalog # noqa: E402 +from qdl.common.v1 import common_pb2 # noqa: E402 +from qdl.consumer.manifest import ConsumerManifestLoader # noqa: E402 +from qdl.data_quality.binding_decision import ( # noqa: E402 + BindingQualityInput, + ComponentEvidence, + evaluate_binding_quality, +) +from qdl.marketdata.v2 import market_data_pb2 # noqa: E402 +from qdl.runtime.mark_index_lineage import paired_mark_index_lineage # noqa: E402 +from qdl.runtime.session_liveness import StableSessionLivenessReader # noqa: E402 +from qdl.runtime.stable_catalog import StableSourceCatalog # noqa: E402 +from qdl.runtime.stable_deployment import StableAcquisitionPlan # noqa: E402 -def newest_by_partition(db: Path) -> dict[tuple[str, str], bytes]: - """The newest stored payload per (stream, partition_key). +def newest_by_partition(db: Path) -> dict[tuple[str, str], tuple[bytes, int, int]]: + """Return newest durable payload, commit and watermark facts per partition.""" - Newest is taken by `logical_offset`, the spool's own append order, which is - what `read_tail` returns; `committed_at_ns` is carried only so the caller - can report append lag separately from event age. - """ - # `mode=ro`, never `immutable=1`: the spool is being written while this - # reads it, and `immutable` tells SQLite the file cannot change, which - # skips locking and silently returns stale or torn pages once the WAL moves. + # Never use immutable=1: this spool may append while the external auditor + # reads it, and immutable mode may return stale or torn WAL pages. connection = sqlite3.connect(f"file:{db}?mode=ro", uri=True) try: rows = connection.execute( - "SELECT e.stream, e.partition_key, e.payload, e.committed_at_ns " + "SELECT e.stream, e.partition_key, e.payload, e.committed_at_ns, e.logical_offset " "FROM events AS e JOIN (" - " SELECT stream, partition_key, MAX(logical_offset) AS top" + " SELECT stream, partition_key, MAX(logical_offset) AS top " " FROM events GROUP BY stream, partition_key" - ") AS t ON e.stream = t.stream AND e.partition_key = t.partition_key" + ") AS t ON e.stream = t.stream AND e.partition_key = t.partition_key " " AND e.logical_offset = t.top" ).fetchall() finally: connection.close() - return {(stream, partition): (payload, committed) - for stream, partition, payload, committed in rows} + return { + (str(stream), str(partition)): ( + bytes(payload), int(committed), int(logical_offset) + ) + for stream, partition, payload, committed, logical_offset in rows + } + + +def _flag_names(envelope: market_data_pb2.EventEnvelope) -> tuple[str, ...]: + return tuple( + common_pb2.QualityFlag.Name(value).removeprefix("QUALITY_FLAG_") + for value in envelope.quality_flags + ) + + +def _market_open(binding, now_ns: int) -> bool: + if binding.continuous_calendar: + return True + from qdl.domain.calendar import trading_calendar_for_id + + return trading_calendar_for_id(binding.instrument.session_calendar_id).is_open_ns(now_ns) -def freshness_observed_ns(binding, envelope) -> int: - """The timestamp the runtime measures this binding's freshness from. +def _requirements_by_key(manifest) -> dict[tuple[str, object, str | None], Any]: + return { + (item.instrument_uid, item.feed, item.interval): item + for item in manifest.requirements + } - Mirrors `qdl/runtime/stable_source.py:483-492`. Any divergence here turns - this report into a second opinion rather than a check. - """ - if binding.freshness_basis == "PROVIDER_CONFIRMATION": - return envelope.received_at_ns - if envelope.WhichOneof("payload") == "bar": - return envelope.bar.close_time_ns - return envelope.source_event_time_ns + +def _default_policy(binding) -> tuple[str, int | None, int]: + """Capability-only rows cannot claim an execution consumer policy.""" + + policy = ( + "OBSERVE" + if binding.feed.value in {"TRADE", "BOOK_DELTA", "MARK_INDEX_PRICE"} + else "BLOCK" + ) + return policy, None, binding.stale_after_ms + + +def _components(binding, acquisition, envelope, *, now_ns: int) -> tuple[ComponentEvidence, ...]: + if binding.feed.value != "MARK_INDEX_PRICE" or acquisition.mark_index is None: + return () + try: + lineage = paired_mark_index_lineage(envelope) + except ValueError: + return () + cadence = dict(acquisition.mark_index.component_quiet_after_ms) + if "BOTH" in cadence: + cadence = {"MARK": cadence["BOTH"], "INDEX": cadence["BOTH"]} + if set(cadence) != {"MARK", "INDEX"}: + return () + return ( + ComponentEvidence( + "MARK", + max(0, (now_ns - lineage.mark_received_at_ns) // 1_000_000), + int(cadence["MARK"]), + ), + ComponentEvidence( + "INDEX", + max(0, (now_ns - lineage.index_received_at_ns) // 1_000_000), + int(cadence["INDEX"]), + ), + ) + + +def decision_row( + *, + binding, + acquisition, + requirement, + stored: tuple[bytes, int, int] | None, + now_ns: int, + session_reader: StableSessionLivenessReader, +) -> dict[str, object]: + """Build one bounded audit row without changing runtime/durable state.""" + + consumer_bound = requirement is not None + if requirement is None: + event_policy, session_limit_ms, event_limit_ms = _default_policy(binding) + else: + event_policy = requirement.effective_event_recency_policy.value + session_limit_ms = requirement.max_session_liveness_ms + event_limit_ms = min( + binding.stale_after_ms, + requirement.max_freshness_ms or binding.stale_after_ms, + ) + + envelope = None + committed_at_ns = None + watermark_offset = 0 + flags: tuple[str, ...] = ("CAPABILITY_ONLY",) if not consumer_bound else () + session_state = "NOT_APPLICABLE" + session_liveness_ms = None + components: tuple[ComponentEvidence, ...] = () + generation_matches = True + config_matches = True + gap_open = False + book_verified = True + final_bar = True + event_age_ms = None + ingest_to_durable_latency_ms = None + if stored is not None: + payload, committed_at_ns, watermark_offset = stored + envelope = market_data_pb2.EventEnvelope() + envelope.ParseFromString(payload) + flags += _flag_names(envelope) + observed_ns = ( + int(envelope.received_at_ns) + if binding.freshness_basis == "PROVIDER_CONFIRMATION" + else ( + int(envelope.bar.close_time_ns) + if envelope.WhichOneof("payload") == "bar" + else int(envelope.source_event_time_ns) + ) + ) + event_age_ms = max(0, (now_ns - observed_ns) // 1_000_000) + ingest_to_durable_latency_ms = max( + 0, (committed_at_ns - int(envelope.received_at_ns)) // 1_000_000 + ) + gap_open = any( + value in {"SEQUENCE_GAP_BEFORE", "OUT_OF_ORDER", "RESYNC_REQUIRED"} + for value in flags + ) + payload_name = envelope.WhichOneof("payload") + if payload_name in {"book_snapshot", "book_delta"}: + book = getattr(envelope, payload_name) + book_verified = bool(book.sequence_verified) and int(book.book_generation) >= 1 + if payload_name == "bar": + final_bar = bool(envelope.bar.is_final) + if session_limit_ms is not None: + session = session_reader.status( + venue=envelope.venue, + market=envelope.market, + source_session_id=envelope.source_session_id, + connection_generation=int(envelope.connection_generation), + config_revision=max(1, int(envelope.config_revision)), + now_ns=now_ns, + ) + session_state = session.state + session_liveness_ms = session.liveness_ms + flags += session.flags + generation_matches = "SOURCE_SESSION_AMBIGUOUS" not in session.flags + config_matches = "SOURCE_SESSION_CONFIG_MISMATCH" not in session.flags + components = _components(binding, acquisition, envelope, now_ns=now_ns) + + decision = evaluate_binding_quality( + BindingQualityInput( + binding_id=binding.binding_id, + instrument_uid=binding.instrument.instrument_uid, + feed=binding.feed.value, + source_role=binding.source_role, + authoritative=binding.authoritative, + acquisition_enabled=acquisition.enabled, + acquisition_mode=acquisition.mode, + market_open=_market_open(binding, now_ns), + event_present=envelope is not None, + event_age_ms=event_age_ms, + event_limit_ms=event_limit_ms, + event_recency_policy=event_policy, + session_state=session_state, + session_liveness_ms=session_liveness_ms, + session_limit_ms=session_limit_ms, + delivery_semantics=binding.delivery_semantics, + components=components, + generation_matches=generation_matches, + config_matches=config_matches, + gap_open=gap_open, + book_verified=book_verified, + final_bar=final_bar, + require_final_bar=binding.require_final_bar, + watermark_offset=watermark_offset, + flags=flags, + ) + ).as_mapping() + decision.update({ + "consumer_scope": "BOUND" if consumer_bound else "CAPABILITY_ONLY", + "event_age_ms": event_age_ms, + "ingest_to_durable_latency_ms": ingest_to_durable_latency_ms, + "last_durable_append_age_ms": ( + max(0, (now_ns - committed_at_ns) // 1_000_000) + if committed_at_ns is not None + else None + ), + }) + return decision def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--spool", default="/st/shared/canonical-cache.sqlite3") parser.add_argument("--catalog", default="/runtime/stable-source-bindings.yaml") - parser.add_argument("--only-stale", action="store_true", - help="print only bindings over their own budget") + parser.add_argument("--acquisition", default="/runtime/stable-acquisition-bindings.yaml") + parser.add_argument( + "--manifest", + default=str(ROOT / "consumers/stable/trading-system-paper.yaml"), + ) + parser.add_argument("--session-liveness-root", default="/st/runtime/session-liveness") + parser.add_argument("--format", choices=("text", "json"), default="text") + parser.add_argument("--only-failures", action="store_true") args = parser.parse_args() catalog = StableSourceCatalog.load(args.catalog) + acquisition = StableAcquisitionPlan.load(args.acquisition, catalog=catalog) + manifest = ConsumerManifestLoader.load(args.manifest) + requirements = _requirements_by_key(manifest) newest = newest_by_partition(Path(args.spool)) now_ns = time.time_ns() - - live: list[tuple] = [] - stale: list[tuple] = [] - empty: list[str] = [] - for binding in catalog.bindings: - key = (binding.canonical_stream, binding.partition_key) - stored = newest.get(key) - if stored is None: - empty.append(binding.binding_id) - continue - payload, committed = stored - envelope = market_data_pb2.EventEnvelope() - envelope.ParseFromString(payload) - age_ms = (now_ns - freshness_observed_ns(binding, envelope)) / 1e6 - budget_ms = float(binding.stale_after_ms) - row = (binding.binding_id, binding.feed.value, binding.interval or "-", - age_ms, budget_ms, age_ms / budget_ms if budget_ms else float("inf"), - (now_ns - committed) / 1e6) - (stale if age_ms > budget_ms else live).append(row) - - print(f"catalog bindings : {len(catalog.bindings)}") - print(f"live : {len(live)}") - print(f"over budget : {len(stale)}") - print(f"no event stored : {len(empty)}") - if not args.only_stale and live: - worst = sorted(live, key=lambda r: -r[5])[:10] - print("\nclosest to their budget (live):") - for bid, feed, interval, age, budget, ratio, append in worst: - print(f" {bid:52s} {feed:16s} {interval:4s} " - f"age={age/1000:9.1f}s budget={budget/1000:9.1f}s {ratio*100:5.1f}%" - f" append_lag={append/1000:6.1f}s") - if stale: - print("\nover their own budget:") - for bid, feed, interval, age, budget, ratio, append in sorted( - stale, key=lambda r: -r[5]): - print(f" {bid:52s} {feed:16s} {interval:4s} " - f"age={age/1000:9.1f}s budget={budget/1000:9.1f}s {ratio*100:5.1f}%" - f" append_lag={append/1000:6.1f}s") - if empty: - print(f"\nno event stored ({len(empty)}):") - for bid in sorted(empty): - print(f" {bid}") - return 1 if stale else 0 + session_reader = StableSessionLivenessReader(args.session_liveness_root) + acquisition_by_id = {item.binding_id: item for item in acquisition.bindings} + rows = [ + decision_row( + binding=binding, + acquisition=acquisition_by_id[binding.binding_id], + requirement=requirements.get(binding.requirement_key), + stored=newest.get((binding.canonical_stream, binding.partition_key)), + now_ns=now_ns, + session_reader=session_reader, + ) + for binding in catalog.bindings + ] + unexpected = [ + row for row in rows + if row["availability"] == "ACTIVE" and row["state"] != "LIVE" + ] + result = { + "schema": "qdl.binding-quality-audit.v1", + "catalog_binding_count": len(rows), + "manifest_consumer_id": manifest.consumer_id, + "manifest_revision": manifest.manifest_revision, + "counts_by_availability": dict(sorted(Counter( + str(row["availability"]) for row in rows + ).items())), + "counts_by_state": dict(sorted(Counter(str(row["state"]) for row in rows).items())), + "unexpected_count": len(unexpected), + "rows": sorted( + unexpected if args.only_failures else rows, + key=lambda value: str(value["binding_id"]), + ), + } + if args.format == "json": + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + else: + print(f"catalog bindings : {result['catalog_binding_count']}") + print(f"consumer : {manifest.consumer_id} revision={manifest.manifest_revision}") + print(f"availability : {result['counts_by_availability']}") + print(f"states : {result['counts_by_state']}") + print(f"unexpected : {result['unexpected_count']}") + for row in result["rows"]: + print( + f"{row['binding_id']:55s} {row['feed']:16s} " + f"{row['availability']:22s} {row['state']:13s} " + f"event_age_ms={row['event_age_ms']} " + f"ingest_to_durable_ms={row['ingest_to_durable_latency_ms']} " + f"last_durable_append_age_ms={row['last_durable_append_age_ms']} " + f"reasons={','.join(row['reason_codes'])}" + ) + return 1 if unexpected else 0 if __name__ == "__main__": diff --git a/tests/test_catalog_regeneration.py b/tests/test_catalog_regeneration.py index 456ec30..f20d1f6 100644 --- a/tests/test_catalog_regeneration.py +++ b/tests/test_catalog_regeneration.py @@ -21,12 +21,6 @@ DEMAND_PATH = ROOT / "config/v2/stable-crypto-demand.yaml" CAPTURES = ROOT / "config/v2/captures" PROVENANCE = CAPTURES / "provenance.json" -# This builder is fed only the stable price/bar demand capture. Dated futures -# used by the independently governed L2/reference product are deliberately -# retained in the stable catalog but are not a false claim that this narrow -# price-plane regeneration owns their metadata. -REGENERATED_PRODUCT_TYPES = {"SPOT", "PERPETUAL"} - # Fields where the committed catalog disagrees with the provider capture that # regenerates it. Each was verified directly against the raw provider response, # and the committed value is the wrong one: the Binance Spot tick and step were @@ -105,11 +99,19 @@ def setUp(self) -> None: self.committed = { item["instrument_id"]: item for item in raw["instruments"] - if item["product_type"] in REGENERATED_PRODUCT_TYPES + if item["instrument_id"] in self.generated + } + self.compatibility_only = { + item["instrument_id"] + for item in raw["instruments"] + if item["instrument_id"] not in self.generated } def test_the_same_instrument_set_is_produced(self): self.assertEqual(set(self.generated), set(self.committed)) + # The stable source catalog intentionally retains V1/dormant venue + # inventory. Regeneration owns only the sealed active demand scope. + self.assertTrue(self.compatibility_only) def test_identity_fields_reproduce_exactly(self): for instrument_id in sorted(self.generated): diff --git a/tests/test_core_binding_identity.py b/tests/test_core_binding_identity.py new file mode 100644 index 0000000..c3dd55b --- /dev/null +++ b/tests/test_core_binding_identity.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import unittest + +from qdl.runtime.core_binding_identity import core_binding_map + + +def _binding( + *, + source_id: str = "okx-mark-index", + component: str | None = None, + channel: str = "mark-price", +) -> dict[str, object]: + value: dict[str, object] = { + "provider": "OKX", + "venue": "OKX", + "market": "SWAP", + "product_type": "PERPETUAL", + "native_symbol": "BTC-USDT-SWAP", + "native_channel": channel, + "instrument_uid": "OKX.SWAP.PERPETUAL.BTC-USDT", + "source_id": source_id, + } + if component is not None: + value["mark_index"] = {"component": component} + return value + + +class CoreBindingIdentityTests(unittest.TestCase): + def test_accepts_one_logical_source_with_mark_and_index_components(self) -> None: + result = core_binding_map( + [ + _binding(component="MARK", channel="mark-price"), + _binding( + component="INDEX", + channel="index-tickers", + ), + ], + field="test", + ) + self.assertEqual(len(result), 2) + + def test_rejects_a_duplicate_component(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicate mark_index component"): + core_binding_map( + [ + _binding(component="MARK"), + _binding(component="MARK", channel="mark-price-secondary"), + ], + field="test", + ) + + def test_rejects_an_ordinary_mark_index_source_collision(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicate ordinary source_id"): + core_binding_map( + [ + _binding(component="MARK", channel="mark-price"), + _binding(component="INDEX", channel="index-tickers"), + _binding(component=None, channel="trades"), + ], + field="test", + ) + + def test_rejects_an_incomplete_component_pair(self) -> None: + with self.assertRaisesRegex(ValueError, "incomplete mark_index"): + core_binding_map( + [_binding(component="INDEX", channel="index-tickers")], + field="test", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_crypto_demand_manifest.py b/tests/test_crypto_demand_manifest.py index 368db73..28ea26e 100644 --- a/tests/test_crypto_demand_manifest.py +++ b/tests/test_crypto_demand_manifest.py @@ -102,26 +102,22 @@ def test_demand_and_catalog_agree_in_both_directions(self): catalog = self._catalog_crypto_keys() self.assertTrue(demand <= catalog) - # Dated contracts are pre-registered for the shared Rust L2 core, but - # only the ten five-liquid perpetuals are active execution demand. A - # new dormant capability must still be an L2 book on a dated leg; no - # price/bar product may silently fall outside the active inventory. + # The stable catalog is a capability/compatibility inventory. Its + # retained Spot, dated, VN and V1 rows are not an activation request; + # only the sealed derivative demand below is V2-primary scope. dormant = catalog - demand self.assertTrue(dormant) - self.assertTrue(all( - feed in {FeedType.BOOK_SNAPSHOT.value, FeedType.BOOK_DELTA.value} - and (market == "FUTURES" or "_" in native_symbol) - for _venue, market, native_symbol, feed, _interval in dormant - )) + self.assertTrue(any(market == "SPOT" for _venue, market, _symbol, _feed, _interval in dormant)) + self.assertTrue(any(market == "FUTURES" for _venue, market, _symbol, _feed, _interval in dormant)) def test_every_crypto_family_in_the_catalog_is_expressible(self): families = { (item.venue, item.market, item.product_type) for item in self.manifest.demands } - self.assertIn(("BINANCE", "SPOT", "SPOT"), families) - self.assertIn(("BINANCE", "USDM", "PERPETUAL"), families) - self.assertIn(("OKX", "SPOT", "SPOT"), families) - self.assertIn(("OKX", "SWAP", "PERPETUAL"), families) + self.assertEqual( + families, + {("BINANCE", "USDM", "PERPETUAL"), ("OKX", "SWAP", "PERPETUAL")}, + ) def test_execution_l2_demand_is_bounded_and_live(self): books = [ diff --git a/tests/test_dlv2_r125_projector_batch_poll.py b/tests/test_dlv2_r125_projector_batch_poll.py index 6d6a6d3..920f9e5 100644 --- a/tests/test_dlv2_r125_projector_batch_poll.py +++ b/tests/test_dlv2_r125_projector_batch_poll.py @@ -7,7 +7,11 @@ """ import asyncio +import os +import subprocess +import sys import unittest +from pathlib import Path from qdl.runtime.stable_projector import poll_projector_records from qdl.transport.kafka_projector import KafkaProjectorRecord @@ -56,6 +60,20 @@ def poll(self, timeout_seconds): class PollProjectorRecordsTests(unittest.IsolatedAsyncioTestCase): + async def test_projector_import_does_not_depend_on_query_package_order(self): + environment = dict(os.environ) + environment["PYTHONDONTWRITEBYTECODE"] = "1" + completed = await asyncio.to_thread( + subprocess.run, + [sys.executable, "-c", "import qdl.runtime.stable_projector"], + cwd=Path(__file__).resolve().parents[1], + env=environment, + capture_output=True, + text=True, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + async def test_batch_broker_is_served_by_one_call(self): broker = _BatchBroker([[_record(1), _record(2), _record(3)]]) records = await poll_projector_records( diff --git a/tests/test_dlv2_r125_spool_wal_bound.py b/tests/test_dlv2_r125_spool_wal_bound.py index c40c7f4..b446eab 100644 --- a/tests/test_dlv2_r125_spool_wal_bound.py +++ b/tests/test_dlv2_r125_spool_wal_bound.py @@ -5,15 +5,20 @@ write then failed closed for two hours. The frames had already been checkpointed - PASSIVE recycles a WAL but never shrinks the file - so the outage was the file, not the retained data. These tests pin both halves of the -repair: retention work reclaims a WAL that outgrew journal_size_limit, and the -physical bound reclaims before it refuses a write. +repair: routine retention remains nonblocking, and the physical bound reclaims +before it refuses a write. """ import tempfile import unittest from pathlib import Path +from unittest.mock import patch -from qdl.transport.contracts import BackpressureRequired, DurableEvent +from qdl.transport.contracts import ( + BackpressureRequired, + DurableEvent, + FINAL_BAR_CLOSE_TIME_NS_HEADER, +) from qdl.transport.sqlite_spool import ( JOURNAL_SIZE_LIMIT_BYTES, SpoolConfig, @@ -31,6 +36,17 @@ def _event(index: int, payload: bytes) -> DurableEvent: ) +def _bar_event(index: int, close_time_ns: int) -> DurableEvent: + return DurableEvent( + stream="md.canonical.v2", + partition_key="OKX/SWAP/BTC-USDT-SWAP/bar/1m", + event_id=index.to_bytes(16, "big"), + payload=f"final-bar-{index}".encode(), + accepted_at_ns=1_700_000_000_000_000_000 + index, + headers={FINAL_BAR_CLOSE_TIME_NS_HEADER: str(close_time_ns)}, + ) + + class SpoolWalBoundTests(unittest.TestCase): def setUp(self): self.temp = tempfile.TemporaryDirectory() @@ -56,6 +72,133 @@ def test_journal_size_limit_is_declared_on_the_connection(self): limit = spool._connection.execute("PRAGMA journal_size_limit").fetchone()[0] self.assertEqual(int(limit), JOURNAL_SIZE_LIMIT_BYTES) + def test_final_bar_watermark_is_atomic_max_and_survives_reopen(self): + spool = self._spool() + partition = "OKX/SWAP/BTC-USDT-SWAP/bar/1m" + other_partition = "OKX/SWAP/ETH-USDT-SWAP/bar/1m" + self.assertIsNone( + spool.final_bar_watermark( + stream="md.canonical.v2", partition_key=partition + ) + ) + spool.append_many((_bar_event(1, 300), _bar_event(2, 200))) + self.assertEqual( + spool.final_bar_watermark( + stream="md.canonical.v2", partition_key=partition + ), + 300, + ) + self.assertEqual( + spool.seed_final_bar_watermark( + stream="md.canonical.v2", partition_key=partition, close_time_ns=250 + ), + 300, + ) + spool.append(DurableEvent( + stream="md.canonical.v2", + partition_key=other_partition, + event_id=b"o" * 16, + payload=b"final-bar-other-partition", + accepted_at_ns=2, + headers={FINAL_BAR_CLOSE_TIME_NS_HEADER: "900"}, + )) + self.assertEqual( + spool.final_bar_watermark( + stream="md.canonical.v2", partition_key=other_partition + ), + 900, + ) + self.assertEqual( + spool.final_bar_watermark( + stream="md.canonical.v2", partition_key=partition + ), + 300, + ) + spool.close() + reopened = self._spool() + self.assertEqual( + reopened.final_bar_watermark( + stream="md.canonical.v2", partition_key=partition + ), + 300, + ) + + def test_duplicate_event_can_hydrate_legacy_final_bar_watermark(self): + spool = self._spool() + legacy = DurableEvent( + stream="md.canonical.v2", + partition_key="OKX/SWAP/BTC-USDT-SWAP/bar/1m", + event_id=b"l" * 16, + payload=b"legacy-final-bar", + accepted_at_ns=1, + ) + spool.append(legacy) + self.assertIsNone( + spool.final_bar_watermark( + stream=legacy.stream, partition_key=legacy.partition_key + ) + ) + hydrated = DurableEvent( + stream=legacy.stream, + partition_key=legacy.partition_key, + event_id=legacy.event_id, + payload=legacy.payload, + accepted_at_ns=legacy.accepted_at_ns, + headers={FINAL_BAR_CLOSE_TIME_NS_HEADER: "400"}, + ) + result = spool.append(hydrated) + self.assertTrue(result.duplicate) + self.assertEqual( + spool.final_bar_watermark( + stream=legacy.stream, partition_key=legacy.partition_key + ), + 400, + ) + + def test_legacy_hydration_is_reused_after_a_second_spool_opens(self): + first = self._spool() + partition = "OKX/SWAP/BTC-USDT-SWAP/bar/1m" + calls = [] + self.assertEqual( + first.hydrate_final_bar_watermark( + stream="md.canonical.v2", + partition_key=partition, + legacy_lookup=lambda: calls.append("first") or 300, + ), + 300, + ) + self.assertEqual(calls, ["first"]) + + second = SQLiteDurableSpool(first.config) + self.addCleanup(second.close) + self.assertEqual( + second.hydrate_final_bar_watermark( + stream="md.canonical.v2", + partition_key=partition, + legacy_lookup=lambda: self.fail("second spool must not rescan legacy tail"), + ), + 300, + ) + + def test_malformed_final_bar_watermark_rolls_back_the_event(self): + spool = self._spool() + malformed = DurableEvent( + stream="md.canonical.v2", + partition_key="OKX/SWAP/BTC-USDT-SWAP/bar/1m", + event_id=b"m" * 16, + payload=b"malformed-final-bar", + accepted_at_ns=1, + headers={FINAL_BAR_CLOSE_TIME_NS_HEADER: "-1"}, + ) + with self.assertRaisesRegex(ValueError, "final BAR watermark header"): + spool.append(malformed) + self.assertEqual( + spool.read_tail( + stream=malformed.stream, partition_key=malformed.partition_key, limit=1 + ), + [], + ) + def test_truncate_checkpoint_reclaims_the_wal_file(self): spool = self._spool() spool.append_many([_event(index, b"p" * 4096) for index in range(200)]) @@ -68,6 +211,20 @@ def test_wal_bytes_reports_zero_without_a_wal_file(self): spool._checkpoint_wal_truncate_locked() self.assertEqual(spool._wal_bytes(), 0) + def test_routine_maintenance_never_runs_blocking_truncate(self): + spool = self._spool() + with patch.object( + spool, "_wal_bytes", return_value=JOURNAL_SIZE_LIMIT_BYTES + 1 + ), patch.object( + spool, "_checkpoint_wal_passive_locked", return_value=False + ) as passive, patch.object( + spool, "_checkpoint_wal_truncate_locked", return_value=True + ) as truncate: + spool.append(_event(1, b"p" * 4096)) + + passive.assert_called_once_with() + truncate.assert_not_called() + def test_storage_bytes_counts_every_physical_file(self): spool = self._spool() spool.append_many([_event(index, b"p" * 2048) for index in range(50)]) diff --git a/tests/test_execution_mark_index_consumer_latency.py b/tests/test_execution_mark_index_consumer_latency.py new file mode 100644 index 0000000..d494272 --- /dev/null +++ b/tests/test_execution_mark_index_consumer_latency.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import unittest +from pathlib import Path +from types import SimpleNamespace + +from qdl_sdk.reference import ReferenceProduct +from scripts.measure_execution_mark_index_consumer_latency import ( + _acceptance_gate, + execution_mark_index_requirements, + validate_live_response, +) + + +def _response( + requirements, + *, + now_ns: int, + endpoint: str = "qdl://stable-stream/internal/v2/execution/mark-index/latest", + **overrides, +): + labels = { + "execution_view": "STABLE_STREAM_GATEWAY", + "source_event_time_ns": str(now_ns - 61_000_000_000), + "provider_confirmation_ns": str(now_ns - 60_000_000_000), + "delivery_stage": "CANONICAL_READ_COMMITTED", + "event_recency_policy": "OBSERVE", + "recency_mode": "COMPONENT_SESSION_LIVE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": "10", + "provider_session_checked_at_ns": str(now_ns - 5_000_000), + "component_mark_received_at_ns": str(now_ns - 10_000_000_000), + "component_index_received_at_ns": str(now_ns - 60_000_000_000), + "component_mark_quiet_after_ms": "15000", + "component_index_quiet_after_ms": "70000", + } + labels.update({key: str(value) for key, value in overrides.items()}) + results = [] + for requirement in requirements: + observation = SimpleNamespace( + instrument_uid=requirement.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + fields=( + SimpleNamespace(name="mark_price"), + SimpleNamespace(name="index_price"), + ), + labels=dict(labels), + ) + data = SimpleNamespace( + status="OK", + instrument_uid=requirement.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + received_at_ns=int(labels["provider_confirmation_ns"]), + observations=(observation,), + lineage=(SimpleNamespace(provider_endpoint=endpoint),), + ) + results.append( + SimpleNamespace( + instrument_uid=requirement.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + status="OK", + problem=None, + data=data, + ) + ) + return SimpleNamespace(partial=False, results=tuple(results)) + + +class ExecutionMarkIndexConsumerLatencyTests(unittest.TestCase): + manifest = Path(__file__).resolve().parents[1] / "consumers/stable/trading-system-paper.yaml" + + def test_manifest_selects_exact_execution_mark_index_scope(self) -> None: + requirements = execution_mark_index_requirements(self.manifest) + self.assertEqual(len(requirements), 10) + self.assertTrue(all( + item.product is ReferenceProduct.MARK_INDEX_PRICE + and item.max_freshness_ms == 2_000 + and item.event_recency_policy.value == "OBSERVE" + and item.max_session_liveness_ms == 45_000 + for item in requirements + )) + + def test_live_response_rejects_non_internal_lineage(self) -> None: + requirements = execution_mark_index_requirements(self.manifest) + now_ns = 90_000_000_000 + response = _response(requirements, now_ns=now_ns) + response.results[0].data.lineage = ( + SimpleNamespace(provider_endpoint="https://provider.example/mark"), + ) + with self.assertRaisesRegex(ValueError, "internal live reader"): + validate_live_response( + requirements, + response, + usable_at_ns=now_ns, + ) + + def test_quiet_live_response_keeps_original_provider_time_without_failing_latency_gate( + self, + ) -> None: + requirements = execution_mark_index_requirements(self.manifest) + now_ns = 90_000_000_000 + values = validate_live_response( + requirements, + _response(requirements, now_ns=now_ns), + usable_at_ns=now_ns, + ) + self.assertEqual(len(values), 10) + for evidence in values.values(): + self.assertEqual(evidence["recency_mode"], "COMPONENT_SESSION_LIVE") + self.assertEqual(evidence["provider_confirmation_to_usable_ms"], 60_000) + self.assertLess(evidence["provider_session_liveness_to_usable_ms"], 45_000) + self.assertLess(evidence["component_mark_age_to_usable_ms"], 15_000) + self.assertLess(evidence["component_index_age_to_usable_ms"], 70_000) + + def test_strict_event_session_response_is_validated_without_changing_lineage(self) -> None: + requirements = execution_mark_index_requirements(self.manifest) + now_ns = 90_000_000_000 + values = validate_live_response( + requirements, + _response( + requirements, + now_ns=now_ns, + provider_confirmation_ns=now_ns - 1_000_000, + source_event_time_ns=now_ns - 2_000_000, + component_mark_received_at_ns=now_ns - 1_000_000, + component_index_received_at_ns=now_ns - 1_000_000, + recency_mode="STRICT_EVENT_SESSION_LIVE", + ), + usable_at_ns=now_ns, + ) + self.assertEqual( + {item["recency_mode"] for item in values.values()}, + {"STRICT_EVENT_SESSION_LIVE"}, + ) + + def test_quiet_response_rejects_session_or_component_failure(self) -> None: + requirements = execution_mark_index_requirements(self.manifest) + now_ns = 90_000_000_000 + with self.subTest("disconnected"): + with self.assertRaisesRegex(ValueError, "not live"): + validate_live_response( + requirements, + _response( + requirements, + now_ns=now_ns, + provider_session_state="DISCONNECTED", + ), + usable_at_ns=now_ns, + ) + with self.subTest("expired_component"): + with self.assertRaisesRegex(ValueError, "quiet cadence"): + validate_live_response( + requirements, + _response( + requirements, + now_ns=now_ns, + component_index_received_at_ns=now_ns - 70_001_000_000, + ), + usable_at_ns=now_ns, + ) + with self.subTest("expired_session"): + with self.assertRaisesRegex(ValueError, "session exceeded"): + validate_live_response( + requirements, + _response( + requirements, + now_ns=now_ns, + provider_session_checked_at_ns=now_ns - 45_001_000_000, + ), + usable_at_ns=now_ns, + ) + + with self.subTest("missing_session"): + response = _response(requirements, now_ns=now_ns) + for item in response.results: + del item.data.observations[0].labels["provider_session_checked_at_ns"] + with self.assertRaisesRegex(ValueError, "malformed"): + validate_live_response( + requirements, + response, + usable_at_ns=now_ns, + ) + + def test_gate_uses_consumer_latency_not_old_immutable_provider_lineage(self) -> None: + requirements = execution_mark_index_requirements(self.manifest) + per_binding = { + requirement.instrument_uid: { + "provider_confirmation_to_usable_ms": {"n": 149, "p99_ms": 70_000}, + "provider_session_liveness_to_usable_ms": {"n": 149}, + "component_mark_age_to_usable_ms": {"n": 149}, + "component_index_age_to_usable_ms": {"n": 149}, + } + for requirement in requirements + } + minimum_samples, passed = _acceptance_gate( + { + "errors": [], + "per_binding": per_binding, + "consumer_call_to_usable_ms": {"n": 149, "p99_ms": 281}, + }, + requirement_count=10, + duration_seconds=300, + cadence_seconds=2, + max_consumer_call_p99_ms=2_000, + ) + self.assertEqual(minimum_samples, 149) + self.assertTrue(passed) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_execution_mark_index_live_view.py b/tests/test_execution_mark_index_live_view.py new file mode 100644 index 0000000..42b003a --- /dev/null +++ b/tests/test_execution_mark_index_live_view.py @@ -0,0 +1,1468 @@ +"""Execution MARK/INDEX live-view contract tests. + +All market values in this file are deterministic test provenance. They prove +the private current-state boundary, not provider latency or a real market read. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +import httpx +from fastapi import FastAPI + +from qdl.common.v1 import common_pb2 +from qdl.domain.capabilities import CapabilityAvailability, FeedCapability +from qdl.domain.decimal import CanonicalDecimal +from qdl.domain.instrument import ( + AssetClass, + InstrumentIdentity, + InstrumentRecord, + InstrumentRegistry, + ProductType, +) +from qdl.marketdata.v2 import market_data_pb2 +from qdl.query import ( + AccessPurpose, + ConsumerGrade, + DataProduct, + EntitlementGrant, + EntitlementPolicy, + FeedType, + InstrumentQuery, + MemoryMarketDataBackend, + V2QueryService, +) +from qdl.query.contracts import StalePolicy +from qdl.query.reference import ReferenceBatchRequirement, ReferenceDataRequirement +from qdl.reference.batch import ReferenceBatch +from qdl.reference.contracts import ( + MarkIndexKind, + ReferenceBatchResult, + ReferenceCoverage, + ReferenceFetch, + ReferenceLineage, + ReferenceObservation, + ReferenceProduct, + ReferenceRequest, + ReferenceStatus, + decimal_field, +) +from qdl.reference.execution_live import HttpExecutionMarkIndexReader +from qdl.runtime.execution_mark_index import ( + ExecutionMarkIndexQuietPolicy, + ExecutionMarkIndexLiveView, + install_execution_mark_index_read, +) +from qdl.runtime.internal_auth import stable_hmac_signature +from qdl.runtime.lease import GatewayFenced +from qdl.runtime.stable_catalog import StableSourceBinding +from qdl.runtime.session_liveness import StableSessionLivenessReader +from qdl.transport import Cursor, DurableEvent, StoredEvent + + +NOW_NS = 1_800_000_000_000_000_000 +SECRET = b"execution-mark-index-live-view-test-secret" +STREAM = "md.canonical.execution-mark-index-test.v2" +LIVE_ENDPOINT = "qdl://stable-stream/internal/v2/execution/mark-index/latest" + + +def _record(*, venue: str, market: str, native_symbol: str, base: str) -> InstrumentRecord: + identity = InstrumentIdentity.create( + venue=venue, + market=market, + product_type=ProductType.PERPETUAL, + canonical_symbol=f"{base}-USDT", + ) + return InstrumentRecord( + identity=identity, + metadata_revision=7, + asset_class=AssetClass.DERIVATIVE, + native_symbol=native_symbol, + base_asset=base, + quote_asset="USDT", + settlement_asset="USDT", + price_tick=CanonicalDecimal.from_text("0.01"), + quantity_step=CanonicalDecimal.from_text("0.001"), + contract_multiplier=CanonicalDecimal.from_text("1"), + session_calendar_id="CRYPTO_24X7", + ) + + +def _binding(record: InstrumentRecord, *, source_policy_id: str = "crypto_liquid_v2"): + provider = "BINANCE_DIRECT" if record.identity.venue == "BINANCE" else "OKX_DIRECT" + return StableSourceBinding( + binding_id=f"execution-mark-index-{record.native_symbol.lower()}", + instrument=record, + provider=provider, + source_id=provider, + source_role="PRIMARY", + source_policy_id=source_policy_id, + authoritative=True, + adapter_version="execution-mark-index-test/1", + normalizer_version="execution-mark-index-core-test/1", + feed=FeedType.MARK_INDEX_PRICE, + interval=None, + stale_after_ms=2_000, + require_final_bar=False, + continuous_calendar=True, + v1_compatibility="NONE", + canonical_stream=STREAM, + freshness_basis="PROVIDER_CONFIRMATION", + ) + + +def _envelope( + binding: StableSourceBinding, + *, + sequence: int, + generation: int = 1, + received_at_ns: int = NOW_NS, + quality_flags: tuple[int, ...] = (), +) -> market_data_pb2.EventEnvelope: + raw = hashlib.sha256( + f"execution-mark-index:{binding.instrument.instrument_uid}:{sequence}:{generation}".encode() + ).digest() + record = binding.instrument + envelope = market_data_pb2.EventEnvelope( + schema_name="qdl.marketdata.v2", + schema_major=2, + schema_minor=0, + event_id=raw[:16], + instrument_uid=record.instrument_uid, + instrument_id=record.instrument_id, + instrument_revision=record.metadata_revision, + venue=record.identity.venue, + market=record.identity.market, + product_type=record.identity.product_type.value, + native_symbol=record.native_symbol, + provider=binding.provider, + source_id=binding.source_id, + source_role=common_pb2.SOURCE_ROLE_PRIMARY, + lease_epoch=5, + source_event_time_ns=received_at_ns - 1_000_000, + received_at_ns=received_at_ns, + normalized_at_ns=received_at_ns + 1, + published_at_ns=received_at_ns + 2, + source_sequence=f"test:{generation}:{sequence}", + partition_sequence=sequence, + normalizer_version=binding.normalizer_version, + adapter_version=binding.adapter_version, + raw_capture_id=raw[:16], + raw_payload_hash=raw, + correlation_id=raw.hex(), + config_revision=11, + source_session_id="execution-mark-index-test-session", + connection_generation=generation, + authority_revision=3, + partition_plan_epoch=1, + ) + envelope.quality_flags.extend(quality_flags) + envelope.mark_index_price.mark_price.source_text = "101.25" + envelope.mark_index_price.index_price.source_text = "101.00" + envelope.canonical_payload_hash = hashlib.sha256( + envelope.mark_index_price.SerializeToString(deterministic=True) + ).digest() + return envelope + + +def _paired_envelope( + binding: StableSourceBinding, + *, + sequence: int, + generation: int = 1, + mark_received_at_ns: int, + index_received_at_ns: int, +) -> market_data_pb2.EventEnvelope: + """Build deterministic Rust-shaped pair lineage for quiet-contract tests.""" + + envelope = _envelope( + binding, + sequence=sequence, + generation=generation, + received_at_ns=min(mark_received_at_ns, index_received_at_ns), + ) + mark_capture = hashlib.sha256( + f"mark:{binding.instrument.instrument_uid}:{sequence}".encode() + ).digest()[:16] + index_capture = hashlib.sha256( + f"index:{binding.instrument.instrument_uid}:{sequence}".encode() + ).digest()[:16] + source_times = ( + mark_received_at_ns // 1_000_000, + index_received_at_ns // 1_000_000, + ) + envelope.source_event_time_ns = min(source_times) * 1_000_000 + envelope.received_at_ns = min(mark_received_at_ns, index_received_at_ns) + envelope.normalized_at_ns = envelope.received_at_ns + 1 + envelope.published_at_ns = envelope.received_at_ns + 2 + envelope.source_sequence = ":".join( + str(value) + for value in ( + *source_times, + mark_received_at_ns, + index_received_at_ns, + mark_capture.hex(), + index_capture.hex(), + ) + ) + capture = hashlib.sha256() + capture.update(b"qdl-mark-index-capture-v1") + capture.update(mark_capture) + capture.update(index_capture) + envelope.raw_capture_id = capture.digest()[:16] + envelope.raw_payload_hash = hashlib.sha256( + b"paired-mark-index-test" + mark_capture + index_capture + ).digest() + return envelope + + +def _write_session( + root: Path, + envelope: market_data_pb2.EventEnvelope, + *, + state: str = "LIVE", + generation: int | None = None, + config_revision: int | None = None, + last_transport_at_ns: int = NOW_NS - 1_000_000, +) -> None: + directory = root / f"{envelope.venue.lower()}-{envelope.market.lower()}" + directory.mkdir(parents=True, exist_ok=True) + (directory / "source.json").write_text( + json.dumps({ + "schema": "qdl.provider-session-liveness.v1", + "source_session_id": envelope.source_session_id, + "connection_generation": ( + envelope.connection_generation if generation is None else generation + ), + "state": state, + "last_transport_at_ns": last_transport_at_ns, + "updated_at_ns": last_transport_at_ns, + "config_revision": ( + envelope.config_revision + if config_revision is None else config_revision + ), + }), + encoding="utf-8", + ) + + +def _stored(envelope: market_data_pb2.EventEnvelope, *, offset: int) -> StoredEvent: + payload = envelope.SerializeToString(deterministic=True) + event = DurableEvent( + stream=STREAM, + partition_key=f"execution/{envelope.instrument_uid}", + event_id=bytes(envelope.event_id), + payload=payload, + accepted_at_ns=envelope.received_at_ns, + ) + return StoredEvent( + event=event, + cursor=Cursor(STREAM, event.partition_key, offset), + committed_at_ns=envelope.received_at_ns, + payload_sha256=hashlib.sha256(payload).hexdigest(), + ) + + +def _hydration_stored( + binding: StableSourceBinding, + envelope: market_data_pb2.EventEnvelope, + *, + offset: int, +) -> StoredEvent: + """One durable canonical row in the exact physical partition under test.""" + + payload = envelope.SerializeToString(deterministic=True) + event = DurableEvent( + stream=STREAM, + partition_key=binding.partition_key, + event_id=bytes(envelope.event_id), + payload=payload, + accepted_at_ns=envelope.received_at_ns, + ) + return StoredEvent( + event=event, + cursor=Cursor(STREAM, event.partition_key, offset), + committed_at_ns=envelope.received_at_ns, + payload_sha256=hashlib.sha256(payload).hexdigest(), + ) + + +class _Gateway: + def __init__(self, epoch: int = 5) -> None: + self.epoch = epoch + self.fenced = False + + def assert_active(self, expected_epoch: int | None = None) -> int: + if self.fenced or (expected_epoch is not None and expected_epoch != self.epoch): + raise GatewayFenced("test gateway fenced") + return self.epoch + + +class _FallbackReferenceAdapter: + def __init__(self) -> None: + self.calls = 0 + + async def fetch(self, request, *, capability, received_at_ns): + del capability + self.calls += 1 + fields = tuple( + field + for field in ( + decimal_field("mark_price", "101.25", "QUOTE_PRICE"), + decimal_field("index_price", "101.00", "QUOTE_PRICE"), + ) + if field is not None + ) + return ReferenceFetch( + observations=(ReferenceObservation( + instrument_uid=request.instrument.instrument_uid, + instrument_revision=request.instrument.metadata_revision, + product=request.product, + observed_at_ns=NOW_NS, + fields=fields, + ),), + lineage=(ReferenceLineage( + provider="BINANCE_DIRECT", + provider_endpoint="TEST_REFERENCE_REST_ADAPTER", + source_role="REFERENCE", + adapter_version="execution-mark-index-test/1", + capability_name="mark_index_price", + ),), + coverage=ReferenceCoverage( + requested_start_ms=None, + requested_end_ms=None, + observed_min_ms=NOW_NS // 1_000_000, + observed_max_ms=NOW_NS // 1_000_000, + complete_left=True, + complete_right=True, + truncated=False, + terminal_reason="TEST_CURRENT", + ), + ) + + +class _LiveReader: + def __init__( + self, + *, + status: ReferenceStatus = ReferenceStatus.OK, + source_event_time_ns: int = NOW_NS, + provider_confirmation_ns: int = NOW_NS, + freshness_basis: str = "SOURCE_EVENT", + ) -> None: + self.status = status + self.calls = 0 + self.calls_by_policy: list[tuple[str, int, int | None]] = [] + self.calls_by_recency: list[tuple[StalePolicy, int | None]] = [] + self.source_event_time_ns = source_event_time_ns + self.provider_confirmation_ns = provider_confirmation_ns + self.freshness_basis = freshness_basis + + async def fetch( + self, + request, + *, + max_freshness_ms, + source_policy_id, + event_recency_policy=StalePolicy.BLOCK, + max_session_liveness_ms=None, + deadline_ms=None, + ): + self.calls += 1 + self.calls_by_policy.append((source_policy_id, max_freshness_ms, deadline_ms)) + self.calls_by_recency.append((event_recency_policy, max_session_liveness_ms)) + capability = FeedCapability(CapabilityAvailability.AVAILABLE, snapshot=True) + lineage = ReferenceLineage( + provider="BINANCE_DIRECT", + provider_endpoint=LIVE_ENDPOINT, + source_role="REFERENCE", + adapter_version="execution-mark-index-live-test/1", + capability_name="mark_index_price", + ) + if self.status is not ReferenceStatus.OK: + return ReferenceBatchResult( + request=request, + status=self.status, + capability=capability, + lineage=(lineage,), + coverage=ReferenceCoverage( + requested_start_ms=None, + requested_end_ms=None, + observed_min_ms=None, + observed_max_ms=None, + complete_left=False, + complete_right=False, + truncated=False, + terminal_reason="LIVE_VIEW_STALE", + ), + received_at_ns=self.provider_confirmation_ns, + error_code="LIVE_VIEW_STALE", + error_detail="test stale live view", + ) + fields = tuple( + field + for field in ( + decimal_field("mark_price", "101.25", "QUOTE_PRICE"), + decimal_field("index_price", "101.00", "QUOTE_PRICE"), + ) + if field is not None + ) + labels = [ + ("freshness_basis", self.freshness_basis), + ("provider_confirmation_ns", str(self.provider_confirmation_ns)), + ] + if event_recency_policy is StalePolicy.OBSERVE: + labels.extend(( + ("event_recency_policy", "OBSERVE"), + ("recency_mode", "COMPONENT_SESSION_LIVE"), + ("provider_session_state", "LIVE"), + ("provider_session_liveness_ms", "1"), + ("provider_session_checked_at_ns", str(NOW_NS)), + ("component_mark_received_at_ns", str(NOW_NS - 10_000_000_000)), + ("component_index_received_at_ns", str(NOW_NS - 60_000_000_000)), + ("component_mark_quiet_after_ms", "15000"), + ("component_index_quiet_after_ms", "70000"), + )) + return ReferenceBatchResult( + request=request, + status=ReferenceStatus.OK, + capability=capability, + lineage=(lineage,), + coverage=ReferenceCoverage( + requested_start_ms=None, + requested_end_ms=None, + observed_min_ms=NOW_NS // 1_000_000, + observed_max_ms=NOW_NS // 1_000_000, + complete_left=True, + complete_right=True, + truncated=False, + terminal_reason="LIVE_EXECUTION_VIEW", + ), + received_at_ns=self.provider_confirmation_ns, + observations=(ReferenceObservation( + instrument_uid=request.instrument.instrument_uid, + instrument_revision=request.instrument.metadata_revision, + product=request.product, + observed_at_ns=self.source_event_time_ns, + fields=fields, + labels=tuple(labels), + ),), + ) + + def stats(self) -> dict[str, int]: + return {"calls": self.calls, "successes": self.calls if self.status is ReferenceStatus.OK else 0, + "failures": 0 if self.status is ReferenceStatus.OK else self.calls} + + +class StableCatalogImportTests(unittest.TestCase): + def test_direct_catalog_import_does_not_form_a_runtime_cycle(self): + result = subprocess.run( + [ + sys.executable, + "-c", + "from qdl.runtime.stable_catalog import StableSourceCatalog; " + "assert StableSourceCatalog.__name__ == 'StableSourceCatalog'", + ], + cwd=Path(__file__).resolve().parents[1], + check=False, + capture_output=True, + text=True, + timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +class ExecutionMarkIndexLiveViewTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.record = _record( + venue="BINANCE", market="USDM", native_symbol="BTCUSDT", base="BTC" + ) + self.binding = _binding(self.record) + self.view = ExecutionMarkIndexLiveView(frozenset({self.record.instrument_uid})) + + async def test_generation_gap_fence_and_exact_identity_are_fail_closed(self): + first = _envelope(self.binding, sequence=1, generation=2) + await self.view.remember( + binding=self.binding, envelope=first, stored=_stored(first, offset=10), gateway_epoch=5 + ) + ready = await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + ) + self.assertIsNotNone(ready.record) + + delayed_old_generation = _envelope( + self.binding, sequence=2, generation=1, received_at_ns=NOW_NS + 1_000_000 + ) + await self.view.remember( + binding=self.binding, + envelope=delayed_old_generation, + stored=_stored(delayed_old_generation, offset=11), + gateway_epoch=5, + ) + unchanged = await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + ) + self.assertEqual(unchanged.record.connection_generation, 2) + + gap = _envelope( + self.binding, + sequence=3, + generation=2, + received_at_ns=NOW_NS + 2_000_000, + quality_flags=(common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE,), + ) + await self.view.remember( + binding=self.binding, envelope=gap, stored=_stored(gap, offset=12), gateway_epoch=5 + ) + self.assertEqual( + (await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).reason, + "GAP_OR_RESYNC", + ) + same_generation = _envelope( + self.binding, sequence=4, generation=2, received_at_ns=NOW_NS + 3_000_000 + ) + await self.view.remember( + binding=self.binding, + envelope=same_generation, + stored=_stored(same_generation, offset=13), + gateway_epoch=5, + ) + self.assertEqual( + (await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).reason, + "GAP_OR_RESYNC", + ) + recovered = _envelope( + self.binding, sequence=5, generation=3, received_at_ns=NOW_NS + 4_000_000 + ) + await self.view.remember( + binding=self.binding, + envelope=recovered, + stored=_stored(recovered, offset=14), + gateway_epoch=5, + ) + self.assertIsNotNone((await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).record) + + self.assertEqual( + (await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=8, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).reason, + "IDENTITY_MISMATCH", + ) + self.assertEqual( + (await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="different-policy", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).reason, + "SOURCE_POLICY_MISMATCH", + ) + self.assertEqual( + (await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 3_000_000_000, + )).reason, + "STALE", + ) + await self.view.fence_all() + self.assertEqual( + (await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).reason, + "NOT_READY", + ) + + async def test_durable_hydration_restores_only_the_exact_latest_binding(self): + envelope = _envelope(self.binding, sequence=11, generation=4) + stored = _hydration_stored(self.binding, envelope, offset=44) + + class _Spool: + def __init__(self): + self.calls = [] + + def read_tail(self, *, stream, partition_key, limit): + self.calls.append((stream, partition_key, limit)) + return [stored] + + view = ExecutionMarkIndexLiveView( + frozenset({self.record.instrument_uid}), + bindings={self.record.instrument_uid: self.binding}, + ) + restored = await view.hydrate_from_spool( + spool=_Spool(), canonical_stream=STREAM, gateway_epoch=9 + ) + self.assertEqual(restored, 1) + result = await view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=self.record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=9, + now_ns=NOW_NS + 500_000_000, + ) + self.assertEqual(result.record.delivery_stage, "SPOOL_CONFIRMED") + self.assertEqual(result.record.spool_watermark_offset, 44) + + async def test_durable_hydration_preserves_gap_and_identity_fences(self): + gap = _envelope( + self.binding, + sequence=12, + generation=4, + quality_flags=(common_pb2.QUALITY_FLAG_SEQUENCE_GAP_BEFORE,), + ) + stored = _hydration_stored(self.binding, gap, offset=45) + + class _Spool: + def read_tail(self, *, stream, partition_key, limit): + return [stored] + + view = ExecutionMarkIndexLiveView( + frozenset({self.record.instrument_uid}), + bindings={self.record.instrument_uid: self.binding}, + ) + self.assertEqual( + await view.hydrate_from_spool( + spool=_Spool(), canonical_stream=STREAM, gateway_epoch=9 + ), + 1, + ) + self.assertEqual( + (await view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=self.record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=9, + now_ns=NOW_NS + 500_000_000, + )).reason, + "GAP_OR_RESYNC", + ) + + async def test_durable_hydration_never_overwrites_a_newer_live_record(self): + durable = _envelope( + self.binding, + sequence=13, + generation=4, + received_at_ns=NOW_NS - 100_000_000, + ) + stored = _hydration_stored(self.binding, durable, offset=46) + + class _Spool: + def read_tail(self, *, stream, partition_key, limit): + return [stored] + + view = ExecutionMarkIndexLiveView( + frozenset({self.record.instrument_uid}), + bindings={self.record.instrument_uid: self.binding}, + ) + await view.hydrate_from_spool( + spool=_Spool(), canonical_stream=STREAM, gateway_epoch=9 + ) + current = _envelope( + self.binding, + sequence=14, + generation=4, + received_at_ns=NOW_NS, + ) + await view.remember( + binding=self.binding, + envelope=current, + stored=_hydration_stored(self.binding, current, offset=47), + gateway_epoch=9, + ) + await view.hydrate_from_spool( + spool=_Spool(), canonical_stream=STREAM, gateway_epoch=9 + ) + result = await view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=self.record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=9, + now_ns=NOW_NS + 500_000_000, + ) + self.assertEqual(result.record.event_id, bytes(current.event_id)) + + async def test_quiet_pair_is_session_bound_and_preserves_component_lineage(self): + """Only a declared, healthy paired provider session may admit quiet data.""" + + record = _record( + venue="OKX", market="SWAP", native_symbol="DOGE-USDT-SWAP", base="DOGE" + ) + binding = _binding(record) + envelope = _paired_envelope( + binding, + sequence=1, + generation=3, + mark_received_at_ns=NOW_NS - 10_000_000_000, + index_received_at_ns=NOW_NS - 60_000_000_000, + ) + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + view = ExecutionMarkIndexLiveView( + frozenset({record.instrument_uid}), + quiet_policies={ + record.instrument_uid: ExecutionMarkIndexQuietPolicy(( + ("MARK", 15_000), ("INDEX", 70_000), + )), + }, + session_liveness_reader=StableSessionLivenessReader(root), + ) + await view.remember( + binding=binding, + envelope=envelope, + stored=_stored(envelope, offset=31), + gateway_epoch=5, + ) + _write_session(root, envelope) + quiet = await view.read( + instrument_uid=record.instrument_uid, + instrument_revision=record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + now_ns=NOW_NS, + ) + self.assertIsNotNone(quiet.record) + self.assertEqual(quiet.recency_mode, "COMPONENT_SESSION_LIVE") + self.assertEqual( + quiet.record.source_event_time_ns, + envelope.source_event_time_ns, + ) + self.assertEqual( + dict(quiet.component_receipts_ns), + {"MARK": NOW_NS - 10_000_000_000, "INDEX": NOW_NS - 60_000_000_000}, + ) + + strict = await view.read( + instrument_uid=record.instrument_uid, + instrument_revision=record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS, + ) + self.assertEqual(strict.reason, "STALE") + + _write_session(root, envelope, state="DISCONNECTED") + self.assertEqual( + (await view.read( + instrument_uid=record.instrument_uid, + instrument_revision=record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + now_ns=NOW_NS, + )).reason, + "SESSION_STATE", + ) + + _write_session(root, envelope, generation=4) + self.assertEqual( + (await view.read( + instrument_uid=record.instrument_uid, + instrument_revision=record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + now_ns=NOW_NS, + )).reason, + "SESSION_STATE", + ) + + _write_session(root, envelope) + expired_mark = _paired_envelope( + binding, + sequence=2, + generation=3, + mark_received_at_ns=NOW_NS - 16_000_000_000, + index_received_at_ns=NOW_NS - 60_000_000_000, + ) + await view.remember( + binding=binding, + envelope=expired_mark, + stored=_stored(expired_mark, offset=32), + gateway_epoch=5, + ) + self.assertEqual( + (await view.read( + instrument_uid=record.instrument_uid, + instrument_revision=record.metadata_revision, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + now_ns=NOW_NS, + )).reason, + "COMPONENT_STALE", + ) + + async def test_pre_spool_record_is_readable_then_withdrawn_or_promoted_exactly(self): + first = _envelope(self.binding, sequence=1, generation=2) + await self.view.remember( + binding=self.binding, envelope=first, stored=None, gateway_epoch=5 + ) + pre_spool = await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + ) + self.assertEqual(pre_spool.record.delivery_stage, "CANONICAL_READ_COMMITTED") + self.assertIsNone(pre_spool.record.spool_watermark_offset) + + # A failed append can only withdraw its own current unconfirmed record; + # a later record must not disappear with it. + later = _envelope( + self.binding, sequence=2, generation=2, received_at_ns=NOW_NS + 1_000_000 + ) + await self.view.remember( + binding=self.binding, envelope=later, stored=None, gateway_epoch=5 + ) + await self.view.withdraw( + instrument_uid=self.record.instrument_uid, + event_id=bytes(first.event_id), + gateway_epoch=5, + ) + retained = await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + ) + self.assertEqual(retained.record.event_id, bytes(later.event_id)) + + await self.view.remember( + binding=self.binding, + envelope=later, + stored=_stored(later, offset=12), + gateway_epoch=5, + ) + confirmed = await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + ) + self.assertEqual(confirmed.record.delivery_stage, "SPOOL_CONFIRMED") + self.assertEqual(confirmed.record.spool_watermark_offset, 12) + await self.view.withdraw( + instrument_uid=self.record.instrument_uid, + event_id=bytes(later.event_id), + gateway_epoch=5, + ) + self.assertIsNotNone((await self.view.read( + instrument_uid=self.record.instrument_uid, + instrument_revision=7, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + gateway_epoch=5, + now_ns=NOW_NS + 500_000_000, + )).record) + + async def test_private_endpoint_and_reader_preserve_pair_and_never_call_venue(self): + envelope = _envelope(self.binding, sequence=1) + await self.view.remember( + binding=self.binding, envelope=envelope, stored=None, gateway_epoch=5 + ) + gateway = _Gateway() + app = FastAPI() + install_execution_mark_index_read(app, gateway=gateway, view=self.view, secret=SECRET) + body = json.dumps( + { + "schema": "qdl.v2.execution-mark-index-read.v1", + "instrument_uid": self.record.instrument_uid, + "instrument_revision": 7, + "source_policy_id": "crypto_liquid_v2", + "max_freshness_ms": 2_000, + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + denied = await client.post("/internal/v2/execution/mark-index/latest", content=body) + self.assertEqual(denied.status_code, 401) + response = await client.post( + "/internal/v2/execution/mark-index/latest", + content=body, + headers={"X-QDL-Stable-Signature": stable_hmac_signature(SECRET, body)}, + ) + self.assertEqual(response.status_code, 200) + payload = response.json() + self.assertEqual(payload["schema"], "qdl.v2.execution-mark-index-view.v2") + self.assertEqual(payload["delivery_stage"], "CANONICAL_READ_COMMITTED") + self.assertIsNone(payload["spool_watermark_offset"]) + self.assertEqual( + response.headers["X-QDL-Execution-Freshness-Basis"], + "PROVIDER_CONFIRMATION", + ) + self.assertEqual( + market_data_pb2.EventEnvelope.FromString(base64.b64decode(payload["canonical"])).instrument_uid, + self.record.instrument_uid, + ) + + venue_calls = [] + + async def stream_handler(request: httpx.Request) -> httpx.Response: + venue_calls.append(str(request.url)) + self.assertEqual(request.url.host, "stream_v2_active") + return httpx.Response( + 200, + json=payload, + headers={"X-QDL-Execution-Freshness-Basis": "PROVIDER_CONFIRMATION"}, + request=request, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(stream_handler)) + reader = HttpExecutionMarkIndexReader( + ("https://stream_v2_active:8200",), SECRET, client=client + ) + try: + result = await reader.fetch( + ReferenceRequest( + self.record, ReferenceProduct.MARK_INDEX_PRICE, mark_index_kind=MarkIndexKind.BOTH + ), + max_freshness_ms=2_000, + source_policy_id="crypto_liquid_v2", + ) + finally: + await client.aclose() + self.assertEqual(result.status, ReferenceStatus.OK) + self.assertEqual({field.name for field in result.observations[0].fields}, {"mark_price", "index_price"}) + self.assertEqual(result.lineage[0].provider_endpoint, LIVE_ENDPOINT) + self.assertEqual( + dict(result.observations[0].labels)["freshness_basis"], + "PROVIDER_CONFIRMATION", + ) + self.assertEqual(len(venue_calls), 1) + + async def test_reader_accepts_both_venues_but_rejects_cross_venue_identity(self): + records = ( + _record( + venue="BINANCE", market="USDM", native_symbol="SOLUSDT", base="SOL" + ), + _record( + venue="OKX", market="SWAP", native_symbol="SOL-USDT-SWAP", base="SOL" + ), + ) + for sequence, record in enumerate(records, start=1): + with self.subTest(venue=record.identity.venue): + binding = _binding(record) + envelope = _envelope(binding, sequence=sequence) + payload = { + "schema": "qdl.v2.execution-mark-index-view.v2", + "lease_epoch": 5, + "spool_watermark_offset": sequence, + "delivery_stage": "SPOOL_CONFIRMED", + "canonical": base64.b64encode( + envelope.SerializeToString(deterministic=True) + ).decode("ascii"), + } + + async def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=payload, request=request) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + reader = HttpExecutionMarkIndexReader( + ("https://stream_v2_active:8200",), SECRET, client=client + ) + try: + result = await reader.fetch( + ReferenceRequest( + record, + ReferenceProduct.MARK_INDEX_PRICE, + mark_index_kind=MarkIndexKind.BOTH, + ), + max_freshness_ms=2_000, + source_policy_id="crypto_liquid_v2", + ) + finally: + await client.aclose() + self.assertEqual(result.status, ReferenceStatus.OK) + self.assertEqual( + result.observations[0].instrument_uid, record.instrument_uid + ) + + expected = records[0] + wrong = _envelope(_binding(records[1]), sequence=99) + payload = { + "schema": "qdl.v2.execution-mark-index-view.v2", + "lease_epoch": 5, + "spool_watermark_offset": 99, + "delivery_stage": "SPOOL_CONFIRMED", + "canonical": base64.b64encode( + wrong.SerializeToString(deterministic=True) + ).decode("ascii"), + } + + async def wrong_handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=payload, request=request) + + client = httpx.AsyncClient(transport=httpx.MockTransport(wrong_handler)) + reader = HttpExecutionMarkIndexReader( + ("https://stream_v2_active:8200",), SECRET, client=client + ) + try: + result = await reader.fetch( + ReferenceRequest(expected, ReferenceProduct.MARK_INDEX_PRICE), + max_freshness_ms=2_000, + source_policy_id="crypto_liquid_v2", + ) + finally: + await client.aclose() + self.assertEqual(result.status, ReferenceStatus.ERROR) + self.assertEqual(result.error_code, "LIVE_VIEW_PROTOCOL") + + async def test_reader_rejects_missing_session_provenance_for_quiet_request(self): + envelope = _envelope(self.binding, sequence=98) + envelope.source_session_id = "" + payload = { + "schema": "qdl.v2.execution-mark-index-view.v2", + "lease_epoch": 5, + "spool_watermark_offset": 98, + "delivery_stage": "SPOOL_CONFIRMED", + "canonical": base64.b64encode( + envelope.SerializeToString(deterministic=True) + ).decode("ascii"), + } + + async def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json=payload, + headers={ + "X-QDL-Execution-Freshness-Basis": "PROVIDER_CONFIRMATION", + "X-QDL-Execution-Recency-Mode": "COMPONENT_SESSION_LIVE", + "X-QDL-Execution-Session-State": "LIVE", + "X-QDL-Execution-Session-Liveness-Ms": "1", + "X-QDL-Execution-Session-Checked-At-Ns": str(NOW_NS), + "X-QDL-Execution-Component-Receipts-Ns": ( + f"INDEX={NOW_NS - 1},MARK={NOW_NS - 1}" + ), + "X-QDL-Execution-Component-Quiet-After-Ms": ( + "INDEX=70000,MARK=15000" + ), + }, + request=request, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + reader = HttpExecutionMarkIndexReader( + ("https://stream_v2_active:8200",), SECRET, client=client + ) + try: + result = await reader.fetch( + ReferenceRequest(self.record, ReferenceProduct.MARK_INDEX_PRICE), + max_freshness_ms=2_000, + source_policy_id="crypto_liquid_v2", + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + ) + finally: + await client.aclose() + self.assertEqual(result.status, ReferenceStatus.ERROR) + self.assertEqual(result.error_code, "LIVE_VIEW_PROTOCOL") + + async def test_reader_uses_one_deadline_across_active_passive_urls(self): + envelope = _envelope(self.binding, sequence=50) + payload = { + "schema": "qdl.v2.execution-mark-index-view.v2", + "lease_epoch": 5, + "spool_watermark_offset": 50, + "delivery_stage": "SPOOL_CONFIRMED", + "canonical": base64.b64encode( + envelope.SerializeToString(deterministic=True) + ).decode("ascii"), + } + calls = [] + + async def handler(request: httpx.Request) -> httpx.Response: + calls.append(request.url.host) + if request.url.host == "stream_v2_active": + await asyncio.sleep(0.05) + return httpx.Response(200, json=payload, request=request) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + reader = HttpExecutionMarkIndexReader( + ( + "https://stream_v2_active:8200", + "https://stream_v2_passive:8200", + ), + SECRET, + timeout_seconds=1.0, + client=client, + ) + try: + result = await reader.fetch( + ReferenceRequest(self.record, ReferenceProduct.MARK_INDEX_PRICE), + max_freshness_ms=2_000, + source_policy_id="crypto_liquid_v2", + deadline_ms=10, + ) + finally: + await client.aclose() + self.assertEqual(result.status, ReferenceStatus.ERROR) + self.assertEqual(result.error_code, "LIVE_VIEW_UNAVAILABLE") + self.assertEqual(calls, ["stream_v2_active"]) + + async def test_reader_fails_over_within_one_deadline(self): + envelope = _envelope(self.binding, sequence=51) + payload = { + "schema": "qdl.v2.execution-mark-index-view.v2", + "lease_epoch": 5, + "spool_watermark_offset": 51, + "delivery_stage": "SPOOL_CONFIRMED", + "canonical": base64.b64encode( + envelope.SerializeToString(deterministic=True) + ).decode("ascii"), + } + calls = [] + + async def handler(request: httpx.Request) -> httpx.Response: + calls.append(request.url.host) + if request.url.host == "stream_v2_active": + return httpx.Response( + 409, + json={"detail": "execution MARK/INDEX gateway fenced"}, + request=request, + ) + return httpx.Response( + 200, + json=payload, + headers={"X-QDL-Execution-Freshness-Basis": "PROVIDER_CONFIRMATION"}, + request=request, + ) + + client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + reader = HttpExecutionMarkIndexReader( + ( + "https://stream_v2_active:8200", + "https://stream_v2_passive:8200", + ), + SECRET, + timeout_seconds=1.0, + client=client, + ) + try: + result = await reader.fetch( + ReferenceRequest(self.record, ReferenceProduct.MARK_INDEX_PRICE), + max_freshness_ms=2_000, + source_policy_id="crypto_liquid_v2", + deadline_ms=100, + ) + finally: + await client.aclose() + self.assertEqual(result.status, ReferenceStatus.OK) + self.assertEqual(calls, ["stream_v2_active", "stream_v2_passive"]) + + +class ExecutionMarkIndexQueryRoutingTests(unittest.IsolatedAsyncioTestCase): + def setUp(self) -> None: + self.record = _record( + venue="BINANCE", market="USDM", native_symbol="BTCUSDT", base="BTC" + ) + registry = InstrumentRegistry() + registry.register(self.record, []) + self.fallback = _FallbackReferenceAdapter() + self.entitlements = EntitlementPolicy((EntitlementGrant( + source_id="BINANCE_DIRECT", + license_revision="execution-mark-index-live-test", + purposes=frozenset({AccessPurpose.INTERNAL_EXECUTION, AccessPurpose.INTERNAL_ALPHA}), + products=frozenset({DataProduct.CANONICAL_SNAPSHOT}), + valid_from_ns=0, + ),)) + self.common = { + "instruments": InstrumentQuery(registry), + "backend": MemoryMarketDataBackend(), + "entitlements": self.entitlements, + "reference_batch": ReferenceBatch({("BINANCE", "USDM"): self.fallback}, clock_ns=lambda: NOW_NS), + "reference_source_id": lambda _record: "BINANCE_DIRECT", + "clock_ns": lambda: NOW_NS, + } + + def _execution_requirement( + self, + *, + source_policy_id: str = "crypto_liquid_v2", + max_freshness_ms: int = 2_000, + event_recency_policy: StalePolicy | None = None, + max_session_liveness_ms: int | None = None, + deadline_ms: int = 20_000, + ) -> ReferenceDataRequirement: + return ReferenceDataRequirement( + instrument_uid=self.record.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id=source_policy_id, + limit=1, + page_size=1, + max_pages=1, + max_freshness_ms=max_freshness_ms, + event_recency_policy=event_recency_policy, + max_session_liveness_ms=max_session_liveness_ms, + deadline_ms=deadline_ms, + ) + + async def test_execution_uses_live_view_but_alpha_reference_keeps_existing_adapter(self): + live = _LiveReader() + service = V2QueryService(**self.common, execution_mark_index_reader=live) + execution = await service.reference_data_batch_async( + ReferenceBatchRequirement("execution-reader", (self._execution_requirement(),)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(execution.partial) + self.assertEqual(live.calls, 1) + self.assertEqual(self.fallback.calls, 0) + self.assertEqual( + execution.results[0].result.lineage[0].provider_endpoint, LIVE_ENDPOINT + ) + + alpha_requirement = ReferenceDataRequirement( + instrument_uid=self.record.instrument_uid, + product=ReferenceProduct.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + ) + alpha = await service.reference_data_batch_async( + ReferenceBatchRequirement("alpha-reader", (alpha_requirement,)), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + self.assertFalse(alpha.partial) + self.assertEqual(live.calls, 1) + self.assertEqual(self.fallback.calls, 1) + self.assertEqual( + alpha.results[0].result.lineage[0].provider_endpoint, + "TEST_REFERENCE_REST_ADAPTER", + ) + + async def test_execution_live_view_stale_is_typed_and_never_falls_back_to_rest(self): + live = _LiveReader(status=ReferenceStatus.ERROR) + service = V2QueryService(**self.common, execution_mark_index_reader=live) + result = await service.reference_data_batch_async( + ReferenceBatchRequirement( + "execution-reader", (self._execution_requirement(),), require_all=False + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertTrue(result.partial) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertEqual(live.calls, 1) + self.assertEqual(self.fallback.calls, 0) + + async def test_provider_confirmation_freshness_is_explicit_and_source_event_remains_lineage(self): + clock = {"ns": NOW_NS + 1_500_000_000} + requirement = self._execution_requirement() + service = V2QueryService( + **{**self.common, "clock_ns": lambda: clock["ns"]}, + execution_mark_index_reader=_LiveReader( + source_event_time_ns=NOW_NS - 3_000_000_000, + provider_confirmation_ns=NOW_NS, + freshness_basis="PROVIDER_CONFIRMATION", + ), + ) + accepted = await service.reference_data_batch_async( + ReferenceBatchRequirement("execution-reader", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(accepted.partial) + labels = dict(accepted.results[0].result.observations[0].labels) + self.assertEqual(labels["freshness_basis"], "PROVIDER_CONFIRMATION") + self.assertEqual(labels["provider_confirmation_ns"], str(NOW_NS)) + self.assertEqual( + accepted.results[0].result.observations[0].observed_at_ns, + NOW_NS - 3_000_000_000, + ) + + source_event_service = V2QueryService( + **{**self.common, "clock_ns": lambda: clock["ns"]}, + execution_mark_index_reader=_LiveReader( + source_event_time_ns=NOW_NS - 3_000_000_000, + provider_confirmation_ns=NOW_NS, + freshness_basis="SOURCE_EVENT", + ), + ) + rejected = await source_event_service.reference_data_batch_async( + ReferenceBatchRequirement( + "execution-reader", (requirement,), require_all=False + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertTrue(rejected.partial) + self.assertEqual(rejected.results[0].problem.code.value, "DATA_STALE") + + async def test_execution_live_snapshot_that_ages_before_assembly_is_never_ok(self): + clock = {"ns": NOW_NS} + + class AgingReader(_LiveReader): + async def fetch(self, *args, **kwargs): + result = await super().fetch(*args, **kwargs) + clock["ns"] += 2_001_000_000 + return result + + service = V2QueryService( + **{**self.common, "clock_ns": lambda: clock["ns"]}, + execution_mark_index_reader=AgingReader(), + ) + result = await service.reference_data_batch_async( + ReferenceBatchRequirement( + "execution-reader", + (self._execution_requirement(),), + require_all=False, + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertTrue(result.partial) + self.assertEqual(result.results[0].problem.code.value, "DATA_STALE") + self.assertEqual(self.fallback.calls, 0) + + async def test_quiet_execution_mark_index_rechecks_component_and_session_evidence(self): + clock = {"ns": NOW_NS} + requirement = self._execution_requirement( + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + ) + service = V2QueryService( + **{**self.common, "clock_ns": lambda: clock["ns"]}, + execution_mark_index_reader=_LiveReader( + source_event_time_ns=NOW_NS - 60_000_000_000, + provider_confirmation_ns=NOW_NS - 60_000_000_000, + freshness_basis="PROVIDER_CONFIRMATION", + ), + ) + accepted = await service.reference_data_batch_async( + ReferenceBatchRequirement("execution-reader", (requirement,)), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertFalse(accepted.partial) + labels = dict(accepted.results[0].result.observations[0].labels) + self.assertEqual(labels["recency_mode"], "COMPONENT_SESSION_LIVE") + self.assertEqual(labels["component_index_quiet_after_ms"], "70000") + + clock["ns"] += 70_001_000_000 + stale = await service.reference_data_batch_async( + ReferenceBatchRequirement( + "execution-reader", (requirement,), require_all=False, + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + ) + self.assertTrue(stale.partial) + self.assertEqual(stale.results[0].problem.code.value, "DATA_STALE") + self.assertEqual(self.fallback.calls, 0) + + async def test_execution_live_singleflight_isolated_by_policy_freshness_and_deadline(self): + class BlockingReader(_LiveReader): + def __init__(self): + super().__init__() + self.entered = 0 + self.started = asyncio.Event() + self.release = asyncio.Event() + + async def fetch(self, *args, **kwargs): + self.entered += 1 + if self.entered == 2: + self.started.set() + await self.release.wait() + return await super().fetch(*args, **kwargs) + + live = BlockingReader() + service = V2QueryService(**self.common, execution_mark_index_reader=live) + first = asyncio.create_task(service.reference_data_batch_async( + ReferenceBatchRequirement( + "execution-reader-a", + (self._execution_requirement(source_policy_id="policy-a"),), + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + )) + second = asyncio.create_task(service.reference_data_batch_async( + ReferenceBatchRequirement( + "execution-reader-b", + (self._execution_requirement( + source_policy_id="policy-b", + max_freshness_ms=1_500, + deadline_ms=1_500, + ),), + ), + purpose=AccessPurpose.INTERNAL_EXECUTION, + )) + await asyncio.wait_for(live.started.wait(), timeout=0.2) + live.release.set() + left, right = await asyncio.gather(first, second) + self.assertFalse(left.partial) + self.assertFalse(right.partial) + self.assertEqual(live.calls, 2) + self.assertEqual( + set(live.calls_by_policy), + {("policy-a", 2_000, 2_000), ("policy-b", 1_500, 1_500)}, + ) diff --git a/tests/test_fund_phase2_transport.py b/tests/test_fund_phase2_transport.py index 5850cc5..de7a7f0 100644 --- a/tests/test_fund_phase2_transport.py +++ b/tests/test_fund_phase2_transport.py @@ -147,6 +147,110 @@ def test_tail_returns_newest_window_without_changing_replay_order(self): [row.event.payload for row in latest], [b"event-4", b"event-5"] ) + def test_batch_tails_use_one_bounded_snapshot_without_cross_partition_mix(self): + with self.spool(max_records=10) as spool: + stream = event(1).stream + primary = event(1).partition_key + secondary = "instrument/bar/source" + spool.append_many([ + event(index) + for index in range(1, 4) + ]) + spool.append_many([ + DurableEvent( + stream=stream, + partition_key=secondary, + event_id=(100 + index).to_bytes(16, "big"), + payload=f"secondary-{index}".encode(), + accepted_at_ns=1_800_000_000_000_000_100 + index, + headers={"schema": "qdl.marketdata.bar/2"}, + ) + for index in range(1, 4) + ]) + + tails = spool.read_tails(requests=( + (stream, primary, 2), + (stream, secondary, 1), + # The largest duplicate request wins at the physical reader; + # callers retain their narrower logical tail above it. + (stream, primary, 1), + )) + + self.assertEqual( + [row.cursor.offset for row in tails[(stream, primary)]], [2, 3] + ) + self.assertEqual( + [row.event.payload for row in tails[(stream, secondary)]], + [b"secondary-3"], + ) + + def test_batch_tails_use_indexed_reads_inside_one_snapshot(self): + with self.spool(max_records=10) as spool: + stream = event(1).stream + first = event(1).partition_key + second = "instrument/quote/source" + spool.append_many([event(index) for index in range(1, 4)]) + spool.append_many([ + DurableEvent( + stream=stream, + partition_key=second, + event_id=(200 + index).to_bytes(16, "big"), + payload=f"quote-{index}".encode(), + accepted_at_ns=1_800_000_000_000_000_200 + index, + headers={"schema": "qdl.marketdata.quote/2"}, + ) + for index in range(1, 4) + ]) + statements: list[str] = [] + spool._connection.set_trace_callback(statements.append) + try: + tails = spool.read_tails(requests=( + (stream, first, 2), + (stream, second, 1), + )) + finally: + spool._connection.set_trace_callback(None) + + self.assertEqual( + [row.event.payload for row in tails[(stream, first)]], + [b"event-2", b"event-3"], + ) + self.assertEqual( + [row.event.payload for row in tails[(stream, second)]], + [b"quote-3"], + ) + trace = "\n".join(statements).upper() + self.assertIn("BEGIN", trace) + self.assertIn("COMMIT", trace) + self.assertNotIn("ROW_NUMBER", trace) + self.assertEqual(trace.count("SELECT * FROM EVENTS"), 2) + + def test_batch_tails_support_the_public_fifty_partition_shape(self): + with self.spool(max_records=100) as spool: + stream = event(1).stream + requests = [] + events = [] + for index in range(50): + partition = f"instrument-{index}/bar/source" + events.append(DurableEvent( + stream=stream, + partition_key=partition, + event_id=(1_000 + index).to_bytes(16, "big"), + payload=f"batch-{index}".encode(), + accepted_at_ns=1_800_000_000_000_001_000 + index, + headers={"schema": "qdl.marketdata.bar/2"}, + )) + requests.append((stream, partition, 1)) + spool.append_many(events) + + tails = spool.read_tails(requests=tuple(requests)) + + self.assertEqual(len(tails), 50) + self.assertEqual( + [row.event.payload for row in tails[(stream, "instrument-49/bar/source")]], + [b"batch-49"], + ) + def test_tail_allows_only_configured_internal_partition_headroom(self): with self.spool(max_partition_records=10_064) as spool: self.assertEqual( diff --git a/tests/test_fund_phase5_api.py b/tests/test_fund_phase5_api.py index fc47ea0..2f40829 100644 --- a/tests/test_fund_phase5_api.py +++ b/tests/test_fund_phase5_api.py @@ -8,6 +8,7 @@ from fastapi.testclient import TestClient from qdl.api_v2 import create_v2_app +from qdl.api_v2.models import BatchResponse from qdl.consumer import ConsumerManifestLoader from qdl.domain.decimal import CanonicalDecimal from qdl.domain.instrument import ( @@ -338,6 +339,41 @@ def test_batch_partial_semantics_and_execution_fail_closed(self): self.assertEqual(denied.headers["content-type"], "application/problem+json") self.assertEqual(denied.json()["code"], "INVALID_ARGUMENT") + def test_batch_completion_returns_the_validated_public_json_contract(self): + requirement = self.requirement.__dict__ | { + "consumer_grade": "ALPHA", + "feed": "BAR", + "stale_policy": "BLOCK", + "gap_policy": "BLOCK", + "recovery": "SNAPSHOT_AND_REPLAY", + "bar_revision_policy": "LATEST", + } + original = self.service.warmup_batch_completed_async + completed = [] + + async def instrumented(*args, **kwargs): + response = await original(*args, **kwargs) + completed.append(response) + return response + + with patch.object(self.service, "warmup_batch_completed_async", instrumented): + response = self.client.post( + "/v2/market-data/warmup:batch", + json={ + "consumer_id": self.consumer_id, + "require_all": True, + "requirements": [requirement], + }, + ) + self.assertEqual(response.status_code, 200, response.text) + self.assertEqual(len(completed), 1) + self.assertEqual(completed[0].media_type, "application/json") + validated = BatchResponse.model_validate_json(response.content) + self.assertEqual( + validated.model_dump(mode="json", by_alias=True), + response.json(), + ) + def test_stale_and_unentitled_sources_return_stable_problem_details(self): stale_requirement = DataRequirement( **{**self.requirement.__dict__, "consumer_grade": ConsumerGrade.EXECUTION} diff --git a/tests/test_fund_phase6_release.py b/tests/test_fund_phase6_release.py index f4eb86e..87c812d 100644 --- a/tests/test_fund_phase6_release.py +++ b/tests/test_fund_phase6_release.py @@ -64,7 +64,16 @@ def test_runtime_image_is_non_root_and_trivy_waiver_is_narrow(self): ci_compose = (ROOT / "docker-compose.ci.yml").read_text(encoding="utf-8") self.assertGreaterEqual(ci_compose.count("container_name: !reset null"), 5) self.assertIn("ports: !reset []", ci_compose) - self.assertGreaterEqual(ci_compose.count("volumes: !reset []"), 4) + self.assertEqual(ci_compose.count("volumes: !reset []"), 3) + test_runner = ci_compose.split(" test_runner:\n", 1)[1].split( + "\n data_source_checker:", 1 + )[0] + self.assertIn("volumes: !override", test_runner) + self.assertIn( + "./upgrade/evidence:/app/upgrade/evidence:ro", + test_runner, + ) + self.assertNotIn("volumes: !reset []", test_runner) ignored = { line.strip() for line in (ROOT / ".trivyignore").read_text(encoding="utf-8").splitlines() diff --git a/tests/test_fund_phase71_beta_runtime.py b/tests/test_fund_phase71_beta_runtime.py index 05de1b4..09cb143 100644 --- a/tests/test_fund_phase71_beta_runtime.py +++ b/tests/test_fund_phase71_beta_runtime.py @@ -210,6 +210,28 @@ async def test_active_passive_uses_monotonic_fencing_epoch(self): with self.assertRaises(GatewayFenced): active.assert_active(first_epoch) + async def test_lease_acquisition_callback_is_fenced_when_initialization_fails(self): + store = InMemoryAsyncGatewayLeaseStore() + fenced = [] + + async def on_acquired(_lease): + raise RuntimeError("durable view unavailable") + + async def on_fenced(): + fenced.append(True) + + lease = ActivePassiveGatewayLease( + store, + shard_id="public", + owner_id="initializer", + on_acquired=on_acquired, + on_fenced=on_fenced, + ) + self.assertFalse(await lease.acquire_once()) + self.assertFalse(lease.active) + self.assertEqual(fenced, [True]) + self.assertIn("activation RuntimeError", lease.last_error) + async def test_replay_registration_barrier_has_no_live_handoff_gap(self): replay_started = threading.Event() release_replay = threading.Event() diff --git a/tests/test_mark_index_paired_lineage.py b/tests/test_mark_index_paired_lineage.py index 7df05ac..7b47ed8 100644 --- a/tests/test_mark_index_paired_lineage.py +++ b/tests/test_mark_index_paired_lineage.py @@ -3,10 +3,14 @@ import asyncio import base64 import hashlib +import hmac +import json import tempfile import unittest +from contextlib import suppress from pathlib import Path from types import SimpleNamespace +from unittest.mock import patch import httpx from fastapi import FastAPI @@ -28,13 +32,14 @@ StableHttpCanonicalSink, install_stable_canonical_ingest, ) +from qdl.runtime.execution_mark_index import ExecutionMarkIndexLiveView from qdl.runtime.stable_projector import ( LocalStableCanonicalSink, StableProjectorEngine, ) from qdl.replay import GapFreeHandoff, SignedHandoffCursorCodec from qdl.stream import DurableStreamGateway -from qdl.transport import DurableEvent, SQLiteDurableSpool, SpoolConfig +from qdl.transport import BackpressureRequired, DurableEvent, SQLiteDurableSpool, SpoolConfig from qdl.transport.kafka_projector import KafkaProjectorRecord @@ -165,6 +170,17 @@ def _pair(*, venue: str = "OKX"): feed=FeedType.MARK_INDEX_PRICE, v1_compatibility="NONE", partition_key="paired/mark-index", + instrument=SimpleNamespace( + instrument_uid=envelope.instrument_uid, + instrument_id=envelope.instrument_id, + metadata_revision=envelope.instrument_revision, + ), + authoritative=True, + source_role="PRIMARY", + source_policy_id="paired-mark-index-v1", + source_id=envelope.source_id, + stale_after_ms=2_000, + freshness_basis="PROVIDER_CONFIRMATION", ) return binding, mark, index, envelope @@ -174,6 +190,10 @@ class _Catalog: def __init__(self, binding): self.binding = binding + # StableProjectorEngine prewarms declared final-BAR partitions before + # polling. This fixture only models its MARK/INDEX lookup contract, so + # it declares no final-BAR partitions for that independent prewarm. + self.bindings = () def binding_for_envelope(self, _envelope): return self.binding @@ -325,6 +345,207 @@ async def test_projector_and_signed_ingest_preserve_valid_component_marker(self) spool.close() temp.cleanup() + async def test_execution_mark_index_view_is_available_before_secondary_spool(self): + """A read-committed canonical MARK/INDEX record must not wait for SQLite. + + The test deliberately holds ``publish_many`` after the signed endpoint + has validated the canonical/raw pair. Query eligibility is visible at + that point; releasing the append then promotes the same event with a + spool offset. This pins the latency boundary rather than merely the + view's in-memory methods. + """ + + binding, _mark, index, envelope = _pair() + catalog = _Catalog(binding) + temp, spool, gateway = self._spool_gateway() + + class _Authority: + current_epoch = 5 + + @staticmethod + def assert_active(expected_epoch=None): + if expected_epoch is not None and expected_epoch != 5: + raise RuntimeError("test gateway fenced") + return 5 + + gateway.authority = _Authority() + view = ExecutionMarkIndexLiveView( + frozenset({binding.instrument.instrument_uid}) + ) + app = FastAPI() + install_stable_canonical_ingest( + app, + gateway=gateway, + catalog=catalog, + spool=spool, + secret=_SECRET, + execution_mark_index_view=view, + ) + body = json.dumps({ + "schema": "qdl.v2.stable-canonical-ingest.v1", + "batch_id": "00000000-0000-4000-8000-000000000003", + "events": [{ + "canonical": base64.b64encode( + envelope.SerializeToString(deterministic=True) + ).decode("ascii"), + "raw_stream": "kafka-header:qdl-raw-provider-envelope", + "raw_event_id": bytes(envelope.raw_capture_id).hex(), + "raw_provider_envelope": base64.b64encode( + index.SerializeToString(deterministic=True) + ).decode("ascii"), + "raw_lineage_kind": DERIVED_MARK_INDEX_COMPONENT_V1, + }], + }, sort_keys=True, separators=(",", ":")).encode() + signature = "sha256=" + hmac.new( + _SECRET, body, hashlib.sha256 + ).hexdigest() + entered = asyncio.Event() + release = asyncio.Event() + original_publish_many = gateway.publish_many + + async def delayed_publish_many(events): + entered.set() + await release.wait() + return await original_publish_many(events) + + request: asyncio.Task[httpx.Response] | None = None + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) + try: + with patch.object(gateway, "publish_many", side_effect=delayed_publish_many): + request = asyncio.create_task(client.post( + "/internal/v2/canonical/events", + content=body, + headers={"X-QDL-Stable-Signature": signature}, + )) + entered_wait = asyncio.create_task(entered.wait()) + done, pending = await asyncio.wait( + (request, entered_wait), timeout=2.0, + return_when=asyncio.FIRST_COMPLETED, + ) + if request in done: + await request + self.assertIn(entered_wait, done, "ingest did not reach spool boundary") + if entered_wait in pending: + entered_wait.cancel() + pre_spool = await view.read( + instrument_uid=binding.instrument.instrument_uid, + instrument_revision=binding.instrument.metadata_revision, + source_policy_id=binding.source_policy_id, + max_freshness_ms=binding.stale_after_ms, + gateway_epoch=gateway.assert_active(), + now_ns=envelope.received_at_ns + 1_000_000, + ) + self.assertEqual( + pre_spool.record.delivery_stage, "CANONICAL_READ_COMMITTED" + ) + self.assertEqual( + spool.read_tail( + stream=_STREAM, partition_key=binding.partition_key, limit=1 + ), + [], + ) + release.set() + response = await request + self.assertEqual(response.status_code, 200) + confirmed = await view.read( + instrument_uid=binding.instrument.instrument_uid, + instrument_revision=binding.instrument.metadata_revision, + source_policy_id=binding.source_policy_id, + max_freshness_ms=binding.stale_after_ms, + gateway_epoch=gateway.assert_active(), + now_ns=envelope.received_at_ns + 1_000_000, + ) + self.assertEqual(confirmed.record.delivery_stage, "SPOOL_CONFIRMED") + self.assertIsNotNone(confirmed.record.spool_watermark_offset) + finally: + if request is not None and not request.done(): + request.cancel() + with suppress(asyncio.CancelledError): + await request + await client.aclose() + spool.close() + temp.cleanup() + + async def test_execution_mark_index_view_withdraws_when_secondary_spool_rejects(self): + """A failed secondary append must not leave an execution value usable.""" + + binding, _mark, index, envelope = _pair() + catalog = _Catalog(binding) + temp, spool, gateway = self._spool_gateway() + + class _Authority: + current_epoch = 5 + + @staticmethod + def assert_active(expected_epoch=None): + if expected_epoch is not None and expected_epoch != 5: + raise RuntimeError("test gateway fenced") + return 5 + + gateway.authority = _Authority() + view = ExecutionMarkIndexLiveView( + frozenset({binding.instrument.instrument_uid}) + ) + app = FastAPI() + install_stable_canonical_ingest( + app, + gateway=gateway, + catalog=catalog, + spool=spool, + secret=_SECRET, + execution_mark_index_view=view, + ) + body = json.dumps({ + "schema": "qdl.v2.stable-canonical-ingest.v1", + "batch_id": "00000000-0000-4000-8000-000000000004", + "events": [{ + "canonical": base64.b64encode( + envelope.SerializeToString(deterministic=True) + ).decode("ascii"), + "raw_stream": "kafka-header:qdl-raw-provider-envelope", + "raw_event_id": bytes(envelope.raw_capture_id).hex(), + "raw_provider_envelope": base64.b64encode( + index.SerializeToString(deterministic=True) + ).decode("ascii"), + "raw_lineage_kind": DERIVED_MARK_INDEX_COMPONENT_V1, + }], + }, sort_keys=True, separators=(",", ":")).encode() + signature = "sha256=" + hmac.new( + _SECRET, body, hashlib.sha256 + ).hexdigest() + + async def reject_append(events): + del events + raise BackpressureRequired("test secondary spool rejection") + + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) + try: + with patch.object(gateway, "publish_many", side_effect=reject_append): + response = await client.post( + "/internal/v2/canonical/events", + content=body, + headers={"X-QDL-Stable-Signature": signature}, + ) + self.assertEqual(response.status_code, 503) + read = await view.read( + instrument_uid=binding.instrument.instrument_uid, + instrument_revision=binding.instrument.metadata_revision, + source_policy_id=binding.source_policy_id, + max_freshness_ms=binding.stale_after_ms, + gateway_epoch=gateway.assert_active(), + now_ns=envelope.received_at_ns + 1_000_000, + ) + self.assertIsNone(read.record) + self.assertEqual(read.reason, "NOT_READY") + finally: + await client.aclose() + spool.close() + temp.cleanup() + async def test_ingest_rejects_marker_without_inline_component(self): binding, _mark, _index, envelope = _pair() catalog = _Catalog(binding) diff --git a/tests/test_phase103_consumer_acceptance.py b/tests/test_phase103_consumer_acceptance.py index 8c53d3e..7025226 100644 --- a/tests/test_phase103_consumer_acceptance.py +++ b/tests/test_phase103_consumer_acceptance.py @@ -416,6 +416,55 @@ def test_quiet_connected_trade_is_accepted_for_no_order_observation_only(self): with self.assertRaisesRegex(ValueError, "provider session"): validate_product_view(product, disconnected) + def test_on_change_quote_uses_session_only_when_query_proves_price_eligibility(self): + product = next( + item + for item in self.scope().products + if ( + item.consumer_id == "trading-system.paper.stable" + and item.binding_id == "okx-swap-bnb-usdt-swap-quote" + ) + ) + quiet = self._view( + product, + freshness_ms=product.requirement.max_freshness_ms + 1, + execution_eligible=True, + ).model_copy( + update={ + "quality": self._view( + product, + freshness_ms=product.requirement.max_freshness_ms + 1, + execution_eligible=True, + ).quality.model_copy( + update={ + "event_recency_state": "STALE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 1, + "flags": ("LAST_EVENT_STALE", "DELIVERY_ON_CHANGE"), + } + ) + } + ) + validate_product_view(product, quiet) + + for fields in ( + {"flags": ("LAST_EVENT_STALE",)}, + {"execution_eligible": False}, + {"provider_session_state": "DISCONNECTED"}, + {"provider_session_liveness_ms": product.requirement.max_session_liveness_ms + 1}, + {"gap_open": True}, + ): + with self.subTest(fields=fields): + with self.assertRaises(ValueError): + validate_product_view( + product, + quiet.model_copy( + update={ + "quality": quiet.quality.model_copy(update=fields) + } + ), + ) + def test_state_replay_keeps_identity_and_gap_checks_but_defers_old_session_quality(self): product = self._quiet_book_delta_product() stale = self._view( diff --git a/tests/test_phase105_consumer_acceptance.py b/tests/test_phase105_consumer_acceptance.py index 3dc3250..8084509 100644 --- a/tests/test_phase105_consumer_acceptance.py +++ b/tests/test_phase105_consumer_acceptance.py @@ -170,7 +170,7 @@ def test_paper_book_delta_routes_declare_observed_continuity_and_session_sla(sel }, ) - def test_paper_quote_routes_keep_strict_freshness_and_session_sla(self): + def test_paper_quote_routes_apply_declared_on_change_session_semantics(self): scope = build_release_consumer_acceptance_scope( self.release, catalog=self.catalog, @@ -179,10 +179,32 @@ def test_paper_quote_routes_keep_strict_freshness_and_session_sla(self): ) quotes = [item for item in scope.products if item.feed is FeedType.QUOTE] self.assertEqual(len(quotes), 20) + execution_quotes = [ + item + for item in quotes + if item.consumer_id == "trading-system.paper.stable" + ] + alpha_quotes = [ + item + for item in quotes + if item.consumer_id != "trading-system.paper.stable" + ] + self.assertEqual(len(execution_quotes), 10) + self.assertEqual(len(alpha_quotes), 10) self.assertTrue(all( - item.requirement.event_recency_policy is None + item.requirement.event_recency_policy is StalePolicy.OBSERVE + and item.requirement.max_session_liveness_ms == 2_000 + and item.requirement.stale_policy is StalePolicy.BLOCK + for item in execution_quotes + )) + self.assertTrue(all( + item.requirement.event_recency_policy is StalePolicy.OBSERVE and item.requirement.max_session_liveness_ms == 45_000 and item.requirement.stale_policy is StalePolicy.BLOCK + for item in alpha_quotes + )) + self.assertTrue(all( + self.catalog.binding_for(item.requirement).delivery_semantics == "ON_CHANGE" for item in quotes )) self.assertEqual( diff --git a/tests/test_phase105_execution_l2_status_matrix.py b/tests/test_phase105_execution_l2_status_matrix.py index ca25f6d..bbd9c68 100644 --- a/tests/test_phase105_execution_l2_status_matrix.py +++ b/tests/test_phase105_execution_l2_status_matrix.py @@ -17,37 +17,52 @@ execution_book_products, ready_book_row, replica_parity, + source_pair_ready, _read_one, ) -def _ready_row(*, source_id: str = "source-1", native_symbol: str = "BTCUSDT") -> dict[str, object]: +def _ready_row( + *, + source_id: str = "source-1", + native_symbol: str = "BTCUSDT", + feed: str = "BOOK_SNAPSHOT", + generation: int = 1, +) -> dict[str, object]: + view = { + "feed": feed, + "sequence_verified": True, + "book_generation": generation, + "watermark_offset": 12, + "complete": True, + "gap_open": False, + "execution_eligible": True, + "sequence_present": True, + } + if feed == "BOOK_SNAPSHOT": + view["depth"] = 100 + else: + view["reset"] = False return { "instrument_uid": "instrument-1", "venue": "BINANCE", "market": "USDM", "native_symbol": native_symbol, - "feed": "BOOK_SNAPSHOT", + "feed": feed, "source_policy_id": "crypto_primary_v2", "source_id": source_id, - "depth": 100, "typed_status": { "quality": { "state": "LIVE", "complete": True, "gap_open": False, "execution_eligible": True, - } - }, - "snapshot": { - "sequence_verified": True, - "book_generation": 1, - "depth": 100, - "watermark_offset": 12, - "complete": True, - "gap_open": False, - "execution_eligible": True, + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 1, + }, + "flags": [], }, + "view": view, } @@ -57,14 +72,14 @@ def setUpClass(cls) -> None: cls.catalog = StableSourceCatalog.load(DEFAULT_CATALOG) cls.acquisition = StableAcquisitionPlan.load(DEFAULT_ACQUISITION, catalog=cls.catalog) - def test_declared_execution_matrix_is_exactly_ten_physical_books(self) -> None: + def test_declared_execution_matrix_is_exactly_ten_snapshot_delta_pairs(self) -> None: products = execution_book_products( catalog=self.catalog, acquisition=self.acquisition, execution_demand=DEFAULT_EXECUTION_DEMAND, trading_manifest=DEFAULT_TRADING_MANIFEST, ) - self.assertEqual(len(products), 10) + self.assertEqual(len(products), 20) self.assertEqual( {(item.venue, item.native_symbol) for item in products}, { @@ -82,6 +97,18 @@ def test_declared_execution_matrix_is_exactly_ten_physical_books(self) -> None: ) }, ) + self.assertEqual( + {(item.venue, item.native_symbol, item.feed.value) for item in products}, + { + (venue, symbol, feed) + for venue, symbols in { + "BINANCE": ("BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "BNBUSDT"), + "OKX": ("BTC-USDT-SWAP", "ETH-USDT-SWAP", "SOL-USDT-SWAP", "DOGE-USDT-SWAP", "BNB-USDT-SWAP"), + }.items() + for symbol in symbols + for feed in ("BOOK_SNAPSHOT", "BOOK_DELTA") + }, + ) def test_quality_matrix_fails_closed_for_gap_or_unverified_resync_view(self) -> None: row = _ready_row() @@ -98,8 +125,11 @@ def test_quality_matrix_fails_closed_for_gap_or_unverified_resync_view(self) -> } }, ), - ("snapshot", {**row["snapshot"], "sequence_verified": False}), - ("snapshot", {**row["snapshot"], "book_generation": 0}), + ("view", {**row["view"], "sequence_verified": False}), + ("view", {**row["view"], "book_generation": 0}), + ("typed_status", { + **row["typed_status"], "flags": ["SOURCE_SESSION_UNAVAILABLE"], + }), ): with self.subTest(field=field): changed = dict(row) @@ -113,6 +143,36 @@ def test_replica_parity_rejects_cross_book_and_preserves_duplicate_ready_view(se cross_book = _ready_row(source_id="source-other", native_symbol="ETHUSDT") self.assertFalse(replica_parity(primary, cross_book)) + def test_source_pair_rejects_missing_delta_or_generation_mismatch(self) -> None: + snapshot = _ready_row(feed="BOOK_SNAPSHOT") + delta = _ready_row(feed="BOOK_DELTA") + self.assertTrue(source_pair_ready((snapshot, delta))) + self.assertFalse(source_pair_ready((snapshot,))) + self.assertFalse(source_pair_ready((snapshot, _ready_row( + feed="BOOK_DELTA", generation=2, + )))) + self.assertFalse(source_pair_ready((snapshot, _ready_row( + feed="BOOK_DELTA", source_id="other-source", + )))) + + def test_quiet_live_delta_remains_continuity_ready_but_disconnect_blocks(self) -> None: + delta = _ready_row(feed="BOOK_DELTA") + delta["typed_status"]["quality"].update({ + "event_recency_state": "STALE", + "execution_eligible": False, + }) + delta["view"]["execution_eligible"] = False + self.assertTrue(ready_book_row(delta)) + blocked = dict(delta) + blocked["typed_status"] = { + **delta["typed_status"], + "quality": { + **delta["typed_status"]["quality"], + "provider_session_state": "DISCONNECTED", + }, + } + self.assertFalse(ready_book_row(blocked)) + class ExecutionL2StatusMatrixReadTests(unittest.IsolatedAsyncioTestCase): async def test_status_transport_error_returns_compact_fail_closed_row(self) -> None: @@ -154,7 +214,7 @@ async def close(self) -> None: "detail": "query replica unavailable", }) self.assertIsNone(row["typed_status"]) - self.assertIsNone(row["snapshot"]) + self.assertIsNone(row["view"]) self.assertFalse(ready_book_row(row)) self.assertFalse(row["payload_recorded"]) diff --git a/tests/test_phase105_identity_acceptance.py b/tests/test_phase105_identity_acceptance.py index db29fbd..72b1ade 100644 --- a/tests/test_phase105_identity_acceptance.py +++ b/tests/test_phase105_identity_acceptance.py @@ -1,18 +1,28 @@ from __future__ import annotations import json +import io import tempfile import unittest import asyncio import httpx +from collections import Counter from dataclasses import replace from pathlib import Path from types import SimpleNamespace from unittest.mock import patch from qdl.certification.phase103_consumer_acceptance import AcceptanceProduct, DeliveryClass -from qdl.query import DataRequirement, FeedType, RecoveryPolicy +from qdl.certification.phase105_consumer_acceptance import ( + build_release_consumer_acceptance_scope, +) +from qdl.certification.phase105_fallback import build_v1_fallback_probes +from qdl.consumer import StableReleaseRoutePlan, requirement_key +from qdl.query import ConsumerGrade, DataRequirement, FeedType, RecoveryPolicy, StalePolicy +from qdl.runtime.stable_catalog import StableSourceCatalog +from qdl.runtime.stable_deployment import StableAcquisitionPlan from qdl_sdk import ( + DataLayerError, DataRequirement as SdkDataRequirement, Feed, FeedStatusResponse, @@ -22,31 +32,182 @@ from scripts.phase103_consumer_receipt_acceptance import C2StatusEvidenceError from scripts.phase105_consumer_v2_identity_acceptance import ( C2ProductAcceptanceError, + C2ReferenceProductError, C2ClosingBatchError, + C2ClosingL2Error, + C2BatchShapeError, + C2OpeningCapacityError, _C2ConsumerRequestPacer, _PacedQueryTransport, + _compact_strict_batch_response, _PacedStreamTransport, IDENTITY_PREFIXES, _authority, + _closing_batch_problem_evidence, _closing_batch_revalidation, _closing_batches, _closing_requirement, + _manifest_maximum_bar_batch, + _strict_bar_batch_shape_matrix, + _strict_bar_batch_windows, + _strict_bar_collocation_matrix, _c2_grpc_targets, _consumer_ids, + _build_c2_opening_operation_plan, + _certify_references, + _effective_c2_opening_timeout_seconds, _identity_files, _identity_files_for_consumers, + _read_plane_preflight_receipt, _route_summary, _reference_batch_concurrency, _reference_transport_timeout_seconds, _run_consumer_groups, + _timing_policy, _paced_client_factory, _wait_for_minimum_observation, _v1_base_url, + main, parser, ) class Phase105IdentityAcceptanceTests(unittest.TestCase): + def test_main_emits_typed_l2_closing_failure(self) -> None: + product = SimpleNamespace( + consumer_id="trading-system.paper.stable", + feed=FeedType.BOOK_DELTA, + evidence=lambda: {"feed": "BOOK_DELTA", "instrument_uid": "book-uid"}, + ) + error = C2ClosingL2Error( + product=product, + replica="primary", + operation="FEED_STATUS", + error=DataLayerError("DATA_STALE", "injected stale L2", retryable=False), + status_evidence={"quality": {"state": "STALE"}}, + ) + output = io.StringIO() + fake_args = SimpleNamespace( + timeout_seconds=15.0, + concurrency=4, + observation_seconds=300.0, + batch_shape_matrix=False, + read_plane_preflight=False, + opening_timeout_seconds=None, + closing_timeout_seconds=120.0, + ) + fake_parser = SimpleNamespace(parse_args=lambda: fake_args) + async def failing_run(_args): + raise error + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance.parser", + return_value=fake_parser, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.run", + new=failing_run, + ), patch("sys.stdout", output): + self.assertEqual(main(), 1) + payload = json.loads(output.getvalue()) + self.assertEqual(payload["status"], "FAIL_TYPED_STATUS") + self.assertEqual(payload["failure"]["operation"], "FEED_STATUS") + self.assertFalse(payload["failure"]["payload_recorded"]) + + def test_timing_policy_separates_final_bar_continuity_from_quiet_execution(self) -> None: + bar = SimpleNamespace(requirement=DataRequirement( + instrument_uid="bar-uid", + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + max_freshness_ms=180_000, + require_final_bars=True, + )) + quiet = SimpleNamespace(requirement=DataRequirement( + instrument_uid="mark-uid", + feed=FeedType.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=2_000, + event_recency_policy=StalePolicy.OBSERVE, + max_session_liveness_ms=45_000, + )) + quote = SimpleNamespace(requirement=DataRequirement( + instrument_uid="quote-uid", + feed=FeedType.QUOTE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=2_000, + max_session_liveness_ms=45_000, + )) + reference_mark = SimpleNamespace(requirement=DataRequirement( + instrument_uid="reference-mark-uid", + feed=FeedType.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_liquid_v2", + max_freshness_ms=2_000, + )) + self.assertEqual(_timing_policy(bar)["semantic_class"], "FINAL_SCHEDULED") + self.assertEqual(_timing_policy(bar)["freshness_role"], "CONTINUITY_DROPOUT_HORIZON") + self.assertEqual(_timing_policy(bar)["interval_ms"], 60_000) + self.assertEqual(_timing_policy(quiet)["semantic_class"], "QUIET_SESSION") + self.assertEqual(_timing_policy(quote)["semantic_class"], "STRICT_EVENT_WITH_SESSION") + self.assertEqual(_timing_policy(reference_mark)["semantic_class"], "REFERENCE_SNAPSHOT") + self.assertEqual( + _timing_policy(reference_mark)["freshness_role"], "PROVIDER_OBSERVATION_AGE" + ) + + def test_timing_policy_fails_closed_for_invalid_bar_or_quiet_session_contract(self) -> None: + short_bar = SimpleNamespace(requirement=DataRequirement( + instrument_uid="bar-uid", + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + max_freshness_ms=59_999, + require_final_bars=True, + )) + strict_without_session = SimpleNamespace(requirement=DataRequirement( + instrument_uid="trade-uid", + feed=FeedType.TRADE, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + max_freshness_ms=2_000, + )) + missing_observed_session = SimpleNamespace(requirement=SimpleNamespace( + feed=FeedType.TRADE, + consumer_grade=ConsumerGrade.ALPHA, + effective_event_recency_policy=StalePolicy.OBSERVE, + max_freshness_ms=2_000, + max_session_liveness_ms=None, + interval=None, + require_final_bars=False, + )) + # The public DataRequirement constructor rejects this malformed shape + # first. Keep a boundary-shaped object here to prove C2 independently + # refuses it if a bad serialized/configured requirement reaches the + # acceptance harness. + missing_execution_mark_session = SimpleNamespace(requirement=SimpleNamespace( + feed=FeedType.MARK_INDEX_PRICE, + consumer_grade=ConsumerGrade.EXECUTION, + effective_event_recency_policy=StalePolicy.OBSERVE, + max_freshness_ms=2_000, + max_session_liveness_ms=None, + interval=None, + require_final_bars=False, + )) + with self.assertRaisesRegex(ValueError, "continuity horizon"): + _timing_policy(short_bar) + self.assertEqual(_timing_policy(strict_without_session)["semantic_class"], "STRICT_EVENT") + self.assertEqual( + _timing_policy(strict_without_session)["session_contract"], + "NOT_DECLARED_STRICT_EVENT", + ) + with self.assertRaisesRegex(ValueError, "session-liveness"): + _timing_policy(missing_observed_session) + with self.assertRaisesRegex(ValueError, "session-liveness"): + _timing_policy(missing_execution_mark_session) + def test_closing_batches_isolate_hot_feeds_without_losing_scope(self) -> None: products = tuple(SimpleNamespace(identity=(venue, symbol, feed), feed=Feed(feed)) for venue in ("BINANCE", "OKX") @@ -63,6 +224,220 @@ def test_closing_batches_isolate_hot_feeds_without_losing_scope(self) -> None: self.assertLessEqual(len(batch), bound) self.assertEqual(tuple(_closing_batches((), 50)), ()) + def test_strict_bar_batch_windows_keep_the_exact_manifest_maximum(self) -> None: + consumer_id = "alpha.okx.paper.stable" + products = tuple( + SimpleNamespace( + consumer_id=consumer_id, + instrument_uid=f"bar-{index}", + feed=FeedType.BAR, + interval=f"{index + 1}m", + source_policy_id="crypto_primary_v2", + identity=( + consumer_id, + f"bar-{index}", + "BAR", + f"{index + 1}m", + "crypto_primary_v2", + ), + ) + for index in range(70) + ) + + maximum = _manifest_maximum_bar_batch(products, max_batch_items=50) + windows = _strict_bar_batch_windows(products, max_batch_items=50) + + self.assertEqual(len(maximum), 50) + self.assertEqual([item[0] for item in windows], [1, 1, 8, 8, 16, 16, 32, 32, 50]) + self.assertEqual(windows[-1][2], maximum) + self.assertEqual({item.identity for item in windows[-1][2]}, { + item.identity for item in maximum + }) + + def test_strict_bar_batch_shape_matrix_is_strict_and_payload_free(self) -> None: + consumer_id = "alpha.okx.paper.stable" + products = tuple( + SimpleNamespace( + consumer_id=consumer_id, + instrument_uid=f"bar-{index}", + feed=FeedType.BAR, + interval=f"{index + 1}m", + source_policy_id="crypto_primary_v2", + identity=( + consumer_id, + f"bar-{index}", + "BAR", + f"{index + 1}m", + "crypto_primary_v2", + ), + ) + for index in range(50) + ) + calls = [] + + async def revalidate(batch, **kwargs): + calls.append((len(batch), kwargs["max_batch_items"])) + return [ + { + "consumer_id": item.consumer_id, + "instrument_uid": item.instrument_uid, + "feed": item.feed.value, + "interval": item.interval, + "source_policy_id": item.source_policy_id, + "primary_latency_ms": 1.0, + "secondary_latency_ms": 2.0, + "primary_content_sha256": f"primary-{item.instrument_uid}", + "secondary_content_sha256": f"secondary-{item.instrument_uid}", + "quality_sha256": {"primary": "quality-primary", "secondary": "quality-secondary"}, + } + for item in batch + ] + + evidence = asyncio.run(_strict_bar_batch_shape_matrix( + products, + identity=object(), + primary_url="https://primary.invalid", + secondary_url="https://secondary.invalid", + grpc_target="unused:8210", + state_dir=Path("/tmp"), + timeout_seconds=5.0, + max_batch_items=50, + client_factory=object(), + revalidate=revalidate, + )) + + self.assertEqual(calls, [(1, 1), (1, 1), (8, 8), (8, 8), (16, 16), (16, 16), (32, 32), (32, 32), (50, 50)]) + self.assertEqual(evidence[-1]["batch_size"], 50) + self.assertTrue(all(item["payload_recorded"] is False for item in evidence)) + + def test_strict_bar_batch_shape_matrix_keeps_typed_failure_context(self) -> None: + product = SimpleNamespace( + consumer_id="alpha.okx.paper.stable", + instrument_uid="bar-0", + feed=FeedType.BAR, + interval="1m", + source_policy_id="crypto_primary_v2", + identity=("alpha.okx.paper.stable", "bar-0", "BAR", "1m", "crypto_primary_v2"), + ) + + async def failing(batch, **kwargs): + raise C2ClosingBatchError( + consumer_id=product.consumer_id, + replica="secondary", + products=batch, + error=DataLayerError("PARTIAL_RESULT", "injected strict batch failure", retryable=True), + status_observations=[], + ) + + with self.assertRaises(C2BatchShapeError) as raised: + asyncio.run(_strict_bar_batch_shape_matrix( + (product,), + identity=object(), + primary_url="https://primary.invalid", + secondary_url="https://secondary.invalid", + grpc_target="unused:8210", + state_dir=Path("/tmp"), + timeout_seconds=5.0, + max_batch_items=1, + client_factory=object(), + revalidate=failing, + )) + self.assertEqual(raised.exception.evidence["stage"], "ISOLATED") + self.assertEqual(raised.exception.evidence["transport_error_code"], "PARTIAL_RESULT") + self.assertFalse(raised.exception.evidence["payload_recorded"]) + + def test_strict_bar_collocation_matrix_skips_consumers_without_durable_bar(self) -> None: + consumer_ids = tuple(IDENTITY_PREFIXES) + non_bar_consumer = consumer_ids[0] + products = tuple( + SimpleNamespace( + consumer_id=consumer_id, + instrument_uid=f"{consumer_id}-{index}", + feed=(FeedType.QUOTE if consumer_id == non_bar_consumer else FeedType.BAR), + interval=f"{index + 1}m", + source_policy_id="crypto_primary_v2", + delivery=DeliveryClass.DURABLE, + identity=( + consumer_id, + f"{consumer_id}-{index}", + "QUOTE" if consumer_id == non_bar_consumer else "BAR", + f"{index + 1}m", + "crypto_primary_v2", + ), + ) + for consumer_id in consumer_ids + for index in range(50) + ) + scope = SimpleNamespace(products=products) + release = SimpleNamespace(consumers=tuple( + SimpleNamespace( + consumer_id=consumer_id, + manifest=SimpleNamespace(quotas=SimpleNamespace(max_batch_items=50)), + ) + for consumer_id in consumer_ids + )) + calls = [] + + async def revalidate(batch, **kwargs): + calls.append((batch[0].consumer_id, len(batch), kwargs["max_batch_items"])) + return [ + { + "consumer_id": item.consumer_id, + "instrument_uid": item.instrument_uid, + "feed": item.feed.value, + "interval": item.interval, + "source_policy_id": item.source_policy_id, + "primary_latency_ms": 1.0, + "secondary_latency_ms": 2.0, + "primary_content_sha256": f"primary-{item.instrument_uid}", + "secondary_content_sha256": f"secondary-{item.instrument_uid}", + "quality_sha256": {"primary": "quality-primary", "secondary": "quality-secondary"}, + } + for item in batch + ] + + bar_consumers = [ + consumer_id for consumer_id in consumer_ids if consumer_id != non_bar_consumer + ] + preferred_consumer_id = bar_consumers[-1] + evidence = asyncio.run(_strict_bar_collocation_matrix( + scope, + release, + consumer_ids=consumer_ids, + identities={consumer_id: object() for consumer_id in consumer_ids}, + primary_url="https://primary.invalid", + secondary_url="https://secondary.invalid", + grpc_target="unused:8210", + state_dir=Path("/tmp"), + timeout_seconds=5.0, + client_factories={consumer_id: object() for consumer_id in consumer_ids}, + preferred_consumer_id=preferred_consumer_id, + revalidate=revalidate, + )) + call_counts = Counter(consumer_id for consumer_id, _size, _limit in calls) + ordered = [preferred_consumer_id, *sorted( + consumer_id for consumer_id in bar_consumers + if consumer_id != preferred_consumer_id + )] + self.assertEqual(call_counts, Counter({ + consumer_id: len(ordered) - index + for index, consumer_id in enumerate(ordered) + })) + self.assertEqual( + [item["parallel_lanes"] for item in evidence["waves"]], + [1, 2, 3], + ) + self.assertEqual( + [item["consumer_ids"] for item in evidence["waves"]], + [ordered[:size] for size in [1, 2, 3]], + ) + self.assertEqual(evidence["not_applicable"], [{ + "consumer_id": non_bar_consumer, + "status": "NOT_APPLICABLE_NO_DURABLE_BAR", + "read_actions": 0, + "payload_recorded": False, + }]) + def test_typed_c2_product_failure_keeps_status_without_market_payload(self) -> None: status = FeedStatusResponse.model_validate({ "schema": "qdl.feed-status.v2", @@ -96,6 +471,93 @@ def test_typed_c2_product_failure_keeps_status_without_market_payload(self) -> N self.assertFalse(failure.evidence["payload_recorded"]) self.assertNotIn("levels", repr(failure.evidence)) + def test_typed_c2_reference_failure_keeps_product_identity_without_payload(self) -> None: + product = SimpleNamespace( + consumer_id="trading-system.paper.stable", + instrument_id="OKX.SWAP.PERPETUAL.DOGE-USDT", + requirement=SimpleNamespace(feed=Feed.MARK_INDEX_PRICE), + evidence=lambda: { + "instrument_uid": "mark-uid", + "feed": "MARK_INDEX_PRICE", + }, + ) + failure = C2ReferenceProductError( + product, + replica="primary", + error=ValueError("quiet execution MARK/INDEX component exceeded its cadence"), + ) + self.assertEqual(failure.evidence["replica"], "primary") + self.assertEqual(failure.evidence["product"]["instrument_uid"], "mark-uid") + self.assertFalse(failure.evidence["payload_recorded"]) + self.assertNotIn("price", repr(failure.evidence)) + + def test_reference_batch_failure_is_bound_to_exact_product_and_replica(self) -> None: + product = SimpleNamespace(consumer_id="trading-system.paper.stable") + reference_product = SimpleNamespace( + consumer_id="trading-system.paper.stable", + instrument_id="OKX.SWAP.PERPETUAL.DOGE-USDT", + requirement=SimpleNamespace(feed=Feed.MARK_INDEX_PRICE), + identity=("trading-system.paper.stable", "mark-uid", "MARK_INDEX_PRICE", "", "crypto"), + evidence=lambda: {"instrument_uid": "mark-uid", "feed": "MARK_INDEX_PRICE"}, + ) + response = SimpleNamespace(results=(object(),)) + + class Client: + def __init__(self, replica: str) -> None: + self.replica = replica + self.closed = False + + async def close(self) -> None: + self.closed = True + + clients = [] + + def client_factory(_identity, *, base_url, **_kwargs): + client = Client("primary" if base_url == "https://primary" else "secondary") + clients.append(client) + return client + + async def batch_for_c2(client, *_args, **_kwargs): + if client.replica == "secondary": + await asyncio.sleep(0.01) + return response, 1, 0 + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._reference_product", + return_value=reference_product, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance._reference_transport_timeout_seconds", + return_value=1.0, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.reference_acceptance_batches", + return_value=((reference_product,),), + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance._reference_batch_for_c2", + side_effect=batch_for_c2, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.reference_evidence", + side_effect=ValueError("quiet execution MARK/INDEX provenance is invalid"), + ): + with self.assertRaises(C2ReferenceProductError) as raised: + asyncio.run(_certify_references( + (product,), + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp"), + timeout_seconds=1.0, + deadline_monotonic=10.0, + semaphore=asyncio.Semaphore(1), + native_basis_semaphore=asyncio.Semaphore(1), + client_factory=client_factory, + )) + self.assertEqual(raised.exception.evidence["replica"], "primary") + self.assertEqual( + raised.exception.evidence["product"]["instrument_uid"], "mark-uid", + ) + self.assertTrue(all(client.closed for client in clients)) + def test_closing_bar_requirement_keeps_policy_and_reduces_only_history_rows(self) -> None: requirement = SdkDataRequirement( instrument_uid="bar-uid", @@ -362,6 +824,194 @@ class Release: class Phase105ConcurrentConsumerGroupTests(unittest.IsolatedAsyncioTestCase): + async def test_opening_pacer_records_sdk_operation_categories(self) -> None: + pacer = _C2ConsumerRequestPacer( + 180, + ) + await pacer.acquire("QUERY_READ") + await pacer.acquire("STREAM_SUBSCRIBE") + self.assertEqual(pacer.evidence()["c2_operation_counts"], { + "QUERY_READ": 1, + "STREAM_SUBSCRIBE": 1, + }) + + def test_stable_scope_opening_budget_is_manifest_derived_and_bounded(self) -> None: + root = Path(__file__).resolve().parents[1] + catalog = StableSourceCatalog.load( + root / "config/v2/stable-source-bindings.yaml" + ) + acquisition = StableAcquisitionPlan.load( + root / "config/v2/stable-acquisition-bindings.yaml", catalog=catalog + ) + release = StableReleaseRoutePlan.load( + root / "config/v2/stable-v2-release-routing.yaml", manifest_root=root + ) + consumer_ids = ( + "monitoring.multivenue.stable", + "trading-system.paper.stable", + "alpha.binance.paper.stable", + "alpha.okx.paper.stable", + ) + scope = build_release_consumer_acceptance_scope( + release, + catalog=catalog, + acquisition=acquisition, + consumer_ids=consumer_ids, + ) + probes = build_v1_fallback_probes( + release, + catalog=catalog, + products=scope.products, + consumer_ids=consumer_ids, + ) + plan = _build_c2_opening_operation_plan( + scope.products, + release, + probes, + consumer_ids, + generic_timeout_seconds=15.0, + reference_now_ns=1_800_000_000_000_000_000, + ) + selected_release_routes = { + (consumer.consumer_id, product.requirement_key): product + for consumer in release.consumers + if consumer.consumer_id in consumer_ids + for product in consumer.products + } + global_release_routes = { + (consumer.consumer_id, product.requirement_key): product + for consumer in release.consumers + for product in consumer.products + } + expected_v2 = { + identity for identity, product in selected_release_routes.items() + if product.route == "V2_PRIMARY" + } + actual_v2 = { + (product.consumer_id, requirement_key(product.requirement)) + for product in scope.products + } + self.assertEqual(actual_v2, expected_v2) + self.assertEqual(plan["global_release_route_count"], len(global_release_routes)) + self.assertEqual( + plan["global_v2_primary_product_count"], + sum(product.route == "V2_PRIMARY" for product in global_release_routes.values()), + ) + self.assertEqual( + plan["global_v1_primary_route_count"], + sum(product.route == "V1_PRIMARY" for product in global_release_routes.values()), + ) + self.assertEqual( + plan["selected_release_route_count"], len(selected_release_routes) + ) + self.assertEqual(plan["selected_v2_primary_product_count"], len(actual_v2)) + self.assertEqual( + plan["selected_v1_primary_excluded_count"], + sum(product.route == "V1_PRIMARY" for product in selected_release_routes.values()), + ) + self.assertEqual( + len(scope.excluded), plan["selected_v1_primary_excluded_count"] + ) + self.assertEqual(plan["global_v2_primary_product_count"], len(actual_v2)) + self.assertEqual(plan["product_count"], len(scope.products)) + self.assertEqual( + { + key: plan[key] + for key in ( + "global_release_route_count", + "global_v2_primary_product_count", + "global_v1_primary_route_count", + "selected_release_route_count", + "selected_v2_primary_product_count", + "selected_v1_primary_excluded_count", + "minimum_deadline_seconds", + ) + }, + { + "global_release_route_count": 303, + "global_v2_primary_product_count": 299, + "global_v1_primary_route_count": 4, + "selected_release_route_count": 301, + "selected_v2_primary_product_count": 299, + "selected_v1_primary_excluded_count": 2, + "minimum_deadline_seconds": 935.0, + }, + ) + self.assertEqual(set(plan["consumers"]), set(consumer_ids)) + self.assertGreater(plan["total_operations"], len(scope.products)) + self.assertGreater(plan["minimum_deadline_seconds"], 0) + for consumer_id, item in plan["consumers"].items(): + self.assertGreater(item["safe_requests_per_minute"], 0, consumer_id) + self.assertGreater(item["max_streams"], 0, consumer_id) + self.assertEqual( + item["opening_total_operations"], + sum(item["opening_operation_budget"].values()), + consumer_id, + ) + + def test_operator_timeout_cannot_undercut_manifest_derived_floor(self) -> None: + plan = { + "minimum_deadline_seconds": 121.0, + "consumers": {}, + } + self.assertEqual(_effective_c2_opening_timeout_seconds(plan, None), 121.0) + self.assertEqual(_effective_c2_opening_timeout_seconds(plan, 121.0), 121.0) + with self.assertRaises(C2OpeningCapacityError) as raised: + _effective_c2_opening_timeout_seconds(plan, 120.0) + self.assertEqual( + raised.exception.evidence["code"], "OPENING_TIMEOUT_BELOW_DERIVED_MINIMUM" + ) + with self.assertRaises(C2OpeningCapacityError) as raised: + _effective_c2_opening_timeout_seconds(plan, 0.0) + self.assertEqual(raised.exception.evidence["code"], "OPENING_TIMEOUT_NOT_POSITIVE") + + def test_read_plane_preflight_receipt_requires_the_exact_release_scope(self) -> None: + product = SimpleNamespace( + consumer_id="alpha.binance.paper.stable", + instrument_uid="uid-doge", + feed=Feed.BAR, + interval="12h", + source_policy_id="crypto_primary_v2", + ) + scope = SimpleNamespace(products=(product,), sha256="scope-sha") + release = SimpleNamespace(digest="release-sha") + observation = { + "consumer_id": product.consumer_id, + "instrument_uid": product.instrument_uid, + "feed": "BAR", + "interval": product.interval, + "source_policy_id": product.source_policy_id, + "primary_latency_ms": 11.0, + "secondary_latency_ms": 13.0, + "timing_policy": {"semantic_class": "FINAL_SCHEDULED"}, + } + receipt = _read_plane_preflight_receipt( + scope=scope, + release=release, + consumer_ids=(product.consumer_id,), + observations=[observation], + authority_revision=12, + elapsed_seconds=0.25, + quota_window_wait_seconds=0.0, + pacers={product.consumer_id: _C2ConsumerRequestPacer(180)}, + ) + self.assertEqual(receipt["status"], "PASS_READ_PLANE_PREFLIGHT") + self.assertEqual(receipt["product_count"], 1) + self.assertEqual(receipt["feed_counts"], {"BAR": 1}) + self.assertEqual(receipt["timing_class_counts"], {"FINAL_SCHEDULED": 1}) + self.assertEqual(receipt["replica_read_count"], 2) + with self.assertRaisesRegex(AssertionError, "scope differs"): + _read_plane_preflight_receipt( + scope=scope, + release=release, + consumer_ids=(product.consumer_id,), + observations=[], + authority_revision=12, + elapsed_seconds=0.25, + quota_window_wait_seconds=0.0, + pacers={product.consumer_id: _C2ConsumerRequestPacer(180)}, + ) + async def test_c2_pacer_aligns_then_spaces_requests_below_manifest_quota(self) -> None: clock = {"value": 100.0} sleeps: list[float] = [] @@ -436,6 +1086,51 @@ async def close(self) -> None: self.assertEqual(sleeps, [30.0]) self.assertEqual(pacer.evidence()["c2_request_count"], 2) + async def test_paced_strict_batch_summary_is_payload_free(self) -> None: + class QueryDelegate: + async def warmup_batch(self, *args, **kwargs): + del args, kwargs + return { + "partial": True, + "success_count": 1, + "error_count": 1, + "results": [ + { + "instrument_uid": "private-instrument", + "status": "DATA_STALE", + "data": {"market_payload": "must-not-be-recorded"}, + "problem": { + "code": "DATA_STALE", + "retryable": True, + "detail": "must-not-be-recorded", + }, + }, + {"instrument_uid": "private-ok", "status": "OK", "data": {"price": "1"}}, + ], + } + + async def close(self) -> None: + return None + + transport = _PacedQueryTransport(QueryDelegate(), _C2ConsumerRequestPacer(60)) + await transport.warmup_batch(object()) + summary = transport.last_warmup_batch_summary() + self.assertEqual(summary, { + "partial": True, + "success_count": 1, + "error_count": 1, + "result_count": 2, + "problem_outcomes": [{ + "index": 0, + "status": "DATA_STALE", + "problem_code": "DATA_STALE", + "retryable": True, + }], + "payload_recorded": False, + }) + self.assertNotIn("private", json.dumps(summary, sort_keys=True)) + self.assertIsNone(_compact_strict_batch_response({"partial": True})) + async def test_stream_open_failure_remains_fail_closed(self) -> None: class StreamDelegate: async def subscribe(self, *args, **kwargs): @@ -511,7 +1206,7 @@ def __init__(self, label: str) -> None: async def warmup_batch(self, requirements, *, require_all: bool): if not require_all: - raise AssertionError("closing batch must require every product") + raise AssertionError("execution closing batch must require all items") self.calls.append(tuple(requirements)) return SimpleNamespace( partial=False, @@ -541,6 +1236,9 @@ def factory(identity, *, base_url, grpc_target, cursor_path, timeout_seconds): ), patch( "scripts.phase105_consumer_v2_identity_acceptance.compact_view_quality", return_value={"state": "LIVE"}, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance._timing_policy", + return_value={"semantic_class": "QUIET_SESSION"}, ): evidence = await _closing_batch_revalidation( products, @@ -550,13 +1248,380 @@ def factory(identity, *, base_url, grpc_target, cursor_path, timeout_seconds): grpc_target="stream:8210", state_dir=Path("/tmp/phase105-closing"), timeout_seconds=15.0, - max_batch_items=1, + max_batch_items=2, client_factory=factory, ) self.assertEqual(len(evidence), 2) self.assertEqual({item["closing_read"] for item in evidence}, {"BATCH_V2_PRIMARY"}) self.assertEqual(len(clients), 2) - self.assertEqual([len(call) for client in clients for call in client.calls], [1, 1, 1, 1]) + self.assertEqual([len(call) for client in clients for call in client.calls], [2, 2]) + + async def test_closing_l2_uses_status_and_snapshot_not_history_batch(self) -> None: + class Product: + def __init__(self, feed: FeedType) -> None: + self.consumer_id = "trading-system.paper.stable" + self.instrument_uid = f"uid-{feed.value.lower()}" + self.instrument_id = f"BINANCE.USDM.PERPETUAL.{feed.value}" + self.feed = feed + self.interval = None + self.source_policy_id = "crypto_liquid_v2" + self.delivery = DeliveryClass.DURABLE + self.requirement = SimpleNamespace(max_session_liveness_ms=45_000) + self.identity = ( + self.consumer_id, + self.instrument_uid, + feed.value, + "", + self.source_policy_id, + ) + + def evidence(self) -> dict[str, object]: + return { + "consumer_id": self.consumer_id, + "instrument_uid": self.instrument_uid, + "feed": self.feed.value, + "interval": None, + "source_policy_id": self.source_policy_id, + } + + products = (Product(FeedType.BOOK_SNAPSHOT), Product(FeedType.BOOK_DELTA)) + + def status_for(product: Product) -> FeedStatusResponse: + return FeedStatusResponse.model_validate({ + "schema": "qdl.feed-status.v2", + "instrument_uid": product.instrument_uid, + "feed": product.feed.value, + "quality": { + "state": "LIVE", + "freshness_ms": 12, + "event_recency_state": "STALE" if product.feed is FeedType.BOOK_DELTA else "LIVE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 4, + "gap_open": False, + "complete": True, + "execution_eligible": True, + "policy_id": product.source_policy_id, + "flags": [], + }, + }) + + class Client: + def __init__(self, label: str) -> None: + self.label = label + self.status_calls: list[object] = [] + self.snapshot_calls: list[object] = [] + self.warmup_calls: list[object] = [] + + async def feed_status(self, requirement): + self.status_calls.append(requirement) + return status_for(requirement) + + async def snapshot(self, requirement): + self.snapshot_calls.append(requirement) + return SimpleNamespace(data=SimpleNamespace()) + + async def warmup_batch(self, *_args, **_kwargs): + self.warmup_calls.append(True) + raise AssertionError("lossless L2 must not enter warmup_batch") + + async def close(self) -> None: + return None + + clients = [] + + def factory(_identity, *, base_url, **_kwargs): + client = Client(base_url) + clients.append(client) + return client + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + side_effect=lambda product: product, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.validate_product_view", + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.validate_replica_views", + return_value=("a" * 64, "b" * 64), + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance.compact_view_quality", + return_value={"state": "LIVE"}, + ), patch( + "scripts.phase105_consumer_v2_identity_acceptance._timing_policy", + return_value={"semantic_class": "LOSSLESS_L2"}, + ): + evidence = await _closing_batch_revalidation( + products, + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp/phase105-closing-l2"), + timeout_seconds=15.0, + max_batch_items=8, + client_factory=factory, + ) + self.assertEqual(len(clients), 2) + self.assertTrue(all(len(client.status_calls) == 2 for client in clients)) + self.assertTrue(all(len(client.snapshot_calls) == 2 for client in clients)) + self.assertTrue(all(not client.warmup_calls for client in clients)) + self.assertEqual( + {item["closing_read"] for item in evidence}, + {"L2_STATUS_SNAPSHOT"}, + ) + self.assertEqual([item["feed"] for item in evidence], [ + "BOOK_SNAPSHOT", "BOOK_DELTA", + ]) + + async def test_closing_l2_fails_with_typed_status_before_snapshot(self) -> None: + product = SimpleNamespace( + consumer_id="trading-system.paper.stable", + instrument_uid="uid-book", + instrument_id="OKX.SWAP.PERPETUAL.BTC-USDT", + feed=FeedType.BOOK_DELTA, + interval=None, + source_policy_id="crypto_liquid_v2", + delivery=DeliveryClass.DURABLE, + requirement=SimpleNamespace(max_session_liveness_ms=45_000), + identity=( + "trading-system.paper.stable", "uid-book", "BOOK_DELTA", "", + "crypto_liquid_v2", + ), + evidence=lambda: { + "consumer_id": "trading-system.paper.stable", + "instrument_uid": "uid-book", + "feed": "BOOK_DELTA", + "interval": None, + "source_policy_id": "crypto_liquid_v2", + }, + ) + stale = FeedStatusResponse.model_validate({ + "schema": "qdl.feed-status.v2", + "instrument_uid": "uid-book", + "feed": "BOOK_DELTA", + "quality": { + "state": "STALE", + "freshness_ms": 2_001, + "event_recency_state": "STALE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": 3, + "gap_open": False, + "complete": True, + "execution_eligible": False, + "policy_id": "crypto_liquid_v2", + "flags": ["EVENT_STALE"], + }, + }) + + class Client: + snapshot_calls = 0 + + async def feed_status(self, requirement): + self.assertIs(requirement, product) + return stale + + async def snapshot(self, _requirement): + self.snapshot_calls += 1 + raise AssertionError("stale L2 status must block before snapshot") + + async def close(self) -> None: + return None + + @staticmethod + def assertIs(actual, expected): + if actual is not expected: + raise AssertionError("requirements differ") + + clients = [] + + def factory(*_args, **_kwargs): + client = Client() + clients.append(client) + return client + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + return_value=product, + ): + with self.assertRaises(C2ClosingL2Error) as raised: + await _closing_batch_revalidation( + (product,), + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp/phase105-closing-l2"), + timeout_seconds=15.0, + max_batch_items=8, + client_factory=factory, + ) + self.assertEqual(raised.exception.evidence["operation"], "FEED_STATUS") + self.assertEqual(raised.exception.evidence["typed_status"]["quality"]["state"], "STALE") + self.assertFalse(raised.exception.evidence["payload_recorded"]) + self.assertTrue(all(client.snapshot_calls == 0 for client in clients)) + + async def test_partial_batch_bisects_only_failing_leaf_for_typed_diagnostic(self) -> None: + class Product: + def __init__(self, name: str) -> None: + self.requirement = name + self.identity = ("alpha.binance.paper.stable", f"uid-{name}", "TRADE", "", "crypto") + + def evidence(self) -> dict[str, object]: + return {"instrument_uid": self.identity[1], "feed": "TRADE"} + + good, bad = Product("good"), Product("bad") + + class Client: + def __init__(self) -> None: + self.batch_calls: list[tuple[str, ...]] = [] + self.warmup_calls: list[str] = [] + + async def warmup_batch(self, requirements, *, require_all: bool): + self.assert_true(require_all) + values = tuple(requirements) + self.batch_calls.append(values) + if bad.requirement in values: + raise DataLayerError("PARTIAL_RESULT", "one item failed", retryable=True) + return SimpleNamespace(partial=False, results=[]) + + async def warmup(self, requirement): + self.warmup_calls.append(requirement) + if requirement == bad.requirement: + raise DataLayerError("DATA_STALE", "only the bad leaf is stale", retryable=True) + raise AssertionError("successful leaf must not receive an individual diagnostic read") + + @staticmethod + def assert_true(value): + if not value: + raise AssertionError("strict batch expected") + + client = Client() + observations = [ + {"product_identity": list(good.identity), "quality_sha256": "a" * 64}, + {"product_identity": list(bad.identity), "quality_sha256": "b" * 64}, + ] + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + side_effect=lambda product: product.requirement, + ): + evidence = await _closing_batch_problem_evidence( + client, + (good, bad), + error=DataLayerError("PARTIAL_RESULT", "strict root failure", retryable=True), + status_observations=observations, + ) + self.assertEqual(client.batch_calls, [(good.requirement,), (bad.requirement,)]) + self.assertEqual(client.warmup_calls, [bad.requirement]) + self.assertEqual([item["instrument_uid"] for item in evidence], ["uid-bad"]) + self.assertEqual(evidence[0]["problem_code"], "DATA_STALE") + self.assertEqual(evidence[0]["quality_sha256"], "b" * 64) + + async def test_closing_batch_partial_retains_typed_item_problem(self) -> None: + product = SimpleNamespace( + consumer_id="alpha.binance.paper.stable", + instrument_uid="uid-doge", + instrument_id="BINANCE.USDM.PERPETUAL.DOGE-USDT", + feed=Feed.BAR, + interval="12h", + source_policy_id="crypto_primary_v2", + delivery=DeliveryClass.DURABLE, + requirement=object(), + identity=("alpha.binance.paper.stable", "uid-doge", "BAR", "12h", "crypto_primary_v2"), + evidence=lambda: { + "consumer_id": "alpha.binance.paper.stable", + "instrument_uid": "uid-doge", + "feed": "BAR", + "interval": "12h", + "source_policy_id": "crypto_primary_v2", + }, + ) + status = FeedStatusResponse.model_validate({ + "schema": "qdl.feed-status.v2", + "instrument_uid": "uid-doge", + "feed": "BAR", + "quality": { + "state": "LIVE", + "freshness_ms": 1_500, + "event_recency_state": "LIVE", + "provider_session_state": "NOT_APPLICABLE", + "provider_session_liveness_ms": None, + "gap_open": False, + "complete": True, + "execution_eligible": True, + "policy_id": "crypto_primary_v2", + "flags": [], + }, + }) + + class Client: + async def warmup_batch(self, requirements, *, require_all: bool): + self.assertTrue(require_all) + self.assertEqual(tuple(requirements), (product.requirement,)) + raise DataLayerError( + "PARTIAL_RESULT", + "required warmup batch contains one or more explicit failures", + retryable=True, + ) + + async def warmup(self, requirement): + self.assertIs(requirement, product.requirement) + raise DataLayerError( + "DATA_NOT_READY", + "cache is intentionally omitted from evidence", + retryable=True, + ) + + async def feed_status(self, requirement): + self.assertIs(requirement, product.requirement) + return status + + async def close(self) -> None: + return None + + def assertFalse(self, value): + if value: + raise AssertionError("expected false") + + def assertTrue(self, value): + if not value: + raise AssertionError("expected true") + + def assertEqual(self, actual, expected): + if actual != expected: + raise AssertionError(f"{actual!r} != {expected!r}") + + def assertIs(self, actual, expected): + if actual is not expected: + raise AssertionError("objects differ") + + def factory(*args, **kwargs): + del args, kwargs + return Client() + + with patch( + "scripts.phase105_consumer_v2_identity_acceptance._closing_requirement", + return_value=product.requirement, + ): + with self.assertRaises(C2ClosingBatchError) as raised: + await _closing_batch_revalidation( + (product,), + identity=object(), + primary_url="https://primary", + secondary_url="https://secondary", + grpc_target="stream:8210", + state_dir=Path("/tmp/phase105-closing"), + timeout_seconds=15.0, + max_batch_items=50, + client_factory=factory, + ) + evidence = raised.exception.evidence + self.assertEqual(evidence["transport_error"], "DataLayerError") + self.assertEqual(evidence["transport_error_code"], "PARTIAL_RESULT") + self.assertEqual(evidence["batch_item_problems"][0]["problem_code"], "DATA_NOT_READY") + self.assertTrue(evidence["batch_item_problems"][0]["retryable"]) + self.assertIn("problem_detail_sha256", evidence["batch_item_problems"][0]) + self.assertIsInstance(evidence["batch_item_problems"][0]["quality_sha256"], str) + self.assertNotIn("cache is intentionally", repr(evidence)) + self.assertFalse(evidence["payload_recorded"]) async def test_closing_batch_rejects_partial_cardinality(self) -> None: product = SimpleNamespace( diff --git a/tests/test_phase105_stable_release.py b/tests/test_phase105_stable_release.py index 47be284..75dc3db 100644 --- a/tests/test_phase105_stable_release.py +++ b/tests/test_phase105_stable_release.py @@ -201,16 +201,25 @@ def test_materialized_v2_product_must_remain_in_declared_demand(self): shutil.copytree(ROOT / "consumers", temporary_root / "consumers") demand_path = temporary_root / "config/v2/stable-crypto-demand.yaml" demand = yaml.safe_load(demand_path.read_text(encoding="utf-8")) - requirements = demand["consumers"][0]["requirements"] - removed = next( - item for item in requirements - if item["venue"] == "BINANCE" - and item["market"] == "USDM" - and item["native_symbol"] == "BTCUSDT" - and item["feed"] == "BAR" - and item["interval"] == "1m" - ) - requirements.remove(removed) + def is_btc_1m_bar(item): + return ( + item["venue"] == "BINANCE" + and item["market"] == "USDM" + and item["native_symbol"] == "BTCUSDT" + and item["feed"] == "BAR" + and item["interval"] == "1m" + ) + + # A demand key may be shared by multiple declared consumers. Remove + # every declaration of this key so the release validator is tested + # against an actually absent materialized product, not one still + # requested by a different paper consumer. + removed = 0 + for consumer in demand["consumers"]: + prior = consumer["requirements"] + consumer["requirements"] = [item for item in prior if not is_btc_1m_bar(item)] + removed += len(prior) - len(consumer["requirements"]) + self.assertGreaterEqual(removed, 2) demand_path.write_text( yaml.safe_dump(demand, sort_keys=False), encoding="utf-8" ) diff --git a/tests/test_phase10_universal_demand.py b/tests/test_phase10_universal_demand.py index a03afdc..7b03025 100644 --- a/tests/test_phase10_universal_demand.py +++ b/tests/test_phase10_universal_demand.py @@ -518,13 +518,11 @@ def test_read_only_provider_admission_covers_every_declared_crypto_slice(self): self.assertEqual( {(item.venue, item.market, item.native_symbol) for item in slices}, { - ("BINANCE", "SPOT", "BTCUSDT"), ("BINANCE", "USDM", "BTCUSDT"), ("BINANCE", "USDM", "ETHUSDT"), ("BINANCE", "USDM", "SOLUSDT"), ("BINANCE", "USDM", "DOGEUSDT"), ("BINANCE", "USDM", "BNBUSDT"), - ("OKX", "SPOT", "BTC-USDT"), ("OKX", "SWAP", "BTC-USDT-SWAP"), ("OKX", "SWAP", "ETH-USDT-SWAP"), ("OKX", "SWAP", "SOL-USDT-SWAP"), @@ -545,6 +543,8 @@ def json(self): def fake_get(url, **_): if "binance.com" in url: + if url.endswith("/premiumIndex"): + return Response({"markPrice": "1", "indexPrice": "1", "time": 1_000}) if url.endswith("/trades"): return Response([{"id": 1, "price": "1", "qty": "1", "time": 1_000}]) if url.endswith("/ticker/bookTicker"): @@ -556,6 +556,10 @@ def fake_get(url, **_): "asks": [["2", "1"]], }) return Response([[0, "1", "2", "1", "1", "1", 1_000]]) + if url.endswith("/mark-price"): + return Response({"code": "0", "data": [{"markPx": "1", "ts": "1000"}]}) + if url.endswith("/index-tickers"): + return Response({"code": "0", "data": [{"idxPx": "1", "ts": "1000"}]}) if url.endswith("/trades"): return Response({"code": "0", "data": [{"px": "1", "sz": "1", "tradeId": "1", "ts": "1000"}]}) if url.endswith("/books"): diff --git a/tests/test_phase10_universal_warmup.py b/tests/test_phase10_universal_warmup.py index 5d6ce7b..fef0a74 100644 --- a/tests/test_phase10_universal_warmup.py +++ b/tests/test_phase10_universal_warmup.py @@ -7,6 +7,7 @@ from datetime import datetime, timezone from pathlib import Path import time +import threading from types import SimpleNamespace import unittest @@ -41,6 +42,7 @@ ) from qdl.domain.calendar import trading_calendar_for_id from qdl.query.results import MarketDataItem +from qdl.query.service import _LocalBatchAdmission from qdl.runtime.closed_bar_cache import ClosedBarWindowCache from qdl.runtime.provider_history import ( ProviderBarHistorySource, @@ -57,6 +59,7 @@ BoundedWarmupExecutor, ProviderBudgetPolicy, RetryableWarmupError, + WarmupExecution, ) from qdl.warmup.handoff import ClosedBarFifo, resample_final_bars from qdl.warmup.planner import UniversalWarmupPlanner @@ -437,6 +440,329 @@ async def work(value): self.assertEqual(executor.provider_policies["OKX"].requests_per_second, 5.0) self.assertEqual(executor.provider_policies["BINANCE"].requests_per_second, 8.0) + async def test_local_cache_collocated_batches_start_read_deadline_after_admission(self): + executor = BoundedWarmupExecutor[int, int]( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + max_pending=4, + deadline_starts_after_admission=True, + ) + } + ) + + async def work(value): + await asyncio.sleep(0.02) + return value + + arguments = dict( + work=work, + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 40, + ) + first = asyncio.create_task(executor.execute((1, 2), **arguments)) + await asyncio.sleep(0) + second = asyncio.create_task(executor.execute((3, 4), **arguments)) + one, two = await asyncio.gather(first, second) + + self.assertEqual([item.value for item in one + two], [1, 2, 3, 4]) + self.assertTrue(all(item.ok for item in one + two)) + # At least one item necessarily waited behind another read. Its elapsed + # request time can exceed 40ms, but its admitted cache read must retain + # its own 40ms execution budget. + self.assertGreater(max(item.elapsed_ms for item in one + two), 40.0) + self.assertEqual(executor._pending, {}) + + async def test_local_batch_gate_gives_a_collocated_batch_a_worker_turn(self): + executor = BoundedWarmupExecutor[str, str]( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=2, + requests_per_second=None, + max_attempts=1, + max_pending=8, + deadline_starts_after_admission=True, + max_batch_concurrency=1, + ) + } + ) + started: list[str] = [] + both_started = asyncio.Event() + release = asyncio.Event() + + async def work(value: str) -> str: + started.append(value) + if len(started) == 2: + both_started.set() + await release.wait() + return value + + arguments = dict( + work=work, + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 1_000, + ) + first = asyncio.create_task(executor.execute(("a1", "a2"), **arguments)) + await asyncio.sleep(0) + second = asyncio.create_task(executor.execute(("b1", "b2"), **arguments)) + await asyncio.wait_for(both_started.wait(), timeout=1) + + # The first batch cannot reserve both global workers while the second + # legal batch is waiting. This is a local fairness contract, not a + # provider pacing policy. + self.assertEqual(set(started[:2]), {"a1", "b1"}) + release.set() + first_result, second_result = await asyncio.gather(first, second) + self.assertTrue(all(item.ok for item in first_result + second_result)) + self.assertEqual(executor._pending, {}) + + async def test_local_batch_gate_wait_does_not_spend_admitted_read_deadline(self): + executor = BoundedWarmupExecutor[int, int]( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=2, + requests_per_second=None, + max_attempts=1, + max_pending=8, + deadline_starts_after_admission=True, + max_batch_concurrency=1, + ) + } + ) + + async def work(value: int) -> int: + await asyncio.sleep(0.02) + return value + + arguments = dict( + work=work, + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 30, + ) + first = asyncio.create_task(executor.execute((1, 2), **arguments)) + await asyncio.sleep(0) + second = asyncio.create_task(executor.execute((3, 4), **arguments)) + one, two = await asyncio.gather(first, second) + + self.assertTrue(all(item.ok for item in one + two)) + self.assertGreater(max(item.elapsed_ms for item in one + two), 30.0) + self.assertEqual(executor._pending, {}) + + def test_local_batch_gate_must_fit_inside_global_concurrency(self): + with self.assertRaisesRegex(ValueError, "batch concurrency"): + ProviderBudgetPolicy( + max_concurrency=2, + requests_per_second=None, + max_batch_concurrency=3, + ) + + async def test_local_cache_execution_deadline_still_fails_after_admission(self): + executor = BoundedWarmupExecutor[int, int]( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + max_pending=1, + deadline_starts_after_admission=True, + ) + } + ) + + async def work(_value): + await asyncio.sleep(0.03) + return 1 + + result = await executor.execute( + (1,), + work=work, + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 10, + ) + self.assertFalse(result[0].ok) + self.assertIsInstance(result[0].error, RetryableWarmupError) + self.assertIn("execution deadline", str(result[0].error)) + self.assertEqual(executor._pending, {}) + + async def test_local_cache_pending_capacity_fails_typed_without_unbounded_queue(self): + executor = BoundedWarmupExecutor[int, int]( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + max_pending=1, + deadline_starts_after_admission=True, + ) + } + ) + started = asyncio.Event() + release = asyncio.Event() + + async def work(value): + started.set() + await release.wait() + return value + + arguments = dict( + work=work, + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 1_000, + ) + first = asyncio.create_task(executor.execute((1,), **arguments)) + await started.wait() + rejected = await executor.execute((2,), **arguments) + self.assertFalse(rejected[0].ok) + self.assertIsInstance(rejected[0].error, RetryableWarmupError) + self.assertIn("admission capacity", str(rejected[0].error)) + self.assertEqual(executor.stats()["admission_rejections"], 1) + release.set() + completed = await first + self.assertTrue(completed[0].ok) + self.assertEqual(executor._pending, {}) + + async def test_local_cache_cancellation_releases_pending_admission(self): + executor = BoundedWarmupExecutor[int, int]( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + max_pending=1, + deadline_starts_after_admission=True, + ) + } + ) + started = asyncio.Event() + cancelled = asyncio.Event() + + async def blocking(_value): + started.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + task = asyncio.create_task(executor.execute( + (1,), + work=blocking, + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 1_000, + )) + await started.wait() + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + await asyncio.wait_for(cancelled.wait(), timeout=1) + self.assertEqual(executor._pending, {}) + + resumed = await executor.execute( + (2,), + work=lambda value: asyncio.sleep(0, result=value), + identity=lambda value: value, + provider=lambda _: "LOCAL_CANONICAL_CACHE", + deadline_ms=lambda _: 1_000, + ) + self.assertTrue(resumed[0].ok) + + async def test_external_provider_deadline_still_includes_queue_admission(self): + executor = BoundedWarmupExecutor[int, int]( + provider_policies={ + "BINANCE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + ) + } + ) + started = asyncio.Event() + release = asyncio.Event() + + async def work(value): + if value == 1: + started.set() + await release.wait() + return value + + arguments = dict( + work=work, + identity=lambda value: value, + provider=lambda _: "BINANCE", + ) + first = asyncio.create_task(executor.execute((1,), deadline_ms=lambda _: 1_000, **arguments)) + await started.wait() + queued = await executor.execute((2,), deadline_ms=lambda _: 10, **arguments) + self.assertFalse(queued[0].ok) + self.assertIn("deadline exceeded", str(queued[0].error)) + release.set() + completed = await first + self.assertTrue(completed[0].ok) + + async def test_internal_stream_has_bounded_concurrency_without_external_pacing_or_retry(self): + sleeps = [] + running = 0 + peak = 0 + + async def sleep(delay): + sleeps.append(delay) + + async def work(value): + nonlocal running, peak + running += 1 + peak = max(peak, running) + await asyncio.sleep(0) + running -= 1 + return value + + executor = BoundedWarmupExecutor(sleep=sleep) + result = await executor.execute( + range(10), + work=work, + identity=lambda value: value, + provider=lambda _: "INTERNAL_STREAM", + deadline_ms=lambda _: 2_000, + ) + policy = executor.provider_policies["INTERNAL_STREAM"] + self.assertEqual([item.value for item in result], list(range(10))) + self.assertTrue(all(item.ok and item.attempts == 1 for item in result)) + self.assertLessEqual(peak, 4) + self.assertEqual(sleeps, []) + self.assertIsNone(policy.requests_per_second) + self.assertEqual(policy.max_attempts, 1) + self.assertEqual(policy.circuit_cooldown_ms, 1_000) + self.assertEqual(executor.provider_policies["OKX"].requests_per_second, 5.0) + self.assertEqual(executor.provider_policies["BINANCE"].requests_per_second, 8.0) + self.assertEqual(executor.provider_policies["DNSE"].requests_per_second, 2.0) + + async def test_internal_stream_retryable_failure_is_one_typed_attempt(self): + calls = 0 + + async def work(_value): + nonlocal calls + calls += 1 + raise RetryableWarmupError("test internal transport failure") + + executor = BoundedWarmupExecutor[int, int]() + result = await executor.execute( + (1,), + work=work, + identity=lambda value: value, + provider=lambda _: "INTERNAL_STREAM", + deadline_ms=lambda _: 2_000, + ) + self.assertEqual(calls, 1) + self.assertFalse(result[0].ok) + self.assertEqual(result[0].attempts, 1) + self.assertEqual(executor.retry_count, 0) + async def test_identical_concurrent_work_is_singleflight(self): executor = BoundedWarmupExecutor[int, int]() started = asyncio.Event() @@ -562,7 +888,7 @@ async def fail(_): arguments = dict( work=fail, - identity=lambda value: value, + identity=lambda _value: "one-route-generation", provider=lambda _: "OKX", deadline_ms=lambda _: 1_000, ) @@ -573,6 +899,50 @@ async def fail(_): self.assertIn("circuit is open", str(result[0].error)) self.assertEqual(executor.source_calls, 2) + async def test_internal_stream_circuit_is_route_generation_scoped_and_success_resets_only_that_key(self): + now = [0.0] + fail_generation_one = [True] + executor = BoundedWarmupExecutor[tuple[str, str], tuple[str, str]]( + provider_policies={ + "INTERNAL_STREAM": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + circuit_failures=1, + circuit_cooldown_ms=1_000, + ) + }, + clock=lambda: now[0], + ) + + async def work(item): + if item == ("BTC", "generation-1") and fail_generation_one[0]: + raise RetryableWarmupError("local stream route unavailable") + return item + + arguments = dict( + work=work, + identity=lambda item: item, + provider=lambda _: "INTERNAL_STREAM", + deadline_ms=lambda _: 2_000, + ) + first = await executor.execute((("BTC", "generation-1"),), **arguments) + self.assertFalse(first[0].ok) + unaffected = await executor.execute((("BTC", "generation-2"),), **arguments) + self.assertTrue(unaffected[0].ok) + rejected = await executor.execute((("BTC", "generation-1"),), **arguments) + self.assertFalse(rejected[0].ok) + self.assertIn("circuit is open", str(rejected[0].error)) + + now[0] = 1.0 + fail_generation_one[0] = False + recovered = await executor.execute((("BTC", "generation-1"),), **arguments) + self.assertTrue(recovered[0].ok) + self.assertEqual( + executor._circuit[("INTERNAL_STREAM", ("BTC", "generation-1"))], + (0, 0.0), + ) + async def test_deadline_cancels_unshared_underlying_work(self): executor = BoundedWarmupExecutor[int, int]() cancelled = asyncio.Event() @@ -1171,37 +1541,29 @@ def warmup(self, requirement, *, purpose, request_id=None): self.assertEqual(set(service.warmup_executor._semaphores), {expected}) self.assertEqual(set(service.warmup_executor._tokens), set() if local else {"OKX"}) - async def test_retryable_local_cache_error_does_not_open_shared_circuit(self): - unavailable_uid = BINANCE_ETH - ready_uid = "ee93fabf-68df-5b50-8924-51bf25a5a758" - + async def test_query_local_batch_collocation_keeps_each_cache_read_deadline(self): class Service(V2QueryService): def __init__(self): self.instruments = SimpleNamespace(get=lambda _: SimpleNamespace( - identity=SimpleNamespace(venue="OKX"))) + identity=SimpleNamespace(venue="OKX") + )) self.backend = SimpleNamespace(warmup_is_local=lambda _: True) self.warmup_executor = BoundedWarmupExecutor( provider_policies={ "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, max_attempts=1, - circuit_failures=1, - circuit_cooldown_ms=60_000, + max_pending=4, + deadline_starts_after_admission=True, ) } ) self.last_batch_evidence = {} def warmup(self, requirement, *, purpose, request_id=None): - del purpose, request_id - if requirement.instrument_uid == unavailable_uid: - raise QueryServiceError( - QueryProblem( - CanonicalErrorCode.DATA_NOT_READY, - "injected local cache not ready", - True, - ), - request_id="local-cache", - ) + del requirement, purpose, request_id + time.sleep(0.15) return "warmup-ok" def requirement(instrument_uid: str) -> DataRequirement: @@ -1211,23 +1573,947 @@ def requirement(instrument_uid: str) -> DataRequirement: consumer_grade=ConsumerGrade.ALPHA, source_policy_id="crypto_primary_v2", interval="1m", - warmup=WarmupSpecification.for_rows(1), + warmup=WarmupSpecification.for_rows(1, deadline_ms=400), ) service = Service() - result = await service.warmup_batch_async( + first = asyncio.create_task(service.warmup_batch_async( BatchRequirement( - consumer_id="phase10-local-cache-isolation", - requirements=(requirement(unavailable_uid), requirement(ready_uid)), + consumer_id="local-batch-a", + requirements=(requirement("local-a-1"), requirement("local-a-2")), ), purpose=AccessPurpose.INTERNAL_ALPHA, - ) - self.assertEqual([item.status for item in result.results], ["DATA_NOT_READY", "OK"]) - self.assertEqual(service.warmup_executor.circuit_rejections, 0) + )) + await asyncio.sleep(0) + second = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-b", + requirements=(requirement("local-b-1"), requirement("local-b-2")), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + first_result, second_result = await asyncio.gather(first, second) self.assertEqual( - service.warmup_executor._circuit["LOCAL_CANONICAL_CACHE"], - (0, 0.0), + [item.status for item in first_result.results + second_result.results], + ["OK", "OK", "OK", "OK"], ) + self.assertEqual(service.warmup_executor._pending, {}) + + async def test_query_local_batch_shares_one_snapshot_and_keeps_item_failure_typed(self): + first_uid = "local-batch-history-a" + second_uid = "local-batch-history-b" + + class Backend: + def __init__(self): + self.history_many_calls = 0 + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.history_many_calls += 1 + return { + requirements[0]: "history-a", + requirements[1]: QueryServiceError( + QueryProblem( + CanonicalErrorCode.DATA_NOT_READY, + "injected local cache absence", + True, + ), + request_id="batch-snapshot", + ), + } + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=2, + requests_per_second=None, + max_attempts=1, + max_pending=4, + deadline_starts_after_admission=True, + max_batch_concurrency=2, + ) + } + ) + self.last_batch_evidence = {} + + def warmup(self, *_args, **_kwargs): + raise AssertionError("local batch must use its shared snapshot") + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1), + ) + + service = Service() + result = await service.warmup_batch_async( + BatchRequirement( + consumer_id="phase10-local-batch-snapshot", + requirements=(requirement(first_uid), requirement(second_uid)), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + + self.assertEqual([item.status for item in result.results], ["OK", "DATA_NOT_READY"]) + self.assertEqual(service.backend.history_many_calls, 1) + self.assertTrue(service.last_batch_evidence["local_batch_snapshot"]) + self.assertEqual(service.last_batch_evidence["local_batch_items"], 2) + self.assertEqual(service.warmup_executor._pending, {}) + + async def test_query_local_batch_snapshot_cancellation_releases_admission(self): + class Backend: + def __init__(self): + self.started = threading.Event() + self.release = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.started.set() + self.release.wait(timeout=1) + return {requirement: "history" for requirement in requirements} + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_concurrency=1, + requests_per_second=None, + max_attempts=1, + max_pending=1, + deadline_starts_after_admission=True, + ) + } + ) + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + requirement = DataRequirement( + instrument_uid="local-batch-cancel", + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + service = Service() + task = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="phase10-local-batch-cancel", + requirements=(requirement,), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + self.assertTrue(await asyncio.to_thread(service.backend.started.wait, 1)) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertEqual(service.warmup_executor._pending, {}) + admission = service._local_batch_admission_for() + self.assertEqual(admission.stats()["active"], 1) + self.assertEqual(admission.stats()["pending"], 1) + service.backend.release.set() + + async def wait_for_drain(): + while admission.stats()["pending"]: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_drain(), timeout=1) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_serializes_collocated_history_materialization(self): + class Backend: + def __init__(self): + self.calls = 0 + self.first_started = threading.Event() + self.second_started = threading.Event() + self.release = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.calls += 1 + if self.calls == 1: + self.first_started.set() + self.release.wait(timeout=1) + else: + self.second_started.set() + return {requirement: "history" for requirement in requirements} + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor() + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + + service = Service() + first = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-serial-a", + requirements=(requirement("serial-a"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + self.assertTrue(await asyncio.to_thread(service.backend.first_started.wait, 1)) + second = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-serial-b", + requirements=(requirement("serial-b"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + admission = service._local_batch_admission_for() + + async def wait_for_queued_second(): + while admission.stats()["pending"] != 2: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_queued_second(), timeout=1) + self.assertEqual(service.backend.calls, 1) + self.assertFalse(service.backend.second_started.is_set()) + service.backend.release.set() + first_result, second_result = await asyncio.gather(first, second) + self.assertEqual( + [item.status for item in first_result.results + second_result.results], + ["OK", "OK"], + ) + self.assertEqual(service.backend.calls, 2) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_holds_admission_through_response_assembly(self): + class Backend: + def __init__(self): + self.calls = 0 + self.second_started = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.calls += 1 + if self.calls > 1: + self.second_started.set() + return {requirement: "history" for requirement in requirements} + + class BlockingExecutor: + def __init__(self): + self.calls = 0 + self.first_assembly_started = asyncio.Event() + self.release = asyncio.Event() + + @staticmethod + def stats(): + return {} + + async def execute(self, items, *, work, identity, provider, deadline_ms): + del identity, provider, deadline_ms + self.calls += 1 + if self.calls == 1: + self.first_assembly_started.set() + await self.release.wait() + executions = [] + for item in items: + executions.append(WarmupExecution( + item, + await work(item), + None, + 1, + False, + 0.0, + )) + return tuple(executions) + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BlockingExecutor() + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + + service = Service() + first = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-assembly-a", + requirements=(requirement("assembly-a"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + await asyncio.wait_for( + service.warmup_executor.first_assembly_started.wait(), timeout=1 + ) + second = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-assembly-b", + requirements=(requirement("assembly-b"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + admission = service._local_batch_admission_for() + + async def wait_for_queued_second(): + while admission.stats()["pending"] != 2: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_queued_second(), timeout=1) + self.assertEqual(service.backend.calls, 1) + self.assertFalse(service.backend.second_started.is_set()) + service.warmup_executor.release.set() + first_result, second_result = await asyncio.gather(first, second) + self.assertEqual( + [item.status for item in first_result.results + second_result.results], + ["OK", "OK"], + ) + self.assertEqual(service.backend.calls, 2) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_holds_admission_through_http_completion(self): + class Backend: + def __init__(self): + self.calls = 0 + self.second_started = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.calls += 1 + if self.calls > 1: + self.second_started.set() + return {requirement: "history" for requirement in requirements} + + class Executor: + @staticmethod + def stats(): + return {} + + async def execute(self, items, *, work, identity, provider, deadline_ms): + del identity, provider, deadline_ms + executions = [] + for item in items: + executions.append( + WarmupExecution(item, await work(item), None, 1, False, 0.0) + ) + return tuple(executions) + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = Executor() + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + + completion_started = asyncio.Event() + release_completion = asyncio.Event() + completion_calls = 0 + + async def completion(result): + nonlocal completion_calls + completion_calls += 1 + if completion_calls == 1: + completion_started.set() + await release_completion.wait() + return result + + service = Service() + first = asyncio.create_task(service.warmup_batch_completed_async( + BatchRequirement( + consumer_id="local-http-completion-a", + requirements=(requirement("http-a"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + completion=completion, + )) + await asyncio.wait_for(completion_started.wait(), timeout=1) + second = asyncio.create_task(service.warmup_batch_completed_async( + BatchRequirement( + consumer_id="local-http-completion-b", + requirements=(requirement("http-b"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + completion=completion, + )) + admission = service._local_batch_admission_for() + + async def wait_for_queued_second(): + while admission.stats()["pending"] != 2: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_queued_second(), timeout=1) + self.assertEqual(service.backend.calls, 1) + self.assertFalse(service.backend.second_started.is_set()) + release_completion.set() + first_result, second_result = await asyncio.gather(first, second) + self.assertEqual( + [item.status for item in first_result.results + second_result.results], + ["OK", "OK"], + ) + self.assertEqual(service.backend.calls, 2) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_cancellation_keeps_http_completion_lease_until_drain(self): + class Backend: + @staticmethod + def warmup_is_local(_requirement): + return True + + @staticmethod + def history_many(requirements): + return {requirement: "history" for requirement in requirements} + + class Executor: + @staticmethod + def stats(): + return {} + + async def execute(self, items, *, work, identity, provider, deadline_ms): + del identity, provider, deadline_ms + executions = [] + for item in items: + executions.append( + WarmupExecution(item, await work(item), None, 1, False, 0.0) + ) + return tuple(executions) + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = Executor() + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + requirement = DataRequirement( + instrument_uid="local-http-completion-cancel", + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + completion_started = asyncio.Event() + release_completion = asyncio.Event() + + async def completion(result): + completion_started.set() + await release_completion.wait() + return result + + service = Service() + task = asyncio.create_task(service.warmup_batch_completed_async( + BatchRequirement( + consumer_id="local-http-completion-cancel", + requirements=(requirement,), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + completion=completion, + )) + await asyncio.wait_for(completion_started.wait(), timeout=1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + admission = service._local_batch_admission_for() + self.assertEqual(admission.stats()["active"], 1) + self.assertEqual(admission.stats()["pending"], 1) + release_completion.set() + + async def wait_for_drain(): + while admission.stats()["pending"]: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_drain(), timeout=1) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_cancellation_keeps_assembly_lease_until_drain(self): + class Backend: + @staticmethod + def warmup_is_local(_requirement): + return True + + @staticmethod + def history_many(requirements): + return {requirement: "history" for requirement in requirements} + + class BlockingExecutor: + def __init__(self): + self.assembly_started = asyncio.Event() + self.release = asyncio.Event() + + @staticmethod + def stats(): + return {} + + async def execute(self, items, *, work, identity, provider, deadline_ms): + del identity, provider, deadline_ms + self.assembly_started.set() + await self.release.wait() + executions = [] + for item in items: + executions.append(WarmupExecution( + item, + await work(item), + None, + 1, + False, + 0.0, + )) + return tuple(executions) + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BlockingExecutor() + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + requirement = DataRequirement( + instrument_uid="local-batch-assembly-cancel", + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + service = Service() + task = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-assembly-cancel", + requirements=(requirement,), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + await asyncio.wait_for(service.warmup_executor.assembly_started.wait(), timeout=1) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + admission = service._local_batch_admission_for() + self.assertEqual(admission.stats()["active"], 1) + self.assertEqual(admission.stats()["pending"], 1) + service.warmup_executor.release.set() + + async def wait_for_drain(): + while admission.stats()["pending"]: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_drain(), timeout=1) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_rejects_at_bounded_admission_then_recovers(self): + class Backend: + def __init__(self): + self.calls = 0 + self.started = threading.Event() + self.release = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.calls += 1 + if self.calls == 1: + self.started.set() + self.release.wait(timeout=1) + return {requirement: "history" for requirement in requirements} + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor() + self.last_batch_evidence = {} + self._local_batch_admission = _LocalBatchAdmission(max_pending=1) + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + + service = Service() + first = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-capacity-a", + requirements=(requirement("capacity-a"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + self.assertTrue(await asyncio.to_thread(service.backend.started.wait, 1)) + rejected = await service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-capacity-b", + requirements=(requirement("capacity-b"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + self.assertEqual([item.status for item in rejected.results], ["RATE_LIMITED"]) + self.assertEqual(service.backend.calls, 1) + self.assertEqual(service._local_batch_admission.stats()["rejected"], 1) + service.backend.release.set() + first_result = await first + self.assertEqual([item.status for item in first_result.results], ["OK"]) + recovered = await service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-capacity-c", + requirements=(requirement("capacity-c"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + self.assertEqual([item.status for item in recovered.results], ["OK"]) + self.assertEqual(service.backend.calls, 2) + self.assertEqual(service._local_batch_admission.stats()["active"], 0) + self.assertEqual(service._local_batch_admission.stats()["pending"], 0) + + async def test_query_local_batch_default_admission_preserves_incumbent_and_four_manifest_lanes(self): + class Backend: + def __init__(self): + self.calls = 0 + self.active = 0 + self.peak = 0 + self.lock = threading.Lock() + self.started = threading.Event() + self.release = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + with self.lock: + self.calls += 1 + self.active += 1 + self.peak = max(self.peak, self.active) + if self.calls == 1: + self.started.set() + self.release.wait(timeout=1) + time.sleep(0.01) + with self.lock: + self.active -= 1 + return {requirement: "history" for requirement in requirements} + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor() + self.last_batch_evidence = {} + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=1_000), + ) + + service = Service() + consumer_ids = ( + "monitoring.multivenue.stable", + "trading-system.paper.stable", + "alpha.binance.paper.stable", + "alpha.okx.paper.stable", + ) + incumbent = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-existing-reader", + requirements=(requirement("existing-reader"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + self.assertTrue(await asyncio.to_thread(service.backend.started.wait, 1)) + tasks = tuple( + asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id=consumer_id, + requirements=(requirement(f"default-{index}"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + for index, consumer_id in enumerate(consumer_ids) + ) + admission = service._local_batch_admission_for() + + async def wait_for_five_lanes(): + while admission.stats()["pending"] != 5: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_five_lanes(), timeout=1) + rejected = await service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-default-overflow", + requirements=(requirement("default-overflow"),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + self.assertEqual([item.status for item in rejected.results], ["RATE_LIMITED"]) + self.assertEqual(service.backend.calls, 1) + self.assertEqual(admission.stats()["rejected"], 1) + service.backend.release.set() + results = await asyncio.gather(incumbent, *tasks) + self.assertTrue(all(item.status == "OK" for result in results for item in result.results)) + self.assertEqual(service.backend.calls, 5) + self.assertEqual(service.backend.peak, 1) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_query_local_batch_queue_wait_expiry_is_typed_and_drains(self): + class Backend: + def __init__(self): + self.calls = 0 + self.started = threading.Event() + self.release = threading.Event() + + @staticmethod + def warmup_is_local(_requirement): + return True + + def history_many(self, requirements): + self.calls += 1 + if self.calls == 1: + self.started.set() + self.release.wait(timeout=1) + return {requirement: "history" for requirement in requirements} + + class Service(V2QueryService): + def __init__(self): + self.backend = Backend() + self.instruments = SimpleNamespace() + self.warmup_executor = BoundedWarmupExecutor() + self.last_batch_evidence = {} + self._local_batch_admission = _LocalBatchAdmission(max_pending=2) + + def _warmup_from_history(self, requirement, history, *, purpose, request_id): + del requirement, purpose, request_id + return history + + def requirement(instrument_uid: str, deadline_ms: int) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1, deadline_ms=deadline_ms), + ) + + service = Service() + first = asyncio.create_task(service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-wait-a", + requirements=(requirement("wait-a", 1_000),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + )) + self.assertTrue(await asyncio.to_thread(service.backend.started.wait, 1)) + expired = await service.warmup_batch_async( + BatchRequirement( + consumer_id="local-batch-wait-b", + requirements=(requirement("wait-b", 100),), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + self.assertEqual([item.status for item in expired.results], ["RATE_LIMITED"]) + self.assertIn("declared deadline", expired.results[0].problem.detail) + admission = service._local_batch_admission_for() + self.assertEqual(admission.stats()["queue_wait_timeouts"], 1) + self.assertEqual(admission.stats()["pending"], 1) + service.backend.release.set() + first_result = await first + self.assertEqual([item.status for item in first_result.results], ["OK"]) + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_local_batch_admission_queued_cancellation_releases_capacity(self): + admission = _LocalBatchAdmission() + active_started = asyncio.Event() + release_active = asyncio.Event() + + async def active_work(): + active_started.set() + await release_active.wait() + return "active" + + async def queued_work(): + return "queued" + + active = asyncio.create_task(admission.run(active_work, wait_timeout_ms=1_000)) + await asyncio.wait_for(active_started.wait(), timeout=1) + queued = asyncio.create_task(admission.run(queued_work, wait_timeout_ms=1_000)) + + async def wait_for_queue(): + while admission.stats()["pending"] != 2: + await asyncio.sleep(0) + + await asyncio.wait_for(wait_for_queue(), timeout=1) + queued.cancel() + with self.assertRaises(asyncio.CancelledError): + await queued + self.assertEqual(admission.stats()["active"], 1) + self.assertEqual(admission.stats()["pending"], 1) + release_active.set() + self.assertEqual(await active, "active") + self.assertEqual(admission.stats()["active"], 0) + self.assertEqual(admission.stats()["pending"], 0) + + async def test_retryable_local_cache_error_does_not_open_shared_circuit(self): + unavailable_uid = BINANCE_ETH + ready_uid = "ee93fabf-68df-5b50-8924-51bf25a5a758" + + class Service(V2QueryService): + def __init__(self): + self.instruments = SimpleNamespace(get=lambda _: SimpleNamespace( + identity=SimpleNamespace(venue="OKX"))) + self.backend = SimpleNamespace(warmup_is_local=lambda _: True) + self.warmup_executor = BoundedWarmupExecutor( + provider_policies={ + "LOCAL_CANONICAL_CACHE": ProviderBudgetPolicy( + max_attempts=1, + circuit_failures=1, + circuit_cooldown_ms=60_000, + ) + } + ) + self.last_batch_evidence = {} + + def warmup(self, requirement, *, purpose, request_id=None): + del purpose, request_id + if requirement.instrument_uid == unavailable_uid: + raise QueryServiceError( + QueryProblem( + CanonicalErrorCode.DATA_NOT_READY, + "injected local cache not ready", + True, + ), + request_id="local-cache", + ) + return "warmup-ok" + + def requirement(instrument_uid: str) -> DataRequirement: + return DataRequirement( + instrument_uid=instrument_uid, + feed=FeedType.BAR, + consumer_grade=ConsumerGrade.ALPHA, + source_policy_id="crypto_primary_v2", + interval="1m", + warmup=WarmupSpecification.for_rows(1), + ) + + service = Service() + result = await service.warmup_batch_async( + BatchRequirement( + consumer_id="phase10-local-cache-isolation", + requirements=(requirement(unavailable_uid), requirement(ready_uid)), + ), + purpose=AccessPurpose.INTERNAL_ALPHA, + ) + self.assertEqual([item.status for item in result.results], ["DATA_NOT_READY", "OK"]) + self.assertEqual(service.warmup_executor.circuit_rejections, 0) + circuit_states = { + key[1].instrument_uid: state + for key, state in service.warmup_executor._circuit.items() + if key[0] == "LOCAL_CANONICAL_CACHE" + } + self.assertEqual(circuit_states[ready_uid], (0, 0.0)) + # The query service converts this local-cache absence directly to the + # typed result; it must neither create nor clear another route's + # circuit state. + self.assertNotIn(unavailable_uid, circuit_states) async def test_single_warmup_is_nonblocking_and_reuses_retry_policy(self): class Service(V2QueryService): diff --git a/tests/test_phase113_reference_v2.py b/tests/test_phase113_reference_v2.py index 64f688e..46cf2c4 100644 --- a/tests/test_phase113_reference_v2.py +++ b/tests/test_phase113_reference_v2.py @@ -189,22 +189,25 @@ async def execute(self, *args, **kwargs): class AdvanceClockAfterBatchedMarkRefreshExecutor(BoundedWarmupExecutor): - """Expose stale response assembly after a multi-item MARK refresh. + """Expose stale response assembly before one concurrent multi-item refresh. - The first two-item execute ages initial snapshots. A legacy second - two-item refresh would age both refreshed snapshots before response - assembly. Per-item refreshes remain current at their own assembly turn. + The first two-item execute ages initial snapshots. The replacement refresh + is concurrent and response-time validation happens after it completes, so + the fixture advances only that initial batch instead of inventing a second + two-second pause after current replacement observations exist. """ def __init__(self, *args, clock, adapter, **kwargs): super().__init__(*args, **kwargs) self._test_clock = clock self._test_adapter = adapter + self._advance_once = True async def execute(self, items, **kwargs): values = tuple(items) result = await super().execute(values, **kwargs) - if len(values) == 2: + if len(values) == 2 and self._advance_once: + self._advance_once = False self._test_clock["ns"] += 2_100_000_000 self._test_adapter.observed_at_ns = self._test_clock["ns"] return result diff --git a/tests/test_phase115c_native_bar_materialization.py b/tests/test_phase115c_native_bar_materialization.py index 8e391a8..27bd663 100644 --- a/tests/test_phase115c_native_bar_materialization.py +++ b/tests/test_phase115c_native_bar_materialization.py @@ -170,7 +170,15 @@ def test_exact_active_native_interval_sets_are_materialized(self) -> None: self._intervals("OKX", "SWAP", symbol), set(OKX_NATIVE_INTERVALS), ) - self.assertEqual(self.summary["demand_additions"], 130) + expected_demand_additions = sum( + len(current["requirements"]) - len(before["requirements"]) + for current, before in zip( + self.current_demand["consumers"], + self.before_demand["consumers"], + strict=True, + ) + ) + self.assertEqual(self.summary["demand_additions"], expected_demand_additions) self.assertEqual(self.summary["bar_binding_counts"], { "binance_usdm": 70, "okx_swap": 70, "dnse": 2, }) diff --git a/tests/test_phase533_alpha_runtime_entitlements.py b/tests/test_phase533_alpha_runtime_entitlements.py index fa52a13..d900ec1 100644 --- a/tests/test_phase533_alpha_runtime_entitlements.py +++ b/tests/test_phase533_alpha_runtime_entitlements.py @@ -8,7 +8,7 @@ import yaml from qdl.consumer import ConsumerManifestLoader, requirement_key -from qdl.query import ConsumerGrade, FeedType +from qdl.query import ConsumerGrade, FeedType, StalePolicy ROOT = Path(__file__).resolve().parents[1] @@ -105,6 +105,13 @@ def test_five_liquid_manifests_are_complete_bounded_and_non_execution(self) -> N self.assertTrue(all(item.recovery.value == "SNAPSHOT_AND_REPLAY" for item in manifest.requirements if item.feed is FeedType.BAR)) self.assertTrue(all(item.require_final_bars for item in manifest.requirements if item.feed is FeedType.BAR)) self.assertTrue(all(item.max_session_liveness_ms == 45_000 for item in manifest.requirements if item.feed in {FeedType.TRADE, FeedType.QUOTE, FeedType.BOOK_SNAPSHOT, FeedType.BOOK_DELTA})) + self.assertTrue( + all( + item.effective_event_recency_policy is StalePolicy.OBSERVE + for item in manifest.requirements + if item.feed in {FeedType.TRADE, FeedType.BOOK_DELTA} + ) + ) def test_native_identity_interval_and_venue_never_cross_mix(self) -> None: catalog_by_uid = { @@ -126,6 +133,86 @@ def test_native_identity_interval_and_venue_never_cross_mix(self) -> None: self.assertEqual(len(bars_by_uid), 5) self.assertTrue(all(len(intervals) == 14 for intervals in bars_by_uid.values())) + def test_declared_on_change_quotes_use_session_semantics_only(self) -> None: + bindings = { + ( + item["instrument_uid"], + item["feed"], + item.get("interval"), + item["source"]["source_policy_id"], + ): item + for item in self.catalog["bindings"] + } + observed = 0 + for payload in self.rendered.values(): + manifest = ConsumerManifestLoader.from_mapping(payload) + for requirement in manifest.requirements: + if requirement.feed is not FeedType.QUOTE: + continue + binding = bindings[( + requirement.instrument_uid, + requirement.feed.value, + requirement.interval, + requirement.source_policy_id, + )] + semantics = binding["quality"].get("delivery_semantics", "STRICT_EVENT") + if semantics == "ON_CHANGE": + observed += 1 + self.assertEqual(requirement.effective_event_recency_policy, StalePolicy.OBSERVE) + self.assertEqual(requirement.max_session_liveness_ms, 45_000) + else: + self.assertEqual(requirement.effective_event_recency_policy, StalePolicy.BLOCK) + self.assertEqual(observed, 10) + + strict_quote = self.tool._manifest_requirement( + { + "feed": "QUOTE", + "source_policy_id": "strict-test", + "interval": None, + }, + instrument_uid="strict-quote", + binding={ + "quality": { + "stale_after_ms": 5_000, + "require_final_bar": False, + "delivery_semantics": "STRICT_EVENT", + }, + }, + ) + self.assertNotIn("event_recency_policy", strict_quote) + + def test_identical_shared_realtime_demand_is_unioned_by_physical_identity(self) -> None: + instrument = next( + item + for item in self.catalog["instruments"] + if item["venue"] == "BINANCE" + and item["market"] == "USDM" + and item["native_symbol"] == "BTCUSDT" + ) + rows = self.tool._demand_rows(self.demand, instrument=instrument) + self.assertEqual(len(rows), 18) + self.assertEqual(len({self.tool._identity(row) for row in rows}), 18) + + def test_conflicting_shared_realtime_demand_fails_closed(self) -> None: + conflicting = deepcopy(self.demand) + for consumer in conflicting["consumers"]: + if consumer["consumer_id"] != "alpha.binance.paper.stable": + continue + for row in consumer["requirements"]: + if row.get("native_symbol") == "BTCUSDT" and row.get("feed") == "TRADE": + row["max_freshness_ms"] = 4_999 + break + break + with self.assertRaisesRegex(ValueError, "conflicting shared runtime identity"): + self.tool.build_documents( + catalog=self.catalog, + demand=conflicting, + reference_manifest=self.reference, + alpha_manifests=self.manifests, + release_route=self.release, + primary_route=self.primary, + ) + def test_release_routes_are_complete_and_only_trade_can_fallback(self) -> None: routes = { item["consumer_id"]: item diff --git a/tests/test_phaseb_stable_deployment.py b/tests/test_phaseb_stable_deployment.py index d33d8d4..dd593db 100644 --- a/tests/test_phaseb_stable_deployment.py +++ b/tests/test_phaseb_stable_deployment.py @@ -16,6 +16,7 @@ from qdl.adapters.intervals import canonical_interval_ms from qdl.adapters.vn import build_dnse_bar_raw_envelope +from qdl.query import FeedType, StalePolicy from qdl.stream import requirement_from_proto from qdl.runtime.stable_bar_edge import StableBinanceBarEdge from qdl.runtime.stable_vn_edge import StableDnseVendorEdge @@ -186,6 +187,139 @@ def test_c39_trade_request_matches_quiet_feed_manifest_entitlement(self): self.assertEqual(requirement.event_recency_policy.value, "OBSERVE") self.assertEqual(requirement.max_session_liveness_ms, 45_000) + def test_trading_paper_quote_routes_are_exact_native_on_change_bbo(self): + manifest = ConsumerManifestLoader.load( + ROOT / "consumers/stable/trading-system-paper.yaml" + ) + acquisition_by_id = { + item.binding_id: item for item in self.acquisition.bindings + } + quote_requirements = tuple( + item for item in manifest.requirements if item.feed is FeedType.QUOTE + ) + self.assertEqual(len(quote_requirements), 10) + expected = { + ("BINANCE", symbol) + for symbol in ("BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "BNBUSDT") + } | { + ("OKX", symbol) + for symbol in ( + "BTC-USDT-SWAP", "ETH-USDT-SWAP", "SOL-USDT-SWAP", + "DOGE-USDT-SWAP", "BNB-USDT-SWAP", + ) + } + observed = set() + for requirement in quote_requirements: + with self.subTest(instrument_uid=requirement.instrument_uid): + binding = self.catalog.binding_for(requirement) + acquisition = acquisition_by_id[binding.binding_id] + observed.add(( + binding.instrument.identity.venue, + binding.instrument.native_symbol, + )) + self.assertIs(requirement.effective_event_recency_policy, StalePolicy.OBSERVE) + self.assertEqual(requirement.max_session_liveness_ms, 2_000) + self.assertEqual(binding.delivery_semantics, "ON_CHANGE") + self.assertEqual(acquisition.mode, "RUST_NATIVE") + self.assertIn( + acquisition.provider_kind, + {"binance_usdm_bbo", "okx_bbo"}, + ) + if acquisition.runtime == "BINANCE": + self.assertTrue(acquisition.native_channel.endswith("@bookTicker")) + else: + self.assertEqual(acquisition.native_channel, "bbo-tbt") + self.assertEqual(observed, expected) + on_change = { + ( + item.instrument.identity.venue, + item.instrument.identity.market, + item.instrument.native_symbol, + ) + for item in self.catalog.bindings + if item.delivery_semantics == "ON_CHANGE" + } + self.assertEqual( + on_change, + { + (venue, "USDM" if venue == "BINANCE" else "SWAP", symbol) + for venue, symbol in expected + }, + ) + self.assertEqual( + next( + item.delivery_semantics + for item in self.catalog.bindings + if item.binding_id == "binance-spot-btcusdt-quote" + ), + "STRICT_EVENT", + ) + self.assertEqual( + next( + item.delivery_semantics + for item in self.catalog.bindings + if item.binding_id == "okx-spot-btcusdt-quote" + ), + "STRICT_EVENT", + ) + + def test_execution_mark_index_bindings_remain_acquired_promoted_and_demanded(self): + """Keep the active MARK/INDEX plane coupled to its declared readers.""" + expected = { + f"binance-usdm-{symbol.lower()}-mark_index_price" + for symbol in ("BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "BNBUSDT") + } | { + f"okx-swap-{symbol.lower()}-mark_index_price" + for symbol in ( + "BTC-USDT-SWAP", "ETH-USDT-SWAP", "SOL-USDT-SWAP", + "DOGE-USDT-SWAP", "BNB-USDT-SWAP", + ) + } + catalog_by_id = {item.binding_id: item for item in self.catalog.bindings} + acquisition_ids = {item.binding_id for item in self.acquisition.bindings} + self.assertTrue(expected <= set(catalog_by_id)) + self.assertTrue(expected <= acquisition_ids) + self.assertTrue(expected <= set(self.promotion_scope.binding_ids)) + self.assertTrue(all( + catalog_by_id[binding_id].feed is FeedType.MARK_INDEX_PRICE + and catalog_by_id[binding_id].freshness_basis == "PROVIDER_CONFIRMATION" + for binding_id in expected + )) + + demand = yaml.safe_load( + (ROOT / "config/v2/stable-crypto-demand.yaml").read_text(encoding="utf-8") + ) + consumers = {item["consumer_id"]: item for item in demand["consumers"]} + self.assertEqual( + set(consumers), + { + "trading-system.paper.stable", + "alpha.binance.paper.stable", + "alpha.okx.paper.stable", + }, + ) + marks = { + ( + requirement["venue"], + requirement["native_symbol"], + ) + for requirement in consumers["trading-system.paper.stable"]["requirements"] + if requirement["feed"] == "MARK_INDEX_PRICE" + } + self.assertEqual( + marks, + { + ("BINANCE", symbol) + for symbol in ("BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "BNBUSDT") + } | { + ("OKX", symbol) + for symbol in ( + "BTC-USDT-SWAP", "ETH-USDT-SWAP", "SOL-USDT-SWAP", + "DOGE-USDT-SWAP", "BNB-USDT-SWAP", + ) + }, + ) + def test_tls_generator_covers_all_published_ingress_aliases(self): script = (ROOT / "scripts/phase80_generate_tls.sh").read_text( encoding="utf-8" @@ -385,6 +519,7 @@ def test_all_catalog_bindings_have_one_capability_truthful_acquisition(self): "BAR": "LOSSLESS", "TRADE": "LOSSLESS", "QUOTE": "LATEST_STATE", + "MARK_INDEX": "LATEST_STATE", "BOOK": "LOSSLESS", }, ) @@ -609,7 +744,7 @@ def test_full_generated_bundle_keeps_every_enabled_binding_on_strict_v2_ingress( ) self.assertEqual( {item["feed"] for item in okx["bindings"]}, - {"BAR", "TRADE", "QUOTE", "BOOK"}, + {"BAR", "TRADE", "QUOTE", "MARK_INDEX", "BOOK"}, ) expected_okx_native = sum( 1 @@ -1172,6 +1307,41 @@ def test_missing_binding_wrong_provider_kind_and_invalid_primary_authority_fail_ with self.assertRaisesRegex(ValueError, "not an isolated shared Rust"): self.acquisition.core_config(catalog=self.catalog, authority=primary) + def test_on_change_delivery_cannot_escape_catalog_or_native_bbo_validation(self): + catalog_payload = yaml.safe_load(CATALOG_PATH.read_text(encoding="utf-8")) + trade = next( + item for item in catalog_payload["bindings"] + if item["binding_id"] == "binance-usdm-btcusdt-trade" + ) + trade["quality"]["delivery_semantics"] = "ON_CHANGE" + with self.assertRaisesRegex(ValueError, "reserved for native BBO QUOTE"): + StableSourceCatalog.from_mapping(catalog_payload) + + payload = yaml.safe_load(ACQUISITION_PATH.read_text(encoding="utf-8")) + with tempfile.TemporaryDirectory(prefix="qdl-on-change-acquisition-") as directory: + path = Path(directory) / "candidate.yaml" + wrong_mode = copy.deepcopy(payload) + bbo = next( + item for item in wrong_mode["bindings"] + if item["binding_id"] == "okx-swap-btcusdt-quote" + ) + bbo["mode"] = "PYTHON_REST" + bbo["websocket_url"] = None + bbo["business_websocket_url"] = None + path.write_text(yaml.safe_dump(wrong_mode, sort_keys=False), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "on-change delivery requires"): + StableAcquisitionPlan.load(path, catalog=self.catalog) + + wrong_channel = copy.deepcopy(payload) + bbo = next( + item for item in wrong_channel["bindings"] + if item["binding_id"] == "binance-usdm-btcusdt-quote" + ) + bbo["native_channel"] = "btcusdt@trade" + path.write_text(yaml.safe_dump(wrong_channel, sort_keys=False), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "on-change delivery channel"): + StableAcquisitionPlan.load(path, catalog=self.catalog) + def test_hot_l2_materialization_is_core_only_and_keeps_provider_refresh(self): core = self.acquisition.core_config( catalog=self.catalog, authority=self.authority @@ -1313,10 +1483,29 @@ def test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route(self): self.assertEqual(healthcheck["interval"], "5s") self.assertEqual(healthcheck["timeout"], "3s") self.assertEqual(healthcheck["retries"], 20) - projector_names = ("projector_v2", "projector_v2_2", "projector_v2_3") - for name in projector_names: + projector_names = ( + "projector_v2", "projector_v2_2", "projector_v2_3", + "projector_v2_4", "projector_v2_5", "projector_v2_6", + ) + self.assertEqual( + len(projector_names), compose["x-kafka-env"]["KAFKA_NUM_PARTITIONS"] + ) + for ordinal, name in enumerate(projector_names, start=1): self.assertNotIn("ports", services[name]) self.assertEqual(services[name]["networks"], ["stable_internal"]) + heartbeat = ( + f"/var/lib/qdl-stable/runtime/heartbeat/projector-{ordinal}.json" + ) + self.assertEqual( + services[name]["environment"]["QDL_STABLE_HEARTBEAT_PATH"], + heartbeat, + ) + healthcheck = services[name]["healthcheck"] + self.assertIn(heartbeat, " ".join(healthcheck["test"])) + self.assertEqual(healthcheck["interval"], "20s") + self.assertEqual(healthcheck["timeout"], "5s") + self.assertEqual(healthcheck["retries"], 3) + self.assertEqual(healthcheck["start_period"], "60s") self.assertEqual( { services[name]["environment"]["QDL_STABLE_CONSUMER_GROUP"] @@ -1329,7 +1518,10 @@ def test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route(self): services[name]["environment"]["QDL_STABLE_KAFKA_CLIENT_ID"] for name in projector_names }, - {"stable-projector-1", "stable-projector-2", "stable-projector-3"}, + { + "stable-projector-1", "stable-projector-2", "stable-projector-3", + "stable-projector-4", "stable-projector-5", "stable-projector-6", + }, ) self.assertEqual( { @@ -1345,6 +1537,20 @@ def test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route(self): }, {"33554432"}, ) + self.assertEqual( + { + services[name]["environment"]["QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS"] + for name in projector_names + }, + {"512"}, + ) + self.assertEqual( + { + services[name]["environment"]["QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS"] + for name in projector_names + }, + {"512"}, + ) for name in ("query_v2_1", "query_v2_2", "stream_v2_active", "stream_v2_passive"): self.assertNotIn("QDL_STABLE_MAX_PENDING_RECORDS", services[name]["environment"]) self.assertNotIn("QDL_STABLE_MAX_PENDING_BYTES", services[name]["environment"]) @@ -1524,6 +1730,7 @@ def test_compose_is_isolated_bounded_nonroot_and_has_no_v1_route(self): for name in ( "query_v2_1", "query_v2_2", "stream_v2_active", "stream_v2_passive", "projector_v2", "projector_v2_2", "projector_v2_3", + "projector_v2_4", "projector_v2_5", "projector_v2_6", ): with self.subTest(authority_reader=name): self.assertEqual( diff --git a/tests/test_phaseb_stable_edge.py b/tests/test_phaseb_stable_edge.py index 047d9f3..6711d5a 100644 --- a/tests/test_phaseb_stable_edge.py +++ b/tests/test_phaseb_stable_edge.py @@ -7,6 +7,7 @@ import json import os import tempfile +from dataclasses import replace from types import SimpleNamespace from urllib.parse import urlsplit import unittest @@ -22,8 +23,10 @@ canonicalize_binance_usdm_bbo, canonicalize_binance_usdm_rest_bar, canonicalize_dnse_bar, + canonicalize_okx_bar, canonicalize_okx_bbo, ) +from qdl.adapters.intervals import canonical_interval_ms from qdl.canonical.trade import ( TradeContext, canonicalize_binance_usdm_trade, @@ -40,6 +43,7 @@ from qdl.domain.decimal import CanonicalDecimal from qdl.query import ( AccessPurpose, + BarRevisionPolicy, ConsumerGrade, DataRequirement, InstrumentQuery, @@ -83,7 +87,13 @@ ) from qdl.runtime.session_liveness import StableSessionLivenessReader from qdl.stream import DurableStreamGateway -from qdl.transport import BackpressureRequired, DurableEvent, SQLiteDurableSpool, SpoolConfig +from qdl.transport import ( + BackpressureRequired, + DurableEvent, + FINAL_BAR_CLOSE_TIME_NS_HEADER, + SQLiteDurableSpool, + SpoolConfig, +) from qdl.transport.kafka_projector import KafkaProjectorRecord from qdl.warmup import WarmupSpecification, WarmupTimeRange @@ -98,6 +108,8 @@ def _canonicalizer(name: str): return canonicalize_dnse_trade if name.startswith("dnse") else canonicalize_binance_usdm_trade if "bbo" in name: return canonicalize_okx_bbo if name.startswith("okx") else canonicalize_binance_usdm_bbo + if name.startswith("okx"): + return canonicalize_okx_bar if name.startswith("dnse"): return canonicalize_dnse_bar return canonicalize_binance_usdm_rest_bar @@ -255,28 +267,33 @@ def _broker_records(binding, raw, event, *, raw_offset=0, canonical_offset=0): return raw_topic, canonical_topic, raw_record, canonical_record -def _append(spool, catalog, event): +def _append(spool, catalog, event, *, final_bar_watermark=False): binding = catalog.binding_for_envelope(event) - return _append_unvalidated(spool, binding, event) + return _append_unvalidated( + spool, binding, event, final_bar_watermark=final_bar_watermark + ) -def _append_unvalidated(spool, binding, event): +def _append_unvalidated(spool, binding, event, *, final_bar_watermark=False): """Persist a controlled legacy-row fixture without catalog admission. Production ingestion always resolves the envelope through the catalog. This helper exists solely to model old retained rows whose lineage was valid at the time they were written but has since been retired. """ + headers = { + "raw_stream": "md.raw.v1.phase-b", + "raw_event_id": event.raw_capture_id.hex(), + } + if final_bar_watermark: + headers[FINAL_BAR_CLOSE_TIME_NS_HEADER] = str(event.bar.close_time_ns) durable = DurableEvent( stream=binding.canonical_stream, partition_key=binding.partition_key, event_id=bytes(event.event_id), payload=event.SerializeToString(deterministic=True), accepted_at_ns=event.received_at_ns, - headers={ - "raw_stream": "md.raw.v1.phase-b", - "raw_event_id": event.raw_capture_id.hex(), - }, + headers=headers, ) return spool.append(durable) @@ -292,6 +309,43 @@ def _requirement(binding, *, grade=ConsumerGrade.ALPHA, warmup=1): ) +def _final_bar_at(catalog, binding, fixture_name, *, offset: int, label: str): + """Build one catalog-valid final-BAR fixture at a deterministic market time.""" + + fixture_binding_id = ( + "okx-swap-btcusdt-bar-1m" + if fixture_name.startswith("okx") + else "binance-usdm-btcusdt-bar-1m" + ) + event = market_data_pb2.EventEnvelope() + event.CopyFrom(_stable_event(catalog, fixture_name, fixture_binding_id)) + event.instrument_uid = binding.instrument.instrument_uid + event.instrument_id = binding.instrument.instrument_id + event.instrument_revision = binding.instrument.metadata_revision + event.venue = binding.instrument.identity.venue + event.market = binding.instrument.identity.market + event.product_type = binding.instrument.identity.product_type.value + event.native_symbol = binding.instrument.native_symbol + event.provider = binding.provider + event.source_id = binding.source_id + event.source_role = getattr(common_pb2, f"SOURCE_ROLE_{binding.source_role}") + event.adapter_version = binding.adapter_version + event.normalizer_version = binding.normalizer_version + interval_ns = canonical_interval_ms(binding.interval) * 1_000_000 + event.event_id = hashlib.sha256(label.encode()).digest()[:16] + event.raw_capture_id = hashlib.sha256(f"{label}-raw".encode()).digest()[:16] + event.bar.open_time_ns += offset * interval_ns + event.bar.close_time_ns += offset * interval_ns + event.source_event_time_ns = event.bar.close_time_ns + event.received_at_ns = event.bar.close_time_ns + 1 + event.normalized_at_ns = event.received_at_ns + 1 + event.published_at_ns = event.received_at_ns + 2 + event.source_sequence = label + event.partition_sequence = abs(offset) + 1 + event.correlation_id = label + return event + + class StableCatalogContractTests(unittest.TestCase): def test_catalog_from_mapping_matches_strict_file_loader_without_io(self): payload = yaml.safe_load(CATALOG_PATH.read_text()) @@ -308,7 +362,10 @@ def test_catalog_covers_equal_source_baseline_with_deterministic_identity(self): # The fixed non-crypto capability plane has 10 rows. Each of the five # liquid Binance USD-M and five OKX Swap instruments contributes TRADE, # QUOTE and every provider-native BAR interval. C3.6 adds the declared - # 18 physical L2 books as 36 snapshot/delta logical bindings. + # 18 physical L2 books as 36 snapshot/delta logical bindings. The same + # ten crypto instruments each have one official mark/index binding; + # keep that inventory explicit so a count change cannot hide a missing + # execution reference route or an unrelated catalog expansion. from qdl.adapters.intervals import ( BINANCE_USDM_NATIVE_INTERVALS, OKX_NATIVE_INTERVALS, @@ -322,19 +379,47 @@ def test_catalog_covers_equal_source_baseline_with_deterministic_identity(self): item for item in catalog.bindings if item.feed.value in {"BOOK_SNAPSHOT", "BOOK_DELTA"} ] + mark_index_bindings = [ + item for item in catalog.bindings + if item.feed.value == "MARK_INDEX_PRICE" + ] self.assertEqual(len(l2_bindings), 36) - self.assertEqual(len(catalog.bindings), baseline + len(l2_bindings)) + self.assertEqual(len(mark_index_bindings), 10) + self.assertEqual( + len(catalog.bindings), + baseline + len(l2_bindings) + len(mark_index_bindings), + ) self.assertEqual( {(item.instrument.identity.venue, item.feed.value) for item in catalog.bindings}, { ("BINANCE", "TRADE"), ("BINANCE", "QUOTE"), ("BINANCE", "BAR"), ("BINANCE", "BOOK_SNAPSHOT"), ("BINANCE", "BOOK_DELTA"), + ("BINANCE", "MARK_INDEX_PRICE"), ("OKX", "TRADE"), ("OKX", "QUOTE"), ("OKX", "BAR"), ("OKX", "BOOK_SNAPSHOT"), ("OKX", "BOOK_DELTA"), + ("OKX", "MARK_INDEX_PRICE"), ("HNX", "TRADE"), ("HNX", "BAR"), ("HOSE", "TRADE"), ("HOSE", "BAR"), }, ) + self.assertEqual( + { + (item.instrument.identity.venue, item.instrument.native_symbol) + for item in mark_index_bindings + }, + { + ("BINANCE", "BTCUSDT"), + ("BINANCE", "ETHUSDT"), + ("BINANCE", "SOLUSDT"), + ("BINANCE", "DOGEUSDT"), + ("BINANCE", "BNBUSDT"), + ("OKX", "BTC-USDT-SWAP"), + ("OKX", "ETH-USDT-SWAP"), + ("OKX", "SOL-USDT-SWAP"), + ("OKX", "DOGE-USDT-SWAP"), + ("OKX", "BNB-USDT-SWAP"), + }, + ) for binding in catalog.bindings: recreated = InstrumentIdentity.create( venue=binding.instrument.identity.venue, @@ -492,6 +577,442 @@ def test_materialized_bar_history_data_as_of_is_the_closed_boundary(self): self.assertIsNotNone(history) self.assertEqual(history.data_as_of_ns, event.bar.close_time_ns) + def test_history_many_matches_single_reads_from_one_snapshot(self): + bindings_and_fixtures = ( + ("binance-usdm-btcusdt-bar-1m", "binance_usdm_rest_bar.json"), + ("okx-swap-btcusdt-bar-1m", "okx_bar.json"), + ) + bindings = tuple( + next( + item + for item in self.catalog.bindings + if item.binding_id == binding_id + ) + for binding_id, _fixture in bindings_and_fixtures + ) + events = tuple( + _stable_event(self.catalog, fixture, binding.binding_id) + for binding, (_binding_id, fixture) in zip(bindings, bindings_and_fixtures) + ) + for event in events: + _append(self.spool, self.catalog, event) + now_ns = max(event.bar.close_time_ns for event in events) + 1_000_000 + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="a" * 64, + clock_ns=lambda: now_ns, + ) + requirements = tuple(_requirement(binding) for binding in bindings) + expected = { + requirement: backend.history(requirement) for requirement in requirements + } + snapshots = [] + visit_tails = self.spool.visit_tails + + def tracked_visit_tails(*, requests, visit): + snapshots.append(tuple(requests)) + return visit_tails(requests=requests, visit=visit) + + self.spool.visit_tails = tracked_visit_tails + actual = backend.history_many(requirements) + + self.assertEqual(len(snapshots), 1) + self.assertEqual(len(snapshots[0]), 2) + self.assertEqual(actual, expected) + self.assertEqual( + actual[requirements[0]].items[-1].instrument_uid, + bindings[0].instrument.instrument_uid, + ) + self.assertEqual( + actual[requirements[1]].items[-1].instrument_uid, + bindings[1].instrument.instrument_uid, + ) + + def test_history_many_visits_one_physical_tail_at_a_time(self): + bindings_and_fixtures = ( + ("binance-usdm-btcusdt-bar-1m", "binance_usdm_rest_bar.json"), + ("okx-swap-btcusdt-bar-1m", "okx_bar.json"), + ) + bindings = tuple( + next(item for item in self.catalog.bindings if item.binding_id == binding_id) + for binding_id, _fixture in bindings_and_fixtures + ) + events = tuple( + _stable_event(self.catalog, fixture, binding.binding_id) + for binding, (_binding_id, fixture) in zip(bindings, bindings_and_fixtures) + ) + for event in events: + _append(self.spool, self.catalog, event) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="a" * 64, + clock_ns=lambda: max(event.bar.close_time_ns for event in events) + 1_000_000, + ) + requirements = tuple(_requirement(binding) for binding in bindings) + observed: list[tuple[str, tuple[str, str]]] = [] + visit_tails = self.spool.visit_tails + + def tracked_visit_tails(*, requests, visit): + def tracked_visit(key, rows): + observed.append(("start", key)) + visit(key, rows) + observed.append(("end", key)) + + return visit_tails(requests=requests, visit=tracked_visit) + + self.spool.visit_tails = tracked_visit_tails + result = backend.history_many(requirements) + + self.assertTrue(all(result[item] is not None for item in requirements)) + self.assertEqual( + [marker for marker, _key in observed], + ["start", "end", "start", "end"], + ) + self.assertNotEqual(observed[0][1], observed[2][1]) + + def test_history_many_preserves_late_backfill_gap_and_missing_item_results(self): + btc = next( + item + for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + eth = next( + item + for item in self.catalog.bindings + if item.binding_id == "binance-usdm-ethusdt-bar-1m" + ) + first = _stable_event(self.catalog, "binance_usdm_rest_bar.json", btc.binding_id) + late = type(first)() + late.CopyFrom(first) + late.event_id = hashlib.sha256(b"batch-history-gap-late").digest()[:16] + late.raw_capture_id = hashlib.sha256(b"batch-history-gap-late-raw").digest()[:16] + # Offset order is intentionally the inverse of market order and leaves + # exactly one governed minute missing. The batch path must preserve the + # single-read PARTIAL outcome instead of hiding it behind its snapshot. + late.bar.open_time_ns += 120 * 1_000_000_000 + late.bar.close_time_ns += 120 * 1_000_000_000 + late.source_event_time_ns += 120 * 1_000_000_000 + late.received_at_ns += 120 * 1_000_000_000 + late.normalized_at_ns += 120 * 1_000_000_000 + late.published_at_ns += 120 * 1_000_000_000 + late.source_sequence = "batch-history-gap-late" + _append(self.spool, self.catalog, late) + _append(self.spool, self.catalog, first) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="b" * 64, + clock_ns=lambda: late.bar.close_time_ns + 1_000_000, + ) + btc_requirement = _requirement(btc, warmup=2) + eth_requirement = _requirement(eth, warmup=2) + expected = { + btc_requirement: backend.history(btc_requirement), + eth_requirement: backend.history(eth_requirement), + } + + actual = backend.history_many((btc_requirement, eth_requirement)) + + self.assertEqual(actual, expected) + self.assertEqual(actual[btc_requirement].coverage.value, "PARTIAL") + self.assertIsNone(actual[eth_requirement]) + + def test_history_many_exact_final_window_matches_full_tail_without_fallback(self): + bindings_and_fixtures = ( + ("binance-usdm-btcusdt-bar-1m", "binance_usdm_rest_bar.json"), + ("okx-swap-btcusdt-bar-1m", "okx_bar.json"), + ) + bindings = tuple( + next(item for item in self.catalog.bindings if item.binding_id == binding_id) + for binding_id, _fixture in bindings_and_fixtures + ) + newest = [] + for binding, (_binding_id, fixture_name) in zip(bindings, bindings_and_fixtures): + older = _final_bar_at( + self.catalog, binding, fixture_name, + offset=-1, label=f"exact-final-{binding.binding_id}-older", + ) + current = _final_bar_at( + self.catalog, binding, fixture_name, + offset=0, label=f"exact-final-{binding.binding_id}-current", + ) + _append(self.spool, self.catalog, current, final_bar_watermark=True) + # A provider repair may arrive after the live final BAR. The exact + # lookup must still return the market-time tail, not append order. + _append(self.spool, self.catalog, older, final_bar_watermark=True) + newest.append(current) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="a" * 64, + clock_ns=lambda: max(item.bar.close_time_ns for item in newest) + 1_000_000, + ) + requirements = tuple(_requirement(binding, warmup=2) for binding in bindings) + expected = {requirement: backend.history(requirement) for requirement in requirements} + fallback_reads = [] + read_tail_rows_locked = self.spool._read_tail_rows_locked + + def tracked_fallback(**kwargs): + fallback_reads.append((kwargs["stream"], kwargs["partition_key"])) + return read_tail_rows_locked(**kwargs) + + self.spool._read_tail_rows_locked = tracked_fallback + actual = backend.history_many(requirements) + + self.assertEqual(actual, expected) + self.assertEqual(fallback_reads, []) + + def test_history_many_exact_final_window_supports_emit_revisions_when_unique(self): + binding = next( + item for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + newest = None + for offset in (-1, 0): + event = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=offset, label=f"emit-revisions-unique-{offset}", + ) + _append(self.spool, self.catalog, event, final_bar_watermark=True) + newest = event + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="a" * 64, + clock_ns=lambda: newest.bar.close_time_ns + 1_000_000, + ) + requirement = replace( + _requirement(binding, warmup=2), + bar_revision_policy=BarRevisionPolicy.EMIT_REVISIONS, + ) + expected = backend.history(requirement) + fallback_reads = [] + read_tail_rows_locked = self.spool._read_tail_rows_locked + + def tracked_fallback(**kwargs): + fallback_reads.append((kwargs["stream"], kwargs["partition_key"])) + return read_tail_rows_locked(**kwargs) + + self.spool._read_tail_rows_locked = tracked_fallback + actual = backend.history_many((requirement,))[requirement] + + self.assertEqual(actual, expected) + self.assertEqual(fallback_reads, []) + + def test_history_many_emit_revisions_duplicate_uses_retained_tail(self): + binding = next( + item for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + values = [] + for offset in (-1, 0): + event = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=offset, label=f"emit-revisions-duplicate-{offset}", + ) + _append(self.spool, self.catalog, event, final_bar_watermark=True) + values.append(event) + revised = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=0, label="emit-revisions-duplicate-revised", + ) + revised.bar.revision = 1 + revised.bar.lifecycle = market_data_pb2.BAR_LIFECYCLE_REVISED + revised.bar.supersedes_event_id = values[-1].event_id + _append(self.spool, self.catalog, revised, final_bar_watermark=True) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="b" * 64, + clock_ns=lambda: revised.bar.close_time_ns + 1_000_000, + ) + requirement = replace( + _requirement(binding, warmup=2), + bar_revision_policy=BarRevisionPolicy.EMIT_REVISIONS, + ) + expected = backend.history(requirement) + fallback_reads = [] + read_tail_rows_locked = self.spool._read_tail_rows_locked + + def tracked_fallback(**kwargs): + fallback_reads.append((kwargs["stream"], kwargs["partition_key"])) + return read_tail_rows_locked(**kwargs) + + self.spool._read_tail_rows_locked = tracked_fallback + actual = backend.history_many((requirement,))[requirement] + + self.assertEqual(actual, expected) + self.assertEqual(len(fallback_reads), 1) + + def test_history_many_final_window_falls_back_for_missing_gap_or_revision(self): + cases = ( + ("binance-usdm-btcusdt-bar-1m", "missing", (0,)), + ("binance-usdm-ethusdt-bar-1m", "gap", (0, 2)), + ("binance-usdm-solusdt-bar-1m", "duplicate", (0, 1)), + ) + requirements = [] + newest = [] + + for binding_id, shape, offsets in cases: + binding = next(item for item in self.catalog.bindings if item.binding_id == binding_id) + values = [] + for offset in offsets: + event = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=offset, label=f"final-fallback-{shape}-{offset}", + ) + _append(self.spool, self.catalog, event, final_bar_watermark=True) + values.append(event) + if shape == "duplicate": + revised = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=offsets[-1], label="final-fallback-duplicate-revised", + ) + revised.bar.revision = 1 + revised.bar.lifecycle = market_data_pb2.BAR_LIFECYCLE_REVISED + revised.bar.supersedes_event_id = values[-1].event_id + _append(self.spool, self.catalog, revised, final_bar_watermark=True) + values.append(revised) + newest.extend(values) + requirements.append(_requirement(binding, warmup=2)) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="b" * 64, + clock_ns=lambda: max(item.bar.close_time_ns for item in newest) + 1_000_000, + ) + expected = {requirement: backend.history(requirement) for requirement in requirements} + fallback_reads = [] + read_tail_rows_locked = self.spool._read_tail_rows_locked + + def tracked_fallback(**kwargs): + fallback_reads.append((kwargs["stream"], kwargs["partition_key"])) + return read_tail_rows_locked(**kwargs) + + self.spool._read_tail_rows_locked = tracked_fallback + actual = backend.history_many(tuple(requirements)) + + self.assertEqual(actual, expected) + self.assertEqual(len(fallback_reads), 3) + self.assertEqual(actual[requirements[1]].coverage.value, "PARTIAL") + + def test_history_many_hybrid_final_window_uses_one_sqlite_snapshot(self): + binance = next( + item for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + okx = next( + item for item in self.catalog.bindings + if item.binding_id == "okx-swap-btcusdt-bar-1m" + ) + newest = [] + for binding, fixture_name, watermarked in ( + (binance, "binance_usdm_rest_bar.json", True), + (okx, "okx_bar.json", False), + ): + for offset in (-1, 0): + event = _final_bar_at( + self.catalog, binding, fixture_name, + offset=offset, label=f"hybrid-{binding.binding_id}-{offset}", + ) + _append(self.spool, self.catalog, event, final_bar_watermark=watermarked) + newest.append(event) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="c" * 64, + clock_ns=lambda: max(item.bar.close_time_ns for item in newest) + 1_000_000, + ) + requirements = (_requirement(binance, warmup=2), _requirement(okx, warmup=2)) + expected = {requirement: backend.history(requirement) for requirement in requirements} + statements = [] + fallback_reads = [] + read_tail_rows_locked = self.spool._read_tail_rows_locked + + def tracked_fallback(**kwargs): + fallback_reads.append((kwargs["stream"], kwargs["partition_key"])) + return read_tail_rows_locked(**kwargs) + + self.spool._read_tail_rows_locked = tracked_fallback + self.spool._connection.set_trace_callback(statements.append) + try: + actual = backend.history_many(requirements) + finally: + self.spool._connection.set_trace_callback(None) + + self.assertEqual(actual, expected) + self.assertEqual(len(fallback_reads), 1) + self.assertEqual(sum(statement == "BEGIN" for statement in statements), 1) + self.assertEqual(sum(statement == "COMMIT" for statement in statements), 1) + + def test_history_many_invalid_final_watermark_keeps_retained_tail_authority(self): + binding = next( + item for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + newest = None + for offset in (-1, 0): + event = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=offset, label=f"invalid-watermark-{offset}", + ) + _append(self.spool, self.catalog, event, final_bar_watermark=True) + newest = event + self.spool._connection.execute( + """ + UPDATE final_bar_watermarks SET close_time_ns = 0 + WHERE stream = ? AND partition_key = ? + """, + (binding.canonical_stream, binding.partition_key), + ) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="d" * 64, + clock_ns=lambda: newest.bar.close_time_ns + 1_000_000, + ) + requirement = _requirement(binding, warmup=2) + + def unexpected_final_window(**_kwargs): + raise AssertionError("invalid watermark must use retained-tail materialization") + + self.spool.visit_final_bar_windows = unexpected_final_window + self.assertEqual( + backend.history_many((requirement,))[requirement], + backend.history(requirement), + ) + + def test_history_many_large_row_warmups_keep_retained_tail_path(self): + binding = next( + item for item in self.catalog.bindings + if item.binding_id == "binance-usdm-btcusdt-bar-1m" + ) + for offset in (-1, 0): + event = _final_bar_at( + self.catalog, binding, "binance_usdm_rest_bar.json", + offset=offset, label=f"large-row-{offset}", + ) + _append(self.spool, self.catalog, event, final_bar_watermark=True) + backend = StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="d" * 64, + clock_ns=lambda: _stable_event( + self.catalog, "binance_usdm_rest_bar.json", binding.binding_id + ).bar.close_time_ns + 1_000_000, + ) + + def unexpected_final_window(**_kwargs): + raise AssertionError("large row warmup must use retained-tail materialization") + + self.spool.visit_final_bar_windows = unexpected_final_window + requirements = tuple(_requirement(binding, warmup=rows) for rows in (2500, 5000, 10000)) + result = backend.history_many(requirements) + + self.assertTrue(all(result[requirement] is not None for requirement in requirements)) + def test_late_bar_backfill_keeps_market_order_and_fences_max_offset(self): binding = next( item @@ -1174,6 +1695,162 @@ def test_block_policy_remains_strict_with_a_live_session(self): self.assertEqual(raised.exception.problem.code.value, "DATA_STALE") +class StableOnChangeQuoteQualityTests(unittest.TestCase): + """Native BBO can be quiet, but only behind the signed source contract.""" + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.catalog = StableSourceCatalog.load(CATALOG_PATH) + self.spool = SQLiteDurableSpool(SpoolConfig( + path=Path(self.temp.name) / "stable.sqlite3", + max_records=100, + max_payload_bytes=2 * 1024 * 1024, + max_storage_bytes=8 * 1024 * 1024, + min_free_disk_bytes=0, + )) + self.binding = next( + item + for item in self.catalog.bindings + if item.binding_id == "okx-swap-btcusdt-quote" + ) + self.event = _stable_event( + self.catalog, "okx_bbo.json", self.binding.binding_id + ) + self.event.config_revision = 7 + _append(self.spool, self.catalog, self.event) + self.now_ns = self.event.source_event_time_ns + 5_000_000_000 + self.root = Path(self.temp.name) / "session-liveness" + + def tearDown(self): + self.spool.close() + self.temp.cleanup() + + def _requirement(self, *, policy=StalePolicy.OBSERVE): + return DataRequirement( + instrument_uid=self.binding.instrument.instrument_uid, + feed=self.binding.feed, + interval=self.binding.interval, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id=self.binding.source_policy_id, + max_freshness_ms=2_000, + event_recency_policy=policy, + max_session_liveness_ms=2_000, + stale_policy=StalePolicy.BLOCK, + ) + + def _write_session( + self, + *, + state="LIVE", + generation=None, + revision=7, + age_ms=1, + ): + directory = self.root / "okx-swap" + directory.mkdir(parents=True, exist_ok=True) + transport_at_ns = self.now_ns - age_ms * 1_000_000 + (directory / "bbo-lane.json").write_text( + json.dumps({ + "schema": "qdl.provider-session-liveness.v1", + "source_session_id": self.event.source_session_id, + "connection_generation": ( + self.event.connection_generation + if generation is None + else generation + ), + "state": state, + "last_transport_at_ns": transport_at_ns, + "updated_at_ns": transport_at_ns, + "config_revision": revision, + }), + encoding="utf-8", + ) + + def _backend(self): + return StableSpoolQueryBackend( + self.spool, + self.catalog, + schema_digest="f" * 64, + config_revision=7, + session_liveness_root=str(self.root), + clock_ns=lambda: self.now_ns, + ) + + def _service(self, backend): + return V2QueryService( + instruments=InstrumentQuery(self.catalog.instrument_registry()), + backend=backend, + entitlements=self.catalog.entitlements(), + clock_ns=lambda: self.now_ns, + ) + + def test_on_change_quote_is_usable_only_with_live_signed_session(self): + self._write_session(age_ms=1_999) + requirement = self._requirement() + item = self._backend().latest(requirement) + self.assertIsNotNone(item) + assert item is not None + self.assertEqual(item.quality.state, "LIVE") + self.assertEqual(item.quality.event_recency_state, "STALE") + self.assertGreater(item.quality.freshness_ms, requirement.max_freshness_ms) + self.assertEqual(item.quality.provider_session_liveness_ms, 1_999) + self.assertIn("LAST_EVENT_STALE", item.quality.flags) + self.assertIn("DELIVERY_ON_CHANGE", item.quality.flags) + self.assertTrue(item.quality.execution_eligible) + result = self._service(self._backend()).snapshot( + requirement, purpose=AccessPurpose.INTERNAL_EXECUTION + ) + self.assertTrue(result.item.quality.execution_eligible) + + def test_on_change_quote_keeps_strict_policy_and_all_fences_fail_closed(self): + self._write_session() + strict = self._requirement(policy=StalePolicy.BLOCK) + item = self._backend().latest(strict) + self.assertIsNotNone(item) + assert item is not None + self.assertEqual(item.quality.state, "STALE") + self.assertFalse(item.quality.execution_eligible) + with self.assertRaises(QueryServiceError) as raised: + self._service(self._backend()).snapshot( + strict, purpose=AccessPurpose.INTERNAL_EXECUTION + ) + self.assertEqual(raised.exception.problem.code.value, "DATA_STALE") + + observed = self._requirement() + for state, generation, revision, age_ms, expected_session in ( + ("DISCONNECTED", None, 7, 1, "DISCONNECTED"), + ("LIVE", None, 7, 2_001, "STALE"), + ("LIVE", None, 6, 1, "UNKNOWN"), + ("LIVE", 2, 7, 1, "UNKNOWN"), + ): + with self.subTest( + state=state, + generation=generation, + revision=revision, + age_ms=age_ms, + ): + self._write_session( + state=state, + generation=generation, + revision=revision, + age_ms=age_ms, + ) + backend = self._backend() + candidate = backend.latest(observed) + self.assertIsNotNone(candidate) + assert candidate is not None + self.assertEqual(candidate.quality.state, "STALE") + self.assertEqual( + candidate.quality.provider_session_state, expected_session + ) + self.assertFalse(candidate.quality.execution_eligible) + with self.assertRaises(QueryServiceError) as raised: + self._service(backend).snapshot( + observed, purpose=AccessPurpose.INTERNAL_EXECUTION + ) + self.assertEqual(raised.exception.problem.code.value, "DATA_STALE") + + class StableCursorScopeValidatorTests(unittest.TestCase): def setUp(self): self.catalog = StableSourceCatalog.load(CATALOG_PATH) @@ -1316,13 +1993,16 @@ async def asyncTearDown(self): self.spool.close() self.temp.cleanup() - def engine(self, broker, target, raw_topic, canonical_topic, *, sink=None): + def engine( + self, broker, target, raw_topic, canonical_topic, *, sink=None, **overrides + ): return StableProjectorEngine( broker=broker, spool=self.spool, catalog=self.catalog, canonical_topic=canonical_topic, raw_topics=(raw_topic,), sink=sink or LocalStableCanonicalSink(self.gateway, self.spool), projector=StableCompatibilityProjector(self.catalog), target=target, max_pending_records=10, max_pending_bytes=1024 * 1024, + **overrides, ) def _native_backfill_overlap(self): @@ -1443,11 +2123,54 @@ def record(partition: int, offset: int) -> KafkaProjectorRecord: [40, 41], ) + def test_projector_six_partition_turn_preserves_fifo_and_no_partition_starves(self): + def record(partition: int, offset: int) -> KafkaProjectorRecord: + return KafkaProjectorRecord( + topic="qdl.stable.canonical.phase-b.v2", + partition=partition, + offset=offset, + key=f"fixture/{partition}", + event_id=(partition.to_bytes(1) + offset.to_bytes(15, "big")), + payload=b"fixture", + accepted_at_ns=1, + ) + + queues = { + ("qdl.stable.canonical.phase-b.v2", partition): deque( + record(partition, offset) for offset in range(3) + ) + for partition in range(6) + } + + selected = StableProjectorEngine._round_robin_candidates(queues, 12) + + self.assertEqual( + [(record.partition, record.offset) for _partition, record in selected], + [(partition, offset) for offset in range(2) for partition in range(6)], + ) + for partition in range(6): + with self.subTest(partition=partition): + self.assertEqual( + [ + record.offset + for queue, record in selected + if queue[1] == partition + ], + [0, 1], + ) + self.assertEqual( + [record.offset for record in queues[( + "qdl.stable.canonical.phase-b.v2", partition + )]], + [0, 1, 2], + ) + async def test_supervisor_recreates_poisoned_generation_with_bounded_backoff(self): stopped = [False] sleeps = [] active = [] brokers = [] + events = [] generations = ["fail", "recover"] class Broker: @@ -1464,6 +2187,9 @@ class Engine: def __init__(self, outcome): self.outcome = outcome + async def prepare_for_polling(self): + events.append(f"prepare:{self.outcome}") + async def run_once(self, timeout_seconds): self.assert_timeout = timeout_seconds if self.outcome == "fail": @@ -1480,16 +2206,27 @@ def factory(): async def sleep(delay): sleeps.append(delay) + def on_broker(value): + events.append("broker:none" if value is None else "broker:ready") + active.append(value) + await supervise_stable_projector( broker_factory=factory, should_stop=lambda: stopped[0], - on_broker=active.append, + on_broker=on_broker, sleep=sleep, ) self.assertEqual(len(brokers), 2) self.assertEqual([broker.closed for broker in brokers], [1, 1]) self.assertEqual(sleeps, [0.25]) self.assertEqual(active, [brokers[0], None, brokers[1], None]) + self.assertEqual( + events, + [ + "prepare:fail", "broker:ready", "broker:none", + "prepare:recover", "broker:ready", "broker:none", + ], + ) async def test_run_once_defers_a_polled_record_at_the_batch_byte_bound(self): class Record: @@ -1744,6 +2481,59 @@ def tracked_apply_many(records): self.assertEqual(engine.stats.canonical_committed, 2) self.assertEqual(engine.stats.pending_canonical, 0) + async def test_projector_limits_each_durable_commit_turn_without_reordering(self): + first_binding, first_raw, first_event = _stable_pair( + self.catalog, "binance_usdm_trade.json", "binance-usdm-btcusdt-trade" + ) + second_binding, second_raw, second_event = _stable_pair( + self.catalog, "okx_bbo.json", "okx-swap-btcusdt-quote" + ) + raw_topic, canonical_topic, raw_first, canonical_first = _broker_records( + first_binding, first_raw, first_event, raw_offset=0, canonical_offset=0 + ) + _, _, raw_second, canonical_second = _broker_records( + second_binding, second_raw, second_event, raw_offset=1, canonical_offset=1 + ) + broker = _Broker() + target = InMemoryStableProjectionTarget() + sink = _CountingStableSink(LocalStableCanonicalSink(self.gateway, self.spool)) + projection_calls = [] + original_apply_many = target.apply_many + + def tracked_apply_many(records): + projection_calls.append(tuple(record.offset for record in records)) + return original_apply_many(records) + + target.apply_many = tracked_apply_many + engine = self.engine( + broker, + target, + raw_topic, + canonical_topic, + sink=sink, + max_batch_records=2, + max_commit_records=1, + ) + + await engine.accept_many( + (raw_first, raw_second, canonical_first, canonical_second) + ) + + self.assertEqual(sink.publish_many_calls, 2) + self.assertEqual(projection_calls, [(1,), (1,)]) + self.assertEqual(broker.checkpoint_batches, [2, 1, 1]) + self.assertEqual( + broker.checkpoints, + [ + (raw_topic, 0, 0), + (raw_topic, 0, 1), + (canonical_topic, 0, 0), + (canonical_topic, 0, 1), + ], + ) + self.assertEqual(engine.stats.canonical_committed, 2) + self.assertEqual(engine.stats.pending_canonical, 0) + async def test_batch_projection_failure_replays_without_premature_checkpoint(self): binding, raw, event = _stable_pair( self.catalog, "binance_usdm_trade.json", "binance-usdm-btcusdt-trade" @@ -2212,6 +3002,200 @@ def pair(open_time_ms): ) self.assertEqual(broker.checkpoints[-1], (canonical_topic, 0, 1)) + async def test_final_bar_watermark_hydrates_once_and_fences_late_restart(self): + binding, raw, event = _stable_pair( + self.catalog, + "binance_usdm_rest_bar.json", + "binance-usdm-btcusdt-bar-1m", + ) + raw_topic, canonical_topic, raw_record, _canonical_record = _broker_records( + binding, raw, event + ) + + def record(envelope, offset, *, kafka_partition=0, key=binding.partition_key, + raw_payload=raw_record.payload): + return KafkaProjectorRecord( + topic=canonical_topic, + partition=kafka_partition, + offset=offset, + key=key, + event_id=bytes(envelope.event_id), + payload=envelope.SerializeToString(deterministic=True), + accepted_at_ns=envelope.received_at_ns, + raw_provider_envelope=raw_payload, + ) + + def changed_bar(label, *, base=event, close_delta_ns=0, revised=False): + value = type(event)() + value.CopyFrom(base) + value.event_id = hashlib.sha256(label.encode()).digest()[:16] + value.source_sequence = f"{base.source_sequence}:{label}" + if close_delta_ns: + value.bar.open_time_ns += close_delta_ns + value.bar.close_time_ns += close_delta_ns + value.source_event_time_ns += close_delta_ns + value.received_at_ns += close_delta_ns + value.normalized_at_ns += close_delta_ns + value.published_at_ns += close_delta_ns + if revised: + value.bar.lifecycle = market_data_pb2.BAR_LIFECYCLE_REVISED + value.bar.revision = 1 + value.canonical_payload_hash = hashlib.sha256( + value.bar.SerializeToString(deterministic=True) + ).digest() + return value + + tail_limits = [] + read_tail = self.spool.read_tail + + def tracked_read_tail(**kwargs): + if kwargs["partition_key"] == binding.partition_key: + tail_limits.append(kwargs["limit"]) + return read_tail(**kwargs) + + # This row models an existing cache written before the additive + # watermark table. It remains valid history but has no table row. + _append(self.spool, self.catalog, event) + self.assertIsNone( + self.spool.final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=binding.partition_key, + ) + ) + self.spool.read_tail = tracked_read_tail + target = InMemoryStableProjectionTarget() + engine = self.engine(_Broker(), target, raw_topic, canonical_topic) + await engine.prepare_for_polling() + self.assertEqual(tail_limits, [10_000]) + self.assertEqual( + self.spool.final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=binding.partition_key, + ), + event.bar.close_time_ns, + ) + + # The first live batch is now O(1): the cache migration finished before + # polling, so a final BAR cannot make the strict quote path scan history. + tail_limits.clear() + newer = changed_bar("newer-after-legacy", close_delta_ns=60_000_000_000) + await engine.accept(record(newer, 0)) + self.assertEqual(tail_limits, []) + self.assertEqual( + self.spool.final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=binding.partition_key, + ), + newer.bar.close_time_ns, + ) + + revision = changed_bar("same-close-revision", base=newer, revised=True) + older_in_same_batch = changed_bar( + "older-in-same-batch", base=event, close_delta_ns=0 + ) + quote_binding, quote_raw, quote_event = _stable_pair( + self.catalog, "binance_usdm_bbo.json", "binance-usdm-btcusdt-quote" + ) + _quote_raw_topic, _quote_canonical_topic, quote_raw_record, _quote_record = ( + _broker_records(quote_binding, quote_raw, quote_event) + ) + await engine.accept_many(( + record(revision, 1), + record(older_in_same_batch, 2), + record( + quote_event, + 0, + kafka_partition=1, + key=quote_binding.partition_key, + raw_payload=quote_raw_record.payload, + ), + )) + self.assertEqual(tail_limits, []) + canonical_value = next( + payload + for key, payload in target.latest.items() + if key.startswith("qdl:stable:v2:latest:bar:") + ) + self.assertEqual( + market_data_pb2.EventEnvelope.FromString(canonical_value).bar.revision, + 1, + ) + self.assertTrue(any( + key.startswith("qdl:stable:v2:latest:quote:") + for key in target.latest + )) + + restarted_target = InMemoryStableProjectionTarget() + restarted = self.engine( + _Broker(), restarted_target, raw_topic, canonical_topic + ) + late = changed_bar("late-after-restart") + await restarted.accept(record(late, 2)) + self.assertEqual(tail_limits, []) + self.assertEqual(restarted_target.latest, {}) + self.assertEqual( + self.spool.final_bar_watermark( + stream=self.catalog.canonical_stream, + partition_key=binding.partition_key, + ), + newer.bar.close_time_ns, + ) + + async def test_http_ingest_derives_final_bar_watermark_from_validated_envelope(self): + binding, raw, event = _stable_pair( + self.catalog, + "binance_usdm_rest_bar.json", + "binance-usdm-btcusdt-bar-1m", + ) + raw_topic, _canonical_topic, raw_record, canonical_record = _broker_records( + binding, raw, event + ) + app = FastAPI() + secret = b"phase-b-stable-final-bar-watermark-32" + install_stable_canonical_ingest( + app, gateway=self.gateway, catalog=self.catalog, + spool=self.spool, secret=secret, + ) + client = httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) + sink = StableHttpCanonicalSink( + ("http://localhost",), secret, self.spool, client=client + ) + durable = DurableEvent( + stream=self.catalog.canonical_stream, + partition_key=binding.partition_key, + event_id=canonical_record.event_id, + payload=canonical_record.payload, + accepted_at_ns=canonical_record.accepted_at_ns, + headers={ + "raw_stream": raw_topic, + "raw_event_id": raw_record.event_id.hex(), + "raw_provider_envelope": base64.b64encode( + raw_record.payload + ).decode("ascii"), + }, + ) + try: + stored = await sink.publish(durable) + self.assertEqual( + self.spool.final_bar_watermark( + stream=stored.event.stream, + partition_key=stored.event.partition_key, + ), + event.bar.close_time_ns, + ) + self.assertNotIn( + "qdl.final_bar_close_time_ns", durable.headers + ) + self.assertEqual( + stored.event.headers["qdl.final_bar_close_time_ns"], + str(event.bar.close_time_ns), + ) + finally: + await sink.close() + await client.aclose() + async def test_same_event_id_with_changed_market_semantics_fails_closed(self): binding, raw, event = _stable_pair( self.catalog, "binance_usdm_trade.json", "binance-usdm-btcusdt-trade" @@ -2710,20 +3694,26 @@ def test_query_role_is_isolated_and_projector_dependencies_fail_closed(self): values.update({ "QDL_STABLE_MAX_PENDING_RECORDS": "2048", "QDL_STABLE_MAX_PENDING_BYTES": "33554432", - "QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS": "1000", + "QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS": "512", "QDL_STABLE_PROJECTOR_MAX_BATCH_BYTES": "8388608", + "QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS": "512", }) bounded_projector = StableRuntimeConfig.from_environment( "projector_v2", values ) self.assertEqual(bounded_projector.max_pending_records, 2048) self.assertEqual(bounded_projector.max_pending_bytes, 33_554_432) - self.assertEqual(bounded_projector.projector_max_batch_records, 1000) + self.assertEqual(bounded_projector.projector_max_batch_records, 512) self.assertEqual(bounded_projector.projector_max_batch_bytes, 8_388_608) + self.assertEqual(bounded_projector.projector_max_commit_records, 512) values["QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS"] = "1001" with self.assertRaisesRegex(ValueError, "projector batch bound"): StableRuntimeConfig.from_environment("projector_v2", values) - values["QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS"] = "1000" + values["QDL_STABLE_PROJECTOR_MAX_BATCH_RECORDS"] = "512" + values["QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS"] = "513" + with self.assertRaisesRegex(ValueError, "commit batch bound"): + StableRuntimeConfig.from_environment("projector_v2", values) + values["QDL_STABLE_PROJECTOR_MAX_COMMIT_RECORDS"] = "512" values["QDL_STABLE_MAX_PENDING_RECORDS"] = "64" with self.assertRaisesRegex(ValueError, "pending records"): StableRuntimeConfig.from_environment("projector_v2", values) diff --git a/tests/test_phaseb_stable_release.py b/tests/test_phaseb_stable_release.py index ce5bb2c..55f4bde 100644 --- a/tests/test_phaseb_stable_release.py +++ b/tests/test_phaseb_stable_release.py @@ -321,10 +321,16 @@ def test_service_openapi_and_sdk_versions_are_explicit(self): self.assertEqual(generated["info"]["version"], "2.0.0") self.assertEqual(snapshot, generated) # ``reference:batch`` is a governed V2 public path in the checked-in - # router and snapshot; keep this count as a regression guard rather - # than silently accepting a stale release assertion. + # router and snapshot. The two namespaced stale-policy schemas are + # intentional: Query accepts UNSPECIFIED internally while the public + # SDK only exposes concrete policy values. Keep both the count and the + # identity fence so a future generator collision cannot look harmless. self.assertEqual(len(generated["paths"]), 11) - self.assertEqual(len(generated["components"]["schemas"]), 67) + schemas = generated["components"]["schemas"] + self.assertEqual(len(schemas), 68) + self.assertNotIn("StalePolicy", schemas) + self.assertIn("qdl__query__contracts__StalePolicy", schemas) + self.assertIn("qdl_sdk__models__StalePolicy", schemas) if __name__ == "__main__": diff --git a/tests/test_production_catalog.py b/tests/test_production_catalog.py index 5bde25c..f8e205c 100644 --- a/tests/test_production_catalog.py +++ b/tests/test_production_catalog.py @@ -320,6 +320,77 @@ def test_book_demand_requires_explicit_bounded_acquisition_fields(self): self.assertEqual(len(demand), 1) self.assertEqual(demand[0].depth_per_side, 100) + def test_mark_index_cadence_is_generated_complete_for_binance_and_okx(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload = { + "schema": "qdl.v2.production-demand.v1", + "revision": 1, + "consumers": [{ + "consumer_id": "execution", "consumer_grade": "EXECUTION", + "requirements": [ + { + "venue": "BINANCE", "market": "USDM", + "product_type": "PERPETUAL", "native_symbol": "BTCUSDT", + "feed": "MARK_INDEX_PRICE", "interval": None, + "source_policy_id": "crypto_liquid_v2", + "max_freshness_ms": 2_000, "require_live": True, + "index_native_symbol": None, + }, + { + "venue": "OKX", "market": "SWAP", + "product_type": "PERPETUAL", "native_symbol": "BTC-USDT-SWAP", + "feed": "MARK_INDEX_PRICE", "interval": None, + "source_policy_id": "crypto_liquid_v2", + "max_freshness_ms": 2_000, "require_live": True, + "index_native_symbol": "BTC-USDT", + }, + ], + }], + } + import yaml + path = root / "mark-index.yaml" + path.write_text(yaml.safe_dump(payload), encoding="utf-8") + demand = ProductionDemandManifest.load_many([path]) + bundle = ProductionCatalogBuilder( + catalog_revision=1, + source_policy_revision=1, + authority_revision=1, + ).build( + demand=demand, + binance_usdm=parse_exchange_info(BINANCE, valid_from_ns=1), + okx_rows=OKX, + ) + acquisition = { + item["binding_id"]: item + for item in bundle.acquisition_plan["bindings"] + } + self.assertEqual( + acquisition["binance-usdm-btcusdt-mark_index_price"]["mark_index"] + ["component_quiet_after_ms"], + {"BOTH": 5_000}, + ) + self.assertEqual( + acquisition["okx-swap-btc-usdt-swap-mark_index_price"]["mark_index"] + ["component_quiet_after_ms"], + {"MARK": 15_000, "INDEX": 70_000}, + ) + paths = bundle.write(root / "out") + catalog = StableSourceCatalog.load(paths["source_catalog"]) + plan = StableAcquisitionPlan.load(paths["acquisition_plan"], catalog=catalog) + physical = plan._physical_entries( + source_by_id={item.binding_id: item for item in catalog.bindings}, + selected_ids=frozenset(item.binding_id for item in catalog.bindings), + ) + by_component = { + (item.source.instrument.identity.venue, item.mark_index_component): + item.mark_index_quiet_after_ms + for item in physical + } + self.assertEqual(by_component[("BINANCE", "BOTH")], 5_000) + self.assertEqual(by_component[("OKX", "MARK")], 15_000) + self.assertEqual(by_component[("OKX", "INDEX")], 70_000) + def test_catalog_accepts_all_certified_fixed_bar_intervals_and_scales_staleness(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_r135_measure_binding_quality.py b/tests/test_r135_measure_binding_quality.py new file mode 100644 index 0000000..01fda97 --- /dev/null +++ b/tests/test_r135_measure_binding_quality.py @@ -0,0 +1,51 @@ +"""R1.35-B: the read-only quality probe honors declared feed semantics.""" + +from __future__ import annotations + +import importlib.util +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + + +def _probe_module(): + script_dir = Path(__file__).resolve().parents[1] / "scripts" + spec = importlib.util.spec_from_file_location( + "r135_measure_binding_quality", script_dir / "measure_binding_quality.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.path.insert(0, str(script_dir)) + try: + spec.loader.exec_module(module) + finally: + sys.path.remove(str(script_dir)) + return module + + +class MeasureBindingQualitySemanticsTests(unittest.TestCase): + def test_strict_and_quiet_requirements_use_their_declared_policy(self): + probe = _probe_module() + quality = SimpleNamespace( + state="LIVE", + complete=True, + execution_eligible=True, + gap_open=False, + freshness_ms=2_001, + ) + strict = SimpleNamespace( + effective_event_recency_policy=SimpleNamespace(value="BLOCK"), + max_freshness_ms=2_000, + ) + quiet = SimpleNamespace( + effective_event_recency_policy=SimpleNamespace(value="OBSERVE"), + max_freshness_ms=2_000, + ) + + self.assertFalse(probe._quality_is_usable(strict, quality)) + self.assertTrue(probe._quality_is_usable(quiet, quality)) + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/tests/test_r135_quality_convergence.py b/tests/test_r135_quality_convergence.py new file mode 100644 index 0000000..ee399b1 --- /dev/null +++ b/tests/test_r135_quality_convergence.py @@ -0,0 +1,187 @@ +"""R1.35-A: one quality policy across core, query, SDK and audit inputs. + +The fixture is deterministic test provenance. It deliberately carries no +provider payload, price, credential or runtime state; the real-provider matrix +belongs to R1.35-B/C after this source-only contract is sealed. +""" + +from __future__ import annotations + +import json +import unittest +from pathlib import Path + +from qdl.data_quality.binding_decision import ( + BindingQualityInput, + ComponentEvidence, + evaluate_binding_quality, + freshness_verdict, +) +from qdl.query import DataRequirement +from qdl.query.contracts import ConsumerGrade, FeedType, StalePolicy +from qdl.query.results import QualityMetadata +from qdl.query.service import _freshness_verdict +from scripts.report_binding_liveness import decision_row + + +FIXTURE = ( + Path(__file__).resolve().parents[1] + / "contracts/golden/quality/binding-quality-decision-v1.json" +) + + +class BindingQualityGoldenTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.fixture = json.loads(FIXTURE.read_text(encoding="utf-8")) + + def test_python_matches_shared_golden_corpus(self) -> None: + self.assertEqual(self.fixture["schema"], "qdl.binding-quality-decision.v1") + self.assertGreaterEqual(len(self.fixture["cases"]), 14) + for case in self.fixture["cases"]: + with self.subTest(case=case["name"]): + raw = dict(case["input"]) + raw["components"] = tuple( + ComponentEvidence(**component) + for component in raw["components"] + ) + raw["flags"] = tuple(raw["flags"]) + decision = evaluate_binding_quality(BindingQualityInput(**raw)) + expected = case["expected"] + self.assertEqual(decision.semantics.value, expected["semantics"]) + self.assertEqual( + decision.delivery_semantics, + expected.get("delivery_semantics", "STRICT_EVENT"), + ) + self.assertEqual(decision.availability.value, expected["availability"]) + self.assertEqual(decision.state, expected["state"]) + self.assertEqual( + decision.event_recency_state, expected["event_recency_state"] + ) + self.assertEqual(decision.complete, expected["complete"]) + self.assertEqual( + decision.execution_eligible, expected["execution_eligible"] + ) + self.assertEqual(list(decision.reason_codes), expected["reason_codes"]) + + def test_quiet_semantics_cannot_relabel_quote_as_executable(self) -> None: + quiet = next( + case for case in self.fixture["cases"] + if case["name"] == "quiet_mark_index_components_live" + ) + raw = dict(quiet["input"]) + raw.update({ + "feed": "QUOTE", + "event_recency_policy": "OBSERVE", + "allow_quiet_execution": True, + "components": (), + }) + decision = evaluate_binding_quality(BindingQualityInput(**raw)) + self.assertEqual(decision.semantics.value, "STRICT_EVENT") + self.assertEqual(decision.state, "STALE") + self.assertFalse(decision.execution_eligible) + + def test_on_change_quote_requires_source_semantics_and_live_fences(self) -> None: + source_authorized = next( + case for case in self.fixture["cases"] + if case["name"] == "on_change_quote_live_session" + ) + raw = dict(source_authorized["input"]) + raw["components"] = tuple( + ComponentEvidence(**component) for component in raw["components"] + ) + raw["flags"] = tuple(raw["flags"]) + self.assertTrue( + evaluate_binding_quality(BindingQualityInput(**raw)).execution_eligible + ) + for field, value in ( + ("delivery_semantics", "STRICT_EVENT"), + ("allow_quiet_execution", False), + ("generation_matches", False), + ("config_matches", False), + ("gap_open", True), + ): + with self.subTest(field=field): + candidate = dict(raw) + candidate[field] = value + self.assertFalse( + evaluate_binding_quality( + BindingQualityInput(**candidate) + ).execution_eligible + ) + + def test_query_admission_matches_quality_metadata_without_erasing_gap(self) -> None: + requirement = DataRequirement( + instrument_uid="BINANCE:USD_M:BTCUSDT", + feed=FeedType.QUOTE, + consumer_grade=ConsumerGrade.EXECUTION, + source_policy_id="crypto_primary_v2", + max_freshness_ms=2_000, + event_recency_policy=StalePolicy.BLOCK, + max_session_liveness_ms=45_000, + ) + metadata = QualityMetadata( + state="GAPPED", + freshness_ms=100, + gap_open=True, + complete=False, + execution_eligible=False, + policy_id=requirement.source_policy_id, + event_recency_state="LIVE", + provider_session_state="LIVE", + provider_session_liveness_ms=10, + ) + self.assertEqual(_freshness_verdict(requirement, metadata), (True, None)) + self.assertEqual( + freshness_verdict( + state=metadata.state, + freshness_ms=metadata.freshness_ms, + event_recency_policy=requirement.effective_event_recency_policy.value, + max_freshness_ms=requirement.max_freshness_ms, + provider_session_state=metadata.provider_session_state, + provider_session_liveness_ms=metadata.provider_session_liveness_ms, + max_session_liveness_ms=requirement.max_session_liveness_ms, + ), + (True, None), + ) + + def test_liveness_audit_preserves_declared_on_change_quote_semantics(self) -> None: + binding = type("Binding", (), { + "binding_id": "okx-swap-bnb-usdt-swap-quote", + "instrument": type("Instrument", (), { + "instrument_uid": "bnb", + "session_calendar_id": "CRYPTO_24_7", + })(), + "feed": type("Feed", (), {"value": "QUOTE"})(), + "source_role": "PRIMARY", + "authoritative": True, + "continuous_calendar": True, + "stale_after_ms": 5_000, + "freshness_basis": "SOURCE_EVENT", + "require_final_bar": False, + "delivery_semantics": "ON_CHANGE", + })() + acquisition = type("Acquisition", (), { + "enabled": True, + "mode": "RUST_NATIVE", + "mark_index": None, + })() + requirement = type("Requirement", (), { + "effective_event_recency_policy": type("Policy", (), {"value": "OBSERVE"})(), + "max_session_liveness_ms": 2_000, + "max_freshness_ms": 2_000, + })() + row = decision_row( + binding=binding, + acquisition=acquisition, + requirement=requirement, + stored=None, + now_ns=1_000_000_000, + session_reader=object(), + ) + self.assertEqual(row["delivery_semantics"], "ON_CHANGE") + self.assertEqual(row["semantics"], "QUIET_SESSION") + + +if __name__ == "__main__": # pragma: no cover + unittest.main() diff --git a/tests/test_reference_l2_consumer_acceptance.py b/tests/test_reference_l2_consumer_acceptance.py index 302b4d9..fd38fc3 100644 --- a/tests/test_reference_l2_consumer_acceptance.py +++ b/tests/test_reference_l2_consumer_acceptance.py @@ -24,10 +24,11 @@ reference_request_for_requirement, ) from qdl.certification.phase103_consumer_acceptance import _validate_payload -from qdl.query import ConsumerGrade, FeedType +from qdl.query import ConsumerGrade, FeedType, StalePolicy from qdl.runtime.stable_catalog import StableSourceCatalog from qdl.runtime.stable_deployment import StableAcquisitionPlan from qdl_sdk import Grade +from qdl_sdk.models import StalePolicy as SdkStalePolicy from qdl_sdk.reference import ReferenceProduct, ReferenceRequirement from scripts.phasec36_reference_l2_consumer_acceptance import _reference_batch_until_terminal @@ -241,6 +242,144 @@ def test_execution_mark_index_request_is_one_complete_snapshot(self): self.assertEqual((request.limit, request.page_size, request.max_pages), (1, 1, 1)) self.assertTrue(request.require_full_coverage) + def _execution_mark_index_product(self): + product = next( + item for item in self.scope.references + if item.requirement.feed is FeedType.MARK_INDEX_PRICE + ) + return replace( + product, + requirement=replace( + product.requirement, + consumer_grade=ConsumerGrade.EXECUTION, + event_recency_policy=StalePolicy.OBSERVE, + max_freshness_ms=2_000, + max_session_liveness_ms=45_000, + ), + sdk_requirement=product.sdk_requirement.model_copy(update={ + "consumer_grade": Grade.EXECUTION, + "event_recency_policy": SdkStalePolicy.OBSERVE, + "max_freshness_ms": 2_000, + "max_session_liveness_ms": 45_000, + "limit": 1, + "page_size": 1, + "max_pages": 1, + }), + ) + + @staticmethod + def _quiet_execution_mark_index_item(product, *, source_age_ms: int = 60_000): + source_event_ns = NOW_NS - source_age_ms * _MILLISECOND_NS + field = SimpleNamespace( + name="mark_price", + unit="QUOTE_PRICE", + value=SimpleNamespace(source_text="101.25", coefficient="10125", scale=2), + ) + labels = { + "execution_view": "STABLE_STREAM_GATEWAY", + "freshness_basis": "PROVIDER_CONFIRMATION", + "source_event_time_ns": str(source_event_ns), + "provider_confirmation_ns": str(source_event_ns), + "connection_generation": "1", + "gateway_lease_epoch": "1", + "delivery_stage": "CANONICAL_READ_COMMITTED", + "spool_watermark_offset": "PENDING", + "event_recency_policy": "OBSERVE", + "recency_mode": "COMPONENT_SESSION_LIVE", + "provider_session_state": "LIVE", + "provider_session_liveness_ms": "4", + "provider_session_checked_at_ns": str(NOW_NS - 1 * _MILLISECOND_NS), + "component_mark_received_at_ns": str(NOW_NS - 10_000 * _MILLISECOND_NS), + "component_index_received_at_ns": str(NOW_NS - 60_000 * _MILLISECOND_NS), + "component_mark_quiet_after_ms": "15000", + "component_index_quiet_after_ms": "70000", + } + observation = SimpleNamespace( + instrument_uid=product.instrument_uid, + product=product.sdk_requirement.product, + observed_at_ns=source_event_ns, + fields=[field], + labels=labels, + ) + data = SimpleNamespace( + instrument_uid=product.instrument_uid, + product=product.sdk_requirement.product, + received_at_ns=source_event_ns, + coverage=SimpleNamespace( + complete_left=True, + complete_right=True, + truncated=False, + terminal_reason="LIVE_EXECUTION_VIEW", + ), + lineage=[SimpleNamespace( + provider="BINANCE_USDM", + provider_endpoint=( + "qdl://stable-stream/internal/v2/execution/mark-index/latest" + ), + capability_name="mark_index_price", + source_role="REFERENCE", + )], + observations=[observation], + ) + return SimpleNamespace( + instrument_uid=product.instrument_uid, + product=product.sdk_requirement.product, + status="OK", + problem=None, + data=data, + ) + + def test_quiet_execution_mark_index_quality_uses_session_and_component_evidence(self): + product = self._execution_mark_index_product() + item = self._quiet_execution_mark_index_item(product) + + quality = reference_quality(product, item, observed_at_ns=NOW_NS) + + self.assertEqual(quality["source_age_ms"], 60_000) + self.assertEqual(quality["session_liveness_ms"], 4) + self.assertEqual(quality["component_mark_age_ms"], 10_000) + self.assertEqual(quality["component_index_age_ms"], 60_000) + self.assertFalse(quality["gap_open"]) + self.assertEqual(len(reference_evidence(product, item, observed_at_ns=NOW_NS)), 64) + + def test_quiet_execution_mark_index_quality_fails_closed_outside_exact_contract(self): + product = self._execution_mark_index_product() + strict_product = replace( + product, + requirement=replace( + product.requirement, + event_recency_policy=StalePolicy.BLOCK, + ), + sdk_requirement=product.sdk_requirement.model_copy(update={ + "event_recency_policy": SdkStalePolicy.BLOCK, + }), + ) + with self.assertRaisesRegex(ValueError, "governed freshness"): + reference_quality( + strict_product, + self._quiet_execution_mark_index_item(strict_product), + observed_at_ns=NOW_NS, + ) + + cases = ( + ("session", lambda item: item.data.observations[0].labels.__setitem__( + "provider_session_state", "DISCONNECTED"), "session evidence"), + ("component", lambda item: item.data.observations[0].labels.__setitem__( + "component_index_received_at_ns", str(NOW_NS - 70_001 * _MILLISECOND_NS)), "component exceeded"), + ("missing", lambda item: item.data.observations[0].labels.pop( + "component_mark_received_at_ns"), "evidence is malformed"), + ("zero-fence", lambda item: item.data.observations[0].labels.__setitem__( + "gateway_lease_epoch", "0"), "provenance is invalid"), + ("lineage", lambda item: setattr( + item.data.lineage[0], "provider_endpoint", "https://provider.invalid"), "live-view lineage"), + ) + for name, mutate, expected in cases: + with self.subTest(case=name): + item = self._quiet_execution_mark_index_item(product) + mutate(item) + with self.assertRaisesRegex(ValueError, expected): + reference_quality(product, item, observed_at_ns=NOW_NS) + def test_reference_quality_uses_provider_observation_not_local_receive_time(self): product = next( item for item in self.scope.references diff --git a/tests/test_refresh_v2_l2_core_runtime.py b/tests/test_refresh_v2_l2_core_runtime.py index 0be1c48..5e388ba 100644 --- a/tests/test_refresh_v2_l2_core_runtime.py +++ b/tests/test_refresh_v2_l2_core_runtime.py @@ -13,6 +13,7 @@ write_stable_runtime_bundle, ) from qdl.runtime.execution_l2 import execution_l2_materialization_plan +from qdl.runtime.core_binding_identity import core_binding_map from scripts.refresh_v2_l2_core_runtime import ( CORE_FILES, refresh, @@ -36,6 +37,11 @@ def _execution_l2_source_ids() -> frozenset[str]: ).source_ids) +@lru_cache(maxsize=1) +def _catalog_revision() -> int: + return StableSourceCatalog.load(CATALOG).catalog_revision + + class L2CoreRuntimeRefreshTests(unittest.TestCase): def _authority(self) -> dict[str, object]: return stable_authority_record( @@ -99,7 +105,7 @@ def test_dry_run_adds_exact_declared_l2_scope_without_mutation(self): item["before_binding_count"], ) self.assertEqual(item["before_catalog_revisions"], [7]) - self.assertEqual(item["after_catalog_revisions"], [8]) + self.assertEqual(item["after_catalog_revisions"], [_catalog_revision()]) self.assertEqual(set(item["added_book_source_ids"]), _execution_l2_source_ids()) self.assertEqual({name: (runtime / name).read_bytes() for name in CORE_FILES}, before) @@ -125,8 +131,8 @@ def test_apply_preserves_existing_bindings_authority_and_modes(self): self.assertEqual((runtime / file_name).stat().st_mode & 0o777, modes[file_name]) active = json.loads(before[file_name])["core"]["bindings"] updated = json.loads((runtime / file_name).read_text(encoding="utf-8"))["core"]["bindings"] - active_by_id = {item["source_id"]: item for item in active} - updated_by_id = {item["source_id"]: item for item in updated} + active_by_id = core_binding_map(active, field="active test core") + updated_by_id = core_binding_map(updated, field="updated test core") self.assertTrue(set(active_by_id).issubset(updated_by_id)) self.assertTrue(all( { @@ -142,11 +148,15 @@ def test_apply_preserves_existing_bindings_authority_and_modes(self): for key in active_by_id )) self.assertTrue(all( - updated_by_id[key]["instrument_catalog_revision"] == 8 + updated_by_id[key]["instrument_catalog_revision"] == _catalog_revision() for key in active_by_id )) self.assertEqual( - set(updated_by_id) - set(active_by_id), + { + item["source_id"] + for identity, item in updated_by_id.items() + if identity not in active_by_id + }, _execution_l2_source_ids(), ) @@ -228,8 +238,11 @@ def test_rejects_unknown_active_binding(self): runtime = self._runtime(root) path = runtime / "core-002.json" payload = json.loads(path.read_text(encoding="utf-8")) - extra = dict(payload["core"]["bindings"][0]) + extra = dict(next( + item for item in payload["core"]["bindings"] if item.get("l2") is not None + )) extra["source_id"] = "unknown-book-source" + extra["native_channel"] = "unknown-book@100ms" payload["core"]["bindings"].append(extra) path.write_text(json.dumps(payload), encoding="utf-8") with self.assertRaisesRegex(ValueError, "absent from current catalog"): diff --git a/tests/test_refresh_v2_native_ingestor_runtime.py b/tests/test_refresh_v2_native_ingestor_runtime.py index ecbd777..44fe7d3 100644 --- a/tests/test_refresh_v2_native_ingestor_runtime.py +++ b/tests/test_refresh_v2_native_ingestor_runtime.py @@ -11,7 +11,7 @@ stable_authority_record, write_stable_runtime_bundle, ) -from scripts.refresh_v2_native_ingestor_runtime import TARGETS, refresh +from scripts.refresh_v2_native_ingestor_runtime import REALTIME_FEEDS, TARGETS, refresh ROOT = Path(__file__).resolve().parents[1] @@ -75,6 +75,15 @@ def _refresh(self, root: Path, *, apply: bool) -> dict: state_root=root / "state", ) + def _expected_realtime_count(self, lane: str) -> int: + catalog = StableSourceCatalog.load(CATALOG) + acquisition = StableAcquisitionPlan.load(ACQUISITION, catalog=catalog) + generated = acquisition.native_ingestor_configs( + catalog=catalog, + authority=self._authority(), + )[lane]["bindings"] + return sum(item["feed"] in REALTIME_FEEDS for item in generated) + def test_dry_run_adds_only_the_three_declared_l2_books_per_venue(self): with tempfile.TemporaryDirectory() as raw: root = Path(raw) @@ -141,8 +150,22 @@ def test_apply_preserves_authority_and_is_idempotent(self): oct(before_modes[file_name]), ) payload = json.loads((runtime / file_name).read_text(encoding="utf-8")) - self.assertEqual(len(payload["bindings"]), 19) + lane = next(name for name, value in TARGETS.items() if value == file_name) + self.assertEqual( + len(payload["bindings"]), + self._expected_realtime_count(lane), + ) self.assertFalse(any(item["feed"] == "BAR" for item in payload["bindings"])) + mark_index = [ + item for item in payload["bindings"] if item["feed"] == "MARK_INDEX" + ] + self.assertEqual( + len({ + (item["subscription_id"], item["native_symbol"], item["native_channel"]) + for item in mark_index + }), + len(mark_index), + ) repeated = refresh( runtime_dir=runtime, diff --git a/tests/test_release_session_observations.py b/tests/test_release_session_observations.py index 4f8b5bc..6fbb4dc 100644 --- a/tests/test_release_session_observations.py +++ b/tests/test_release_session_observations.py @@ -59,6 +59,28 @@ def test_fresh_price_still_requires_typed_live_complete_session(self): self.assertFalse(v2_observation_is_current(requirement, replace(observation, v2_session_state="DISCONNECTED"))) + def test_old_on_change_quote_requires_signed_delivery_and_execution_eligibility(self): + requirement = self.requirement(FeedType.QUOTE, StalePolicy.OBSERVE) + observation = replace( + self.observation(), + v2_execution_eligible=True, + v2_delivery_semantics="ON_CHANGE", + ) + self.assertTrue(v2_observation_is_current(requirement, observation)) + for fields in ( + {"v2_delivery_semantics": "STRICT_EVENT"}, + {"v2_delivery_semantics": None}, + {"v2_execution_eligible": False}, + {"v2_session_state": "DISCONNECTED"}, + {"v2_session_liveness_ms": 45_001}, + {"v2_gap_open": True}, + {"v2_complete": False}, + ): + with self.subTest(fields=fields): + self.assertFalse( + v2_observation_is_current(requirement, replace(observation, **fields)) + ) + def test_compact_quality_does_not_discard_session_or_make_quiet_data_executable(self): quality = SimpleNamespace(freshness_ms=12000, gap_open=False, state="LIVE", provider_session_state="LIVE", provider_session_liveness_ms=900, @@ -69,6 +91,19 @@ def test_compact_quality_does_not_discard_session_or_make_quiet_data_executable( self.assertEqual(evidence["provider_session_liveness_ms"], 900) self.assertFalse(evidence["execution_eligible"]) + def test_compact_quality_preserves_on_change_delivery_provenance(self): + quality = SimpleNamespace( + freshness_ms=12000, gap_open=False, state="LIVE", + provider_session_state="LIVE", provider_session_liveness_ms=900, + complete=True, execution_eligible=True, + flags=("LAST_EVENT_STALE", "DELIVERY_ON_CHANGE"), + ) + evidence = compact_view_quality( + SimpleNamespace(quality=quality, received_at_ns=1000000), + observed_at_ns=13000000, + ) + self.assertEqual(evidence["delivery_semantics"], "ON_CHANGE") + def test_typed_evidence_rejects_bad_types_and_extra_fields(self): for fields in ({"v2_session_liveness_ms": True}, {"v2_complete": 1}, {"v2_quality_state": "UNVALIDATED"}, {"v2_session_liveness_ms": -1}): diff --git a/tests/test_trading_consumer_scope.py b/tests/test_trading_consumer_scope.py index c24f094..601e448 100644 --- a/tests/test_trading_consumer_scope.py +++ b/tests/test_trading_consumer_scope.py @@ -31,7 +31,7 @@ def test_complete_native_five_symbol_matrix(self): def test_sealed_roundtrip_and_manifest_revision(self): parsed = ConsumerRouteBinding.from_canonical_mapping(self.binding.canonical_mapping()) self.assertEqual(parsed, self.binding) - self.assertEqual(parsed.consumer_manifest_revision, 9) + self.assertEqual(parsed.consumer_manifest_revision, 10) value = self.binding.canonical_mapping() value["products"][0]["native_symbol"] = "WRONG" with self.assertRaises(ValueError): diff --git a/upgrade/evidence/releases/v2.0.26/RELEASE_NOTES.md b/upgrade/evidence/releases/v2.0.26/RELEASE_NOTES.md new file mode 100644 index 0000000..b9689bd --- /dev/null +++ b/upgrade/evidence/releases/v2.0.26/RELEASE_NOTES.md @@ -0,0 +1,50 @@ +# Quant Data Layer v2.0.26 + +## Certified Scope + +`v2.0.26` certifies the current declared V2 consumer plane for Binance USD-M +and OKX Swap. The sealed no-order C2 receipt passed all `299` active products: +`234` durable and `65` on-demand. It exercised both Query replicas, public V2 +Query/Stream SDK paths, signed cursor/reconnect, final BAR/history, reference +batch, L2 status/snapshot and manifest-governed V1 fallback. + +The acceptance observed for `300.089s`, completed every opening and closing +product read, made zero order actions and zero provider-direct client +connections, retained no secrets or raw market payload, and removed its client +cursor directory. Seven allowed `TRADE` routes passed `V2_PRIMARY -> +V1_FALLBACK -> V2_PRIMARY`; no blocked route was downgraded. + +## Runtime Provenance + +The active Query and Stream roles run +`qdl-v2-python:2.0.26-137633b@sha256:0a69fbf0c883cad27a433ea529555269ab7386706de6e1c635fa5fef2107a545`. +Both Query and both Stream roles were healthy with restart count `0` and no +OOM kill at certification. The Stream image override is stored in external +release state, not `/tmp`; the temporary source path was removed after a +byte-identical Compose resolution and serial Stream roll. + +The release is component-attested. The active reader image contains all +serving runtime changes in this closure. Source changes after that image are +only the acceptance harness, its regression tests and release journal. Rust +post-image work is a pure shared golden/parity oracle and is not a new serving +path in `qdl-realtime-core`. + +## Consumer-Call Latency + +The certificate retains per-binding/per-replica p50/p95/p99/max measurements. +They measure authenticated consumer request start to SDK-usable response under +the concurrent 299-product C2 workload. They must not be read as venue event +age or final-bar close latency. The full bounded figures and semantics are in +`DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md` R1.35-C and the sealed runtime +receipt identified by SHA-256 +`6cbbbac45c7f8f9e321677499dc698addd5cce62f74201f261a1dce68a5b53aa`. + +## Declared Boundaries + +- DNSE/VN remains `V1_PRIMARY` until its separate market-hours certificate. +- Dark or unentitled Spot/catalog rows are not claimed as V2 execution scope. +- This data-plane release does not grant broker-order authority; Trading System + and alpha paper/live policy remain their own control plane. + +The machine-readable [certificate](./certificate.json) and +[scope evidence](./scope-evidence.json) are part of this release. diff --git a/upgrade/evidence/releases/v2.0.26/certificate.json b/upgrade/evidence/releases/v2.0.26/certificate.json new file mode 100644 index 0000000..f08ad72 --- /dev/null +++ b/upgrade/evidence/releases/v2.0.26/certificate.json @@ -0,0 +1,110 @@ +{ + "schema": "qdl.v2.r135-release-certificate.v1", + "status": "PASS", + "release": "v2.0.26", + "classification": "R1.35 full active-V2 endpoint, binding and consumer certificate with release-provenance reconciliation.", + "release_ref": "annotated git tag v2.0.26", + "scope": { + "active_v2_products": 299, + "durable_products": 234, + "on_demand_products": 65, + "venues": { + "BINANCE_USDM": 157, + "OKX_SWAP": 142 + }, + "feeds": { + "BAR": 150, + "BASIS": 5, + "BOOK_DELTA": 20, + "BOOK_SNAPSHOT": 20, + "CONTRACT_METADATA": 10, + "FUNDING_RATE": 10, + "LONG_SHORT_RATIO": 5, + "MARK_INDEX_PRICE": 20, + "OPEN_INTEREST": 10, + "QUOTE": 20, + "TAKER_FLOW": 5, + "TRADE": 24 + } + }, + "acceptance": { + "receipt_schema": "qdl.phase105.v2-identity-acceptance.v1", + "receipt_sha256": "6cbbbac45c7f8f9e321677499dc698addd5cce62f74201f261a1dce68a5b53aa", + "status": "PASS_V2_DATA_PLANE_ONLY", + "opening_products": 299, + "opening_seconds": 903.722, + "opening_budget_seconds": 935, + "observation_seconds": 300.089, + "closing_products": 299, + "closing_seconds": 30.111, + "elapsed_seconds": 1233.925, + "quota_window_wait_seconds": 34.165, + "order_actions": 0, + "provider_direct_connections": 0, + "cursor_directory_removed": true, + "secret_values_recorded": false, + "test_provenance": false, + "fallback_drill": { + "status": "PASS", + "allowed_trade_routes": 7, + "transition": "V2_PRIMARY -> V1_FALLBACK -> V2_PRIMARY", + "blocked_route_downgrade": false + }, + "closing_transports": { + "BATCH_V2_PRIMARY": 194, + "L2_STATUS_SNAPSHOT": 40, + "ON_DEMAND_NO_CLOSING_DURABLE_READ": 65 + }, + "latency_semantics": "Per-binding primary and secondary values in the sealed C2 receipt measure authenticated consumer request start to SDK-usable response under concurrent 299-product workload. They are not venue event age, provider timestamp age, or final-bar-close latency." + }, + "runtime": { + "reader_image": { + "reference": "qdl-v2-python:2.0.26-137633b", + "digest": "sha256:0a69fbf0c883cad27a433ea529555269ab7386706de6e1c635fa5fef2107a545", + "source_commit": "137633b9d5458b28ceb35ca34defe9f55738f722", + "roles": [ + "query_v2_1", + "query_v2_2", + "stream_v2_active", + "stream_v2_passive" + ] + }, + "reader_rollbacks": { + "query": "sha256:3b06543620dda25ab70db566d808841e137dfa55e54bd5bc366a372cf665c6ee", + "stream": "sha256:f2489160923d65c076b7cadc8c9bba2da6e9c862567428ce87ab264e957b6ad7" + }, + "projector_image": { + "digest": "sha256:d724764b17dc9f21e681c2ebeac4fb48588d45010b0f1ff3a00b6cfa408f2a3b", + "source_commit": "987b1a2827338f98eb41318d8536a28808ff80d0" + }, + "binance_bar_edge_image": { + "digest": "sha256:9039236e7a8e570f2364b470b33386ab702bc1dde5ae9d5e7d90a4dda531e8f0", + "source_commit": "95d95957f2ee41a8af7068857c15916b9cb082c7" + }, + "rust_core_image": { + "digest": "sha256:389753b37c4f3935193acc739687c4a18c5b307f3ea6a427153f5689d31c9762", + "source_commit": "f1c9e1dd145c159934fb418d96fdac50d21c1d3b" + }, + "runtime_provenance": { + "stream_override_sha256": "01e341ecb94a9b037500d49f0c76d22fcda5f771d28194a5abb4a4fb911c3711", + "stream_override_path": "/home/bobby/.local/state/qdl-v2/releases/r1.35-137633b/stream-reader-image.override.yml", + "temporary_override_removed": true, + "reader_restart_count": 0, + "reader_oom_killed": false + } + }, + "component_reconciliation": { + "reader_delta_after_137633b": [ + "DATA_LAYER_UNIFIED_IMPLEMENTATION_PLAN.md", + "scripts/phase105_consumer_v2_identity_acceptance.py", + "tests/test_phase105_identity_acceptance.py" + ], + "rust_delta_after_f1c9e1d": "qdl-core quality golden/parity module only; qdl-realtime-core and provider-admission serving paths have no post-image source delta.", + "projector_and_bar_edge": "The active projector implementation and Binance bar-edge entrypoint/path have no later serving-code delta for this release scope." + }, + "declared_exclusions": [ + "DNSE/VN remains V1_PRIMARY pending its separate market-hours certificate.", + "Dark or unentitled Spot/catalog inventory is not claimed as V2 execution coverage.", + "This certificate covers declared Binance USD-M and OKX Swap V2 consumer products only; it does not grant broker-order authority." + ] +} diff --git a/upgrade/evidence/releases/v2.0.26/scope-evidence.json b/upgrade/evidence/releases/v2.0.26/scope-evidence.json new file mode 100644 index 0000000..0362521 --- /dev/null +++ b/upgrade/evidence/releases/v2.0.26/scope-evidence.json @@ -0,0 +1,27 @@ +{ + "schema": "qdl.v2.r135-release-scope.v1", + "release": "v2.0.26", + "certificate_sha256_source": "upgrade/evidence/releases/v2.0.26/certificate.json", + "active_scope": { + "products": 299, + "durable": 234, + "on_demand": 65, + "binance_usdm": 157, + "okx_swap": 142 + }, + "quality_fences": [ + "identity", + "venue isolation", + "declared feed semantics", + "final-bar validity", + "sequence/gap/generation", + "two-query-replica parity", + "signed cursor replay", + "manifest-governed V1 fallback" + ], + "acceptance_receipt_sha256": "6cbbbac45c7f8f9e321677499dc698addd5cce62f74201f261a1dce68a5b53aa", + "order_actions": 0, + "provider_direct_connections": 0, + "raw_market_payload_recorded": false, + "secrets_recorded": false +} diff --git a/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md b/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md index ee4c2a8..0247d15 100644 --- a/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md +++ b/upgrade/quant-data-layer-fund-grade-upgrade-architecture.md @@ -5836,6 +5836,36 @@ runtime image, bundle, role, provider session, durable store or consumer was changed by this evidence. This is not a C2 certificate; one separately approved runtime packet remains required. +#### J.7.2 MARK/INDEX component recency on quiet provider channels + +`MARK_INDEX_PRICE` is a paired execution reference, not a generic sparse +trade. Its original component timestamps and receipts remain immutable +lineage. A provider may legitimately leave an unchanged mark or index value +quiet for its documented cadence, so an execution consumer may use a pair +whose older component is beyond the ordinary event-age bound only when all of +the following are true: + +- the signed acquisition binding declares a finite quiet window for every + physical component; +- Rust has retained the exact same-session, same-generation pair and refuses + to materialize a pair whose component exceeds its own window; +- the V2 request explicitly sets `event_recency_policy=OBSERVE` and a bounded + `max_session_liveness_ms`; +- the stream gateway reads a matching `LIVE` provider-session record in the + current config revision and proves no gap/resync or gateway fence; and +- the query result carries the original component receipt times, component + cadence, session check time and liveness evidence for the consumer to audit. + +This is a narrow execution-reference exception, not a relaxation of +`max_freshness_ms` for trade, quote, book, BAR or arbitrary reference data. +For an admitted quiet MARK/INDEX pair, the two-second consumer limit remains +the bounded request-to-usable-read budget; component age is evaluated against +the signed component cadence and is never rewritten as a fresh event. Missing +cadence, stale component, stopped/disconnected/ambiguous session, generation +or config mismatch, clock skew, gap/resync, or a legacy strict request fails +closed. The same model represents a combined Binance frame and separate OKX +MARK/INDEX frames without a symbol-specific Python exception. + ### J.8 Phase 11.5 consumer-scoped universal route binding The universal release manifest is a release-control artifact, not an