From 26934a6d44d6d7f1317358bbe2e90cf8a252fb38 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 07:39:58 -0600 Subject: [PATCH 01/63] perf(db): serve eq-filtered live queries from shared source partitions (WIP) Work in progress toward Redux-level mount and update cost for many small eq-filtered live queries (#445). Not yet pushed. - Pool single-source queries filtered only by eq(field, literal) through a per-source partition keyed by eq-normalized literal tuples; a lean wholesale observer reads the bucket, and the live-query Collection is built only when the app touches result.collection. - Resolve adapter query values through one core resolveLiveQueryValue; React uses it and skips identity hashing without DbClient or Suspense. - Remove per-subscription change routing and the unindexed snapshot prefilter. - Cheaper query building: ref proxies are branded instead of registered in a WeakSet and cache children by property; CollectionRef and PropRef avoid defineProperty. - SortedMap updates of existing keys skip re-sorting without a comparator. - Pooled live query oracle compares pooled observers with the live-query Collection over sync, optimistic, and mount/unmount histories. Co-authored-by: Isaac --- packages/db/package.json | 2 +- packages/db/src/SortedMap.ts | 5 + packages/db/src/collection/change-events.ts | 136 ---- packages/db/src/collection/changes.ts | 84 +-- packages/db/src/collection/index.ts | 4 +- packages/db/src/collection/state.ts | 17 - packages/db/src/collection/subscription.ts | 26 - packages/db/src/live-query-observer.ts | 28 +- packages/db/src/live-query-options.ts | 37 +- .../src/query/builder/ref-proxy-identity.ts | 17 +- packages/db/src/query/builder/ref-proxy.ts | 41 +- packages/db/src/query/ir.ts | 19 +- packages/db/src/query/pooled-live-query.ts | 600 ++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + .../pooled-live-query-oracle.property.test.ts | 506 +++++++++++++++ packages/react-db/src/useLiveQuery.ts | 86 +-- 16 files changed, 1252 insertions(+), 357 deletions(-) create mode 100644 packages/db/src/query/pooled-live-query.ts create mode 100644 packages/db/tests/query/pooled-live-query-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index 630343add..864d5acb5 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:dist": "vitest run --config vitest.dist.config.ts", "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts", - "test:oracles": "vitest --run --coverage.enabled=false tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-truncate-ownership-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts", + "test:oracles": "vitest --run --coverage.enabled=false tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-truncate-ownership-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts", "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run" }, "type": "module", diff --git a/packages/db/src/SortedMap.ts b/packages/db/src/SortedMap.ts index c0b31b14c..dbcd830e5 100644 --- a/packages/db/src/SortedMap.ts +++ b/packages/db/src/SortedMap.ts @@ -106,6 +106,11 @@ export class SortedMap { * @returns This SortedMap instance for chaining */ set(key: TKey, value: TValue, deferOrder = false): this { + // Key order cannot change when an existing key gets a new value. + if (!this.comparator && this.map.has(key)) { + this.map.set(key, value) + return this + } // Grouped Collections can produce nullish keys at runtime. compareKeys // is not a total order there, so retain the existing binary-insert path. const runtimeKey = typeof key !== `string` && typeof key !== `number` diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index 6409b883f..f22bcda97 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -7,8 +7,6 @@ import { optimizeExpressionWithIndexes, } from '../utils/index-optimization.js' import { ensureIndexForField } from '../indexes/auto-index.js' -import { getPropRefPropertyPath } from '../query/ir.js' -import { isVirtualPropName } from '../virtual-props.js' import { makeComparator } from '../utils/comparison.js' import { buildCompareOptions } from '../query/compiler/order-by' import type { @@ -21,14 +19,6 @@ import type { CollectionImpl } from './index.js' import type { BasicExpression, OrderBy } from '../query/ir.js' import type { WithVirtualProps } from '../virtual-props.js' -/** - * Yields visible entries, enriched with virtual properties, whose stored row - * passes `prefilter`. - */ -export type StoredRowScan = ( - prefilter: (row: object) => boolean, -) => Iterable<[TKey, WithVirtualProps]> - /** * Returns the current state of the collection as an array of changes * @param collection - The collection to get changes from @@ -66,23 +56,12 @@ export function currentStateAsChanges< >( collection: CollectionLike, TKey>, options: CurrentStateAsChangesOptions = {}, - scanStoredRows?: StoredRowScan, ): Array, TKey>> | void { // Helper function to collect filtered results const collectFilteredResults = ( filterFn?: (value: WithVirtualProps) => boolean, ): Array, TKey>> => { const result: Array, TKey>> = [] - // Reject rows by one stored field before copying them to add virtual - // properties. Survivors still pass through the full predicate. - const prefilter = - scanStoredRows && options.where && compileEqualityPrefilter(options.where) - if (filterFn && scanStoredRows && prefilter) { - for (const [key, value] of scanStoredRows(prefilter)) { - if (filterFn(value)) result.push({ type: `insert`, key, value }) - } - return result - } for (const [key, value] of collection.entries()) { // If no filter function is provided, include all items if (filterFn?.(value) ?? true) { @@ -219,121 +198,6 @@ export function createFilterFunctionFromExpression( } } -/** A field and the string or boolean literal a top-level `eq` requires. */ -export type EqualityRoute = { - path: Array - /** Stable identity of `path`, for grouping routes by field. */ - pathKey: string - expected: string | boolean -} - -/** Read result for a route path whose property access threw. */ -export const UNREADABLE_ROUTE_VALUE: unique symbol = Symbol( - `unreadable route value`, -) - -/** - * Finds a cheap necessary condition for `expression` to be TRUE, or returns - * undefined when the expression has none. - * - * A top-level conjunct `eq(field, literal)` with a string or boolean literal is - * TRUE only when the field holds the identical string or boolean: equality - * normalization never maps another type onto a plain string or boolean. A - * row whose field holds anything else therefore fails the whole expression. - * - * With `storedRows`, the condition is read from a stored row instead of its - * enriched copy, so conjuncts on virtual fields are skipped: stored rows need - * not carry them. - */ -export function findEqualityRoute( - expression: BasicExpression, - { storedRows = false }: { storedRows?: boolean } = {}, -): EqualityRoute | undefined { - const conjuncts: Array = [] - const collect = (node: BasicExpression) => { - if (node.type === `func` && node.name === `and`) node.args.forEach(collect) - else conjuncts.push(node) - } - collect(expression) - - // A string literal usually rejects more rows than a boolean one. - let best: EqualityRoute | undefined - for (const conjunct of conjuncts) { - if (conjunct.type !== `func` || conjunct.name !== `eq`) continue - const [left, right] = conjunct.args - const ref = - left?.type === `ref` ? left : right?.type === `ref` ? right : undefined - const literal = - left?.type === `val` ? left : right?.type === `val` ? right : undefined - if (!ref || !literal) continue - const expected: unknown = literal.value - if (typeof expected !== `string` && typeof expected !== `boolean`) continue - - const path = getPropRefPropertyPath(ref) - if (storedRows && (path.length === 0 || isVirtualPropName(path[0]!))) { - continue - } - if (best === undefined || typeof best.expected === `boolean`) { - best = { path, pathKey: JSON.stringify(path), expected } - } - if (typeof expected === `string`) break - } - return best -} - -/** - * Reads a route field the way the single-row evaluator does. A throwing read - * returns UNREADABLE_ROUTE_VALUE so callers leave the decision to the full - * predicate. - */ -export function readRouteValue( - row: unknown, - path: ReadonlyArray, -): unknown { - try { - let value: unknown = row - for (const segment of path) { - if (value === null || value === undefined) return undefined - value = (value as Record)[segment] - } - return value - } catch { - return UNREADABLE_ROUTE_VALUE - } -} - -/** - * Compiles the route of `expression` as a row test that is false only when - * the full predicate must be false. - * - * The test reads a stored row instead of its enriched copy. The copy holds - * each enumerable own root property of the stored row and lacks the others, so its field is either the stored value or `undefined`, - * which never equals the literal. A read that throws passes the row to the - * full predicate. - */ -export function compileEqualityPrefilter( - expression: BasicExpression, -): ((row: object) => boolean) | undefined { - const route = findEqualityRoute(expression, { storedRows: true }) - if (route === undefined) return undefined - const { path, expected } = route - // Most routes name one top-level field; read it without walking a path. - if (path.length === 1) { - const field = path[0]! - return (row) => { - try { - return (row as Record)[field] === expected - } catch { - return true - } - } - } - return (row) => { - const value = readRouteValue(row, path) - return value === UNREADABLE_ROUTE_VALUE || value === expected - } -} - /** * Creates a filtered callback that only calls the original callback with changes that match the where clause * @param originalCallback - The original callback to filter diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index f6b1403f8..5bf5be736 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -6,7 +6,6 @@ import { toExpression, } from '../query/builder/ref-proxy.js' import { CollectionSubscription } from './subscription.js' -import { readRouteValue } from './change-events.js' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessage, SubscribeChangesOptions } from '../types' import type { CollectionLifecycleManager } from './lifecycle.js' @@ -260,25 +259,9 @@ export class CollectionChangesManager< const layoutListeners = [...this.layoutChangeListeners] const subscriptions = [...this.changeSubscriptions] withPublicationContext(() => { - // An empty batch signals readiness to every subscriber. - const routed = - rawEvents.length > 0 - ? routeChanges(enrichedEvents, subscriptions) - : undefined - const callbacks: Array<() => void> = [] - for (const subscription of subscriptions) { - const own = routed?.get(subscription) - callbacks.push(() => { - // An earlier callback in this publication can end routing for this - // subscription, for example by leaving stale rows to reconcile. - if (own === undefined || !subscription.changeRoute) { - subscription.emitEvents(enrichedEvents) - } else if (own.length > 0) { - // A routed subscription with no candidate change cannot publish. - subscription.emitEvents(own) - } - }) - } + const callbacks: Array<() => void> = subscriptions.map( + (subscription) => () => subscription.emitEvents(enrichedEvents), + ) if (rawEvents.length === 0) { callbacks.unshift(...layoutListeners) } @@ -442,64 +425,3 @@ export class CollectionChangesManager< this.deferral = undefined } } - -/** - * Gives each routed subscription only the changes whose value or previous - * value holds its route literal, in batch order. Subscriptions without a - * route are absent from the result and receive the whole batch; with no - * routed subscription the result is undefined. - */ -function routeChanges( - changes: Array>, - subscriptions: Array, -): Map>> | undefined { - const routes = subscriptions.map((subscription) => subscription.changeRoute) - if (routes.every((route) => route === undefined)) return undefined - const routed = new Map< - CollectionSubscription, - Array> - >() - const groups = new Map< - string, - { - path: Array - byLiteral: Map> - } - >() - for (const [index, subscription] of subscriptions.entries()) { - const route = routes[index] - if (!route) continue - routed.set(subscription, []) - let group = groups.get(route.pathKey) - if (!group) { - group = { path: route.path, byLiteral: new Map() } - groups.set(route.pathKey, group) - } - const peers = group.byLiteral.get(route.expected) - if (peers) peers.push(subscription) - else group.byLiteral.set(route.expected, [subscription]) - } - - const deliver = ( - targets: Array | undefined, - change: ChangeMessage, - ) => { - for (const subscription of targets ?? []) { - routed.get(subscription)!.push(change) - } - } - for (const change of changes) { - for (const group of groups.values()) { - const value = readRouteValue(change.value, group.path) - const previous = - change.previousValue === undefined - ? undefined - : readRouteValue(change.previousValue, group.path) - // The where filter reads these same values, and a read that throws makes - // its predicate false, so an unreadable value matches no literal here. - deliver(group.byLiteral.get(value), change) - if (previous !== value) deliver(group.byLiteral.get(previous), change) - } - } - return routed -} diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 239778486..589b61980 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -1051,9 +1051,7 @@ export class CollectionImpl< public currentStateAsChanges( options: CurrentStateAsChangesOptions = {}, ): Array, TKey>> | void { - return currentStateAsChanges(this, options, (prefilter) => - this._state.entriesPassing(prefilter), - ) + return currentStateAsChanges(this, options) } /** diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index a7de166e1..2caa91d33 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -500,23 +500,6 @@ export class CollectionStateManager< } } - /** - * Visible entries whose stored row passes `prefilter`, enriched with virtual - * properties. Rows that fail are never copied. - */ - public *entriesPassing( - prefilter: (row: object) => boolean, - ): IterableIterator<[TKey, WithVirtualProps]> { - // Without optimistic state, the visible rows are the synced rows in order. - const rows = - this.optimisticUpserts.size === 0 && this.optimisticDeletes.size === 0 - ? this.syncedData - : this.entries() - for (const [key, row] of rows) { - if (prefilter(row)) yield [key, this.enrichWithVirtualProps(row, key)] - } - } - /** * Get all entries (virtual derived state) */ diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 4f76ee879..8285bad58 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -12,9 +12,7 @@ import { LoadSubsetOperationAbortedError } from '../errors.js' import { createFilterFunctionFromExpression, createFilteredCallback, - findEqualityRoute, } from './change-events.js' -import type { EqualityRoute } from './change-events.js' import type { BasicExpression, OrderBy } from '../query/ir.js' import type { IndexReader } from '../indexes/base-index.js' import type { @@ -174,9 +172,6 @@ export class CollectionSubscription private filteredCallback: (changes: Array>) => boolean - /** Field and literal the where clause requires, if it has a cheap one. */ - private readonly equalityRoute: EqualityRoute | undefined - private orderByIndex: IndexReader | undefined // Status tracking @@ -239,10 +234,6 @@ export class CollectionSubscription this.callback = callbackWithSentKeysTracking - this.equalityRoute = options.whereExpression - ? findEqualityRoute(options.whereExpression) - : undefined - // Create a filtered callback if where clause is provided this.filteredCallback = options.whereExpression ? createFilteredCallback(this.callback, options) @@ -1133,23 +1124,6 @@ export class CollectionSubscription return this.filteredCallback(newChanges) } - /** - * The route through which this subscription may receive only the changes - * whose value or previous value holds the route's literal. A change reaches - * the where filter only through those values, so the others cannot publish, - * and sent-key records cover published rows only. Stale published rows and - * truncate replay consume unfiltered changes, so no route applies then. - */ - get changeRoute(): EqualityRoute | undefined { - if ( - this.stalePublishedRows.size > 0 || - this.truncateReplayState !== undefined - ) { - return undefined - } - return this.equalityRoute - } - /** Keep direct snapshot reads private while an authoritative replay is open. */ private publishSnapshot(changes: Array>): void { if (!this.bufferPrivately(changes)) this.callback(changes) diff --git a/packages/db/src/live-query-observer.ts b/packages/db/src/live-query-observer.ts index 08dce318a..e596eff67 100644 --- a/packages/db/src/live-query-observer.ts +++ b/packages/db/src/live-query-observer.ts @@ -5,6 +5,7 @@ import { } from './live-query-adapter.js' import { getBuilderFromConfig } from './query/live/collection-registry.js' import { getPersistedReadinessSource } from './persisted-readiness.js' +import { createPooledObserver } from './query/pooled-live-query.js' import type { Collection } from './collection/index.js' import type { DbClient, DehydratedLiveQueryResult } from './client.js' import type { PersistedReadinessSource } from './persisted-readiness.js' @@ -307,7 +308,11 @@ class LiveQueryObserverImpl< this.cachedSnapshot = { state, data: singleResult ? data[0] : data, - collection, + // A pooled view observes its bucket directly and hands users a + // Collection that is built only when touched. + collection: + (collection as { publicCollection?: Collection }) + .publicCollection ?? collection, layoutRevision: this.layoutRevision, status, ...getLiveQueryStatusFlags(status), @@ -816,8 +821,19 @@ class LiveQueryObserverImpl< // listener delivery: useSyncExternalStore performs its consistency read // immediately after subscribe returns. this.flushPublications(!this.wholesale) - const { entries, revision } = this.readEntries(collection) - this.updateCachedEntries(entries, revision) + // The render-time read is still current unless the handshake published. + const revision = this.getCollectionRevision(collection) + const layoutRevision = this.getCollectionLayoutRevision(collection) + if ( + revision === undefined || + this.cachedEntries === undefined || + revision !== this.cachedCollectionRevision || + layoutRevision !== this.cachedCollectionLayoutRevision + ) { + const { entries } = this.readEntries(collection) + this.updateCachedEntries(entries, revision) + this.cachedCollectionLayoutRevision = layoutRevision + } } if (this.hasHydrationSeed()) { if (!this.wholesale) this.seed(Array.from(this.subscriptions)[0]!) @@ -1150,6 +1166,12 @@ export function createLiveQueryObserver< collection: Collection | null | undefined, options: CreateLiveQueryObserverOptions = {}, ): LiveQueryObserver { + const pooled = createPooledObserver(collection, { + wholesale: options.mode === `wholesale`, + client: options.client, + onPreload: options.onPreload, + }) + if (pooled) return pooled return new LiveQueryObserverImpl( collection ?? null, options.mode === `wholesale`, diff --git a/packages/db/src/live-query-options.ts b/packages/db/src/live-query-options.ts index 987810b30..76529fb96 100644 --- a/packages/db/src/live-query-options.ts +++ b/packages/db/src/live-query-options.ts @@ -1,11 +1,13 @@ import { BaseQueryBuilder } from './query/builder/index.js' import { isCollection } from './live-query-adapter.js' +import { createLiveQueryCollection } from './query/live-query-collection.js' +import { createPooledLiveQuery } from './query/pooled-live-query.js' import { getStableQueryBuilderHash, getStableValueHash, } from './query/ir-stable-identity.js' import { getStringCollationIdentity } from './query/runtime-reference-identity.js' -import type { CollectionImpl } from './collection/index.js' +import type { Collection, CollectionImpl } from './collection/index.js' import type { CollectionOptionsIdentity } from './collection-options.js' import type { CollectionOptions, DbClient } from './client.js' import type { @@ -142,3 +144,36 @@ export function getLiveQueryHash( return getStableValueHash(identity, `queryKey`) } + +/** + * Resolve an adapter's query value to what its observer watches: `null` for a + * disabled query, an existing Collection with sync started, or a live query. + * A query builder whose shape a shared partition can serve gets a pooled view + * instead of its own live-query Collection. + */ +export function resolveLiveQueryValue( + value: unknown, + { gcTime, pool = true }: { gcTime?: number; pool?: boolean } = {}, +): Collection | null { + if (value === undefined || value === null) return null + if (isCollection(value)) { + value.startSyncImmediate() + return value + } + if (value instanceof BaseQueryBuilder) { + return ( + (pool ? createPooledLiveQuery(value) : undefined) ?? + createLiveQueryCollection({ query: value, startSync: true, gcTime }) + ) + } + if (typeof value === `object`) { + return createLiveQueryCollection({ + startSync: true, + gcTime, + ...(value as LiveQueryCollectionConfig), + }) + } + throw new Error( + `A live query must be a QueryBuilder, LiveQueryCollectionConfig, Collection, undefined, or null. Got: ${typeof value}`, + ) +} diff --git a/packages/db/src/query/builder/ref-proxy-identity.ts b/packages/db/src/query/builder/ref-proxy-identity.ts index 57e354fb2..ba13c923b 100644 --- a/packages/db/src/query/builder/ref-proxy-identity.ts +++ b/packages/db/src/query/builder/ref-proxy-identity.ts @@ -1,11 +1,20 @@ import type { RefProxy } from './ref-proxy.js' -const refProxies = new WeakSet() +/** + * Ref proxy traps answer this module-private key, so a user object shaped + * like a ref proxy is never mistaken for one. + */ +export const REF_PROXY_BRAND: unique symbol = Symbol(`refProxy`) -export function registerRefProxy(value: object): void { - refProxies.add(value) +function hasRefProxyBrand(value: object): boolean { + try { + return (value as Record)[REF_PROXY_BRAND] === true + } catch { + // A revoked proxy, such as a finished Immer draft, is not a ref proxy. + return false + } } export function isRefProxy(value: any): value is RefProxy { - return value && typeof value === `object` && refProxies.has(value) + return value && typeof value === `object` && hasRefProxyBrand(value) } diff --git a/packages/db/src/query/builder/ref-proxy.ts b/packages/db/src/query/builder/ref-proxy.ts index 0244cc41b..e6744b151 100644 --- a/packages/db/src/query/builder/ref-proxy.ts +++ b/packages/db/src/query/builder/ref-proxy.ts @@ -1,5 +1,5 @@ import { PropRef, Value, isBasicOrAggregateExpression } from '../ir.js' -import { isRefProxy, registerRefProxy } from './ref-proxy-identity.js' +import { REF_PROXY_BRAND, isRefProxy } from './ref-proxy-identity.js' import { getWrapperExpressionName } from './wrapper-identity.js' import type { BasicExpression } from '../ir.js' import type { IsPlainObject, RefLeaf } from './types.js' @@ -86,6 +86,7 @@ export function createSingleRowRefProxy< if (prop === `__path`) return path if (prop === `__sourceAlias`) return undefined if (prop === `__type`) return undefined // Type is only for TypeScript inference + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) const newPath = [...path, String(prop)] @@ -119,8 +120,6 @@ export function createSingleRowRefProxy< return Reflect.getOwnPropertyDescriptor(target, prop) }, }) - - registerRefProxy(proxy) cache.set(pathKey, proxy) return proxy } @@ -136,25 +135,28 @@ export function createSingleRowRefProxy< export function createRefProxy>( aliases: Array, ): RefProxy & T { - const cache = new Map() + // Each path has one proxy, cached by its parent under the property name. + const aliasProxies = new Map() let accessId = 0 // Monotonic counter to record evaluation order function createProxy(path: Array): any { - const pathKey = JSON.stringify(path) - if (cache.has(pathKey)) { - return cache.get(pathKey) - } - + let children: Map | undefined const proxy = new Proxy({} as any, { get(target, prop, receiver) { if (prop === `__refProxy`) return true if (prop === `__path`) return path if (prop === `__sourceAlias`) return path[0] if (prop === `__type`) return undefined // Type is only for TypeScript inference + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) - const newPath = [...path, String(prop)] - return createProxy(newPath) + children ??= new Map() + let child = children.get(prop) + if (child === undefined) { + child = createProxy([...path, prop]) + children.set(prop, child) + } + return child }, has(target, prop) { @@ -193,9 +195,6 @@ export function createRefProxy>( return Reflect.getOwnPropertyDescriptor(target, prop) }, }) - - registerRefProxy(proxy) - cache.set(pathKey, proxy) return proxy } @@ -206,11 +205,17 @@ export function createRefProxy>( if (prop === `__path`) return [] if (prop === `__sourceAlias`) return undefined if (prop === `__type`) return undefined // Type is only for TypeScript inference + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) const propStr = String(prop) if (aliases.includes(propStr) || aliases.includes(`*`)) { - return createProxy([propStr]) + let proxy = aliasProxies.get(propStr) + if (proxy === undefined) { + proxy = createProxy([propStr]) + aliasProxies.set(propStr, proxy) + } + return proxy } return undefined @@ -247,8 +252,6 @@ export function createRefProxy>( return undefined }, }) - - registerRefProxy(rootProxy) return rootProxy } @@ -283,6 +286,7 @@ export function createRefProxyWithSelected>( if (prop === `__path`) return [`$selected`, ...path] if (prop === `__sourceAlias`) return `$selected` if (prop === `__type`) return undefined + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) const newPath = [...path, String(prop)] @@ -316,8 +320,6 @@ export function createRefProxyWithSelected>( return Reflect.getOwnPropertyDescriptor(target, prop) }, }) - - registerRefProxy(proxy) cache.set(pathKey, proxy) return proxy } @@ -356,7 +358,6 @@ export function createRefProxyWithSelected>( T & { $selected: SingleRowRefProxy } - registerRefProxy(selectedRootProxy) return selectedRootProxy } diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 90cba8ec3..f145a0ef2 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -88,17 +88,18 @@ abstract class BaseExpression { export class CollectionRef extends BaseExpression { public type = `collectionRef` as const - /** Opaque runtime identity; aliases are lexical names only. */ - public readonly sourceId!: string + // Not an own property, so structural identity and hashing ignore it. + readonly #sourceId = `source-${++nextCollectionSourceId}` constructor( public collection: CollectionImpl, public alias: string, ) { super() - Object.defineProperty(this, `sourceId`, { - value: `source-${++nextCollectionSourceId}`, - enumerable: false, - }) + } + + /** Opaque runtime identity; aliases are lexical names only. */ + get sourceId(): string { + return this.#sourceId } } @@ -148,11 +149,9 @@ export class PropRef extends BaseExpression { sourceAlias?: string, ) { super() + // Present only when given, so unqualified refs keep their shape. if (sourceAlias !== undefined) { - Object.defineProperty(this, `sourceAlias`, { - value: sourceAlias, - enumerable: true, - }) + ;(this as { sourceAlias?: string }).sourceAlias = sourceAlias } } } diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts new file mode 100644 index 000000000..61021e964 --- /dev/null +++ b/packages/db/src/query/pooled-live-query.ts @@ -0,0 +1,600 @@ +import { SortedMap } from '../SortedMap.js' +import { normalizeValue } from '../utils/comparison.js' +import { isVirtualPropName } from '../virtual-props.js' +import { getPersistedReadinessSource } from '../persisted-readiness.js' +import { LiveQueryObserverDisposedError } from '../errors.js' +import { getLiveQueryStatusFlags } from '../live-query-adapter.js' +import { getWhereExpression } from './ir.js' +import { createLiveQueryCollection } from './live-query-collection.js' +import type { BasicExpression, QueryIR } from './ir.js' +import type { BaseQueryBuilder } from './builder/index.js' +import type { Collection, CollectionImpl } from '../collection/index.js' +import type { ChangeMessage, CollectionStatus } from '../types.js' +import type { CollectionEventHandler } from '../collection/events.js' +import type { + LiveQueryObserver, + LiveQueryObserverListener, + LiveQuerySnapshot, +} from '../live-query-observer.js' +import type { DehydratedLiveQueryResult } from '../client.js' + +/** + * Live queries that filter one source Collection only by `eq(field, literal)` + * share one partition of that source per filtered field set. Each query reads + * the bucket for its literal tuple, so mounting many queries of one shape + * costs a lookup each instead of a compiled graph and a source subscription. + * + * A bucket holds the rows the partition's source subscription has published, + * keyed by the same normalized equality that `eq` uses: a Date equals its + * timestamp, `NaN` equals `NaN`, `-0` equals `0`, and nullish values match no + * literal. + */ + +type Row = Record +type Listener = (changes: Array>) => void +type StatusListener = CollectionEventHandler<`status:change`> + +interface Bucket { + // Key order, as in a live-query Collection without orderBy. + rows: SortedMap + listeners: Set + revision: number + layoutRevision: number + // Rows as entries, rebuilt after a change. + entries: Array<[string | number, Row]> | undefined +} + +const partitionsBySource = new WeakMap>() + +// Typed so that 1, '1', and true stay distinct. +function equalityKey(value: unknown): string | undefined { + const normalized = normalizeValue(value) + const type = typeof normalized + return type === `string` || type === `number` || type === `boolean` + ? `${type}:${String(normalized)}` + : undefined +} + +function readPath(row: Row, path: Array): unknown { + try { + let value: unknown = row + for (const segment of path) value = (value as Row | undefined)?.[segment] + return value + } catch { + // The full predicate treats a throwing read as false. + return undefined + } +} + +class Partition { + private readonly buckets = new Map() + private subscription: { unsubscribe: () => void } | undefined + private stopStatusEvents: (() => void) | undefined + // One source status listener serves every view of this partition. + readonly statusListeners = new Set() + private listenerCount = 0 + private releaseTimer: ReturnType | undefined + + constructor( + private readonly source: CollectionImpl, + private readonly paths: Array>, + private readonly onEmpty: () => void, + ) {} + + bucketKeyOf(row: Row | undefined): string | undefined { + if (row === undefined) return undefined + const parts: Array = [] + for (const path of this.paths) { + const key = equalityKey(readPath(row, path)) + if (key === undefined) return undefined + parts.push(key) + } + return JSON.stringify(parts) + } + + bucket(key: string): Bucket { + let bucket = this.buckets.get(key) + if (!bucket) { + bucket = { + rows: new SortedMap(), + listeners: new Set(), + revision: 0, + layoutRevision: 0, + entries: undefined, + } + this.buckets.set(key, bucket) + } + return bucket + } + + /** Start the shared source subscription; release it when unused. */ + retain(): void { + if (!this.subscription) { + this.subscription = this.source.subscribeChanges( + (changes) => + this.apply(changes as Array>), + { includeInitialState: true }, + ) + this.stopStatusEvents = this.source.on(`status:change`, (event) => { + for (const listener of [...this.statusListeners]) listener(event) + }) + } + this.scheduleRelease() + } + + listen(bucket: Bucket, listener: Listener): () => void { + this.addListener(bucket, listener) + return () => this.removeListener(bucket, listener) + } + + addListener(bucket: Bucket, listener: Listener): void { + this.retain() + bucket.listeners.add(listener) + this.listenerCount++ + } + + removeListener(bucket: Bucket, listener: Listener): void { + if (!bucket.listeners.delete(listener)) return + this.listenerCount-- + this.scheduleRelease() + } + + private scheduleRelease(): void { + if (this.releaseTimer !== undefined) clearTimeout(this.releaseTimer) + this.releaseTimer = undefined + if (this.listenerCount > 0) return + // A rendered query may subscribe shortly after construction. + this.releaseTimer = setTimeout(() => { + this.releaseTimer = undefined + if (this.listenerCount > 0) return + this.subscription?.unsubscribe() + this.subscription = undefined + this.stopStatusEvents?.() + this.stopStatusEvents = undefined + this.buckets.clear() + this.onEmpty() + }, 1000) + } + + private apply(changes: Array>): void { + const touched = new Map< + Bucket, + Array> + >() + const record = ( + bucket: Bucket, + change: ChangeMessage, + ) => { + const list = touched.get(bucket) + if (list) list.push(change) + else touched.set(bucket, [change]) + if (change.type !== `update`) bucket.layoutRevision++ + } + for (const change of changes) { + const next = + change.type === `delete` ? undefined : this.bucketKeyOf(change.value) + const previous = + change.type === `insert` + ? undefined + : this.bucketKeyOf( + change.type === `delete` ? change.value : change.previousValue, + ) + if (previous !== undefined && previous !== next) { + const bucket = this.bucket(previous) + const old = bucket.rows.get(change.key) + if (bucket.rows.delete(change.key)) { + record(bucket, { type: `delete`, key: change.key, value: old! }) + } + } + if (next !== undefined) { + const bucket = this.bucket(next) + const existed = bucket.rows.has(change.key) + bucket.rows.set(change.key, change.value) + record( + bucket, + existed + ? { ...change, type: `update` } + : { type: `insert`, key: change.key, value: change.value }, + ) + } + } + for (const [bucket, bucketChanges] of touched) { + bucket.revision++ + bucket.entries = undefined + for (const listener of [...bucket.listeners]) listener(bucketChanges) + } + } +} + +type Conjunct = { path: Array; pathKey: string; literalKey: string } + +// Adds `eq(alias.field, literal)` conjuncts to `out`; false for anything else. +function collectConjuncts( + expression: BasicExpression, + alias: string, + out: Array, +): boolean { + if (expression.type !== `func`) return false + const args = expression.args + if (expression.name === `and`) { + for (const arg of args) if (!collectConjuncts(arg, alias, out)) return false + return true + } + if (expression.name !== `eq` || args.length !== 2) return false + const left = args[0]! + const right = args[1]! + const ref = left.type === `ref` ? left : right + const literal = left.type === `val` ? left : right + if (ref.type !== `ref` || literal.type !== `val`) return false + const refPath = ref.path + if ( + refPath[0] !== alias || + refPath.length < 2 || + isVirtualPropName(refPath[1]!) + ) { + return false + } + const literalKey = equalityKey(literal.value) + if (literalKey === undefined) return false + const path = refPath.slice(1) + out.push({ path, pathKey: JSON.stringify(path), literalKey }) + return true +} + +/** + * The equality conjuncts of a query that a partition can serve, or undefined + * when any other clause or operand is present. + */ +function poolableShape( + query: QueryIR, +): + | { paths: Array>; shapeKey: string; bucketKey: string } + | undefined { + if ( + query.from.type !== `collectionRef` || + query.select || + query.join || + query.groupBy || + query.having || + query.orderBy || + query.limit !== undefined || + query.offset !== undefined || + query.distinct || + query.singleResult || + query.fnSelect || + query.fnWhere?.length || + query.fnHaving?.length || + !query.where?.length + ) { + return undefined + } + const conjuncts: Array = [] + for (const where of query.where) { + if ( + !collectConjuncts(getWhereExpression(where), query.from.alias, conjuncts) + ) { + return undefined + } + } + if (conjuncts.length > 1) { + conjuncts.sort((a, b) => (a.pathKey < b.pathKey ? -1 : 1)) + } + return { + paths: conjuncts.map(({ path }) => path), + shapeKey: conjuncts.map(({ pathKey }) => pathKey).join(`,`), + bucketKey: JSON.stringify(conjuncts.map(({ literalKey }) => literalKey)), + } +} + +/** + * One query's view of its bucket. It answers the calls the live-query observer + * makes; any other Collection member builds the query's live-query Collection + * once and forwards to it, so `result.collection` keeps its full API. + */ +class PooledLiveQuery { + readonly isLoadingSubset = false + // No persisted readiness, single-result config, or layout channel. + readonly config = undefined + readonly _subscribeLayoutChanges = undefined + private readonly bucket: Bucket + private collection: Collection | undefined = undefined + + constructor( + private readonly source: CollectionImpl, + private readonly query: BaseQueryBuilder, + private readonly partition: Partition, + bucketKey: string, + ) { + this.bucket = partition.bucket(bucketKey) + partition.retain() + } + + get status(): CollectionStatus { + return this.source.status + } + + get _stateRevision(): number { + return this.bucket.revision + } + + get _layoutRevision(): number { + return this.bucket.layoutRevision + } + + entries(): Array<[string | number, Row]> { + return (this.bucket.entries ??= [...this.bucket.rows.entries()]) + } + + subscribeChanges( + callback: Listener, + options: { includeInitialState?: boolean } = {}, + ): { unsubscribe: () => void } { + const unsubscribe = this.partition.listen(this.bucket, callback) + if (options.includeInitialState) { + callback( + [...this.bucket.rows].map(([key, value]) => ({ + type: `insert`, + key, + value, + })), + ) + } + return { unsubscribe } + } + + on(...args: Parameters): () => void { + const [event, listener] = args + if (event !== `status:change`) return this.source.on(...args) + const listeners = this.partition.statusListeners + listeners.add(listener as StatusListener) + return () => listeners.delete(listener as StatusListener) + } + + preload(): Promise { + return this.source.preload() + } + + /** Observe changes and status without allocating unsubscribe closures. */ + watch(onChanges: Listener, onStatus: StatusListener): void { + this.partition.addListener(this.bucket, onChanges) + this.partition.statusListeners.add(onStatus) + } + + unwatch(onChanges: Listener, onStatus: StatusListener): void { + this.partition.removeListener(this.bucket, onChanges) + this.partition.statusListeners.delete(onStatus) + } + + cleanup(): Promise { + return this.collection?.cleanup() ?? Promise.resolve() + } + + private proxy: Collection | undefined = undefined + + /** This view as the Collection it stands in for. */ + get publicCollection(): Collection { + return (this.proxy ??= new Proxy( + this, + forwardToCollection, + ) as unknown as Collection) + } + + materialize(): Collection { + return (this.collection ??= createLiveQueryCollection({ + query: this.query, + startSync: true, + })) + } +} + +/** + * The wholesale observer for a pooled view without a `DbClient`. Pooled views + * have no hydration, persisted restore, or single-result mode, so this keeps + * only the snapshot, subscription, preload, and disposal parts of the general + * observer contract. + */ +class PooledWholesaleObserver implements LiveQueryObserver< + Row, + string | number +> { + private snapshot: LiveQuerySnapshot | undefined + private snapshotRevision = -1 + private snapshotStatus: CollectionStatus | undefined + private layoutKeys: Array = [] + private layoutRevision = 0 + private readonly records = new Set<{ + listener: LiveQueryObserverListener + }>() + private watching = false + private readonly onChanges: Listener = (changes) => this.deliver(changes) + private readonly onStatus: StatusListener = () => this.deliver(undefined) + private preloadPromise: Promise | undefined + private disposed = false + + constructor( + private readonly view: PooledLiveQuery, + private readonly onPreload: (() => void) | undefined, + ) {} + + getSnapshot(): LiveQuerySnapshot { + const status = this.view.status + if ( + this.snapshot && + this.snapshotRevision === this.view._stateRevision && + this.snapshotStatus === status + ) { + return this.snapshot + } + const entries = this.view.entries() + const keys = entries.map(([key]) => key) + if ( + keys.length !== this.layoutKeys.length || + keys.some((key, index) => key !== this.layoutKeys[index]) + ) { + this.layoutKeys = keys + this.layoutRevision++ + } + this.snapshotRevision = this.view._stateRevision + this.snapshotStatus = status + return (this.snapshot = { + state: new Map(entries), + data: entries.map(([, value]) => value), + collection: this.view.publicCollection, + layoutRevision: this.layoutRevision, + status, + ...getLiveQueryStatusFlags(status), + persistedStatus: `unavailable`, + isPersistedReady: false, + persistedError: undefined, + isEnabled: true, + }) + } + + getServerSnapshot(): LiveQuerySnapshot { + return this.getSnapshot() + } + + subscribe( + listener: LiveQueryObserverListener, + ): () => void { + if (this.disposed) throw new LiveQueryObserverDisposedError() + // A record per call, so one listener subscribed twice tears down twice. + const record = { listener } + this.records.add(record) + if (!this.watching) { + this.watching = true + this.view.watch(this.onChanges, this.onStatus) + } + return () => { + if (this.records.delete(record) && this.records.size === 0) { + this.stopWatching() + } + } + } + + private deliver( + changes: Array> | undefined, + ): void { + const records = this.records.size === 1 ? this.records : [...this.records] + for (const { listener } of records) listener(changes) + } + + private stopWatching(): void { + if (!this.watching) return + this.watching = false + this.view.unwatch(this.onChanges, this.onStatus) + } + + preload(): Promise { + if (this.preloadPromise) return this.preloadPromise + this.onPreload?.() + const promise = this.view.preload() + this.preloadPromise = promise + const clear = () => { + if (this.preloadPromise === promise) this.preloadPromise = undefined + } + void promise.then(clear, clear) + return promise + } + + preloadForInitialRender(): Promise { + if (this.disposed) { + return Promise.reject(new LiveQueryObserverDisposedError()) + } + return this.preload() + } + + isInitialRenderReady(): boolean { + return false + } + + getError(): unknown { + return undefined + } + + dehydrate(): DehydratedLiveQueryResult { + return { + rows: this.view.entries().map(([key, value]) => ({ key, value })), + } + } + + dispose(): void { + if (this.disposed) return + this.disposed = true + this.records.clear() + this.stopWatching() + } +} + +/** A lean observer for a pooled view, or undefined for anything else. */ +export function createPooledObserver< + T extends object, + TKey extends string | number, +>( + collection: unknown, + { + wholesale, + client, + onPreload, + }: { + wholesale: boolean + client: unknown + onPreload: (() => void) | undefined + }, +): LiveQueryObserver | undefined { + if (!(collection instanceof PooledLiveQuery) || !wholesale || client) { + return undefined + } + return new PooledWholesaleObserver( + collection, + onPreload, + ) as unknown as LiveQueryObserver +} + +const forwardToCollection: ProxyHandler = { + get(view, property) { + if (property in view) return Reflect.get(view, property, view) + const collection = view.materialize() + const value = Reflect.get(collection, property, collection) + return typeof value === `function` ? value.bind(collection) : value + }, +} + +/** + * A pooled view for a query a partition can serve, or undefined. The view is + * typed as the Collection it stands in for. + */ +export function createPooledLiveQuery( + query: BaseQueryBuilder, +): Collection | undefined { + const ir = query._getQuery() + const shape = poolableShape(ir) + if (!shape || ir.from.type !== `collectionRef`) return undefined + const source = ir.from.collection + // Persisted restore and on-demand loading need the live-query Collection. + if ( + source.config.syncMode === `on-demand` || + getPersistedReadinessSource(source.config) + ) { + return undefined + } + let partitions = partitionsBySource.get(source) + if (!partitions) { + partitions = new Map() + partitionsBySource.set(source, partitions) + } + const { shapeKey } = shape + let partition = partitions.get(shapeKey) + if (!partition) { + const owner = partitions + partition = new Partition(source, shape.paths, () => owner.delete(shapeKey)) + partitions.set(shapeKey, partition) + } + // Observers read the view directly; users get its `publicCollection`. + return new PooledLiveQuery( + source, + query, + partition, + shape.bucketKey, + ) as unknown as Collection +} diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 9b51b818b..0f5bfc147 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -48,6 +48,7 @@ const staticOracleProperties = [ `query-identity.equality-partition`, `where-predicate.publication`, `join-result-key.pairs`, + `pooled-live-query.publication`, `derived-publication.membership-work`, `collection-publication.metadata-cancellation`, `collection-publication.metadata-only`, diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts new file mode 100644 index 000000000..514d310e9 --- /dev/null +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -0,0 +1,506 @@ +/** + * # Does a pooled live query publish what its live-query Collection would? + * + * Law and source: a live query that filters one source Collection only by + * `eq(field, literal)` conjuncts is served from a partition of that source + * shared by every query with the same fields. Its observer must publish the + * rows the live-query Collection for the same query publishes: the visible + * source rows whose fields equal the literals under `eq` semantics + * (`src/query/compiler/evaluators.ts`: nullish is UNKNOWN, a Date equals its + * timestamp, `NaN` equals `NaN`, `-0` equals `0`, other types differ), in + * key order, with the same row values and status. + * + * Why an example can miss the failure: one query over static rows passes even + * if rows never move between buckets, a peer bucket never sees a row leave, a + * remounted query reads a stale bucket, or a rollback leaves an optimistic row + * behind. + * + * Model: `expectedKeys` filters the model's visible rows with an independent + * `eq` over plain values. It does not import the evaluator, normalization, or + * the partition. Order, row values, and status come from a second + * formulation: a live-query Collection compiled for the same query. + * + * History grammar: rows have a field `f` from strings, numbers and their + * look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, `0`, `null`, and a + * missing value, and a field `g` of `x` or `y`. Up to three peer queries use + * `eq(f, literal)`, optionally with `eq(g, literal)`. Steps commit sync + * transactions of one or two inserts, updates, or deletes; apply one + * optimistic insert, update, or delete and then confirm or roll it back; or + * mount or unmount a peer. + * + * Production driver: `createPooledLiveQuery` builds each peer's view from the + * query builder's IR, and `createLiveQueryObserver` observes it in wholesale + * and granular mode, as the framework adapters do. + * + * Refinement check: after every step, each mounted peer's wholesale snapshot + * equals its live-query Collection's keys in order, row values, and status; + * both observers' key sets equal the model. + * + * Calibration: a partition that ignored the previous value, kept bucket rows + * in arrival order, or compared literals without normalization fails the + * pinned histories and both campaigns. + * + * Known omissions: on-demand and persisted sources, `DbClient` hydration, + * Suspense, `select`, and every other clause keep the live-query Collection + * and are outside this owner. + */ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createLiveQueryObserver } from '../../src/live-query-observer.js' +import { Query } from '../../src/query/builder/index.js' +import { and, createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' +import { + oraclePropertyOptions, + oracleRuns, + readOracleRunConfig, +} from '../oracle-config.js' +import { + flushPromises, + mockSyncCollectionOptions, + withExpectedRejection, + withOracleCleanup, +} from '../utils.js' +import type { ChangeMessage } from '../../src/types.js' + +const property = `pooled-live-query.publication` +const requestedReplayProperty = readOracleRunConfig().replayProperty + +const MISSING = Symbol(`missing`) +type FieldValue = string | number | boolean | Date | null | typeof MISSING +type Row = { id: string; f?: unknown; g: string } +type Peer = { f: string | number | boolean; g?: string } +type Step = + | { kind: `sync`; ops: Array } + | { kind: `optimistic`; op: Op; confirm: boolean } + | { kind: `mount`; peer: number } + | { kind: `unmount`; peer: number } +type Op = + | { type: `insert`; id: number; f: FieldValue; g: string } + | { type: `update`; id: number; f: FieldValue; g: string } + | { type: `delete`; id: number } +type History = { + rows: Array<{ f: FieldValue; g: string }> + peers: Array + steps: Array +} + +const DATE_ONE = new Date(1) +const fieldValues: ReadonlyArray = [ + `a`, + `b`, + 1, + `1`, + true, + DATE_ONE, + Number.NaN, + -0, + 0, + null, + MISSING, +] +const literals: ReadonlyArray = [`a`, 1, `1`, true, Number.NaN, 0] + +// --------------------------------------------------------------------------- +// Model +// --------------------------------------------------------------------------- + +// `eq` keeps a row only when TRUE: nullish is UNKNOWN, a Date is its +// timestamp, NaN equals NaN, and -0 equals 0. +function modelEq(value: unknown, literal: unknown): boolean { + if (value === null || value === undefined) return false + const left = value instanceof Date ? value.getTime() : value + if (typeof left === `number` && typeof literal === `number`) { + return (Number.isNaN(left) && Number.isNaN(literal)) || left === literal + } + return typeof left === typeof literal && left === literal +} + +function expectedKeys( + rows: ReadonlyMap, + peer: Peer, +): Array { + return [...rows.values()] + .filter( + (row) => + modelEq(row.f, peer.f) && (peer.g === undefined || row.g === peer.g), + ) + .map((row) => row.id) + .sort() +} + +// Collection change detection treats 0 and -0, and NaN and NaN, as equal. +function sameValueZero(a: unknown, b: unknown): boolean { + return a === b || (Number.isNaN(a) && Number.isNaN(b)) +} + +function sourceRow(id: string, f: FieldValue, g: string): Row { + return f === MISSING ? { id, g } : { id, f, g } +} + +// --------------------------------------------------------------------------- +// History grammar +// --------------------------------------------------------------------------- + +const fieldArbitrary = fc.constantFrom(...fieldValues) +const gArbitrary = fc.constantFrom(`x`, `y`) +const opArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + type: fc.constant(`insert` as const), + id: fc.nat({ max: 5 }), + f: fieldArbitrary, + g: gArbitrary, + }), + { + weight: 2, + arbitrary: fc.record({ + type: fc.constant(`update` as const), + id: fc.nat({ max: 5 }), + f: fieldArbitrary, + g: gArbitrary, + }), + }, + fc.record({ type: fc.constant(`delete` as const), id: fc.nat({ max: 5 }) }), +) +const peerArbitrary: fc.Arbitrary = fc.record( + { f: fc.constantFrom(...literals), g: gArbitrary }, + { requiredKeys: [`f`] }, +) +const historyArbitrary: fc.Arbitrary = fc.record({ + rows: fc.array(fc.record({ f: fieldArbitrary, g: gArbitrary }), { + maxLength: 4, + }), + peers: fc.array(peerArbitrary, { minLength: 1, maxLength: 3 }), + steps: fc.array( + fc.oneof( + { + weight: 3, + arbitrary: fc.record({ + kind: fc.constant(`sync` as const), + ops: fc.array(opArbitrary, { minLength: 1, maxLength: 2 }), + }), + }, + fc.record({ + kind: fc.constant(`optimistic` as const), + op: opArbitrary, + confirm: fc.boolean(), + }), + fc.record({ + kind: fc.constant(`mount` as const), + peer: fc.nat({ max: 2 }), + }), + fc.record({ + kind: fc.constant(`unmount` as const), + peer: fc.nat({ max: 2 }), + }), + ), + { maxLength: 8 }, + ), +}) + +// Each pinned history moves a row across buckets a short example would keep +// still. +const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ + { + name: `a Date row joins the bucket of its timestamp`, + history: { + rows: [{ f: `a`, g: `x` }], + peers: [{ f: 1 }, { f: `a` }], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: DATE_ONE, g: `x` }] }, + ], + }, + }, + { + name: `rows keep key order across strings and numbers`, + history: { + rows: [ + { f: `a`, g: `x` }, + { f: `a`, g: `x` }, + ], + peers: [{ f: `a` }], + steps: [ + { kind: `sync`, ops: [{ type: `insert`, id: 4, f: `a`, g: `x` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `b`, g: `x` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `a`, g: `x` }] }, + ], + }, + }, + { + name: `a rolled-back optimistic insert leaves its bucket`, + history: { + rows: [], + peers: [{ f: true, g: `y` }], + steps: [ + { + kind: `optimistic`, + op: { type: `insert`, id: 2, f: true, g: `y` }, + confirm: false, + }, + ], + }, + }, + { + name: `a remounted peer reads a bucket that changed while it was away`, + history: { + rows: [{ f: 0, g: `x` }], + peers: [{ f: 0 }, { f: Number.NaN }], + steps: [ + { kind: `unmount`, peer: 1 }, + { + kind: `sync`, + ops: [{ type: `update`, id: 0, f: Number.NaN, g: `x` }], + }, + { kind: `mount`, peer: 1 }, + { kind: `sync`, ops: [{ type: `update`, id: 0, f: -0, g: `x` }] }, + ], + }, + }, +] + +// --------------------------------------------------------------------------- +// Production driver and refinement check +// --------------------------------------------------------------------------- + +let serial = 0 +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) + +function peerQuery(source: any, peer: Peer) { + return (q: any) => + q + .from({ r: source }) + .where(({ r }: any) => + peer.g === undefined + ? eq(r.f, peer.f) + : and(eq(r.f, peer.f), eq(r.g, peer.g)), + ) +} + +function describeRow(row: Record) { + const f = row.f instanceof Date ? `Date(${row.f.getTime()})` : String(row.f) + return `${String(row.id)}:${typeof row.f}:${f}:${String(row.g)}:${String(row.$synced)}` +} + +async function runHistory(history: History): Promise { + const id = (n: number) => `r${n}` + const rows = new Map() + history.rows.forEach((row, n) => + rows.set(id(n), sourceRow(id(n), row.f, row.g)), + ) + const source = createCollection( + mockSyncCollectionOptions({ + id: `pooled-${serial++}`, + getKey: (row) => row.id, + initialData: [...rows.values()].map((row) => ({ ...row })), + }), + ) + await source.stateWhenReady() + const references = history.peers.map((peer) => + createLiveQueryCollection(peerQuery(source, peer)), + ) + type Mounted = { + view: { collection?: unknown } + layout: { keys: string; revision: number } | undefined + wholesale: ReturnType> + granular: ReturnType> + granularKeys: Set + unsubscribe: () => void + } + const mounted = new Map() + const mount = (index: number) => { + const peer = history.peers[index] + if (!peer || mounted.has(index)) return + const view = createPooledLiveQuery(peerQuery(source, peer)(new Query())) + expect(view, `peer ${index} is poolable`).toBeDefined() + const wholesale = createLiveQueryObserver(view as any, { + mode: `wholesale`, + }) + const granular = createLiveQueryObserver(view as any) + const granularKeys = new Set() + const offWholesale = wholesale.subscribe(() => {}) + const offGranular = granular.subscribe((changes) => { + for (const change of (changes ?? []) as Array>) { + if (change.type === `delete`) granularKeys.delete(change.key) + else granularKeys.add(change.key) + } + }) + mounted.set(index, { + view: view as unknown as Mounted[`view`], + layout: undefined, + wholesale, + granular, + granularKeys, + unsubscribe: () => { + offWholesale() + offGranular() + wholesale.dispose() + granular.dispose() + }, + }) + } + const check = (checkpoint: string) => { + for (const [index, entry] of mounted) { + const { view, wholesale, granularKeys } = entry + const peer = history.peers[index]! + const snapshot = wholesale.getSnapshot() + const reference = references[index]! + const label = `${checkpoint}, peer ${index}` + expect( + (snapshot.data as Array>).map(describeRow), + `${label} rows`, + ).toEqual(reference.toArray.map((row) => describeRow(row))) + expect(snapshot.status, `${label} status`).toBe(reference.status) + const model = expectedKeys(rows, peer) + expect( + [...snapshot.state!.keys()].map(String).sort(), + `${label} model`, + ).toEqual(model) + expect([...granularKeys].map(String).sort(), `${label} granular`).toEqual( + model, + ) + // A change in the ordered keys always advances the layout revision. + const keys = JSON.stringify([...snapshot.state!.keys()]) + if (entry.layout && entry.layout.keys !== keys) { + expect(snapshot.layoutRevision, `${label} layout`).toBeGreaterThan( + entry.layout.revision, + ) + } + entry.layout = { keys, revision: snapshot.layoutRevision } + // Observing a pooled view never builds its live-query Collection. + expect(view.collection, `${label} materialized`).toBeUndefined() + } + } + const applyOp = (op: Op): boolean => { + const key = id(op.id) + if (op.type === `insert`) { + if (rows.has(key)) return false + rows.set(key, sourceRow(key, op.f, op.g)) + } else if (op.type === `update`) { + if (!rows.has(key)) return false + rows.set(key, sourceRow(key, op.f, op.g)) + } else { + if (!rows.delete(key)) return false + } + return true + } + const write = (op: Op) => { + const key = id(op.id) + source.utils.write( + op.type === `delete` + ? { type: `delete`, key } + : op.type === `update` && op.f === MISSING + ? // A synced update merges fields, so clear `f` explicitly. + { type: `update`, value: { id: key, f: undefined, g: op.g } } + : { type: op.type, value: sourceRow(key, op.f, op.g) }, + ) + } + + await withOracleCleanup(async () => { + await Promise.all(references.map((reference) => reference.preload())) + history.peers.forEach((_, index) => mount(index)) + check(`after mount`) + for (const [n, step] of history.steps.entries()) { + const checkpoint = `after step ${n} (${step.kind})` + if (step.kind === `mount`) mount(step.peer) + else if (step.kind === `unmount`) { + mounted.get(step.peer)?.unsubscribe() + mounted.delete(step.peer) + } else if (step.kind === `sync`) { + const accepted = step.ops.filter((op) => { + const before = new Map(rows) + if (applyOp(op)) return true + rows.clear() + for (const [k, v] of before) rows.set(k, v) + return false + }) + if (accepted.length === 0) continue + source.utils.begin() + for (const op of accepted) write(op) + source.utils.commit() + } else { + const { op, confirm } = step + const key = id(op.id) + const before = new Map(rows) + const previous = rows.get(key) + // An update to the same value creates no pending transaction. + if ( + op.type === `update` && + previous !== undefined && + `f` in previous === (op.f !== MISSING) && + sameValueZero(previous.f, op.f === MISSING ? undefined : op.f) && + previous.g === op.g + ) { + continue + } + if (!applyOp(op)) continue + const transaction = + op.type === `insert` + ? source.insert(sourceRow(key, op.f, op.g)) + : op.type === `update` + ? source.update(key, (draft) => { + if (op.f === MISSING) delete draft.f + else draft.f = op.f + draft.g = op.g + }) + : source.delete(key) + const persisted = transaction.isPersisted.promise.catch(() => undefined) + check(`${checkpoint} pending`) + if (confirm) { + source.utils.begin() + write(op) + source.utils.commit() + source.utils.resolveSync() + } else { + rows.clear() + for (const [k, v] of before) rows.set(k, v) + await withExpectedRejection(`rolled back`, async () => { + source.utils.rejectSync(new Error(`rolled back`)) + await persisted + await flushPromises() + }) + } + await tick() + } + check(checkpoint) + } + // Any other Collection member builds the live-query Collection. + for (const [index, { wholesale }] of mounted) { + const collection = wholesale.getSnapshot().collection! + expect( + (collection.toArray as Array>).map(describeRow), + `peer ${index} forwarded toArray`, + ).toEqual(references[index]!.toArray.map((row) => describeRow(row))) + } + }, [ + () => { + for (const entry of mounted.values()) entry.unsubscribe() + }, + () => Promise.all(references.map((reference) => reference.cleanup())), + () => source.cleanup(), + ]) +} + +describe(`pooled live query oracle`, () => { + if (requestedReplayProperty === undefined) { + for (const { name, history } of pinnedHistories) { + it(`matches the live-query Collection when ${name}`, () => + runHistory(history)) + } + fcTest.prop([historyArbitrary], { + seed: 44_502_001, + numRuns: oracleRuns(80), + })(`matches the live-query Collection (fixed)`, runHistory) + fcTest.prop([historyArbitrary], oraclePropertyOptions(80, property))( + `matches the live-query Collection (random)`, + runHistory, + ) + } else if (requestedReplayProperty === property) { + fcTest.prop([historyArbitrary], oraclePropertyOptions(80, property))( + `matches the live-query Collection (replay)`, + runHistory, + ) + } else { + it.skip(`runs only when its replay property is selected`, () => {}) + } +}) diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index acd676cd2..aaea5e304 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -5,13 +5,13 @@ import { BaseQueryBuilder, IR, UnhashableQueryIRError, - createLiveQueryCollection, createLiveQueryObserver, deepEquals, getPreparedLiveQueryIdentity, getStableValueHash, isCollection, prepareLiveQueryValue, + resolveLiveQueryValue, } from '@tanstack/db' import { useOptionalDbClient } from './DbProvider' import { setLiveQueryResultInfo } from './live-query-internals' @@ -336,40 +336,6 @@ export function warnUnhashableDerivedIdentity( ) } -function createCollectionFromPreparedQuery( - value: unknown, - defaultGcTime = DEFAULT_GC_TIME_MS, -) { - if (value === undefined || value === null) { - return null - } - - if (isCollection(value)) { - value.startSyncImmediate() - return value - } - - if (value instanceof BaseQueryBuilder) { - return createLiveQueryCollection({ - query: value, - startSync: true, - gcTime: defaultGcTime, - }) - } - - if (typeof value === `object`) { - return createLiveQueryCollection({ - startSync: true, - gcTime: defaultGcTime, - ...(value as LiveQueryCollectionConfig), - }) - } - - throw new Error( - `useLiveQuery callback must return a QueryBuilder, LiveQueryCollectionConfig, Collection, undefined, or null. Got: ${typeof value}`, - ) -} - /** * Create a live query using a query function. * @param queryFn - Query function that defines what data to fetch @@ -825,22 +791,29 @@ function useLiveQueryImpl( streamIdentity = [`queryKey`, queryKey] } else if (deps !== undefined) { identityDeps = resolvedDeps - try { - preparedQueryValue = prepareQueryValue( - configOrQueryOrCollection, - dbClient, - deferredCollectionsRef.current, - ) - streamIdentity = [ - `deps`, - resolvedDeps, - getPreparedLiveQueryIdentity(preparedQueryValue), - ] - } catch (error) { - if (!(error instanceof UnhashableQueryIRError)) throw error - warnUnhashableDerivedIdentity(error) - identityError = error - } + // Deps decide reuse. Only hydration and Suspense read the query hash; the + // development warning about unhashable queries still derives it. + if ( + dbClient || + forSuspense || + shouldWarnInDevelopment(`TANSTACK_DB_DISABLE_QUERY_IDENTITY_WARNINGS`) + ) + try { + preparedQueryValue = prepareQueryValue( + configOrQueryOrCollection, + dbClient, + deferredCollectionsRef.current, + ) + streamIdentity = [ + `deps`, + resolvedDeps, + getPreparedLiveQueryIdentity(preparedQueryValue), + ] + } catch (error) { + if (!(error instanceof UnhashableQueryIRError)) throw error + warnUnhashableDerivedIdentity(error) + identityError = error + } } else if (inputIsCollection) { identityDeps = [] streamIdentity = [`collection`, configOrQueryOrCollection.id] @@ -976,10 +949,13 @@ function useLiveQueryImpl( deferredCollectionsRef.current, ) } - collectionRef.current = createCollectionFromPreparedQuery( - preparedQueryValue, - forSuspense ? DEFAULT_SUSPENSE_GC_TIME_MS : DEFAULT_GC_TIME_MS, - ) as SuspenseCollection | null + collectionRef.current = resolveLiveQueryValue(preparedQueryValue, { + gcTime: forSuspense + ? DEFAULT_SUSPENSE_GC_TIME_MS + : DEFAULT_GC_TIME_MS, + // Hydration and Suspense key the live-query Collection by identity. + pool: !forSuspense && !dbClient, + }) as SuspenseCollection | null if (suspenseCollections && suspenseKey && collectionRef.current) { const collection = collectionRef.current const removeCleanupListener = collection.on(`status:cleaned-up`, () => From 4a386bb3431b0f192f8e2466d7c22f375d958695 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 08:50:44 -0600 Subject: [PATCH 02/63] perf(db): faster local-only direct writes and cheaper drafts (WIP) - A local-only Collection without a handler for an operation type writes direct mutations as synced rows and returns a completed transaction, unless another transaction on the Collection is pending or persisting. - Draft change tracking: the root draft compares against the stored row instead of a clone, assigned keys live in a Map instead of a dictionary-mode object, draft handles resolve through a trap-answered brand instead of a global WeakMap, clone bookkeeping uses Maps, and plain objects skip the special-type checks when cloned. - Mutation ids are a per-runtime random prefix plus a counter, so a mutation no longer costs a random UUID while ids stay unique across tabs and sessions. Co-authored-by: Isaac --- packages/db/src/collection/mutations.ts | 57 ++++++++- packages/db/src/collection/state.ts | 12 ++ packages/db/src/local-only.ts | 17 ++- packages/db/src/proxy.ts | 163 ++++++++++++------------ 4 files changed, 162 insertions(+), 87 deletions(-) diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 5e9c1fa83..0d1d9c7f5 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -25,6 +25,7 @@ import type { CollectionConfig, InsertConfig, OperationConfig, + OperationType, PendingMutation, StandardSchema, TransactionConfig, @@ -37,6 +38,14 @@ import type { TransactionScope } from '../transactions' import type { CollectionLifecycleManager } from './lifecycle' import type { CollectionStateManager } from './state' +// One random prefix per runtime keeps mutation ids unique across tabs and +// sessions; the counter avoids generating a random UUID per mutation. +const mutationIdPrefix = safeRandomUUID() +let mutationCount = 0 +function createMutationId(): string { + return `${mutationIdPrefix}-${++mutationCount}` +} + export class CollectionMutationsManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -187,6 +196,40 @@ export class CollectionMutationsManager< } } + /** + * A local-only Collection confirms its own writes. Without a user handler + * for this operation type, and with no other transaction unsettled, write + * the mutations as synced rows and return a completed transaction instead + * of publishing an optimistic overlay and confirming it a tick later. + */ + private commitLocalOnlyDirect( + mutations: Array>, + type: OperationType, + ): TransactionType | undefined { + const direct = this.state.localOnlyDirectWrite + if (!direct?.types.has(type)) return undefined + for (const transaction of this.state.transactions.values()) { + // A persisting transaction holds sync commits, and a pending one + // overlays them. + if ( + transaction.state === `pending` || + transaction.state === `persisting` + ) { + return undefined + } + } + const transaction = this.createTransaction({ + autoCommit: false, + metadata: { [DIRECT_TRANSACTION_METADATA_KEY]: true }, + mutationFn: () => Promise.resolve(), + }) + transaction.applyMutations(mutations) + direct.write(mutations) + transaction.setState(`completed`) + transaction.isPersisted.resolve(transaction) + return transaction + } + /** * Inserts one or more items into the collection */ @@ -218,7 +261,7 @@ export class CollectionMutationsManager< const globalKey = this.generateGlobalKey(key, item) const mutation: PendingMutation = { - mutationId: safeRandomUUID(), + mutationId: createMutationId(), original: {}, modified: validatedData, // Pick the values from validatedData based on what's passed in - this is for cases @@ -262,6 +305,8 @@ export class CollectionMutationsManager< return ambientTransaction } else { + const localOnly = this.commitLocalOnlyDirect(mutations, `insert`) + if (localOnly) return localOnly // Create a new transaction with a mutation function that calls the onInsert handler const directOpTransaction = this.createTransaction({ metadata: { [DIRECT_TRANSACTION_METADATA_KEY]: true }, @@ -403,7 +448,7 @@ export class CollectionMutationsManager< const globalKey = this.generateGlobalKey(modifiedItemId, modifiedItem) return { - mutationId: safeRandomUUID(), + mutationId: createMutationId(), original: originalItem, modified: modifiedItem, // Pick the values from modifiedItem based on what's passed in - this is for cases @@ -463,6 +508,9 @@ export class CollectionMutationsManager< // No need to check for onUpdate handler here as we've already checked at the beginning + const localOnly = this.commitLocalOnlyDirect(mutations, `update`) + if (localOnly) return localOnly + // Create a new transaction with a mutation function that calls the onUpdate handler const directOpTransaction = this.createTransaction({ metadata: { [DIRECT_TRANSACTION_METADATA_KEY]: true }, @@ -540,7 +588,7 @@ export class CollectionMutationsManager< `delete`, CollectionImpl > = { - mutationId: safeRandomUUID(), + mutationId: createMutationId(), original: this.state.get(key)!, modified: this.state.get(key)!, changes: this.state.get(key)!, @@ -572,6 +620,9 @@ export class CollectionMutationsManager< return ambientTransaction } + const localOnly = this.commitLocalOnlyDirect(mutations, `delete`) + if (localOnly) return localOnly + // Create a new transaction with a mutation function that calls the onDelete handler const directOpTransaction = this.createTransaction({ autoCommit: true, diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 2caa91d33..0679bac16 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -13,6 +13,7 @@ import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessage, CollectionConfig, + OperationType, OptimisticChangeMessage, PendingMutation, } from '../types' @@ -191,6 +192,16 @@ export class CollectionStateManager< private isDrainingSyncTransactions = false private syncRunGeneration = 0 public isLocalOnly = false + /** + * Set by a local-only Collection for operation types without a user handler. + * Their direct mutations can be written as synced rows at once. + */ + public localOnlyDirectWrite: + | { + types: ReadonlySet + write: (mutations: Array>) => void + } + | undefined /** * Creates a new CollectionState manager @@ -2137,6 +2148,7 @@ export class CollectionStateManager< this.hasAppliedAdapterTruncate = false this.clearOriginTrackingState() this.isLocalOnly = false + this.localOnlyDirectWrite = undefined this.size = 0 this.pendingSyncedTransactions = [] this.pendingSyncedProjection = { states: new Map(), truncated: false } diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 9de92f4d6..923586db0 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -187,7 +187,11 @@ export function localOnlyCollectionOptions< const collectionId = id ?? safeRandomUUID() // Create the sync configuration with transaction confirmation capability - const syncResult = createLocalOnlySync(initialData) + const directTypes = new Set() + if (!onInsert) directTypes.add(`insert`) + if (!onUpdate) directTypes.add(`update`) + if (!onDelete) directTypes.add(`delete`) + const syncResult = createLocalOnlySync(initialData, directTypes) /** * Create wrapper handlers that call user handlers first, then confirm transactions @@ -304,7 +308,9 @@ export function localOnlyCollectionOptions< * @returns Object with sync configuration and confirmOperationsSync function */ function createLocalOnlySync( - initialData?: Array, + initialData: Array | undefined, + // Operation types without a user handler, which confirm synchronously. + directTypes: ReadonlySet, ) { // Capture sync functions and collection for transaction confirmation let syncBegin: (() => void) | null = null @@ -329,6 +335,13 @@ function createLocalOnlySync( syncCommit = commit collection = params.collection params.collection._state.isLocalOnly = true + if (directTypes.size > 0) { + params.collection._state.localOnlyDirectWrite = { + types: directTypes, + write: (mutations) => + confirmOperationsSync(mutations), + } + } // Apply initial data if provided if (initialData && initialData.length > 0) { diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 48bd0c84b..38942ab32 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -5,8 +5,9 @@ import { deepEquals, isTemporal } from './utils' -// Resolve draft handles before calling native Map/Set membership methods. -const draftCopies = new WeakMap() +// A draft proxy's get trap answers this key with its private copy, so draft +// handles resolve before native Map/Set membership methods. +const DRAFT_COPY: unique symbol = Symbol(`draftCopy`) function defineDataProperty( object: object, key: PropertyKey, @@ -25,9 +26,13 @@ function defineDataProperty( } function unwrapDraft(value: unknown): unknown { - return value !== null && typeof value === `object` - ? (draftCopies.get(value) ?? value) - : value + if (value === null || typeof value !== `object`) return value + try { + return (value as Record)[DRAFT_COPY] ?? value + } catch { + // A revoked proxy is not a draft. + return value + } } /** @@ -235,11 +240,14 @@ interface ChangeParent { } interface ChangeTracker { - valueCopies: WeakMap + // Shared by one update's drafts and dropped with them. + valueCopies: Map originalObject: T modified: boolean copy_: T - assigned_: Record + // A Map, not a null-prototype object: those start in dictionary mode, + // where reading their keys is slow. + assigned_: Map parent?: ChangeParent target: T } @@ -250,7 +258,8 @@ interface ChangeTracker { function deepClone( obj: T, - visited = new WeakMap(), + // Lives only for this clone, so a Map avoids weak-reference bookkeeping. + visited = new Map(), detach = false, ): T { // A draft handle and its underlying copy must share one cycle identity. @@ -270,6 +279,12 @@ function deepClone( return visited.get(obj as object) as T } + // Plain objects, the common case, skip the special-type checks below. + const prototype = Object.getPrototypeOf(obj) + if (prototype === Object.prototype || prototype === null) { + return clonePlainObject(obj, visited, detach) + } + if (obj instanceof Date) { const clone = new Date(obj.getTime()) visited.set(obj, clone) @@ -342,25 +357,28 @@ function deepClone( // Arbitrary instances may carry private/native state we cannot reconstruct. // Keep them by reference at publication, rather than silently flattening them. - if (detach) { - const prototype = Object.getPrototypeOf(obj) - if (prototype !== Object.prototype && prototype !== null) return obj - } + if (detach) return obj + return clonePlainObject(obj, visited, detach) +} +function clonePlainObject( + obj: T, + visited: Map, + detach: boolean, +): T { const clone = {} as Record visited.set(obj as object, clone) for (const key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { // Copy data properties without invoking Object.prototype.__proto__. + const value = (obj as Record)[key] defineDataProperty( clone, key, - deepClone( - (obj as Record)[key], - visited, - detach, - ), + value === null || typeof value !== `object` + ? value + : deepClone(value, visited, detach), ) } } @@ -501,23 +519,24 @@ export function createChangeProxy< return changeProxy } } - // Create a WeakMap to cache proxies for nested objects + // Cache proxies for nested objects // This prevents creating multiple proxies for the same object // and handles circular references const proxyCache = new Map() // Existing values share one private copy per row. Newly inserted objects // retain normal references during the callback; the result is detached below. - const valueCopies = - parent?.tracker.valueCopies ?? new WeakMap() + const valueCopies = parent?.tracker.valueCopies ?? new Map() const changeTracker: ChangeTracker = { valueCopies, copy_: parent ? ((valueCopies.get(target) ?? target) as T) : deepClone(target, valueCopies), - originalObject: deepClone(target), + // The root target is the stored row, which the draft never writes, so it + // is its own baseline. A nested target is a draft copy that writes reach. + originalObject: parent ? deepClone(target) : target, modified: false, - assigned_: Object.create(null), + assigned_: new Map(), parent, target, // Store reference to the target object } @@ -537,7 +556,7 @@ export function createChangeProxy< ) { // Only mark an edge that still points to this child. A retained handle // must not reinstall itself after the callback replaces or deletes it. - state.parent.tracker.assigned_[state.parent.prop] = true + state.parent.tracker.assigned_.set(state.parent.prop, true) } // Mark parent as changed @@ -561,43 +580,19 @@ export function createChangeProxy< ) } // If there are no assigned properties, object is unchanged - if ( - Object.keys(state.assigned_).length === 0 && - Object.getOwnPropertySymbols(state.assigned_).length === 0 - ) { + if (state.assigned_.size === 0) { return true } - // Check each assigned regular property - for (const prop in state.assigned_) { - // If this property is marked as assigned - if (state.assigned_[prop] === true) { - const currentValue = state.copy_[prop] - const originalValue = (state.originalObject as any)[prop] - - // If the value is not equal to original, something is still changed - if (!draftValuesEqual(currentValue, originalValue)) { - return false - } - } else if (state.assigned_[prop] === false) { - // Property was deleted, so it's different from original - return false - } - } - - // Check each assigned symbol property - const symbolProps = Object.getOwnPropertySymbols(state.assigned_) - for (const sym of symbolProps) { - if (state.assigned_[sym] === true) { - const currentValue = (state.copy_ as any)[sym] - const originalValue = (state.originalObject as any)[sym] + // Check each assigned property + for (const [prop, assigned] of state.assigned_) { + // Property was deleted, so it's different from original + if (!assigned) return false + const currentValue = (state.copy_ as any)[prop] + const originalValue = (state.originalObject as any)[prop] - // If the value is not equal to original, something is still changed - if (!draftValuesEqual(currentValue, originalValue)) { - return false - } - } else if (state.assigned_[sym] === false) { - // Property was deleted, so it's different from original + // If the value is not equal to original, something is still changed + if (!draftValuesEqual(currentValue, originalValue)) { return false } } @@ -616,7 +611,7 @@ export function createChangeProxy< if (isReverted) { // If everything is reverted, clear the tracking parentState.modified = false - parentState.assigned_ = Object.create(null) + parentState.assigned_ = new Map() // Continue up the chain if (parentState.parent) { @@ -635,6 +630,7 @@ export function createChangeProxy< // Create a proxy for the object const proxy = new Proxy(obj, { get(ptarget, prop, receiver) { + if (prop === DRAFT_COPY) return changeTracker.copy_ const value = changeTracker.copy_[prop as keyof T] // If it's a getter, return the value directly @@ -778,7 +774,7 @@ export function createChangeProxy< if (isRevertToOriginal) { // If the value is reverted to its original state, remove it from changes - delete changeTracker.assigned_[prop.toString()] + changeTracker.assigned_.delete(prop.toString()) // Make sure the copy is updated with the original value changeTracker.copy_[prop as keyof T] = deepClone(originalValue) @@ -789,7 +785,7 @@ export function createChangeProxy< if (allReverted) { // If all have been reverted, clear tracking changeTracker.modified = false - changeTracker.assigned_ = Object.create(null) + changeTracker.assigned_ = new Map() // If we're a nested object, check if the parent needs updating if (parent) { @@ -804,7 +800,7 @@ export function createChangeProxy< changeTracker.copy_[prop as keyof T] = value // Track that this property was assigned - store using the actual property (symbol or string) - changeTracker.assigned_[prop.toString()] = true + changeTracker.assigned_.set(prop.toString(), true) // Mark this object and its ancestors as modified markChanged(changeTracker) @@ -820,7 +816,7 @@ export function createChangeProxy< const result = Reflect.defineProperty(ptarget, prop, descriptor) if (result && `value` in descriptor) { changeTracker.copy_[prop as keyof T] = deepClone(descriptor.value) - changeTracker.assigned_[prop.toString()] = true + changeTracker.assigned_.set(prop.toString(), true) markChanged(changeTracker) } return result @@ -859,15 +855,11 @@ export function createChangeProxy< // If the property didn't exist in the original object, removing it // should revert to the original state if (!hadPropertyInOriginal) { - delete changeTracker.assigned_[stringProp] + changeTracker.assigned_.delete(stringProp) // If this is the last change and we're not a nested object, // mark the object as unmodified - if ( - Object.keys(changeTracker.assigned_).length === 0 && - Object.getOwnPropertySymbols(changeTracker.assigned_).length === - 0 - ) { + if (changeTracker.assigned_.size === 0) { changeTracker.modified = false } else { // We still have changes, keep as modified @@ -875,7 +867,7 @@ export function createChangeProxy< } } else { // Mark this property as deleted - changeTracker.assigned_[stringProp] = false + changeTracker.assigned_.set(stringProp, false) markChanged(changeTracker) } } @@ -889,7 +881,6 @@ export function createChangeProxy< // Cache the proxy proxyCache.set(obj, proxy) - draftCopies.set(proxy, changeTracker.copy_) return proxy } @@ -917,19 +908,27 @@ export function createChangeProxy< return changeTracker.copy_ } - if (Object.keys(changeTracker.assigned_).length === 0) { + const assigned = changeTracker.assigned_ + if (assigned.size === 0) { return changeTracker.copy_ } const result: Record = {} - const mayHaveChangedAliases = Object.keys(changeTracker.assigned_).some( - (key) => - typeof changeTracker.copy_[key] === `object` || - typeof changeTracker.originalObject[key] === `object`, - ) - const pairedRoots = new Map([ - [changeTracker.copy_, changeTracker.originalObject], - ]) + let mayHaveChangedAliases = false + for (const key of assigned.keys()) { + if ( + typeof (changeTracker.copy_ as any)[key] === `object` || + typeof (changeTracker.originalObject as any)[key] === `object` + ) { + mayHaveChangedAliases = true + break + } + } + const pairedRoots = mayHaveChangedAliases + ? new Map([ + [changeTracker.copy_, changeTracker.originalObject], + ]) + : undefined // Iterate through keys in keyObj for (const key in changeTracker.copy_) { @@ -938,7 +937,7 @@ export function createChangeProxy< // Compare child contents, stopping only at paired root backedges. A // child's own changes still count even when it also points to this row. if ( - (changeTracker.assigned_[key] === true || + (assigned.get(key) === true || (mayHaveChangedAliases && !draftValuesEqual( value instanceof Set ? Array.from(value) : value, @@ -951,9 +950,9 @@ export function createChangeProxy< } } - for (const key of Object.keys(changeTracker.assigned_)) { - if (changeTracker.assigned_[key] === false) { - defineDataProperty(result, key, undefined) + for (const [key, isAssigned] of assigned) { + if (!isAssigned) { + defineDataProperty(result, key as string, undefined) } } From b29f56ffad9c7ba9cd6a1c05ca7fb335d0ca6300 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 09:57:15 -0600 Subject: [PATCH 03/63] fix(db): keep fields added as undefined when another field reverts The draft proxy treated a field added with an undefined value as reverted, because its original value also reads undefined. Setting any other field back to its original value then cleared the whole change set and dropped the added field. A field the original row lacks is now always a change. Found by the flat change tracking oracle. Co-authored-by: Isaac --- packages/db/src/proxy.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 38942ab32..20afccf8d 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -588,6 +588,8 @@ export function createChangeProxy< for (const [prop, assigned] of state.assigned_) { // Property was deleted, so it's different from original if (!assigned) return false + // A field the original lacks was added, even when it holds undefined. + if (!Object.hasOwn(state.originalObject, prop)) return false const currentValue = (state.copy_ as any)[prop] const originalValue = (state.originalObject as any)[prop] From 9ecf0a1be4c4e6b9b23f242797109280ffd6f7c1 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 09:57:18 -0600 Subject: [PATCH 04/63] perf(db): track flat-row updates without proxies A row whose own fields are all primitives or functions, with a plain or null prototype and no symbol keys, gets a shallow copy as its draft. Its changes are the fields that differ afterwards under the draft proxy's equality, plus deleted fields; other rows keep the proxy. The flat change tracking oracle runs generated callbacks (assignments, reverts, deletions, added fields, assigned objects) through both trackers and an independent model and requires the same change sets. Diffs using `!==`, `Object.is` alone, or no deletions fail it. Co-authored-by: Isaac --- packages/db/package.json | 2 +- packages/db/src/collection/mutations.ts | 36 +- packages/db/src/proxy.ts | 48 +++ ...at-change-tracking-oracle.property.test.ts | 331 ++++++++++++++++++ packages/db/tests/oracle-config.ts | 1 + 5 files changed, 403 insertions(+), 15 deletions(-) create mode 100644 packages/db/tests/flat-change-tracking-oracle.property.test.ts diff --git a/packages/db/package.json b/packages/db/package.json index 864d5acb5..fe0968e41 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:dist": "vitest run --config vitest.dist.config.ts", "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts", - "test:oracles": "vitest --run --coverage.enabled=false tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-truncate-ownership-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts", + "test:oracles": "vitest --run --coverage.enabled=false tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-truncate-ownership-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/flat-change-tracking-oracle.property.test.ts", "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run" }, "type": "module", diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 0d1d9c7f5..ad9ee14dd 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -1,4 +1,8 @@ -import { withArrayChangeTracking, withChangeTracking } from '../proxy' +import { + withArrayChangeTracking, + withChangeTracking, + withFlatChangeTracking, +} from '../proxy' import { safeRandomUUID } from '../utils/uuid' import { createTransaction, getActiveTransaction } from '../transactions' import { @@ -395,20 +399,24 @@ export class CollectionMutationsManager< return item }) as unknown as Array - let changesArray - if (isArray) { - // Use the proxy to track changes for all objects - changesArray = withArrayChangeTracking( + // Flat rows need no proxy; nested rows track changes through drafts. + const changesArray = + withFlatChangeTracking( currentObjects, - callback as (draft: Array) => void, - ) - } else { - const result = withChangeTracking( - currentObjects[0]!, - callback as (draft: TInput) => void, - ) - changesArray = [result] - } + callback as (drafts: Array | TInput) => void, + isArray, + ) ?? + (isArray + ? withArrayChangeTracking( + currentObjects, + callback as (draft: Array) => void, + ) + : [ + withChangeTracking( + currentObjects[0]!, + callback as (draft: TInput) => void, + ), + ]) // Create mutations for each object that has changes const mutations: Array< diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 20afccf8d..d730ebd28 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -1020,3 +1020,51 @@ export function withArrayChangeTracking( return deepClone(getChanges(), undefined, true) } + +// Whether every own field of a plain object is a primitive or a function. +function isFlatPlainObject(value: object): boolean { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + for (const key in value) { + const field = (value as Record)[key] + if (field !== null && typeof field === `object`) return false + } + return Object.getOwnPropertySymbols(value).length === 0 +} + +/** + * Change tracking for flat rows without proxies. A draft is a shallow copy, + * and its changes are the fields that differ from the row afterwards under + * the same equality the draft proxy uses for primitives. Returns undefined + * when any row has a nested object, a symbol key, or a class prototype, so + * the caller falls back to the proxy. + */ +export function withFlatChangeTracking( + targets: Array, + callback: (drafts: Array | T) => void, + asArray: boolean, +): Array> | undefined { + if (!targets.every(isFlatPlainObject)) return undefined + const drafts = targets.map((target) => ({ ...target })) + callback(asArray ? drafts : drafts[0]!) + return drafts.map((draft, index) => { + const original = targets[index] as Record + const changes: Record = {} + for (const key in draft) { + const value = (draft as Record)[key] + const before = original[key] + if ( + !Object.hasOwn(original, key) || + !(value === before || Object.is(value, before)) + ) { + defineDataProperty(changes, key, value) + } + } + for (const key in original) { + if (!Object.hasOwn(draft, key)) + defineDataProperty(changes, key, undefined) + } + // A callback may assign objects; detach them as the proxy path does. + return deepClone(changes, undefined, true) + }) +} diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts new file mode 100644 index 000000000..be425768b --- /dev/null +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -0,0 +1,331 @@ +/** + * # Does flat-row change tracking report what the draft proxy reports? + * + * Law and source: `collection.update` passes each row to the callback as a + * draft and records the fields whose final value differs from the row, plus + * deleted fields as `undefined` (`src/proxy.ts`). A row whose own fields are + * all primitives or functions, with a plain or null prototype and no symbol + * keys, is tracked with a shallow copy instead of a proxy. Both trackers must + * report the same change set for every callback, and any other row must fall + * back to the proxy. + * + * Why an example can miss the failure: a single assignment of a new value + * passes any diff. The trackers can disagree only on equal-but-not-identical + * values (`0` and `-0`, two `NaN`s), on a field set back to its original + * value, on a field set to `undefined` versus deleted, on an added field, and + * on an assigned object that later changes outside the callback. + * + * Model: `expectedChanges` folds the operations over a plain copy and keeps + * fields whose final value differs under `===` or `Object.is`, plus deleted + * fields. It does not import either tracker. + * + * History grammar: one to three rows with fields `a`, `b`, and `c` drawn from + * `0`, `-0`, `1`, `NaN`, `''`, `'x'`, `true`, `false`, `null`, `undefined`, a + * function, or missing, with a plain or null prototype. Each row gets up to + * five operations: assign a field (`a` to `d`) a value from that domain or a + * fresh object, set a field back to its original value, delete a field, or + * read it. Histories call the tracker with an array or with a single row. + * + * Production driver: `withFlatChangeTracking` and the proxy trackers + * `withArrayChangeTracking` and `withChangeTracking` run the same operations. + * + * Refinement check: the flat result, the proxy result, and the model are + * strictly equal, including present `undefined` fields, except that `0` and + * `-0` compare equal: collection equality does not distinguish them, and the + * proxy skips writing a value equal to the current one. Mutating an assigned + * object after the callback leaves both results unchanged. + * + * Calibration: a flat diff with `!==` reports unchanged `NaN` fields, one with + * `Object.is` alone reports `-0` written over `0`, and one that skips + * deletions loses removed fields; each fails the pinned histories and both + * campaigns. The proxy used to treat a field added with `undefined` as + * reverted when another field went back to its original value, and dropped + * the added field; the pinned history for that case failed before the fix. + * + * Known omissions: nested objects, arrays, Dates, Maps, Sets, class + * instances, and symbol keys are outside this owner; the proxy oracles and + * contracts own them, and this file checks only that they fall back. + */ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { + withArrayChangeTracking, + withChangeTracking, + withFlatChangeTracking, +} from '../src/proxy.js' +import { + oraclePropertyOptions, + oracleRuns, + readOracleRunConfig, +} from './oracle-config.js' + +const property = `flat-change-tracking.equivalence` +const requestedReplayProperty = readOracleRunConfig().replayProperty + +const MISSING = Symbol(`missing`) +const FRESH_OBJECT = Symbol(`fresh object`) +const fn = () => 1 +const values: ReadonlyArray = [ + 0, + -0, + 1, + Number.NaN, + ``, + `x`, + true, + false, + null, + undefined, + fn, +] +const fields = [`a`, `b`, `c`] as const +type Row = Record +type Operation = + | { kind: `set`; field: string; value: unknown } + | { kind: `revert`; field: string } + | { kind: `delete`; field: string } + | { kind: `read`; field: string } +type History = { + rows: Array<{ fields: Array; nullPrototype: boolean }> + operations: Array> + single: boolean +} + +// --------------------------------------------------------------------------- +// Model +// --------------------------------------------------------------------------- + +function sameValue(a: unknown, b: unknown): boolean { + return a === b || Object.is(a, b) +} + +function expectedChanges(original: Row, operations: Array): Row { + const draft: Row = { ...original } + for (const operation of operations) applyOperation(draft, original, operation) + const changes: Row = {} + for (const key of Object.keys(draft)) { + if ( + !Object.hasOwn(original, key) || + !sameValue(draft[key], original[key]) + ) { + changes[key] = draft[key] + } + } + for (const key of Object.keys(original)) { + if (!Object.hasOwn(draft, key)) changes[key] = undefined + } + return changes +} + +function applyOperation(draft: Row, original: Row, operation: Operation) { + switch (operation.kind) { + case `set`: + draft[operation.field] = + operation.value === FRESH_OBJECT ? { nested: 1 } : operation.value + break + case `revert`: + if (Object.hasOwn(original, operation.field)) { + draft[operation.field] = original[operation.field] + } + break + case `delete`: + delete draft[operation.field] + break + case `read`: + void draft[operation.field] + break + } +} + +// Collection equality does not distinguish -0 from 0. +function normalizeZeros(rows: Array | undefined) { + return rows?.map((row) => + Object.fromEntries( + Object.entries(row).map(([key, value]) => [key, value === 0 ? 0 : value]), + ), + ) +} + +function buildRow({ + fields: fieldValues, + nullPrototype, +}: History[`rows`][number]): Row { + const row: Row = nullPrototype ? Object.create(null) : {} + fieldValues.forEach((value, index) => { + if (value !== MISSING) row[fields[index]!] = value + }) + return row +} + +// --------------------------------------------------------------------------- +// History grammar +// --------------------------------------------------------------------------- + +const fieldArbitrary = fc.constantFrom(`a`, `b`, `c`, `d`) +const operationArbitrary: fc.Arbitrary = fc.oneof( + { + weight: 3, + arbitrary: fc.record({ + kind: fc.constant(`set` as const), + field: fieldArbitrary, + value: fc.constantFrom(...values, FRESH_OBJECT), + }), + }, + fc.record({ kind: fc.constant(`revert` as const), field: fieldArbitrary }), + fc.record({ kind: fc.constant(`delete` as const), field: fieldArbitrary }), + fc.record({ kind: fc.constant(`read` as const), field: fieldArbitrary }), +) +const historyArbitrary: fc.Arbitrary = fc + .array( + fc.record({ + fields: fc.tuple( + ...fields.map(() => fc.constantFrom(...values, MISSING)), + ), + nullPrototype: fc.boolean(), + }), + { minLength: 1, maxLength: 3 }, + ) + .chain((rows) => + fc.record({ + rows: fc.constant(rows), + operations: fc.tuple( + ...rows.map(() => fc.array(operationArbitrary, { maxLength: 5 })), + ), + single: rows.length === 1 ? fc.boolean() : fc.constant(false), + }), + ) + +// Each history isolates one place a plausible flat diff goes wrong. +const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ + { + name: `NaN written over NaN is not a change`, + history: { + rows: [{ fields: [Number.NaN, 1, MISSING], nullPrototype: false }], + operations: [[{ kind: `set`, field: `a`, value: Number.NaN }]], + single: false, + }, + }, + { + name: `-0 written over 0 is not a change`, + history: { + rows: [{ fields: [0, 1, MISSING], nullPrototype: false }], + operations: [[{ kind: `set`, field: `a`, value: -0 }]], + single: true, + }, + }, + { + name: `a field added as undefined survives another field's revert`, + history: { + rows: [{ fields: [`x`, 1, MISSING], nullPrototype: false }], + operations: [ + [ + { kind: `set`, field: `a`, value: `y` }, + { kind: `set`, field: `d`, value: undefined }, + { kind: `revert`, field: `a` }, + ], + ], + single: false, + }, + }, + { + name: `deleted, added, and reverted fields`, + history: { + rows: [{ fields: [1, `x`, true], nullPrototype: true }], + operations: [ + [ + { kind: `delete`, field: `a` }, + { kind: `set`, field: `d`, value: undefined }, + { kind: `set`, field: `b`, value: `y` }, + { kind: `revert`, field: `b` }, + ], + ], + single: false, + }, + }, +] + +// --------------------------------------------------------------------------- +// Production driver and refinement check +// --------------------------------------------------------------------------- + +function runHistory(history: History): void { + const originals = history.rows.map(buildRow) + const run = (drafts: Array | Row) => { + const list = Array.isArray(drafts) ? drafts : [drafts] + list.forEach((draft, index) => { + for (const operation of history.operations[index]!) { + applyOperation(draft, originals[index]!, operation) + } + }) + } + const flat = withFlatChangeTracking( + history.rows.map(buildRow), + run, + !history.single, + ) + expect(flat, `flat rows take the flat path`).toBeDefined() + const proxy = history.single + ? [withChangeTracking(buildRow(history.rows[0]!), run)] + : withArrayChangeTracking(history.rows.map(buildRow), run) + const model = originals.map((original, index) => + expectedChanges(original, history.operations[index]!), + ) + expect(normalizeZeros(flat), `flat vs model`).toStrictEqual( + normalizeZeros(model), + ) + expect(normalizeZeros(proxy), `proxy vs model`).toStrictEqual( + normalizeZeros(model), + ) +} + +describe(`flat change tracking oracle`, () => { + if (requestedReplayProperty === undefined) { + for (const { name, history } of pinnedHistories) { + it(`matches the draft proxy when ${name}`, () => runHistory(history)) + } + + it(`falls back to the proxy for rows that are not flat`, () => { + const callback = () => {} + for (const row of [ + { a: { nested: 1 } }, + { a: [1] }, + { a: new Date(0) }, + { [Symbol(`s`)]: 1 }, + new (class Row { + a = 1 + })(), + ]) { + expect(withFlatChangeTracking([row], callback, true)).toBeUndefined() + } + }) + + it(`detaches an assigned object from later mutation`, () => { + const assigned = { nested: 1 } + const [changes] = withFlatChangeTracking( + [{ a: 1 }], + (drafts) => { + ;(drafts as Array)[0]!.a = assigned + }, + true, + )! + assigned.nested = 2 + expect(changes).toStrictEqual({ a: { nested: 1 } }) + }) + + fcTest.prop([historyArbitrary], { + seed: 44_502_101, + numRuns: oracleRuns(200), + })(`matches the draft proxy (fixed)`, runHistory) + fcTest.prop([historyArbitrary], oraclePropertyOptions(200, property))( + `matches the draft proxy (random)`, + runHistory, + ) + } else if (requestedReplayProperty === property) { + fcTest.prop([historyArbitrary], oraclePropertyOptions(200, property))( + `matches the draft proxy (replay)`, + runHistory, + ) + } else { + it.skip(`runs only when its replay property is selected`, () => {}) + } +}) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index 0f5bfc147..1166dffce 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -49,6 +49,7 @@ const staticOracleProperties = [ `where-predicate.publication`, `join-result-key.pairs`, `pooled-live-query.publication`, + `flat-change-tracking.equivalence`, `derived-publication.membership-work`, `collection-publication.metadata-cancellation`, `collection-publication.metadata-only`, From a0ea7e4a57c58bcb2c31b9a4760489a9de69da0d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 10:08:04 -0600 Subject: [PATCH 05/63] test(db): witness local-only direct write fallbacks Pin when a local-only direct write skips the optimistic stage and when it keeps it: a pending transaction, a persisting transaction, an ambient transaction, and a user handler. Removing the guard, or guarding only persisting transactions, fails these witnesses. Co-authored-by: Isaac --- .../db/tests/local-only-direct-write.test.ts | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 packages/db/tests/local-only-direct-write.test.ts diff --git a/packages/db/tests/local-only-direct-write.test.ts b/packages/db/tests/local-only-direct-write.test.ts new file mode 100644 index 000000000..a90346d61 --- /dev/null +++ b/packages/db/tests/local-only-direct-write.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/index' +import { localOnlyCollectionOptions } from '../src/local-only' +import { createTransaction } from '../src/transactions' +import { createDeferred } from '../src/deferred' + +/** + * # When does a local-only direct write skip the optimistic stage? + * + * Law: a local-only Collection confirms its own writes. A direct `insert`, + * `update`, or `delete` for an operation type without a user handler, with no + * other transaction on the Collection pending or persisting, publishes the + * synced row once and returns a completed transaction. Otherwise the write + * keeps the optimistic path and its visibility: it overlays a pending + * transaction, is visible while another persists, joins an ambient + * transaction, and runs the user's handler. + * + * The change-event history oracle drives this path through generated + * histories and checks every publication. These witnesses pin the guard's + * fallback cases, which that oracle does not generate: removing the guard + * hides a direct update under a pending overlay and holds it behind a + * persisting transaction. + */ +type Row = { id: number; value: string } + +function createOrders( + handlers: Partial<{ onUpdate: () => Promise }> = {}, +) { + return createCollection( + localOnlyCollectionOptions({ + id: `local-only-direct-${Math.random()}`, + getKey: (row) => row.id, + initialData: [{ id: 1, value: `a` }], + ...handlers, + }), + ) +} + +describe(`local-only direct writes`, () => { + it(`publishes each direct write once and returns a completed transaction`, async () => { + const orders = createOrders() + const batches: Array> = [] + orders.subscribeChanges( + (changes) => + batches.push(changes.map((change) => `${change.type}:${change.key}`)), + { includeInitialState: true }, + ) + + const inserted = orders.insert({ id: 2, value: `b` }) + const updated = orders.update(1, (draft) => { + draft.value = `c` + }) + const deleted = orders.delete(2) + + for (const transaction of [inserted, updated, deleted]) { + expect(transaction.state).toBe(`completed`) + await expect(transaction.isPersisted.promise).resolves.toBe(transaction) + } + expect(batches).toEqual([ + [`insert:1`], + [`insert:2`], + [`update:1`], + [`delete:2`], + ]) + expect(orders.get(1)).toMatchObject({ + value: `c`, + $synced: true, + $origin: `local`, + }) + expect(orders.has(2)).toBe(false) + }) + + it(`overlays a pending transaction instead of writing beneath it`, () => { + const orders = createOrders() + const pending = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + pending.mutate(() => + orders.update(1, (draft) => { + draft.value = `pending` + }), + ) + + const direct = orders.update(1, (draft) => { + draft.value = `direct` + }) + + expect(direct.state).not.toBe(`completed`) + expect(orders.get(1)?.value).toBe(`direct`) + pending.rollback() + }) + + it(`stays visible while another transaction persists`, async () => { + const orders = createOrders() + const release = createDeferred() + const persisting = createTransaction({ + mutationFn: () => release.promise, + }) + persisting.mutate(() => orders.insert({ id: 3, value: `persisting` })) + expect(persisting.state).toBe(`persisting`) + + orders.update(1, (draft) => { + draft.value = `direct` + }) + + expect(orders.get(1)?.value).toBe(`direct`) + release.resolve() + await persisting.isPersisted.promise + }) + + it(`joins an ambient transaction`, () => { + const orders = createOrders() + const ambient = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + ambient.mutate(() => + orders.update(1, (draft) => { + draft.value = `ambient` + }), + ) + + expect(ambient.state).toBe(`pending`) + expect(ambient.mutations).toHaveLength(1) + expect(orders.get(1)?.value).toBe(`ambient`) + ambient.rollback() + expect(orders.get(1)?.value).toBe(`a`) + }) + + it(`runs the user's handler for that operation type`, async () => { + let calls = 0 + const orders = createOrders({ + onUpdate: () => { + calls++ + return Promise.resolve() + }, + }) + + const transaction = orders.update(1, (draft) => { + draft.value = `handled` + }) + + expect(transaction.state).not.toBe(`completed`) + await transaction.isPersisted.promise + expect(calls).toBe(1) + expect(orders.get(1)?.value).toBe(`handled`) + // Inserts have no handler, so they still take the direct path. + expect(orders.insert({ id: 4, value: `d` }).state).toBe(`completed`) + }) +}) From 39f24544fcf9246c3122d53a7d7a493f25907bbe Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 10:14:37 -0600 Subject: [PATCH 06/63] fix(db): fail pooled live queries when their source starts cleanup A live query enters a terminal error when its source starts cleanup. A pooled view instead reported the source's status, became ready again after a restart, and kept publishing later writes. The partition now terminates on cleanup: its views report `error` with their last rows, and queries mounted afterwards get a new partition on the restarted source. The pooled live query oracle gains a cleanup-and-restart step and a pinned history; a view that follows the source's status, or a partition that never terminates, fails it. Co-authored-by: Isaac --- packages/db/src/query/pooled-live-query.ts | 22 +++++- .../pooled-live-query-oracle.property.test.ts | 72 ++++++++++++++----- 2 files changed, 76 insertions(+), 18 deletions(-) diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 61021e964..182392ffb 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -73,6 +73,8 @@ class Partition { // One source status listener serves every view of this partition. readonly statusListeners = new Set() private listenerCount = 0 + /** Set when the source starts cleanup; buckets keep their last rows. */ + terminated = false private releaseTimer: ReturnType | undefined constructor( @@ -109,6 +111,7 @@ class Partition { /** Start the shared source subscription; release it when unused. */ retain(): void { + if (this.terminated) return if (!this.subscription) { this.subscription = this.source.subscribeChanges( (changes) => @@ -116,7 +119,14 @@ class Partition { { includeInitialState: true }, ) this.stopStatusEvents = this.source.on(`status:change`, (event) => { - for (const listener of [...this.statusListeners]) listener(event) + // Like a live query, a pooled view fails for good when its source + // starts cleanup; queries mounted later get a new partition. + if (event.status === `cleaned-up`) this.terminate() + const delivered = this.terminated + ? { ...event, status: `error` as const } + : event + for (const listener of [...this.statusListeners]) listener(delivered) + if (this.terminated) this.stopStatusEvents?.() }) } this.scheduleRelease() @@ -139,6 +149,14 @@ class Partition { this.scheduleRelease() } + private terminate(): void { + this.terminated = true + if (this.releaseTimer !== undefined) clearTimeout(this.releaseTimer) + this.subscription?.unsubscribe() + this.subscription = undefined + this.onEmpty() + } + private scheduleRelease(): void { if (this.releaseTimer !== undefined) clearTimeout(this.releaseTimer) this.releaseTimer = undefined @@ -310,7 +328,7 @@ class PooledLiveQuery { } get status(): CollectionStatus { - return this.source.status + return this.partition.terminated ? `error` : this.source.status } get _stateRevision(): number { diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 514d310e9..55e781d57 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -25,8 +25,8 @@ * missing value, and a field `g` of `x` or `y`. Up to three peer queries use * `eq(f, literal)`, optionally with `eq(g, literal)`. Steps commit sync * transactions of one or two inserts, updates, or deletes; apply one - * optimistic insert, update, or delete and then confirm or roll it back; or - * mount or unmount a peer. + * optimistic insert, update, or delete and then confirm or roll it back; + * mount or unmount a peer; or clean up the source and restart it. * * Production driver: `createPooledLiveQuery` builds each peer's view from the * query builder's IR, and `createLiveQueryObserver` observes it in wholesale @@ -34,11 +34,15 @@ * * Refinement check: after every step, each mounted peer's wholesale snapshot * equals its live-query Collection's keys in order, row values, and status; - * both observers' key sets equal the model. + * both observers' key sets equal the model. A peer mounted when its source + * starts cleanup is terminal like its live query: status `error` with the + * rows it had, through the restart and later writes. A peer mounted after + * the restart follows the restarted source. * * Calibration: a partition that ignored the previous value, kept bucket rows * in arrival order, or compared literals without normalization fails the - * pinned histories and both campaigns. + * pinned histories and both campaigns. A view that kept reporting the + * source's status after cleanup fails the cleanup history. * * Known omissions: on-demand and persisted sources, `DbClient` hydration, * Suspense, `select`, and every other clause keep the live-query Collection @@ -76,6 +80,7 @@ type Step = | { kind: `optimistic`; op: Op; confirm: boolean } | { kind: `mount`; peer: number } | { kind: `unmount`; peer: number } + | { kind: `cleanup-restart` } type Op = | { type: `insert`; id: number; f: FieldValue; g: string } | { type: `update`; id: number; f: FieldValue; g: string } @@ -194,6 +199,7 @@ const historyArbitrary: fc.Arbitrary = fc.record({ kind: fc.constant(`unmount` as const), peer: fc.nat({ max: 2 }), }), + fc.constant({ kind: `cleanup-restart` as const }), ), { maxLength: 8 }, ), @@ -241,6 +247,20 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ ], }, }, + { + name: `a peer mounted at cleanup stays failed while a new one follows the restart`, + history: { + rows: [{ f: `a`, g: `x` }], + peers: [{ f: `a` }, { f: `a` }], + steps: [ + { kind: `unmount`, peer: 1 }, + { kind: `sync`, ops: [{ type: `insert`, id: 2, f: `a`, g: `y` }] }, + { kind: `cleanup-restart` }, + { kind: `mount`, peer: 1 }, + { kind: `sync`, ops: [{ type: `insert`, id: 3, f: `a`, g: `x` }] }, + ], + }, + }, { name: `a remounted peer reads a bucket that changed while it was away`, history: { @@ -296,10 +316,11 @@ async function runHistory(history: History): Promise { }), ) await source.stateWhenReady() - const references = history.peers.map((peer) => - createLiveQueryCollection(peerQuery(source, peer)), - ) + const references: Array> = [] type Mounted = { + reference: ReturnType + // The model keys when the source started cleanup, if it has since. + frozen: Array | undefined view: { collection?: unknown } layout: { keys: string; revision: number } | undefined wholesale: ReturnType> @@ -308,9 +329,12 @@ async function runHistory(history: History): Promise { unsubscribe: () => void } const mounted = new Map() - const mount = (index: number) => { + const mount = async (index: number) => { const peer = history.peers[index] if (!peer || mounted.has(index)) return + const reference = createLiveQueryCollection(peerQuery(source, peer)) + references.push(reference) + await reference.preload() const view = createPooledLiveQuery(peerQuery(source, peer)(new Query())) expect(view, `peer ${index} is poolable`).toBeDefined() const wholesale = createLiveQueryObserver(view as any, { @@ -326,6 +350,8 @@ async function runHistory(history: History): Promise { } }) mounted.set(index, { + reference, + frozen: undefined, view: view as unknown as Mounted[`view`], layout: undefined, wholesale, @@ -344,14 +370,14 @@ async function runHistory(history: History): Promise { const { view, wholesale, granularKeys } = entry const peer = history.peers[index]! const snapshot = wholesale.getSnapshot() - const reference = references[index]! + const reference = entry.reference const label = `${checkpoint}, peer ${index}` expect( (snapshot.data as Array>).map(describeRow), `${label} rows`, ).toEqual(reference.toArray.map((row) => describeRow(row))) expect(snapshot.status, `${label} status`).toBe(reference.status) - const model = expectedKeys(rows, peer) + const model = entry.frozen ?? expectedKeys(rows, peer) expect( [...snapshot.state!.keys()].map(String).sort(), `${label} model`, @@ -397,13 +423,24 @@ async function runHistory(history: History): Promise { } await withOracleCleanup(async () => { - await Promise.all(references.map((reference) => reference.preload())) - history.peers.forEach((_, index) => mount(index)) + for (const index of history.peers.keys()) await mount(index) check(`after mount`) for (const [n, step] of history.steps.entries()) { const checkpoint = `after step ${n} (${step.kind})` - if (step.kind === `mount`) mount(step.peer) - else if (step.kind === `unmount`) { + if (step.kind === `mount`) await mount(step.peer) + else if (step.kind === `cleanup-restart`) { + for (const [index, entry] of mounted) { + entry.frozen ??= expectedKeys(rows, history.peers[index]!) + } + await source.cleanup() + check(`${checkpoint} cleaned up`) + // The mock source re-syncs its initial rows when it restarts. + rows.clear() + history.rows.forEach((row, rowIndex) => + rows.set(id(rowIndex), sourceRow(id(rowIndex), row.f, row.g)), + ) + await source.preload() + } else if (step.kind === `unmount`) { mounted.get(step.peer)?.unsubscribe() mounted.delete(step.peer) } else if (step.kind === `sync`) { @@ -465,12 +502,15 @@ async function runHistory(history: History): Promise { check(checkpoint) } // Any other Collection member builds the live-query Collection. - for (const [index, { wholesale }] of mounted) { + // A terminal peer's Collection would be a new query on the restarted + // source, so only live peers compare forwarded rows. + for (const [index, { wholesale, reference, frozen }] of mounted) { + if (frozen) continue const collection = wholesale.getSnapshot().collection! expect( (collection.toArray as Array>).map(describeRow), `peer ${index} forwarded toArray`, - ).toEqual(references[index]!.toArray.map((row) => describeRow(row))) + ).toEqual(reference.toArray.map((row) => describeRow(row))) } }, [ () => { From 5d2ca2a408c7a9b68a1f25bf06097be9d4e9e99e Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 10:21:43 -0600 Subject: [PATCH 07/63] test(db): conformance scenarios for single-source eq filters Every adapter now runs an eq-filtered query whose rows move in and out, and three eq-filtered peers on one source. React serves these from a shared partition while the other adapters compile live queries, so the suite compares the two paths; a partition that ignores a row's previous bucket fails both scenarios under React. Co-authored-by: Isaac --- packages/db/tests/conformance/suite.ts | 77 +++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/conformance/suite.ts b/packages/db/tests/conformance/suite.ts index 9ac852674..84367f785 100644 --- a/packages/db/tests/conformance/suite.ts +++ b/packages/db/tests/conformance/suite.ts @@ -262,6 +262,81 @@ export function runSuite(rawDriver: LiveQueryDriver) { }, ) + // Single-source eq filters may be served from a partition shared by every + // query of that shape; adapters must publish what the live query would. + scenario( + `eq-filter-rows`, + `an eq filter publishes matching rows in key order as rows move in and out`, + async () => { + const source = driver.makeSource(SEED) + const h = driver.mount((q) => + q + .from({ items: source.collection }) + .where(({ items }: any) => ops.eq(items.team, `a`)), + ) + await h.flush() + const ids = () => + (h.current().data as Array<{ id: string }>).map((row) => row.id) + expect(ids()).toEqual([`1`, `3`]) + + source.update({ id: `2`, name: `Jane Doe`, age: 25, team: `a` }) + await h.flush() + expect(ids()).toEqual([`1`, `2`, `3`]) + + source.update({ id: `1`, name: `John Doe`, age: 30, team: `b` }) + source.insert({ id: `4`, name: `Dave`, age: 40, team: `a` }) + await h.flush() + expect(ids()).toEqual([`2`, `3`, `4`]) + + source.remove({ id: `3`, name: `John Smith`, age: 35, team: `a` }) + await h.flush() + expectOrderedRows(h.current().data, [ + { id: `2`, name: `Jane Doe`, age: 25, team: `a` }, + { id: `4`, name: `Dave`, age: 40, team: `a` }, + ]) + expectKeyedRows(h.current().state, [ + { id: `2`, name: `Jane Doe`, age: 25, team: `a` }, + { id: `4`, name: `Dave`, age: 40, team: `a` }, + ]) + expect(h.current().status).toBe(`ready`) + h.unmount() + }, + ) + + scenario( + `eq-filter-peers`, + `eq-filtered peers on one source each see only their rows`, + async () => { + const source = driver.makeSource(SEED) + const query = (team: string) => (q: any) => + q + .from({ items: source.collection }) + .where(({ items }: any) => ops.eq(items.team, team)) + const teamA = driver.mount(query(`a`)) + const teamB = driver.mount(query(`b`)) + const olderA = driver.mount((q) => + query(`a`)(q).where(({ items }: any) => ops.eq(items.age, 35)), + ) + for (const h of [teamA, teamB, olderA]) await h.flush() + const ids = (h: typeof teamA) => + (h.current().data as Array<{ id: string }>).map((row) => row.id) + expect([ids(teamA), ids(teamB), ids(olderA)]).toEqual([ + [`1`, `3`], + [`2`], + [`3`], + ]) + + source.update({ id: `3`, name: `John Smith`, age: 35, team: `b` }) + for (const h of [teamA, teamB, olderA]) await h.flush() + expect([ids(teamA), ids(teamB), ids(olderA)]).toEqual([ + [`1`], + [`2`, `3`], + [], + ]) + for (const h of [teamA, teamB, olderA]) h.unmount() + }, + ) + scenario(`live-insert`, `a sync insert appears in the result`, async () => { const source = driver.makeSource(SEED) const h = driver.mount((q) => @@ -941,7 +1016,7 @@ export function runSuite(rawDriver: LiveQueryDriver) { ) it(`registers every distinct scenario without whole-test waivers`, () => { - expect(registry.size).toBe(28) + expect(registry.size).toBe(30) }) }) } From 9f0a239cd47d405f8abf6b815195fc20e8f96874 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 10:30:28 -0600 Subject: [PATCH 08/63] docs(db): document pooled live queries and their coverage - Name the shared structure an equality partition with partition groups, distinct from includes buckets, in code, tests, and the glossary. - The live-query architecture document states when an adapter may serve a query from an equality partition and what it must still publish. - The coverage map adds owners for pooled live queries, flat-row change tracking, and local-only direct writes, and notes that the routing and prefilter mutants in the WHERE row describe removed code. - Changesets for pooled live queries, local-only direct writes, cheaper mutations, and the draft fix; the unreleased filtered-query changeset no longer advertises the removed routing and prefilter. - Mangle map gains the new private member names. Co-authored-by: Isaac --- .changeset/cheaper-mutations.md | 5 + .changeset/fix-draft-added-undefined.md | 5 + .changeset/local-only-direct-writes.md | 5 + .changeset/perf-many-filtered-live-queries.md | 2 +- .changeset/perf-pooled-live-queries.md | 8 ++ docs/contributing/glossary.md | 3 + docs/contributing/oracle-coverage.md | 7 +- packages/db/mangle-cache.json | 14 ++- packages/db/src/live-query-observer.ts | 2 +- packages/db/src/query/live/ARCHITECTURE.md | 16 +++ packages/db/src/query/pooled-live-query.ts | 106 +++++++++--------- packages/db/tests/conformance/suite.ts | 10 +- .../pooled-live-query-oracle.property.test.ts | 14 +-- 13 files changed, 127 insertions(+), 70 deletions(-) create mode 100644 .changeset/cheaper-mutations.md create mode 100644 .changeset/fix-draft-added-undefined.md create mode 100644 .changeset/local-only-direct-writes.md create mode 100644 .changeset/perf-pooled-live-queries.md diff --git a/.changeset/cheaper-mutations.md b/.changeset/cheaper-mutations.md new file mode 100644 index 000000000..0387daf14 --- /dev/null +++ b/.changeset/cheaper-mutations.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Make updates and query building cheaper. Updates to rows whose fields are all primitives track changes without a proxy, drafts of other rows allocate less, sorted Collections no longer re-sort when an existing row changes value, and query builder references allocate less. Mutation ids are now a random per-runtime prefix plus a counter instead of a random UUID per mutation; they stay unique across tabs and sessions, but are no longer bare UUIDs. diff --git a/.changeset/fix-draft-added-undefined.md b/.changeset/fix-draft-added-undefined.md new file mode 100644 index 000000000..d01a32458 --- /dev/null +++ b/.changeset/fix-draft-added-undefined.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Fix an update that drops a field added as `undefined`. When a callback added a field with the value `undefined` and set another field back to its original value, the draft treated every change as reverted and reported nothing. diff --git a/.changeset/local-only-direct-writes.md b/.changeset/local-only-direct-writes.md new file mode 100644 index 000000000..d4112d1ed --- /dev/null +++ b/.changeset/local-only-direct-writes.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Local-only Collections apply direct `insert`, `update`, and `delete` calls without an optimistic stage when no user handler is configured for that operation and no other transaction on the Collection is pending or persisting. The write is published once, and the returned transaction is already `completed` with `isPersisted.promise` resolved. Writes inside an ambient transaction, with a handler, or beside another unsettled transaction behave as before. diff --git a/.changeset/perf-many-filtered-live-queries.md b/.changeset/perf-many-filtered-live-queries.md index d11323746..ba0fa9b8d 100644 --- a/.changeset/perf-many-filtered-live-queries.md +++ b/.changeset/perf-many-filtered-live-queries.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Speed up apps that mount many small filtered live queries. Queries without includes keep their compiled pipeline instead of paying for include materialization, eager subscriptions no longer build an abort error on every unsubscribe, a filtered subscription skips source batches that cannot match its `eq` condition, and unindexed snapshots reject rows by that condition before copying them. With 240 `eq`-filtered live queries, mounting is about 2x faster (indexed) to 2.5x faster (unindexed), and a 50-row update batch is about 2.7x faster. +Speed up apps that mount many small filtered live queries. Queries without includes keep their compiled pipeline instead of paying for include materialization, and eager subscriptions no longer build an abort error on every unsubscribe. diff --git a/.changeset/perf-pooled-live-queries.md b/.changeset/perf-pooled-live-queries.md new file mode 100644 index 000000000..315035e62 --- /dev/null +++ b/.changeset/perf-pooled-live-queries.md @@ -0,0 +1,8 @@ +--- +'@tanstack/db': patch +'@tanstack/react-db': patch +--- + +Mount and update many small filtered live queries at Redux-level cost. In React, a `useLiveQuery` that reads one eager source Collection and filters it only by `eq(field, literal)` conjuncts, without a `DbClient` or Suspense, is served from an equality partition shared by every query on those fields. Each component reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. + +Results are unchanged: the same rows in key order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. diff --git a/docs/contributing/glossary.md b/docs/contributing/glossary.md index 1612f2a40..c11c9e8ee 100644 --- a/docs/contributing/glossary.md +++ b/docs/contributing/glossary.md @@ -19,6 +19,9 @@ production queues, caches, or semantic helpers merely to share their names. | Collection | The public keyed data container. Capitalize it when referring to the TanStack DB type. | Relation, table, or query result. | | source Collection | A Collection read by a query or adapter. | Source relation when the value is a public Collection. | | live-query Collection | A Collection whose rows are produced by a live query. | Query, observer, or result set. | +| pooled live query | A live query filtered only by `eq(field, literal)` on one source Collection and served from an equality partition; its live-query Collection is built only when read. | Cached query or shared live-query Collection. | +| equality partition | Source rows grouped by the `eq`-normalized values of one set of fields, shared by every pooled live query that filters on those fields. | Index or bucket relation. | +| partition group | The rows of an equality partition whose fields equal one tuple of literals. | Bucket or active bucket. | | relation | An internal weighted multiset maintained by D2. | Collection. | | row | One keyed public Collection value or one relation value. Qualify source row, relation row, or public row when more than one kind appears. | Event or transaction. | | change message | One insert, update, or delete delivered through the Collection sync boundary. | Transaction or publication. | diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index d66b3d3d7..92354dbc8 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -65,6 +65,8 @@ that test identifiers must copy production's private data structures. | WHERE predicate publication | Complete for bounded predicate and sync-transaction grammar | The contract, Kleene reference evaluator, snapshot and change-history grammar, three subscriber consumers plus direct snapshot, and per-commit key-set refinement check are literate. A fixed same-key update checks live-row and direct-subscriber payloads; a focused descriptor boundary checks stored-row prefilter safety. Generated cleanup and restart histories for filtered subscribers remain open. | | Joined result keys | Complete for bounded two-source key grammar | The contract, nested-loop pair model, delimiter-, number-like, infinite, and `NaN` key grammar, public join driver, and per-checkpoint pair and key-count check are literate. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations remain outside this owner. | | D2 Index storage | Complete for bounded prefix grammar | The contract, plain-`Map` multiset model, prefixed and unprefixed value grammar, `Index` driver, and per-addition `get`/`has` check are literate. Compaction, presence tracking, and structural payloads remain outside this owner. | +| Pooled live queries | Complete for bounded eq-filter grammar | The contract, independent `eq` model, live-query Collection second formulation, sync/optimistic/mount/cleanup-restart grammar, observer driver, and per-step refinement check are literate. On-demand and persisted sources, `DbClient`, Suspense, and every clause beyond `eq` conjuncts keep the live-query Collection. | +| Flat-row change tracking | Complete for bounded flat-row grammar | Flat and proxy trackers and an independent change model run the same generated callbacks. Nested values fall back to the proxy, which its own oracles own. | | Lazy target path identity | Focused compiler boundary | A same-source union/coalesce witness keeps dotted and nested demand paths distinct during target deduplication. | | Correlated include path identity | Focused public route-context witnesses | One-level and nested includes keep dotted and nested parent paths, including ancestor aliases, distinct. Conditional result paths receive separate routes. Fixed fixtures cover initial reads and selected source updates; other recursive source forms and arbitrary path segments remain outside this witness. | | Join equality and cold acquisition | Complete | Independent cold relational recomputation, acquisition evidence, established equality domains, replacements, and scan/index routes are literate. | @@ -260,9 +262,12 @@ comment and the current API/architecture contract before extending its model. | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites), [Vue synchronous publication](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/useLiveQuery-publication-oracle.test.ts) | Exact exposed rows/pages and each framework's own lifecycle cuts. Vue's synchronous watcher checks insert, update, and nonterminal delete through supplied Collection and identity query inputs; it does not cover an empty final result, multiple changes in one transaction, or query recompilation. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Window-controller overlap | [shared infinite-query conformance](https://github.com/TanStack/db/blob/main/packages/db/tests/conformance/infinite-suite.ts), [React driver](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [Vue driver](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/infinite-query-conformance.test.ts), [Svelte driver](https://github.com/TanStack/db/blob/main/packages/svelte-db/tests/infinite-query-conformance.svelte.test.ts), [review record](oracle-reviews/dec-03-window-overlap.md) | Four or five ordered source rows, two window-settlement orders, and public snapshots after each settlement, a subscribed observer update, a detached controller read, and explicit recovery. Insertion and removal of the fifth row distinguish both continuation directions while an earlier error remains visible. The detached read follows a source change with no controller subscriber; its preloaded Collection retains the five-row window. The exact overlap uses the exported DB controller in each package realm. Framework hooks expose no controller preload, so this cell does not establish a hook scheduling path; direct hook paging remains in the adjacent shared scenarios. An unsubscribed controller has no notification claim. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash identity](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-identity-oracle.property.test.ts), [MultiSet consolidation](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/multiset-consolidate-oracle.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [BTree Map](https://github.com/TanStack/db/blob/main/packages/db/tests/btree-map-oracle.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts), [LIKE semantics](https://github.com/TanStack/db/blob/main/packages/db/tests/query/compiler/evaluators.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, custom comparator dispatch/reference identity/index compatibility, LIKE wildcard refinement, compiled output bags, declared value-pair agreement between D2 equality keys and IR equality operands, and Collection collation with exact public scan/index order and one-index reuse across repeated reads. The auto-index cells cross BasicIndex and BTreeIndex; scan cells omit indexes. Both paths cover lexical and numeric `en-US` locale defaults plus an explicit ordinary locale override of a numeric default. An unindexed six-row work cell bounds Collection collation resolution to at most two reads per ordered scan: one index-path check and one scan clause. Other locales and ordered histories are outside this owner. The identity pair grammar covers primitives, signed zero, NaN/invalid Date, valid Date/timestamp, binary/Buffer, Temporal, nested references, symbols, and functions; fixed and sampled pairs do not prove every JavaScript value or hash collision freedom. Ref proxies are expressions rather than literal values in this lane. The hash-values owner samples Set/object and Map/object type-marker distinctions in fixed and random campaigns with seed-and-path replay; collision freedom is not promised. The hash-identity owner checks `hash` and `equalHashValues` against one spec-level identity model: primitives with signed zero, NaN, and bigint; reference leaves (functions, registered handles, `File`, binary over 128 bytes); Date, binary, Temporal, and RegExp headers; array length and holes; Map and Set insertion order; and object keys in any order, for acyclic values and for cycles through arrays, Map values, Sets, and plain objects. Map/Set order sensitivity is pinned current behavior, not a promise. Arrays from another realm, RegExps with a `NaN` `lastIndex`, getters, proxies, other cycle carriers, and pairs that differ only in a back-edge target are outside its grammar; pinned cases keep equality's cross-realm length check and strict RegExp field comparison. `hash-work.test.ts` pins how visited values and array/RegExp headers count toward the structural work cap, and that equality copies a shared Map's entries once per distinct pair ([code-weight hash dispatch review](oracle-reviews/code-weight-hash-dispatch.md)). Custom string indexes do not optimize ordinary ranges, and custom cursors remain unsupported. The LIKE owner covers boolean string matching and bounded work, not nullish three-valued logic or a general Unicode collation contract. Unsupported composite cursors reject. The MultiSet consolidation owner checks keyed, single-type, and structural identity, signed zero and NaN in keyed and single-number data, the retained first record, zero-sum removal, and input record contents. It does not assert output order. Its keyed grammar includes cross-type key and value collisions, non-finite numeric keys, the `\|` delimiter, symbol and function identity, and a direct-object/object-tuple delimiter control from #1948. Eight named collision witnesses and both generated campaigns are RED on `c879ba6d` and GREEN with the repair ([#1948 review record](oracle-reviews/issue-1948-keyed-consolidation.md)); the [code-weight consolidation review](oracle-reviews/code-weight-multiset-consolidation.md) records the earlier exclusion. The keyed reference uses direct value/reference equality, including SameValueZero for numeric keys. The unkeyed fallback keeps its original key and value domains. The direct `MultiSet` owner does not prove which `@tanstack/db` query shapes produce primitive keyed values or mixed-type source keys; that needs a live-query production witness. The index owner enforces one invalid-comparator law across both BasicIndex and BTreeIndex: for each comparator, accepted numeric prefix (including the empty prefix), and probe operation (add, update, eq/range lookup, take, build), the operation throws exactly when it receives a `NaN` or non-number result and never when every comparison is valid. A `signed infinity` comparator is the valid control, and a custom collation `compare` supplied through `compareOptions` is checked the same way. Two BTree Map split histories check inserted-key reads, exact whole-range payloads/order, size, and extrema, proving split placement follows the insertion index without a post-mutation comparison. A rejected index add or remove leaves the index refining its accepted rows; update and build are not atomic. At the Collection boundary, both index types and both write paths (optimistic insert, sync commit) crash the collection: status becomes `error` and the next mutation throws, so a usable collection never holds rows its subscribers were not told about. Open: a sync source can still commit writes on a collection in `error` state, which stores unpublished rows; this is existing `markError` behavior and needs a lifecycle-owner witness. Comparator transitivity is outside this evidence. | -| WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | +| WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart. Wholesale and granular observers must match after every step, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React. | +| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields) run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history failed before that fix. Non-flat rows must fall back. | +| Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | | Minified DB public API | [consumer bundle check](https://github.com/TanStack/db/blob/main/scripts/test-minified-db.mjs) | CI builds `@tanstack/db`, then bundles its built `dist` entry with db-ivm inlined through esbuild `minify: true`. The build renames TypeScript-private members from `packages/db/mangle-cache.json`, so this lane runs against the renamed output that consumers install. It rejects a `dist` that is older than `src`, the cache, or the package's build configuration. `pnpm --filter @tanstack/db test:dist` also runs five db test files against the built `dist`, chosen for coverage of the modules with the most renamed members. `pnpm check:mangle` fails when a cached name is used other than as a private member in db src, is read by another package, appears as a string, or has a short name that collides with a source identifier ([private-member mangling review](oracle-reviews/code-weight-private-member-mangling.md)). It checks every exported error class's current public `name`, the built-in `BasicIndex` resolver name in index metadata and its event, complete query rows (after removing the four documented virtual fields) against an independent array filter/sort/projection, and one live update. The `new.target.name`, public-member mangle, and extra-output-field calibration modes fail at the intended observations. This fixed slice does not replace the unminified generated oracles, exercise framework adapters, or establish stable names for custom index resolvers. | | Boundary refinements | [cleanup/restart](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-cleanup-restart-oracle.test.ts), [issue #1891 async-cleanup review](oracle-reviews/issue-1891-async-cleanup.md), [issue #1891 cleanup-start follow-up](oracle-reviews/issue-1891-cleanup-start-followup.md), [metadata publication](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-metadata-publication-oracle.property.test.ts), [state retention](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-state-retention-oracle.property.test.ts), [acquisition cells](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-subscription-lifecycle-oracle.test.ts), [D2 source reconciliation](https://github.com/TanStack/db/blob/main/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts), [top-K support windows](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topk-support-window-oracle.test.ts), [nested Query work](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/includes-work-counter-oracle.test.ts) | Explicit lifecycle products, independent source maps and weighted relations, exact publication cuts, support/multiplicity, and value-plus-work observations. These refine the larger subsystem models; they do not replace them. | diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index c5df07ef3..4ec2e6940 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -366,5 +366,17 @@ "canRetryRepair": "f0", "restore": "f1", "attach": "f2", - "compilations": "f3" + "compilations": "f3", + "releaseTimer": "f4", + "commitLocalOnlyDirect": "f5", + "stopStatusEvents": "f6", + "snapshotRevision": "f7", + "scheduleRelease": "f8", + "snapshotStatus": "f9", + "layoutKeys": "ga", + "watching": "gb", + "stopWatching": "gc", + "onChanges": "gd", + "onStatus": "ge", + "onEmpty": "gf" } diff --git a/packages/db/src/live-query-observer.ts b/packages/db/src/live-query-observer.ts index e596eff67..d513a877c 100644 --- a/packages/db/src/live-query-observer.ts +++ b/packages/db/src/live-query-observer.ts @@ -308,7 +308,7 @@ class LiveQueryObserverImpl< this.cachedSnapshot = { state, data: singleResult ? data[0] : data, - // A pooled view observes its bucket directly and hands users a + // A pooled view observes its partition group directly and hands users a // Collection that is built only when touched. collection: (collection as { publicCollection?: Collection }) diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index a6ef6d918..2b30a46d7 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1235,6 +1235,21 @@ active query acquisition, and persisted retention are distinct owner tokens. The live-query graph publishes coherent rows but does not own query-db cache or listener lifetime. +### Pooled live queries + +A framework adapter may serve a live query from an equality partition instead +of this graph. That applies only when the query reads one eager, +non-persisted source Collection, filters it only by `eq(field, literal)` +conjuncts, and has no other clause, `DbClient`, or Suspense key. Such a pooled +live query reads the partition group for its literal tuple +(`packages/db/src/query/pooled-live-query.ts`). It builds its live-query +Collection only when the application reads it. + +A pooled live query must publish what its live-query Collection would: the +same rows, in key order, with the same values and status. When its source +starts cleanup, it enters the same terminal error and keeps its last rows. +The pooled live query oracle compares the two over generated histories. + ### Physical planning and work Correct relation state does not prove efficient work. When an applicable index @@ -1343,6 +1358,7 @@ keep the meanings defined there. | Replay lease balance, reference-counted peers, and failed-start recovery | `packages/db/tests/replay-adapter-ownership.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | | Cleanup-start invalidation, settlement, and restart admission | `packages/db/tests/collection-cleanup-restart-oracle.test.ts` | +| Pooled live queries match their live-query Collection, including cleanup | `packages/db/tests/query/pooled-live-query-oracle.property.test.ts` | Each oracle identifies the first divergent checkpoint and compares either the whole result or one exact structural difference. Correlated-materialization diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 182392ffb..863b1e3f0 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -21,10 +21,10 @@ import type { DehydratedLiveQueryResult } from '../client.js' /** * Live queries that filter one source Collection only by `eq(field, literal)` * share one partition of that source per filtered field set. Each query reads - * the bucket for its literal tuple, so mounting many queries of one shape + * the group for its literal tuple, so mounting many queries of one shape * costs a lookup each instead of a compiled graph and a source subscription. * - * A bucket holds the rows the partition's source subscription has published, + * A group holds the rows the partition's source subscription has published, * keyed by the same normalized equality that `eq` uses: a Date equals its * timestamp, `NaN` equals `NaN`, `-0` equals `0`, and nullish values match no * literal. @@ -34,7 +34,7 @@ type Row = Record type Listener = (changes: Array>) => void type StatusListener = CollectionEventHandler<`status:change`> -interface Bucket { +interface PartitionGroup { // Key order, as in a live-query Collection without orderBy. rows: SortedMap listeners: Set @@ -67,13 +67,13 @@ function readPath(row: Row, path: Array): unknown { } class Partition { - private readonly buckets = new Map() + private readonly groups = new Map() private subscription: { unsubscribe: () => void } | undefined private stopStatusEvents: (() => void) | undefined // One source status listener serves every view of this partition. readonly statusListeners = new Set() private listenerCount = 0 - /** Set when the source starts cleanup; buckets keep their last rows. */ + /** Set when the source starts cleanup; groups keep their last rows. */ terminated = false private releaseTimer: ReturnType | undefined @@ -83,7 +83,7 @@ class Partition { private readonly onEmpty: () => void, ) {} - bucketKeyOf(row: Row | undefined): string | undefined { + groupKeyOf(row: Row | undefined): string | undefined { if (row === undefined) return undefined const parts: Array = [] for (const path of this.paths) { @@ -94,19 +94,19 @@ class Partition { return JSON.stringify(parts) } - bucket(key: string): Bucket { - let bucket = this.buckets.get(key) - if (!bucket) { - bucket = { + group(key: string): PartitionGroup { + let group = this.groups.get(key) + if (!group) { + group = { rows: new SortedMap(), listeners: new Set(), revision: 0, layoutRevision: 0, entries: undefined, } - this.buckets.set(key, bucket) + this.groups.set(key, group) } - return bucket + return group } /** Start the shared source subscription; release it when unused. */ @@ -132,19 +132,19 @@ class Partition { this.scheduleRelease() } - listen(bucket: Bucket, listener: Listener): () => void { - this.addListener(bucket, listener) - return () => this.removeListener(bucket, listener) + listen(group: PartitionGroup, listener: Listener): () => void { + this.addListener(group, listener) + return () => this.removeListener(group, listener) } - addListener(bucket: Bucket, listener: Listener): void { + addListener(group: PartitionGroup, listener: Listener): void { this.retain() - bucket.listeners.add(listener) + group.listeners.add(listener) this.listenerCount++ } - removeListener(bucket: Bucket, listener: Listener): void { - if (!bucket.listeners.delete(listener)) return + removeListener(group: PartitionGroup, listener: Listener): void { + if (!group.listeners.delete(listener)) return this.listenerCount-- this.scheduleRelease() } @@ -169,57 +169,57 @@ class Partition { this.subscription = undefined this.stopStatusEvents?.() this.stopStatusEvents = undefined - this.buckets.clear() + this.groups.clear() this.onEmpty() }, 1000) } private apply(changes: Array>): void { const touched = new Map< - Bucket, + PartitionGroup, Array> >() const record = ( - bucket: Bucket, + group: PartitionGroup, change: ChangeMessage, ) => { - const list = touched.get(bucket) + const list = touched.get(group) if (list) list.push(change) - else touched.set(bucket, [change]) - if (change.type !== `update`) bucket.layoutRevision++ + else touched.set(group, [change]) + if (change.type !== `update`) group.layoutRevision++ } for (const change of changes) { const next = - change.type === `delete` ? undefined : this.bucketKeyOf(change.value) + change.type === `delete` ? undefined : this.groupKeyOf(change.value) const previous = change.type === `insert` ? undefined - : this.bucketKeyOf( + : this.groupKeyOf( change.type === `delete` ? change.value : change.previousValue, ) if (previous !== undefined && previous !== next) { - const bucket = this.bucket(previous) - const old = bucket.rows.get(change.key) - if (bucket.rows.delete(change.key)) { - record(bucket, { type: `delete`, key: change.key, value: old! }) + const group = this.group(previous) + const old = group.rows.get(change.key) + if (group.rows.delete(change.key)) { + record(group, { type: `delete`, key: change.key, value: old! }) } } if (next !== undefined) { - const bucket = this.bucket(next) - const existed = bucket.rows.has(change.key) - bucket.rows.set(change.key, change.value) + const group = this.group(next) + const existed = group.rows.has(change.key) + group.rows.set(change.key, change.value) record( - bucket, + group, existed ? { ...change, type: `update` } : { type: `insert`, key: change.key, value: change.value }, ) } } - for (const [bucket, bucketChanges] of touched) { - bucket.revision++ - bucket.entries = undefined - for (const listener of [...bucket.listeners]) listener(bucketChanges) + for (const [group, groupChanges] of touched) { + group.revision++ + group.entries = undefined + for (const listener of [...group.listeners]) listener(groupChanges) } } } @@ -266,7 +266,7 @@ function collectConjuncts( function poolableShape( query: QueryIR, ): - | { paths: Array>; shapeKey: string; bucketKey: string } + | { paths: Array>; shapeKey: string; groupKey: string } | undefined { if ( query.from.type !== `collectionRef` || @@ -300,12 +300,12 @@ function poolableShape( return { paths: conjuncts.map(({ path }) => path), shapeKey: conjuncts.map(({ pathKey }) => pathKey).join(`,`), - bucketKey: JSON.stringify(conjuncts.map(({ literalKey }) => literalKey)), + groupKey: JSON.stringify(conjuncts.map(({ literalKey }) => literalKey)), } } /** - * One query's view of its bucket. It answers the calls the live-query observer + * One query's view of its group. It answers the calls the live-query observer * makes; any other Collection member builds the query's live-query Collection * once and forwards to it, so `result.collection` keeps its full API. */ @@ -314,16 +314,16 @@ class PooledLiveQuery { // No persisted readiness, single-result config, or layout channel. readonly config = undefined readonly _subscribeLayoutChanges = undefined - private readonly bucket: Bucket + private readonly group: PartitionGroup private collection: Collection | undefined = undefined constructor( private readonly source: CollectionImpl, private readonly query: BaseQueryBuilder, private readonly partition: Partition, - bucketKey: string, + groupKey: string, ) { - this.bucket = partition.bucket(bucketKey) + this.group = partition.group(groupKey) partition.retain() } @@ -332,25 +332,25 @@ class PooledLiveQuery { } get _stateRevision(): number { - return this.bucket.revision + return this.group.revision } get _layoutRevision(): number { - return this.bucket.layoutRevision + return this.group.layoutRevision } entries(): Array<[string | number, Row]> { - return (this.bucket.entries ??= [...this.bucket.rows.entries()]) + return (this.group.entries ??= [...this.group.rows.entries()]) } subscribeChanges( callback: Listener, options: { includeInitialState?: boolean } = {}, ): { unsubscribe: () => void } { - const unsubscribe = this.partition.listen(this.bucket, callback) + const unsubscribe = this.partition.listen(this.group, callback) if (options.includeInitialState) { callback( - [...this.bucket.rows].map(([key, value]) => ({ + [...this.group.rows].map(([key, value]) => ({ type: `insert`, key, value, @@ -374,12 +374,12 @@ class PooledLiveQuery { /** Observe changes and status without allocating unsubscribe closures. */ watch(onChanges: Listener, onStatus: StatusListener): void { - this.partition.addListener(this.bucket, onChanges) + this.partition.addListener(this.group, onChanges) this.partition.statusListeners.add(onStatus) } unwatch(onChanges: Listener, onStatus: StatusListener): void { - this.partition.removeListener(this.bucket, onChanges) + this.partition.removeListener(this.group, onChanges) this.partition.statusListeners.delete(onStatus) } @@ -613,6 +613,6 @@ export function createPooledLiveQuery( source, query, partition, - shape.bucketKey, + shape.groupKey, ) as unknown as Collection } diff --git a/packages/db/tests/conformance/suite.ts b/packages/db/tests/conformance/suite.ts index 84367f785..67afd4034 100644 --- a/packages/db/tests/conformance/suite.ts +++ b/packages/db/tests/conformance/suite.ts @@ -290,14 +290,12 @@ export function runSuite(rawDriver: LiveQueryDriver) { source.remove({ id: `3`, name: `John Smith`, age: 35, team: `a` }) await h.flush() - expectOrderedRows(h.current().data, [ - { id: `2`, name: `Jane Doe`, age: 25, team: `a` }, - { id: `4`, name: `Dave`, age: 40, team: `a` }, - ]) - expectKeyedRows(h.current().state, [ + const remaining: Array = [ { id: `2`, name: `Jane Doe`, age: 25, team: `a` }, { id: `4`, name: `Dave`, age: 40, team: `a` }, - ]) + ] + expectOrderedRows(h.current().data, remaining) + expectKeyedRows(h.current().state, remaining) expect(h.current().status).toBe(`ready`) h.unmount() }, diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 55e781d57..36b23f92c 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -11,8 +11,8 @@ * key order, with the same row values and status. * * Why an example can miss the failure: one query over static rows passes even - * if rows never move between buckets, a peer bucket never sees a row leave, a - * remounted query reads a stale bucket, or a rollback leaves an optimistic row + * if rows never move between groups, a peer group never sees a row leave, a + * remounted query reads a stale group, or a rollback leaves an optimistic row * behind. * * Model: `expectedKeys` filters the model's visible rows with an independent @@ -39,7 +39,7 @@ * rows it had, through the restart and later writes. A peer mounted after * the restart follows the restarted source. * - * Calibration: a partition that ignored the previous value, kept bucket rows + * Calibration: a partition that ignored the previous value, kept group rows * in arrival order, or compared literals without normalization fails the * pinned histories and both campaigns. A view that kept reporting the * source's status after cleanup fails the cleanup history. @@ -205,11 +205,11 @@ const historyArbitrary: fc.Arbitrary = fc.record({ ), }) -// Each pinned history moves a row across buckets a short example would keep +// Each pinned history moves a row across groups a short example would keep // still. const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ { - name: `a Date row joins the bucket of its timestamp`, + name: `a Date row joins the group of its timestamp`, history: { rows: [{ f: `a`, g: `x` }], peers: [{ f: 1 }, { f: `a` }], @@ -234,7 +234,7 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ }, }, { - name: `a rolled-back optimistic insert leaves its bucket`, + name: `a rolled-back optimistic insert leaves its group`, history: { rows: [], peers: [{ f: true, g: `y` }], @@ -262,7 +262,7 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ }, }, { - name: `a remounted peer reads a bucket that changed while it was away`, + name: `a remounted peer reads a group that changed while it was away`, history: { rows: [{ f: 0, g: `x` }], peers: [{ f: 0 }, { f: Number.NaN }], From e4da7e203b1d80fb25381ed225e5b13f793a4ef4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 10:33:37 -0600 Subject: [PATCH 09/63] docs(db): review the pooled live query and flat tracking oracles Record each ORC-001 to ORC-014 outcome with classified mutant runs at this head, and link the record from the coverage map. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 4 +- .../2026-10-01-pooled-live-queries.md | 102 ++++++++++++++++++ 2 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 92354dbc8..4a7d71b56 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -265,8 +265,8 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart. Wholesale and granular observers must match after every step, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React. | -| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields) run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history failed before that fix. Non-flat rows must fall back. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart. Wholesale and granular observers must match after every step, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields) run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history failed before that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | | Minified DB public API | [consumer bundle check](https://github.com/TanStack/db/blob/main/scripts/test-minified-db.mjs) | CI builds `@tanstack/db`, then bundles its built `dist` entry with db-ivm inlined through esbuild `minify: true`. The build renames TypeScript-private members from `packages/db/mangle-cache.json`, so this lane runs against the renamed output that consumers install. It rejects a `dist` that is older than `src`, the cache, or the package's build configuration. `pnpm --filter @tanstack/db test:dist` also runs five db test files against the built `dist`, chosen for coverage of the modules with the most renamed members. `pnpm check:mangle` fails when a cached name is used other than as a private member in db src, is read by another package, appears as a string, or has a short name that collides with a source identifier ([private-member mangling review](oracle-reviews/code-weight-private-member-mangling.md)). It checks every exported error class's current public `name`, the built-in `BasicIndex` resolver name in index metadata and its event, complete query rows (after removing the four documented virtual fields) against an independent array filter/sort/projection, and one live update. The `new.target.name`, public-member mangle, and extra-output-field calibration modes fail at the intended observations. This fixed slice does not replace the unminified generated oracles, exercise framework adapters, or establish stable names for custom index resolvers. | diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md new file mode 100644 index 000000000..28fde758f --- /dev/null +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -0,0 +1,102 @@ +# Pooled live query and flat change tracking oracle review + +## Reviewed state and claim + +Base: `84b828c7e`, merged with `origin/main` at `18abceee4`. This record reviews +the Git tree that contains it; the final commit or pull request identifies that +tree. + +The pooled live query oracle claims that a live query filtered only by +`eq(field, literal)` on one eager, non-persisted source, served from an +equality partition, publishes what its live-query Collection would: the same +rows in key order, values, and status. That holds through sync transactions, +optimistic writes confirmed or rolled back, peer mounts and unmounts, and +source cleanup and restart. The claim excludes on-demand and persisted +sources, `DbClient` hydration, Suspense, and every clause beyond `eq` +conjuncts, which keep the live-query Collection. + +The flat change tracking oracle claims that, for a row whose own fields are +all primitives or functions with a plain or null prototype and no symbol +keys, the flat tracker reports the same change set as the draft proxy and an +independent model, with `0` and `-0` equal. Nested values, Dates, Maps, Sets, +class instances, and symbol keys are outside it, except that they must fall +back to the proxy. + +Both oracles use `mockSyncCollectionOptions` or plain rows; neither claims a +real sync adapter's behavior. + +## RED and GREEN evidence + +Mutants ran through each oracle file alone on the reviewed tree. Each file was +restored after each run. Every outcome below is an assertion failure. + +| Mutant | Oracle | Tests failed | +| --- | --- | --- | +| Partition ignores a row's previous group | Pooled | 6 of 7 | +| Groups keep rows in arrival order | Pooled | 2 of 7 (pinned key-order history and one campaign) | +| Literals and fields compared without `eq` normalization | Pooled | 3 of 7 | +| Partition skips the source's initial state | Pooled | 6 of 7 | +| View reports the source's status after cleanup | Pooled | 3 of 7 | +| Partition never terminates on cleanup | Pooled | 3 of 7 | +| Flat diff with `!==` | Flat | 3 of 8 | +| Flat diff with `Object.is` alone | Flat | 3 of 8 | +| Flat diff without deletions | Flat | 3 of 8 | +| Draft proxy without the added-field revert fix | Flat | 1 of 8 (pinned history only) | + +The pooled oracle found that a pooled view followed its source's status after +cleanup instead of entering the live query's terminal error; the repair makes +the partition terminate. The flat oracle found that the draft proxy dropped a +field added as `undefined` when another field reverted, on `main` as well; the +repair treats a field the original lacks as changed. The generated flat +campaigns did not reach that history; only its pinned case kills the +unrepaired proxy. + +Two shared conformance scenarios, `eq-filter-rows` and `eq-filter-peers`, run +the pooled path under React and compiled live queries under Vue, Solid, +Svelte, and Angular. A partition that ignores a row's previous group fails +both under React and none elsewhere. + +## Pooled live query oracle + +| Requirement | Outcome | +| --- | --- | +| ORC-001 Contract authority and limits | Pass. The `eq` operand rules come from `src/query/compiler/evaluators.ts`; the pooled boundary and the terminal-error rule come from the live-query architecture document's pooled section and cleanup law. The opening prose lists the omissions. | +| ORC-002 Independent judgment | Pass. `expectedKeys` uses a local `eq` over plain values. Order, values, and status come from a live-query Collection, which compiles a D2 pipeline and does not use the partition. | +| ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses only domain values and step kinds. Ablation: removing Dates, `NaN`, or `-0` loses the normalization mutant; removing optimistic steps loses rollback; removing mount and unmount loses remounted groups; removing cleanup-restart loses the terminal-error mutants; removing the second conjunct loses multi-field groups. Range: at most four rows, three peers, eight steps, and ids 0 through 5. Exclusion: an optimistic update to an equal value, an insert of an existing id, and an update or delete of a missing id are dropped. | +| ORC-005 Production path and observation | Pass. `createPooledLiveQuery` and `createLiveQueryObserver`, the adapter seam, run in wholesale and granular mode; the conformance scenarios run through React's `useLiveQuery`. Rows, keyed state, status, layout revision, and non-materialization are observed after every step. | +| ORC-006 Checker calibration | Pass. Six mutants, classified above. | +| ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_001`, an unseeded campaign, and a replay entry share one property and budget; the file is in `test:oracles`. | +| ORC-008 Stateful-model minimality | Pass. The model keeps source rows and, per mounted peer, the frozen keys at cleanup. The pinned cleanup history distinguishes a frozen peer from one mounted after the restart. | +| ORC-009 Vocabulary mapping | Pass. Equality partition, partition group, and pooled live query are glossary terms; a peer is one mounted pooled live query. | +| ORC-010 Failure fidelity and cleanup | Pass. `withOracleCleanup` releases observers, references, and the source and keeps the check failure. | +| ORC-011 Independent second formulation | Pass. The live-query Collection is the second formulation for order, values, and status. | +| ORC-012 Review evidence | This record; the coverage map links it. | +| ORC-013 Reusable boundary law | Pass. Normalization is rejected by the Date history, arrival order by the key-order history, and following the source's status by the cleanup history. | +| ORC-014 Controlled-premise handoff | Not triggered. The claim is limited to the mock source's sync transactions and lifecycle. | + +## Flat change tracking oracle + +| Requirement | Outcome | +| --- | --- | +| ORC-001 Contract authority and limits | Pass. The change-set rules come from `src/proxy.ts`'s `getChanges` contract: changed fields with their final value and deleted fields as `undefined`. | +| ORC-002 Independent judgment | Pass. `expectedChanges` folds the operations over a plain copy and does not import either tracker. | +| ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses domain values and operations. Ablation: removing `NaN`, `-0`, reverts, or deletions each loses a mutant. Range: one to three rows, three fields plus one added field, five operations per row. Exclusion: non-flat rows are rejected by the fallback witness. | +| ORC-005 Production path and observation | Pass. `withFlatChangeTracking`, `withArrayChangeTracking`, and `withChangeTracking` run the same callbacks; the result change sets are observed. `collection.update` selects between them. | +| ORC-006 Checker calibration | Pass. Four mutants, classified above. | +| ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_101`, an unseeded campaign, and a replay entry; the file is in `test:oracles`. | +| ORC-008 Stateful-model minimality | Not triggered. The model recomputes from the operations. | +| ORC-009 Vocabulary mapping | Pass. Draft, change set, and revert follow the proxy's terms. | +| ORC-010 Failure fidelity and cleanup | Not triggered. The oracle holds no resources. | +| ORC-011 Independent second formulation | Pass. The draft proxy is the second formulation. | +| ORC-012 Review evidence | This record; the coverage map links it. | +| ORC-013 Reusable boundary law | Pass. `!==` is rejected by the `NaN` history, `Object.is` by the `-0` history, and missing deletions by the deleted-field history. | +| ORC-014 Controlled-premise handoff | Not triggered. No provider is involved. | + +## Open work + +- The generated flat campaigns do not reach the added-field revert history; + only its pinned case covers it. +- Pooled live queries run only in React; the other adapters keep compiled live + queries until they use the shared resolver. From f8ade57cdbc804d9ad48150dd2c4c81d239318ac Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 10:51:07 -0600 Subject: [PATCH 10/63] fix(db): forward every pooled `result.collection` member to its Collection The public Collection of a pooled live query answered members the observer also reads from the internal view: `subscribeChanges` ignored a `whereExpression`, `entries()` returned an array, and `config` was undefined. The observer reads the view itself, so the public proxy now forwards every member to the live-query Collection it builds on first use. The pooled live query oracle now compares forwarded `entries()`, a filtered `currentStateAsChanges`, and `config` with the live-query Collection; the view-first handler fails every history. Found by a loss audit of old versus new behavior. Co-authored-by: Isaac --- packages/db/src/query/pooled-live-query.ts | 11 ++++++---- .../pooled-live-query-oracle.property.test.ts | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 863b1e3f0..207e84afa 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -305,9 +305,9 @@ function poolableShape( } /** - * One query's view of its group. It answers the calls the live-query observer - * makes; any other Collection member builds the query's live-query Collection - * once and forwards to it, so `result.collection` keeps its full API. + * One query's view of its group, read by the live-query observer. Users get + * `publicCollection` instead, which builds the query's live-query Collection + * on first use and forwards every member to it. */ class PooledLiveQuery { readonly isLoadingSubset = false @@ -569,13 +569,16 @@ export function createPooledObserver< ) as unknown as LiveQueryObserver } +// The observer reads the view itself; users get the live-query Collection. const forwardToCollection: ProxyHandler = { get(view, property) { - if (property in view) return Reflect.get(view, property, view) const collection = view.materialize() const value = Reflect.get(collection, property, collection) return typeof value === `function` ? value.bind(collection) : value }, + has(view, property) { + return Reflect.has(view.materialize(), property) + }, } /** diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 36b23f92c..ec2ec2ced 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -55,6 +55,7 @@ import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { Query } from '../../src/query/builder/index.js' import { and, createLiveQueryCollection, eq } from '../../src/query/index.js' import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' +import { Func, PropRef, Value } from '../../src/query/ir.js' import { oraclePropertyOptions, oracleRuns, @@ -511,6 +512,25 @@ async function runHistory(history: History): Promise { (collection.toArray as Array>).map(describeRow), `peer ${index} forwarded toArray`, ).toEqual(reference.toArray.map((row) => describeRow(row))) + // Members the observer also reads must still behave as the Collection's. + expect( + [...collection.entries()].map(([key]) => key), + `peer ${index} forwarded entries`, + ).toEqual([...reference.entries()].map(([key]) => key)) + const narrower = new Func(`eq`, [new PropRef([`g`]), new Value(`x`)]) + const keysOf = (target: { + currentStateAsChanges: (options: { + where: typeof narrower + }) => Array<{ key: unknown }> | void + }) => + [...(target.currentStateAsChanges({ where: narrower }) || [])].map( + (change) => change.key, + ) + expect( + keysOf(collection as unknown as Parameters[0]), + `peer ${index} forwarded filter`, + ).toEqual(keysOf(reference as unknown as Parameters[0])) + expect(collection.config, `peer ${index} forwarded config`).toBeDefined() } }, [ () => { From 2d36ddbe486255695d2eee0be7657bfced74f8c9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 11:05:15 -0600 Subject: [PATCH 11/63] fix(db): honor gcTime for pooled live queries A partition released its source a fixed second after its last listener. It now waits for the longest gcTime among its views, never releases for gcTime 0 or Infinity, and gives a never-subscribed query the same 50 ms floor as the live-query Collection lifecycle. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- .../2026-10-01-pooled-live-queries.md | 8 ++ packages/db/src/live-query-options.ts | 2 +- packages/db/src/query/pooled-live-query.ts | 35 +++++- .../tests/query/pooled-live-query-gc.test.ts | 107 ++++++++++++++++++ 5 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 packages/db/tests/query/pooled-live-query-gc.test.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 4a7d71b56..4a18da449 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -265,7 +265,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart. Wholesale and granular observers must match after every step, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart. Wholesale and granular observers must match after every step, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and checks that a partition waits for its views' longest `gcTime`; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields) run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history failed before that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md index 28fde758f..ef470c174 100644 --- a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -56,6 +56,14 @@ the pooled path under React and compiled live queries under Vue, Solid, Svelte, and Angular. A partition that ignores a row's previous group fails both under React and none elsewhere. +The loss audit found that a partition released its source one second after +its last listener, whatever `gcTime` its views had. A focused release-timing +test, `pooled-live-query-gc.test.ts`, now compares pooled and live-query +Collection release times. A fixed delay failed 5 of 10 cases, a missing 50 ms +floor for unsubscribed queries 1, a last-`gcTime`-wins rule 1, and releasing +at `gcTime` 0 2. Release timing is resource lifetime, so the publication +oracle does not observe it. + ## Pooled live query oracle | Requirement | Outcome | diff --git a/packages/db/src/live-query-options.ts b/packages/db/src/live-query-options.ts index 76529fb96..ee8fe1d22 100644 --- a/packages/db/src/live-query-options.ts +++ b/packages/db/src/live-query-options.ts @@ -162,7 +162,7 @@ export function resolveLiveQueryValue( } if (value instanceof BaseQueryBuilder) { return ( - (pool ? createPooledLiveQuery(value) : undefined) ?? + (pool ? createPooledLiveQuery(value, { gcTime }) : undefined) ?? createLiveQueryCollection({ query: value, startSync: true, gcTime }) ) } diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 207e84afa..2473cfb65 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -44,6 +44,9 @@ interface PartitionGroup { entries: Array<[string | number, Row]> | undefined } +// Matches the Collection lifecycle's floor for a never-subscribed Collection. +const UNSUBSCRIBED_RELEASE_FLOOR_MS = 50 + const partitionsBySource = new WeakMap>() // Typed so that 1, '1', and true stay distinct. @@ -73,6 +76,8 @@ class Partition { // One source status listener serves every view of this partition. readonly statusListeners = new Set() private listenerCount = 0 + private gcTime = 0 + private hadListener = false /** Set when the source starts cleanup; groups keep their last rows. */ terminated = false private releaseTimer: ReturnType | undefined @@ -110,7 +115,17 @@ class Partition { } /** Start the shared source subscription; release it when unused. */ - retain(): void { + /** + * Keep the shared source subscription open. Each view brings its query's + * `gcTime`; the partition keeps the longest, so it never releases before + * one of its views' own live-query Collection would have. + */ + retain(gcTime?: number): void { + if (gcTime !== undefined) { + // As for a Collection, a non-positive or non-finite gcTime disables GC. + const delay = gcTime > 0 && Number.isFinite(gcTime) ? gcTime : Infinity + this.gcTime = Math.max(this.gcTime, delay) + } if (this.terminated) return if (!this.subscription) { this.subscription = this.source.subscribeChanges( @@ -139,6 +154,7 @@ class Partition { addListener(group: PartitionGroup, listener: Listener): void { this.retain() + this.hadListener = true group.listeners.add(listener) this.listenerCount++ } @@ -160,8 +176,12 @@ class Partition { private scheduleRelease(): void { if (this.releaseTimer !== undefined) clearTimeout(this.releaseTimer) this.releaseTimer = undefined - if (this.listenerCount > 0) return - // A rendered query may subscribe shortly after construction. + if (this.listenerCount > 0 || !Number.isFinite(this.gcTime)) return + // Like a Collection that synced before anything subscribed, a view built + // during a render gets a grace period to subscribe when it commits. + const delay = this.hadListener + ? this.gcTime + : Math.max(this.gcTime, UNSUBSCRIBED_RELEASE_FLOOR_MS) this.releaseTimer = setTimeout(() => { this.releaseTimer = undefined if (this.listenerCount > 0) return @@ -171,7 +191,7 @@ class Partition { this.stopStatusEvents = undefined this.groups.clear() this.onEmpty() - }, 1000) + }, delay) } private apply(changes: Array>): void { @@ -322,9 +342,10 @@ class PooledLiveQuery { private readonly query: BaseQueryBuilder, private readonly partition: Partition, groupKey: string, + private readonly gcTime: number, ) { this.group = partition.group(groupKey) - partition.retain() + partition.retain(gcTime) } get status(): CollectionStatus { @@ -401,6 +422,7 @@ class PooledLiveQuery { return (this.collection ??= createLiveQueryCollection({ query: this.query, startSync: true, + gcTime: this.gcTime, })) } } @@ -587,6 +609,8 @@ const forwardToCollection: ProxyHandler = { */ export function createPooledLiveQuery( query: BaseQueryBuilder, + // A Collection's default when the adapter gives none. + { gcTime = 300_000 }: { gcTime?: number } = {}, ): Collection | undefined { const ir = query._getQuery() const shape = poolableShape(ir) @@ -617,5 +641,6 @@ export function createPooledLiveQuery( query, partition, shape.groupKey, + gcTime, ) as unknown as Collection } diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts new file mode 100644 index 000000000..a22bc72b0 --- /dev/null +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -0,0 +1,107 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createLiveQueryObserver } from '../../src/live-query-observer.js' +import { Query } from '../../src/query/builder/index.js' +import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' +import { mockSyncCollectionOptions } from '../utils.js' + +/** + * # Does a pooled live query hold its source as long as its Collection would? + * + * Law: a pooled live query keeps its source subscribed for the same time as + * the live-query Collection it stands in for. After the last subscriber + * leaves, that is exactly `gcTime`; a `gcTime` of 0 or `Infinity` never + * releases; and a query built but never subscribed waits at least the + * Collection lifecycle's 50 ms floor. Views of one partition with different + * `gcTime`s release with the longest. + * + * Each case runs the same timeline against a pooled query and a live-query + * Collection on separate sources and compares when each source loses its + * subscriber, allowing the one tick the Collection's cleanup queue adds. + * Release timing is resource lifetime, not data, so the pooled + * live query oracle does not observe it. + */ +type Row = { id: string; g: string } +let serial = 0 + +function makeSource() { + return createCollection( + mockSyncCollectionOptions({ + id: `pooled-gc-${serial++}`, + getKey: (row) => row.id, + initialData: [{ id: `a`, g: `x` }], + }), + ) +} + +const query = (source: ReturnType) => (q: any) => + q.from({ r: source }).where(({ r }: any) => eq(r.g, `x`)) + +// Milliseconds after which the source no longer has a subscriber, checking +// up to `horizon`; undefined when it keeps one throughout. +async function releaseTime( + source: ReturnType, + horizon: number, +): Promise { + for (let elapsed = 0; elapsed <= horizon; elapsed++) { + if (source.subscriberCount === 0) return elapsed + // Collection cleanup settles through promises after its timer fires. + await vi.advanceTimersByTimeAsync(1) + } + return undefined +} + +function pooled(gcTime: number, subscribe: boolean) { + const source = makeSource() + const view = createPooledLiveQuery(query(source)(new Query()), { gcTime })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + if (subscribe) observer.subscribe(() => {})() + return source +} + +function compiled(gcTime: number, subscribe: boolean) { + const source = makeSource() + const live = createLiveQueryCollection({ + query: query(source), + startSync: true, + gcTime, + }) + const observer = createLiveQueryObserver(live, { mode: `wholesale` }) + if (subscribe) observer.subscribe(() => {})() + return source +} + +describe(`pooled live query gcTime`, () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + for (const gcTime of [1, 100, 0, Number.POSITIVE_INFINITY]) { + for (const subscribe of [true, false]) { + it(`matches a live-query Collection for gcTime ${gcTime}${subscribe ? `` : ` without a subscriber`}`, async () => { + const expected = await releaseTime(compiled(gcTime, subscribe), 400) + const actual = await releaseTime(pooled(gcTime, subscribe), 400) + if (expected === undefined) expect(actual).toBeUndefined() + // The Collection releases its source one cleanup-queue tick after + // its GC timer fires; the partition releases in the timer itself. + else expect([expected - 1, expected]).toContain(actual) + }) + } + } + + it.each([[[5, 120]], [[120, 5]]])( + `releases with the longest gcTime among a partition's views (%j)`, + async (gcTimes) => { + const source = makeSource() + for (const gcTime of gcTimes) { + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime, + })! + createLiveQueryObserver(view, { mode: `wholesale` }).subscribe( + () => {}, + )() + } + expect(await releaseTime(source, 400)).toBe(120) + }, + ) +}) From a52ddf4d855c7c58d7798ca762c4c153ee62eff0 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 11:08:19 -0600 Subject: [PATCH 12/63] fix(db): keep PropRef.sourceAlias read-only Restores the non-writable own property from main. Costs about 0.15 us per four-ref query build. Co-authored-by: Isaac --- packages/db/src/query/ir.ts | 6 ++++-- packages/db/tests/query/ir-stable-identity.test.ts | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 8940ad287..e2210e6ca 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -149,9 +149,11 @@ export class PropRef extends BaseExpression { sourceAlias?: string, ) { super() - // Present only when given, so unqualified refs keep their shape. if (sourceAlias !== undefined) { - ;(this as { sourceAlias?: string }).sourceAlias = sourceAlias + Object.defineProperty(this, `sourceAlias`, { + value: sourceAlias, + enumerable: true, + }) } } } diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index bf4add9a5..60962e1be 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -642,6 +642,18 @@ describe(`loadSubset demand identity`, () => { ) }) + it(`keeps an explicit source alias read-only`, () => { + const qualified = new PropRef([`profile`, `score`], `profile`) + + expect(Object.getOwnPropertyDescriptor(qualified, `sourceAlias`)).toEqual({ + value: `profile`, + enumerable: true, + writable: false, + configurable: false, + }) + expect(Object.hasOwn(new PropRef([`profile`]), `sourceAlias`)).toBe(false) + }) + const id = new PropRef([`id`]) const group = new PropRef([`group`]) const first = new Func(`eq`, [id, new Value(`a`)]) From bc24f183a31c5d0e8c11997dab8768e5e8b9bcc7 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 11:13:19 -0600 Subject: [PATCH 13/63] revert(db): let PropRef.sourceAlias be a plain own property Nothing writes it, readers already check it against path[0], and the TypeScript declaration stays readonly; the defineProperty lock cost about 0.15 us per query build. Co-authored-by: Isaac --- packages/db/src/query/ir.ts | 6 ++---- .../db/tests/query/ir-stable-identity.test.ts | 16 +++++++--------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index e2210e6ca..8940ad287 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -149,11 +149,9 @@ export class PropRef extends BaseExpression { sourceAlias?: string, ) { super() + // Present only when given, so unqualified refs keep their shape. if (sourceAlias !== undefined) { - Object.defineProperty(this, `sourceAlias`, { - value: sourceAlias, - enumerable: true, - }) + ;(this as { sourceAlias?: string }).sourceAlias = sourceAlias } } } diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index 60962e1be..a220c79b2 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -642,16 +642,14 @@ describe(`loadSubset demand identity`, () => { ) }) - it(`keeps an explicit source alias read-only`, () => { - const qualified = new PropRef([`profile`, `score`], `profile`) - - expect(Object.getOwnPropertyDescriptor(qualified, `sourceAlias`)).toEqual({ - value: `profile`, - enumerable: true, - writable: false, - configurable: false, - }) + it(`gives an unqualified ref no source alias property`, () => { expect(Object.hasOwn(new PropRef([`profile`]), `sourceAlias`)).toBe(false) + expect( + Object.hasOwn( + new PropRef([`profile`, `score`], `profile`), + `sourceAlias`, + ), + ).toBe(true) }) const id = new PropRef([`id`]) From c8a48615257c54180639234d65e73a561dc19681 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 11:16:12 -0600 Subject: [PATCH 14/63] perf(react-db): attach live query result info only for Suspense Only useLiveSuspenseQuery reads it, and defining it cost about 120 ns on every render of every useLiveQuery. Co-authored-by: Isaac --- packages/react-db/src/useLiveQuery.ts | 15 +++++++++------ packages/react-db/tests/useLiveQuery.test.tsx | 14 +++++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index aaea5e304..36335c0c9 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -1038,11 +1038,14 @@ function useLiveQueryImpl( () => observer.getSnapshot(), () => observer.getServerSnapshot(), ) - setLiveQueryResultInfo(returned, { - client: dbClient, - queryHash: queryHashRef.current, - identityError: identityErrorRef.current, - observer, - }) + // Only useLiveSuspenseQuery reads this, and it costs a define per render. + if (forSuspense) { + setLiveQueryResultInfo(returned, { + client: dbClient, + queryHash: queryHashRef.current, + identityError: identityErrorRef.current, + observer, + }) + } return returned as any } diff --git a/packages/react-db/tests/useLiveQuery.test.tsx b/packages/react-db/tests/useLiveQuery.test.tsx index ad2574193..7edca3327 100644 --- a/packages/react-db/tests/useLiveQuery.test.tsx +++ b/packages/react-db/tests/useLiveQuery.test.tsx @@ -16,7 +16,7 @@ import { toArray, } from '@tanstack/db' import { useEffect } from 'react' -import { useLiveQuery } from '../src/useLiveQuery' +import { useLiveQuery, useLiveQueryForSuspense } from '../src/useLiveQuery' import { getLiveQueryResultInfo } from '../src/live-query-internals' import { DbProvider } from '../src/DbProvider' import { @@ -3621,15 +3621,19 @@ describe(`Query Collections`, () => { initialData: initialPersons, }), ) + // Suspense is the reader of this identity; plain useLiveQuery skips it. const first = renderHook(() => - useLiveQuery((q) => q.from({ people: collection }), [1]), + useLiveQueryForSuspense( + (q: any) => q.from({ people: collection }), + [1], + ), ) const second = renderHook(() => - useLiveQuery( - (q) => + useLiveQueryForSuspense( + (q: any) => q .from({ people: collection }) - .where(({ people }) => gt(people.age, 30)), + .where(({ people }: any) => gt(people.age, 30)), [1], ), ) From ff7079addacfe4c6d11e8f809815610fe47de1d3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 11:16:16 -0600 Subject: [PATCH 15/63] test(db): compare local-only direct writes with the handler path Covers insert, update, and delete fallbacks, schema rejection, handler rollback, and mixed multi-key batches; each guard's mutant fails. Co-authored-by: Isaac --- .../db/tests/local-only-direct-write.test.ts | 277 +++++++++++++++++- 1 file changed, 272 insertions(+), 5 deletions(-) diff --git a/packages/db/tests/local-only-direct-write.test.ts b/packages/db/tests/local-only-direct-write.test.ts index a90346d61..2f2847402 100644 --- a/packages/db/tests/local-only-direct-write.test.ts +++ b/packages/db/tests/local-only-direct-write.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest' +import { z } from 'zod' import { createCollection } from '../src/index' import { localOnlyCollectionOptions } from '../src/local-only' import { createTransaction } from '../src/transactions' @@ -18,14 +19,22 @@ import { createDeferred } from '../src/deferred' * The change-event history oracle drives this path through generated * histories and checks every publication. These witnesses pin the guard's * fallback cases, which that oracle does not generate: removing the guard - * hides a direct update under a pending overlay and holds it behind a - * persisting transaction. + * hides a direct write under a pending overlay and holds it behind a + * persisting transaction, for every operation type. + * + * A local-only Collection whose handlers all resolve is the reference for + * the direct path: it confirms the same writes through the optimistic stage. + * Both must leave the same rows, publish the same batches (order within a + * batch aside), throw the same error for a rejected write without publishing + * it, and end with completed transactions and none left on the Collection. */ type Row = { id: number; value: string } -function createOrders( - handlers: Partial<{ onUpdate: () => Promise }> = {}, -) { +type Handlers = Partial< + Record<`onInsert` | `onUpdate` | `onDelete`, () => Promise> +> + +function createOrders(handlers: Handlers = {}) { return createCollection( localOnlyCollectionOptions({ id: `local-only-direct-${Math.random()}`, @@ -149,3 +158,261 @@ describe(`local-only direct writes`, () => { expect(orders.insert({ id: 4, value: `d` }).state).toBe(`completed`) }) }) + +const confirmEverything: Handlers = { + onInsert: () => Promise.resolve(), + onUpdate: () => Promise.resolve(), + onDelete: () => Promise.resolve(), +} + +type Orders = ReturnType +type Write = (orders: Orders) => unknown + +// Runs writes in order and records what a caller could observe. A write +// that throws is recorded and the remaining writes still run. +async function observe(handlers: Handlers, writes: Array) { + const orders = createOrders(handlers) + const batches: Array> = [] + orders.subscribeChanges( + (changes) => + batches.push( + changes + .map( + ({ type, key, value }) => + `${type}:${key}:${value.value}:${String(value.$synced)}`, + ) + .sort(), + ), + { includeInitialState: true }, + ) + const errors: Array = [] + const transactions = [] + for (const write of writes) { + try { + transactions.push(write(orders) as ReturnType) + } catch (error) { + errors.push(`${(error as Error).name}: ${(error as Error).message}`) + } + } + await Promise.all(transactions.map((t) => t.isPersisted.promise)) + return { + rows: orders.toArray.map(({ id, value }) => ({ id, value })), + batches, + errors, + states: transactions.map((t) => t.state), + remaining: orders._state.transactions.size, + } +} + +describe(`local-only direct writes match the confirmed optimistic path`, () => { + const histories: Record> = { + 'a mixed multi-key batch': [ + (orders) => + orders.insert([ + { id: 2, value: `b` }, + { id: 3, value: `c` }, + ]), + (orders) => + orders.update([1, 2], (drafts) => { + for (const draft of drafts) draft.value += `!` + }), + (orders) => orders.delete([1, 3]), + ], + 'an insert batch with an existing key': [ + (orders) => + orders.insert([ + { id: 4, value: `d` }, + { id: 1, value: `dup` }, + ]), + (orders) => orders.insert({ id: 5, value: `e` }), + ], + 'an insert batch that repeats a key': [ + (orders) => + orders.insert([ + { id: 4, value: `d` }, + { id: 4, value: `again` }, + ]), + ], + 'an update batch with a missing key': [ + (orders) => + orders.update([1, 99], (drafts) => { + for (const draft of drafts) draft.value = `x` + }), + (orders) => + orders.update(1, (draft) => { + draft.value = `y` + }), + ], + 'an update callback that throws midway': [ + (orders) => + orders.update(1, (draft) => { + draft.value = `half` + throw new Error(`callback failed`) + }), + ], + 'a delete batch with a missing key': [ + (orders) => orders.insert({ id: 2, value: `b` }), + (orders) => orders.delete([2, 99]), + (orders) => orders.delete(1), + ], + } + + for (const [name, writes] of Object.entries(histories)) { + it(`for ${name}`, async () => { + const direct = await observe({}, writes) + const reference = await observe(confirmEverything, writes) + + expect(direct).toEqual(reference) + expect(direct.states.every((state) => state === `completed`)).toBe(true) + expect(direct.remaining).toBe(0) + }) + } + + it(`for a write the schema rejects`, () => { + const results = [] + for (const handlers of [{}, confirmEverything]) { + const orders = createCollection( + localOnlyCollectionOptions({ + id: `local-only-direct-schema-${Math.random()}`, + getKey: (row) => row.id, + schema: z.object({ id: z.number(), value: z.string().min(1) }), + initialData: [{ id: 1, value: `a` }], + ...handlers, + }), + ) + const batches: Array = [] + orders.subscribeChanges((changes) => batches.push(changes.length)) + const errors = [] + for (const write of [ + () => orders.insert({ id: 2, value: `` }), + () => + orders.update(1, (draft) => { + draft.value = `` + }), + ]) { + try { + write() + } catch (error) { + errors.push(`${(error as Error).name}: ${(error as Error).message}`) + } + } + results.push({ + errors, + batches, + rows: orders.toArray.map(({ id, value }) => ({ id, value })), + remaining: orders._state.transactions.size, + }) + } + + expect(results[0]!.errors).toHaveLength(2) + expect(results[0]!.batches).toEqual([]) + expect(results[0]).toEqual(results[1]) + }) +}) + +describe(`local-only fallbacks for every operation type`, () => { + type Kind = `insert` | `update` | `delete` + // However the fallback is reached, the write must stay visible and must + // not report completion before the optimistic stage confirms it. + const write: Record ReturnType> = + { + insert: (orders) => orders.insert({ id: 5, value: `direct` }), + update: (orders) => + orders.update(1, (draft) => { + draft.value = `direct` + }), + delete: (orders) => orders.delete(1), + } + const rollbackBatches: Record>> = { + insert: [[`insert:1`], [`insert:5`], [`delete:5`]], + update: [[`insert:1`], [`update:1`], [`update:1`]], + delete: [[`insert:1`], [`delete:1`], [`insert:1`]], + } + const visible: Record boolean> = { + insert: (orders) => orders.get(5)?.value === `direct`, + update: (orders) => orders.get(1)?.value === `direct`, + delete: (orders) => !orders.has(1), + } + + for (const kind of [`insert`, `update`, `delete`] as const) { + it(`keeps a ${kind} visible while another transaction persists`, async () => { + const orders = createOrders() + const release = createDeferred() + const persisting = createTransaction({ + mutationFn: () => release.promise, + }) + persisting.mutate(() => orders.insert({ id: 3, value: `persisting` })) + + const transaction = write[kind](orders) + + expect(transaction.state).not.toBe(`completed`) + expect(visible[kind](orders)).toBe(true) + release.resolve() + await persisting.isPersisted.promise + await transaction.isPersisted.promise + expect(visible[kind](orders)).toBe(true) + }) + + it(`overlays a pending transaction with a ${kind}`, async () => { + const orders = createOrders() + const pending = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + pending.mutate(() => + orders.update(1, (draft) => { + draft.value = `pending` + }), + ) + + const transaction = write[kind](orders) + + expect(transaction.state).not.toBe(`completed`) + expect(visible[kind](orders)).toBe(true) + pending.rollback() + await transaction.isPersisted.promise + expect(visible[kind](orders)).toBe(true) + }) + + it(`joins an ambient transaction with a ${kind}`, () => { + const orders = createOrders() + const ambient = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + ambient.mutate(() => write[kind](orders)) + + expect(ambient.mutations.map((m) => m.type)).toEqual([kind]) + expect(visible[kind](orders)).toBe(true) + ambient.rollback() + expect(visible[kind](orders)).toBe(false) + }) + + it(`rolls back a ${kind} whose handler rejects`, async () => { + const handler = `on${kind[0]!.toUpperCase()}${kind.slice(1)}` as const + const orders = createOrders({ + [handler]: () => Promise.reject(new Error(`handler failed`)), + }) + const batches: Array> = [] + orders.subscribeChanges( + (changes) => + batches.push(changes.map((change) => `${change.type}:${change.key}`)), + { includeInitialState: true }, + ) + + const transaction = write[kind](orders) + + expect(visible[kind](orders)).toBe(true) + await expect(transaction.isPersisted.promise).rejects.toThrow( + `handler failed`, + ) + expect(transaction.state).toBe(`failed`) + expect(visible[kind](orders)).toBe(false) + expect(orders.get(1)?.value ?? `a`).toBe(`a`) + expect(batches).toEqual(rollbackBatches[kind]) + // Operation types without a handler still write directly. + const other = kind === `insert` ? `update` : `insert` + expect(write[other](orders).state).toBe(`completed`) + }) + } +}) From a0f537e9f1413c4b556ed8ff78393ed5c10d3917 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 11:21:27 -0600 Subject: [PATCH 16/63] test(db): generate exotic rows and the added-undefined revert in the flat oracle Adds frozen rows, non-enumerable fields, defineProperty in callbacks, stored drafts, and throwing callbacks, and biases generation to reach the added-undefined revert. Four tracker divergences are recorded as open markers pending a contract decision. Co-authored-by: Isaac --- ...at-change-tracking-oracle.property.test.ts | 477 ++++++++++++++---- 1 file changed, 390 insertions(+), 87 deletions(-) diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts index be425768b..608205203 100644 --- a/packages/db/tests/flat-change-tracking-oracle.property.test.ts +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -12,39 +12,62 @@ * Why an example can miss the failure: a single assignment of a new value * passes any diff. The trackers can disagree only on equal-but-not-identical * values (`0` and `-0`, two `NaN`s), on a field set back to its original - * value, on a field set to `undefined` versus deleted, on an added field, and - * on an assigned object that later changes outside the callback. + * value, on a field set to `undefined` versus deleted, on an added field, on + * an assigned object that later changes outside the callback, and on the + * shapes a shallow copy treats differently from a proxy: a frozen row, a + * non-enumerable field, a property defined in the callback, another row's + * draft stored in a field, and a callback that throws. * - * Model: `expectedChanges` folds the operations over a plain copy and keeps - * fields whose final value differs under `===` or `Object.is`, plus deleted - * fields. It does not import either tracker. + * Model: `expectedChanges` folds every row's operations, in callback order, + * over plain copies of the rows. It keeps enumerable fields whose final value + * differs under `===` or `Object.is` from the row's own field, plus deleted + * enumerable fields. A copy holds no non-enumerable field, so reading one + * gives `undefined`, deleting one does nothing, and writing one is a change + * unless it writes the row's value. A stored draft is the model's copy, read + * when the callback returns. It does not import either tracker. When the + * callback throws, both trackers must rethrow the same error and leave the + * rows unchanged. * * History grammar: one to three rows with fields `a`, `b`, and `c` drawn from * `0`, `-0`, `1`, `NaN`, `''`, `'x'`, `true`, `false`, `null`, `undefined`, a - * function, or missing, with a plain or null prototype. Each row gets up to - * five operations: assign a field (`a` to `d`) a value from that domain or a - * fresh object, set a field back to its original value, delete a field, or - * read it. Histories call the tracker with an array or with a single row. + * function, or missing, with a plain or null prototype, optionally frozen, + * and optionally with a non-enumerable field `h` holding a domain value. Each + * row gets up to six operations: assign a field (`a` to `d`, or `h`) a value + * from that domain or a fresh object, assign `d` `undefined`, set a field + * back to its original value, delete a field, read it, define it with + * `Object.defineProperty` as an enumerable or non-enumerable data property, + * or store a row's draft in it. A weighted run changes a field, adds `d` as + * `undefined`, and reverts the field. Histories call the tracker with an + * array or with a single row, and may throw after any operation. * * Production driver: `withFlatChangeTracking` and the proxy trackers * `withArrayChangeTracking` and `withChangeTracking` run the same operations. * * Refinement check: the flat result, the proxy result, and the model are - * strictly equal, including present `undefined` fields, except that `0` and - * `-0` compare equal: collection equality does not distinguish them, and the - * proxy skips writing a value equal to the current one. Mutating an assigned - * object after the callback leaves both results unchanged. + * strictly equal, including present `undefined` fields and the cycles a + * stored draft creates, except that `0` and `-0` compare equal and + * prototypes are ignored: collection equality does not distinguish them, and + * the proxy skips writing a value equal to the current one. Mutating an + * assigned object after the callback leaves both results unchanged. * * Calibration: a flat diff with `!==` reports unchanged `NaN` fields, one with - * `Object.is` alone reports `-0` written over `0`, and one that skips - * deletions loses removed fields; each fails the pinned histories and both - * campaigns. The proxy used to treat a field added with `undefined` as + * `Object.is` alone reports `-0` written over `0`, one that skips deletions + * loses removed fields, one that ignores whether the row owns a field drops + * added `undefined` fields, and one that edits rows in place breaks the throw + * and stored-draft histories; each fails a pinned history and the fixed + * campaign. The proxy used to treat a field added with `undefined` as * reverted when another field went back to its original value, and dropped - * the added field; the pinned history for that case failed before the fix. + * the added field; the pinned history and both campaigns fail without the + * fix, because the grammar weights that run. * * Known omissions: nested objects, arrays, Dates, Maps, Sets, class * instances, and symbol keys are outside this owner; the proxy oracles and * contracts own them, and this file checks only that they fall back. + * The trackers disagree on four shapes, which the grammar excludes and + * `openDivergences` records as open decisions: an accessor defined in the + * callback, a field defined with the row's own value, a deep-equal write to a + * non-enumerable object field, and a non-enumerable field written and then + * deleted. Each marker fails once the trackers agree. */ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' @@ -85,10 +108,20 @@ type Operation = | { kind: `revert`; field: string } | { kind: `delete`; field: string } | { kind: `read`; field: string } + | { kind: `define`; field: string; value: unknown; enumerable: boolean } + | { kind: `store-draft`; field: string; row: number } +type RowShape = { + fields: Array + nullPrototype: boolean + frozen: boolean + hidden: unknown +} type History = { - rows: Array<{ fields: Array; nullPrototype: boolean }> + rows: Array operations: Array> single: boolean + // The callback throws after this many operations, counted across rows. + throwAfter: number | undefined } // --------------------------------------------------------------------------- @@ -99,29 +132,58 @@ function sameValue(a: unknown, b: unknown): boolean { return a === b || Object.is(a, b) } -function expectedChanges(original: Row, operations: Array): Row { - const draft: Row = { ...original } - for (const operation of operations) applyOperation(draft, original, operation) - const changes: Row = {} - for (const key of Object.keys(draft)) { - if ( - !Object.hasOwn(original, key) || - !sameValue(draft[key], original[key]) - ) { - changes[key] = draft[key] +function expectedChanges( + originals: Array, + operations: Array>, +): Array { + const drafts: Array = originals.map((original) => ({ ...original })) + drafts.forEach((draft, index) => { + for (const operation of operations[index]!) { + applyOperation(draft, originals[index]!, operation, drafts) } - } - for (const key of Object.keys(original)) { - if (!Object.hasOwn(draft, key)) changes[key] = undefined - } - return changes + }) + return drafts.map((draft, index) => { + const original = originals[index]! + const changes: Row = {} + for (const key of Object.keys(draft)) { + if ( + !Object.hasOwn(original, key) || + !sameValue(draft[key], original[key]) + ) { + changes[key] = draft[key] + } + } + for (const key of Object.keys(original)) { + if (!Object.hasOwn(draft, key)) changes[key] = undefined + } + return changes + }) } -function applyOperation(draft: Row, original: Row, operation: Operation) { +const domainValue = (value: unknown) => + value === FRESH_OBJECT ? { nested: 1 } : value + +// Shared by the model and the driver; `drafts` are the callback's drafts. +function applyOperation( + draft: Row, + original: Row, + operation: Operation, + drafts: Array, +) { switch (operation.kind) { case `set`: - draft[operation.field] = - operation.value === FRESH_OBJECT ? { nested: 1 } : operation.value + draft[operation.field] = domainValue(operation.value) + break + case `define`: + Object.defineProperty(draft, operation.field, { + value: domainValue(operation.value), + enumerable: operation.enumerable, + writable: true, + configurable: true, + }) + break + case `store-draft`: + draft[operation.field] = drafts[operation.row % drafts.length] break case `revert`: if (Object.hasOwn(original, operation.field)) { @@ -137,86 +199,241 @@ function applyOperation(draft: Row, original: Row, operation: Operation) { } } -// Collection equality does not distinguish -0 from 0. -function normalizeZeros(rows: Array | undefined) { - return rows?.map((row) => - Object.fromEntries( - Object.entries(row).map(([key, value]) => [key, value === 0 ? 0 : value]), - ), - ) +// Collection equality does not distinguish -0 from 0 or prototypes. A +// stored draft can make a change set cyclic, so the copy keeps cycles. +function normalize(value: unknown, copies = new Map()): unknown { + if (value === 0) return 0 + if (value === null || typeof value !== `object`) return value + const existing = copies.get(value) + if (existing) return existing + const copy: Row = {} + copies.set(value, copy) + for (const [key, field] of Object.entries(value)) { + copy[key] = normalize(field, copies) + } + return copy } -function buildRow({ - fields: fieldValues, - nullPrototype, -}: History[`rows`][number]): Row { - const row: Row = nullPrototype ? Object.create(null) : {} - fieldValues.forEach((value, index) => { +function buildRow(shape: RowShape): Row { + const row: Row = shape.nullPrototype ? Object.create(null) : {} + shape.fields.forEach((value, index) => { if (value !== MISSING) row[fields[index]!] = value }) - return row + if (shape.hidden !== MISSING) { + Object.defineProperty(row, `h`, { + value: domainValue(shape.hidden), + enumerable: false, + writable: true, + configurable: true, + }) + } + return shape.frozen ? Object.freeze(row) : row } +// Captures a row's own properties, including non-enumerable ones. +const rowState = (row: Row) => Object.getOwnPropertyDescriptors(row) + // --------------------------------------------------------------------------- // History grammar // --------------------------------------------------------------------------- -const fieldArbitrary = fc.constantFrom(`a`, `b`, `c`, `d`) +const fieldArbitrary = fc.constantFrom(`a`, `b`, `c`, `d`, `h`) +const valueArbitrary = fc.constantFrom(...values, FRESH_OBJECT) const operationArbitrary: fc.Arbitrary = fc.oneof( { weight: 3, arbitrary: fc.record({ kind: fc.constant(`set` as const), field: fieldArbitrary, - value: fc.constantFrom(...values, FRESH_OBJECT), + value: valueArbitrary, + }), + }, + fc.constant({ kind: `set`, field: `d`, value: undefined }), + { + weight: 2, + arbitrary: fc.record({ + kind: fc.constant(`revert` as const), + field: fieldArbitrary, }), }, - fc.record({ kind: fc.constant(`revert` as const), field: fieldArbitrary }), fc.record({ kind: fc.constant(`delete` as const), field: fieldArbitrary }), fc.record({ kind: fc.constant(`read` as const), field: fieldArbitrary }), + fc.record({ + kind: fc.constant(`define` as const), + field: fieldArbitrary, + value: valueArbitrary, + enumerable: fc.boolean(), + }), + fc.record({ + kind: fc.constant(`store-draft` as const), + field: fieldArbitrary, + row: fc.nat({ max: 2 }), + }), ) -const historyArbitrary: fc.Arbitrary = fc +// An added `undefined` while another field changes and reverts is where the +// proxy once dropped the added field, so the grammar also emits that run. +const excursionArbitrary: fc.Arbitrary> = fc + .tuple(fc.constantFrom(...fields), valueArbitrary) + .map(([field, value]) => [ + { kind: `set`, field, value }, + { kind: `set`, field: `d`, value: undefined }, + { kind: `revert`, field }, + ]) +const operationsArbitrary: fc.Arbitrary> = fc .array( - fc.record({ - fields: fc.tuple( - ...fields.map(() => fc.constantFrom(...values, MISSING)), - ), - nullPrototype: fc.boolean(), - }), - { minLength: 1, maxLength: 3 }, + fc.oneof( + { + weight: 4, + arbitrary: operationArbitrary.map((operation) => [operation]), + }, + excursionArbitrary, + ), + { maxLength: 5 }, ) + .map((chunks) => chunks.flat().slice(0, maxOperations)) +const maxOperations = 6 + +const rowArbitrary: fc.Arbitrary = fc.record({ + fields: fc.tuple(...fields.map(() => fc.constantFrom(...values, MISSING))), + nullPrototype: fc.boolean(), + frozen: fc.boolean(), + hidden: fc.oneof( + { weight: 2, arbitrary: fc.constant(MISSING) }, + fc.constantFrom(...values), + ), +}) +const historyArbitrary: fc.Arbitrary = fc + .array(rowArbitrary, { minLength: 1, maxLength: 3 }) .chain((rows) => fc.record({ rows: fc.constant(rows), - operations: fc.tuple( - ...rows.map(() => fc.array(operationArbitrary, { maxLength: 5 })), - ), + operations: fc.tuple(...rows.map(() => operationsArbitrary)), single: rows.length === 1 ? fc.boolean() : fc.constant(false), + throwAfter: fc.oneof( + { weight: 5, arbitrary: fc.constant(undefined) }, + fc.nat({ max: maxOperations }), + ), }), ) + .filter((history) => !isExcluded(history)) + +// Excluded, each recorded in `openDivergences`: the proxy reports a field +// defined with the row's own value as changed, though it skips the same +// write through assignment, and reports a non-enumerable field written and +// then deleted as a deletion, though a plain delete reports nothing. +function isExcluded(history: History): boolean { + return history.rows.some((shape, index) => { + const original = buildRow(shape) + const operations = history.operations[index]! + const definesOriginalValue = operations.some( + (operation) => + operation.kind === `define` && + Object.hasOwn(original, operation.field) && + sameValue(operation.value, original[operation.field]), + ) + const firstHiddenWrite = operations.findIndex( + (operation) => + operation.field === `h` && + operation.kind !== `delete` && + operation.kind !== `read`, + ) + const deletesWrittenHidden = + shape.hidden !== MISSING && + firstHiddenWrite >= 0 && + operations + .slice(firstHiddenWrite) + .some( + (operation) => operation.kind === `delete` && operation.field === `h`, + ) + return definesOriginalValue || deletesWrittenHidden + }) +} + +const plainRow = (rowFields: Array, nullPrototype = false) => ({ + fields: rowFields, + nullPrototype, + frozen: false, + hidden: MISSING, +}) + +const firstDraft = (draft: Array | Row) => + Array.isArray(draft) ? draft[0]! : draft +const openDivergences: ReadonlyArray<{ + name: string + shape: RowShape + callback: (draft: Array | Row) => void +}> = [ + { + // The flat tracker reports the getter's value; the proxy records only + // data descriptors, as on `main`. + name: `the callback defines an accessor`, + shape: plainRow([1, MISSING, MISSING]), + callback: (draft) => { + Object.defineProperty(firstDraft(draft), `g`, { + get: () => 7, + enumerable: true, + configurable: true, + }) + }, + }, + { + // The proxy, as on `main`, reports the field; the flat tracker does not. + name: `the callback defines a field with its own value`, + shape: plainRow([1, MISSING, MISSING]), + callback: (draft) => { + Object.defineProperty(firstDraft(draft), `a`, { + value: 1, + enumerable: true, + writable: true, + configurable: true, + }) + }, + }, + { + // The flat check skips non-enumerable fields, so this row takes the flat + // path, which compares by identity; the proxy compares deeply. + name: `a non-enumerable object field gets a deep-equal object`, + shape: { ...plainRow([1, MISSING, MISSING]), hidden: FRESH_OBJECT }, + callback: (draft) => { + firstDraft(draft).h = { nested: 1 } + }, + }, + { + // The proxy reports a deletion; the flat tracker, whose copy never held + // the field, reports nothing. + name: `a non-enumerable field is written and then deleted`, + shape: { ...plainRow([1, MISSING, MISSING]), hidden: `x` }, + callback: (draft) => { + firstDraft(draft).h = `y` + delete firstDraft(draft).h + }, + }, +] // Each history isolates one place a plausible flat diff goes wrong. const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ { name: `NaN written over NaN is not a change`, history: { - rows: [{ fields: [Number.NaN, 1, MISSING], nullPrototype: false }], + rows: [plainRow([Number.NaN, 1, MISSING])], operations: [[{ kind: `set`, field: `a`, value: Number.NaN }]], single: false, + throwAfter: undefined, }, }, { name: `-0 written over 0 is not a change`, history: { - rows: [{ fields: [0, 1, MISSING], nullPrototype: false }], + rows: [plainRow([0, 1, MISSING])], operations: [[{ kind: `set`, field: `a`, value: -0 }]], single: true, + throwAfter: undefined, }, }, { name: `a field added as undefined survives another field's revert`, history: { - rows: [{ fields: [`x`, 1, MISSING], nullPrototype: false }], + rows: [plainRow([`x`, 1, MISSING])], operations: [ [ { kind: `set`, field: `a`, value: `y` }, @@ -225,12 +442,13 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ ], ], single: false, + throwAfter: undefined, }, }, { name: `deleted, added, and reverted fields`, history: { - rows: [{ fields: [1, `x`, true], nullPrototype: true }], + rows: [plainRow([1, `x`, true], true)], operations: [ [ { kind: `delete`, field: `a` }, @@ -240,6 +458,64 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ ], ], single: false, + throwAfter: undefined, + }, + }, + { + name: `a frozen row with a hidden field is written, defined, and deleted`, + history: { + rows: [ + { + fields: [1, 2, MISSING], + nullPrototype: false, + frozen: true, + hidden: 5, + }, + ], + operations: [ + [ + { kind: `set`, field: `a`, value: 3 }, + { kind: `define`, field: `c`, value: `x`, enumerable: true }, + { kind: `define`, field: `d`, value: 1, enumerable: false }, + { kind: `set`, field: `h`, value: 6 }, + { kind: `delete`, field: `b` }, + ], + ], + single: true, + throwAfter: undefined, + }, + }, + { + name: `rows store each other's drafts`, + history: { + rows: [plainRow([1, 2, MISSING]), plainRow([3, 4, MISSING])], + operations: [ + [ + { kind: `store-draft`, field: `d`, row: 1 }, + { kind: `store-draft`, field: `c`, row: 0 }, + ], + [ + { kind: `store-draft`, field: `d`, row: 0 }, + { kind: `delete`, field: `b` }, + ], + ], + single: false, + throwAfter: undefined, + }, + }, + { + name: `the callback throws after writing`, + history: { + rows: [plainRow([1, 2, MISSING]), plainRow([3, 4, MISSING])], + operations: [ + [ + { kind: `set`, field: `a`, value: 5 }, + { kind: `delete`, field: `b` }, + ], + [{ kind: `set`, field: `d`, value: 1 }], + ], + single: false, + throwAfter: 2, }, }, ] @@ -248,34 +524,51 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ // Production driver and refinement check // --------------------------------------------------------------------------- +class CallbackError extends Error {} + function runHistory(history: History): void { const originals = history.rows.map(buildRow) - const run = (drafts: Array | Row) => { + // A revert reads the row the tracker was given. + const run = (rows: Array) => (drafts: Array | Row) => { const list = Array.isArray(drafts) ? drafts : [drafts] + let applied = 0 list.forEach((draft, index) => { for (const operation of history.operations[index]!) { - applyOperation(draft, originals[index]!, operation) + if (applied++ === history.throwAfter) throw new CallbackError() + applyOperation(draft, rows[index]!, operation, list) } }) + if (applied === history.throwAfter) throw new CallbackError() } - const flat = withFlatChangeTracking( - history.rows.map(buildRow), - run, - !history.single, - ) + const flatRows = history.rows.map(buildRow) + const proxyRows = history.rows.map(buildRow) + const track = { + flat: () => + withFlatChangeTracking(flatRows, run(flatRows), !history.single), + proxy: () => + history.single + ? [withChangeTracking(proxyRows[0]!, run(proxyRows))] + : withArrayChangeTracking(proxyRows, run(proxyRows)), + } + const throws = + history.throwAfter !== undefined && + history.throwAfter <= history.operations.flat().length + if (throws) { + expect(track.flat, `flat rethrows`).toThrow(CallbackError) + expect(track.proxy, `proxy rethrows`).toThrow(CallbackError) + for (const rows of [flatRows, proxyRows]) { + expect(rows.map(rowState), `rows unchanged`).toStrictEqual( + originals.map(rowState), + ) + } + return + } + const flat = track.flat() expect(flat, `flat rows take the flat path`).toBeDefined() - const proxy = history.single - ? [withChangeTracking(buildRow(history.rows[0]!), run)] - : withArrayChangeTracking(history.rows.map(buildRow), run) - const model = originals.map((original, index) => - expectedChanges(original, history.operations[index]!), - ) - expect(normalizeZeros(flat), `flat vs model`).toStrictEqual( - normalizeZeros(model), - ) - expect(normalizeZeros(proxy), `proxy vs model`).toStrictEqual( - normalizeZeros(model), - ) + const proxy = track.proxy() + const model = expectedChanges(originals, history.operations) + expect(normalize(flat), `flat vs model`).toStrictEqual(normalize(model)) + expect(normalize(proxy), `proxy vs model`).toStrictEqual(normalize(model)) } describe(`flat change tracking oracle`, () => { @@ -312,6 +605,16 @@ describe(`flat change tracking oracle`, () => { expect(changes).toStrictEqual({ a: { nested: 1 } }) }) + // Open decisions, not laws: the trackers disagree on these shapes, so + // the grammar excludes them. Each marker fails once the two agree. + for (const { name, shape, callback } of openDivergences) { + it.fails(`agrees with the draft proxy when ${name}`, () => { + expect( + withFlatChangeTracking([buildRow(shape)], callback, false), + ).toStrictEqual([withChangeTracking(buildRow(shape), callback)]) + }) + } + fcTest.prop([historyArbitrary], { seed: 44_502_101, numRuns: oracleRuns(200), From 6543a5ab8e3ef0a4d34265b87edd8e7dba8a432f Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:06:13 -0600 Subject: [PATCH 17/63] test(db): check pooled change payloads and pending-write cleanup The pooled oracle now compares granular changes by type, key, value, and previous value with a reference observer and replays them against the model. It generates cleanup while a write is pending and mounts between cleanup and restart, and reaches within-group updates. The eq-filter conformance scenario checks that React shares one source subscription. Co-authored-by: Isaac --- packages/db/tests/conformance/contract.ts | 10 +- packages/db/tests/conformance/suite.ts | 4 + .../pooled-live-query-oracle.property.test.ts | 360 ++++++++++++++---- packages/react-db/tests/conformance.test.tsx | 2 +- 4 files changed, 293 insertions(+), 83 deletions(-) diff --git a/packages/db/tests/conformance/contract.ts b/packages/db/tests/conformance/contract.ts index b88ace6dc..2138dbc5f 100644 --- a/packages/db/tests/conformance/contract.ts +++ b/packages/db/tests/conformance/contract.ts @@ -183,5 +183,13 @@ export interface LiveQueryDriver { * to catch (e.g. Solid's `createResource`/`` model). */ errorSurface?: `flag` | `throw` - features?: { serverSnapshot?: boolean; suspense?: boolean } + features?: { + serverSnapshot?: boolean + suspense?: boolean + /** + * Public sharing policy: queries filtered only by `eq` on one source share + * one source subscription per filtered field set, instead of one each. + */ + pooledEqFilters?: boolean + } } diff --git a/packages/db/tests/conformance/suite.ts b/packages/db/tests/conformance/suite.ts index 67afd4034..173af2f13 100644 --- a/packages/db/tests/conformance/suite.ts +++ b/packages/db/tests/conformance/suite.ts @@ -316,6 +316,10 @@ export function runSuite(rawDriver: LiveQueryDriver) { query(`a`)(q).where(({ items }: any) => ops.eq(items.age, 35)), ) for (const h of [teamA, teamB, olderA]) await h.flush() + // Pooling shares one subscription between the two team queries. + expect(source.collection.subscriberCount).toBe( + driver.features?.pooledEqFilters ? 2 : 3, + ) const ids = (h: typeof teamA) => (h.current().data as Array<{ id: string }>).map((row) => row.id) expect([ids(teamA), ids(teamB), ids(olderA)]).toEqual([ diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index ec2ec2ced..746fd28d5 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -20,29 +20,44 @@ * the partition. Order, row values, and status come from a second * formulation: a live-query Collection compiled for the same query. * - * History grammar: rows have a field `f` from strings, numbers and their - * look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, `0`, `null`, and a - * missing value, and a field `g` of `x` or `y`. Up to three peer queries use - * `eq(f, literal)`, optionally with `eq(g, literal)`. Steps commit sync - * transactions of one or two inserts, updates, or deletes; apply one - * optimistic insert, update, or delete and then confirm or roll it back; - * mount or unmount a peer; or clean up the source and restart it. + * History grammar: rows have ids 0 through 3, a field `f` from strings, + * numbers and their look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, + * `0`, `null`, and a missing value, and a field `g` of `x` or `y`. Up to + * three peer queries use `eq(f, literal)`, optionally with `eq(g, literal)`. + * Values are weighted toward `a`, and toward the normalized values against + * numeric literals, so groups hold rows that stay, move, and normalize. + * Steps commit sync transactions of one or two inserts, updates, or deletes, + * where an update may keep `f` so the row stays in its group; apply one + * optimistic insert, update, or delete and then confirm or roll it back, + * optionally after cleaning up and restarting the source while it is + * pending; mount or unmount a peer; or clean up the source and restart it, + * optionally (re)mounting a peer on the cleaned-up source first. * * Production driver: `createPooledLiveQuery` builds each peer's view from the * query builder's IR, and `createLiveQueryObserver` observes it in wholesale - * and granular mode, as the framework adapters do. + * and granular mode, as the framework adapters do. The view subscribes + * before its live-query Collection preloads, so a peer mounted after cleanup + * is the one that restarts the source. * * Refinement check: after every step, each mounted peer's wholesale snapshot - * equals its live-query Collection's keys in order, row values, and status; - * both observers' key sets equal the model. A peer mounted when its source - * starts cleanup is terminal like its live query: status `error` with the - * rows it had, through the restart and later writes. A peer mounted after - * the restart follows the restarted source. + * equals its live-query Collection's keys in order, row values, and status, + * and its rows' fields equal the model's. The granular changes delivered since + * the last checkpoint equal those of a granular observer of the live-query + * Collection, by type, key, value, and previous value; folded in order, they + * never insert a held key, update or delete an unheld one, or carry a stale + * previous value, and they leave the model's rows. A peer mounted when its + * source starts cleanup is terminal like its live query: status `error` with + * the rows it had, pending optimistic rows included, through the restart and + * later writes. A peer mounted after cleanup follows the restarted source. * * Calibration: a partition that ignored the previous value, kept group rows * in arrival order, or compared literals without normalization fails the - * pinned histories and both campaigns. A view that kept reporting the - * source's status after cleanup fails the cleanup history. + * pinned histories and campaigns. A view that kept reporting the source's + * status after cleanup fails the cleanup histories. An update delivered as a + * delete and insert, with a stale previous value, or with the previous row as + * its value fails the in-group update history and both campaigns; so do a + * partition born terminal on a cleaned-up source and a freeze that drops + * pending optimistic rows. * * Known omissions: on-demand and persisted sources, `DbClient` hydration, * Suspense, `select`, and every other clause keep the live-query Collection @@ -78,13 +93,21 @@ type Row = { id: string; f?: unknown; g: string } type Peer = { f: string | number | boolean; g?: string } type Step = | { kind: `sync`; ops: Array } - | { kind: `optimistic`; op: Op; confirm: boolean } + | { + kind: `optimistic` + op: Op + confirm: boolean + // Clean up and restart the source before settling the write. + cleanupFirst?: boolean + } | { kind: `mount`; peer: number } | { kind: `unmount`; peer: number } - | { kind: `cleanup-restart` } + // `mount` (re)mounts that peer after cleanup, before the restart. + | { kind: `cleanup-restart`; mount?: number } type Op = | { type: `insert`; id: number; f: FieldValue; g: string } - | { type: `update`; id: number; f: FieldValue; g: string } + // `keepF` keeps the row's current `f`, so the row stays in its group. + | { type: `update`; id: number; f: FieldValue; g: string; keepF?: boolean } | { type: `delete`; id: number } type History = { rows: Array<{ f: FieldValue; g: string }> @@ -136,6 +159,14 @@ function expectedKeys( .sort() } +// The model's matching rows, described by id and fields. +function expectedRows( + rows: ReadonlyMap, + peer: Peer, +): Array { + return expectedKeys(rows, peer).map((key) => describeFields(rows.get(key)!)) +} + // Collection change detection treats 0 and -0, and NaN and NaN, as equal. function sameValueZero(a: unknown, b: unknown): boolean { return a === b || (Number.isNaN(a) && Number.isNaN(b)) @@ -149,28 +180,45 @@ function sourceRow(id: string, f: FieldValue, g: string): Row { // History grammar // --------------------------------------------------------------------------- -const fieldArbitrary = fc.constantFrom(...fieldValues) +// Most rows and peers share `a`, so groups hold rows that stay or move; +// the normalized values have their own weight against numeric literals. +const fieldArbitrary = fc.oneof( + { weight: 2, arbitrary: fc.constant(`a`) }, + fc.constantFrom(DATE_ONE, Number.NaN, -0), + fc.constantFrom(...fieldValues), +) const gArbitrary = fc.constantFrom(`x`, `y`) const opArbitrary: fc.Arbitrary = fc.oneof( fc.record({ type: fc.constant(`insert` as const), - id: fc.nat({ max: 5 }), + id: fc.nat({ max: 3 }), f: fieldArbitrary, g: gArbitrary, }), { weight: 2, - arbitrary: fc.record({ - type: fc.constant(`update` as const), - id: fc.nat({ max: 5 }), - f: fieldArbitrary, - g: gArbitrary, - }), + arbitrary: fc.record( + { + type: fc.constant(`update` as const), + id: fc.nat({ max: 3 }), + f: fieldArbitrary, + g: gArbitrary, + keepF: fc.boolean(), + }, + { requiredKeys: [`type`, `id`, `f`, `g`] }, + ), }, - fc.record({ type: fc.constant(`delete` as const), id: fc.nat({ max: 5 }) }), + fc.record({ type: fc.constant(`delete` as const), id: fc.nat({ max: 3 }) }), ) const peerArbitrary: fc.Arbitrary = fc.record( - { f: fc.constantFrom(...literals), g: gArbitrary }, + { + f: fc.oneof( + { weight: 2, arbitrary: fc.constant(`a`) }, + fc.constantFrom(1, Number.NaN, 0), + fc.constantFrom(...literals), + ), + g: gArbitrary, + }, { requiredKeys: [`f`] }, ) const historyArbitrary: fc.Arbitrary = fc.record({ @@ -187,11 +235,15 @@ const historyArbitrary: fc.Arbitrary = fc.record({ ops: fc.array(opArbitrary, { minLength: 1, maxLength: 2 }), }), }, - fc.record({ - kind: fc.constant(`optimistic` as const), - op: opArbitrary, - confirm: fc.boolean(), - }), + fc.record( + { + kind: fc.constant(`optimistic` as const), + op: opArbitrary, + confirm: fc.boolean(), + cleanupFirst: fc.boolean(), + }, + { requiredKeys: [`kind`, `op`, `confirm`] }, + ), fc.record({ kind: fc.constant(`mount` as const), peer: fc.nat({ max: 2 }), @@ -200,7 +252,13 @@ const historyArbitrary: fc.Arbitrary = fc.record({ kind: fc.constant(`unmount` as const), peer: fc.nat({ max: 2 }), }), - fc.constant({ kind: `cleanup-restart` as const }), + fc.record( + { + kind: fc.constant(`cleanup-restart` as const), + mount: fc.nat({ max: 2 }), + }, + { requiredKeys: [`kind`] }, + ), ), { maxLength: 8 }, ), @@ -262,6 +320,42 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ ], }, }, + { + name: `peers clean up with a pending write and one mounts before the restart`, + history: { + rows: [{ f: `a`, g: `x` }], + peers: [{ f: `a` }, { f: `a`, g: `y` }], + steps: [ + { + kind: `optimistic`, + op: { type: `insert`, id: 2, f: `a`, g: `y` }, + confirm: false, + cleanupFirst: true, + }, + { kind: `sync`, ops: [{ type: `insert`, id: 3, f: `a`, g: `y` }] }, + { kind: `cleanup-restart`, mount: 1 }, + { kind: `sync`, ops: [{ type: `update`, id: 3, f: `a`, g: `x` }] }, + ], + }, + }, + { + name: `a row updated within its group reaches peers as one update`, + history: { + rows: [ + { f: 1, g: `x` }, + { f: `b`, g: `x` }, + ], + peers: [{ f: 1 }, { f: 1, g: `y` }], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: DATE_ONE, g: `y` }] }, + { + kind: `optimistic`, + op: { type: `update`, id: 0, f: 1, g: `y` }, + confirm: true, + }, + ], + }, + }, { name: `a remounted peer reads a group that changed while it was away`, history: { @@ -298,9 +392,49 @@ function peerQuery(source: any, peer: Peer) { ) } -function describeRow(row: Record) { +// A row's id and fields, which the model also knows. +function describeFields(row: Record) { const f = row.f instanceof Date ? `Date(${row.f.getTime()})` : String(row.f) - return `${String(row.id)}:${typeof row.f}:${f}:${String(row.g)}:${String(row.$synced)}` + return `${String(row.id)}:${typeof row.f}:${f}:${String(row.g)}` +} + +function describeRow(row: Record) { + return `${describeFields(row)}:${String(row.$synced)}` +} + +type Change = ChangeMessage, string | number> + +function describeChange(change: Change) { + const previous = change.previousValue + return `${change.type}:${String(change.key)}:${describeRow(change.value)}:${previous ? describeRow(previous) : `-`}` +} + +// Applies a granular batch to the rows a consumer has folded so far, +// recording each change that contradicts them. +function foldChanges( + rows: Map>, + changes: Array, + violations: Array, +) { + for (const change of changes) { + const held = rows.get(change.key) + if (change.type === `insert`) { + if (held) violations.push(`insert of held ${describeChange(change)}`) + rows.set(change.key, change.value) + } else if (!held) { + violations.push(`${change.type} of unheld ${describeChange(change)}`) + } else if (change.type === `delete`) { + rows.delete(change.key) + } else { + if ( + !change.previousValue || + describeFields(change.previousValue) !== describeFields(held) + ) { + violations.push(`stale previousValue in ${describeChange(change)}`) + } + rows.set(change.key, change.value) + } + } } async function runHistory(history: History): Promise { @@ -320,55 +454,69 @@ async function runHistory(history: History): Promise { const references: Array> = [] type Mounted = { reference: ReturnType - // The model keys when the source started cleanup, if it has since. + // The model rows when the source started cleanup, if it has since. frozen: Array | undefined view: { collection?: unknown } layout: { keys: string; revision: number } | undefined wholesale: ReturnType> - granular: ReturnType> - granularKeys: Set + // Rows folded from the pooled granular stream, and contradictions. + granularRows: Map> + violations: Array + // Granular changes since the last checkpoint, pooled and reference. + pooledChanges: Array + referenceChanges: Array unsubscribe: () => void } const mounted = new Map() const mount = async (index: number) => { const peer = history.peers[index] if (!peer || mounted.has(index)) return - const reference = createLiveQueryCollection(peerQuery(source, peer)) - references.push(reference) - await reference.preload() + // The pooled view subscribes first, so after cleanup it is the one that + // restarts the source. const view = createPooledLiveQuery(peerQuery(source, peer)(new Query())) expect(view, `peer ${index} is poolable`).toBeDefined() const wholesale = createLiveQueryObserver(view as any, { mode: `wholesale`, }) const granular = createLiveQueryObserver(view as any) - const granularKeys = new Set() - const offWholesale = wholesale.subscribe(() => {}) - const offGranular = granular.subscribe((changes) => { - for (const change of (changes ?? []) as Array>) { - if (change.type === `delete`) granularKeys.delete(change.key) - else granularKeys.add(change.key) - } - }) - mounted.set(index, { - reference, + const entry: Mounted = { + reference: createLiveQueryCollection(peerQuery(source, peer)), frozen: undefined, view: view as unknown as Mounted[`view`], layout: undefined, wholesale, - granular, - granularKeys, - unsubscribe: () => { - offWholesale() - offGranular() - wholesale.dispose() - granular.dispose() - }, + granularRows: new Map(), + violations: [], + pooledChanges: [], + referenceChanges: [], + unsubscribe: () => {}, + } + references.push(entry.reference) + const offWholesale = wholesale.subscribe(() => {}) + const offGranular = granular.subscribe((changes) => { + const batch = (changes ?? []) as Array + foldChanges(entry.granularRows, batch, entry.violations) + entry.pooledChanges.push(...batch.map(describeChange)) + }) + await entry.reference.preload() + const referenceGranular = createLiveQueryObserver(entry.reference as any) + const offReference = referenceGranular.subscribe((changes) => { + const batch = (changes ?? []) as Array + entry.referenceChanges.push(...batch.map(describeChange)) }) + entry.unsubscribe = () => { + offWholesale() + offGranular() + offReference() + wholesale.dispose() + granular.dispose() + referenceGranular.dispose() + } + mounted.set(index, entry) } const check = (checkpoint: string) => { for (const [index, entry] of mounted) { - const { view, wholesale, granularKeys } = entry + const { view, wholesale } = entry const peer = history.peers[index]! const snapshot = wholesale.getSnapshot() const reference = entry.reference @@ -378,14 +526,23 @@ async function runHistory(history: History): Promise { `${label} rows`, ).toEqual(reference.toArray.map((row) => describeRow(row))) expect(snapshot.status, `${label} status`).toBe(reference.status) - const model = entry.frozen ?? expectedKeys(rows, peer) + const model = entry.frozen ?? expectedRows(rows, peer) expect( - [...snapshot.state!.keys()].map(String).sort(), + [...snapshot.state!.values()].map(describeFields).sort(), `${label} model`, ).toEqual(model) - expect([...granularKeys].map(String).sort(), `${label} granular`).toEqual( - model, + // The granular stream delivers each change once, with the type and + // values the live-query Collection's stream has, and folds to the model. + expect(entry.violations, `${label} granular contradictions`).toEqual([]) + expect( + [...entry.granularRows.values()].map(describeFields).sort(), + `${label} granular`, + ).toEqual(model) + expect(entry.pooledChanges.sort(), `${label} granular changes`).toEqual( + entry.referenceChanges.sort(), ) + entry.pooledChanges.length = 0 + entry.referenceChanges.length = 0 // A change in the ordered keys always advances the layout revision. const keys = JSON.stringify([...snapshot.state!.keys()]) if (entry.layout && entry.layout.keys !== keys) { @@ -398,6 +555,12 @@ async function runHistory(history: History): Promise { expect(view.collection, `${label} materialized`).toBeUndefined() } } + // Resolves `keepF` against the model's current row. + const resolveOp = (op: Op): Op => { + const row = rows.get(id(op.id)) + if (op.type !== `update` || !op.keepF || !row) return op + return { ...op, f: `f` in row ? (row.f as FieldValue) : MISSING } + } const applyOp = (op: Op): boolean => { const key = id(op.id) if (op.type === `insert`) { @@ -423,6 +586,30 @@ async function runHistory(history: History): Promise { ) } + const unmount = (index: number) => { + mounted.get(index)?.unsubscribe() + mounted.delete(index) + } + // Every mounted peer freezes with the rows it had, pending writes included. + // `beforeRestart` runs once the source is cleaned up. + const cleanupAndRestart = async ( + checkpoint: string, + beforeRestart: () => Promise, + ) => { + for (const [index, entry] of mounted) { + entry.frozen ??= expectedRows(rows, history.peers[index]!) + } + await source.cleanup() + check(`${checkpoint} cleaned up`) + // The mock source re-syncs its initial rows when it restarts. + rows.clear() + history.rows.forEach((row, rowIndex) => + rows.set(id(rowIndex), sourceRow(id(rowIndex), row.f, row.g)), + ) + await beforeRestart() + await source.preload() + } + await withOracleCleanup(async () => { for (const index of history.peers.keys()) await mount(index) check(`after mount`) @@ -430,22 +617,17 @@ async function runHistory(history: History): Promise { const checkpoint = `after step ${n} (${step.kind})` if (step.kind === `mount`) await mount(step.peer) else if (step.kind === `cleanup-restart`) { - for (const [index, entry] of mounted) { - entry.frozen ??= expectedKeys(rows, history.peers[index]!) - } - await source.cleanup() - check(`${checkpoint} cleaned up`) - // The mock source re-syncs its initial rows when it restarts. - rows.clear() - history.rows.forEach((row, rowIndex) => - rows.set(id(rowIndex), sourceRow(id(rowIndex), row.f, row.g)), - ) - await source.preload() + const between = step.mount + await cleanupAndRestart(checkpoint, async () => { + // A peer mounted on the cleaned-up source restarts it. + if (between === undefined) return + unmount(between) + await mount(between) + }) } else if (step.kind === `unmount`) { - mounted.get(step.peer)?.unsubscribe() - mounted.delete(step.peer) + unmount(step.peer) } else if (step.kind === `sync`) { - const accepted = step.ops.filter((op) => { + const accepted = step.ops.map(resolveOp).filter((op) => { const before = new Map(rows) if (applyOp(op)) return true rows.clear() @@ -457,7 +639,8 @@ async function runHistory(history: History): Promise { for (const op of accepted) write(op) source.utils.commit() } else { - const { op, confirm } = step + const { confirm, cleanupFirst } = step + const op = resolveOp(step.op) const key = id(op.id) const before = new Map(rows) const previous = rows.get(key) @@ -484,7 +667,22 @@ async function runHistory(history: History): Promise { : source.delete(key) const persisted = transaction.isPersisted.promise.catch(() => undefined) check(`${checkpoint} pending`) - if (confirm) { + if (cleanupFirst) { + // The write settles after cleanup, before the restart. The mock + // server keeps no data, so either outcome leaves the initial rows. + await cleanupAndRestart(checkpoint, async () => { + if (confirm) { + source.utils.resolveSync() + await persisted + return + } + await withExpectedRejection(`rolled back`, async () => { + source.utils.rejectSync(new Error(`rolled back`)) + await persisted + await flushPromises() + }) + }) + } else if (confirm) { source.utils.begin() write(op) source.utils.commit() diff --git a/packages/react-db/tests/conformance.test.tsx b/packages/react-db/tests/conformance.test.tsx index 5367455d8..1af95556d 100644 --- a/packages/react-db/tests/conformance.test.tsx +++ b/packages/react-db/tests/conformance.test.tsx @@ -214,7 +214,7 @@ const reactDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: true, suspense: true }, + features: { serverSnapshot: true, suspense: true, pooledEqFilters: true }, } runSuite(reactDriver) From 94c22c6df03070f13c2e82af3961a9efc09ba2bd Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:20:42 -0600 Subject: [PATCH 18/63] fix(db): make defineProperty in update callbacks act as assignment The draft proxy recorded data defines as changes even when they restored the original value, ignored accessors, accepted a getter-only field's own value, and reported deleting a hidden field it had written. The flat tracker compared a hidden object field by identity. Both now follow one rule, which the flat oracle generates instead of excluding. Co-authored-by: Isaac --- .changeset/fix-draft-define-property.md | 5 + packages/db/src/proxy.ts | 107 +++++----- ...at-change-tracking-oracle.property.test.ts | 198 +++++++++++------- 3 files changed, 190 insertions(+), 120 deletions(-) create mode 100644 .changeset/fix-draft-define-property.md diff --git a/.changeset/fix-draft-define-property.md b/.changeset/fix-draft-define-property.md new file mode 100644 index 000000000..58c843488 --- /dev/null +++ b/.changeset/fix-draft-define-property.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Make `Object.defineProperty` inside an update callback report what assignment would. Defining a field back to its original value is no longer a change, an enumerable getter reports its value, and assigning a field the callback gave only a getter throws as it would on a plain object. Deleting a non-enumerable field the callback had written is no longer reported as a deletion, matching a plain delete. diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index d730ebd28..4b6f22dc2 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -622,6 +622,33 @@ export function createChangeProxy< } } + // Whether a value equals the original's own value for a field. + function isOriginalValue(prop: string | symbol, value: unknown): boolean { + const original = changeTracker.originalObject + return ( + Object.hasOwn(original, prop) && + draftValuesEqual(value, original[prop as keyof T]) + ) + } + + // Records a write the draft now holds. Assignment and defineProperty share + // it so they report the same change. + function recordWrite(prop: string | symbol, reverted: boolean) { + if (!reverted) { + changeTracker.assigned_.set(prop.toString(), true) + markChanged(changeTracker) + return + } + changeTracker.assigned_.delete(prop.toString()) + if (checkIfReverted(changeTracker)) { + changeTracker.modified = false + changeTracker.assigned_ = new Map() + if (parent) checkParentStatus(parent.tracker) + } else { + changeTracker.modified = true + } + } + // Create a proxy for the target object function createObjectProxy(obj: TObj): TObj { // If we've already created a proxy for this object, return it @@ -759,7 +786,12 @@ export function createChangeProxy< return value }, - set(_sobj, prop, value) { + set(ptarget, prop, value) { + // An accessor the callback defined behaves as on a plain object; a + // getter without a setter rejects even a write of its own value. + if (Reflect.getOwnPropertyDescriptor(ptarget, prop)?.get) { + return Reflect.set(ptarget, prop, value) + } const currentValue = changeTracker.copy_[prop as keyof T] // Only track the change if the value is actually different @@ -767,46 +799,12 @@ export function createChangeProxy< !Object.hasOwn(changeTracker.copy_, prop) || !draftValuesEqual(currentValue, value) ) { - // Check if the new value is equal to the original value - // Important: Use the originalObject to get the true original value - const originalValue = changeTracker.originalObject[prop as keyof T] - const isRevertToOriginal = - Object.hasOwn(changeTracker.originalObject, prop) && - draftValuesEqual(value, originalValue) - - if (isRevertToOriginal) { - // If the value is reverted to its original state, remove it from changes - changeTracker.assigned_.delete(prop.toString()) - - // Make sure the copy is updated with the original value - changeTracker.copy_[prop as keyof T] = deepClone(originalValue) - - // Check if all properties in this object have been reverted - const allReverted = checkIfReverted(changeTracker) - - if (allReverted) { - // If all have been reverted, clear tracking - changeTracker.modified = false - changeTracker.assigned_ = new Map() - - // If we're a nested object, check if the parent needs updating - if (parent) { - checkParentStatus(parent.tracker) - } - } else { - // Some properties are still changed - changeTracker.modified = true - } - } else { - // Set the value on the copy - changeTracker.copy_[prop as keyof T] = value - - // Track that this property was assigned - store using the actual property (symbol or string) - changeTracker.assigned_.set(prop.toString(), true) - - // Mark this object and its ancestors as modified - markChanged(changeTracker) - } + const reverted = isOriginalValue(prop, value) + // A revert restores a copy so the draft never aliases the row. + changeTracker.copy_[prop as keyof T] = reverted + ? deepClone(changeTracker.originalObject[prop as keyof T]) + : value + recordWrite(prop, reverted) } return true @@ -816,10 +814,9 @@ export function createChangeProxy< // Forward the defineProperty to the target to maintain Proxy invariants // This allows Object.seal() and Object.freeze() to work on the proxy const result = Reflect.defineProperty(ptarget, prop, descriptor) - if (result && `value` in descriptor) { - changeTracker.copy_[prop as keyof T] = deepClone(descriptor.value) - changeTracker.assigned_.set(prop.toString(), true) - markChanged(changeTracker) + // Accessors count by the value they read, as assignment would. + if (result) { + recordWrite(prop, isOriginalValue(prop, Reflect.get(ptarget, prop))) } return result }, @@ -844,10 +841,13 @@ export function createChangeProxy< if (Object.hasOwn(dobj, prop)) { // Check if the property exists in the original object - const hadPropertyInOriginal = Object.hasOwn( - changeTracker.originalObject, - prop, - ) + // A hidden original field is not row data, so as with a plain + // delete of it, removing it is not a change. + const hadPropertyInOriginal = + Object.prototype.propertyIsEnumerable.call( + changeTracker.originalObject, + prop, + ) // Forward the delete to the target using Reflect // This respects Object.seal/preventExtensions constraints @@ -1053,9 +1053,16 @@ export function withFlatChangeTracking( for (const key in draft) { const value = (draft as Record)[key] const before = original[key] + // Only a hidden field can hold an object; compare it as the proxy does. if ( !Object.hasOwn(original, key) || - !(value === before || Object.is(value, before)) + !( + value === before || + Object.is(value, before) || + (typeof before === `object` && + before !== null && + draftValuesEqual(value, before)) + ) ) { defineDataProperty(changes, key, value) } diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts index 608205203..897bbaf71 100644 --- a/packages/db/tests/flat-change-tracking-oracle.property.test.ts +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -20,22 +20,26 @@ * * Model: `expectedChanges` folds every row's operations, in callback order, * over plain copies of the rows. It keeps enumerable fields whose final value - * differs under `===` or `Object.is` from the row's own field, plus deleted - * enumerable fields. A copy holds no non-enumerable field, so reading one - * gives `undefined`, deleting one does nothing, and writing one is a change - * unless it writes the row's value. A stored draft is the model's copy, read + * differs under `===` or `Object.is` from the row's own field, or, for the + * object a non-enumerable field holds, by contents; plus deleted enumerable + * fields. A copy holds no non-enumerable field, so reading one gives + * `undefined`, deleting one does nothing, and writing one is a change unless + * it writes the row's value. Defining a field acts as assigning it: an + * enumerable accessor reports the value it reads. A stored draft is the model's copy, read * when the callback returns. It does not import either tracker. When the - * callback throws, both trackers must rethrow the same error and leave the - * rows unchanged. + * callback throws, or a plain object rejects one of its operations, both + * trackers must throw the same error and leave the rows unchanged. * * History grammar: one to three rows with fields `a`, `b`, and `c` drawn from * `0`, `-0`, `1`, `NaN`, `''`, `'x'`, `true`, `false`, `null`, `undefined`, a * function, or missing, with a plain or null prototype, optionally frozen, - * and optionally with a non-enumerable field `h` holding a domain value. Each + * and optionally with a non-enumerable field `h` holding a domain value or an + * object. Each * row gets up to six operations: assign a field (`a` to `d`, or `h`) a value * from that domain or a fresh object, assign `d` `undefined`, set a field * back to its original value, delete a field, read it, define it with - * `Object.defineProperty` as an enumerable or non-enumerable data property, + * `Object.defineProperty` as an enumerable or non-enumerable data or accessor + * property, * or store a row's draft in it. A weighted run changes a field, adds `d` as * `undefined`, and reverts the field. Histories call the tracker with an * array or with a single row, and may throw after any operation. @@ -63,11 +67,6 @@ * Known omissions: nested objects, arrays, Dates, Maps, Sets, class * instances, and symbol keys are outside this owner; the proxy oracles and * contracts own them, and this file checks only that they fall back. - * The trackers disagree on four shapes, which the grammar excludes and - * `openDivergences` records as open decisions: an accessor defined in the - * callback, a field defined with the row's own value, a deep-equal write to a - * non-enumerable object field, and a non-enumerable field written and then - * deleted. Each marker fails once the trackers agree. */ import { fc, test as fcTest } from '@fast-check/vitest' import { describe, expect, it } from 'vitest' @@ -109,6 +108,12 @@ type Operation = | { kind: `delete`; field: string } | { kind: `read`; field: string } | { kind: `define`; field: string; value: unknown; enumerable: boolean } + | { + kind: `define-getter` + field: string + value: unknown + enumerable: boolean + } | { kind: `store-draft`; field: string; row: number } type RowShape = { fields: Array @@ -132,6 +137,27 @@ function sameValue(a: unknown, b: unknown): boolean { return a === b || Object.is(a, b) } +// Only a non-enumerable field holds an object, always `{ nested: 1 }`; an +// equal object written over it is not a change. +function sameContents(a: unknown, b: unknown): boolean { + if (sameValue(a, b)) return true + if ( + a === null || + b === null || + typeof a !== `object` || + typeof b !== `object` + ) + return false + const keys = Object.keys(a) + return ( + keys.length === Object.keys(b).length && + keys.every( + (key) => + Object.hasOwn(b, key) && sameValue((a as Row)[key], (b as Row)[key]), + ) + ) +} + function expectedChanges( originals: Array, operations: Array>, @@ -148,7 +174,7 @@ function expectedChanges( for (const key of Object.keys(draft)) { if ( !Object.hasOwn(original, key) || - !sameValue(draft[key], original[key]) + !sameContents(draft[key], original[key]) ) { changes[key] = draft[key] } @@ -182,6 +208,15 @@ function applyOperation( configurable: true, }) break + case `define-getter`: { + const value = domainValue(operation.value) + Object.defineProperty(draft, operation.field, { + get: () => value, + enumerable: operation.enumerable, + configurable: true, + }) + break + } case `store-draft`: draft[operation.field] = drafts[operation.row % drafts.length] break @@ -264,6 +299,12 @@ const operationArbitrary: fc.Arbitrary = fc.oneof( value: valueArbitrary, enumerable: fc.boolean(), }), + fc.record({ + kind: fc.constant(`define-getter` as const), + field: fieldArbitrary, + value: valueArbitrary, + enumerable: fc.boolean(), + }), fc.record({ kind: fc.constant(`store-draft` as const), field: fieldArbitrary, @@ -299,7 +340,7 @@ const rowArbitrary: fc.Arbitrary = fc.record({ frozen: fc.boolean(), hidden: fc.oneof( { weight: 2, arbitrary: fc.constant(MISSING) }, - fc.constantFrom(...values), + fc.constantFrom(...values, FRESH_OBJECT), ), }) const historyArbitrary: fc.Arbitrary = fc @@ -315,39 +356,6 @@ const historyArbitrary: fc.Arbitrary = fc ), }), ) - .filter((history) => !isExcluded(history)) - -// Excluded, each recorded in `openDivergences`: the proxy reports a field -// defined with the row's own value as changed, though it skips the same -// write through assignment, and reports a non-enumerable field written and -// then deleted as a deletion, though a plain delete reports nothing. -function isExcluded(history: History): boolean { - return history.rows.some((shape, index) => { - const original = buildRow(shape) - const operations = history.operations[index]! - const definesOriginalValue = operations.some( - (operation) => - operation.kind === `define` && - Object.hasOwn(original, operation.field) && - sameValue(operation.value, original[operation.field]), - ) - const firstHiddenWrite = operations.findIndex( - (operation) => - operation.field === `h` && - operation.kind !== `delete` && - operation.kind !== `read`, - ) - const deletesWrittenHidden = - shape.hidden !== MISSING && - firstHiddenWrite >= 0 && - operations - .slice(firstHiddenWrite) - .some( - (operation) => operation.kind === `delete` && operation.field === `h`, - ) - return definesOriginalValue || deletesWrittenHidden - }) -} const plainRow = (rowFields: Array, nullPrototype = false) => ({ fields: rowFields, @@ -358,15 +366,17 @@ const plainRow = (rowFields: Array, nullPrototype = false) => ({ const firstDraft = (draft: Array | Row) => Array.isArray(draft) ? draft[0]! : draft -const openDivergences: ReadonlyArray<{ +// Each witness pins a shape the trackers once disagreed on to the change +// set both must now report: defining a field acts as assigning it, and a +// non-enumerable field is not row data unless the callback writes it. +const descriptorLaws: ReadonlyArray<{ name: string shape: RowShape callback: (draft: Array | Row) => void + expected: Row }> = [ { - // The flat tracker reports the getter's value; the proxy records only - // data descriptors, as on `main`. - name: `the callback defines an accessor`, + name: `an enumerable accessor defined in the callback reports its value`, shape: plainRow([1, MISSING, MISSING]), callback: (draft) => { Object.defineProperty(firstDraft(draft), `g`, { @@ -375,10 +385,10 @@ const openDivergences: ReadonlyArray<{ configurable: true, }) }, + expected: { g: 7 }, }, { - // The proxy, as on `main`, reports the field; the flat tracker does not. - name: `the callback defines a field with its own value`, + name: `defining a field with its own value is not a change`, shape: plainRow([1, MISSING, MISSING]), callback: (draft) => { Object.defineProperty(firstDraft(draft), `a`, { @@ -388,30 +398,57 @@ const openDivergences: ReadonlyArray<{ configurable: true, }) }, + expected: {}, }, { - // The flat check skips non-enumerable fields, so this row takes the flat - // path, which compares by identity; the proxy compares deeply. - name: `a non-enumerable object field gets a deep-equal object`, + name: `an equal object written to a non-enumerable object field is not a change`, shape: { ...plainRow([1, MISSING, MISSING]), hidden: FRESH_OBJECT }, callback: (draft) => { firstDraft(draft).h = { nested: 1 } }, + expected: {}, }, { - // The proxy reports a deletion; the flat tracker, whose copy never held - // the field, reports nothing. - name: `a non-enumerable field is written and then deleted`, + name: `a non-enumerable field written and then deleted is not a change`, shape: { ...plainRow([1, MISSING, MISSING]), hidden: `x` }, callback: (draft) => { firstDraft(draft).h = `y` delete firstDraft(draft).h }, + expected: {}, }, ] // Each history isolates one place a plausible flat diff goes wrong. const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ + { + name: `a changed field defined back to its own value is not a change`, + history: { + rows: [plainRow([1, `x`, MISSING])], + operations: [ + [ + { kind: `set`, field: `a`, value: 0 }, + { kind: `define`, field: `a`, value: 1, enumerable: true }, + ], + ], + single: true, + throwAfter: undefined, + }, + }, + { + name: `assigning a getter-only field its own value throws`, + history: { + rows: [plainRow([1, MISSING, MISSING])], + operations: [ + [ + { kind: `define-getter`, field: `a`, value: 2, enumerable: true }, + { kind: `set`, field: `a`, value: 2 }, + ], + ], + single: false, + throwAfter: undefined, + }, + }, { name: `NaN written over NaN is not a change`, history: { @@ -550,12 +587,31 @@ function runHistory(history: History): void { ? [withChangeTracking(proxyRows[0]!, run(proxyRows))] : withArrayChangeTracking(proxyRows, run(proxyRows)), } + // A plain object rejects some callbacks itself, such as assigning a field + // the callback gave only a getter; the trackers must reject them too. + let model: Array | undefined + let modelRejects = false + try { + model = expectedChanges(originals, history.operations) + } catch (error) { + if (!(error instanceof TypeError)) throw error + modelRejects = true + } const throws = - history.throwAfter !== undefined && - history.throwAfter <= history.operations.flat().length + modelRejects || + (history.throwAfter !== undefined && + history.throwAfter <= history.operations.flat().length) if (throws) { - expect(track.flat, `flat rethrows`).toThrow(CallbackError) - expect(track.proxy, `proxy rethrows`).toThrow(CallbackError) + const errors = [track.flat, track.proxy].map((tracker) => { + try { + tracker() + } catch (error) { + return (error as Error).constructor + } + return undefined + }) + expect(errors[0], `flat rethrows`).toBeOneOf([CallbackError, TypeError]) + expect(errors[1], `proxy throws the same error`).toBe(errors[0]) for (const rows of [flatRows, proxyRows]) { expect(rows.map(rowState), `rows unchanged`).toStrictEqual( originals.map(rowState), @@ -566,7 +622,6 @@ function runHistory(history: History): void { const flat = track.flat() expect(flat, `flat rows take the flat path`).toBeDefined() const proxy = track.proxy() - const model = expectedChanges(originals, history.operations) expect(normalize(flat), `flat vs model`).toStrictEqual(normalize(model)) expect(normalize(proxy), `proxy vs model`).toStrictEqual(normalize(model)) } @@ -605,13 +660,16 @@ describe(`flat change tracking oracle`, () => { expect(changes).toStrictEqual({ a: { nested: 1 } }) }) - // Open decisions, not laws: the trackers disagree on these shapes, so - // the grammar excludes them. Each marker fails once the two agree. - for (const { name, shape, callback } of openDivergences) { - it.fails(`agrees with the draft proxy when ${name}`, () => { + for (const { name, shape, callback, expected } of descriptorLaws) { + it(`reports the same change set when ${name}`, () => { expect( withFlatChangeTracking([buildRow(shape)], callback, false), - ).toStrictEqual([withChangeTracking(buildRow(shape), callback)]) + `flat`, + ).toStrictEqual([expected]) + expect( + withChangeTracking(buildRow(shape), callback), + `proxy`, + ).toStrictEqual(expected) }) } From 1073a95a34b9f80970928f4f00b193558297db90 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:20:45 -0600 Subject: [PATCH 19/63] docs: record widened pooled, flat, and local-only oracle evidence Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 6 +- .../2026-10-01-pooled-live-queries.md | 81 ++++++++++++++----- 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 4a18da449..7f0f404a6 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -265,9 +265,9 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart. Wholesale and granular observers must match after every step, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and checks that a partition waits for its views' longest `gcTime`; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | -| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields) run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history failed before that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | -| Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React, and `eq-filter-peers` checks through `subscriberCount` that React shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and checks that a partition waits for its views' longest `gcTime`; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | | Minified DB public API | [consumer bundle check](https://github.com/TanStack/db/blob/main/scripts/test-minified-db.mjs) | CI builds `@tanstack/db`, then bundles its built `dist` entry with db-ivm inlined through esbuild `minify: true`. The build renames TypeScript-private members from `packages/db/mangle-cache.json`, so this lane runs against the renamed output that consumers install. It rejects a `dist` that is older than `src`, the cache, or the package's build configuration. `pnpm --filter @tanstack/db test:dist` also runs five db test files against the built `dist`, chosen for coverage of the modules with the most renamed members. `pnpm check:mangle` fails when a cached name is used other than as a private member in db src, is read by another package, appears as a string, or has a short name that collides with a source identifier ([private-member mangling review](oracle-reviews/code-weight-private-member-mangling.md)). It checks every exported error class's current public `name`, the built-in `BasicIndex` resolver name in index metadata and its event, complete query rows (after removing the four documented virtual fields) against an independent array filter/sort/projection, and one live update. The `new.target.name`, public-member mangle, and extra-output-field calibration modes fail at the intended observations. This fixed slice does not replace the unminified generated oracles, exercise framework adapters, or establish stable names for custom index resolvers. | | Boundary refinements | [cleanup/restart](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-cleanup-restart-oracle.test.ts), [issue #1891 async-cleanup review](oracle-reviews/issue-1891-async-cleanup.md), [issue #1891 cleanup-start follow-up](oracle-reviews/issue-1891-cleanup-start-followup.md), [metadata publication](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-metadata-publication-oracle.property.test.ts), [state retention](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-state-retention-oracle.property.test.ts), [acquisition cells](https://github.com/TanStack/db/blob/main/packages/db/tests/collection-subscription-lifecycle-oracle.test.ts), [D2 source reconciliation](https://github.com/TanStack/db/blob/main/packages/db/tests/d2-source-reconciliation-oracle.property.test.ts), [top-K support windows](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/operators/topk-support-window-oracle.test.ts), [nested Query work](https://github.com/TanStack/db/blob/main/packages/query-db-collection/tests/includes-work-counter-oracle.test.ts) | Explicit lifecycle products, independent source maps and weighted relations, exact publication cuts, support/multiplicity, and value-plus-work observations. These refine the larger subsystem models; they do not replace them. | diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md index ef470c174..612ec3bb8 100644 --- a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -32,29 +32,68 @@ restored after each run. Every outcome below is an assertion failure. | Mutant | Oracle | Tests failed | | --- | --- | --- | -| Partition ignores a row's previous group | Pooled | 6 of 7 | -| Groups keep rows in arrival order | Pooled | 2 of 7 (pinned key-order history and one campaign) | -| Literals and fields compared without `eq` normalization | Pooled | 3 of 7 | -| Partition skips the source's initial state | Pooled | 6 of 7 | -| View reports the source's status after cleanup | Pooled | 3 of 7 | -| Partition never terminates on cleanup | Pooled | 3 of 7 | -| Flat diff with `!==` | Flat | 3 of 8 | -| Flat diff with `Object.is` alone | Flat | 3 of 8 | -| Flat diff without deletions | Flat | 3 of 8 | -| Draft proxy without the added-field revert fix | Flat | 1 of 8 (pinned history only) | +| Partition ignores a row's previous group | Pooled | 6 of 9 | +| Groups keep rows in arrival order | Pooled | 2 of 9 (pinned key-order history and one campaign) | +| Literals and fields compared without `eq` normalization | Pooled | 3–4 of 9 | +| Partition skips the source's initial state | Pooled | 8 of 9 | +| View reports the source's status after cleanup | Pooled | 4 of 9 | +| Partition never terminates on cleanup | Pooled | 4 of 9 | +| Partition keys groups by the first field only | Pooled | 5 of 9 | +| Update published as delete then insert | Pooled | 3 of 9 (0 of 7 before payload checks) | +| Update carries a stale `previousValue` | Pooled | 3 of 9 (0 of 7 before) | +| Update carries the old row as its value | Pooled | 3 of 9 (0 of 7 before) | +| Within-group updates dropped | Pooled | 2 of 9 | +| Partition built on a cleaned-up source starts terminal | Pooled | 3 of 9 (0 of 7 before) | +| Frozen peers drop pending optimistic rows | Pooled | 2 of 9 (0 of 7 before) | +| Flat diff with `!==` | Flat | 3 of 17 | +| Flat diff with `Object.is` alone | Flat | 3 of 17 | +| Flat diff without deletions | Flat | 5 of 17 | +| Draft proxy without the added-field revert fix | Flat | pinned history and both campaigns | +| Flat diff ignores whether the row owns a field | Flat | 4 of 17 | +| Flat drafts edit the rows in place | Flat | 9 of 17 | +| Assigned objects not detached | Flat | 1 of 17 (detach witness) | +| Frozen rows sent to the proxy | Flat | 3 of 17 | +| Proxy ignores accessors defined in the callback | Flat | 3 of 17 | +| Proxy reports a field defined back to its own value | Flat | 4 of 17 | +| Proxy accepts a getter-only field's own value | Flat | 2 of 17 | +| Proxy reports a written hidden field's delete | Flat | 1 of 17 (pinned witness only) | +| Flat compares a hidden object field by identity | Flat | 1 of 17 (pinned witness only) | The pooled oracle found that a pooled view followed its source's status after cleanup instead of entering the live query's terminal error; the repair makes the partition terminate. The flat oracle found that the draft proxy dropped a field added as `undefined` when another field reverted, on `main` as well; the -repair treats a field the original lacks as changed. The generated flat -campaigns did not reach that history; only its pinned case kills the -unrepaired proxy. +repair treats a field the original lacks as changed. The grammar now weights +that run, so both campaigns also kill the unrepaired proxy. + +The loss audit then widened both oracles. The pooled granular observer had +been checked by key membership only, so the payload and lifecycle mutants +marked "0 of 7 before" survived it. The flat grammar gained frozen rows, +non-enumerable fields, accessor and data `defineProperty`, stored drafts, and +throwing callbacks. Four shapes split the trackers, three of them on `main`'s +proxy too. The adopted rule is that defining a field acts as assigning it, +and that a non-enumerable field is row data only once the callback writes +it. The proxy now records accessors and data defines through the assignment +path, rejects a getter-only write of its own value, and ignores the delete of +a hidden field it wrote. The flat tracker now compares a hidden object field +by contents. + +Local-only direct writes are compared with a local-only Collection whose +handlers resolve, across mixed multi-key batches, failing batches, schema +rejection, and every fallback for insert, update, and delete. Mutants that +ignore handler types, drop the pending or persisting check, drop the whole +transaction check, run before the ambient branch, write only the first +mutation, or never complete the transaction fail 4, 4, 4, 8, 12, 1, and 9 +tests. Two shared conformance scenarios, `eq-filter-rows` and `eq-filter-peers`, run the pooled path under React and compiled live queries under Vue, Solid, Svelte, and Angular. A partition that ignores a row's previous group fails -both under React and none elsewhere. +both under React and none elsewhere. `eq-filter-peers` also checks the +source's public `subscriberCount`: an adapter that declares `pooledEqFilters` +must share one subscription for two queries on the same fields. Disabling +pooling or using one partition per query fails it under React; before this +check the first passed. The loss audit found that a partition released its source one second after its last listener, whatever `gcTime` its views had. A focused release-timing @@ -71,9 +110,9 @@ oracle does not observe it. | ORC-001 Contract authority and limits | Pass. The `eq` operand rules come from `src/query/compiler/evaluators.ts`; the pooled boundary and the terminal-error rule come from the live-query architecture document's pooled section and cleanup law. The opening prose lists the omissions. | | ORC-002 Independent judgment | Pass. `expectedKeys` uses a local `eq` over plain values. Order, values, and status come from a live-query Collection, which compiles a D2 pipeline and does not use the partition. | | ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | -| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses only domain values and step kinds. Ablation: removing Dates, `NaN`, or `-0` loses the normalization mutant; removing optimistic steps loses rollback; removing mount and unmount loses remounted groups; removing cleanup-restart loses the terminal-error mutants; removing the second conjunct loses multi-field groups. Range: at most four rows, three peers, eight steps, and ids 0 through 5. Exclusion: an optimistic update to an equal value, an insert of an existing id, and an update or delete of a missing id are dropped. | -| ORC-005 Production path and observation | Pass. `createPooledLiveQuery` and `createLiveQueryObserver`, the adapter seam, run in wholesale and granular mode; the conformance scenarios run through React's `useLiveQuery`. Rows, keyed state, status, layout revision, and non-materialization are observed after every step. | -| ORC-006 Checker calibration | Pass. Six mutants, classified above. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses only domain values and step kinds. Ablation, run per axis on the campaigns with pinned cases skipped: without Dates, `NaN`, and `-0` the normalization mutant survives; without optimistic steps, frozen peers dropping optimistic rows survives; without mounts and unmounts, the terminal-at-creation mutant survives; without cleanup-restart, the status, termination, and both pending-cleanup mutants survive; without the second conjunct, first-field grouping survives. Range: at most four rows, three peers, eight steps, and ids 0 through 3; field values weight toward the literal so rows update within their group. Exclusion: an optimistic update to an equal value, an insert of an existing id, and an update or delete of a missing id are dropped. | +| ORC-005 Production path and observation | Pass. `createPooledLiveQuery` and `createLiveQueryObserver`, the adapter seam, run in wholesale and granular mode; the conformance scenarios run through React's `useLiveQuery`. Rows, keyed state, status, layout revision, and non-materialization are observed after every step, and granular changes are compared with a reference observer by type, key, value, and previous value, then replayed against the model. | +| ORC-006 Checker calibration | Pass. Thirteen mutants, classified above. | | ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_001`, an unseeded campaign, and a replay entry share one property and budget; the file is in `test:oracles`. | | ORC-008 Stateful-model minimality | Pass. The model keeps source rows and, per mounted peer, the frozen keys at cleanup. The pinned cleanup history distinguishes a frozen peer from one mounted after the restart. | | ORC-009 Vocabulary mapping | Pass. Equality partition, partition group, and pooled live query are glossary terms; a peer is one mounted pooled live query. | @@ -90,9 +129,9 @@ oracle does not observe it. | ORC-001 Contract authority and limits | Pass. The change-set rules come from `src/proxy.ts`'s `getChanges` contract: changed fields with their final value and deleted fields as `undefined`. | | ORC-002 Independent judgment | Pass. `expectedChanges` folds the operations over a plain copy and does not import either tracker. | | ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | -| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses domain values and operations. Ablation: removing `NaN`, `-0`, reverts, or deletions each loses a mutant. Range: one to three rows, three fields plus one added field, five operations per row. Exclusion: non-flat rows are rejected by the fallback witness. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses domain values and operations. Ablation: removing `NaN`, `-0`, reverts, or deletions each loses a mutant. Range: one to three rows, three fields plus one added and one non-enumerable field, frozen or not, up to six operations per row including data and accessor defines, stored drafts, and a throw. Exclusion: none in generation; non-flat rows are rejected by the fallback witness. Hidden-field deletes and hidden object fields are killed only by pinned witnesses. | | ORC-005 Production path and observation | Pass. `withFlatChangeTracking`, `withArrayChangeTracking`, and `withChangeTracking` run the same callbacks; the result change sets are observed. `collection.update` selects between them. | -| ORC-006 Checker calibration | Pass. Four mutants, classified above. | +| ORC-006 Checker calibration | Pass. Thirteen mutants, classified above. | | ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_101`, an unseeded campaign, and a replay entry; the file is in `test:oracles`. | | ORC-008 Stateful-model minimality | Not triggered. The model recomputes from the operations. | | ORC-009 Vocabulary mapping | Pass. Draft, change set, and revert follow the proxy's terms. | @@ -104,7 +143,7 @@ oracle does not observe it. ## Open work -- The generated flat campaigns do not reach the added-field revert history; - only its pinned case covers it. +- The generated flat campaigns rarely reach a hidden object field or a + written hidden field's delete; their pinned witnesses carry those laws. - Pooled live queries run only in React; the other adapters keep compiled live queries until they use the shared resolver. From a5d376ef11272d9d5edda2aba0d133464520b56e Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:25:47 -0600 Subject: [PATCH 20/63] test(db): weight the flat grammar toward the descriptor laws Hidden write-then-delete, equal objects over a hidden object field, and getter-only own-value writes now fail both campaigns, not only pinned witnesses. Co-authored-by: Isaac --- ...at-change-tracking-oracle.property.test.ts | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts index 897bbaf71..881c10163 100644 --- a/packages/db/tests/flat-change-tracking-oracle.property.test.ts +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -41,7 +41,9 @@ * `Object.defineProperty` as an enumerable or non-enumerable data or accessor * property, * or store a row's draft in it. A weighted run changes a field, adds `d` as - * `undefined`, and reverts the field. Histories call the tracker with an + * `undefined`, and reverts the field; others write and delete `h`, write an + * equal object over an object `h`, or define a getter and assign it its own + * value. Histories call the tracker with an * array or with a single row, and may throw after any operation. * * Production driver: `withFlatChangeTracking` and the proxy trackers @@ -320,6 +322,26 @@ const excursionArbitrary: fc.Arbitrary> = fc { kind: `set`, field: `d`, value: undefined }, { kind: `revert`, field }, ]) +// Runs that reach the descriptor laws, each of which a single operation +// rarely forms: a hidden field written then deleted, an equal object written +// over a hidden object field, and a getter-only field assigned its own value. +const descriptorRunArbitrary: fc.Arbitrary> = fc.oneof( + valueArbitrary.map( + (value): Array => [ + { kind: `set`, field: `h`, value }, + { kind: `delete`, field: `h` }, + ], + ), + fc.constant>([ + { kind: `set`, field: `h`, value: FRESH_OBJECT }, + ]), + fc.tuple(fc.constantFrom(...fields), valueArbitrary, fc.boolean()).map( + ([field, value, enumerable]): Array => [ + { kind: `define-getter`, field, value, enumerable }, + { kind: `set`, field, value }, + ], + ), +) const operationsArbitrary: fc.Arbitrary> = fc .array( fc.oneof( @@ -328,6 +350,7 @@ const operationsArbitrary: fc.Arbitrary> = fc arbitrary: operationArbitrary.map((operation) => [operation]), }, excursionArbitrary, + descriptorRunArbitrary, ), { maxLength: 5 }, ) @@ -340,7 +363,8 @@ const rowArbitrary: fc.Arbitrary = fc.record({ frozen: fc.boolean(), hidden: fc.oneof( { weight: 2, arbitrary: fc.constant(MISSING) }, - fc.constantFrom(...values, FRESH_OBJECT), + fc.constant(FRESH_OBJECT), + fc.constantFrom(...values), ), }) const historyArbitrary: fc.Arbitrary = fc From 035e8a0949441732639f52af56d82306e77ac7df Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:31:55 -0600 Subject: [PATCH 21/63] test(db): reach arrival order and frozen optimistic rows in pooled campaigns Initial rows may arrive in reverse key order, and a weighted run inserts a row most peers see before cleanup, so every pooled mutant now fails both campaigns. Co-authored-by: Isaac --- .../2026-10-01-pooled-live-queries.md | 29 +++++++---- .../pooled-live-query-oracle.property.test.ts | 49 ++++++++++++++----- 2 files changed, 55 insertions(+), 23 deletions(-) diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md index 612ec3bb8..4d4778e17 100644 --- a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -33,8 +33,8 @@ restored after each run. Every outcome below is an assertion failure. | Mutant | Oracle | Tests failed | | --- | --- | --- | | Partition ignores a row's previous group | Pooled | 6 of 9 | -| Groups keep rows in arrival order | Pooled | 2 of 9 (pinned key-order history and one campaign) | -| Literals and fields compared without `eq` normalization | Pooled | 3–4 of 9 | +| Groups keep rows in arrival order | Pooled | 3 of 9 | +| Literals and fields compared without `eq` normalization | Pooled | 4 of 9 | | Partition skips the source's initial state | Pooled | 8 of 9 | | View reports the source's status after cleanup | Pooled | 4 of 9 | | Partition never terminates on cleanup | Pooled | 4 of 9 | @@ -42,9 +42,9 @@ restored after each run. Every outcome below is an assertion failure. | Update published as delete then insert | Pooled | 3 of 9 (0 of 7 before payload checks) | | Update carries a stale `previousValue` | Pooled | 3 of 9 (0 of 7 before) | | Update carries the old row as its value | Pooled | 3 of 9 (0 of 7 before) | -| Within-group updates dropped | Pooled | 2 of 9 | +| Within-group updates dropped | Pooled | 3 of 9 | | Partition built on a cleaned-up source starts terminal | Pooled | 3 of 9 (0 of 7 before) | -| Frozen peers drop pending optimistic rows | Pooled | 2 of 9 (0 of 7 before) | +| Frozen peers drop pending optimistic rows | Pooled | 3 of 9 (0 of 7 before) | | Flat diff with `!==` | Flat | 3 of 17 | | Flat diff with `Object.is` alone | Flat | 3 of 17 | | Flat diff without deletions | Flat | 5 of 17 | @@ -55,9 +55,20 @@ restored after each run. Every outcome below is an assertion failure. | Frozen rows sent to the proxy | Flat | 3 of 17 | | Proxy ignores accessors defined in the callback | Flat | 3 of 17 | | Proxy reports a field defined back to its own value | Flat | 4 of 17 | -| Proxy accepts a getter-only field's own value | Flat | 2 of 17 | -| Proxy reports a written hidden field's delete | Flat | 1 of 17 (pinned witness only) | -| Flat compares a hidden object field by identity | Flat | 1 of 17 (pinned witness only) | +| Proxy accepts a getter-only field's own value | Flat | 3 of 17 | +| Proxy reports a written hidden field's delete | Flat | 3 of 17 | +| Flat compares a hidden object field by identity | Flat | 3 of 17 | + +Every pooled and flat mutant above fails its pinned history and both +campaigns. The pooled grammar delivers initial rows in key order or in +reverse and weights a pending insert most peers see before cleanup; the flat +grammar weights runs that write and delete a hidden field, write an equal +object over a hidden object field, and assign a getter-only field its own +value. Before that weighting, arrival order and frozen optimistic rows +escaped about one random campaign in three, and the three descriptor +mutants escaped both campaigns. The `Object.is`-alone flat mutant still +escapes about one random campaign in two; its pinned `-0` history and the +fixed campaign always kill it. The pooled oracle found that a pooled view followed its source's status after cleanup instead of entering the live query's terminal error; the repair makes @@ -129,7 +140,7 @@ oracle does not observe it. | ORC-001 Contract authority and limits | Pass. The change-set rules come from `src/proxy.ts`'s `getChanges` contract: changed fields with their final value and deleted fields as `undefined`. | | ORC-002 Independent judgment | Pass. `expectedChanges` folds the operations over a plain copy and does not import either tracker. | | ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | -| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses domain values and operations. Ablation: removing `NaN`, `-0`, reverts, or deletions each loses a mutant. Range: one to three rows, three fields plus one added and one non-enumerable field, frozen or not, up to six operations per row including data and accessor defines, stored drafts, and a throw. Exclusion: none in generation; non-flat rows are rejected by the fallback witness. Hidden-field deletes and hidden object fields are killed only by pinned witnesses. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses domain values and operations. Ablation: removing `NaN`, `-0`, reverts, or deletions each loses a mutant. Range: one to three rows, three fields plus one added and one non-enumerable field, frozen or not, up to six operations per row including data and accessor defines, stored drafts, and a throw. Exclusion: none in generation; non-flat rows are rejected by the fallback witness. | | ORC-005 Production path and observation | Pass. `withFlatChangeTracking`, `withArrayChangeTracking`, and `withChangeTracking` run the same callbacks; the result change sets are observed. `collection.update` selects between them. | | ORC-006 Checker calibration | Pass. Thirteen mutants, classified above. | | ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_101`, an unseeded campaign, and a replay entry; the file is in `test:oracles`. | @@ -143,7 +154,5 @@ oracle does not observe it. ## Open work -- The generated flat campaigns rarely reach a hidden object field or a - written hidden field's delete; their pinned witnesses carry those laws. - Pooled live queries run only in React; the other adapters keep compiled live queries until they use the shared resolver. diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 746fd28d5..80e214bef 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -20,7 +20,8 @@ * the partition. Order, row values, and status come from a second * formulation: a live-query Collection compiled for the same query. * - * History grammar: rows have ids 0 through 3, a field `f` from strings, + * History grammar: rows have ids 0 through 3, delivered initially in key + * order or in reverse, a field `f` from strings, * numbers and their look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, * `0`, `null`, and a missing value, and a field `g` of `x` or `y`. Up to * three peer queries use `eq(f, literal)`, optionally with `eq(g, literal)`. @@ -29,8 +30,8 @@ * Steps commit sync transactions of one or two inserts, updates, or deletes, * where an update may keep `f` so the row stays in its group; apply one * optimistic insert, update, or delete and then confirm or roll it back, - * optionally after cleaning up and restarting the source while it is - * pending; mount or unmount a peer; or clean up the source and restart it, + * optionally after cleaning up and restarting the source while it is pending, + * with a weighted run that inserts a row most peers see before that cleanup; mount or unmount a peer; or clean up the source and restart it, * optionally (re)mounting a peer on the cleaned-up source first. * * Production driver: `createPooledLiveQuery` builds each peer's view from the @@ -111,6 +112,8 @@ type Op = | { type: `delete`; id: number } type History = { rows: Array<{ f: FieldValue; g: string }> + // The source delivers its initial rows in reverse key order. + reverseInitial?: boolean peers: Array steps: Array } @@ -225,6 +228,7 @@ const historyArbitrary: fc.Arbitrary = fc.record({ rows: fc.array(fc.record({ f: fieldArbitrary, g: gArbitrary }), { maxLength: 4, }), + reverseInitial: fc.boolean(), peers: fc.array(peerArbitrary, { minLength: 1, maxLength: 3 }), steps: fc.array( fc.oneof( @@ -235,15 +239,31 @@ const historyArbitrary: fc.Arbitrary = fc.record({ ops: fc.array(opArbitrary, { minLength: 1, maxLength: 2 }), }), }, - fc.record( - { - kind: fc.constant(`optimistic` as const), - op: opArbitrary, - confirm: fc.boolean(), - cleanupFirst: fc.boolean(), - }, - { requiredKeys: [`kind`, `op`, `confirm`] }, - ), + { + weight: 2, + arbitrary: fc.record( + { + kind: fc.constant(`optimistic` as const), + op: opArbitrary, + confirm: fc.boolean(), + cleanupFirst: fc.boolean(), + }, + { requiredKeys: [`kind`, `op`, `confirm`] }, + ), + }, + // A pending write that most peers see, settled after cleanup, is what + // a freeze must keep; independent choices rarely line it up. + fc.record({ + kind: fc.constant(`optimistic` as const), + op: fc.record({ + type: fc.constant(`insert` as const), + id: fc.nat({ max: 3 }), + f: fc.constant(`a`), + g: gArbitrary, + }), + confirm: fc.boolean(), + cleanupFirst: fc.constant(true), + }), fc.record({ kind: fc.constant(`mount` as const), peer: fc.nat({ max: 2 }), @@ -447,7 +467,10 @@ async function runHistory(history: History): Promise { mockSyncCollectionOptions({ id: `pooled-${serial++}`, getKey: (row) => row.id, - initialData: [...rows.values()].map((row) => ({ ...row })), + initialData: (history.reverseInitial + ? [...rows.values()].reverse() + : [...rows.values()] + ).map((row) => ({ ...row })), }), ) await source.stateWhenReady() From 6d08832db1448e66b32b97e821f006a5e2ca8bd4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:36:32 -0600 Subject: [PATCH 22/63] test(db): flip signed zeros in flat campaigns The Object.is-alone diff escaped about half of random campaigns because a zero field rarely received the other zero by chance. Co-authored-by: Isaac --- .../2026-10-01-pooled-live-queries.md | 9 ++++----- .../flat-change-tracking-oracle.property.test.ts | 15 +++++++++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md index 4d4778e17..7c4b90d84 100644 --- a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -63,12 +63,11 @@ Every pooled and flat mutant above fails its pinned history and both campaigns. The pooled grammar delivers initial rows in key order or in reverse and weights a pending insert most peers see before cleanup; the flat grammar weights runs that write and delete a hidden field, write an equal -object over a hidden object field, and assign a getter-only field its own -value. Before that weighting, arrival order and frozen optimistic rows +object over a hidden object field, assign a getter-only field its own value, and write the opposite-signed zero +over each zero field. Before that weighting, arrival order and frozen optimistic rows escaped about one random campaign in three, and the three descriptor -mutants escaped both campaigns. The `Object.is`-alone flat mutant still -escapes about one random campaign in two; its pinned `-0` history and the -fixed campaign always kill it. +mutants escaped both campaigns. The `Object.is`-alone flat mutant escaped about one random campaign in two +until the zero-flip run. The pooled oracle found that a pooled view followed its source's status after cleanup instead of entering the live query's terminal error; the repair makes diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts index 881c10163..fae011236 100644 --- a/packages/db/tests/flat-change-tracking-oracle.property.test.ts +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -42,8 +42,8 @@ * property, * or store a row's draft in it. A weighted run changes a field, adds `d` as * `undefined`, and reverts the field; others write and delete `h`, write an - * equal object over an object `h`, or define a getter and assign it its own - * value. Histories call the tracker with an + * equal object over an object `h`, or define a getter and assign it its own value; and one writes the + * opposite-signed zero over each zero field. Histories call the tracker with an * array or with a single row, and may throw after any operation. * * Production driver: `withFlatChangeTracking` and the proxy trackers @@ -117,6 +117,8 @@ type Operation = enumerable: boolean } | { kind: `store-draft`; field: string; row: number } + // Writes the opposite-signed zero over every field the row holds as a zero. + | { kind: `flip-zeros` } type RowShape = { fields: Array nullPrototype: boolean @@ -219,6 +221,13 @@ function applyOperation( }) break } + case `flip-zeros`: + for (const key of Object.keys(original)) { + if (original[key] === 0) { + draft[key] = Object.is(original[key], 0) ? -0 : 0 + } + } + break case `store-draft`: draft[operation.field] = drafts[operation.row % drafts.length] break @@ -351,6 +360,8 @@ const operationsArbitrary: fc.Arbitrary> = fc }, excursionArbitrary, descriptorRunArbitrary, + // A field drawn as a zero rarely gets the other zero by chance. + fc.constant>([{ kind: `flip-zeros` }]), ), { maxLength: 5 }, ) From edca2bc9f7dabd8180c2484bf352d9a1707654b2 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 12:45:22 -0600 Subject: [PATCH 23/63] fix: run development checks in browser bundles The duplicate-instance check and React development warnings returned early when no process global existed, which disabled them in every browser bundle even though bundlers inline process.env.NODE_ENV. Co-authored-by: Isaac --- .changeset/fix-browser-development-checks.md | 6 ++ packages/db/src/duplicate-instance-check.ts | 16 +++-- .../db/tests/duplicate-instance-check.test.ts | 65 +++++++++++++++++++ packages/react-db/src/development.ts | 14 ++++ packages/react-db/src/useLiveQuery.ts | 11 +--- packages/react-db/tests/development.test.ts | 50 ++++++++++++++ 6 files changed, 148 insertions(+), 14 deletions(-) create mode 100644 .changeset/fix-browser-development-checks.md create mode 100644 packages/db/tests/duplicate-instance-check.test.ts create mode 100644 packages/react-db/src/development.ts create mode 100644 packages/react-db/tests/development.test.ts diff --git a/.changeset/fix-browser-development-checks.md b/.changeset/fix-browser-development-checks.md new file mode 100644 index 000000000..89c15fe0e --- /dev/null +++ b/.changeset/fix-browser-development-checks.md @@ -0,0 +1,6 @@ +--- +'@tanstack/db': patch +'@tanstack/react-db': patch +--- + +Run development-only checks in browser development builds. The duplicate `@tanstack/db` instance check and React's development warnings (deprecated dependency arrays, unhashable query identity) skipped themselves whenever there was no `process` global, which is the case in Vite and other browser bundles even though they inline `process.env.NODE_ENV`. They now read `process.env.NODE_ENV` as bundlers expect, so an app that loads two copies of `@tanstack/db` in development throws `DuplicateDbInstanceError` as documented. Set `process.env.TANSTACK_DB_DISABLE_DUP_CHECK` to `'1'` through your bundler's `define` to turn the check off. diff --git a/packages/db/src/duplicate-instance-check.ts b/packages/db/src/duplicate-instance-check.ts index 2f38a562f..0010359e1 100644 --- a/packages/db/src/duplicate-instance-check.ts +++ b/packages/db/src/duplicate-instance-check.ts @@ -18,11 +18,19 @@ function isBrowserTopWindow(): boolean { // Detect duplicate @tanstack/db instances (dev-only, browser top-window only) const DB_INSTANCE_MARKER = Symbol.for(`@tanstack/db/instance-marker`) -const DEV = - typeof process !== `undefined` && process.env.NODE_ENV !== `production` +// Bundlers inline these reads even where `process` does not exist, so they +// are read directly; without a bundler or `process`, the check stays off. +function readEnv(read: () => string | undefined): string | undefined | null { + try { + return read() + } catch { + return null + } +} +const nodeEnv = readEnv(() => process.env.NODE_ENV) +const DEV = nodeEnv !== null && nodeEnv !== `production` const DISABLED = - typeof process !== `undefined` && - process.env.TANSTACK_DB_DISABLE_DUP_CHECK === `1` + readEnv(() => process.env.TANSTACK_DB_DISABLE_DUP_CHECK) === `1` if (DEV && !DISABLED && isBrowserTopWindow()) { if ((globalThis as any)[DB_INSTANCE_MARKER]) { diff --git a/packages/db/tests/duplicate-instance-check.test.ts b/packages/db/tests/duplicate-instance-check.test.ts new file mode 100644 index 000000000..3338cdbcd --- /dev/null +++ b/packages/db/tests/duplicate-instance-check.test.ts @@ -0,0 +1,65 @@ +// @vitest-environment node +import { readFileSync } from 'node:fs' +import { createContext, runInContext } from 'node:vm' +import { transformSync } from 'esbuild' +import { describe, expect, it } from 'vitest' + +const source = readFileSync( + new URL(`../src/duplicate-instance-check.ts`, import.meta.url), + `utf8`, +) + +class DuplicateDbInstanceError extends Error {} + +// Loads the check twice in one browser top window, as two bundled copies of +// the package would: `define` is what the bundler inlines, and the window +// has no `process` global. +function loadTwice(define: Record): unknown { + const { code } = transformSync(source, { + loader: `ts`, + format: `cjs`, + define, + }) + const window: Record = { document: {} } + window.top = window + const context = createContext({ window }) + // Each copy runs in its own module scope, sharing the window's globals. + const load = runInContext( + `(function (module, exports, require) {\n${code}\n})`, + context, + ) as ( + module: { exports: object }, + exports: object, + require: () => object, + ) => void + try { + for (let copy = 0; copy < 2; copy++) { + const bundle = { exports: {} } + load(bundle, bundle.exports, () => ({ DuplicateDbInstanceError })) + } + } catch (error) { + return error + } + return undefined +} + +describe(`duplicate @tanstack/db instance check`, () => { + it(`rejects a second copy in a browser development bundle`, () => { + expect( + loadTwice({ 'process.env.NODE_ENV': `"development"` }), + ).toBeInstanceOf(DuplicateDbInstanceError) + }) + + it(`stays off in production, when disabled, and without a bundler`, () => { + expect( + loadTwice({ 'process.env.NODE_ENV': `"production"` }), + ).toBeUndefined() + expect( + loadTwice({ + 'process.env.NODE_ENV': `"development"`, + 'process.env.TANSTACK_DB_DISABLE_DUP_CHECK': `"1"`, + }), + ).toBeUndefined() + expect(loadTwice({})).toBeUndefined() + }) +}) diff --git a/packages/react-db/src/development.ts b/packages/react-db/src/development.ts new file mode 100644 index 000000000..05269bb80 --- /dev/null +++ b/packages/react-db/src/development.ts @@ -0,0 +1,14 @@ +// Bundlers inline `process.env.NODE_ENV` even where `process` does not exist, +// so it is read directly; without either, warnings stay off. +export function shouldWarnInDevelopment(disableEnvVar: string): boolean { + let development: boolean + try { + development = process.env.NODE_ENV !== `production` + } catch { + return false + } + return ( + development && + (typeof process === `undefined` || process.env[disableEnvVar] !== `1`) + ) +} diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index 36335c0c9..a12fad85d 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -15,6 +15,7 @@ import { } from '@tanstack/db' import { useOptionalDbClient } from './DbProvider' import { setLiveQueryResultInfo } from './live-query-internals' +import { shouldWarnInDevelopment } from './development' import type { Collection, CollectionImpl, @@ -172,16 +173,6 @@ export function warnDeprecatedDepsArray( ) } -function shouldWarnInDevelopment(disableEnvVar: string): boolean { - if (typeof process === `undefined`) { - return false - } - - return ( - process.env.NODE_ENV !== `production` && process.env[disableEnvVar] !== `1` - ) -} - function getCurrentTime(): number { return typeof performance !== `undefined` && typeof performance.now === `function` diff --git a/packages/react-db/tests/development.test.ts b/packages/react-db/tests/development.test.ts new file mode 100644 index 000000000..15a62363c --- /dev/null +++ b/packages/react-db/tests/development.test.ts @@ -0,0 +1,50 @@ +// @vitest-environment node +import { readFileSync } from 'node:fs' +import { createContext, runInContext } from 'node:vm' +import { transformSync } from 'esbuild' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { shouldWarnInDevelopment } from '../src/development' + +const source = readFileSync( + new URL(`../src/development.ts`, import.meta.url), + `utf8`, +) + +// Evaluates the module as a browser bundle would: `NODE_ENV` inlined by the +// bundler, or left alone, and no `process` global. +function inBrowser(nodeEnv: string | undefined): (name: string) => boolean { + const { code } = transformSync(source, { + loader: `ts`, + format: `cjs`, + define: + nodeEnv === undefined + ? {} + : { 'process.env.NODE_ENV': JSON.stringify(nodeEnv) }, + }) + const bundle = { exports: {} as Record } + runInContext(code, createContext({ module: bundle, exports: bundle.exports })) + return bundle.exports.shouldWarnInDevelopment as (name: string) => boolean +} + +describe(`development warnings`, () => { + afterEach(() => vi.unstubAllEnvs()) + + it(`warn in a browser development bundle without a process global`, () => { + expect(inBrowser(`development`)(`DISABLE`)).toBe(true) + }) + + it(`stay off in a browser production bundle or without a bundler`, () => { + expect(inBrowser(`production`)(`DISABLE`)).toBe(false) + expect(inBrowser(undefined)(`DISABLE`)).toBe(false) + }) + + it(`honor the disable variable and production under Node`, () => { + vi.stubEnv(`NODE_ENV`, `development`) + expect(shouldWarnInDevelopment(`DISABLE`)).toBe(true) + vi.stubEnv(`DISABLE`, `1`) + expect(shouldWarnInDevelopment(`DISABLE`)).toBe(false) + vi.unstubAllEnvs() + vi.stubEnv(`NODE_ENV`, `production`) + expect(shouldWarnInDevelopment(`DISABLE`)).toBe(false) + }) +}) From 2918b231f9b226771f1a57466985e691845c922c Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:20:01 -0600 Subject: [PATCH 24/63] fix(db): refill a released partition's groups when a view resubscribes A release cleared the partition's groups, so a view that subscribed again afterwards, as a hidden React Activity does, read a group the revived subscription no longer filled and kept its old rows. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- packages/db/src/query/pooled-live-query.ts | 9 +++- .../tests/query/pooled-live-query-gc.test.ts | 46 ++++++++++++++++++- 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 7f0f404a6..f8a9fbd81 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -265,7 +265,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React, and `eq-filter-peers` checks through `subscriberCount` that React shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and checks that a partition waits for its views' longest `gcTime`; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React, and `eq-filter-peers` checks through `subscriberCount` that React shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 2473cfb65..0523a7b77 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -189,7 +189,14 @@ class Partition { this.subscription = undefined this.stopStatusEvents?.() this.stopStatusEvents = undefined - this.groups.clear() + // Views outlive a release and may subscribe again, so they keep their + // groups for the next subscription to refill. + for (const group of this.groups.values()) { + group.rows.clear() + group.revision++ + group.layoutRevision++ + group.entries = undefined + } this.onEmpty() }, delay) } diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index a22bc72b0..8fc7de350 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -13,8 +13,9 @@ import { mockSyncCollectionOptions } from '../utils.js' * the live-query Collection it stands in for. After the last subscriber * leaves, that is exactly `gcTime`; a `gcTime` of 0 or `Infinity` never * releases; and a query built but never subscribed waits at least the - * Collection lifecycle's 50 ms floor. Views of one partition with different - * `gcTime`s release with the longest. + * Collection lifecycle's 50 ms floor. Views of one partition with different `gcTime`s release with the longest. + * A view that subscribes again after its partition released, as a hidden + * React Activity does, follows the source like its restarted Collection. * * Each case runs the same timeline against a pooled query and a live-query * Collection on separate sources and compares when each source loses its @@ -89,6 +90,47 @@ describe(`pooled live query gcTime`, () => { } } + it(`follows its source again when resubscribed after its partition released`, async () => { + const run = async ( + build: (source: ReturnType) => any, + ) => { + const source = makeSource() + const observer = createLiveQueryObserver(build(source), { + mode: `wholesale`, + }) + observer.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(50) + // Hidden past gcTime, as under a hidden React Activity, then shown. + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `b`, g: `x` } }) + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.commit() + const stop = observer.subscribe(() => {}) + await vi.advanceTimersByTimeAsync(1) + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `c`, g: `x` } }) + source.utils.commit() + await vi.advanceTimersByTimeAsync(1) + const keys = [...observer.getSnapshot().state!.keys()] + stop() + return keys + } + const compiledKeys = await run((source) => + createLiveQueryCollection({ + query: query(source), + startSync: true, + gcTime: 1, + }), + ) + expect(compiledKeys).toEqual([`b`, `c`]) + expect( + await run( + (source) => + createPooledLiveQuery(query(source)(new Query()), { gcTime: 1 })!, + ), + ).toEqual(compiledKeys) + }) + it.each([[[5, 120]], [[120, 5]]])( `releases with the longest gcTime among a partition's views (%j)`, async (gcTimes) => { From 1d5a816ef7e9c4c83c58400d70849ee85c16f5e0 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:22:41 -0600 Subject: [PATCH 25/63] perf(db): build pooled snapshots in one pass from the group's rows Reads keys and rows straight from the group and takes its layout revision instead of comparing key arrays per snapshot. Co-authored-by: Isaac --- packages/db/src/query/pooled-live-query.ts | 30 ++++++++++++---------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 0523a7b77..01b1d8c37 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -367,6 +367,11 @@ class PooledLiveQuery { return this.group.layoutRevision } + /** The group's rows, in key order. */ + get rows(): SortedMap { + return this.group.rows + } + entries(): Array<[string | number, Row]> { return (this.group.entries ??= [...this.group.rows.entries()]) } @@ -447,8 +452,7 @@ class PooledWholesaleObserver implements LiveQueryObserver< private snapshot: LiveQuerySnapshot | undefined private snapshotRevision = -1 private snapshotStatus: CollectionStatus | undefined - private layoutKeys: Array = [] - private layoutRevision = 0 + private readonly records = new Set<{ listener: LiveQueryObserverListener }>() @@ -472,22 +476,22 @@ class PooledWholesaleObserver implements LiveQueryObserver< ) { return this.snapshot } - const entries = this.view.entries() - const keys = entries.map(([key]) => key) - if ( - keys.length !== this.layoutKeys.length || - keys.some((key, index) => key !== this.layoutKeys[index]) - ) { - this.layoutKeys = keys - this.layoutRevision++ + const rows = this.view.rows + const state = new Map() + const data: Array = [] + for (const key of rows.keys()) { + const value = rows.get(key)! + state.set(key, value) + data.push(value) } this.snapshotRevision = this.view._stateRevision this.snapshotStatus = status return (this.snapshot = { - state: new Map(entries), - data: entries.map(([, value]) => value), + state, + data, collection: this.view.publicCollection, - layoutRevision: this.layoutRevision, + // Rows stay in key order, so only inserts and deletes move keys. + layoutRevision: this.view._layoutRevision, status, ...getLiveQueryStatusFlags(status), persistedStatus: `unavailable`, From f038f5665ed7420a91738ffa525c696cae1fad0d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:25:22 -0600 Subject: [PATCH 26/63] perf(react-db): keep useLiveQuery's refs in one hook slot A render no longer looks up twelve refs or allocates their unused initial values. Co-authored-by: Isaac --- packages/react-db/src/useLiveQuery.ts | 74 ++++++++++++++++----------- 1 file changed, 45 insertions(+), 29 deletions(-) diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index a12fad85d..24a05d443 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -728,6 +728,33 @@ export function useLiveQueryForSuspense( return useLiveQueryImpl(configOrQueryOrCollection, deps, true) } +const ref = (current: T): { current: T } => ({ current }) + +// Every ref the hook keeps, created once per hook instance. +function createHookRefs(dbClient: DbClient | undefined) { + return { + collectionRef: ref | null>(null), + depsRef: ref | null>(null), + configRef: ref(null), + clientRef: ref(dbClient), + legacyUnhashableIdentityRef: ref>([`legacy-unhashable`]), + derivedIdentityProfilerRef: ref({ + renderCount: 0, + totalMs: 0, + maxMs: 0, + warned: false, + }), + deferredCollectionsRef: ref( + new Set>(), + ), + observerRef: ref | null>(null), + queryHashRef: ref(undefined), + suspenseKeyRef: ref(undefined), + identityErrorRef: ref(undefined), + subscribeRef: ref<((onStoreChange: () => void) => () => void) | null>(null), + } +} + function useLiveQueryImpl( configOrQueryOrCollection: any, deps: Array | undefined, @@ -741,32 +768,23 @@ function useLiveQueryImpl( : (getExplicitDbClient(configOrQueryOrCollection) ?? contextDbClient) const resolvedDeps = deps ?? [] - // Use refs to cache collection and track dependencies - const collectionRef = useRef | null>( - null, - ) - const depsRef = useRef | null>(null) - const configRef = useRef(null) - const clientRef = useRef(dbClient) - const legacyUnhashableIdentityRef = useRef>([ - `legacy-unhashable`, - ]) - - const derivedIdentityProfilerRef = useRef({ - renderCount: 0, - totalMs: 0, - maxMs: 0, - warned: false, - }) - const deferredCollectionsRef = useRef( - new Set>(), - ) - const observerRef = useRef | null>( - null, - ) - const queryHashRef = useRef(undefined) - const suspenseKeyRef = useRef(undefined) - const identityErrorRef = useRef(undefined) + // One hook slot holds every ref, so a render neither looks up nor + // allocates the others. + const refsRef = useRef | null>(null) + const { + collectionRef, + depsRef, + configRef, + clientRef, + legacyUnhashableIdentityRef, + derivedIdentityProfilerRef, + deferredCollectionsRef, + observerRef, + queryHashRef, + suspenseKeyRef, + identityErrorRef, + subscribeRef, + } = (refsRef.current ??= createHookRefs(dbClient)) const queryKey = !inputIsCollection ? getExplicitQueryKey(configOrQueryOrCollection) @@ -1013,9 +1031,7 @@ function useLiveQueryImpl( // Stable subscribe bound to the current observer; the observer owns the // subscription, ready-race, and disposal. - const subscribeRef = useRef< - ((onStoreChange: () => void) => () => void) | null - >(null) + if (!subscribeRef.current || needsNewCollection) { subscribeRef.current = (onStoreChange: () => void) => { const unsubscribe = observer.subscribe(() => onStoreChange()) From 03dd1e160488b39c0f355d54babb387695fd6a9e Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:34:46 -0600 Subject: [PATCH 27/63] perf(db): reuse the tracker's change object and one timestamp per write Without a schema, an update's changes are the tracker's fresh change object, not a rebuilt copy, and each insert, update, or delete call shares one Date. Saves about 94 KiB per 200-row update batch. Co-authored-by: Isaac --- packages/db/src/collection/mutations.ts | 34 ++++++++++++++++--------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index ad9ee14dd..a413c015f 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -248,6 +248,8 @@ export class CollectionMutationsManager< } const items = Array.isArray(data) ? data : [data] + // One timestamp per call; mutations replace these rather than mutate them. + const now = new Date() const mutations: Array> = [] const keysInCurrentBatch = new Set() @@ -283,8 +285,8 @@ export class CollectionMutationsManager< syncMetadata: this.config.sync.getSyncMetadata?.() || {}, optimistic: config?.optimistic ?? true, type: `insert`, - createdAt: new Date(), - updatedAt: new Date(), + createdAt: now, + updatedAt: now, collection: this.collection, } @@ -419,6 +421,8 @@ export class CollectionMutationsManager< ]) // Create mutations for each object that has changes + // One timestamp per call; mutations replace these rather than mutate them. + const now = new Date() const mutations: Array< PendingMutation< TOutput, @@ -463,12 +467,16 @@ export class CollectionMutationsManager< // where a schema has default values or transforms. The modified data has the extra // default or transformed values but for changes, we just want to show the data that // was actually passed in. - changes: Object.fromEntries( - Object.keys(itemChanges).map((k) => [ - k, - modifiedItem[k as keyof typeof modifiedItem], - ]), - ) as TInput, + // Without a schema, validation returns the tracker's fresh change + // object, which already holds exactly these values. + changes: (validatedUpdatePayload === itemChanges + ? itemChanges + : Object.fromEntries( + Object.keys(itemChanges).map((k) => [ + k, + modifiedItem[k as keyof typeof modifiedItem], + ]), + )) as TInput, globalKey, key, metadata: config.metadata as unknown, @@ -478,8 +486,8 @@ export class CollectionMutationsManager< >, optimistic: config.optimistic ?? true, type: `update`, - createdAt: new Date(), - updatedAt: new Date(), + createdAt: now, + updatedAt: now, collection: this.collection, } }) @@ -578,6 +586,8 @@ export class CollectionMutationsManager< const keysArray = Array.isArray(keys) ? keys : [keys] this.collection._sync.startSync() + // One timestamp per call; mutations replace these rather than mutate them. + const now = new Date() const mutations: Array< PendingMutation< TOutput, @@ -609,8 +619,8 @@ export class CollectionMutationsManager< >, optimistic: config?.optimistic ?? true, type: `delete`, - createdAt: new Date(), - updatedAt: new Date(), + createdAt: now, + updatedAt: now, collection: this.collection, } From 10d0309edc0df487b1ec92aada440a96ee09ab26 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:37:48 -0600 Subject: [PATCH 28/63] perf(db): allocate deepEquals' cycle map only for nested containers Comparing flat rows no longer creates a Map or registers each object. Saves about 59 KiB per 200-row update batch. Co-authored-by: Isaac --- packages/db/src/utils.ts | 49 +++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index 1bca8e9a9..3bb2f0a7f 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -27,7 +27,7 @@ interface TypedArray { * ``` */ export function deepEquals(a: any, b: any): boolean { - return deepEqualsInternal(a, b, new Map()) + return deepEqualsInternal(a, b, undefined) } function enumerableOwnKeys(value: object): Array { @@ -45,7 +45,8 @@ function enumerableOwnKeys(value: object): Array { export function deepEqualsInternal( a: any, b: any, - visited: Map, + // Created on the first container that can hold a cycle. + visited: Map | undefined, ): boolean { // Handle strict equality (primitives, same reference) if (a === b || Object.is(a, b)) return true @@ -78,9 +79,10 @@ export function deepEqualsInternal( if (a.size !== b.size) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) const entries = Array.from(a.entries()) @@ -100,9 +102,10 @@ export function deepEqualsInternal( if (a.size !== b.size) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) // Convert to arrays for comparison @@ -205,9 +208,10 @@ export function deepEqualsInternal( if (!Array.isArray(b) || a.length !== b.length) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) const result = a.every((item, index) => @@ -222,10 +226,9 @@ export function deepEqualsInternal( // Handle objects if (typeof a === `object`) { // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } - visited.set(a, b) // Compare enumerable symbol keys as well as string keys. Query results may // use user-owned symbols, and a symbol-only update is still a value change. @@ -233,19 +236,29 @@ export function deepEqualsInternal( const keysB = enumerableOwnKeys(b) // Check if they have the same number of keys - if (keysA.length !== keysB.length) { - visited.delete(a) - return false - } + if (keysA.length !== keysB.length) return false // Check if all keys exist in both objects and their values are equal - const result = keysA.every( - (key) => - Object.prototype.propertyIsEnumerable.call(b, key) && - deepEqualsInternal(a[key], b[key], visited), - ) - - visited.delete(a) + // Register for cycles only before descending, so a flat object + // allocates no cycle map. + let registered = false + let result = true + for (const key of keysA) { + const value = a[key] + if (!registered && value !== null && typeof value === `object`) { + visited ??= new Map() + visited.set(a, b) + registered = true + } + if ( + !Object.prototype.propertyIsEnumerable.call(b, key) || + !deepEqualsInternal(value, b[key], visited) + ) { + result = false + break + } + } + if (registered) visited!.delete(a) return result } From ecc4046abae2fae3f986fbe0de8b5b544d5b45b3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:41:08 -0600 Subject: [PATCH 29/63] perf(db): cheaper partition group keys and in-group updates Group keys are length-prefixed strings instead of JSON arrays, an update that keeps every filtered field reuses its group key, and an in-group update is published as the source's message instead of a copy. Saves about 77 KiB per 200-row update batch. Co-authored-by: Isaac --- packages/db/src/query/pooled-live-query.ts | 37 +++++++++++++++++----- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 01b1d8c37..0d998aa29 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -58,6 +58,11 @@ function equalityKey(value: unknown): string | undefined { : undefined } +// Length-prefixed, so no two part lists share an encoding. +function appendGroupKeyPart(groupKey: string, part: string): string { + return `${groupKey}${part.length}:${part}` +} + function readPath(row: Row, path: Array): unknown { try { let value: unknown = row @@ -90,13 +95,20 @@ class Partition { groupKeyOf(row: Row | undefined): string | undefined { if (row === undefined) return undefined - const parts: Array = [] + let groupKey = `` for (const path of this.paths) { const key = equalityKey(readPath(row, path)) if (key === undefined) return undefined - parts.push(key) + groupKey = appendGroupKeyPart(groupKey, key) } - return JSON.stringify(parts) + return groupKey + } + + // Whether two versions of a row hold the same value in every field. + private sameFields(a: Row, b: Row): boolean { + return this.paths.every((path) => + Object.is(readPath(a, path), readPath(b, path)), + ) } group(key: string): PartitionGroup { @@ -221,9 +233,13 @@ class Partition { const previous = change.type === `insert` ? undefined - : this.groupKeyOf( - change.type === `delete` ? change.value : change.previousValue, - ) + : change.type === `update` && + change.previousValue !== undefined && + this.sameFields(change.value, change.previousValue) + ? next + : this.groupKeyOf( + change.type === `delete` ? change.value : change.previousValue, + ) if (previous !== undefined && previous !== next) { const group = this.group(previous) const old = group.rows.get(change.key) @@ -238,7 +254,9 @@ class Partition { record( group, existed - ? { ...change, type: `update` } + ? change.type === `update` + ? change + : { ...change, type: `update` } : { type: `insert`, key: change.key, value: change.value }, ) } @@ -327,7 +345,10 @@ function poolableShape( return { paths: conjuncts.map(({ path }) => path), shapeKey: conjuncts.map(({ pathKey }) => pathKey).join(`,`), - groupKey: JSON.stringify(conjuncts.map(({ literalKey }) => literalKey)), + groupKey: conjuncts.reduce( + (groupKey, { literalKey }) => appendGroupKeyPart(groupKey, literalKey), + ``, + ), } } From 2a153fe645372f4ebdacc7ec360164df1f962643 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:42:21 -0600 Subject: [PATCH 30/63] perf(db): skip the flat tracker's detach copy without assigned objects Saves about 58 KiB per 200-row update batch. Co-authored-by: Isaac --- packages/db/src/proxy.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 4b6f22dc2..199c31c49 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -1050,6 +1050,7 @@ export function withFlatChangeTracking( return drafts.map((draft, index) => { const original = targets[index] as Record const changes: Record = {} + let assignedObject = false for (const key in draft) { const value = (draft as Record)[key] const before = original[key] @@ -1065,6 +1066,7 @@ export function withFlatChangeTracking( ) ) { defineDataProperty(changes, key, value) + if (value !== null && typeof value === `object`) assignedObject = true } } for (const key in original) { @@ -1072,6 +1074,6 @@ export function withFlatChangeTracking( defineDataProperty(changes, key, undefined) } // A callback may assign objects; detach them as the proxy path does. - return deepClone(changes, undefined, true) + return assignedObject ? deepClone(changes, undefined, true) : changes }) } From 15b517818a28496a05650f4cdc09b90520997b91 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 13:45:51 -0600 Subject: [PATCH 31/63] perf(db): convert literals and refs to expressions with fewer traps toExpression returns a Value for a primitive before any proxy check, and an alias-qualified ref proxy answers its brand with its path, so one trap gives the PropRef. Query building is about 12% faster. Co-authored-by: Isaac --- .../src/query/builder/ref-proxy-identity.ts | 13 ++++++++---- packages/db/src/query/builder/ref-proxy.ts | 20 +++++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/packages/db/src/query/builder/ref-proxy-identity.ts b/packages/db/src/query/builder/ref-proxy-identity.ts index ba13c923b..bc963425c 100644 --- a/packages/db/src/query/builder/ref-proxy-identity.ts +++ b/packages/db/src/query/builder/ref-proxy-identity.ts @@ -6,15 +6,20 @@ import type { RefProxy } from './ref-proxy.js' */ export const REF_PROXY_BRAND: unique symbol = Symbol(`refProxy`) -function hasRefProxyBrand(value: object): boolean { +/** The brand a ref proxy answers, or undefined for any other value. */ +export function readRefProxyBrand(value: unknown): unknown { + if (!value || typeof value !== `object`) return undefined try { - return (value as Record)[REF_PROXY_BRAND] === true + const brand = (value as Record)[REF_PROXY_BRAND] + return brand === true || Array.isArray(brand) ? brand : undefined } catch { // A revoked proxy, such as a finished Immer draft, is not a ref proxy. - return false + return undefined } } export function isRefProxy(value: any): value is RefProxy { - return value && typeof value === `object` && hasRefProxyBrand(value) + return ( + value && typeof value === `object` && readRefProxyBrand(value) !== undefined + ) } diff --git a/packages/db/src/query/builder/ref-proxy.ts b/packages/db/src/query/builder/ref-proxy.ts index e6744b151..02bdeafd5 100644 --- a/packages/db/src/query/builder/ref-proxy.ts +++ b/packages/db/src/query/builder/ref-proxy.ts @@ -1,5 +1,5 @@ import { PropRef, Value, isBasicOrAggregateExpression } from '../ir.js' -import { REF_PROXY_BRAND, isRefProxy } from './ref-proxy-identity.js' +import { REF_PROXY_BRAND, readRefProxyBrand } from './ref-proxy-identity.js' import { getWrapperExpressionName } from './wrapper-identity.js' import type { BasicExpression } from '../ir.js' import type { IsPlainObject, RefLeaf } from './types.js' @@ -147,7 +147,8 @@ export function createRefProxy>( if (prop === `__path`) return path if (prop === `__sourceAlias`) return path[0] if (prop === `__type`) return undefined // Type is only for TypeScript inference - if (prop === REF_PROXY_BRAND) return true + // Answers with the path so toExpression reads it in one trap. + if (prop === REF_PROXY_BRAND) return path if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) children ??= new Map() @@ -370,8 +371,19 @@ export function createRefProxyWithSelected>( export function toExpression(value: T): BasicExpression export function toExpression(value: RefProxy): BasicExpression export function toExpression(value: any): BasicExpression { - if (isRefProxy(value)) { - return new PropRef(value.__path, value.__sourceAlias) + // A primitive cannot be a ref proxy, a wrapper, or an expression. + if ( + value === null || + (typeof value !== `object` && typeof value !== `function`) + ) { + return new Value(value) + } + const brand = readRefProxyBrand(value) + if (brand !== undefined) { + // An alias-qualified ref proxy answers the brand with its path. + return Array.isArray(brand) + ? new PropRef(brand, brand[0]) + : new PropRef(value.__path, value.__sourceAlias) } // toArray(), concat(toArray()), and materialize() must be used as direct // select fields, not inside expressions From 7ab82769a4fcef7310def441295350abfe1458bc Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 14:21:39 -0600 Subject: [PATCH 32/63] perf(db): build pooled shape keys in one pass Two conjuncts swap instead of sorting, and the paths, shape key, and group key come from one loop. A 240-query mount allocates about 11% less. Co-authored-by: Isaac --- packages/db/src/query/pooled-live-query.ts | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 0d998aa29..f5e8b2b17 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -339,17 +339,22 @@ function poolableShape( return undefined } } - if (conjuncts.length > 1) { + // Most shapes have one or two fields; a general sort costs more than both. + if (conjuncts.length === 2) { + if (conjuncts[1]!.pathKey < conjuncts[0]!.pathKey) conjuncts.reverse() + } else if (conjuncts.length > 2) { conjuncts.sort((a, b) => (a.pathKey < b.pathKey ? -1 : 1)) } - return { - paths: conjuncts.map(({ path }) => path), - shapeKey: conjuncts.map(({ pathKey }) => pathKey).join(`,`), - groupKey: conjuncts.reduce( - (groupKey, { literalKey }) => appendGroupKeyPart(groupKey, literalKey), - ``, - ), + const paths: Array> = [] + let shapeKey = `` + let groupKey = `` + for (const { path, pathKey, literalKey } of conjuncts) { + paths.push(path) + // Each JSON path delimits itself, so concatenation stays unambiguous. + shapeKey += pathKey + groupKey = appendGroupKeyPart(groupKey, literalKey) } + return { paths, shapeKey, groupKey } } /** From 969d03bae63b23422037a4cea496be921a66b76b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 14:22:23 -0600 Subject: [PATCH 33/63] perf(react-db): skip empty deferred-sync work on subscribe Co-authored-by: Isaac --- packages/react-db/src/useLiveQuery.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index 24a05d443..2d7e65479 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -917,6 +917,7 @@ function useLiveQueryImpl( (!inputIsCollection && (clientRef.current !== dbClient || identityChanged)) const resumeDeferredCollections = () => { + if (deferredCollectionsRef.current.size === 0) return for (const collection of deferredCollectionsRef.current) { collection._resumeSyncStart() } @@ -1031,10 +1032,9 @@ function useLiveQueryImpl( // Stable subscribe bound to the current observer; the observer owns the // subscription, ready-race, and disposal. - if (!subscribeRef.current || needsNewCollection) { subscribeRef.current = (onStoreChange: () => void) => { - const unsubscribe = observer.subscribe(() => onStoreChange()) + const unsubscribe = observer.subscribe(onStoreChange) resumeDeferredCollections() return unsubscribe } From cb08c96b97cfa2a9f7dec6fa6b27ff866145a1a5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 14:38:44 -0600 Subject: [PATCH 34/63] perf(db): key the virtual props cache by row key Adding a WeakMap entry for each published row cost more than the copy it saved. The cache now holds each key's latest row; a change enriches its previous value first so the published value stays the row later reads return, and deletes drop the key. Updates are about 8-14% faster. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 1 + packages/db/src/collection/state.ts | 28 ++++-- packages/db/tests/virtual-props-cache.test.ts | 88 +++++++++++++++++++ 3 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 packages/db/tests/virtual-props-cache.test.ts diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index f8a9fbd81..123341222 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,6 +267,7 @@ comment and the current API/architecture contract before extending its model. | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | | Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React, and `eq-filter-peers` checks through `subscriberCount` that React shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | | Minified DB public API | [consumer bundle check](https://github.com/TanStack/db/blob/main/scripts/test-minified-db.mjs) | CI builds `@tanstack/db`, then bundles its built `dist` entry with db-ivm inlined through esbuild `minify: true`. The build renames TypeScript-private members from `packages/db/mangle-cache.json`, so this lane runs against the renamed output that consumers install. It rejects a `dist` that is older than `src`, the cache, or the package's build configuration. `pnpm --filter @tanstack/db test:dist` also runs five db test files against the built `dist`, chosen for coverage of the modules with the most renamed members. `pnpm check:mangle` fails when a cached name is used other than as a private member in db src, is read by another package, appears as a string, or has a short name that collides with a source identifier ([private-member mangling review](oracle-reviews/code-weight-private-member-mangling.md)). It checks every exported error class's current public `name`, the built-in `BasicIndex` resolver name in index metadata and its event, complete query rows (after removing the four documented virtual fields) against an independent array filter/sort/projection, and one live update. The `new.target.name`, public-member mangle, and extra-output-field calibration modes fail at the intended observations. This fixed slice does not replace the unminified generated oracles, exercise framework adapters, or establish stable names for custom index resolvers. | diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 0679bac16..a1f69ed12 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -167,9 +167,13 @@ export class CollectionStateManager< // failed mutations must not add to, or erase a sibling's entry in, this set. public pendingLocalOrigins = new Set() - private virtualPropsCache = new WeakMap< - object, + // Keyed by row key, not row object: adding a WeakMap entry for each + // published row cost more than the copy it saves. Sync writes, deletes, + // and cleanup drop a key's entry. + private virtualPropsCache = new Map< + TKey, { + row: TOutput synced: boolean origin: VirtualOrigin key: TKey @@ -334,9 +338,10 @@ export class CollectionStateManager< const resolvedKey = existingRow.$key ?? virtualProps.$key const collectionId = existingRow.$collectionId ?? virtualProps.$collectionId - const cached = this.virtualPropsCache.get(row as object) + const cached = this.virtualPropsCache.get(resolvedKey) if ( cached && + cached.row === row && cached.synced === synced && cached.origin === origin && cached.key === resolvedKey && @@ -353,7 +358,8 @@ export class CollectionStateManager< $collectionId: collectionId, } as WithVirtualProps - this.virtualPropsCache.set(row as object, { + this.virtualPropsCache.set(resolvedKey, { + row, synced, origin, key: resolvedKey, @@ -365,6 +371,7 @@ export class CollectionStateManager< } private clearOriginTrackingState(): void { + this.virtualPropsCache.clear() this.rowOrigins.clear() this.pendingLocalChanges.clear() this.pendingLocalOrigins.clear() @@ -394,9 +401,8 @@ export class CollectionStateManager< change: ChangeMessage, ): ChangeMessage, TKey> { const { __virtualProps } = change as InternalChangeMessage - const enrichedValue = __virtualProps?.value - ? this.enrichWithVirtualPropsSnapshot(change.value, __virtualProps.value) - : this.enrichWithVirtualProps(change.value, change.key) + // The cache holds one row per key, so the previous row goes first and + // the published value stays the row that later reads return. const enrichedPreviousValue = change.previousValue ? __virtualProps?.previousValue ? this.enrichWithVirtualPropsSnapshot( @@ -405,6 +411,11 @@ export class CollectionStateManager< ) : this.enrichWithVirtualProps(change.previousValue, change.key) : undefined + const enrichedValue = __virtualProps?.value + ? this.enrichWithVirtualPropsSnapshot(change.value, __virtualProps.value) + : this.enrichWithVirtualProps(change.value, change.key) + // A deleted key, such as a rolled-back insert, has no row to read again. + if (change.type === `delete`) this.virtualPropsCache.delete(change.key) return { key: change.key, @@ -1584,8 +1595,7 @@ export class CollectionStateManager< // A sync source may reuse a live-reading row object, making an // enriched snapshot cached for an earlier publication stale. - if (operation.type !== `delete`) - this.virtualPropsCache.delete(operation.value) + this.virtualPropsCache.delete(key) // Update synced data switch (operation.type) { diff --git a/packages/db/tests/virtual-props-cache.test.ts b/packages/db/tests/virtual-props-cache.test.ts new file mode 100644 index 000000000..15734132a --- /dev/null +++ b/packages/db/tests/virtual-props-cache.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { localOnlyCollectionOptions } from '../src/local-only.js' +import { mockSyncCollectionOptions } from './utils.js' + +/** + * A row read with virtual props is a copy, cached so repeated reads and + * publications share one object. These laws pin what that cache must keep: + * the value a change publishes is the row later reads return, for every + * subscriber, and a key that leaves the collection leaves the cache. + */ +type Row = { id: string; a: number } +const cacheSize = (collection: unknown) => + (collection as { _state: { virtualPropsCache: Map } }) + ._state.virtualPropsCache.size + +describe(`virtual props cache`, () => { + it(`publishes the row that later reads return, to every subscriber`, async () => { + const collection = createCollection( + mockSyncCollectionOptions({ + id: `virtual-props-cache-sync`, + getKey: (row) => row.id, + initialData: [{ id: `x`, a: 0 }], + }), + ) + await collection.stateWhenReady() + const first: Array = [] + const second: Array = [] + collection.subscribeChanges((changes) => + first.push(...changes.map((c) => c.value)), + ) + collection.subscribeChanges((changes) => + second.push(...changes.map((c) => c.value)), + ) + for (const a of [1, 2]) { + collection.utils.begin() + collection.utils.write({ type: `update`, value: { id: `x`, a } }) + collection.utils.commit() + expect(first.at(-1)).toBe(collection.get(`x`)) + expect(second.at(-1)).toBe(first.at(-1)) + expect(collection.toArray[0]).toBe(first.at(-1)) + } + }) + + it(`publishes the row that later reads return after local writes`, () => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `virtual-props-cache-local`, + getKey: (row) => row.id, + initialData: [{ id: `x`, a: 0 }], + }), + ) + const published: Array = [] + collection.subscribeChanges((changes) => + published.push(...changes.map((c) => c.value)), + ) + for (const a of [1, 2]) { + collection.update(`x`, (draft) => { + draft.a = a + }) + expect(published.at(-1)).toBe(collection.get(`x`)) + } + }) + + it(`drops a key's entry when the key is deleted or rolled back`, async () => { + const collection = createCollection( + mockSyncCollectionOptions({ + id: `virtual-props-cache-delete`, + getKey: (row) => row.id, + initialData: [{ id: `x`, a: 0 }], + }), + ) + await collection.stateWhenReady() + collection.subscribeChanges(() => {}) + collection.get(`x`) + collection.utils.begin() + collection.utils.write({ type: `delete`, value: { id: `x`, a: 0 } }) + collection.utils.commit() + expect(cacheSize(collection)).toBe(0) + + const transaction = collection.insert({ id: `y`, a: 1 }) + collection.get(`y`) + collection.utils.rejectSync(new Error(`rolled back`)) + await transaction.isPersisted.promise.catch(() => undefined) + expect(collection.has(`y`)).toBe(false) + expect(cacheSize(collection)).toBe(0) + }) +}) From 53cd676828a3011c8e94edb156b01141cfc4e438 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 14:58:35 -0600 Subject: [PATCH 35/63] test(db): settle local-only direct writes through when('settled') Co-authored-by: Isaac --- packages/db/tests/local-only-direct-write.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/db/tests/local-only-direct-write.test.ts b/packages/db/tests/local-only-direct-write.test.ts index 2f2847402..eb2664a2d 100644 --- a/packages/db/tests/local-only-direct-write.test.ts +++ b/packages/db/tests/local-only-direct-write.test.ts @@ -64,6 +64,7 @@ describe(`local-only direct writes`, () => { for (const transaction of [inserted, updated, deleted]) { expect(transaction.state).toBe(`completed`) await expect(transaction.isPersisted.promise).resolves.toBe(transaction) + await expect(transaction.when(`settled`)).resolves.toBe(transaction) } expect(batches).toEqual([ [`insert:1`], From 25523d0892af37499113e0c739ca6fd3105d1b57 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 15:13:01 -0600 Subject: [PATCH 36/63] refactor(react-db): keep useLiveQuery's state in one plain object The hook's twelve ref wrappers become fields of one instance object in a single ref slot, and the development check returns early. Co-authored-by: Isaac --- packages/react-db/src/development.ts | 8 +- packages/react-db/src/useLiveQuery.ts | 138 ++++++++++++-------------- 2 files changed, 63 insertions(+), 83 deletions(-) diff --git a/packages/react-db/src/development.ts b/packages/react-db/src/development.ts index 05269bb80..ba47d3802 100644 --- a/packages/react-db/src/development.ts +++ b/packages/react-db/src/development.ts @@ -1,14 +1,10 @@ // Bundlers inline `process.env.NODE_ENV` even where `process` does not exist, // so it is read directly; without either, warnings stay off. export function shouldWarnInDevelopment(disableEnvVar: string): boolean { - let development: boolean try { - development = process.env.NODE_ENV !== `production` + if (process.env.NODE_ENV === `production`) return false } catch { return false } - return ( - development && - (typeof process === `undefined` || process.env[disableEnvVar] !== `1`) - ) + return typeof process === `undefined` || process.env[disableEnvVar] !== `1` } diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index 2d7e65479..edb25f056 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -728,30 +728,29 @@ export function useLiveQueryForSuspense( return useLiveQueryImpl(configOrQueryOrCollection, deps, true) } -const ref = (current: T): { current: T } => ({ current }) - -// Every ref the hook keeps, created once per hook instance. -function createHookRefs(dbClient: DbClient | undefined) { +// What one hook instance keeps across renders. It lives in a single ref +// slot, so a render neither looks up nor allocates more. +function createHookInstance(dbClient: DbClient | undefined) { return { - collectionRef: ref | null>(null), - depsRef: ref | null>(null), - configRef: ref(null), - clientRef: ref(dbClient), - legacyUnhashableIdentityRef: ref>([`legacy-unhashable`]), - derivedIdentityProfilerRef: ref({ + collection: null as Collection | null, + deps: null as Array | null, + config: null as unknown, + client: dbClient, + legacyUnhashableIdentity: [`legacy-unhashable`] as Array, + derivedIdentityProfiler: { renderCount: 0, totalMs: 0, maxMs: 0, warned: false, - }), - deferredCollectionsRef: ref( - new Set>(), - ), - observerRef: ref | null>(null), - queryHashRef: ref(undefined), - suspenseKeyRef: ref(undefined), - identityErrorRef: ref(undefined), - subscribeRef: ref<((onStoreChange: () => void) => () => void) | null>(null), + } as DerivedIdentityProfiler, + deferredCollections: new Set< + CollectionImpl + >(), + observer: null as LiveQueryObserver | null, + queryHash: undefined as string | undefined, + suspenseKey: undefined as string | undefined, + identityError: undefined as UnhashableQueryIRError | undefined, + subscribe: null as ((onStoreChange: () => void) => () => void) | null, } } @@ -768,23 +767,8 @@ function useLiveQueryImpl( : (getExplicitDbClient(configOrQueryOrCollection) ?? contextDbClient) const resolvedDeps = deps ?? [] - // One hook slot holds every ref, so a render neither looks up nor - // allocates the others. - const refsRef = useRef | null>(null) - const { - collectionRef, - depsRef, - configRef, - clientRef, - legacyUnhashableIdentityRef, - derivedIdentityProfilerRef, - deferredCollectionsRef, - observerRef, - queryHashRef, - suspenseKeyRef, - identityErrorRef, - subscribeRef, - } = (refsRef.current ??= createHookRefs(dbClient)) + const instanceRef = useRef | null>(null) + const instance = (instanceRef.current ??= createHookInstance(dbClient)) const queryKey = !inputIsCollection ? getExplicitQueryKey(configOrQueryOrCollection) @@ -811,7 +795,7 @@ function useLiveQueryImpl( preparedQueryValue = prepareQueryValue( configOrQueryOrCollection, dbClient, - deferredCollectionsRef.current, + instance.deferredCollections, ) streamIdentity = [ `deps`, @@ -830,8 +814,8 @@ function useLiveQueryImpl( const preparation = prepareDerivedQuery( configOrQueryOrCollection, dbClient, - derivedIdentityProfilerRef.current, - deferredCollectionsRef.current, + instance.derivedIdentityProfiler, + instance.deferredCollections, ) preparedQueryValue = preparation.value if (preparation.status === `hashable`) { @@ -839,7 +823,7 @@ function useLiveQueryImpl( streamIdentity = preparation.identityDeps } else { warnUnhashableDerivedIdentity(preparation.error) - identityDeps = legacyUnhashableIdentityRef.current + identityDeps = instance.legacyUnhashableIdentity identityError = preparation.error } } @@ -867,10 +851,10 @@ function useLiveQueryImpl( !inputIsCollection && queryHash !== undefined && !dbClient && - collectionRef.current !== null && - clientRef.current === dbClient && - queryHashRef.current === queryHash && - suspenseKeyRef.current !== undefined + instance.collection !== null && + instance.client === dbClient && + instance.queryHash === queryHash && + instance.suspenseKey !== undefined if ( forSuspense && @@ -883,14 +867,14 @@ function useLiveQueryImpl( preparedQueryValue = prepareQueryValue( configOrQueryOrCollection, dbClient, - deferredCollectionsRef.current, + instance.deferredCollections, ) } const suspenseKey = queryHash && !dbClient ? canReuseSuspenseKey - ? suspenseKeyRef.current + ? instance.suspenseKey : getUnscopedSuspenseKey(preparedQueryValue, queryHash) : queryHash @@ -904,24 +888,24 @@ function useLiveQueryImpl( const suspenseCollection = suspenseEntry?.collection const identityChanged = - depsRef.current === null || + instance.deps === null || (deps !== undefined - ? depsRef.current.length !== identityDeps.length || - depsRef.current.some((dep, index) => dep !== identityDeps[index]) - : !deepEquals(depsRef.current, identityDeps)) + ? instance.deps.length !== identityDeps.length || + instance.deps.some((dep, index) => dep !== identityDeps[index]) + : !deepEquals(instance.deps, identityDeps)) // Check if we need to create/recreate the collection const needsNewCollection = - !collectionRef.current || - (inputIsCollection && configRef.current !== configOrQueryOrCollection) || - (!inputIsCollection && (clientRef.current !== dbClient || identityChanged)) + !instance.collection || + (inputIsCollection && instance.config !== configOrQueryOrCollection) || + (!inputIsCollection && (instance.client !== dbClient || identityChanged)) const resumeDeferredCollections = () => { - if (deferredCollectionsRef.current.size === 0) return - for (const collection of deferredCollectionsRef.current) { + if (instance.deferredCollections.size === 0) return + for (const collection of instance.deferredCollections) { collection._resumeSyncStart() } - deferredCollectionsRef.current.clear() + instance.deferredCollections.clear() } if (needsNewCollection) { @@ -946,28 +930,28 @@ function useLiveQueryImpl( } // It's already a collection, ensure sync is started for React hooks configOrQueryOrCollection.startSyncImmediate() - collectionRef.current = configOrQueryOrCollection - configRef.current = configOrQueryOrCollection + instance.collection = configOrQueryOrCollection + instance.config = configOrQueryOrCollection } else { if (suspenseCollection) { - collectionRef.current = suspenseCollection + instance.collection = suspenseCollection } else { if (preparedQueryValue === unpreparedQueryValue) { preparedQueryValue = prepareQueryValue( configOrQueryOrCollection, dbClient, - deferredCollectionsRef.current, + instance.deferredCollections, ) } - collectionRef.current = resolveLiveQueryValue(preparedQueryValue, { + instance.collection = resolveLiveQueryValue(preparedQueryValue, { gcTime: forSuspense ? DEFAULT_SUSPENSE_GC_TIME_MS : DEFAULT_GC_TIME_MS, // Hydration and Suspense key the live-query Collection by identity. pool: !forSuspense && !dbClient, }) as SuspenseCollection | null - if (suspenseCollections && suspenseKey && collectionRef.current) { - const collection = collectionRef.current + if (suspenseCollections && suspenseKey && instance.collection) { + const collection = instance.collection const removeCleanupListener = collection.on(`status:cleaned-up`, () => releaseSuspenseCollection( suspenseCollections, @@ -999,13 +983,13 @@ function useLiveQueryImpl( suspenseCollections.set(suspenseKey, entry) } } - configRef.current = configOrQueryOrCollection - depsRef.current = [...identityDeps] + instance.config = configOrQueryOrCollection + instance.deps = [...identityDeps] } - clientRef.current = dbClient - queryHashRef.current = queryHash - suspenseKeyRef.current = suspenseKey - identityErrorRef.current = identityError + instance.client = dbClient + instance.queryHash = queryHash + instance.suspenseKey = suspenseKey + instance.identityError = identityError } // Recreate the observer when the underlying collection changes. The observer @@ -1021,19 +1005,19 @@ function useLiveQueryImpl( // hook's pre-observer loading policy, and — because wholesale delivers // nothing synchronously during subscribe — never notifies // useSyncExternalStore inside its own subscribe call. - observerRef.current = createLiveQueryObserver(collectionRef.current, { + instance.observer = createLiveQueryObserver(instance.collection, { mode: `wholesale`, client: dbClient, - queryHash: queryHashRef.current, + queryHash: instance.queryHash, onPreload: resumeDeferredCollections, }) } - const observer = observerRef.current! + const observer = instance.observer! // Stable subscribe bound to the current observer; the observer owns the // subscription, ready-race, and disposal. - if (!subscribeRef.current || needsNewCollection) { - subscribeRef.current = (onStoreChange: () => void) => { + if (!instance.subscribe || needsNewCollection) { + instance.subscribe = (onStoreChange: () => void) => { const unsubscribe = observer.subscribe(onStoreChange) resumeDeferredCollections() return unsubscribe @@ -1041,7 +1025,7 @@ function useLiveQueryImpl( } const returned = useSyncExternalStore( - subscribeRef.current, + instance.subscribe, () => observer.getSnapshot(), () => observer.getServerSnapshot(), ) @@ -1049,8 +1033,8 @@ function useLiveQueryImpl( if (forSuspense) { setLiveQueryResultInfo(returned, { client: dbClient, - queryHash: queryHashRef.current, - identityError: identityErrorRef.current, + queryHash: instance.queryHash, + identityError: instance.identityError, observer, }) } From 052532586a95ed5a15494056996aaf874251a5f4 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 15:14:15 -0600 Subject: [PATCH 37/63] refactor(db): drop the pooled group's entries cache and listen helper The generic observer already caches entries per revision, so the group no longer keeps its own copy; terminate and release share one teardown. Co-authored-by: Isaac --- packages/db/src/query/pooled-live-query.ts | 35 +++++++++------------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index f5e8b2b17..481ef4095 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -40,8 +40,6 @@ interface PartitionGroup { listeners: Set revision: number layoutRevision: number - // Rows as entries, rebuilt after a change. - entries: Array<[string | number, Row]> | undefined } // Matches the Collection lifecycle's floor for a never-subscribed Collection. @@ -119,14 +117,12 @@ class Partition { listeners: new Set(), revision: 0, layoutRevision: 0, - entries: undefined, } this.groups.set(key, group) } return group } - /** Start the shared source subscription; release it when unused. */ /** * Keep the shared source subscription open. Each view brings its query's * `gcTime`; the partition keeps the longest, so it never releases before @@ -159,11 +155,6 @@ class Partition { this.scheduleRelease() } - listen(group: PartitionGroup, listener: Listener): () => void { - this.addListener(group, listener) - return () => this.removeListener(group, listener) - } - addListener(group: PartitionGroup, listener: Listener): void { this.retain() this.hadListener = true @@ -179,14 +170,18 @@ class Partition { private terminate(): void { this.terminated = true - if (this.releaseTimer !== undefined) clearTimeout(this.releaseTimer) + clearTimeout(this.releaseTimer) + this.release() + } + + private release(): void { this.subscription?.unsubscribe() this.subscription = undefined this.onEmpty() } private scheduleRelease(): void { - if (this.releaseTimer !== undefined) clearTimeout(this.releaseTimer) + clearTimeout(this.releaseTimer) this.releaseTimer = undefined if (this.listenerCount > 0 || !Number.isFinite(this.gcTime)) return // Like a Collection that synced before anything subscribed, a view built @@ -197,8 +192,6 @@ class Partition { this.releaseTimer = setTimeout(() => { this.releaseTimer = undefined if (this.listenerCount > 0) return - this.subscription?.unsubscribe() - this.subscription = undefined this.stopStatusEvents?.() this.stopStatusEvents = undefined // Views outlive a release and may subscribe again, so they keep their @@ -207,9 +200,8 @@ class Partition { group.rows.clear() group.revision++ group.layoutRevision++ - group.entries = undefined } - this.onEmpty() + this.release() }, delay) } @@ -263,7 +255,6 @@ class Partition { } for (const [group, groupChanges] of touched) { group.revision++ - group.entries = undefined for (const listener of [...group.listeners]) listener(groupChanges) } } @@ -398,15 +389,15 @@ class PooledLiveQuery { return this.group.rows } - entries(): Array<[string | number, Row]> { - return (this.group.entries ??= [...this.group.rows.entries()]) + entries(): IterableIterator<[string | number, Row]> { + return this.group.rows.entries() } subscribeChanges( callback: Listener, options: { includeInitialState?: boolean } = {}, ): { unsubscribe: () => void } { - const unsubscribe = this.partition.listen(this.group, callback) + this.partition.addListener(this.group, callback) if (options.includeInitialState) { callback( [...this.group.rows].map(([key, value]) => ({ @@ -416,7 +407,9 @@ class PooledLiveQuery { })), ) } - return { unsubscribe } + return { + unsubscribe: () => this.partition.removeListener(this.group, callback), + } } on(...args: Parameters): () => void { @@ -591,7 +584,7 @@ class PooledWholesaleObserver implements LiveQueryObserver< dehydrate(): DehydratedLiveQueryResult { return { - rows: this.view.entries().map(([key, value]) => ({ key, value })), + rows: Array.from(this.view.entries(), ([key, value]) => ({ key, value })), } } From 2ab1afcbe12c560b738cd201b061fc4a3a0bd23b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 15:18:29 -0600 Subject: [PATCH 38/63] refactor(db): trim the draft delete path, cache entry, and direct-write setup The added-field delete sets modified from the assigned size, the virtual props cache entry drops a key its map already holds, and local-only collections always register direct writes, since an empty type set already falls through. Co-authored-by: Isaac --- packages/db/src/collection/state.ts | 3 --- packages/db/src/local-only.ts | 9 +++------ packages/db/src/proxy.ts | 11 +---------- 3 files changed, 4 insertions(+), 19 deletions(-) diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 55a246abd..b53f1cb75 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -176,7 +176,6 @@ export class CollectionStateManager< row: TOutput synced: boolean origin: VirtualOrigin - key: TKey collectionId: string enriched: WithVirtualProps } @@ -346,7 +345,6 @@ export class CollectionStateManager< cached.row === row && cached.synced === synced && cached.origin === origin && - cached.key === resolvedKey && cached.collectionId === collectionId ) { return cached.enriched @@ -365,7 +363,6 @@ export class CollectionStateManager< row, synced, origin, - key: resolvedKey, collectionId, enriched, }) diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 923586db0..5aece5cb7 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -335,12 +335,9 @@ function createLocalOnlySync( syncCommit = commit collection = params.collection params.collection._state.isLocalOnly = true - if (directTypes.size > 0) { - params.collection._state.localOnlyDirectWrite = { - types: directTypes, - write: (mutations) => - confirmOperationsSync(mutations), - } + params.collection._state.localOnlyDirectWrite = { + types: directTypes, + write: confirmOperationsSync, } // Apply initial data if provided diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 199c31c49..dbd97c2cd 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -840,7 +840,6 @@ export function createChangeProxy< const stringProp = typeof prop === `symbol` ? prop.toString() : prop if (Object.hasOwn(dobj, prop)) { - // Check if the property exists in the original object // A hidden original field is not row data, so as with a plain // delete of it, removing it is not a change. const hadPropertyInOriginal = @@ -858,15 +857,7 @@ export function createChangeProxy< // should revert to the original state if (!hadPropertyInOriginal) { changeTracker.assigned_.delete(stringProp) - - // If this is the last change and we're not a nested object, - // mark the object as unmodified - if (changeTracker.assigned_.size === 0) { - changeTracker.modified = false - } else { - // We still have changes, keep as modified - changeTracker.modified = true - } + changeTracker.modified = changeTracker.assigned_.size > 0 } else { // Mark this property as deleted changeTracker.assigned_.set(stringProp, false) From 54de6d7fb87abac38329fdca45bdb1f4e833d168 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 16:14:50 -0600 Subject: [PATCH 39/63] feat: pool eq-filtered live queries in Vue, Solid, Svelte, and Angular Each adapter resolves query builders through the shared resolver and exposes a pooled query's public Collection as `collection`. Svelte pools only without a DbClient, as React does. The eq-filter conformance scenario now checks that every adapter shares one source subscription. Co-authored-by: Isaac --- packages/angular-db/src/index.ts | 20 ++++----- packages/angular-db/tests/conformance.test.ts | 2 +- packages/db/src/live-query-adapter.ts | 13 ++++++ packages/db/src/live-query-observer.ts | 7 +-- packages/solid-db/src/useLiveQuery.ts | 9 ++-- packages/solid-db/tests/conformance.test.tsx | 2 +- packages/svelte-db/src/useLiveQuery.svelte.ts | 10 ++--- .../tests/conformance.svelte.test.ts | 2 +- packages/vue-db/src/useLiveQuery.ts | 43 ++++++------------- packages/vue-db/tests/conformance.test.ts | 2 +- 10 files changed, 48 insertions(+), 62 deletions(-) diff --git a/packages/angular-db/src/index.ts b/packages/angular-db/src/index.ts index 8264b4647..1682546ae 100644 --- a/packages/angular-db/src/index.ts +++ b/packages/angular-db/src/index.ts @@ -10,8 +10,10 @@ import { BaseQueryBuilder, createLiveQueryCollection, createLiveQueryObserver, + getPublicCollection, isCollection, isSingleResultCollection, + resolveLiveQueryValue, } from '@tanstack/db' import type { Collection, @@ -180,11 +182,7 @@ export function injectLiveQuery(opts: any) { return null } - return createLiveQueryCollection({ - query: opts, - startSync: true, - gcTime: 0, - }) + return resolveLiveQueryValue(result, { gcTime: 0 }) } // Check if it's reactive query options @@ -207,11 +205,7 @@ export function injectLiveQuery(opts: any) { return null } - return createLiveQueryCollection({ - query: () => result, - startSync: true, - gcTime: 0, - }) + return resolveLiveQueryValue(result, { gcTime: 0 }) } // Handle LiveQueryCollectionConfig objects. Default startSync/gcTime to @@ -250,7 +244,7 @@ export function injectLiveQuery(opts: any) { observer: LiveQueryObserver, ) => { const newState = new Map(currentCollection.entries()) - const newData = Array.from(currentCollection.values()) + const newData = Array.from(newState.values()) state.set(newState) internalData.set(newData) @@ -312,7 +306,9 @@ export function injectLiveQuery(opts: any) { data, // Loosely typed so the impl return stays compatible with every overload // (the shared `isCollection` guard narrows the computed to `Collection | null`). - collection: collection as Signal, + collection: computed(() => + getPublicCollection(collection()), + ) as Signal, status, isLoading: computed(() => status() === `loading`), isReady: computed(() => status() === `ready` || status() === `disabled`), diff --git a/packages/angular-db/tests/conformance.test.ts b/packages/angular-db/tests/conformance.test.ts index 546e49f09..7501f3d14 100644 --- a/packages/angular-db/tests/conformance.test.ts +++ b/packages/angular-db/tests/conformance.test.ts @@ -235,7 +235,7 @@ const angularDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: false, suspense: false }, + features: { serverSnapshot: false, suspense: false, pooledEqFilters: true }, } describe(`owned native scope setup`, () => { diff --git a/packages/db/src/live-query-adapter.ts b/packages/db/src/live-query-adapter.ts index b13001d0a..6979829a2 100644 --- a/packages/db/src/live-query-adapter.ts +++ b/packages/db/src/live-query-adapter.ts @@ -31,6 +31,19 @@ export function isCollection( ) } +/** + * The Collection an adapter hands users as `collection`. A pooled live query + * stands in for its live-query Collection and builds it only when touched. + */ +export function getPublicCollection< + T extends Collection | null | undefined, +>(collection: T): T { + return ( + (collection as { publicCollection?: T } | null | undefined) + ?.publicCollection ?? collection + ) +} + /** Whether a collection yields a single result (`findOne`) rather than an array. */ export function isSingleResultCollection( collection: Collection, diff --git a/packages/db/src/live-query-observer.ts b/packages/db/src/live-query-observer.ts index d513a877c..11f42a708 100644 --- a/packages/db/src/live-query-observer.ts +++ b/packages/db/src/live-query-observer.ts @@ -1,6 +1,7 @@ import { LiveQueryObserverDisposedError } from './errors.js' import { getLiveQueryStatusFlags, + getPublicCollection, isSingleResultCollection, } from './live-query-adapter.js' import { getBuilderFromConfig } from './query/live/collection-registry.js' @@ -308,11 +309,7 @@ class LiveQueryObserverImpl< this.cachedSnapshot = { state, data: singleResult ? data[0] : data, - // A pooled view observes its partition group directly and hands users a - // Collection that is built only when touched. - collection: - (collection as { publicCollection?: Collection }) - .publicCollection ?? collection, + collection: getPublicCollection(collection), layoutRevision: this.layoutRevision, status, ...getLiveQueryStatusFlags(status), diff --git a/packages/solid-db/src/useLiveQuery.ts b/packages/solid-db/src/useLiveQuery.ts index 3644e4a37..21963fa94 100644 --- a/packages/solid-db/src/useLiveQuery.ts +++ b/packages/solid-db/src/useLiveQuery.ts @@ -11,8 +11,10 @@ import { BaseQueryBuilder, createLiveQueryCollection, createLiveQueryObserver, + getPublicCollection, isCollection, isSingleResultCollection, + resolveLiveQueryValue, } from '@tanstack/db' import { createStore, reconcile } from 'solid-js/store' import type { Accessor } from 'solid-js' @@ -321,10 +323,7 @@ export function useLiveQuery( return null } - return createLiveQueryCollection({ - query: configOrQueryOrCollection, - startSync: true, - }) + return resolveLiveQueryValue(result) } const innerCollection = configOrQueryOrCollection() @@ -558,7 +557,7 @@ export function useLiveQuery( }, collection: { get() { - return collection() + return getPublicCollection(collection()) }, }, state: { diff --git a/packages/solid-db/tests/conformance.test.tsx b/packages/solid-db/tests/conformance.test.tsx index 956f0435c..013cd3a3c 100644 --- a/packages/solid-db/tests/conformance.test.tsx +++ b/packages/solid-db/tests/conformance.test.tsx @@ -236,7 +236,7 @@ const solidDriver: LiveQueryDriver = { // gap — the error-status scenario is parametrized to assert it via the boundary. errorSurface: `throw`, knownGaps: [], - features: { serverSnapshot: false, suspense: true }, + features: { serverSnapshot: false, suspense: true, pooledEqFilters: true }, } describe(`owned native scope setup`, () => { diff --git a/packages/svelte-db/src/useLiveQuery.svelte.ts b/packages/svelte-db/src/useLiveQuery.svelte.ts index f58bbf95d..0feab95a9 100644 --- a/packages/svelte-db/src/useLiveQuery.svelte.ts +++ b/packages/svelte-db/src/useLiveQuery.svelte.ts @@ -8,10 +8,12 @@ import { createLiveQueryCollection, createLiveQueryObserver, getLiveQueryHash, + getPublicCollection, getStableValueHash, isCollection, isSingleResultCollection, prepareLiveQueryValue, + resolveLiveQueryValue, } from '@tanstack/db' import { useOptionalDbClient } from './db-context.js' import type { @@ -415,10 +417,8 @@ export function useLiveQuery( } else if (isCollection(preparedValue)) { collection = preparedValue } else if (preparedValue instanceof BaseQueryBuilder) { - collection = createLiveQueryCollection({ - query: preparedValue, - startSync: true, - }) + // Hydration keys the live-query Collection by identity. + collection = resolveLiveQueryValue(preparedValue, { pool: !dbClient }) } else { collection = createLiveQueryCollection({ ...(preparedValue as LiveQueryCollectionConfig), @@ -535,7 +535,7 @@ export function useLiveQuery( return internalData }, get collection() { - return resolved.collection + return getPublicCollection(resolved.collection) }, get status() { return status as CollectionStatus diff --git a/packages/svelte-db/tests/conformance.svelte.test.ts b/packages/svelte-db/tests/conformance.svelte.test.ts index 5cdbde0bf..f8674ec35 100644 --- a/packages/svelte-db/tests/conformance.svelte.test.ts +++ b/packages/svelte-db/tests/conformance.svelte.test.ts @@ -222,7 +222,7 @@ const svelteDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: false, suspense: false }, + features: { serverSnapshot: false, suspense: false, pooledEqFilters: true }, } runSuite(svelteDriver) diff --git a/packages/vue-db/src/useLiveQuery.ts b/packages/vue-db/src/useLiveQuery.ts index 8a1f5e221..6eeea473f 100644 --- a/packages/vue-db/src/useLiveQuery.ts +++ b/packages/vue-db/src/useLiveQuery.ts @@ -9,10 +9,13 @@ import { watchEffect, } from 'vue' import { + BaseQueryBuilder, createLiveQueryCollection, createLiveQueryObserver, + getPublicCollection, isCollection, isSingleResultCollection, + resolveLiveQueryValue, } from '@tanstack/db' import type { ChangeMessage, @@ -329,29 +332,10 @@ export function useLiveQuery( // Ensure we always start sync for Vue hooks if (typeof unwrappedParam === `function`) { - // To avoid calling the query function twice, we wrap it to handle null/undefined returns - // The wrapper will be called once by createLiveQueryCollection - const disabledQuery = Symbol() - const wrappedQuery = (q: InitialQueryBuilder) => { - const result = unwrappedParam(q) - if (result === undefined || result === null) { - throw disabledQuery - } - return result - } - - try { - return createLiveQueryCollection({ - query: wrappedQuery, - startSync: true, - }) - } catch (error) { - if (error === disabledQuery) { - return null - } - // Re-throw other errors - throw error - } + // A query function returning null or undefined disables the query. + return resolveLiveQueryValue( + unwrappedParam(new BaseQueryBuilder() as InitialQueryBuilder), + ) } else { return createLiveQueryCollection({ ...unwrappedParam, @@ -389,15 +373,12 @@ export function useLiveQuery( // materializes into its own reactive map (granular) + ordered array. let currentObserver: LiveQueryObserver | null = null - const syncFromObserver = ( - observer: LiveQueryObserver, - currentCollection: Collection, - ) => { + const syncFromObserver = (observer: LiveQueryObserver) => { const snapshot = observer.getSnapshot() status.value = snapshot.status as CollectionStatus persistedStatus.value = snapshot.persistedStatus persistedError.value = snapshot.persistedError - internalData.value = Array.from(currentCollection.values()) + internalData.value = Array.from(snapshot.state?.values() ?? []) } // Watch for collection changes and subscribe to updates @@ -447,10 +428,10 @@ export function useLiveQuery( state.set(key, value) } } - syncFromObserver(observer, currentCollection) + syncFromObserver(observer) }, ) - syncFromObserver(observer, currentCollection) + syncFromObserver(observer) // Cleanup when effect is invalidated onInvalidate(() => { @@ -469,7 +450,7 @@ export function useLiveQuery( return { state: computed(() => state), data, - collection: computed(() => collection.value), + collection: computed(() => getPublicCollection(collection.value)), status: computed(() => status.value), isLoading: computed(() => status.value === `loading`), isReady: computed( diff --git a/packages/vue-db/tests/conformance.test.ts b/packages/vue-db/tests/conformance.test.ts index b15d95a27..1f1ce818c 100644 --- a/packages/vue-db/tests/conformance.test.ts +++ b/packages/vue-db/tests/conformance.test.ts @@ -229,7 +229,7 @@ const vueDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: false, suspense: false }, + features: { serverSnapshot: false, suspense: false, pooledEqFilters: true }, } describe(`owned native scope setup`, () => { From f2e0a0e6c12c89765f329cfedd508d3c0723b218 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 16:21:58 -0600 Subject: [PATCH 40/63] refactor(db): serve pooled views through the generic live-query observer Deletes the pooled-only wholesale observer. The generic observer skips its persisted-source walk for views without config and reuses shared no-op teardowns. The pooled oracle now also checks each view's rows and status directly, since the observer reads them only when notified. Costs about 0.3 us per mount in Node for 181 fewer lines. Co-authored-by: Isaac --- packages/db/src/live-query-observer.ts | 21 +- packages/db/src/query/pooled-live-query.ts | 182 ------------------ .../pooled-live-query-oracle.property.test.ts | 12 +- 3 files changed, 22 insertions(+), 193 deletions(-) diff --git a/packages/db/src/live-query-observer.ts b/packages/db/src/live-query-observer.ts index 11f42a708..4326c4f18 100644 --- a/packages/db/src/live-query-observer.ts +++ b/packages/db/src/live-query-observer.ts @@ -6,7 +6,6 @@ import { } from './live-query-adapter.js' import { getBuilderFromConfig } from './query/live/collection-registry.js' import { getPersistedReadinessSource } from './persisted-readiness.js' -import { createPooledObserver } from './query/pooled-live-query.js' import type { Collection } from './collection/index.js' import type { DbClient, DehydratedLiveQueryResult } from './client.js' import type { PersistedReadinessSource } from './persisted-readiness.js' @@ -23,6 +22,12 @@ export type LiveQueryPersistedStatus = // must remain observable without treating source readiness as query readiness. const INITIAL_RENDER_PRELOADS = new WeakSet>() +const NO_PERSISTED_READINESS = { + status: `unavailable`, + error: undefined, +} as const +const noop = () => {} + interface PersistedSourceEntry { collection: Collection readiness: PersistedReadinessSource @@ -31,6 +36,8 @@ interface PersistedSourceEntry { function collectPersistedReadinessSources( root: Collection, ): ReadonlyArray | undefined { + // A few integrations provide Collection-compatible objects without config. + if (!(root as { config?: unknown }).config) return undefined const seen = new Set>() const sources: Array = [] const visit = (collection: Collection): boolean => { @@ -357,7 +364,7 @@ class LiveQueryObserverImpl< error: unknown | undefined } { const sources = this.persistedSources - if (!sources) return { status: `unavailable`, error: undefined } + if (!sources) return NO_PERSISTED_READINESS let loading = false let error: unknown | undefined let failed = false @@ -753,7 +760,7 @@ class LiveQueryObserverImpl< ? subscribeLayoutChanges.call(collection, () => notify([], collection.status, true), ) - : () => {} + : noop const persistedUnsubs = this.persistedSources?.map((source) => source.readiness.subscribe(() => { if (this.disposed || this.subscriptions.size === 0) return @@ -794,7 +801,7 @@ class LiveQueryObserverImpl< : this.diffEntries(previousEntries, nextEntries), ) }) - : () => {} + : noop const release = () => { clientUnsub() statusUnsub() @@ -1163,12 +1170,6 @@ export function createLiveQueryObserver< collection: Collection | null | undefined, options: CreateLiveQueryObserverOptions = {}, ): LiveQueryObserver { - const pooled = createPooledObserver(collection, { - wholesale: options.mode === `wholesale`, - client: options.client, - onPreload: options.onPreload, - }) - if (pooled) return pooled return new LiveQueryObserverImpl( collection ?? null, options.mode === `wholesale`, diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 481ef4095..0413aaea8 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -2,8 +2,6 @@ import { SortedMap } from '../SortedMap.js' import { normalizeValue } from '../utils/comparison.js' import { isVirtualPropName } from '../virtual-props.js' import { getPersistedReadinessSource } from '../persisted-readiness.js' -import { LiveQueryObserverDisposedError } from '../errors.js' -import { getLiveQueryStatusFlags } from '../live-query-adapter.js' import { getWhereExpression } from './ir.js' import { createLiveQueryCollection } from './live-query-collection.js' import type { BasicExpression, QueryIR } from './ir.js' @@ -11,12 +9,6 @@ import type { BaseQueryBuilder } from './builder/index.js' import type { Collection, CollectionImpl } from '../collection/index.js' import type { ChangeMessage, CollectionStatus } from '../types.js' import type { CollectionEventHandler } from '../collection/events.js' -import type { - LiveQueryObserver, - LiveQueryObserverListener, - LiveQuerySnapshot, -} from '../live-query-observer.js' -import type { DehydratedLiveQueryResult } from '../client.js' /** * Live queries that filter one source Collection only by `eq(field, literal)` @@ -424,17 +416,6 @@ class PooledLiveQuery { return this.source.preload() } - /** Observe changes and status without allocating unsubscribe closures. */ - watch(onChanges: Listener, onStatus: StatusListener): void { - this.partition.addListener(this.group, onChanges) - this.partition.statusListeners.add(onStatus) - } - - unwatch(onChanges: Listener, onStatus: StatusListener): void { - this.partition.removeListener(this.group, onChanges) - this.partition.statusListeners.delete(onStatus) - } - cleanup(): Promise { return this.collection?.cleanup() ?? Promise.resolve() } @@ -458,169 +439,6 @@ class PooledLiveQuery { } } -/** - * The wholesale observer for a pooled view without a `DbClient`. Pooled views - * have no hydration, persisted restore, or single-result mode, so this keeps - * only the snapshot, subscription, preload, and disposal parts of the general - * observer contract. - */ -class PooledWholesaleObserver implements LiveQueryObserver< - Row, - string | number -> { - private snapshot: LiveQuerySnapshot | undefined - private snapshotRevision = -1 - private snapshotStatus: CollectionStatus | undefined - - private readonly records = new Set<{ - listener: LiveQueryObserverListener - }>() - private watching = false - private readonly onChanges: Listener = (changes) => this.deliver(changes) - private readonly onStatus: StatusListener = () => this.deliver(undefined) - private preloadPromise: Promise | undefined - private disposed = false - - constructor( - private readonly view: PooledLiveQuery, - private readonly onPreload: (() => void) | undefined, - ) {} - - getSnapshot(): LiveQuerySnapshot { - const status = this.view.status - if ( - this.snapshot && - this.snapshotRevision === this.view._stateRevision && - this.snapshotStatus === status - ) { - return this.snapshot - } - const rows = this.view.rows - const state = new Map() - const data: Array = [] - for (const key of rows.keys()) { - const value = rows.get(key)! - state.set(key, value) - data.push(value) - } - this.snapshotRevision = this.view._stateRevision - this.snapshotStatus = status - return (this.snapshot = { - state, - data, - collection: this.view.publicCollection, - // Rows stay in key order, so only inserts and deletes move keys. - layoutRevision: this.view._layoutRevision, - status, - ...getLiveQueryStatusFlags(status), - persistedStatus: `unavailable`, - isPersistedReady: false, - persistedError: undefined, - isEnabled: true, - }) - } - - getServerSnapshot(): LiveQuerySnapshot { - return this.getSnapshot() - } - - subscribe( - listener: LiveQueryObserverListener, - ): () => void { - if (this.disposed) throw new LiveQueryObserverDisposedError() - // A record per call, so one listener subscribed twice tears down twice. - const record = { listener } - this.records.add(record) - if (!this.watching) { - this.watching = true - this.view.watch(this.onChanges, this.onStatus) - } - return () => { - if (this.records.delete(record) && this.records.size === 0) { - this.stopWatching() - } - } - } - - private deliver( - changes: Array> | undefined, - ): void { - const records = this.records.size === 1 ? this.records : [...this.records] - for (const { listener } of records) listener(changes) - } - - private stopWatching(): void { - if (!this.watching) return - this.watching = false - this.view.unwatch(this.onChanges, this.onStatus) - } - - preload(): Promise { - if (this.preloadPromise) return this.preloadPromise - this.onPreload?.() - const promise = this.view.preload() - this.preloadPromise = promise - const clear = () => { - if (this.preloadPromise === promise) this.preloadPromise = undefined - } - void promise.then(clear, clear) - return promise - } - - preloadForInitialRender(): Promise { - if (this.disposed) { - return Promise.reject(new LiveQueryObserverDisposedError()) - } - return this.preload() - } - - isInitialRenderReady(): boolean { - return false - } - - getError(): unknown { - return undefined - } - - dehydrate(): DehydratedLiveQueryResult { - return { - rows: Array.from(this.view.entries(), ([key, value]) => ({ key, value })), - } - } - - dispose(): void { - if (this.disposed) return - this.disposed = true - this.records.clear() - this.stopWatching() - } -} - -/** A lean observer for a pooled view, or undefined for anything else. */ -export function createPooledObserver< - T extends object, - TKey extends string | number, ->( - collection: unknown, - { - wholesale, - client, - onPreload, - }: { - wholesale: boolean - client: unknown - onPreload: (() => void) | undefined - }, -): LiveQueryObserver | undefined { - if (!(collection instanceof PooledLiveQuery) || !wholesale || client) { - return undefined - } - return new PooledWholesaleObserver( - collection, - onPreload, - ) as unknown as LiveQueryObserver -} - // The observer reads the view itself; users get the live-query Collection. const forwardToCollection: ProxyHandler = { get(view, property) { diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 80e214bef..1295758b8 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -479,7 +479,11 @@ async function runHistory(history: History): Promise { reference: ReturnType // The model rows when the source started cleanup, if it has since. frozen: Array | undefined - view: { collection?: unknown } + view: { + collection?: unknown + status: string + entries: () => Iterable<[string | number, Record]> + } layout: { keys: string; revision: number } | undefined wholesale: ReturnType> // Rows folded from the pooled granular stream, and contradictions. @@ -549,6 +553,12 @@ async function runHistory(history: History): Promise { `${label} rows`, ).toEqual(reference.toArray.map((row) => describeRow(row))) expect(snapshot.status, `${label} status`).toBe(reference.status) + // The view itself, which the observer reads only when notified. + expect( + [...view.entries()].map(([, row]) => describeRow(row)), + `${label} view rows`, + ).toEqual(reference.toArray.map((row) => describeRow(row))) + expect(view.status, `${label} view status`).toBe(reference.status) const model = entry.frozen ?? expectedRows(rows, peer) expect( [...snapshot.state!.values()].map(describeFields).sort(), From 9c7c96669d465ee6e07f7803b672ec52934845d5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 16:24:43 -0600 Subject: [PATCH 41/63] refactor(db): release pooled partitions on the Collections' cleanup queue The partition schedules its release on the shared CleanupQueue with the lifecycle's never-subscribed floor, replacing its own timer and listener flag. The queue unrefs its timer, so a Node process holding a pooled view exits like one holding a Collection. Co-authored-by: Isaac --- packages/db/src/collection/lifecycle.ts | 2 +- packages/db/src/query/pooled-live-query.ts | 65 +++++++++++----------- 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 20dbce7a2..db812205b 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -28,7 +28,7 @@ import type { CollectionStateManager } from './state' * to the timer armed when the last subscriber leaves, which still honours * `gcTime` exactly. */ -const UNSUBSCRIBED_GC_FLOOR_MS = 50 +export const UNSUBSCRIBED_GC_FLOOR_MS = 50 export class CollectionLifecycleManager< TOutput extends object = Record, diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 0413aaea8..adc10f463 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -1,4 +1,6 @@ import { SortedMap } from '../SortedMap.js' +import { CleanupQueue } from '../collection/cleanup-queue.js' +import { UNSUBSCRIBED_GC_FLOOR_MS } from '../collection/lifecycle.js' import { normalizeValue } from '../utils/comparison.js' import { isVirtualPropName } from '../virtual-props.js' import { getPersistedReadinessSource } from '../persisted-readiness.js' @@ -34,9 +36,6 @@ interface PartitionGroup { layoutRevision: number } -// Matches the Collection lifecycle's floor for a never-subscribed Collection. -const UNSUBSCRIBED_RELEASE_FLOOR_MS = 50 - const partitionsBySource = new WeakMap>() // Typed so that 1, '1', and true stay distinct. @@ -72,10 +71,8 @@ class Partition { readonly statusListeners = new Set() private listenerCount = 0 private gcTime = 0 - private hadListener = false /** Set when the source starts cleanup; groups keep their last rows. */ terminated = false - private releaseTimer: ReturnType | undefined constructor( private readonly source: CollectionImpl, @@ -127,7 +124,14 @@ class Partition { this.gcTime = Math.max(this.gcTime, delay) } if (this.terminated) return - if (!this.subscription) { + this.subscribe() + // Like a Collection that synced before anything subscribed, a view built + // during a render gets a grace period to subscribe when it commits. + this.scheduleRelease(UNSUBSCRIBED_GC_FLOOR_MS) + } + + private subscribe(): void { + if (!this.terminated && !this.subscription) { this.subscription = this.source.subscribeChanges( (changes) => this.apply(changes as Array>), @@ -144,12 +148,10 @@ class Partition { if (this.terminated) this.stopStatusEvents?.() }) } - this.scheduleRelease() } addListener(group: PartitionGroup, listener: Listener): void { - this.retain() - this.hadListener = true + this.subscribe() group.listeners.add(listener) this.listenerCount++ } @@ -157,12 +159,12 @@ class Partition { removeListener(group: PartitionGroup, listener: Listener): void { if (!group.listeners.delete(listener)) return this.listenerCount-- - this.scheduleRelease() + this.scheduleRelease(0) } private terminate(): void { this.terminated = true - clearTimeout(this.releaseTimer) + CleanupQueue.getInstance().cancel(this) this.release() } @@ -172,29 +174,26 @@ class Partition { this.onEmpty() } - private scheduleRelease(): void { - clearTimeout(this.releaseTimer) - this.releaseTimer = undefined + // Releases on the Collections' shared GC queue, after the longest + // `gcTime` of this partition's views. + private scheduleRelease(minDelay: number): void { if (this.listenerCount > 0 || !Number.isFinite(this.gcTime)) return - // Like a Collection that synced before anything subscribed, a view built - // during a render gets a grace period to subscribe when it commits. - const delay = this.hadListener - ? this.gcTime - : Math.max(this.gcTime, UNSUBSCRIBED_RELEASE_FLOOR_MS) - this.releaseTimer = setTimeout(() => { - this.releaseTimer = undefined - if (this.listenerCount > 0) return - this.stopStatusEvents?.() - this.stopStatusEvents = undefined - // Views outlive a release and may subscribe again, so they keep their - // groups for the next subscription to refill. - for (const group of this.groups.values()) { - group.rows.clear() - group.revision++ - group.layoutRevision++ - } - this.release() - }, delay) + const delay = Math.max(this.gcTime, minDelay) + CleanupQueue.getInstance().schedule(this, delay, this.releaseIfUnused) + } + + private readonly releaseIfUnused = (): void => { + if (this.listenerCount > 0) return + this.stopStatusEvents?.() + this.stopStatusEvents = undefined + // Views outlive a release and may subscribe again, so they keep their + // groups for the next subscription to refill. + for (const group of this.groups.values()) { + group.rows.clear() + group.revision++ + group.layoutRevision++ + } + this.release() } private apply(changes: Array>): void { From 6a790bfae0b6673e09938433b455aae3de1e9133 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 16:31:24 -0600 Subject: [PATCH 42/63] perf(db): let query builder clones keep their fresh query Every builder step spread the query twice, once for the clone and again in the constructor. Clones now take ownership of the object they were built from; external construction still copies. About 4% faster query building. Co-authored-by: Isaac --- packages/db/src/query/builder/index.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 186038267..29265926f 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -135,19 +135,26 @@ type FnSelectQueryResult = : QueryBuilder> export class BaseQueryBuilder { - private readonly query: Partial = {} + private readonly query: Partial constructor( query: Partial = {}, private readonly resolveCollection?: CollectionResolver, + /** @internal Whether the builder may keep `query` without copying it. */ + owned = false, ) { - this.query = { ...query } + this.query = owned ? query : { ...query } } private _clone( query: Partial, ): BaseQueryBuilder { - return new BaseQueryBuilder(query, this.resolveCollection) + // Every clone receives a freshly spread query, so it needs no copy. + return new BaseQueryBuilder( + query, + this.resolveCollection, + true, + ) } /** From df928a8b64beadb53ab3e6106b3ae840b7b2ec0b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 16:31:54 -0600 Subject: [PATCH 43/63] docs: describe pooling in every framework adapter Co-authored-by: Isaac --- .changeset/perf-pooled-live-queries.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.changeset/perf-pooled-live-queries.md b/.changeset/perf-pooled-live-queries.md index 315035e62..824b7b4e4 100644 --- a/.changeset/perf-pooled-live-queries.md +++ b/.changeset/perf-pooled-live-queries.md @@ -1,8 +1,12 @@ --- '@tanstack/db': patch '@tanstack/react-db': patch +'@tanstack/vue-db': patch +'@tanstack/solid-db': patch +'@tanstack/svelte-db': patch +'@tanstack/angular-db': patch --- -Mount and update many small filtered live queries at Redux-level cost. In React, a `useLiveQuery` that reads one eager source Collection and filters it only by `eq(field, literal)` conjuncts, without a `DbClient` or Suspense, is served from an equality partition shared by every query on those fields. Each component reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. +Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection and filters it only by `eq(field, literal)` conjuncts is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. Results are unchanged: the same rows in key order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. From 9d1dd621066dcc76c28163fa85daccc532bed4d6 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 16:36:07 -0600 Subject: [PATCH 44/63] docs: record pooling conformance across every adapter Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- .../2026-10-01-pooled-live-queries.md | 15 +++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index c39edcf23..982f919be 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React and the compiled path under the other adapters; a partition that ignores a row's previous group fails both under React, and `eq-filter-peers` checks through `subscriberCount` that React shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md index 7c4b90d84..86e2c44a8 100644 --- a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -97,13 +97,12 @@ mutation, or never complete the transaction fail 4, 4, 4, 8, 12, 1, and 9 tests. Two shared conformance scenarios, `eq-filter-rows` and `eq-filter-peers`, run -the pooled path under React and compiled live queries under Vue, Solid, -Svelte, and Angular. A partition that ignores a row's previous group fails -both under React and none elsewhere. `eq-filter-peers` also checks the +the pooled path under React, Vue, Solid, Svelte, and Angular. A partition that +ignores a row's previous group fails both under every adapter. `eq-filter-peers` also checks the source's public `subscriberCount`: an adapter that declares `pooledEqFilters` -must share one subscription for two queries on the same fields. Disabling -pooling or using one partition per query fails it under React; before this -check the first passed. +must share one subscription for two queries on the same fields. Disabling pooling or using one partition per query fails it under React; +before this check the first passed. Each adapter's driver declares +`pooledEqFilters`. The loss audit found that a partition released its source one second after its last listener, whatever `gcTime` its views had. A focused release-timing @@ -153,5 +152,5 @@ oracle does not observe it. ## Open work -- Pooled live queries run only in React; the other adapters keep compiled live - queries until they use the shared resolver. +- Svelte's suite reads `@tanstack/db` from its built `dist`, so a mutant + must type-check and be rebuilt before Svelte can observe it. From adbdd49fee813c37446305f0adaba2c85125562d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 17:01:36 -0600 Subject: [PATCH 45/63] perf(db): allocate deepEquals' cycle map only for nested containers Re-applies this branch's lazy cycle map to main's deepEquals: containers create the map before descending, and a plain object registers only before its first object-valued child. Saves about 85 KiB per 200-row update batch. Co-authored-by: Isaac --- packages/db/src/proxy.ts | 2 +- packages/db/src/utils.ts | 52 ++++++++++++++++++++++++---------------- 2 files changed, 33 insertions(+), 21 deletions(-) diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 3622f90dd..56799fbb4 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -306,7 +306,7 @@ function deepClone( function draftValuesEqual( left: unknown, right: unknown, - paired = new Map(), + paired?: Map, ): boolean { return deepEqualsInternal(left, right, paired, true) } diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index 967650ea9..fe991f4ac 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -27,7 +27,7 @@ interface TypedArray { * ``` */ export function deepEquals(a: any, b: any): boolean { - return deepEqualsInternal(a, b, new Map()) + return deepEqualsInternal(a, b, undefined) } function isPlainPrototype(prototype: object | null): boolean { @@ -53,7 +53,8 @@ function enumerableOwnKeys(value: object): Array { export function deepEqualsInternal( a: any, b: any, - visited: Map, + // Created on the first container that descends into a child. + visited: Map | undefined, draft = false, ): boolean { // Handle strict equality (primitives, same reference) @@ -91,9 +92,10 @@ export function deepEqualsInternal( if (a.size !== b.size) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) // A draft compares entries in order; general equality looks keys up. @@ -117,9 +119,10 @@ export function deepEqualsInternal( if (a.size !== b.size) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) // Convert to arrays for comparison @@ -238,9 +241,10 @@ export function deepEqualsInternal( if (Array.isArray(a) && a.length !== b.length) return false if (Array.isArray(a) && !draft) { // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) const result = a.every((item, index) => @@ -261,10 +265,9 @@ export function deepEqualsInternal( if (prototype !== prototypeB && !plain && !plainB) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } - visited.set(a, b) // Compare enumerable symbol keys as well as string keys. Query results may // use user-owned symbols, and a symbol-only update is still a value change. @@ -272,27 +275,36 @@ export function deepEqualsInternal( const keysB = enumerableOwnKeys(b) // Check if they have the same number of keys - if (keysA.length !== keysB.length) { - visited.delete(a) - return false - } + if (keysA.length !== keysB.length) return false // A class instance without enumerable keys (a File, an object with // private fields) keeps its state elsewhere, so it equals only itself. // A draft copies a URL by its href, so URLs compare by href. if (keysA.length === 0 && !Array.isArray(a) && !(plain && plainB)) { - visited.delete(a) return a instanceof URL && a.href === b.href } - // Check if all keys exist in both objects and their values are equal - const result = keysA.every( - (key) => - Object.prototype.propertyIsEnumerable.call(b, key) && - deepEqualsInternal(a[key], b[key], visited, draft), - ) - - visited.delete(a) + // Check if all keys exist in both objects and their values are equal. + // Register for cycles only before descending, so a flat object + // allocates no cycle map. + let registered = false + let result = true + for (const key of keysA) { + const value = a[key] + if (!registered && value !== null && typeof value === `object`) { + visited ??= new Map() + visited.set(a, b) + registered = true + } + if ( + !Object.prototype.propertyIsEnumerable.call(b, key) || + !deepEqualsInternal(value, b[key], visited, draft) + ) { + result = false + break + } + } + if (registered) visited!.delete(a) return result } From d3543451bf235f5b337881afa503445458a2ae81 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 17:14:12 -0600 Subject: [PATCH 46/63] test(query-db-collection): assert settled update rows through collection.base The test expected the visible row to show the server's revision once an update settled. Core keeps a settled optimistic snapshot over the row until a later source acknowledgement, as the optimistic history and state retention oracles pin, so it passed only when the persisted runtime delivered another commit in time. It now checks the authoritative base rows. Co-authored-by: Isaac --- packages/query-db-collection/tests/query.test.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index fb4f05c5f..728353704 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -4805,11 +4805,10 @@ describe(`QueryCollection`, () => { }) await first.isPersisted.promise expect(adapter.rows.get(`p`)?.revision).toBe(2) - expect(collection.get(`p`)).toMatchObject({ - a: 10, - revision: 2, - $synced: true, - }) + // Settlement guarantees the stored server response, not that the + // accepted optimistic snapshot has stopped overlaying the row: a + // later source acknowledgement retires it. + expect(collection.base.get(`p`)).toMatchObject({ a: 10, revision: 2 }) const second = collection.update(`p`, (draft) => { draft.b = 1 @@ -4817,11 +4816,10 @@ describe(`QueryCollection`, () => { await second.isPersisted.promise expect(requestRevisions).toEqual([1, 2]) expect(adapter.rows.get(`p`)?.revision).toBe(3) - expect(collection.get(`p`)).toMatchObject({ + expect(collection.base.get(`p`)).toMatchObject({ a: 10, b: 1, revision: 3, - $synced: true, }) } finally { await collection.cleanup() From c7225f01a58dc5732754709ca6a0f881c5adf8f9 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 17:34:06 -0600 Subject: [PATCH 47/63] feat(db): pool eq-filtered live queries with other row conditions A query whose where has at least one eq conjunct and whose other conjuncts read only its own row now pools: the eq conjuncts pick its partition group and each view evaluates the rest with the compiler's evaluator, tracking which rows pass per subscription. The pooled oracle generates a residual not(eq(...)) conjunct and pins rows moving in and out of a view within one group. Co-authored-by: Isaac --- .changeset/perf-pooled-live-queries.md | 2 +- docs/contributing/oracle-coverage.md | 2 +- .../2026-10-01-pooled-live-queries.md | 13 ++ packages/db/src/query/live/ARCHITECTURE.md | 8 +- packages/db/src/query/pooled-live-query.ts | 138 ++++++++++++++---- .../pooled-live-query-oracle.property.test.ts | 72 ++++++--- 6 files changed, 185 insertions(+), 50 deletions(-) diff --git a/.changeset/perf-pooled-live-queries.md b/.changeset/perf-pooled-live-queries.md index 824b7b4e4..d7efcf3d3 100644 --- a/.changeset/perf-pooled-live-queries.md +++ b/.changeset/perf-pooled-live-queries.md @@ -7,6 +7,6 @@ '@tanstack/angular-db': patch --- -Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection and filters it only by `eq(field, literal)` conjuncts is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. +Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection, filters it by at least one `eq(field, literal)`, and has no clause besides `where` is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Its other `where` conditions on the row, such as `not`, `gt`, or `like`, are evaluated per query over its group. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. Results are unchanged: the same rows in key order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 9fa683fac..0ef72ac5f 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md index 86e2c44a8..ebbfc6117 100644 --- a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -112,6 +112,19 @@ floor for unsubscribed queries 1, a last-`gcTime`-wins rule 1, and releasing at `gcTime` 0 2. Release timing is resource lifetime, so the publication oracle does not observe it. +Pooling later admitted residual conjuncts: any `where` conjunct that reads +only the query's own row, beside at least one `eq`, is evaluated per view +with the compiler's evaluator. Peers may add `not(eq(g, literal))`, which the +model evaluates itself, and a pinned history moves rows in and out of a view +within one group. A view that ignores the residual, reports unfiltered +entries, or sends a row entering or leaving it as an update fails the +pinned history and both campaigns. Hiding part of a group makes some +earlier histories rarer, so in one full run the arrival-order, +normalization, delete-plus-insert, and dropped in-group update mutants +escaped the random campaign; the fixed campaign and their pinned histories +still kill each. Peers' literals were weighted toward the normalized values +to keep the normalization mutant in the fixed campaign. + ## Pooled live query oracle | Requirement | Outcome | diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index cbfc5b741..404737ff5 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1248,9 +1248,11 @@ listener lifetime. A framework adapter may serve a live query from an equality partition instead of this graph. That applies only when the query reads one eager, -non-persisted source Collection, filters it only by `eq(field, literal)` -conjuncts, and has no other clause, `DbClient`, or Suspense key. Such a pooled -live query reads the partition group for its literal tuple +non-persisted source Collection, its `where` has at least one +`eq(field, literal)` conjunct, every other conjunct reads only that row's +own fields, and it has no other clause, `DbClient`, or Suspense key. Such a +pooled live query reads the partition group for its `eq` literal tuple and +filters it by its remaining conjuncts with the compiler's evaluator (`packages/db/src/query/pooled-live-query.ts`). It builds its live-query Collection only when the application reads it. diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index adc10f463..8b9e2cd3b 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -5,6 +5,7 @@ import { normalizeValue } from '../utils/comparison.js' import { isVirtualPropName } from '../virtual-props.js' import { getPersistedReadinessSource } from '../persisted-readiness.js' import { getWhereExpression } from './ir.js' +import { compileExpression, toBooleanPredicate } from './compiler/evaluators.js' import { createLiveQueryCollection } from './live-query-collection.js' import type { BasicExpression, QueryIR } from './ir.js' import type { BaseQueryBuilder } from './builder/index.js' @@ -253,48 +254,93 @@ class Partition { type Conjunct = { path: Array; pathKey: string; literalKey: string } -// Adds `eq(alias.field, literal)` conjuncts to `out`; false for anything else. +type PoolableShape = { + paths: Array> + shapeKey: string + groupKey: string + // Conjuncts each view evaluates over its group's rows. + residual: Array +} + +// Whether an expression reads only this query's own row fields, so a view +// can evaluate it with the compiler's evaluator. +function readsOnlyRow(expression: BasicExpression, alias: string): boolean { + if (expression.type === `val`) return true + if (expression.type === `ref`) { + const [root, field] = expression.path + return root === alias && field !== undefined && !isVirtualPropName(field) + } + return expression.args.every((arg) => readsOnlyRow(arg, alias)) +} + +// Splits a conjunct into `eq(alias.field, literal)` groups and residual +// conjuncts; false for an expression a view cannot evaluate. function collectConjuncts( expression: BasicExpression, alias: string, out: Array, + residual: Array, ): boolean { - if (expression.type !== `func`) return false - const args = expression.args - if (expression.name === `and`) { - for (const arg of args) if (!collectConjuncts(arg, alias, out)) return false + if (expression.type === `func` && expression.name === `and`) { + for (const arg of expression.args) { + if (!collectConjuncts(arg, alias, out, residual)) return false + } return true } - if (expression.name !== `eq` || args.length !== 2) return false + const conjunct = equalityConjunct(expression, alias) + if (conjunct) out.push(conjunct) + else if (readsOnlyRow(expression, alias)) residual.push(expression) + else return false + return true +} + +function equalityConjunct( + expression: BasicExpression, + alias: string, +): Conjunct | undefined { + if (expression.type !== `func` || expression.name !== `eq`) return undefined + const args = expression.args + if (args.length !== 2) return undefined const left = args[0]! const right = args[1]! const ref = left.type === `ref` ? left : right const literal = left.type === `val` ? left : right - if (ref.type !== `ref` || literal.type !== `val`) return false + if (ref.type !== `ref` || literal.type !== `val`) return undefined const refPath = ref.path if ( refPath[0] !== alias || refPath.length < 2 || isVirtualPropName(refPath[1]!) ) { - return false + return undefined } const literalKey = equalityKey(literal.value) - if (literalKey === undefined) return false + if (literalKey === undefined) return undefined const path = refPath.slice(1) - out.push({ path, pathKey: JSON.stringify(path), literalKey }) - return true + return { path, pathKey: JSON.stringify(path), literalKey } +} + +// Whether a row passes every residual conjunct, as a WHERE filter decides. +function rowPredicate( + residual: Array, + alias: string, +): (row: Row) => boolean { + const conjuncts = residual.map((expression) => compileExpression(expression)) + // One namespaced row, reused so a check allocates nothing. + const namespaced: Record = {} + return (row) => { + namespaced[alias] = row + return conjuncts.every((conjunct) => + toBooleanPredicate(conjunct(namespaced as any)), + ) + } } /** * The equality conjuncts of a query that a partition can serve, or undefined * when any other clause or operand is present. */ -function poolableShape( - query: QueryIR, -): - | { paths: Array>; shapeKey: string; groupKey: string } - | undefined { +function poolableShape(query: QueryIR): PoolableShape | undefined { if ( query.from.type !== `collectionRef` || query.select || @@ -314,13 +360,21 @@ function poolableShape( return undefined } const conjuncts: Array = [] + const residual: Array = [] for (const where of query.where) { if ( - !collectConjuncts(getWhereExpression(where), query.from.alias, conjuncts) + !collectConjuncts( + getWhereExpression(where), + query.from.alias, + conjuncts, + residual, + ) ) { return undefined } } + // A partition needs at least one equality to group by. + if (conjuncts.length === 0) return undefined // Most shapes have one or two fields; a general sort costs more than both. if (conjuncts.length === 2) { if (conjuncts[1]!.pathKey < conjuncts[0]!.pathKey) conjuncts.reverse() @@ -336,7 +390,7 @@ function poolableShape( shapeKey += pathKey groupKey = appendGroupKeyPart(groupKey, literalKey) } - return { paths, shapeKey, groupKey } + return { paths, shapeKey, groupKey, residual } } /** @@ -358,6 +412,8 @@ class PooledLiveQuery { private readonly partition: Partition, groupKey: string, private readonly gcTime: number, + // The query's conjuncts beyond its group's equalities, if any. + private readonly passes: ((row: Row) => boolean) | undefined, ) { this.group = partition.group(groupKey) partition.retain(gcTime) @@ -375,23 +431,21 @@ class PooledLiveQuery { return this.group.layoutRevision } - /** The group's rows, in key order. */ - get rows(): SortedMap { - return this.group.rows - } - - entries(): IterableIterator<[string | number, Row]> { - return this.group.rows.entries() + entries(): Iterable<[string | number, Row]> { + const rows = this.group.rows.entries() + const passes = this.passes + return passes ? [...rows].filter(([, row]) => passes(row)) : rows } subscribeChanges( callback: Listener, options: { includeInitialState?: boolean } = {}, ): { unsubscribe: () => void } { - this.partition.addListener(this.group, callback) + const listener = this.passes ? this.filterChanges(callback) : callback + this.partition.addListener(this.group, listener) if (options.includeInitialState) { callback( - [...this.group.rows].map(([key, value]) => ({ + Array.from(this.entries(), ([key, value]) => ({ type: `insert`, key, value, @@ -399,7 +453,32 @@ class PooledLiveQuery { ) } return { - unsubscribe: () => this.partition.removeListener(this.group, callback), + unsubscribe: () => this.partition.removeListener(this.group, listener), + } + } + + // Turns the group's changes into this query's, tracking which rows have + // passed its residual conjuncts for this subscription. + private filterChanges(callback: Listener): Listener { + const passes = this.passes! + const visible = new Map(this.entries()) + return (changes) => { + const out: Array> = [] + for (const change of changes) { + const { key, value } = change + const previous = visible.get(key) + const next = change.type !== `delete` && passes(value) + if (next) visible.set(key, value) + else visible.delete(key) + if (previous && next) { + out.push({ type: `update`, key, value, previousValue: previous }) + } else if (previous) { + out.push({ type: `delete`, key, value: previous }) + } else if (next) { + out.push({ type: `insert`, key, value }) + } + } + if (out.length > 0) callback(out) } } @@ -489,5 +568,8 @@ export function createPooledLiveQuery( partition, shape.groupKey, gcTime, + shape.residual.length > 0 + ? rowPredicate(shape.residual, ir.from.alias) + : undefined, ) as unknown as Collection } diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 1295758b8..1e96df8a4 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -1,9 +1,10 @@ /** * # Does a pooled live query publish what its live-query Collection would? * - * Law and source: a live query that filters one source Collection only by - * `eq(field, literal)` conjuncts is served from a partition of that source - * shared by every query with the same fields. Its observer must publish the + * Law and source: a live query that filters one source Collection by + * `eq(field, literal)` conjuncts, plus any conjuncts that read only its row, + * is served from a partition of that source shared by every query with the + * same `eq` fields; each view evaluates its other conjuncts itself. Its observer must publish the * rows the live-query Collection for the same query publishes: the visible * source rows whose fields equal the literals under `eq` semantics * (`src/query/compiler/evaluators.ts`: nullish is UNKNOWN, a Date equals its @@ -16,15 +17,15 @@ * behind. * * Model: `expectedKeys` filters the model's visible rows with an independent - * `eq` over plain values. It does not import the evaluator, normalization, or + * `eq` over plain values and the peer's negated `g` equality. It does not import the evaluator, normalization, or * the partition. Order, row values, and status come from a second * formulation: a live-query Collection compiled for the same query. * * History grammar: rows have ids 0 through 3, delivered initially in key * order or in reverse, a field `f` from strings, * numbers and their look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, - * `0`, `null`, and a missing value, and a field `g` of `x` or `y`. Up to - * three peer queries use `eq(f, literal)`, optionally with `eq(g, literal)`. + * `0`, `null`, and a missing value, and a field `g` of `x` or `y`. Up to three peer queries use `eq(f, literal)`, optionally with + * `eq(g, literal)` and a residual `not(eq(g, literal))`. * Values are weighted toward `a`, and toward the normalized values against * numeric literals, so groups hold rows that stay, move, and normalize. * Steps commit sync transactions of one or two inserts, updates, or deletes, @@ -69,7 +70,12 @@ import { describe, expect, it } from 'vitest' import { createCollection } from '../../src/collection/index.js' import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { Query } from '../../src/query/builder/index.js' -import { and, createLiveQueryCollection, eq } from '../../src/query/index.js' +import { + and, + createLiveQueryCollection, + eq, + not, +} from '../../src/query/index.js' import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' import { Func, PropRef, Value } from '../../src/query/ir.js' import { @@ -91,7 +97,12 @@ const requestedReplayProperty = readOracleRunConfig().replayProperty const MISSING = Symbol(`missing`) type FieldValue = string | number | boolean | Date | null | typeof MISSING type Row = { id: string; f?: unknown; g: string } -type Peer = { f: string | number | boolean; g?: string } +type Peer = { + f: string | number | boolean + g?: string + // A residual conjunct, `not(eq(r.g, notG))`, each view evaluates itself. + notG?: string +} type Step = | { kind: `sync`; ops: Array } | { @@ -156,7 +167,10 @@ function expectedKeys( return [...rows.values()] .filter( (row) => - modelEq(row.f, peer.f) && (peer.g === undefined || row.g === peer.g), + modelEq(row.f, peer.f) && + (peer.g === undefined || row.g === peer.g) && + // `g` is never nullish here, so the negated equality is two-valued. + (peer.notG === undefined || row.g !== peer.notG), ) .map((row) => row.id) .sort() @@ -217,10 +231,14 @@ const peerArbitrary: fc.Arbitrary = fc.record( { f: fc.oneof( { weight: 2, arbitrary: fc.constant(`a`) }, - fc.constantFrom(1, Number.NaN, 0), + { + weight: 2, + arbitrary: fc.constantFrom(1, Number.NaN, 0), + }, fc.constantFrom(...literals), ), g: gArbitrary, + notG: gArbitrary, }, { requiredKeys: [`f`] }, ) @@ -358,6 +376,26 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ ], }, }, + { + name: `a residual conjunct moves rows in and out of a view within one group`, + history: { + rows: [ + { f: `a`, g: `x` }, + { f: `a`, g: `y` }, + ], + peers: [{ f: `a`, notG: `y` }, { f: `a` }], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `a`, g: `y` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 1, f: `a`, g: `x` }] }, + { + kind: `optimistic`, + op: { type: `update`, id: 1, f: `a`, g: `y` }, + confirm: false, + }, + { kind: `sync`, ops: [{ type: `delete`, id: 1 }] }, + ], + }, + }, { name: `a row updated within its group reaches peers as one update`, history: { @@ -403,13 +441,13 @@ const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) function peerQuery(source: any, peer: Peer) { return (q: any) => - q - .from({ r: source }) - .where(({ r }: any) => - peer.g === undefined - ? eq(r.f, peer.f) - : and(eq(r.f, peer.f), eq(r.g, peer.g)), - ) + q.from({ r: source }).where(({ r }: any) => { + const conjuncts = [eq(r.f, peer.f)] + if (peer.g !== undefined) conjuncts.push(eq(r.g, peer.g)) + if (peer.notG !== undefined) conjuncts.push(not(eq(r.g, peer.notG))) + const [first, second, ...rest] = conjuncts + return second ? and(first, second, ...rest) : first + }) } // A row's id and fields, which the model also knows. From ff262aef396c3453b8f1686c4fde944777d2d98d Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 19:43:47 -0600 Subject: [PATCH 48/63] test(db): drop the index-usage helper's dead entriesPassing patch The routing prefilter it instrumented was removed; unindexed scans all go through collection.entries, which the helper still tracks. Co-authored-by: Isaac --- packages/db/tests/utils.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index c8ebed421..53345f955 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -215,12 +215,6 @@ export function createIndexUsageTracker(collection: any): { recordFullScan() yield* originalEntries.call(this) } - const state = collection._state - const originalEntriesPassing = state.entriesPassing - state.entriesPassing = function* (prefilter: (row: object) => boolean) { - recordFullScan() - yield* originalEntriesPassing.call(this, prefilter) - } const restore = () => { // Remove the instance getter so the prototype getter applies again @@ -230,7 +224,6 @@ export function createIndexUsageTracker(collection: any): { if (rangeQuery) index.rangeQuery = rangeQuery } collection.entries = originalEntries - state.entriesPassing = originalEntriesPassing } return { stats, restore } From eb79accc0be6cfe7e03bd1178f88727d3b7da55a Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 20:07:48 -0600 Subject: [PATCH 49/63] docs: describe every-Collection update speedups in the changeset Co-authored-by: Isaac --- .changeset/cheaper-mutations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/cheaper-mutations.md b/.changeset/cheaper-mutations.md index 0387daf14..daaa395c6 100644 --- a/.changeset/cheaper-mutations.md +++ b/.changeset/cheaper-mutations.md @@ -2,4 +2,4 @@ '@tanstack/db': patch --- -Make updates and query building cheaper. Updates to rows whose fields are all primitives track changes without a proxy, drafts of other rows allocate less, sorted Collections no longer re-sort when an existing row changes value, and query builder references allocate less. Mutation ids are now a random per-runtime prefix plus a counter instead of a random UUID per mutation; they stay unique across tabs and sessions, but are no longer bare UUIDs. +Make updates and query building cheaper. Updates to rows whose fields are all primitives track changes without a proxy, drafts of other rows allocate less, sorted Collections no longer re-sort when an existing row changes value, published rows reuse one cached copy per key, equality checks on flat rows allocate nothing, and query building copies less. Mutation ids are now a random per-runtime prefix plus a counter instead of a random UUID per mutation; they stay unique across tabs and sessions, but are no longer bare UUIDs. From 9e615589236095ed02a0d550aef636e1c455de7b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 20:22:58 -0600 Subject: [PATCH 50/63] chore(db): refresh the private-member mangle cache Drops entries for private members that the pooled observer and release timer removals deleted. Co-authored-by: Isaac --- packages/db/mangle-cache.json | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index 4ec2e6940..78c5b50d7 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -367,16 +367,12 @@ "restore": "f1", "attach": "f2", "compilations": "f3", - "releaseTimer": "f4", "commitLocalOnlyDirect": "f5", "stopStatusEvents": "f6", - "snapshotRevision": "f7", "scheduleRelease": "f8", - "snapshotStatus": "f9", - "layoutKeys": "ga", - "watching": "gb", - "stopWatching": "gc", - "onChanges": "gd", - "onStatus": "ge", - "onEmpty": "gf" + "onEmpty": "gf", + "releaseIfUnused": "f4", + "filterChanges": "f7", + "passes": "f9", + "sameFields": "ga" } From 0eaca7f2dc37b8d9fe6cc64aa874b6e3d31b7b1c Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 20:22:59 -0600 Subject: [PATCH 51/63] fix(db): draw the mutation id prefix on the first mutation Generating it at module load broke Cloudflare Workers, which reject random values in global scope, so the Durable Object E2E suite could not start its worker. Co-authored-by: Isaac --- packages/db/src/collection/mutations.ts | 7 +++-- packages/db/tests/mutation-id.test.ts | 41 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 packages/db/tests/mutation-id.test.ts diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index a413c015f..893405cbf 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -43,10 +43,13 @@ import type { CollectionLifecycleManager } from './lifecycle' import type { CollectionStateManager } from './state' // One random prefix per runtime keeps mutation ids unique across tabs and -// sessions; the counter avoids generating a random UUID per mutation. -const mutationIdPrefix = safeRandomUUID() +// sessions; the counter avoids generating a random UUID per mutation. The +// prefix waits for the first mutation, because some runtimes reject random +// values at module scope. +let mutationIdPrefix: string | undefined let mutationCount = 0 function createMutationId(): string { + mutationIdPrefix ??= safeRandomUUID() return `${mutationIdPrefix}-${++mutationCount}` } diff --git a/packages/db/tests/mutation-id.test.ts b/packages/db/tests/mutation-id.test.ts new file mode 100644 index 000000000..76ab902a0 --- /dev/null +++ b/packages/db/tests/mutation-id.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from 'vitest' + +// Runtimes such as Cloudflare Workers reject random values generated at +// module scope, so a mutation id's per-runtime prefix waits for the first +// mutation. +describe(`mutation ids`, () => { + it(`do not draw random values during module evaluation`, async () => { + let next = 0 + const randomUUID = vi.fn(() => `prefix-${++next}`) + vi.stubGlobal(`crypto`, { randomUUID }) + vi.resetModules() + + try { + const { createCollection } = await import(`../src/collection/index.js`) + const { localOnlyCollectionOptions } = await import( + `../src/local-only.js` + ) + expect(randomUUID).not.toHaveBeenCalled() + + const collection = createCollection( + localOnlyCollectionOptions<{ id: string }>({ + id: `mutation-ids`, + getKey: (row) => row.id, + }), + ) + const first = collection.insert({ id: `a` }) + const second = collection.insert({ id: `b` }) + const [firstId, secondId] = [first, second].map( + (transaction) => transaction.mutations[0]!.mutationId, + ) + expect(firstId).not.toBe(secondId) + expect(firstId!.startsWith(`prefix-`)).toBe(true) + // One prefix per runtime; the counter makes each id unique. + expect(firstId!.split(`-`).slice(0, 2)).toEqual( + secondId!.split(`-`).slice(0, 2), + ) + } finally { + vi.unstubAllGlobals() + } + }) +}) From 83eb853a33e4d4ab73adac4a48ddc6ddac27b417 Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 20:44:07 -0600 Subject: [PATCH 52/63] fix(db): address review findings on pooled live query lifecycle - A filtered view resubscribing after its partition released seeds its row filter after the group refills, so later deletes reach it. - A released partition that subscribes again re-registers for new mounts, and never removes a newer partition under its key. - A pooled query's public Collection stays subscribed while its view is observed, so it does not clean itself up under a mounted view. - The public Collection reports the Collection prototype, so it passes instanceof and works as a query source. - React's development check tolerates a process shim without env. - The glossary describes residual conjuncts. Co-authored-by: Isaac --- docs/contributing/glossary.md | 2 +- docs/contributing/oracle-coverage.md | 2 +- packages/db/mangle-cache.json | 6 +- packages/db/src/query/pooled-live-query.ts | 64 +++++++++-- .../tests/query/pooled-live-query-gc.test.ts | 106 +++++++++++++++++- packages/react-db/src/development.ts | 7 +- packages/react-db/tests/development.test.ts | 15 ++- 7 files changed, 183 insertions(+), 19 deletions(-) diff --git a/docs/contributing/glossary.md b/docs/contributing/glossary.md index a7078bc30..4d48703ff 100644 --- a/docs/contributing/glossary.md +++ b/docs/contributing/glossary.md @@ -19,7 +19,7 @@ production queues, caches, or semantic helpers merely to share their names. | Collection | The public keyed data container. Capitalize it when referring to the TanStack DB type. | Relation, table, or query result. | | source Collection | A Collection read by a query or adapter. | Source relation when the value is a public Collection. | | live-query Collection | A Collection whose rows are produced by a live query. | Query, observer, or result set. | -| pooled live query | A live query filtered only by `eq(field, literal)` on one source Collection and served from an equality partition; its live-query Collection is built only when read. | Cached query or shared live-query Collection. | +| pooled live query | A live query on one source Collection whose `where` has at least one `eq(field, literal)` conjunct and otherwise reads only the row, served from an equality partition; its live-query Collection is built only when read. | Cached query or shared live-query Collection. | | equality partition | Source rows grouped by the `eq`-normalized values of one set of fields, shared by every pooled live query that filters on those fields. | Index or bucket relation. | | partition group | The rows of an equality partition whose fields equal one tuple of literals. | Bucket or active bucket. | | relation | An internal weighted multiset maintained by D2. | Collection. | diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 0ef72ac5f..76f677b83 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, and checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed and accept it as a query source. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index 78c5b50d7..2f7f34475 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -370,9 +370,11 @@ "commitLocalOnlyDirect": "f5", "stopStatusEvents": "f6", "scheduleRelease": "f8", - "onEmpty": "gf", "releaseIfUnused": "f4", "filterChanges": "f7", "passes": "f9", - "sameFields": "ga" + "sameFields": "ga", + "collectionHold": "gb", + "holdCollection": "gc", + "registry": "gd" } diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 8b9e2cd3b..eae689d11 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -78,7 +78,8 @@ class Partition { constructor( private readonly source: CollectionImpl, private readonly paths: Array>, - private readonly onEmpty: () => void, + // The partition's entry in its source's map, which a mount looks up. + private readonly registry: { add: () => void; remove: () => void }, ) {} groupKeyOf(row: Row | undefined): string | undefined { @@ -131,8 +132,10 @@ class Partition { this.scheduleRelease(UNSUBSCRIBED_GC_FLOOR_MS) } - private subscribe(): void { + subscribe(): void { if (!this.terminated && !this.subscription) { + // A released partition that subscribes again serves new mounts too. + this.registry.add() this.subscription = this.source.subscribeChanges( (changes) => this.apply(changes as Array>), @@ -172,7 +175,7 @@ class Partition { private release(): void { this.subscription?.unsubscribe() this.subscription = undefined - this.onEmpty() + this.registry.remove() } // Releases on the Collections' shared GC queue, after the longest @@ -405,6 +408,8 @@ class PooledLiveQuery { readonly _subscribeLayoutChanges = undefined private readonly group: PartitionGroup private collection: Collection | undefined = undefined + private listenerCount = 0 + private collectionHold: { unsubscribe: () => void } | undefined = undefined constructor( private readonly source: CollectionImpl, @@ -441,8 +446,12 @@ class PooledLiveQuery { callback: Listener, options: { includeInitialState?: boolean } = {}, ): { unsubscribe: () => void } { + // A released partition refills its group here, so seed the filter after. + this.partition.subscribe() const listener = this.passes ? this.filterChanges(callback) : callback this.partition.addListener(this.group, listener) + this.listenerCount++ + this.holdCollection() if (options.includeInitialState) { callback( Array.from(this.entries(), ([key, value]) => ({ @@ -452,8 +461,25 @@ class PooledLiveQuery { })), ) } + let subscribed = true return { - unsubscribe: () => this.partition.removeListener(this.group, listener), + unsubscribe: () => { + if (!subscribed) return + subscribed = false + this.partition.removeListener(this.group, listener) + if (--this.listenerCount === 0) { + this.collectionHold?.unsubscribe() + this.collectionHold = undefined + } + }, + } + } + + // While the view is observed, its built Collection stays subscribed, as + // the Collection would be if it served the observer itself. + private holdCollection(): void { + if (this.collection && this.listenerCount > 0) { + this.collectionHold ??= this.collection.subscribeChanges(() => {}) } } @@ -509,11 +535,15 @@ class PooledLiveQuery { } materialize(): Collection { - return (this.collection ??= createLiveQueryCollection({ - query: this.query, - startSync: true, - gcTime: this.gcTime, - })) + if (!this.collection) { + this.collection = createLiveQueryCollection({ + query: this.query, + startSync: true, + gcTime: this.gcTime, + }) + this.holdCollection() + } + return this.collection } } @@ -527,6 +557,10 @@ const forwardToCollection: ProxyHandler = { has(view, property) { return Reflect.has(view.materialize(), property) }, + // So `instanceof` and query sources accept it as the Collection it is. + getPrototypeOf(view) { + return Reflect.getPrototypeOf(view.materialize()) + }, } /** @@ -558,7 +592,17 @@ export function createPooledLiveQuery( let partition = partitions.get(shapeKey) if (!partition) { const owner = partitions - partition = new Partition(source, shape.paths, () => owner.delete(shapeKey)) + // A released partition may subscribe again; it must not then replace or + // remove a newer partition created under its key. + const created: Partition = new Partition(source, shape.paths, { + add: () => { + if (!owner.has(shapeKey)) owner.set(shapeKey, created) + }, + remove: () => { + if (owner.get(shapeKey) === created) owner.delete(shapeKey) + }, + }) + partition = created partitions.set(shapeKey, partition) } // Observers read the view directly; users get its `publicCollection`. diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index 8fc7de350..bee51ede1 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -1,8 +1,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { createCollection } from '../../src/collection/index.js' +import { CollectionImpl, createCollection } from '../../src/collection/index.js' import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { Query } from '../../src/query/builder/index.js' -import { createLiveQueryCollection, eq } from '../../src/query/index.js' +import { createLiveQueryCollection, eq, not } from '../../src/query/index.js' import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' import { mockSyncCollectionOptions } from '../utils.js' @@ -131,6 +131,108 @@ describe(`pooled live query gcTime`, () => { ).toEqual(compiledKeys) }) + it(`seeds a filtered view from its refilled group after a release`, async () => { + const source = makeSource() + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `b`, g: `x` } }) + source.utils.commit() + // `not(eq(r.id, 'b'))` is evaluated per view, so this view keeps `a`. + const view = createPooledLiveQuery( + query(source)(new Query()).where(({ r }: any) => not(eq(r.id, `b`))), + { gcTime: 1 }, + )! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + observer.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(60) + const stop = observer.subscribe(() => {}) + await vi.advanceTimersByTimeAsync(1) + expect([...observer.getSnapshot().state!.keys()]).toEqual([`a`]) + source.utils.begin() + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.commit() + await vi.advanceTimersByTimeAsync(1) + expect([...observer.getSnapshot().state!.keys()]).toEqual([]) + stop() + }) + + it(`shares one partition after a released partition subscribes again`, async () => { + const source = makeSource() + const mount = () => { + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + return { observer, stop: observer.subscribe(() => {}) } + } + const first = mount() + first.stop() + await vi.advanceTimersByTimeAsync(60) + // The released partition's view subscribes again, beside a new view. + const again = first.observer.subscribe(() => {}) + const second = mount() + // The new view joins the partition that subscribed again. + expect(source.subscriberCount).toBe(1) + again() + await vi.advanceTimersByTimeAsync(60) + const third = mount() + expect(source.subscriberCount).toBe(1) + second.stop() + third.stop() + }) + + it(`keeps a newer partition when an older one releases again`, async () => { + const source = makeSource() + const mount = () => { + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + return { observer, stop: observer.subscribe(() => {}) } + } + const first = mount() + first.stop() + await vi.advanceTimersByTimeAsync(60) + // A new partition takes the key before the old view subscribes again. + const second = mount() + first.observer.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(60) + const third = mount() + expect(source.subscriberCount).toBe(1) + second.stop() + third.stop() + }) + + it(`keeps its public Collection live while observed`, async () => { + const source = makeSource() + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + const stop = observer.subscribe(() => {}) + const collection = observer.getSnapshot().collection! + expect(collection.toArray).toHaveLength(1) + await vi.advanceTimersByTimeAsync(500) + expect(collection.status).toBe(`ready`) + expect(collection.toArray).toHaveLength(1) + stop() + }) + + it(`hands users a Collection that queries accept as a source`, () => { + const source = makeSource() + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const collection = createLiveQueryObserver(view, { + mode: `wholesale`, + }).getSnapshot().collection! + expect(collection).toBeInstanceOf(CollectionImpl) + const nested = createLiveQueryCollection({ + query: (q) => q.from({ c: collection }), + startSync: true, + }) + expect(nested.toArray.map((row) => row.id)).toEqual([`a`]) + }) + it.each([[[5, 120]], [[120, 5]]])( `releases with the longest gcTime among a partition's views (%j)`, async (gcTimes) => { diff --git a/packages/react-db/src/development.ts b/packages/react-db/src/development.ts index ba47d3802..af9da217e 100644 --- a/packages/react-db/src/development.ts +++ b/packages/react-db/src/development.ts @@ -6,5 +6,10 @@ export function shouldWarnInDevelopment(disableEnvVar: string): boolean { } catch { return false } - return typeof process === `undefined` || process.env[disableEnvVar] !== `1` + // A page may supply a `process` shim without `env`; nothing disabled it. + try { + return process.env[disableEnvVar] !== `1` + } catch { + return true + } } diff --git a/packages/react-db/tests/development.test.ts b/packages/react-db/tests/development.test.ts index 15a62363c..d4519afe0 100644 --- a/packages/react-db/tests/development.test.ts +++ b/packages/react-db/tests/development.test.ts @@ -12,7 +12,11 @@ const source = readFileSync( // Evaluates the module as a browser bundle would: `NODE_ENV` inlined by the // bundler, or left alone, and no `process` global. -function inBrowser(nodeEnv: string | undefined): (name: string) => boolean { +function inBrowser( + nodeEnv: string | undefined, + // Globals of the page, such as a `process` shim from another library. + globals: Record = {}, +): (name: string) => boolean { const { code } = transformSync(source, { loader: `ts`, format: `cjs`, @@ -22,7 +26,10 @@ function inBrowser(nodeEnv: string | undefined): (name: string) => boolean { : { 'process.env.NODE_ENV': JSON.stringify(nodeEnv) }, }) const bundle = { exports: {} as Record } - runInContext(code, createContext({ module: bundle, exports: bundle.exports })) + runInContext( + code, + createContext({ ...globals, module: bundle, exports: bundle.exports }), + ) return bundle.exports.shouldWarnInDevelopment as (name: string) => boolean } @@ -33,6 +40,10 @@ describe(`development warnings`, () => { expect(inBrowser(`development`)(`DISABLE`)).toBe(true) }) + it(`stay on when a page's process shim has no env`, () => { + expect(inBrowser(`development`, { process: {} })(`DISABLE`)).toBe(true) + }) + it(`stay off in a browser production bundle or without a bundler`, () => { expect(inBrowser(`production`)(`DISABLE`)).toBe(false) expect(inBrowser(undefined)(`DISABLE`)).toBe(false) From 693c1ebe03c02d5484d44c22223d7e1f11bd646b Mon Sep 17 00:00:00 2001 From: Isaac Date: Thu, 1 Oct 2026 22:02:51 -0600 Subject: [PATCH 53/63] fix(db): turn pooled queries terminal when source cleanup starts A partition terminated only on the source's cleaned-up status, which an adapter's pending cleanup delays, so pooled queries reported ready while live-query Collections already reported error. The partition now terminates from the source's cleanup-start hook, as live-query Collections do. A pinned oracle witness also checks that an eq path whose getter throws excludes the row on both paths. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- packages/db/mangle-cache.json | 3 +- packages/db/src/query/pooled-live-query.ts | 44 ++++++++++++++----- .../tests/query/pooled-live-query-gc.test.ts | 41 +++++++++++++++++ .../pooled-live-query-oracle.property.test.ts | 37 ++++++++++++++++ 5 files changed, 115 insertions(+), 12 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 76f677b83..05bb96717 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed and accept it as a query source. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index 2f7f34475..fd6171d92 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -376,5 +376,6 @@ "sameFields": "ga", "collectionHold": "gb", "holdCollection": "gc", - "registry": "gd" + "registry": "gd", + "deliverStatus": "ge" } diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index eae689d11..6580e87a7 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -11,7 +11,10 @@ import type { BasicExpression, QueryIR } from './ir.js' import type { BaseQueryBuilder } from './builder/index.js' import type { Collection, CollectionImpl } from '../collection/index.js' import type { ChangeMessage, CollectionStatus } from '../types.js' -import type { CollectionEventHandler } from '../collection/events.js' +import type { + CollectionEventHandler, + CollectionStatusChangeEvent, +} from '../collection/events.js' /** * Live queries that filter one source Collection only by `eq(field, literal)` @@ -141,16 +144,37 @@ class Partition { this.apply(changes as Array>), { includeInitialState: true }, ) - this.stopStatusEvents = this.source.on(`status:change`, (event) => { - // Like a live query, a pooled view fails for good when its source - // starts cleanup; queries mounted later get a new partition. - if (event.status === `cleaned-up`) this.terminate() - const delivered = this.terminated - ? { ...event, status: `error` as const } - : event - for (const listener of [...this.statusListeners]) listener(delivered) - if (this.terminated) this.stopStatusEvents?.() + const stopStatus = this.source.on(`status:change`, (event) => { + this.deliverStatus(event) }) + // Like a live query, a pooled view fails for good when its source + // starts cleanup, before the adapter's cleanup settles; queries + // mounted later get a new partition. + const stopCleanupStart = this.source._onCleanupStart(() => { + const previousStatus = this.source.status + this.terminate() + this.deliverStatus({ + type: `status:change`, + collection: this.source as unknown as Collection, + previousStatus, + status: `error`, + }) + }) + this.stopStatusEvents = () => { + stopStatus() + stopCleanupStart() + } + } + } + + private deliverStatus(event: CollectionStatusChangeEvent): void { + const delivered = this.terminated + ? { ...event, status: `error` as const } + : event + for (const listener of [...this.statusListeners]) listener(delivered) + if (this.terminated) { + this.stopStatusEvents?.() + this.stopStatusEvents = undefined } } diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index bee51ede1..5c07d38bd 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -202,6 +202,47 @@ describe(`pooled live query gcTime`, () => { third.stop() }) + it(`turns terminal when source cleanup starts, before it settles`, async () => { + let release!: () => void + const source = createCollection({ + id: `pooled-gc-${serial++}`, + getKey: (row) => row.id, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `a`, g: `x` } }) + commit() + markReady() + // The adapter's cleanup stays pending until released. + return () => + new Promise((resolve) => { + release = resolve + }) + }, + }, + }) + const observe = (view: any) => { + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + return { observer, stop: observer.subscribe(() => {}) } + } + const pooledView = observe( + createPooledLiveQuery(query(source)(new Query()), { gcTime: 1 })!, + ) + const compiledView = observe( + createLiveQueryCollection({ query: query(source), startSync: true }), + ) + await vi.advanceTimersByTimeAsync(1) + const cleanup = source.cleanup() + await vi.advanceTimersByTimeAsync(1) + expect(compiledView.observer.getSnapshot().status).toBe(`error`) + expect(pooledView.observer.getSnapshot().status).toBe(`error`) + release() + await cleanup + pooledView.stop() + compiledView.stop() + }) + it(`keeps its public Collection live while observed`, async () => { const source = makeSource() const view = createPooledLiveQuery(query(source)(new Query()), { diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index 1e96df8a4..e298feffd 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -812,6 +812,43 @@ async function runHistory(history: History): Promise { describe(`pooled live query oracle`, () => { if (requestedReplayProperty === undefined) { + // Generated fields are plain values; this pins a field whose getter + // throws, which both paths treat as a row that does not match. + it(`matches the live-query Collection when an eq path getter throws`, async () => { + const source = createCollection( + mockSyncCollectionOptions({ + id: `pooled-${serial++}`, + getKey: (row) => row.id, + initialData: [ + { id: `a`, profile: { code: 1 } }, + { + id: `b`, + profile: { + get code(): number { + throw new Error(`getter failed`) + }, + }, + }, + ], + }), + ) + await source.stateWhenReady() + const query = (q: any) => + q.from({ r: source }).where(({ r }: any) => eq(r.profile.code, 1)) + const pooled = createLiveQueryObserver( + createPooledLiveQuery(query(new Query())), + { mode: `wholesale` }, + ) + const stop = pooled.subscribe(() => {}) + const reference = createLiveQueryCollection({ query, startSync: true }) + await reference.preload() + const snapshot = pooled.getSnapshot() + expect(snapshot.status).toBe(reference.status) + expect([...snapshot.state!.keys()]).toEqual([...reference.keys()]) + stop() + await source.cleanup() + }) + for (const { name, history } of pinnedHistories) { it(`matches the live-query Collection when ${name}`, () => runHistory(history)) From 880b105cd8497267bb7735797497e7812d3b6686 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 07:36:26 -0600 Subject: [PATCH 54/63] fix(db): send rows with getter fields to the draft proxy The flat tracker read a getter field once to copy the row and again to diff it, so a getter that returns a new value per read reported a change the draft proxy does not. Rows with accessor fields now fall back to the proxy. Also corrects a comment that called tracker output user-provided changes. Co-authored-by: Isaac --- packages/db/src/collection/mutations.ts | 2 +- packages/db/src/proxy.ts | 11 ++++++----- .../flat-change-tracking-oracle.property.test.ts | 3 +++ 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index 893405cbf..50b5081cf 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -434,7 +434,7 @@ export class CollectionMutationsManager< > > = keysArray .map((key, index) => { - const itemChanges = changesArray[index] // User-provided changes for this specific item + const itemChanges = changesArray[index] // A fresh object the tracker recorded for this item // Skip items with no changes if (!itemChanges || Object.keys(itemChanges).length === 0) { diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 56799fbb4..b10ff85dc 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -835,13 +835,14 @@ export function withArrayChangeTracking( return deepClone(getChanges(), undefined, true) } -// Whether every own field of a plain object is a primitive or a function. +// Whether every own field of a plain object holds a primitive or a function. function isFlatPlainObject(value: object): boolean { const prototype = Object.getPrototypeOf(value) if (prototype !== Object.prototype && prototype !== null) return false for (const key in value) { - const field = (value as Record)[key] - if (field !== null && typeof field === `object`) return false + // A getter may return a new value on each read; the proxy reads it once. + const { value: field, get } = Object.getOwnPropertyDescriptor(value, key)! + if (get || (field !== null && typeof field === `object`)) return false } return Object.getOwnPropertySymbols(value).length === 0 } @@ -850,8 +851,8 @@ function isFlatPlainObject(value: object): boolean { * Change tracking for flat rows without proxies. A draft is a shallow copy, * and its changes are the fields that differ from the row afterwards under * the same equality the draft proxy uses for primitives. Returns undefined - * when any row has a nested object, a symbol key, or a class prototype, so - * the caller falls back to the proxy. + * when any row has a nested object, a getter, a symbol key, or a class + * prototype, so the caller falls back to the proxy. */ export function withFlatChangeTracking( targets: Array, diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts index fae011236..1d5a97965 100644 --- a/packages/db/tests/flat-change-tracking-oracle.property.test.ts +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -677,6 +677,9 @@ describe(`flat change tracking oracle`, () => { new (class Row { a = 1 })(), + // A getter can return a new value on each read, so only the proxy, + // which reads it once, reports a stable change set. + Object.defineProperty({}, `a`, { get: () => 1, enumerable: true }), ]) { expect(withFlatChangeTracking([row], callback, true)).toBeUndefined() } From 24565589e7c3988cb69d1d653b401875f0e54be5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 08:52:40 -0600 Subject: [PATCH 55/63] perf(db): pool live query configs that set only their query useLiveQuery({ query }), the form the deps-array deprecation recommends, resolved every config object to a compiled live-query Collection, so it never pooled: 240 such queries took 14 ms to mount instead of 2.3 ms. A config whose only options are query and gcTime now pools like its builder. Configs with an id, getKey, schema, handlers, or startSync still compile their own Collection. Co-authored-by: Isaac --- .changeset/perf-pooled-live-queries.md | 2 +- packages/db/src/live-query-options.ts | 27 +++++++++++++++---- .../tests/query/pooled-live-query-gc.test.ts | 18 +++++++++++++ 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.changeset/perf-pooled-live-queries.md b/.changeset/perf-pooled-live-queries.md index d7efcf3d3..8ef5b0635 100644 --- a/.changeset/perf-pooled-live-queries.md +++ b/.changeset/perf-pooled-live-queries.md @@ -7,6 +7,6 @@ '@tanstack/angular-db': patch --- -Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection, filters it by at least one `eq(field, literal)`, and has no clause besides `where` is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Its other `where` conditions on the row, such as `not`, `gt`, or `like`, are evaluated per query over its group. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. +Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection, filters it by at least one `eq(field, literal)`, and has no clause besides `where` is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Its other `where` conditions on the row, such as `not`, `gt`, or `like`, are evaluated per query over its group. This applies to a query function, a query builder, and a `{ query }` config that sets no other option besides `queryKey` or `gcTime`. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. Results are unchanged: the same rows in key order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. diff --git a/packages/db/src/live-query-options.ts b/packages/db/src/live-query-options.ts index ee8fe1d22..c14200cca 100644 --- a/packages/db/src/live-query-options.ts +++ b/packages/db/src/live-query-options.ts @@ -151,6 +151,23 @@ export function getLiveQueryHash( * A query builder whose shape a shared partition can serve gets a pooled view * instead of its own live-query Collection. */ +// A config that names only its query and lifetime describes the same live +// query as its builder. Any other option shapes its Collection, so compiles. +function poolConfig( + config: LiveQueryCollectionConfig, + gcTime: number | undefined, +): Collection | undefined { + if ( + !(config.query instanceof BaseQueryBuilder) || + !Object.keys(config).every((key) => key === `query` || key === `gcTime`) + ) { + return undefined + } + return createPooledLiveQuery(config.query, { + gcTime: config.gcTime ?? gcTime, + }) +} + export function resolveLiveQueryValue( value: unknown, { gcTime, pool = true }: { gcTime?: number; pool?: boolean } = {}, @@ -167,11 +184,11 @@ export function resolveLiveQueryValue( ) } if (typeof value === `object`) { - return createLiveQueryCollection({ - startSync: true, - gcTime, - ...(value as LiveQueryCollectionConfig), - }) + const config = value as LiveQueryCollectionConfig + return ( + (pool ? poolConfig(config, gcTime) : undefined) ?? + createLiveQueryCollection({ startSync: true, gcTime, ...config }) + ) } throw new Error( `A live query must be a QueryBuilder, LiveQueryCollectionConfig, Collection, undefined, or null. Got: ${typeof value}`, diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index 5c07d38bd..fdb0c842f 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -4,6 +4,7 @@ import { createLiveQueryObserver } from '../../src/live-query-observer.js' import { Query } from '../../src/query/builder/index.js' import { createLiveQueryCollection, eq, not } from '../../src/query/index.js' import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' +import { resolveLiveQueryValue } from '../../src/live-query-options.js' import { mockSyncCollectionOptions } from '../utils.js' /** @@ -243,6 +244,23 @@ describe(`pooled live query gcTime`, () => { compiledView.stop() }) + it(`pools a query config that names only its query`, () => { + const source = makeSource() + const observe = (value: unknown) => + createLiveQueryObserver(resolveLiveQueryValue(value, { gcTime: 1 }), { + mode: `wholesale`, + }).subscribe(() => {}) + const stops = [ + observe({ query: query(source)(new Query()) }), + observe({ query: query(source)(new Query()), gcTime: 5 }), + ] + expect(source.subscriberCount).toBe(1) + // An id names a distinct Collection, so that config compiles its own. + stops.push(observe({ query: query(source)(new Query()), id: `own` })) + expect(source.subscriberCount).toBe(2) + for (const stop of stops) stop() + }) + it(`keeps its public Collection live while observed`, async () => { const source = makeSource() const view = createPooledLiveQuery(query(source)(new Query()), { From b87aee508f640df4b33d5e066e05110d5a49b6af Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 09:01:10 -0600 Subject: [PATCH 56/63] perf(db): identify pooled queries by their source, fields, and literals A live query with no deps or queryKey rebuilt and canonicalized its whole query IR on every render to detect changes. For a query a partition can serve without a residual conjunct, its source and eq fields and literals determine its rows, so they now form its identity. Other queries keep the structural identity. With 240 useLiveQuery({ query }) cells, mount drops from about 3.1 to 2.6 ms and a 200-update batch from 2.0 to 1.45 ms. Co-authored-by: Isaac --- packages/db/src/live-query-options.ts | 12 +- packages/db/src/query/pooled-live-query.ts | 15 +++ .../tests/query/pooled-query-identity.test.ts | 114 ++++++++++++++++++ 3 files changed, 139 insertions(+), 2 deletions(-) create mode 100644 packages/db/tests/query/pooled-query-identity.test.ts diff --git a/packages/db/src/live-query-options.ts b/packages/db/src/live-query-options.ts index c14200cca..2901c0526 100644 --- a/packages/db/src/live-query-options.ts +++ b/packages/db/src/live-query-options.ts @@ -1,7 +1,10 @@ import { BaseQueryBuilder } from './query/builder/index.js' import { isCollection } from './live-query-adapter.js' import { createLiveQueryCollection } from './query/live-query-collection.js' -import { createPooledLiveQuery } from './query/pooled-live-query.js' +import { + createPooledLiveQuery, + getPooledQueryIdentity, +} from './query/pooled-live-query.js' import { getStableQueryBuilderHash, getStableValueHash, @@ -112,7 +115,12 @@ export function prepareLiveQueryValue( export function getPreparedLiveQueryIdentity(value: unknown): unknown { if (isCollection(value)) return [`collection`, value.id] if (value instanceof BaseQueryBuilder) { - return [`query`, getStableQueryBuilderHash(value)] + // A pooled query's fields and literals identify its rows without + // canonicalizing its whole IR on every render. + const pooled = getPooledQueryIdentity(value) + return pooled === undefined + ? [`query`, getStableQueryBuilderHash(value)] + : [`pooled`, pooled] } if (value && typeof value === `object` && `query` in value) { const config = value as LiveQueryCollectionConfig diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 6580e87a7..8ab791735 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -587,6 +587,21 @@ const forwardToCollection: ProxyHandler = { }, } +/** + * The identity of a query a partition can serve with no residual conjunct: + * its source and its `eq` fields and literals, which determine its rows. + * Undefined for any other query, which keeps the full structural identity. + */ +export function getPooledQueryIdentity( + query: BaseQueryBuilder, +): string | undefined { + const ir = query._getQuery() + if (ir.from.type !== `collectionRef`) return undefined + const shape = poolableShape(ir) + if (!shape || shape.residual.length > 0) return undefined + return JSON.stringify([ir.from.collection.id, shape.shapeKey, shape.groupKey]) +} + /** * A pooled view for a query a partition can serve, or undefined. The view is * typed as the Collection it stands in for. diff --git a/packages/db/tests/query/pooled-query-identity.test.ts b/packages/db/tests/query/pooled-query-identity.test.ts new file mode 100644 index 000000000..0f05ebe89 --- /dev/null +++ b/packages/db/tests/query/pooled-query-identity.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { getPreparedLiveQueryIdentity } from '../../src/live-query-options.js' +import { Query } from '../../src/query/builder/index.js' +import { and, eq, gt } from '../../src/query/index.js' +import { mockSyncCollectionOptions } from '../utils.js' + +/** + * A query a partition can serve is identified by its source and its `eq` + * fields and literals, which the partition already extracts. Two queries + * with the same identity must publish the same rows, and queries that can + * publish different rows must have different identities. Queries the + * partition cannot serve keep the full structural identity. + */ +type Row = { id: string; a: string; b: string; n: number } +let serial = 0 +const makeSource = () => + createCollection( + mockSyncCollectionOptions({ + id: `pooled-identity-${serial++}`, + getKey: (row) => row.id, + initialData: [], + }), + ) +const identity = (build: (q: any) => unknown) => + getPreparedLiveQueryIdentity(build(new Query())) + +describe(`pooled query identity`, () => { + const source = makeSource() + + it(`ignores conjunct and operand order`, () => { + expect( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(eq(r.a, `x`), eq(r.b, `y`))), + ), + ).toEqual( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => eq(`y`, r.b)) + .where(({ r }: any) => eq(r.a, `x`)), + ), + ) + }) + + it(`ignores the source alias`, () => { + expect( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ), + ).toEqual( + identity((q) => + q.from({ s: source }).where(({ s }: any) => eq(s.a, `x`)), + ), + ) + }) + + it(`separates literals, fields, and sources`, () => { + const base = identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ) + expect( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `y`)), + ), + ).not.toEqual(base) + expect( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.b, `x`)), + ), + ).not.toEqual(base) + // A literal 1 and '1' match different rows. + expect( + identity((q) => q.from({ r: source }).where(({ r }: any) => eq(r.n, 1))), + ).not.toEqual( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.n, `1`)), + ), + ) + const other = makeSource() + expect( + identity((q) => q.from({ r: other }).where(({ r }: any) => eq(r.a, `x`))), + ).not.toEqual(base) + }) + + it(`keeps the structural identity for queries a partition cannot serve`, () => { + const residual = identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(eq(r.a, `x`), gt(r.n, 1))), + ) + expect(residual).toEqual( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(gt(r.n, 1), eq(r.a, `x`))), + ), + ) + expect(residual).not.toEqual( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(eq(r.a, `x`), gt(r.n, 2))), + ), + ) + expect((residual as Array)[0]).toBe(`query`) + const pooled = identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ) + expect((pooled as Array)[0]).toBe(`pooled`) + }) +}) From cc2015414577d68895816589342074521c0f5ed3 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 09:34:36 -0600 Subject: [PATCH 57/63] perf(db): skip enriching rows an unindexed snapshot rejects Compiled live queries that do not pool, such as those with orderBy, a DbClient, or Suspense, scanned every source row of an unindexed source and enriched it with virtual properties before running their predicate. The scan now tests one eq conjunct with a string or boolean literal on the stored row first and enriches only rows that pass, as main's prefilter did. The enriched copy's field is the stored value or undefined, so the test rejects only rows the full predicate rejects. The property-visibility test now also checks the reverse direction. With 240 such queries, mount drops from about 15.5 ms to 11.7 ms (main: 13.1 ms). Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- packages/db/src/collection/change-events.ts | 65 ++++++++++++++++++- packages/db/src/collection/index.ts | 4 +- packages/db/src/collection/state.ts | 17 +++++ ...here-prefilter-property-visibility.test.ts | 37 +++++++++++ packages/db/tests/utils.ts | 8 +++ 6 files changed, 130 insertions(+), 3 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 05bb96717..74762817f 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -264,7 +264,7 @@ comment and the current API/architecture contract before extending its model. | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites), [Vue synchronous publication](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/useLiveQuery-publication-oracle.test.ts) | Exact exposed rows/pages and each framework's own lifecycle cuts. Vue's synchronous watcher checks insert, update, and nonterminal delete through supplied Collection and identity query inputs; it does not cover an empty final result, multiple changes in one transaction, or query recompilation. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Window-controller overlap | [shared infinite-query conformance](https://github.com/TanStack/db/blob/main/packages/db/tests/conformance/infinite-suite.ts), [React driver](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [Vue driver](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/infinite-query-conformance.test.ts), [Svelte driver](https://github.com/TanStack/db/blob/main/packages/svelte-db/tests/infinite-query-conformance.svelte.test.ts), [review record](oracle-reviews/dec-03-window-overlap.md) | Four or five ordered source rows, two window-settlement orders, and public snapshots after each settlement, a subscribed observer update, a detached controller read, and explicit recovery. Insertion and removal of the fifth row distinguish both continuation directions while an earlier error remains visible. The detached read follows a source change with no controller subscriber; its preloaded Collection retains the five-row window. The exact overlap uses the exported DB controller in each package realm. Framework hooks expose no controller preload, so this cell does not establish a hook scheduling path; direct hook paging remains in the adjacent shared scenarios. An unsubscribed controller has no notification claim. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash identity](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-identity-oracle.property.test.ts), [MultiSet consolidation](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/multiset-consolidate-oracle.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [BTree Map](https://github.com/TanStack/db/blob/main/packages/db/tests/btree-map-oracle.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts), [LIKE semantics](https://github.com/TanStack/db/blob/main/packages/db/tests/query/compiler/evaluators.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, custom comparator dispatch/reference identity/index compatibility, LIKE wildcard refinement, compiled output bags, declared value-pair agreement between D2 equality keys and IR equality operands, and Collection collation with exact public scan/index order and one-index reuse across repeated reads. The auto-index cells cross BasicIndex and BTreeIndex; scan cells omit indexes. Both paths cover lexical and numeric `en-US` locale defaults plus an explicit ordinary locale override of a numeric default. An unindexed six-row work cell bounds Collection collation resolution to at most two reads per ordered scan: one index-path check and one scan clause. Other locales and ordered histories are outside this owner. The identity pair grammar covers primitives, signed zero, NaN/invalid Date, valid Date/timestamp, binary/Buffer, Temporal, nested references, symbols, and functions; fixed and sampled pairs do not prove every JavaScript value or hash collision freedom. Ref proxies are expressions rather than literal values in this lane. The deep-equality owner also pins that `deepEquals` ignores Map and Set insertion order, RegExp `lastIndex`, array holes, and typed-array class, the rules draft equality keeps. It also pins that objects of another class differ, that plain and null-prototype objects are one class, that a class instance without enumerable keys equals only itself, that URLs compare by `href`, and that typed-array elements compare like numbers. The hash-values owner samples Set/object and Map/object type-marker distinctions in fixed and random campaigns with seed-and-path replay; collision freedom is not promised. The hash-identity owner checks `hash` and `equalHashValues` against one spec-level identity model: primitives with signed zero, NaN, and bigint; reference leaves (functions, registered handles, `File`, binary over 128 bytes); Date, binary, Temporal, and RegExp headers; array length and holes; Map and Set insertion order; and object keys in any order, for acyclic values and for cycles through arrays, Map values, Sets, and plain objects. Map/Set order sensitivity is pinned current behavior, not a promise. Arrays from another realm, RegExps with a `NaN` `lastIndex`, getters, proxies, other cycle carriers, and pairs that differ only in a back-edge target are outside its grammar; pinned cases keep equality's cross-realm length check and strict RegExp field comparison. `hash-work.test.ts` pins how visited values and array/RegExp headers count toward the structural work cap, and that equality copies a shared Map's entries once per distinct pair ([code-weight hash dispatch review](oracle-reviews/code-weight-hash-dispatch.md)). Custom string indexes do not optimize ordinary ranges, and custom cursors remain unsupported. The LIKE owner covers boolean string matching and bounded work, not nullish three-valued logic or a general Unicode collation contract. Unsupported composite cursors reject. The MultiSet consolidation owner checks keyed, single-type, and structural identity, signed zero and NaN in keyed and single-number data, the retained first record, zero-sum removal, and input record contents. It does not assert output order. Its keyed grammar includes cross-type key and value collisions, non-finite numeric keys, the `\|` delimiter, symbol and function identity, and a direct-object/object-tuple delimiter control from #1948. Eight named collision witnesses and both generated campaigns are RED on `c879ba6d` and GREEN with the repair ([#1948 review record](oracle-reviews/issue-1948-keyed-consolidation.md)); the [code-weight consolidation review](oracle-reviews/code-weight-multiset-consolidation.md) records the earlier exclusion. The keyed reference uses direct value/reference equality, including SameValueZero for numeric keys. The unkeyed fallback keeps its original key and value domains. The direct `MultiSet` owner does not prove which `@tanstack/db` query shapes produce primitive keyed values or mixed-type source keys; that needs a live-query production witness. The index owner enforces one invalid-comparator law across both BasicIndex and BTreeIndex: for each comparator, accepted numeric prefix (including the empty prefix), and probe operation (add, update, eq/range lookup, take, build), the operation throws exactly when it receives a `NaN` or non-number result and never when every comparison is valid. A `signed infinity` comparator is the valid control, and a custom collation `compare` supplied through `compareOptions` is checked the same way. Two BTree Map split histories check inserted-key reads, exact whole-range payloads/order, size, and extrema, proving split placement follows the insertion index without a post-mutation comparison. A rejected index add or remove leaves the index refining its accepted rows; update and build are not atomic. At the Collection boundary, both index types and both write paths (optimistic insert, sync commit) crash the collection: status becomes `error` and the next mutation throws, so a usable collection never holds rows its subscribers were not told about. Open: a sync source can still commit writes on a collection in `error` state, which stores unpublished rows; this is existing `markError` behavior and needs a lifecycle-owner witness. Comparator transitivity is outside this evidence. | -| WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing and the unindexed snapshot prefilter were later removed in favor of pooled live queries; the routing and prefilter mutants above are historical, and the same peer and snapshot laws now check plain subscription dispatch and scans. | +| WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing was later removed in favor of pooled live queries; the routing mutants above are historical, and the same peer laws now check plain subscription dispatch. The unindexed snapshot prefilter was restored for compiled queries as one stored-row `eq` test on a string or boolean literal. Its property-visibility test also covers the reverse direction: a stored row with an inherited or non-enumerable field must still match `isUndefined` on that field, which a scan that evaluated the full predicate on stored rows fails. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | | Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index f22bcda97..c48f64971 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -7,6 +7,8 @@ import { optimizeExpressionWithIndexes, } from '../utils/index-optimization.js' import { ensureIndexForField } from '../indexes/auto-index.js' +import { getPropRefPropertyPath } from '../query/ir.js' +import { isVirtualPropName } from '../virtual-props.js' import { makeComparator } from '../utils/comparison.js' import { buildCompareOptions } from '../query/compiler/order-by' import type { @@ -19,6 +21,59 @@ import type { CollectionImpl } from './index.js' import type { BasicExpression, OrderBy } from '../query/ir.js' import type { WithVirtualProps } from '../virtual-props.js' +/** + * Yields visible entries, enriched with virtual properties, whose stored row + * passes `prefilter`. + */ +export type StoredRowScan = ( + prefilter: (row: object) => boolean, +) => Iterable<[TKey, WithVirtualProps]> + +/** + * A test on a stored row that is false only when `expression` must be false + * on the row's enriched copy, so a scan can skip enriching rows that fail. + * + * It reads one `eq(field, literal)` conjunct with a string or boolean + * literal. The copy holds each enumerable own root field of the stored row + * and lacks the others, so its field is the stored value or `undefined`, and + * equality normalization maps no other value onto a plain string or boolean. + * A read that throws passes the row to the full predicate. + */ +export function compileStoredRowPrefilter( + expression: BasicExpression, +): ((row: object) => boolean) | undefined { + const conjuncts: Array = [] + const collect = (node: BasicExpression) => { + if (node.type === `func` && node.name === `and`) node.args.forEach(collect) + else conjuncts.push(node) + } + collect(expression) + for (const conjunct of conjuncts) { + if (conjunct.type !== `func` || conjunct.name !== `eq`) continue + const [left, right] = conjunct.args + const ref = left?.type === `ref` ? left : right + const literal = left?.type === `val` ? left : right + if (ref?.type !== `ref` || literal?.type !== `val`) continue + const expected: unknown = literal.value + if (typeof expected !== `string` && typeof expected !== `boolean`) continue + const path = getPropRefPropertyPath(ref) + if (path.length === 0 || isVirtualPropName(path[0]!)) continue + return (row) => { + try { + let value: unknown = row + for (const segment of path) { + if (value === null || value === undefined) return false + value = (value as Record)[segment] + } + return value === expected + } catch { + return true + } + } + } + return undefined +} + /** * Returns the current state of the collection as an array of changes * @param collection - The collection to get changes from @@ -56,13 +111,21 @@ export function currentStateAsChanges< >( collection: CollectionLike, TKey>, options: CurrentStateAsChangesOptions = {}, + scanStoredRows?: StoredRowScan, ): Array, TKey>> | void { // Helper function to collect filtered results const collectFilteredResults = ( filterFn?: (value: WithVirtualProps) => boolean, ): Array, TKey>> => { const result: Array, TKey>> = [] - for (const [key, value] of collection.entries()) { + // Reject rows by a stored field before enriching them; survivors still + // pass through the full predicate. + const prefilter = + filterFn && scanStoredRows && options.where + ? compileStoredRowPrefilter(options.where) + : undefined + const rows = prefilter ? scanStoredRows!(prefilter) : collection.entries() + for (const [key, value] of rows) { // If no filter function is provided, include all items if (filterFn?.(value) ?? true) { result.push({ diff --git a/packages/db/src/collection/index.ts b/packages/db/src/collection/index.ts index 4c81a0b9f..3dc01061c 100644 --- a/packages/db/src/collection/index.ts +++ b/packages/db/src/collection/index.ts @@ -1072,7 +1072,9 @@ export class CollectionImpl< public currentStateAsChanges( options: CurrentStateAsChangesOptions = {}, ): Array, TKey>> | void { - return currentStateAsChanges(this, options) + return currentStateAsChanges(this, options, (prefilter) => + this._state.entriesPassing(prefilter), + ) } /** diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index b53f1cb75..39f8914e4 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -392,6 +392,23 @@ export class CollectionStateManager< ) } + /** + * Visible entries whose stored row passes `prefilter`, enriched with virtual + * properties. Rows that fail are never enriched. + */ + public *entriesPassing( + prefilter: (row: object) => boolean, + ): IterableIterator<[TKey, WithVirtualProps]> { + // Without optimistic state, the visible rows are the synced rows in order. + const rows = + this.optimisticUpserts.size === 0 && this.optimisticDeletes.size === 0 + ? this.syncedData + : this.entries() + for (const [key, row] of rows) { + if (prefilter(row)) yield [key, this.enrichWithVirtualProps(row, key)] + } + } + /** * Creates a change message with virtual properties. * Uses the "add-if-missing" pattern so that pass-through from upstream diff --git a/packages/db/tests/query/where-prefilter-property-visibility.test.ts b/packages/db/tests/query/where-prefilter-property-visibility.test.ts index 7efb54a57..95d9ee819 100644 --- a/packages/db/tests/query/where-prefilter-property-visibility.test.ts +++ b/packages/db/tests/query/where-prefilter-property-visibility.test.ts @@ -134,3 +134,40 @@ it('lets the full filter handle a throwing nested getter in a change', async () await collection.cleanup() } }) + +it.each(['inherited', 'non-enumerable'] as const)( + 'keeps a row whose %s field the enriched row omits', + async (kind) => { + // The enriched row lacks `v`, so `isUndefined(v)` holds there although + // the stored row reads a value. A scan that evaluated the predicate on + // stored rows would drop it. + const row = + kind === 'inherited' + ? Object.assign(Object.create({ v: 'x' }) as Row, { id: 'hidden' }) + : Object.defineProperty({ id: 'hidden' } as Row, 'v', { + value: 'x', + enumerable: false, + }) + const collection = createCollection( + mockSyncCollectionOptions({ + id: `prefilter-hidden-${kind}`, + getKey: (item) => item.id, + initialData: [row], + }), + ) + try { + await collection.stateWhenReady() + const hidden = new Func('and', [ + new Func('eq', [new PropRef(['id']), new Value('hidden')]), + new Func('isUndefined', [new PropRef(['v'])]), + ]) + expect( + collection + .currentStateAsChanges({ where: hidden }) + ?.map((change) => change.key), + ).toEqual(['hidden']) + } finally { + await collection.cleanup() + } + }, +) diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index 53345f955..a3e7c0b5a 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -215,6 +215,13 @@ export function createIndexUsageTracker(collection: any): { recordFullScan() yield* originalEntries.call(this) } + // The unindexed snapshot scan reads stored rows through the state. + const state = collection._state + const originalEntriesPassing = state.entriesPassing + state.entriesPassing = function* (prefilter: (row: object) => boolean) { + recordFullScan() + yield* originalEntriesPassing.call(this, prefilter) + } const restore = () => { // Remove the instance getter so the prototype getter applies again @@ -224,6 +231,7 @@ export function createIndexUsageTracker(collection: any): { if (rangeQuery) index.rangeQuery = rangeQuery } collection.entries = originalEntries + state.entriesPassing = originalEntriesPassing } return { stats, restore } From 01a225c07ac396e3a9a935b83e5b8688b1755976 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 09:35:59 -0600 Subject: [PATCH 58/63] perf(db): pool eq-filtered live queries ordered by their own fields A query whose orderBy reads only its own row fields, without a custom string comparator, limit, or offset, now pools. Each order gets its own partition, whose groups sort rows with the compiler's comparator and break ties by key, so order matches a live-query Collection. The pooled identity includes the order. With 240 ordered cells, mount drops from about 16 ms to 2.6 ms and a 200-update batch from about 5 ms to 1.1 ms. The pooled oracle generates orders by id and by a field that can be null, with explicit nulls, and weights peers that order a whole group. Co-authored-by: Isaac --- .changeset/perf-pooled-live-queries.md | 4 +- docs/contributing/glossary.md | 2 +- docs/contributing/oracle-coverage.md | 2 +- packages/db/mangle-cache.json | 3 +- packages/db/src/query/live/ARCHITECTURE.md | 12 +- packages/db/src/query/pooled-live-query.ts | 88 +++++++++++-- .../pooled-live-query-oracle.property.test.ts | 121 +++++++++++++++--- .../tests/query/pooled-query-identity.test.ts | 17 +++ 8 files changed, 209 insertions(+), 40 deletions(-) diff --git a/.changeset/perf-pooled-live-queries.md b/.changeset/perf-pooled-live-queries.md index 8ef5b0635..d5e8d2f21 100644 --- a/.changeset/perf-pooled-live-queries.md +++ b/.changeset/perf-pooled-live-queries.md @@ -7,6 +7,6 @@ '@tanstack/angular-db': patch --- -Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection, filters it by at least one `eq(field, literal)`, and has no clause besides `where` is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Its other `where` conditions on the row, such as `not`, `gt`, or `like`, are evaluated per query over its group. This applies to a query function, a query builder, and a `{ query }` config that sets no other option besides `queryKey` or `gcTime`. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. +Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection, filters it by at least one `eq(field, literal)`, and has no clause besides `where` and an `orderBy` on its own fields is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Its other `where` conditions on the row, such as `not`, `gt`, or `like`, are evaluated per query over its group. This applies to a query function, a query builder, and a `{ query }` config that sets no other option besides `queryKey` or `gcTime`. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. -Results are unchanged: the same rows in key order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. +Results are unchanged: the same rows in the same order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. diff --git a/docs/contributing/glossary.md b/docs/contributing/glossary.md index 4d48703ff..c7012c5eb 100644 --- a/docs/contributing/glossary.md +++ b/docs/contributing/glossary.md @@ -19,7 +19,7 @@ production queues, caches, or semantic helpers merely to share their names. | Collection | The public keyed data container. Capitalize it when referring to the TanStack DB type. | Relation, table, or query result. | | source Collection | A Collection read by a query or adapter. | Source relation when the value is a public Collection. | | live-query Collection | A Collection whose rows are produced by a live query. | Query, observer, or result set. | -| pooled live query | A live query on one source Collection whose `where` has at least one `eq(field, literal)` conjunct and otherwise reads only the row, served from an equality partition; its live-query Collection is built only when read. | Cached query or shared live-query Collection. | +| pooled live query | A live query on one source Collection whose `where` has at least one `eq(field, literal)` conjunct and otherwise reads only the row, optionally ordered by the row's own fields without a limit, served from an equality partition; its live-query Collection is built only when read. | Cached query or shared live-query Collection. | | equality partition | Source rows grouped by the `eq`-normalized values of one set of fields, shared by every pooled live query that filters on those fields. | Index or bucket relation. | | partition group | The rows of an equality partition whose fields equal one tuple of literals. | Bucket or active bucket. | | relation | An internal weighted multiset maintained by D2. | Collection. | diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 74762817f..239125e97 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing was later removed in favor of pooled live queries; the routing mutants above are historical, and the same peer laws now check plain subscription dispatch. The unindexed snapshot prefilter was restored for compiled queries as one stored-row `eq` test on a string or boolean literal. Its property-visibility test also covers the reverse direction: a stored row with an inherited or non-enumerable field must still match `isUndefined` on that field, which a scan that evaluated the full predicate on stored rows fails. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index fd6171d92..8553c6a3a 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -377,5 +377,6 @@ "collectionHold": "gb", "holdCollection": "gc", "registry": "gd", - "deliverStatus": "ge" + "deliverStatus": "ge", + "compareRows": "gf" } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index 404737ff5..ac20f1981 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1249,15 +1249,17 @@ listener lifetime. A framework adapter may serve a live query from an equality partition instead of this graph. That applies only when the query reads one eager, non-persisted source Collection, its `where` has at least one -`eq(field, literal)` conjunct, every other conjunct reads only that row's -own fields, and it has no other clause, `DbClient`, or Suspense key. Such a +`eq(field, literal)` conjunct, every other conjunct reads only that row's own fields, any `orderBy` reads +only that row's own fields without a custom string comparator, and it has no +other clause, `limit`, `offset`, `DbClient`, or Suspense key. Such a pooled live query reads the partition group for its `eq` literal tuple and -filters it by its remaining conjuncts with the compiler's evaluator +filters it by its remaining conjuncts with the compiler's evaluator. Queries +with different orders use different partitions, whose groups sort rows with +the compiler's comparator and break ties by key (`packages/db/src/query/pooled-live-query.ts`). It builds its live-query Collection only when the application reads it. -A pooled live query must publish what its live-query Collection would: the -same rows, in key order, with the same values and status. When its source +A pooled live query must publish what its live-query Collection would: the same rows, in its order or else key order, with the same values and status. When its source starts cleanup, it enters the same terminal error and keeps its last rows. The pooled live query oracle compares the two over generated histories. diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 8ab791735..2afdebd99 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -1,13 +1,14 @@ import { SortedMap } from '../SortedMap.js' import { CleanupQueue } from '../collection/cleanup-queue.js' import { UNSUBSCRIBED_GC_FLOOR_MS } from '../collection/lifecycle.js' -import { normalizeValue } from '../utils/comparison.js' +import { makeComparator, normalizeValue } from '../utils/comparison.js' import { isVirtualPropName } from '../virtual-props.js' import { getPersistedReadinessSource } from '../persisted-readiness.js' import { getWhereExpression } from './ir.js' import { compileExpression, toBooleanPredicate } from './compiler/evaluators.js' +import { buildCompareOptions } from './compiler/order-by.js' import { createLiveQueryCollection } from './live-query-collection.js' -import type { BasicExpression, QueryIR } from './ir.js' +import type { BasicExpression, OrderBy, QueryIR } from './ir.js' import type { BaseQueryBuilder } from './builder/index.js' import type { Collection, CollectionImpl } from '../collection/index.js' import type { ChangeMessage, CollectionStatus } from '../types.js' @@ -81,6 +82,8 @@ class Partition { constructor( private readonly source: CollectionImpl, private readonly paths: Array>, + // Row order for an `orderBy` shape; key order otherwise. + private readonly compareRows: ((a: Row, b: Row) => number) | undefined, // The partition's entry in its source's map, which a mount looks up. private readonly registry: { add: () => void; remove: () => void }, ) {} @@ -107,7 +110,7 @@ class Partition { let group = this.groups.get(key) if (!group) { group = { - rows: new SortedMap(), + rows: new SortedMap(this.compareRows), listeners: new Set(), revision: 0, layoutRevision: 0, @@ -287,6 +290,7 @@ type PoolableShape = { groupKey: string // Conjuncts each view evaluates over its group's rows. residual: Array + orderBy: OrderBy | undefined } // Whether an expression reads only this query's own row fields, so a view @@ -374,7 +378,6 @@ function poolableShape(query: QueryIR): PoolableShape | undefined { query.join || query.groupBy || query.having || - query.orderBy || query.limit !== undefined || query.offset !== undefined || query.distinct || @@ -402,6 +405,11 @@ function poolableShape(query: QueryIR): PoolableShape | undefined { } // A partition needs at least one equality to group by. if (conjuncts.length === 0) return undefined + const orderBy = query.orderBy?.length ? query.orderBy : undefined + const orderKey = orderBy + ? orderByKey(orderBy, query.from.alias, query.from.collection) + : `` + if (orderKey === undefined) return undefined // Most shapes have one or two fields; a general sort costs more than both. if (conjuncts.length === 2) { if (conjuncts[1]!.pathKey < conjuncts[0]!.pathKey) conjuncts.reverse() @@ -417,7 +425,58 @@ function poolableShape(query: QueryIR): PoolableShape | undefined { shapeKey += pathKey groupKey = appendGroupKeyPart(groupKey, literalKey) } - return { paths, shapeKey, groupKey, residual } + // Groups of one shape share a row order, so the order is part of it. + if (orderKey) shapeKey += `|${orderKey}` + return { paths, shapeKey, groupKey, residual, orderBy } +} + +// A key for an `orderBy` over this query's own row fields, or undefined for +// one a partition cannot share by value, such as a custom string comparator. +function orderByKey( + orderBy: OrderBy, + alias: string, + source: CollectionImpl, +): string | undefined { + let key = `` + for (const clause of orderBy) { + const { expression } = clause + if ( + expression.type !== `ref` || + !readsOnlyRow(expression, alias) || + expression.path.length < 2 + ) { + return undefined + } + const options = buildCompareOptions(clause, source) + if (options.stringSort === `custom`) return undefined + key += JSON.stringify([expression.path.slice(1), options]) + } + return key +} + +// Orders rows as a live-query Collection's `orderBy` does; the group's +// SortedMap breaks ties by key. +function rowComparator( + orderBy: OrderBy, + alias: string, + source: CollectionImpl, +): (a: Row, b: Row) => number { + const terms = orderBy.map((clause) => ({ + read: compileExpression(clause.expression), + compare: makeComparator(buildCompareOptions(clause, source)), + })) + // One namespaced row, reused so a comparison allocates nothing. + const namespaced: Record = {} + return (a, b) => { + for (const { read, compare } of terms) { + namespaced[alias] = a + const left = read(namespaced as any) + namespaced[alias] = b + const result = compare(left, read(namespaced as any)) + if (result !== 0) return result + } + return 0 + } } /** @@ -633,14 +692,19 @@ export function createPooledLiveQuery( const owner = partitions // A released partition may subscribe again; it must not then replace or // remove a newer partition created under its key. - const created: Partition = new Partition(source, shape.paths, { - add: () => { - if (!owner.has(shapeKey)) owner.set(shapeKey, created) + const created: Partition = new Partition( + source, + shape.paths, + shape.orderBy && rowComparator(shape.orderBy, ir.from.alias, source), + { + add: () => { + if (!owner.has(shapeKey)) owner.set(shapeKey, created) + }, + remove: () => { + if (owner.get(shapeKey) === created) owner.delete(shapeKey) + }, }, - remove: () => { - if (owner.get(shapeKey) === created) owner.delete(shapeKey) - }, - }) + ) partition = created partitions.set(shapeKey, partition) } diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts index e298feffd..694f1ab8a 100644 --- a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -4,12 +4,11 @@ * Law and source: a live query that filters one source Collection by * `eq(field, literal)` conjuncts, plus any conjuncts that read only its row, * is served from a partition of that source shared by every query with the - * same `eq` fields; each view evaluates its other conjuncts itself. Its observer must publish the + * same `eq` fields and order; each view evaluates its other conjuncts itself. Its observer must publish the * rows the live-query Collection for the same query publishes: the visible * source rows whose fields equal the literals under `eq` semantics * (`src/query/compiler/evaluators.ts`: nullish is UNKNOWN, a Date equals its - * timestamp, `NaN` equals `NaN`, `-0` equals `0`, other types differ), in - * key order, with the same row values and status. + * timestamp, `NaN` equals `NaN`, `-0` equals `0`, other types differ), in its order or else key order, with the same row values and status. * * Why an example can miss the failure: one query over static rows passes even * if rows never move between groups, a peer group never sees a row leave, a @@ -24,8 +23,9 @@ * History grammar: rows have ids 0 through 3, delivered initially in key * order or in reverse, a field `f` from strings, * numbers and their look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, - * `0`, `null`, and a missing value, and a field `g` of `x` or `y`. Up to three peer queries use `eq(f, literal)`, optionally with - * `eq(g, literal)` and a residual `not(eq(g, literal))`. + * `0`, `null`, and a missing value, and a field `g` of `x`, `y`, or `null`. Up to three peer queries use `eq(f, literal)`, optionally with + * `eq(g, literal)`, a residual `not(eq(g, literal))`, and an order by `id` or + * by `g` with explicit `nulls`; a weighted peer orders a whole `f` group. * Values are weighted toward `a`, and toward the normalized values against * numeric literals, so groups hold rows that stay, move, and normalize. * Steps commit sync transactions of one or two inserts, updates, or deletes, @@ -96,13 +96,23 @@ const requestedReplayProperty = readOracleRunConfig().replayProperty const MISSING = Symbol(`missing`) type FieldValue = string | number | boolean | Date | null | typeof MISSING -type Row = { id: string; f?: unknown; g: string } +type Row = { id: string; f?: unknown; g: string | null } type Peer = { f: string | number | boolean g?: string // A residual conjunct, `not(eq(r.g, notG))`, each view evaluates itself. notG?: string + // Orders a peer's rows; the reference gives the expected order. A group's + // rows share `f`, so orders read `g`, which may be null, and the id. + order?: PeerOrder } +type PeerOrder = + | `id-desc` + | `g-desc` + | `g-asc-id-desc` + | `g-asc-nulls-first` + | `g-asc-nulls-last` + | `g-desc-nulls-last` type Step = | { kind: `sync`; ops: Array } | { @@ -117,12 +127,18 @@ type Step = // `mount` (re)mounts that peer after cleanup, before the restart. | { kind: `cleanup-restart`; mount?: number } type Op = - | { type: `insert`; id: number; f: FieldValue; g: string } + | { type: `insert`; id: number; f: FieldValue; g: string | null } // `keepF` keeps the row's current `f`, so the row stays in its group. - | { type: `update`; id: number; f: FieldValue; g: string; keepF?: boolean } + | { + type: `update` + id: number + f: FieldValue + g: string | null + keepF?: boolean + } | { type: `delete`; id: number } type History = { - rows: Array<{ f: FieldValue; g: string }> + rows: Array<{ f: FieldValue; g: string | null }> // The source delivers its initial rows in reverse key order. reverseInitial?: boolean peers: Array @@ -169,8 +185,8 @@ function expectedKeys( (row) => modelEq(row.f, peer.f) && (peer.g === undefined || row.g === peer.g) && - // `g` is never nullish here, so the negated equality is two-valued. - (peer.notG === undefined || row.g !== peer.notG), + // A null `g` makes the negated equality UNKNOWN, which excludes. + (peer.notG === undefined || (row.g !== null && row.g !== peer.notG)), ) .map((row) => row.id) .sort() @@ -189,7 +205,7 @@ function sameValueZero(a: unknown, b: unknown): boolean { return a === b || (Number.isNaN(a) && Number.isNaN(b)) } -function sourceRow(id: string, f: FieldValue, g: string): Row { +function sourceRow(id: string, f: FieldValue, g: string | null): Row { return f === MISSING ? { id, g } : { id, f, g } } @@ -205,12 +221,14 @@ const fieldArbitrary = fc.oneof( fc.constantFrom(...fieldValues), ) const gArbitrary = fc.constantFrom(`x`, `y`) +// Rows may hold a null `g`, which orders by its `nulls` option. +const rowGArbitrary = fc.constantFrom(`x`, `y`, null) const opArbitrary: fc.Arbitrary = fc.oneof( fc.record({ type: fc.constant(`insert` as const), id: fc.nat({ max: 3 }), f: fieldArbitrary, - g: gArbitrary, + g: rowGArbitrary, }), { weight: 2, @@ -219,7 +237,7 @@ const opArbitrary: fc.Arbitrary = fc.oneof( type: fc.constant(`update` as const), id: fc.nat({ max: 3 }), f: fieldArbitrary, - g: gArbitrary, + g: rowGArbitrary, keepF: fc.boolean(), }, { requiredKeys: [`type`, `id`, `f`, `g`] }, @@ -227,6 +245,14 @@ const opArbitrary: fc.Arbitrary = fc.oneof( }, fc.record({ type: fc.constant(`delete` as const), id: fc.nat({ max: 3 }) }), ) +const orderArbitrary = fc.constantFrom( + `id-desc`, + `g-desc`, + `g-asc-id-desc`, + `g-asc-nulls-first`, + `g-asc-nulls-last`, + `g-desc-nulls-last`, +) const peerArbitrary: fc.Arbitrary = fc.record( { f: fc.oneof( @@ -239,15 +265,24 @@ const peerArbitrary: fc.Arbitrary = fc.record( ), g: gArbitrary, notG: gArbitrary, + order: orderArbitrary, }, { requiredKeys: [`f`] }, ) const historyArbitrary: fc.Arbitrary = fc.record({ - rows: fc.array(fc.record({ f: fieldArbitrary, g: gArbitrary }), { + rows: fc.array(fc.record({ f: fieldArbitrary, g: rowGArbitrary }), { maxLength: 4, }), reverseInitial: fc.boolean(), - peers: fc.array(peerArbitrary, { minLength: 1, maxLength: 3 }), + peers: fc.array( + fc.oneof( + { weight: 2, arbitrary: peerArbitrary }, + // An ordered peer over a whole group, so rows with several `g` + // values, including null, share one ordered view. + fc.record({ f: fc.constant(`a`), order: orderArbitrary }), + ), + { minLength: 1, maxLength: 3 }, + ), steps: fc.array( fc.oneof( { @@ -376,6 +411,28 @@ const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ ], }, }, + { + name: `an update reorders rows within one ordered group`, + history: { + rows: [ + { f: `a`, g: `x` }, + { f: `a`, g: `y` }, + ], + peers: [ + { f: `a`, order: `g-desc` }, + { f: `a`, order: `g-asc-nulls-last` }, + ], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `a`, g: `y` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 1, f: `a`, g: null }] }, + { + kind: `optimistic`, + op: { type: `update`, id: 0, f: `a`, g: null }, + confirm: false, + }, + ], + }, + }, { name: `a residual conjunct moves rows in and out of a view within one group`, history: { @@ -440,14 +497,42 @@ let serial = 0 const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) function peerQuery(source: any, peer: Peer) { - return (q: any) => - q.from({ r: source }).where(({ r }: any) => { + return (q: any) => { + const query = q.from({ r: source }).where(({ r }: any) => { const conjuncts = [eq(r.f, peer.f)] if (peer.g !== undefined) conjuncts.push(eq(r.g, peer.g)) if (peer.notG !== undefined) conjuncts.push(not(eq(r.g, peer.notG))) const [first, second, ...rest] = conjuncts return second ? and(first, second, ...rest) : first }) + switch (peer.order) { + case undefined: + return query + case `id-desc`: + return query.orderBy(({ r }: any) => r.id, `desc`) + case `g-desc`: + return query.orderBy(({ r }: any) => r.g, `desc`) + case `g-asc-id-desc`: + return query + .orderBy(({ r }: any) => r.g) + .orderBy(({ r }: any) => r.id, `desc`) + case `g-asc-nulls-first`: + return query.orderBy(({ r }: any) => r.g, { + direction: `asc`, + nulls: `first`, + }) + case `g-asc-nulls-last`: + return query.orderBy(({ r }: any) => r.g, { + direction: `asc`, + nulls: `last`, + }) + case `g-desc-nulls-last`: + return query.orderBy(({ r }: any) => r.g, { + direction: `desc`, + nulls: `last`, + }) + } + } } // A row's id and fields, which the model also knows. diff --git a/packages/db/tests/query/pooled-query-identity.test.ts b/packages/db/tests/query/pooled-query-identity.test.ts index 0f05ebe89..aab60ccf6 100644 --- a/packages/db/tests/query/pooled-query-identity.test.ts +++ b/packages/db/tests/query/pooled-query-identity.test.ts @@ -85,6 +85,23 @@ describe(`pooled query identity`, () => { ).not.toEqual(base) }) + it(`separates orders`, () => { + const ordered = (direction: `asc` | `desc`) => + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => eq(r.a, `x`)) + .orderBy(({ r }: any) => r.n, direction), + ) + expect(ordered(`asc`)).toEqual(ordered(`asc`)) + expect(ordered(`asc`)).not.toEqual(ordered(`desc`)) + expect(ordered(`asc`)).not.toEqual( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ), + ) + }) + it(`keeps the structural identity for queries a partition cannot serve`, () => { const residual = identity((q) => q From be8fa8a580cb705910042cd51490ac00f8c24bb5 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 09:42:01 -0600 Subject: [PATCH 59/63] test(db): type the gc test's query helper for any Collection The cleanup-start witness builds its own Collection, which vitest's type check rejected against the mock-source helper type. Co-authored-by: Isaac --- packages/db/tests/query/pooled-live-query-gc.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index fdb0c842f..784e53b5b 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -6,6 +6,7 @@ import { createLiveQueryCollection, eq, not } from '../../src/query/index.js' import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' import { resolveLiveQueryValue } from '../../src/live-query-options.js' import { mockSyncCollectionOptions } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' /** * # Does a pooled live query hold its source as long as its Collection would? @@ -37,7 +38,7 @@ function makeSource() { ) } -const query = (source: ReturnType) => (q: any) => +const query = (source: Collection) => (q: any) => q.from({ r: source }).where(({ r }: any) => eq(r.g, `x`)) // Milliseconds after which the source no longer has a subscriber, checking From 0f6bcc6e085d6626b3b361b03ccc81b3d01d5191 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 10:51:40 -0600 Subject: [PATCH 60/63] refactor(db): share eq conjunct parsing between pooling and the prefilter The pooled partition and the snapshot prefilter each parsed eq(field, literal), normalized values, and read row paths. Both now use one module. The prefilter compares normalized keys, so it also covers number and Date literals, not only strings and booleans. Co-authored-by: Isaac --- packages/db/src/collection/change-events.ts | 50 +++++++------------- packages/db/src/query/equality-conjunct.ts | 50 ++++++++++++++++++++ packages/db/src/query/pooled-live-query.ts | 51 ++++----------------- 3 files changed, 75 insertions(+), 76 deletions(-) create mode 100644 packages/db/src/query/equality-conjunct.ts diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index c48f64971..f74a7d971 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -8,7 +8,11 @@ import { } from '../utils/index-optimization.js' import { ensureIndexForField } from '../indexes/auto-index.js' import { getPropRefPropertyPath } from '../query/ir.js' -import { isVirtualPropName } from '../virtual-props.js' +import { + equalityConjunct, + equalityKey, + readPath, +} from '../query/equality-conjunct.js' import { makeComparator } from '../utils/comparison.js' import { buildCompareOptions } from '../query/compiler/order-by' import type { @@ -32,43 +36,23 @@ export type StoredRowScan = ( /** * A test on a stored row that is false only when `expression` must be false * on the row's enriched copy, so a scan can skip enriching rows that fail. - * - * It reads one `eq(field, literal)` conjunct with a string or boolean - * literal. The copy holds each enumerable own root field of the stored row - * and lacks the others, so its field is the stored value or `undefined`, and - * equality normalization maps no other value onto a plain string or boolean. - * A read that throws passes the row to the full predicate. + * It reads one `eq(field, literal)` conjunct. The copy holds each enumerable + * own root field of the stored row and lacks the others, so its field is the + * stored value or `undefined`, which matches no literal. */ export function compileStoredRowPrefilter( expression: BasicExpression, ): ((row: object) => boolean) | undefined { - const conjuncts: Array = [] - const collect = (node: BasicExpression) => { - if (node.type === `func` && node.name === `and`) node.args.forEach(collect) - else conjuncts.push(node) - } - collect(expression) + const conjuncts = + expression.type === `func` && expression.name === `and` + ? expression.args + : [expression] for (const conjunct of conjuncts) { - if (conjunct.type !== `func` || conjunct.name !== `eq`) continue - const [left, right] = conjunct.args - const ref = left?.type === `ref` ? left : right - const literal = left?.type === `val` ? left : right - if (ref?.type !== `ref` || literal?.type !== `val`) continue - const expected: unknown = literal.value - if (typeof expected !== `string` && typeof expected !== `boolean`) continue - const path = getPropRefPropertyPath(ref) - if (path.length === 0 || isVirtualPropName(path[0]!)) continue - return (row) => { - try { - let value: unknown = row - for (const segment of path) { - if (value === null || value === undefined) return false - value = (value as Record)[segment] - } - return value === expected - } catch { - return true - } + const eq = equalityConjunct(conjunct, getPropRefPropertyPath) + if (eq) { + return (row) => + equalityKey(readPath(row as Record, eq.path)) === + eq.literalKey } } return undefined diff --git a/packages/db/src/query/equality-conjunct.ts b/packages/db/src/query/equality-conjunct.ts new file mode 100644 index 000000000..a0765a8f2 --- /dev/null +++ b/packages/db/src/query/equality-conjunct.ts @@ -0,0 +1,50 @@ +import { normalizeValue } from '../utils/comparison.js' +import { isVirtualPropName } from '../virtual-props.js' +import type { BasicExpression, PropRef } from './ir.js' + +type Row = Record + +/** An `eq(field, literal)` conjunct on a row's own field. */ +export type EqualityConjunct = { path: Array; literalKey: string } + +// Typed so that 1, '1', and true stay distinct. +export function equalityKey(value: unknown): string | undefined { + const normalized = normalizeValue(value) + const type = typeof normalized + return type === `string` || type === `number` || type === `boolean` + ? `${type}:${String(normalized)}` + : undefined +} + +export function readPath(row: Row, path: Array): unknown { + try { + let value: unknown = row + for (const segment of path) value = (value as Row | undefined)?.[segment] + return value + } catch { + // The full predicate treats a throwing read as false. + return undefined + } +} + +/** + * The `eq(field, literal)` conjunct `expression` is, with the field's path in + * the row as `rowPath` gives it, or undefined for any other expression. + */ +export function equalityConjunct( + expression: BasicExpression, + rowPath: (ref: PropRef) => Array | undefined, +): EqualityConjunct | undefined { + if (expression.type !== `func` || expression.name !== `eq`) return undefined + const args = expression.args + if (args.length !== 2) return undefined + const left = args[0]! + const right = args[1]! + const ref = left.type === `ref` ? left : right + const literal = left.type === `val` ? left : right + if (ref.type !== `ref` || literal.type !== `val`) return undefined + const path = rowPath(ref) + if (!path?.length || isVirtualPropName(path[0]!)) return undefined + const literalKey = equalityKey(literal.value) + return literalKey === undefined ? undefined : { path, literalKey } +} diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index 2afdebd99..c59dd96ca 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -1,10 +1,11 @@ import { SortedMap } from '../SortedMap.js' import { CleanupQueue } from '../collection/cleanup-queue.js' import { UNSUBSCRIBED_GC_FLOOR_MS } from '../collection/lifecycle.js' -import { makeComparator, normalizeValue } from '../utils/comparison.js' +import { makeComparator } from '../utils/comparison.js' import { isVirtualPropName } from '../virtual-props.js' import { getPersistedReadinessSource } from '../persisted-readiness.js' import { getWhereExpression } from './ir.js' +import { equalityConjunct, equalityKey, readPath } from './equality-conjunct.js' import { compileExpression, toBooleanPredicate } from './compiler/evaluators.js' import { buildCompareOptions } from './compiler/order-by.js' import { createLiveQueryCollection } from './live-query-collection.js' @@ -43,31 +44,11 @@ interface PartitionGroup { const partitionsBySource = new WeakMap>() -// Typed so that 1, '1', and true stay distinct. -function equalityKey(value: unknown): string | undefined { - const normalized = normalizeValue(value) - const type = typeof normalized - return type === `string` || type === `number` || type === `boolean` - ? `${type}:${String(normalized)}` - : undefined -} - // Length-prefixed, so no two part lists share an encoding. function appendGroupKeyPart(groupKey: string, part: string): string { return `${groupKey}${part.length}:${part}` } -function readPath(row: Row, path: Array): unknown { - try { - let value: unknown = row - for (const segment of path) value = (value as Row | undefined)?.[segment] - return value - } catch { - // The full predicate treats a throwing read as false. - return undefined - } -} - class Partition { private readonly groups = new Map() private subscription: { unsubscribe: () => void } | undefined @@ -318,37 +299,21 @@ function collectConjuncts( } return true } - const conjunct = equalityConjunct(expression, alias) + const conjunct = equalityConjunctOf(expression, alias) if (conjunct) out.push(conjunct) else if (readsOnlyRow(expression, alias)) residual.push(expression) else return false return true } -function equalityConjunct( +function equalityConjunctOf( expression: BasicExpression, alias: string, ): Conjunct | undefined { - if (expression.type !== `func` || expression.name !== `eq`) return undefined - const args = expression.args - if (args.length !== 2) return undefined - const left = args[0]! - const right = args[1]! - const ref = left.type === `ref` ? left : right - const literal = left.type === `val` ? left : right - if (ref.type !== `ref` || literal.type !== `val`) return undefined - const refPath = ref.path - if ( - refPath[0] !== alias || - refPath.length < 2 || - isVirtualPropName(refPath[1]!) - ) { - return undefined - } - const literalKey = equalityKey(literal.value) - if (literalKey === undefined) return undefined - const path = refPath.slice(1) - return { path, pathKey: JSON.stringify(path), literalKey } + const conjunct = equalityConjunct(expression, (ref) => + ref.path[0] === alias ? ref.path.slice(1) : undefined, + ) + return conjunct && { ...conjunct, pathKey: JSON.stringify(conjunct.path) } } // Whether a row passes every residual conjunct, as a WHERE filter decides. From 699fe3a7d95e556702282e9a246ab300b3a0612e Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 11:26:16 -0600 Subject: [PATCH 61/63] fix(db): default pooled queries to the live-query Collection gcTime Pooled queries defaulted to the plain Collection gcTime of 5 minutes, while the live-query Collections they stand in for default to 5 seconds. Adapters that pass no gcTime (Solid, Vue, Svelte, and query-only configs) kept an unmounted query's source subscription for 5 minutes. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- packages/db/src/query/pooled-live-query.ts | 4 +-- .../tests/query/pooled-live-query-gc.test.ts | 25 +++++++++++++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index a3459b23c..360e6e763 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing was later removed in favor of pooled live queries; the routing mutants above are historical, and the same peer laws now check plain subscription dispatch. The unindexed snapshot prefilter was restored for compiled queries as one stored-row `eq` test on a string or boolean literal. Its property-visibility test also covers the reverse direction: a stored row with an inherited or non-enumerable field must still match `isUndefined` on that field, which a scan that evaluated the full predicate on stored rows fails. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and with no `gcTime`, where both use the live-query Collection default, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index c59dd96ca..bc3e26888 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -632,8 +632,8 @@ export function getPooledQueryIdentity( */ export function createPooledLiveQuery( query: BaseQueryBuilder, - // A Collection's default when the adapter gives none. - { gcTime = 300_000 }: { gcTime?: number } = {}, + // A live-query Collection's default when the adapter gives none. + { gcTime = 5_000 }: { gcTime?: number } = {}, ): Collection | undefined { const ir = query._getQuery() const shape = poolableShape(ir) diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index 784e53b5b..dbcfa3664 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -245,6 +245,31 @@ describe(`pooled live query gcTime`, () => { compiledView.stop() }) + it(`defaults to a live-query Collection's gcTime when the adapter gives none`, async () => { + // A pooled view and a compiled one, each on its own source. + const pooledOnOwn = makeSource() + const pooledObserver = createLiveQueryObserver( + resolveLiveQueryValue(query(pooledOnOwn)(new Query())), + { mode: `wholesale` }, + ) + pooledObserver.subscribe(() => {})() + const compiledOnOwn = makeSource() + const compiledObserver = createLiveQueryObserver( + createLiveQueryCollection({ + query: query(compiledOnOwn), + startSync: true, + }), + { mode: `wholesale` }, + ) + compiledObserver.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(4_900) + expect(compiledOnOwn.subscriberCount).toBe(1) + expect(pooledOnOwn.subscriberCount).toBe(1) + await vi.advanceTimersByTimeAsync(300) + expect(compiledOnOwn.subscriberCount).toBe(0) + expect(pooledOnOwn.subscriberCount).toBe(0) + }) + it(`pools a query config that names only its query`, () => { const source = makeSource() const observe = (value: unknown) => From 9e4e971e4289770b6b87f36cba840830e14161ac Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 11:50:34 -0600 Subject: [PATCH 62/63] test(query-db-collection): restore the settled update row assertions Reverts d3543451b. The visible-row assertions describe the intended settlement law; the flake they expose is a core race that main shares and that a separate fix addresses. Co-authored-by: Isaac --- packages/query-db-collection/tests/query.test.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/query-db-collection/tests/query.test.ts b/packages/query-db-collection/tests/query.test.ts index 728353704..fb4f05c5f 100644 --- a/packages/query-db-collection/tests/query.test.ts +++ b/packages/query-db-collection/tests/query.test.ts @@ -4805,10 +4805,11 @@ describe(`QueryCollection`, () => { }) await first.isPersisted.promise expect(adapter.rows.get(`p`)?.revision).toBe(2) - // Settlement guarantees the stored server response, not that the - // accepted optimistic snapshot has stopped overlaying the row: a - // later source acknowledgement retires it. - expect(collection.base.get(`p`)).toMatchObject({ a: 10, revision: 2 }) + expect(collection.get(`p`)).toMatchObject({ + a: 10, + revision: 2, + $synced: true, + }) const second = collection.update(`p`, (draft) => { draft.b = 1 @@ -4816,10 +4817,11 @@ describe(`QueryCollection`, () => { await second.isPersisted.promise expect(requestRevisions).toEqual([1, 2]) expect(adapter.rows.get(`p`)?.revision).toBe(3) - expect(collection.base.get(`p`)).toMatchObject({ + expect(collection.get(`p`)).toMatchObject({ a: 10, b: 1, revision: 3, + $synced: true, }) } finally { await collection.cleanup() From d9f25b275d9355b10eb99d4520838facc0eda0c6 Mon Sep 17 00:00:00 2001 From: Isaac Date: Fri, 2 Oct 2026 11:55:12 -0600 Subject: [PATCH 63/63] fix(db): drop pooled partition groups with no rows and no watchers A partition kept a group for every eq value any source row ever held, so a churning high-cardinality source grew without bound while the partition stayed subscribed. Views now read their group by key, a group with no rows and no listeners is dropped, and every group draws its revisions from one partition clock, so a recreated group cannot repeat a revision a detached reader cached. Co-authored-by: Isaac --- docs/contributing/oracle-coverage.md | 2 +- packages/db/mangle-cache.json | 4 +- packages/db/src/query/pooled-live-query.ts | 60 +++++++++++---- .../tests/query/pooled-live-query-gc.test.ts | 77 +++++++++++++++++++ 4 files changed, 125 insertions(+), 18 deletions(-) diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index 360e6e763..ff122d796 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -267,7 +267,7 @@ comment and the current API/architecture contract before extending its model. | WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing was later removed in favor of pooled live queries; the routing mutants above are historical, and the same peer laws now check plain subscription dispatch. The unindexed snapshot prefilter was restored for compiled queries as one stored-row `eq` test on a string or boolean literal. Its property-visibility test also covers the reverse direction: a stored row with an inherited or non-enumerable field must still match `isUndefined` on that field, which a scan that evaluated the full predicate on stored rows fails. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | -| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and with no `gcTime`, where both use the live-query Collection default, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and with no `gcTime`, where both use the live-query Collection default, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses check that a partition keeps groups only for values that have rows or watchers, that a detached reader never sees a dropped group's revision reused, keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | | Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | | Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index af86175d9..96388dbec 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -380,5 +380,7 @@ "passes": "ge", "registry": "gf", "compareRows": "gg", - "sameFields": "gh" + "sameFields": "gh", + "dropIfUnused": "gi", + "clock": "gj" } diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts index bc3e26888..6ad3a4331 100644 --- a/packages/db/src/query/pooled-live-query.ts +++ b/packages/db/src/query/pooled-live-query.ts @@ -35,6 +35,7 @@ type Listener = (changes: Array>) => void type StatusListener = CollectionEventHandler<`status:change`> interface PartitionGroup { + key: string // Key order, as in a live-query Collection without orderBy. rows: SortedMap listeners: Set @@ -44,6 +45,16 @@ interface PartitionGroup { const partitionsBySource = new WeakMap>() +// What a view reads for a value with no rows and no watchers. Real groups +// draw revisions from their partition's clock, which starts above 0. +const EMPTY_GROUP: PartitionGroup = { + key: ``, + rows: new SortedMap(), + listeners: new Set(), + revision: 0, + layoutRevision: 0, +} + // Length-prefixed, so no two part lists share an encoding. function appendGroupKeyPart(groupKey: string, part: string): string { return `${groupKey}${part.length}:${part}` @@ -51,6 +62,8 @@ function appendGroupKeyPart(groupKey: string, part: string): string { class Partition { private readonly groups = new Map() + // Revisions for every group, so a recreated group never repeats one. + private clock = 0 private subscription: { unsubscribe: () => void } | undefined private stopStatusEvents: (() => void) | undefined // One source status listener serves every view of this partition. @@ -90,17 +103,31 @@ class Partition { group(key: string): PartitionGroup { let group = this.groups.get(key) if (!group) { + const revision = ++this.clock group = { + key, rows: new SortedMap(this.compareRows), listeners: new Set(), - revision: 0, - layoutRevision: 0, + revision, + layoutRevision: revision, } this.groups.set(key, group) } return group } + /** A group for reading, without creating one. */ + peek(key: string): PartitionGroup { + return this.groups.get(key) ?? EMPTY_GROUP + } + + // A group with no rows and no watchers holds nothing anyone can read. + private dropIfUnused(group: PartitionGroup): void { + if (group.rows.size === 0 && group.listeners.size === 0) { + this.groups.delete(group.key) + } + } + /** * Keep the shared source subscription open. Each view brings its query's * `gcTime`; the partition keeps the longest, so it never releases before @@ -171,6 +198,7 @@ class Partition { removeListener(group: PartitionGroup, listener: Listener): void { if (!group.listeners.delete(listener)) return this.listenerCount-- + this.dropIfUnused(group) this.scheduleRelease(0) } @@ -198,13 +226,8 @@ class Partition { if (this.listenerCount > 0) return this.stopStatusEvents?.() this.stopStatusEvents = undefined - // Views outlive a release and may subscribe again, so they keep their - // groups for the next subscription to refill. - for (const group of this.groups.values()) { - group.rows.clear() - group.revision++ - group.layoutRevision++ - } + // Views read groups by key, so a later subscription refills new ones. + this.groups.clear() this.release() } @@ -220,7 +243,7 @@ class Partition { const list = touched.get(group) if (list) list.push(change) else touched.set(group, [change]) - if (change.type !== `update`) group.layoutRevision++ + if (change.type !== `update`) group.layoutRevision = ++this.clock } for (const change of changes) { const next = @@ -257,8 +280,9 @@ class Partition { } } for (const [group, groupChanges] of touched) { - group.revision++ + group.revision = ++this.clock for (const listener of [...group.listeners]) listener(groupChanges) + this.dropIfUnused(group) } } } @@ -454,7 +478,6 @@ class PooledLiveQuery { // No persisted readiness, single-result config, or layout channel. readonly config = undefined readonly _subscribeLayoutChanges = undefined - private readonly group: PartitionGroup private collection: Collection | undefined = undefined private listenerCount = 0 private collectionHold: { unsubscribe: () => void } | undefined = undefined @@ -463,12 +486,11 @@ class PooledLiveQuery { private readonly source: CollectionImpl, private readonly query: BaseQueryBuilder, private readonly partition: Partition, - groupKey: string, + private readonly groupKey: string, private readonly gcTime: number, // The query's conjuncts beyond its group's equalities, if any. private readonly passes: ((row: Row) => boolean) | undefined, ) { - this.group = partition.group(groupKey) partition.retain(gcTime) } @@ -476,6 +498,11 @@ class PooledLiveQuery { return this.partition.terminated ? `error` : this.source.status } + // Read by key: the partition drops a group nobody watches once it empties. + private get group(): PartitionGroup { + return this.partition.peek(this.groupKey) + } + get _stateRevision(): number { return this.group.revision } @@ -497,7 +524,8 @@ class PooledLiveQuery { // A released partition refills its group here, so seed the filter after. this.partition.subscribe() const listener = this.passes ? this.filterChanges(callback) : callback - this.partition.addListener(this.group, listener) + const group = this.partition.group(this.groupKey) + this.partition.addListener(group, listener) this.listenerCount++ this.holdCollection() if (options.includeInitialState) { @@ -514,7 +542,7 @@ class PooledLiveQuery { unsubscribe: () => { if (!subscribed) return subscribed = false - this.partition.removeListener(this.group, listener) + this.partition.removeListener(group, listener) if (--this.listenerCount === 0) { this.collectionHold?.unsubscribe() this.collectionHold = undefined diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts index dbcfa3664..89e3e223d 100644 --- a/packages/db/tests/query/pooled-live-query-gc.test.ts +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -287,6 +287,83 @@ describe(`pooled live query gcTime`, () => { for (const stop of stops) stop() }) + it(`keeps groups only for values with rows or watchers`, async () => { + const source = makeSource() + source.utils.begin() + for (let n = 0; n < 50; n++) { + source.utils.write({ type: `insert`, value: { id: `r${n}`, g: `v${n}` } }) + } + source.utils.commit() + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + const stop = observer.subscribe(() => {}) + const groups = () => + (view as unknown as { partition: { groups: Map } }) + .partition.groups.size + // One group per distinct value, plus the watched `x`. + expect(groups()).toBe(51) + source.utils.begin() + for (let n = 0; n < 50; n++) { + source.utils.write({ type: `delete`, value: { id: `r${n}`, g: `v${n}` } }) + } + source.utils.commit() + expect(groups()).toBe(1) + // The watched group empties and is dropped; new rows recreate it, and + // the observer still sees them. + source.utils.begin() + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.commit() + expect(groups()).toBe(1) + expect(observer.getSnapshot().data).toEqual([]) + stop() + expect(groups()).toBe(0) + const again = observer.subscribe(() => {}) + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `b`, g: `x` } }) + source.utils.commit() + expect([...observer.getSnapshot().state!.keys()]).toEqual([`b`]) + again() + }) + + it(`never reuses a dropped group's revision for a detached reader`, () => { + const source = makeSource() + const write = (type: `insert` | `delete`, id: string, g: string) => { + source.utils.begin() + source.utils.write({ type, value: { id, g } }) + source.utils.commit() + } + // A watched `y` view keeps the partition subscribed. + write(`insert`, `z`, `y`) + const keepAlive = createLiveQueryObserver( + createPooledLiveQuery( + new Query() + .from({ r: source }) + .where(({ r }: any) => eq(r.g, `y`)) as any, + { gcTime: 1 }, + ), + { mode: `wholesale` }, + ).subscribe(() => {}) + // The `x` view is read without subscribing, so it compares revisions. + const reader = createLiveQueryObserver( + createPooledLiveQuery(query(source)(new Query()), { gcTime: 1 }), + { mode: `wholesale` }, + ) + write(`insert`, `b`, `x`) + const keys = () => [...reader.getSnapshot().state!.keys()] + expect(keys()).toEqual([`a`, `b`]) + // Emptying the unwatched group drops it; new rows build a new one. + source.utils.begin() + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.write({ type: `delete`, value: { id: `b`, g: `x` } }) + source.utils.commit() + write(`insert`, `c`, `x`) + write(`insert`, `d`, `x`) + expect(keys()).toEqual([`c`, `d`]) + keepAlive() + }) + it(`keeps its public Collection live while observed`, async () => { const source = makeSource() const view = createPooledLiveQuery(query(source)(new Query()), {