diff --git a/.changeset/cheaper-mutations.md b/.changeset/cheaper-mutations.md new file mode 100644 index 0000000000..daaa395c66 --- /dev/null +++ b/.changeset/cheaper-mutations.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Make updates and query building cheaper. Updates to rows whose fields are all primitives track changes without a proxy, drafts of other rows allocate less, sorted Collections no longer re-sort when an existing row changes value, published rows reuse one cached copy per key, equality checks on flat rows allocate nothing, and query building copies less. Mutation ids are now a random per-runtime prefix plus a counter instead of a random UUID per mutation; they stay unique across tabs and sessions, but are no longer bare UUIDs. diff --git a/.changeset/fix-browser-development-checks.md b/.changeset/fix-browser-development-checks.md new file mode 100644 index 0000000000..89c15fe0ea --- /dev/null +++ b/.changeset/fix-browser-development-checks.md @@ -0,0 +1,6 @@ +--- +'@tanstack/db': patch +'@tanstack/react-db': patch +--- + +Run development-only checks in browser development builds. The duplicate `@tanstack/db` instance check and React's development warnings (deprecated dependency arrays, unhashable query identity) skipped themselves whenever there was no `process` global, which is the case in Vite and other browser bundles even though they inline `process.env.NODE_ENV`. They now read `process.env.NODE_ENV` as bundlers expect, so an app that loads two copies of `@tanstack/db` in development throws `DuplicateDbInstanceError` as documented. Set `process.env.TANSTACK_DB_DISABLE_DUP_CHECK` to `'1'` through your bundler's `define` to turn the check off. diff --git a/.changeset/fix-draft-added-undefined.md b/.changeset/fix-draft-added-undefined.md new file mode 100644 index 0000000000..d01a324586 --- /dev/null +++ b/.changeset/fix-draft-added-undefined.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Fix an update that drops a field added as `undefined`. When a callback added a field with the value `undefined` and set another field back to its original value, the draft treated every change as reverted and reported nothing. diff --git a/.changeset/fix-draft-define-property.md b/.changeset/fix-draft-define-property.md new file mode 100644 index 0000000000..58c843488d --- /dev/null +++ b/.changeset/fix-draft-define-property.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Make `Object.defineProperty` inside an update callback report what assignment would. Defining a field back to its original value is no longer a change, an enumerable getter reports its value, and assigning a field the callback gave only a getter throws as it would on a plain object. Deleting a non-enumerable field the callback had written is no longer reported as a deletion, matching a plain delete. diff --git a/.changeset/local-only-direct-writes.md b/.changeset/local-only-direct-writes.md new file mode 100644 index 0000000000..d4112d1ed4 --- /dev/null +++ b/.changeset/local-only-direct-writes.md @@ -0,0 +1,5 @@ +--- +'@tanstack/db': patch +--- + +Local-only Collections apply direct `insert`, `update`, and `delete` calls without an optimistic stage when no user handler is configured for that operation and no other transaction on the Collection is pending or persisting. The write is published once, and the returned transaction is already `completed` with `isPersisted.promise` resolved. Writes inside an ambient transaction, with a handler, or beside another unsettled transaction behave as before. diff --git a/.changeset/perf-pooled-live-queries.md b/.changeset/perf-pooled-live-queries.md new file mode 100644 index 0000000000..d5e8d2f21a --- /dev/null +++ b/.changeset/perf-pooled-live-queries.md @@ -0,0 +1,12 @@ +--- +'@tanstack/db': patch +'@tanstack/react-db': patch +'@tanstack/vue-db': patch +'@tanstack/solid-db': patch +'@tanstack/svelte-db': patch +'@tanstack/angular-db': patch +--- + +Mount and update many small filtered live queries at Redux-level cost. A live query that reads one eager source Collection, filters it by at least one `eq(field, literal)`, and has no clause besides `where` and an `orderBy` on its own fields is served from an equality partition shared by every query on those fields, in React, Vue, Solid, Svelte, and Angular. Its other `where` conditions on the row, such as `not`, `gt`, or `like`, are evaluated per query over its group. This applies to a query function, a query builder, and a `{ query }` config that sets no other option besides `queryKey` or `gcTime`. Queries with a `DbClient` (React, Svelte) or React Suspense keep a live-query Collection. Each query reads its group of rows instead of compiling a live query and subscribing to the source. With 240 such queries in React, mounting takes about 2.3 ms instead of 8.2 ms, and is the same with or without an index. + +Results are unchanged: the same rows in the same order with the same values and status, including a terminal error when the source is cleaned up. Two things can differ. Rows are the source Collection's row objects rather than copies. The returned `collection` is built only when your code reads it, so its automatic id may differ, and tools that list live Collections do not see a pooled query until then. diff --git a/docs/contributing/glossary.md b/docs/contributing/glossary.md index 1504c094b6..c7012c5eb6 100644 --- a/docs/contributing/glossary.md +++ b/docs/contributing/glossary.md @@ -19,6 +19,9 @@ production queues, caches, or semantic helpers merely to share their names. | Collection | The public keyed data container. Capitalize it when referring to the TanStack DB type. | Relation, table, or query result. | | source Collection | A Collection read by a query or adapter. | Source relation when the value is a public Collection. | | live-query Collection | A Collection whose rows are produced by a live query. | Query, observer, or result set. | +| pooled live query | A live query on one source Collection whose `where` has at least one `eq(field, literal)` conjunct and otherwise reads only the row, optionally ordered by the row's own fields without a limit, served from an equality partition; its live-query Collection is built only when read. | Cached query or shared live-query Collection. | +| equality partition | Source rows grouped by the `eq`-normalized values of one set of fields, shared by every pooled live query that filters on those fields. | Index or bucket relation. | +| partition group | The rows of an equality partition whose fields equal one tuple of literals. | Bucket or active bucket. | | relation | An internal weighted multiset maintained by D2. | Collection. | | row | One keyed public Collection value or one relation value. Qualify source row, relation row, or public row when more than one kind appears. | Event or transaction. | | change message | One insert, update, or delete delivered through the Collection sync boundary. | Transaction or publication. | diff --git a/docs/contributing/oracle-coverage.md b/docs/contributing/oracle-coverage.md index a5f4b1708d..82bf5a92ac 100644 --- a/docs/contributing/oracle-coverage.md +++ b/docs/contributing/oracle-coverage.md @@ -65,6 +65,8 @@ that test identifiers must copy production's private data structures. | WHERE predicate publication | Complete for bounded predicate and sync-transaction grammar | The contract, Kleene reference evaluator, snapshot and change-history grammar, three subscriber consumers plus direct snapshot, and per-commit key-set refinement check are literate. A fixed same-key update checks live-row and direct-subscriber payloads; a focused descriptor boundary checks stored-row prefilter safety. Generated cleanup and restart histories for filtered subscribers remain open. | | Joined result keys | Complete for bounded two-source key grammar | The contract, nested-loop pair model, delimiter-, number-like, infinite, and `NaN` key grammar, public join driver, and per-checkpoint pair and key-count check are literate. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations remain outside this owner. | | D2 Index storage | Complete for bounded prefix grammar | The contract, plain-`Map` multiset model, prefixed and unprefixed value grammar, `Index` driver, and per-addition `get`/`has` check are literate. Compaction, presence tracking, and structural payloads remain outside this owner. | +| Pooled live queries | Complete for bounded eq-filter grammar | The contract, independent `eq` model, live-query Collection second formulation, sync/optimistic/mount/cleanup-restart grammar, observer driver, and per-step refinement check are literate. On-demand and persisted sources, `DbClient`, Suspense, and every clause beyond `eq` conjuncts keep the live-query Collection. | +| Flat-row change tracking | Complete for bounded flat-row grammar | Flat and proxy trackers and an independent change model run the same generated callbacks. Nested values fall back to the proxy, which its own oracles own. | | Lazy target path identity | Focused compiler boundary | A same-source union/coalesce witness keeps dotted and nested demand paths distinct during target deduplication. | | Correlated include path identity | Focused public route-context witnesses | One-level and nested includes keep dotted and nested parent paths, including ancestor aliases, distinct. Conditional result paths receive separate routes. Fixed fixtures cover initial reads and selected source updates; other recursive source forms and arbitrary path segments remain outside this witness. | | Alias scope identity | Generated cross-scope alpha-renaming with an independent model | Optimizer copies, wraps, and collapses keep `SourceId`; compilation binds inputs by `SourceId` only; includes cannot shadow a parent subquery alias. Unreached forms and channels are listed in the owner row. | @@ -262,9 +264,13 @@ comment and the current API/architecture contract before extending its model. | Frameworks | [React conformance](https://github.com/TanStack/db/blob/main/packages/react-db/tests/conformance.test.tsx), [React pagination](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [shared suites](https://github.com/TanStack/db/tree/main/packages/db-collection-e2e/src/suites), [Vue synchronous publication](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/useLiveQuery-publication-oracle.test.ts) | Exact exposed rows/pages and each framework's own lifecycle cuts. Vue's synchronous watcher checks insert, update, and nonterminal delete through supplied Collection and identity query inputs; it does not cover an empty final result, multiple changes in one transaction, or query recompilation. A React witness does not prove Vue/Solid/Angular/Svelte scheduling. Preserve their receiving registrations. | | Window-controller overlap | [shared infinite-query conformance](https://github.com/TanStack/db/blob/main/packages/db/tests/conformance/infinite-suite.ts), [React driver](https://github.com/TanStack/db/blob/main/packages/react-db/tests/infinite-query-conformance.test.tsx), [Vue driver](https://github.com/TanStack/db/blob/main/packages/vue-db/tests/infinite-query-conformance.test.ts), [Svelte driver](https://github.com/TanStack/db/blob/main/packages/svelte-db/tests/infinite-query-conformance.svelte.test.ts), [review record](oracle-reviews/dec-03-window-overlap.md) | Four or five ordered source rows, two window-settlement orders, and public snapshots after each settlement, a subscribed observer update, a detached controller read, and explicit recovery. Insertion and removal of the fifth row distinguish both continuation directions while an earlier error remains visible. The detached read follows a source change with no controller subscriber; its preloaded Collection retains the five-row window. The exact overlap uses the exported DB controller in each package realm. Framework hooks expose no controller preload, so this cell does not establish a hook scheduling path; direct hook paging remains in the adjacent shared scenarios. An unsubscribed controller has no notification claim. | | Structural values and ordered primitives | [hash values](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash.property.test.ts), [hash identity](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-identity-oracle.property.test.ts), [MultiSet consolidation](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/multiset-consolidate-oracle.property.test.ts), [hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-graph.property.test.ts), [mixed hash graphs](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-mixed-graph.property.test.ts), [hash retry](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/hash-failure-retry.property.test.ts), [comparison](https://github.com/TanStack/db/blob/main/packages/db/tests/comparison.property.test.ts), [deep equality](https://github.com/TanStack/db/blob/main/packages/db/tests/utils.property.test.ts), [cursor](https://github.com/TanStack/db/blob/main/packages/db/tests/cursor.property.test.ts), [indexes](https://github.com/TanStack/db/blob/main/packages/db/tests/index-update.property.test.ts), [BTree Map](https://github.com/TanStack/db/blob/main/packages/db/tests/btree-map-oracle.test.ts), [query identity](https://github.com/TanStack/db/blob/main/packages/db/tests/query/identity-output-shape-oracle.test.ts), [LIKE semantics](https://github.com/TanStack/db/blob/main/packages/db/tests/query/compiler/evaluators.test.ts) | Independent flat values, graph topology, algebraic laws, Map/group/sort recomputation, expression denotation, custom comparator dispatch/reference identity/index compatibility, LIKE wildcard refinement, compiled output bags, declared value-pair agreement between D2 equality keys and IR equality operands, and Collection collation with exact public scan/index order and one-index reuse across repeated reads. The auto-index cells cross BasicIndex and BTreeIndex; scan cells omit indexes. Both paths cover lexical and numeric `en-US` locale defaults plus an explicit ordinary locale override of a numeric default. An unindexed six-row work cell bounds Collection collation resolution to at most two reads per ordered scan: one index-path check and one scan clause. Other locales and ordered histories are outside this owner. The identity pair grammar covers primitives, signed zero, NaN/invalid Date, valid Date/timestamp, binary/Buffer, Temporal, nested references, symbols, and functions; fixed and sampled pairs do not prove every JavaScript value or hash collision freedom. Ref proxies are expressions rather than literal values in this lane. The deep-equality owner also pins that `deepEquals` ignores Map and Set insertion order, RegExp `lastIndex`, array holes, and typed-array class, the rules draft equality keeps. It also pins that objects of another class differ, that plain and null-prototype objects are one class, that a class instance without enumerable keys equals only itself, that URLs compare by `href`, and that typed-array elements compare like numbers. The hash-values owner samples Set/object and Map/object type-marker distinctions in fixed and random campaigns with seed-and-path replay; collision freedom is not promised. The hash-identity owner checks `hash` and `equalHashValues` against one spec-level identity model: primitives with signed zero, NaN, and bigint; reference leaves (functions, registered handles, `File`, binary over 128 bytes); Date, binary, Temporal, and RegExp headers; array length and holes; Map and Set insertion order; and object keys in any order, for acyclic values and for cycles through arrays, Map values, Sets, and plain objects. Map/Set order sensitivity is pinned current behavior, not a promise. Arrays from another realm, RegExps with a `NaN` `lastIndex`, getters, proxies, other cycle carriers, and pairs that differ only in a back-edge target are outside its grammar; pinned cases keep equality's cross-realm length check and strict RegExp field comparison. `hash-work.test.ts` pins how visited values and array/RegExp headers count toward the structural work cap, and that equality copies a shared Map's entries once per distinct pair ([code-weight hash dispatch review](oracle-reviews/code-weight-hash-dispatch.md)). Custom string indexes do not optimize ordinary ranges, and custom cursors remain unsupported. The LIKE owner covers boolean string matching and bounded work, not nullish three-valued logic or a general Unicode collation contract. Unsupported composite cursors reject. The MultiSet consolidation owner checks keyed, single-type, and structural identity, signed zero and NaN in keyed and single-number data, the retained first record, zero-sum removal, and input record contents. It does not assert output order. Its keyed grammar includes cross-type key and value collisions, non-finite numeric keys, the `\|` delimiter, symbol and function identity, and a direct-object/object-tuple delimiter control from #1948. Eight named collision witnesses and both generated campaigns are RED on `c879ba6d` and GREEN with the repair ([#1948 review record](oracle-reviews/issue-1948-keyed-consolidation.md)); the [code-weight consolidation review](oracle-reviews/code-weight-multiset-consolidation.md) records the earlier exclusion. The keyed reference uses direct value/reference equality, including SameValueZero for numeric keys. The unkeyed fallback keeps its original key and value domains. The direct `MultiSet` owner does not prove which `@tanstack/db` query shapes produce primitive keyed values or mixed-type source keys; that needs a live-query production witness. The index owner enforces one invalid-comparator law across both BasicIndex and BTreeIndex: for each comparator, accepted numeric prefix (including the empty prefix), and probe operation (add, update, eq/range lookup, take, build), the operation throws exactly when it receives a `NaN` or non-number result and never when every comparison is valid. A `signed infinity` comparator is the valid control, and a custom collation `compare` supplied through `compareOptions` is checked the same way. Two BTree Map split histories check inserted-key reads, exact whole-range payloads/order, size, and extrema, proving split placement follows the insertion index without a post-mutation comparison. A rejected index add or remove leaves the index refining its accepted rows; update and build are not atomic. At the Collection boundary, both index types and both write paths (optimistic insert, sync commit) crash the collection: status becomes `error` and the next mutation throws, so a usable collection never holds rows its subscribers were not told about. Open: a sync source can still commit writes on a collection in `error` state, which stores unpublished rows; this is existing `markError` behavior and needs a lifecycle-owner witness. Comparator transitivity is outside this evidence. | -| WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | +| WHERE predicate publication | [WHERE predicate publication oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-predicate-publication-oracle.property.test.ts) | An independent Kleene evaluator judges `eq`/`not`/`and`/`or` predicates over strings, a normalization-prefixed string, booleans, numbers, `NaN`, a valid Date, `null`, a missing field, and virtual fields. Snapshot histories with pending optimistic inserts compare a live query, direct subscribers with and without initial state, and `currentStateAsChanges`; change histories apply multi-key sync transactions, including reinsertion of a key deleted earlier, and compare every consumer's key set after each commit, on scan and `BasicIndex` paths. Peer subscribers share one field with different literals, plus one on a virtual field, so every published change must reach exactly the subscribers whose predicate it can satisfy; routing mutants that ignored the previous value, ignored the field, delivered a change twice, or routed while stale published rows awaited reconciliation fail here. Fixed witnesses cover a layout-only publication reaching a filtered subscriber as one empty batch, a filtered subscriber's empty Collection-readiness batch, retraction of a vanished row after eager cleanup and restart, and the new row plus exact subscriber update payload when a same-key row stays TRUE. A stale-live-value mutant survives the generated membership checks but fails the payload witness. A focused [property-visibility test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/where-prefilter-property-visibility.test.ts) compares the public unindexed snapshot with the enriched-row contract for inherited, non-enumerable, enumerable-own, and nested getter paths; the pre-fix stored-row shortcut failed three of its four controls. Other property-descriptor and stateful-getter histories remain outside those fixed controls. Hostile mutants for FALSE-for-UNKNOWN `eq`, `or` or number-literal subscription prefilters, a prefilter that ignores the previous value, a skipped readiness batch, a skip while stale published rows await reconciliation, and an unindexed snapshot scan that reads only synced rows while an optimistic insert or delete changes visibility passed the prior `@tanstack/db` suite and fail here. Change routing withholds a dropped row from an `eq` subscription before its sent key could be recorded, so a mutant that records dropped rows as sent is equivalent for routed predicates in this oracle. A focused `collection-subscription.test.ts` witness checks that a dropped insert or update, alone or beside a matching change, cannot advance a limited subscription's page offset under a routed `eq` and an unrouted `or` predicate; the unrouted cases beside a matching change kill that mutant, and the unrouted `alone` case does not. A loose-equality prefilter is an equivalent mutant: it only skips less. Skipping during truncate replay also survives; the replay's baseline diff re-derives the same retraction, so the guard keeps the prior dataflow without its own witness. Generated cleanup and restart histories for filtered subscribers remain open for the lifecycle publication owner. No-op updates, other comparison operators, Temporal and binary operands, joins, ordering, optimistic updates, and truncate are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. Change routing was later removed in favor of pooled live queries; the routing mutants above are historical, and the same peer laws now check plain subscription dispatch. The unindexed snapshot prefilter was restored for compiled queries as one stored-row `eq` test on a string or boolean literal. Its property-visibility test also covers the reverse direction: a stored row with an inherited or non-enumerable field must still match `isUndefined` on that field, which a scan that evaluated the full predicate on stored rows fails. | | Joined result keys | [Joined result key oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/join-result-key-oracle.property.test.ts) | A nested-loop model recomputes the pair set of an inner, left, or full join over source keys drawn from plain, comma-bearing, bracket-bearing, and quoted strings, numbers beside the strings that print the same, both infinities, and `NaN`. After preload and each synced group change, the published rows must equal the model's pairs and the result key count must equal the row count. The comma-joined key encoding fails its two pinned histories and both campaigns; plain `JSON.stringify`, which prints infinities as the missing-side `null`, fails the pinned infinity history and both campaigns. A pinned history in which a `NaN`-keyed pair leaves and re-forms fails when the join index compares source-key prefixes with `===`. Joins over subqueries, more than two sources, custom `getKey`, and optimistic mutations are outside this owner. It runs in `@tanstack/db`'s `test:oracles` campaign. The [2026-09-30 review](https://github.com/TanStack/db/blob/main/docs/contributing/oracle-reviews/2026-09-30-where-predicate-and-join-keys.md) records each ORC outcome. | | D2 Index storage | [Index refinement oracle](https://github.com/TanStack/db/blob/main/packages/db-ivm/tests/index-refinement-oracle.property.test.ts) | A plain-`Map` model sums multiplicities under a type-and-value identity in which `NaN` equals `NaN` and `-0` equals `0`. Histories of up to twelve additions to two keys mix prefixed arrays with `NaN`, `0`, `-0`, `1`, `'1'`, `1n`, and `'a'` prefixes and unprefixed values, with cancellation and reappearance. After each addition, `get` and `has` must equal the model. Comparing prefixes with `===`, which disagrees with the `Map` that holds them, fails three pinned histories and both campaigns; the `-0` control passes under both. Compaction, presence tracking, and structural payloads are outside this owner; the join operator tests and incrementalization law own operator behavior. | +| Pooled live queries | [pooled live query oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-oracle.property.test.ts) | An independent `eq` model (nullish UNKNOWN, Date as timestamp, `NaN` equal to `NaN`, `-0` equal to `0`) gives membership; a live-query Collection compiled for the same query gives order, values, and status. Histories mix sync transactions, optimistic writes confirmed or rolled back, peer mounts and unmounts, and source cleanup and restart, including cleanup while a write is pending and a mount between cleanup and restart. Wholesale observers must match after every step, granular changes must match a reference observer by type, key, value, and previous value and replay to the model's rows, the layout revision must advance when keys reorder, and observing must never build the live-query Collection. Peers may add a residual `not(eq(...))` conjunct, which the model evaluates itself; a view that ignores it, reports an unfiltered `entries()`, or delivers a row entering or leaving it as an update fails it. Other residual operators use the same compiler evaluator but are not generated. Peers may also order their rows by `id` or by a `g` that can be null, with explicit `nulls`; order comes from the reference. A comparator that ignores direction or `nulls`, an order left out of the partition key, and key order instead of the query's order each fail the pinned reorder history and both campaigns. A partition that ignores a row's previous group, orders by arrival, skips normalization, skips the initial snapshot, follows the source's status after cleanup, or never terminates fails it. Two shared conformance scenarios (`eq-filter-rows`, `eq-filter-peers`) run the pooled path under React, Vue, Solid, Svelte, and Angular; a partition that ignores a row's previous group fails both under every adapter, and `eq-filter-peers` checks through `subscriberCount` that each adapter shares one source subscription per partition. A focused [release-timing test](https://github.com/TanStack/db/blob/main/packages/db/tests/query/pooled-live-query-gc.test.ts) compares when a pooled query and a live-query Collection release their sources across `gcTime` 1, 100, 0, and `Infinity`, with and without a subscriber, and with no `gcTime`, where both use the live-query Collection default, checks that a partition waits for its views' longest `gcTime`, checks that a view resubscribed after its partition released follows later source writes as its live-query Collection does, including a view with a residual conjunct, and checks that a partition that subscribes again still serves new mounts from one source subscription. Further focused witnesses check that a partition keeps groups only for values that have rows or watchers, that a detached reader never sees a dropped group's revision reused, keep a pooled query's public Collection live while the view is observed, accept it as a query source, and check that a pooled query turns terminal when source cleanup starts while the adapter's cleanup is still pending, as a live-query Collection does. A pinned oracle witness checks that an `eq` path whose getter throws excludes the row on both paths; a pooled path that rethrows fails it. The pooled oracle's grammar never releases a partition, so release and resubscribe histories rely on these fixed witnesses; a fixed delay, a missing unsubscribed floor, a last-`gcTime`-wins rule, and releasing at `gcTime` 0 each fail it. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Flat-row change tracking | [flat change tracking oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/flat-change-tracking-oracle.property.test.ts) | Generated callbacks over flat rows, frozen or not and with or without a non-enumerable field (assignments of `0`, `-0`, `NaN`, `undefined`, functions, and fresh objects; reverts; deletions; added fields; data and accessor `defineProperty`; stored drafts; throws), run through the flat tracker, the draft proxy, and an independent model, which must report the same change sets with `0` and `-0` equal, or throw the same error and leave the rows unchanged. Defining a field acts as assigning it, and a non-enumerable field is row data only once written; the proxy broke three of those laws on `main` and the flat tracker one. Diffs using `!==`, `Object.is` alone, or no deletions fail it. It found that the proxy dropped a field added as `undefined` when another field reverted; the pinned history and both campaigns fail without that fix. Non-flat rows must fall back. The [2026-10-01 review](oracle-reviews/2026-10-01-pooled-live-queries.md) records each ORC outcome. | +| Virtual props cache | [cache laws](https://github.com/TanStack/db/blob/main/packages/db/tests/virtual-props-cache.test.ts) | Focused laws for the per-key cache of rows read with virtual props: a change's published value is the row later `get` and `toArray` reads return, for every subscriber, after sync and local writes, and a key that is deleted or rolled back leaves the cache. Enriching the value before the previous value fails the first two; skipping the drop on delete events fails the third. Rows read with virtual props through other paths, such as joins, are outside these laws. | +| Local-only direct writes | [direct write witnesses](https://github.com/TanStack/db/blob/main/packages/db/tests/local-only-direct-write.test.ts) | Focused witnesses pin when a local-only direct write skips the optimistic stage: a completed transaction and one publication per write, and the fallbacks for a pending, persisting, or ambient transaction and a user handler, for insert, update, and delete. Each history also runs on a local-only Collection whose handlers resolve, which must match its rows, change batches, errors, and transaction states, across mixed multi-key batches, a batch that fails midway, schema rejection, and handler rollback. Mutants that drop each guard, write only the first mutation, or never complete the transaction fail them. The change-event history oracle drives the direct path through generated histories (about two thousand writes per run); removing the guard, or guarding only persisting transactions, fails these witnesses. | | Reused sync row objects | [reused row witness](https://github.com/TanStack/db/blob/main/packages/db/tests/sync-reused-row.test.ts) | Focused witnesses for a sync source that changes a stored row object in place: in development, writing it again as an update without `previousValue` throws, while the same write with `previousValue`, or with a new object, moves the row out of an `eq` live query. Production skips the check. The check compares a shallow snapshot taken when the object was written, so it misses changes to nested fields; a write that names `previousValue` and a row whose fields throw on read are not checked. | | Index suggestions | [collection-size suggestion oracle](https://github.com/TanStack/db/blob/main/packages/db/tests/index-suggestion-oracle.test.ts) | A public filtered live-query Collection checks manual/default, eager, threshold, matching-index, and unrelated-index cases against the documented collection-size suggestion policy. The original manual-silence/eager-noise implementation failed two expected observations. The owner does not judge repeated-query warning volume, slow-query timing, query work, production-build suppression, or every query shape. It runs in `@tanstack/db`'s `test:oracles` campaign. | | Minified DB public API | [consumer bundle check](https://github.com/TanStack/db/blob/main/scripts/test-minified-db.mjs) | CI builds `@tanstack/db`, then bundles its built `dist` entry with db-ivm inlined through esbuild `minify: true`. The build renames TypeScript-private members from `packages/db/mangle-cache.json`, so this lane runs against the renamed output that consumers install. It rejects a `dist` that is older than `src`, the cache, or the package's build configuration. `pnpm --filter @tanstack/db test:dist` also runs five db test files against the built `dist`, chosen for coverage of the modules with the most renamed members. `pnpm check:mangle` fails when a cached name is used other than as a private member in db src, is read by another package, appears as a string, or has a short name that collides with a source identifier ([private-member mangling review](oracle-reviews/code-weight-private-member-mangling.md)). It checks every exported error class's current public `name`, the built-in `BasicIndex` resolver name in index metadata and its event, complete query rows (after removing the four documented virtual fields) against an independent array filter/sort/projection, and one live update. The `new.target.name`, public-member mangle, and extra-output-field calibration modes fail at the intended observations. This fixed slice does not replace the unminified generated oracles, exercise framework adapters, or establish stable names for custom index resolvers. | diff --git a/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md new file mode 100644 index 0000000000..ebbfc6117c --- /dev/null +++ b/docs/contributing/oracle-reviews/2026-10-01-pooled-live-queries.md @@ -0,0 +1,169 @@ +# Pooled live query and flat change tracking oracle review + +## Reviewed state and claim + +Base: `84b828c7e`, merged with `origin/main` at `18abceee4`. This record reviews +the Git tree that contains it; the final commit or pull request identifies that +tree. + +The pooled live query oracle claims that a live query filtered only by +`eq(field, literal)` on one eager, non-persisted source, served from an +equality partition, publishes what its live-query Collection would: the same +rows in key order, values, and status. That holds through sync transactions, +optimistic writes confirmed or rolled back, peer mounts and unmounts, and +source cleanup and restart. The claim excludes on-demand and persisted +sources, `DbClient` hydration, Suspense, and every clause beyond `eq` +conjuncts, which keep the live-query Collection. + +The flat change tracking oracle claims that, for a row whose own fields are +all primitives or functions with a plain or null prototype and no symbol +keys, the flat tracker reports the same change set as the draft proxy and an +independent model, with `0` and `-0` equal. Nested values, Dates, Maps, Sets, +class instances, and symbol keys are outside it, except that they must fall +back to the proxy. + +Both oracles use `mockSyncCollectionOptions` or plain rows; neither claims a +real sync adapter's behavior. + +## RED and GREEN evidence + +Mutants ran through each oracle file alone on the reviewed tree. Each file was +restored after each run. Every outcome below is an assertion failure. + +| Mutant | Oracle | Tests failed | +| --- | --- | --- | +| Partition ignores a row's previous group | Pooled | 6 of 9 | +| Groups keep rows in arrival order | Pooled | 3 of 9 | +| Literals and fields compared without `eq` normalization | Pooled | 4 of 9 | +| Partition skips the source's initial state | Pooled | 8 of 9 | +| View reports the source's status after cleanup | Pooled | 4 of 9 | +| Partition never terminates on cleanup | Pooled | 4 of 9 | +| Partition keys groups by the first field only | Pooled | 5 of 9 | +| Update published as delete then insert | Pooled | 3 of 9 (0 of 7 before payload checks) | +| Update carries a stale `previousValue` | Pooled | 3 of 9 (0 of 7 before) | +| Update carries the old row as its value | Pooled | 3 of 9 (0 of 7 before) | +| Within-group updates dropped | Pooled | 3 of 9 | +| Partition built on a cleaned-up source starts terminal | Pooled | 3 of 9 (0 of 7 before) | +| Frozen peers drop pending optimistic rows | Pooled | 3 of 9 (0 of 7 before) | +| Flat diff with `!==` | Flat | 3 of 17 | +| Flat diff with `Object.is` alone | Flat | 3 of 17 | +| Flat diff without deletions | Flat | 5 of 17 | +| Draft proxy without the added-field revert fix | Flat | pinned history and both campaigns | +| Flat diff ignores whether the row owns a field | Flat | 4 of 17 | +| Flat drafts edit the rows in place | Flat | 9 of 17 | +| Assigned objects not detached | Flat | 1 of 17 (detach witness) | +| Frozen rows sent to the proxy | Flat | 3 of 17 | +| Proxy ignores accessors defined in the callback | Flat | 3 of 17 | +| Proxy reports a field defined back to its own value | Flat | 4 of 17 | +| Proxy accepts a getter-only field's own value | Flat | 3 of 17 | +| Proxy reports a written hidden field's delete | Flat | 3 of 17 | +| Flat compares a hidden object field by identity | Flat | 3 of 17 | + +Every pooled and flat mutant above fails its pinned history and both +campaigns. The pooled grammar delivers initial rows in key order or in +reverse and weights a pending insert most peers see before cleanup; the flat +grammar weights runs that write and delete a hidden field, write an equal +object over a hidden object field, assign a getter-only field its own value, and write the opposite-signed zero +over each zero field. Before that weighting, arrival order and frozen optimistic rows +escaped about one random campaign in three, and the three descriptor +mutants escaped both campaigns. The `Object.is`-alone flat mutant escaped about one random campaign in two +until the zero-flip run. + +The pooled oracle found that a pooled view followed its source's status after +cleanup instead of entering the live query's terminal error; the repair makes +the partition terminate. The flat oracle found that the draft proxy dropped a +field added as `undefined` when another field reverted, on `main` as well; the +repair treats a field the original lacks as changed. The grammar now weights +that run, so both campaigns also kill the unrepaired proxy. + +The loss audit then widened both oracles. The pooled granular observer had +been checked by key membership only, so the payload and lifecycle mutants +marked "0 of 7 before" survived it. The flat grammar gained frozen rows, +non-enumerable fields, accessor and data `defineProperty`, stored drafts, and +throwing callbacks. Four shapes split the trackers, three of them on `main`'s +proxy too. The adopted rule is that defining a field acts as assigning it, +and that a non-enumerable field is row data only once the callback writes +it. The proxy now records accessors and data defines through the assignment +path, rejects a getter-only write of its own value, and ignores the delete of +a hidden field it wrote. The flat tracker now compares a hidden object field +by contents. + +Local-only direct writes are compared with a local-only Collection whose +handlers resolve, across mixed multi-key batches, failing batches, schema +rejection, and every fallback for insert, update, and delete. Mutants that +ignore handler types, drop the pending or persisting check, drop the whole +transaction check, run before the ambient branch, write only the first +mutation, or never complete the transaction fail 4, 4, 4, 8, 12, 1, and 9 +tests. + +Two shared conformance scenarios, `eq-filter-rows` and `eq-filter-peers`, run +the pooled path under React, Vue, Solid, Svelte, and Angular. A partition that +ignores a row's previous group fails both under every adapter. `eq-filter-peers` also checks the +source's public `subscriberCount`: an adapter that declares `pooledEqFilters` +must share one subscription for two queries on the same fields. Disabling pooling or using one partition per query fails it under React; +before this check the first passed. Each adapter's driver declares +`pooledEqFilters`. + +The loss audit found that a partition released its source one second after +its last listener, whatever `gcTime` its views had. A focused release-timing +test, `pooled-live-query-gc.test.ts`, now compares pooled and live-query +Collection release times. A fixed delay failed 5 of 10 cases, a missing 50 ms +floor for unsubscribed queries 1, a last-`gcTime`-wins rule 1, and releasing +at `gcTime` 0 2. Release timing is resource lifetime, so the publication +oracle does not observe it. + +Pooling later admitted residual conjuncts: any `where` conjunct that reads +only the query's own row, beside at least one `eq`, is evaluated per view +with the compiler's evaluator. Peers may add `not(eq(g, literal))`, which the +model evaluates itself, and a pinned history moves rows in and out of a view +within one group. A view that ignores the residual, reports unfiltered +entries, or sends a row entering or leaving it as an update fails the +pinned history and both campaigns. Hiding part of a group makes some +earlier histories rarer, so in one full run the arrival-order, +normalization, delete-plus-insert, and dropped in-group update mutants +escaped the random campaign; the fixed campaign and their pinned histories +still kill each. Peers' literals were weighted toward the normalized values +to keep the normalization mutant in the fixed campaign. + +## Pooled live query oracle + +| Requirement | Outcome | +| --- | --- | +| ORC-001 Contract authority and limits | Pass. The `eq` operand rules come from `src/query/compiler/evaluators.ts`; the pooled boundary and the terminal-error rule come from the live-query architecture document's pooled section and cleanup law. The opening prose lists the omissions. | +| ORC-002 Independent judgment | Pass. `expectedKeys` uses a local `eq` over plain values. Order, values, and status come from a live-query Collection, which compiles a D2 pipeline and does not use the partition. | +| ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses only domain values and step kinds. Ablation, run per axis on the campaigns with pinned cases skipped: without Dates, `NaN`, and `-0` the normalization mutant survives; without optimistic steps, frozen peers dropping optimistic rows survives; without mounts and unmounts, the terminal-at-creation mutant survives; without cleanup-restart, the status, termination, and both pending-cleanup mutants survive; without the second conjunct, first-field grouping survives. Range: at most four rows, three peers, eight steps, and ids 0 through 3; field values weight toward the literal so rows update within their group. Exclusion: an optimistic update to an equal value, an insert of an existing id, and an update or delete of a missing id are dropped. | +| ORC-005 Production path and observation | Pass. `createPooledLiveQuery` and `createLiveQueryObserver`, the adapter seam, run in wholesale and granular mode; the conformance scenarios run through React's `useLiveQuery`. Rows, keyed state, status, layout revision, and non-materialization are observed after every step, and granular changes are compared with a reference observer by type, key, value, and previous value, then replayed against the model. | +| ORC-006 Checker calibration | Pass. Thirteen mutants, classified above. | +| ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_001`, an unseeded campaign, and a replay entry share one property and budget; the file is in `test:oracles`. | +| ORC-008 Stateful-model minimality | Pass. The model keeps source rows and, per mounted peer, the frozen keys at cleanup. The pinned cleanup history distinguishes a frozen peer from one mounted after the restart. | +| ORC-009 Vocabulary mapping | Pass. Equality partition, partition group, and pooled live query are glossary terms; a peer is one mounted pooled live query. | +| ORC-010 Failure fidelity and cleanup | Pass. `withOracleCleanup` releases observers, references, and the source and keeps the check failure. | +| ORC-011 Independent second formulation | Pass. The live-query Collection is the second formulation for order, values, and status. | +| ORC-012 Review evidence | This record; the coverage map links it. | +| ORC-013 Reusable boundary law | Pass. Normalization is rejected by the Date history, arrival order by the key-order history, and following the source's status by the cleanup history. | +| ORC-014 Controlled-premise handoff | Not triggered. The claim is limited to the mock source's sync transactions and lifecycle. | + +## Flat change tracking oracle + +| Requirement | Outcome | +| --- | --- | +| ORC-001 Contract authority and limits | Pass. The change-set rules come from `src/proxy.ts`'s `getChanges` contract: changed fields with their final value and deleted fields as `undefined`. | +| ORC-002 Independent judgment | Pass. `expectedChanges` folds the operations over a plain copy and does not import either tracker. | +| ORC-003 Distinguishable responsibilities | Pass. Contract, model, grammar, driver, and refinement check are separate marked sections. | +| ORC-004 Generated-history controls | Pass. Reconstruction: every pinned history uses domain values and operations. Ablation: removing `NaN`, `-0`, reverts, or deletions each loses a mutant. Range: one to three rows, three fields plus one added and one non-enumerable field, frozen or not, up to six operations per row including data and accessor defines, stored drafts, and a throw. Exclusion: none in generation; non-flat rows are rejected by the fallback witness. | +| ORC-005 Production path and observation | Pass. `withFlatChangeTracking`, `withArrayChangeTracking`, and `withChangeTracking` run the same callbacks; the result change sets are observed. `collection.update` selects between them. | +| ORC-006 Checker calibration | Pass. Thirteen mutants, classified above. | +| ORC-007 Fixed/random replay | Pass. Fixed seed `44_502_101`, an unseeded campaign, and a replay entry; the file is in `test:oracles`. | +| ORC-008 Stateful-model minimality | Not triggered. The model recomputes from the operations. | +| ORC-009 Vocabulary mapping | Pass. Draft, change set, and revert follow the proxy's terms. | +| ORC-010 Failure fidelity and cleanup | Not triggered. The oracle holds no resources. | +| ORC-011 Independent second formulation | Pass. The draft proxy is the second formulation. | +| ORC-012 Review evidence | This record; the coverage map links it. | +| ORC-013 Reusable boundary law | Pass. `!==` is rejected by the `NaN` history, `Object.is` by the `-0` history, and missing deletions by the deleted-field history. | +| ORC-014 Controlled-premise handoff | Not triggered. No provider is involved. | + +## Open work + +- Svelte's suite reads `@tanstack/db` from its built `dist`, so a mutant + must type-check and be rebuilt before Svelte can observe it. diff --git a/packages/angular-db/src/index.ts b/packages/angular-db/src/index.ts index 8264b4647b..1682546ae7 100644 --- a/packages/angular-db/src/index.ts +++ b/packages/angular-db/src/index.ts @@ -10,8 +10,10 @@ import { BaseQueryBuilder, createLiveQueryCollection, createLiveQueryObserver, + getPublicCollection, isCollection, isSingleResultCollection, + resolveLiveQueryValue, } from '@tanstack/db' import type { Collection, @@ -180,11 +182,7 @@ export function injectLiveQuery(opts: any) { return null } - return createLiveQueryCollection({ - query: opts, - startSync: true, - gcTime: 0, - }) + return resolveLiveQueryValue(result, { gcTime: 0 }) } // Check if it's reactive query options @@ -207,11 +205,7 @@ export function injectLiveQuery(opts: any) { return null } - return createLiveQueryCollection({ - query: () => result, - startSync: true, - gcTime: 0, - }) + return resolveLiveQueryValue(result, { gcTime: 0 }) } // Handle LiveQueryCollectionConfig objects. Default startSync/gcTime to @@ -250,7 +244,7 @@ export function injectLiveQuery(opts: any) { observer: LiveQueryObserver, ) => { const newState = new Map(currentCollection.entries()) - const newData = Array.from(currentCollection.values()) + const newData = Array.from(newState.values()) state.set(newState) internalData.set(newData) @@ -312,7 +306,9 @@ export function injectLiveQuery(opts: any) { data, // Loosely typed so the impl return stays compatible with every overload // (the shared `isCollection` guard narrows the computed to `Collection | null`). - collection: collection as Signal, + collection: computed(() => + getPublicCollection(collection()), + ) as Signal, status, isLoading: computed(() => status() === `loading`), isReady: computed(() => status() === `ready` || status() === `disabled`), diff --git a/packages/angular-db/tests/conformance.test.ts b/packages/angular-db/tests/conformance.test.ts index 546e49f09e..7501f3d14f 100644 --- a/packages/angular-db/tests/conformance.test.ts +++ b/packages/angular-db/tests/conformance.test.ts @@ -235,7 +235,7 @@ const angularDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: false, suspense: false }, + features: { serverSnapshot: false, suspense: false, pooledEqFilters: true }, } describe(`owned native scope setup`, () => { diff --git a/packages/db/mangle-cache.json b/packages/db/mangle-cache.json index 5e8c5034ed..96388dbec4 100644 --- a/packages/db/mangle-cache.json +++ b/packages/db/mangle-cache.json @@ -368,6 +368,19 @@ "attach": "f2", "compilations": "f3", "writtenRows": "f4", - "equalityRoute": "f5", - "checkReusedRow": "f6" + "checkReusedRow": "f6", + "stopStatusEvents": "f5", + "commitLocalOnlyDirect": "f7", + "collectionHold": "f8", + "scheduleRelease": "f9", + "holdCollection": "ga", + "deliverStatus": "gb", + "releaseIfUnused": "gc", + "filterChanges": "gd", + "passes": "ge", + "registry": "gf", + "compareRows": "gg", + "sameFields": "gh", + "dropIfUnused": "gi", + "clock": "gj" } diff --git a/packages/db/package.json b/packages/db/package.json index be81bc13b9..c1830b3b72 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -23,7 +23,7 @@ "test": "vitest --run", "test:dist": "vitest run --config vitest.dist.config.ts", "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts", - "test:oracles": "vitest --run --coverage.enabled=false tests/cleanup-queue.property.test.ts tests/comparison.property.test.ts tests/cursor.property.test.ts tests/index-update.property.test.ts tests/utils.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-truncate-ownership-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts", + "test:oracles": "vitest --run --coverage.enabled=false tests/cleanup-queue.property.test.ts tests/comparison.property.test.ts tests/cursor.property.test.ts tests/index-update.property.test.ts tests/utils.property.test.ts tests/change-event-history-oracle.test.ts tests/index-suggestion-oracle.test.ts tests/btree-map-oracle.test.ts tests/paced-mutations-oracle.test.ts tests/db-client-hydration-authority-oracle.test.ts tests/collection-mutation-startup-oracle.test.ts tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-truncate-ownership-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-reentrancy-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/cold-join-reconciliation-oracle.test.ts tests/query/identity-output-shape-oracle.test.ts tests/query/optimizer-semantics-oracle.test.ts tests/query/index-path-collision-oracle.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/subquery-user-value-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts tests/query/virtual-row-fields-oracle.test.ts tests/query/where-predicate-publication-oracle.property.test.ts tests/query/join-result-key-oracle.property.test.ts tests/query/pooled-live-query-oracle.property.test.ts tests/flat-change-tracking-oracle.property.test.ts", "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run" }, "type": "module", diff --git a/packages/db/src/SortedMap.ts b/packages/db/src/SortedMap.ts index c0b31b14ce..dbcd830e57 100644 --- a/packages/db/src/SortedMap.ts +++ b/packages/db/src/SortedMap.ts @@ -106,6 +106,11 @@ export class SortedMap { * @returns This SortedMap instance for chaining */ set(key: TKey, value: TValue, deferOrder = false): this { + // Key order cannot change when an existing key gets a new value. + if (!this.comparator && this.map.has(key)) { + this.map.set(key, value) + return this + } // Grouped Collections can produce nullish keys at runtime. compareKeys // is not a total order there, so retain the existing binary-insert path. const runtimeKey = typeof key !== `string` && typeof key !== `number` diff --git a/packages/db/src/collection/change-events.ts b/packages/db/src/collection/change-events.ts index 6409b883f7..f74a7d971e 100644 --- a/packages/db/src/collection/change-events.ts +++ b/packages/db/src/collection/change-events.ts @@ -8,7 +8,11 @@ import { } from '../utils/index-optimization.js' import { ensureIndexForField } from '../indexes/auto-index.js' import { getPropRefPropertyPath } from '../query/ir.js' -import { isVirtualPropName } from '../virtual-props.js' +import { + equalityConjunct, + equalityKey, + readPath, +} from '../query/equality-conjunct.js' import { makeComparator } from '../utils/comparison.js' import { buildCompareOptions } from '../query/compiler/order-by' import type { @@ -29,6 +33,31 @@ export type StoredRowScan = ( prefilter: (row: object) => boolean, ) => Iterable<[TKey, WithVirtualProps]> +/** + * A test on a stored row that is false only when `expression` must be false + * on the row's enriched copy, so a scan can skip enriching rows that fail. + * It reads one `eq(field, literal)` conjunct. The copy holds each enumerable + * own root field of the stored row and lacks the others, so its field is the + * stored value or `undefined`, which matches no literal. + */ +export function compileStoredRowPrefilter( + expression: BasicExpression, +): ((row: object) => boolean) | undefined { + const conjuncts = + expression.type === `func` && expression.name === `and` + ? expression.args + : [expression] + for (const conjunct of conjuncts) { + const eq = equalityConjunct(conjunct, getPropRefPropertyPath) + if (eq) { + return (row) => + equalityKey(readPath(row as Record, eq.path)) === + eq.literalKey + } + } + return undefined +} + /** * Returns the current state of the collection as an array of changes * @param collection - The collection to get changes from @@ -73,17 +102,14 @@ export function currentStateAsChanges< filterFn?: (value: WithVirtualProps) => boolean, ): Array, TKey>> => { const result: Array, TKey>> = [] - // Reject rows by one stored field before copying them to add virtual - // properties. Survivors still pass through the full predicate. + // Reject rows by a stored field before enriching them; survivors still + // pass through the full predicate. const prefilter = - scanStoredRows && options.where && compileEqualityPrefilter(options.where) - if (filterFn && scanStoredRows && prefilter) { - for (const [key, value] of scanStoredRows(prefilter)) { - if (filterFn(value)) result.push({ type: `insert`, key, value }) - } - return result - } - for (const [key, value] of collection.entries()) { + filterFn && scanStoredRows && options.where + ? compileStoredRowPrefilter(options.where) + : undefined + const rows = prefilter ? scanStoredRows!(prefilter) : collection.entries() + for (const [key, value] of rows) { // If no filter function is provided, include all items if (filterFn?.(value) ?? true) { result.push({ @@ -219,121 +245,6 @@ export function createFilterFunctionFromExpression( } } -/** A field and the string or boolean literal a top-level `eq` requires. */ -export type EqualityRoute = { - path: Array - /** Stable identity of `path`, for grouping routes by field. */ - pathKey: string - expected: string | boolean -} - -/** Read result for a route path whose property access threw. */ -export const UNREADABLE_ROUTE_VALUE: unique symbol = Symbol( - `unreadable route value`, -) - -/** - * Finds a cheap necessary condition for `expression` to be TRUE, or returns - * undefined when the expression has none. - * - * A top-level conjunct `eq(field, literal)` with a string or boolean literal is - * TRUE only when the field holds the identical string or boolean: equality - * normalization never maps another type onto a plain string or boolean. A - * row whose field holds anything else therefore fails the whole expression. - * - * With `storedRows`, the condition is read from a stored row instead of its - * enriched copy, so conjuncts on virtual fields are skipped: stored rows need - * not carry them. - */ -export function findEqualityRoute( - expression: BasicExpression, - { storedRows = false }: { storedRows?: boolean } = {}, -): EqualityRoute | undefined { - const conjuncts: Array = [] - const collect = (node: BasicExpression) => { - if (node.type === `func` && node.name === `and`) node.args.forEach(collect) - else conjuncts.push(node) - } - collect(expression) - - // A string literal usually rejects more rows than a boolean one. - let best: EqualityRoute | undefined - for (const conjunct of conjuncts) { - if (conjunct.type !== `func` || conjunct.name !== `eq`) continue - const [left, right] = conjunct.args - const ref = - left?.type === `ref` ? left : right?.type === `ref` ? right : undefined - const literal = - left?.type === `val` ? left : right?.type === `val` ? right : undefined - if (!ref || !literal) continue - const expected: unknown = literal.value - if (typeof expected !== `string` && typeof expected !== `boolean`) continue - - const path = getPropRefPropertyPath(ref) - if (storedRows && (path.length === 0 || isVirtualPropName(path[0]!))) { - continue - } - if (best === undefined || typeof best.expected === `boolean`) { - best = { path, pathKey: JSON.stringify(path), expected } - } - if (typeof expected === `string`) break - } - return best -} - -/** - * Reads a route field the way the single-row evaluator does. A throwing read - * returns UNREADABLE_ROUTE_VALUE so callers leave the decision to the full - * predicate. - */ -export function readRouteValue( - row: unknown, - path: ReadonlyArray, -): unknown { - try { - let value: unknown = row - for (const segment of path) { - if (value === null || value === undefined) return undefined - value = (value as Record)[segment] - } - return value - } catch { - return UNREADABLE_ROUTE_VALUE - } -} - -/** - * Compiles the route of `expression` as a row test that is false only when - * the full predicate must be false. - * - * The test reads a stored row instead of its enriched copy. The copy holds - * each enumerable own root property of the stored row and lacks the others, so its field is either the stored value or `undefined`, - * which never equals the literal. A read that throws passes the row to the - * full predicate. - */ -export function compileEqualityPrefilter( - expression: BasicExpression, -): ((row: object) => boolean) | undefined { - const route = findEqualityRoute(expression, { storedRows: true }) - if (route === undefined) return undefined - const { path, expected } = route - // Most routes name one top-level field; read it without walking a path. - if (path.length === 1) { - const field = path[0]! - return (row) => { - try { - return (row as Record)[field] === expected - } catch { - return true - } - } - } - return (row) => { - const value = readRouteValue(row, path) - return value === UNREADABLE_ROUTE_VALUE || value === expected - } -} - /** * Creates a filtered callback that only calls the original callback with changes that match the where clause * @param originalCallback - The original callback to filter diff --git a/packages/db/src/collection/changes.ts b/packages/db/src/collection/changes.ts index e9deb69c43..4068653ac4 100644 --- a/packages/db/src/collection/changes.ts +++ b/packages/db/src/collection/changes.ts @@ -6,7 +6,6 @@ import { toExpression, } from '../query/builder/ref-proxy.js' import { CollectionSubscription } from './subscription.js' -import { readRouteValue } from './change-events.js' import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessage, SubscribeChangesOptions } from '../types' import type { CollectionLifecycleManager } from './lifecycle.js' @@ -260,25 +259,9 @@ export class CollectionChangesManager< const layoutListeners = [...this.layoutChangeListeners] const subscriptions = [...this.changeSubscriptions] withPublicationContext(() => { - // An empty batch signals readiness to every subscriber. - const routed = - rawEvents.length > 0 - ? routeChanges(enrichedEvents, subscriptions) - : undefined - const callbacks: Array<() => void> = [] - for (const subscription of subscriptions) { - const own = routed?.get(subscription) - callbacks.push(() => { - // An earlier callback in this publication can end routing for this - // subscription, for example by leaving stale rows to reconcile. - if (own === undefined || !subscription.changeRoute) { - subscription.emitEvents(enrichedEvents) - } else if (own.length > 0) { - // A routed subscription with no candidate change cannot publish. - subscription.emitEvents(own) - } - }) - } + const callbacks: Array<() => void> = subscriptions.map( + (subscription) => () => subscription.emitEvents(enrichedEvents), + ) if (rawEvents.length === 0) { callbacks.unshift(...layoutListeners) } @@ -442,64 +425,3 @@ export class CollectionChangesManager< this.deferral = undefined } } - -/** - * Gives each routed subscription only the changes whose value or previous - * value holds its route literal, in batch order. Subscriptions without a - * route are absent from the result and receive the whole batch; with no - * routed subscription the result is undefined. - */ -function routeChanges( - changes: Array>, - subscriptions: Array, -): Map>> | undefined { - const routes = subscriptions.map((subscription) => subscription.changeRoute) - if (routes.every((route) => route === undefined)) return undefined - const routed = new Map< - CollectionSubscription, - Array> - >() - const groups = new Map< - string, - { - path: Array - byLiteral: Map> - } - >() - for (const [index, subscription] of subscriptions.entries()) { - const route = routes[index] - if (!route) continue - routed.set(subscription, []) - let group = groups.get(route.pathKey) - if (!group) { - group = { path: route.path, byLiteral: new Map() } - groups.set(route.pathKey, group) - } - const peers = group.byLiteral.get(route.expected) - if (peers) peers.push(subscription) - else group.byLiteral.set(route.expected, [subscription]) - } - - const deliver = ( - targets: Array | undefined, - change: ChangeMessage, - ) => { - for (const subscription of targets ?? []) { - routed.get(subscription)!.push(change) - } - } - for (const change of changes) { - for (const group of groups.values()) { - const value = readRouteValue(change.value, group.path) - const previous = - change.previousValue === undefined - ? undefined - : readRouteValue(change.previousValue, group.path) - // The where filter reads these same values, and a read that throws makes - // its predicate false, so an unreadable value matches no literal here. - deliver(group.byLiteral.get(value), change) - if (previous !== value) deliver(group.byLiteral.get(previous), change) - } - } - return routed -} diff --git a/packages/db/src/collection/lifecycle.ts b/packages/db/src/collection/lifecycle.ts index 20dbce7a2b..db812205bf 100644 --- a/packages/db/src/collection/lifecycle.ts +++ b/packages/db/src/collection/lifecycle.ts @@ -28,7 +28,7 @@ import type { CollectionStateManager } from './state' * to the timer armed when the last subscriber leaves, which still honours * `gcTime` exactly. */ -const UNSUBSCRIBED_GC_FLOOR_MS = 50 +export const UNSUBSCRIBED_GC_FLOOR_MS = 50 export class CollectionLifecycleManager< TOutput extends object = Record, diff --git a/packages/db/src/collection/mutations.ts b/packages/db/src/collection/mutations.ts index cd75a55f46..d22c4d6716 100644 --- a/packages/db/src/collection/mutations.ts +++ b/packages/db/src/collection/mutations.ts @@ -1,4 +1,8 @@ -import { withArrayChangeTracking, withChangeTracking } from '../proxy' +import { + withArrayChangeTracking, + withChangeTracking, + withFlatChangeTracking, +} from '../proxy' import { safeRandomUUID } from '../utils/uuid' import { createTransaction, getActiveTransaction } from '../transactions' import { @@ -25,6 +29,7 @@ import type { CollectionConfig, InsertConfig, OperationConfig, + OperationType, PendingMutation, StandardSchema, TransactionConfig, @@ -37,6 +42,17 @@ import type { TransactionScope } from '../transactions' import type { CollectionLifecycleManager } from './lifecycle' import type { CollectionStateManager } from './state' +// One random prefix per runtime keeps mutation ids unique across tabs and +// sessions; the counter avoids generating a random UUID per mutation. The +// prefix waits for the first mutation, because some runtimes reject random +// values at module scope. +let mutationIdPrefix: string | undefined +let mutationCount = 0 +function createMutationId(): string { + mutationIdPrefix ??= safeRandomUUID() + return `${mutationIdPrefix}-${++mutationCount}` +} + export class CollectionMutationsManager< TOutput extends object = Record, TKey extends string | number = string | number, @@ -187,6 +203,40 @@ export class CollectionMutationsManager< } } + /** + * A local-only Collection confirms its own writes. Without a user handler + * for this operation type, and with no other transaction unsettled, write + * the mutations as synced rows and return a completed transaction instead + * of publishing an optimistic overlay and confirming it a tick later. + */ + private commitLocalOnlyDirect( + mutations: Array>, + type: OperationType, + ): TransactionType | undefined { + const direct = this.state.localOnlyDirectWrite + if (!direct?.types.has(type)) return undefined + for (const transaction of this.state.transactions.values()) { + // A persisting transaction holds sync commits, and a pending one + // overlays them. + if ( + transaction.state === `pending` || + transaction.state === `persisting` + ) { + return undefined + } + } + const transaction = this.createTransaction({ + autoCommit: false, + metadata: { [DIRECT_TRANSACTION_METADATA_KEY]: true }, + mutationFn: () => Promise.resolve(), + }) + transaction.applyMutations(mutations) + direct.write(mutations) + transaction.setState(`completed`) + transaction.isPersisted.resolve(transaction) + return transaction + } + /** * Inserts one or more items into the collection */ @@ -201,6 +251,8 @@ export class CollectionMutationsManager< } const items = Array.isArray(data) ? data : [data] + // One timestamp per call; mutations replace these rather than mutate them. + const now = new Date() const mutations: Array> = [] const keysInCurrentBatch = new Set() @@ -218,7 +270,7 @@ export class CollectionMutationsManager< const globalKey = this.generateGlobalKey(key, item) const mutation: PendingMutation = { - mutationId: safeRandomUUID(), + mutationId: createMutationId(), original: {}, modified: validatedData, // Pick the values from validatedData based on what's passed in - this is for cases @@ -236,8 +288,8 @@ export class CollectionMutationsManager< syncMetadata: this.config.sync.getSyncMetadata?.() || {}, optimistic: config?.optimistic ?? true, type: `insert`, - createdAt: new Date(), - updatedAt: new Date(), + createdAt: now, + updatedAt: now, collection: this.collection, } @@ -262,6 +314,8 @@ export class CollectionMutationsManager< return ambientTransaction } else { + const localOnly = this.commitLocalOnlyDirect(mutations, `insert`) + if (localOnly) return localOnly // Create a new transaction with a mutation function that calls the onInsert handler const directOpTransaction = this.createTransaction({ metadata: { [DIRECT_TRANSACTION_METADATA_KEY]: true }, @@ -350,22 +404,28 @@ export class CollectionMutationsManager< return item }) as unknown as Array - let changesArray - if (isArray) { - // Use the proxy to track changes for all objects - changesArray = withArrayChangeTracking( + // Flat rows need no proxy; nested rows track changes through drafts. + const changesArray = + withFlatChangeTracking( currentObjects, - callback as (draft: Array) => void, - ) - } else { - const result = withChangeTracking( - currentObjects[0]!, - callback as (draft: TInput) => void, - ) - changesArray = [result] - } + callback as (drafts: Array | TInput) => void, + isArray, + ) ?? + (isArray + ? withArrayChangeTracking( + currentObjects, + callback as (draft: Array) => void, + ) + : [ + withChangeTracking( + currentObjects[0]!, + callback as (draft: TInput) => void, + ), + ]) // Create mutations for each object that has changes + // One timestamp per call; mutations replace these rather than mutate them. + const now = new Date() const mutations: Array< PendingMutation< TOutput, @@ -374,7 +434,7 @@ export class CollectionMutationsManager< > > = keysArray .map((key, index) => { - const itemChanges = changesArray[index] // User-provided changes for this specific item + const itemChanges = changesArray[index] // A fresh object the tracker recorded for this item // Skip items with no changes if (!itemChanges || Object.keys(itemChanges).length === 0) { @@ -403,19 +463,23 @@ export class CollectionMutationsManager< const globalKey = this.generateGlobalKey(modifiedItemId, modifiedItem) return { - mutationId: safeRandomUUID(), + mutationId: createMutationId(), original: originalItem, modified: modifiedItem, // Pick the values from modifiedItem based on what's passed in - this is for cases // where a schema has default values or transforms. The modified data has the extra // default or transformed values but for changes, we just want to show the data that // was actually passed in. - changes: Object.fromEntries( - Object.keys(itemChanges).map((k) => [ - k, - modifiedItem[k as keyof typeof modifiedItem], - ]), - ) as TInput, + // Without a schema, validation returns the tracker's fresh change + // object, which already holds exactly these values. + changes: (validatedUpdatePayload === itemChanges + ? itemChanges + : Object.fromEntries( + Object.keys(itemChanges).map((k) => [ + k, + modifiedItem[k as keyof typeof modifiedItem], + ]), + )) as TInput, globalKey, key, metadata: config.metadata as unknown, @@ -425,8 +489,8 @@ export class CollectionMutationsManager< >, optimistic: config.optimistic ?? true, type: `update`, - createdAt: new Date(), - updatedAt: new Date(), + createdAt: now, + updatedAt: now, collection: this.collection, } }) @@ -463,6 +527,9 @@ export class CollectionMutationsManager< // No need to check for onUpdate handler here as we've already checked at the beginning + const localOnly = this.commitLocalOnlyDirect(mutations, `update`) + if (localOnly) return localOnly + // Create a new transaction with a mutation function that calls the onUpdate handler const directOpTransaction = this.createTransaction({ metadata: { [DIRECT_TRANSACTION_METADATA_KEY]: true }, @@ -520,6 +587,8 @@ export class CollectionMutationsManager< const keysArray = Array.isArray(keys) ? keys : [keys] this.collection._sync.startSync() + // One timestamp per call; mutations replace these rather than mutate them. + const now = new Date() const mutations: Array< PendingMutation< TOutput, @@ -538,7 +607,7 @@ export class CollectionMutationsManager< `delete`, CollectionImpl > = { - mutationId: safeRandomUUID(), + mutationId: createMutationId(), original: this.state.get(key)!, modified: this.state.get(key)!, changes: this.state.get(key)!, @@ -551,8 +620,8 @@ export class CollectionMutationsManager< >, optimistic: config?.optimistic ?? true, type: `delete`, - createdAt: new Date(), - updatedAt: new Date(), + createdAt: now, + updatedAt: now, collection: this.collection, } @@ -570,6 +639,9 @@ export class CollectionMutationsManager< return ambientTransaction } + const localOnly = this.commitLocalOnlyDirect(mutations, `delete`) + if (localOnly) return localOnly + // Create a new transaction with a mutation function that calls the onDelete handler const directOpTransaction = this.createTransaction({ autoCommit: true, diff --git a/packages/db/src/collection/state.ts b/packages/db/src/collection/state.ts index 65419163cd..12607a9000 100644 --- a/packages/db/src/collection/state.ts +++ b/packages/db/src/collection/state.ts @@ -13,6 +13,7 @@ import type { StandardSchemaV1 } from '@standard-schema/spec' import type { ChangeMessage, CollectionConfig, + OperationType, OptimisticChangeMessage, PendingMutation, } from '../types' @@ -166,12 +167,15 @@ export class CollectionStateManager< // failed mutations must not add to, or erase a sibling's entry in, this set. public pendingLocalOrigins = new Set() - private virtualPropsCache = new WeakMap< - object, + // Keyed by row key, not row object: adding a WeakMap entry for each + // published row cost more than the copy it saves. Sync writes, deletes, + // and cleanup drop a key's entry. + private virtualPropsCache = new Map< + TKey, { + row: TOutput synced: boolean origin: VirtualOrigin - key: TKey collectionId: string enriched: WithVirtualProps } @@ -191,6 +195,16 @@ export class CollectionStateManager< private isDrainingSyncTransactions = false private syncRunGeneration = 0 public isLocalOnly = false + /** + * Set by a local-only Collection for operation types without a user handler. + * Their direct mutations can be written as synced rows at once. + */ + public localOnlyDirectWrite: + | { + types: ReadonlySet + write: (mutations: Array>) => void + } + | undefined /** * Creates a new CollectionState manager @@ -325,12 +339,12 @@ export class CollectionStateManager< const resolvedKey = existingRow.$key ?? virtualProps.$key const collectionId = existingRow.$collectionId ?? virtualProps.$collectionId - const cached = this.virtualPropsCache.get(row as object) + const cached = this.virtualPropsCache.get(resolvedKey) if ( cached && + cached.row === row && cached.synced === synced && cached.origin === origin && - cached.key === resolvedKey && cached.collectionId === collectionId ) { return cached.enriched @@ -345,10 +359,10 @@ export class CollectionStateManager< $collectionId: collectionId, } as WithVirtualProps - this.virtualPropsCache.set(row as object, { + this.virtualPropsCache.set(resolvedKey, { + row, synced, origin, - key: resolvedKey, collectionId, enriched, }) @@ -357,6 +371,7 @@ export class CollectionStateManager< } private clearOriginTrackingState(): void { + this.virtualPropsCache.clear() this.rowOrigins.clear() this.pendingLocalChanges.clear() this.pendingLocalOrigins.clear() @@ -377,6 +392,23 @@ export class CollectionStateManager< ) } + /** + * Visible entries whose stored row passes `prefilter`, enriched with virtual + * properties. Rows that fail are never enriched. + */ + public *entriesPassing( + prefilter: (row: object) => boolean, + ): IterableIterator<[TKey, WithVirtualProps]> { + // Without optimistic state, the visible rows are the synced rows in order. + const rows = + this.optimisticUpserts.size === 0 && this.optimisticDeletes.size === 0 + ? this.syncedData + : this.entries() + for (const [key, row] of rows) { + if (prefilter(row)) yield [key, this.enrichWithVirtualProps(row, key)] + } + } + /** * Creates a change message with virtual properties. * Uses the "add-if-missing" pattern so that pass-through from upstream @@ -386,9 +418,8 @@ export class CollectionStateManager< change: ChangeMessage, ): ChangeMessage, TKey> { const { __virtualProps } = change as InternalChangeMessage - const enrichedValue = __virtualProps?.value - ? this.enrichWithVirtualPropsSnapshot(change.value, __virtualProps.value) - : this.enrichWithVirtualProps(change.value, change.key) + // The cache holds one row per key, so the previous row goes first and + // the published value stays the row that later reads return. const enrichedPreviousValue = change.previousValue ? __virtualProps?.previousValue ? this.enrichWithVirtualPropsSnapshot( @@ -397,6 +428,11 @@ export class CollectionStateManager< ) : this.enrichWithVirtualProps(change.previousValue, change.key) : undefined + const enrichedValue = __virtualProps?.value + ? this.enrichWithVirtualPropsSnapshot(change.value, __virtualProps.value) + : this.enrichWithVirtualProps(change.value, change.key) + // A deleted key, such as a rolled-back insert, has no row to read again. + if (change.type === `delete`) this.virtualPropsCache.delete(change.key) return { key: change.key, @@ -503,23 +539,6 @@ export class CollectionStateManager< } } - /** - * Visible entries whose stored row passes `prefilter`, enriched with virtual - * properties. Rows that fail are never copied. - */ - public *entriesPassing( - prefilter: (row: object) => boolean, - ): IterableIterator<[TKey, WithVirtualProps]> { - // Without optimistic state, the visible rows are the synced rows in order. - const rows = - this.optimisticUpserts.size === 0 && this.optimisticDeletes.size === 0 - ? this.syncedData - : this.entries() - for (const [key, row] of rows) { - if (prefilter(row)) yield [key, this.enrichWithVirtualProps(row, key)] - } - } - /** * Get all entries (virtual derived state) */ @@ -1587,8 +1606,7 @@ export class CollectionStateManager< // A sync source may reuse a live-reading row object, making an // enriched snapshot cached for an earlier publication stale. - if (operation.type !== `delete`) - this.virtualPropsCache.delete(operation.value) + this.virtualPropsCache.delete(key) // Update synced data switch (operation.type) { @@ -2135,6 +2153,7 @@ export class CollectionStateManager< this.hasAppliedAdapterTruncate = false this.clearOriginTrackingState() this.isLocalOnly = false + this.localOnlyDirectWrite = undefined this.size = 0 this.pendingSyncedTransactions = [] this.pendingSyncedProjection = { states: new Map(), truncated: false } diff --git a/packages/db/src/collection/subscription.ts b/packages/db/src/collection/subscription.ts index 4f76ee8797..8285bad587 100644 --- a/packages/db/src/collection/subscription.ts +++ b/packages/db/src/collection/subscription.ts @@ -12,9 +12,7 @@ import { LoadSubsetOperationAbortedError } from '../errors.js' import { createFilterFunctionFromExpression, createFilteredCallback, - findEqualityRoute, } from './change-events.js' -import type { EqualityRoute } from './change-events.js' import type { BasicExpression, OrderBy } from '../query/ir.js' import type { IndexReader } from '../indexes/base-index.js' import type { @@ -174,9 +172,6 @@ export class CollectionSubscription private filteredCallback: (changes: Array>) => boolean - /** Field and literal the where clause requires, if it has a cheap one. */ - private readonly equalityRoute: EqualityRoute | undefined - private orderByIndex: IndexReader | undefined // Status tracking @@ -239,10 +234,6 @@ export class CollectionSubscription this.callback = callbackWithSentKeysTracking - this.equalityRoute = options.whereExpression - ? findEqualityRoute(options.whereExpression) - : undefined - // Create a filtered callback if where clause is provided this.filteredCallback = options.whereExpression ? createFilteredCallback(this.callback, options) @@ -1133,23 +1124,6 @@ export class CollectionSubscription return this.filteredCallback(newChanges) } - /** - * The route through which this subscription may receive only the changes - * whose value or previous value holds the route's literal. A change reaches - * the where filter only through those values, so the others cannot publish, - * and sent-key records cover published rows only. Stale published rows and - * truncate replay consume unfiltered changes, so no route applies then. - */ - get changeRoute(): EqualityRoute | undefined { - if ( - this.stalePublishedRows.size > 0 || - this.truncateReplayState !== undefined - ) { - return undefined - } - return this.equalityRoute - } - /** Keep direct snapshot reads private while an authoritative replay is open. */ private publishSnapshot(changes: Array>): void { if (!this.bufferPrivately(changes)) this.callback(changes) diff --git a/packages/db/src/duplicate-instance-check.ts b/packages/db/src/duplicate-instance-check.ts index 2f38a562f8..0010359e14 100644 --- a/packages/db/src/duplicate-instance-check.ts +++ b/packages/db/src/duplicate-instance-check.ts @@ -18,11 +18,19 @@ function isBrowserTopWindow(): boolean { // Detect duplicate @tanstack/db instances (dev-only, browser top-window only) const DB_INSTANCE_MARKER = Symbol.for(`@tanstack/db/instance-marker`) -const DEV = - typeof process !== `undefined` && process.env.NODE_ENV !== `production` +// Bundlers inline these reads even where `process` does not exist, so they +// are read directly; without a bundler or `process`, the check stays off. +function readEnv(read: () => string | undefined): string | undefined | null { + try { + return read() + } catch { + return null + } +} +const nodeEnv = readEnv(() => process.env.NODE_ENV) +const DEV = nodeEnv !== null && nodeEnv !== `production` const DISABLED = - typeof process !== `undefined` && - process.env.TANSTACK_DB_DISABLE_DUP_CHECK === `1` + readEnv(() => process.env.TANSTACK_DB_DISABLE_DUP_CHECK) === `1` if (DEV && !DISABLED && isBrowserTopWindow()) { if ((globalThis as any)[DB_INSTANCE_MARKER]) { diff --git a/packages/db/src/live-query-adapter.ts b/packages/db/src/live-query-adapter.ts index b13001d0a8..6979829a25 100644 --- a/packages/db/src/live-query-adapter.ts +++ b/packages/db/src/live-query-adapter.ts @@ -31,6 +31,19 @@ export function isCollection( ) } +/** + * The Collection an adapter hands users as `collection`. A pooled live query + * stands in for its live-query Collection and builds it only when touched. + */ +export function getPublicCollection< + T extends Collection | null | undefined, +>(collection: T): T { + return ( + (collection as { publicCollection?: T } | null | undefined) + ?.publicCollection ?? collection + ) +} + /** Whether a collection yields a single result (`findOne`) rather than an array. */ export function isSingleResultCollection( collection: Collection, diff --git a/packages/db/src/live-query-observer.ts b/packages/db/src/live-query-observer.ts index 08dce318ac..4326c4f188 100644 --- a/packages/db/src/live-query-observer.ts +++ b/packages/db/src/live-query-observer.ts @@ -1,6 +1,7 @@ import { LiveQueryObserverDisposedError } from './errors.js' import { getLiveQueryStatusFlags, + getPublicCollection, isSingleResultCollection, } from './live-query-adapter.js' import { getBuilderFromConfig } from './query/live/collection-registry.js' @@ -21,6 +22,12 @@ export type LiveQueryPersistedStatus = // must remain observable without treating source readiness as query readiness. const INITIAL_RENDER_PRELOADS = new WeakSet>() +const NO_PERSISTED_READINESS = { + status: `unavailable`, + error: undefined, +} as const +const noop = () => {} + interface PersistedSourceEntry { collection: Collection readiness: PersistedReadinessSource @@ -29,6 +36,8 @@ interface PersistedSourceEntry { function collectPersistedReadinessSources( root: Collection, ): ReadonlyArray | undefined { + // A few integrations provide Collection-compatible objects without config. + if (!(root as { config?: unknown }).config) return undefined const seen = new Set>() const sources: Array = [] const visit = (collection: Collection): boolean => { @@ -307,7 +316,7 @@ class LiveQueryObserverImpl< this.cachedSnapshot = { state, data: singleResult ? data[0] : data, - collection, + collection: getPublicCollection(collection), layoutRevision: this.layoutRevision, status, ...getLiveQueryStatusFlags(status), @@ -355,7 +364,7 @@ class LiveQueryObserverImpl< error: unknown | undefined } { const sources = this.persistedSources - if (!sources) return { status: `unavailable`, error: undefined } + if (!sources) return NO_PERSISTED_READINESS let loading = false let error: unknown | undefined let failed = false @@ -751,7 +760,7 @@ class LiveQueryObserverImpl< ? subscribeLayoutChanges.call(collection, () => notify([], collection.status, true), ) - : () => {} + : noop const persistedUnsubs = this.persistedSources?.map((source) => source.readiness.subscribe(() => { if (this.disposed || this.subscriptions.size === 0) return @@ -792,7 +801,7 @@ class LiveQueryObserverImpl< : this.diffEntries(previousEntries, nextEntries), ) }) - : () => {} + : noop const release = () => { clientUnsub() statusUnsub() @@ -816,8 +825,19 @@ class LiveQueryObserverImpl< // listener delivery: useSyncExternalStore performs its consistency read // immediately after subscribe returns. this.flushPublications(!this.wholesale) - const { entries, revision } = this.readEntries(collection) - this.updateCachedEntries(entries, revision) + // The render-time read is still current unless the handshake published. + const revision = this.getCollectionRevision(collection) + const layoutRevision = this.getCollectionLayoutRevision(collection) + if ( + revision === undefined || + this.cachedEntries === undefined || + revision !== this.cachedCollectionRevision || + layoutRevision !== this.cachedCollectionLayoutRevision + ) { + const { entries } = this.readEntries(collection) + this.updateCachedEntries(entries, revision) + this.cachedCollectionLayoutRevision = layoutRevision + } } if (this.hasHydrationSeed()) { if (!this.wholesale) this.seed(Array.from(this.subscriptions)[0]!) diff --git a/packages/db/src/live-query-options.ts b/packages/db/src/live-query-options.ts index 987810b301..2901c05264 100644 --- a/packages/db/src/live-query-options.ts +++ b/packages/db/src/live-query-options.ts @@ -1,11 +1,16 @@ import { BaseQueryBuilder } from './query/builder/index.js' import { isCollection } from './live-query-adapter.js' +import { createLiveQueryCollection } from './query/live-query-collection.js' +import { + createPooledLiveQuery, + getPooledQueryIdentity, +} from './query/pooled-live-query.js' import { getStableQueryBuilderHash, getStableValueHash, } from './query/ir-stable-identity.js' import { getStringCollationIdentity } from './query/runtime-reference-identity.js' -import type { CollectionImpl } from './collection/index.js' +import type { Collection, CollectionImpl } from './collection/index.js' import type { CollectionOptionsIdentity } from './collection-options.js' import type { CollectionOptions, DbClient } from './client.js' import type { @@ -110,7 +115,12 @@ export function prepareLiveQueryValue( export function getPreparedLiveQueryIdentity(value: unknown): unknown { if (isCollection(value)) return [`collection`, value.id] if (value instanceof BaseQueryBuilder) { - return [`query`, getStableQueryBuilderHash(value)] + // A pooled query's fields and literals identify its rows without + // canonicalizing its whole IR on every render. + const pooled = getPooledQueryIdentity(value) + return pooled === undefined + ? [`query`, getStableQueryBuilderHash(value)] + : [`pooled`, pooled] } if (value && typeof value === `object` && `query` in value) { const config = value as LiveQueryCollectionConfig @@ -142,3 +152,53 @@ export function getLiveQueryHash( return getStableValueHash(identity, `queryKey`) } + +/** + * Resolve an adapter's query value to what its observer watches: `null` for a + * disabled query, an existing Collection with sync started, or a live query. + * A query builder whose shape a shared partition can serve gets a pooled view + * instead of its own live-query Collection. + */ +// A config that names only its query and lifetime describes the same live +// query as its builder. Any other option shapes its Collection, so compiles. +function poolConfig( + config: LiveQueryCollectionConfig, + gcTime: number | undefined, +): Collection | undefined { + if ( + !(config.query instanceof BaseQueryBuilder) || + !Object.keys(config).every((key) => key === `query` || key === `gcTime`) + ) { + return undefined + } + return createPooledLiveQuery(config.query, { + gcTime: config.gcTime ?? gcTime, + }) +} + +export function resolveLiveQueryValue( + value: unknown, + { gcTime, pool = true }: { gcTime?: number; pool?: boolean } = {}, +): Collection | null { + if (value === undefined || value === null) return null + if (isCollection(value)) { + value.startSyncImmediate() + return value + } + if (value instanceof BaseQueryBuilder) { + return ( + (pool ? createPooledLiveQuery(value, { gcTime }) : undefined) ?? + createLiveQueryCollection({ query: value, startSync: true, gcTime }) + ) + } + if (typeof value === `object`) { + const config = value as LiveQueryCollectionConfig + return ( + (pool ? poolConfig(config, gcTime) : undefined) ?? + createLiveQueryCollection({ startSync: true, gcTime, ...config }) + ) + } + throw new Error( + `A live query must be a QueryBuilder, LiveQueryCollectionConfig, Collection, undefined, or null. Got: ${typeof value}`, + ) +} diff --git a/packages/db/src/local-only.ts b/packages/db/src/local-only.ts index 9de92f4d63..5aece5cb77 100644 --- a/packages/db/src/local-only.ts +++ b/packages/db/src/local-only.ts @@ -187,7 +187,11 @@ export function localOnlyCollectionOptions< const collectionId = id ?? safeRandomUUID() // Create the sync configuration with transaction confirmation capability - const syncResult = createLocalOnlySync(initialData) + const directTypes = new Set() + if (!onInsert) directTypes.add(`insert`) + if (!onUpdate) directTypes.add(`update`) + if (!onDelete) directTypes.add(`delete`) + const syncResult = createLocalOnlySync(initialData, directTypes) /** * Create wrapper handlers that call user handlers first, then confirm transactions @@ -304,7 +308,9 @@ export function localOnlyCollectionOptions< * @returns Object with sync configuration and confirmOperationsSync function */ function createLocalOnlySync( - initialData?: Array, + initialData: Array | undefined, + // Operation types without a user handler, which confirm synchronously. + directTypes: ReadonlySet, ) { // Capture sync functions and collection for transaction confirmation let syncBegin: (() => void) | null = null @@ -329,6 +335,10 @@ function createLocalOnlySync( syncCommit = commit collection = params.collection params.collection._state.isLocalOnly = true + params.collection._state.localOnlyDirectWrite = { + types: directTypes, + write: confirmOperationsSync, + } // Apply initial data if provided if (initialData && initialData.length > 0) { diff --git a/packages/db/src/proxy.ts b/packages/db/src/proxy.ts index 13e5188c48..b10ff85dcf 100644 --- a/packages/db/src/proxy.ts +++ b/packages/db/src/proxy.ts @@ -306,7 +306,7 @@ function deepClone( function draftValuesEqual( left: unknown, right: unknown, - paired = new Map(), + paired?: Map, ): boolean { return deepEqualsInternal(left, right, paired, true) } @@ -359,7 +359,9 @@ export function createChangeProxy< copy_: parent ? ((valueCopies.get(target) ?? target) as T) : deepClone(target, valueCopies), - originalObject: deepClone(target), + // The root target is the stored row, which the draft never writes, so it + // is its own baseline. A nested target is a draft copy that writes reach. + originalObject: parent ? deepClone(target) : target, modified: false, assigned_: Object.create(null), parent, @@ -455,6 +457,30 @@ export function createChangeProxy< // Create a proxy for the target object. // Use the unfrozen copy_ as the proxy target to avoid Proxy invariant violations // when the original target is frozen (e.g., from Immer) + // Whether a value equals the original's own value for a field. + function isOriginalValue(prop: string | symbol, value: unknown): boolean { + const original = changeTracker.originalObject + return ( + Object.hasOwn(original, prop) && + draftValuesEqual(value, original[prop as keyof T]) + ) + } + + // Records a write the draft now holds. Assignment and defineProperty share + // it so they report the same change. + function recordWrite(prop: string | symbol, reverted: boolean) { + if (reverted) { + delete changeTracker.assigned_[prop.toString()] + // Some properties may still be changed; checkParentStatus clears + // tracking here and up the chain once everything is reverted. + changeTracker.modified = true + checkParentStatus(changeTracker) + } else { + changeTracker.assigned_[prop.toString()] = true + markChanged(changeTracker) + } + } + const proxy = new Proxy(changeTracker.copy_, { get(ptarget, prop, receiver) { const value = changeTracker.copy_[prop as keyof T] @@ -594,7 +620,12 @@ export function createChangeProxy< return value }, - set(_sobj, prop, value) { + set(ptarget, prop, value) { + // An accessor the callback defined behaves as on a plain object; a + // getter without a setter rejects even a write of its own value. + if (Reflect.getOwnPropertyDescriptor(ptarget, prop)?.get) { + return Reflect.set(ptarget, prop, value) + } const currentValue = changeTracker.copy_[prop as keyof T] // Only track the change if the value is actually different @@ -602,34 +633,12 @@ export function createChangeProxy< !Object.hasOwn(changeTracker.copy_, prop) || !draftValuesEqual(currentValue, value) ) { - // Check if the new value is equal to the original value - // Important: Use the originalObject to get the true original value - const originalValue = changeTracker.originalObject[prop as keyof T] - const isRevertToOriginal = - Object.hasOwn(changeTracker.originalObject, prop) && - draftValuesEqual(value, originalValue) - - if (isRevertToOriginal) { - // If the value is reverted to its original state, remove it from changes - delete changeTracker.assigned_[prop.toString()] - - // Make sure the copy is updated with the original value - changeTracker.copy_[prop as keyof T] = deepClone(originalValue) - - // Some properties may still be changed; checkParentStatus clears - // tracking here and up the chain once everything is reverted. - changeTracker.modified = true - checkParentStatus(changeTracker) - } else { - // Set the value on the copy - changeTracker.copy_[prop as keyof T] = value - - // Track that this property was assigned - store using the actual property (symbol or string) - changeTracker.assigned_[prop.toString()] = true - - // Mark this object and its ancestors as modified - markChanged(changeTracker) - } + const reverted = isOriginalValue(prop, value) + // A revert restores a copy so the draft never aliases the row. + changeTracker.copy_[prop as keyof T] = reverted + ? deepClone(changeTracker.originalObject[prop as keyof T]) + : value + recordWrite(prop, reverted) } return true @@ -639,13 +648,13 @@ export function createChangeProxy< // Forward the defineProperty to the target to maintain Proxy invariants // This allows Object.seal() and Object.freeze() to work on the proxy const result = Reflect.defineProperty(ptarget, prop, descriptor) - // A value or an accessor changes what the key reads. Sealing does not. + // A value or an accessor changes what the key reads, counted by the + // value it reads as an assignment would be. Sealing does not. if ( result && (`value` in descriptor || descriptor.get || descriptor.set) ) { - changeTracker.assigned_[prop.toString()] = true - markChanged(changeTracker) + recordWrite(prop, isOriginalValue(prop, Reflect.get(ptarget, prop))) } return result }, @@ -669,11 +678,13 @@ export function createChangeProxy< const stringProp = typeof prop === `symbol` ? prop.toString() : prop if (Object.hasOwn(dobj, prop)) { - // Check if the property exists in the original object - const hadPropertyInOriginal = Object.hasOwn( - changeTracker.originalObject, - prop, - ) + // A hidden original field is not row data, so as with a plain + // delete of it, removing it is not a change. + const hadPropertyInOriginal = + Object.prototype.propertyIsEnumerable.call( + changeTracker.originalObject, + prop, + ) // Forward the delete to the target using Reflect // This respects Object.seal/preventExtensions constraints @@ -823,3 +834,61 @@ export function withArrayChangeTracking( return deepClone(getChanges(), undefined, true) } + +// Whether every own field of a plain object holds a primitive or a function. +function isFlatPlainObject(value: object): boolean { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) return false + for (const key in value) { + // A getter may return a new value on each read; the proxy reads it once. + const { value: field, get } = Object.getOwnPropertyDescriptor(value, key)! + if (get || (field !== null && typeof field === `object`)) return false + } + return Object.getOwnPropertySymbols(value).length === 0 +} + +/** + * Change tracking for flat rows without proxies. A draft is a shallow copy, + * and its changes are the fields that differ from the row afterwards under + * the same equality the draft proxy uses for primitives. Returns undefined + * when any row has a nested object, a getter, a symbol key, or a class + * prototype, so the caller falls back to the proxy. + */ +export function withFlatChangeTracking( + targets: Array, + callback: (drafts: Array | T) => void, + asArray: boolean, +): Array> | undefined { + if (!targets.every(isFlatPlainObject)) return undefined + const drafts = targets.map((target) => ({ ...target })) + callback(asArray ? drafts : drafts[0]!) + return drafts.map((draft, index) => { + const original = targets[index] as Record + const changes: Record = {} + let assignedObject = false + for (const key in draft) { + const value = (draft as Record)[key] + const before = original[key] + // Only a hidden field can hold an object; compare it as the proxy does. + if ( + !Object.hasOwn(original, key) || + !( + value === before || + Object.is(value, before) || + (typeof before === `object` && + before !== null && + draftValuesEqual(value, before)) + ) + ) { + defineDataProperty(changes, key, value) + if (value !== null && typeof value === `object`) assignedObject = true + } + } + for (const key in original) { + if (!Object.hasOwn(draft, key)) + defineDataProperty(changes, key, undefined) + } + // A callback may assign objects; detach them as the proxy path does. + return assignedObject ? deepClone(changes, undefined, true) : changes + }) +} diff --git a/packages/db/src/query/builder/index.ts b/packages/db/src/query/builder/index.ts index 1860382673..29265926f7 100644 --- a/packages/db/src/query/builder/index.ts +++ b/packages/db/src/query/builder/index.ts @@ -135,19 +135,26 @@ type FnSelectQueryResult = : QueryBuilder> export class BaseQueryBuilder { - private readonly query: Partial = {} + private readonly query: Partial constructor( query: Partial = {}, private readonly resolveCollection?: CollectionResolver, + /** @internal Whether the builder may keep `query` without copying it. */ + owned = false, ) { - this.query = { ...query } + this.query = owned ? query : { ...query } } private _clone( query: Partial, ): BaseQueryBuilder { - return new BaseQueryBuilder(query, this.resolveCollection) + // Every clone receives a freshly spread query, so it needs no copy. + return new BaseQueryBuilder( + query, + this.resolveCollection, + true, + ) } /** diff --git a/packages/db/src/query/builder/ref-proxy-identity.ts b/packages/db/src/query/builder/ref-proxy-identity.ts index 57e354fb26..bc963425ca 100644 --- a/packages/db/src/query/builder/ref-proxy-identity.ts +++ b/packages/db/src/query/builder/ref-proxy-identity.ts @@ -1,11 +1,25 @@ import type { RefProxy } from './ref-proxy.js' -const refProxies = new WeakSet() +/** + * Ref proxy traps answer this module-private key, so a user object shaped + * like a ref proxy is never mistaken for one. + */ +export const REF_PROXY_BRAND: unique symbol = Symbol(`refProxy`) -export function registerRefProxy(value: object): void { - refProxies.add(value) +/** The brand a ref proxy answers, or undefined for any other value. */ +export function readRefProxyBrand(value: unknown): unknown { + if (!value || typeof value !== `object`) return undefined + try { + const brand = (value as Record)[REF_PROXY_BRAND] + return brand === true || Array.isArray(brand) ? brand : undefined + } catch { + // A revoked proxy, such as a finished Immer draft, is not a ref proxy. + return undefined + } } export function isRefProxy(value: any): value is RefProxy { - return value && typeof value === `object` && refProxies.has(value) + return ( + value && typeof value === `object` && readRefProxyBrand(value) !== undefined + ) } diff --git a/packages/db/src/query/builder/ref-proxy.ts b/packages/db/src/query/builder/ref-proxy.ts index 5ca672f567..af19b68fb5 100644 --- a/packages/db/src/query/builder/ref-proxy.ts +++ b/packages/db/src/query/builder/ref-proxy.ts @@ -1,5 +1,5 @@ import { PropRef, Value, isBasicOrAggregateExpression } from '../ir.js' -import { isRefProxy, registerRefProxy } from './ref-proxy-identity.js' +import { REF_PROXY_BRAND, readRefProxyBrand } from './ref-proxy-identity.js' import { getWrapperExpressionName } from './wrapper-identity.js' import type { BasicExpression } from '../ir.js' import type { IsPlainObject, RefLeaf } from './types.js' @@ -88,6 +88,7 @@ export function createSingleRowRefProxy< if (prop === `__path`) return path if (prop === `__sourceAlias`) return undefined if (prop === `__type`) return undefined // Type is only for TypeScript inference + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) const newPath = [...path, String(prop)] @@ -121,8 +122,6 @@ export function createSingleRowRefProxy< return Reflect.getOwnPropertyDescriptor(target, prop) }, }) - - registerRefProxy(proxy) cache.set(pathKey, proxy) return proxy } @@ -138,25 +137,29 @@ export function createSingleRowRefProxy< export function createRefProxy>( aliases: Array, ): RefProxy & T { - const cache = new Map() + // Each path has one proxy, cached by its parent under the property name. + const aliasProxies = new Map() let accessId = 0 // Monotonic counter to record evaluation order function createProxy(path: Array): any { - const pathKey = JSON.stringify(path) - if (cache.has(pathKey)) { - return cache.get(pathKey) - } - + let children: Map | undefined const proxy = new Proxy({} as any, { get(target, prop, receiver) { if (prop === `__refProxy`) return true if (prop === `__path`) return path if (prop === `__sourceAlias`) return path[0] if (prop === `__type`) return undefined // Type is only for TypeScript inference + // Answers with the path so toExpression reads it in one trap. + if (prop === REF_PROXY_BRAND) return path if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) - const newPath = [...path, String(prop)] - return createProxy(newPath) + children ??= new Map() + let child = children.get(prop) + if (child === undefined) { + child = createProxy([...path, prop]) + children.set(prop, child) + } + return child }, has(target, prop) { @@ -195,9 +198,6 @@ export function createRefProxy>( return Reflect.getOwnPropertyDescriptor(target, prop) }, }) - - registerRefProxy(proxy) - cache.set(pathKey, proxy) return proxy } @@ -208,11 +208,17 @@ export function createRefProxy>( if (prop === `__path`) return [] if (prop === `__sourceAlias`) return undefined if (prop === `__type`) return undefined // Type is only for TypeScript inference + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) const propStr = String(prop) if (aliases.includes(propStr) || aliases.includes(`*`)) { - return createProxy([propStr]) + let proxy = aliasProxies.get(propStr) + if (proxy === undefined) { + proxy = createProxy([propStr]) + aliasProxies.set(propStr, proxy) + } + return proxy } return undefined @@ -249,8 +255,6 @@ export function createRefProxy>( return undefined }, }) - - registerRefProxy(rootProxy) return rootProxy } @@ -285,6 +289,7 @@ export function createRefProxyWithSelected>( if (prop === `__path`) return [`$selected`, ...path] if (prop === `__sourceAlias`) return `$selected` if (prop === `__type`) return undefined + if (prop === REF_PROXY_BRAND) return true if (typeof prop === `symbol`) return Reflect.get(target, prop, receiver) const newPath = [...path, String(prop)] @@ -318,8 +323,6 @@ export function createRefProxyWithSelected>( return Reflect.getOwnPropertyDescriptor(target, prop) }, }) - - registerRefProxy(proxy) cache.set(pathKey, proxy) return proxy } @@ -358,7 +361,6 @@ export function createRefProxyWithSelected>( T & { $selected: SingleRowRefProxy } - registerRefProxy(selectedRootProxy) return selectedRootProxy } @@ -371,8 +373,19 @@ export function createRefProxyWithSelected>( export function toExpression(value: T): BasicExpression export function toExpression(value: RefProxy): BasicExpression export function toExpression(value: any): BasicExpression { - if (isRefProxy(value)) { - return new PropRef(value.__path, value.__sourceAlias) + // A primitive cannot be a ref proxy, a wrapper, or an expression. + if ( + value === null || + (typeof value !== `object` && typeof value !== `function`) + ) { + return new Value(value) + } + const brand = readRefProxyBrand(value) + if (brand !== undefined) { + // An alias-qualified ref proxy answers the brand with its path. + return Array.isArray(brand) + ? new PropRef(brand, brand[0]) + : new PropRef(value.__path, value.__sourceAlias) } // toArray(), concat(toArray()), and materialize() must be used as direct // select fields, not inside expressions diff --git a/packages/db/src/query/equality-conjunct.ts b/packages/db/src/query/equality-conjunct.ts new file mode 100644 index 0000000000..a0765a8f24 --- /dev/null +++ b/packages/db/src/query/equality-conjunct.ts @@ -0,0 +1,50 @@ +import { normalizeValue } from '../utils/comparison.js' +import { isVirtualPropName } from '../virtual-props.js' +import type { BasicExpression, PropRef } from './ir.js' + +type Row = Record + +/** An `eq(field, literal)` conjunct on a row's own field. */ +export type EqualityConjunct = { path: Array; literalKey: string } + +// Typed so that 1, '1', and true stay distinct. +export function equalityKey(value: unknown): string | undefined { + const normalized = normalizeValue(value) + const type = typeof normalized + return type === `string` || type === `number` || type === `boolean` + ? `${type}:${String(normalized)}` + : undefined +} + +export function readPath(row: Row, path: Array): unknown { + try { + let value: unknown = row + for (const segment of path) value = (value as Row | undefined)?.[segment] + return value + } catch { + // The full predicate treats a throwing read as false. + return undefined + } +} + +/** + * The `eq(field, literal)` conjunct `expression` is, with the field's path in + * the row as `rowPath` gives it, or undefined for any other expression. + */ +export function equalityConjunct( + expression: BasicExpression, + rowPath: (ref: PropRef) => Array | undefined, +): EqualityConjunct | undefined { + if (expression.type !== `func` || expression.name !== `eq`) return undefined + const args = expression.args + if (args.length !== 2) return undefined + const left = args[0]! + const right = args[1]! + const ref = left.type === `ref` ? left : right + const literal = left.type === `val` ? left : right + if (ref.type !== `ref` || literal.type !== `val`) return undefined + const path = rowPath(ref) + if (!path?.length || isVirtualPropName(path[0]!)) return undefined + const literalKey = equalityKey(literal.value) + return literalKey === undefined ? undefined : { path, literalKey } +} diff --git a/packages/db/src/query/ir.ts b/packages/db/src/query/ir.ts index 446f92019d..8940ad287f 100644 --- a/packages/db/src/query/ir.ts +++ b/packages/db/src/query/ir.ts @@ -88,17 +88,18 @@ abstract class BaseExpression { export class CollectionRef extends BaseExpression { public type = `collectionRef` as const - /** Opaque runtime identity; aliases are lexical names only. */ - public readonly sourceId!: string + // Not an own property, so structural identity and hashing ignore it. + readonly #sourceId = `source-${++nextCollectionSourceId}` constructor( public collection: CollectionImpl, public alias: string, ) { super() - Object.defineProperty(this, `sourceId`, { - value: `source-${++nextCollectionSourceId}`, - enumerable: false, - }) + } + + /** Opaque runtime identity; aliases are lexical names only. */ + get sourceId(): string { + return this.#sourceId } } @@ -148,11 +149,9 @@ export class PropRef extends BaseExpression { sourceAlias?: string, ) { super() + // Present only when given, so unqualified refs keep their shape. if (sourceAlias !== undefined) { - Object.defineProperty(this, `sourceAlias`, { - value: sourceAlias, - enumerable: true, - }) + ;(this as { sourceAlias?: string }).sourceAlias = sourceAlias } } } diff --git a/packages/db/src/query/live/ARCHITECTURE.md b/packages/db/src/query/live/ARCHITECTURE.md index e91d6202ce..ac20f19818 100644 --- a/packages/db/src/query/live/ARCHITECTURE.md +++ b/packages/db/src/query/live/ARCHITECTURE.md @@ -1244,6 +1244,25 @@ active query acquisition, and persisted retention are distinct owner tokens. The live-query graph publishes coherent rows but does not own query-db cache or listener lifetime. +### Pooled live queries + +A framework adapter may serve a live query from an equality partition instead +of this graph. That applies only when the query reads one eager, +non-persisted source Collection, its `where` has at least one +`eq(field, literal)` conjunct, every other conjunct reads only that row's own fields, any `orderBy` reads +only that row's own fields without a custom string comparator, and it has no +other clause, `limit`, `offset`, `DbClient`, or Suspense key. Such a +pooled live query reads the partition group for its `eq` literal tuple and +filters it by its remaining conjuncts with the compiler's evaluator. Queries +with different orders use different partitions, whose groups sort rows with +the compiler's comparator and break ties by key +(`packages/db/src/query/pooled-live-query.ts`). It builds its live-query +Collection only when the application reads it. + +A pooled live query must publish what its live-query Collection would: the same rows, in its order or else key order, with the same values and status. When its source +starts cleanup, it enters the same terminal error and keeps its last rows. +The pooled live query oracle compares the two over generated histories. + ### Physical planning and work Correct relation state does not prove efficient work. When an applicable index @@ -1354,6 +1373,7 @@ keep the meanings defined there. | Replay lease balance, reference-counted peers, and failed-start recovery | `packages/db/tests/replay-adapter-ownership.test.ts` | | Reachable nested shape | `packages/query-db-collection/tests/includes-work-counter-oracle.test.ts` | | Cleanup-start invalidation, settlement, and restart admission | `packages/db/tests/collection-cleanup-restart-oracle.test.ts` | +| Pooled live queries match their live-query Collection, including cleanup | `packages/db/tests/query/pooled-live-query-oracle.property.test.ts` | Each oracle identifies the first divergent checkpoint and compares either the whole result or one exact structural difference. Correlated-materialization diff --git a/packages/db/src/query/pooled-live-query.ts b/packages/db/src/query/pooled-live-query.ts new file mode 100644 index 0000000000..6ad3a43317 --- /dev/null +++ b/packages/db/src/query/pooled-live-query.ts @@ -0,0 +1,715 @@ +import { SortedMap } from '../SortedMap.js' +import { CleanupQueue } from '../collection/cleanup-queue.js' +import { UNSUBSCRIBED_GC_FLOOR_MS } from '../collection/lifecycle.js' +import { makeComparator } from '../utils/comparison.js' +import { isVirtualPropName } from '../virtual-props.js' +import { getPersistedReadinessSource } from '../persisted-readiness.js' +import { getWhereExpression } from './ir.js' +import { equalityConjunct, equalityKey, readPath } from './equality-conjunct.js' +import { compileExpression, toBooleanPredicate } from './compiler/evaluators.js' +import { buildCompareOptions } from './compiler/order-by.js' +import { createLiveQueryCollection } from './live-query-collection.js' +import type { BasicExpression, OrderBy, QueryIR } from './ir.js' +import type { BaseQueryBuilder } from './builder/index.js' +import type { Collection, CollectionImpl } from '../collection/index.js' +import type { ChangeMessage, CollectionStatus } from '../types.js' +import type { + CollectionEventHandler, + CollectionStatusChangeEvent, +} from '../collection/events.js' + +/** + * Live queries that filter one source Collection only by `eq(field, literal)` + * share one partition of that source per filtered field set. Each query reads + * the group for its literal tuple, so mounting many queries of one shape + * costs a lookup each instead of a compiled graph and a source subscription. + * + * A group holds the rows the partition's source subscription has published, + * keyed by the same normalized equality that `eq` uses: a Date equals its + * timestamp, `NaN` equals `NaN`, `-0` equals `0`, and nullish values match no + * literal. + */ + +type Row = Record +type Listener = (changes: Array>) => void +type StatusListener = CollectionEventHandler<`status:change`> + +interface PartitionGroup { + key: string + // Key order, as in a live-query Collection without orderBy. + rows: SortedMap + listeners: Set + revision: number + layoutRevision: number +} + +const partitionsBySource = new WeakMap>() + +// What a view reads for a value with no rows and no watchers. Real groups +// draw revisions from their partition's clock, which starts above 0. +const EMPTY_GROUP: PartitionGroup = { + key: ``, + rows: new SortedMap(), + listeners: new Set(), + revision: 0, + layoutRevision: 0, +} + +// Length-prefixed, so no two part lists share an encoding. +function appendGroupKeyPart(groupKey: string, part: string): string { + return `${groupKey}${part.length}:${part}` +} + +class Partition { + private readonly groups = new Map() + // Revisions for every group, so a recreated group never repeats one. + private clock = 0 + private subscription: { unsubscribe: () => void } | undefined + private stopStatusEvents: (() => void) | undefined + // One source status listener serves every view of this partition. + readonly statusListeners = new Set() + private listenerCount = 0 + private gcTime = 0 + /** Set when the source starts cleanup; groups keep their last rows. */ + terminated = false + + constructor( + private readonly source: CollectionImpl, + private readonly paths: Array>, + // Row order for an `orderBy` shape; key order otherwise. + private readonly compareRows: ((a: Row, b: Row) => number) | undefined, + // The partition's entry in its source's map, which a mount looks up. + private readonly registry: { add: () => void; remove: () => void }, + ) {} + + groupKeyOf(row: Row | undefined): string | undefined { + if (row === undefined) return undefined + let groupKey = `` + for (const path of this.paths) { + const key = equalityKey(readPath(row, path)) + if (key === undefined) return undefined + groupKey = appendGroupKeyPart(groupKey, key) + } + return groupKey + } + + // Whether two versions of a row hold the same value in every field. + private sameFields(a: Row, b: Row): boolean { + return this.paths.every((path) => + Object.is(readPath(a, path), readPath(b, path)), + ) + } + + group(key: string): PartitionGroup { + let group = this.groups.get(key) + if (!group) { + const revision = ++this.clock + group = { + key, + rows: new SortedMap(this.compareRows), + listeners: new Set(), + revision, + layoutRevision: revision, + } + this.groups.set(key, group) + } + return group + } + + /** A group for reading, without creating one. */ + peek(key: string): PartitionGroup { + return this.groups.get(key) ?? EMPTY_GROUP + } + + // A group with no rows and no watchers holds nothing anyone can read. + private dropIfUnused(group: PartitionGroup): void { + if (group.rows.size === 0 && group.listeners.size === 0) { + this.groups.delete(group.key) + } + } + + /** + * Keep the shared source subscription open. Each view brings its query's + * `gcTime`; the partition keeps the longest, so it never releases before + * one of its views' own live-query Collection would have. + */ + retain(gcTime?: number): void { + if (gcTime !== undefined) { + // As for a Collection, a non-positive or non-finite gcTime disables GC. + const delay = gcTime > 0 && Number.isFinite(gcTime) ? gcTime : Infinity + this.gcTime = Math.max(this.gcTime, delay) + } + if (this.terminated) return + this.subscribe() + // Like a Collection that synced before anything subscribed, a view built + // during a render gets a grace period to subscribe when it commits. + this.scheduleRelease(UNSUBSCRIBED_GC_FLOOR_MS) + } + + subscribe(): void { + if (!this.terminated && !this.subscription) { + // A released partition that subscribes again serves new mounts too. + this.registry.add() + this.subscription = this.source.subscribeChanges( + (changes) => + this.apply(changes as Array>), + { includeInitialState: true }, + ) + const stopStatus = this.source.on(`status:change`, (event) => { + this.deliverStatus(event) + }) + // Like a live query, a pooled view fails for good when its source + // starts cleanup, before the adapter's cleanup settles; queries + // mounted later get a new partition. + const stopCleanupStart = this.source._onCleanupStart(() => { + const previousStatus = this.source.status + this.terminate() + this.deliverStatus({ + type: `status:change`, + collection: this.source as unknown as Collection, + previousStatus, + status: `error`, + }) + }) + this.stopStatusEvents = () => { + stopStatus() + stopCleanupStart() + } + } + } + + private deliverStatus(event: CollectionStatusChangeEvent): void { + const delivered = this.terminated + ? { ...event, status: `error` as const } + : event + for (const listener of [...this.statusListeners]) listener(delivered) + if (this.terminated) { + this.stopStatusEvents?.() + this.stopStatusEvents = undefined + } + } + + addListener(group: PartitionGroup, listener: Listener): void { + this.subscribe() + group.listeners.add(listener) + this.listenerCount++ + } + + removeListener(group: PartitionGroup, listener: Listener): void { + if (!group.listeners.delete(listener)) return + this.listenerCount-- + this.dropIfUnused(group) + this.scheduleRelease(0) + } + + private terminate(): void { + this.terminated = true + CleanupQueue.getInstance().cancel(this) + this.release() + } + + private release(): void { + this.subscription?.unsubscribe() + this.subscription = undefined + this.registry.remove() + } + + // Releases on the Collections' shared GC queue, after the longest + // `gcTime` of this partition's views. + private scheduleRelease(minDelay: number): void { + if (this.listenerCount > 0 || !Number.isFinite(this.gcTime)) return + const delay = Math.max(this.gcTime, minDelay) + CleanupQueue.getInstance().schedule(this, delay, this.releaseIfUnused) + } + + private readonly releaseIfUnused = (): void => { + if (this.listenerCount > 0) return + this.stopStatusEvents?.() + this.stopStatusEvents = undefined + // Views read groups by key, so a later subscription refills new ones. + this.groups.clear() + this.release() + } + + private apply(changes: Array>): void { + const touched = new Map< + PartitionGroup, + Array> + >() + const record = ( + group: PartitionGroup, + change: ChangeMessage, + ) => { + const list = touched.get(group) + if (list) list.push(change) + else touched.set(group, [change]) + if (change.type !== `update`) group.layoutRevision = ++this.clock + } + for (const change of changes) { + const next = + change.type === `delete` ? undefined : this.groupKeyOf(change.value) + const previous = + change.type === `insert` + ? undefined + : change.type === `update` && + change.previousValue !== undefined && + this.sameFields(change.value, change.previousValue) + ? next + : this.groupKeyOf( + change.type === `delete` ? change.value : change.previousValue, + ) + if (previous !== undefined && previous !== next) { + const group = this.group(previous) + const old = group.rows.get(change.key) + if (group.rows.delete(change.key)) { + record(group, { type: `delete`, key: change.key, value: old! }) + } + } + if (next !== undefined) { + const group = this.group(next) + const existed = group.rows.has(change.key) + group.rows.set(change.key, change.value) + record( + group, + existed + ? change.type === `update` + ? change + : { ...change, type: `update` } + : { type: `insert`, key: change.key, value: change.value }, + ) + } + } + for (const [group, groupChanges] of touched) { + group.revision = ++this.clock + for (const listener of [...group.listeners]) listener(groupChanges) + this.dropIfUnused(group) + } + } +} + +type Conjunct = { path: Array; pathKey: string; literalKey: string } + +type PoolableShape = { + paths: Array> + shapeKey: string + groupKey: string + // Conjuncts each view evaluates over its group's rows. + residual: Array + orderBy: OrderBy | undefined +} + +// Whether an expression reads only this query's own row fields, so a view +// can evaluate it with the compiler's evaluator. +function readsOnlyRow(expression: BasicExpression, alias: string): boolean { + if (expression.type === `val`) return true + if (expression.type === `ref`) { + const [root, field] = expression.path + return root === alias && field !== undefined && !isVirtualPropName(field) + } + return expression.args.every((arg) => readsOnlyRow(arg, alias)) +} + +// Splits a conjunct into `eq(alias.field, literal)` groups and residual +// conjuncts; false for an expression a view cannot evaluate. +function collectConjuncts( + expression: BasicExpression, + alias: string, + out: Array, + residual: Array, +): boolean { + if (expression.type === `func` && expression.name === `and`) { + for (const arg of expression.args) { + if (!collectConjuncts(arg, alias, out, residual)) return false + } + return true + } + const conjunct = equalityConjunctOf(expression, alias) + if (conjunct) out.push(conjunct) + else if (readsOnlyRow(expression, alias)) residual.push(expression) + else return false + return true +} + +function equalityConjunctOf( + expression: BasicExpression, + alias: string, +): Conjunct | undefined { + const conjunct = equalityConjunct(expression, (ref) => + ref.path[0] === alias ? ref.path.slice(1) : undefined, + ) + return conjunct && { ...conjunct, pathKey: JSON.stringify(conjunct.path) } +} + +// Whether a row passes every residual conjunct, as a WHERE filter decides. +function rowPredicate( + residual: Array, + alias: string, +): (row: Row) => boolean { + const conjuncts = residual.map((expression) => compileExpression(expression)) + // One namespaced row, reused so a check allocates nothing. + const namespaced: Record = {} + return (row) => { + namespaced[alias] = row + return conjuncts.every((conjunct) => + toBooleanPredicate(conjunct(namespaced as any)), + ) + } +} + +/** + * The equality conjuncts of a query that a partition can serve, or undefined + * when any other clause or operand is present. + */ +function poolableShape(query: QueryIR): PoolableShape | undefined { + if ( + query.from.type !== `collectionRef` || + query.select || + query.join || + query.groupBy || + query.having || + query.limit !== undefined || + query.offset !== undefined || + query.distinct || + query.singleResult || + query.fnSelect || + query.fnWhere?.length || + query.fnHaving?.length || + !query.where?.length + ) { + return undefined + } + const conjuncts: Array = [] + const residual: Array = [] + for (const where of query.where) { + if ( + !collectConjuncts( + getWhereExpression(where), + query.from.alias, + conjuncts, + residual, + ) + ) { + return undefined + } + } + // A partition needs at least one equality to group by. + if (conjuncts.length === 0) return undefined + const orderBy = query.orderBy?.length ? query.orderBy : undefined + const orderKey = orderBy + ? orderByKey(orderBy, query.from.alias, query.from.collection) + : `` + if (orderKey === undefined) return undefined + // Most shapes have one or two fields; a general sort costs more than both. + if (conjuncts.length === 2) { + if (conjuncts[1]!.pathKey < conjuncts[0]!.pathKey) conjuncts.reverse() + } else if (conjuncts.length > 2) { + conjuncts.sort((a, b) => (a.pathKey < b.pathKey ? -1 : 1)) + } + const paths: Array> = [] + let shapeKey = `` + let groupKey = `` + for (const { path, pathKey, literalKey } of conjuncts) { + paths.push(path) + // Each JSON path delimits itself, so concatenation stays unambiguous. + shapeKey += pathKey + groupKey = appendGroupKeyPart(groupKey, literalKey) + } + // Groups of one shape share a row order, so the order is part of it. + if (orderKey) shapeKey += `|${orderKey}` + return { paths, shapeKey, groupKey, residual, orderBy } +} + +// A key for an `orderBy` over this query's own row fields, or undefined for +// one a partition cannot share by value, such as a custom string comparator. +function orderByKey( + orderBy: OrderBy, + alias: string, + source: CollectionImpl, +): string | undefined { + let key = `` + for (const clause of orderBy) { + const { expression } = clause + if ( + expression.type !== `ref` || + !readsOnlyRow(expression, alias) || + expression.path.length < 2 + ) { + return undefined + } + const options = buildCompareOptions(clause, source) + if (options.stringSort === `custom`) return undefined + key += JSON.stringify([expression.path.slice(1), options]) + } + return key +} + +// Orders rows as a live-query Collection's `orderBy` does; the group's +// SortedMap breaks ties by key. +function rowComparator( + orderBy: OrderBy, + alias: string, + source: CollectionImpl, +): (a: Row, b: Row) => number { + const terms = orderBy.map((clause) => ({ + read: compileExpression(clause.expression), + compare: makeComparator(buildCompareOptions(clause, source)), + })) + // One namespaced row, reused so a comparison allocates nothing. + const namespaced: Record = {} + return (a, b) => { + for (const { read, compare } of terms) { + namespaced[alias] = a + const left = read(namespaced as any) + namespaced[alias] = b + const result = compare(left, read(namespaced as any)) + if (result !== 0) return result + } + return 0 + } +} + +/** + * One query's view of its group, read by the live-query observer. Users get + * `publicCollection` instead, which builds the query's live-query Collection + * on first use and forwards every member to it. + */ +class PooledLiveQuery { + readonly isLoadingSubset = false + // No persisted readiness, single-result config, or layout channel. + readonly config = undefined + readonly _subscribeLayoutChanges = undefined + private collection: Collection | undefined = undefined + private listenerCount = 0 + private collectionHold: { unsubscribe: () => void } | undefined = undefined + + constructor( + private readonly source: CollectionImpl, + private readonly query: BaseQueryBuilder, + private readonly partition: Partition, + private readonly groupKey: string, + private readonly gcTime: number, + // The query's conjuncts beyond its group's equalities, if any. + private readonly passes: ((row: Row) => boolean) | undefined, + ) { + partition.retain(gcTime) + } + + get status(): CollectionStatus { + return this.partition.terminated ? `error` : this.source.status + } + + // Read by key: the partition drops a group nobody watches once it empties. + private get group(): PartitionGroup { + return this.partition.peek(this.groupKey) + } + + get _stateRevision(): number { + return this.group.revision + } + + get _layoutRevision(): number { + return this.group.layoutRevision + } + + entries(): Iterable<[string | number, Row]> { + const rows = this.group.rows.entries() + const passes = this.passes + return passes ? [...rows].filter(([, row]) => passes(row)) : rows + } + + subscribeChanges( + callback: Listener, + options: { includeInitialState?: boolean } = {}, + ): { unsubscribe: () => void } { + // A released partition refills its group here, so seed the filter after. + this.partition.subscribe() + const listener = this.passes ? this.filterChanges(callback) : callback + const group = this.partition.group(this.groupKey) + this.partition.addListener(group, listener) + this.listenerCount++ + this.holdCollection() + if (options.includeInitialState) { + callback( + Array.from(this.entries(), ([key, value]) => ({ + type: `insert`, + key, + value, + })), + ) + } + let subscribed = true + return { + unsubscribe: () => { + if (!subscribed) return + subscribed = false + this.partition.removeListener(group, listener) + if (--this.listenerCount === 0) { + this.collectionHold?.unsubscribe() + this.collectionHold = undefined + } + }, + } + } + + // While the view is observed, its built Collection stays subscribed, as + // the Collection would be if it served the observer itself. + private holdCollection(): void { + if (this.collection && this.listenerCount > 0) { + this.collectionHold ??= this.collection.subscribeChanges(() => {}) + } + } + + // Turns the group's changes into this query's, tracking which rows have + // passed its residual conjuncts for this subscription. + private filterChanges(callback: Listener): Listener { + const passes = this.passes! + const visible = new Map(this.entries()) + return (changes) => { + const out: Array> = [] + for (const change of changes) { + const { key, value } = change + const previous = visible.get(key) + const next = change.type !== `delete` && passes(value) + if (next) visible.set(key, value) + else visible.delete(key) + if (previous && next) { + out.push({ type: `update`, key, value, previousValue: previous }) + } else if (previous) { + out.push({ type: `delete`, key, value: previous }) + } else if (next) { + out.push({ type: `insert`, key, value }) + } + } + if (out.length > 0) callback(out) + } + } + + on(...args: Parameters): () => void { + const [event, listener] = args + if (event !== `status:change`) return this.source.on(...args) + const listeners = this.partition.statusListeners + listeners.add(listener as StatusListener) + return () => listeners.delete(listener as StatusListener) + } + + preload(): Promise { + return this.source.preload() + } + + cleanup(): Promise { + return this.collection?.cleanup() ?? Promise.resolve() + } + + private proxy: Collection | undefined = undefined + + /** This view as the Collection it stands in for. */ + get publicCollection(): Collection { + return (this.proxy ??= new Proxy( + this, + forwardToCollection, + ) as unknown as Collection) + } + + materialize(): Collection { + if (!this.collection) { + this.collection = createLiveQueryCollection({ + query: this.query, + startSync: true, + gcTime: this.gcTime, + }) + this.holdCollection() + } + return this.collection + } +} + +// The observer reads the view itself; users get the live-query Collection. +const forwardToCollection: ProxyHandler = { + get(view, property) { + const collection = view.materialize() + const value = Reflect.get(collection, property, collection) + return typeof value === `function` ? value.bind(collection) : value + }, + has(view, property) { + return Reflect.has(view.materialize(), property) + }, + // So `instanceof` and query sources accept it as the Collection it is. + getPrototypeOf(view) { + return Reflect.getPrototypeOf(view.materialize()) + }, +} + +/** + * The identity of a query a partition can serve with no residual conjunct: + * its source and its `eq` fields and literals, which determine its rows. + * Undefined for any other query, which keeps the full structural identity. + */ +export function getPooledQueryIdentity( + query: BaseQueryBuilder, +): string | undefined { + const ir = query._getQuery() + if (ir.from.type !== `collectionRef`) return undefined + const shape = poolableShape(ir) + if (!shape || shape.residual.length > 0) return undefined + return JSON.stringify([ir.from.collection.id, shape.shapeKey, shape.groupKey]) +} + +/** + * A pooled view for a query a partition can serve, or undefined. The view is + * typed as the Collection it stands in for. + */ +export function createPooledLiveQuery( + query: BaseQueryBuilder, + // A live-query Collection's default when the adapter gives none. + { gcTime = 5_000 }: { gcTime?: number } = {}, +): Collection | undefined { + const ir = query._getQuery() + const shape = poolableShape(ir) + if (!shape || ir.from.type !== `collectionRef`) return undefined + const source = ir.from.collection + // Persisted restore and on-demand loading need the live-query Collection. + if ( + source.config.syncMode === `on-demand` || + getPersistedReadinessSource(source.config) + ) { + return undefined + } + let partitions = partitionsBySource.get(source) + if (!partitions) { + partitions = new Map() + partitionsBySource.set(source, partitions) + } + const { shapeKey } = shape + let partition = partitions.get(shapeKey) + if (!partition) { + const owner = partitions + // A released partition may subscribe again; it must not then replace or + // remove a newer partition created under its key. + const created: Partition = new Partition( + source, + shape.paths, + shape.orderBy && rowComparator(shape.orderBy, ir.from.alias, source), + { + add: () => { + if (!owner.has(shapeKey)) owner.set(shapeKey, created) + }, + remove: () => { + if (owner.get(shapeKey) === created) owner.delete(shapeKey) + }, + }, + ) + partition = created + partitions.set(shapeKey, partition) + } + // Observers read the view directly; users get its `publicCollection`. + return new PooledLiveQuery( + source, + query, + partition, + shape.groupKey, + gcTime, + shape.residual.length > 0 + ? rowPredicate(shape.residual, ir.from.alias) + : undefined, + ) as unknown as Collection +} diff --git a/packages/db/src/utils.ts b/packages/db/src/utils.ts index 967650ea9d..fe991f4ace 100644 --- a/packages/db/src/utils.ts +++ b/packages/db/src/utils.ts @@ -27,7 +27,7 @@ interface TypedArray { * ``` */ export function deepEquals(a: any, b: any): boolean { - return deepEqualsInternal(a, b, new Map()) + return deepEqualsInternal(a, b, undefined) } function isPlainPrototype(prototype: object | null): boolean { @@ -53,7 +53,8 @@ function enumerableOwnKeys(value: object): Array { export function deepEqualsInternal( a: any, b: any, - visited: Map, + // Created on the first container that descends into a child. + visited: Map | undefined, draft = false, ): boolean { // Handle strict equality (primitives, same reference) @@ -91,9 +92,10 @@ export function deepEqualsInternal( if (a.size !== b.size) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) // A draft compares entries in order; general equality looks keys up. @@ -117,9 +119,10 @@ export function deepEqualsInternal( if (a.size !== b.size) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) // Convert to arrays for comparison @@ -238,9 +241,10 @@ export function deepEqualsInternal( if (Array.isArray(a) && a.length !== b.length) return false if (Array.isArray(a) && !draft) { // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } + visited ??= new Map() visited.set(a, b) const result = a.every((item, index) => @@ -261,10 +265,9 @@ export function deepEqualsInternal( if (prototype !== prototypeB && !plain && !plainB) return false // Check for circular references - if (visited.has(a)) { + if (visited?.has(a)) { return visited.get(a) === b } - visited.set(a, b) // Compare enumerable symbol keys as well as string keys. Query results may // use user-owned symbols, and a symbol-only update is still a value change. @@ -272,27 +275,36 @@ export function deepEqualsInternal( const keysB = enumerableOwnKeys(b) // Check if they have the same number of keys - if (keysA.length !== keysB.length) { - visited.delete(a) - return false - } + if (keysA.length !== keysB.length) return false // A class instance without enumerable keys (a File, an object with // private fields) keeps its state elsewhere, so it equals only itself. // A draft copies a URL by its href, so URLs compare by href. if (keysA.length === 0 && !Array.isArray(a) && !(plain && plainB)) { - visited.delete(a) return a instanceof URL && a.href === b.href } - // Check if all keys exist in both objects and their values are equal - const result = keysA.every( - (key) => - Object.prototype.propertyIsEnumerable.call(b, key) && - deepEqualsInternal(a[key], b[key], visited, draft), - ) - - visited.delete(a) + // Check if all keys exist in both objects and their values are equal. + // Register for cycles only before descending, so a flat object + // allocates no cycle map. + let registered = false + let result = true + for (const key of keysA) { + const value = a[key] + if (!registered && value !== null && typeof value === `object`) { + visited ??= new Map() + visited.set(a, b) + registered = true + } + if ( + !Object.prototype.propertyIsEnumerable.call(b, key) || + !deepEqualsInternal(value, b[key], visited, draft) + ) { + result = false + break + } + } + if (registered) visited!.delete(a) return result } diff --git a/packages/db/tests/conformance/contract.ts b/packages/db/tests/conformance/contract.ts index b88ace6dc2..2138dbc5f6 100644 --- a/packages/db/tests/conformance/contract.ts +++ b/packages/db/tests/conformance/contract.ts @@ -183,5 +183,13 @@ export interface LiveQueryDriver { * to catch (e.g. Solid's `createResource`/`` model). */ errorSurface?: `flag` | `throw` - features?: { serverSnapshot?: boolean; suspense?: boolean } + features?: { + serverSnapshot?: boolean + suspense?: boolean + /** + * Public sharing policy: queries filtered only by `eq` on one source share + * one source subscription per filtered field set, instead of one each. + */ + pooledEqFilters?: boolean + } } diff --git a/packages/db/tests/conformance/suite.ts b/packages/db/tests/conformance/suite.ts index 9ac852674e..173af2f135 100644 --- a/packages/db/tests/conformance/suite.ts +++ b/packages/db/tests/conformance/suite.ts @@ -262,6 +262,83 @@ export function runSuite(rawDriver: LiveQueryDriver) { }, ) + // Single-source eq filters may be served from a partition shared by every + // query of that shape; adapters must publish what the live query would. + scenario( + `eq-filter-rows`, + `an eq filter publishes matching rows in key order as rows move in and out`, + async () => { + const source = driver.makeSource(SEED) + const h = driver.mount((q) => + q + .from({ items: source.collection }) + .where(({ items }: any) => ops.eq(items.team, `a`)), + ) + await h.flush() + const ids = () => + (h.current().data as Array<{ id: string }>).map((row) => row.id) + expect(ids()).toEqual([`1`, `3`]) + + source.update({ id: `2`, name: `Jane Doe`, age: 25, team: `a` }) + await h.flush() + expect(ids()).toEqual([`1`, `2`, `3`]) + + source.update({ id: `1`, name: `John Doe`, age: 30, team: `b` }) + source.insert({ id: `4`, name: `Dave`, age: 40, team: `a` }) + await h.flush() + expect(ids()).toEqual([`2`, `3`, `4`]) + + source.remove({ id: `3`, name: `John Smith`, age: 35, team: `a` }) + await h.flush() + const remaining: Array = [ + { id: `2`, name: `Jane Doe`, age: 25, team: `a` }, + { id: `4`, name: `Dave`, age: 40, team: `a` }, + ] + expectOrderedRows(h.current().data, remaining) + expectKeyedRows(h.current().state, remaining) + expect(h.current().status).toBe(`ready`) + h.unmount() + }, + ) + + scenario( + `eq-filter-peers`, + `eq-filtered peers on one source each see only their rows`, + async () => { + const source = driver.makeSource(SEED) + const query = (team: string) => (q: any) => + q + .from({ items: source.collection }) + .where(({ items }: any) => ops.eq(items.team, team)) + const teamA = driver.mount(query(`a`)) + const teamB = driver.mount(query(`b`)) + const olderA = driver.mount((q) => + query(`a`)(q).where(({ items }: any) => ops.eq(items.age, 35)), + ) + for (const h of [teamA, teamB, olderA]) await h.flush() + // Pooling shares one subscription between the two team queries. + expect(source.collection.subscriberCount).toBe( + driver.features?.pooledEqFilters ? 2 : 3, + ) + const ids = (h: typeof teamA) => + (h.current().data as Array<{ id: string }>).map((row) => row.id) + expect([ids(teamA), ids(teamB), ids(olderA)]).toEqual([ + [`1`, `3`], + [`2`], + [`3`], + ]) + + source.update({ id: `3`, name: `John Smith`, age: 35, team: `b` }) + for (const h of [teamA, teamB, olderA]) await h.flush() + expect([ids(teamA), ids(teamB), ids(olderA)]).toEqual([ + [`1`], + [`2`, `3`], + [], + ]) + for (const h of [teamA, teamB, olderA]) h.unmount() + }, + ) + scenario(`live-insert`, `a sync insert appears in the result`, async () => { const source = driver.makeSource(SEED) const h = driver.mount((q) => @@ -941,7 +1018,7 @@ export function runSuite(rawDriver: LiveQueryDriver) { ) it(`registers every distinct scenario without whole-test waivers`, () => { - expect(registry.size).toBe(28) + expect(registry.size).toBe(30) }) }) } diff --git a/packages/db/tests/duplicate-instance-check.test.ts b/packages/db/tests/duplicate-instance-check.test.ts new file mode 100644 index 0000000000..3338cdbcd5 --- /dev/null +++ b/packages/db/tests/duplicate-instance-check.test.ts @@ -0,0 +1,65 @@ +// @vitest-environment node +import { readFileSync } from 'node:fs' +import { createContext, runInContext } from 'node:vm' +import { transformSync } from 'esbuild' +import { describe, expect, it } from 'vitest' + +const source = readFileSync( + new URL(`../src/duplicate-instance-check.ts`, import.meta.url), + `utf8`, +) + +class DuplicateDbInstanceError extends Error {} + +// Loads the check twice in one browser top window, as two bundled copies of +// the package would: `define` is what the bundler inlines, and the window +// has no `process` global. +function loadTwice(define: Record): unknown { + const { code } = transformSync(source, { + loader: `ts`, + format: `cjs`, + define, + }) + const window: Record = { document: {} } + window.top = window + const context = createContext({ window }) + // Each copy runs in its own module scope, sharing the window's globals. + const load = runInContext( + `(function (module, exports, require) {\n${code}\n})`, + context, + ) as ( + module: { exports: object }, + exports: object, + require: () => object, + ) => void + try { + for (let copy = 0; copy < 2; copy++) { + const bundle = { exports: {} } + load(bundle, bundle.exports, () => ({ DuplicateDbInstanceError })) + } + } catch (error) { + return error + } + return undefined +} + +describe(`duplicate @tanstack/db instance check`, () => { + it(`rejects a second copy in a browser development bundle`, () => { + expect( + loadTwice({ 'process.env.NODE_ENV': `"development"` }), + ).toBeInstanceOf(DuplicateDbInstanceError) + }) + + it(`stays off in production, when disabled, and without a bundler`, () => { + expect( + loadTwice({ 'process.env.NODE_ENV': `"production"` }), + ).toBeUndefined() + expect( + loadTwice({ + 'process.env.NODE_ENV': `"development"`, + 'process.env.TANSTACK_DB_DISABLE_DUP_CHECK': `"1"`, + }), + ).toBeUndefined() + expect(loadTwice({})).toBeUndefined() + }) +}) diff --git a/packages/db/tests/flat-change-tracking-oracle.property.test.ts b/packages/db/tests/flat-change-tracking-oracle.property.test.ts new file mode 100644 index 0000000000..1d5a979651 --- /dev/null +++ b/packages/db/tests/flat-change-tracking-oracle.property.test.ts @@ -0,0 +1,730 @@ +/** + * # Does flat-row change tracking report what the draft proxy reports? + * + * Law and source: `collection.update` passes each row to the callback as a + * draft and records the fields whose final value differs from the row, plus + * deleted fields as `undefined` (`src/proxy.ts`). A row whose own fields are + * all primitives or functions, with a plain or null prototype and no symbol + * keys, is tracked with a shallow copy instead of a proxy. Both trackers must + * report the same change set for every callback, and any other row must fall + * back to the proxy. + * + * Why an example can miss the failure: a single assignment of a new value + * passes any diff. The trackers can disagree only on equal-but-not-identical + * values (`0` and `-0`, two `NaN`s), on a field set back to its original + * value, on a field set to `undefined` versus deleted, on an added field, on + * an assigned object that later changes outside the callback, and on the + * shapes a shallow copy treats differently from a proxy: a frozen row, a + * non-enumerable field, a property defined in the callback, another row's + * draft stored in a field, and a callback that throws. + * + * Model: `expectedChanges` folds every row's operations, in callback order, + * over plain copies of the rows. It keeps enumerable fields whose final value + * differs under `===` or `Object.is` from the row's own field, or, for the + * object a non-enumerable field holds, by contents; plus deleted enumerable + * fields. A copy holds no non-enumerable field, so reading one gives + * `undefined`, deleting one does nothing, and writing one is a change unless + * it writes the row's value. Defining a field acts as assigning it: an + * enumerable accessor reports the value it reads. A stored draft is the model's copy, read + * when the callback returns. It does not import either tracker. When the + * callback throws, or a plain object rejects one of its operations, both + * trackers must throw the same error and leave the rows unchanged. + * + * History grammar: one to three rows with fields `a`, `b`, and `c` drawn from + * `0`, `-0`, `1`, `NaN`, `''`, `'x'`, `true`, `false`, `null`, `undefined`, a + * function, or missing, with a plain or null prototype, optionally frozen, + * and optionally with a non-enumerable field `h` holding a domain value or an + * object. Each + * row gets up to six operations: assign a field (`a` to `d`, or `h`) a value + * from that domain or a fresh object, assign `d` `undefined`, set a field + * back to its original value, delete a field, read it, define it with + * `Object.defineProperty` as an enumerable or non-enumerable data or accessor + * property, + * or store a row's draft in it. A weighted run changes a field, adds `d` as + * `undefined`, and reverts the field; others write and delete `h`, write an + * equal object over an object `h`, or define a getter and assign it its own value; and one writes the + * opposite-signed zero over each zero field. Histories call the tracker with an + * array or with a single row, and may throw after any operation. + * + * Production driver: `withFlatChangeTracking` and the proxy trackers + * `withArrayChangeTracking` and `withChangeTracking` run the same operations. + * + * Refinement check: the flat result, the proxy result, and the model are + * strictly equal, including present `undefined` fields and the cycles a + * stored draft creates, except that `0` and `-0` compare equal and + * prototypes are ignored: collection equality does not distinguish them, and + * the proxy skips writing a value equal to the current one. Mutating an + * assigned object after the callback leaves both results unchanged. + * + * Calibration: a flat diff with `!==` reports unchanged `NaN` fields, one with + * `Object.is` alone reports `-0` written over `0`, one that skips deletions + * loses removed fields, one that ignores whether the row owns a field drops + * added `undefined` fields, and one that edits rows in place breaks the throw + * and stored-draft histories; each fails a pinned history and the fixed + * campaign. The proxy used to treat a field added with `undefined` as + * reverted when another field went back to its original value, and dropped + * the added field; the pinned history and both campaigns fail without the + * fix, because the grammar weights that run. + * + * Known omissions: nested objects, arrays, Dates, Maps, Sets, class + * instances, and symbol keys are outside this owner; the proxy oracles and + * contracts own them, and this file checks only that they fall back. + */ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { + withArrayChangeTracking, + withChangeTracking, + withFlatChangeTracking, +} from '../src/proxy.js' +import { + oraclePropertyOptions, + oracleRuns, + readOracleRunConfig, +} from './oracle-config.js' + +const property = `flat-change-tracking.equivalence` +const requestedReplayProperty = readOracleRunConfig().replayProperty + +const MISSING = Symbol(`missing`) +const FRESH_OBJECT = Symbol(`fresh object`) +const fn = () => 1 +const values: ReadonlyArray = [ + 0, + -0, + 1, + Number.NaN, + ``, + `x`, + true, + false, + null, + undefined, + fn, +] +const fields = [`a`, `b`, `c`] as const +type Row = Record +type Operation = + | { kind: `set`; field: string; value: unknown } + | { kind: `revert`; field: string } + | { kind: `delete`; field: string } + | { kind: `read`; field: string } + | { kind: `define`; field: string; value: unknown; enumerable: boolean } + | { + kind: `define-getter` + field: string + value: unknown + enumerable: boolean + } + | { kind: `store-draft`; field: string; row: number } + // Writes the opposite-signed zero over every field the row holds as a zero. + | { kind: `flip-zeros` } +type RowShape = { + fields: Array + nullPrototype: boolean + frozen: boolean + hidden: unknown +} +type History = { + rows: Array + operations: Array> + single: boolean + // The callback throws after this many operations, counted across rows. + throwAfter: number | undefined +} + +// --------------------------------------------------------------------------- +// Model +// --------------------------------------------------------------------------- + +function sameValue(a: unknown, b: unknown): boolean { + return a === b || Object.is(a, b) +} + +// Only a non-enumerable field holds an object, always `{ nested: 1 }`; an +// equal object written over it is not a change. +function sameContents(a: unknown, b: unknown): boolean { + if (sameValue(a, b)) return true + if ( + a === null || + b === null || + typeof a !== `object` || + typeof b !== `object` + ) + return false + const keys = Object.keys(a) + return ( + keys.length === Object.keys(b).length && + keys.every( + (key) => + Object.hasOwn(b, key) && sameValue((a as Row)[key], (b as Row)[key]), + ) + ) +} + +function expectedChanges( + originals: Array, + operations: Array>, +): Array { + const drafts: Array = originals.map((original) => ({ ...original })) + drafts.forEach((draft, index) => { + for (const operation of operations[index]!) { + applyOperation(draft, originals[index]!, operation, drafts) + } + }) + return drafts.map((draft, index) => { + const original = originals[index]! + const changes: Row = {} + for (const key of Object.keys(draft)) { + if ( + !Object.hasOwn(original, key) || + !sameContents(draft[key], original[key]) + ) { + changes[key] = draft[key] + } + } + for (const key of Object.keys(original)) { + if (!Object.hasOwn(draft, key)) changes[key] = undefined + } + return changes + }) +} + +const domainValue = (value: unknown) => + value === FRESH_OBJECT ? { nested: 1 } : value + +// Shared by the model and the driver; `drafts` are the callback's drafts. +function applyOperation( + draft: Row, + original: Row, + operation: Operation, + drafts: Array, +) { + switch (operation.kind) { + case `set`: + draft[operation.field] = domainValue(operation.value) + break + case `define`: + Object.defineProperty(draft, operation.field, { + value: domainValue(operation.value), + enumerable: operation.enumerable, + writable: true, + configurable: true, + }) + break + case `define-getter`: { + const value = domainValue(operation.value) + Object.defineProperty(draft, operation.field, { + get: () => value, + enumerable: operation.enumerable, + configurable: true, + }) + break + } + case `flip-zeros`: + for (const key of Object.keys(original)) { + if (original[key] === 0) { + draft[key] = Object.is(original[key], 0) ? -0 : 0 + } + } + break + case `store-draft`: + draft[operation.field] = drafts[operation.row % drafts.length] + break + case `revert`: + if (Object.hasOwn(original, operation.field)) { + draft[operation.field] = original[operation.field] + } + break + case `delete`: + delete draft[operation.field] + break + case `read`: + void draft[operation.field] + break + } +} + +// Collection equality does not distinguish -0 from 0 or prototypes. A +// stored draft can make a change set cyclic, so the copy keeps cycles. +function normalize(value: unknown, copies = new Map()): unknown { + if (value === 0) return 0 + if (value === null || typeof value !== `object`) return value + const existing = copies.get(value) + if (existing) return existing + const copy: Row = {} + copies.set(value, copy) + for (const [key, field] of Object.entries(value)) { + copy[key] = normalize(field, copies) + } + return copy +} + +function buildRow(shape: RowShape): Row { + const row: Row = shape.nullPrototype ? Object.create(null) : {} + shape.fields.forEach((value, index) => { + if (value !== MISSING) row[fields[index]!] = value + }) + if (shape.hidden !== MISSING) { + Object.defineProperty(row, `h`, { + value: domainValue(shape.hidden), + enumerable: false, + writable: true, + configurable: true, + }) + } + return shape.frozen ? Object.freeze(row) : row +} + +// Captures a row's own properties, including non-enumerable ones. +const rowState = (row: Row) => Object.getOwnPropertyDescriptors(row) + +// --------------------------------------------------------------------------- +// History grammar +// --------------------------------------------------------------------------- + +const fieldArbitrary = fc.constantFrom(`a`, `b`, `c`, `d`, `h`) +const valueArbitrary = fc.constantFrom(...values, FRESH_OBJECT) +const operationArbitrary: fc.Arbitrary = fc.oneof( + { + weight: 3, + arbitrary: fc.record({ + kind: fc.constant(`set` as const), + field: fieldArbitrary, + value: valueArbitrary, + }), + }, + fc.constant({ kind: `set`, field: `d`, value: undefined }), + { + weight: 2, + arbitrary: fc.record({ + kind: fc.constant(`revert` as const), + field: fieldArbitrary, + }), + }, + fc.record({ kind: fc.constant(`delete` as const), field: fieldArbitrary }), + fc.record({ kind: fc.constant(`read` as const), field: fieldArbitrary }), + fc.record({ + kind: fc.constant(`define` as const), + field: fieldArbitrary, + value: valueArbitrary, + enumerable: fc.boolean(), + }), + fc.record({ + kind: fc.constant(`define-getter` as const), + field: fieldArbitrary, + value: valueArbitrary, + enumerable: fc.boolean(), + }), + fc.record({ + kind: fc.constant(`store-draft` as const), + field: fieldArbitrary, + row: fc.nat({ max: 2 }), + }), +) +// An added `undefined` while another field changes and reverts is where the +// proxy once dropped the added field, so the grammar also emits that run. +const excursionArbitrary: fc.Arbitrary> = fc + .tuple(fc.constantFrom(...fields), valueArbitrary) + .map(([field, value]) => [ + { kind: `set`, field, value }, + { kind: `set`, field: `d`, value: undefined }, + { kind: `revert`, field }, + ]) +// Runs that reach the descriptor laws, each of which a single operation +// rarely forms: a hidden field written then deleted, an equal object written +// over a hidden object field, and a getter-only field assigned its own value. +const descriptorRunArbitrary: fc.Arbitrary> = fc.oneof( + valueArbitrary.map( + (value): Array => [ + { kind: `set`, field: `h`, value }, + { kind: `delete`, field: `h` }, + ], + ), + fc.constant>([ + { kind: `set`, field: `h`, value: FRESH_OBJECT }, + ]), + fc.tuple(fc.constantFrom(...fields), valueArbitrary, fc.boolean()).map( + ([field, value, enumerable]): Array => [ + { kind: `define-getter`, field, value, enumerable }, + { kind: `set`, field, value }, + ], + ), +) +const operationsArbitrary: fc.Arbitrary> = fc + .array( + fc.oneof( + { + weight: 4, + arbitrary: operationArbitrary.map((operation) => [operation]), + }, + excursionArbitrary, + descriptorRunArbitrary, + // A field drawn as a zero rarely gets the other zero by chance. + fc.constant>([{ kind: `flip-zeros` }]), + ), + { maxLength: 5 }, + ) + .map((chunks) => chunks.flat().slice(0, maxOperations)) +const maxOperations = 6 + +const rowArbitrary: fc.Arbitrary = fc.record({ + fields: fc.tuple(...fields.map(() => fc.constantFrom(...values, MISSING))), + nullPrototype: fc.boolean(), + frozen: fc.boolean(), + hidden: fc.oneof( + { weight: 2, arbitrary: fc.constant(MISSING) }, + fc.constant(FRESH_OBJECT), + fc.constantFrom(...values), + ), +}) +const historyArbitrary: fc.Arbitrary = fc + .array(rowArbitrary, { minLength: 1, maxLength: 3 }) + .chain((rows) => + fc.record({ + rows: fc.constant(rows), + operations: fc.tuple(...rows.map(() => operationsArbitrary)), + single: rows.length === 1 ? fc.boolean() : fc.constant(false), + throwAfter: fc.oneof( + { weight: 5, arbitrary: fc.constant(undefined) }, + fc.nat({ max: maxOperations }), + ), + }), + ) + +const plainRow = (rowFields: Array, nullPrototype = false) => ({ + fields: rowFields, + nullPrototype, + frozen: false, + hidden: MISSING, +}) + +const firstDraft = (draft: Array | Row) => + Array.isArray(draft) ? draft[0]! : draft +// Each witness pins a shape the trackers once disagreed on to the change +// set both must now report: defining a field acts as assigning it, and a +// non-enumerable field is not row data unless the callback writes it. +const descriptorLaws: ReadonlyArray<{ + name: string + shape: RowShape + callback: (draft: Array | Row) => void + expected: Row +}> = [ + { + name: `an enumerable accessor defined in the callback reports its value`, + shape: plainRow([1, MISSING, MISSING]), + callback: (draft) => { + Object.defineProperty(firstDraft(draft), `g`, { + get: () => 7, + enumerable: true, + configurable: true, + }) + }, + expected: { g: 7 }, + }, + { + name: `defining a field with its own value is not a change`, + shape: plainRow([1, MISSING, MISSING]), + callback: (draft) => { + Object.defineProperty(firstDraft(draft), `a`, { + value: 1, + enumerable: true, + writable: true, + configurable: true, + }) + }, + expected: {}, + }, + { + name: `an equal object written to a non-enumerable object field is not a change`, + shape: { ...plainRow([1, MISSING, MISSING]), hidden: FRESH_OBJECT }, + callback: (draft) => { + firstDraft(draft).h = { nested: 1 } + }, + expected: {}, + }, + { + name: `a non-enumerable field written and then deleted is not a change`, + shape: { ...plainRow([1, MISSING, MISSING]), hidden: `x` }, + callback: (draft) => { + firstDraft(draft).h = `y` + delete firstDraft(draft).h + }, + expected: {}, + }, +] + +// Each history isolates one place a plausible flat diff goes wrong. +const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ + { + name: `a changed field defined back to its own value is not a change`, + history: { + rows: [plainRow([1, `x`, MISSING])], + operations: [ + [ + { kind: `set`, field: `a`, value: 0 }, + { kind: `define`, field: `a`, value: 1, enumerable: true }, + ], + ], + single: true, + throwAfter: undefined, + }, + }, + { + name: `assigning a getter-only field its own value throws`, + history: { + rows: [plainRow([1, MISSING, MISSING])], + operations: [ + [ + { kind: `define-getter`, field: `a`, value: 2, enumerable: true }, + { kind: `set`, field: `a`, value: 2 }, + ], + ], + single: false, + throwAfter: undefined, + }, + }, + { + name: `NaN written over NaN is not a change`, + history: { + rows: [plainRow([Number.NaN, 1, MISSING])], + operations: [[{ kind: `set`, field: `a`, value: Number.NaN }]], + single: false, + throwAfter: undefined, + }, + }, + { + name: `-0 written over 0 is not a change`, + history: { + rows: [plainRow([0, 1, MISSING])], + operations: [[{ kind: `set`, field: `a`, value: -0 }]], + single: true, + throwAfter: undefined, + }, + }, + { + name: `a field added as undefined survives another field's revert`, + history: { + rows: [plainRow([`x`, 1, MISSING])], + operations: [ + [ + { kind: `set`, field: `a`, value: `y` }, + { kind: `set`, field: `d`, value: undefined }, + { kind: `revert`, field: `a` }, + ], + ], + single: false, + throwAfter: undefined, + }, + }, + { + name: `deleted, added, and reverted fields`, + history: { + rows: [plainRow([1, `x`, true], true)], + operations: [ + [ + { kind: `delete`, field: `a` }, + { kind: `set`, field: `d`, value: undefined }, + { kind: `set`, field: `b`, value: `y` }, + { kind: `revert`, field: `b` }, + ], + ], + single: false, + throwAfter: undefined, + }, + }, + { + name: `a frozen row with a hidden field is written, defined, and deleted`, + history: { + rows: [ + { + fields: [1, 2, MISSING], + nullPrototype: false, + frozen: true, + hidden: 5, + }, + ], + operations: [ + [ + { kind: `set`, field: `a`, value: 3 }, + { kind: `define`, field: `c`, value: `x`, enumerable: true }, + { kind: `define`, field: `d`, value: 1, enumerable: false }, + { kind: `set`, field: `h`, value: 6 }, + { kind: `delete`, field: `b` }, + ], + ], + single: true, + throwAfter: undefined, + }, + }, + { + name: `rows store each other's drafts`, + history: { + rows: [plainRow([1, 2, MISSING]), plainRow([3, 4, MISSING])], + operations: [ + [ + { kind: `store-draft`, field: `d`, row: 1 }, + { kind: `store-draft`, field: `c`, row: 0 }, + ], + [ + { kind: `store-draft`, field: `d`, row: 0 }, + { kind: `delete`, field: `b` }, + ], + ], + single: false, + throwAfter: undefined, + }, + }, + { + name: `the callback throws after writing`, + history: { + rows: [plainRow([1, 2, MISSING]), plainRow([3, 4, MISSING])], + operations: [ + [ + { kind: `set`, field: `a`, value: 5 }, + { kind: `delete`, field: `b` }, + ], + [{ kind: `set`, field: `d`, value: 1 }], + ], + single: false, + throwAfter: 2, + }, + }, +] + +// --------------------------------------------------------------------------- +// Production driver and refinement check +// --------------------------------------------------------------------------- + +class CallbackError extends Error {} + +function runHistory(history: History): void { + const originals = history.rows.map(buildRow) + // A revert reads the row the tracker was given. + const run = (rows: Array) => (drafts: Array | Row) => { + const list = Array.isArray(drafts) ? drafts : [drafts] + let applied = 0 + list.forEach((draft, index) => { + for (const operation of history.operations[index]!) { + if (applied++ === history.throwAfter) throw new CallbackError() + applyOperation(draft, rows[index]!, operation, list) + } + }) + if (applied === history.throwAfter) throw new CallbackError() + } + const flatRows = history.rows.map(buildRow) + const proxyRows = history.rows.map(buildRow) + const track = { + flat: () => + withFlatChangeTracking(flatRows, run(flatRows), !history.single), + proxy: () => + history.single + ? [withChangeTracking(proxyRows[0]!, run(proxyRows))] + : withArrayChangeTracking(proxyRows, run(proxyRows)), + } + // A plain object rejects some callbacks itself, such as assigning a field + // the callback gave only a getter; the trackers must reject them too. + let model: Array | undefined + let modelRejects = false + try { + model = expectedChanges(originals, history.operations) + } catch (error) { + if (!(error instanceof TypeError)) throw error + modelRejects = true + } + const throws = + modelRejects || + (history.throwAfter !== undefined && + history.throwAfter <= history.operations.flat().length) + if (throws) { + const errors = [track.flat, track.proxy].map((tracker) => { + try { + tracker() + } catch (error) { + return (error as Error).constructor + } + return undefined + }) + expect(errors[0], `flat rethrows`).toBeOneOf([CallbackError, TypeError]) + expect(errors[1], `proxy throws the same error`).toBe(errors[0]) + for (const rows of [flatRows, proxyRows]) { + expect(rows.map(rowState), `rows unchanged`).toStrictEqual( + originals.map(rowState), + ) + } + return + } + const flat = track.flat() + expect(flat, `flat rows take the flat path`).toBeDefined() + const proxy = track.proxy() + expect(normalize(flat), `flat vs model`).toStrictEqual(normalize(model)) + expect(normalize(proxy), `proxy vs model`).toStrictEqual(normalize(model)) +} + +describe(`flat change tracking oracle`, () => { + if (requestedReplayProperty === undefined) { + for (const { name, history } of pinnedHistories) { + it(`matches the draft proxy when ${name}`, () => runHistory(history)) + } + + it(`falls back to the proxy for rows that are not flat`, () => { + const callback = () => {} + for (const row of [ + { a: { nested: 1 } }, + { a: [1] }, + { a: new Date(0) }, + { [Symbol(`s`)]: 1 }, + new (class Row { + a = 1 + })(), + // A getter can return a new value on each read, so only the proxy, + // which reads it once, reports a stable change set. + Object.defineProperty({}, `a`, { get: () => 1, enumerable: true }), + ]) { + expect(withFlatChangeTracking([row], callback, true)).toBeUndefined() + } + }) + + it(`detaches an assigned object from later mutation`, () => { + const assigned = { nested: 1 } + const [changes] = withFlatChangeTracking( + [{ a: 1 }], + (drafts) => { + ;(drafts as Array)[0]!.a = assigned + }, + true, + )! + assigned.nested = 2 + expect(changes).toStrictEqual({ a: { nested: 1 } }) + }) + + for (const { name, shape, callback, expected } of descriptorLaws) { + it(`reports the same change set when ${name}`, () => { + expect( + withFlatChangeTracking([buildRow(shape)], callback, false), + `flat`, + ).toStrictEqual([expected]) + expect( + withChangeTracking(buildRow(shape), callback), + `proxy`, + ).toStrictEqual(expected) + }) + } + + fcTest.prop([historyArbitrary], { + seed: 44_502_101, + numRuns: oracleRuns(200), + })(`matches the draft proxy (fixed)`, runHistory) + fcTest.prop([historyArbitrary], oraclePropertyOptions(200, property))( + `matches the draft proxy (random)`, + runHistory, + ) + } else if (requestedReplayProperty === property) { + fcTest.prop([historyArbitrary], oraclePropertyOptions(200, property))( + `matches the draft proxy (replay)`, + runHistory, + ) + } else { + it.skip(`runs only when its replay property is selected`, () => {}) + } +}) diff --git a/packages/db/tests/local-only-direct-write.test.ts b/packages/db/tests/local-only-direct-write.test.ts new file mode 100644 index 0000000000..eb2664a2dd --- /dev/null +++ b/packages/db/tests/local-only-direct-write.test.ts @@ -0,0 +1,419 @@ +import { describe, expect, it } from 'vitest' +import { z } from 'zod' +import { createCollection } from '../src/index' +import { localOnlyCollectionOptions } from '../src/local-only' +import { createTransaction } from '../src/transactions' +import { createDeferred } from '../src/deferred' + +/** + * # When does a local-only direct write skip the optimistic stage? + * + * Law: a local-only Collection confirms its own writes. A direct `insert`, + * `update`, or `delete` for an operation type without a user handler, with no + * other transaction on the Collection pending or persisting, publishes the + * synced row once and returns a completed transaction. Otherwise the write + * keeps the optimistic path and its visibility: it overlays a pending + * transaction, is visible while another persists, joins an ambient + * transaction, and runs the user's handler. + * + * The change-event history oracle drives this path through generated + * histories and checks every publication. These witnesses pin the guard's + * fallback cases, which that oracle does not generate: removing the guard + * hides a direct write under a pending overlay and holds it behind a + * persisting transaction, for every operation type. + * + * A local-only Collection whose handlers all resolve is the reference for + * the direct path: it confirms the same writes through the optimistic stage. + * Both must leave the same rows, publish the same batches (order within a + * batch aside), throw the same error for a rejected write without publishing + * it, and end with completed transactions and none left on the Collection. + */ +type Row = { id: number; value: string } + +type Handlers = Partial< + Record<`onInsert` | `onUpdate` | `onDelete`, () => Promise> +> + +function createOrders(handlers: Handlers = {}) { + return createCollection( + localOnlyCollectionOptions({ + id: `local-only-direct-${Math.random()}`, + getKey: (row) => row.id, + initialData: [{ id: 1, value: `a` }], + ...handlers, + }), + ) +} + +describe(`local-only direct writes`, () => { + it(`publishes each direct write once and returns a completed transaction`, async () => { + const orders = createOrders() + const batches: Array> = [] + orders.subscribeChanges( + (changes) => + batches.push(changes.map((change) => `${change.type}:${change.key}`)), + { includeInitialState: true }, + ) + + const inserted = orders.insert({ id: 2, value: `b` }) + const updated = orders.update(1, (draft) => { + draft.value = `c` + }) + const deleted = orders.delete(2) + + for (const transaction of [inserted, updated, deleted]) { + expect(transaction.state).toBe(`completed`) + await expect(transaction.isPersisted.promise).resolves.toBe(transaction) + await expect(transaction.when(`settled`)).resolves.toBe(transaction) + } + expect(batches).toEqual([ + [`insert:1`], + [`insert:2`], + [`update:1`], + [`delete:2`], + ]) + expect(orders.get(1)).toMatchObject({ + value: `c`, + $synced: true, + $origin: `local`, + }) + expect(orders.has(2)).toBe(false) + }) + + it(`overlays a pending transaction instead of writing beneath it`, () => { + const orders = createOrders() + const pending = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + pending.mutate(() => + orders.update(1, (draft) => { + draft.value = `pending` + }), + ) + + const direct = orders.update(1, (draft) => { + draft.value = `direct` + }) + + expect(direct.state).not.toBe(`completed`) + expect(orders.get(1)?.value).toBe(`direct`) + pending.rollback() + }) + + it(`stays visible while another transaction persists`, async () => { + const orders = createOrders() + const release = createDeferred() + const persisting = createTransaction({ + mutationFn: () => release.promise, + }) + persisting.mutate(() => orders.insert({ id: 3, value: `persisting` })) + expect(persisting.state).toBe(`persisting`) + + orders.update(1, (draft) => { + draft.value = `direct` + }) + + expect(orders.get(1)?.value).toBe(`direct`) + release.resolve() + await persisting.isPersisted.promise + }) + + it(`joins an ambient transaction`, () => { + const orders = createOrders() + const ambient = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + ambient.mutate(() => + orders.update(1, (draft) => { + draft.value = `ambient` + }), + ) + + expect(ambient.state).toBe(`pending`) + expect(ambient.mutations).toHaveLength(1) + expect(orders.get(1)?.value).toBe(`ambient`) + ambient.rollback() + expect(orders.get(1)?.value).toBe(`a`) + }) + + it(`runs the user's handler for that operation type`, async () => { + let calls = 0 + const orders = createOrders({ + onUpdate: () => { + calls++ + return Promise.resolve() + }, + }) + + const transaction = orders.update(1, (draft) => { + draft.value = `handled` + }) + + expect(transaction.state).not.toBe(`completed`) + await transaction.isPersisted.promise + expect(calls).toBe(1) + expect(orders.get(1)?.value).toBe(`handled`) + // Inserts have no handler, so they still take the direct path. + expect(orders.insert({ id: 4, value: `d` }).state).toBe(`completed`) + }) +}) + +const confirmEverything: Handlers = { + onInsert: () => Promise.resolve(), + onUpdate: () => Promise.resolve(), + onDelete: () => Promise.resolve(), +} + +type Orders = ReturnType +type Write = (orders: Orders) => unknown + +// Runs writes in order and records what a caller could observe. A write +// that throws is recorded and the remaining writes still run. +async function observe(handlers: Handlers, writes: Array) { + const orders = createOrders(handlers) + const batches: Array> = [] + orders.subscribeChanges( + (changes) => + batches.push( + changes + .map( + ({ type, key, value }) => + `${type}:${key}:${value.value}:${String(value.$synced)}`, + ) + .sort(), + ), + { includeInitialState: true }, + ) + const errors: Array = [] + const transactions = [] + for (const write of writes) { + try { + transactions.push(write(orders) as ReturnType) + } catch (error) { + errors.push(`${(error as Error).name}: ${(error as Error).message}`) + } + } + await Promise.all(transactions.map((t) => t.isPersisted.promise)) + return { + rows: orders.toArray.map(({ id, value }) => ({ id, value })), + batches, + errors, + states: transactions.map((t) => t.state), + remaining: orders._state.transactions.size, + } +} + +describe(`local-only direct writes match the confirmed optimistic path`, () => { + const histories: Record> = { + 'a mixed multi-key batch': [ + (orders) => + orders.insert([ + { id: 2, value: `b` }, + { id: 3, value: `c` }, + ]), + (orders) => + orders.update([1, 2], (drafts) => { + for (const draft of drafts) draft.value += `!` + }), + (orders) => orders.delete([1, 3]), + ], + 'an insert batch with an existing key': [ + (orders) => + orders.insert([ + { id: 4, value: `d` }, + { id: 1, value: `dup` }, + ]), + (orders) => orders.insert({ id: 5, value: `e` }), + ], + 'an insert batch that repeats a key': [ + (orders) => + orders.insert([ + { id: 4, value: `d` }, + { id: 4, value: `again` }, + ]), + ], + 'an update batch with a missing key': [ + (orders) => + orders.update([1, 99], (drafts) => { + for (const draft of drafts) draft.value = `x` + }), + (orders) => + orders.update(1, (draft) => { + draft.value = `y` + }), + ], + 'an update callback that throws midway': [ + (orders) => + orders.update(1, (draft) => { + draft.value = `half` + throw new Error(`callback failed`) + }), + ], + 'a delete batch with a missing key': [ + (orders) => orders.insert({ id: 2, value: `b` }), + (orders) => orders.delete([2, 99]), + (orders) => orders.delete(1), + ], + } + + for (const [name, writes] of Object.entries(histories)) { + it(`for ${name}`, async () => { + const direct = await observe({}, writes) + const reference = await observe(confirmEverything, writes) + + expect(direct).toEqual(reference) + expect(direct.states.every((state) => state === `completed`)).toBe(true) + expect(direct.remaining).toBe(0) + }) + } + + it(`for a write the schema rejects`, () => { + const results = [] + for (const handlers of [{}, confirmEverything]) { + const orders = createCollection( + localOnlyCollectionOptions({ + id: `local-only-direct-schema-${Math.random()}`, + getKey: (row) => row.id, + schema: z.object({ id: z.number(), value: z.string().min(1) }), + initialData: [{ id: 1, value: `a` }], + ...handlers, + }), + ) + const batches: Array = [] + orders.subscribeChanges((changes) => batches.push(changes.length)) + const errors = [] + for (const write of [ + () => orders.insert({ id: 2, value: `` }), + () => + orders.update(1, (draft) => { + draft.value = `` + }), + ]) { + try { + write() + } catch (error) { + errors.push(`${(error as Error).name}: ${(error as Error).message}`) + } + } + results.push({ + errors, + batches, + rows: orders.toArray.map(({ id, value }) => ({ id, value })), + remaining: orders._state.transactions.size, + }) + } + + expect(results[0]!.errors).toHaveLength(2) + expect(results[0]!.batches).toEqual([]) + expect(results[0]).toEqual(results[1]) + }) +}) + +describe(`local-only fallbacks for every operation type`, () => { + type Kind = `insert` | `update` | `delete` + // However the fallback is reached, the write must stay visible and must + // not report completion before the optimistic stage confirms it. + const write: Record ReturnType> = + { + insert: (orders) => orders.insert({ id: 5, value: `direct` }), + update: (orders) => + orders.update(1, (draft) => { + draft.value = `direct` + }), + delete: (orders) => orders.delete(1), + } + const rollbackBatches: Record>> = { + insert: [[`insert:1`], [`insert:5`], [`delete:5`]], + update: [[`insert:1`], [`update:1`], [`update:1`]], + delete: [[`insert:1`], [`delete:1`], [`insert:1`]], + } + const visible: Record boolean> = { + insert: (orders) => orders.get(5)?.value === `direct`, + update: (orders) => orders.get(1)?.value === `direct`, + delete: (orders) => !orders.has(1), + } + + for (const kind of [`insert`, `update`, `delete`] as const) { + it(`keeps a ${kind} visible while another transaction persists`, async () => { + const orders = createOrders() + const release = createDeferred() + const persisting = createTransaction({ + mutationFn: () => release.promise, + }) + persisting.mutate(() => orders.insert({ id: 3, value: `persisting` })) + + const transaction = write[kind](orders) + + expect(transaction.state).not.toBe(`completed`) + expect(visible[kind](orders)).toBe(true) + release.resolve() + await persisting.isPersisted.promise + await transaction.isPersisted.promise + expect(visible[kind](orders)).toBe(true) + }) + + it(`overlays a pending transaction with a ${kind}`, async () => { + const orders = createOrders() + const pending = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + pending.mutate(() => + orders.update(1, (draft) => { + draft.value = `pending` + }), + ) + + const transaction = write[kind](orders) + + expect(transaction.state).not.toBe(`completed`) + expect(visible[kind](orders)).toBe(true) + pending.rollback() + await transaction.isPersisted.promise + expect(visible[kind](orders)).toBe(true) + }) + + it(`joins an ambient transaction with a ${kind}`, () => { + const orders = createOrders() + const ambient = createTransaction({ + autoCommit: false, + mutationFn: () => Promise.resolve(), + }) + ambient.mutate(() => write[kind](orders)) + + expect(ambient.mutations.map((m) => m.type)).toEqual([kind]) + expect(visible[kind](orders)).toBe(true) + ambient.rollback() + expect(visible[kind](orders)).toBe(false) + }) + + it(`rolls back a ${kind} whose handler rejects`, async () => { + const handler = `on${kind[0]!.toUpperCase()}${kind.slice(1)}` as const + const orders = createOrders({ + [handler]: () => Promise.reject(new Error(`handler failed`)), + }) + const batches: Array> = [] + orders.subscribeChanges( + (changes) => + batches.push(changes.map((change) => `${change.type}:${change.key}`)), + { includeInitialState: true }, + ) + + const transaction = write[kind](orders) + + expect(visible[kind](orders)).toBe(true) + await expect(transaction.isPersisted.promise).rejects.toThrow( + `handler failed`, + ) + expect(transaction.state).toBe(`failed`) + expect(visible[kind](orders)).toBe(false) + expect(orders.get(1)?.value ?? `a`).toBe(`a`) + expect(batches).toEqual(rollbackBatches[kind]) + // Operation types without a handler still write directly. + const other = kind === `insert` ? `update` : `insert` + expect(write[other](orders).state).toBe(`completed`) + }) + } +}) diff --git a/packages/db/tests/mutation-id.test.ts b/packages/db/tests/mutation-id.test.ts new file mode 100644 index 0000000000..76ab902a03 --- /dev/null +++ b/packages/db/tests/mutation-id.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, it, vi } from 'vitest' + +// Runtimes such as Cloudflare Workers reject random values generated at +// module scope, so a mutation id's per-runtime prefix waits for the first +// mutation. +describe(`mutation ids`, () => { + it(`do not draw random values during module evaluation`, async () => { + let next = 0 + const randomUUID = vi.fn(() => `prefix-${++next}`) + vi.stubGlobal(`crypto`, { randomUUID }) + vi.resetModules() + + try { + const { createCollection } = await import(`../src/collection/index.js`) + const { localOnlyCollectionOptions } = await import( + `../src/local-only.js` + ) + expect(randomUUID).not.toHaveBeenCalled() + + const collection = createCollection( + localOnlyCollectionOptions<{ id: string }>({ + id: `mutation-ids`, + getKey: (row) => row.id, + }), + ) + const first = collection.insert({ id: `a` }) + const second = collection.insert({ id: `b` }) + const [firstId, secondId] = [first, second].map( + (transaction) => transaction.mutations[0]!.mutationId, + ) + expect(firstId).not.toBe(secondId) + expect(firstId!.startsWith(`prefix-`)).toBe(true) + // One prefix per runtime; the counter makes each id unique. + expect(firstId!.split(`-`).slice(0, 2)).toEqual( + secondId!.split(`-`).slice(0, 2), + ) + } finally { + vi.unstubAllGlobals() + } + }) +}) diff --git a/packages/db/tests/oracle-config.ts b/packages/db/tests/oracle-config.ts index bfccf89b77..9011900773 100644 --- a/packages/db/tests/oracle-config.ts +++ b/packages/db/tests/oracle-config.ts @@ -61,6 +61,8 @@ const staticOracleProperties = [ `query-identity.equality-partition`, `where-predicate.publication`, `join-result-key.pairs`, + `pooled-live-query.publication`, + `flat-change-tracking.equivalence`, `derived-publication.membership-work`, `collection-publication.metadata-cancellation`, `collection-publication.metadata-only`, diff --git a/packages/db/tests/query/ir-stable-identity.test.ts b/packages/db/tests/query/ir-stable-identity.test.ts index bf4add9a53..a220c79b2a 100644 --- a/packages/db/tests/query/ir-stable-identity.test.ts +++ b/packages/db/tests/query/ir-stable-identity.test.ts @@ -642,6 +642,16 @@ describe(`loadSubset demand identity`, () => { ) }) + it(`gives an unqualified ref no source alias property`, () => { + expect(Object.hasOwn(new PropRef([`profile`]), `sourceAlias`)).toBe(false) + expect( + Object.hasOwn( + new PropRef([`profile`, `score`], `profile`), + `sourceAlias`, + ), + ).toBe(true) + }) + const id = new PropRef([`id`]) const group = new PropRef([`group`]) const first = new Func(`eq`, [id, new Value(`a`)]) diff --git a/packages/db/tests/query/pooled-live-query-gc.test.ts b/packages/db/tests/query/pooled-live-query-gc.test.ts new file mode 100644 index 0000000000..89e3e223d3 --- /dev/null +++ b/packages/db/tests/query/pooled-live-query-gc.test.ts @@ -0,0 +1,413 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { CollectionImpl, createCollection } from '../../src/collection/index.js' +import { createLiveQueryObserver } from '../../src/live-query-observer.js' +import { Query } from '../../src/query/builder/index.js' +import { createLiveQueryCollection, eq, not } from '../../src/query/index.js' +import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' +import { resolveLiveQueryValue } from '../../src/live-query-options.js' +import { mockSyncCollectionOptions } from '../utils.js' +import type { Collection } from '../../src/collection/index.js' + +/** + * # Does a pooled live query hold its source as long as its Collection would? + * + * Law: a pooled live query keeps its source subscribed for the same time as + * the live-query Collection it stands in for. After the last subscriber + * leaves, that is exactly `gcTime`; a `gcTime` of 0 or `Infinity` never + * releases; and a query built but never subscribed waits at least the + * Collection lifecycle's 50 ms floor. Views of one partition with different `gcTime`s release with the longest. + * A view that subscribes again after its partition released, as a hidden + * React Activity does, follows the source like its restarted Collection. + * + * Each case runs the same timeline against a pooled query and a live-query + * Collection on separate sources and compares when each source loses its + * subscriber, allowing the one tick the Collection's cleanup queue adds. + * Release timing is resource lifetime, not data, so the pooled + * live query oracle does not observe it. + */ +type Row = { id: string; g: string } +let serial = 0 + +function makeSource() { + return createCollection( + mockSyncCollectionOptions({ + id: `pooled-gc-${serial++}`, + getKey: (row) => row.id, + initialData: [{ id: `a`, g: `x` }], + }), + ) +} + +const query = (source: Collection) => (q: any) => + q.from({ r: source }).where(({ r }: any) => eq(r.g, `x`)) + +// Milliseconds after which the source no longer has a subscriber, checking +// up to `horizon`; undefined when it keeps one throughout. +async function releaseTime( + source: ReturnType, + horizon: number, +): Promise { + for (let elapsed = 0; elapsed <= horizon; elapsed++) { + if (source.subscriberCount === 0) return elapsed + // Collection cleanup settles through promises after its timer fires. + await vi.advanceTimersByTimeAsync(1) + } + return undefined +} + +function pooled(gcTime: number, subscribe: boolean) { + const source = makeSource() + const view = createPooledLiveQuery(query(source)(new Query()), { gcTime })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + if (subscribe) observer.subscribe(() => {})() + return source +} + +function compiled(gcTime: number, subscribe: boolean) { + const source = makeSource() + const live = createLiveQueryCollection({ + query: query(source), + startSync: true, + gcTime, + }) + const observer = createLiveQueryObserver(live, { mode: `wholesale` }) + if (subscribe) observer.subscribe(() => {})() + return source +} + +describe(`pooled live query gcTime`, () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + for (const gcTime of [1, 100, 0, Number.POSITIVE_INFINITY]) { + for (const subscribe of [true, false]) { + it(`matches a live-query Collection for gcTime ${gcTime}${subscribe ? `` : ` without a subscriber`}`, async () => { + const expected = await releaseTime(compiled(gcTime, subscribe), 400) + const actual = await releaseTime(pooled(gcTime, subscribe), 400) + if (expected === undefined) expect(actual).toBeUndefined() + // The Collection releases its source one cleanup-queue tick after + // its GC timer fires; the partition releases in the timer itself. + else expect([expected - 1, expected]).toContain(actual) + }) + } + } + + it(`follows its source again when resubscribed after its partition released`, async () => { + const run = async ( + build: (source: ReturnType) => any, + ) => { + const source = makeSource() + const observer = createLiveQueryObserver(build(source), { + mode: `wholesale`, + }) + observer.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(50) + // Hidden past gcTime, as under a hidden React Activity, then shown. + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `b`, g: `x` } }) + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.commit() + const stop = observer.subscribe(() => {}) + await vi.advanceTimersByTimeAsync(1) + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `c`, g: `x` } }) + source.utils.commit() + await vi.advanceTimersByTimeAsync(1) + const keys = [...observer.getSnapshot().state!.keys()] + stop() + return keys + } + const compiledKeys = await run((source) => + createLiveQueryCollection({ + query: query(source), + startSync: true, + gcTime: 1, + }), + ) + expect(compiledKeys).toEqual([`b`, `c`]) + expect( + await run( + (source) => + createPooledLiveQuery(query(source)(new Query()), { gcTime: 1 })!, + ), + ).toEqual(compiledKeys) + }) + + it(`seeds a filtered view from its refilled group after a release`, async () => { + const source = makeSource() + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `b`, g: `x` } }) + source.utils.commit() + // `not(eq(r.id, 'b'))` is evaluated per view, so this view keeps `a`. + const view = createPooledLiveQuery( + query(source)(new Query()).where(({ r }: any) => not(eq(r.id, `b`))), + { gcTime: 1 }, + )! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + observer.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(60) + const stop = observer.subscribe(() => {}) + await vi.advanceTimersByTimeAsync(1) + expect([...observer.getSnapshot().state!.keys()]).toEqual([`a`]) + source.utils.begin() + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.commit() + await vi.advanceTimersByTimeAsync(1) + expect([...observer.getSnapshot().state!.keys()]).toEqual([]) + stop() + }) + + it(`shares one partition after a released partition subscribes again`, async () => { + const source = makeSource() + const mount = () => { + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + return { observer, stop: observer.subscribe(() => {}) } + } + const first = mount() + first.stop() + await vi.advanceTimersByTimeAsync(60) + // The released partition's view subscribes again, beside a new view. + const again = first.observer.subscribe(() => {}) + const second = mount() + // The new view joins the partition that subscribed again. + expect(source.subscriberCount).toBe(1) + again() + await vi.advanceTimersByTimeAsync(60) + const third = mount() + expect(source.subscriberCount).toBe(1) + second.stop() + third.stop() + }) + + it(`keeps a newer partition when an older one releases again`, async () => { + const source = makeSource() + const mount = () => { + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + return { observer, stop: observer.subscribe(() => {}) } + } + const first = mount() + first.stop() + await vi.advanceTimersByTimeAsync(60) + // A new partition takes the key before the old view subscribes again. + const second = mount() + first.observer.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(60) + const third = mount() + expect(source.subscriberCount).toBe(1) + second.stop() + third.stop() + }) + + it(`turns terminal when source cleanup starts, before it settles`, async () => { + let release!: () => void + const source = createCollection({ + id: `pooled-gc-${serial++}`, + getKey: (row) => row.id, + startSync: true, + sync: { + sync: ({ begin, write, commit, markReady }) => { + begin() + write({ type: `insert`, value: { id: `a`, g: `x` } }) + commit() + markReady() + // The adapter's cleanup stays pending until released. + return () => + new Promise((resolve) => { + release = resolve + }) + }, + }, + }) + const observe = (view: any) => { + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + return { observer, stop: observer.subscribe(() => {}) } + } + const pooledView = observe( + createPooledLiveQuery(query(source)(new Query()), { gcTime: 1 })!, + ) + const compiledView = observe( + createLiveQueryCollection({ query: query(source), startSync: true }), + ) + await vi.advanceTimersByTimeAsync(1) + const cleanup = source.cleanup() + await vi.advanceTimersByTimeAsync(1) + expect(compiledView.observer.getSnapshot().status).toBe(`error`) + expect(pooledView.observer.getSnapshot().status).toBe(`error`) + release() + await cleanup + pooledView.stop() + compiledView.stop() + }) + + it(`defaults to a live-query Collection's gcTime when the adapter gives none`, async () => { + // A pooled view and a compiled one, each on its own source. + const pooledOnOwn = makeSource() + const pooledObserver = createLiveQueryObserver( + resolveLiveQueryValue(query(pooledOnOwn)(new Query())), + { mode: `wholesale` }, + ) + pooledObserver.subscribe(() => {})() + const compiledOnOwn = makeSource() + const compiledObserver = createLiveQueryObserver( + createLiveQueryCollection({ + query: query(compiledOnOwn), + startSync: true, + }), + { mode: `wholesale` }, + ) + compiledObserver.subscribe(() => {})() + await vi.advanceTimersByTimeAsync(4_900) + expect(compiledOnOwn.subscriberCount).toBe(1) + expect(pooledOnOwn.subscriberCount).toBe(1) + await vi.advanceTimersByTimeAsync(300) + expect(compiledOnOwn.subscriberCount).toBe(0) + expect(pooledOnOwn.subscriberCount).toBe(0) + }) + + it(`pools a query config that names only its query`, () => { + const source = makeSource() + const observe = (value: unknown) => + createLiveQueryObserver(resolveLiveQueryValue(value, { gcTime: 1 }), { + mode: `wholesale`, + }).subscribe(() => {}) + const stops = [ + observe({ query: query(source)(new Query()) }), + observe({ query: query(source)(new Query()), gcTime: 5 }), + ] + expect(source.subscriberCount).toBe(1) + // An id names a distinct Collection, so that config compiles its own. + stops.push(observe({ query: query(source)(new Query()), id: `own` })) + expect(source.subscriberCount).toBe(2) + for (const stop of stops) stop() + }) + + it(`keeps groups only for values with rows or watchers`, async () => { + const source = makeSource() + source.utils.begin() + for (let n = 0; n < 50; n++) { + source.utils.write({ type: `insert`, value: { id: `r${n}`, g: `v${n}` } }) + } + source.utils.commit() + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + const stop = observer.subscribe(() => {}) + const groups = () => + (view as unknown as { partition: { groups: Map } }) + .partition.groups.size + // One group per distinct value, plus the watched `x`. + expect(groups()).toBe(51) + source.utils.begin() + for (let n = 0; n < 50; n++) { + source.utils.write({ type: `delete`, value: { id: `r${n}`, g: `v${n}` } }) + } + source.utils.commit() + expect(groups()).toBe(1) + // The watched group empties and is dropped; new rows recreate it, and + // the observer still sees them. + source.utils.begin() + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.commit() + expect(groups()).toBe(1) + expect(observer.getSnapshot().data).toEqual([]) + stop() + expect(groups()).toBe(0) + const again = observer.subscribe(() => {}) + source.utils.begin() + source.utils.write({ type: `insert`, value: { id: `b`, g: `x` } }) + source.utils.commit() + expect([...observer.getSnapshot().state!.keys()]).toEqual([`b`]) + again() + }) + + it(`never reuses a dropped group's revision for a detached reader`, () => { + const source = makeSource() + const write = (type: `insert` | `delete`, id: string, g: string) => { + source.utils.begin() + source.utils.write({ type, value: { id, g } }) + source.utils.commit() + } + // A watched `y` view keeps the partition subscribed. + write(`insert`, `z`, `y`) + const keepAlive = createLiveQueryObserver( + createPooledLiveQuery( + new Query() + .from({ r: source }) + .where(({ r }: any) => eq(r.g, `y`)) as any, + { gcTime: 1 }, + ), + { mode: `wholesale` }, + ).subscribe(() => {}) + // The `x` view is read without subscribing, so it compares revisions. + const reader = createLiveQueryObserver( + createPooledLiveQuery(query(source)(new Query()), { gcTime: 1 }), + { mode: `wholesale` }, + ) + write(`insert`, `b`, `x`) + const keys = () => [...reader.getSnapshot().state!.keys()] + expect(keys()).toEqual([`a`, `b`]) + // Emptying the unwatched group drops it; new rows build a new one. + source.utils.begin() + source.utils.write({ type: `delete`, value: { id: `a`, g: `x` } }) + source.utils.write({ type: `delete`, value: { id: `b`, g: `x` } }) + source.utils.commit() + write(`insert`, `c`, `x`) + write(`insert`, `d`, `x`) + expect(keys()).toEqual([`c`, `d`]) + keepAlive() + }) + + it(`keeps its public Collection live while observed`, async () => { + const source = makeSource() + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const observer = createLiveQueryObserver(view, { mode: `wholesale` }) + const stop = observer.subscribe(() => {}) + const collection = observer.getSnapshot().collection! + expect(collection.toArray).toHaveLength(1) + await vi.advanceTimersByTimeAsync(500) + expect(collection.status).toBe(`ready`) + expect(collection.toArray).toHaveLength(1) + stop() + }) + + it(`hands users a Collection that queries accept as a source`, () => { + const source = makeSource() + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime: 1, + })! + const collection = createLiveQueryObserver(view, { + mode: `wholesale`, + }).getSnapshot().collection! + expect(collection).toBeInstanceOf(CollectionImpl) + const nested = createLiveQueryCollection({ + query: (q) => q.from({ c: collection }), + startSync: true, + }) + expect(nested.toArray.map((row) => row.id)).toEqual([`a`]) + }) + + it.each([[[5, 120]], [[120, 5]]])( + `releases with the longest gcTime among a partition's views (%j)`, + async (gcTimes) => { + const source = makeSource() + for (const gcTime of gcTimes) { + const view = createPooledLiveQuery(query(source)(new Query()), { + gcTime, + })! + createLiveQueryObserver(view, { mode: `wholesale` }).subscribe( + () => {}, + )() + } + expect(await releaseTime(source, 400)).toBe(120) + }, + ) +}) diff --git a/packages/db/tests/query/pooled-live-query-oracle.property.test.ts b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts new file mode 100644 index 0000000000..694f1ab8a1 --- /dev/null +++ b/packages/db/tests/query/pooled-live-query-oracle.property.test.ts @@ -0,0 +1,957 @@ +/** + * # Does a pooled live query publish what its live-query Collection would? + * + * Law and source: a live query that filters one source Collection by + * `eq(field, literal)` conjuncts, plus any conjuncts that read only its row, + * is served from a partition of that source shared by every query with the + * same `eq` fields and order; each view evaluates its other conjuncts itself. Its observer must publish the + * rows the live-query Collection for the same query publishes: the visible + * source rows whose fields equal the literals under `eq` semantics + * (`src/query/compiler/evaluators.ts`: nullish is UNKNOWN, a Date equals its + * timestamp, `NaN` equals `NaN`, `-0` equals `0`, other types differ), in its order or else key order, with the same row values and status. + * + * Why an example can miss the failure: one query over static rows passes even + * if rows never move between groups, a peer group never sees a row leave, a + * remounted query reads a stale group, or a rollback leaves an optimistic row + * behind. + * + * Model: `expectedKeys` filters the model's visible rows with an independent + * `eq` over plain values and the peer's negated `g` equality. It does not import the evaluator, normalization, or + * the partition. Order, row values, and status come from a second + * formulation: a live-query Collection compiled for the same query. + * + * History grammar: rows have ids 0 through 3, delivered initially in key + * order or in reverse, a field `f` from strings, + * numbers and their look-alikes, `true`, a Date equal to 1, `NaN`, `-0`, + * `0`, `null`, and a missing value, and a field `g` of `x`, `y`, or `null`. Up to three peer queries use `eq(f, literal)`, optionally with + * `eq(g, literal)`, a residual `not(eq(g, literal))`, and an order by `id` or + * by `g` with explicit `nulls`; a weighted peer orders a whole `f` group. + * Values are weighted toward `a`, and toward the normalized values against + * numeric literals, so groups hold rows that stay, move, and normalize. + * Steps commit sync transactions of one or two inserts, updates, or deletes, + * where an update may keep `f` so the row stays in its group; apply one + * optimistic insert, update, or delete and then confirm or roll it back, + * optionally after cleaning up and restarting the source while it is pending, + * with a weighted run that inserts a row most peers see before that cleanup; mount or unmount a peer; or clean up the source and restart it, + * optionally (re)mounting a peer on the cleaned-up source first. + * + * Production driver: `createPooledLiveQuery` builds each peer's view from the + * query builder's IR, and `createLiveQueryObserver` observes it in wholesale + * and granular mode, as the framework adapters do. The view subscribes + * before its live-query Collection preloads, so a peer mounted after cleanup + * is the one that restarts the source. + * + * Refinement check: after every step, each mounted peer's wholesale snapshot + * equals its live-query Collection's keys in order, row values, and status, + * and its rows' fields equal the model's. The granular changes delivered since + * the last checkpoint equal those of a granular observer of the live-query + * Collection, by type, key, value, and previous value; folded in order, they + * never insert a held key, update or delete an unheld one, or carry a stale + * previous value, and they leave the model's rows. A peer mounted when its + * source starts cleanup is terminal like its live query: status `error` with + * the rows it had, pending optimistic rows included, through the restart and + * later writes. A peer mounted after cleanup follows the restarted source. + * + * Calibration: a partition that ignored the previous value, kept group rows + * in arrival order, or compared literals without normalization fails the + * pinned histories and campaigns. A view that kept reporting the source's + * status after cleanup fails the cleanup histories. An update delivered as a + * delete and insert, with a stale previous value, or with the previous row as + * its value fails the in-group update history and both campaigns; so do a + * partition born terminal on a cleaned-up source and a freeze that drops + * pending optimistic rows. + * + * Known omissions: on-demand and persisted sources, `DbClient` hydration, + * Suspense, `select`, and every other clause keep the live-query Collection + * and are outside this owner. + */ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { createLiveQueryObserver } from '../../src/live-query-observer.js' +import { Query } from '../../src/query/builder/index.js' +import { + and, + createLiveQueryCollection, + eq, + not, +} from '../../src/query/index.js' +import { createPooledLiveQuery } from '../../src/query/pooled-live-query.js' +import { Func, PropRef, Value } from '../../src/query/ir.js' +import { + oraclePropertyOptions, + oracleRuns, + readOracleRunConfig, +} from '../oracle-config.js' +import { + flushPromises, + mockSyncCollectionOptions, + withExpectedRejection, + withOracleCleanup, +} from '../utils.js' +import type { ChangeMessage } from '../../src/types.js' + +const property = `pooled-live-query.publication` +const requestedReplayProperty = readOracleRunConfig().replayProperty + +const MISSING = Symbol(`missing`) +type FieldValue = string | number | boolean | Date | null | typeof MISSING +type Row = { id: string; f?: unknown; g: string | null } +type Peer = { + f: string | number | boolean + g?: string + // A residual conjunct, `not(eq(r.g, notG))`, each view evaluates itself. + notG?: string + // Orders a peer's rows; the reference gives the expected order. A group's + // rows share `f`, so orders read `g`, which may be null, and the id. + order?: PeerOrder +} +type PeerOrder = + | `id-desc` + | `g-desc` + | `g-asc-id-desc` + | `g-asc-nulls-first` + | `g-asc-nulls-last` + | `g-desc-nulls-last` +type Step = + | { kind: `sync`; ops: Array } + | { + kind: `optimistic` + op: Op + confirm: boolean + // Clean up and restart the source before settling the write. + cleanupFirst?: boolean + } + | { kind: `mount`; peer: number } + | { kind: `unmount`; peer: number } + // `mount` (re)mounts that peer after cleanup, before the restart. + | { kind: `cleanup-restart`; mount?: number } +type Op = + | { type: `insert`; id: number; f: FieldValue; g: string | null } + // `keepF` keeps the row's current `f`, so the row stays in its group. + | { + type: `update` + id: number + f: FieldValue + g: string | null + keepF?: boolean + } + | { type: `delete`; id: number } +type History = { + rows: Array<{ f: FieldValue; g: string | null }> + // The source delivers its initial rows in reverse key order. + reverseInitial?: boolean + peers: Array + steps: Array +} + +const DATE_ONE = new Date(1) +const fieldValues: ReadonlyArray = [ + `a`, + `b`, + 1, + `1`, + true, + DATE_ONE, + Number.NaN, + -0, + 0, + null, + MISSING, +] +const literals: ReadonlyArray = [`a`, 1, `1`, true, Number.NaN, 0] + +// --------------------------------------------------------------------------- +// Model +// --------------------------------------------------------------------------- + +// `eq` keeps a row only when TRUE: nullish is UNKNOWN, a Date is its +// timestamp, NaN equals NaN, and -0 equals 0. +function modelEq(value: unknown, literal: unknown): boolean { + if (value === null || value === undefined) return false + const left = value instanceof Date ? value.getTime() : value + if (typeof left === `number` && typeof literal === `number`) { + return (Number.isNaN(left) && Number.isNaN(literal)) || left === literal + } + return typeof left === typeof literal && left === literal +} + +function expectedKeys( + rows: ReadonlyMap, + peer: Peer, +): Array { + return [...rows.values()] + .filter( + (row) => + modelEq(row.f, peer.f) && + (peer.g === undefined || row.g === peer.g) && + // A null `g` makes the negated equality UNKNOWN, which excludes. + (peer.notG === undefined || (row.g !== null && row.g !== peer.notG)), + ) + .map((row) => row.id) + .sort() +} + +// The model's matching rows, described by id and fields. +function expectedRows( + rows: ReadonlyMap, + peer: Peer, +): Array { + return expectedKeys(rows, peer).map((key) => describeFields(rows.get(key)!)) +} + +// Collection change detection treats 0 and -0, and NaN and NaN, as equal. +function sameValueZero(a: unknown, b: unknown): boolean { + return a === b || (Number.isNaN(a) && Number.isNaN(b)) +} + +function sourceRow(id: string, f: FieldValue, g: string | null): Row { + return f === MISSING ? { id, g } : { id, f, g } +} + +// --------------------------------------------------------------------------- +// History grammar +// --------------------------------------------------------------------------- + +// Most rows and peers share `a`, so groups hold rows that stay or move; +// the normalized values have their own weight against numeric literals. +const fieldArbitrary = fc.oneof( + { weight: 2, arbitrary: fc.constant(`a`) }, + fc.constantFrom(DATE_ONE, Number.NaN, -0), + fc.constantFrom(...fieldValues), +) +const gArbitrary = fc.constantFrom(`x`, `y`) +// Rows may hold a null `g`, which orders by its `nulls` option. +const rowGArbitrary = fc.constantFrom(`x`, `y`, null) +const opArbitrary: fc.Arbitrary = fc.oneof( + fc.record({ + type: fc.constant(`insert` as const), + id: fc.nat({ max: 3 }), + f: fieldArbitrary, + g: rowGArbitrary, + }), + { + weight: 2, + arbitrary: fc.record( + { + type: fc.constant(`update` as const), + id: fc.nat({ max: 3 }), + f: fieldArbitrary, + g: rowGArbitrary, + keepF: fc.boolean(), + }, + { requiredKeys: [`type`, `id`, `f`, `g`] }, + ), + }, + fc.record({ type: fc.constant(`delete` as const), id: fc.nat({ max: 3 }) }), +) +const orderArbitrary = fc.constantFrom( + `id-desc`, + `g-desc`, + `g-asc-id-desc`, + `g-asc-nulls-first`, + `g-asc-nulls-last`, + `g-desc-nulls-last`, +) +const peerArbitrary: fc.Arbitrary = fc.record( + { + f: fc.oneof( + { weight: 2, arbitrary: fc.constant(`a`) }, + { + weight: 2, + arbitrary: fc.constantFrom(1, Number.NaN, 0), + }, + fc.constantFrom(...literals), + ), + g: gArbitrary, + notG: gArbitrary, + order: orderArbitrary, + }, + { requiredKeys: [`f`] }, +) +const historyArbitrary: fc.Arbitrary = fc.record({ + rows: fc.array(fc.record({ f: fieldArbitrary, g: rowGArbitrary }), { + maxLength: 4, + }), + reverseInitial: fc.boolean(), + peers: fc.array( + fc.oneof( + { weight: 2, arbitrary: peerArbitrary }, + // An ordered peer over a whole group, so rows with several `g` + // values, including null, share one ordered view. + fc.record({ f: fc.constant(`a`), order: orderArbitrary }), + ), + { minLength: 1, maxLength: 3 }, + ), + steps: fc.array( + fc.oneof( + { + weight: 3, + arbitrary: fc.record({ + kind: fc.constant(`sync` as const), + ops: fc.array(opArbitrary, { minLength: 1, maxLength: 2 }), + }), + }, + { + weight: 2, + arbitrary: fc.record( + { + kind: fc.constant(`optimistic` as const), + op: opArbitrary, + confirm: fc.boolean(), + cleanupFirst: fc.boolean(), + }, + { requiredKeys: [`kind`, `op`, `confirm`] }, + ), + }, + // A pending write that most peers see, settled after cleanup, is what + // a freeze must keep; independent choices rarely line it up. + fc.record({ + kind: fc.constant(`optimistic` as const), + op: fc.record({ + type: fc.constant(`insert` as const), + id: fc.nat({ max: 3 }), + f: fc.constant(`a`), + g: gArbitrary, + }), + confirm: fc.boolean(), + cleanupFirst: fc.constant(true), + }), + fc.record({ + kind: fc.constant(`mount` as const), + peer: fc.nat({ max: 2 }), + }), + fc.record({ + kind: fc.constant(`unmount` as const), + peer: fc.nat({ max: 2 }), + }), + fc.record( + { + kind: fc.constant(`cleanup-restart` as const), + mount: fc.nat({ max: 2 }), + }, + { requiredKeys: [`kind`] }, + ), + ), + { maxLength: 8 }, + ), +}) + +// Each pinned history moves a row across groups a short example would keep +// still. +const pinnedHistories: ReadonlyArray<{ name: string; history: History }> = [ + { + name: `a Date row joins the group of its timestamp`, + history: { + rows: [{ f: `a`, g: `x` }], + peers: [{ f: 1 }, { f: `a` }], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: DATE_ONE, g: `x` }] }, + ], + }, + }, + { + name: `rows keep key order across strings and numbers`, + history: { + rows: [ + { f: `a`, g: `x` }, + { f: `a`, g: `x` }, + ], + peers: [{ f: `a` }], + steps: [ + { kind: `sync`, ops: [{ type: `insert`, id: 4, f: `a`, g: `x` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `b`, g: `x` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `a`, g: `x` }] }, + ], + }, + }, + { + name: `a rolled-back optimistic insert leaves its group`, + history: { + rows: [], + peers: [{ f: true, g: `y` }], + steps: [ + { + kind: `optimistic`, + op: { type: `insert`, id: 2, f: true, g: `y` }, + confirm: false, + }, + ], + }, + }, + { + name: `a peer mounted at cleanup stays failed while a new one follows the restart`, + history: { + rows: [{ f: `a`, g: `x` }], + peers: [{ f: `a` }, { f: `a` }], + steps: [ + { kind: `unmount`, peer: 1 }, + { kind: `sync`, ops: [{ type: `insert`, id: 2, f: `a`, g: `y` }] }, + { kind: `cleanup-restart` }, + { kind: `mount`, peer: 1 }, + { kind: `sync`, ops: [{ type: `insert`, id: 3, f: `a`, g: `x` }] }, + ], + }, + }, + { + name: `peers clean up with a pending write and one mounts before the restart`, + history: { + rows: [{ f: `a`, g: `x` }], + peers: [{ f: `a` }, { f: `a`, g: `y` }], + steps: [ + { + kind: `optimistic`, + op: { type: `insert`, id: 2, f: `a`, g: `y` }, + confirm: false, + cleanupFirst: true, + }, + { kind: `sync`, ops: [{ type: `insert`, id: 3, f: `a`, g: `y` }] }, + { kind: `cleanup-restart`, mount: 1 }, + { kind: `sync`, ops: [{ type: `update`, id: 3, f: `a`, g: `x` }] }, + ], + }, + }, + { + name: `an update reorders rows within one ordered group`, + history: { + rows: [ + { f: `a`, g: `x` }, + { f: `a`, g: `y` }, + ], + peers: [ + { f: `a`, order: `g-desc` }, + { f: `a`, order: `g-asc-nulls-last` }, + ], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `a`, g: `y` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 1, f: `a`, g: null }] }, + { + kind: `optimistic`, + op: { type: `update`, id: 0, f: `a`, g: null }, + confirm: false, + }, + ], + }, + }, + { + name: `a residual conjunct moves rows in and out of a view within one group`, + history: { + rows: [ + { f: `a`, g: `x` }, + { f: `a`, g: `y` }, + ], + peers: [{ f: `a`, notG: `y` }, { f: `a` }], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: `a`, g: `y` }] }, + { kind: `sync`, ops: [{ type: `update`, id: 1, f: `a`, g: `x` }] }, + { + kind: `optimistic`, + op: { type: `update`, id: 1, f: `a`, g: `y` }, + confirm: false, + }, + { kind: `sync`, ops: [{ type: `delete`, id: 1 }] }, + ], + }, + }, + { + name: `a row updated within its group reaches peers as one update`, + history: { + rows: [ + { f: 1, g: `x` }, + { f: `b`, g: `x` }, + ], + peers: [{ f: 1 }, { f: 1, g: `y` }], + steps: [ + { kind: `sync`, ops: [{ type: `update`, id: 0, f: DATE_ONE, g: `y` }] }, + { + kind: `optimistic`, + op: { type: `update`, id: 0, f: 1, g: `y` }, + confirm: true, + }, + ], + }, + }, + { + name: `a remounted peer reads a group that changed while it was away`, + history: { + rows: [{ f: 0, g: `x` }], + peers: [{ f: 0 }, { f: Number.NaN }], + steps: [ + { kind: `unmount`, peer: 1 }, + { + kind: `sync`, + ops: [{ type: `update`, id: 0, f: Number.NaN, g: `x` }], + }, + { kind: `mount`, peer: 1 }, + { kind: `sync`, ops: [{ type: `update`, id: 0, f: -0, g: `x` }] }, + ], + }, + }, +] + +// --------------------------------------------------------------------------- +// Production driver and refinement check +// --------------------------------------------------------------------------- + +let serial = 0 +const tick = () => new Promise((resolve) => setTimeout(resolve, 0)) + +function peerQuery(source: any, peer: Peer) { + return (q: any) => { + const query = q.from({ r: source }).where(({ r }: any) => { + const conjuncts = [eq(r.f, peer.f)] + if (peer.g !== undefined) conjuncts.push(eq(r.g, peer.g)) + if (peer.notG !== undefined) conjuncts.push(not(eq(r.g, peer.notG))) + const [first, second, ...rest] = conjuncts + return second ? and(first, second, ...rest) : first + }) + switch (peer.order) { + case undefined: + return query + case `id-desc`: + return query.orderBy(({ r }: any) => r.id, `desc`) + case `g-desc`: + return query.orderBy(({ r }: any) => r.g, `desc`) + case `g-asc-id-desc`: + return query + .orderBy(({ r }: any) => r.g) + .orderBy(({ r }: any) => r.id, `desc`) + case `g-asc-nulls-first`: + return query.orderBy(({ r }: any) => r.g, { + direction: `asc`, + nulls: `first`, + }) + case `g-asc-nulls-last`: + return query.orderBy(({ r }: any) => r.g, { + direction: `asc`, + nulls: `last`, + }) + case `g-desc-nulls-last`: + return query.orderBy(({ r }: any) => r.g, { + direction: `desc`, + nulls: `last`, + }) + } + } +} + +// A row's id and fields, which the model also knows. +function describeFields(row: Record) { + const f = row.f instanceof Date ? `Date(${row.f.getTime()})` : String(row.f) + return `${String(row.id)}:${typeof row.f}:${f}:${String(row.g)}` +} + +function describeRow(row: Record) { + return `${describeFields(row)}:${String(row.$synced)}` +} + +type Change = ChangeMessage, string | number> + +function describeChange(change: Change) { + const previous = change.previousValue + return `${change.type}:${String(change.key)}:${describeRow(change.value)}:${previous ? describeRow(previous) : `-`}` +} + +// Applies a granular batch to the rows a consumer has folded so far, +// recording each change that contradicts them. +function foldChanges( + rows: Map>, + changes: Array, + violations: Array, +) { + for (const change of changes) { + const held = rows.get(change.key) + if (change.type === `insert`) { + if (held) violations.push(`insert of held ${describeChange(change)}`) + rows.set(change.key, change.value) + } else if (!held) { + violations.push(`${change.type} of unheld ${describeChange(change)}`) + } else if (change.type === `delete`) { + rows.delete(change.key) + } else { + if ( + !change.previousValue || + describeFields(change.previousValue) !== describeFields(held) + ) { + violations.push(`stale previousValue in ${describeChange(change)}`) + } + rows.set(change.key, change.value) + } + } +} + +async function runHistory(history: History): Promise { + const id = (n: number) => `r${n}` + const rows = new Map() + history.rows.forEach((row, n) => + rows.set(id(n), sourceRow(id(n), row.f, row.g)), + ) + const source = createCollection( + mockSyncCollectionOptions({ + id: `pooled-${serial++}`, + getKey: (row) => row.id, + initialData: (history.reverseInitial + ? [...rows.values()].reverse() + : [...rows.values()] + ).map((row) => ({ ...row })), + }), + ) + await source.stateWhenReady() + const references: Array> = [] + type Mounted = { + reference: ReturnType + // The model rows when the source started cleanup, if it has since. + frozen: Array | undefined + view: { + collection?: unknown + status: string + entries: () => Iterable<[string | number, Record]> + } + layout: { keys: string; revision: number } | undefined + wholesale: ReturnType> + // Rows folded from the pooled granular stream, and contradictions. + granularRows: Map> + violations: Array + // Granular changes since the last checkpoint, pooled and reference. + pooledChanges: Array + referenceChanges: Array + unsubscribe: () => void + } + const mounted = new Map() + const mount = async (index: number) => { + const peer = history.peers[index] + if (!peer || mounted.has(index)) return + // The pooled view subscribes first, so after cleanup it is the one that + // restarts the source. + const view = createPooledLiveQuery(peerQuery(source, peer)(new Query())) + expect(view, `peer ${index} is poolable`).toBeDefined() + const wholesale = createLiveQueryObserver(view as any, { + mode: `wholesale`, + }) + const granular = createLiveQueryObserver(view as any) + const entry: Mounted = { + reference: createLiveQueryCollection(peerQuery(source, peer)), + frozen: undefined, + view: view as unknown as Mounted[`view`], + layout: undefined, + wholesale, + granularRows: new Map(), + violations: [], + pooledChanges: [], + referenceChanges: [], + unsubscribe: () => {}, + } + references.push(entry.reference) + const offWholesale = wholesale.subscribe(() => {}) + const offGranular = granular.subscribe((changes) => { + const batch = (changes ?? []) as Array + foldChanges(entry.granularRows, batch, entry.violations) + entry.pooledChanges.push(...batch.map(describeChange)) + }) + await entry.reference.preload() + const referenceGranular = createLiveQueryObserver(entry.reference as any) + const offReference = referenceGranular.subscribe((changes) => { + const batch = (changes ?? []) as Array + entry.referenceChanges.push(...batch.map(describeChange)) + }) + entry.unsubscribe = () => { + offWholesale() + offGranular() + offReference() + wholesale.dispose() + granular.dispose() + referenceGranular.dispose() + } + mounted.set(index, entry) + } + const check = (checkpoint: string) => { + for (const [index, entry] of mounted) { + const { view, wholesale } = entry + const peer = history.peers[index]! + const snapshot = wholesale.getSnapshot() + const reference = entry.reference + const label = `${checkpoint}, peer ${index}` + expect( + (snapshot.data as Array>).map(describeRow), + `${label} rows`, + ).toEqual(reference.toArray.map((row) => describeRow(row))) + expect(snapshot.status, `${label} status`).toBe(reference.status) + // The view itself, which the observer reads only when notified. + expect( + [...view.entries()].map(([, row]) => describeRow(row)), + `${label} view rows`, + ).toEqual(reference.toArray.map((row) => describeRow(row))) + expect(view.status, `${label} view status`).toBe(reference.status) + const model = entry.frozen ?? expectedRows(rows, peer) + expect( + [...snapshot.state!.values()].map(describeFields).sort(), + `${label} model`, + ).toEqual(model) + // The granular stream delivers each change once, with the type and + // values the live-query Collection's stream has, and folds to the model. + expect(entry.violations, `${label} granular contradictions`).toEqual([]) + expect( + [...entry.granularRows.values()].map(describeFields).sort(), + `${label} granular`, + ).toEqual(model) + expect(entry.pooledChanges.sort(), `${label} granular changes`).toEqual( + entry.referenceChanges.sort(), + ) + entry.pooledChanges.length = 0 + entry.referenceChanges.length = 0 + // A change in the ordered keys always advances the layout revision. + const keys = JSON.stringify([...snapshot.state!.keys()]) + if (entry.layout && entry.layout.keys !== keys) { + expect(snapshot.layoutRevision, `${label} layout`).toBeGreaterThan( + entry.layout.revision, + ) + } + entry.layout = { keys, revision: snapshot.layoutRevision } + // Observing a pooled view never builds its live-query Collection. + expect(view.collection, `${label} materialized`).toBeUndefined() + } + } + // Resolves `keepF` against the model's current row. + const resolveOp = (op: Op): Op => { + const row = rows.get(id(op.id)) + if (op.type !== `update` || !op.keepF || !row) return op + return { ...op, f: `f` in row ? (row.f as FieldValue) : MISSING } + } + const applyOp = (op: Op): boolean => { + const key = id(op.id) + if (op.type === `insert`) { + if (rows.has(key)) return false + rows.set(key, sourceRow(key, op.f, op.g)) + } else if (op.type === `update`) { + if (!rows.has(key)) return false + rows.set(key, sourceRow(key, op.f, op.g)) + } else { + if (!rows.delete(key)) return false + } + return true + } + const write = (op: Op) => { + const key = id(op.id) + source.utils.write( + op.type === `delete` + ? { type: `delete`, key } + : op.type === `update` && op.f === MISSING + ? // A synced update merges fields, so clear `f` explicitly. + { type: `update`, value: { id: key, f: undefined, g: op.g } } + : { type: op.type, value: sourceRow(key, op.f, op.g) }, + ) + } + + const unmount = (index: number) => { + mounted.get(index)?.unsubscribe() + mounted.delete(index) + } + // Every mounted peer freezes with the rows it had, pending writes included. + // `beforeRestart` runs once the source is cleaned up. + const cleanupAndRestart = async ( + checkpoint: string, + beforeRestart: () => Promise, + ) => { + for (const [index, entry] of mounted) { + entry.frozen ??= expectedRows(rows, history.peers[index]!) + } + await source.cleanup() + check(`${checkpoint} cleaned up`) + // The mock source re-syncs its initial rows when it restarts. + rows.clear() + history.rows.forEach((row, rowIndex) => + rows.set(id(rowIndex), sourceRow(id(rowIndex), row.f, row.g)), + ) + await beforeRestart() + await source.preload() + } + + await withOracleCleanup(async () => { + for (const index of history.peers.keys()) await mount(index) + check(`after mount`) + for (const [n, step] of history.steps.entries()) { + const checkpoint = `after step ${n} (${step.kind})` + if (step.kind === `mount`) await mount(step.peer) + else if (step.kind === `cleanup-restart`) { + const between = step.mount + await cleanupAndRestart(checkpoint, async () => { + // A peer mounted on the cleaned-up source restarts it. + if (between === undefined) return + unmount(between) + await mount(between) + }) + } else if (step.kind === `unmount`) { + unmount(step.peer) + } else if (step.kind === `sync`) { + const accepted = step.ops.map(resolveOp).filter((op) => { + const before = new Map(rows) + if (applyOp(op)) return true + rows.clear() + for (const [k, v] of before) rows.set(k, v) + return false + }) + if (accepted.length === 0) continue + source.utils.begin() + for (const op of accepted) write(op) + source.utils.commit() + } else { + const { confirm, cleanupFirst } = step + const op = resolveOp(step.op) + const key = id(op.id) + const before = new Map(rows) + const previous = rows.get(key) + // An update to the same value creates no pending transaction. + if ( + op.type === `update` && + previous !== undefined && + `f` in previous === (op.f !== MISSING) && + sameValueZero(previous.f, op.f === MISSING ? undefined : op.f) && + previous.g === op.g + ) { + continue + } + if (!applyOp(op)) continue + const transaction = + op.type === `insert` + ? source.insert(sourceRow(key, op.f, op.g)) + : op.type === `update` + ? source.update(key, (draft) => { + if (op.f === MISSING) delete draft.f + else draft.f = op.f + draft.g = op.g + }) + : source.delete(key) + const persisted = transaction.isPersisted.promise.catch(() => undefined) + check(`${checkpoint} pending`) + if (cleanupFirst) { + // The write settles after cleanup, before the restart. The mock + // server keeps no data, so either outcome leaves the initial rows. + await cleanupAndRestart(checkpoint, async () => { + if (confirm) { + source.utils.resolveSync() + await persisted + return + } + await withExpectedRejection(`rolled back`, async () => { + source.utils.rejectSync(new Error(`rolled back`)) + await persisted + await flushPromises() + }) + }) + } else if (confirm) { + source.utils.begin() + write(op) + source.utils.commit() + source.utils.resolveSync() + } else { + rows.clear() + for (const [k, v] of before) rows.set(k, v) + await withExpectedRejection(`rolled back`, async () => { + source.utils.rejectSync(new Error(`rolled back`)) + await persisted + await flushPromises() + }) + } + await tick() + } + check(checkpoint) + } + // Any other Collection member builds the live-query Collection. + // A terminal peer's Collection would be a new query on the restarted + // source, so only live peers compare forwarded rows. + for (const [index, { wholesale, reference, frozen }] of mounted) { + if (frozen) continue + const collection = wholesale.getSnapshot().collection! + expect( + (collection.toArray as Array>).map(describeRow), + `peer ${index} forwarded toArray`, + ).toEqual(reference.toArray.map((row) => describeRow(row))) + // Members the observer also reads must still behave as the Collection's. + expect( + [...collection.entries()].map(([key]) => key), + `peer ${index} forwarded entries`, + ).toEqual([...reference.entries()].map(([key]) => key)) + const narrower = new Func(`eq`, [new PropRef([`g`]), new Value(`x`)]) + const keysOf = (target: { + currentStateAsChanges: (options: { + where: typeof narrower + }) => Array<{ key: unknown }> | void + }) => + [...(target.currentStateAsChanges({ where: narrower }) || [])].map( + (change) => change.key, + ) + expect( + keysOf(collection as unknown as Parameters[0]), + `peer ${index} forwarded filter`, + ).toEqual(keysOf(reference as unknown as Parameters[0])) + expect(collection.config, `peer ${index} forwarded config`).toBeDefined() + } + }, [ + () => { + for (const entry of mounted.values()) entry.unsubscribe() + }, + () => Promise.all(references.map((reference) => reference.cleanup())), + () => source.cleanup(), + ]) +} + +describe(`pooled live query oracle`, () => { + if (requestedReplayProperty === undefined) { + // Generated fields are plain values; this pins a field whose getter + // throws, which both paths treat as a row that does not match. + it(`matches the live-query Collection when an eq path getter throws`, async () => { + const source = createCollection( + mockSyncCollectionOptions({ + id: `pooled-${serial++}`, + getKey: (row) => row.id, + initialData: [ + { id: `a`, profile: { code: 1 } }, + { + id: `b`, + profile: { + get code(): number { + throw new Error(`getter failed`) + }, + }, + }, + ], + }), + ) + await source.stateWhenReady() + const query = (q: any) => + q.from({ r: source }).where(({ r }: any) => eq(r.profile.code, 1)) + const pooled = createLiveQueryObserver( + createPooledLiveQuery(query(new Query())), + { mode: `wholesale` }, + ) + const stop = pooled.subscribe(() => {}) + const reference = createLiveQueryCollection({ query, startSync: true }) + await reference.preload() + const snapshot = pooled.getSnapshot() + expect(snapshot.status).toBe(reference.status) + expect([...snapshot.state!.keys()]).toEqual([...reference.keys()]) + stop() + await source.cleanup() + }) + + for (const { name, history } of pinnedHistories) { + it(`matches the live-query Collection when ${name}`, () => + runHistory(history)) + } + fcTest.prop([historyArbitrary], { + seed: 44_502_001, + numRuns: oracleRuns(80), + })(`matches the live-query Collection (fixed)`, runHistory) + fcTest.prop([historyArbitrary], oraclePropertyOptions(80, property))( + `matches the live-query Collection (random)`, + runHistory, + ) + } else if (requestedReplayProperty === property) { + fcTest.prop([historyArbitrary], oraclePropertyOptions(80, property))( + `matches the live-query Collection (replay)`, + runHistory, + ) + } else { + it.skip(`runs only when its replay property is selected`, () => {}) + } +}) diff --git a/packages/db/tests/query/pooled-query-identity.test.ts b/packages/db/tests/query/pooled-query-identity.test.ts new file mode 100644 index 0000000000..aab60ccf61 --- /dev/null +++ b/packages/db/tests/query/pooled-query-identity.test.ts @@ -0,0 +1,131 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { getPreparedLiveQueryIdentity } from '../../src/live-query-options.js' +import { Query } from '../../src/query/builder/index.js' +import { and, eq, gt } from '../../src/query/index.js' +import { mockSyncCollectionOptions } from '../utils.js' + +/** + * A query a partition can serve is identified by its source and its `eq` + * fields and literals, which the partition already extracts. Two queries + * with the same identity must publish the same rows, and queries that can + * publish different rows must have different identities. Queries the + * partition cannot serve keep the full structural identity. + */ +type Row = { id: string; a: string; b: string; n: number } +let serial = 0 +const makeSource = () => + createCollection( + mockSyncCollectionOptions({ + id: `pooled-identity-${serial++}`, + getKey: (row) => row.id, + initialData: [], + }), + ) +const identity = (build: (q: any) => unknown) => + getPreparedLiveQueryIdentity(build(new Query())) + +describe(`pooled query identity`, () => { + const source = makeSource() + + it(`ignores conjunct and operand order`, () => { + expect( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(eq(r.a, `x`), eq(r.b, `y`))), + ), + ).toEqual( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => eq(`y`, r.b)) + .where(({ r }: any) => eq(r.a, `x`)), + ), + ) + }) + + it(`ignores the source alias`, () => { + expect( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ), + ).toEqual( + identity((q) => + q.from({ s: source }).where(({ s }: any) => eq(s.a, `x`)), + ), + ) + }) + + it(`separates literals, fields, and sources`, () => { + const base = identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ) + expect( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `y`)), + ), + ).not.toEqual(base) + expect( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.b, `x`)), + ), + ).not.toEqual(base) + // A literal 1 and '1' match different rows. + expect( + identity((q) => q.from({ r: source }).where(({ r }: any) => eq(r.n, 1))), + ).not.toEqual( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.n, `1`)), + ), + ) + const other = makeSource() + expect( + identity((q) => q.from({ r: other }).where(({ r }: any) => eq(r.a, `x`))), + ).not.toEqual(base) + }) + + it(`separates orders`, () => { + const ordered = (direction: `asc` | `desc`) => + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => eq(r.a, `x`)) + .orderBy(({ r }: any) => r.n, direction), + ) + expect(ordered(`asc`)).toEqual(ordered(`asc`)) + expect(ordered(`asc`)).not.toEqual(ordered(`desc`)) + expect(ordered(`asc`)).not.toEqual( + identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ), + ) + }) + + it(`keeps the structural identity for queries a partition cannot serve`, () => { + const residual = identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(eq(r.a, `x`), gt(r.n, 1))), + ) + expect(residual).toEqual( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(gt(r.n, 1), eq(r.a, `x`))), + ), + ) + expect(residual).not.toEqual( + identity((q) => + q + .from({ r: source }) + .where(({ r }: any) => and(eq(r.a, `x`), gt(r.n, 2))), + ), + ) + expect((residual as Array)[0]).toBe(`query`) + const pooled = identity((q) => + q.from({ r: source }).where(({ r }: any) => eq(r.a, `x`)), + ) + expect((pooled as Array)[0]).toBe(`pooled`) + }) +}) diff --git a/packages/db/tests/query/where-prefilter-property-visibility.test.ts b/packages/db/tests/query/where-prefilter-property-visibility.test.ts index 7efb54a574..95d9ee8196 100644 --- a/packages/db/tests/query/where-prefilter-property-visibility.test.ts +++ b/packages/db/tests/query/where-prefilter-property-visibility.test.ts @@ -134,3 +134,40 @@ it('lets the full filter handle a throwing nested getter in a change', async () await collection.cleanup() } }) + +it.each(['inherited', 'non-enumerable'] as const)( + 'keeps a row whose %s field the enriched row omits', + async (kind) => { + // The enriched row lacks `v`, so `isUndefined(v)` holds there although + // the stored row reads a value. A scan that evaluated the predicate on + // stored rows would drop it. + const row = + kind === 'inherited' + ? Object.assign(Object.create({ v: 'x' }) as Row, { id: 'hidden' }) + : Object.defineProperty({ id: 'hidden' } as Row, 'v', { + value: 'x', + enumerable: false, + }) + const collection = createCollection( + mockSyncCollectionOptions({ + id: `prefilter-hidden-${kind}`, + getKey: (item) => item.id, + initialData: [row], + }), + ) + try { + await collection.stateWhenReady() + const hidden = new Func('and', [ + new Func('eq', [new PropRef(['id']), new Value('hidden')]), + new Func('isUndefined', [new PropRef(['v'])]), + ]) + expect( + collection + .currentStateAsChanges({ where: hidden }) + ?.map((change) => change.key), + ).toEqual(['hidden']) + } finally { + await collection.cleanup() + } + }, +) diff --git a/packages/db/tests/utils.ts b/packages/db/tests/utils.ts index c8ebed421e..a3e7c0b5ac 100644 --- a/packages/db/tests/utils.ts +++ b/packages/db/tests/utils.ts @@ -215,6 +215,7 @@ export function createIndexUsageTracker(collection: any): { recordFullScan() yield* originalEntries.call(this) } + // The unindexed snapshot scan reads stored rows through the state. const state = collection._state const originalEntriesPassing = state.entriesPassing state.entriesPassing = function* (prefilter: (row: object) => boolean) { diff --git a/packages/db/tests/virtual-props-cache.test.ts b/packages/db/tests/virtual-props-cache.test.ts new file mode 100644 index 0000000000..058efd643d --- /dev/null +++ b/packages/db/tests/virtual-props-cache.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from 'vitest' +import { createCollection } from '../src/collection/index.js' +import { localOnlyCollectionOptions } from '../src/local-only.js' +import { mockSyncCollectionOptions, withExpectedRejection } from './utils.js' + +/** + * A row read with virtual props is a copy, cached so repeated reads and + * publications share one object. These laws pin what that cache must keep: + * the value a change publishes is the row later reads return, for every + * subscriber, and a key that leaves the collection leaves the cache. + */ +type Row = { id: string; a: number } +const cacheSize = (collection: unknown) => + (collection as { _state: { virtualPropsCache: Map } }) + ._state.virtualPropsCache.size + +describe(`virtual props cache`, () => { + it(`publishes the row that later reads return, to every subscriber`, async () => { + const collection = createCollection( + mockSyncCollectionOptions({ + id: `virtual-props-cache-sync`, + getKey: (row) => row.id, + initialData: [{ id: `x`, a: 0 }], + }), + ) + await collection.stateWhenReady() + const first: Array = [] + const second: Array = [] + collection.subscribeChanges((changes) => + first.push(...changes.map((c) => c.value)), + ) + collection.subscribeChanges((changes) => + second.push(...changes.map((c) => c.value)), + ) + for (const a of [1, 2]) { + collection.utils.begin() + collection.utils.write({ type: `update`, value: { id: `x`, a } }) + collection.utils.commit() + expect(first.at(-1)).toBe(collection.get(`x`)) + expect(second.at(-1)).toBe(first.at(-1)) + expect(collection.toArray[0]).toBe(first.at(-1)) + } + }) + + it(`publishes the row that later reads return after local writes`, () => { + const collection = createCollection( + localOnlyCollectionOptions({ + id: `virtual-props-cache-local`, + getKey: (row) => row.id, + initialData: [{ id: `x`, a: 0 }], + }), + ) + const published: Array = [] + collection.subscribeChanges((changes) => + published.push(...changes.map((c) => c.value)), + ) + for (const a of [1, 2]) { + collection.update(`x`, (draft) => { + draft.a = a + }) + expect(published.at(-1)).toBe(collection.get(`x`)) + } + }) + + it(`drops a key's entry when the key is deleted or rolled back`, async () => { + const collection = createCollection( + mockSyncCollectionOptions({ + id: `virtual-props-cache-delete`, + getKey: (row) => row.id, + initialData: [{ id: `x`, a: 0 }], + }), + ) + await collection.stateWhenReady() + collection.subscribeChanges(() => {}) + collection.get(`x`) + collection.utils.begin() + collection.utils.write({ type: `delete`, value: { id: `x`, a: 0 } }) + collection.utils.commit() + expect(cacheSize(collection)).toBe(0) + + await withExpectedRejection(`rolled back`, async () => { + const transaction = collection.insert({ id: `y`, a: 1 }) + collection.get(`y`) + collection.utils.rejectSync(new Error(`rolled back`)) + await transaction.isPersisted.promise.catch(() => undefined) + }) + expect(collection.has(`y`)).toBe(false) + expect(cacheSize(collection)).toBe(0) + }) +}) diff --git a/packages/react-db/src/development.ts b/packages/react-db/src/development.ts new file mode 100644 index 0000000000..af9da217eb --- /dev/null +++ b/packages/react-db/src/development.ts @@ -0,0 +1,15 @@ +// Bundlers inline `process.env.NODE_ENV` even where `process` does not exist, +// so it is read directly; without either, warnings stay off. +export function shouldWarnInDevelopment(disableEnvVar: string): boolean { + try { + if (process.env.NODE_ENV === `production`) return false + } catch { + return false + } + // A page may supply a `process` shim without `env`; nothing disabled it. + try { + return process.env[disableEnvVar] !== `1` + } catch { + return true + } +} diff --git a/packages/react-db/src/useLiveQuery.ts b/packages/react-db/src/useLiveQuery.ts index acd676cd25..edb25f0569 100644 --- a/packages/react-db/src/useLiveQuery.ts +++ b/packages/react-db/src/useLiveQuery.ts @@ -5,16 +5,17 @@ import { BaseQueryBuilder, IR, UnhashableQueryIRError, - createLiveQueryCollection, createLiveQueryObserver, deepEquals, getPreparedLiveQueryIdentity, getStableValueHash, isCollection, prepareLiveQueryValue, + resolveLiveQueryValue, } from '@tanstack/db' import { useOptionalDbClient } from './DbProvider' import { setLiveQueryResultInfo } from './live-query-internals' +import { shouldWarnInDevelopment } from './development' import type { Collection, CollectionImpl, @@ -172,16 +173,6 @@ export function warnDeprecatedDepsArray( ) } -function shouldWarnInDevelopment(disableEnvVar: string): boolean { - if (typeof process === `undefined`) { - return false - } - - return ( - process.env.NODE_ENV !== `production` && process.env[disableEnvVar] !== `1` - ) -} - function getCurrentTime(): number { return typeof performance !== `undefined` && typeof performance.now === `function` @@ -336,40 +327,6 @@ export function warnUnhashableDerivedIdentity( ) } -function createCollectionFromPreparedQuery( - value: unknown, - defaultGcTime = DEFAULT_GC_TIME_MS, -) { - if (value === undefined || value === null) { - return null - } - - if (isCollection(value)) { - value.startSyncImmediate() - return value - } - - if (value instanceof BaseQueryBuilder) { - return createLiveQueryCollection({ - query: value, - startSync: true, - gcTime: defaultGcTime, - }) - } - - if (typeof value === `object`) { - return createLiveQueryCollection({ - startSync: true, - gcTime: defaultGcTime, - ...(value as LiveQueryCollectionConfig), - }) - } - - throw new Error( - `useLiveQuery callback must return a QueryBuilder, LiveQueryCollectionConfig, Collection, undefined, or null. Got: ${typeof value}`, - ) -} - /** * Create a live query using a query function. * @param queryFn - Query function that defines what data to fetch @@ -771,6 +728,32 @@ export function useLiveQueryForSuspense( return useLiveQueryImpl(configOrQueryOrCollection, deps, true) } +// What one hook instance keeps across renders. It lives in a single ref +// slot, so a render neither looks up nor allocates more. +function createHookInstance(dbClient: DbClient | undefined) { + return { + collection: null as Collection | null, + deps: null as Array | null, + config: null as unknown, + client: dbClient, + legacyUnhashableIdentity: [`legacy-unhashable`] as Array, + derivedIdentityProfiler: { + renderCount: 0, + totalMs: 0, + maxMs: 0, + warned: false, + } as DerivedIdentityProfiler, + deferredCollections: new Set< + CollectionImpl + >(), + observer: null as LiveQueryObserver | null, + queryHash: undefined as string | undefined, + suspenseKey: undefined as string | undefined, + identityError: undefined as UnhashableQueryIRError | undefined, + subscribe: null as ((onStoreChange: () => void) => () => void) | null, + } +} + function useLiveQueryImpl( configOrQueryOrCollection: any, deps: Array | undefined, @@ -784,32 +767,8 @@ function useLiveQueryImpl( : (getExplicitDbClient(configOrQueryOrCollection) ?? contextDbClient) const resolvedDeps = deps ?? [] - // Use refs to cache collection and track dependencies - const collectionRef = useRef | null>( - null, - ) - const depsRef = useRef | null>(null) - const configRef = useRef(null) - const clientRef = useRef(dbClient) - const legacyUnhashableIdentityRef = useRef>([ - `legacy-unhashable`, - ]) - - const derivedIdentityProfilerRef = useRef({ - renderCount: 0, - totalMs: 0, - maxMs: 0, - warned: false, - }) - const deferredCollectionsRef = useRef( - new Set>(), - ) - const observerRef = useRef | null>( - null, - ) - const queryHashRef = useRef(undefined) - const suspenseKeyRef = useRef(undefined) - const identityErrorRef = useRef(undefined) + const instanceRef = useRef | null>(null) + const instance = (instanceRef.current ??= createHookInstance(dbClient)) const queryKey = !inputIsCollection ? getExplicitQueryKey(configOrQueryOrCollection) @@ -825,22 +784,29 @@ function useLiveQueryImpl( streamIdentity = [`queryKey`, queryKey] } else if (deps !== undefined) { identityDeps = resolvedDeps - try { - preparedQueryValue = prepareQueryValue( - configOrQueryOrCollection, - dbClient, - deferredCollectionsRef.current, - ) - streamIdentity = [ - `deps`, - resolvedDeps, - getPreparedLiveQueryIdentity(preparedQueryValue), - ] - } catch (error) { - if (!(error instanceof UnhashableQueryIRError)) throw error - warnUnhashableDerivedIdentity(error) - identityError = error - } + // Deps decide reuse. Only hydration and Suspense read the query hash; the + // development warning about unhashable queries still derives it. + if ( + dbClient || + forSuspense || + shouldWarnInDevelopment(`TANSTACK_DB_DISABLE_QUERY_IDENTITY_WARNINGS`) + ) + try { + preparedQueryValue = prepareQueryValue( + configOrQueryOrCollection, + dbClient, + instance.deferredCollections, + ) + streamIdentity = [ + `deps`, + resolvedDeps, + getPreparedLiveQueryIdentity(preparedQueryValue), + ] + } catch (error) { + if (!(error instanceof UnhashableQueryIRError)) throw error + warnUnhashableDerivedIdentity(error) + identityError = error + } } else if (inputIsCollection) { identityDeps = [] streamIdentity = [`collection`, configOrQueryOrCollection.id] @@ -848,8 +814,8 @@ function useLiveQueryImpl( const preparation = prepareDerivedQuery( configOrQueryOrCollection, dbClient, - derivedIdentityProfilerRef.current, - deferredCollectionsRef.current, + instance.derivedIdentityProfiler, + instance.deferredCollections, ) preparedQueryValue = preparation.value if (preparation.status === `hashable`) { @@ -857,7 +823,7 @@ function useLiveQueryImpl( streamIdentity = preparation.identityDeps } else { warnUnhashableDerivedIdentity(preparation.error) - identityDeps = legacyUnhashableIdentityRef.current + identityDeps = instance.legacyUnhashableIdentity identityError = preparation.error } } @@ -885,10 +851,10 @@ function useLiveQueryImpl( !inputIsCollection && queryHash !== undefined && !dbClient && - collectionRef.current !== null && - clientRef.current === dbClient && - queryHashRef.current === queryHash && - suspenseKeyRef.current !== undefined + instance.collection !== null && + instance.client === dbClient && + instance.queryHash === queryHash && + instance.suspenseKey !== undefined if ( forSuspense && @@ -901,14 +867,14 @@ function useLiveQueryImpl( preparedQueryValue = prepareQueryValue( configOrQueryOrCollection, dbClient, - deferredCollectionsRef.current, + instance.deferredCollections, ) } const suspenseKey = queryHash && !dbClient ? canReuseSuspenseKey - ? suspenseKeyRef.current + ? instance.suspenseKey : getUnscopedSuspenseKey(preparedQueryValue, queryHash) : queryHash @@ -922,23 +888,24 @@ function useLiveQueryImpl( const suspenseCollection = suspenseEntry?.collection const identityChanged = - depsRef.current === null || + instance.deps === null || (deps !== undefined - ? depsRef.current.length !== identityDeps.length || - depsRef.current.some((dep, index) => dep !== identityDeps[index]) - : !deepEquals(depsRef.current, identityDeps)) + ? instance.deps.length !== identityDeps.length || + instance.deps.some((dep, index) => dep !== identityDeps[index]) + : !deepEquals(instance.deps, identityDeps)) // Check if we need to create/recreate the collection const needsNewCollection = - !collectionRef.current || - (inputIsCollection && configRef.current !== configOrQueryOrCollection) || - (!inputIsCollection && (clientRef.current !== dbClient || identityChanged)) + !instance.collection || + (inputIsCollection && instance.config !== configOrQueryOrCollection) || + (!inputIsCollection && (instance.client !== dbClient || identityChanged)) const resumeDeferredCollections = () => { - for (const collection of deferredCollectionsRef.current) { + if (instance.deferredCollections.size === 0) return + for (const collection of instance.deferredCollections) { collection._resumeSyncStart() } - deferredCollectionsRef.current.clear() + instance.deferredCollections.clear() } if (needsNewCollection) { @@ -963,25 +930,28 @@ function useLiveQueryImpl( } // It's already a collection, ensure sync is started for React hooks configOrQueryOrCollection.startSyncImmediate() - collectionRef.current = configOrQueryOrCollection - configRef.current = configOrQueryOrCollection + instance.collection = configOrQueryOrCollection + instance.config = configOrQueryOrCollection } else { if (suspenseCollection) { - collectionRef.current = suspenseCollection + instance.collection = suspenseCollection } else { if (preparedQueryValue === unpreparedQueryValue) { preparedQueryValue = prepareQueryValue( configOrQueryOrCollection, dbClient, - deferredCollectionsRef.current, + instance.deferredCollections, ) } - collectionRef.current = createCollectionFromPreparedQuery( - preparedQueryValue, - forSuspense ? DEFAULT_SUSPENSE_GC_TIME_MS : DEFAULT_GC_TIME_MS, - ) as SuspenseCollection | null - if (suspenseCollections && suspenseKey && collectionRef.current) { - const collection = collectionRef.current + instance.collection = resolveLiveQueryValue(preparedQueryValue, { + gcTime: forSuspense + ? DEFAULT_SUSPENSE_GC_TIME_MS + : DEFAULT_GC_TIME_MS, + // Hydration and Suspense key the live-query Collection by identity. + pool: !forSuspense && !dbClient, + }) as SuspenseCollection | null + if (suspenseCollections && suspenseKey && instance.collection) { + const collection = instance.collection const removeCleanupListener = collection.on(`status:cleaned-up`, () => releaseSuspenseCollection( suspenseCollections, @@ -1013,13 +983,13 @@ function useLiveQueryImpl( suspenseCollections.set(suspenseKey, entry) } } - configRef.current = configOrQueryOrCollection - depsRef.current = [...identityDeps] + instance.config = configOrQueryOrCollection + instance.deps = [...identityDeps] } - clientRef.current = dbClient - queryHashRef.current = queryHash - suspenseKeyRef.current = suspenseKey - identityErrorRef.current = identityError + instance.client = dbClient + instance.queryHash = queryHash + instance.suspenseKey = suspenseKey + instance.identityError = identityError } // Recreate the observer when the underlying collection changes. The observer @@ -1035,38 +1005,38 @@ function useLiveQueryImpl( // hook's pre-observer loading policy, and — because wholesale delivers // nothing synchronously during subscribe — never notifies // useSyncExternalStore inside its own subscribe call. - observerRef.current = createLiveQueryObserver(collectionRef.current, { + instance.observer = createLiveQueryObserver(instance.collection, { mode: `wholesale`, client: dbClient, - queryHash: queryHashRef.current, + queryHash: instance.queryHash, onPreload: resumeDeferredCollections, }) } - const observer = observerRef.current! + const observer = instance.observer! // Stable subscribe bound to the current observer; the observer owns the // subscription, ready-race, and disposal. - const subscribeRef = useRef< - ((onStoreChange: () => void) => () => void) | null - >(null) - if (!subscribeRef.current || needsNewCollection) { - subscribeRef.current = (onStoreChange: () => void) => { - const unsubscribe = observer.subscribe(() => onStoreChange()) + if (!instance.subscribe || needsNewCollection) { + instance.subscribe = (onStoreChange: () => void) => { + const unsubscribe = observer.subscribe(onStoreChange) resumeDeferredCollections() return unsubscribe } } const returned = useSyncExternalStore( - subscribeRef.current, + instance.subscribe, () => observer.getSnapshot(), () => observer.getServerSnapshot(), ) - setLiveQueryResultInfo(returned, { - client: dbClient, - queryHash: queryHashRef.current, - identityError: identityErrorRef.current, - observer, - }) + // Only useLiveSuspenseQuery reads this, and it costs a define per render. + if (forSuspense) { + setLiveQueryResultInfo(returned, { + client: dbClient, + queryHash: instance.queryHash, + identityError: instance.identityError, + observer, + }) + } return returned as any } diff --git a/packages/react-db/tests/conformance.test.tsx b/packages/react-db/tests/conformance.test.tsx index 5367455d8d..1af95556d5 100644 --- a/packages/react-db/tests/conformance.test.tsx +++ b/packages/react-db/tests/conformance.test.tsx @@ -214,7 +214,7 @@ const reactDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: true, suspense: true }, + features: { serverSnapshot: true, suspense: true, pooledEqFilters: true }, } runSuite(reactDriver) diff --git a/packages/react-db/tests/development.test.ts b/packages/react-db/tests/development.test.ts new file mode 100644 index 0000000000..d4519afe0f --- /dev/null +++ b/packages/react-db/tests/development.test.ts @@ -0,0 +1,61 @@ +// @vitest-environment node +import { readFileSync } from 'node:fs' +import { createContext, runInContext } from 'node:vm' +import { transformSync } from 'esbuild' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { shouldWarnInDevelopment } from '../src/development' + +const source = readFileSync( + new URL(`../src/development.ts`, import.meta.url), + `utf8`, +) + +// Evaluates the module as a browser bundle would: `NODE_ENV` inlined by the +// bundler, or left alone, and no `process` global. +function inBrowser( + nodeEnv: string | undefined, + // Globals of the page, such as a `process` shim from another library. + globals: Record = {}, +): (name: string) => boolean { + const { code } = transformSync(source, { + loader: `ts`, + format: `cjs`, + define: + nodeEnv === undefined + ? {} + : { 'process.env.NODE_ENV': JSON.stringify(nodeEnv) }, + }) + const bundle = { exports: {} as Record } + runInContext( + code, + createContext({ ...globals, module: bundle, exports: bundle.exports }), + ) + return bundle.exports.shouldWarnInDevelopment as (name: string) => boolean +} + +describe(`development warnings`, () => { + afterEach(() => vi.unstubAllEnvs()) + + it(`warn in a browser development bundle without a process global`, () => { + expect(inBrowser(`development`)(`DISABLE`)).toBe(true) + }) + + it(`stay on when a page's process shim has no env`, () => { + expect(inBrowser(`development`, { process: {} })(`DISABLE`)).toBe(true) + }) + + it(`stay off in a browser production bundle or without a bundler`, () => { + expect(inBrowser(`production`)(`DISABLE`)).toBe(false) + expect(inBrowser(undefined)(`DISABLE`)).toBe(false) + }) + + it(`honor the disable variable and production under Node`, () => { + vi.stubEnv(`NODE_ENV`, `development`) + expect(shouldWarnInDevelopment(`DISABLE`)).toBe(true) + vi.stubEnv(`DISABLE`, `1`) + expect(shouldWarnInDevelopment(`DISABLE`)).toBe(false) + vi.unstubAllEnvs() + vi.stubEnv(`NODE_ENV`, `production`) + expect(shouldWarnInDevelopment(`DISABLE`)).toBe(false) + }) +}) diff --git a/packages/react-db/tests/useLiveQuery.test.tsx b/packages/react-db/tests/useLiveQuery.test.tsx index ad25741938..7edca3327c 100644 --- a/packages/react-db/tests/useLiveQuery.test.tsx +++ b/packages/react-db/tests/useLiveQuery.test.tsx @@ -16,7 +16,7 @@ import { toArray, } from '@tanstack/db' import { useEffect } from 'react' -import { useLiveQuery } from '../src/useLiveQuery' +import { useLiveQuery, useLiveQueryForSuspense } from '../src/useLiveQuery' import { getLiveQueryResultInfo } from '../src/live-query-internals' import { DbProvider } from '../src/DbProvider' import { @@ -3621,15 +3621,19 @@ describe(`Query Collections`, () => { initialData: initialPersons, }), ) + // Suspense is the reader of this identity; plain useLiveQuery skips it. const first = renderHook(() => - useLiveQuery((q) => q.from({ people: collection }), [1]), + useLiveQueryForSuspense( + (q: any) => q.from({ people: collection }), + [1], + ), ) const second = renderHook(() => - useLiveQuery( - (q) => + useLiveQueryForSuspense( + (q: any) => q .from({ people: collection }) - .where(({ people }) => gt(people.age, 30)), + .where(({ people }: any) => gt(people.age, 30)), [1], ), ) diff --git a/packages/solid-db/src/useLiveQuery.ts b/packages/solid-db/src/useLiveQuery.ts index 3644e4a37e..21963fa948 100644 --- a/packages/solid-db/src/useLiveQuery.ts +++ b/packages/solid-db/src/useLiveQuery.ts @@ -11,8 +11,10 @@ import { BaseQueryBuilder, createLiveQueryCollection, createLiveQueryObserver, + getPublicCollection, isCollection, isSingleResultCollection, + resolveLiveQueryValue, } from '@tanstack/db' import { createStore, reconcile } from 'solid-js/store' import type { Accessor } from 'solid-js' @@ -321,10 +323,7 @@ export function useLiveQuery( return null } - return createLiveQueryCollection({ - query: configOrQueryOrCollection, - startSync: true, - }) + return resolveLiveQueryValue(result) } const innerCollection = configOrQueryOrCollection() @@ -558,7 +557,7 @@ export function useLiveQuery( }, collection: { get() { - return collection() + return getPublicCollection(collection()) }, }, state: { diff --git a/packages/solid-db/tests/conformance.test.tsx b/packages/solid-db/tests/conformance.test.tsx index 956f0435ca..013cd3a3c5 100644 --- a/packages/solid-db/tests/conformance.test.tsx +++ b/packages/solid-db/tests/conformance.test.tsx @@ -236,7 +236,7 @@ const solidDriver: LiveQueryDriver = { // gap — the error-status scenario is parametrized to assert it via the boundary. errorSurface: `throw`, knownGaps: [], - features: { serverSnapshot: false, suspense: true }, + features: { serverSnapshot: false, suspense: true, pooledEqFilters: true }, } describe(`owned native scope setup`, () => { diff --git a/packages/svelte-db/src/useLiveQuery.svelte.ts b/packages/svelte-db/src/useLiveQuery.svelte.ts index f58bbf95dc..0feab95a98 100644 --- a/packages/svelte-db/src/useLiveQuery.svelte.ts +++ b/packages/svelte-db/src/useLiveQuery.svelte.ts @@ -8,10 +8,12 @@ import { createLiveQueryCollection, createLiveQueryObserver, getLiveQueryHash, + getPublicCollection, getStableValueHash, isCollection, isSingleResultCollection, prepareLiveQueryValue, + resolveLiveQueryValue, } from '@tanstack/db' import { useOptionalDbClient } from './db-context.js' import type { @@ -415,10 +417,8 @@ export function useLiveQuery( } else if (isCollection(preparedValue)) { collection = preparedValue } else if (preparedValue instanceof BaseQueryBuilder) { - collection = createLiveQueryCollection({ - query: preparedValue, - startSync: true, - }) + // Hydration keys the live-query Collection by identity. + collection = resolveLiveQueryValue(preparedValue, { pool: !dbClient }) } else { collection = createLiveQueryCollection({ ...(preparedValue as LiveQueryCollectionConfig), @@ -535,7 +535,7 @@ export function useLiveQuery( return internalData }, get collection() { - return resolved.collection + return getPublicCollection(resolved.collection) }, get status() { return status as CollectionStatus diff --git a/packages/svelte-db/tests/conformance.svelte.test.ts b/packages/svelte-db/tests/conformance.svelte.test.ts index 5cdbde0bfd..f8674ec357 100644 --- a/packages/svelte-db/tests/conformance.svelte.test.ts +++ b/packages/svelte-db/tests/conformance.svelte.test.ts @@ -222,7 +222,7 @@ const svelteDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: false, suspense: false }, + features: { serverSnapshot: false, suspense: false, pooledEqFilters: true }, } runSuite(svelteDriver) diff --git a/packages/vue-db/src/useLiveQuery.ts b/packages/vue-db/src/useLiveQuery.ts index 8a1f5e2216..6eeea473f6 100644 --- a/packages/vue-db/src/useLiveQuery.ts +++ b/packages/vue-db/src/useLiveQuery.ts @@ -9,10 +9,13 @@ import { watchEffect, } from 'vue' import { + BaseQueryBuilder, createLiveQueryCollection, createLiveQueryObserver, + getPublicCollection, isCollection, isSingleResultCollection, + resolveLiveQueryValue, } from '@tanstack/db' import type { ChangeMessage, @@ -329,29 +332,10 @@ export function useLiveQuery( // Ensure we always start sync for Vue hooks if (typeof unwrappedParam === `function`) { - // To avoid calling the query function twice, we wrap it to handle null/undefined returns - // The wrapper will be called once by createLiveQueryCollection - const disabledQuery = Symbol() - const wrappedQuery = (q: InitialQueryBuilder) => { - const result = unwrappedParam(q) - if (result === undefined || result === null) { - throw disabledQuery - } - return result - } - - try { - return createLiveQueryCollection({ - query: wrappedQuery, - startSync: true, - }) - } catch (error) { - if (error === disabledQuery) { - return null - } - // Re-throw other errors - throw error - } + // A query function returning null or undefined disables the query. + return resolveLiveQueryValue( + unwrappedParam(new BaseQueryBuilder() as InitialQueryBuilder), + ) } else { return createLiveQueryCollection({ ...unwrappedParam, @@ -389,15 +373,12 @@ export function useLiveQuery( // materializes into its own reactive map (granular) + ordered array. let currentObserver: LiveQueryObserver | null = null - const syncFromObserver = ( - observer: LiveQueryObserver, - currentCollection: Collection, - ) => { + const syncFromObserver = (observer: LiveQueryObserver) => { const snapshot = observer.getSnapshot() status.value = snapshot.status as CollectionStatus persistedStatus.value = snapshot.persistedStatus persistedError.value = snapshot.persistedError - internalData.value = Array.from(currentCollection.values()) + internalData.value = Array.from(snapshot.state?.values() ?? []) } // Watch for collection changes and subscribe to updates @@ -447,10 +428,10 @@ export function useLiveQuery( state.set(key, value) } } - syncFromObserver(observer, currentCollection) + syncFromObserver(observer) }, ) - syncFromObserver(observer, currentCollection) + syncFromObserver(observer) // Cleanup when effect is invalidated onInvalidate(() => { @@ -469,7 +450,7 @@ export function useLiveQuery( return { state: computed(() => state), data, - collection: computed(() => collection.value), + collection: computed(() => getPublicCollection(collection.value)), status: computed(() => status.value), isLoading: computed(() => status.value === `loading`), isReady: computed( diff --git a/packages/vue-db/tests/conformance.test.ts b/packages/vue-db/tests/conformance.test.ts index b15d95a27d..1f1ce818c8 100644 --- a/packages/vue-db/tests/conformance.test.ts +++ b/packages/vue-db/tests/conformance.test.ts @@ -229,7 +229,7 @@ const vueDriver: LiveQueryDriver = { mountConfig, mountDisabled, knownGaps: [], - features: { serverSnapshot: false, suspense: false }, + features: { serverSnapshot: false, suspense: false, pooledEqFilters: true }, } describe(`owned native scope setup`, () => {