fix(db): preserve optimistic collection order

.changeset/fix-optimistic-collection-order.md

@@ -0,0 +1,5 @@
+---
+'@tanstack/db': patch
+---
+
+Preserve a collection's configured sort order while optimistic inserts and updates are visible.

docs/contributing/oracle-coverage.md

@@ -31,7 +31,7 @@ comment and the current API/architecture contract before extending its model.
 | Ordered relations and BTree | [top-K relation oracle](../../packages/db-ivm/tests/operators/topk-relation-oracle.test.ts), [BTree/Map](../../packages/db/tests/btree-map-oracle.test.ts), [incrementalization laws](../../packages/db-ivm/tests/incrementalization-law.property.test.ts) | Independent ordered relations and cumulative signed output. Top-K consolidation compares same-key values without hashing, including cyclic replacements and fresh transient cancellation. Other hash-based operators retain hashing's declared domain. Algebra does not specify client readiness. |
 | Includes and publication | [cross-formulation](../../packages/db/tests/query/includes-cross-formulation-oracle.property.test.ts), [temporal](../../packages/db/tests/query/includes-temporal-oracle.test.ts), [Collection includes](../../packages/db/tests/query/includes-collection-oracle.property.test.ts), [architecture and complete suite map](../../packages/db/src/query/live/ARCHITECTURE.md#executable-contracts) | Per-parent/flat-join/partition relations, callback-time rows, nested values, and route histories. Observe raw promised order; fresh queries do not establish continuous publication safety. |
 | Collection lifecycle | [history](../../packages/db/tests/collection-subscription-lifecycle-history.property.test.ts), [publication](../../packages/db/tests/collection-subscription-lifecycle-publication.property.test.ts), [replay](../../packages/db/tests/collection-subscription-replay-oracle.property.test.ts), [effect disposal](../../packages/db/tests/effect-disposal-oracle.test.ts) | Ownership and phase histories, exact caller/error/publication evidence, late completion and restart. Effect self-dependent disposal remains a separate contract question. |
-| Optimistic state | [history model](../../packages/db/tests/optimistic-history-oracle.ts), [generated histories](../../packages/db/tests/optimistic-transaction-oracle.property.test.ts), [outcomes](../../packages/db/tests/optimistic-history-outcomes.test.ts), [publication](../../packages/db/tests/optimistic-history-publication.test.ts) | Independent whole-row snapshots, rollback dependencies, metadata and prior-value events. Never rebase a pending snapshot merely to simplify the model. |
+| Optimistic state | [history model](../../packages/db/tests/optimistic-history-oracle.ts), [generated histories](../../packages/db/tests/optimistic-transaction-oracle.property.test.ts), [outcomes](../../packages/db/tests/optimistic-history-outcomes.test.ts), [publication](../../packages/db/tests/optimistic-history-publication.test.ts), [visible order](../../packages/db/tests/collection-visible-order-oracle.test.ts) | Independent whole-row snapshots, rollback dependencies, metadata, prior-value events, and declared Collection order over the complete visible optimistic overlay. Visible-order coverage spans Collection iteration surfaces; it does not promise change-message order or inherited live-query order. Never rebase a pending snapshot merely to simplify the model. |
 | Drafts and native values | [proxy](../../packages/db/tests/proxy.test.ts), [detachment](../../packages/db/tests/proxy-detachment-contract.test.ts), [iteration](../../packages/db/tests/proxy-iteration-contract.test.ts) | Native-operation controls, exact patches and actual stored rows; alias/cycle/adversarial-key histories. General native-mutator and symbol-write support is not established by a plain-object oracle. |
 | Query DB and observer | [ownership](../../packages/query-db-collection/tests/ownership-lifecycle.oracle.test.ts), [load lifecycle](../../packages/query-db-collection/tests/load-subset-lifecycle-oracle.test.ts), [observer histories](../../packages/db/tests/live-query-observer-history.property.test.ts) | Real QueryClient boundary and a per-listener eligibility ledger, not a duplicate dispatch queue. Check reentry, peer survival, FIFO and disposal independently of final rows. |
 | Ordered acquisition | [pagination](../../packages/db/tests/query/pagination-oracle.property.test.ts), [ordered work](../../packages/db/tests/query/ordered-work-oracle.property.test.ts), [ordered lifecycle](../../packages/db/tests/query/ordered-lifecycle-oracle.property.test.ts) | Complete finite provider results, pending windows, ties/nulls, ownership and documented repair timing. Request completion is not proof of unrequested source extent. |

packages/db/package.json

@@ -22,7 +22,7 @@
     "lint": "eslint . --fix",
     "test": "vitest --run",
     "test:facade-retention": "node --expose-gc --import tsx tests/facade-retention.probe.ts",
-    "test:oracles": "vitest --run tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts",
+    "test:oracles": "vitest --run tests/collection-cleanup-restart-oracle.test.ts tests/effect-disposal-oracle.test.ts tests/optimistic-transaction-oracle.property.test.ts tests/optimistic-settlement-boundaries.test.ts tests/optimistic-history-publication.test.ts tests/optimistic-history-outcomes.test.ts tests/collection-visible-order-oracle.test.ts tests/collection-metadata-publication-oracle.property.test.ts tests/collection-state-retention-oracle.property.test.ts tests/collection-subscription-lifecycle-history.property.test.ts tests/collection-subscription-lifecycle-oracle.test.ts tests/collection-subscription-lifecycle-publication.property.test.ts tests/collection-subscription-replay-oracle.property.test.ts tests/d2-source-reconciliation-oracle.property.test.ts tests/live-query-observer-history.property.test.ts tests/query/includes-collection-oracle.property.test.ts tests/query/includes-functional-projection-oracle.test.ts tests/query/includes-functional-input-boundary.test.ts tests/query/includes-context-transport-oracle.test.ts tests/query/includes-cross-formulation-oracle.property.test.ts tests/query/includes-optimistic-oracle.property.test.ts tests/query/includes-oracle.property.test.ts tests/query/includes-publication-oracle.test.ts tests/query/includes-query-shape-oracle.test.ts tests/query/includes-temporal-oracle.test.ts tests/query/includes-work-counter-oracle.test.ts tests/query/load-subset-oracle.property.test.ts tests/query/load-subset-replay-refinement-oracle.test.ts tests/query/load-subset-source-readiness-refinement-oracle.test.ts tests/query/load-subset-transaction-refinement-oracle.test.ts tests/query/ordered-source-loader-state.test.ts tests/query/ordered-demand-retirement.test.ts tests/query/ordered-default-work.test.ts tests/query/ordered-lifecycle-oracle.property.test.ts tests/query/ordered-work-oracle.property.test.ts tests/query/pagination-oracle.property.test.ts tests/query/includes-space-oracle.test.ts",
     "bench:nested-includes": "vitest bench tests/query/includes-performance.bench.ts --run"
   },
   "type": "module",

packages/db/src/collection/state.ts

@@ -1,3 +1,4 @@
+import { compareKeys } from '@tanstack/db-ivm'
 import { deepEquals } from '../utils'
 import { SortedMap } from '../SortedMap'
 import { enrichRowWithVirtualProps } from '../virtual-props.js'
@@ -414,20 +415,64 @@ export class CollectionStateManager<
    */
   public *keys(): IterableIterator<TKey> {
     const { syncedData, optimisticDeletes, optimisticUpserts } = this
-    // Yield keys from synced data, skipping any that are deleted.
-    for (const key of syncedData.keys()) {
-      if (!optimisticDeletes.has(key)) {
-        yield key
+    const compare = this.config.compare
+
+    if (!compare || optimisticUpserts.size === 0) {
+      // Yield keys from synced data, skipping any that are deleted.
+      for (const key of syncedData.keys()) {
+        if (!optimisticDeletes.has(key)) {
+          yield key
+        }
+      }
+      // Without a comparator, preserve insertion order for optimistic keys.
+      for (const key of optimisticUpserts.keys()) {
+        if (!syncedData.has(key) && !optimisticDeletes.has(key)) {
+          yield key
+        }
+      }
+      return
+    }
+
+    const compareEntries = (
+      [leftKey, leftValue]: [TKey, TOutput],
+      [rightKey, rightValue]: [TKey, TOutput],
+    ) => compare(leftValue, rightValue) || compareKeys(leftKey, rightKey)
+    const optimisticEntries = [...optimisticUpserts.entries()]
+      .filter(([key]) => !optimisticDeletes.has(key))
+      .sort(compareEntries)
+    const syncedEntries = syncedData.entries()
+    let syncedEntry = syncedEntries.next()
+    const advanceSynced = () => {
+      while (
+        !syncedEntry.done &&
+        (optimisticDeletes.has(syncedEntry.value[0]) ||
+          optimisticUpserts.has(syncedEntry.value[0]))
+      ) {
+        syncedEntry = syncedEntries.next()
       }
     }
-    // Yield keys from upserts that were not already in synced data.
-    for (const key of optimisticUpserts.keys()) {
-      if (!syncedData.has(key) && !optimisticDeletes.has(key)) {
-        // The optimisticDeletes check is technically redundant if inserts/updates always remove from deletes,
-        // but it's safer to keep it.
-        yield key
+    advanceSynced()
+
+    let optimisticIndex = 0
+    while (!syncedEntry.done && optimisticIndex < optimisticEntries.length) {
+      const optimisticEntry = optimisticEntries[optimisticIndex]!
+      if (compareEntries(optimisticEntry, syncedEntry.value) < 0) {
+        yield optimisticEntry[0]
+        optimisticIndex++
+      } else {
+        yield syncedEntry.value[0]
+        syncedEntry = syncedEntries.next()
+        advanceSynced()
       }
     }
+    while (!syncedEntry.done) {
+      yield syncedEntry.value[0]
+      syncedEntry = syncedEntries.next()
+      advanceSynced()
+    }
+    while (optimisticIndex < optimisticEntries.length) {
+      yield optimisticEntries[optimisticIndex++]![0]
+    }
   }
 
   /**

packages/db/tests/collection-sync-reentrancy.test.ts

@@ -856,7 +856,7 @@ describe(`sync publication reentrancy`, () => {
     },
   )
 
-  it(`honors a parked layout mark when truncate drains its causal prefix`, async () => {
+  it(`keeps a parked layout mark hidden when truncate preserves visible order`, async () => {
     const updatePersistence = createDeferred<void>()
     const insertPersistence = createDeferred<void>()
     let sync!: OrderedSync
@@ -930,22 +930,22 @@ describe(`sync publication reentrancy`, () => {
       const truncateReceipt = sync.commit()
 
       expect(truncateReceipt).toBe(true)
-      expect([...collection.keys()]).toEqual([2, 1, 3])
+      expect([...collection.keys()]).toEqual([1, 2, 3])
       expect(collection.toArray.map(({ value }) => value)).toEqual([
-        `two`,
         `optimistic-one`,
+        `two`,
         `optimistic-three`,
       ])
-      expect(collection._layoutRevision).toBe(revisionBeforeDrain + 1)
+      expect(collection._layoutRevision).toBe(revisionBeforeDrain)
       expect(callbacks).toEqual([
         {
           // Delete the prior public layout [1, 2, 3], not the unpublished
-          // rank update's intermediate [2, 1, 3], then replay whole snapshots.
+          // server-only rank update, then replay whole visible snapshots.
           changes: [1, 2, 3, 1, 3, 1, 2],
-          keys: [2, 1, 3],
-          values: [`two`, `optimistic-one`, `optimistic-three`],
+          keys: [1, 2, 3],
+          values: [`optimistic-one`, `two`, `optimistic-three`],
           markedReceiptSettled: false,
-          revision: revisionBeforeDrain + 1,
+          revision: revisionBeforeDrain,
         },
       ])
       if (firstReceipt !== true) await firstReceipt