diff --git a/packages/db/tests/query/includes-publication-oracle.test.ts b/packages/db/tests/query/includes-publication-oracle.test.ts new file mode 100644 index 000000000..27d7c8134 --- /dev/null +++ b/packages/db/tests/query/includes-publication-oracle.test.ts @@ -0,0 +1,634 @@ +import { fc, test as fcTest } from '@fast-check/vitest' +import { describe, expect } from 'vitest' +import { createCollection } from '../../src/collection/index.js' +import { BasicIndex } from '../../src/indexes/basic-index.js' +import { + createLiveQueryCollection, + eq, + materialize, +} from '../../src/query/index.js' +import { expectAssertionFailure } from '../expected-failure.js' +import { TraceAssertionError, runTrace } from '../trace-runner.js' +import { + flushPromises, + mockSyncCollectionOptions, + withExpectedRejection, +} from '../utils.js' +import type { AssertionDifference } from '../expected-failure.js' +import type { TraceDriver, TraceProjection } from '../trace-runner.js' + +type ParentRow = { + id: number + group: number + value: number +} + +type ChildRow = { + id: number + parentGroup: number + value: number +} + +type MetadataRow = { + id: number + parentId: number +} + +type PublishedRow = { + item: ParentRow + children: Array + otherChildren: Array +} + +type Q2Shape = `passThrough` | `where` | `orderBy` | `select` +type Q1Shape = `direct` | `joined` + +const initialParent: ParentRow = { id: 1, group: 10, value: 0 } +const initialChild: ChildRow = { id: 100, parentGroup: 10, value: 1 } +const initialChildren: ReadonlyArray = [ + initialChild, + { id: 200, parentGroup: 20, value: 2 }, +] +const initialOtherChildren: ReadonlyArray = [ + { id: 300, parentGroup: 10, value: 3 }, + { id: 400, parentGroup: 20, value: 4 }, +] + +type SyncChange = { + type: `insert` | `update` | `delete` + value: T +} + +type PublicationAction = + | { type: `parentScalar`; value: number } + | { type: `childScalar`; value: number } + | { type: `parentRoute`; group: number } + | { type: `atomicReplace`; group: number; value: number } + | { type: `optimisticConfirm`; value: number } + | { type: `optimisticRollback`; value: number } + | { type: `parentThenChild`; parentValue: number; childValue: number } + +let nextCollectionId = 0 + +function createControlledCollection( + name: string, + initialData: ReadonlyArray, +) { + const options = mockSyncCollectionOptions({ + id: `${name}-${nextCollectionId++}`, + getKey: (row) => row.id, + initialData: initialData.map((row) => ({ ...row })), + }) + const collection = createCollection(options) + + const writeBatch = (changes: ReadonlyArray>): void => { + options.utils.begin() + for (const change of changes) options.utils.write(change) + options.utils.commit() + } + + return { + collection, + write(type: SyncChange[`type`], value: T): void { + writeBatch([{ type, value: { ...value } }]) + }, + writeBatch, + resolveSync: options.utils.resolveSync, + rejectSync: options.utils.rejectSync, + } +} + +function createLayeredQuery( + parents: ReturnType>, + children: ReturnType>, + otherChildren: ReturnType>, + metadata: ReturnType>, + q1Shape: Q1Shape, + q2Shape: Q2Shape, +) { + const q1 = createLiveQueryCollection({ + id: `publication-q1-${nextCollectionId++}`, + query: (q) => { + const source = q.from({ item: parents.collection }) + const parentRows = + q1Shape === `direct` + ? source + : source.join( + { metadata: metadata.collection }, + ({ item, metadata: rowMetadata }) => + eq(item.id, rowMetadata.parentId), + `inner`, + ) + + return parentRows.select(({ item }) => ({ + item, + children: materialize( + q + .from({ child: children.collection }) + .where(({ child }) => eq(child.parentGroup, item.group)) + .orderBy(({ child }) => child.id) + .select(({ child }) => ({ + id: child.id, + parentGroup: child.parentGroup, + value: child.value, + })), + ), + otherChildren: materialize( + q + .from({ otherChild: otherChildren.collection }) + .where(({ otherChild }) => eq(otherChild.parentGroup, item.group)) + .orderBy(({ otherChild }) => otherChild.id) + .select(({ otherChild }) => ({ + id: otherChild.id, + parentGroup: otherChild.parentGroup, + value: otherChild.value, + })), + ), + })) + }, + getKey: (row) => row.item.id, + }) + const id = `publication-q2-${nextCollectionId++}` + const q2 = (() => { + switch (q2Shape) { + case `passThrough`: + return createLiveQueryCollection({ + id, + query: (q) => q.from({ row: q1 }), + getKey: (row) => row.item.id, + }) + case `where`: + return createLiveQueryCollection({ + id, + query: (q) => + q + .from({ row: q1 }) + .where(({ row }) => eq(row.item.id, initialParent.id)), + getKey: (row) => row.item.id, + }) + case `orderBy`: + return createLiveQueryCollection({ + id, + query: (q) => + q.from({ row: q1 }).orderBy(({ row }) => row.item.value), + getKey: (row) => row.item.id, + }) + case `select`: + return createLiveQueryCollection({ + id, + query: (q) => + q.from({ row: q1 }).select(({ row }) => ({ + item: row.item, + children: row.children, + otherChildren: row.otherChildren, + })), + getKey: (row) => row.item.id, + }) + } + })() + return { q1, q2 } +} + +function stripVirtualProperties(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripVirtualProperties) + if (!value || typeof value !== `object`) return value + + return Object.fromEntries( + Object.entries(value) + .filter(([key]) => !key.startsWith(`$`)) + .map(([key, entry]) => [key, stripVirtualProperties(entry)]), + ) +} + +type PublicationObservation = { + q1: Array + q2: Array +} + +type PublicationContext = { + sources: { + parents: ReturnType> + children: ReturnType> + otherChildren: ReturnType> + metadata: ReturnType> + } + queries: ReturnType + model: { + parents: Map + children: Map + otherChildren: Map + } +} + +function recomputeRows(context: PublicationContext): Array { + return [...context.model.parents.values()] + .sort((left, right) => left.id - right.id) + .map((parent) => ({ + item: { ...parent }, + children: [...context.model.children.values()] + .filter((child) => child.parentGroup === parent.group) + .sort((left, right) => left.id - right.id) + .map((child) => ({ ...child })), + otherChildren: [...context.model.otherChildren.values()] + .filter((child) => child.parentGroup === parent.group) + .sort((left, right) => left.id - right.id) + .map((child) => ({ ...child })), + })) +} + +const publicationProjection: TraceProjection< + PublicationContext, + PublicationObservation +> = { + observe: ({ queries }) => ({ + q1: stripVirtualProperties(queries.q1.toArray) as Array, + q2: stripVirtualProperties(queries.q2.toArray) as Array, + }), + recompute: (context) => { + const expected = recomputeRows(context) + return { + q1: expected.map((row) => structuredClone(row)), + q2: expected.map((row) => structuredClone(row)), + } + }, + assertEqual: (observed, expected) => { + expect(observed).toEqual(expected) + return undefined + }, +} + +function sameValue(left: unknown, right: unknown): boolean { + try { + expect(left).toEqual(right) + return true + } catch { + return false + } +} + +// #1713 is specifically publication of Q1's compiled null placeholder to Q2: +// Q1 has already patched its materialization, while Q2 persists the placeholder. +function classifyDroppedQ2Materialization({ + actual, + expected, +}: AssertionDifference): boolean { + if ( + typeof actual !== `object` || + actual === null || + typeof expected !== `object` || + expected === null || + !(`q1` in actual) || + !(`q2` in actual) || + !(`q1` in expected) || + !(`q2` in expected) || + !Array.isArray(actual.q2) || + !Array.isArray(expected.q2) + ) { + return false + } + + const expectedWithDroppedChildren = expected.q2.map((row) => + typeof row === `object` && row !== null + ? { ...row, children: null, otherChildren: null } + : row, + ) + + return ( + sameValue(actual.q1, expected.q1) && + sameValue(actual.q2, expectedWithDroppedChildren) + ) +} + +function expectDroppedQ2FailureAt(error: unknown, checkpoint: number): void { + expect(error).toMatchObject({ + name: `TraceAssertionError`, + checkpoint, + cause: { name: `AssertionError` }, + }) + if ( + !(error instanceof TraceAssertionError) || + typeof error.cause !== `object` || + error.cause === null || + !(`actual` in error.cause) || + !(`expected` in error.cause) + ) { + throw error + } + expect( + classifyDroppedQ2Materialization({ + actual: error.cause.actual, + expected: error.cause.expected, + }), + ).toBe(true) +} + +async function settleRollback( + rejectSync: (error: Error) => void, + persisted: Promise, +): Promise { + const message = `publication oracle rollback` + const outcome = persisted.catch(() => undefined) + await withExpectedRejection(message, async () => { + rejectSync(new Error(message)) + await outcome + await flushPromises() + }) +} + +function createPublicationDriver( + q1Shape: Q1Shape, + q2Shape: Q2Shape, + checkpointOptimistic = false, +): TraceDriver { + return { + setup: () => { + const parents = createControlledCollection(`publication-parents`, [ + initialParent, + ]) + const children = createControlledCollection( + `publication-children`, + initialChildren, + ) + const otherChildren = createControlledCollection( + `publication-other-children`, + initialOtherChildren, + ) + const metadata = createControlledCollection(`publication-metadata`, [ + { id: initialParent.id, parentId: initialParent.id }, + ]) + if (q1Shape === `joined`) { + parents.collection.createIndex((row) => row.id, { + indexType: BasicIndex, + }) + } + return { + sources: { parents, children, otherChildren, metadata }, + queries: createLayeredQuery( + parents, + children, + otherChildren, + metadata, + q1Shape, + q2Shape, + ), + model: { + parents: new Map([[initialParent.id, { ...initialParent }]]), + children: new Map( + initialChildren.map((child) => [child.id, { ...child }]), + ), + otherChildren: new Map( + initialOtherChildren.map((child) => [child.id, { ...child }]), + ), + }, + } + }, + start: async ({ queries }) => { + await queries.q1.preload() + await queries.q2.preload() + }, + apply: async (action, context, checkpoint) => { + if (action.type === `parentThenChild`) { + const parent = context.model.parents.get(initialParent.id) + const child = context.model.children.get(initialChild.id) + if (!parent || !child) throw new Error(`Missing publication fixture`) + + const nextParent = { ...parent, value: action.parentValue } + context.sources.parents.write(`update`, nextParent) + context.model.parents.set(nextParent.id, { ...nextParent }) + + let publicationFailure: unknown + try { + checkpoint() + } catch (error) { + publicationFailure = error + } + expectDroppedQ2FailureAt(publicationFailure, 1) + + const nextChild = { ...child, value: action.childValue } + context.sources.children.write(`update`, nextChild) + context.model.children.set(nextChild.id, { ...nextChild }) + return + } + + if (action.type === `childScalar`) { + const currentChild = context.model.children.get(initialChild.id) + if (!currentChild) throw new Error(`Missing publication child`) + const nextChild = { ...currentChild, value: action.value } + context.sources.children.write(`update`, nextChild) + context.model.children.set(nextChild.id, { ...nextChild }) + return + } + + const current = context.model.parents.get(initialParent.id) + if (!current) throw new Error(`Missing publication parent`) + + const next: ParentRow = { + ...current, + group: + action.type === `parentRoute` || action.type === `atomicReplace` + ? action.group + : current.group, + value: `value` in action ? action.value : current.value, + } + + if (action.type === `atomicReplace`) { + context.sources.parents.writeBatch([ + { type: `delete`, value: { ...current } }, + { type: `insert`, value: { ...next } }, + ]) + context.model.parents.set(next.id, { ...next }) + return + } + + if ( + action.type === `optimisticConfirm` || + action.type === `optimisticRollback` + ) { + const transaction = context.sources.parents.collection.update( + next.id, + (draft) => { + draft.value = next.value + }, + ) + const previous = { ...current } + context.model.parents.set(next.id, { ...next }) + + let optimisticFailure: unknown + if (checkpointOptimistic) { + try { + checkpoint() + } catch (error) { + optimisticFailure = error + } + } + + if (action.type === `optimisticConfirm`) { + context.sources.parents.write(`update`, next) + context.sources.parents.resolveSync() + await transaction.isPersisted.promise + } else { + await settleRollback( + context.sources.parents.rejectSync, + transaction.isPersisted.promise, + ) + context.model.parents.set(previous.id, previous) + } + + if (optimisticFailure) throw optimisticFailure + return + } + + context.sources.parents.write(`update`, next) + context.model.parents.set(next.id, { ...next }) + }, + cleanup: async ({ queries, sources }) => { + await queries.q2.cleanup() + await queries.q1.cleanup() + await Promise.all([ + sources.parents.collection.cleanup(), + sources.children.collection.cleanup(), + sources.otherChildren.collection.cleanup(), + sources.metadata.collection.cleanup(), + ]) + }, + } +} + +async function expectPublicationMatches( + action: PublicationAction, + checkpointOptimistic = false, + q1Shape: Q1Shape = `direct`, + q2Shape: Q2Shape = `passThrough`, +): Promise { + await runTrace({ + steps: [action], + driver: createPublicationDriver(q1Shape, q2Shape, checkpointOptimistic), + projection: publicationProjection, + }) +} + +async function expectDroppedQ2Materialization( + action: PublicationAction, + checkpointOptimistic = false, + q1Shape: Q1Shape = `direct`, + q2Shape: Q2Shape = `passThrough`, +): Promise { + await expectAssertionFailure( + () => + expectPublicationMatches(action, checkpointOptimistic, q1Shape, q2Shape), + { + checkpoint: 1, + classify: classifyDroppedQ2Materialization, + }, + )() +} + +const q2Shapes = [`passThrough`, `where`, `orderBy`, `select`] as const +const q1Shapes = [`direct`, `joined`] as const + +describe(`layered-query publication oracle`, () => { + const changedValueArbitrary = fc.oneof( + fc.integer({ min: -100, max: -1 }), + fc.integer({ min: 1, max: 100 }), + ) + const changedChildValueArbitrary = fc.oneof( + fc.integer({ min: -100, max: 0 }), + fc.integer({ min: 2, max: 100 }), + ) + + for (const q1Shape of q1Shapes) { + for (const q2Shape of q2Shapes) { + fcTest.prop([changedValueArbitrary], { numRuns: 12 })( + `classifies #1713 through a ${q1Shape} Q1 and ${q2Shape} Q2`, + async (value) => { + await expectDroppedQ2Materialization( + { type: `parentScalar`, value }, + false, + q1Shape, + q2Shape, + ) + }, + ) + + fcTest.prop([changedValueArbitrary, changedChildValueArbitrary], { + numRuns: 12, + })( + `recovers a ${q1Shape} Q1 and ${q2Shape} Q2 after a child update`, + async (parentValue, childValue) => { + await expectPublicationMatches( + { type: `parentThenChild`, parentValue, childValue }, + false, + q1Shape, + q2Shape, + ) + }, + ) + + fcTest.prop([changedValueArbitrary], { numRuns: 8 })( + `classifies optimistic publication before confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, + async (value) => { + await expectDroppedQ2Materialization( + { type: `optimisticConfirm`, value }, + true, + q1Shape, + q2Shape, + ) + }, + ) + + fcTest.prop([changedValueArbitrary], { numRuns: 8 })( + `classifies publication after optimistic confirmation through a ${q1Shape} Q1 and ${q2Shape} Q2`, + async (value) => { + await expectDroppedQ2Materialization( + { type: `optimisticConfirm`, value }, + false, + q1Shape, + q2Shape, + ) + }, + ) + } + } + + fcTest.prop([changedChildValueArbitrary])( + `publishes child-only scalar updates through both layers`, + async (value) => { + await expectPublicationMatches({ type: `childScalar`, value }) + }, + ) + + fcTest.prop([fc.constantFrom(20, 30)])( + `compares route transitions at both query layers`, + async (group) => { + await expectPublicationMatches({ type: `parentRoute`, group }) + }, + ) + + fcTest.prop([ + fc.record({ + group: fc.constantFrom(10, 20, 30), + value: changedValueArbitrary, + }), + ])( + `compares atomic parent replacements at both query layers`, + async (row) => { + // Changing the route rebuilds the materialization before publication and + // is green. A same-route replacement republishes the null placeholder. + const assertion = + row.group === 10 + ? expectDroppedQ2Materialization + : expectPublicationMatches + await assertion({ type: `atomicReplace`, ...row }) + }, + ) + + fcTest.prop([changedValueArbitrary])( + `classifies stale publication after optimistic rollback`, + async (value) => { + await expectDroppedQ2Materialization({ + type: `optimisticRollback`, + value, + }) + }, + ) +})