diff --git a/package.json b/package.json index e4702844..3a5b43ae 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "data-monorepo", - "version": "0.9.88", + "version": "0.9.89", "private": true, "engines": { "node": ">=22" diff --git a/packages/data-ai/.claude-plugin/plugin.json b/packages/data-ai/.claude-plugin/plugin.json index cb0b5901..98aefbd3 100644 --- a/packages/data-ai/.claude-plugin/plugin.json +++ b/packages/data-ai/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "adobe-data-ai", - "version": "0.9.88", + "version": "0.9.89", "description": "Architecture skills for @adobe/data — data-oriented modelling, archetype iteration, hot-path performance, and related conventions.", "author": { "name": "Adobe" diff --git a/packages/data-ai/package.json b/packages/data-ai/package.json index 40df0e04..349289e4 100644 --- a/packages/data-ai/package.json +++ b/packages/data-ai/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-ai", - "version": "0.9.88", + "version": "0.9.89", "description": "Cross-agent architecture skills for @adobe/data — installable as a Claude Code plugin or copied into any Agent-Skills-compatible agent (Cursor, Codex).", "type": "module", "private": false, diff --git a/packages/data-gpu-hopper/package.json b/packages/data-gpu-hopper/package.json index da6dd581..f8c8b760 100644 --- a/packages/data-gpu-hopper/package.json +++ b/packages/data-gpu-hopper/package.json @@ -1,6 +1,6 @@ { "name": "data-gpu-hopper", - "version": "0.9.88", + "version": "0.9.89", "description": "Hopper sample - real-time ECS game rendered as colored cubes via @adobe/data-gpu", "type": "module", "private": true, diff --git a/packages/data-gpu-samples/package.json b/packages/data-gpu-samples/package.json index be8192bd..d0d2651f 100644 --- a/packages/data-gpu-samples/package.json +++ b/packages/data-gpu-samples/package.json @@ -1,6 +1,6 @@ { "name": "data-gpu-samples", - "version": "0.9.88", + "version": "0.9.89", "description": "WebGPU samples built on @adobe/data-gpu", "type": "module", "private": true, diff --git a/packages/data-gpu/package.json b/packages/data-gpu/package.json index e48f97c2..6c3ee24e 100644 --- a/packages/data-gpu/package.json +++ b/packages/data-gpu/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-gpu", - "version": "0.9.88", + "version": "0.9.89", "description": "Adobe data WebGPU plugins and types for graphics and compute", "type": "module", "private": false, diff --git a/packages/data-gpu/src/graphics/rendering/bone-collider-plugin.ts b/packages/data-gpu/src/graphics/rendering/bone-collider-plugin.ts index 4d91a8ed..3a93e90e 100644 --- a/packages/data-gpu/src/graphics/rendering/bone-collider-plugin.ts +++ b/packages/data-gpu/src/graphics/rendering/bone-collider-plugin.ts @@ -53,10 +53,10 @@ interface SkinMesh { export const boneColliders = Database.Plugin.create({ extends: Database.Plugin.combine(physicsData, jointData, ragdollTrigger, pbrSkinning, modelLoader, transform), components: { - _boneJoint: { ...Entity.schema, nonPersistent: true }, // the skeleton joint this capsule tracks - _boneOffsetPos: { ...Vec3.schema, nonPersistent: true }, // capsule offset in the bone's bind-local frame - _boneOffsetRot: { ...Quat.schema, nonPersistent: true }, - _ragdollBuilt: { ...True.schema, nonPersistent: true }, // tag: this skeleton's bone capsules have been generated + _boneJoint: { ...Entity.schema, default: undefined, nonPersistent: true }, // the skeleton joint this capsule tracks + _boneOffsetPos: { ...Vec3.schema, default: undefined, nonPersistent: true }, // capsule offset in the bone's bind-local frame + _boneOffsetRot: { ...Quat.schema, default: undefined, nonPersistent: true }, + _ragdollBuilt: { ...True.schema, default: undefined, nonPersistent: true }, // tag: this skeleton's bone capsules have been generated }, archetypes: { // a kinematic capsule body bound to a skeleton joint (collisionGroup 1 ⇒ the diff --git a/packages/data-gpu/src/graphics/rendering/display-transform-plugin.ts b/packages/data-gpu/src/graphics/rendering/display-transform-plugin.ts index 8d8c18d4..d6119d5d 100644 --- a/packages/data-gpu/src/graphics/rendering/display-transform-plugin.ts +++ b/packages/data-gpu/src/graphics/rendering/display-transform-plugin.ts @@ -20,7 +20,7 @@ import { Vec3, Quat } from "@adobe/data/math"; */ export const displayTransform = Database.Plugin.create({ components: { - _renderPosition: { ...Vec3.schema, nonPersistent: true }, // derived: interpolated pose to render at (else use `position`) - _renderRotation: { ...Quat.schema, nonPersistent: true }, + _renderPosition: { ...Vec3.schema, default: undefined, nonPersistent: true }, // derived: interpolated pose to render at (else use `position`) + _renderRotation: { ...Quat.schema, default: undefined, nonPersistent: true }, }, }); diff --git a/packages/data-gpu/src/graphics/rendering/material-palette-gpu/material-palette-gpu-plugin.ts b/packages/data-gpu/src/graphics/rendering/material-palette-gpu/material-palette-gpu-plugin.ts index 33c14ea3..21f36223 100644 --- a/packages/data-gpu/src/graphics/rendering/material-palette-gpu/material-palette-gpu-plugin.ts +++ b/packages/data-gpu/src/graphics/rendering/material-palette-gpu/material-palette-gpu-plugin.ts @@ -27,7 +27,7 @@ const FACTOR_MATERIAL_COMPONENTS = [ export const materialPaletteGpu = Database.Plugin.create({ extends: Database.Plugin.combine(Material.plugin, core), components: { - _paletteIndex: { ...U32.schema, nonPersistent: true }, + _paletteIndex: { ...U32.schema, default: undefined, nonPersistent: true }, }, resources: { _factorPalette: { default: null as GPUBuffer | null, nonPersistent: true }, diff --git a/packages/data-gpu/src/graphics/rendering/pbr-core-plugin.ts b/packages/data-gpu/src/graphics/rendering/pbr-core-plugin.ts index f072e3ce..e973f3c9 100644 --- a/packages/data-gpu/src/graphics/rendering/pbr-core-plugin.ts +++ b/packages/data-gpu/src/graphics/rendering/pbr-core-plugin.ts @@ -15,19 +15,19 @@ export const pbrCore = Database.Plugin.create({ * null for static primitives. Drives skinned-pipeline dispatch. */ _skinVertexBuffer: { default: null as GPUBuffer | null, nonPersistent: true }, _indexBuffer: { default: null as GPUBuffer | null, nonPersistent: true }, - _indexCount: { ...U32.schema, nonPersistent: true }, + _indexCount: { ...U32.schema, default: undefined, nonPersistent: true }, _indexFormat: { default: "uint16" as GPUIndexFormat, nonPersistent: true }, _materialBindGroup: { default: null as GPUBindGroup | null, nonPersistent: true }, /** Back-reference from a _PbrPrimitive / _VisibleMaterial to the * baked mesh asset that owns it. */ - _mesh: { ...Entity.schema, nonPersistent: true }, - _material: { ...Entity.schema, nonPersistent: true }, + _mesh: { ...Entity.schema, default: undefined, nonPersistent: true }, + _material: { ...Entity.schema, default: undefined, nonPersistent: true }, /** Node-local-to-model-root matrix baked at load time. The renderer * pre-multiplies it with the per-instance model-root world matrix. */ - _nodeLocalMatrix: { ...Mat4x4.schema, nonPersistent: true }, + _nodeLocalMatrix: { ...Mat4x4.schema, default: undefined, nonPersistent: true }, /** Skeleton-side back-references the renderer reads when dispatching * the skinned pipeline. Populated by `pbrSkinning`. */ - _skeletonModelRef: { ...Entity.schema, nonPersistent: true }, + _skeletonModelRef: { ...Entity.schema, default: undefined, nonPersistent: true }, _skeletonJointMatrixBindGroup: { default: null as GPUBindGroup | null, nonPersistent: true }, }, archetypes: { diff --git a/packages/data-gpu/src/graphics/rendering/skinning/skinning-plugin.ts b/packages/data-gpu/src/graphics/rendering/skinning/skinning-plugin.ts index 31131cf5..31510779 100644 --- a/packages/data-gpu/src/graphics/rendering/skinning/skinning-plugin.ts +++ b/packages/data-gpu/src/graphics/rendering/skinning/skinning-plugin.ts @@ -16,7 +16,7 @@ export const pbrSkinning = Database.Plugin.create({ extends: Database.Plugin.combine(modelLoader, pbrCore, transform, animation), components: { _skeletonJoints: { default: [] as number[], nonPersistent: true }, - _skeletonMesh: { ...Entity.schema, nonPersistent: true }, + _skeletonMesh: { ...Entity.schema, default: undefined, nonPersistent: true }, _skeletonInstanceBuffer: { default: null as GPUBuffer | null, nonPersistent: true }, _skeletonJointMatrixBuffer: { default: null as GPUBuffer | null, nonPersistent: true }, }, diff --git a/packages/data-gpu/src/graphics/scene/node/node-data-plugin.ts b/packages/data-gpu/src/graphics/scene/node/node-data-plugin.ts index 27083276..855d1518 100644 --- a/packages/data-gpu/src/graphics/scene/node/node-data-plugin.ts +++ b/packages/data-gpu/src/graphics/scene/node/node-data-plugin.ts @@ -23,8 +23,8 @@ export const nodeData = Database.Plugin.create({ rotation: Quat.schema, scale: Vec3.schema, parent: Entity.schema, - _worldMatrix: { ...Mat4x4.schema, nonPersistent: true }, - _worldBounds: { ...Aabb.schema, nonPersistent: true }, + _worldMatrix: { ...Mat4x4.schema, default: undefined, nonPersistent: true }, + _worldBounds: { ...Aabb.schema, default: undefined, nonPersistent: true }, }, archetypes: { Node: ["position", "rotation", "scale", "parent", "visible"], diff --git a/packages/data-gpu/src/physics/physics-data-plugin.ts b/packages/data-gpu/src/physics/physics-data-plugin.ts index c47042f1..2ff47775 100644 --- a/packages/data-gpu/src/physics/physics-data-plugin.ts +++ b/packages/data-gpu/src/physics/physics-data-plugin.ts @@ -57,8 +57,8 @@ export const physicsData = Database.Plugin.create({ rotation: Quat.schema, // unified with Node.rotation so a body renders directly linearVelocity: Vec3.schema, angularVelocity: Vec3.schema, - _prevPosition: { ...Vec3.schema, nonPersistent: true }, // derived: pose before the last fixed step (render interpolation) - _prevRotation: { ...Quat.schema, nonPersistent: true }, + _prevPosition: { ...Vec3.schema, default: undefined, nonPersistent: true }, // derived: pose before the last fixed step (render interpolation) + _prevRotation: { ...Quat.schema, default: undefined, nonPersistent: true }, // Bodies sharing the same non-zero collisionGroup do not collide with each // other (they still collide with group 0 / the world) — e.g. a ragdoll's // bones. 0 = default (collide with everything). Honored by both solvers diff --git a/packages/data-gpu/src/physics/solvers/jolt-solver-plugin.ts b/packages/data-gpu/src/physics/solvers/jolt-solver-plugin.ts index eb091a63..ed2ece8a 100644 --- a/packages/data-gpu/src/physics/solvers/jolt-solver-plugin.ts +++ b/packages/data-gpu/src/physics/solvers/jolt-solver-plugin.ts @@ -85,8 +85,8 @@ export interface JoltContext { export const joltSolver = Database.Plugin.create({ extends: Database.Plugin.combine(physicsData, jointData, physicsClock), components: { - _joltBody: { ...True.schema, nonPersistent: true }, // tag: this body has been mirrored into the Jolt world - _joltJoint: { ...True.schema, nonPersistent: true }, // tag: this joint has been mirrored into the Jolt world + _joltBody: { ...True.schema, default: undefined, nonPersistent: true }, // tag: this body has been mirrored into the Jolt world + _joltJoint: { ...True.schema, default: undefined, nonPersistent: true }, // tag: this joint has been mirrored into the Jolt world }, resources: { _joltContext: { default: null as JoltContext | null, nonPersistent: true }, diff --git a/packages/data-gpu/src/physics/solvers/rapier-solver-plugin.ts b/packages/data-gpu/src/physics/solvers/rapier-solver-plugin.ts index 6c09f557..149e42ab 100644 --- a/packages/data-gpu/src/physics/solvers/rapier-solver-plugin.ts +++ b/packages/data-gpu/src/physics/solvers/rapier-solver-plugin.ts @@ -53,8 +53,8 @@ interface MatProps { density: number; restitution: number; friction: number } export const rapierSolver = Database.Plugin.create({ extends: Database.Plugin.combine(physicsData, jointData, physicsClock), components: { - _rapierBody: { ...True.schema, nonPersistent: true }, // tag: this body has been mirrored into the Rapier world - _rapierJoint: { ...True.schema, nonPersistent: true }, // tag: this joint has been mirrored into the Rapier world + _rapierBody: { ...True.schema, default: undefined, nonPersistent: true }, // tag: this body has been mirrored into the Rapier world + _rapierJoint: { ...True.schema, default: undefined, nonPersistent: true }, // tag: this joint has been mirrored into the Rapier world }, systems: { rapierStep: { diff --git a/packages/data-lit-space-rock-game/package.json b/packages/data-lit-space-rock-game/package.json index c51701bc..678e90a7 100644 --- a/packages/data-lit-space-rock-game/package.json +++ b/packages/data-lit-space-rock-game/package.json @@ -1,6 +1,6 @@ { "name": "data-lit-space-rock-game", - "version": "0.9.88", + "version": "0.9.89", "description": "Space Rock Game sample - real-time ECS game with Lit and @adobe/data", "type": "module", "private": true, diff --git a/packages/data-lit-tictactoe/package.json b/packages/data-lit-tictactoe/package.json index 4d8f5f13..11e0f2ef 100644 --- a/packages/data-lit-tictactoe/package.json +++ b/packages/data-lit-tictactoe/package.json @@ -1,6 +1,6 @@ { "name": "data-lit-tictactoe", - "version": "0.9.88", + "version": "0.9.89", "description": "Tic-Tac-Toe sample - Lit web components with @adobe/data-lit and AgenticService", "type": "module", "private": true, diff --git a/packages/data-lit-todo/package.json b/packages/data-lit-todo/package.json index b8674beb..0a645ad4 100644 --- a/packages/data-lit-todo/package.json +++ b/packages/data-lit-todo/package.json @@ -1,6 +1,6 @@ { "name": "data-lit-todo", - "version": "0.9.88", + "version": "0.9.89", "description": "Todo application - Lit web components with @adobe/data ECS", "type": "module", "private": true, diff --git a/packages/data-lit/package.json b/packages/data-lit/package.json index 823f48a7..d6411415 100644 --- a/packages/data-lit/package.json +++ b/packages/data-lit/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-lit", - "version": "0.9.88", + "version": "0.9.89", "description": "Adobe data Lit bindings - hooks, elements, decorators", "type": "module", "private": false, diff --git a/packages/data-p2p-tictactoe/package.json b/packages/data-p2p-tictactoe/package.json index c403c0b8..65be93fd 100644 --- a/packages/data-p2p-tictactoe/package.json +++ b/packages/data-p2p-tictactoe/package.json @@ -1,6 +1,6 @@ { "name": "data-p2p-tictactoe", - "version": "0.9.88", + "version": "0.9.89", "description": "Serverless P2P tic-tac-toe — WebRTC DataChannel + @adobe/data-sync", "type": "module", "private": true, diff --git a/packages/data-persistence/package.json b/packages/data-persistence/package.json index 3e6d08e4..bc975fbc 100644 --- a/packages/data-persistence/package.json +++ b/packages/data-persistence/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-persistence", - "version": "0.9.88", + "version": "0.9.89", "description": "Worker-based incremental persistence layer for @adobe/data ECS over OPFS (browser) and node:fs (server).", "type": "module", "sideEffects": false, diff --git a/packages/data-persistence/src/node/node-worker-transport.node.test.ts b/packages/data-persistence/src/node/node-worker-transport.node.test.ts index f5a53597..051aab5b 100644 --- a/packages/data-persistence/src/node/node-worker-transport.node.test.ts +++ b/packages/data-persistence/src/node/node-worker-transport.node.test.ts @@ -104,7 +104,7 @@ describe("createNodeWorkerTransport (E2E)", () => { // Verify on-disk artifacts exist. const metaStat = await stat(join(tmpRoot, "meta.json")); expect(metaStat.size).toBeGreaterThan(0); - const eltStat = await stat(join(tmpRoot, "entity-location.bin")); + const eltStat = await stat(join(tmpRoot, "entity-location-0.bin")); expect(eltStat.size).toBeGreaterThan(0); // Loading phase: fresh database, same plugin, recover via load(). @@ -136,7 +136,7 @@ describe("createNodeWorkerTransport (E2E)", () => { } }); - it("writes column files and entity-location.bin via the worker", async () => { + it("writes column files and entity-location files via the worker", async () => { const workerScript = ensureBuilt(); const db = Database.create(particlePlugin); @@ -161,8 +161,8 @@ describe("createNodeWorkerTransport (E2E)", () => { const archetypeDir = archetypeDirs[0]!; const cols = [...(await backend.list(`archetypes/${archetypeDir}`))].sort(); // position, velocity, mass — all .bin files. The implicit `id` - // column is intentionally not persisted (it's recoverable from - // entity-location.bin). + // column is intentionally not persisted (it is recoverable from the + // entity-location files). expect(cols).toEqual(["mass.bin", "position.bin", "velocity.bin"]); await service.dispose(); diff --git a/packages/data-persistence/src/service/component-nonpersistent-durability.test.ts b/packages/data-persistence/src/service/component-nonpersistent-durability.test.ts new file mode 100644 index 00000000..6024866b --- /dev/null +++ b/packages/data-persistence/src/service/component-nonpersistent-durability.test.ts @@ -0,0 +1,65 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. +// +// A component whose schema is marked `nonPersistent: true` must not be saved. +// On load it is reconstructed via the hybrid rule: a defaulted one is reset to +// its default (present); a no-default one is stripped (the entity restores into +// the reduced archetype). This exercises the full worker/checkpoint/reload path. + +import { Database } from "@adobe/data/ecs"; +import type { Schema } from "@adobe/data/schema"; +import { F32 } from "@adobe/data/math"; +import { beforeEach, describe, expect, it } from "vitest"; +import { createMemoryBackend } from "../backend/memory-backend.js"; +import { createWorkerPersistenceService } from "./create-worker-persistence-service.js"; + +const plugin = Database.Plugin.create({ + components: { + value: F32.schema, + // Defaulted nonPersistent component → reset to default on load. + cache: { type: "number", default: 7, nonPersistent: true } as const, + // No-default nonPersistent component → stripped on load. + derived: { type: "number", nonPersistent: true } as const, + // undefined-default (type-only placeholder, e.g. a GPUBuffer handle) → + // treated as no default → stripped on load. + gpuRef: { default: undefined as unknown, nonPersistent: true } satisfies Schema, + }, + archetypes: { + Item: ["value", "cache", "derived", "gpuRef"], + }, + transactions: { + spawn(t, args: { value: number; cache: number; derived: number; gpuRef: unknown }) { + return t.archetypes.Item.insert(args); + }, + }, +}); + +const noCheckpoint = { everyNTransactions: 0, idleMs: 0 } as const; + +describe("component-level nonPersistent durability", () => { + let backend: ReturnType; + + beforeEach(() => { + backend = createMemoryBackend(); + }); + + it("does not persist nonPersistent columns: defaulted resets, no-default strips", async () => { + const db1 = Database.create(plugin); + const svc1 = await createWorkerPersistenceService({ database: db1, backend, checkpoint: noCheckpoint }); + const e = db1.transactions.spawn({ value: 5, cache: 99, derived: 42, gpuRef: 12345 })!; + await svc1.flush(); + await svc1.checkpoint(); + await svc1.dispose(); + + const db2 = Database.create(plugin); + const svc2 = await createWorkerPersistenceService({ database: db2, backend, checkpoint: noCheckpoint }); + await svc2.load(); + await svc2.dispose(); + + const view = db2.read(e) as { value: number; cache?: number; derived?: number; gpuRef?: unknown } | null; + expect(view).not.toBeNull(); + expect(view!.value).toBe(5); // persistent component preserved + expect(view!.cache).toBe(7); // defaulted nonPersistent → reset to default + expect("derived" in view!).toBe(false); // no-default nonPersistent → stripped + expect("gpuRef" in view!).toBe(false); // null-default nonPersistent → stripped + }); +}); diff --git a/packages/data-persistence/src/service/create-incremental-persistence-service.ts b/packages/data-persistence/src/service/create-incremental-persistence-service.ts index 5b33b38f..a458112c 100644 --- a/packages/data-persistence/src/service/create-incremental-persistence-service.ts +++ b/packages/data-persistence/src/service/create-incremental-persistence-service.ts @@ -1,6 +1,6 @@ // © 2026 Adobe. MIT License. See /LICENSE for details. -import { ECS_SNAPSHOT_VERSION, type Archetype } from "@adobe/data/ecs"; +import { ECS_SNAPSHOT_VERSION, Entity, serializedEntityLocationTables, type Archetype, type EntityLocationEntry } from "@adobe/data/ecs"; import { createColumnEncoder } from "../encoder/create-column-encoder.js"; import { decodeJournalSnapshot, @@ -11,12 +11,13 @@ import { decodeJournalStream, encodeJournalEntry } from "../journal/journal-code import type { JournalEntry, JournalEntryKind } from "../journal/journal-entry.js"; import { createInprocessTransport } from "../transport/inprocess-transport.js"; import type { ListDirResult, PersistOp, ReadFileResult, Transport } from "../transport/transport.js"; -import { createEntityLocationCache } from "./entity-location-cache.js"; import { asMutableArchetype, getColumn, getIdColumn, getMutableStore } from "./internal-access.js"; import type { IncrementalPersistenceService, IncrementalPersistenceServiceOptions } from "./incremental-persistence-service.js"; const META_FILE = "meta.json"; -const ENTITY_LOCATION_FILE = "entity-location.bin"; +// One entity-location file per persistent quadrant (matches router.ts), each +// densely indexed by the entity's per-quadrant local index. +const entityLocationFile = (quadrant: number): string => `entity-location-${quadrant}.bin`; const JOURNAL_FILE = "journal.bin"; const ELT_STRIDE = 8; const columnPath = (archetypeId: number, component: string): string => @@ -72,6 +73,12 @@ export const createIncrementalPersistenceService = async ( const store = getMutableStore(database); + // A component whose SCHEMA is marked nonPersistent (distinct from the + // built-in `nonPersistent` marker component) is never journaled/checkpointed. + const isNonPersistentComponent = (component: string): boolean => + component !== "nonPersistent" && component !== "nonShared" && + (store.componentSchemas as Record)[component]?.nonPersistent === true; + // Cache per-archetype context. We discover archetypes lazily as // they appear in changedEntities so that newly-introduced // archetypes (created by an extend after service init) are picked @@ -124,9 +131,13 @@ export const createIncrementalPersistenceService = async ( const encoders = new Map(); for (const component of archetype.components) { // Skip the implicit `id` column — entity ids are recovered - // from entity-location.bin (which is keyed by entity id), so - // storing them again per archetype row would be redundant. + // from the per-quadrant entity-location files, so storing them + // again per archetype row would be redundant. if (component === "id") continue; + // Skip nonPersistent-schema components — their values are never + // saved; on load they're reset to default or the component is + // stripped (see store.reconstructNonPersistentColumns). + if (isNonPersistentComponent(component)) continue; const buffer = getColumn(archetype, component); if (buffer === undefined) continue; encoders.set(component, createColumnEncoder(component, buffer)); @@ -135,7 +146,7 @@ export const createIncrementalPersistenceService = async ( const componentIds = new Map(); for (const component of archetype.components) { - if (component === "id") continue; + if (component === "id" || isNonPersistentComponent(component)) continue; componentIds.set(component, internComponent(component)); } @@ -303,58 +314,16 @@ export const createIncrementalPersistenceService = async ( await checkpointInFlight; }; - // Track every persisted entity's last known on-disk location. - // Required to detect swap-remove side effects: when an entity at - // row X is deleted (or moves to a different archetype), the table - // moves the last row into row X. The transaction observer does NOT - // report that incidental move, so we have to detect it ourselves - // by re-reading the id column at the vacated slot and treating any - // entity now living there as a synthetic change. - // - // Backed by a flat Uint32Array indexed by entity id (two slots - // per entity) — avoids the per-change Map lookup + per-entry - // object allocation a `Map` would require. - const entityLocations = createEntityLocationCache(); - - const detectSwapRemoveAt = ( - archetypeId: number, - row: number, - txId: number, - alreadyHandled: Set, - ): void => { - // archetypeContexts is keyed by archetype id; if we recorded an - // entity at (archetypeId, row) at any point, the context for - // archetypeId is guaranteed to be in the cache by the time we - // try to detect a swap-remove there. Direct map lookup is - // O(1); the previous linear scan over `store.archetypes` was - // O(archetypes) per delete and showed up under workloads that - // delete many entities. - const ctx = archetypeContexts.get(archetypeId); - if (ctx === undefined) return; - const archetype = ctx.archetype; - if (row >= archetype.rowCount) return; // nothing was moved into this slot - const idColumn = getIdColumn(archetype); - if (idColumn === undefined) return; - const movedEntity = idColumn.get(row); - if (alreadyHandled.has(movedEntity)) return; - alreadyHandled.add(movedEntity); - // The swap-moved entity now lives at a fresh row whose - // contents are wholesale new — every column needs writing, - // not just the columns the user transaction touched. - handleEntityUpdate(movedEntity, null, txId, alreadyHandled); - }; - const handleEntityDelete = ( entity: number, txId: number, - alreadyHandled: Set, ): void => { - const prevArchetype = entityLocations.getArchetypeId(entity); - const prevRow = prevArchetype >= 0 ? entityLocations.getRow(entity) : -1; // WAL discipline: append the journal entry FIRST, then update - // the on-disk snapshot (entity-location.bin). On crash, replay + // the on-disk snapshot (the entity-location files). On crash, replay // applies the journal — which means the snapshot is always - // catching up to the WAL, never running ahead of it. + // catching up to the WAL, never running ahead of it. The delete + // op is keyed by entity id alone; the on-disk layer resolves the + // vacated slot, so no prior-location lookup is needed. sendJournal({ txId, timestampMs: clock(), @@ -366,23 +335,21 @@ export const createIncrementalPersistenceService = async ( bytes: new Uint8Array(0), }); sendEntityDelete(entity); - entityLocations.delete(entity); - // The deleted entity may have been swap-removed: another - // entity may now occupy its old row. - if (prevArchetype >= 0) { - detectSwapRemoveAt(prevArchetype, prevRow, txId, alreadyHandled); - } }; /** * Persist a present entity. `changedComponents`: - * - `Set` → write only the named columns. Used for - * pure same-row updates where the user + * - `Set` → write only the named columns. Used for pure + * same-row value updates where the user * transaction touched a known subset. - * - `null` → write all columns. Used for new entities, - * archetype migrations, and the synthetic - * swap-remove path, where the destination - * row's bytes are wholesale new. + * - `null` → write all columns. Used for new entities and + * for any RELOCATED entity (migration or + * swap-move), whose destination row's bytes are + * wholesale new. + * + * The caller decides null vs. subset from the ECS-emitted relocation + * signal (TransactionResult.relocatedEntities) — the persistence layer + * no longer keeps a per-entity location shadow to infer it. * * Send order within a tx: every journal entry for this entity is * appended FIRST (one per emitColumn call), then the @@ -393,29 +360,19 @@ export const createIncrementalPersistenceService = async ( entity: number, changedComponents: ReadonlySet | null, txId: number, - alreadyHandled: Set, ): void => { - const prevArchetype = entityLocations.getArchetypeId(entity); - const prevRow = prevArchetype >= 0 ? entityLocations.getRow(entity) : -1; const located = store.locate(entity); if (located === null) { - // Edge case: the user transaction reported the entity as - // present but the store no longer has it. Fall back to - // delete (which already follows WAL ordering internally). - handleEntityDelete(entity, txId, alreadyHandled); + // The entity reported as present/relocated is no longer in the + // store (e.g. created then deleted across skipped intermediate + // transactions). Fall back to delete (WAL-ordered internally); + // deleting an entity that was never persisted is a no-op. + handleEntityDelete(entity, txId); return; } const ctx = getArchetypeContext(located.archetype); - - // Decide whether we can trust changedComponents: only when the - // entity stayed at the SAME (archetype, row). Anything else - // means the destination row's underlying memory is freshly - // populated (insert / migrate / swap-into-this-row) and every - // column must be flushed even if the user touched just one. - const sameLocation = - prevArchetype === ctx.id && prevRow === located.row; - const writeAll = changedComponents === null || !sameLocation; + const writeAll = changedComponents === null; if (writeAll) { for (const [component, encoder] of ctx.encoders) { @@ -428,14 +385,10 @@ export const createIncrementalPersistenceService = async ( emitColumn(entity, ctx, component, encoder, located.row, txId); } } - // Snapshot writes go AFTER journal entries for this entity. + // Snapshot writes go AFTER journal entries for this entity. The row + // is read live from the store, so it is always the current row even + // for a swap-moved neighbor. sendEntityLocation(entity, ctx.id, located.row); - - const moved = prevArchetype >= 0 && !sameLocation; - entityLocations.set(entity, ctx.id, located.row); - if (moved) { - detectSwapRemoveAt(prevArchetype, prevRow, txId, alreadyHandled); - } }; const emitColumn = ( @@ -494,30 +447,65 @@ export const createIncrementalPersistenceService = async ( } }); + // Persistent entities relocated by transactions the persistence layer + // SKIPPED (intermediate ones) since the last persisted flush. An + // intermediate transaction is not persisted step-by-step, but a + // relocation it caused leaves that entity's on-disk row stale — so we + // remember it and full-write it at the next persisted flush. This is + // the transient, per-flush replacement for the old per-entity location + // shadow: bounded by inter-flush churn, not by world size. + const pendingRelocations = new Set(); + let unsubscribe: (() => void) | null = null; if (autoPersist) { unsubscribe = database.observe.transactions((result) => { + // Accumulate relocations from EVERY observed transaction — even + // skipped intermediate ones — so a reshuffle hidden inside an + // intermediate transaction is reconciled at the next flush. + // Persistent partitions never share a location table with + // non-persistent ones, so a relocation of a persistent entity + // only ever rides a persistent transaction; filter defensively. + for (const entity of result.relocatedEntities) { + if (Entity.isPersistent(entity)) pendingRelocations.add(entity); + } if (result.intermediate || !result.persistent) return; // One txId per observer firing — every entity-level entry // we emit for this user transaction shares it, and the // trailing commit entry uses the same id. Replay groups by // txId so this is what makes torn-tail recovery atomic. const txId = allocTxId(); - const alreadyHandled = new Set(); - for (const entity of result.changedEntities.keys()) alreadyHandled.add(entity); + const relocated = pendingRelocations; + const handled = new Set(); + for (const entity of result.changedEntities.keys()) handled.add(entity); for (const [entity, values] of result.changedEntities) { + // Only persistent entities are written to disk. A persistent + // transaction can also touch non-persistent (e.g. presence) + // entities; skip those — they live in a separate quadrant and + // are never persisted. + if (Entity.isNonPersistent(entity)) continue; if (values === null) { - handleEntityDelete(entity, txId, alreadyHandled); + handleEntityDelete(entity, txId); + } else if (relocated.has(entity)) { + // Relocated (migration): its new row's other columns are + // carried over and not in `values` — write the whole row. + handleEntityUpdate(entity, null, txId); } else { - // The patched values map's keys are the union of - // every component the transaction touched for this - // entity. For pure same-row updates this is a strict - // subset of all columns — emitting only those is - // the per-component-write optimization. + // Pure same-row update: the patched values map's keys are + // the union of every component the transaction touched for + // this entity — a strict subset of all columns. Emitting + // only those is the per-component-write optimization. const components = Object.keys(values) as readonly string[]; - handleEntityUpdate(entity, new Set(components), txId, alreadyHandled); + handleEntityUpdate(entity, new Set(components), txId); } } + // Swap-moved neighbors (and entities relocated by skipped + // intermediate transactions) that the user did not directly + // touch: their backing row is freshly established → full-write. + for (const entity of relocated) { + if (handled.has(entity)) continue; + handleEntityUpdate(entity, null, txId); + } + pendingRelocations.clear(); sendCommit(txId); txsSinceCheckpoint += 1; if (everyNTransactions > 0 && txsSinceCheckpoint >= everyNTransactions) { @@ -535,15 +523,13 @@ export const createIncrementalPersistenceService = async ( // initial snapshot or none of it. const txId = allocTxId(); const named = store.archetypes; - const handled = new Set(); for (const key in named) { const archetype = named[key]!; const idColumn = getIdColumn(archetype); if (idColumn === undefined) continue; for (let row = 0; row < archetype.rowCount; row++) { const entity = idColumn.get(row); - handled.add(entity); - handleEntityUpdate(entity, null, txId, handled); + handleEntityUpdate(entity, null, txId); } } sendCommit(txId); @@ -560,7 +546,7 @@ export const createIncrementalPersistenceService = async ( * grow column buffers to the persisted capacity, bulk-copy * each column file's bytes into the typed array, and restore * `rowCount`. - * 4. Read `entity-location.bin` and build an in-memory snapshot + * 4. Read the per-quadrant entity-location files and build an in-memory snapshot * of the entity-location table (entities Int32Array + free * list). * 5. Replay the journal forward over the snapshot — this both @@ -590,9 +576,9 @@ export const createIncrementalPersistenceService = async ( await restoreArchetype(aMan); } - const eltState = await readEntityLocationSnapshot(); - await replayJournal(manifest, eltState); - finalizeEntityLocationTable(manifest, eltState); + const eltStates = await readEntityLocationSnapshots(); + await replayJournal(manifest, eltStates); + finalizeEntityLocationTable(manifest, eltStates); }; const rehydrateComponentIds = (manifest: Manifest): void => { @@ -739,43 +725,59 @@ export const createIncrementalPersistenceService = async ( * Mutable in-memory snapshot of the entity-location table, used * during load() before journal replay. */ + // One EltState per persistent quadrant, indexed by per-quadrant local index + // (Entity.toLocalIndex), mirroring that quadrant's on-disk file. Two Int32 + // per slot: [archetypeId, rowIndex]. interface EltState { entities: Int32Array; capacity: number; - nextIndex: number; + nextSlot: number; freeListHead: number; } + type EltStates = Map; - const readEntityLocationSnapshot = async (): Promise => { + const readEntityLocationSnapshot = async (quadrant: number): Promise => { const reply = await transport.request({ id: allocOpId(), kind: "readFile", - path: ENTITY_LOCATION_FILE, + path: entityLocationFile(quadrant), }); const eltBytes = new Uint8Array(reply.bytes); - const nextIndex = Math.floor(eltBytes.byteLength / ELT_STRIDE); + const nextSlot = Math.floor(eltBytes.byteLength / ELT_STRIDE); let capacity = 16; - while (capacity < Math.max(nextIndex, 16)) capacity *= 2; + while (capacity < Math.max(nextSlot, 16)) capacity *= 2; const entities = new Int32Array(new ArrayBuffer(capacity * 2 * 4)); const view = new DataView(eltBytes.buffer, eltBytes.byteOffset, eltBytes.byteLength); let freeListHead = -1; - for (let entity = 0; entity < nextIndex; entity++) { - const offset = entity * ELT_STRIDE; - const archetypeId = view.getInt32(offset + 0, true); + // Each 8-byte record is one local slot, in the same order the router + // wrote them (offset = localIndex * ELT_STRIDE), so this is a straight scan. + for (let slot = 0; slot < nextSlot; slot++) { + const offset = slot * ELT_STRIDE; + // Stored archetype is biased by +1: 0 = empty (never written or + // deleted), N+1 = archetype N. + const storedArchetype = view.getUint32(offset + 0, true); const rowIndex = view.getInt32(offset + 4, true); - if (archetypeId === -1) { - entities[entity * 2 + 0] = -1; - entities[entity * 2 + 1] = freeListHead; - freeListHead = entity; + if (storedArchetype === 0) { + entities[slot * 2 + 0] = -1; + entities[slot * 2 + 1] = freeListHead; + freeListHead = slot; } else { - entities[entity * 2 + 0] = archetypeId; - entities[entity * 2 + 1] = rowIndex; + entities[slot * 2 + 0] = storedArchetype - 1; + entities[slot * 2 + 1] = rowIndex; } } - return { entities, capacity, nextIndex, freeListHead }; + return { entities, capacity, nextSlot, freeListHead }; + }; + + const readEntityLocationSnapshots = async (): Promise => { + const states: EltStates = new Map(); + for (const quadrant of Entity.persistentQuadrants) { + states.set(quadrant, await readEntityLocationSnapshot(quadrant)); + } + return states; }; /** @@ -790,7 +792,7 @@ export const createIncrementalPersistenceService = async ( * emitted post-load entries don't recycle ids and collide on a * subsequent crash-recovery pass. */ - const replayJournal = async (manifest: Manifest, eltState: EltState): Promise => { + const replayJournal = async (manifest: Manifest, eltStates: EltStates): Promise => { const reply = await transport.request({ id: allocOpId(), kind: "readFile", @@ -806,7 +808,7 @@ export const createIncrementalPersistenceService = async ( const flushIfMatch = (commitTxId: number): void => { if (bufferedTxId === commitTxId) { - for (const e of buffered) applyJournalEntry(manifest, eltState, e); + for (const e of buffered) applyJournalEntry(manifest, eltStates, e); } buffered = []; bufferedTxId = null; @@ -834,64 +836,69 @@ export const createIncrementalPersistenceService = async ( if (maxTxId >= nextTxId) nextTxId = maxTxId + 1; }; - const ensureEntityCapacity = (eltState: EltState, entity: number): void => { - // Grow `entities` to fit `entity` if needed. The persistent + const ensureEntityCapacity = (eltState: EltState, slot: number): void => { + // Grow `entities` to fit `slot` if needed. The persistent // location table grows by *2; mirror that policy so any later // round-trip through fromData matches what create() would do. - while (entity >= eltState.capacity) { + while (slot >= eltState.capacity) { const grown = new Int32Array(new ArrayBuffer(eltState.capacity * 2 * 2 * 4)); grown.set(eltState.entities); eltState.entities = grown; eltState.capacity *= 2; } - if (entity >= eltState.nextIndex) { + if (slot >= eltState.nextSlot) { // Any slots between the previous high-water-mark and this - // entity were never allocated and become free-list entries. - for (let slot = eltState.nextIndex; slot < entity; slot++) { - eltState.entities[slot * 2 + 0] = -1; - eltState.entities[slot * 2 + 1] = eltState.freeListHead; - eltState.freeListHead = slot; + // slot were never allocated and become free-list entries. + for (let s = eltState.nextSlot; s < slot; s++) { + eltState.entities[s * 2 + 0] = -1; + eltState.entities[s * 2 + 1] = eltState.freeListHead; + eltState.freeListHead = s; } - eltState.nextIndex = entity + 1; + eltState.nextSlot = slot + 1; } }; - const applyJournalEntry = (manifest: Manifest, eltState: EltState, entry: JournalEntry): void => { + const applyJournalEntry = (manifest: Manifest, states: EltStates, entry: JournalEntry): void => { if (entry.kind === "commit") return; // tx-end markers carry no state to apply - if (entry.entity < 0) return; // non-persistent / sentinel — should not appear + if (Entity.isNonPersistent(entry.entity)) return; // non-persistent — should not appear + + // Route to the entity's quadrant state, indexed by its local index. + const eltState = states.get(Entity.quadrantOf(entry.entity)); + if (eltState === undefined) return; + const slot = Entity.toLocalIndex(entry.entity); if (entry.kind === "delete") { - ensureEntityCapacity(eltState, entry.entity); - const prevArch = eltState.entities[entry.entity * 2 + 0]!; + ensureEntityCapacity(eltState, slot); + const prevArch = eltState.entities[slot * 2 + 0]!; if (prevArch >= 0) { - eltState.entities[entry.entity * 2 + 0] = -1; - eltState.entities[entry.entity * 2 + 1] = eltState.freeListHead; - eltState.freeListHead = entry.entity; + eltState.entities[slot * 2 + 0] = -1; + eltState.entities[slot * 2 + 1] = eltState.freeListHead; + eltState.freeListHead = slot; } return; } // insert / update / migrate — set the location and re-apply // the component bytes. - ensureEntityCapacity(eltState, entry.entity); - const prevArch = eltState.entities[entry.entity * 2 + 0]!; - eltState.entities[entry.entity * 2 + 0] = entry.archetypeId; - eltState.entities[entry.entity * 2 + 1] = entry.rowIndex; + ensureEntityCapacity(eltState, slot); + const prevArch = eltState.entities[slot * 2 + 0]!; + eltState.entities[slot * 2 + 0] = entry.archetypeId; + eltState.entities[slot * 2 + 1] = entry.rowIndex; if (prevArch === -1) { - // Entity is moving out of the free list. Splice it out so + // Slot is moving out of the free list. Splice it out so // the free chain stays consistent. The list is short in // practice (bounded by the number of holes since the last // checkpoint), so a linear scan is fine. - if (eltState.freeListHead === entry.entity) { - eltState.freeListHead = eltState.entities[entry.entity * 2 + 1]!; - eltState.entities[entry.entity * 2 + 1] = entry.rowIndex; + if (eltState.freeListHead === slot) { + eltState.freeListHead = eltState.entities[slot * 2 + 1]!; + eltState.entities[slot * 2 + 1] = entry.rowIndex; } else { let cursor = eltState.freeListHead; while (cursor !== -1) { const next = eltState.entities[cursor * 2 + 1]!; - if (next === entry.entity) { - eltState.entities[cursor * 2 + 1] = eltState.entities[entry.entity * 2 + 1]!; - eltState.entities[entry.entity * 2 + 1] = entry.rowIndex; + if (next === slot) { + eltState.entities[cursor * 2 + 1] = eltState.entities[slot * 2 + 1]!; + eltState.entities[slot * 2 + 1] = entry.rowIndex; break; } cursor = next; @@ -964,27 +971,41 @@ export const createIncrementalPersistenceService = async ( * column from the final ELT, then hand the rebuilt table back * through `store.fromData`. */ - const finalizeEntityLocationTable = (manifest: Manifest, eltState: EltState): void => { - const { entities, nextIndex, capacity, freeListHead } = eltState; - for (let entity = 0; entity < nextIndex; entity++) { - const archetypeId = entities[entity * 2 + 0]; - if (archetypeId === undefined || archetypeId < 0) continue; - const rowIndex = entities[entity * 2 + 1]!; - const aMan = manifest.archetypes[archetypeId]; - if (aMan === undefined) continue; - const liveArchetype = store.archetypes[aMan.name]; - if (liveArchetype === undefined) continue; - const idColumn = getIdColumn(liveArchetype); - if (idColumn === undefined) continue; - idColumn.set(rowIndex, entity); + const finalizeEntityLocationTable = (manifest: Manifest, eltStates: EltStates): void => { + // Rebuild each archetype's implicit `id` column and collect a flat list + // of live persistent entity locations. Each quadrant's state is indexed + // by local index, so recover the entity id from (localIndex, quadrant). + // The ECS buckets these into its quadrant-partitioned location tables. + const locations: EntityLocationEntry[] = []; + for (const [quadrant, eltState] of eltStates) { + const { entities, nextSlot } = eltState; + for (let slot = 0; slot < nextSlot; slot++) { + const archetypeId = entities[slot * 2 + 0]; + if (archetypeId === undefined || archetypeId < 0) continue; + const rowIndex = entities[slot * 2 + 1]!; + const aMan = manifest.archetypes[archetypeId]; + if (aMan === undefined) continue; + const liveArchetype = store.archetypes[aMan.name]; + if (liveArchetype === undefined) continue; + const idColumn = getIdColumn(liveArchetype); + if (idColumn === undefined) continue; + const entity = Entity.toEntity(slot, quadrant); + idColumn.set(rowIndex, entity); + locations.push({ entity, archetype: archetypeId, row: rowIndex }); + } } store.fromData({ version: ECS_SNAPSHOT_VERSION, componentSchemas: {}, - entityLocationTableData: { entities, freeListHead, nextIndex, capacity }, + entityLocationTables: serializedEntityLocationTables(locations), archetypesData: [], }); + + // nonPersistent-schema columns were never checkpointed/journaled, so the + // freshly-restored archetypes have them empty: reset defaulted ones to + // their default and strip no-default ones. + store.reconstructNonPersistentColumns(); }; const dispose = async (): Promise => { diff --git a/packages/data-persistence/src/service/create-worker-persistence-service.test.ts b/packages/data-persistence/src/service/create-worker-persistence-service.test.ts index be2b4275..519b8abf 100644 --- a/packages/data-persistence/src/service/create-worker-persistence-service.test.ts +++ b/packages/data-persistence/src/service/create-worker-persistence-service.test.ts @@ -53,8 +53,8 @@ describe("createWorkerPersistenceService", () => { const archetypeDirs = await backend.list("archetypes"); expect(archetypeDirs.length).toBe(1); - // Entity-location.bin records the new entity at row 0. - const eltFile = await backend.open("entity-location.bin"); + // The document-quadrant entity-location file records the new entity at row 0. + const eltFile = await backend.open("entity-location-0.bin"); const eltSize = await eltFile.size(); expect(eltSize).toBeGreaterThan(0); await eltFile.close(); diff --git a/packages/data-persistence/src/service/entity-location-cache.ts b/packages/data-persistence/src/service/entity-location-cache.ts deleted file mode 100644 index 5b53cafa..00000000 --- a/packages/data-persistence/src/service/entity-location-cache.ts +++ /dev/null @@ -1,83 +0,0 @@ -// © 2026 Adobe. MIT License. See /LICENSE for details. - -/** - * In-memory mirror of every persisted entity's last-known on-disk - * location, keyed by entity id. Used by the persistence service to - * detect swap-remove side effects (an unrelated entity moving into - * a vacated row) and to decide whether a per-component update path - * can be taken vs. a full row write. - * - * Implementation: two `Uint32Array` slots per entity (archetype id, - * row), packed flat and indexed directly by entity id. This avoids - * a `Map` — both a hash lookup and a per-entry - * object allocation per change. - * - * The sentinel 0xFFFFFFFF means "absent". Archetype ids are u16 in - * the on-disk manifest so the sentinel never collides with a real - * archetype id. - * - * Backing buffer grows geometrically when the entity id outruns the - * current capacity. Shrinking is intentionally not supported; the - * worst case is bounded by the high-water-mark entity id. - */ - -const SENTINEL = 0xffff_ffff; -const SLOTS_PER_ENTITY = 2; -const INITIAL_CAPACITY_ENTITIES = 256; - -// Hidden-class friendly: assign every field once in the constructor, -// in the same order, and don't mutate the shape. -class EntityLocationCache { - private buffer: Uint32Array; - private capacityEntities: number; - - constructor() { - this.capacityEntities = INITIAL_CAPACITY_ENTITIES; - this.buffer = new Uint32Array(this.capacityEntities * SLOTS_PER_ENTITY); - this.buffer.fill(SENTINEL); - } - - /** Returns the archetype id for `entity`, or -1 if absent. */ - getArchetypeId(entity: number): number { - if (entity < 0 || entity >= this.capacityEntities) return -1; - const v = this.buffer[entity * SLOTS_PER_ENTITY]!; - return v === SENTINEL ? -1 : v; - } - - /** Returns the row index for `entity`. Caller must guard with has(). */ - getRow(entity: number): number { - return this.buffer[entity * SLOTS_PER_ENTITY + 1]!; - } - - has(entity: number): boolean { - if (entity < 0 || entity >= this.capacityEntities) return false; - return this.buffer[entity * SLOTS_PER_ENTITY]! !== SENTINEL; - } - - set(entity: number, archetypeId: number, row: number): void { - if (entity < 0) return; - if (entity >= this.capacityEntities) this.grow(entity + 1); - const i = entity * SLOTS_PER_ENTITY; - this.buffer[i] = archetypeId; - this.buffer[i + 1] = row; - } - - delete(entity: number): void { - if (entity < 0 || entity >= this.capacityEntities) return; - this.buffer[entity * SLOTS_PER_ENTITY] = SENTINEL; - } - - private grow(neededEntities: number): void { - let nextCap = this.capacityEntities; - while (nextCap < neededEntities) nextCap *= 2; - const next = new Uint32Array(nextCap * SLOTS_PER_ENTITY); - next.fill(SENTINEL); - next.set(this.buffer); - this.buffer = next; - this.capacityEntities = nextCap; - } -} - -export const createEntityLocationCache = (): EntityLocationCache => new EntityLocationCache(); - -export type { EntityLocationCache }; diff --git a/packages/data-persistence/src/service/entity-location-density.test.ts b/packages/data-persistence/src/service/entity-location-density.test.ts new file mode 100644 index 00000000..266d06c9 --- /dev/null +++ b/packages/data-persistence/src/service/entity-location-density.test.ts @@ -0,0 +1,47 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. +// +// The on-disk entity-location file for each persistent quadrant is indexed by +// the entity's per-quadrant local index, so it packs with exactly one 8-byte +// slot per entity — no gaps from the quadrant bits in the entity id. + +import { Database } from "@adobe/data/ecs"; +import { F32 } from "@adobe/data/math"; +import { describe, expect, it } from "vitest"; +import { createMemoryBackend } from "../backend/memory-backend.js"; +import { createWorkerPersistenceService } from "./create-worker-persistence-service.js"; + +const plugin = Database.Plugin.create({ + components: { value: F32.schema }, + archetypes: { Item: ["value"] }, + transactions: { + spawn(t, v: number) { + return t.archetypes.Item.insert({ value: v }); + }, + }, +}); + +const ELT_STRIDE = 8; + +describe("entity-location file density", () => { + it("packs the document quadrant with exactly one slot per entity (no gaps)", async () => { + const backend = createMemoryBackend(); + const db = Database.create(plugin); + const svc = await createWorkerPersistenceService({ + database: db, + backend, + checkpoint: { everyNTransactions: 0, idleMs: 0 }, + }); + + const N = 50; + for (let i = 0; i < N; i++) db.transactions.spawn(i); + await svc.flush(); + + // Document entities get dense local indices 0..N-1 — the file is exactly + // N slots, not stretched out by the entity id's quadrant bits. + const eltFile = await backend.open("entity-location-0.bin"); + expect(await eltFile.size()).toBe(N * ELT_STRIDE); + await eltFile.close(); + + await svc.dispose(); + }); +}); diff --git a/packages/data-persistence/src/service/intermediate-relocation-durability.test.ts b/packages/data-persistence/src/service/intermediate-relocation-durability.test.ts new file mode 100644 index 00000000..8ea49ab1 --- /dev/null +++ b/packages/data-persistence/src/service/intermediate-relocation-durability.test.ts @@ -0,0 +1,105 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. +// +// Probe: does an INTERMEDIATE transaction that relocates a persistent +// entity (via swap-remove) desync the persisted image? +// +// An async-generator transaction applies each non-final yield as an +// INTERMEDIATE step, rolls it back, then commits the return value. The +// persistence service skips intermediate transactions wholesale. The +// yield below deletes entity A, which swap-moves the last row (C) into +// A's vacated row; the rollback then re-inserts A at a fresh row. So by +// the time the (persisted) commit runs, A and C sit at different +// physical rows than the last checkpoint saw — a relocation the +// persistence layer never observed as part of a persisted transaction. +// +// The commit does a value-only update of A. If persistence decides a +// partial (per-component) write is safe, it writes A's one column at +// A's *current* row while the on-disk image still has a different +// entity's bytes there → corruption of the neighbor. + +import { Database } from "@adobe/data/ecs"; +import { F32 } from "@adobe/data/math"; +import { beforeEach, describe, expect, it } from "vitest"; +import { createMemoryBackend } from "../backend/memory-backend.js"; +import { createWorkerPersistenceService } from "./create-worker-persistence-service.js"; + +type Arg = + | { kind: "kill"; entity: number } + | { kind: "set"; entity: number; value: number }; + +const plugin = Database.Plugin.create({ + components: { + value: F32.schema, + }, + archetypes: { + Item: ["value"], + }, + transactions: { + spawn(t, value: number) { + return t.archetypes.Item.insert({ value }); + }, + killOrSet(t, arg: Arg) { + if (arg.kind === "kill") { + t.delete(arg.entity); + } else { + t.update(arg.entity, { value: arg.value }); + } + }, + }, +}); + +const noCheckpoint = { everyNTransactions: 0, idleMs: 0 } as const; + +describe("intermediate relocation durability", () => { + let backend: ReturnType; + + beforeEach(() => { + backend = createMemoryBackend(); + }); + + it("keeps a swap-moved neighbor intact when a relocation happens inside an intermediate transaction", async () => { + const db1 = Database.create(plugin); + const svc1 = await createWorkerPersistenceService({ + database: db1, + backend, + checkpoint: noCheckpoint, + }); + + const a = db1.transactions.spawn(1)!; + const b = db1.transactions.spawn(2)!; + const c = db1.transactions.spawn(3)!; + await svc1.flush(); + await svc1.checkpoint(); + + // Intermediate delete of A (swap-moves C into A's row), rolled + // back, then a persisted commit that value-updates A. + db1.transactions.killOrSet( + () => + (async function* (): AsyncGenerator { + yield { kind: "kill", entity: a }; + return { kind: "set", entity: a, value: 99 }; + })(), + ); + await new Promise((resolve) => setTimeout(resolve, 25)); + await svc1.flush(); + await svc1.checkpoint(); + await svc1.dispose(); + + const db2 = Database.create(plugin); + const svc2 = await createWorkerPersistenceService({ + database: db2, + backend, + checkpoint: noCheckpoint, + }); + await svc2.load(); + await svc2.dispose(); + + const va = db2.read(a) as { value: number } | null; + const vb = db2.read(b) as { value: number } | null; + const vc = db2.read(c) as { value: number } | null; + + expect(va?.value).toBe(99); + expect(vb?.value).toBe(2); + expect(vc?.value).toBe(3); + }); +}); diff --git a/packages/data-persistence/src/service/journal-replay.test.ts b/packages/data-persistence/src/service/journal-replay.test.ts index f6f5b985..ec094379 100644 --- a/packages/data-persistence/src/service/journal-replay.test.ts +++ b/packages/data-persistence/src/service/journal-replay.test.ts @@ -8,7 +8,7 @@ import { createWorkerPersistenceService } from "./create-worker-persistence-serv /** * Journal-replay focused tests. These exercise paths where the - * snapshot (manifest + column files + entity-location.bin) is *not* + * snapshot (manifest + column files + entity-location files) is *not* * up to date and the journal must roll the database forward, plus * crash-recovery scenarios where the journal itself is partially * written or where the checkpoint was killed mid-step. diff --git a/packages/data-persistence/src/transport/inprocess-transport.test.ts b/packages/data-persistence/src/transport/inprocess-transport.test.ts index 3bd16df0..f673732b 100644 --- a/packages/data-persistence/src/transport/inprocess-transport.test.ts +++ b/packages/data-persistence/src/transport/inprocess-transport.test.ts @@ -49,7 +49,8 @@ describe("createInprocessTransport", () => { const off = transport.onMessage(msg => { if (msg.kind === "ack") acks.push({ id: msg.id, error: msg.error }); }); - transport.send({ id: 42, kind: "writeEntityLocation", entity: 1, archetypeId: 5, rowIndex: 3 }); + // entity 4 is persistent (quadrant 0); a non-persistent id would be rejected. + transport.send({ id: 42, kind: "writeEntityLocation", entity: 4, archetypeId: 5, rowIndex: 3 }); await new Promise(resolve => setTimeout(resolve, 0)); expect(acks).toEqual([{ id: 42, error: undefined }]); off(); diff --git a/packages/data-persistence/src/transport/router.ts b/packages/data-persistence/src/transport/router.ts index 3a3f710d..e480ec0b 100644 --- a/packages/data-persistence/src/transport/router.ts +++ b/packages/data-persistence/src/transport/router.ts @@ -1,16 +1,19 @@ // © 2026 Adobe. MIT License. See /LICENSE for details. +import { Entity } from "@adobe/data/ecs"; import type { PersistenceBackend } from "../backend/persistence-backend.js"; import type { RandomAccessFile } from "../backend/random-access-file.js"; import type { PersistOp } from "./transport.js"; -const ENTITY_LOCATION_FILE = "entity-location.bin"; +// One entity-location file per persistent quadrant (document = 0, settings = 2), +// each indexed by the entity's per-quadrant local index so it packs fully +// densely with no gaps. +const entityLocationFile = (quadrant: number): string => `entity-location-${quadrant}.bin`; const JOURNAL_FILE = "journal.bin"; const META_FILE = "meta.json"; -// Packed entity-location entry: u32 archetypeId (we reserve the high -// bit as "deleted") + u32 rowIndex. 8 bytes per entity, indexed by the -// absolute value of the entity id - 1 (entities are 1-indexed in the -// store). Deletions are recorded by setting all bytes to 0xff. +// Packed entity-location entry: u32 (archetypeId + 1, so 0 = empty) + u32 +// rowIndex. 8 bytes per persistent entity. An unwritten slot reads as the zero +// (empty) entry; deletions write a zero entry. const ELT_STRIDE = 8; const columnPath = (archetypeId: number, component: string): string => @@ -60,25 +63,28 @@ export const createPersistRouter = (backend: PersistenceBackend): PersistRouter return file.appendAt(new Uint8Array(op.bytes)); } case "writeEntityLocation": { - if (op.entity < 0) { + if (Entity.isNonPersistent(op.entity)) { throw new Error(`writeEntityLocation: non-persistent entity not persistable (entity=${op.entity})`); } - const file = await openCached(ENTITY_LOCATION_FILE); + const file = await openCached(entityLocationFile(Entity.quadrantOf(op.entity))); const entry = new ArrayBuffer(ELT_STRIDE); const view = new DataView(entry); - view.setUint32(0, op.archetypeId, true); + // Store archetypeId biased by +1 so a zero-filled (never-written) + // slot reads as "empty" rather than a valid archetype 0 (the + // backend zero-fills any gap left before the first write). + view.setUint32(0, op.archetypeId + 1, true); view.setUint32(4, op.rowIndex, true); - // Entities are 0-indexed; offset = entity * stride. - await file.writeAt(op.entity * ELT_STRIDE, new Uint8Array(entry)); + await file.writeAt(Entity.toLocalIndex(op.entity) * ELT_STRIDE, new Uint8Array(entry)); return undefined; } case "deleteEntityLocation": { - if (op.entity < 0) { + if (Entity.isNonPersistent(op.entity)) { throw new Error(`deleteEntityLocation: non-persistent entity not persistable (entity=${op.entity})`); } - const file = await openCached(ENTITY_LOCATION_FILE); - const tombstone = new Uint8Array(ELT_STRIDE).fill(0xff); - await file.writeAt(op.entity * ELT_STRIDE, tombstone); + const file = await openCached(entityLocationFile(Entity.quadrantOf(op.entity))); + // Zero entry = empty slot (biased archetype 0 means "absent"). + const tombstone = new Uint8Array(ELT_STRIDE); + await file.writeAt(Entity.toLocalIndex(op.entity) * ELT_STRIDE, tombstone); return undefined; } case "writeJournalSnapshot": { diff --git a/packages/data-react-hello/package.json b/packages/data-react-hello/package.json index 27ecffd9..9a6157a5 100644 --- a/packages/data-react-hello/package.json +++ b/packages/data-react-hello/package.json @@ -1,6 +1,6 @@ { "name": "data-react-hello", - "version": "0.9.88", + "version": "0.9.89", "description": "Hello World sample - click counter using @adobe/data-react", "type": "module", "private": true, diff --git a/packages/data-react-pixie/package.json b/packages/data-react-pixie/package.json index a42db880..45e54fc4 100644 --- a/packages/data-react-pixie/package.json +++ b/packages/data-react-pixie/package.json @@ -1,6 +1,6 @@ { "name": "data-react-pixie", - "version": "0.9.88", + "version": "0.9.89", "description": "PixiJS React sample - ECS sprites (bunny, fox) with @adobe/data-react", "type": "module", "private": true, diff --git a/packages/data-react/package.json b/packages/data-react/package.json index bb7d1d87..eeef9606 100644 --- a/packages/data-react/package.json +++ b/packages/data-react/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-react", - "version": "0.9.88", + "version": "0.9.89", "description": "Adobe data React bindings — hooks and context for ECS database", "type": "module", "private": false, diff --git a/packages/data-solid-dashboard/package.json b/packages/data-solid-dashboard/package.json index f86fd955..89a03ddf 100644 --- a/packages/data-solid-dashboard/package.json +++ b/packages/data-solid-dashboard/package.json @@ -1,6 +1,6 @@ { "name": "data-solid-dashboard", - "version": "0.9.88", + "version": "0.9.89", "description": "Mini dashboard sample — multiple components sharing one @adobe/data ECS database with SolidJS", "type": "module", "private": true, diff --git a/packages/data-solid/package.json b/packages/data-solid/package.json index 840287a0..ad39b29b 100644 --- a/packages/data-solid/package.json +++ b/packages/data-solid/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-solid", - "version": "0.9.88", + "version": "0.9.89", "description": "Adobe data SolidJS bindings — context and provider for ECS database", "type": "module", "private": false, diff --git a/packages/data-sync/package.json b/packages/data-sync/package.json index d5f07371..6f6dfd04 100644 --- a/packages/data-sync/package.json +++ b/packages/data-sync/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data-sync", - "version": "0.9.88", + "version": "0.9.89", "description": "Multi-user real-time synchronisation for @adobe/data ECS — server, client, and in-process loopback.", "type": "module", "sideEffects": false, diff --git a/packages/data-sync/src/create-sync-service.ts b/packages/data-sync/src/create-sync-service.ts index 5132f5e6..885ad13c 100644 --- a/packages/data-sync/src/create-sync-service.ts +++ b/packages/data-sync/src/create-sync-service.ts @@ -1,6 +1,6 @@ // © 2026 Adobe. MIT License. See /LICENSE for details. -import type { Database } from "@adobe/data/ecs"; +import { Entity, type Database } from "@adobe/data/ecs"; import type { ClientTransport, ServerMessage } from "./transport.js"; /** @@ -203,10 +203,36 @@ export const createSyncService = (options: SyncServiceOptions): SyncService => { } }); + // A component whose SCHEMA is marked nonShared is local to this client. + const isNonSharedComponent = (component: string): boolean => + (database.componentSchemas as Record)[component]?.nonShared === true; + + // Whether a transaction has any effect that must reach peers: it changed a + // shared entity through a shared (not nonShared) component. A transaction + // whose effects are entirely non-shared — only non-shared entities, or only + // nonShared components — stays local. NOTE: peers replay the WHOLE + // transaction, so a mixed shared/non-shared transaction cannot be partially + // stripped; keep non-shared mutations in their own transactions to keep them + // off the wire. + const hasSharedEffect = (result: { + readonly changedEntities: ReadonlyMap; + readonly changedComponents: ReadonlySet; + }): boolean => { + let sharedEntity = false; + for (const entity of result.changedEntities.keys()) { + if (Entity.isShared(entity)) { sharedEntity = true; break; } + } + if (!sharedEntity) return false; + for (const component of result.changedComponents) { + if (!isNonSharedComponent(component)) return true; + } + return false; + }; + const unsubscribeOutbound = database.observe.envelopes(({ envelope, result, intent }) => { // Skip envelopes whose only effect was on non-persistent entities/ - // resources — those stay local-only by design. - if (result !== undefined && !result.persistent) return; + // resources, or entirely on non-shared (client-local) entities/components. + if (result !== undefined && (!result.persistent || !hasSharedEffect(result))) return; if (intent === "commit") { log(`propose out (id=${envelope.id}, name=${envelope.name})`); diff --git a/packages/data-sync/src/sync.test.ts b/packages/data-sync/src/sync.test.ts index 8cc01c4a..be771353 100644 --- a/packages/data-sync/src/sync.test.ts +++ b/packages/data-sync/src/sync.test.ts @@ -29,6 +29,9 @@ const plugin = Database.Plugin.create({ // Local-only resource — never replicates because of `nonPersistent: true`. // (Verified by the "nonPersistent resource never replicates" test below.) bannerText: { default: "" as string, nonPersistent: true }, + // Local + durable resource — persisted locally, but never replicated to + // peers because of `nonShared: true`. (Verified by the nonShared test.) + localPref: { default: "" as string, nonShared: true }, }, archetypes: { Point: ["x", "y", "label"], @@ -46,6 +49,9 @@ const plugin = Database.Plugin.create({ setBanner(s, args: { text: string }) { s.resources.bannerText = args.text; }, + setLocalPref(s, args: { text: string }) { + s.resources.localPref = args.text; + }, }, }); @@ -224,6 +230,39 @@ describe("sync soundness", () => { server.dispose(); }); + // ----------------------------------------------------------------------- + // 5b. NonShared resource never replicates (but is still local + durable). + // The sync service skips envelopes with no shared effect. + // ----------------------------------------------------------------------- + + it("nonShared resource mutations are never replicated to peers", () => { + const server = createSyncServer(); + const { client: c1t, server: s1t } = createLoopbackTransport(); + const { client: c2t, server: s2t } = createLoopbackTransport(); + + const db1 = makeDb("peer1"); + const db2 = makeDb("peer2"); + + server.connect(s1t); + server.connect(s2t); + + const sync1 = createSyncService({ database: db1, transport: c1t }); + createSyncService({ database: db2, transport: c2t }); + + db1.transactions.setLocalPref({ text: "local to peer1" }); + + // Local DB sees the change; peer 2 must NOT. + expect(db1.resources.localPref).toBe("local to peer1"); + expect(db2.resources.localPref).toBe(""); + + // Sanity: a shared mutation in the same session DOES replicate. + db1.transactions.bumpScore({ delta: 3 }); + expect(db2.resources.score).toBe(3); + + sync1.dispose(); + server.dispose(); + }); + // ----------------------------------------------------------------------- // 6. (userId, id) compound key keeps two peers' independent local // counters from colliding on the wire. diff --git a/packages/data/package.json b/packages/data/package.json index 84a88be0..f3e9fb49 100644 --- a/packages/data/package.json +++ b/packages/data/package.json @@ -1,6 +1,6 @@ { "name": "@adobe/data", - "version": "0.9.88", + "version": "0.9.89", "description": "Adobe data oriented programming library", "type": "module", "sideEffects": false, diff --git a/packages/data/src/ecs/README.md b/packages/data/src/ecs/README.md index 51977ab7..bc7f5a4b 100644 --- a/packages/data/src/ecs/README.md +++ b/packages/data/src/ecs/README.md @@ -26,7 +26,7 @@ db.transactions.addPoints(10); ## Core Concepts -**Entity** — a unique integer ID. Persistent entities have positive IDs; nonPersistent entities have negative IDs. +**Entity** — a unique integer ID. Its low 2 bits encode a quadrant (persistence × sharing): bit 0 set = nonPersistent, bit 1 set = nonShared. Use `Entity.isPersistent` / `Entity.isShared` (etc.) rather than reading the bits directly. **Component** — a named data column. Each component has a schema that describes its type. Numeric schemas (F32, Vec3, etc.) are stored in tightly packed typed arrays for cache-friendly performance. @@ -668,7 +668,9 @@ const data = db.toData(); db.fromData(data); ``` -nonPersistent entities, and components/resources whose schema is marked `nonPersistent: true`, are excluded from serialization. +**nonPersistent** — nonPersistent entities are excluded from serialization entirely. An individual **component/resource** whose schema is marked `nonPersistent: true` is also never serialized; on load its value is reconstructed: if the schema has a real (non-`null`/`undefined`) `default` it is reset to that default, otherwise the component is stripped (the entity restores into the reduced archetype, so a system can re-add it on demand). A `default` of `null`/`undefined` — the type-only-placeholder pattern for opaque runtime types like `{ default: null as unknown as GPUBuffer }` — counts as no default and strips. + +**nonShared** — a component/resource marked `nonShared: true` is still persisted locally but is not replicated to peers by `@adobe/data-sync`. A transaction whose effects are entirely non-shared (only non-shared entities, or only nonShared components) is never sent. Note: peers replay whole transactions, so a transaction that mixes shared and non-shared mutations cannot be partially stripped — keep non-shared mutations in their own transactions to keep them local. ## Type Utilities @@ -688,11 +690,15 @@ The ECS separates two orthogonal ideas. **nonPersistent** means *not persisted* ### nonPersistent Component -A built-in optional component that can only be set at entity creation time. It cannot be added to or removed from an existing entity. Entities created with this component are allocated negative IDs and stored in a separate entity table that is never serialized. +A built-in optional component that can only be set at entity creation time. It cannot be added to or removed from an existing entity. Entities created with this component are allocated ids in a non-persistent quadrant, backed by a separate entity table that is never serialized. ### nonPersistent Entities -Entities created with the `nonPersistent` component. They always have negative IDs and are never persisted. Use them for session-only or UI-local state (selections, hover states, panel positions, etc.). +Entities created with the `nonPersistent` component (`Entity.isNonPersistent` is true). They are never persisted. Use them for session-only or UI-local state (selections, hover states, panel positions, etc.). + +### nonShared Component / Entities + +The sharing counterpart of `nonPersistent`, also creation-only. `nonShared` marks state as local to this client (never replicated to peers), orthogonal to durability. Together the two flags select one of four quadrants — document (shared+persistent), settings (nonShared+persistent), presence (shared+nonPersistent), session (nonShared+nonPersistent) — each with its own entity-id space. Query with `Entity.isShared` / `Entity.isNonShared`. ### nonPersistent Schema @@ -704,7 +710,7 @@ resources: { }, ``` -Setting `nonPersistent: true` on a **resource** schema also places that resource's singleton entity in the negative-ID space (it gets the `nonPersistent` component), so the resource resets to its default on load. +Setting `nonPersistent: true` on a **resource** schema also places that resource's singleton entity in a non-persistent entity-id quadrant (it gets the `nonPersistent` component), so the resource resets to its default on load. ### Intermediate Transaction diff --git a/packages/data/src/ecs/archetype/archetype.ts b/packages/data/src/ecs/archetype/archetype.ts index db307e2f..dbd60163 100644 --- a/packages/data/src/ecs/archetype/archetype.ts +++ b/packages/data/src/ecs/archetype/archetype.ts @@ -31,15 +31,19 @@ export interface ReadonlyArchetype extends BaseArc * detached (`.copy()`) so the snapshot survives later mutation of the live * archetype; otherwise the snapshot references the live column buffers * (faster, but only valid until the next mutation). + * + * `omit` names columns to exclude from the snapshot (e.g. nonPersistent + * components). `fromData` rebuilds any omitted column fresh, so the archetype + * stays structurally intact. */ - toData: (copy?: boolean) => unknown + toData: (copy?: boolean, omit?: ReadonlySet) => unknown } export interface Archetype extends BaseArchetype, Table { readonly components: ComponentSet>; insert: >(rowData: Exact, T>) => Entity; /** See {@link ReadonlyArchetype.toData}. */ - toData: (copy?: boolean) => unknown + toData: (copy?: boolean, omit?: ReadonlySet) => unknown fromData: (data: unknown) => void } diff --git a/packages/data/src/ecs/archetype/create-archetype.test.ts b/packages/data/src/ecs/archetype/create-archetype.test.ts index 45765eaa..8d0f2eb2 100644 --- a/packages/data/src/ecs/archetype/create-archetype.test.ts +++ b/packages/data/src/ecs/archetype/create-archetype.test.ts @@ -43,9 +43,9 @@ describe('createArchetype', () => { // Create second entity const entity2 = archetype.insert({ value: 100 }); - expect(entity2).toBe(1); // Second entity should have id 1 + expect(entity2).toBe(4); // Second entity: local index 1 in quadrant 0 → id 4 expect(archetype.rowCount).toBe(2); - expect(archetype.columns.id.get(1)).toBe(1); + expect(archetype.columns.id.get(1)).toBe(entity2); expect(archetype.columns.value.get(1)).toBe(100); // Verify entity locations in EntityLocationTable @@ -129,7 +129,7 @@ describe('createArchetype', () => { expect(newArchetype.columns.mana.get(0)).toBe(50); expect(newArchetype.columns.mana.get(1)).toBe(25); expect(newArchetype.columns.id.get(0)).toBe(0); - expect(newArchetype.columns.id.get(1)).toBe(1); + expect(newArchetype.columns.id.get(1)).toBe(4); }); it('should preserve component set during serialization/deserialization', () => { diff --git a/packages/data/src/ecs/archetype/create-archetype.ts b/packages/data/src/ecs/archetype/create-archetype.ts index 70b69737..b5f63788 100644 --- a/packages/data/src/ecs/archetype/create-archetype.ts +++ b/packages/data/src/ecs/archetype/create-archetype.ts @@ -8,7 +8,7 @@ import { EntityLocationTable } from "../entity-location-table/entity-location-ta import { Entity } from "../entity/entity.js"; import { StringKeyof } from "../../types/types.js"; import { ensureCapacity } from "../../table/ensure-capacity.js"; -import { TypedBuffer } from "../../typed-buffer/index.js"; +import { TypedBuffer, createTypedBuffer } from "../../typed-buffer/index.js"; // ─────────────────────────────────────────────────────────────────────────── // Specialized insert function per archetype. @@ -192,21 +192,32 @@ export const createArchetype = ( ...table, components: componentSet as Set>, insert: createEntity, - toData: (copy = false) => ({ - columns: copy - ? Object.fromEntries( - Object.entries(archetype.columns as Record unknown }>) - .map(([name, column]) => [name, column.copy()]), - ) - : archetype.columns, + toData: (copy = false, omit?: ReadonlySet) => ({ + columns: Object.fromEntries( + Object.entries(archetype.columns as Record>) + .filter(([name]) => omit === undefined || !omit.has(name)) + .map(([name, column]) => [name, copy ? column.copy() : column]), + ), rowCount: archetype.rowCount, rowCapacity: archetype.rowCapacity, }), fromData: (data: unknown) => { + // Capture the current column schemas before Object.assign replaces the + // whole `columns` object. Any component omitted from `data` (a + // nonPersistent column that wasn't serialized) is then rebuilt fresh + // rather than left missing — keeping the archetype structurally intact. + const columnSchemas: Record = {}; + for (const name in archetype.columns) { + columnSchemas[name] = (archetype.columns as Record>)[name]!.schema; + } Object.assign(archetype, data); - // Restoring archetype state replaces archetype.columns with the - // deserialized columns object. Rebuild insertImpl so the baked - // column refs match the new live columns. + for (const name of archetype.components) { + if (!(name in archetype.columns)) { + (archetype.columns as Record>)[name] = + createTypedBuffer(columnSchemas[name]!, archetype.rowCapacity); + } + } + // Rebuild insertImpl so the baked column refs match the new live columns. refreshInsertImpl(); // component set cannot be changed by this as the archetype components should be the same. } diff --git a/packages/data/src/ecs/archetype/delete-row.test.ts b/packages/data/src/ecs/archetype/delete-row.test.ts index 7d641f14..1d545dc7 100644 --- a/packages/data/src/ecs/archetype/delete-row.test.ts +++ b/packages/data/src/ecs/archetype/delete-row.test.ts @@ -22,8 +22,8 @@ describe('Archetype_deleteRow', () => { const entity2 = archetype.insert({ value: 200 }); const entity3 = archetype.insert({ value: 300 }); - // Delete the middle entity (entity2) - deleteRow(archetype, entity2, entityLocationTable); + // Delete the middle entity (entity2), which lives at row index 1 + deleteRow(archetype, 1, entityLocationTable); // Verify entity3 was moved to position 1 expect(archetype.columns.id.get(1)).toBe(entity3); @@ -56,8 +56,8 @@ describe('Archetype_deleteRow', () => { const entity1 = archetype.insert({ value: 100 }); const entity2 = archetype.insert({ value: 200 }); - // Delete the last entity (entity2) - deleteRow(archetype, entity2, entityLocationTable); + // Delete the last entity (entity2), which lives at row index 1 + deleteRow(archetype, 1, entityLocationTable); // Verify total rows decreased expect(archetype.rowCount).toBe(1); diff --git a/packages/data/src/ecs/archetype/delete-row.ts b/packages/data/src/ecs/archetype/delete-row.ts index 856a032d..ad51df5b 100644 --- a/packages/data/src/ecs/archetype/delete-row.ts +++ b/packages/data/src/ecs/archetype/delete-row.ts @@ -3,15 +3,22 @@ import * as TABLE from "../../table/index.js"; import { Archetype } from "./archetype.js"; import { RequiredComponents } from "../required-components.js"; import { EntityLocationTable } from "../entity-location-table/entity-location-table.js"; +import { Entity } from "../entity/entity.js"; /** * Deletes a row from the archetype and updates the entity location table for any row which may have been moved into it's position. * Does NOT modify the deleted row's entity location. + * + * Returns the entity that was swap-moved into the vacated row, or `undefined` + * when the deleted row was the last row (no move). Callers surface this so + * observers/persistence learn about the relocation the swap caused. */ -export const deleteRow = (archetype: Archetype, row: number, entityLocationTable: EntityLocationTable): void => { +export const deleteRow = (archetype: Archetype, row: number, entityLocationTable: EntityLocationTable): Entity | undefined => { const movedARowToFillHole = TABLE.deleteRow(archetype, row); if (movedARowToFillHole) { const movedId = archetype.columns.id.get(row); entityLocationTable.update(movedId, { archetype: archetype.id, row }); + return movedId; } + return undefined; } diff --git a/packages/data/src/ecs/database/concurrency/README.md b/packages/data/src/ecs/database/concurrency/README.md index 5faf37b6..a67ff658 100644 --- a/packages/data/src/ecs/database/concurrency/README.md +++ b/packages/data/src/ecs/database/concurrency/README.md @@ -130,7 +130,7 @@ The two built-ins are the reference implementations and bracket the design space `db.toData()` rolls back, serializes, and replays so a snapshot excludes in-flight transients. The snapshot is otherwise backed by **live** store buffers, so the replay would corrupt it — therefore, when `onAfterToData` is present, the -database serializes a **detached copy** (`store.toData(true)`, which clones +database serializes a **detached copy** (`store.toData({ copy: true })`, which clones columns and the entity table) between `onBeforeToData` and `onAfterToData`. If your strategy keeps a pending buffer: diff --git a/packages/data/src/ecs/database/database.ts b/packages/data/src/ecs/database/database.ts index 15948661..3a93085b 100644 --- a/packages/data/src/ecs/database/database.ts +++ b/packages/data/src/ecs/database/database.ts @@ -7,6 +7,7 @@ import { Entity } from "../entity/entity.js"; import { EntityReadValues } from "../store/core/index.js"; import { Observe } from "../../observe/index.js"; import { TransactionResult } from "./transactional-store/index.js"; +import { PersistenceScope } from "../persistence-scope.js"; import { TransactionEnvelope } from "./reconciling/reconciling-database.js"; import { StringKeyof, RemoveIndex } from "../../types/types.js"; import { Components } from "../store/components.js"; @@ -258,8 +259,8 @@ export interface Database< * in-flight transients into persisted state. The old reconciler tests missed * it because they only asserted the snapshot was truthy, never round-tripped it. */ - toData(): unknown - fromData(data: unknown): void + toData(options?: { readonly scope?: PersistenceScope }): unknown + fromData(data: unknown, scope?: PersistenceScope): void extend

(plugin: P): Database< C & FromSchemas>, R & FromSchemas>, diff --git a/packages/data/src/ecs/database/observed/create-observed-database.ts b/packages/data/src/ecs/database/observed/create-observed-database.ts index 3ab88061..a4eaa9a8 100644 --- a/packages/data/src/ecs/database/observed/create-observed-database.ts +++ b/packages/data/src/ecs/database/observed/create-observed-database.ts @@ -9,6 +9,7 @@ import { ArchetypeComponents } from "../../store/archetype-components.js"; import { Archetype, ArchetypeId, ReadonlyArchetype } from "../../archetype/index.js"; import { Store } from "../../store/index.js"; import { TransactionResult } from "../transactional-store/index.js"; +import { PersistenceScope, ToDataOptions } from "../../persistence-scope.js"; import { observeSelectEntities } from "../observe-select-entities.js"; import { createDerive } from "../observe-derive.js"; import { createTransactionalStore } from "../transactional-store/create-transactional-store.js"; @@ -116,10 +117,11 @@ export function createObservedDatabase< const observeComponent = mapEntries(store.componentSchemas, ([component]) => addToMapSet(component, componentObservers)); const resourceArchetypeComponents = (resource: string): StringKeyof[] => { - const isNonPersistent = (store.componentSchemas as any)[resource]?.nonPersistent; - return isNonPersistent - ? ["id" as StringKeyof, resource as unknown as StringKeyof, "nonPersistent" as StringKeyof] - : ["id" as StringKeyof, resource as unknown as StringKeyof]; + const schema = (store.componentSchemas as any)[resource]; + const names: StringKeyof[] = ["id" as StringKeyof, resource as unknown as StringKeyof]; + if (schema?.nonPersistent) names.push("nonPersistent" as StringKeyof); + if (schema?.nonShared) names.push("nonShared" as StringKeyof); + return names; }; const observeResource = Object.fromEntries( @@ -168,6 +170,9 @@ export function createObservedDatabase< undo: [], redo: [], undoable: null, + // A full-store reload is not a swap/migration side effect; every + // present entity is already re-materialized via changedEntities. + relocatedEntities: new Set(), }; notifyObservers(notifyResult); }; @@ -182,9 +187,9 @@ export function createObservedDatabase< store.reset(); notifyAllObserversStoreReloaded(); }, - toData: (copy = false) => store.toData(copy), - fromData: (data: unknown) => { - store.fromData(data); + toData: (options?: ToDataOptions) => store.toData(options), + fromData: (data: unknown, scope?: PersistenceScope) => { + store.fromData(data, scope); notifyAllObserversStoreReloaded(); }, extend: (plugin: any) => { diff --git a/packages/data/src/ecs/database/observed/observed-database.ts b/packages/data/src/ecs/database/observed/observed-database.ts index 0313e891..764fed1b 100644 --- a/packages/data/src/ecs/database/observed/observed-database.ts +++ b/packages/data/src/ecs/database/observed/observed-database.ts @@ -13,6 +13,7 @@ import { Entity } from "../../entity/entity.js"; import { EntityReadValues } from "../../store/core/index.js"; import { Database } from "../database.js"; import { FromSchemas } from "../../../schema/from-schemas.js"; +import { PersistenceScope, ToDataOptions } from "../../persistence-scope.js"; export interface ObservedDatabase< C extends Components, R extends ResourceComponents, @@ -43,8 +44,8 @@ export interface ObservedDatabase< ): Observe; readonly execute: (handler: (ctx: Store) => Entity | void, options?: { transient?: boolean; userId?: number | string }) => TransactionResult; readonly reset: () => void; - readonly toData: (copy?: boolean) => unknown; - readonly fromData: (data: unknown) => void; + readonly toData: (options?: ToDataOptions) => unknown; + readonly fromData: (data: unknown, scope?: PersistenceScope) => void; readonly extend: < P extends Database.Plugin >( diff --git a/packages/data/src/ecs/database/public/create-database.ts b/packages/data/src/ecs/database/public/create-database.ts index 1cc92f6c..8d86d4ca 100644 --- a/packages/data/src/ecs/database/public/create-database.ts +++ b/packages/data/src/ecs/database/public/create-database.ts @@ -2,6 +2,7 @@ import { ReadonlyStore, Store } from "../../store/index.js"; import { Database, FromServiceFactories } from "../database.js"; +import { PersistenceScope } from "../../persistence-scope.js"; import { calculateSystemOrder } from "../calculate-system-order.js"; import { createTransactionDispatcher } from "./create-transaction-dispatcher.js"; import { observeSelectEntities } from "../observe-select-entities.js"; @@ -131,22 +132,23 @@ function createEmptyDatabase(concurrency: ConcurrencyStrategyFactory | undefined // `db.indexes.` and `t.indexes.` pointing at one source of // truth. - const toData = () => { + const toData = (options?: { readonly scope?: PersistenceScope }) => { + const scope = options?.scope; // Fast path: a strategy with no replay hook leaves the store untouched // after serialization, so a live-reference snapshot is safe. if (!strategy.onAfterToData) { - return observedDatabase.toData(); + return observedDatabase.toData({ copy: false, scope }); } // A replay strategy mutates the live buffers in `onAfterToData`, which // would corrupt a live-reference snapshot. Capture a detached copy of // the committed (rolled-back) state before replaying. strategy.onBeforeToData?.(); - const data = observedDatabase.toData(true); + const data = observedDatabase.toData({ copy: true, scope }); strategy.onAfterToData(); return data; }; - const fromData = (data: unknown) => { - observedDatabase.fromData(data); + const fromData = (data: unknown, scope?: PersistenceScope) => { + observedDatabase.fromData(data, scope); strategy.onAfterFromData?.(); }; diff --git a/packages/data/src/ecs/database/reconciling/create-reconciling-database.ts b/packages/data/src/ecs/database/reconciling/create-reconciling-database.ts index 931b0338..c33aa4af 100644 --- a/packages/data/src/ecs/database/reconciling/create-reconciling-database.ts +++ b/packages/data/src/ecs/database/reconciling/create-reconciling-database.ts @@ -53,7 +53,7 @@ export function createReconcilingDatabase< // Detach the snapshot from live buffers: replayAllTransients below // mutates them and would otherwise corrupt the returned snapshot. applier.rollbackAllTransients(); - const data = observedDatabase.toData(true); + const data = observedDatabase.toData({ copy: true }); applier.replayAllTransients(); return data; }, diff --git a/packages/data/src/ecs/database/transactional-store/coalesce-actions.test.ts b/packages/data/src/ecs/database/transactional-store/coalesce-actions.test.ts index 51569287..c206ff19 100644 --- a/packages/data/src/ecs/database/transactional-store/coalesce-actions.test.ts +++ b/packages/data/src/ecs/database/transactional-store/coalesce-actions.test.ts @@ -24,6 +24,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -36,6 +37,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; expect(shouldCoalesceTransactions(previous, current)).toBe(true); @@ -52,6 +54,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -64,6 +67,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; expect(shouldCoalesceTransactions(previous, current)).toBe(false); @@ -80,6 +84,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -92,6 +97,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; expect(shouldCoalesceTransactions(previous, current)).toBe(false); @@ -108,6 +114,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -120,6 +127,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; expect(shouldCoalesceTransactions(previous, current)).toBe(false); @@ -136,6 +144,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -148,6 +157,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; expect(shouldCoalesceTransactions(previous, current)).toBe(false); @@ -164,6 +174,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -176,6 +187,7 @@ describe("shouldCoalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; expect(shouldCoalesceTransactions(previous, current)).toBe(false); @@ -194,6 +206,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -206,6 +219,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { y: 2 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -236,6 +250,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -248,6 +263,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[456, { name: "test" }]]), changedComponents: new Set(["name"]), changedArchetypes: new Set([2]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -276,6 +292,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -288,6 +305,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map(), changedComponents: new Set(), changedArchetypes: new Set(), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -308,6 +326,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -320,6 +339,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { y: 2 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -348,6 +368,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -360,6 +381,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, null]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -386,6 +408,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -404,6 +427,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { y: 2 }, name: "test" }]]), changedComponents: new Set(["position", "name"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -435,6 +459,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -447,6 +472,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, null]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); @@ -483,6 +509,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[123, { position: { x: 1 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const current: TransactionResult = { @@ -495,6 +522,7 @@ describe("coalesceTransactions", () => { changedEntities: new Map([[456, { position: { y: 2 } }]]), changedComponents: new Set(["position"]), changedArchetypes: new Set([1]), + relocatedEntities: new Set(), }; const result = coalesceTransactions(previous, current); diff --git a/packages/data/src/ecs/database/transactional-store/coalesce-actions.ts b/packages/data/src/ecs/database/transactional-store/coalesce-actions.ts index fccbcb96..9c8b55ee 100644 --- a/packages/data/src/ecs/database/transactional-store/coalesce-actions.ts +++ b/packages/data/src/ecs/database/transactional-store/coalesce-actions.ts @@ -97,6 +97,11 @@ export function coalesceTransactions( combinedChangedArchetypes.add(archetype); } + const combinedRelocatedEntities = new Set(previous.relocatedEntities); + for (const entity of current.relocatedEntities) { + combinedRelocatedEntities.add(entity); + } + return { value: current.value, intermediate: current.intermediate, @@ -107,5 +112,6 @@ export function coalesceTransactions( changedEntities: combinedChangedEntities, changedComponents: combinedChangedComponents, changedArchetypes: combinedChangedArchetypes, + relocatedEntities: combinedRelocatedEntities, }; -} \ No newline at end of file +} \ No newline at end of file diff --git a/packages/data/src/ecs/database/transactional-store/create-transactional-store.test.ts b/packages/data/src/ecs/database/transactional-store/create-transactional-store.test.ts index ffd0d397..e16ca196 100644 --- a/packages/data/src/ecs/database/transactional-store/create-transactional-store.test.ts +++ b/packages/data/src/ecs/database/transactional-store/create-transactional-store.test.ts @@ -841,4 +841,64 @@ describe("createTransactionalStore", () => { expect(baseStore.read(entity)?.health).toBeUndefined(); }); }); + + describe("relocatedEntities (swap-move / migration signal)", () => { + const make = () => createTransactionalStore(Store.create({ + components: { value: F32.schema, tag: F32.schema }, + resources: {}, + archetypes: {}, + })); + + it("reports the swap-moved neighbor when a non-last entity is deleted", () => { + const store = make(); + let a: Entity, b: Entity, c: Entity; + store.execute((t) => { + const ar = t.ensureArchetype(["id", "value"]); + a = ar.insert({ value: 1 }); + b = ar.insert({ value: 2 }); + c = ar.insert({ value: 3 }); + }); + // Deleting row 0 (a) swap-moves the last row (c) into it. + const result = store.execute((t) => { t.delete(a!); }); + expect([...result.relocatedEntities]).toEqual([c!]); + expect(result.relocatedEntities.has(b!)).toBe(false); + }); + + it("reports none when the last entity is deleted (no swap)", () => { + const store = make(); + let b: Entity; + store.execute((t) => { + const ar = t.ensureArchetype(["id", "value"]); + ar.insert({ value: 1 }); + b = ar.insert({ value: 2 }); + }); + const result = store.execute((t) => { t.delete(b!); }); + expect(result.relocatedEntities.size).toBe(0); + }); + + it("reports the migrated entity and its old-archetype neighbor on a component-add migration", () => { + const store = make(); + let x: Entity, y: Entity; + store.execute((t) => { + const ar = t.ensureArchetype(["id", "value"]); + x = ar.insert({ value: 1 }); + y = ar.insert({ value: 2 }); + }); + // Adding `tag` migrates x out of ["id","value"]; its vacated row 0 + // is backfilled by y (the last row of the old archetype). + const result = store.execute((t) => { t.update(x!, { tag: 9 }); }); + expect(result.relocatedEntities.has(x!)).toBe(true); + expect(result.relocatedEntities.has(y!)).toBe(true); + }); + + it("reports none for a value-only update (no relocation)", () => { + const store = make(); + let x: Entity; + store.execute((t) => { + x = t.ensureArchetype(["id", "value"]).insert({ value: 1 }); + }); + const result = store.execute((t) => { t.update(x!, { value: 5 }); }); + expect(result.relocatedEntities.size).toBe(0); + }); + }); }); \ No newline at end of file diff --git a/packages/data/src/ecs/database/transactional-store/create-transactional-store.ts b/packages/data/src/ecs/database/transactional-store/create-transactional-store.ts index 2ebc1f85..463957f7 100644 --- a/packages/data/src/ecs/database/transactional-store/create-transactional-store.ts +++ b/packages/data/src/ecs/database/transactional-store/create-transactional-store.ts @@ -39,6 +39,12 @@ export function createTransactionalStore< entities: new Map | null>(), components: new Set(), archetypes: new Set(), + // Entities relocated to a new backing row as a SIDE EFFECT of this + // transaction (swap-moved neighbors of a delete/migration, and the + // migrated entity itself) — NOT the entities the transaction directly + // touched. Their columns are not covered by `entities`, so a consumer + // must full-write them. See TransactionResult.relocatedEntities. + moves: new Set(), }; // Wrap archetype creation to track operations @@ -77,7 +83,7 @@ export function createTransactionalStore< return wrappedArchetypes.get(archetype.id); }; - const updateEntity = (entity: Entity, values: EntityUpdateValues) => { + const updateEntity = (entity: Entity, values: EntityUpdateValues): Entity | undefined => { trackEntity(entity); const oldValues = store.read(entity); if (!oldValues) { @@ -111,19 +117,31 @@ export function createTransactionalStore< const redoValues = { ...values }; // Perform the actual update - store.update(entity, values as any); + const swapped = store.update(entity, values as any); + if (swapped !== undefined) { + // The update migrated `entity` to a new archetype; its old + // archetype swap-moved `swapped` into the vacated row. + changed.moves.add(swapped); + } // Check if archetype changed after update const newLocation = store.locate(entity); if (newLocation) { changed.archetypes.add(newLocation.archetype.id); + // A migration relocates `entity` itself to a fresh row whose other + // columns are carried over (not in `values`), so it must be + // full-written by consumers even though it is a "changed" entity. + if (!location || newLocation.archetype.id !== location.archetype.id || newLocation.row !== location.row) { + changed.moves.add(entity); + } } // Add operations with potential combining addUpdateOperationsMaybeCombineLast(undoOperationsInReverseOrder, redoOperations, entity, redoValues, replacedValues); + return swapped; }; - const deleteEntity = (entity: Entity) => { + const deleteEntity = (entity: Entity): Entity | undefined => { trackEntity(entity); const location = store.locate(entity); if (location) { @@ -141,18 +159,28 @@ export function createTransactionalStore< changed.components.add(key); } - store.delete(entity); + const swapped = store.delete(entity); + if (swapped !== undefined) { + // Deleting `entity` swap-moved `swapped` into the vacated row. + changed.moves.add(swapped); + } redoOperations.push({ type: "delete", entity }); undoOperationsInReverseOrder.push({ type: "insert", values: oldValuesWithoutId }); + return swapped; + }; + + const resourceComponentNames = (name: string): StringKeyof[] => { + const schema = (store.componentSchemas as any)[name]; + const names = ["id", name] as StringKeyof[]; + if (schema?.nonPersistent) names.push("nonPersistent" as StringKeyof); + if (schema?.nonShared) names.push("nonShared" as StringKeyof); + return names; }; const resources = {} as { [K in keyof R]: R[K] }; for (const name of Object.keys(store.resources)) { const resourceId = name as keyof C; - const isNonPersistent = (store.componentSchemas as any)[name]?.nonPersistent; - const componentNames = isNonPersistent - ? ["id", resourceId, "nonPersistent"] as StringKeyof[] - : ["id", resourceId] as StringKeyof[]; + const componentNames = resourceComponentNames(name); const archetype = store.ensureArchetype(componentNames); const entityId = archetype.columns.id.get(0); Object.defineProperty(resources, name, { @@ -202,6 +230,7 @@ export function createTransactionalStore< changed.entities.clear(); changed.components.clear(); changed.archetypes.clear(); + changed.moves.clear(); try { // Execute the transaction @@ -221,6 +250,7 @@ export function createTransactionalStore< changedEntities: new Map(changed.entities), changedComponents: new Set(changed.components), changedArchetypes: new Set(changed.archetypes), + relocatedEntities: new Set(changed.moves), }; return result; @@ -236,6 +266,7 @@ export function createTransactionalStore< changed.entities.clear(); changed.components.clear(); changed.archetypes.clear(); + changed.moves.clear(); wrappedArchetypes.clear(); } }; @@ -257,10 +288,7 @@ export function createTransactionalStore< for (const name of Object.keys(store.resources)) { if (!Object.hasOwn(resources, name)) { const resourceId = name as keyof C; - const isNonPersistent = (store.componentSchemas as any)[name]?.nonPersistent; - const componentNames = isNonPersistent - ? ["id", resourceId, "nonPersistent"] as StringKeyof[] - : ["id", resourceId] as StringKeyof[]; + const componentNames = resourceComponentNames(name); const archetype = store.ensureArchetype(componentNames); const entityId = archetype.columns.id.get(0); Object.defineProperty(resources, name, { diff --git a/packages/data/src/ecs/database/transactional-store/transactional-store.ts b/packages/data/src/ecs/database/transactional-store/transactional-store.ts index b56d284b..06317beb 100644 --- a/packages/data/src/ecs/database/transactional-store/transactional-store.ts +++ b/packages/data/src/ecs/database/transactional-store/transactional-store.ts @@ -76,12 +76,22 @@ export interface TransactionResult { readonly value: Entity | void; /** True when the transaction is a non-final intermediate operation within a sequence. */ readonly intermediate: boolean; - /** True when at least one changed entity is persistent (entity id >= 0). */ + /** True when at least one changed entity is persistent (see Entity.isPersistent). */ readonly persistent: boolean; readonly undoable: null | Undoable; readonly redo: TransactionWriteOperation[]; readonly undo: TransactionWriteOperation[]; readonly changedEntities: Map | null>; + /** + * Entities relocated to a new backing row as a SIDE EFFECT of this + * transaction — swap-moved neighbors of a delete/migration, plus the + * migrated entity itself. Distinct from `changedEntities` (the directly + * touched entities): a relocated entity's columns are freshly established + * at its new row and are NOT covered by `changedEntities`, so a consumer + * that mirrors row layout (e.g. persistence) must full-write it. Empty for + * transactions with no swap/migration. + */ + readonly relocatedEntities: Set; // Component names are always strings. Keeping this `Set` (rather // than `Set`) avoids widening to `string | number | // symbol` for a generic `C`, which otherwise makes `TransactionResult` diff --git a/packages/data/src/ecs/entity-location-table/create-entity-location-table.test.ts b/packages/data/src/ecs/entity-location-table/create-entity-location-table.test.ts index 5f605d5d..4e00df2a 100644 --- a/packages/data/src/ecs/entity-location-table/create-entity-location-table.test.ts +++ b/packages/data/src/ecs/entity-location-table/create-entity-location-table.test.ts @@ -4,16 +4,17 @@ import { createEntityLocationTable } from './create-entity-location-table.js'; import { Entity } from '../entity/entity.js'; describe('createEntityLocationTable', () => { - it('should create entities with increasing ids starting from 0', () => { + it('should create entities with quadrant-encoded ids (quadrant 0)', () => { const table = createEntityLocationTable(); const entity0 = table.create({ archetype: 1, row: 10 }); const entity1 = table.create({ archetype: 2, row: 20 }); const entity2 = table.create({ archetype: 3, row: 30 }); + // quadrant 0 packs local index n into id (n << 2): 0, 4, 8, … expect(entity0).toBe(0); - expect(entity1).toBe(1); - expect(entity2).toBe(2); + expect(entity1).toBe(4); + expect(entity2).toBe(8); }); it('should store and retrieve entity locations correctly', () => { @@ -37,7 +38,7 @@ describe('createEntityLocationTable', () => { // Create a new entity - should reuse entity1's id const entity3 = table.create({ archetype: 4, row: 40 }); - expect(entity3).toBe(1); + expect(entity3).toBe(entity1); // reuses entity1's freed slot (same id) // Verify the new entity's location const location = table.locate(entity3); @@ -60,20 +61,20 @@ describe('createEntityLocationTable', () => { // Recreate entities - should get IDs in reverse deletion order const newEntity1 = table.create({ archetype: 5, row: 50 }); - expect(newEntity1).toBe(0); // Should get entity0's id (last deleted) + expect(newEntity1).toBe(entity0); // Should get entity0's id (last deleted) expect(table.locate(newEntity1)).toEqual({ archetype: 5, row: 50 }); const newEntity2 = table.create({ archetype: 6, row: 60 }); - expect(newEntity2).toBe(2); // Should get entity2's id (second-to-last deleted) + expect(newEntity2).toBe(entity2); // Should get entity2's id (second-to-last deleted) expect(table.locate(newEntity2)).toEqual({ archetype: 6, row: 60 }); const newEntity3 = table.create({ archetype: 7, row: 70 }); - expect(newEntity3).toBe(1); // Should get entity1's id (first deleted) + expect(newEntity3).toBe(entity1); // Should get entity1's id (first deleted) expect(table.locate(newEntity3)).toEqual({ archetype: 7, row: 70 }); // Creating one more should create a new ID since free list is empty const newEntity4 = table.create({ archetype: 8, row: 80 }); - expect(newEntity4).toBe(4); // Should get a new ID + expect(newEntity4).toBe(entity3 + 4); // fresh id: next local index in quadrant 0 expect(table.locate(newEntity4)).toEqual({ archetype: 8, row: 80 }); }); @@ -100,16 +101,16 @@ describe('createEntityLocationTable', () => { for (let i = 0; i < initialCapacity + 5; i++) { const entity = table.create({ archetype: i, row: i * 10 }); entities.push(entity); - // Verify entity was created correctly - expect(entity).toBe(i); + // Verify entity was created in quadrant 0 and round-trips + expect(entity & 0b11).toBe(0); const location = table.locate(entity); expect(location).toEqual({ archetype: i, row: i * 10 }); } // Delete some entities that span across the initial capacity boundary - table.delete(initialCapacity - 1); - table.delete(initialCapacity); - table.delete(initialCapacity + 1); + table.delete(entities[initialCapacity - 1]); + table.delete(entities[initialCapacity]); + table.delete(entities[initialCapacity + 1]); // Create new entities and verify they reuse the deleted IDs const newEntity1 = table.create({ archetype: 100, row: 1000 }); @@ -117,9 +118,9 @@ describe('createEntityLocationTable', () => { const newEntity3 = table.create({ archetype: 102, row: 1020 }); // Verify the entities were reused in LIFO order - expect(newEntity1).toBe(initialCapacity + 1); - expect(newEntity2).toBe(initialCapacity); - expect(newEntity3).toBe(initialCapacity - 1); + expect(newEntity1).toBe(entities[initialCapacity + 1]); + expect(newEntity2).toBe(entities[initialCapacity]); + expect(newEntity3).toBe(entities[initialCapacity - 1]); // Verify their locations are correct expect(table.locate(newEntity1)).toEqual({ archetype: 100, row: 1000 }); @@ -148,26 +149,35 @@ describe('createEntityLocationTable', () => { expect(table.locate(entity2)).toEqual({ archetype: 4, row: 40 }); }); - it("should return null when locating entity with invalid id -1", () => { + it("should return null when locating an id that was never created", () => { const table = createEntityLocationTable(); - expect(() => table.locate(-1)).toThrow("locate entity must be >= 0"); + expect(table.locate(400)).toBeNull(); }); }); -describe('createNonPersistentEntityLocationTable', () => { - it('should create entities with increasing ids starting from -1', () => { - const table = createEntityLocationTable(16, true); +describe('quadrant-encoded entity location table', () => { + it('packs the quadrant into each id (quadrant 1)', () => { + const table = createEntityLocationTable(16, 1); const entity0 = table.create({ archetype: 1, row: 10 }); const entity1 = table.create({ archetype: 2, row: 20 }); const entity2 = table.create({ archetype: 3, row: 30 }); - expect(entity0).toBe(-1); - expect(entity1).toBe(-2); - expect(entity2).toBe(-3); + // quadrant 1 packs local index n into id ((n << 2) | 1): 1, 5, 9, … + expect(entity0).toBe(1); + expect(entity1).toBe(5); + expect(entity2).toBe(9); expect(table.locate(entity0)).toEqual({ archetype: 1, row: 10 }); expect(table.locate(entity1)).toEqual({ archetype: 2, row: 20 }); expect(table.locate(entity2)).toEqual({ archetype: 3, row: 30 }); }); + + it('keeps quadrant id-spaces disjoint under the & 0x3 mask', () => { + for (const q of [0, 1, 2, 3]) { + const table = createEntityLocationTable(16, q); + const e = table.create({ archetype: 1, row: 0 }); + expect(e & 0b11).toBe(q); + } + }); }); \ No newline at end of file diff --git a/packages/data/src/ecs/entity-location-table/create-entity-location-table.ts b/packages/data/src/ecs/entity-location-table/create-entity-location-table.ts index c86f2426..ef1a2d0b 100644 --- a/packages/data/src/ecs/entity-location-table/create-entity-location-table.ts +++ b/packages/data/src/ecs/entity-location-table/create-entity-location-table.ts @@ -3,25 +3,28 @@ import { resize } from "../../internal/array-buffer-like/resize.js"; import { EntityLocationTable } from "./entity-location-table.js"; import { EntityLocation } from "./entity-location.js"; import { Entity } from "../entity/entity.js"; +import { toEntity, toLocalIndex } from "../entity/persistence-sharing.js"; import { createSharedArrayBuffer } from "../../internal/shared-array-buffer/create-shared-array-buffer.js"; -export const createEntityLocationTable = (initialCapacity: number = 16, nonPersistent: boolean = false): EntityLocationTable => { - return nonPersistent ? createNegativeEntityLocationTable(initialCapacity) : createPositiveEntityLocationTable(initialCapacity); -} - -const createNegativeEntityLocationTable = (initialCapacity: number = 16): EntityLocationTable => { - const table = createEntityLocationTable(initialCapacity); +/** + * A location table for one entity quadrant (0..3). The backing store allocates + * dense per-quadrant local indices; this wrapper packs the quadrant into each + * id's low bits (and strips it on the way back in) so an entity id alone names + * its quadrant. `toData`/`fromData` serialize the raw local-index array + * (quadrant-agnostic); the quadrant is re-established by table position on load. + */ +export const createEntityLocationTable = (initialCapacity: number = 16, quadrant: number = 0): EntityLocationTable => { + const table = createLocalIndexEntityLocationTable(initialCapacity); return { ...table, - create: (location: EntityLocation): Entity => -1 - table.create(location), - delete: (entity: Entity) => table.delete(-1 - entity), - locate: (entity: Entity) => table.locate(-1 - entity), - update: (entity: Entity, location: EntityLocation) => table.update(-1 - entity, location), - reset: () => table.reset(), + create: (location: EntityLocation): Entity => toEntity(table.create(location), quadrant), + delete: (entity: Entity) => table.delete(toLocalIndex(entity)), + locate: (entity: Entity) => table.locate(toLocalIndex(entity)), + update: (entity: Entity, location: EntityLocation) => table.update(toLocalIndex(entity), location), }; } -const createPositiveEntityLocationTable = (initialCapacity: number = 16): EntityLocationTable => { +const createLocalIndexEntityLocationTable = (initialCapacity: number = 16): EntityLocationTable => { let freeListHead = -1; let nextIndex = 0; let capacity = initialCapacity; diff --git a/packages/data/src/ecs/entity-location-table/serialized-entity-location-tables.ts b/packages/data/src/ecs/entity-location-table/serialized-entity-location-tables.ts new file mode 100644 index 00000000..5bb7f02b --- /dev/null +++ b/packages/data/src/ecs/entity-location-table/serialized-entity-location-tables.ts @@ -0,0 +1,70 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. +import { isPersistentQuadrant, quadrantOf, toLocalIndex } from "../entity/persistence-sharing.js"; + +/** A live persistent entity's location, as mirrored by a persistence layer. */ +export interface EntityLocationEntry { + readonly entity: number; + readonly archetype: number; + readonly row: number; +} + +/** + * Build the per-quadrant serialized location tables (the shape core `fromData` + * consumes) from a flat list of live persistent entity locations. + * + * A persistence layer mirrors entity locations keyed by entity id; on load it + * hands that flat set here and this buckets each entity into its persistent + * quadrant's local-index table. This keeps all quadrant/id-encoding knowledge + * inside `@adobe/data` — callers never touch the bit layout. + */ +export const serializedEntityLocationTables = ( + entries: Iterable, +): { [quadrant: number]: unknown } => { + const buckets = new Map(); + for (const { entity, archetype, row } of entries) { + const quadrant = quadrantOf(entity); + if (!isPersistentQuadrant(quadrant)) continue; + let bucket = buckets.get(quadrant); + if (bucket === undefined) { + bucket = []; + buckets.set(quadrant, bucket); + } + bucket.push({ local: toLocalIndex(entity), archetype, row }); + } + + const tables: { [quadrant: number]: unknown } = {}; + for (const [quadrant, bucket] of buckets) { + tables[quadrant] = buildQuadrantTable(bucket); + } + return tables; +}; + +/** + * Rebuild one quadrant's location-table snapshot (indexed by per-quadrant local + * index). Holes below the high-water mark are threaded into the free list so + * post-load allocations reuse them and `nextIndex` never collides with a + * restored id. + */ +const buildQuadrantTable = ( + entries: { local: number; archetype: number; row: number }[], +): { entities: Int32Array; freeListHead: number; nextIndex: number; capacity: number } => { + let nextIndex = 0; + for (const e of entries) nextIndex = Math.max(nextIndex, e.local + 1); + let capacity = 16; + while (capacity < Math.max(nextIndex, 16)) capacity *= 2; + const entities = new Int32Array(new ArrayBuffer(capacity * 2 * 4)); + const occupied = new Uint8Array(nextIndex); + for (const e of entries) { + entities[e.local * 2 + 0] = e.archetype; + entities[e.local * 2 + 1] = e.row; + occupied[e.local] = 1; + } + let freeListHead = -1; + for (let local = 0; local < nextIndex; local++) { + if (occupied[local] === 1) continue; + entities[local * 2 + 0] = -1; + entities[local * 2 + 1] = freeListHead; + freeListHead = local; + } + return { entities, freeListHead, nextIndex, capacity }; +}; diff --git a/packages/data/src/ecs/entity/is-non-persistent.test.ts b/packages/data/src/ecs/entity/is-non-persistent.test.ts deleted file mode 100644 index 7deccca0..00000000 --- a/packages/data/src/ecs/entity/is-non-persistent.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -// © 2026 Adobe. MIT License. See /LICENSE for details. -import { describe, it, expect } from "vitest"; -import { Entity } from "./entity.js"; - -describe("Entity.isNonPersistent", () => { - it("should return true for negative entity IDs", () => { - expect(Entity.isNonPersistent(-1)).toBe(true); - expect(Entity.isNonPersistent(-100)).toBe(true); - }); - - it("should return false for non-negative entity IDs", () => { - expect(Entity.isNonPersistent(0)).toBe(false); - expect(Entity.isNonPersistent(1)).toBe(false); - expect(Entity.isNonPersistent(100)).toBe(false); - }); -}); - -describe("Entity.isPersistent", () => { - it("should return true for non-negative entity IDs", () => { - expect(Entity.isPersistent(0)).toBe(true); - expect(Entity.isPersistent(1)).toBe(true); - expect(Entity.isPersistent(100)).toBe(true); - }); - - it("should return false for negative entity IDs", () => { - expect(Entity.isPersistent(-1)).toBe(false); - expect(Entity.isPersistent(-100)).toBe(false); - }); -}); diff --git a/packages/data/src/ecs/entity/is-non-persistent.ts b/packages/data/src/ecs/entity/is-non-persistent.ts deleted file mode 100644 index 270e7fc0..00000000 --- a/packages/data/src/ecs/entity/is-non-persistent.ts +++ /dev/null @@ -1,5 +0,0 @@ -// © 2026 Adobe. MIT License. See /LICENSE for details. -import type { Entity } from "./entity.js"; - -export const isNonPersistent = (entity: Entity): boolean => entity < 0; -export const isPersistent = (entity: Entity): boolean => entity >= 0; diff --git a/packages/data/src/ecs/entity/persistence-sharing.test.ts b/packages/data/src/ecs/entity/persistence-sharing.test.ts new file mode 100644 index 00000000..aa9de9fb --- /dev/null +++ b/packages/data/src/ecs/entity/persistence-sharing.test.ts @@ -0,0 +1,53 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. +import { describe, it, expect } from "vitest"; +import { Entity } from "./entity.js"; +import { quadrantFor, quadrantOf, toEntity, toLocalIndex } from "./persistence-sharing.js"; + +// Quadrant encoding: bit0 = non-persistent, bit1 = non-shared. +// 0 = persistent + shared (document) +// 1 = non-persistent + shared (presence) +// 2 = persistent + non-shared (settings) +// 3 = non-persistent + non-shared (session) +describe("Entity persistence/sharing quadrant", () => { + it("isPersistent / isNonPersistent read bit 0", () => { + expect(Entity.isPersistent(0)).toBe(true); + expect(Entity.isNonPersistent(0)).toBe(false); + expect(Entity.isPersistent(2)).toBe(true); // settings: persistent + expect(Entity.isNonPersistent(1)).toBe(true); // presence: non-persistent + expect(Entity.isNonPersistent(3)).toBe(true); // session: non-persistent + }); + + it("isShared / isNonShared read bit 1", () => { + expect(Entity.isShared(0)).toBe(true); // document: shared + expect(Entity.isShared(1)).toBe(true); // presence: shared + expect(Entity.isNonShared(2)).toBe(true); // settings: non-shared + expect(Entity.isNonShared(3)).toBe(true); // session: non-shared + expect(Entity.isShared(2)).toBe(false); + }); + + it("classifies every quadrant of a shared local index", () => { + const cases: Array<[boolean, boolean]> = [ + [false, false], // document + [true, false], // presence + [false, true], // settings + [true, true], // session + ]; + for (const [nonPersistent, nonShared] of cases) { + const q = quadrantFor(nonPersistent, nonShared); + const e = toEntity(37, q); + expect(quadrantOf(e)).toBe(q); + expect(Entity.isNonPersistent(e)).toBe(nonPersistent); + expect(Entity.isNonShared(e)).toBe(nonShared); + expect(toLocalIndex(e)).toBe(37); + } + }); + + it("round-trips local index through the top 30 bits", () => { + for (const q of [0, 1, 2, 3]) { + for (const idx of [0, 1, 2, 1000, 1 << 20]) { + expect(toLocalIndex(toEntity(idx, q))).toBe(idx); + expect(quadrantOf(toEntity(idx, q))).toBe(q); + } + } + }); +}); diff --git a/packages/data/src/ecs/entity/persistence-sharing.ts b/packages/data/src/ecs/entity/persistence-sharing.ts new file mode 100644 index 00000000..70bfdedb --- /dev/null +++ b/packages/data/src/ecs/entity/persistence-sharing.ts @@ -0,0 +1,52 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. +import type { Entity } from "./entity.js"; + +// Entity ids carry a 2-bit quadrant in their low bits, partitioning entity +// space along two orthogonal axes: +// bit 0 — durability: set = non-persistent (not saved), clear = persistent +// bit 1 — sharing: set = non-shared (local), clear = shared (replicated) +// The remaining high bits are the entity's per-quadrant local index. THIS FILE +// IS THE ONLY PLACE THAT KNOWS THIS LAYOUT — everything else goes through the +// helpers below (predicates for consumers; encode/decode for the location +// table and core). To change the layout, change it here. + +/** Number of low bits reserved for the quadrant. */ +const QUADRANT_BITS = 2; +/** Mask selecting the quadrant bits: `entity & QUADRANT_MASK` → quadrant. */ +const QUADRANT_MASK = 0b11; +/** Quadrant bit 0 — set when the entity is non-persistent. */ +const PERSISTENCE_BIT = 0b01; +/** Quadrant bit 1 — set when the entity is non-shared. */ +const SHARING_BIT = 0b10; + +export const isNonPersistent = (entity: Entity): boolean => (entity & PERSISTENCE_BIT) !== 0; +export const isPersistent = (entity: Entity): boolean => (entity & PERSISTENCE_BIT) === 0; +export const isNonShared = (entity: Entity): boolean => (entity & SHARING_BIT) !== 0; +export const isShared = (entity: Entity): boolean => (entity & SHARING_BIT) === 0; + +/** Total number of quadrants (persistence × sharing). */ +export const QUADRANT_COUNT = 1 << QUADRANT_BITS; + +/** The quadrant (0..3) an entity id belongs to. */ +export const quadrantOf = (entity: Entity): number => entity & QUADRANT_MASK; + +/** Whether a quadrant (0..3) is a persistent one (its entities are serialized). */ +export const isPersistentQuadrant = (quadrant: number): boolean => (quadrant & PERSISTENCE_BIT) === 0; + +/** Compose the quadrant (0..3) from the two axis flags. */ +export const quadrantFor = (nonPersistent: boolean, nonShared: boolean): number => + (nonPersistent ? PERSISTENCE_BIT : 0) | (nonShared ? SHARING_BIT : 0); + +/** Pack a per-quadrant local index + quadrant into an entity id. */ +export const toEntity = (localIndex: number, quadrant: number): Entity => (localIndex << QUADRANT_BITS) | quadrant; + +/** Recover the per-quadrant local index from an entity id (unsigned). */ +export const toLocalIndex = (entity: Entity): number => entity >>> QUADRANT_BITS; + +/** + * The quadrants whose entities are persisted (document = 0, settings = 2). A + * persistence layer keeps one gap-free durable array per persistent quadrant, + * each indexed by `toLocalIndex`, so every quadrant packs fully densely. + */ +export const persistentQuadrants: readonly number[] = + Array.from({ length: QUADRANT_COUNT }, (_, quadrant) => quadrant).filter(isPersistentQuadrant); diff --git a/packages/data/src/ecs/entity/public.ts b/packages/data/src/ecs/entity/public.ts index 8b71b9a7..dfd941c1 100644 --- a/packages/data/src/ecs/entity/public.ts +++ b/packages/data/src/ecs/entity/public.ts @@ -1,3 +1,3 @@ // © 2026 Adobe. MIT License. See /LICENSE for details. export { schema } from "./schema.js"; -export { isNonPersistent, isPersistent } from "./is-non-persistent.js"; +export { isNonPersistent, isPersistent, isNonShared, isShared, quadrantOf, toLocalIndex, toEntity, persistentQuadrants } from "./persistence-sharing.js"; diff --git a/packages/data/src/ecs/index.ts b/packages/data/src/ecs/index.ts index a84bd254..198007c1 100644 --- a/packages/data/src/ecs/index.ts +++ b/packages/data/src/ecs/index.ts @@ -2,6 +2,8 @@ export * from "./store/index.js"; export * from "./database/index.js"; export { type EntityLocationTable } from "./entity-location-table/entity-location-table.js"; +export { serializedEntityLocationTables, type EntityLocationEntry } from "./entity-location-table/serialized-entity-location-tables.js"; +export { type PersistenceScope, type ToDataOptions } from "./persistence-scope.js"; export * from "./archetype/index.js"; export * from "./required-components.js"; export * from "./optional-components.js"; diff --git a/packages/data/src/ecs/optional-components.ts b/packages/data/src/ecs/optional-components.ts index 02dab2bb..1fa5725e 100644 --- a/packages/data/src/ecs/optional-components.ts +++ b/packages/data/src/ecs/optional-components.ts @@ -2,7 +2,8 @@ export type OptionalComponents = { nonPersistent: true; // Marks an entity as local to this client — never replicated to peers. - // Declared for modelling; the database does not yet act on it (no separate - // id space or sync exclusion). Orthogonal to `nonPersistent` (durability). + // Orthogonal to `nonPersistent` (durability): together they place an entity + // in one of four quadrants, each backed by its own entity-id space + // (see entity/persistence-sharing). nonShared: true; }; diff --git a/packages/data/src/ecs/persistence-scope.ts b/packages/data/src/ecs/persistence-scope.ts new file mode 100644 index 00000000..8b97e65e --- /dev/null +++ b/packages/data/src/ecs/persistence-scope.ts @@ -0,0 +1,32 @@ +// © 2026 Adobe. MIT License. See /LICENSE for details. + +/** + * Selects which persistent quadrants a `toData` / `fromData` operates on. Both + * quadrants are persistent; they differ only by sharing: + * - `shared` → the shared + persistent quadrant ("document") + * - `nonShared` → the nonShared + persistent quadrant ("settings") + * + * Omit the scope entirely to operate on the **whole** persistent snapshot (both + * quadrants) — the default, whole-database behavior. Passing a scope makes the + * operation surgical: `toData` emits row data only for the in-scope quadrant(s) + * (every archetype's structure is still emitted so archetype ids stay aligned), + * and `fromData` restores only the in-scope quadrant(s), leaving every other + * quadrant — persistent or not — untouched. This lets one service own + * `settings` while another owns `document`, each persisting to its own backend. + */ +export interface PersistenceScope { + readonly shared?: boolean; + readonly nonShared?: boolean; +} + +/** Options for serializing a store/database snapshot via `toData`. */ +export interface ToDataOptions { + /** + * Detach the snapshot from the live store (clone column/entity buffers) so + * it survives later mutation. Otherwise the snapshot references live buffers + * and is only valid until the next mutation. + */ + readonly copy?: boolean; + /** Restrict the snapshot to specific persistent quadrants; omit for all. */ + readonly scope?: PersistenceScope; +} diff --git a/packages/data/src/ecs/persistence-service/create-storage-persistence-service.ts b/packages/data/src/ecs/persistence-service/create-storage-persistence-service.ts index 163ccdce..a13c6a34 100644 --- a/packages/data/src/ecs/persistence-service/create-storage-persistence-service.ts +++ b/packages/data/src/ecs/persistence-service/create-storage-persistence-service.ts @@ -2,7 +2,8 @@ import { deserializeFromStorage, serializeToStorage } from "../../functions/serialization/serialize-to-storage.js"; import { debounce } from "../../internal/function/debounce.js"; -import { Database } from "../index.js"; +import { Database, Entity } from "../index.js"; +import { PersistenceScope } from "../persistence-scope.js"; import { PersistenceService } from "./persistence-service.js"; export const createStoragePersistenceService = async (options: { @@ -10,18 +11,25 @@ export const createStoragePersistenceService = async (options: { defaultFileId: string, autoSaveOnChange: boolean, autoLoadOnStart: boolean, - storage?: Storage + storage?: Storage, + /** + * Restrict this service to specific persistent quadrant(s) — e.g. + * `{ nonShared: true }` for a settings-only service. Omit to own the whole + * persistent snapshot (both document and settings). The same scope is used + * for both save and load. + */ + scope?: PersistenceScope, }): Promise => { - const { database, defaultFileId, autoSaveOnChange: autoSave, autoLoadOnStart, storage = sessionStorage } = options; + const { database, defaultFileId, autoSaveOnChange: autoSave, autoLoadOnStart, storage = sessionStorage, scope } = options; const service: PersistenceService = { serviceName: "SessionPersistenceService", save: async (fileId = defaultFileId) => { - await serializeToStorage(database.toData(), fileId, storage); + await serializeToStorage(database.toData({ scope }), fileId, storage); }, load: async (fileId = defaultFileId) => { const data = await deserializeFromStorage(fileId, storage); if (data) { - database.fromData(data); + database.fromData(data, scope); } } } @@ -29,9 +37,19 @@ export const createStoragePersistenceService = async (options: { await service.load(); } if (autoSave) { + // Whether an entity falls within this service's scope. Unscoped ⇒ any + // persistent entity; scoped ⇒ only the persistent quadrant(s) it owns. + const entityInScope = (entity: number): boolean => { + if (!Entity.isPersistent(entity)) return false; + if (scope === undefined) return true; + return Boolean((scope.shared && Entity.isShared(entity)) || (scope.nonShared && Entity.isNonShared(entity))); + }; const debouncedSave = debounce(() => service.save(), 300); database.observe.transactions(t => { - if (!t.intermediate && t.persistent) debouncedSave(); + if (t.intermediate || !t.persistent) return; + for (const entity of t.changedEntities.keys()) { + if (entityInScope(entity)) { debouncedSave(); return; } + } }); } return service; diff --git a/packages/data/src/ecs/store/core/core.ts b/packages/data/src/ecs/store/core/core.ts index 480d0c63..e6f439e7 100644 --- a/packages/data/src/ecs/store/core/core.ts +++ b/packages/data/src/ecs/store/core/core.ts @@ -8,6 +8,7 @@ import { StringKeyof } from "../../../types/index.js"; import { Components } from "../components.js"; import { OptionalComponents } from "../../optional-components.js"; import { HasPartitionKey } from "../partition.js"; +import { PersistenceScope, ToDataOptions } from "../../persistence-scope.js"; export type EntityValues = { readonly [K in (RequiredComponents & StringKeyof)]: (C & OptionalComponents)[K] } export type EntityReadValues = RequiredComponents & { readonly [K in StringKeyof as string extends K ? never : K]?: (C & OptionalComponents)[K] } @@ -79,8 +80,11 @@ export interface ReadonlyCore< * live store (column and entity buffers are copied) so it survives later * mutation; otherwise it references live buffers — faster, but only valid * until the next mutation. See {@link Store.toData} bug notes. + * + * `options.scope` selects which persistent quadrants to emit (see + * PersistenceScope); omit it for the whole persistent snapshot. */ - toData(copy?: boolean): unknown + toData(options?: ToDataOptions): unknown } /** @@ -106,10 +110,37 @@ export interface Core< partitionValues: { readonly [K in Extract]: (C & RequiredComponents & OptionalComponents)[K] }, ): Archetype; locate: (entity: Entity) => { archetype: Archetype, row: number } | null; - delete: (entity: Entity) => void; - update: (entity: Entity, values: EntityUpdateValues) => void; + /** + * Deletes the entity. Returns the entity that was swap-moved into the + * vacated row (a relocation side effect), or `undefined` when the deleted + * row was the last row. The return is optional info — callers that don't + * track relocations may ignore it. + */ + delete: (entity: Entity) => Entity | undefined; + /** + * Updates the entity. When the update migrates the entity to another + * archetype, its old archetype swap-moves a neighbor into the vacated row; + * that neighbor is returned (or `undefined` for an in-place update with no + * migration). The migrated entity's own relocation is observable via + * `locate`. The return is optional info — callers may ignore it. + */ + update: (entity: Entity, values: EntityUpdateValues) => Entity | undefined; compact: () => void; /** Wipe all entities. O(num_archetypes). Location tables and row counts reset to empty. */ reset(): void; - fromData(data: unknown): void + /** + * Reconcile nonPersistent-schema columns after an external restore (e.g. a + * persistence layer that rebuilt the store from disk column-by-column): reset + * defaulted ones to their schema default and strip no-default ones (the entity + * migrates out of the component). Idempotent. `fromData` already applies this + * to its own loads; this is for callers that restore columns another way. + */ + reconstructNonPersistentColumns(): void; + /** + * Restore from a snapshot. With no `scope`, performs a whole-database load + * (restores both persistent quadrants and resets the non-persistent ones). + * With a `scope`, restores only the in-scope persistent quadrant(s) and + * leaves every other quadrant untouched (see PersistenceScope). + */ + fromData(data: unknown, scope?: PersistenceScope): void } diff --git a/packages/data/src/ecs/store/core/create-core.partition.test.ts b/packages/data/src/ecs/store/core/create-core.partition.test.ts index 121fa050..098179fc 100644 --- a/packages/data/src/ecs/store/core/create-core.partition.test.ts +++ b/packages/data/src/ecs/store/core/create-core.partition.test.ts @@ -159,7 +159,7 @@ describe("partition components", () => { const a = router.insert({ cell: 1, position: { x: 1, y: 1 } }); const b = router.insert({ cell: 2, position: { x: 2, y: 2 } }); - const snapshot = core.toData(true); + const snapshot = core.toData({ copy: true }); const restored = makeCore(); restored.fromData(snapshot); diff --git a/packages/data/src/ecs/store/core/create-core.test.ts b/packages/data/src/ecs/store/core/create-core.test.ts index 404e94b2..69a2ed7c 100644 --- a/packages/data/src/ecs/store/core/create-core.test.ts +++ b/packages/data/src/ecs/store/core/create-core.test.ts @@ -2,7 +2,8 @@ import { describe, it, expect } from "vitest"; import { createCore } from "./create-core.js"; import { Schema } from "../../../schema/index.js"; -import type { Entity } from "../../entity/entity.js"; +import { Entity } from "../../entity/entity.js"; +import { quadrantOf } from "../../entity/persistence-sharing.js"; import { F32 } from "../../../math/f32/index.js"; // Shared test schemas @@ -446,7 +447,7 @@ export function createCoreTestSuite( } }); - it("should create nonPersistent entities with negative ids", () => { + it("should create nonPersistent entities in the non-persistent quadrant", () => { const core = factory({ position: positionSchema, health: healthSchema, @@ -454,7 +455,7 @@ export function createCoreTestSuite( const ephemeralPositionTable = core.ensureArchetype(["id", "position", "nonPersistent"]); const writeId = ephemeralPositionTable.insert({ position: { x: 1, y: 2, z: 3 }, nonPersistent: true }); - expect(writeId).toBe(-1); + expect(Entity.isNonPersistent(writeId)).toBe(true); const readId = ephemeralPositionTable.columns.id.get(0); expect(readId).toBe(writeId); @@ -700,13 +701,14 @@ export function createCoreTestSuite( const archetype = core.ensureArchetype(["id", "position"]); // Add many entities + const ids: Entity[] = []; for (let i = 0; i < 20; i++) { - archetype.insert({ position: { x: i, y: i * 2, z: i * 3 } }); + ids.push(archetype.insert({ position: { x: i, y: i * 2, z: i * 3 } })); } - // Delete most of them + // Delete most of them (keep the first two) for (let i = 2; i < 20; i++) { - core.delete(i); + core.delete(ids[i]); } expect(archetype.rowCount).toBe(2); @@ -768,8 +770,8 @@ export function createCoreTestSuite( nonPersistent: true }); - // Verify nonPersistent entity exists and has negative id - expect(ephemeralEntity).toBeLessThan(0); + // Verify nonPersistent entity exists and is non-persistent + expect(Entity.isNonPersistent(ephemeralEntity)).toBe(true); expect(core.locate(ephemeralEntity)).not.toBeNull(); expect(core.read(ephemeralEntity)).not.toBeNull(); @@ -794,8 +796,8 @@ export function createCoreTestSuite( nonPersistent: true }); - // Verify nonPersistent entity exists and has negative id - expect(ephemeralEntity).toBeLessThan(0); + // Verify nonPersistent entity exists and is non-persistent + expect(Entity.isNonPersistent(ephemeralEntity)).toBe(true); expect(core.locate(ephemeralEntity)).not.toBeNull(); // Add health component to trigger archetype change @@ -812,6 +814,134 @@ export function createCoreTestSuite( expect(data?.nonPersistent).toBe(true); }); + it("partitions entities into four disjoint quadrants by persistence × sharing", () => { + const core = factory({ position: positionSchema }); + const doc = core.ensureArchetype(["id", "position"]).insert({ position: { x: 0, y: 0, z: 0 } }); + const settings = core.ensureArchetype(["id", "position", "nonShared"]).insert({ position: { x: 0, y: 0, z: 0 }, nonShared: true }); + const presence = core.ensureArchetype(["id", "position", "nonPersistent"]).insert({ position: { x: 0, y: 0, z: 0 }, nonPersistent: true }); + const session = core.ensureArchetype(["id", "position", "nonPersistent", "nonShared"]).insert({ position: { x: 0, y: 0, z: 0 }, nonPersistent: true, nonShared: true }); + + expect(Entity.isPersistent(doc) && Entity.isShared(doc)).toBe(true); + expect(Entity.isPersistent(settings) && Entity.isNonShared(settings)).toBe(true); + expect(Entity.isNonPersistent(presence) && Entity.isShared(presence)).toBe(true); + expect(Entity.isNonPersistent(session) && Entity.isNonShared(session)).toBe(true); + + const quadrants = new Set([doc, settings, presence, session].map(quadrantOf)); + expect(quadrants.size).toBe(4); + }); + + it("serializes persistent quadrants and resets non-persistent ones on load", () => { + const core = factory({ position: positionSchema }); + const doc = core.ensureArchetype(["id", "position"]).insert({ position: { x: 1, y: 0, z: 0 } }); + const settings = core.ensureArchetype(["id", "position", "nonShared"]).insert({ position: { x: 2, y: 0, z: 0 }, nonShared: true }); + core.ensureArchetype(["id", "position", "nonPersistent"]).insert({ position: { x: 3, y: 0, z: 0 }, nonPersistent: true }); + + const data = core.toData(); + + const restored = factory({ position: positionSchema }); + restored.fromData(data); + + // document (persistent+shared) and settings (persistent+nonShared) survive. + expect(restored.read(doc)?.position).toEqual({ x: 1, y: 0, z: 0 }); + expect(restored.read(settings)?.position).toEqual({ x: 2, y: 0, z: 0 }); + // presence (non-persistent) is not serialized: its archetype loads empty. + expect(restored.ensureArchetype(["id", "position", "nonPersistent"]).rowCount).toBe(0); + }); + + it("should throw when trying to update nonShared component", () => { + const core = factory({ position: positionSchema }); + const entity = core.ensureArchetype(["id", "position", "nonShared"]).insert({ position: { x: 0, y: 0, z: 0 }, nonShared: true }); + expect(() => core.update(entity, { nonShared: true } as never)).toThrow("Cannot update nonShared component"); + }); + + it("scopes toData/fromData to selected persistent quadrants", () => { + const core = factory({ position: positionSchema }); + const doc = core.ensureArchetype(["id", "position"]).insert({ position: { x: 1, y: 0, z: 0 } }); + const settings = core.ensureArchetype(["id", "position", "nonShared"]).insert({ position: { x: 2, y: 0, z: 0 }, nonShared: true }); + + // One scoped snapshot per persistent quadrant. + const docData = core.toData({ scope: { shared: true } }); + const settingsData = core.toData({ scope: { nonShared: true } }); + + // Each scoped load restores only its own quadrant. + const onlyDoc = factory({ position: positionSchema }); + onlyDoc.fromData(docData, { shared: true }); + expect(onlyDoc.read(doc)?.position).toEqual({ x: 1, y: 0, z: 0 }); + expect(onlyDoc.locate(settings)).toBeNull(); + + const onlySettings = factory({ position: positionSchema }); + onlySettings.fromData(settingsData, { nonShared: true }); + expect(onlySettings.read(settings)?.position).toEqual({ x: 2, y: 0, z: 0 }); + expect(onlySettings.locate(doc)).toBeNull(); + + // Two independent scoped loads into one store compose without either + // clobbering the other — the "settings service + document service" model. + const combined = factory({ position: positionSchema }); + combined.fromData(settingsData, { nonShared: true }); + combined.fromData(docData, { shared: true }); + expect(combined.read(doc)?.position).toEqual({ x: 1, y: 0, z: 0 }); + expect(combined.read(settings)?.position).toEqual({ x: 2, y: 0, z: 0 }); + }); + + it("honors component-level nonPersistent: defaulted resets, no-default is stripped", () => { + const cacheSchema = { type: "number", default: 7, nonPersistent: true } as const satisfies Schema; + const derivedSchema = { type: "number", nonPersistent: true } as const satisfies Schema; + const make = () => factory({ position: positionSchema, cache: cacheSchema, derived: derivedSchema }); + + const core = make(); + const e = core.ensureArchetype(["id", "position", "cache", "derived"]).insert({ + position: { x: 1, y: 2, z: 3 }, cache: 99, derived: 42, + }); + + const data = core.toData(); + const restored = make(); + restored.fromData(data); + + const view = restored.read(e) as { position: { x: number }; cache?: number; derived?: number } | null; + expect(view).not.toBeNull(); + // Persistent component survives. + expect(view!.position).toEqual({ x: 1, y: 2, z: 3 }); + // Defaulted nonPersistent component: not persisted, reset to default. + expect(view!.cache).toBe(7); + // No-default nonPersistent component: stripped — the entity restored + // into the reduced archetype without it. + expect("derived" in view!).toBe(false); + expect(restored.locate(e)!.archetype.components.has("derived")).toBe(false); + expect(restored.locate(e)!.archetype.components.has("cache")).toBe(true); + }); + + it("strips a nonPersistent component only when its default is undefined; retains null and falsy defaults", () => { + // `{ default: undefined }` is the type-only-placeholder pattern (e.g. + // an opaque GPUBuffer handle): the key carries a type, not a usable + // value — so it must be stripped. + const gpuRefSchema = { default: undefined as unknown, nonPersistent: true } satisfies Schema; + // `null` is a real, valid value → retained and reset to null. + const maybeSchema = { default: null as unknown, nonPersistent: true } satisfies Schema; + // A falsy-but-real default (0) → retained and reset. + const counterSchema = { type: "number", default: 0, nonPersistent: true } as const satisfies Schema; + const make = () => factory({ position: positionSchema, gpuRef: gpuRefSchema, maybe: maybeSchema, counter: counterSchema }); + + const core = make(); + const e = core.ensureArchetype(["id", "position", "gpuRef", "maybe", "counter"]).insert({ + position: { x: 1, y: 2, z: 3 }, gpuRef: 12345, maybe: 999, counter: 99, + }); + + const data = core.toData(); + const restored = make(); + restored.fromData(data); + + const view = restored.read(e) as { position: unknown; gpuRef?: unknown; maybe?: unknown; counter?: number } | null; + expect(view).not.toBeNull(); + // undefined default → stripped. + expect("gpuRef" in view!).toBe(false); + expect(restored.locate(e)!.archetype.components.has("gpuRef")).toBe(false); + // null default → retained, reset to null (null is a valid value). + expect(view!.maybe).toBe(null); + expect(restored.locate(e)!.archetype.components.has("maybe")).toBe(true); + // falsy-but-real default (0) → retained, reset to 0. + expect(view!.counter).toBe(0); + }); + }); } diff --git a/packages/data/src/ecs/store/core/create-core.ts b/packages/data/src/ecs/store/core/create-core.ts index 41fb7bf3..0e756767 100644 --- a/packages/data/src/ecs/store/core/create-core.ts +++ b/packages/data/src/ecs/store/core/create-core.ts @@ -7,6 +7,8 @@ import { Table, getRowData, addRow, updateRow } from "../../../table/index.js"; import { Archetype, ReadonlyArchetype } from "../../archetype/archetype.js"; import { RequiredComponents } from "../../required-components.js"; import { Entity } from "../../entity/entity.js"; +import { QUADRANT_COUNT, isPersistentQuadrant, quadrantFor, quadrantOf } from "../../entity/persistence-sharing.js"; +import { PersistenceScope, ToDataOptions } from "../../persistence-scope.js"; import { Core, EntityUpdateValues, ArchetypeQueryOptions } from "./core.js"; import { Assert, Equal, Simplify, StringKeyof } from "../../../types/index.js"; import { ComponentSchemas } from "../../component-schemas.js"; @@ -19,19 +21,20 @@ import { PartitionKeysOf } from "../partition.js"; * checked by `fromData`. A mismatch is thrown rather than silently * mis-reconstructed, so an incompatible snapshot fails loudly at load. * - * Version 1 is the first *versioned* format. Snapshots produced before this - * field existed carry no `version` and are therefore rejected — they used an - * incompatible archetype-entry shape. Bump this whenever the snapshot shape + * Version 1 was the first *versioned* format. Version 2 changed the entity-id + * encoding: durability + sharing are now a 2-bit quadrant in the id's low bits + * (see entity/persistence-sharing), so persisted ids and the set of serialized + * location tables both changed shape. Bump this whenever the snapshot shape * changes in a way older readers cannot load. */ -export const ECS_SNAPSHOT_VERSION = 1; +export const ECS_SNAPSHOT_VERSION = 2; /** * One archetype's entry in a serialized snapshot. Every archetype * contributes an entry so its `id` (a dense index into `archetypes`, stored - * by value in the persistent location table) is reproduced exactly on load. + * by value in the persistent location tables) is reproduced exactly on load. * Only persistent archetypes carry `data`: nonPersistent archetypes back the - * negative-ID entity space, whose location table is never serialized, so + * non-persistent quadrants, whose location tables are never serialized, so * their rows are not persistent state. */ type SerializedArchetype = { @@ -47,7 +50,10 @@ type SerializedArchetype = { type SerializedCore = { readonly version: number; readonly componentSchemas: object; - readonly entityLocationTableData: unknown; + // Serialized location tables for the persistent quadrants only, keyed by + // quadrant (0 = document, 2 = settings). Non-persistent quadrants (1, 3) + // are never serialized — they reset on load. + readonly entityLocationTables: { readonly [quadrant: number]: unknown }; readonly archetypesData: readonly SerializedArchetype[]; }; @@ -67,14 +73,17 @@ export function createCore( const componentSchemas: { readonly [K in StringKeyof]: Schema } = { id: Entity.schema, nonPersistent: True.schema, - // Declared built-in (no behavior wired yet) — mirrors nonPersistent so - // apps can model local vs. shared scope; the store does not act on it. + // Built-in sharing tag, mirror of nonPersistent. Together they place an + // archetype's entities into one of four quadrants (persistence × sharing); + // see entity/persistence-sharing. nonShared: True.schema, ...newComponentSchemas }; - const persistentLocationTable = createEntityLocationTable(16, false); - const nonPersistentLocationTable = createEntityLocationTable(16, true); - const getLocationTable = (entity: Entity) => entity < 0 ? nonPersistentLocationTable : persistentLocationTable; + // One location table per quadrant, indexed by an entity id's low 2 bits. + // Each table owns a disjoint id space (quadrant packed into the low bits), + // so an entity id alone names its quadrant. + const locationTables = Array.from({ length: QUADRANT_COUNT }, (_, quadrant) => createEntityLocationTable(16, quadrant)); + const getLocationTable = (entity: Entity) => locationTables[quadrantOf(entity)]!; const archetypes = [] as unknown as Archetype[] & { readonly [x: string]: Archetype }; // A component declared `partition: true`: every distinct runtime value gets @@ -164,9 +173,11 @@ export function createCore( const archetypeComponentSchemas: Record = {}; let hasId = false; let isNonPersistent = false; + let isNonShared = false; for (const comp of namesArr) { if (comp === "id") hasId = true; if (comp === "nonPersistent") isNonPersistent = true; + if (comp === "nonShared") isNonShared = true; const base = componentSchemas[comp as StringKeyof]; archetypeComponentSchemas[comp] = isPartition(comp) ? { ...base, const: partitionValues![comp] } @@ -178,7 +189,7 @@ export function createCore( const archetype = ARCHETYPE.createArchetype( archetypeComponentSchemas as any, id, - isNonPersistent ? nonPersistentLocationTable : persistentLocationTable + locationTables[quadrantFor(isNonPersistent, isNonShared)]! ); archetypes.push(archetype as unknown as Archetype); archetypeByIdentity.set(key, archetype); @@ -221,7 +232,7 @@ export function createCore( }) as Core["ensureArchetype"]; const locateInternal = (entity: Entity) => { - return (entity < 0 ? nonPersistentLocationTable : persistentLocationTable).locate(entity); + return getLocationTable(entity).locate(entity); } const readEntity = ( @@ -256,7 +267,7 @@ export function createCore( return getRowData(archetype, location.row); } - const deleteEntity = (entity: Entity) => { + const deleteEntity = (entity: Entity): Entity | undefined => { const locationTable = getLocationTable(entity); const location = locationTable.locate(entity); if (location !== null) { @@ -264,12 +275,14 @@ export function createCore( if (!archetype) { throw new Error("Archetype not found: " + JSON.stringify(location)); } - ARCHETYPE.deleteRow(archetype, location.row, locationTable); + const swapped = ARCHETYPE.deleteRow(archetype, location.row, locationTable); locationTable.delete(entity); + return swapped; } + return undefined; } - const updateEntity = (entity: Entity, components: EntityUpdateValues) => { + const updateEntity = (entity: Entity, components: EntityUpdateValues): Entity | undefined => { const currentLocation = locateInternal(entity); if (currentLocation === null) { throw new Error(`Entity not found ${entity}`); @@ -277,6 +290,9 @@ export function createCore( if ("nonPersistent" in components) { throw new Error("Cannot update nonPersistent component"); } + if ("nonShared" in components) { + throw new Error("Cannot update nonShared component"); + } const currentArchetype = archetypes[currentLocation.archetype]; let newArchetype = currentArchetype; let addComponents: null | StringKeyof[] = null; @@ -335,12 +351,14 @@ export function createCore( const currentData = getRowData(currentArchetype, currentLocation.row); const currentLocationTable = getLocationTable(entity); // deletes the row from the current archetype (this will update the entity location table for any row which may have been moved into it's position) - ARCHETYPE.deleteRow(currentArchetype, currentLocation.row, currentLocationTable); + const swapped = ARCHETYPE.deleteRow(currentArchetype, currentLocation.row, currentLocationTable); const newRow = addRow(newArchetype, { ...currentData, ...components }); // update the entity location table for the entity so it points to the new archetype and row currentLocationTable.update(entity, { archetype: newArchetype.id, row: newRow }); + return swapped; } else { updateRow(newArchetype, currentLocation.row, components as any); + return undefined; } } @@ -361,13 +379,80 @@ export function createCore( }; const resetCore = () => { - persistentLocationTable.reset(); - nonPersistentLocationTable.reset(); + for (const table of locationTables) { + table.reset(); + } for (const archetype of archetypes) { archetype.rowCount = 0; } }; + // The persistent quadrants a toData/fromData operates on. Omitted scope ⇒ + // all persistent quadrants (whole-database behavior). + const scopeQuadrants = (scope?: PersistenceScope): number[] => { + if (scope === undefined) { + return locationTables.map((_, quadrant) => quadrant).filter(isPersistentQuadrant); + } + const quadrants: number[] = []; + if (scope.shared) quadrants.push(quadrantFor(false, false)); + if (scope.nonShared) quadrants.push(quadrantFor(false, true)); + return quadrants; + }; + const archetypeQuadrant = (archetype: Archetype): number => + quadrantFor(archetype.components.has("nonPersistent"), archetype.components.has("nonShared")); + + // Component names whose SCHEMA is marked `nonPersistent` (distinct from the + // built-in `nonPersistent`/`nonShared` marker components). Their column data + // is never serialized; on load each is reconstructed — reset to its schema + // default, or (when it has no default) stripped from the entity entirely. + const nonPersistentComponents = (): Set => { + const names = new Set(); + for (const name in componentSchemas) { + if (name === "id" || name === "nonPersistent" || name === "nonShared") continue; + if ((componentSchemas as Record)[name]?.nonPersistent === true) names.add(name); + } + return names; + }; + + // After a restore, put nonPersistent columns back into a valid state for the + // archetypes whose data was just loaded (their nonPersistent columns were + // omitted from the snapshot and rebuilt empty by archetype.fromData). + const reconstructNonPersistentColumns = (restored: readonly Archetype[]): void => { + const nonPersistent = nonPersistentComponents(); + if (nonPersistent.size === 0) return; + // An `undefined` default (explicit, or an absent `default` key) means + // "no usable default" — including the type-only-placeholder pattern + // `{ default: undefined as unknown as GPUBuffer }` — so the component is + // stripped on load. Every real value is retained and reset, INCLUDING + // `null` (a valid value) and falsy values like 0, false, "". + const noDefault = new Set(); + for (const name of nonPersistent) { + if ((componentSchemas as Record)[name]!.default === undefined) noDefault.add(name); + } + // Defaulted nonPersistent columns → reset every row to the schema default. + for (const archetype of restored) { + for (const name of nonPersistent) { + if (noDefault.has(name) || !archetype.components.has(name)) continue; + const column = archetype.columns[name]!; + const def = (componentSchemas as Record)[name]!.default; + for (let row = 0; row < archetype.rowCount; row++) column.set(row, def); + } + } + // No-default nonPersistent components → strip them so the entity restores + // without the component (a system re-adds it on demand). Uses the normal + // remove-component migration, which naturally merges into reduced archetypes. + if (noDefault.size > 0) { + for (const archetype of restored) { + const present = [...noDefault].filter((n) => archetype.components.has(n)); + if (present.length === 0) continue; + const removal = Object.fromEntries(present.map((n) => [n, undefined])) as EntityUpdateValues; + while (archetype.rowCount > 0) { + updateEntity(archetype.columns.id!.get(0), removal); + } + } + } + }; + const core: Core = { componentSchemas: componentSchemas, queryArchetypes, @@ -385,27 +470,39 @@ export function createCore( update: updateEntity, compact, reset: resetCore, - toData: (copy = false): SerializedCore => ({ - version: ECS_SNAPSHOT_VERSION, - componentSchemas, - entityLocationTableData: persistentLocationTable.toData(copy), - // Every archetype contributes an entry so its id (this array - // index, stored by value in the persistent location table) is - // reproduced on load. Only persistent archetypes carry `data`; - // nonPersistent ones back the negative-ID space, whose location - // table is never serialized, so their rows aren't persistent. - archetypesData: archetypes.map((archetype): SerializedArchetype => { - const componentNames = [...archetype.components]; - const partitionNames = partitionNamesIn(componentNames.slice().sort()); - const partitionValues = partitionNames.length > 0 - ? Object.fromEntries(partitionNames.map((n) => [n, archetype.columns[n]!.get(0)])) - : undefined; - return archetype.components.has("nonPersistent") - ? { componentNames, partitionValues } - : { componentNames, partitionValues, data: archetype.toData(copy) }; - }) - }), - fromData: (data: SerializedCore) => { + reconstructNonPersistentColumns: () => reconstructNonPersistentColumns(archetypes), + toData: (options?: ToDataOptions): SerializedCore => { + const { copy = false, scope } = options ?? {}; + const inScope = new Set(scopeQuadrants(scope)); + // nonPersistent-schema components are never written; their column + // data is omitted and reconstructed on load. + const omit = nonPersistentComponents(); + return { + version: ECS_SNAPSHOT_VERSION, + componentSchemas, + // Serialize the in-scope persistent quadrants' location tables. + entityLocationTables: Object.fromEntries( + locationTables.flatMap((table, quadrant) => + inScope.has(quadrant) ? [[quadrant, table.toData(copy)]] : []), + ), + // Every archetype contributes a structural entry so its id (this + // array index, stored by value in the location tables) is + // reproduced on load — this is why a scoped snapshot can still be + // loaded without shifting ids. Only in-scope (persistent) quadrant + // archetypes carry `data`; everything else is structure only. + archetypesData: archetypes.map((archetype): SerializedArchetype => { + const componentNames = [...archetype.components]; + const partitionNames = partitionNamesIn(componentNames.slice().sort()); + const partitionValues = partitionNames.length > 0 + ? Object.fromEntries(partitionNames.map((n) => [n, archetype.columns[n]!.get(0)])) + : undefined; + return inScope.has(archetypeQuadrant(archetype)) + ? { componentNames, partitionValues, data: archetype.toData(copy, omit) } + : { componentNames, partitionValues }; + }) + }; + }, + fromData: (data: SerializedCore, scope?: PersistenceScope) => { if (data.version !== ECS_SNAPSHOT_VERSION) { // Incompatible (or legacy, unversioned) snapshot. Skip the load // rather than throw: callers treat this as "no saved data" and @@ -417,30 +514,46 @@ export function createCore( return; } Object.assign(componentSchemas, data.componentSchemas); - // The non-persistent (negative-ID) space is never captured by - // toData, so a load must revert it to defaults rather than leak the - // loading store's pre-load live values across the load. Clear it the - // same way reset() does; the persistent side below is fully - // overwritten by the restore, so only the nonPersistent rows need - // clearing here. The store layer re-seeds nonPersistent resource - // defaults afterward (its rowCount === 0 re-init guard). - nonPersistentLocationTable.reset(); - for (const archetype of archetypes) { - if (archetype.components.has("nonPersistent")) { - archetype.rowCount = 0; + // A whole-database load (no scope) also reverts the non-persistent + // quadrants to defaults, so the loading store's pre-load transient + // values don't leak across the load. A scoped load is surgical: it + // touches only its persistent quadrant(s) and leaves everything else + // — other persistent quadrants and the transient ones — alone. + if (scope === undefined) { + for (let quadrant = 0; quadrant < QUADRANT_COUNT; quadrant++) { + if (!isPersistentQuadrant(quadrant)) locationTables[quadrant]!.reset(); + } + for (const archetype of archetypes) { + if (archetype.components.has("nonPersistent")) { + archetype.rowCount = 0; + } + } + } + for (const quadrant of scopeQuadrants(scope)) { + const restored = data.entityLocationTables[quadrant]; + // Restore the quadrant's table, or reset it if the snapshot + // carried no entities for it. + if (restored !== undefined) { + locationTables[quadrant]!.fromData(restored); + } else { + locationTables[quadrant]!.reset(); } } - persistentLocationTable.fromData(data.entityLocationTableData); + const restoredArchetypes: Archetype[] = []; for (const { componentNames, partitionValues, data: archetypeData } of data.archetypesData) { - // Recreating the archetype reserves its id and leaves it - // empty; only persistent entries carry data to restore. - // resolveArchetype (not the public ensureArchetype) so a + // Recreating the archetype reserves its id and leaves it empty + // (keeping ids aligned); only in-scope entries carry data to + // restore. resolveArchetype (not the public ensureArchetype) so a // partition archetype restores as its concrete value-child. const archetype = resolveArchetype(componentNames, partitionValues); if (archetypeData !== undefined) { archetype.fromData(archetypeData); + restoredArchetypes.push(archetype as unknown as Archetype); } } + // The archetypes just loaded had their nonPersistent columns omitted + // and rebuilt empty; put those columns back into a valid state. + reconstructNonPersistentColumns(restoredArchetypes); } }; return core as any; diff --git a/packages/data/src/ecs/store/public/create-store.test.ts b/packages/data/src/ecs/store/public/create-store.test.ts index bf850080..2ccd6e16 100644 --- a/packages/data/src/ecs/store/public/create-store.test.ts +++ b/packages/data/src/ecs/store/public/create-store.test.ts @@ -635,11 +635,11 @@ describe("createStore", () => { (store.resources as any).score = 999; (store.resources as any).persistentScore = 123; - const serializedData: any = store.toData(true); + const serializedData: any = store.toData({ copy: true }); // The nonPersistent resource's value must never appear in the - // snapshot: its entity lives in the negative-ID space, which - // entityLocationTableData never covers either. Its archetype slot + // snapshot: its entity lives in a non-persistent quadrant, whose + // location table is never serialized either. Its archetype slot // is still present (to keep archetype ids stable) but carries no // `data` — only its component names. const scoreEntry = serializedData.archetypesData.find( @@ -675,7 +675,7 @@ describe("createStore", () => { (store.resources as any).score = 999; (store.resources as any).persistentScore = 123; - const encoded = serialize(store.toData(true)); + const encoded = serialize(store.toData({ copy: true })); // The nonPersistent value must not survive into the encoded bytes. expect(encoded.json).not.toContain("999"); expect(encoded.json).toContain("123"); @@ -702,7 +702,7 @@ describe("createStore", () => { // Save a snapshot from a source store. const source = createStore(schema as any); (source.resources as any).persistentScore = 123; - const snapshot = source.toData(true); + const snapshot = source.toData({ copy: true }); // Load into a store that ALREADY holds non-default state — the // same-instance Save-As -> Load path. The nonPersistent resource @@ -728,11 +728,11 @@ describe("createStore", () => { }); const source = makeStore(); - const snapshot = source.toData(true); + const snapshot = source.toData({ copy: true }); - // Target holds a nonPersistent entity (negative-ID space) before the - // load. fromData must clear it — the nonPersistent space is never - // serialized, so a load resets it exactly as reset() would. + // Target holds a nonPersistent entity (non-persistent quadrant) before + // the load. fromData must clear it — the non-persistent quadrants are + // never serialized, so a load resets them exactly as reset() would. const target = makeStore(); const selectionArchetype = target.ensureArchetype(["id", "selection", "nonPersistent"]); const selectionEntity = selectionArchetype.insert({ selection: true, nonPersistent: true }); @@ -763,7 +763,7 @@ describe("createStore", () => { const positionArchetype = store.ensureArchetype(["id", "position"]); const positionEntity = positionArchetype.insert({ position: 42 }); - const serializedData = store.toData(true); + const serializedData = store.toData({ copy: true }); const newStore = makeStore(); newStore.fromData(serializedData); @@ -777,7 +777,7 @@ describe("createStore", () => { it("stamps a version and skips (warns, does not throw) snapshots of an incompatible or legacy format", () => { const store = createStore({ components: { position: positionSchema }, resources: {}, archetypes: {} }); const entity = store.ensureArchetype(["id", "position"]).insert({ position: { x: 1, y: 2, z: 3 } }); - const snapshot: any = store.toData(true); + const snapshot: any = store.toData({ copy: true }); expect(snapshot.version).toBe(ECS_SNAPSHOT_VERSION); const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); @@ -820,7 +820,7 @@ describe("createStore", () => { // Detached: mutating after the snapshot must NOT change it. const detached = makePopulatedStore(); - const detachedSnapshot = detached.store.toData(true); + const detachedSnapshot = detached.store.toData({ copy: true }); detached.store.update(detached.entity, { health: { current: 1, max: 100 } }); expect(restore(detachedSnapshot)).toBe(100); diff --git a/packages/data/src/ecs/store/public/create-store.ts b/packages/data/src/ecs/store/public/create-store.ts index 8e1fa423..faed3256 100644 --- a/packages/data/src/ecs/store/public/create-store.ts +++ b/packages/data/src/ecs/store/public/create-store.ts @@ -4,6 +4,7 @@ import { ComponentSchemas } from "../../component-schemas.js"; import { StringKeyof } from "../../../types/types.js"; import { RequiredComponents } from "../../required-components.js"; import { Store } from "../store.js"; +import { PersistenceScope, ToDataOptions } from "../../persistence-scope.js"; import { Schema } from "../../../schema/index.js"; import { FromSchemas } from "../../../schema/from-schemas.js"; import { createCore } from "../core/create-core.js"; @@ -175,14 +176,15 @@ export function createStore< const ensureResourceInitialized = (name: string, resourceSchema: Schema & { default: unknown }) => { const resourceId = name as StringKeyof; const isNonPersistent = resourceSchema.nonPersistent; - const componentNames: StringKeyof[] = isNonPersistent - ? ["id" as StringKeyof, resourceId, "nonPersistent" as StringKeyof] - : ["id" as StringKeyof, resourceId]; + const isNonShared = resourceSchema.nonShared; + const componentNames: StringKeyof[] = ["id" as StringKeyof, resourceId]; + if (isNonPersistent) componentNames.push("nonPersistent" as StringKeyof); + if (isNonShared) componentNames.push("nonShared" as StringKeyof); const archetype = core.ensureArchetype(componentNames); if (archetype.rowCount === 0) { - const insertValues = isNonPersistent - ? { [resourceId]: resourceSchema.default, nonPersistent: true } - : { [resourceId]: resourceSchema.default }; + const insertValues: Record = { [resourceId]: resourceSchema.default }; + if (isNonPersistent) insertValues.nonPersistent = true; + if (isNonShared) insertValues.nonShared = true; // Resource singleton inserts bypass index pre-check because // resources are not typically indexed by their schema name and // the singleton row is created exactly once. @@ -232,20 +234,22 @@ export function createStore< }; }; - const updateEntity = (entity: Entity, values: any) => { + const updateEntity = (entity: Entity, values: any): Entity | undefined => { indexRegistry.checkUniqueAvailableForUpdate(entity, values); // `update` can move the entity to a different archetype (when it // adds/removes components), so capture both ends for dispatch. const from = core.locate(entity)?.archetype ?? null; - core.update(entity, values); + const swapped = core.update(entity, values); const to = core.locate(entity)?.archetype ?? null; indexRegistry.applyUpdate(entity, from, to); + return swapped; }; - const deleteEntity = (entity: Entity) => { + const deleteEntity = (entity: Entity): Entity | undefined => { const archetype = core.locate(entity)?.archetype ?? null; - core.delete(entity); + const swapped = core.delete(entity); indexRegistry.applyDelete(entity, archetype); + return swapped; }; const extend = (schema: Store.Schema) => { @@ -342,9 +346,9 @@ export function createStore< seedIndexFromArchetypes(idx); } }, - toData: (copy = false) => core.toData(copy), - fromData: (data: unknown) => { - core.fromData(data); + toData: (options?: ToDataOptions) => core.toData(options), + fromData: (data: unknown, scope?: PersistenceScope) => { + core.fromData(data, scope); for (const [name, resourceSchema] of Object.entries(resourceSchemas)) { ensureResourceInitialized(name, resourceSchema as any); } diff --git a/packages/data/src/ecs/store/store.ts b/packages/data/src/ecs/store/store.ts index 7a451137..38744137 100644 --- a/packages/data/src/ecs/store/store.ts +++ b/packages/data/src/ecs/store/store.ts @@ -13,6 +13,7 @@ import { Undoable } from "../database/undoable.js"; import { Assert } from "../../types/assert.js"; import { Equal } from "../../types/equal.js"; import { FromSchemas } from "../../schema/from-schemas.js"; +import { PersistenceScope, ToDataOptions } from "../persistence-scope.js"; import { ComponentSchemas } from "../component-schemas.js"; import { ResourceSchemas } from "../resource-schemas.js"; import { createStore } from "./public/create-store.js"; @@ -33,8 +34,10 @@ interface BaseStore { * remains valid after subsequent mutations; otherwise it references live * buffers and is only valid until the next mutation. See `Database.toData()` * for the bug this `copy` flag addresses for transient-bearing strategies. + * + * `options.scope` selects which persistent quadrants to emit; omit for all. */ - toData(copy?: boolean): unknown + toData(options?: ToDataOptions): unknown } export interface ReadonlyStore< @@ -92,7 +95,7 @@ export interface Store< readonly indexes: { readonly [K in keyof IX]: Index.Handle }; /** Wipe all entities and reset resources to plugin defaults. O(num_archetypes + num_resources). */ reset(): void; - fromData(data: unknown): void + fromData(data: unknown, scope?: PersistenceScope): void extend(schema: S): S extends Store.Schema ? Store, R & FromSchemas, A & XA, IX & XIX, PK | PartitionKeysOf> : never; } diff --git a/packages/data/src/schema/schema.ts b/packages/data/src/schema/schema.ts index 1be6e33e..7e085d04 100644 --- a/packages/data/src/schema/schema.ts +++ b/packages/data/src/schema/schema.ts @@ -27,9 +27,9 @@ export interface Schema { conditionals?: readonly Conditional[]; nonPersistent?: boolean; // Marks state as local to this client — never replicated to peers. Orthogonal - // to nonPersistent (which is about durability). The database does not yet act - // on this flag; it is declared so application schemas can model local vs. - // shared scope now. See also the built-in `nonShared` component. + // to nonPersistent (which is about durability): together they place an entity + // in one of four quadrants, each with its own entity-id space. See also the + // built-in `nonShared` component and entity/persistence-sharing. nonShared?: boolean; // When true (only valid on a primitive schema), every distinct runtime value // of this component is stored in its own archetype: the value is lifted into