diff --git a/.changeset/22937-cli-organization-ownership-apply.md b/.changeset/22937-cli-organization-ownership-apply.md new file mode 100644 index 00000000000..4b2c410f716 --- /dev/null +++ b/.changeset/22937-cli-organization-ownership-apply.md @@ -0,0 +1,25 @@ +--- +'@objectstack/cli': minor +--- + +feat(cli): `os migrate organization-ownership --apply` — the v18 organization-ownership ceremony's apply step and post-check (ADR-0131 D10, ceremony items 2 to 4) + +Clause-②: yes (widening) + +The command that planned the ceremony now performs it. Without `--apply` nothing changes: the read-only plan, written to a file. With `--apply`: + +- **The backup comes first.** It is the operator's act: pass `--backup-taken`, or answer yes at the prompt. A non-interactive run without the flag is refused before the database is opened. +- **The fates run in order**, each step stopping the run where it meets a blocker: + 1. **Attribution** — every NULL `organization_id` row the plan derives an owner for takes it (a parent, subject or holder anchor; under `single`, the Default Organization). A row with no derivable owner is never assigned one: it stays NULL and is listed by id. + 2. **Authored catalog rows** — permission sets and positions an organization authored in Setup become environment metadata through the metadata door (under `single`; under a wall they are reported), with each position's binding rows carried into its `permissionSets`. + 3. **The verified id→name rewrite** — every `sys_user_permission_set` grant names its set in `permission_set`, then every grant is verified to name a set the security catalog resolves. `permission_set_id` stays. + 4. **Mirror deletion** — only after that verification, read again just before the first delete: the seeded catalog copies (per organization and NULL residue), the binding rows, seeded capabilities and seeded templates. Authored, customized and organization-stamped rows stay. + 5. **Column drops** — `organization_id` leaves the deployment-level objects through the schema-drift ceremony (`os migrate apply`'s machinery), never a hand `ALTER`. A table whose rows still need another step keeps its column, and the run says why. +- **Idempotent and resumable.** Every write is conditioned on the state it changes, and each finished step is checkpointed in `sys_migration` (row `adr-0131-organization-ownership`); a run interrupted mid-step resumes at that step and never re-derives an owner already written. +- **The post-check** re-reads the plan and reports, per table, what remains. The ledger row records `applied_at` once every step has finished and `verified_at` when the post-check also reads zero discrepancies. Exit code 1 while the ceremony is unfinished or the post-check finds a discrepancy. +- A JSON report is written beside the plan (`organization-ownership-apply-TIMESTAMP.json`, or `--out`); an existing file is never overwritten. + +```bash +os migrate organization-ownership --out plan.json # read the plan first +os migrate organization-ownership --apply --backup-taken # then perform it +``` diff --git a/packages/cli/src/commands/migrate/organization-ownership.apply-flags.test.ts b/packages/cli/src/commands/migrate/organization-ownership.apply-flags.test.ts new file mode 100644 index 00000000000..beeef571143 --- /dev/null +++ b/packages/cli/src/commands/migrate/organization-ownership.apply-flags.test.ts @@ -0,0 +1,77 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `os migrate organization-ownership --apply` asks about the backup before it + * opens the database (ADR-0131 D10 ceremony item 2): without `--backup-taken` + * a non-interactive run is refused, and the boot never happens. The flag means + * nothing without `--apply`, and is refused there too. + */ + +import { afterEach, beforeAll, describe, expect, it, vi } from 'vitest'; +import { fileURLToPath } from 'node:url'; + +const boot = vi.hoisted(() => vi.fn(async () => { + throw new Error('the database was opened'); +})); +vi.mock('../../utils/schema-migrate.js', () => ({ bootSchemaStack: boot })); + +type Command = (typeof import('./organization-ownership.js'))['default']; +let MigrateOrganizationOwnership: Command; + +// The command's module graph is large, and oclif reads the package's config on +// the first run: both are paid outside the 5 s default (CASE_TIMEOUT_MS). +beforeAll(async () => { + ({ default: MigrateOrganizationOwnership } = await import('./organization-ownership.js')); +}, 60_000); + +const CASE_TIMEOUT_MS = 60_000; + +const CLI_ROOT = fileURLToPath(new URL('../../..', import.meta.url)); + +async function runJson(argv: string[]): Promise<{ payload: Record; exitCode: unknown }> { + const chunks: string[] = []; + const savedExit = process.exitCode; + vi.spyOn(process.stdout, 'write').mockImplementation(((chunk: unknown, ...rest: unknown[]) => { + chunks.push(String(chunk)); + const cb = rest.find((a) => typeof a === 'function') as (() => void) | undefined; + if (cb) cb(); + return true; + }) as typeof process.stdout.write); + vi.spyOn(process.stderr, 'write').mockImplementation((() => true) as typeof process.stderr.write); + try { + await MigrateOrganizationOwnership.run(argv, { root: CLI_ROOT }); + } catch (error) { + if ((error as { oclif?: unknown }).oclif === undefined) throw error; + } + const exitCode = process.exitCode; + process.exitCode = savedExit; + vi.restoreAllMocks(); + return { payload: JSON.parse(chunks.join('').trim()) as Record, exitCode }; +} + +afterEach(() => { + boot.mockClear(); +}); + +describe('os migrate organization-ownership --apply — the backup precondition', () => { + it('refuses a non-interactive --apply without --backup-taken, before the boot', async () => { + const { payload, exitCode } = await runJson(['--apply', '--json']); + expect(payload).toMatchObject({ error: 'apply_refused', reason: 'backup-unacknowledged' }); + expect(exitCode).toBe(1); + expect(boot).not.toHaveBeenCalled(); + }, CASE_TIMEOUT_MS); + + it('refuses --backup-taken without --apply, before the boot', async () => { + const { payload, exitCode } = await runJson(['--backup-taken', '--json']); + expect(payload).toMatchObject({ error: 'plan_refused', reason: 'flag-without-apply' }); + expect(exitCode).toBe(1); + expect(boot).not.toHaveBeenCalled(); + }, CASE_TIMEOUT_MS); + + it('with --backup-taken, goes on to the boot', async () => { + const { payload } = await runJson(['--apply', '--backup-taken', '--json']); + expect(payload).toMatchObject({ error: 'boot_failed', reason: 'boot-failed' }); + expect(boot).toHaveBeenCalledTimes(1); + expect(boot.mock.calls[0]).toEqual([expect.objectContaining({ deferSchemaDdl: true, composeHostStack: true })]); + }, CASE_TIMEOUT_MS); +}); diff --git a/packages/cli/src/commands/migrate/organization-ownership.ts b/packages/cli/src/commands/migrate/organization-ownership.ts index 327c136b882..66235e2f5c4 100644 --- a/packages/cli/src/commands/migrate/organization-ownership.ts +++ b/packages/cli/src/commands/migrate/organization-ownership.ts @@ -2,6 +2,7 @@ import { writeFile } from 'node:fs/promises'; import { resolve } from 'node:path'; +import { createInterface } from 'node:readline'; import { Command, Flags } from '@oclif/core'; import chalk from 'chalk'; import { resolveTenancyPosture } from '@objectstack/types'; @@ -24,6 +25,29 @@ import { OrganizationOwnershipPlanRefusal, type OrganizationOwnershipPlan, } from '../../utils/organization-ownership-plan.js'; +import { + applyOrganizationOwnership, + checkpointFromDetails, + ORGANIZATION_OWNERSHIP_MIGRATION_ID, + type ApplyExecutor, + type CeremonyLedgerPort, + type ColumnDropOutcome, + type OrganizationOwnershipApplyReport, +} from '../../utils/organization-ownership-apply.js'; + +/** Who the ceremony's metadata saves name as their actor. */ +const CEREMONY_ACTOR = 'os migrate organization-ownership'; + +async function confirm(question: string): Promise { + if (!process.stdin.isTTY) return false; // non-interactive → require --backup-taken + const rl = createInterface({ input: process.stdin, output: process.stdout }); + try { + const answer: string = await new Promise((resolve) => rl.question(question, resolve)); + return /^y(es)?$/i.test(answer.trim()); + } finally { + rl.close(); + } +} /** * `os migrate organization-ownership` — the v18 organization-ownership @@ -53,20 +77,43 @@ import { * Bare `os migrate` is the schema-drift plan (#2186) and stays exactly that. * Every data step in this family is a named subcommand whose bare run is the * read-only preview and whose `--apply` is the only writing mode; the ADR's - * `--plan` names this mode. C7b adds `--apply` (fate order: attribution, the - * verified id-to-name rewrite, mirror deletion, column drops) and the - * post-check to THIS command; until then it has no writing mode at all. + * `--plan` names this mode. + * + * ## `--apply` (ADR-0131 D10 ceremony items 2 to 4) + * + * The writing mode. It asks about the backup first — the operator's act and + * the precondition (D10 item 2): `--backup-taken`, or a yes at the prompt; a + * non-interactive run without the flag is refused before the boot. It then + * performs the fates in order — attribution, the verified id→name rewrite, + * mirror deletion, column drops — through `../../utils/organization-ownership-apply.ts`, + * checkpointing each finished step in the deployment ledger (`sys_migration`, + * row {@link ORGANIZATION_OWNERSHIP_MIGRATION_ID}) so an interrupted run + * resumes where it stopped, and ends with the post-check. The ledger row is + * the ceremony's marker: `applied_at` once every step has finished, + * `verified_at` when the post-check also reads zero discrepancies. A report + * file is written beside the plan's, never over an existing one. + * + * The apply boot composes the deployment's served object set + * (`composeHostStack`, with the auth-gated security declarations and the + * served platform) so the security catalog it verifies names against is the + * one the deployment serves, and the schema-drift ceremony proposes the + * column drops of exactly the objects this release declares without the + * column. It still holds the boot's own DDL back (`deferSchemaDdl`): the only + * schema change the ceremony makes is the drops it asked for. */ export default class MigrateOrganizationOwnership extends Command { static override description = - 'The v18 organization-ownership ceremony\'s read-only plan (ADR-0131 D10): per table, the fate, the row counts, the ' + - 'rows whose owner cannot be derived (by id), and which tables will receive NOT NULL. Writes the plan to a file; ' + - 'never changes a row or a column. Refuses — naming it — any table it cannot enumerate.'; + 'The v18 organization-ownership ceremony (ADR-0131 D10). Without --apply, the read-only plan: per table, the fate, ' + + 'the row counts, the rows whose owner cannot be derived (by id), and which tables will receive NOT NULL, written ' + + 'to a file — never a row or a column changed; it refuses, naming it, any table it cannot enumerate. With --apply ' + + '(and a backup): attribution, the verified id-to-name rewrite, mirror deletion and column drops, in that order, ' + + 'resumable from a recorded checkpoint, then the post-check.'; static override examples = [ '$ os migrate organization-ownership', '$ os migrate organization-ownership --out plans/v18-ownership.json', '$ os migrate organization-ownership --database-url postgres://… --json', + '$ os migrate organization-ownership --apply --backup-taken', ]; static override flags = { @@ -76,10 +123,23 @@ export default class MigrateOrganizationOwnership extends Command { }), out: Flags.string({ description: - 'Where to write the plan (default: organization-ownership-plan-TIMESTAMP.json in the current directory). ' + + 'Where to write the plan, or with --apply the apply report (default: organization-ownership-plan-TIMESTAMP.json, ' + + 'or organization-ownership-apply-TIMESTAMP.json, in the current directory). ' + 'An existing file is never overwritten', }), - json: Flags.boolean({ description: 'Also print the plan document on stdout' }), + json: Flags.boolean({ description: 'Also print the plan (or, with --apply, the apply report) on stdout' }), + apply: Flags.boolean({ + description: + 'Perform the ceremony: attribution, the verified id-to-name rewrite, mirror deletion, column drops, then the ' + + 'post-check. Idempotent; an interrupted run resumes from its recorded checkpoint. Requires a backup', + default: false, + }), + 'backup-taken': Flags.boolean({ + description: + 'With --apply: state that a backup of this database has been taken. The platform takes none and assumes ' + + 'none; without this flag the run asks, and a non-interactive run is refused', + default: false, + }), }; async run(): Promise { @@ -88,7 +148,12 @@ export default class MigrateOrganizationOwnership extends Command { const json = Boolean(flags.json); const refuse = async (payload: Record, message: string): Promise => { if (json) { - await emitJson({ error: payload.reason === 'boot-failed' ? 'boot_failed' : 'plan_refused', ...payload }, 1, { compact: true }); + const error = payload.reason === 'boot-failed' + ? 'boot_failed' + : payload.reason === 'apply-failed' + ? 'apply_failed' + : flags.apply ? 'apply_refused' : 'plan_refused'; + await emitJson({ error, ...payload }, 1, { compact: true }); return; } printError(message); @@ -103,6 +168,15 @@ export default class MigrateOrganizationOwnership extends Command { return; } + if (flags.apply) { + await this.runApply(flags, posture, refuse); + return; + } + if (flags['backup-taken']) { + await refuse({ reason: 'flag-without-apply', detail: '--backup-taken is only meaningful with --apply' }, '--backup-taken is only meaningful with --apply.'); + return; + } + if (!json) { printHeader('Migrate · organization-ownership (plan)'); printStep('Booting data stack (read-only)…'); @@ -199,4 +273,212 @@ export default class MigrateOrganizationOwnership extends Command { printSuccess(`Plan written to ${target} — nothing in the database was changed.`); console.log(chalk.dim(` ${timer.display()}`)); } + + /** `--apply`: the backup precondition, the boot, the ceremony, the report file. */ + private async runApply( + flags: { 'database-url'?: string; out?: string; json?: boolean; 'backup-taken'?: boolean }, + posture: string, + refuse: (payload: Record, message: string) => Promise, + ): Promise { + const json = Boolean(flags.json); + const timer = createTimer(); + if (!flags['backup-taken']) { + const target = flags['database-url'] ?? process.env.OS_DATABASE_URL ?? 'the project database'; + const yes = !json && (await confirm( + `${chalk.yellow('Backup')}: this run rewrites ownership, deletes mirror rows and drops columns in ${target}. ` + + 'Has a backup of it been taken? [y/N] ', + )); + if (!yes) { + await refuse( + { reason: 'backup-unacknowledged', detail: 'pass --backup-taken once a backup of this database exists' }, + 'Refused: --apply needs a backup of this database first. Take one, then re-run with --backup-taken ' + + '(or answer yes at the prompt). The platform takes no backup and assumes none.', + ); + return; + } + } + + if (!json) { + printHeader('Migrate · organization-ownership (apply)'); + printStep('Booting the deployment\'s served stack (schema DDL held back)…'); + } + let stack; + try { + stack = await bootSchemaStack({ + jsonOutput: json, + ...(flags['database-url'] ? { databaseUrl: flags['database-url'] } : {}), + deferSchemaDdl: true, + composeHostStack: true, + composeServedPlatform: true, + composeAuthGatedSecurity: true, + }); + } catch (error: unknown) { + await refuse({ reason: 'boot-failed', detail: String((error as Error)?.message ?? error) }, String((error as Error)?.message ?? error)); + return; + } + + let report: OrganizationOwnershipApplyReport; + try { + const exec = resolvePlannedDriverExec(stack.driver); + if (!stack.driver || !exec) { + await refuse( + { reason: 'driver-unsupported', detail: 'no SQL driver with a raw-SQL seam is active' }, + 'No SQL driver with a raw-SQL seam is active. The ceremony supports SQLite, PostgreSQL and MySQL.', + ); + return; + } + if (stack.tableAbsent('sys_migration')) { + await refuse( + { reason: 'ledger-absent', table: 'sys_migration', detail: 'the deployment ledger the ceremony checkpoints into does not exist' }, + 'Refused: this database has no sys_migration table, where the ceremony records its checkpoint. Boot this ' + + 'release against it once (its schema sync creates the table), then re-run.', + ); + return; + } + const { normalizeRows } = await import('@objectstack/metadata-protocol'); + const { createSecurityCatalogReader } = await import('@objectstack/core'); + const { readDataMigrationFlag, recordDataMigrationRun } = await import('@objectstack/platform-objects/system'); + const kernel = stack.kernel as { getService(name: string): unknown }; + const engine = kernel.getService('objectql') as Parameters[0] & { registry?: unknown }; + const protocol = kernel.getService('protocol') as { + saveMetaItem?: (request: Record) => Promise; + } | undefined; + const catalog = createSecurityCatalogReader({ + registry: engine.registry as never, + metadata: kernel.getService('metadata') as never, + }); + const client = (stack.driver.config as { client?: unknown } | undefined)?.client; + const executor: ApplyExecutor = { + client: client === undefined ? undefined : String(client), + query: async (sql, params) => normalizeRows(await exec(sql, params ? [...params] : [])), + execute: async (sql, params) => { + await exec(sql, params ? [...params] : []); + }, + }; + const ledger: CeremonyLedgerPort = { + read: async () => checkpointFromDetails((await readDataMigrationFlag(engine, ORGANIZATION_OWNERSHIP_MIGRATION_ID))?.details), + write: async (checkpoint, verdict) => { + await recordDataMigrationRun(engine, { + migrationId: ORGANIZATION_OWNERSHIP_MIGRATION_ID, + passed: verdict.complete && verdict.blocking === 0, + applied: verdict.complete, + blocking: verdict.blocking, + advisory: verdict.advisory, + details: { checkpoint }, + }); + }, + }; + const liveStack = stack; + report = await applyOrganizationOwnership({ + executor, + posture, + database: stack.dbLabel, + catalog, + door: { + save: async (type, name, body) => { + if (typeof protocol?.saveMetaItem !== 'function') { + throw new Error('no metadata protocol is composed on this stack'); + } + await protocol.saveMetaItem({ type, name, item: body, actor: CEREMONY_ACTOR }); + }, + }, + columnDrops: { + drop: async (tables) => { + const drift = await liveStack.detectManagedDrift(); + const wanted = new Set(tables); + const entries = drift.filter((d) => { + const op = (d as { op?: { type?: string; column?: string } }).op; + return op?.type === 'drop_column' && op.column === 'organization_id' && wanted.has(String(d.table)); + }); + const outcomes: ColumnDropOutcome[] = []; + const proposed = new Set(entries.map((d) => String(d.table))); + for (const table of tables) { + if (!proposed.has(table)) { + outcomes.push({ table, outcome: 'not-proposed', detail: 'the drift differ proposed no drop for it' }); + } + } + if (entries.length === 0) return outcomes; + const { applied, skipped } = await liveStack.applyMigrationEntries(entries, { allowDestructive: true }); + for (const d of applied) outcomes.push({ table: String(d.table), outcome: 'dropped' }); + for (const d of skipped) { + outcomes.push({ table: String(d.table), outcome: 'failed', detail: String((d as { message?: unknown }).message ?? 'skipped by the schema-drift ceremony') }); + } + return outcomes; + }, + }, + ledger, + ...(json ? {} : { onProgress: (message: string) => printStep(message) }), + }); + } catch (error: unknown) { + if (isExitSignal(error)) throw error; + if (error instanceof OrganizationOwnershipPlanRefusal) { + await refuse( + { reason: error.reason, ...(error.table ? { table: error.table } : {}), detail: error.message }, + `Refused${error.table ? ` (${error.table})` : ''}: ${error.message}`, + ); + return; + } + await refuse( + { reason: 'apply-failed', detail: String((error as Error)?.message ?? error), ...errorCodeFields(error) }, + `${String((error as Error)?.message ?? error)} — every step finished before this one is checkpointed; ` + + 're-run --apply to resume.', + ); + return; + } finally { + await stack.shutdown(); + } + + const target = resolve(flags.out ?? `organization-ownership-apply-${report.generatedAt.replace(/[:.]/g, '-')}.json`); + try { + await writeFile(target, `${JSON.stringify(report, null, 2)}\n`, { flag: 'wx' }); + } catch (error) { + await refuse( + { reason: 'report-file-unwritable', file: target, detail: String((error as Error).message) }, + `The ceremony ran, and its report could not be written to ${target}: ${(error as Error).message}. ` + + 'An existing file is never overwritten; re-run --apply (it resumes, writing nothing new) with --out.', + ); + return; + } + const verdictCode = report.complete && report.blocking === 0 ? 0 : 1; + if (json) { + await emitJson({ file: target, report }, verdictCode); + return; + } + console.log(''); + for (const step of report.steps) { + const mark = step.status === 'done' || step.status === 'checkpointed' ? chalk.green('✓') : step.status === 'blocked' ? chalk.red('✗') : chalk.dim('·'); + console.log(` ${mark} ${step.step.padEnd(22)} ${step.status.padEnd(12)} ${chalk.dim(JSON.stringify(step.summary))}`); + for (const blocker of step.blockers) console.log(` ${chalk.red(blocker)}`); + for (const note of step.notes) console.log(` ${chalk.dim(note)}`); + } + console.log(''); + for (const table of report.tables) { + if (!table.before) continue; + if (table.attributed + table.deleted + table.unattributable.length + table.remaining + table.departures.length === 0 && !table.columnDropped) continue; + console.log( + ` ${chalk.white(table.object.padEnd(34))} ${table.fate.padEnd(16)} attributed ${String(table.attributed).padStart(6)} ` + + `deleted ${String(table.deleted).padStart(6)} ${table.columnDropped ? chalk.green('column dropped ') : ''}` + + `${table.unattributable.length ? chalk.yellow(`reported ${table.unattributable.length} `) : ''}` + + `${table.remaining ? chalk.red(`remaining ${table.remaining}`) : ''}`, + ); + for (const departure of table.departures) { + console.log(chalk.yellow(` ${departure.rows} row(s) still to leave by a step not yet run — ${departure.label}`)); + } + } + console.log(''); + if (report.advisory > 0) { + printWarning(`${report.advisory} row(s) whose owner cannot be derived stay NULL; they are listed by id in the report file.`); + } + if (verdictCode === 0) { + printSuccess(`The ceremony is complete; recorded as ${ORGANIZATION_OWNERSHIP_MIGRATION_ID}. Report: ${target}`); + } else { + printError( + report.complete + ? `The post-check found ${report.blocking} discrepancy(ies). Report: ${target}` + : `The ceremony stopped at a blocker (above). Resolve it and re-run --apply; finished steps are not repeated. Report: ${target}`, + ); + } + console.log(chalk.dim(` ${timer.display()}`)); + if (verdictCode !== 0) this.exit(1); + } } diff --git a/packages/cli/src/utils/organization-ownership-apply.test.ts b/packages/cli/src/utils/organization-ownership-apply.test.ts new file mode 100644 index 00000000000..11608ee836d --- /dev/null +++ b/packages/cli/src/utils/organization-ownership-apply.test.ts @@ -0,0 +1,484 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The ADR-0131 D10 ceremony's acceptance: a fixture database carrying every + * fate comes through `--apply` with effective access IDENTICAL (and a positive + * control: a grant the rewrite would lose reddens the comparison), mirrors + * gone, attributed rows carrying their organization, the unattributable row + * still present and reported, and row counts reconciling. Plus the ceremony's + * guarantees: idempotent, resumable from the recorded checkpoint, gated + * (nothing deleted while a reference does not resolve by name), and no + * deletion outside fate 2. + * + * The shape is the post-stage-8 one: the catalog objects are no longer + * registered, so every read and write here goes through the raw-SQL seam, and + * `permission_set_id` is a plain text column beside `permission_set`. + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import Database from 'better-sqlite3'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { permissionSetDefinitionFromRow } from '@objectstack/core'; +import { + applyOrganizationOwnership, + type ApplyExecutor, + type CatalogPort, + type CeremonyCheckpoint, + type CeremonyLedgerPort, + type ColumnDropPort, + type MetadataDoorPort, + type OrganizationOwnershipApplyReport, +} from './organization-ownership-apply.js'; + +let dir: string; +let file: string; +let db: Database.Database; +/** Every write statement the ceremony issued, in order. */ +let writes: string[]; + +function executor(opts: { failOnceOn?: RegExp } = {}): ApplyExecutor { + let failed = false; + return { + client: 'better-sqlite3', + async query(sql, params = []) { + return db.prepare(sql).all(...(params as unknown[])) as Record[]; + }, + async execute(sql, params = []) { + if (opts.failOnceOn && !failed && opts.failOnceOn.test(sql)) { + failed = true; + throw new Error('SQLITE_IOERR: the process was killed mid-step'); + } + writes.push(sql); + db.prepare(sql).run(...(params as unknown[])); + }, + }; +} + +/** + * The declared catalog (code) plus the environment metadata the door saved. + * Like the registry, it serves a saved definition only from the next boot on + * ({@link boot}), never the moment it is saved. + */ +function catalog(): CatalogPort & MetadataDoorPort & { saved: Map>; boot(): void } { + const declared = new Map>([ + ['permission|member_default', permissionSetDefinitionFromRow(MEMBER_DEFAULT_ROW)], + ['position|org_admin', { name: 'org_admin', label: 'Org Admin', delegatable: false, permissionSets: ['member_default'] }], + ]); + const saved = new Map>(); + let hydrated = new Map>(); + return { + saved, + boot() { + hydrated = new Map(saved); + }, + async resolve(type, name) { + const definition = hydrated.get(`${type}|${name}`) ?? declared.get(`${type}|${name}`); + return definition ? { definition } : undefined; + }, + async save(type, name, body) { + if (declared.has(`${type}|${name}`)) throw new Error(`NAMESPACE_CONFLICT: ${type} ${name} is held by a package`); + saved.set(`${type}|${name}`, body); + }, + }; +} + +function ledger(): CeremonyLedgerPort & { checkpoint: CeremonyCheckpoint | null; verdicts: Array<{ complete: boolean; blocking: number; advisory: number }> } { + const state = { + checkpoint: null as CeremonyCheckpoint | null, + verdicts: [] as Array<{ complete: boolean; blocking: number; advisory: number }>, + async read() { + return state.checkpoint ? (JSON.parse(JSON.stringify(state.checkpoint)) as CeremonyCheckpoint) : null; + }, + async write(checkpoint: CeremonyCheckpoint, verdict: { complete: boolean; blocking: number; advisory: number }) { + state.checkpoint = JSON.parse(JSON.stringify(checkpoint)) as CeremonyCheckpoint; + state.verdicts.push(verdict); + }, + }; + return state; +} + +/** The schema-drift ceremony, as SQLite answers it: the column goes. */ +const columnDrops: ColumnDropPort = { + async drop(tables) { + return tables.map((table) => { + writes.push(`ALTER TABLE ${table} DROP COLUMN organization_id`); + db.exec(`ALTER TABLE "${table}" DROP COLUMN organization_id`); + return { table, outcome: 'dropped' as const }; + }); + }, +}; + +const MEMBER_DEFAULT_ROW = { + name: 'member_default', + label: 'Member', + object_permissions: '{"crm_deal":{"allowRead":true}}', + system_permissions: '[]', +}; + +/** A 17.x-shaped database carrying every fate, on the post-stage-8 shape. */ +function seed(): void { + db.exec(` + CREATE TABLE sys_organization (id TEXT PRIMARY KEY, slug TEXT); + INSERT INTO sys_organization VALUES ('org_default', 'default'), ('org_b', 'b'); + + -- fate 2: the catalog. Per-organization mirrors and NULL residue of one + -- declared set and one declared position, beside Setup-authored rows. + CREATE TABLE sys_permission_set (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT, label TEXT, managed_by TEXT, + active INTEGER, object_permissions TEXT, system_permissions TEXT); + INSERT INTO sys_permission_set VALUES + ('ps_null', NULL, 'member_default', 'Member', 'platform', 1, '{"crm_deal":{"allowRead":true}}', '[]'), + ('ps_org', 'org_default', 'member_default', 'Member', 'package', 1, '{"crm_deal":{"allowRead":true}}', '[]'), + ('ps_auth', 'org_default', 'field_auditor', 'Field Auditor', 'admin', 1, + '{"crm_deal":{"allowRead":true,"allowEdit":true}}', '["export_reports"]'); + CREATE TABLE sys_position (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT, label TEXT, managed_by TEXT, + delegatable INTEGER, active INTEGER); + INSERT INTO sys_position VALUES + ('pos_null', NULL, 'org_admin', 'Org Admin', 'platform', 0, 1), + ('pos_org', 'org_default', 'org_admin', 'Org Admin', 'platform', 0, 1), + ('pos_auth', 'org_default', 'sales_lead', 'Sales Lead', 'admin', 1, 1); + CREATE TABLE sys_position_permission_set (id TEXT PRIMARY KEY, organization_id TEXT, position_id TEXT, permission_set_id TEXT); + INSERT INTO sys_position_permission_set VALUES + ('bind_1', NULL, 'pos_null', 'ps_null'), + ('bind_2', 'org_default', 'pos_org', 'ps_org'), + ('bind_3', 'org_default', 'pos_auth', 'ps_auth'); + CREATE TABLE sys_capability (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT, managed_by TEXT); + INSERT INTO sys_capability VALUES ('cap_seeded', NULL, 'manage_users', 'platform'), ('cap_authored', 'org_default', 'audit_fields', 'admin'); + -- fate 2: templates — seeded mirrors, one customized, one organization-stamped, + -- and one an admin authored (never a mirror: kept for the promotion) + CREATE TABLE sys_email_template (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT, customized INTEGER, managed_by TEXT); + INSERT INTO sys_email_template VALUES + ('tpl_seeded', NULL, 'welcome', 0, 'platform'), + ('tpl_customized', NULL, 'reset_password', 1, 'platform'), + ('tpl_org', 'org_b', 'welcome', 0, 'platform'), + ('tpl_admin', NULL, 'crm.note', 0, 'admin'); + + -- the references: grants by id (the rewrite names them) and assignments by name + CREATE TABLE sys_user_permission_set (id TEXT PRIMARY KEY, organization_id TEXT, user_id TEXT, + permission_set_id TEXT, permission_set TEXT); + INSERT INTO sys_user_permission_set VALUES + ('grant_1', 'org_default', 'usr_1', 'ps_org', NULL), + ('grant_2', NULL, 'usr_2', 'ps_null', NULL), + ('grant_3', 'org_default', 'usr_3', 'ps_auth', NULL), + ('grant_4', 'org_default', 'usr_4', 'ps_null', 'member_default'); + CREATE TABLE sys_business_unit (id TEXT PRIMARY KEY, organization_id TEXT, parent_business_unit_id TEXT); + INSERT INTO sys_business_unit VALUES ('bu_b', 'org_b', NULL); + CREATE TABLE sys_user_position (id TEXT PRIMARY KEY, organization_id TEXT, user_id TEXT, position TEXT, business_unit_id TEXT); + INSERT INTO sys_user_position VALUES + ('up_1', 'org_default', 'usr_5', 'sales_lead', NULL), + ('up_2', NULL, 'usr_6', 'org_admin', NULL); + + -- fate 3: NULL sys_file rows whose sys_attachment holder carries the organization + CREATE TABLE crm_deal (id TEXT PRIMARY KEY, organization_id TEXT, name TEXT); + INSERT INTO crm_deal VALUES ('deal_1', 'org_b', 'Acme'), ('deal_2', NULL, 'Orphan'); + CREATE TABLE sys_attachment (id TEXT PRIMARY KEY, organization_id TEXT, file_id TEXT, parent_object TEXT, parent_id TEXT); + INSERT INTO sys_attachment VALUES ('att_1', NULL, 'file_1', 'crm_deal', 'deal_1'); + CREATE TABLE sys_file (id TEXT PRIMARY KEY, organization_id TEXT, ref_object TEXT, ref_id TEXT); + INSERT INTO sys_file VALUES ('file_1', NULL, NULL, NULL), ('file_2', 'org_b', NULL, NULL); + + -- fate 4: one genuinely unattributable row + CREATE TABLE sys_saved_report (id TEXT PRIMARY KEY, organization_id TEXT); + INSERT INTO sys_saved_report VALUES ('report_orphan', NULL); + + -- fate 1: a D7 object whose column is still there + CREATE TABLE sys_job (id TEXT PRIMARY KEY, organization_id TEXT); + INSERT INTO sys_job VALUES ('job_1', NULL), ('job_2', 'org_b'); + `); +} + +const counts = (): Record => { + const tables = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name`).all() as { name: string }[]; + return Object.fromEntries(tables.map(({ name }) => [name, (db.prepare(`SELECT COUNT(*) AS n FROM "${name}"`).get() as { n: number }).n])); +}; +const rows = (sql: string): Record[] => db.prepare(sql).all() as Record[]; +const digestOf = (definition: Record): string => { + const keys = ['name', 'label', 'objects', 'fields', 'systemPermissions', 'rowLevelSecurity', 'tabPermissions']; + return JSON.stringify(keys.map((k) => definition[k] ?? null)); +}; + +/** + * Effective access BEFORE — the 17.x reading: a grant reaches its set row by + * id, a position reaches its sets through its rows' bindings. Under `single` + * an organization-less row is the one organization's. + */ +function accessBefore(): string[] { + const out = new Set(); + const sets = new Map(rows('SELECT * FROM sys_permission_set').map((r) => [String(r.id), r])); + const sees = (row: Record, org: string): boolean => row.organization_id === null || row.organization_id === org; + for (const grant of rows('SELECT * FROM sys_user_permission_set')) { + const org = String(grant.organization_id ?? 'org_default'); + const set = sets.get(String(grant.permission_set_id)); + if (set && sees(set, org)) out.add(`${grant.user_id}|${org}|${set.name}|${digestOf(permissionSetDefinitionFromRow(set))}`); + } + const bindings = rows('SELECT * FROM sys_position_permission_set'); + for (const up of rows('SELECT * FROM sys_user_position')) { + const org = String(up.organization_id ?? 'org_default'); + for (const position of rows('SELECT * FROM sys_position').filter((p) => p.name === up.position && sees(p, org))) { + for (const binding of bindings.filter((b) => b.position_id === position.id)) { + const set = sets.get(String(binding.permission_set_id)); + if (set) out.add(`${up.user_id}|${org}|${set.name}|${digestOf(permissionSetDefinitionFromRow(set))}`); + } + } + } + return [...out].sort(); +} + +/** Effective access AFTER — the v18 reading, on the next boot: every reference is a name, resolved in the catalog. */ +async function accessAfter(c: ReturnType): Promise { + c.boot(); + const out = new Set(); + const add = async (user: unknown, org: string, name: string): Promise => { + const set = await c.resolve('permission', name); + if (set) out.add(`${user}|${org}|${name}|${digestOf(set.definition as Record)}`); + }; + for (const grant of rows('SELECT * FROM sys_user_permission_set')) { + if (grant.permission_set) await add(grant.user_id, String(grant.organization_id), String(grant.permission_set)); + } + for (const up of rows('SELECT * FROM sys_user_position')) { + const position = await c.resolve('position', String(up.position)); + for (const name of (position?.definition.permissionSets as string[] | undefined) ?? []) { + await add(up.user_id, String(up.organization_id), name); + } + } + return [...out].sort(); +} + +const apply = ( + c: ReturnType, + l: ReturnType, + opts: { posture?: string; executor?: ApplyExecutor } = {}, +): Promise => { + c.boot(); + return applyOrganizationOwnership({ + executor: opts.executor ?? executor(), + posture: opts.posture ?? 'single', + database: file, + catalog: c, + door: c, + columnDrops, + ledger: l, + now: () => new Date('2026-10-12T00:00:00Z'), + }); +}; + +const tableOf = (report: OrganizationOwnershipApplyReport, object: string) => { + const found = report.tables.find((t) => t.object === object); + if (!found) throw new Error(`no ${object} in the report`); + return found; +}; + +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'os-c7b-apply-')); + file = join(dir, 'fixture.db'); + db = new Database(file); + writes = []; + seed(); +}); + +afterEach(() => { + db.close(); + rmSync(dir, { recursive: true, force: true }); +}); + +describe('ADR-0131 D10 --apply — the acceptance fixture', () => { + it('comes through with effective access identical, mirrors gone, owners written, the orphan reported, counts reconciling', async () => { + const before = accessBefore(); + const countsBefore = counts(); + const c = catalog(); + const l = ledger(); + + const report = await apply(c, l); + + expect(report.steps.map((s) => [s.step, s.status])).toEqual([ + ['attribution', 'done'], + ['authored-conversion', 'done'], + ['rewrite', 'done'], + ['mirror-deletion', 'done'], + ['column-drops', 'done'], + ]); + expect(report.complete).toBe(true); + expect(report.blocking).toBe(0); + expect(l.verdicts.at(-1)).toEqual({ complete: true, blocking: 0, advisory: 1 }); + + // Effective access: identical, read the 17.x way before and the v18 way after. + expect(before.length).toBeGreaterThan(0); + expect(await accessAfter(c)).toEqual(before); + + // The authored catalog rows became environment metadata, through the door. + expect(c.saved.get('permission|field_auditor')).toMatchObject({ name: 'field_auditor', systemPermissions: ['export_reports'] }); + expect(c.saved.get('position|sales_lead')).toMatchObject({ name: 'sales_lead', delegatable: true, permissionSets: ['field_auditor'] }); + expect(c.saved.has('position|org_admin')).toBe(false); + + // Mirrors gone; authored, customized and organization-stamped rows stay. + expect(rows('SELECT id FROM sys_permission_set').map((r) => r.id)).toEqual(['ps_auth']); + expect(rows('SELECT id FROM sys_position').map((r) => r.id)).toEqual(['pos_auth']); + expect(rows('SELECT id FROM sys_position_permission_set')).toEqual([]); + expect(rows('SELECT id FROM sys_capability').map((r) => r.id)).toEqual(['cap_authored']); + expect(rows('SELECT id FROM sys_email_template ORDER BY id').map((r) => r.id)).toEqual(['tpl_admin', 'tpl_customized', 'tpl_org']); + + // Every grant names its set; the id column is still there (C7b's later drop). + expect(rows('SELECT id, permission_set, permission_set_id FROM sys_user_permission_set ORDER BY id')).toEqual([ + { id: 'grant_1', permission_set: 'member_default', permission_set_id: 'ps_org' }, + { id: 'grant_2', permission_set: 'member_default', permission_set_id: 'ps_null' }, + { id: 'grant_3', permission_set: 'field_auditor', permission_set_id: 'ps_auth' }, + { id: 'grant_4', permission_set: 'member_default', permission_set_id: 'ps_null' }, + ]); + + // Attributed rows carry their organization — the anchor's, else the Default Organization. + expect(rows(`SELECT organization_id FROM sys_file WHERE id = 'file_1'`)).toEqual([{ organization_id: 'org_b' }]); + expect(rows(`SELECT organization_id FROM sys_attachment WHERE id = 'att_1'`)).toEqual([{ organization_id: 'org_b' }]); + expect(rows(`SELECT organization_id FROM crm_deal WHERE id = 'deal_2'`)).toEqual([{ organization_id: 'org_default' }]); + expect(rows(`SELECT organization_id FROM sys_user_permission_set WHERE id = 'grant_2'`)).toEqual([{ organization_id: 'org_default' }]); + + // The unattributable row is still present, still NULL, and reported by id. + expect(rows(`SELECT organization_id FROM sys_saved_report`)).toEqual([{ organization_id: null }]); + expect(tableOf(report, 'sys_saved_report').unattributable.map((u) => u.id)).toEqual(['report_orphan']); + + // Fate 1: the column is gone through the column-drop port. + expect(tableOf(report, 'sys_job').columnDropped).toBe(true); + expect(rows(`SELECT name FROM pragma_table_info('sys_job')`).map((r) => r.name)).toEqual(['id']); + + // Row counts reconcile: only fate 2 deletes, and exactly what it reports. + const countsAfter = counts(); + for (const table of report.tables) { + if (!table.before || !table.after) continue; + expect(table.before.total - table.deleted, table.object).toBe(table.after.total); + expect(countsAfter[table.object], table.object).toBe(table.after.total); + expect(countsBefore[table.object], table.object).toBe(table.before.total); + if (table.fate !== 'mirror-deletion') expect(table.deleted, table.object).toBe(0); + } + expect(writes.filter((w) => /^DELETE/.test(w)).every((w) => + /sys_permission_set|sys_position|sys_position_permission_set|sys_capability|sys_email_template/.test(w))).toBe(true); + }); + + it('positive control: a grant the rewrite would lose reddens the effective-access pin', async () => { + const before = accessBefore(); + const c = catalog(); + await apply(c, ledger()); + expect(await accessAfter(c)).toEqual(before); + + db.exec(`UPDATE sys_user_permission_set SET permission_set = NULL WHERE id = 'grant_3'`); + const after = await accessAfter(c); + expect(after).not.toEqual(before); + expect(before.filter((g) => !after.includes(g))).toEqual([expect.stringMatching(/^usr_3\|org_default\|field_auditor\|/)]); + }); +}); + +describe('ADR-0131 D10 --apply — idempotent, resumable, gated', () => { + it('a second run resumes from the checkpoint and writes nothing', async () => { + const c = catalog(); + const l = ledger(); + await apply(c, l); + const hash = createHash('sha256').update(readFileSync(file)).digest('hex'); + writes = []; + + const again = await apply(c, l); + + expect(again.resumedFrom).toEqual(['attribution', 'authored-conversion', 'rewrite', 'mirror-deletion', 'column-drops']); + expect(again.steps.every((s) => s.status === 'checkpointed')).toBe(true); + expect(again.complete).toBe(true); + expect(writes).toEqual([]); + expect(createHash('sha256').update(readFileSync(file)).digest('hex')).toBe(hash); + }); + + it('a run killed inside mirror deletion resumes there, never re-deriving an owner it already wrote', async () => { + const c = catalog(); + const l = ledger(); + await expect(apply(c, l, { executor: executor({ failOnceOn: /^DELETE/ }) })).rejects.toThrow(/killed mid-step/); + expect(Object.keys(l.checkpoint!.completed)).toEqual(['attribution', 'authored-conversion', 'rewrite']); + expect(rows('SELECT COUNT(*) AS n FROM sys_position_permission_set')).toEqual([{ n: 3 }]); + writes = []; + + const resumed = await apply(c, l); + + expect(resumed.resumedFrom).toEqual(['attribution', 'authored-conversion', 'rewrite']); + expect(resumed.steps.map((s) => s.status)).toEqual(['checkpointed', 'checkpointed', 'checkpointed', 'done', 'done']); + expect(writes.some((w) => /^UPDATE/.test(w))).toBe(false); + expect(resumed.complete).toBe(true); + expect(rows('SELECT COUNT(*) AS n FROM sys_position_permission_set')).toEqual([{ n: 0 }]); + }); + + it('reads the gate again before the first delete: a grant written between runs keeps every mirror', async () => { + const c = catalog(); + const l = ledger(); + await expect(apply(c, l, { executor: executor({ failOnceOn: /^DELETE/ }) })).rejects.toThrow(/killed mid-step/); + db.exec(`INSERT INTO sys_user_permission_set VALUES ('grant_late', 'org_default', 'usr_8', 'ps_gone', NULL)`); + const countsBefore = counts(); + + const resumed = await apply(c, l); + + const deletion = resumed.steps.find((s) => s.step === 'mirror-deletion')!; + expect(deletion.status).toBe('blocked'); + expect(deletion.blockers.join('\n')).toMatch(/grant_late/); + expect(counts()).toEqual(countsBefore); + expect(resumed.complete).toBe(false); + }); + + it('deletes nothing while a reference does not resolve by name — the gate of fate 2', async () => { + // Another organization's set row: a name carries no organization, so it is never written. + db.exec(` + INSERT INTO sys_permission_set VALUES ('ps_b', 'org_b', 'b_only', 'B', 'admin', 1, '{}', '[]'); + INSERT INTO sys_user_permission_set VALUES ('grant_x', 'org_default', 'usr_9', 'ps_b', NULL); + `); + const countsBefore = counts(); + const l = ledger(); + + const report = await apply(catalog(), l); + + const rewrite = report.steps.find((s) => s.step === 'rewrite')!; + expect(rewrite.status).toBe('blocked'); + expect(rewrite.blockers.join('\n')).toMatch(/grant_x/); + expect(report.steps.find((s) => s.step === 'mirror-deletion')!.status).toBe('not-reached'); + expect(writes.filter((w) => /^DELETE/.test(w))).toEqual([]); + expect(counts()).toEqual(countsBefore); + expect(report.complete).toBe(false); + expect(report.blocking).toBeGreaterThan(0); + expect(l.verdicts.at(-1)!.complete).toBe(false); + }); + + it('under a wall: an anchorless row is reported, never assigned, and authored rows are not converted', async () => { + const c = catalog(); + const report = await apply(c, ledger(), { posture: 'isolated' }); + + expect(rows(`SELECT organization_id FROM crm_deal WHERE id = 'deal_2'`)).toEqual([{ organization_id: null }]); + expect(tableOf(report, 'crm_deal').unattributable.map((u) => u.id)).toEqual(['deal_2']); + expect(rows(`SELECT organization_id FROM sys_file WHERE id = 'file_1'`)).toEqual([{ organization_id: 'org_b' }]); + expect(c.saved.size).toBe(0); + expect(report.steps.find((s) => s.step === 'mirror-deletion')!.status).toBe('not-reached'); + expect(rows('SELECT COUNT(*) AS n FROM sys_permission_set')).toEqual([{ n: 3 }]); + }); + + it('does not certify a ceremony whose departures have not left: the global settings rung keeps verified_at null', async () => { + db.exec(` + CREATE TABLE sys_setting (id TEXT PRIMARY KEY, organization_id TEXT, namespace TEXT, key TEXT, scope TEXT); + INSERT INTO sys_setting VALUES ('set_global', NULL, 'mail', 'transport', 'global'), ('set_tenant', 'org_b', 'mail', 'from', 'tenant'); + `); + const l = ledger(); + const report = await apply(catalog(), l); + + expect(report.complete).toBe(true); + expect(report.blocking).toBe(1); + expect(l.verdicts.at(-1)).toMatchObject({ complete: true, blocking: 1 }); + const setting = tableOf(report, 'sys_setting'); + expect(setting.remaining).toBe(1); + expect(setting.departures).toEqual([expect.objectContaining({ id: 'global-rung-move', as: 'moved', rows: 1 })]); + expect(setting.notNull.willReceive).toBe(false); + expect(rows(`SELECT id FROM sys_setting ORDER BY id`).map((r) => r.id)).toEqual(['set_global', 'set_tenant']); + }); + + it('holds a column whose rows another stage owns, and reports it', async () => { + db.exec(` + CREATE TABLE sys_metadata (id TEXT PRIMARY KEY, organization_id TEXT, type TEXT, name TEXT, state TEXT, package_id TEXT); + INSERT INTO sys_metadata VALUES ('md_1', 'org_b', 'view', 'deal_list', 'active', NULL); + `); + const report = await apply(catalog(), ledger()); + + const drops = report.steps.find((s) => s.step === 'column-drops')!; + expect(drops.status).toBe('blocked'); + expect(drops.blockers.join('\n')).toMatch(/sys_metadata\.organization_id is not dropped/); + expect(rows(`SELECT name FROM pragma_table_info('sys_metadata') WHERE name = 'organization_id'`)).toHaveLength(1); + expect(rows(`SELECT name FROM pragma_table_info('sys_job') WHERE name = 'organization_id'`)).toHaveLength(0); + }); +}); diff --git a/packages/cli/src/utils/organization-ownership-apply.ts b/packages/cli/src/utils/organization-ownership-apply.ts new file mode 100644 index 00000000000..d4fa3986d13 --- /dev/null +++ b/packages/cli/src/utils/organization-ownership-apply.ts @@ -0,0 +1,972 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { + convertPositionBindingRows, + permissionSetDefinitionFromRow, + positionDefinitionFromRow, +} from '@objectstack/core'; +import { + inventoryEntryFor, + type InventoryEntry, + type RowPredicate, +} from './organization-ownership-inventory.js'; +import { + buildOrganizationOwnershipPlan, + columnListSql, + columnNameOf, + dialectOf, + predicateSql, + Sql, + type AttributionSink, + type OrganizationOwnershipPlan, + type PlanReader, + type TablePlan, + type UnattributableRow, +} from './organization-ownership-plan.js'; + +/** + * The ADR-0131 D10 ceremony's apply step — ceremony items 2 to 4: the backup + * precondition is the command's, and this module performs the fates, in fate + * order, then the post-check. + * + * 1. `attribution` (fate 3) — every NULL row the plan derives an owner for + * takes it. The owners come from the plan's own walk + * ({@link AttributionSink}): the apply writes exactly what the plan + * reported, derived once, never a second derivation. Repeated to a fixed + * point, so a child whose parent was itself NULL in the first pass takes + * the parent's owner in the next. + * 2. `authored-conversion` — the catalog rows an organization authored + * (the inventory's `authored` categories) become environment metadata + * through the metadata door, by core's row→definition mappers; the + * position bindings are carried into the position definitions + * (`convertPositionBindingRows`). Only under `single` (ADR-0131 D3: a + * Setup catalog write is an environment metadata write under `single` and + * refused under a wall); under a wall the rows are reported. + * 3. `rewrite` — the verified id→name rewrite (D4): + * `sys_user_permission_set.permission_set` is named from the set row its + * `permission_set_id` points at, then EVERY grant is verified to name a set + * the security catalog resolves. The id column stays (C7b drops it). + * 4. `mirror-deletion` (fate 2) — gated on the rewrite's verification, which + * is read again immediately before the first delete. Only rows of a + * `mirror-deletion` object that match its mirror selector and none of its + * named categories are deleted. + * 5. `column-drops` (fate 1) — `organization_id` leaves the D7 objects + * through the schema-drift ceremony ({@link ColumnDropPort}), never a hand + * `ALTER`. + * + * Then the post-check re-runs the plan and reports, per table, what remains. + * + * ## Idempotent and resumable + * + * Every write is conditioned on the state it changes — an owner is written + * only where `organization_id IS NULL`, a name only where `permission_set` is + * empty, a delete selects only mirrors — so a second run writes nothing the + * first one wrote. On top of that, each step that finishes is recorded in the + * ceremony's checkpoint ({@link CeremonyLedgerPort}) and is not run again: a + * run interrupted inside step N resumes at step N. A step that stops on a + * blocker stops the ceremony there — fate order is the contract — and the + * report names the blocker and its remedy. + * + * ## What it never does + * + * ⛔ Delete a row outside fate 2. ⛔ Give an unattributable row an owner: + * owners come only from the plan's anchors and, under `single`, the Default + * Organization. ⛔ Run at boot: the command is the only caller. + */ + +/** The `sys_migration` row the ceremony checkpoints into and completes. */ +export const ORGANIZATION_OWNERSHIP_MIGRATION_ID = 'adr-0131-organization-ownership'; + +export const CEREMONY_STEPS = ['attribution', 'authored-conversion', 'rewrite', 'mirror-deletion', 'column-drops'] as const; +export type CeremonyStep = (typeof CEREMONY_STEPS)[number]; + +/** The raw-SQL seam: the plan's reader plus the one write verb. */ +export interface ApplyExecutor extends PlanReader { + execute(sql: string, params?: readonly unknown[]): Promise; +} + +/** The security catalog read (core's `SecurityCatalogReader` satisfies it). */ +export interface CatalogPort { + resolve(type: 'position' | 'permission', name: string): Promise<{ definition: Readonly> } | undefined>; +} + +/** The metadata door: an environment metadata save. Throws when the door refuses. */ +export interface MetadataDoorPort { + save(type: 'position' | 'permission', name: string, body: Record): Promise; +} + +/** One table's column drop, as the schema-drift ceremony answered it. */ +export interface ColumnDropOutcome { + table: string; + outcome: 'dropped' | 'not-proposed' | 'failed'; + detail?: string; +} + +/** Drops `organization_id` on the named tables through the schema-drift ceremony (ADR-0120 D4). */ +export interface ColumnDropPort { + drop(tables: readonly string[]): Promise; +} + +/** What the ceremony records after each finished step. */ +export interface CeremonyCheckpoint { + ceremony: 'ADR-0131 D10'; + inventoryDigest: string; + completed: Partial }>>; +} + +/** Where the checkpoint lives (the command: the `sys_migration` row). */ +export interface CeremonyLedgerPort { + read(): Promise; + write(checkpoint: CeremonyCheckpoint, verdict: { complete: boolean; blocking: number; advisory: number }): Promise; +} + +export interface StepReport { + step: CeremonyStep; + /** `checkpointed`: finished by an earlier run; `not-reached`: an earlier step stopped the run. */ + status: 'done' | 'checkpointed' | 'blocked' | 'not-reached'; + summary: Record; + /** Each blocker, with its remedy, in the operator's terms. */ + blockers: string[]; + /** Reported, never blocking. */ + notes: string[]; +} + +export interface TableOutcome { + object: string; + fate: TablePlan['fate']; + before: { total: number; organizationNull: number } | null; + after: { total: number; organizationNull: number } | null; + attributed: number; + deleted: number; + /** Fate 1: the column is gone after this run. */ + columnDropped: boolean; + /** Rows still NULL and listed by id — reported, never guessed. */ + unattributable: UnattributableRow[]; + /** + * Rows that leave this table by a step this ceremony has not yet performed + * (a move, or a mirror its own stage removes): each counts as remaining + * until it is gone, and its label is the remedy. + */ + departures: Array<{ id: string; label: string; as: 'mirror-deletion' | 'moved'; rows: number }>; + /** Post-check: what is left for this table's fate, `0` when the fate is done. */ + remaining: number; + notNull: TablePlan['notNull']; +} + +export interface OrganizationOwnershipApplyReport { + kind: 'organization-ownership-apply'; + ceremony: 'ADR-0131 D10'; + migrationId: typeof ORGANIZATION_OWNERSHIP_MIGRATION_ID; + generatedAt: string; + database: string; + posture: string; + inventoryDigest: string; + /** Steps an earlier run finished, read from the checkpoint. */ + resumedFrom: CeremonyStep[]; + steps: StepReport[]; + tables: TableOutcome[]; + /** Every step finished. */ + complete: boolean; + /** Blockers (an unfinished ceremony) or post-check discrepancies (a finished one). Zero is the verdict. */ + blocking: number; + /** Rows reported, never guessed: unattributable rows and rows held for another stage. */ + advisory: number; +} + +export interface ApplyOptions { + executor: ApplyExecutor; + posture: string; + database: string; + catalog: CatalogPort; + door: MetadataDoorPort; + columnDrops: ColumnDropPort; + ledger: CeremonyLedgerPort; + inventory?: readonly InventoryEntry[]; + now?: () => Date; + /** One line per milestone, for the command's human face. */ + onProgress?: (message: string) => void; +} + +const ORG = 'organization_id'; +const IN_CHUNK = 200; +const PAGE = 500; +const MAX_ATTRIBUTION_PASSES = 8; + +const GRANTS = 'sys_user_permission_set'; +const SETS = 'sys_permission_set'; +const POSITIONS = 'sys_position'; +const BINDINGS = 'sys_position_permission_set'; +const CAPABILITIES = 'sys_capability'; + +const text = (value: unknown): string | null => + value === null || value === undefined || String(value) === '' ? null : String(value); + +/** Shared state of one run. */ +interface Run { + opts: ApplyOptions; + sql: Sql; + plan: OrganizationOwnershipPlan; + columns: Map | null>; + attributed: Map; + deleted: Map; + /** + * What the door accepted this run. The registry hydrates environment + * metadata at boot, so a definition saved a moment ago is not in it until + * the next one; the door's acceptance is what makes it resolve now. + */ + saved: Map>>; +} + +/** The catalog read, then what the door accepted this run. */ +async function resolveName(run: Run, type: 'position' | 'permission', name: string): Promise<{ definition: Readonly> } | undefined> { + const found = await run.opts.catalog.resolve(type, name); + if (found) return found; + const saved = run.saved.get(`${type}|${name}`); + return saved ? { definition: saved } : undefined; +} + +async function saveDefinition(run: Run, type: 'position' | 'permission', name: string, body: Record): Promise { + await run.opts.door.save(type, name, body); + run.saved.set(`${type}|${name}`, body); +} + +async function columnsOf(run: Run, table: string): Promise | null> { + if (run.columns.has(table)) return run.columns.get(table)!; + const present = run.plan.tables.some((t) => t.tables.includes(table)); + let cols: Set | null = null; + if (present) { + const rows = await run.sql.read(table, columnListSql(run.sql.dialect), [table]); + cols = new Set(rows.map(columnNameOf).filter((c): c is string => c !== null)); + } + run.columns.set(table, cols); + return cols; +} + +/** Every row of `table` matching `where`, keyset-paged by id. */ +async function readAll(run: Run, table: string, select: readonly string[], where: { text: string; params: unknown[] } = { text: '1 = 1', params: [] }): Promise[]> { + const { sql } = run; + const t = sql.q(table, table); + const id = sql.q('id', table); + const cols = [...new Set(['id', ...select])].map((c) => `${sql.q(c, table)} AS ${sql.q(c, table)}`).join(', '); + const out: Record[] = []; + let after: unknown; + for (;;) { + const rows = await sql.read( + table, + `SELECT ${cols} FROM ${t} WHERE (${where.text})${after === undefined ? '' : ` AND ${id} > ?`} ORDER BY ${id} LIMIT ${PAGE}`, + after === undefined ? where.params : [...where.params, after], + ); + out.push(...rows); + if (rows.length < PAGE) return out; + after = rows[rows.length - 1]!.id; + } +} + +async function execute(run: Run, table: string, statement: string, params: readonly unknown[]): Promise { + try { + await run.opts.executor.execute(statement, params); + } catch (error) { + throw new Error(`The write to ${table} failed: ${error instanceof Error ? error.message : String(error)}`, { cause: error }); + } +} + +const planFor = (run: Run, extra: { onAttribution?: AttributionSink } = {}): Promise => + buildOrganizationOwnershipPlan({ + reader: run.opts.executor, + posture: run.opts.posture, + database: run.opts.database, + ...(run.opts.inventory ? { inventory: run.opts.inventory } : {}), + ...(run.opts.now ? { now: run.opts.now() } : {}), + ...extra, + }); + +const nullsUnderAttribution = (plan: OrganizationOwnershipPlan): number => + plan.tables.filter((t) => t.fate === 'attribution').reduce((n, t) => n + (t.rows?.organizationNull ?? 0), 0); + +// ── Step 1: attribution ────────────────────────────────────────────────────── + +async function runAttribution(run: Run): Promise { + let passes = 0; + let total = 0; + let lastNulls = Number.POSITIVE_INFINITY; + for (;;) { + passes++; + let wrote = 0; + const plan = await planFor(run, { + onAttribution: async ({ object, assignments }) => { + const groups = new Map(); + for (const a of assignments) { + const key = `${a.table}\u0000${a.organizationId}`; + const ids = groups.get(key) ?? []; + ids.push(a.id); + groups.set(key, ids); + } + for (const [key, ids] of groups) { + const [table, organizationId] = key.split('\u0000') as [string, string]; + const t = run.sql.q(table, table); + for (let i = 0; i < ids.length; i += IN_CHUNK) { + const chunk = ids.slice(i, i + IN_CHUNK); + await execute( + run, + table, + `UPDATE ${t} SET ${run.sql.q(ORG, table)} = ? WHERE ${run.sql.q(ORG, table)} IS NULL AND ${run.sql.q('id', table)} IN (${chunk.map(() => '?').join(', ')})`, + [organizationId, ...chunk], + ); + } + } + wrote += assignments.length; + run.attributed.set(object, (run.attributed.get(object) ?? 0) + assignments.length); + }, + }); + const nulls = nullsUnderAttribution(plan); + if (wrote === 0) break; + if (nulls >= lastNulls || passes >= MAX_ATTRIBUTION_PASSES) { + return { + step: 'attribution', + status: 'blocked', + summary: { rowsAttributed: total, passes }, + blockers: [ + `Attribution did not reach a fixed point: pass ${passes} still derived ${wrote} owner(s) while the NULL rows did ` + + `not decrease (${nulls}). The writes are not landing — check that the database accepts the UPDATE, then re-run.`, + ], + notes: [], + }; + } + total += wrote; + lastNulls = nulls; + run.opts.onProgress?.(`attribution pass ${passes}: ${wrote} row(s) took their organization`); + } + return { step: 'attribution', status: 'done', summary: { rowsAttributed: total, passes }, blockers: [], notes: [] }; +} + +// ── Step 2: authored catalog rows → environment metadata ──────────────────── + +/** The inventory's `authored` selector for a catalog object. */ +function authoredWhere(object: string, inventory: readonly InventoryEntry[] | undefined): RowPredicate | null { + const entry = inventory ? inventory.find((e) => e.object === object) : inventoryEntryFor(object); + return entry?.categories?.find((c) => c.id === 'authored')?.where ?? null; +} + +const rowIsOff = (value: unknown): boolean => value === false || value === 0 || value === '0' || value === 'false'; + +/** The JSON of a definition's own keys, for comparing two bodies. */ +const bodyKey = (body: Readonly>, keys: readonly string[]): string => + JSON.stringify(keys.map((k) => [k, body[k] ?? null])); + +interface CatalogRows { + positions: Record[]; + sets: Record[]; + bindings: Record[]; +} + +async function readCatalogRows(run: Run): Promise { + const read = async (table: string, select: string[]): Promise[]> => { + const cols = await columnsOf(run, table); + if (!cols) return []; + return readAll(run, table, select.filter((c) => cols.has(c))); + }; + return { + positions: await read(POSITIONS, ['name', ORG]), + sets: await read(SETS, ['name', ORG]), + bindings: await read(BINDINGS, ['position_id', 'permission_set_id']), + }; +} + +/** The `permissionSets` each catalog position definition names today, by name. */ +async function declaredPositionSets(run: Run, names: Iterable): Promise> { + const declared = new Map(); + for (const name of new Set(names)) { + const entry = await resolveName(run, 'position', name); + const sets = entry?.definition.permissionSets; + if (Array.isArray(sets)) declared.set(name, sets.map(String)); + } + return declared; +} + +async function runAuthoredConversion(run: Run): Promise { + const blockers: string[] = []; + const notes: string[] = []; + const summary: Record = { converted: 0, alreadyConverted: 0, reported: 0 }; + const authoredRows = async (table: string): Promise[]> => { + const cols = await columnsOf(run, table); + const where = authoredWhere(table, run.opts.inventory); + if (!cols || !where) return []; + return readAll(run, table, [...cols], predicateSql(run.sql, table, cols, where)); + }; + const sets = await authoredRows(SETS); + const positions = await authoredRows(POSITIONS); + const capabilities = await authoredRows(CAPABILITIES); + + if (capabilities.length > 0) { + summary.reported += capabilities.length; + notes.push( + `${capabilities.length} authored ${CAPABILITIES} row(s) are reported, not converted: no runtime reader honours a ` + + 'capability row, so nothing is converted for them (fate 4).', + ); + } + if (sets.length + positions.length === 0 && (await columnsOf(run, BINDINGS)) === null) { + return { step: 'authored-conversion', status: 'done', summary, blockers, notes }; + } + if (run.opts.posture !== 'single') { + summary.reported += sets.length + positions.length; + if (sets.length + positions.length > 0) { + notes.push( + `${sets.length + positions.length} organization-authored catalog row(s) are reported, not converted: under the ` + + `${run.opts.posture} posture a catalog definition is not an environment write (ADR-0131 D3). A grant that ` + + 'names one of them does not resolve, and the rewrite step reports it.', + ); + } + return { step: 'authored-conversion', status: 'done', summary, blockers, notes }; + } + + // Bindings first: an authored position's definition carries the sets its rows bind. + const catalogRows = await readCatalogRows(run); + const conversion = convertPositionBindingRows({ + positions: catalogRows.positions, + permissionSets: catalogRows.sets, + bindings: catalogRows.bindings, + declared: await declaredPositionSets(run, catalogRows.positions.map((r) => String(r.name))), + }); + const boundSets = new Map(); + for (const position of conversion.positions) { + if (position.fate === 'conflicting') { + blockers.push( + `Position ${position.name}: organizations bind different permission sets to it ` + + `(${JSON.stringify(position.byOrganization)}). Merging them would grant one organization's members another's ` + + `sets, so it is never guessed: make the ${BINDINGS} rows agree (or remove the extra ones), then re-run.`, + ); + } else if (position.fate === 'converted') { + boundSets.set(position.name, position.permissionSets); + } + } + + const convert = async ( + type: 'permission' | 'position', + rows: Record[], + toBody: (row: Record) => Record, + ): Promise> => { + const handled = new Set(); + const byName = new Map[]>(); + for (const row of rows) { + const name = text(row.name); + if (!name) { + summary.reported++; + notes.push(`An authored ${type} row (${String(row.id)}) carries no name and is reported, never converted.`); + continue; + } + byName.set(name, [...(byName.get(name) ?? []), row]); + } + for (const [name, group] of byName) { + handled.add(name); + if (group.some((r) => 'active' in r && rowIsOff(r.active))) { + summary.reported += group.length; + notes.push( + `The authored ${type} ${name} is deactivated and is not converted: converting it would make it grant. A grant ` + + 'that names it does not resolve, and the rewrite step reports it.', + ); + continue; + } + const bodies = group.map((row) => { + const body = toBody(row); + const extra = type === 'position' ? boundSets.get(name) : undefined; + return extra ? { ...body, permissionSets: [...extra] } : body; + }); + const keys = Object.keys(bodies[0]!); + if (new Set(bodies.map((b) => bodyKey(b, [...new Set(bodies.flatMap(Object.keys))]))).size > 1) { + blockers.push( + `The authored ${type} ${name} is held by ${group.length} rows (${group.map((r) => String(r.id)).join(', ')}) ` + + 'that define it differently. Which one the environment keeps is never guessed: make them agree, or ' + + 'rename all but one, then re-run.', + ); + continue; + } + const body = bodies[0]!; + const existing = await resolveName(run, type, name); + if (existing) { + if (bodyKey(existing.definition, keys) === bodyKey(body, keys)) { + summary.alreadyConverted++; + continue; + } + blockers.push( + `The authored ${type} ${name} differs from the definition the catalog already holds under that name. The ` + + 'catalog\'s definition stays; rename the authored row (and the grants that name it), then re-run.', + ); + continue; + } + try { + await saveDefinition(run, type, name, body); + summary.converted++; + run.opts.onProgress?.(`converted the authored ${type} ${name} to environment metadata`); + } catch (error) { + blockers.push( + `The metadata door refused the authored ${type} ${name}: ${error instanceof Error ? error.message : String(error)}. ` + + 'Correct the row so the definition validates, then re-run.', + ); + } + } + return handled; + }; + + await convert('permission', sets, (row) => permissionSetDefinitionFromRow(row)); + const authoredPositions = await convert('position', positions, (row) => positionDefinitionFromRow(row)); + + // A declared position whose binding rows add sets its definition lacks. + for (const [name, permissionSets] of boundSets) { + if (authoredPositions.has(name)) continue; + const existing = await resolveName(run, 'position', name); + if (!existing) { + blockers.push( + `Binding rows give position ${name} the sets ${permissionSets.join(', ')}, and the catalog holds no position of ` + + 'that name to carry them. Declare the position (or remove the bindings), then re-run.', + ); + continue; + } + try { + await saveDefinition(run, 'position', name, { ...existing.definition, permissionSets: [...permissionSets] }); + summary.converted++; + } catch (error) { + blockers.push( + `Binding rows give the declared position ${name} the sets ${permissionSets.join(', ')}, which its definition ` + + `does not name, and the metadata door refused the change: ${error instanceof Error ? error.message : String(error)}. ` + + `Declare those sets on the position (permissionSets), or remove the ${BINDINGS} rows, then re-run.`, + ); + } + } + if (conversion.dangling.length > 0) { + notes.push( + `${conversion.dangling.length} ${BINDINGS} row(s) name a position or set row that does not exist; they bind nothing ` + + 'and are deleted with the other binding mirrors.', + ); + } + return { step: 'authored-conversion', status: blockers.length ? 'blocked' : 'done', summary, blockers, notes }; +} + +// ── Step 3: the verified id→name rewrite (D4) ─────────────────────────────── + +interface Verification { + /** Blockers: a reference that does not resolve by name, each with its remedy. */ + blockers: string[]; + notes: string[]; + grants: number; + verified: number; +} + +/** Read-only: does every reference resolve by name? The gate of fate 2. */ +async function verifyReferences(run: Run): Promise { + const out: Verification = { blockers: [], notes: [], grants: 0, verified: 0 }; + const cols = await columnsOf(run, GRANTS); + if (cols) { + if (!cols.has('permission_set')) { + out.blockers.push( + `${GRANTS} has no permission_set column, so no grant can name its set. Boot this release once against the ` + + 'database (its schema sync adds the column), then re-run.', + ); + return out; + } + const setCols = await columnsOf(run, SETS); + const setNames = new Map(); + if (setCols) for (const row of await readAll(run, SETS, ['name'])) setNames.set(String(row.id), text(row.name)); + const grants = await readAll(run, GRANTS, ['permission_set', ...(cols.has('permission_set_id') ? ['permission_set_id'] : [])]); + const resolved = new Map(); + const unnamed: string[] = []; + const unresolved = new Map(); + const disagree: string[] = []; + for (const grant of grants) { + out.grants++; + const name = text(grant.permission_set); + if (!name) { + unnamed.push(String(grant.id)); + continue; + } + if (!resolved.has(name)) resolved.set(name, (await resolveName(run, 'permission', name)) !== undefined); + if (!resolved.get(name)) { + unresolved.set(name, [...(unresolved.get(name) ?? []), String(grant.id)]); + continue; + } + const id = text(grant.permission_set_id); + if (id && setNames.has(id) && setNames.get(id) !== name) { + disagree.push(String(grant.id)); + continue; + } + out.verified++; + } + if (unnamed.length) { + out.blockers.push( + `${unnamed.length} grant(s) still name no permission set (${unnamed.slice(0, 20).join(', ')}${unnamed.length > 20 ? ', …' : ''}). ` + + 'The rewrite could not derive their name (see its notes); name or remove each, then re-run.', + ); + } + for (const [name, ids] of unresolved) { + out.blockers.push( + `${ids.length} grant(s) name the permission set ${name}, which the security catalog does not resolve ` + + `(${ids.slice(0, 20).join(', ')}${ids.length > 20 ? ', …' : ''}). Declare or author the set, or remove the grants, then re-run.`, + ); + } + if (disagree.length) { + out.blockers.push( + `${disagree.length} grant(s) name a set other than the one their permission_set_id points at (${disagree.slice(0, 20).join(', ')}). ` + + 'Correct the name, then re-run.', + ); + } + } + if (await columnsOf(run, BINDINGS)) { + const rows = await readCatalogRows(run); + const conversion = convertPositionBindingRows({ + positions: rows.positions, + permissionSets: rows.sets, + bindings: rows.bindings, + declared: await declaredPositionSets(run, rows.positions.map((r) => String(r.name))), + }); + for (const position of conversion.positions) { + if (position.fate === 'declared') continue; + out.blockers.push( + position.fate === 'conflicting' + ? `Position ${position.name}: organizations bind different permission sets to it; the bindings cannot be carried into one definition.` + : `Binding rows give position ${position.name} sets its catalog definition does not name (${position.permissionSets.join(', ')}); ` + + 'deleting them would lose a grant. The authored-conversion step carries them; resolve its blocker, then re-run.', + ); + } + } + return out; +} + +async function runRewrite(run: Run): Promise { + const summary: Record = { named: 0, grants: 0, verified: 0 }; + const notes: string[] = []; + const cols = await columnsOf(run, GRANTS); + if (cols?.has('permission_set') && cols.has('permission_set_id')) { + const sets = new Map(); + if (await columnsOf(run, SETS)) { + const setCols = (await columnsOf(run, SETS))!; + for (const row of await readAll(run, SETS, ['name', ...(setCols.has(ORG) ? [ORG] : [])])) { + sets.set(String(row.id), { name: text(row.name), organization: text(row[ORG]) }); + } + } + const t = run.sql.q(GRANTS, GRANTS); + const nameCol = run.sql.q('permission_set', GRANTS); + const unnamed = await readAll(run, GRANTS, ['permission_set_id', ...(cols.has(ORG) ? [ORG] : [])], { + text: `${nameCol} IS NULL OR ${nameCol} = ''`, + params: [], + }); + const reasons = { noId: [] as string[], dangling: [] as string[], crossOrganization: [] as string[], unresolved: [] as string[] }; + for (const grant of unnamed) { + const gid = String(grant.id); + const setId = text(grant.permission_set_id); + if (!setId) { reasons.noId.push(gid); continue; } + const set = sets.get(setId); + if (!set?.name) { reasons.dangling.push(gid); continue; } + // A name carries no organization: an organization's set row may be named + // only by that organization's grants (the grant rule the resolver applies). + if (set.organization !== null && set.organization !== text(grant[ORG])) { reasons.crossOrganization.push(gid); continue; } + if (!(await resolveName(run, 'permission', set.name))) { reasons.unresolved.push(gid); continue; } + await execute( + run, + GRANTS, + `UPDATE ${t} SET ${nameCol} = ? WHERE ${run.sql.q('id', GRANTS)} = ? AND (${nameCol} IS NULL OR ${nameCol} = '')`, + [set.name, gid], + ); + summary.named++; + } + const describe: Record = { + noId: 'carry neither a name nor a permission_set_id', + dangling: 'point at a set row that does not exist', + crossOrganization: 'point at another organization\'s set row — a name carries no organization, so it is never written', + unresolved: 'point at a set row whose name the security catalog does not resolve', + }; + for (const key of Object.keys(reasons) as (keyof typeof reasons)[]) { + if (reasons[key].length) notes.push(`${reasons[key].length} unnamed grant(s) ${describe[key]}: ${reasons[key].slice(0, 20).join(', ')}.`); + } + if (summary.named) run.opts.onProgress?.(`rewrite: ${summary.named} grant(s) now name their permission set`); + } + const verification = await verifyReferences(run); + summary.grants = verification.grants; + summary.verified = verification.verified; + return { + step: 'rewrite', + status: verification.blockers.length ? 'blocked' : 'done', + summary, + blockers: verification.blockers, + notes: [...notes, ...verification.notes], + }; +} + +// ── Step 4: mirror deletion (fate 2) ───────────────────────────────────────── + +/** Provenance the seeders stamped: only such a row can be a copy of code. */ +const SEEDED_BY = ['package', 'platform', 'system']; + +/** + * The rows of a fate-2 object that are deleted: its mirror selector (every + * row when it names none), minus every row a claim keeps — + * + * - each named category of the object (authored catalog rows, customized or + * organization-stamped templates awaiting promotion); + * - on a table that records provenance, every row no seeder wrote + * (`managed_by` NULL or not package / platform / system): an admin-authored + * row is never a copy of code, whatever the object's inventory row selects. + * It stays and is reported for the step that carries it. + * + * A claim that evaluates neither true nor false on a row (NULL) claims it: + * an unknown is kept, never deleted. + */ +function deletableWhere(run: Run, table: string, cols: ReadonlySet, entry: InventoryEntry): { text: string; params: unknown[] } { + const parts: string[] = []; + const params: unknown[] = []; + if (entry.mirror) { + const mirror = predicateSql(run.sql, table, cols, entry.mirror); + parts.push(`(${mirror.text})`); + params.push(...mirror.params); + } + const keep = (claimed: { text: string; params: unknown[] }): void => { + parts.push(`(CASE WHEN ${claimed.text} THEN 1 WHEN NOT (${claimed.text}) THEN 0 ELSE 1 END) = 0`); + params.push(...claimed.params, ...claimed.params); + }; + for (const category of entry.categories ?? []) { + if (!category.where || category.conflictBy) continue; + keep(predicateSql(run.sql, table, cols, category.where)); + } + if (cols.has('managed_by')) { + keep(predicateSql(run.sql, table, cols, [{ column: 'managed_by', notIn: SEEDED_BY }])); + } + return { text: parts.length ? parts.join(' AND ') : '1 = 1', params }; +} + +const mirrorEntry = (run: Run, object: string): InventoryEntry | undefined => + (run.opts.inventory ? run.opts.inventory.find((e) => e.object === object) : inventoryEntryFor(object)); + +async function countWhere(run: Run, table: string, where: { text: string; params: unknown[] }): Promise { + return run.sql.count(table, `SELECT COUNT(*) AS n FROM ${run.sql.q(table, table)} WHERE ${where.text}`, where.params); +} + +async function runMirrorDeletion(run: Run): Promise { + // The gate, read again now: a grant written since the rewrite ran is a reference too. + const gate = await verifyReferences(run); + if (gate.blockers.length) { + return { + step: 'mirror-deletion', + status: 'blocked', + summary: { deleted: 0 }, + blockers: [`Mirror deletion is gated on every reference resolving by name, and it no longer does:`, ...gate.blockers], + notes: [], + }; + } + const notes: string[] = []; + let total = 0; + for (const table of run.plan.tables) { + if (table.fate !== 'mirror-deletion' || table.physical !== 'present') continue; + const entry = mirrorEntry(run, table.object); + if (!entry) continue; + for (const physical of table.tables) { + const cols = (await columnsOf(run, physical))!; + const where = deletableWhere(run, physical, cols, entry); + const before = await countWhere(run, physical, where); + if (before === 0) continue; + await execute(run, physical, `DELETE FROM ${run.sql.q(physical, physical)} WHERE ${where.text}`, where.params); + const left = await countWhere(run, physical, where); + if (left !== 0) { + throw new Error(`The mirror delete on ${physical} left ${left} mirror row(s) behind; nothing past this table was deleted.`); + } + run.deleted.set(table.object, (run.deleted.get(table.object) ?? 0) + before); + total += before; + run.opts.onProgress?.(`mirror deletion: ${before} row(s) of ${physical}`); + } + const kept = Math.max(0, table.rows!.total - (run.deleted.get(table.object) ?? 0)); + if (kept > 0) notes.push(`${table.object}: ${kept} row(s) are not mirrors (a named category or their non-seeded provenance claims them) and stay.`); + } + return { step: 'mirror-deletion', status: 'done', summary: { deleted: total }, blockers: [], notes }; +} + +// ── Step 5: column drops (fate 1) ──────────────────────────────────────────── + +async function runColumnDrops(run: Run): Promise { + const plan = await planFor(run); + const blockers: string[] = []; + const due: string[] = []; + for (const table of plan.tables) { + if (table.fate !== 'column-drop' || table.physical !== 'present' || table.organizationColumn !== 'present') continue; + const held = table.categories.filter((c) => c.rows > 0); + if (held.length) { + blockers.push( + `${table.object}.${ORG} is not dropped: ${held.map((c) => `${c.rows} row(s) of "${c.label}"`).join('; ')}. ` + + 'Those rows need their own step before the column can go; resolve them, then re-run.', + ); + continue; + } + due.push(...table.tables); + } + let dropped = 0; + if (due.length) { + for (const outcome of await run.opts.columnDrops.drop(due)) { + if (outcome.outcome === 'dropped') { + dropped++; + continue; + } + blockers.push( + outcome.outcome === 'not-proposed' + ? `${outcome.table}.${ORG} was not proposed for dropping by the schema-drift ceremony: the object this ` + + `runtime declares still carries the column${outcome.detail ? ` (${outcome.detail})` : ''}. Run the ceremony ` + + 'with the release whose declaration drops it.' + : `Dropping ${outcome.table}.${ORG} failed: ${outcome.detail ?? 'no detail'}. Re-run once the cause is fixed.`, + ); + } + } + return { step: 'column-drops', status: blockers.length ? 'blocked' : 'done', summary: { dropped, due: due.length }, blockers, notes: [] }; +} + +// ── The run ────────────────────────────────────────────────────────────────── + +const RUNNERS: Record Promise> = { + attribution: runAttribution, + 'authored-conversion': runAuthoredConversion, + rewrite: runRewrite, + 'mirror-deletion': runMirrorDeletion, + 'column-drops': runColumnDrops, +}; + +/** NULL rows a departure still has to carry out of the table. */ +const departingRows = (table: TablePlan): number => table.departures.reduce((n, d) => n + d.rows, 0); + +/** What a table still owes its fate, read from the post-check plan. */ +async function remainingFor(run: Run, table: TablePlan, complete: boolean): Promise { + if (table.physical !== 'present' || !table.rows) return 0; + switch (table.fate) { + case 'attribution': + return Number(table.fateCounts.rowsToAttribute ?? 0) + departingRows(table); + case 'report': + return departingRows(table); + case 'mirror-deletion': { + const entry = mirrorEntry(run, table.object); + if (!entry) return 0; + let left = 0; + for (const physical of table.tables) { + left += await countWhere(run, physical, deletableWhere(run, physical, (await columnsOf(run, physical))!, entry)); + } + return left; + } + case 'column-drop': + return complete && table.organizationColumn === 'present' && table.categories.every((c) => c.rows === 0) ? 1 : 0; + default: + return 0; + } +} + +/** + * Run the ceremony's apply: the fates in order, resuming from the checkpoint, + * then the post-check. Throws only for a failure mid-write (the checkpoint + * holds every step finished before it) or a plan refusal; every blocker is in + * the report. + */ +export async function applyOrganizationOwnership(opts: ApplyOptions): Promise { + const dialect = dialectOf(opts.executor.client); + const now = opts.now ?? (() => new Date()); + const run: Run = { + opts, + sql: new Sql(opts.executor, dialect ?? 'sqlite'), + plan: undefined as unknown as OrganizationOwnershipPlan, + columns: new Map(), + attributed: new Map(), + deleted: new Map(), + saved: new Map(), + }; + // Refuses (driver, catalog, an uninventoried table) before anything is written. + run.plan = await planFor(run); + const before = run.plan; + + const stored = await opts.ledger.read(); + const checkpoint: CeremonyCheckpoint = + stored && stored.ceremony === 'ADR-0131 D10' && stored.inventoryDigest === before.inventoryDigest + ? { ceremony: 'ADR-0131 D10', inventoryDigest: before.inventoryDigest, completed: { ...stored.completed } } + : { ceremony: 'ADR-0131 D10', inventoryDigest: before.inventoryDigest, completed: {} }; + const resumedFrom = CEREMONY_STEPS.filter((s) => checkpoint.completed[s]); + const steps: StepReport[] = []; + if (stored && resumedFrom.length === 0 && Object.keys(stored.completed ?? {}).length > 0) { + opts.onProgress?.('the inventory changed since the recorded checkpoint, so every step runs again (each is idempotent)'); + } + + let stopped = false; + for (const step of CEREMONY_STEPS) { + if (stopped) { + steps.push({ step, status: 'not-reached', summary: {}, blockers: [], notes: [] }); + continue; + } + const done = checkpoint.completed[step]; + if (done) { + steps.push({ step, status: 'checkpointed', summary: done.summary, blockers: [], notes: [] }); + continue; + } + opts.onProgress?.(`step ${step}…`); + const report = await RUNNERS[step](run); + steps.push(report); + if (report.status !== 'done') { + stopped = true; + continue; + } + checkpoint.completed[step] = { at: now().toISOString(), summary: report.summary }; + await opts.ledger.write(checkpoint, { complete: false, blocking: 0, advisory: 0 }); + // Later steps read the tables as this one left them. + run.columns.clear(); + } + + // Post-check: the plan again, read-only. + run.columns.clear(); + const after = await planFor(run); + const complete = CEREMONY_STEPS.every((s) => checkpoint.completed[s]); + const tables: TableOutcome[] = []; + const objects = new Set([...before.tables, ...after.tables].map((t) => t.object)); + let discrepancies = 0; + for (const object of [...objects].sort()) { + const b = before.tables.find((t) => t.object === object); + const a = after.tables.find((t) => t.object === object); + const ref = a ?? b!; + const remaining = a ? await remainingFor(run, a, complete) : 0; + if (complete) discrepancies += remaining; + tables.push({ + object, + fate: ref.fate, + before: b?.rows ? { total: b.rows.total, organizationNull: b.rows.organizationNull } : null, + after: a?.rows ? { total: a.rows.total, organizationNull: a.rows.organizationNull } : null, + attributed: run.attributed.get(object) ?? 0, + deleted: run.deleted.get(object) ?? 0, + columnDropped: b?.organizationColumn === 'present' && a?.organizationColumn === 'absent', + unattributable: a?.unattributable ?? [], + departures: (a?.departures ?? []).filter((d) => d.rows > 0).map((d) => ({ id: d.id, label: d.label, as: d.as, rows: d.rows })), + remaining, + notNull: a && departingRows(a) > 0 + ? { willReceive: false, reason: `${departingRows(a)} row(s) still wait for a step not yet run: ${a.departures.filter((d) => d.rows > 0).map((d) => d.label).join('; ')}` } + : ref.notNull, + }); + } + const advisory = after.summary.unattributableRows; + const blocking = complete ? discrepancies : steps.reduce((n, s) => n + s.blockers.length, 0) || 1; + await opts.ledger.write(checkpoint, { complete, blocking, advisory }); + return { + kind: 'organization-ownership-apply', + ceremony: 'ADR-0131 D10', + migrationId: ORGANIZATION_OWNERSHIP_MIGRATION_ID, + generatedAt: now().toISOString(), + database: opts.database, + posture: opts.posture, + inventoryDigest: before.inventoryDigest, + resumedFrom, + steps, + tables, + complete, + blocking, + advisory, + }; +} + +/** Parse a checkpoint from the ledger row's `details`, or `null` when it holds none. */ +export function checkpointFromDetails(details: unknown): CeremonyCheckpoint | null { + let parsed: unknown = details; + if (typeof details === 'string') { + try { + parsed = JSON.parse(details); + } catch { + return null; + } + } + const cp = (parsed as { checkpoint?: unknown } | null)?.checkpoint as CeremonyCheckpoint | undefined; + if (!cp || cp.ceremony !== 'ADR-0131 D10' || typeof cp.inventoryDigest !== 'string' || typeof cp.completed !== 'object') return null; + return { ceremony: cp.ceremony, inventoryDigest: cp.inventoryDigest, completed: { ...cp.completed } }; +} diff --git a/packages/cli/src/utils/organization-ownership-plan.ts b/packages/cli/src/utils/organization-ownership-plan.ts index 697069c86d8..f3a384ebd8d 100644 --- a/packages/cli/src/utils/organization-ownership-plan.ts +++ b/packages/cli/src/utils/organization-ownership-plan.ts @@ -85,6 +85,27 @@ export interface UnattributableRow { shard?: string; } +/** + * One NULL-organization row the walk derived an owner for. The plan only + * counts these; `--apply` hands a sink to the same walk so the owner it writes + * is the owner the plan reported, derived once ({@link AttributionSink}). + */ +export interface AttributionAssignment { + id: string; + /** The physical table the row lives in (a rotation shard, or the object's own table). */ + table: string; + organizationId: string; + /** The anchor that answered, as the plan's `attribution` counts label it. */ + via: string; +} + +/** + * Receives each page of derived owners, after the page is classified and + * before the walk reads the next. The plan itself never writes; a sink is the + * one place a caller may. + */ +export type AttributionSink = (batch: { object: string; assignments: AttributionAssignment[] }) => Promise; + export interface PlannedCategory { id: string; label: string; @@ -176,7 +197,8 @@ export function inventoryDigest(inventory: readonly InventoryEntry[] = ORGANIZAT return createHash('sha256').update(JSON.stringify(inventory)).digest('hex'); } -class Sql { +/** Identifier quoting, value binding and refusing reads, shared with the apply step. */ +export class Sql { constructor( private readonly reader: PlanReader, readonly dialect: PlanDialect, @@ -231,7 +253,7 @@ function firstValue(row: Record): unknown { return Object.values(row)[0]; } -function columnNameOf(row: Record): string | null { +export function columnNameOf(row: Record): string | null { for (const key of ['column_name', 'COLUMN_NAME', 'name']) { const value = row[key]; if (typeof value === 'string' && value) return value; @@ -247,7 +269,7 @@ function tableNameOf(row: Record): string | null { return null; } -function columnListSql(dialect: PlanDialect): string { +export function columnListSql(dialect: PlanDialect): string { if (dialect === 'sqlite') return 'SELECT name AS column_name FROM pragma_table_info(?)'; if (dialect === 'postgres') { return 'SELECT column_name FROM information_schema.columns WHERE table_schema = ANY (current_schemas(false)) AND table_name = ?'; @@ -256,7 +278,7 @@ function columnListSql(dialect: PlanDialect): string { } /** The SQL of one predicate, its columns required present on `table`. */ -function predicateSql(sql: Sql, table: string, columns: ReadonlySet, predicate: RowPredicate): { text: string; params: unknown[] } { +export function predicateSql(sql: Sql, table: string, columns: ReadonlySet, predicate: RowPredicate): { text: string; params: unknown[] } { const parts: string[] = []; const params: unknown[] = []; for (const condition of predicate) { @@ -281,7 +303,7 @@ function predicateSql(sql: Sql, table: string, columns: ReadonlySet, pre return { text: parts.length ? parts.join(' AND ') : '1 = 1', params }; } -function requireColumn(table: string, columns: ReadonlySet, column: string): void { +export function requireColumn(table: string, columns: ReadonlySet, column: string): void { if (!columns.has(column)) { throw new OrganizationOwnershipPlanRefusal( 'missing-column', @@ -378,6 +400,7 @@ interface PlanContext { catalog: Catalog; posture: string; defaultOrganizationId: string | null; + onAttribution?: AttributionSink; } /** The organization of each id in `ids` on `object`'s own table, or a reason it cannot be read. */ @@ -412,9 +435,19 @@ interface PendingRow { } /** Resolve rows through one anchor; returns the rows it could not derive. */ -async function applyAnchor(ctx: PlanContext, anchor: Anchor, rows: PendingRow[], derived: Record): Promise { +async function applyAnchor( + ctx: PlanContext, + anchor: Anchor, + rows: PendingRow[], + derived: Record, + assigned: AttributionAssignment[], +): Promise { const label = anchorLabel(anchor); const left: PendingRow[] = []; + const derive = (row: PendingRow, organizationId: string): void => { + derived[label] = (derived[label] ?? 0) + 1; + assigned.push({ id: row.id, table: row.shard, organizationId, via: label }); + }; if (anchor.kind === 'parent') { const keys = [...new Set(rows.map((r) => nonEmpty(r.values[anchor.childKey])).filter((k): k is string => k !== null))]; const orgs = keys.length ? await organizationsById(ctx, anchor.parentObject, keys) : new Map(); @@ -424,7 +457,7 @@ async function applyAnchor(ctx: PlanContext, anchor: Anchor, rows: PendingRow[], else if (typeof orgs === 'string') left.push({ ...row, reason: orgs }); else if (!orgs.has(key)) left.push({ ...row, reason: `${anchor.parentObject} row ${key} not found` }); else if (orgs.get(key) === null) left.push({ ...row, reason: `${anchor.parentObject} row ${key} has no organization` }); - else derived[label] = (derived[label] ?? 0) + 1; + else derive(row, orgs.get(key)!); } return left; } @@ -450,7 +483,7 @@ async function applyAnchor(ctx: PlanContext, anchor: Anchor, rows: PendingRow[], if (typeof orgs === 'string') left.push({ ...row, reason: orgs }); else if (!orgs.has(id)) left.push({ ...row, reason: `${object} row ${id} not found` }); else if (orgs.get(id) === null) left.push({ ...row, reason: `${object} row ${id} has no organization` }); - else derived[label] = (derived[label] ?? 0) + 1; + else derive(row, orgs.get(id)!); } } return left; @@ -482,7 +515,7 @@ async function applyAnchor(ctx: PlanContext, anchor: Anchor, rows: PendingRow[], const orgs = holders.get(row.id); if (!orgs) left.push({ ...row, reason: `no stamped ${anchor.holderObject} holder` }); else if (orgs.size > 1) left.push({ ...row, reason: `${anchor.holderObject} holders disagree (${orgs.size} organizations)` }); - else derived[label] = (derived[label] ?? 0) + 1; + else derive(row, [...orgs][0]!); } return left; } @@ -669,16 +702,21 @@ async function walkNullRows( } pending.push({ id: String(row.row_id), values: row, reason: entry.fate === 'report' ? 'fate 4: reported, never guessed' : 'no anchor names the owner', shard: table }); } + const assigned: AttributionAssignment[] = []; if (entry.fate === 'attribution') { for (const anchor of entry.anchors ?? []) { if (pending.length === 0) break; - pending = await applyAnchor(ctx, anchor, pending, derived); + pending = await applyAnchor(ctx, anchor, pending, derived, assigned); } if (pending.length > 0 && ctx.posture === 'single' && ctx.defaultOrganizationId !== null) { derived['default-organization'] = (derived['default-organization'] ?? 0) + pending.length; + for (const row of pending) { + assigned.push({ id: row.id, table: row.shard, organizationId: ctx.defaultOrganizationId, via: 'default-organization' }); + } pending = []; } } + if (ctx.onAttribution && assigned.length > 0) await ctx.onAttribution({ object: entry.object, assignments: assigned }); for (const row of pending) { plan.unattributable.push({ id: row.id, @@ -725,6 +763,11 @@ export async function buildOrganizationOwnershipPlan(opts: { database: string; now?: Date; inventory?: readonly InventoryEntry[]; + /** + * `--apply`'s attribution step: receives every owner the walk derives. The + * plan's own run passes none and stays SELECT-only. + */ + onAttribution?: AttributionSink; }): Promise { const inventory = opts.inventory ?? ORGANIZATION_OWNERSHIP_INVENTORY; const dialect = dialectOf(opts.reader.client); @@ -754,7 +797,13 @@ export async function buildOrganizationOwnershipPlan(opts: { ); defaultOrganizationId = rows.length > 0 ? nonEmpty(rows[0]!.id) : null; } - const ctx: PlanContext = { sql, catalog, posture: opts.posture, defaultOrganizationId }; + const ctx: PlanContext = { + sql, + catalog, + posture: opts.posture, + defaultOrganizationId, + ...(opts.onAttribution ? { onAttribution: opts.onAttribution } : {}), + }; const tables: TablePlan[] = []; let outsideCeremony = 0; diff --git a/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts b/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts index 6dd09036b82..9d794dc173a 100644 --- a/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts +++ b/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts @@ -192,8 +192,11 @@ const CALLERS: Record = { label: 'migrate organization-ownership', argv: ['--out', '@DIR@/organization-ownership-plan.json', '--database-url', '@DB@', '--json'], }], - write: [], - note: 'the ADR-0131 D10 preflight: it has no writing mode until the ceremony\'s apply lands', + // `--apply` writes the ceremony's fates and its report file; never a seed. + write: [{ + label: 'migrate organization-ownership --apply', + argv: ['--apply', '--backup-taken', '--out', '@DIR@/organization-ownership-apply.json', '--database-url', '@DB@', '--json'], + }], }, 'commands/migrate/plan.ts': { run: invoke(MigratePlan), diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index 95a7327f97f..05bb4da34c3 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -681,7 +681,10 @@ export async function bootSchemaStack( * Compose the SAME object set this deployment's `os serve` boot registers — * its `objectstack.config.ts` and the platform floor `serve` composes * unconditionally (#12938). Set by the two SCHEMA commands, `os migrate - * plan` and `os migrate apply`, and by nothing else. + * plan` and `os migrate apply`; by the commands whose subject is that same + * set (`unmapped-columns`, `security-catalog-overlays`); and by `os migrate + * organization-ownership --apply`, whose column drops are that set's drift + * and whose security catalog is the one the deployment serves. * * Off by default and opted into at the call site rather than deduced here: * the DATA subcommands declare their own, narrower set through @@ -777,7 +780,8 @@ export async function bootSchemaStack( * * Off by default and opted into at the call site, like `composeHostStack`: * `os migrate plan` / `apply` compose no tier-gated plugin, and this does - * not change that. See `buildSchemaMigrationPlugins`'s `authGatedSecurity`. + * not change that. `os migrate organization-ownership --apply` sets it so + * the platform's shipped permission sets are names its catalog resolves. See `buildSchemaMigrationPlugins`'s `authGatedSecurity`. */ composeAuthGatedSecurity?: boolean; /** @@ -797,9 +801,10 @@ export async function bootSchemaStack( * family behind its auth gate, the provider of every capability its * resolver mounts (the stack's `requires` and the always-on slate), and * the REST API plugin. Set by `os migrate plan` and `os migrate apply`, - * whose subject is the deployment's whole object set, and by `os migrate - * unmapped-columns`, which reads the plan's own findings over that set — - * and by nothing else. + * whose subject is the deployment's whole object set, by `os migrate + * unmapped-columns`, which reads the plan's own findings over that set, + * and by `os migrate organization-ownership --apply`, which drops the + * ceremony's columns through that set's drift — and by nothing else. * See `buildSchemaMigrationPlugins`'s `servedPlatform`. */ composeServedPlatform?: boolean; diff --git a/packages/cli/test/json-stdout-purity.e2e.test.ts b/packages/cli/test/json-stdout-purity.e2e.test.ts index 0bb95fb871e..84bb341f834 100644 --- a/packages/cli/test/json-stdout-purity.e2e.test.ts +++ b/packages/cli/test/json-stdout-purity.e2e.test.ts @@ -99,8 +99,9 @@ const FAMILY: Record = { 'migrate files-to-references': [], 'migrate meta': ['--stored'], 'migrate multi-value-columns': [], - // Its bare form is its only mode — a read-only plan (`deferSchemaDdl` + - // `readOnlyProbe`). The fixture's database holds none of the platform tables + // Its bare form — a read-only plan (`deferSchemaDdl` + `readOnlyProbe`) — + // is the one driven here; `--apply` emits through the same two `emitJson` + // paths after the same shutdown. The fixture's database holds none of the platform tables // the ceremony inventories, so the face driven here is its REFUSAL face // (`plan_refused`, `emitJson(…, 1, { compact: true })`), emitted after the // stack is shut down. A refused plan writes no plan file, so the fixture