From 06000822af0fffafadaea18cc541f776312f800c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 21:39:05 +0000 Subject: [PATCH 01/11] feat(service-storage): retire the sys_file scope option public and add its one-time rewrite to user The sys_file scope select no longer lists public: the value promised a public file and stored a private one, and the upload doors already refuse it. Rows a deployment already stored with it are rewritten to user by an operator-run sweep in the shape of the organization backfill (plan / apply / run, dry run first and by default, counts before it writes, a no-op at zero, idempotent). One column moves; the storage key and the bytes do not. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../backfill-sys-file-public-scope.test.ts | 249 +++++++++++++ .../src/backfill-sys-file-public-scope.ts | 326 ++++++++++++++++++ .../src/objects/system-file.object.ts | 8 +- 3 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts create mode 100644 packages/services/service-storage/src/backfill-sys-file-public-scope.ts diff --git a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts new file mode 100644 index 00000000000..5c50de93fe9 --- /dev/null +++ b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts @@ -0,0 +1,249 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// #22443, ruling B — the `sys_file.scope` option `public` retires, and the +// rows already stored with it are rewritten to `user` by an operator-run +// sweep (`backfill-sys-file-public-scope.ts`). +// +// Every pin runs on a REAL ObjectQL over a REAL SqlDriver on sqlite `:memory:` +// (the project's ruled test backend), with the real `SystemFile` object and +// the real field-reference hooks: what retiring the option does to a stored +// row is the engine's select-option check, and a double would only restate +// what this file has to measure. A stored `public` row is seeded through the +// DRIVER, because the engine on this release refuses to write one — which is +// exactly the state a deployment upgrading from the previous release is in. +// +// What is pinned: +// +// - the ruling's two pins — a copy of a rewritten row succeeds +// (`copyOwnedFile`, reached through the copy-on-claim hook); a deployment +// with no `public` rows runs clean, with zero writes; +// - counts before it writes, dry run by default, and a second run writes zero; +// - the operator step's reason, as behaviour: BEFORE the sweep runs, a copy +// of a stored `public` row is refused, while reads and scope-free updates +// of that row are unaffected; +// - the rollback's precondition: on this release the inverse write +// (`user` → `public`) is refused by the engine. + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { SystemFile } from './objects/system-file.object.js'; +import { installFileReferenceHooks } from './file-reference-lifecycle.js'; +import { + REWRITTEN_SYS_FILE_SCOPE, + RETIRED_SYS_FILE_SCOPE, + applySysFilePublicScopeBackfill, + formatSysFilePublicScopeBackfillReport, + planSysFilePublicScopeBackfill, + runSysFilePublicScopeBackfill, +} from './backfill-sys-file-public-scope.js'; + +const SYSTEM = { context: { isSystem: true } }; + +const silentLogger = () => ({ + info: vi.fn(), warn: vi.fn(), debug: vi.fn(), error: vi.fn(), + trace: vi.fn(), fatal: vi.fn(), child() { return this; }, +}); + +/** A stored row as the previous release wrote it: `public`, owned by `product/p0.image`. */ +const legacyPublicRow = (id: string) => ({ + id, + key: `public/${id}.png`, + name: 'logo.png', + mime_type: 'image/png', + size: 5, + scope: RETIRED_SYS_FILE_SCOPE, + acl: 'private', + status: 'committed', + ref_object: 'product', + ref_id: 'p0', + ref_field: 'image', +}); + +describe('sys_file public-scope backfill (#22443 ruling B) — a real ObjectQL over SqlDriver (sqlite :memory:)', () => { + let sql: SqlDriver; + let engine: ObjectQL; + let storage: { upload: ReturnType; download: ReturnType; delete: ReturnType; exists: ReturnType }; + /** `sys_file` writes that reached the engine, by verb. */ + let sysFileUpdates: number; + + beforeEach(async () => { + sql = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }); + engine = new ObjectQL({ logger: silentLogger() } as any); + engine.registerDriver(sql as any, true); + await engine.init(); + engine.registry.registerObject(SystemFile as any, 'com.objectstack.storage'); + engine.registry.registerObject({ + name: 'product', + fields: { title: { type: 'text' }, image: { type: 'image' } }, + } as any, 'test-22443'); + await engine.syncSchemas(); + + storage = { + upload: vi.fn(async () => {}), + download: vi.fn(async () => Buffer.from('bytes')), + delete: vi.fn(async () => {}), + exists: vi.fn(async () => true), + }; + installFileReferenceHooks(engine as any, () => storage as any, silentLogger()); + + sysFileUpdates = 0; + const realUpdate = engine.update.bind(engine); + (engine as any).update = (object: string, data: any, options: any) => { + if (object === 'sys_file') sysFileUpdates += 1; + return realUpdate(object, data, options); + }; + }); + + afterEach(async () => { + try { await engine?.destroy(); } catch { /* already torn down */ } + }); + + const sysFile = (id: string) => engine.findOne('sys_file', { where: { id }, ...SYSTEM } as any); + const allFiles = () => engine.find('sys_file', { orderBy: [{ field: 'id', order: 'asc' }], ...SYSTEM } as any); + + /** + * A record write that names a file already owned by ANOTHER slot — the one + * condition that reaches `copyOwnedFile` (copy-on-claim), which re-inserts + * the source row's scope on the copy. + */ + const copyBySecondReference = (recordId: string, fileId: string) => + engine.insert('product', { id: recordId, title: recordId, image: fileId }); + + it('a deployment with no public rows runs clean: it counts zero and writes nothing', async () => { + await engine.insert('sys_file', { ...legacyPublicRow('f_user'), scope: 'user' }, SYSTEM as any); + await engine.insert('sys_file', { ...legacyPublicRow('f_att'), scope: 'attachments', ref_object: null, ref_id: null, ref_field: null }, SYSTEM as any); + const before = await allFiles(); + sysFileUpdates = 0; + + const report = await runSysFilePublicScopeBackfill(engine as any, { dryRun: false }); + + expect(report).toMatchObject({ dryRun: false, scanned: 0, planned: 0, written: 0, rows: [], failures: [], notes: [] }); + expect(sysFileUpdates).toBe(0); + expect(await allFiles()).toEqual(before); + }); + + it('counts before it writes: the default run is a dry run that names every row and writes none', async () => { + await sql.create('sys_file', legacyPublicRow('f_pub')); + sysFileUpdates = 0; + + const dry = await runSysFilePublicScopeBackfill(engine as any); + + expect(dry).toMatchObject({ dryRun: true, scanned: 1, planned: 1, written: 0, failures: [], notes: [] }); + expect(dry.rows).toEqual([{ id: 'f_pub', key: 'public/f_pub.png', from: 'public', to: 'user' }]); + expect(sysFileUpdates).toBe(0); + expect((await sysFile('f_pub'))?.scope).toBe(RETIRED_SYS_FILE_SCOPE); + expect(formatSysFilePublicScopeBackfillReport(dry)).toContain('f_pub'); + }); + + it('before the sweep runs, a copy of a stored public row is refused — the operator step the changeset names', async () => { + await sql.create('sys_file', legacyPublicRow('f_pub')); + + // The rest of the row's life is unaffected: it reads, and a scope-free + // update lands. + expect((await sysFile('f_pub'))?.scope).toBe(RETIRED_SYS_FILE_SCOPE); + await engine.update('sys_file', { id: 'f_pub', name: 'renamed.png' }, SYSTEM as any); + expect((await sysFile('f_pub'))?.name).toBe('renamed.png'); + + // The copy re-inserts `scope: 'public'`, which the select no longer declares. + await expect(copyBySecondReference('p1', 'f_pub')).rejects.toMatchObject({ code: 'ERR_FILE_REFERENCE_COPY' }); + expect((await allFiles()).map((f: any) => f.id)).toEqual(['f_pub']); + + // …and that refusal is the engine's option check on `scope`, the same one + // an insert naming the retired value meets directly. + await expect( + engine.insert('sys_file', { ...legacyPublicRow('f_direct'), ref_object: null, ref_id: null, ref_field: null }, SYSTEM as any), + ).rejects.toMatchObject({ + code: 'VALIDATION_FAILED', + fields: [expect.objectContaining({ field: 'scope', code: 'invalid_option' })], + }); + }); + + it('a copy of a rewritten row succeeds — scope user, the key and the bytes untouched (copyOwnedFile)', async () => { + await sql.create('sys_file', legacyPublicRow('f_pub')); + + const plan = await planSysFilePublicScopeBackfill(engine as any); + const applied = await applySysFilePublicScopeBackfill(engine as any, plan); + expect(applied).toMatchObject({ dryRun: false, scanned: 1, planned: 1, written: 1, failures: [] }); + + const rewritten = await sysFile('f_pub'); + expect(rewritten).toMatchObject({ + scope: REWRITTEN_SYS_FILE_SCOPE, + key: 'public/f_pub.png', + ref_object: 'product', + ref_id: 'p0', + ref_field: 'image', + acl: 'private', + status: 'committed', + }); + + const record = await copyBySecondReference('p1', 'f_pub'); + const copyId = (record as any).image; + expect(copyId).not.toBe('f_pub'); + expect(storage.download).toHaveBeenCalledWith('public/f_pub.png'); + const copy = await sysFile(copyId); + expect(copy).toMatchObject({ scope: REWRITTEN_SYS_FILE_SCOPE, ref_object: 'product', ref_id: 'p1', ref_field: 'image' }); + expect(String(copy?.key)).toMatch(/^user\//); + }); + + it('a second run writes zero', async () => { + await sql.create('sys_file', legacyPublicRow('f_pub_1')); + await sql.create('sys_file', legacyPublicRow('f_pub_2')); + + const first = await runSysFilePublicScopeBackfill(engine as any, { dryRun: false }); + expect(first).toMatchObject({ scanned: 2, written: 2, failures: [] }); + sysFileUpdates = 0; + + const second = await runSysFilePublicScopeBackfill(engine as any, { dryRun: false }); + expect(second).toMatchObject({ scanned: 0, planned: 0, written: 0, rows: [], failures: [], notes: [] }); + expect(sysFileUpdates).toBe(0); + }); + + it('pages through the population by id without skipping a row', async () => { + for (const id of ['f_a', 'f_b', 'f_c', 'f_d', 'f_e']) await sql.create('sys_file', legacyPublicRow(id)); + + const plan = await planSysFilePublicScopeBackfill(engine as any, { pageSize: 2 }); + + expect(plan.rows.map((r) => r.id)).toEqual(['f_a', 'f_b', 'f_c', 'f_d', 'f_e']); + expect(plan.notes).toEqual([]); + }); + + it('says so when the scan stops at its ceiling, and the next run picks up the rest', async () => { + for (const id of ['f_a', 'f_b', 'f_c']) await sql.create('sys_file', legacyPublicRow(id)); + + const first = await runSysFilePublicScopeBackfill(engine as any, { dryRun: false, pageSize: 2, maxRows: 2 }); + expect(first).toMatchObject({ scanned: 2, written: 2 }); + expect(first.notes).toHaveLength(1); + + const second = await runSysFilePublicScopeBackfill(engine as any, { dryRun: false, pageSize: 2, maxRows: 2 }); + expect(second).toMatchObject({ scanned: 1, written: 1, notes: [] }); + }); + + it('reports a scan it could not make instead of reading as a clean deployment', async () => { + const bare = new ObjectQL({ logger: silentLogger() } as any); + bare.registerDriver(new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) as any, true); + await bare.init(); + try { + const report = await runSysFilePublicScopeBackfill(bare as any, { dryRun: false }); + expect(report).toMatchObject({ scanned: 0, written: 0 }); + expect(report.notes).toHaveLength(1); + } finally { + await bare.destroy(); + } + }); + + it('the rollback needs the previous release: on this one the inverse write is refused by the engine', async () => { + await sql.create('sys_file', legacyPublicRow('f_pub')); + const applied = await runSysFilePublicScopeBackfill(engine as any, { dryRun: false }); + expect(applied.rows.map((r) => r.id)).toEqual(['f_pub']); + expect(formatSysFilePublicScopeBackfillReport(applied)).toContain('f_pub'); + + await expect( + engine.update('sys_file', { id: 'f_pub', scope: RETIRED_SYS_FILE_SCOPE }, SYSTEM as any), + ).rejects.toMatchObject({ + code: 'VALIDATION_FAILED', + fields: [expect.objectContaining({ field: 'scope', code: 'invalid_option' })], + }); + expect((await sysFile('f_pub'))?.scope).toBe(REWRITTEN_SYS_FILE_SCOPE); + }); +}); diff --git a/packages/services/service-storage/src/backfill-sys-file-public-scope.ts b/packages/services/service-storage/src/backfill-sys-file-public-scope.ts new file mode 100644 index 00000000000..63147888043 --- /dev/null +++ b/packages/services/service-storage/src/backfill-sys-file-public-scope.ts @@ -0,0 +1,326 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * backfill-sys-file-public-scope — the ONE-OFF rewrite of every stored + * `sys_file` row whose `scope` is the retired `public` to `user`. + * + * ## Why the rows have to move + * + * `scope: 'public'` promised a public file and never delivered one: the + * download doors judge `acl: 'public_read'`, the attachments scope and field + * ownership alone, so a `public`-scoped file with the default acl needs a + * signed-in caller like any other. The value is retired (#22443): the upload + * doors refuse it, the spec's `StorageScopeSchema` refuses it, and the + * `sys_file.scope` select no longer lists it. Ruling B (#22443, 2026-10-09) + * settled what happens to the rows already stored with it — they are rewritten + * to `user`, and this module is that rewrite. + * + * Retiring the option alone was measured infeasible: the field-reference copy + * path (`copyOwnedFile` in `file-reference-lifecycle.ts`) re-inserts the source + * row's scope on the copy, and the engine refuses an insert carrying a value the + * select does not declare. So until this sweep runs on a deployment, a record + * write that names a `public` file another field already owns fails: the + * engine throws `ERR_FILE_REFERENCE_COPY` around the `invalid_option` refusal + * on `scope`, and the data REST doors answer it `500 INTERNAL_ERROR` with the + * sentence withheld from the client (the server logs it). Reads, downloads and + * scope-free updates of those rows are unaffected. That window is what makes + * this sweep the upgrade's operator step, not an optional tidy-up. + * + * ## Why `user`, and why nothing else changes + * + * `user` is the value the copy path itself defaults a missing scope to, and the + * one the upload doors default an omitted scope to. No reader distinguishes it + * from `public`: the only scope value any code reads is `attachments` (the + * download doors, the attachment lifecycle, the orphan inventory, the reference + * verifier). So the rewrite changes no access behaviour. ONE column is written — + * `scope` — and nothing else on the row: the storage `key` keeps its `public/` + * prefix and no byte in the backend moves. + * + * ## Counts first, a no-op at zero, idempotent by construction + * + * The scan is `WHERE scope = 'public'`, read in full before anything is + * written; the count is the report's `scanned`. A deployment with no such rows + * plans nothing and writes nothing. Every write moves its row out of the + * predicate, so a second run over an unchanged database scans 0 and writes 0. + * `backfill-sys-file-public-scope.test.ts` asserts all three on a real engine. + * + * ## Rollback + * + * The inverse is `user` → `public` on exactly the ids an applied run touched — + * every one of them is listed in the applied report + * ({@link SysFilePublicScopeBackfillReport.rows}, printed by + * {@link formatSysFilePublicScopeBackfillReport}), so keep that output. On the + * release that ships this module the inverse CANNOT be written through the + * engine: `public` is no longer a declared option, and the engine refuses the + * write as it refuses any undeclared select value (pinned). That is by design — + * a stored `public` row on this release is exactly the row a copy cannot be + * made of. The rollback therefore goes with a code rollback: on the previous + * release, which still declares the option, write `scope = 'public'` back to + * the recorded ids (through the engine there, or as one raw driver statement + * against `sys_file` filtered to those ids). + * + * ## Usage + * + * An operator step, not a boot hook — the same posture as the organization + * backfill beside it (`backfill-sys-file-organizations.ts`): not exported from + * the package index, run server-side from a context that holds an engine, dry + * run first and by default: + * + * ```ts + * const plan = await planSysFilePublicScopeBackfill(engine); + * console.log(formatSysFilePublicScopeBackfillReport(plan)); // writes nothing + * // …read it, then: + * const applied = await applySysFilePublicScopeBackfill(engine, plan); + * console.log(formatSysFilePublicScopeBackfillReport(applied)); // keep it: it is the rollback list + * ``` + */ + +/** The ONE object this sweep rewrites. */ +export const SYS_FILE_PUBLIC_SCOPE_BACKFILL_OBJECT = 'sys_file'; + +/** The retired value the scan selects on. */ +export const RETIRED_SYS_FILE_SCOPE = 'public'; + +/** + * The value written in its place — the copy path's own default for a missing + * scope, and the upload doors' default for an omitted one. + */ +export const REWRITTEN_SYS_FILE_SCOPE = 'user'; + +const SCOPE_FIELD = 'scope'; +const SYSTEM_CONTEXT = { isSystem: true, positions: [], permissions: [] }; +const DEFAULT_PAGE_SIZE = 200; +const DEFAULT_MAX_ROWS = 100_000; + +/** + * The engine surface the sweep needs — a structural subset of the ObjectQL + * engine, declared here so a test can hand it a real engine or a double. + */ +export interface SysFilePublicScopeBackfillEngine { + find(object: string, options?: unknown): Promise; + update(object: string, data: unknown, options?: unknown): Promise; +} + +/** One row the sweep rewrites, named in full so the dry run is auditable. */ +export interface PlannedSysFileScopeRow { + id: string; + /** The storage key — reported, never written: it keeps its prefix. */ + key: string | null; + from: typeof RETIRED_SYS_FILE_SCOPE; + to: typeof REWRITTEN_SYS_FILE_SCOPE; +} + +/** The whole sweep's plan / outcome. */ +export interface SysFilePublicScopeBackfillReport { + /** `true` when nothing was written. */ + dryRun: boolean; + /** Rows matching `scope = 'public'` at scan time — the count taken before any write. */ + scanned: number; + /** Rows the sweep would write (dry run) or attempted to write (applied). */ + planned: number; + /** Rows actually written. Always 0 on a dry run. */ + written: number; + /** Every planned row; on an applied run, the ids the rollback writes back. */ + rows: PlannedSysFileScopeRow[]; + /** Scanned rows that carry no id the sweep can address — counted, never written. */ + unaddressable: number; + /** Planned rows whose write threw. Reported, never retried, never fatal. */ + failures: Array<{ id: string; error: string }>; + /** Conditions a reader must see, e.g. "the scan failed" or "the ceiling was hit". */ + notes: string[]; +} + +/** Options both halves of the sweep accept. */ +export interface SysFilePublicScopeBackfillOptions { + /** + * Execution context for every read and write. Defaults to a system context: + * the sweep has to see rows across every organization. + */ + context?: unknown; + /** Rows per page while scanning. */ + pageSize?: number; + /** Hard ceiling, so a pathological table cannot spin forever; reported when hit. */ + maxRows?: number; + /** + * `false` writes. Defaults to `true`: a sweep over existing data that defaults + * to writing is one typo away from an unplanned migration. + */ + dryRun?: boolean; +} + +function rowId(row: unknown): string | null { + const raw = (row as Record | null)?.id; + if (typeof raw === 'string' && raw.length > 0) return raw; + if (typeof raw === 'number') return String(raw); + return null; +} + +/** + * Build the sweep's plan — the DRY RUN. Reads only; `written` is 0. + * + * Paged by `id` so the pages partition the population, and read in full BEFORE + * anything is written — a plan built while writing would move rows out from + * under its own offset. + */ +export async function planSysFilePublicScopeBackfill( + engine: SysFilePublicScopeBackfillEngine, + options: SysFilePublicScopeBackfillOptions = {}, +): Promise { + const context = options.context ?? SYSTEM_CONTEXT; + const pageSize = options.pageSize ?? DEFAULT_PAGE_SIZE; + const maxRows = options.maxRows ?? DEFAULT_MAX_ROWS; + const notes: string[] = []; + const rows: PlannedSysFileScopeRow[] = []; + let scanned = 0; + let unaddressable = 0; + + let offset = 0; + for (; offset < maxRows; offset += pageSize) { + let page: unknown[]; + try { + page = await engine.find(SYS_FILE_PUBLIC_SCOPE_BACKFILL_OBJECT, { + where: { [SCOPE_FIELD]: RETIRED_SYS_FILE_SCOPE }, + fields: ['id', 'key'], + limit: pageSize, + offset, + orderBy: [{ field: 'id', order: 'asc' }], + context, + }); + } catch (err) { + // Named, not thrown: a reader has to be able to tell "no public rows" + // from "never looked". + notes.push( + `scan of '${SYS_FILE_PUBLIC_SCOPE_BACKFILL_OBJECT}' failed — ${String((err as Error)?.message ?? err)}. ` + + 'Nothing was counted; this is not a clean deployment, it is an unread one.', + ); + break; + } + const batch = Array.isArray(page) ? page : []; + for (const row of batch) { + scanned += 1; + const id = rowId(row); + if (!id) { + unaddressable += 1; + continue; + } + const key = (row as Record).key; + rows.push({ + id, + key: typeof key === 'string' ? key : null, + from: RETIRED_SYS_FILE_SCOPE, + to: REWRITTEN_SYS_FILE_SCOPE, + }); + } + if (batch.length < pageSize) break; + } + if (offset >= maxRows) { + notes.push( + `the scan stopped at its ceiling of ${maxRows} rows — more '${RETIRED_SYS_FILE_SCOPE}' rows may remain. ` + + 'Apply this plan, then run the sweep again: it picks up exactly the rows still matching.', + ); + } + + return { + dryRun: true, + scanned, + planned: rows.length, + written: 0, + rows, + unaddressable, + failures: [], + notes, + }; +} + +/** + * Write the plan: ONE update per planned row, carrying its id and + * `scope: 'user'` and nothing else — which is what keeps the key and the bytes + * untouched and the inverse expressible as "write `public` back to these ids". + * + * A row whose write throws is RECORDED and the sweep continues: one refused row + * must not cost the others their rewrite, and a half-done sweep is safe because + * the next run picks up exactly what still matches. + * + * ⛔ Takes a plan rather than building one, so the rows written are the rows a + * human read in the dry run. A plan with zero rows writes nothing. + */ +export async function applySysFilePublicScopeBackfill( + engine: SysFilePublicScopeBackfillEngine, + plan: SysFilePublicScopeBackfillReport, + options: SysFilePublicScopeBackfillOptions = {}, +): Promise { + const context = options.context ?? SYSTEM_CONTEXT; + const failures: SysFilePublicScopeBackfillReport['failures'] = []; + let written = 0; + for (const row of plan.rows) { + try { + await engine.update( + SYS_FILE_PUBLIC_SCOPE_BACKFILL_OBJECT, + { id: row.id, [SCOPE_FIELD]: REWRITTEN_SYS_FILE_SCOPE }, + { context }, + ); + written += 1; + } catch (err) { + failures.push({ id: row.id, error: String((err as Error)?.message ?? err) }); + } + } + return { ...plan, dryRun: false, written, failures }; +} + +/** + * Plan, then (only when `dryRun: false`) write — the whole sweep in one call. + * + * Idempotent by construction rather than by a guard: every write moves its row + * out of `scope = 'public'`, so a second call plans nothing and writes nothing. + */ +export async function runSysFilePublicScopeBackfill( + engine: SysFilePublicScopeBackfillEngine, + options: SysFilePublicScopeBackfillOptions = {}, +): Promise { + const plan = await planSysFilePublicScopeBackfill(engine, options); + if (options.dryRun !== false) return plan; + return applySysFilePublicScopeBackfill(engine, plan, options); +} + +/** + * Render a report as the operator-facing text. On an applied run the listed + * ids are the rollback list (see the module doc), so the output is worth + * keeping. + */ +export function formatSysFilePublicScopeBackfillReport(report: SysFilePublicScopeBackfillReport): string { + const lines: string[] = []; + lines.push( + report.dryRun + ? `sys_file scope '${RETIRED_SYS_FILE_SCOPE}' → '${REWRITTEN_SYS_FILE_SCOPE}' — DRY RUN (nothing written)` + : `sys_file scope '${RETIRED_SYS_FILE_SCOPE}' → '${REWRITTEN_SYS_FILE_SCOPE}' — APPLIED`, + ); + lines.push('='.repeat(66)); + lines.push(`scanned (scope = '${RETIRED_SYS_FILE_SCOPE}') : ${report.scanned}`); + lines.push(`${report.dryRun ? 'would write' : 'written '} : ${report.dryRun ? report.planned : report.written}`); + for (const row of report.rows) { + const failed = report.failures.find((f) => f.id === row.id); + lines.push( + ` ${row.id} scope ${row.from} -> ${row.to} (key ${row.key ?? '(none)'} unchanged)` + + (failed ? ` ✗ NOT written — ${failed.error}` : ''), + ); + } + if (report.unaddressable > 0) { + lines.push(` ⚠️ ${report.unaddressable} matching row(s) carry no usable id and were not written`); + } + for (const note of report.notes) lines.push(` ⚠️ ${note}`); + if (!report.dryRun && report.written > 0) { + lines.push(''); + lines.push( + `Rollback: on the previous release, write scope '${RETIRED_SYS_FILE_SCOPE}' back to exactly the ids above ` + + `that were written. This release refuses '${RETIRED_SYS_FILE_SCOPE}' as a sys_file scope.`, + ); + } + lines.push(''); + lines.push('-'.repeat(66)); + lines.push( + `TOTAL scanned=${report.scanned} ` + + `${report.dryRun ? 'would-write' : 'written'}=${report.dryRun ? report.planned : report.written} ` + + `failed=${report.failures.length}`, + ); + return lines.join('\n'); +} diff --git a/packages/services/service-storage/src/objects/system-file.object.ts b/packages/services/service-storage/src/objects/system-file.object.ts index 4917efba2cd..5fdfd8af40f 100644 --- a/packages/services/service-storage/src/objects/system-file.object.ts +++ b/packages/services/service-storage/src/objects/system-file.object.ts @@ -55,12 +55,18 @@ export const SystemFile = ObjectSchema.create({ label: 'Size (bytes)', }), + // A logical key prefix and a lifecycle discriminator — never an access + // grant: the download doors judge `acl`, the attachments scope and field + // ownership alone. `public` is retired (#22443, ruling B): it promised a + // public file and stored a private one. A deployment's stored `public` rows + // are rewritten to `user` by `backfill-sys-file-public-scope.ts`, the + // operator step that must run before a copy of such a row can succeed. + // Anonymous download is `acl: 'public_read'` and nothing else. scope: Field.select({ label: 'Scope', options: [ { label: 'User', value: 'user' }, { label: 'Tenant', value: 'tenant' }, - { label: 'Public', value: 'public' }, { label: 'Private', value: 'private' }, { label: 'Temp', value: 'temp' }, // Files uploaded through the generic Attachments surface (#2727). From 7ee6884d44d7b79ae5330bc3adca481edbcbf4bf Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:10:30 +0000 Subject: [PATCH 02/11] chore(changeset): state the sys_file public-scope rewrite, its operator step and its rollback The test's engine reads now pass typed query options instead of erasing them. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../22443-sys-file-public-scope-rewrite.md | 46 +++++++++++++++++++ .../backfill-sys-file-public-scope.test.ts | 4 +- 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 .changeset/22443-sys-file-public-scope-rewrite.md diff --git a/.changeset/22443-sys-file-public-scope-rewrite.md b/.changeset/22443-sys-file-public-scope-rewrite.md new file mode 100644 index 00000000000..beb8e37887c --- /dev/null +++ b/.changeset/22443-sys-file-public-scope-rewrite.md @@ -0,0 +1,46 @@ +--- +'@objectstack/service-storage': minor +--- + +feat(service-storage)!: the `sys_file` scope option `public` is retired, and rows already stored with it are rewritten to `user` by a one-time operator sweep (#22443) + +Clause-②: no (narrowing) + + + +**BREAKING** — an accept-set narrowing on the `sys_file` object's `scope` select, shipped as `minor` under the launch-window convention for accept-set narrowings. It completes the retirement registered as `storage-scope-public-retired`: `StorageScopeSchema` and both upload doors already refuse `public`, and now the stored vocabulary does too. + +### What changes + +- **`sys_file.scope` no longer lists `public`.** Its options are `user`, `tenant`, `private`, `temp` and `attachments`. The engine refuses a write of `public` to the column as it refuses any undeclared option: `VALIDATION_FAILED`, `invalid_option` on `scope`. No access behaviour changes: the only scope value any code reads is `attachments`, and whether a file can be read before sign-in is decided by `acl: 'public_read'` alone. +- **Rows already stored with scope `public` are rewritten to `user`**, by a one-time sweep the operator runs (below), never at boot. `user` is the scope the upload doors and the field-reference copy path default to, and no reader tells it apart from `public`. One column moves: the storage key keeps its `public/` prefix, the acl and ownership columns are untouched, and no byte moves in the storage backend. + +### FROM → TO + +| before | write instead | +| --- | --- | +| a `sys_file` row written with `scope: 'public'` | `scope: 'user'`, or no scope; and `acl: 'public_read'` on the row if the file must be readable before sign-in | +| rows already stored with `scope: 'public'` | nothing by hand: the sweep below rewrites them to `user` | + +**The one-line fix: stop writing scope `public` on `sys_file`, and run the sweep once on each deployment that stored it.** + +### The operator step: run the sweep once after upgrading + +Until the sweep has run on a deployment that stored `public` rows, a record write that names a `public` file another field already owns is refused. The field-reference copy path copies the file into a new row with the source row's scope, the engine refuses `public` there, and the write fails with `ERR_FILE_REFERENCE_COPY`. The data REST doors answer that `500 INTERNAL_ERROR` ("Internal server error"); the server log carries the full sentence, which ends `Scope must be one of: user, tenant, private, temp, attachments`. Reads, downloads and updates that do not write `scope` are unaffected. + +The sweep is `packages/services/service-storage/src/backfill-sys-file-public-scope.ts`, an operator module in the shape of the `sys_file` organization backfill. Like that module it is not exported from the package index and is not in the published `dist`, so run it from a source checkout of this release, server-side, from a context that holds the engine. It is a dry run first, and by default: + +```ts +const plan = await planSysFilePublicScopeBackfill(engine); // counts, writes nothing +console.log(formatSysFilePublicScopeBackfillReport(plan)); +const applied = await applySysFilePublicScopeBackfill(engine, plan); // one scope write per row +console.log(formatSysFilePublicScopeBackfillReport(applied)); // keep it: it is the rollback list +``` + +It counts before it writes (`scanned`), writes nothing where the count is zero, and is idempotent: every write moves its row out of `scope = 'public'`, so a second run scans and writes zero. A row whose write fails is reported, never retried, and is picked up by the next run. + +### Rollback + +The inverse is `user` back to `public` on exactly the ids the applied report lists. This release cannot write it through the engine, which refuses `public` like any undeclared option. So the inverse goes with a code rollback: on the previous release, which still declares the option, write `scope = 'public'` back to those ids, through the engine there or as one raw driver statement against `sys_file` filtered to them. + +The earlier note of `storage-scope-public-retired` that files already stored with scope `public` are not touched no longer holds for the `scope` column: after the sweep they read `user`. They still download exactly as before. diff --git a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts index 5c50de93fe9..a8c658f52f7 100644 --- a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts +++ b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts @@ -99,8 +99,8 @@ describe('sys_file public-scope backfill (#22443 ruling B) — a real ObjectQL o try { await engine?.destroy(); } catch { /* already torn down */ } }); - const sysFile = (id: string) => engine.findOne('sys_file', { where: { id }, ...SYSTEM } as any); - const allFiles = () => engine.find('sys_file', { orderBy: [{ field: 'id', order: 'asc' }], ...SYSTEM } as any); + const sysFile = (id: string) => engine.findOne('sys_file', { where: { id } }); + const allFiles = () => engine.find('sys_file', { orderBy: [{ field: 'id', order: 'asc' }] }); /** * A record write that names a file already owned by ANOTHER slot — the one From 0f1766d92fb8234715e3b41a8866461d99098112 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:12:19 +0000 Subject: [PATCH 03/11] chore(i18n): drop the retired sys_file scope option public from the storage bundles Regenerated with check-i18n-bundles --write; no other key moves. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../service-storage/src/translations/en.objects.generated.ts | 1 - .../service-storage/src/translations/es-ES.objects.generated.ts | 1 - .../service-storage/src/translations/ja-JP.objects.generated.ts | 1 - .../service-storage/src/translations/zh-CN.objects.generated.ts | 1 - 4 files changed, 4 deletions(-) diff --git a/packages/services/service-storage/src/translations/en.objects.generated.ts b/packages/services/service-storage/src/translations/en.objects.generated.ts index c6e281ecec6..a8bda94427b 100644 --- a/packages/services/service-storage/src/translations/en.objects.generated.ts +++ b/packages/services/service-storage/src/translations/en.objects.generated.ts @@ -40,7 +40,6 @@ export const enObjects: NonNullable = { options: { user: "User", tenant: "Tenant", - public: "Public", private: "Private", temp: "Temp", attachments: "Attachments" diff --git a/packages/services/service-storage/src/translations/es-ES.objects.generated.ts b/packages/services/service-storage/src/translations/es-ES.objects.generated.ts index 2aa6b04216a..3d3a86ee5bf 100644 --- a/packages/services/service-storage/src/translations/es-ES.objects.generated.ts +++ b/packages/services/service-storage/src/translations/es-ES.objects.generated.ts @@ -40,7 +40,6 @@ export const esESObjects: NonNullable = { options: { user: "Usuario", tenant: "Inquilino", - public: "Público", private: "Privado", temp: "Temporal", attachments: "Adjuntos" diff --git a/packages/services/service-storage/src/translations/ja-JP.objects.generated.ts b/packages/services/service-storage/src/translations/ja-JP.objects.generated.ts index 09f6f704206..2a08c1d720a 100644 --- a/packages/services/service-storage/src/translations/ja-JP.objects.generated.ts +++ b/packages/services/service-storage/src/translations/ja-JP.objects.generated.ts @@ -40,7 +40,6 @@ export const jaJPObjects: NonNullable = { options: { user: "ユーザー", tenant: "テナント", - public: "公開", private: "非公開", temp: "一時", attachments: "添付ファイル" diff --git a/packages/services/service-storage/src/translations/zh-CN.objects.generated.ts b/packages/services/service-storage/src/translations/zh-CN.objects.generated.ts index 9825711310e..bf7922c8bb4 100644 --- a/packages/services/service-storage/src/translations/zh-CN.objects.generated.ts +++ b/packages/services/service-storage/src/translations/zh-CN.objects.generated.ts @@ -40,7 +40,6 @@ export const zhCNObjects: NonNullable = { options: { user: "用户", tenant: "租户", - public: "公开", private: "私有", temp: "临时", attachments: "附件" From dec868f9f824af5e89cd97e3ea02f59349b0d010 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:13:15 +0000 Subject: [PATCH 04/11] test(service-storage): the copy-of-a-rewritten-row pin asserts the copy first Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../backfill-sys-file-public-scope.test.ts | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts index a8c658f52f7..3020c7a39eb 100644 --- a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts +++ b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts @@ -164,10 +164,19 @@ describe('sys_file public-scope backfill (#22443 ruling B) — a real ObjectQL o const plan = await planSysFilePublicScopeBackfill(engine as any); const applied = await applySysFilePublicScopeBackfill(engine as any, plan); - expect(applied).toMatchObject({ dryRun: false, scanned: 1, planned: 1, written: 1, failures: [] }); - const rewritten = await sysFile('f_pub'); - expect(rewritten).toMatchObject({ + // The pin: the copy the un-swept row refuses (the case above) now lands. + const record = await copyBySecondReference('p1', 'f_pub'); + const copyId = (record as any).image; + expect(copyId).not.toBe('f_pub'); + expect(storage.download).toHaveBeenCalledWith('public/f_pub.png'); + const copy = await sysFile(copyId); + expect(copy).toMatchObject({ scope: REWRITTEN_SYS_FILE_SCOPE, ref_object: 'product', ref_id: 'p1', ref_field: 'image' }); + expect(String(copy?.key)).toMatch(/^user\//); + + // One column moved on the source row; the key, the acl and ownership did not. + expect(applied).toMatchObject({ dryRun: false, scanned: 1, planned: 1, written: 1, failures: [] }); + expect(await sysFile('f_pub')).toMatchObject({ scope: REWRITTEN_SYS_FILE_SCOPE, key: 'public/f_pub.png', ref_object: 'product', @@ -176,14 +185,6 @@ describe('sys_file public-scope backfill (#22443 ruling B) — a real ObjectQL o acl: 'private', status: 'committed', }); - - const record = await copyBySecondReference('p1', 'f_pub'); - const copyId = (record as any).image; - expect(copyId).not.toBe('f_pub'); - expect(storage.download).toHaveBeenCalledWith('public/f_pub.png'); - const copy = await sysFile(copyId); - expect(copy).toMatchObject({ scope: REWRITTEN_SYS_FILE_SCOPE, ref_object: 'product', ref_id: 'p1', ref_field: 'image' }); - expect(String(copy?.key)).toMatch(/^user\//); }); it('a second run writes zero', async () => { From aef5ff0146ba7d6b820d3e2b7787960cfd5cf765 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 22:36:07 +0000 Subject: [PATCH 05/11] chore(changeset): spell the ADR-0087 disposition as not-required (already-registered storage-scope-public-retired) Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .changeset/22443-sys-file-public-scope-rewrite.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/22443-sys-file-public-scope-rewrite.md b/.changeset/22443-sys-file-public-scope-rewrite.md index beb8e37887c..6602c530029 100644 --- a/.changeset/22443-sys-file-public-scope-rewrite.md +++ b/.changeset/22443-sys-file-public-scope-rewrite.md @@ -6,7 +6,7 @@ feat(service-storage)!: the `sys_file` scope option `public` is retired, and row Clause-②: no (narrowing) - + **BREAKING** — an accept-set narrowing on the `sys_file` object's `scope` select, shipped as `minor` under the launch-window convention for accept-set narrowings. It completes the retirement registered as `storage-scope-public-retired`: `StorageScopeSchema` and both upload doors already refuse `public`, and now the stored vocabulary does too. From 7b8ef8559ad50e91e7b60386efe72222374431be Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 23:23:52 +0000 Subject: [PATCH 06/11] feat(service-storage): export the sys_file public-scope sweep from the package entry plan / apply / run / format and their report types are exported beside backfillFileReferences. Until the sweep runs, a copy of a stored public row is refused, and a deployment consumes the published package, not a source checkout, so the operator step has to ship in the release that asks for it. The module docblock's usage section now imports from the package. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../src/backfill-sys-file-public-scope.ts | 15 ++++++++++++--- packages/services/service-storage/src/index.ts | 17 +++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/packages/services/service-storage/src/backfill-sys-file-public-scope.ts b/packages/services/service-storage/src/backfill-sys-file-public-scope.ts index 63147888043..361eebe99b9 100644 --- a/packages/services/service-storage/src/backfill-sys-file-public-scope.ts +++ b/packages/services/service-storage/src/backfill-sys-file-public-scope.ts @@ -62,11 +62,20 @@ * ## Usage * * An operator step, not a boot hook — the same posture as the organization - * backfill beside it (`backfill-sys-file-organizations.ts`): not exported from - * the package index, run server-side from a context that holds an engine, dry - * run first and by default: + * backfill beside it (`backfill-sys-file-organizations.ts`): run server-side + * from a context that holds an engine, dry run first and by default. Unlike + * that backfill, the four functions and their report types ARE exported from + * the package index: until this sweep runs, a copy of a stored `public` row is + * refused (above), and a deployment runs the published package, not a source + * checkout, so the step has to ship in the release that asks for it: * * ```ts + * import { + * planSysFilePublicScopeBackfill, + * applySysFilePublicScopeBackfill, + * formatSysFilePublicScopeBackfillReport, + * } from '@objectstack/service-storage'; + * * const plan = await planSysFilePublicScopeBackfill(engine); * console.log(formatSysFilePublicScopeBackfillReport(plan)); // writes nothing * // …read it, then: diff --git a/packages/services/service-storage/src/index.ts b/packages/services/service-storage/src/index.ts index 629eb88c1ea..17fb305b211 100644 --- a/packages/services/service-storage/src/index.ts +++ b/packages/services/service-storage/src/index.ts @@ -92,6 +92,23 @@ export type { BackfillOptions, BackfillReport, } from './backfill-file-references.js'; +// The one-time operator sweep that rewrites stored `sys_file` rows whose scope +// is the retired `public` to `user`. Exported, unlike the organization backfill +// beside it, because a deployment that stored such rows must run it after +// upgrading (until it does, a copy of one of those rows is refused) and a +// deployment consumes this package, not a source checkout. +export { + planSysFilePublicScopeBackfill, + applySysFilePublicScopeBackfill, + runSysFilePublicScopeBackfill, + formatSysFilePublicScopeBackfillReport, +} from './backfill-sys-file-public-scope.js'; +export type { + PlannedSysFileScopeRow, + SysFilePublicScopeBackfillEngine, + SysFilePublicScopeBackfillOptions, + SysFilePublicScopeBackfillReport, +} from './backfill-sys-file-public-scope.js'; export { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js'; export type { AttachmentSharingLike, AttachmentSecurityLike } from './attachment-access-hooks.js'; export { runFilesToReferencesMigration } from './files-to-references-migration.js'; From a5093538d1fa64b8d15a3533538e2af10f7558cd Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 23:23:58 +0000 Subject: [PATCH 07/11] fix(spec): the step-18 entry storage-scope-public-retired names the sys_file option's retirement and its operator sweep Its reason said files already stored with scope public are not touched. The sys_file scope select now retires the option too, and the operator sweep @objectstack/service-storage exports rewrites those rows to user with no access change. The reason says so, and the acceptance criteria add the sweep's end state. The changeset names the package import for the operator step and gains a patch line for @objectstack/spec. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../22443-sys-file-public-scope-rewrite.md | 13 +++++++++++-- .../18.storage-scope-public-retired.ts | 18 ++++++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.changeset/22443-sys-file-public-scope-rewrite.md b/.changeset/22443-sys-file-public-scope-rewrite.md index 6602c530029..ed312ff6ee9 100644 --- a/.changeset/22443-sys-file-public-scope-rewrite.md +++ b/.changeset/22443-sys-file-public-scope-rewrite.md @@ -1,5 +1,6 @@ --- '@objectstack/service-storage': minor +'@objectstack/spec': patch --- feat(service-storage)!: the `sys_file` scope option `public` is retired, and rows already stored with it are rewritten to `user` by a one-time operator sweep (#22443) @@ -28,9 +29,15 @@ Clause-②: no (narrowing) Until the sweep has run on a deployment that stored `public` rows, a record write that names a `public` file another field already owns is refused. The field-reference copy path copies the file into a new row with the source row's scope, the engine refuses `public` there, and the write fails with `ERR_FILE_REFERENCE_COPY`. The data REST doors answer that `500 INTERNAL_ERROR` ("Internal server error"); the server log carries the full sentence, which ends `Scope must be one of: user, tenant, private, temp, attachments`. Reads, downloads and updates that do not write `scope` are unaffected. -The sweep is `packages/services/service-storage/src/backfill-sys-file-public-scope.ts`, an operator module in the shape of the `sys_file` organization backfill. Like that module it is not exported from the package index and is not in the published `dist`, so run it from a source checkout of this release, server-side, from a context that holds the engine. It is a dry run first, and by default: +The sweep ships in `@objectstack/service-storage` itself: `planSysFilePublicScopeBackfill`, `applySysFilePublicScopeBackfill`, `runSysFilePublicScopeBackfill` and `formatSysFilePublicScopeBackfillReport` are exported from the package, with their report types. It has the shape of the `sys_file` organization backfill: an operator step, never a boot hook. Import it from the package you upgraded to and run it server-side, from a context that holds the engine. It is a dry run first, and by default: ```ts +import { + planSysFilePublicScopeBackfill, + applySysFilePublicScopeBackfill, + formatSysFilePublicScopeBackfillReport, +} from '@objectstack/service-storage'; + const plan = await planSysFilePublicScopeBackfill(engine); // counts, writes nothing console.log(formatSysFilePublicScopeBackfillReport(plan)); const applied = await applySysFilePublicScopeBackfill(engine, plan); // one scope write per row @@ -43,4 +50,6 @@ It counts before it writes (`scanned`), writes nothing where the count is zero, The inverse is `user` back to `public` on exactly the ids the applied report lists. This release cannot write it through the engine, which refuses `public` like any undeclared option. So the inverse goes with a code rollback: on the previous release, which still declares the option, write `scope = 'public'` back to those ids, through the engine there or as one raw driver statement against `sys_file` filtered to them. -The earlier note of `storage-scope-public-retired` that files already stored with scope `public` are not touched no longer holds for the `scope` column: after the sweep they read `user`. They still download exactly as before. +### The step-18 entry says so too (`@objectstack/spec`) + +The D3 entry `storage-scope-public-retired`, which the protocol upgrade guide is built from, said that files already stored with scope `public` are not touched. That no longer holds for the `scope` column: after the sweep they read `user`. Its reason now names the `sys_file` option's retirement and the operator sweep above, and says that the rewrite changes no access: the storage key and the bytes stay as they are, and those files still download exactly as before. Its acceptance criteria add the sweep's end state: a dry run on each deployment scans zero `sys_file` records with scope `public`. diff --git a/packages/spec/src/migrations/entries/semantic/18.storage-scope-public-retired.ts b/packages/spec/src/migrations/entries/semantic/18.storage-scope-public-retired.ts index 10958ebdb04..a47ca00a7a3 100644 --- a/packages/spec/src/migrations/entries/semantic/18.storage-scope-public-retired.ts +++ b/packages/spec/src/migrations/entries/semantic/18.storage-scope-public-retired.ts @@ -8,7 +8,9 @@ import type { SemanticMigration } from '../../types.js'; // `400 INVALID_REQUEST`. Semantic only: no metadata type carries either // surface, so there is no authored source for a D2 conversion to rewrite — // what moves is caller code and the intent behind it, which only the caller -// can judge. +// can judge. The `sys_file.scope` select retires the option too, and the rows +// already stored with it are rewritten to `user` by the operator sweep +// `@objectstack/service-storage` exports (`backfill-sys-file-public-scope.ts`). export const entry: SemanticMigration = { id: 'storage-scope-public-retired', // No backticks and no pipes in `surface` — build-upgrade-guide.ts renders it @@ -29,11 +31,19 @@ export const entry: SemanticMigration = { + 'anonymous download. Whether a given file must be readable before sign-in is the caller\'s ' + 'call, so no rewrite can make it: an upload that meant public needs its stored file record ' + 'marked, and one that did not needs only another scope. The upload request itself carries no ' - + 'acl, and every upload is stored private. Files already stored with scope public are not ' - + 'touched and download exactly as before. ADR-0049', + + 'acl, and every upload is stored private. The stored file record retires the value too: the ' + + 'scope select of sys_file no longer lists public, so a deployment that stored files with it runs ' + + 'the one-time operator sweep that @objectstack/service-storage exports ' + + '(`planSysFilePublicScopeBackfill` for the dry run, then `applySysFilePublicScopeBackfill`), ' + + 'which rewrites each of those records to scope user. No access changes: no reader tells user ' + + 'apart from public, the storage key and the file bytes are not touched, and those files ' + + 'download exactly as before. Until the sweep has run, a record write that names such a file ' + + 'while another field already owns it is refused, because the copy it makes carries scope ' + + 'public. ADR-0049', acceptanceCriteria: 'No upload call names scope public and no ObjectStorageConfig declares it; each upload that did ' + 'now names another scope or none and is answered 200. Each file that must render before ' + "sign-in has acl 'public_read' on its stored file record, and fetching it with no session " - + 'serves it; fetching any other uploaded file with no session is answered 401.', + + 'serves it; fetching any other uploaded file with no session is answered 401. On each ' + + 'deployment, a dry run of the sweep scans zero sys_file records with scope public.', }; From fe229c03c9b84958f39c6e914ebef8f3ca4ab986 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 23:24:05 +0000 Subject: [PATCH 08/11] chore(spec): regenerate the migration registry for the step-18 entry edit pnpm --filter @objectstack/spec gen:migration-registry; the only change is the storage-scope-public-retired entry. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- packages/spec/src/migrations/registry.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/packages/spec/src/migrations/registry.ts b/packages/spec/src/migrations/registry.ts index 57bd7409cab..65774f4e98f 100644 --- a/packages/spec/src/migrations/registry.ts +++ b/packages/spec/src/migrations/registry.ts @@ -19290,7 +19290,9 @@ const step18: MigrationStep = { // `400 INVALID_REQUEST`. Semantic only: no metadata type carries either // surface, so there is no authored source for a D2 conversion to rewrite — // what moves is caller code and the intent behind it, which only the caller - // can judge. + // can judge. The `sys_file.scope` select retires the option too, and the rows + // already stored with it are rewritten to `user` by the operator sweep + // `@objectstack/service-storage` exports (`backfill-sys-file-public-scope.ts`). { id: 'storage-scope-public-retired', // No backticks and no pipes in `surface` — build-upgrade-guide.ts renders it @@ -19311,13 +19313,21 @@ const step18: MigrationStep = { + 'anonymous download. Whether a given file must be readable before sign-in is the caller\'s ' + 'call, so no rewrite can make it: an upload that meant public needs its stored file record ' + 'marked, and one that did not needs only another scope. The upload request itself carries no ' - + 'acl, and every upload is stored private. Files already stored with scope public are not ' - + 'touched and download exactly as before. ADR-0049', + + 'acl, and every upload is stored private. The stored file record retires the value too: the ' + + 'scope select of sys_file no longer lists public, so a deployment that stored files with it runs ' + + 'the one-time operator sweep that @objectstack/service-storage exports ' + + '(`planSysFilePublicScopeBackfill` for the dry run, then `applySysFilePublicScopeBackfill`), ' + + 'which rewrites each of those records to scope user. No access changes: no reader tells user ' + + 'apart from public, the storage key and the file bytes are not touched, and those files ' + + 'download exactly as before. Until the sweep has run, a record write that names such a file ' + + 'while another field already owns it is refused, because the copy it makes carries scope ' + + 'public. ADR-0049', acceptanceCriteria: 'No upload call names scope public and no ObjectStorageConfig declares it; each upload that did ' + 'now names another scope or none and is answered 200. Each file that must render before ' + "sign-in has acl 'public_read' on its stored file record, and fetching it with no session " - + 'serves it; fetching any other uploaded file with no session is answered 401.', + + 'serves it; fetching any other uploaded file with no session is answered 401. On each ' + + 'deployment, a dry run of the sweep scans zero sys_file records with scope public.', }, { id: 'strategy-context-aggregation-method-narrowed', From 15e5891c778a6f17958fa345f0fa6c9f0339a5bb Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 23:24:06 +0000 Subject: [PATCH 09/11] chore(census): place the sys_file public-scope sweep's write in the tenant-audit census node scripts/tenant-audit-census.mjs --write over the generated region and the counts ledger, plus the hand-written page counts the gate holds to it: 241 write call sites (from 240), 160 decidable, 105 of undecidable elevation, 53 named through a const. The new site is placed as the organization backfill's is: update, sys_file, tenancy enabled, context of undecidable elevation. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../docs/permissions/tenant-audit-census.mdx | 36 +++++++++---------- ...08-tenant-audit-write-call-sites.counts.md | 19 +++++----- 2 files changed, 28 insertions(+), 27 deletions(-) diff --git a/content/docs/permissions/tenant-audit-census.mdx b/content/docs/permissions/tenant-audit-census.mdx index d1f40735b38..d441f7454cc 100644 --- a/content/docs/permissions/tenant-audit-census.mdx +++ b/content/docs/permissions/tenant-audit-census.mdx @@ -122,7 +122,7 @@ are reported as `undecidable` rather than assumed either way. The same holds twice over for the context. An options argument spelled as a literal can be read; one spelled `options`, `{ ...opts }`, or handed through a -forwarding shim cannot, and **53 of the 240 sites are spelled that way**. A +forwarding shim cannot, and **53 of the 241 sites are spelled that way**. A context resolved from an inline literal or a local `const` can be tested for `isSystem`; one arriving from a helper call cannot. @@ -187,10 +187,10 @@ reproduce them. Where it disagrees, it disagrees on the page: | carried figure | where it survives | this census | | :--- | :--- | ---: | -| 175 write call sites | quoted in the merged changeset | **240** | +| 175 write call sites | quoted in the merged changeset | **241** | | 24 carrying no tenant context | quoted in the merged changeset | **2** provable and tenancy-enabled; **32** more whose options argument is unreadable | -| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **159 of 240** decidable, **81** undecidable | -| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 128 decidably elevated, 0 decidably not, 104 undecidable | +| 127 of 175 statically decidable, 48 runtime-parameter-name sites | restated on the `isSystem`-scoping card | **160 of 241** decidable, **81** undecidable | +| 135 (77%) silenced by the `isSystem` guard before the posture gate | the lost issue body — **no surviving corroboration** | **not reproduced**: 128 decidably elevated, 0 decidably not, 105 undecidable | | 141 and 132, two independent re-derivations | the card that filed this work | — | **The differences are not reconciled, and deliberately so.** The old census's @@ -201,17 +201,17 @@ at any commit. Two structural facts do plausibly widen this reading against any hand or regex one, and both are counted in the generated tables below: the 53 sites reached -through an erased (`any`) receiver, and the 52 that name their object through a +through an erased (`any`) receiver, and the 53 that name their object through a `const` rather than inline. An instrument that read either the way a person does would report a smaller number and would not say so. The fourth row is the one worth flagging to anyone citing it. **The 135 / 77% figure has no surviving corroboration anywhere in the tree.** This census reads -128 of 240 (53%) as decidably elevated, with 104 more whose elevation is a +128 of 241 (53%) as decidably elevated, with 105 more whose elevation is a run-time fact — so the claim is neither confirmed nor refuted, and the honest answer is that a static reading cannot settle it. -⇒ **Cite `2 / 240`, and say what it is**: the sites whose options argument was +⇒ **Cite `2 / 241`, and say what it is**: the sites whose options argument was READ and holds no tenant context, against a decidably tenancy-enabled object. That is the control's provable yield surface. ⛔ Do not cite it as "the sites without tenant context" — **32 further sites** have an options argument this @@ -223,29 +223,29 @@ cannot read, and they are neither in nor out. | what | count | | :--- | ---: | -| write call sites on the application surface | **240** | -| …whose object name is statically decidable | 159 | +| write call sites on the application surface | **241** | +| …whose object name is statically decidable | 160 | | …whose object name is chosen at run time | 81 | -| …against an object with tenancy ENABLED | 158 | +| …against an object with tenancy ENABLED | 159 | | …against an object that declares tenancy off | 1 | -| threading a tenant context | 179 | +| threading a tenant context | 180 | | PROVABLY carrying none (options read, no context key) | **8** | | …of those, against a decidably tenancy-enabled object | **2** | | options argument UNREADABLE — may or may not carry one | 53 | | …of those, against a decidably tenancy-enabled object | 32 | | threading a decidably ELEVATED (`isSystem`) context | 128 | | threading a context that is decidably NOT elevated | 0 | -| threading a context whose elevation is a run-time fact | 104 | +| threading a context whose elevation is a run-time fact | 105 | | how the instrument reached the site | count | | :--- | ---: | -| receiver carried a readable engine type | 187 | +| receiver carried a readable engine type | 188 | | receiver erased, placed by the object NAME | 33 | | receiver erased, placed by an `object: string` PARAMETER | 15 | | receiver erased, placed by an `UNTYPED_RECEIVERS` row | 5 | | object name spelled inline | 107 | -| object name spelled through a `const` | 52 | +| object name spelled through a `const` | 53 | | object name is an `object: string` parameter | 19 | | object name is some other run-time expression | 62 | @@ -297,13 +297,13 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-08 at `715ba6f44`. +Measured on 2026-10-09 at `5551f8d8d`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 623 | -| engine-shaped types recognised | 71 | -| declared objects in the registry | 117 | +| tracked non-test sources scanned | 625 | +| engine-shaped types recognised | 72 | +| declared objects in the registry | 116 | | same-named calls subtracted as non-engine | 162 | {/* END GENERATED: tenant-audit-census */} diff --git a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md index 3053e77f8fa..afd8e12c5eb 100644 --- a/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md +++ b/docs/audits/2026-08-tenant-audit-write-call-sites.counts.md @@ -33,19 +33,19 @@ silent, and `node scripts/tenant-audit-census.mjs --write` is the resolution. | Measure | Value | |---|---:| -| Write call sites | 240 | -| Object name statically decidable | 159 | +| Write call sites | 241 | +| Object name statically decidable | 160 | | Object name chosen at run time | 81 | -| Against a tenancy-enabled object | 158 | +| Against a tenancy-enabled object | 159 | | Against an object declaring tenancy off | 1 | -| Threading a tenant context | 179 | +| Threading a tenant context | 180 | | Provably carrying none | 8 | | …and decidably tenancy-enabled | 2 | | Options argument unreadable | 53 | | …and decidably tenancy-enabled | 32 | | Threading a decidably elevated context | 128 | | Threading a decidably non-elevated context | 0 | -| Threading a context of undecidable elevation | 104 | +| Threading a context of undecidable elevation | 105 | ## Subtractions the census could NOT defend — enforced @@ -90,13 +90,13 @@ holds still. They are required to be HERE and to say WHEN they were true; their values are not compared. The reasoning, and the measurement behind it, are in `scripts/check-tenant-audit-census.mjs`. -Measured on 2026-10-08 at `715ba6f44`. +Measured on 2026-10-09 at `5551f8d8d`. | corpus scale (not enforced) | count | | :--- | ---: | -| tracked non-test sources scanned | 623 | -| engine-shaped types recognised | 71 | -| declared objects in the registry | 117 | +| tracked non-test sources scanned | 625 | +| engine-shaped types recognised | 72 | +| declared objects in the registry | 116 | | same-named calls subtracted as non-engine | 162 | ## Every site @@ -251,6 +251,7 @@ Measured on 2026-10-08 at `715ba6f44`. | `packages/services/service-storage/src/backfill-file-references.ts` | `update` | `object` | undecidable | elevated | 1 | | `packages/services/service-storage/src/backfill-file-references.ts` | `insert` | `sys_file` | enabled | context, elevation undecidable | 1 | | `packages/services/service-storage/src/backfill-sys-file-organizations.ts` | `update` | `sys_file` | enabled | context, elevation undecidable | 1 | +| `packages/services/service-storage/src/backfill-sys-file-public-scope.ts` | `update` | `sys_file` | enabled | context, elevation undecidable | 1 | | `packages/services/service-storage/src/file-reference-lifecycle.ts` | `insert` | `sys_file` | enabled | context, elevation undecidable | 1 | | `packages/services/service-storage/src/file-reference-lifecycle.ts` | `update` | `sys_file` | enabled | elevated | 2 | | `packages/services/service-storage/src/metadata-store.ts` | `delete` | `sys_file` | enabled | options unreadable | 1 | From b1079535558d92b1fcd92d8cdcc25f30a71b105e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 23:34:07 +0000 Subject: [PATCH 10/11] test(service-storage): pin the sweep's package-entry exports through the operator step itself Dropping one of the four exports from the entry left every test and every derived export gate green (measured), so this case imports the sweep and its report types from ./index.js, asserts the entry hands out the very functions this file drives, and runs plan, apply and a second run through the entry against a stored public row. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .../backfill-sys-file-public-scope.test.ts | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts index 3020c7a39eb..c0cd83e45ff 100644 --- a/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts +++ b/packages/services/service-storage/src/backfill-sys-file-public-scope.test.ts @@ -22,7 +22,10 @@ // of a stored `public` row is refused, while reads and scope-free updates // of that row are unaffected; // - the rollback's precondition: on this release the inverse write -// (`user` → `public`) is refused by the engine. +// (`user` → `public`) is refused by the engine; +// - the operator step as the changeset writes it: the four functions and +// their report types come from the package entry (`./index.ts`), because a +// deployment runs the published package, never a source checkout. import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; import { ObjectQL } from '@objectstack/objectql'; @@ -37,6 +40,16 @@ import { planSysFilePublicScopeBackfill, runSysFilePublicScopeBackfill, } from './backfill-sys-file-public-scope.js'; +// The operator step imports the sweep from the PACKAGE, not from this module: +// the published `dist` is built from `./index.ts`, so the pin below reads the +// entry itself, values and report types both. +import * as packageEntry from './index.js'; +import type { + PlannedSysFileScopeRow as EntryPlannedRow, + SysFilePublicScopeBackfillEngine as EntryEngine, + SysFilePublicScopeBackfillOptions as EntryOptions, + SysFilePublicScopeBackfillReport as EntryReport, +} from './index.js'; const SYSTEM = { context: { isSystem: true } }; @@ -247,4 +260,28 @@ describe('sys_file public-scope backfill (#22443 ruling B) — a real ObjectQL o }); expect((await sysFile('f_pub'))?.scope).toBe(REWRITTEN_SYS_FILE_SCOPE); }); + + it('the operator step as the changeset writes it: the sweep imported from the package entry plans, applies, and then writes zero', async () => { + // The entry exports the very functions this file drives, not copies of them. + expect(packageEntry.planSysFilePublicScopeBackfill).toBe(planSysFilePublicScopeBackfill); + expect(packageEntry.applySysFilePublicScopeBackfill).toBe(applySysFilePublicScopeBackfill); + expect(packageEntry.runSysFilePublicScopeBackfill).toBe(runSysFilePublicScopeBackfill); + expect(packageEntry.formatSysFilePublicScopeBackfillReport).toBe(formatSysFilePublicScopeBackfillReport); + + await sql.create('sys_file', legacyPublicRow('f_pub')); + const operatorEngine = engine as unknown as EntryEngine; + const options: EntryOptions = { pageSize: 50 }; + + const plan: EntryReport = await packageEntry.planSysFilePublicScopeBackfill(operatorEngine, options); + const planned: EntryPlannedRow[] = plan.rows; + expect(planned).toEqual([{ id: 'f_pub', key: 'public/f_pub.png', from: 'public', to: 'user' }]); + expect(packageEntry.formatSysFilePublicScopeBackfillReport(plan)).toContain('DRY RUN'); + + const applied: EntryReport = await packageEntry.applySysFilePublicScopeBackfill(operatorEngine, plan, options); + expect(applied).toMatchObject({ dryRun: false, scanned: 1, written: 1, failures: [] }); + expect((await sysFile('f_pub'))?.scope).toBe(REWRITTEN_SYS_FILE_SCOPE); + + const again: EntryReport = await packageEntry.runSysFilePublicScopeBackfill(operatorEngine, { ...options, dryRun: false }); + expect(again).toMatchObject({ scanned: 0, planned: 0, written: 0 }); + }); }); From 4d77d54a544d9f9c3d51a4e6b7ec96e34d259fbf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:58:02 +0000 Subject: [PATCH 11/11] chore(changeset): declare the sys_file scope changeset as widening and narrowing The diff adds four functions and four types to the service-storage package entry, a widening of its published surface, beside the narrowing of the sys_file scope accept set. The repo's reader of the line spells that case as yes (narrowing). The levels are unchanged: service-storage minor already meets what yes requires, spec stays patch. Claude-Session: https://claude.ai/code/session_01VZqqwTj2wsihZEbfT6yyYN Co-authored-by: Claude --- .changeset/22443-sys-file-public-scope-rewrite.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/22443-sys-file-public-scope-rewrite.md b/.changeset/22443-sys-file-public-scope-rewrite.md index ed312ff6ee9..98881e9b62f 100644 --- a/.changeset/22443-sys-file-public-scope-rewrite.md +++ b/.changeset/22443-sys-file-public-scope-rewrite.md @@ -5,7 +5,7 @@ feat(service-storage)!: the `sys_file` scope option `public` is retired, and rows already stored with it are rewritten to `user` by a one-time operator sweep (#22443) -Clause-②: no (narrowing) +Clause-②: yes (narrowing)