From 08de5f478545c9ea2a256a98cd2ca98dacf1bf07 Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:22:48 +0800 Subject: [PATCH 1/5] Compact File authority history and migrate formats through verified upgrade backups Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- loopx/cli_commands/authority_archive.py | 43 ++++- .../coordination/authority_format_upgrade.ts | 104 +++++++++++ .../coordination/file_authority_journal.ts | 171 ++++++++++++++++++ .../coordination/file_authority_migration.ts | 90 +++++++++ .../coordination/file_authority_store.ts | 79 +++----- .../coordination/local_authority_archive.ts | 7 + .../sqlite_authority_migration.ts | 10 +- loopx/self_update.py | 21 ++- loopx/windows_install.py | 9 + scripts/install-local.sh | 6 + 10 files changed, 480 insertions(+), 60 deletions(-) create mode 100644 loopx/control_plane/coordination/authority_format_upgrade.ts create mode 100644 loopx/control_plane/coordination/file_authority_journal.ts create mode 100644 loopx/control_plane/coordination/file_authority_migration.ts diff --git a/loopx/cli_commands/authority_archive.py b/loopx/cli_commands/authority_archive.py index 52253176c2..5268be1a80 100644 --- a/loopx/cli_commands/authority_archive.py +++ b/loopx/cli_commands/authority_archive.py @@ -2,11 +2,12 @@ from __future__ import annotations import argparse +import os from collections.abc import Callable from pathlib import Path from ..control_plane.effect_runtime import effect_runtime_result -from ..paths import resolve_runtime_root +from ..paths import DEFAULT_RUNTIME_ROOT, global_registry_path, resolve_runtime_root from ..history import load_registry @@ -19,6 +20,10 @@ def register_authority_archive_command( ) add_subcommand_format(parser) actions = parser.add_subparsers(dest="authority_archive_action", required=True) + upgrade = actions.add_parser("upgrade", help="Back up, verify and migrate local authority formats.") + upgrade.add_argument("--execute", action="store_true") + upgrade.add_argument("--all-known", action="store_true", help="Include runtime roots of registered projects.") + upgrade.add_argument("--require-current", action="store_true", help="Fail if a format upgrade is needed; never write.") for name in ("export", "verify", "restore"): action = actions.add_parser(name) action.add_argument("--archive", type=Path, required=True) @@ -42,8 +47,14 @@ def handle_authority_archive_command( request: dict[str, object] = { "schema_version": "loopx_authority_archive_admin_request_v0", "action": args.authority_archive_action, - "archive": str(args.archive.expanduser().resolve()), } + if args.authority_archive_action == "upgrade": + if args.execute and args.require_current: + raise ValueError("--require-current cannot be combined with --execute") + request.update(runtime_roots=authority_upgrade_roots( + registry_path, runtime_root_arg, all_known=args.all_known), execute=args.execute) + else: + request["archive"] = str(args.archive.expanduser().resolve()) if args.authority_archive_action == "export": request.update(goal_id=args.goal_id, runtime_root=str(resolve_runtime_root( load_registry(registry_path), runtime_root_arg, registry_path=registry_path))) @@ -56,8 +67,36 @@ def handle_authority_archive_command( ) except (RuntimeError, ValueError) as error: result = {"status": "failed", "reason": str(error), "authority_changed": False} + if (args.authority_archive_action == "upgrade" and args.require_current + and any(row.get("status") == "planned" for row in result.get("results", []))): + result.update(status="failed", reason="Authority format upgrade required before activating this runtime.") print_payload(result, output_format(args), lambda value: ( f"Authority archive: {value.get('status')}\n" f"{value.get('reason', 'Active authority selection is unchanged.')}" )) return 1 if result.get("status") == "failed" else 0 + + +def authority_upgrade_roots(registry_path: Path, runtime_root_arg: str | None, + *, all_known: bool) -> list[str]: + """Bounded discovery from existing registries; never scan arbitrary home paths.""" + registry = load_registry(registry_path) if registry_path.exists() else {} + override = runtime_root_arg or os.environ.get("LOOPX_RUNTIME_ROOT") + selected = resolve_runtime_root(registry, override, registry_path=registry_path).resolve() + roots = {selected} + if all_known: + common = Path(override).expanduser().resolve() if override else DEFAULT_RUNTIME_ROOT.resolve() + roots.add(common) + global_path = global_registry_path(common) + global_registry = load_registry(global_path) if global_path.exists() else {} + for goal in global_registry.get("goals", []): + source = goal.get("source_registry") if isinstance(goal, dict) else None + if not source: + continue + path = Path(str(source)).expanduser() + if not path.is_absolute(): + raise ValueError("Registered source_registry must be absolute for automatic upgrade") + # A disconnected checkout does not hide the common runtime store. + if path.exists(): + roots.add(resolve_runtime_root(load_registry(path), registry_path=path).resolve()) + return sorted(str(root) for root in roots) diff --git a/loopx/control_plane/coordination/authority_format_upgrade.ts b/loopx/control_plane/coordination/authority_format_upgrade.ts new file mode 100644 index 0000000000..4dcbee4a84 --- /dev/null +++ b/loopx/control_plane/coordination/authority_format_upgrade.ts @@ -0,0 +1,104 @@ +/** Upgrade physical formats without changing provider selection or Goal authority. + * Provider-to-provider movement uses authority_archive's portable logical log. */ +import {copyFile, mkdir, mkdtemp, open, readFile, readdir, rm, chmod} from "node:fs/promises"; +import {join} from "node:path"; +import {createHash} from "node:crypto"; +import type {JsonObject} from "../effect_program.ts"; +import {withFileMutationLock} from "../effect_runtime_io.ts"; +import {canonicalAuthorityBytes} from "./authority_store_codec.ts"; +import {migrateFileAuthorityStore} from "./file_authority_migration.ts"; +import {FileAuthorityStore, replaceFileAuthorityDurably, syncAuthorityDirectory} from "./file_authority_store.ts"; +import {migrateSqliteAuthorityStoreV1ToV2} from "./sqlite_authority_migration.ts"; +import {sqliteAuthorityRuntime} from "./sqlite_runtime.ts"; +import {sqliteAuthorityPath} from "./sqlite_authority_store.ts"; +import {requireLocalAuthorityRuntimeRoot} from "./local_authority_provider.ts"; + +async function entries(directory: string): Promise { + try { return await readdir(directory); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; throw error; } +} + +async function upgradeSqlite(directory: string, goal: string, execute: boolean): Promise { + const path = sqliteAuthorityPath(directory, goal); + return await withFileMutationLock(`${path}.upgrade`, async () => { + const planned = migrateSqliteAuthorityStoreV1ToV2(directory, goal); + if (planned.status === "failed") throw new Error(planned.reason); + if (planned.status !== "planned" || !execute) return {...planned, provider: "sqlite"}; + const backupRoot = join(directory, "format-backups"); + await mkdir(backupRoot, {recursive: true, mode: 0o700}); + const backupDirectory = await mkdtemp(join(backupRoot, "sqlite-")); + const snapshot = sqliteAuthorityPath(backupDirectory, goal); + const {driver} = sqliteAuthorityRuntime(); + const db = new driver.DatabaseSync(path, {readOnly: true}); + try { await driver.backup(db, snapshot); } finally { db.close(); } + await chmod(snapshot, 0o600); + const handle = await open(snapshot, "r"); + try { await handle.sync(); } finally { await handle.close(); } + // The source may continue committing during online backup. Verify a separate + // copy through the real converter; compare its logical digest inside the + // source's BEGIN IMMEDIATE before table adoption. A race fails, never drops writes. + const proofDirectory = await mkdtemp(join(backupDirectory, "verify-")); + let proof; + try { + await copyFile(snapshot, sqliteAuthorityPath(proofDirectory, goal)); + proof = migrateSqliteAuthorityStoreV1ToV2(proofDirectory, goal, + {execute: true, expectedIdentity: planned.identity}); + if (proof.status !== "migrated") throw new Error(proof.reason ?? "SQLite backup verification failed"); + } finally { await rm(proofDirectory, {recursive: true, force: true}); } + const facts = {schema_version: "loopx_authority_format_upgrade_v0", provider: "sqlite", + goal_id: goal, store_identity: planned.identity!, from_version: 1, to_version: 2, + sequence_digest: proof.sequence_digest!, backup_directory: backupDirectory, + source_sha256: createHash("sha256").update(await readFile(snapshot)).digest("hex")}; + await replaceFileAuthorityDurably(join(backupDirectory, "manifest.json"), canonicalAuthorityBytes(facts)); + await syncAuthorityDirectory(backupRoot); + await syncAuthorityDirectory(directory); + const result = migrateSqliteAuthorityStoreV1ToV2(directory, goal, {execute: true, + expectedIdentity: planned.identity, expectedSequenceDigest: proof.sequence_digest}); + if (result.status !== "migrated" && result.status !== "already_current") { + throw new Error(`${result.reason}; verified backup: ${backupDirectory}`); + } + return {...result, ...facts}; + }); +} + +/** Known local stores only; selectors, registries and writer fences are not mutated. + * All stores, including unselected shadows, must be readable by the new binary. */ +export async function upgradeAuthorityFormats(roots: readonly string[], execute: boolean): Promise { + const results: JsonObject[] = []; + for (const root of [...new Set(roots.map(requireLocalAuthorityRuntimeRoot))]) { + for (const provider of ["file", "sqlite"] as const) { + const directory = join(root, "authority", `${provider}-v0`); + const names = (await entries(directory)).filter(name => + (provider === "file" ? /^authority-store-[0-9a-f]{16}\.json$/ : /^authority-[0-9a-f]{64}\.sqlite$/).test(name)); + if (provider === "file") names.push(...(await entries(join(directory, "rollback"))) + .filter(name => /^authority-store-[0-9a-f]{24}\.json$/.test(name)).map(name => join("rollback", name))); + for (const name of names.sort()) { + const path = join(directory, name); + let goal: string | null = null; + try { + if (provider === "file") { + const value: unknown = JSON.parse(await readFile(path, "utf8")); + goal = (value as {goal_id: string}).goal_id; + if (!name.startsWith("rollback") && new FileAuthorityStore(directory, goal, {existingOnly: true}).path !== path) { + throw new Error("File authority filename does not match its goal"); + } + } else { + const db = new (sqliteAuthorityRuntime().driver.DatabaseSync)(path, {readOnly: true}); + try { goal = String(db.prepare("SELECT goal_id FROM metadata WHERE singleton=1").get()?.goal_id ?? ""); } + finally { db.close(); } + if (sqliteAuthorityPath(directory, goal) !== path) throw new Error("SQLite filename does not match its goal"); + } + const result = provider === "file" ? await migrateFileAuthorityStore(directory, goal, execute, {}, name.startsWith("rollback") ? path : undefined) + : await upgradeSqlite(directory, goal, execute); + results.push({goal_id: goal, ...result}); + } catch (error) { + // Earlier stores may already have migrated. Never report global rollback + // or overwrite their later commits. Retry resumes per-store publication. + return {status: "failed", results, failed_provider: provider, failed_goal_id: goal, + reason: error instanceof Error ? error.message : "Format upgrade failed", retry_safe: true}; + } + } + } + } + return {status: execute ? "upgraded" : "planned", results, authority_changed: false}; +} diff --git a/loopx/control_plane/coordination/file_authority_journal.ts b/loopx/control_plane/coordination/file_authority_journal.ts new file mode 100644 index 0000000000..36caca25f6 --- /dev/null +++ b/loopx/control_plane/coordination/file_authority_journal.ts @@ -0,0 +1,171 @@ +/** File's physical journal codec. Logical revisions, receipts and transactions + * stay unchanged; only repeated projections become checkpoints and deltas. */ +import type {JsonObject} from "../effect_program.ts"; +import type {AuthorityStoreCommit, AuthorityStoreCommittedTransaction} from "./authority_store.ts"; +import {AuthorityStoreProtocolError, canonicalAuthorityBytes, canonicalAuthorityObject, + hasExactAuthorityKeys, isAuthorityJsonObject, parseAuthorityCursor, + requireAuthorityStoreId} from "./authority_store_codec.ts"; +import {decodeAuthorityTransaction, transactionForRevision, + type JournalRevision} from "./authority_store_transactions.ts"; +import {applyAuthorityStateDelta, authorityStateCheckpointCursor, authorityStateDelta, + decodeAuthorityStateDelta, authorityStateDeltaReconstructs, isAuthorityStateCheckpoint, type AuthorityStateDelta} from "./authority_state_log.ts"; + +export const FILE_AUTHORITY_JOURNAL_SCHEMA = "loopx_file_authority_store_v1"; +type StoredState = {kind: "checkpoint"; projection: JsonObject} | + {kind: "delta"; delta: AuthorityStateDelta}; +type StoredCommit = Omit & {state: StoredState}; +type TransactionMetadata = Omit; +type History = {rows: readonly StoredCommit[]}; +const HEADER_KEYS = ["schema_version", "goal_id", "store_identity", "provider_revision", "cursor", "head", "committed"]; + +function invalid(message: string): never { + throw new AuthorityStoreProtocolError(`file authority store ${message}`); +} + +function storedState(value: unknown, cursor: bigint): StoredState { + if (!isAuthorityJsonObject(value)) return invalid("state is invalid"); + if (isAuthorityStateCheckpoint(cursor)) { + if (value.kind !== "checkpoint" || !hasExactAuthorityKeys(value, ["kind", "projection"])) { + return invalid("checkpoint is missing or invalid"); + } + return {kind: "checkpoint", projection: canonicalAuthorityObject(value.projection, "file checkpoint")}; + } + if (value.kind !== "delta" || !hasExactAuthorityKeys(value, ["kind", "delta"])) { + return invalid("delta is missing or invalid"); + } + return {kind: "delta", delta: decodeAuthorityStateDelta(value.delta)}; +} + +function project(state: StoredState, previous: JsonObject | null): JsonObject { + if (state.kind === "checkpoint") return state.projection; + if (previous === null) return invalid("delta has no predecessor"); + return applyAuthorityStateDelta(previous, state.delta); +} + +function retain(transaction: AuthorityStoreCommittedTransaction, previous: JsonObject | null): StoredCommit { + const {projection, ...metadata} = transaction; + if (isAuthorityStateCheckpoint(parseAuthorityCursor(transaction.cursor))) { + return {...metadata, state: {kind: "checkpoint", projection}}; + } + if (previous === null) return invalid("retained delta has no predecessor"); + const delta = authorityStateDelta(previous, projection); + if (!authorityStateDeltaReconstructs(previous, delta, projection)) return invalid("delta reconstruction mismatch"); + return {...metadata, state: {kind: "delta", delta}}; +} + +/** Only verified or locally committed rows enter this object. V1 retains compact + * history. Old formats are accepted only by the explicit migration owner. */ +export class FileAuthorityJournal { + readonly goal_id: string; + readonly store_identity: string; + readonly head: JsonObject; + readonly cursor: string; + readonly provider_revision: string; + private readonly history: History; + private readonly operations: ReadonlyMap; + + private constructor(goal: string, identity: string, head: JsonObject, history: History) { + const rows = history.rows; + this.goal_id = goal; this.store_identity = identity; this.head = head; + this.history = history; + this.cursor = rows.at(-1)!.cursor; + this.provider_revision = rows.at(-1)!.provider_revision; + this.operations = new Map(rows.map(row => [row.operation_id, row])); + } + + static decode(value: unknown, goal: string, identity: string, revisionFor: JournalRevision): FileAuthorityJournal { + if (!isAuthorityJsonObject(value) || !hasExactAuthorityKeys(value, HEADER_KEYS) || + value.schema_version !== FILE_AUTHORITY_JOURNAL_SCHEMA) { + return invalid("schema mismatch; run loopx authority-archive upgrade --execute before opening this store"); + } + if (value.goal_id !== goal) return invalid("goal mismatch"); + if (value.store_identity !== identity) return invalid("lineage mismatch"); + const cursor = requireAuthorityStoreId(value.cursor, "provider cursor"); + const revision = requireAuthorityStoreId(value.provider_revision, "provider revision"); + const head = canonicalAuthorityObject(value.head, "file authority store head"); + if (!Array.isArray(value.committed) || value.committed.length === 0 || + parseAuthorityCursor(cursor) !== BigInt(value.committed.length)) return invalid("lineage is invalid"); + const rows: StoredCommit[] = [], operations = new Set(); + let previous: JsonObject | null = null, previousRevision: string | null = null; + for (const [index, raw] of value.committed.entries()) { + if (!isAuthorityJsonObject(raw) || !hasExactAuthorityKeys(raw, + ["cursor", "provider_revision", "operation_id", "events", "receipts", "state"])) { + return invalid("committed transaction is invalid"); + } + const {state: rawState, ...metadata} = raw; + const state = storedState(rawState, BigInt(index + 1)); + const transaction = decodeAuthorityTransaction({...metadata, projection: project(state, previous)}); + if (parseAuthorityCursor(transaction.cursor) !== BigInt(index + 1)) return invalid("cursor lineage is invalid"); + if (operations.has(transaction.operation_id)) return invalid("operation identity is duplicated"); + if (transaction.provider_revision !== revisionFor(previousRevision, transactionForRevision(transaction))) { + return invalid("revision lineage is invalid"); + } + operations.add(transaction.operation_id); + const {projection, ...entry} = transaction; + rows.push({...entry, state}); + previous = projection; previousRevision = transaction.provider_revision; + } + if (rows.at(-1)!.cursor !== cursor || previousRevision !== revision || + !canonicalAuthorityBytes(previous).equals(canonicalAuthorityBytes(head))) return invalid("head lineage is invalid"); + return new FileAuthorityJournal(goal, identity, head, {rows}); + } + + static append(current: FileAuthorityJournal | null, goal: string, identity: string, + commit: AuthorityStoreCommit, revisionFor: JournalRevision): FileAuthorityJournal { + const transaction = {cursor: (parseAuthorityCursor(current?.cursor ?? null) + 1n).toString(), + operation_id: commit.operation_id, events: commit.events, projection: commit.next_projection, + receipts: commit.receipts}; + const row = retain({...transaction, + provider_revision: revisionFor(current?.provider_revision ?? null, transaction)}, current?.head ?? null); + return new FileAuthorityJournal(goal, identity, commit.next_projection, + {rows: [...(current?.history.rows ?? []), row]}); + } + + /** Migration boundary: callers supply fully verified logical transactions. + * Decode the resulting wire format again before it can be adopted. */ + static fromTransactions(goal: string, identity: string, rows: readonly AuthorityStoreCommittedTransaction[], + revisionFor: JournalRevision): FileAuthorityJournal { + let previous: JsonObject | null = null; + const compact = rows.map(transaction => { + const encoded = retain(transaction, previous); + previous = transaction.projection; + return encoded; + }); + const journal = new FileAuthorityJournal(goal, identity, rows.at(-1)!.projection, {rows: compact}); + return FileAuthorityJournal.decode(journal.toDocument(), goal, identity, revisionFor); + } + + receiptEntries(): Iterable { + return this.operations.values(); + } + + receipt(operation: string): TransactionMetadata | undefined { + const row = this.operations.get(operation); + if (!row) return undefined; + return {cursor: row.cursor, provider_revision: row.provider_revision, + operation_id: row.operation_id, events: row.events, receipts: row.receipts}; + } + + /** At most 63 predecessor deltas plus the requested page (and lookahead). + * Full-file integrity validation still happens before this verified view. */ + scan(offset: number, limit: number): AuthorityStoreCommittedTransaction[] { + const {rows} = this.history; + if (offset >= rows.length) return []; + const checkpoint = Number(authorityStateCheckpointCursor(BigInt(offset + 1))) - 1; + const end = Math.min(rows.length, offset + limit); + const result: AuthorityStoreCommittedTransaction[] = []; + let previous: JsonObject | null = null; + for (let i = checkpoint; i < end; i++) { + const {state, ...metadata} = this.history.rows[i]!; + previous = project(state, previous); + if (i >= offset) result.push({...metadata, projection: previous}); + } + return result; + } + + toDocument(): JsonObject { + return {schema_version: FILE_AUTHORITY_JOURNAL_SCHEMA, goal_id: this.goal_id, + store_identity: this.store_identity, provider_revision: this.provider_revision, + cursor: this.cursor, head: this.head, committed: this.history.rows}; + } +} diff --git a/loopx/control_plane/coordination/file_authority_migration.ts b/loopx/control_plane/coordination/file_authority_migration.ts new file mode 100644 index 0000000000..cac788b960 --- /dev/null +++ b/loopx/control_plane/coordination/file_authority_migration.ts @@ -0,0 +1,90 @@ +/** Explicit physical upgrade. Legacy parsing never participates in business reads. */ +import {mkdir, readFile} from "node:fs/promises"; +import {join, dirname, basename} from "node:path"; +import type {JsonObject} from "../effect_program.ts"; +import {withFileMutationLock} from "../effect_runtime_io.ts"; +import {canonicalAuthorityBytes, canonicalAuthoritySha256, hasExactAuthorityKeys, + isAuthorityJsonObject} from "./authority_store_codec.ts"; +import {decodeRetainedAuthorityJournal} from "./authority_store_transactions.ts"; +import {FILE_AUTHORITY_JOURNAL_SCHEMA, FileAuthorityJournal} from "./file_authority_journal.ts"; +import {FileAuthorityStore, fileAuthorityRevision, replaceFileAuthorityDurably, syncAuthorityDirectory} from "./file_authority_store.ts"; +import {createHash} from "node:crypto"; + +const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); +const LEGACY_SCHEMA = "loopx_file_authority_store_v0"; + +/** Test-only crash seam around the real durable publication boundary. */ +export interface FileAuthorityMigrationEffects { + beforePublish?: () => Promise; + afterPublish?: () => Promise; +} + +export async function migrateFileAuthorityStore(directory: string, goal: string, execute = false, + effects: FileAuthorityMigrationEffects = {}, archivedPath?: string): Promise { + const store = new FileAuthorityStore(directory, goal, {existingOnly: true}); + const sourcePath = archivedPath ?? store.path; + if (archivedPath && (dirname(archivedPath) !== join(store.directory, "rollback") || + !/^authority-store-[0-9a-f]{24}\.json$/.test(basename(archivedPath)))) { + throw new Error("Invalid File rollback document path"); + } + // Same lock as ordinary commits: the backup and source are one exact lineage. + return await withFileMutationLock(store.path, async () => { + let source: Buffer; + try { source = await readFile(sourcePath); } + catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return {status: "missing"}; throw error; } + const identity = await readFile(store.identityPath, "utf8"); + if (!/^file:[0-9a-f]{32}$/.test(identity)) throw new Error("Invalid file store identity"); + const revisionFor = (previous: string | null, transaction: Parameters[3]) => + fileAuthorityRevision(goal, identity, previous, transaction); + const value: unknown = JSON.parse(source.toString("utf8")); + if (!isAuthorityJsonObject(value)) throw new Error("Invalid authority document"); + if (value.schema_version === FILE_AUTHORITY_JOURNAL_SCHEMA) { + const current = FileAuthorityJournal.decode(value, goal, identity, revisionFor); + return {status: "already_current", provider: "file", cursor: current.cursor, + provider_revision: current.provider_revision}; + } + if (value.schema_version !== LEGACY_SCHEMA || value.goal_id !== goal || value.store_identity !== identity || + !hasExactAuthorityKeys(value, ["schema_version", "goal_id", "store_identity", "provider_revision", "cursor", "head", "committed"])) { + throw new Error("Unsupported file authority format or mismatched lineage; source was not changed"); + } + const legacy = decodeRetainedAuthorityJournal(value, "file migration source", revisionFor); + const compact = FileAuthorityJournal.fromTransactions(goal, identity, legacy.committed, revisionFor); + // Compare complete logical history, not only the head or receipt count. + const logicalDigest = canonicalAuthoritySha256(legacy.committed); + if (canonicalAuthoritySha256(compact.scan(0, legacy.committed.length)) !== logicalDigest) { + throw new Error("Migrated logical history differs from source"); + } + const target = canonicalAuthorityBytes(compact.toDocument()); + const sourceDigest = sha256(source), targetDigest = sha256(target); + const backup = join(directory, "format-backups", sourceDigest); + const facts = {provider: "file", from_schema: LEGACY_SCHEMA, to_schema: FILE_AUTHORITY_JOURNAL_SCHEMA, + source_sha256: sourceDigest, target_sha256: targetDigest, logical_sha256: logicalDigest, + store_identity: identity, goal_id: goal, cursor: compact.cursor, provider_revision: compact.provider_revision, + bytes_before: source.length, bytes_after: target.length, backup_directory: backup}; + if (!execute) return {status: "planned", ...facts}; + await mkdir(backup, {recursive: true, mode: 0o700}); + // Content-addressed backups are never overwritten. A damaged existing + // backup stops the upgrade instead of silently replacing its evidence. + for (const [name, bytes] of [["source.json", source], ["store-identity", Buffer.from(identity)]] as const) { + const path = join(backup, name); + let existing: Buffer | undefined; + try { existing = await readFile(path); } + catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; } + if (existing && !existing.equals(bytes)) throw new Error("Existing migration backup is corrupt"); + if (!existing) await replaceFileAuthorityDurably(path, bytes); + if (!(await readFile(path)).equals(bytes)) throw new Error("Migration backup readback mismatch"); + } + // Sync the newly created directory entries as well as their file contents. + await syncAuthorityDirectory(join(directory, "format-backups")); + await syncAuthorityDirectory(directory); + const manifest = {schema_version: "loopx_authority_format_upgrade_v0", ...facts}; + await replaceFileAuthorityDurably(join(backup, "manifest.json"), canonicalAuthorityBytes(manifest)); + await effects.beforePublish?.(); + await replaceFileAuthorityDurably(sourcePath, target); + await effects.afterPublish?.(); + if (!(await readFile(sourcePath)).equals(target)) throw new Error("Migration publication readback mismatch"); + return {status: "migrated", ...facts}; + // No mutable 'done' flag: source/target digests and the actual store are the + // recovery proof. A crash after rename is already_current on retry. + }); +} diff --git a/loopx/control_plane/coordination/file_authority_store.ts b/loopx/control_plane/coordination/file_authority_store.ts index d781c04ed3..6eff136fd9 100644 --- a/loopx/control_plane/coordination/file_authority_store.ts +++ b/loopx/control_plane/coordination/file_authority_store.ts @@ -18,19 +18,16 @@ import type { } from "./authority_store.ts"; import { AuthorityStoreProtocolError, - isAuthorityJsonObject, - hasExactAuthorityKeys, canonicalAuthorityBytes, normalizeAuthorityStoreCommit, requireAuthorityStoreId, } from "./authority_store_codec.ts"; -import {appendRetainedAuthorityJournal, decodeRetainedAuthorityJournal, - type RetainedAuthorityJournal, transactionForRevision} from "./authority_store_transactions.ts"; +import {transactionForRevision} from "./authority_store_transactions.ts"; +import {FileAuthorityJournal} from "./file_authority_journal.ts"; import {AuthorityJournalScan} from "./authority_journal_scan.ts"; -const FILE_AUTHORITY_STORE_SCHEMA = "loopx_file_authority_store_v0"; const STORE_IDENTITY_PATTERN = /^file:[0-9a-f]{32}$/; -// File-v0 retains every projection in one envelope. A managed Effect server +// File retains a checkpoint/delta journal in one durable envelope. A managed Effect server // opens a new store handle for each request. Keep one verified read view across // handles, keyed by exact bytes and store identity. Large journals retain only // the head and receipt index in memory; commits and scans still load and verify @@ -50,7 +47,7 @@ interface VerifiedDocument { providerRevision: string; receipts: readonly JsonObject[]; }>; - document?: FileAuthorityStoreDocument; + document?: FileAuthorityJournal; } let verifiedDocument: VerifiedDocument | null = null; @@ -59,7 +56,7 @@ function documentDigest(raw: Uint8Array): string { } function rememberVerifiedDocument(path: string, identity: string, raw: Uint8Array, - digest: string, document: FileAuthorityStoreDocument, + digest: string, document: FileAuthorityJournal, maxDocumentBytes: number): VerifiedDocument { const receipts = new Map(); let viewBytes = Buffer.byteLength(JSON.stringify(document.head)); - for (const entry of document.committed) { + for (const entry of document.receiptEntries()) { receipts.set(entry.operation_id, {cursor: entry.cursor, providerRevision: entry.provider_revision, receipts: entry.receipts}); viewBytes += Buffer.byteLength(entry.operation_id) + Buffer.byteLength(entry.cursor) + @@ -83,13 +80,6 @@ function rememberVerifiedDocument(path: string, identity: string, raw: Uint8Arra return view; } -interface FileAuthorityStoreDocument extends JsonObject, RetainedAuthorityJournal { - schema_version: typeof FILE_AUTHORITY_STORE_SCHEMA; - goal_id: string; - store_identity: string; - -} - class FileStoreUnavailableError extends Error {} export type FileAuthorityArchiveResult = @@ -125,12 +115,12 @@ export type FileAuthorityArchiveResult = reason: string; }; -function providerRevision(goalId: string, storeIdentity: string, previousRevision: string | null, transaction: ReturnType): string { +export function fileAuthorityRevision(goalId: string, storeIdentity: string, previousRevision: string | null, transaction: ReturnType): string { const digest = createHash("sha256").update(canonicalAuthorityBytes({ goal_id: goalId, store_identity: storeIdentity, previous_provider_revision: previousRevision, transaction })).digest("hex").slice(0, 24); return `file:${transaction.cursor}:${digest}`; } -async function syncDirectory(directory: string): Promise { +export async function syncAuthorityDirectory(directory: string): Promise { if (process.platform === "win32") return; const handle = await open(directory, "r"); try { @@ -140,7 +130,7 @@ async function syncDirectory(directory: string): Promise { } } -async function durableReplace(path: string, payload: Uint8Array): Promise { +export async function replaceFileAuthorityDurably(path: string, payload: Uint8Array): Promise { const directory = dirname(path); await mkdir(directory, { recursive: true, mode: 0o700 }); const temporary = `${path}.tmp-${process.pid}-${randomUUID()}`; @@ -153,7 +143,7 @@ async function durableReplace(path: string, payload: Uint8Array): Promise await handle.close(); } await rename(temporary, path); - await syncDirectory(directory); + await syncAuthorityDirectory(directory); } finally { await rm(temporary, { force: true }); } @@ -163,20 +153,9 @@ function decodeDocument( value: unknown, goalId: string, storeIdentity: string, -): FileAuthorityStoreDocument { - if (!isAuthorityJsonObject(value) || !hasExactAuthorityKeys(value, [ - "schema_version", "goal_id", "provider_revision", "cursor", "store_identity", - "head", "committed", - ]) || value.schema_version !== FILE_AUTHORITY_STORE_SCHEMA) { - throw new AuthorityStoreProtocolError("file authority store schema mismatch"); - } - if (value.goal_id !== goalId) throw new AuthorityStoreProtocolError("file authority store goal mismatch"); - if (value.store_identity !== storeIdentity) { - throw new AuthorityStoreProtocolError("file authority store lineage mismatch"); - } - return {schema_version: FILE_AUTHORITY_STORE_SCHEMA, goal_id: goalId, store_identity: storeIdentity, - ...decodeRetainedAuthorityJournal(value, "file authority store", (previous, transaction) => - providerRevision(goalId, storeIdentity, previous, transaction))}; +): FileAuthorityJournal { + return FileAuthorityJournal.decode(value, goalId, storeIdentity, (previous, transaction) => + fileAuthorityRevision(goalId, storeIdentity, previous, transaction)); } function readFailure(error: unknown): AuthorityStoreReadFailure { @@ -213,14 +192,14 @@ export class FileAuthorityStore implements AuthorityStore { /** Narrow effect seam for crash-window qualification; not a semantic hook. */ protected async replaceDurably(path: string, payload: Uint8Array): Promise { - await durableReplace(path, payload); + await replaceFileAuthorityDurably(path, payload); } /** Filesystem-only crash seam; the archive owner must still fsync both parents. */ protected async archiveRenamed(): Promise {} /** Full-history verification seam; unchanged byte-identical reads may reuse it. */ - protected decodeStoredDocument(value: unknown, identity: string): FileAuthorityStoreDocument { + protected decodeStoredDocument(value: unknown, identity: string): FileAuthorityJournal { return decodeDocument(value, this.goalId, identity); } @@ -235,7 +214,7 @@ export class FileAuthorityStore implements AuthorityStore { if (!STORE_IDENTITY_PATTERN.test(identity)) { throw new AuthorityStoreProtocolError("store identity does not match file:<32 lowercase hex>"); } - if (createIfMissing) await syncDirectory(this.directory); + if (createIfMissing) await syncAuthorityDirectory(this.directory); return identity; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; @@ -247,7 +226,7 @@ export class FileAuthorityStore implements AuthorityStore { if (!STORE_IDENTITY_PATTERN.test(identity)) { throw new AuthorityStoreProtocolError("store identity does not match file:<32 lowercase hex>"); } - await syncDirectory(this.directory); + await syncAuthorityDirectory(this.directory); return identity; } catch (error) { if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; @@ -304,7 +283,7 @@ export class FileAuthorityStore implements AuthorityStore { } } - private async readDocument(knownIdentity?: string): Promise { + private async readDocument(knownIdentity?: string): Promise { return (await this.readVerified(knownIdentity, true))?.document ?? null; } @@ -352,7 +331,7 @@ export class FileAuthorityStore implements AuthorityStore { try { return await withFileMutationLock(this.path, async () => { let identity: string; - let current: FileAuthorityStoreDocument | null; + let current: FileAuthorityJournal | null; try { // Read the identity under the same document lock used by the commit. // A restored directory must not race a missing-head bootstrap and @@ -379,7 +358,7 @@ export class FileAuthorityStore implements AuthorityStore { current_cursor: current?.cursor ?? null, }; } - if (current?.committed.some((entry) => entry.operation_id === normalized.operation_id)) { + if (current?.receipt(normalized.operation_id)) { return { status: "conflict", conflict_kind: "operation_id_exists", @@ -387,13 +366,11 @@ export class FileAuthorityStore implements AuthorityStore { current_cursor: current.cursor, }; } - const journal = appendRetainedAuthorityJournal(current, normalized, (previous, transaction) => - providerRevision(this.goalId, identity, previous, transaction)); - const {cursor, provider_revision: revision} = journal; - const document: FileAuthorityStoreDocument = {schema_version: FILE_AUTHORITY_STORE_SCHEMA, - goal_id: this.goalId, store_identity: identity, ...journal}; + const document = FileAuthorityJournal.append(current, this.goalId, identity, normalized, (previous, transaction) => + fileAuthorityRevision(this.goalId, identity, previous, transaction)); + const {cursor, provider_revision: revision} = document; try { - const bytes = canonicalAuthorityBytes(document); + const bytes = canonicalAuthorityBytes(document.toDocument()); await this.replaceDurably(this.path, bytes); rememberVerifiedDocument(this.path, identity, bytes, documentDigest(bytes), document, this.fullDocumentCacheLimitBytes()); @@ -456,7 +433,7 @@ export class FileAuthorityStore implements AuthorityStore { const range = scan.rangeFailure(document.cursor); if (range) return range; const start = Number(scan.offset); - return scan.page(document.committed.slice(start, start + limit + 1), document); + return scan.page(document.scan(start, limit + 1), document); } catch (error) { return readFailure(error); } } @@ -494,7 +471,7 @@ export class FileAuthorityStore implements AuthorityStore { try { return await withFileMutationLock(this.path, async () => { const identity = await this.readStoreIdentity(false); - let archived: FileAuthorityStoreDocument | null = null; + let archived: FileAuthorityJournal | null = null; try { archived = decodeDocument( JSON.parse(await readFile(archivePath, "utf8")), @@ -542,8 +519,8 @@ export class FileAuthorityStore implements AuthorityStore { renameStarted = true; await rename(this.path, archivePath); await this.archiveRenamed(); - await syncDirectory(this.directory); - await syncDirectory(archiveDirectory); + await syncAuthorityDirectory(this.directory); + await syncAuthorityDirectory(archiveDirectory); return { status: "applied", archived_provider_revision: current.provider_revision, diff --git a/loopx/control_plane/coordination/local_authority_archive.ts b/loopx/control_plane/coordination/local_authority_archive.ts index 86253d46ef..c151717565 100644 --- a/loopx/control_plane/coordination/local_authority_archive.ts +++ b/loopx/control_plane/coordination/local_authority_archive.ts @@ -1,5 +1,6 @@ /** Administrative archive transport. Large private state stays in local files; * the managed effect runtime returns only compact integrity/readback facts. */ +import {upgradeAuthorityFormats} from "./authority_format_upgrade.ts"; import {mkdir, readFile} from "node:fs/promises"; import {isAbsolute, join} from "node:path"; import type {JsonObject} from "../effect_program.ts"; @@ -24,6 +25,12 @@ export async function manageLocalAuthorityArchive(value: unknown, try { const request = requireJsonObject(value, "authority archive request"); if (request.schema_version !== "loopx_authority_archive_admin_request_v0") throw new Error("archive request schema mismatch"); + if (request.action === "upgrade") { + if (!Array.isArray(request.runtime_roots) || request.runtime_roots.some(root => typeof root !== "string")) { + throw new Error("Upgrade requires explicit runtime roots"); + } + return {...base, ...await upgradeAuthorityFormats(request.runtime_roots as string[], request.execute === true)}; + } const archive = path(request.archive, "archive path"); if (request.action === "verify") return {...base, status: "verified", archive: await verifyAuthorityArchive(archive)}; const goalId = requireAuthorityStoreId(request.goal_id, "goal id"); diff --git a/loopx/control_plane/coordination/sqlite_authority_migration.ts b/loopx/control_plane/coordination/sqlite_authority_migration.ts index edc12a7f6a..2eefc75f37 100644 --- a/loopx/control_plane/coordination/sqlite_authority_migration.ts +++ b/loopx/control_plane/coordination/sqlite_authority_migration.ts @@ -71,7 +71,7 @@ interface SqliteAuthorityMigrationInspection { export function migrateSqliteAuthorityStoreV1ToV2( directory: string, goalId: string, - options: {execute?: boolean; expectedIdentity?: string} = {}, + options: {execute?: boolean; expectedIdentity?: string; expectedSequenceDigest?: string} = {}, ): SqliteAuthorityMigrationResult { const path = sqliteAuthorityPath(directory, goalId); const base: SqliteAuthorityMigrationResult = {schema_version: "loopx_sqlite_authority_migration_v0", status: "failed"}; @@ -98,7 +98,7 @@ export function migrateSqliteAuthorityStoreV1ToV2( return {...base, status: "planned", database_path: path, identity: inspection.value.identity, commits: inspection.value.commits, database_bytes_before: before}; } - return executeSqliteAuthorityMigration(path, goalId, inspection.value, before); + return executeSqliteAuthorityMigration(path, goalId, inspection.value, before, options.expectedSequenceDigest); } function inspectSqliteAuthorityStore( @@ -146,6 +146,7 @@ function executeSqliteAuthorityMigration( goalId: string, inspection: SqliteAuthorityMigrationInspection, before: number, + expectedSequenceDigest?: string, ): SqliteAuthorityMigrationResult { const base: SqliteAuthorityMigrationResult = {schema_version: "loopx_sqlite_authority_migration_v0", status: "failed"}; const {driver: sqlite} = sqliteAuthorityRuntime(); @@ -227,6 +228,10 @@ function executeSqliteAuthorityMigration( // store reads with, so the swap cannot publish a log only its writer can // decode. verifyWrittenStateLogReadable(db, commits); + const sequenceDigest = identityDigest.digest("hex"); + if (expectedSequenceDigest !== undefined && sequenceDigest !== expectedSequenceDigest) { + throw new AuthorityStoreProtocolError("SQLite source changed after its verified backup; retry upgrade"); + } // Swap only after every retained row was proved and re-published. db.exec("DROP TABLE head"); db.exec("DROP TABLE commits"); @@ -237,7 +242,6 @@ function executeSqliteAuthorityMigration( db.exec("PRAGMA user_version = 2"); db.exec("COMMIT"); transactionOpen = false; - const sequenceDigest = identityDigest.digest("hex"); const verification = verifyMigratedStore(path, goalId, inspection.identity, sequenceDigest); if (verification !== null) { return {...base, reason_code: verification.reason_code, reason: verification.reason, diff --git a/loopx/self_update.py b/loopx/self_update.py index 17f4186eb2..806b830a8d 100644 --- a/loopx/self_update.py +++ b/loopx/self_update.py @@ -894,6 +894,8 @@ def _execute_python_distribution_update( ) commands = { "install": install_command, + "authority_upgrade": [sys.executable, "-m", "loopx.cli", "--format", "json", + "authority-archive", "upgrade", "--all-known", "--execute"], "workflow_skills": [ sys.executable, "-m", @@ -940,16 +942,18 @@ def _execute_python_distribution_update( timeout=timeout_seconds, ) if results["install"].returncode == 0: - for step in ("workflow_skills", "slash_commands", "doctor"): + for step in ("authority_upgrade", "workflow_skills", "slash_commands", "doctor"): results[step] = subprocess.run( commands[step], text=True, encoding="utf-8", errors="replace", capture_output=True, timeout=timeout_seconds, ) + if results[step].returncode != 0: + break if all( - results[step].returncode == 0 - for step in ("workflow_skills", "slash_commands", "doctor") + step in results and results[step].returncode == 0 + for step in ("authority_upgrade", "workflow_skills", "slash_commands", "doctor") ): results["extension_doctor"] = subprocess.run( commands["extension_doctor"], @@ -965,7 +969,7 @@ def _execute_python_distribution_update( "install_stdout_tail": results["install"].stdout[-2000:], "install_stderr_tail": results["install"].stderr[-2000:], } - for step in ("workflow_skills", "slash_commands", "doctor", "extension_doctor"): + for step in ("authority_upgrade", "workflow_skills", "slash_commands", "doctor", "extension_doctor"): result = results.get(step) if result is None: execution[f"{step}_status"] = "skipped_prior_step_failed" @@ -976,6 +980,7 @@ def _execute_python_distribution_update( runtime_steps = ( "install", + "authority_upgrade", "workflow_skills", "slash_commands", "doctor", @@ -1232,6 +1237,14 @@ def execute_rollback_plan( } updated = dict(payload) try: + compatible = subprocess.run( + [str(target_script), "--format", "json", "authority-archive", "upgrade", + "--all-known", "--require-current"], + capture_output=True, text=True, timeout=timeout_seconds, + ) + if compatible.returncode != 0: + raise RuntimeError("Rollback target cannot read current authority formats. " + "Keep the current runtime; recover a verified backup into an isolated store first.") loopx_bin.parent.mkdir(parents=True, exist_ok=True) temp_link = loopx_bin.with_name(f".{loopx_bin.name}.rollback.{os.getpid()}") if temp_link.exists() or temp_link.is_symlink(): diff --git a/loopx/windows_install.py b/loopx/windows_install.py index 0296190076..e78cfff69d 100644 --- a/loopx/windows_install.py +++ b/loopx/windows_install.py @@ -328,6 +328,15 @@ def install_windows( shutil.rmtree(temporary, ignore_errors=True) raise + upgrade = subprocess.run( + _entry_command(release_root, python, ["--format", "json", "authority-archive", "upgrade", + "--all-known", "--execute"]), + capture_output=True, text=True, timeout=600, + ) + if upgrade.returncode != 0: + raise RuntimeError("Authority format upgrade failed before launcher activation; " + "backups and candidate retained. " + upgrade.stdout[-2000:]) + launcher = bin_dir / "loopx.ps1" pointer = install_root / "current-release.json" launcher_pointer = bin_dir / LAUNCHER_POINTER_FILENAME diff --git a/scripts/install-local.sh b/scripts/install-local.sh index c344d1c77d..6119a61d40 100755 --- a/scripts/install-local.sh +++ b/scripts/install-local.sh @@ -775,6 +775,12 @@ if ! preflight_workflow_skills "$release_dir/skills" "$release_dir" "$bin_dir/lo rm -rf "$release_dir" exit 1 fi +# Data upgrade is a separate, resumable operation. Never delete its backups or +# roll migrated stores back merely because a later launcher/skill step fails. +if ! "$release_dir/scripts/loopx" --format json authority-archive upgrade --all-known --execute; then + echo "loopx installer error: authority format upgrade failed; backups are retained. Retry with this candidate before activation." >&2 + exit 1 +fi install_symlink "$release_dir/scripts/loopx" "$bin_dir/loopx" install_symlink "$release_dir/scripts/loopx-apply-rrule" "$bin_dir/loopx-apply-rrule" verify_default_promotion From 154b2bdb20242270abed1f84e688751c871c4bd7 Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:22:49 +0800 Subject: [PATCH 2/5] Qualify backed-up upgrades, retained history and provider interchange Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- ...26-09-24-default-cutover-reconciliation.md | 25 ++- ...24-default-cutover-reconciliation.zh-CN.md | 18 ++ ...shared-goal-authority-state-provider-v0.md | 10 + ...-goal-authority-state-provider-v0.zh-CN.md | 7 + docs/reference/file-authority-state-log.md | 137 ++++++++++++ tests/control_plane/test_authority_archive.py | 32 +++ .../authority_format_upgrade.test.ts | 85 +++++++ .../control_plane_ts/authority_store.test.ts | 4 +- .../authority_store_transactions.test.ts | 16 +- .../file_authority_journal.test.ts | 207 ++++++++++++++++++ tests/test_self_update_runtime_activation.py | 27 ++- 11 files changed, 558 insertions(+), 10 deletions(-) create mode 100644 docs/reference/file-authority-state-log.md create mode 100644 tests/control_plane_ts/authority_format_upgrade.test.ts create mode 100644 tests/control_plane_ts/file_authority_journal.test.ts diff --git a/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.md b/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.md index 7ef0be0b3e..ae0465067a 100644 --- a/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.md +++ b/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.md @@ -3,7 +3,7 @@ - Baseline: `41ba6f4d9` on `main`, 2026-09-25; open PR states are a snapshot, not merge promises. - Owners: overall roadmap #4574 R5/G2; shared authority L2–L9/D1–D3; TS migration T1–T4. - Delivered #5040: current registration admission, complete saved migration intent and truthful fence recovery. -- Current increment: long-history closeout reuse and TS-owned monitor evidence; the migration packages below remain open. +- Current increment: File retained-state compaction stacked on #5063; the migration packages below remain open. #5063 has now merged; rebased onto main `eaa0c0fd0`. - This checkpoint supersedes numerical remaining-PR estimates in earlier delivery entries. ## Correct the accounting @@ -32,6 +32,27 @@ transaction, complete-source and source-witness owners. The latest formal #4224 (801.81 ms versus 250 ms). #4931 has not supplied a formal exact-head rerun. Reaching the planned ten-day soak end date is not a passing report. +## File history cost: delivered slice, separate acceptance + +A detached long-history snapshot exposed File's repeated full-projection write +cost. #5063 bounds RPC waits and retains verified read views; it does not remove +that physical duplication. This stack reuses the SQLite-owned shared TS state-log +codec for File checkpoints/deltas, preserving logical revisions, receipts and +full scan results. See [format, upgrade and limits](../../../../reference/file-authority-state-log.md). +Normal reads/writes accept only v1. Explicit upgrade automatically backs up and +verifies File/SQLite before physical migration, and installation invokes it before +activation. Cross-provider movement reuses logical archive recovery. Older binaries +cannot read v1. Conversion, cold verification, +steady writes and warm reads require separate evidence; cache limits are unchanged. + +Before this slice, the audited implementation plan therefore contains **four +named packages**: this evidenced File cost repair plus the three below. After +this slice it contains those **three planned packages**, not a new unchanged +“5–8 PRs” estimate. #5063, #5054 and #4931 are existing PRs, not three new tasks. +D1–D3 and an exact total PR count remain unqualified. This storage repair retires +no Python business owner; bounded Python deletion belongs to actual caller +migration in the packages below. + ## Three concrete next code boundaries This delivery repairs integrated migration admission: stale registry snapshots @@ -41,7 +62,7 @@ not implement another store or close the whole migration package or D2 gate. | Proposed PR | Observable result and owner | Exit | | --- | --- | --- | | 1. External-effect execution fencing | Lease/effect owners protect the actual execution interval, takeover, timeout, exit and uncertain completion. Reuse merged #4994/#4995. | Stale executors cannot continue or settle; real executor and receipt recovery matrix passes. A point-in-time proof check is insufficient. | -| 2. Event-writer binding and whole-Goal migration/rollback | Bind event writer locks/atomic publication to existing outbox; integrate Markdown/event/lease capture, drain, saved cutover, consumers and fenced export/rollback; delete Python decisions replaced by TS. | Reuse #5003. Retain `event_log_writer_not_bound` until binding passes; close D1, command inventory and D3 cohort. One Goal without an event overlay does not prove this package. | +| 2. Whole-Goal migration/rollback and retained source closure | Reconcile open #5054, which retires the legacy Todo event path and isolates supervisor logging; do not build another capture writer for a retired source. Integrate remaining supported sources, drain, saved cutover, consumers and fenced export/rollback; delete Python decisions replaced by TS. | Prove the supported command/source inventory after #5054, D1 and the D3 cohort; reject retired input explicitly. One Goal without a legacy event overlay does not prove every retained caller or rollback path. | | 3. Default entrypoints and bounded Python retirement | New Goals, settings, installation and packaged frontend/Lark/CLI select a qualified profile consistently; existing Goals have explicit migration/disable flows. | 1/2 and applicable D1–D3 pass; user entrypoints work; delete business writers only after their last callers migrate. Retain rendering, host IO and lawful import/export. | **Plan three named future implementation PRs, plus existing #4931 and outstanding diff --git a/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.zh-CN.md b/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.zh-CN.md index dd7bb82427..1f61f18d29 100644 --- a/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.zh-CN.md +++ b/docs/architecture/rfcs/ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.zh-CN.md @@ -6,6 +6,24 @@ - 当前增量:长历史 closeout 读取复用与 TS monitor 回执归一;没有完成下列迁移工作包。 - 本检查点取代此前交付记录中的剩余 PR 数量估算。 +## File 历史编码与自动升级增量 + +本次最初 stack 在 #5063 上;#5063 合入后已接到 main `eaa0c0fd0`。 +#5063 解决读取缓存和 RPC 预算,File 每次写入仍复制全历史完整投影。本次复用 +SQLite 已有 TS checkpoint/delta 编码,保留原始版本、回执和完整扫描结果。 +正常读写只接受新格式;安装/更新通过统一命令先自动备份、验证再迁移,旧解析 +只留在迁移工具。跨 provider 则复用逻辑归档,不新增两两转换器。 +[格式、备份迁移及成本边界](../../../../reference/file-authority-state-log.md)。 + +当前增量前是四个具名开发包:本次有实际证据的 File 成本/升级修复,加下文三个 +业务边界;完成本次后仍剩三个规划包,不是继续复述“5–8 PR”。#5063、#5054、 +#4931 是已有 PR,不能重复计为新任务。D1–D3 的未通过证据另列,不能保证总 PR 数。 +本次没有删除 Python 业务 owner,只有升级命令的薄适配。 + +整 Goal 来源闭环须按 #5054 当前方向核对:它退役旧 Todo event 路径并分离 supervisor +日志,不应为已经退役的来源重建捕获 writer。剩余支持来源、consumer、回退与 cohort +仍需完整验证。 + ## 先纠正统计口径 此前“5–8”“6–8”“7–9”把宽泛工作包写成剩余 PR 数,部分实现合入、额外前置项 diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md index b6db7a1371..56a05268d8 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.md @@ -34,6 +34,16 @@ bounded Python retirement. #4931 and outstanding D2 evidence are tracked separately. Three is a delivery plan, not a guaranteed total PR count. [Current inventory and exits](ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.md). +File retained-state storage now reuses the existing TS checkpoint/delta codec, +stacked on #5063's verified read cache and RPC budgets. Original revisions, +receipts and full historical projections survive the physical format upgrade. +Normal reads/writes require v1. Installation runs explicit, verified backup and +format migration; legacy decoding exists only in the migration owner. File and +SQLite reuse logical archives for cross-provider isolated recovery. +This adds no provider/default promotion and retires no Python business owner. +[Automatic backup/migration, cold costs and qualification limits](../../reference/file-authority-state-log.md). + + ## Persistence route for steward scale (2026-09-16) [Roadmap](loopx-overall-roadmap-v0.md) R5 reuses D1 projection, D2 real-backend/capacity/applicable ten-day soak and D3 fenced cutover. R6 connects the selected shared profile to authenticated local/cloud execution. R1–R3 can advance on supported profiles without waiting for PostgreSQL or whole-Goal default promotion. diff --git a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md index 167b5f17df..59544d4abd 100644 --- a/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md +++ b/docs/architecture/rfcs/shared-goal-authority-state-provider-v0.zh-CN.md @@ -28,6 +28,13 @@ 默认启用与最后一批有界 Python 退役。#4931 与 D2 的剩余资格证据单列;三个是 可命名的开发批次,不是保证总 PR 数。[唯一当前清单与退出条件](ledger/shared-goal-authority-state-provider-v0/2026-09-24-default-cutover-reconciliation.zh-CN.md)。 +File 历史存储在 #5063 的读取缓存和 RPC 预算之上,复用现有 TS checkpoint/delta +编码;物理格式升级保留原版本、回执和每条完整历史投影。正常读写只接受 v1, +安装入口调用显式升级流程,先自动备份、验证再迁移;旧解析器仅用于迁移。File/ +SQLite 跨 provider 恢复复用逻辑归档。这不晋升 provider/默认值,也不算删除 Python +业务 owner。[自动备份迁移、冷读成本与验收边界](../../reference/file-authority-state-log.md)。 + + ## 旧观测退役检查点(2026-09-24) [当前交付清单](ledger/shared-goal-authority-state-provider-v0/2026-09-24-observation-retirement.zh-CN.md) diff --git a/docs/reference/file-authority-state-log.md b/docs/reference/file-authority-state-log.md new file mode 100644 index 0000000000..934d092ca5 --- /dev/null +++ b/docs/reference/file-authority-state-log.md @@ -0,0 +1,137 @@ +# Authority format upgrade and File retained state + +The File provider retains its `AuthorityStore` contract and `file_v0` routing +identity. The physical document changes from `loopx_file_authority_store_v0` to +`loopx_file_authority_store_v1`. This does not promote a Goal or select a provider. + +## Storage and semantics + +| | Old File document | Current File document | +| --- | --- | --- | +| Each committed row | Complete projection | Checkpoint at cursors 1, 65, 129, …; exact delta otherwise | +| Events, operation ID, original receipts | Retained | Retained without rewriting | +| Cursor and provider revision | Logical transaction identity | Identical after physical upgrade | +| Historical reads | Full stored projection | Reconstruct the same projection from nearest checkpoint | +| Runtime acceptance | Migration input only | Normal reads and writes | + +File reuses the TypeScript state-delta codec already used by SQLite. Object-key +changes and array splices preserve all JSON data, including empty and `__proto__` +keys. Writers prove reconstruction. Revisions still hash the logical full +transaction, previous revision and store identity. The ledger remains logically +append-only even though File atomically replaces its physical envelope. + +Cold reads verify every retained transaction and the final head; a valid head +cannot hide a corrupt old delta or receipt. Verified pagination reconstructs at +most 63 predecessor deltas plus the requested page. The exact-byte cache remains +bounded. File still reads/hashes and rewrites one retained file: this reduces +repeated data, not asymptotic growth. Cold verification can be slower. Measure +upgrade, cold verification, warm reads and steady writes separately. + +## Automatic upgrade and backups + +Normal readers and writers **do not accept the old File format**. Old parsing +belongs only to migration. There is no migration-on-read or first-business-write +conversion. SQLite's existing v1-to-v2 converter follows the same explicit gate. + +Default local installation and Windows installation run the candidate's upgrade +command before launcher activation. `loopx update apply` for pip/pipx runs it +after package installation and before host updates or service restart. Canary +installation does not migrate stores. Source checkouts and externally managed +package updates use the same explicit command: + +```bash +# Preview selected runtime; global --runtime-root and --registry are supported. +loopx --format json authority-archive upgrade +# Back up and migrate known runtime roots from existing project registrations. +loopx --format json authority-archive upgrade --all-known --execute +# Read-only compatibility gate, also used before binary rollback. +loopx --format json authority-archive upgrade --all-known --require-current +``` + +Discovery includes File and SQLite stores in each known runtime's authority +folders, including unselected shadows and File rollback documents. It does not +scan arbitrary home directories. Disconnected custom runtime roots must be +supplied explicitly. Selectors, registries, writer fences and execution leases +are not changed by format upgrade. + +Each store has its own durable publication boundary: + +- File holds the ordinary writer lock, verifies the entire old history, encodes + and decodes the target, and compares logical history digests. It saves exact + source bytes plus store identity under `format-backups//`, + verifies the backup and syncs directory entries before atomic replacement. +- SQLite makes an online consistent backup, including committed WAL data. A + disposable copy proves the backup through the actual v1-to-v2 converter. + The source migration compares that history digest under `BEGIN IMMEDIATE` + before adopting new tables. Concurrent source advancement aborts the upgrade; + retry takes a fresh backup. It never silently discards intervening commits. +- A manifest records source/target format, identity and integrity evidence. + File recovery uses manifest hashes and actual source/target bytes, not a + mutable completion flag. Interrupted conversion is retriable: before publish + the old store remains; after publish the new store is already current. + +Unknown formats, corrupt history, failed backups or failed validation stop the +upgrade. A multi-store upgrade can have completed earlier stores when a later +one fails; the report retains those results. Retry resumes per store. It does +not pretend to roll back the whole runtime or overwrite later business writes. +For package-manager updates, a failed data upgrade does not undo the package +installation; service activation remains blocked until repair. + +## Provider migration and recovery + +Physical format upgrade preserves provider identity and old revisions. Changing +providers uses the existing portable logical archive as the interchange format: + +```bash +loopx --format json authority-archive export --goal-id example --archive /absolute/history.ndjson +loopx --format json authority-archive verify --archive /absolute/history.ndjson +loopx --format json authority-archive restore --goal-id example --archive /absolute/history.ndjson \ + --destination /absolute/new-isolated-store --provider sqlite --archive-sha256 DIGEST --execute +``` + +File/SQLite exports restore into either File or SQLite. Restore creates a new +provider identity/revisions while preserving logical history and receipts; it +never selects the restored directory as live authority. This avoids one +converter for every pair of storage formats. PostgreSQL's existing archive +source contract remains unchanged; authenticated service activation, cutover +and PostgreSQL destination administration are separate work. + +Old raw backups can be copied into an **isolated** provider directory, with their +original identity and canonical filename, then upgraded and exported. Never +rewrite a schema label or restore an old backup over newer acknowledged writes. +Binary rollback requires the target runtime to pass `--require-current`; an old +binary without that gate is not automatically activated. Data rollback and +provider cutover require their own reviewed, fenced recovery operation. + +## Qualification and limits + +Tests cover legacy rejection, original historical identity/receipts, checkpoint +pagination, malformed history, backup damage, interruptions around rename, +competing migration processes, real SQLite backup/migration, and File/SQLite +archive interchange. CLI validation uses the managed TS runtime. An authorized +detached long-history snapshot additionally checks source immutability, exact +backup bytes and every historical transaction/receipt against the parent. + +The CLI/install surfaces change; frontend and Lark business commands continue +using the unchanged provider contract and need no new settings. This does not +close default-provider promotion, D1–D3, PostgreSQL production qualification or +Python business-owner retirement. Native Windows execution still requires its +platform CI evidence; POSIX validation does not substitute for it. + +## 中文要点 + +旧 File 每次提交都复制完整状态;新格式每 64 条保留完整检查点,其余保存差量。 +事件、原始回执、操作身份、版本号和历史内容不变。复用 SQLite 的 TS 编码规则, +不删除 append-only ledger 抽象,也不切换 provider。 + +正常读写只接受新格式。安装/更新调用统一升级入口:先验证、自动备份并核对, +再迁移和读回;源码开发也可显式调用 `authority-archive upgrade --execute`。 +迁移解析旧数据属于升级工具,不是长期运行的旧格式兼容分支。 + +同 provider 的格式升级保持身份和版本;跨 provider 则用逻辑归档导出、验证、隔离 +恢复,生成目标 provider 的新身份和版本,保留历史事实及原始回执。迁移数据不等于 +获得执行权,恢复目录不会自动成为线上 authority。 + +升级失败时可能已有部分 store 完成,必须据实报告并重试,不能覆盖之后产生的写入。 +备份仍是旧格式,恢复时应先在隔离目录升级。只回退二进制并不等于安全回退数据。 +该方案减少重复存储和后续写入耗时,但冷校验仍验证全历史,可能更慢。 diff --git a/tests/control_plane/test_authority_archive.py b/tests/control_plane/test_authority_archive.py index d64308b944..034428ace8 100644 --- a/tests/control_plane/test_authority_archive.py +++ b/tests/control_plane/test_authority_archive.py @@ -81,3 +81,35 @@ def cli(*args, exit_code=0): finally: subprocess.run([sys.executable, "-c", "from loopx.control_plane.effect_runtime import effect_runtime_result; effect_runtime_result('runtime.shutdown',{},retry_safe=False)"], cwd=REPO, capture_output=True, text=True, timeout=30, check=True) + + +def test_upgrade_cli_requires_migration_and_keeps_verified_backup(tmp_path, monkeypatch): + isolate_sqlite_runtime(tmp_path, monkeypatch) + runtime, registry, state = tmp_path / "runtime", tmp_path / "registry.json", tmp_path / "state.md" + state.write_text("# Synthetic state\n") + goal = "upgrade-cli-goal" + registry.write_text(json.dumps({"common_runtime_root": str(runtime), "goals": []})) + initialize_canonical_authority(runtime, goal, {"goal_id": goal, "value": 1}, state_path=state, provider="file") + store = next((runtime / "authority" / "file-v0").glob("authority-store-*.json")) + document = json.loads(store.read_text()) + # Single-commit legacy envelope, independently expressed on disk. + row = document["committed"][0] + row["projection"] = row.pop("state")["projection"] + document["schema_version"] = "loopx_file_authority_store_v0" + store.write_text(json.dumps(document)) + before = store.read_bytes() + command = [sys.executable, "-m", "loopx.cli", "--registry", str(registry), + "--runtime-root", str(runtime), "--format", "json", "authority-archive", "upgrade"] + checked = subprocess.run([*command, "--require-current"], capture_output=True, text=True, timeout=60) + assert checked.returncode == 1 + assert store.read_bytes() == before + preview = subprocess.run(command, capture_output=True, text=True, check=True, timeout=60) + assert json.loads(preview.stdout)["results"][0]["status"] == "planned" + executed = subprocess.run([*command, "--execute"], capture_output=True, text=True, check=True, timeout=60) + result = json.loads(executed.stdout) + assert result["status"] == "upgraded" + backup = Path(result["results"][0]["backup_directory"]) + assert (backup / "source.json").read_bytes() == before + assert json.loads((backup / "manifest.json").read_text())["cursor"] == document["cursor"] + assert json.loads(store.read_text())["provider_revision"] == document["provider_revision"] + subprocess.run([*command, "--require-current"], capture_output=True, text=True, check=True, timeout=60) diff --git a/tests/control_plane_ts/authority_format_upgrade.test.ts b/tests/control_plane_ts/authority_format_upgrade.test.ts new file mode 100644 index 0000000000..201b1f55de --- /dev/null +++ b/tests/control_plane_ts/authority_format_upgrade.test.ts @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import {mkdtemp, readFile, rm, writeFile, mkdir} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import {upgradeAuthorityFormats} from "../../loopx/control_plane/coordination/authority_format_upgrade.ts"; +import {FileAuthorityStore} from "../../loopx/control_plane/coordination/file_authority_store.ts"; +import {SqliteAuthorityStore} from "../../loopx/control_plane/coordination/sqlite_authority_store.ts"; +import {migrateSqliteAuthorityStoreV1ToV2} from "../../loopx/control_plane/coordination/sqlite_authority_migration.ts"; +import {createSqliteAuthorityStoreV1} from "./sqlite_authority_v1_fixture.ts"; +import {authorityStoreCommitFixture as commit} from "./authority_store_conformance.ts"; +import {exportAuthorityArchive, restoreAuthorityArchive, verifyAuthorityArchive} from + "../../loopx/control_plane/coordination/authority_archive.ts"; + +async function root(t: test.TestContext) { + const value = await mkdtemp(join(tmpdir(), "authority-upgrade-")); + t.after(() => rm(value, {recursive: true, force: true})); return value; +} + +test("runtime upgrade discovers old File and SQLite, backs up and preserves history across provider restore", async t => { + const runtime = await root(t), goal = "upgrade-goal"; + const file = new FileAuthorityStore(join(runtime, "authority", "file-v0"), goal); + const c = {...commit(null, "original-operation", 1, 1), next_projection: {goal_id: goal, value: 1}}; + assert.equal((await file.commitAuthority(c)).status, "applied"); + const history = await file.scanCommitted(null, 10); + assert.equal(history.status, "page"); if (history.status !== "page") return; + const raw = JSON.parse(await readFile(file.path, "utf8")); + await writeFile(file.path, JSON.stringify({...raw, schema_version: "loopx_file_authority_store_v0", committed: history.transactions})); + const source = await readFile(file.path); + const sqliteDir = join(runtime, "authority", "sqlite-v0"); + createSqliteAuthorityStoreV1(sqliteDir, goal, [{operation_id: c.operation_id, projection: c.next_projection, receipts: c.receipts}]); + const planned = await upgradeAuthorityFormats([runtime], false); + assert.equal(planned.status, "planned"); + assert.deepEqual((planned.results as any[]).map(row => row.status), ["planned", "planned"]); + assert.deepEqual(await readFile(file.path), source); + const upgraded = await upgradeAuthorityFormats([runtime], true); + assert.equal(upgraded.status, "upgraded", JSON.stringify(upgraded)); + const rows = upgraded.results as any[]; + assert.deepEqual(rows.map(row => row.status), ["migrated", "migrated"]); + assert.deepEqual(await readFile(join(rows[0].backup_directory, "source.json")), source); + assert.equal(migrateSqliteAuthorityStoreV1ToV2(rows[1].backup_directory, goal).status, "planned"); + assert.deepEqual((await file.scanCommitted(null, 10)), history); + const sqlite = new SqliteAuthorityStore(sqliteDir, goal); + assert.equal((await sqlite.readReceipt(c.operation_id)).status, "found"); + const again = await upgradeAuthorityFormats([runtime], true); + assert.deepEqual((again.results as any[]).map(row => row.status), ["already_current", "already_current"]); + // One logical interchange protocol, not a growing set of pairwise converters. + for (const [label, store] of [["file", file], ["sqlite", sqlite]] as const) { + const archive = join(runtime, `${label}.ndjson`); + await exportAuthorityArchive(store, goal, archive); + const inspected = await verifyAuthorityArchive(archive); + for (const provider of ["file", "sqlite"] as const) { + const directory = join(runtime, `restore-${label}-${provider}`); + await mkdir(directory); + const target = provider === "file" ? new FileAuthorityStore(directory, goal) : new SqliteAuthorityStore(directory, goal); + await restoreAuthorityArchive(archive, target, inspected.archive_sha256); + const receipt = await target.readReceipt(c.operation_id); + assert.equal(receipt.status, "found"); + if (receipt.status === "found") assert.deepEqual(receipt.receipts, c.receipts); + const head = await target.loadAuthority(); + assert.equal(head.status, "loaded"); if (head.status === "loaded") assert.deepEqual(head.head, c.next_projection); + } + } +}); + +test("backup digest fence rejects changed SQLite source before adoption", async t => { + const directory = await root(t), goal = "fenced-goal"; + createSqliteAuthorityStoreV1(directory, goal, [{operation_id: "one", projection: {value: 1}}]); + const result = migrateSqliteAuthorityStoreV1ToV2(directory, goal, + {execute: true, expectedSequenceDigest: "0".repeat(64)}); + assert.equal(result.status, "failed"); + assert.match(result.reason!, /changed after its verified backup/); + assert.equal(migrateSqliteAuthorityStoreV1ToV2(directory, goal).status, "planned"); +}); + +test("unsupported formats fail closed and do not bootstrap fallback stores", async t => { + const runtime = await root(t), goal = "unknown-format"; + const store = new FileAuthorityStore(join(runtime, "authority", "file-v0"), goal); + await store.commitAuthority(commit(null, "seed", 1, 1)); + const raw = JSON.parse(await readFile(store.path, "utf8")); raw.schema_version = "future_format"; + const bytes = JSON.stringify(raw); await writeFile(store.path, bytes); + const result = await upgradeAuthorityFormats([runtime], true); + assert.equal(result.status, "failed"); + assert.equal(await readFile(store.path, "utf8"), bytes); +}); diff --git a/tests/control_plane_ts/authority_store.test.ts b/tests/control_plane_ts/authority_store.test.ts index 694b0558e0..b6613fa6bd 100644 --- a/tests/control_plane_ts/authority_store.test.ts +++ b/tests/control_plane_ts/authority_store.test.ts @@ -103,7 +103,7 @@ test("corrupt, cross-goal, or revision-divergent documents fail closed", async ( assert.equal((await store.loadAuthority()).status, "failed"); const changed = structuredClone(original); - changed.committed[0].projection.authority_revision = 99; + changed.committed[0].state.projection.authority_revision = 99; changed.head.authority_revision = 99; await writeFile(store.path, JSON.stringify(changed), "utf8"); const divergent = await store.loadAuthority(); @@ -168,7 +168,7 @@ test("large file read view reuses verified head and receipts without retaining h assert.equal(CompactStore.validations, 2, "history scans still verify the complete journal"); const changed = JSON.parse(original); - changed.committed[0].projection.authority_revision = 99; + changed.committed[0].state.projection.authority_revision = 99; changed.head.authority_revision = 99; await writeFile(store.path, JSON.stringify(changed), "utf8"); assert.equal((await second.loadAuthority()).status, "failed"); diff --git a/tests/control_plane_ts/authority_store_transactions.test.ts b/tests/control_plane_ts/authority_store_transactions.test.ts index 4c267215f2..c3595d2f66 100644 --- a/tests/control_plane_ts/authority_store_transactions.test.ts +++ b/tests/control_plane_ts/authority_store_transactions.test.ts @@ -176,10 +176,22 @@ async function createFileProvider(): Promise { name: "file", store, async readDocument() { - return JSON.parse(await readFile(store.path, "utf8")) as MutableRecord; + const document = JSON.parse(await readFile(store.path, "utf8")) as MutableRecord; + // These metadata mutants deliberately repeat the same projection. Map + // physical rows to the shared logical fixture without invoking validation + // so the malformed bytes remain observable after a failed read. + return {...document, committed: (document.committed as MutableRecord[]).map(row => { + const {state, ...metadata} = row; + return {...metadata, projection: (state as MutableRecord).projection ?? seedCommit.next_projection}; + })}; }, async writeDocument(document) { - await writeFile(store.path, JSON.stringify(document)); + const rows = (document.committed as MutableRecord[]).map(row => { + const {projection, ...metadata} = row; + return {...metadata, state: row.cursor === "1" ? {kind: "checkpoint", projection} + : {kind: "delta", delta: {schema_version: "loopx_authority_state_delta_v0", operations: []}}}; + }); + await writeFile(store.path, JSON.stringify({...document, committed: rows})); }, cleanup: () => rm(root, { recursive: true, force: true }), }; diff --git a/tests/control_plane_ts/file_authority_journal.test.ts b/tests/control_plane_ts/file_authority_journal.test.ts new file mode 100644 index 0000000000..fc7f6ba76b --- /dev/null +++ b/tests/control_plane_ts/file_authority_journal.test.ts @@ -0,0 +1,207 @@ +import assert from "node:assert/strict"; +import {execFile} from "node:child_process"; +import {promisify} from "node:util"; +import {createHash} from "node:crypto"; +import {mkdtemp, readFile, rm, writeFile} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; +import test from "node:test"; +import type {JsonObject} from "../../loopx/control_plane/effect_program.ts"; +import type {AuthorityStoreCommittedTransaction} from "../../loopx/control_plane/coordination/authority_store.ts"; +import {migrateFileAuthorityStore} from "../../loopx/control_plane/coordination/file_authority_migration.ts"; +import {FileAuthorityStore} from "../../loopx/control_plane/coordination/file_authority_store.ts"; +import {canonicalAuthorityBytes} from "../../loopx/control_plane/coordination/authority_store_codec.ts"; +import {authorityStoreCommitFixture as commit} from "./authority_store_conformance.ts"; +import {productionScaleHistoryProjection} from "./production_scale_coordination_fixture.ts"; + +const GOAL = "goal-a"; +async function fixture(t: test.TestContext) { + const directory = await mkdtemp(join(tmpdir(), "file-state-log-")); + t.after(() => rm(directory, {recursive: true, force: true})); + return new FileAuthorityStore(directory, GOAL); +} + +/** Independent pre-upgrade wire writer: preserves the documented File revision + * formula, without calling the candidate append, delta or reconstruction code. */ +async function legacyHistory(store: FileAuthorityStore, projections: JsonObject[]) { + const identity = await store.storeIdentity(); + assert.equal(identity.status, "available"); + if (identity.status !== "available") throw new Error("identity unavailable"); + let revision: string | null = null; + const committed: AuthorityStoreCommittedTransaction[] = []; + for (const [i, projection] of projections.entries()) { + const transaction = {cursor: String(i + 1), operation_id: `operation-${i + 1}`, + events: [{kind: "observed", sequence: i}], receipts: [{original: i}], projection}; + const digest = createHash("sha256").update(canonicalAuthorityBytes({goal_id: GOAL, + store_identity: identity.store_identity, previous_provider_revision: revision, transaction})).digest("hex").slice(0, 24); + revision = `file:${i + 1}:${digest}`; + committed.push({...transaction, provider_revision: revision}); + } + const document = {schema_version: "loopx_file_authority_store_v0", goal_id: GOAL, + store_identity: identity.store_identity, head: projections.at(-1)!, cursor: String(projections.length), + provider_revision: revision!, committed}; + const bytes = canonicalAuthorityBytes(document); + await writeFile(store.path, bytes); + return {document, bytes}; +} + +function projection(i: number): JsonObject { + // Array insertion/removal/reorder, deleted object members and JSON keys that + // ordinary property assignment mishandles must survive historical replay. + const special = JSON.parse('{"__proto__":{"stored":true},"":42}'); + return {authority_revision: i, special, nested: i % 2 ? {retained: i} : {removed: true}, + values: i % 3 ? [i, "unchanged", {value: i}] : [{value: i}, "unchanged"], + stable: "retained-".repeat(1000)}; +} + +test("v0 is rejected until explicit backed-up upgrade; all original revisions and receipts survive", async t => { + const store = await fixture(t); + const {document, bytes} = await legacyHistory(store, Array.from({length: 66}, (_, i) => projection(i))); + for (const response of [await store.loadAuthority(), await store.scanCommitted(null, 100), + await store.readReceipt("operation-1"), await store.commitAuthority(commit(document.provider_revision, "blocked", 67, 1))]) { + assert.equal(response.status, "failed"); + } + assert.deepEqual(await readFile(store.path), bytes); + const preview = await migrateFileAuthorityStore(store.directory, GOAL); + assert.equal(preview.status, "planned"); + assert.deepEqual(await readFile(store.path), bytes); + const upgrade = await migrateFileAuthorityStore(store.directory, GOAL, true); + assert.equal(upgrade.status, "migrated"); + assert.deepEqual(await readFile(join(String(upgrade.backup_directory), "source.json")), bytes); + assert.deepEqual(await store.loadAuthority(), {status: "loaded", head: document.head, + cursor: document.cursor, provider_revision: document.provider_revision}); + const next = commit(document.provider_revision, "after-upgrade", 67, 1); + const result = await store.commitAuthority(next); + assert.equal(result.status, "applied"); + const stored = JSON.parse(await readFile(store.path, "utf8")); + assert.equal(stored.schema_version, "loopx_file_authority_store_v1"); + assert.equal(stored.store_identity, document.store_identity); + assert.equal(stored.cursor, "67"); + assert.deepEqual(stored.committed.flatMap((row: any, i: number) => row.state.kind === "checkpoint" ? [i + 1] : []), [1, 65]); + // Force cold verification instead of trusting the writer's verified cache. + await writeFile(store.path, (await readFile(store.path, "utf8")) + "\n"); + const reopened = new FileAuthorityStore(store.directory, GOAL, {existingOnly: true}); + const all = await reopened.scanCommitted(null, 100); + assert.equal(all.status, "page"); + if (all.status !== "page") return; + assert.deepEqual(all.transactions.slice(0, 66), document.committed); + assert.deepEqual(all.transactions[66]!.projection, next.next_projection); + assert.equal(all.has_more, false); + for (const row of document.committed) { + assert.deepEqual(await reopened.readReceipt(row.operation_id), {status: "found", cursor: row.cursor, + provider_revision: row.provider_revision, receipts: row.receipts}); + } + for (const after of ["1", "62", "63", "64", "65", "66", "67"]) { + const page = await reopened.scanCommitted(after, 2); + assert.equal(page.status, "page"); + if (page.status === "page") assert.deepEqual(page.transactions, all.transactions.slice(Number(after), Number(after) + 2)); + } + all.transactions[0]!.projection.stable = "mutated caller copy"; + const readAgain = await reopened.scanCommitted(null, 1); + assert.equal(readAgain.status, "page"); + if (readAgain.status === "page") assert.deepEqual(readAgain.transactions[0], document.committed[0]); +}); + +test("cold compact reads reject broken historical state even when head and requested receipt are unchanged", async t => { + const store = await fixture(t); + let revision: string | null = null; + for (let i = 0; i < 66; i++) { + const result = await store.commitAuthority({...commit(revision, `operation-${i}`, i, 1), next_projection: projection(i)}); + assert.equal(result.status, "applied"); + if (result.status === "applied") revision = result.provider_revision; + } + const original = JSON.parse(await readFile(store.path, "utf8")); + const mutations: [string, (value: any) => void][] = [ + ["old delta", d => {d.committed[1].state.delta.operations = [];}], + ["old receipt", d => {d.committed[0].receipts = [{forged: true}];}], + ["checkpoint", d => {d.committed[64].state.projection.stable = "forged";}], + ["missing checkpoint", d => {d.committed[64].state = d.committed[63].state;}], + ["unexpected state key", d => {d.committed[1].state.extra = true;}], + ["missing middle", d => {d.committed.splice(30, 1);}], + ["duplicate operation", d => {d.committed[1].operation_id = d.committed[0].operation_id;}], + ["unsupported schema", d => {d.schema_version = "unknown";}], + ]; + for (const [label, mutate] of mutations) { + const broken = structuredClone(original); mutate(broken); + const bytes = JSON.stringify(broken); + await writeFile(store.path, bytes); + for (const result of [await store.loadAuthority(), await store.readReceipt("operation-65"), + await store.scanCommitted("64", 2), await store.commitAuthority(commit(revision, "after-corruption", 67, 1))]) { + assert.equal(result.status, "failed", label); + if (result.status === "failed") assert.equal(result.reason_code, "provider_protocol_violation", label); + } + assert.equal(await readFile(store.path, "utf8"), bytes); + } +}); + +test("upgrade interruption before/after rename resumes without a new logical commit", async t => { + const base = await fixture(t); + const {document, bytes} = await legacyHistory(base, [projection(0), projection(1)]); + await assert.rejects(migrateFileAuthorityStore(base.directory, GOAL, true, { + beforePublish: async () => {throw new Error("before rename");}, + }), /before rename/); + assert.deepEqual(await readFile(base.path), bytes); + await assert.rejects(migrateFileAuthorityStore(base.directory, GOAL, true, { + afterPublish: async () => {throw new Error("after rename");}, + }), /after rename/); + assert.equal((await migrateFileAuthorityStore(base.directory, GOAL, true)).status, "already_current"); + assert.equal((await base.readReceipt("operation-1")).status, "found"); + const history = await base.scanCommitted(null, 10); + assert.equal(history.status, "page"); + if (history.status === "page") assert.deepEqual(history.transactions, document.committed); + const archived = await base.loadAuthority(); + assert.equal(archived.status, "loaded"); + if (archived.status !== "loaded") return; + assert.equal((await base.archiveAuthorityDocument(archived.provider_revision, "archive")).status, "applied"); + assert.equal((await base.archiveAuthorityDocument(archived.provider_revision, "archive")).status, "replayed"); +}); + +test("production-shaped retained history shrinks without changing mixed Todo/lease facts", async t => { + const store = await fixture(t); + const source = productionScaleHistoryProjection(GOAL, "native").projection; + const expected = Array.from({length: 70}, (_, i) => ({...source, observation_sequence: i})); + const {document, bytes} = await legacyHistory(store, expected); + await migrateFileAuthorityStore(store.directory, GOAL, true); + const result = await store.commitAuthority({...commit(document.provider_revision, "next-observation", 71, 1), + next_projection: {...source, observation_sequence: 70}}); + assert.equal(result.status, "applied"); + const compactBytes = await readFile(store.path); + // A structural budget: this workload changes one scalar in a large live + // projection, so two checkpoints plus head must not retain seventy copies. + assert.ok(compactBytes.length < bytes.length / 8, `${compactBytes.length} vs ${bytes.length}`); + await writeFile(store.path, Buffer.concat([compactBytes, Buffer.from("\n")])); + const history = await store.scanCommitted(null, 100); + assert.equal(history.status, "page"); + if (history.status === "page") { + assert.deepEqual(history.transactions.slice(0, 70), document.committed); + assert.deepEqual(history.transactions[70]!.projection, {...source, observation_sequence: 70}); + } +}); + + +test("competing upgrade processes preserve one lineage and one backup", async t => { + const store = await fixture(t); + const {document} = await legacyHistory(store, [projection(0), projection(1)]); + const module = new URL("../../loopx/control_plane/coordination/file_authority_migration.ts", import.meta.url).href; + const script = `import {migrateFileAuthorityStore} from ${JSON.stringify(module)}; + console.log(JSON.stringify(await migrateFileAuthorityStore(${JSON.stringify(store.directory)}, "goal-a", true)));`; + const commands = [0, 1].map(() => promisify(execFile)(process.execPath, + ["--no-warnings", "--experimental-strip-types", "--input-type=module", "-e", script], {timeout: 20000})); + const results = (await Promise.all(commands)).map(result => JSON.parse(result.stdout)); + assert.deepEqual(results.map(result => result.status).sort(), ["already_current", "migrated"]); + const page = await store.scanCommitted(null, 10); + assert.equal(page.status, "page"); + if (page.status === "page") assert.deepEqual(page.transactions, document.committed); +}); + +test("a damaged backup blocks retry before source publication", async t => { + const store = await fixture(t); + const {bytes} = await legacyHistory(store, [projection(0)]); + const plan = await migrateFileAuthorityStore(store.directory, GOAL); + await assert.rejects(migrateFileAuthorityStore(store.directory, GOAL, true, { + beforePublish: async () => {throw new Error("stop");}, + }), /stop/); + await writeFile(join(String(plan.backup_directory), "source.json"), "damaged backup"); + await assert.rejects(migrateFileAuthorityStore(store.directory, GOAL, true), /backup is corrupt/); + assert.deepEqual(await readFile(store.path), bytes); +}); diff --git a/tests/test_self_update_runtime_activation.py b/tests/test_self_update_runtime_activation.py index 30b7e0689b..fd901a023a 100644 --- a/tests/test_self_update_runtime_activation.py +++ b/tests/test_self_update_runtime_activation.py @@ -460,7 +460,7 @@ def test_python_distribution_apply_uses_the_owning_interpreter_pip() -> None: with ( mock.patch( "loopx.self_update.subprocess.run", - side_effect=[passed, passed, passed, passed, passed], + side_effect=[passed, passed, passed, passed, passed, passed], ) as run, mock.patch( "loopx.runtime_activation.restart_managed_loopx_services", @@ -481,8 +481,9 @@ def test_python_distribution_apply_uses_the_owning_interpreter_pip() -> None: "--upgrade", "loopx", ] - assert run.call_args_list[1].args[0][3:5] == ["workflow-skills", "--install"] - assert run.call_args_list[2].args[0][3:5] == ["slash-commands", "--install"] + assert run.call_args_list[1].args[0][-4:] == ["authority-archive", "upgrade", "--all-known", "--execute"] + assert run.call_args_list[2].args[0][3:5] == ["workflow-skills", "--install"] + assert run.call_args_list[3].args[0][3:5] == ["slash-commands", "--install"] def test_pipx_distribution_apply_preserves_the_pipx_environment() -> None: @@ -508,7 +509,7 @@ def test_pipx_distribution_apply_preserves_the_pipx_environment() -> None: with ( mock.patch( "loopx.self_update.subprocess.run", - side_effect=[passed, passed, passed, passed, passed], + side_effect=[passed, passed, passed, passed, passed, passed], ) as run, mock.patch( "loopx.runtime_activation.restart_managed_loopx_services", @@ -779,3 +780,21 @@ def test_windows_execute_update_fails_closed_without_launching_bash() -> None: assert updated["ok"] is False assert updated["execution"]["status"] == "unsupported_platform" assert "install-windows.ps1" in updated["recommended_action"] + + +def test_failed_authority_upgrade_does_not_activate_services_or_continue_host_updates(): + doctor = doctor_payload() + doctor["package"] = {"install_kind": "python_distribution", "release_root": None} + doctor["install_freshness"].update(install_kind="python_distribution", python_distribution_installer="pip") + payload = build_update_plan(action="apply", doctor_payload=doctor) + with mock.patch("loopx.self_update.subprocess.run", side_effect=[ + subprocess.CompletedProcess([], 0, "installed", ""), + subprocess.CompletedProcess([], 1, '{"status":"failed","reason":"backup failed"}', ""), + ]) as run, mock.patch("loopx.runtime_activation.restart_managed_loopx_services") as restart: + result = execute_update_plan(payload) + assert not result["ok"] + assert result["changes_applied"] # Package installed; data migration failure is not a rollback. + assert result["execution"]["authority_upgrade_returncode"] == 1 + assert result["execution"]["doctor_status"] == "skipped_prior_step_failed" + assert run.call_count == 2 + restart.assert_not_called() From 3336d853169fede7fc035da7899063c94e19ab26 Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:30:57 +0800 Subject: [PATCH 3/5] Identify storage artifacts before migration and guard update recovery Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- docs/reference/file-authority-state-log.md | 24 ++++ loopx/cli_commands/authority_archive.py | 43 +++---- .../authority_format_inspection.ts | 105 ++++++++++++++++++ .../coordination/authority_format_upgrade.ts | 21 ++-- .../coordination/file_authority_migration.ts | 6 +- .../coordination/local_authority_archive.ts | 3 + .../sqlite_authority_migration.ts | 4 + loopx/self_update.py | 22 ++-- loopx/windows_install.py | 4 +- tests/control_plane/test_authority_archive.py | 7 ++ .../authority_format_upgrade.test.ts | 66 +++++++++++ 11 files changed, 253 insertions(+), 52 deletions(-) create mode 100644 loopx/control_plane/coordination/authority_format_inspection.ts diff --git a/docs/reference/file-authority-state-log.md b/docs/reference/file-authority-state-log.md index 934d092ca5..7cbea5b1b3 100644 --- a/docs/reference/file-authority-state-log.md +++ b/docs/reference/file-authority-state-log.md @@ -27,6 +27,30 @@ bounded. File still reads/hashes and rewrites one retained file: this reduces repeated data, not asymptotic growth. Cold verification can be slower. Measure upgrade, cold verification, warm reads and steady writes separately. +## Format recognition + +```bash +loopx --format json authority-archive inspect --source /absolute/store-or-backup +``` + +Recognition uses JSON schema tags or the SQLite file header plus database +metadata and `user_version`, not filename extensions. It reports artifact kind, +provider, physical format, Goal/store identity and migration route. Unknown +versions and inconsistent SQLite version pairs are rejected by automatic +upgrade. A recognized provider selector is only a routing record; PostgreSQL +and NoKV still require their configured provider service for export. + +`metadata_only` recognition is not full history verification. Logical archives +are verified through their complete digest/seal contract; backup packages check +source bytes and lineage. Upgrade subsequently validates the complete store +under its publication boundary. Multiple local stores are reported/upgraded +independently, never silently chosen as a new live authority. + +Legacy Markdown/sidecar capture, project registry envelopes and shadow/outbox +control records have different owners. They are not alternate File database +encodings and are not rewritten by this command. Whole-Goal migration must use +its source capture and writer-fence workflow. + ## Automatic upgrade and backups Normal readers and writers **do not accept the old File format**. Old parsing diff --git a/loopx/cli_commands/authority_archive.py b/loopx/cli_commands/authority_archive.py index 5268be1a80..c2a1b5a0ae 100644 --- a/loopx/cli_commands/authority_archive.py +++ b/loopx/cli_commands/authority_archive.py @@ -20,10 +20,13 @@ def register_authority_archive_command( ) add_subcommand_format(parser) actions = parser.add_subparsers(dest="authority_archive_action", required=True) + inspect = actions.add_parser("inspect", help="Identify store, archive or backup format from its content.") + inspect.add_argument("--source", type=Path, required=True) upgrade = actions.add_parser("upgrade", help="Back up, verify and migrate local authority formats.") - upgrade.add_argument("--execute", action="store_true") + mode = upgrade.add_mutually_exclusive_group() + mode.add_argument("--execute", action="store_true") upgrade.add_argument("--all-known", action="store_true", help="Include runtime roots of registered projects.") - upgrade.add_argument("--require-current", action="store_true", help="Fail if a format upgrade is needed; never write.") + mode.add_argument("--require-current", action="store_true", help="Fail if a format upgrade is needed; never write.") for name in ("export", "verify", "restore"): action = actions.add_parser(name) action.add_argument("--archive", type=Path, required=True) @@ -44,28 +47,28 @@ def handle_authority_archive_command( ) -> int | None: if args.command != "authority-archive": return None - request: dict[str, object] = { - "schema_version": "loopx_authority_archive_admin_request_v0", - "action": args.authority_archive_action, - } - if args.authority_archive_action == "upgrade": - if args.execute and args.require_current: - raise ValueError("--require-current cannot be combined with --execute") - request.update(runtime_roots=authority_upgrade_roots( - registry_path, runtime_root_arg, all_known=args.all_known), execute=args.execute) - else: - request["archive"] = str(args.archive.expanduser().resolve()) - if args.authority_archive_action == "export": - request.update(goal_id=args.goal_id, runtime_root=str(resolve_runtime_root( - load_registry(registry_path), runtime_root_arg, registry_path=registry_path))) - elif args.authority_archive_action == "restore": - request.update(goal_id=args.goal_id, destination=str(args.destination.expanduser().resolve()), - provider=args.provider, archive_sha256=args.archive_sha256, execute=args.execute) try: + request: dict[str, object] = { + "schema_version": "loopx_authority_archive_admin_request_v0", + "action": args.authority_archive_action, + } + if args.authority_archive_action == "inspect": + request["source"] = str(args.source.expanduser().resolve()) + elif args.authority_archive_action == "upgrade": + request.update(runtime_roots=authority_upgrade_roots( + registry_path, runtime_root_arg, all_known=args.all_known), execute=args.execute) + else: + request["archive"] = str(args.archive.expanduser().resolve()) + if args.authority_archive_action == "export": + request.update(goal_id=args.goal_id, runtime_root=str(resolve_runtime_root( + load_registry(registry_path), runtime_root_arg, registry_path=registry_path))) + elif args.authority_archive_action == "restore": + request.update(goal_id=args.goal_id, destination=str(args.destination.expanduser().resolve()), + provider=args.provider, archive_sha256=args.archive_sha256, execute=args.execute) result = effect_runtime_result( "coordination.authority_archive.manage", request, timeout=300.0, retry_safe=False ) - except (RuntimeError, ValueError) as error: + except (OSError, RuntimeError, ValueError) as error: result = {"status": "failed", "reason": str(error), "authority_changed": False} if (args.authority_archive_action == "upgrade" and args.require_current and any(row.get("status") == "planned" for row in result.get("results", []))): diff --git a/loopx/control_plane/coordination/authority_format_inspection.ts b/loopx/control_plane/coordination/authority_format_inspection.ts new file mode 100644 index 0000000000..ee04a16476 --- /dev/null +++ b/loopx/control_plane/coordination/authority_format_inspection.ts @@ -0,0 +1,105 @@ +/** Content-based administrative recognition. Identification is not migration + * permission and does not certify a store's complete transaction history. */ +import {open, readFile, stat} from "node:fs/promises"; +import {join} from "node:path"; +import {createHash} from "node:crypto"; +import type {JsonObject} from "../effect_program.ts"; +import {isAuthorityJsonObject, requireAuthorityStoreId} from "./authority_store_codec.ts"; +import {FILE_AUTHORITY_LEGACY_SCHEMA} from "./file_authority_migration.ts"; +import {FILE_AUTHORITY_JOURNAL_SCHEMA} from "./file_authority_journal.ts"; +import {sqliteAuthorityRuntime} from "./sqlite_runtime.ts"; +import {SQLITE_AUTHORITY_STORE_SCHEMA, sqliteAuthorityPath} from "./sqlite_authority_store.ts"; +import {SQLITE_AUTHORITY_STORE_V1_SCHEMA} from "./sqlite_authority_migration.ts"; +import {verifyAuthorityArchive} from "./authority_archive.ts"; + +type StoreInspection = { + artifact_kind: "authority_store"; status: "recognized"; provider: "file" | "sqlite"; + format: string; goal_id: string; store_identity: string; upgrade_required: boolean; + verification: "metadata_only"; migration_route: "physical_upgrade" | "logical_archive"; +}; +export type AuthorityFormatInspection = StoreInspection | (JsonObject & { + artifact_kind: "logical_archive" | "format_backup" | "provider_selector" | "remote_store_envelope" | "unknown"; + status: "recognized" | "unsupported"; +}); + +function store(provider: "file" | "sqlite", format: string, goal: unknown, identity: unknown, + old: boolean): StoreInspection { + const storeIdentity = requireAuthorityStoreId(identity, "store identity"); + if (!new RegExp(`^${provider}:[0-9a-f]{32}$`).test(storeIdentity)) throw new Error("Invalid store identity"); + return {artifact_kind: "authority_store", status: "recognized", provider, format, + goal_id: requireAuthorityStoreId(goal, "goal id"), store_identity: storeIdentity, + upgrade_required: old, verification: "metadata_only", + migration_route: old ? "physical_upgrade" : "logical_archive"}; +} + +export async function inspectAuthorityFormat(path: string): Promise { + if ((await stat(path)).isDirectory()) { + const manifest: unknown = JSON.parse(await readFile(join(path, "manifest.json"), "utf8")); + if (!isAuthorityJsonObject(manifest) || manifest.schema_version !== "loopx_authority_format_upgrade_v0" || + (manifest.provider !== "file" && manifest.provider !== "sqlite")) throw new Error("Not a recognized format backup package"); + const goal = requireAuthorityStoreId(manifest.goal_id, "backup goal id"); + const source = manifest.provider === "file" ? join(path, "source.json") : sqliteAuthorityPath(path, goal); + const digest = createHash("sha256").update(await readFile(source)).digest("hex"); + if (digest !== manifest.source_sha256) throw new Error("Backup content digest mismatch"); + const detected = await inspectAuthorityFormat(source); + if (detected.artifact_kind !== "authority_store" || detected.provider !== manifest.provider || + detected.goal_id !== goal || detected.store_identity !== manifest.store_identity) throw new Error("Backup lineage mismatch"); + if (manifest.provider === "file" && await readFile(join(path, "store-identity"), "utf8") !== manifest.store_identity) { + throw new Error("Backup store identity mismatch"); + } + return {artifact_kind: "format_backup", status: "recognized", format: manifest.schema_version, + source: detected, verification: "backup_bytes_verified", migration_route: "isolated_restore_then_upgrade"}; + } + const handle = await open(path, "r"); + const prefix = Buffer.alloc(65536); + let length: number; + try { length = (await handle.read(prefix, 0, prefix.length, 0)).bytesRead; } finally { await handle.close(); } + if (prefix.subarray(0, 16).equals(Buffer.from("SQLite format 3\0"))) { + const db = new (sqliteAuthorityRuntime().driver.DatabaseSync)(path, {readOnly: true}); + try { + const version = Number(db.prepare("PRAGMA user_version").get()?.user_version); + const row = db.prepare("SELECT schema_version,goal_id,store_identity FROM metadata WHERE singleton=1").get(); + if (row?.schema_version === SQLITE_AUTHORITY_STORE_V1_SCHEMA && version === 1) { + return store("sqlite", String(row.schema_version), row.goal_id, row.store_identity, true); + } + if (row?.schema_version === SQLITE_AUTHORITY_STORE_SCHEMA && version === 2) { + return store("sqlite", String(row.schema_version), row.goal_id, row.store_identity, false); + } + return {artifact_kind: "unknown", status: "unsupported", format: String(row?.schema_version ?? "unknown"), + database_version: version, reason: "SQLite schema metadata and user_version are unsupported or inconsistent"}; + } finally { db.close(); } + } + // Recognize an archive from its first complete JSON record, never its suffix. + // Full digest/seal validation remains streaming in the established archive owner. + const firstLine = prefix.subarray(0, length).toString("utf8").split("\n", 1)[0]!; + let first: unknown; + try { first = JSON.parse(firstLine); } catch { first = null; } + if (isAuthorityJsonObject(first) && first.kind === "header" && first.schema_version === "loopx_authority_archive_v0") { + return {artifact_kind: "logical_archive", status: "recognized", ...await verifyAuthorityArchive(path), + verification: "archive_verified", migration_route: "isolated_provider_restore"}; + } + let value: unknown; + try { value = JSON.parse(await readFile(path, "utf8")); } + catch { return {artifact_kind: "unknown", status: "unsupported", reason: "Not a supported authority JSON, SQLite store or logical archive"}; } + if (isAuthorityJsonObject(value)) { + if (value.schema_version === FILE_AUTHORITY_LEGACY_SCHEMA || value.schema_version === FILE_AUTHORITY_JOURNAL_SCHEMA) { + return store("file", value.schema_version, value.goal_id, value.store_identity, + value.schema_version !== FILE_AUTHORITY_JOURNAL_SCHEMA); + } + if (value.schema_version === "loopx_local_authority_provider_v0") { + if (value.provider !== "sqlite" && value.provider !== "postgresql") throw new Error("Unknown selector provider"); + requireAuthorityStoreId(value.store_identity, "selector store identity"); + if (value.provider === "postgresql") requireAuthorityStoreId(value.tenant_id, "selector tenant id"); + return {artifact_kind: "provider_selector", status: "recognized", format: value.schema_version, + goal_id: requireAuthorityStoreId(value.goal_id, "goal id"), provider: value.provider ?? null, + verification: "metadata_only", migration_route: "resolve_selected_provider_before_migration"}; + } + if (value.schema_version === "loopx_nokv_authority_store_v0") { + return {artifact_kind: "remote_store_envelope", status: "recognized", provider: "nokv", format: value.schema_version, + goal_id: requireAuthorityStoreId(value.goal_id, "goal id"), verification: "metadata_only", + migration_route: "export_through_configured_provider_service"}; + } + } + return {artifact_kind: "unknown", status: "unsupported", format: isAuthorityJsonObject(value) ? value.schema_version ?? null : null, + reason: "No registered migration route for this artifact"}; +} diff --git a/loopx/control_plane/coordination/authority_format_upgrade.ts b/loopx/control_plane/coordination/authority_format_upgrade.ts index 4dcbee4a84..a70f247a21 100644 --- a/loopx/control_plane/coordination/authority_format_upgrade.ts +++ b/loopx/control_plane/coordination/authority_format_upgrade.ts @@ -5,6 +5,7 @@ import {join} from "node:path"; import {createHash} from "node:crypto"; import type {JsonObject} from "../effect_program.ts"; import {withFileMutationLock} from "../effect_runtime_io.ts"; +import {inspectAuthorityFormat} from "./authority_format_inspection.ts"; import {canonicalAuthorityBytes} from "./authority_store_codec.ts"; import {migrateFileAuthorityStore} from "./file_authority_migration.ts"; import {FileAuthorityStore, replaceFileAuthorityDurably, syncAuthorityDirectory} from "./file_authority_store.ts"; @@ -76,21 +77,17 @@ export async function upgradeAuthorityFormats(roots: readonly string[], execute: const path = join(directory, name); let goal: string | null = null; try { - if (provider === "file") { - const value: unknown = JSON.parse(await readFile(path, "utf8")); - goal = (value as {goal_id: string}).goal_id; - if (!name.startsWith("rollback") && new FileAuthorityStore(directory, goal, {existingOnly: true}).path !== path) { - throw new Error("File authority filename does not match its goal"); + const detected = await inspectAuthorityFormat(path); + if (detected.artifact_kind !== "authority_store" || detected.provider !== provider) { + throw new Error("Detected format does not match this provider directory; no migration selected"); } - } else { - const db = new (sqliteAuthorityRuntime().driver.DatabaseSync)(path, {readOnly: true}); - try { goal = String(db.prepare("SELECT goal_id FROM metadata WHERE singleton=1").get()?.goal_id ?? ""); } - finally { db.close(); } - if (sqliteAuthorityPath(directory, goal) !== path) throw new Error("SQLite filename does not match its goal"); - } + goal = detected.goal_id; + const expected = provider === "file" ? new FileAuthorityStore(directory, goal, {existingOnly: true}).path + : sqliteAuthorityPath(directory, goal); + if (!name.startsWith("rollback") && expected !== path) throw new Error("Authority filename does not match its goal"); const result = provider === "file" ? await migrateFileAuthorityStore(directory, goal, execute, {}, name.startsWith("rollback") ? path : undefined) : await upgradeSqlite(directory, goal, execute); - results.push({goal_id: goal, ...result}); + results.push({goal_id: goal, detected_format: detected.format, ...result}); } catch (error) { // Earlier stores may already have migrated. Never report global rollback // or overwrite their later commits. Retry resumes per-store publication. diff --git a/loopx/control_plane/coordination/file_authority_migration.ts b/loopx/control_plane/coordination/file_authority_migration.ts index cac788b960..6444e68752 100644 --- a/loopx/control_plane/coordination/file_authority_migration.ts +++ b/loopx/control_plane/coordination/file_authority_migration.ts @@ -11,7 +11,7 @@ import {FileAuthorityStore, fileAuthorityRevision, replaceFileAuthorityDurably, import {createHash} from "node:crypto"; const sha256 = (bytes: Uint8Array) => createHash("sha256").update(bytes).digest("hex"); -const LEGACY_SCHEMA = "loopx_file_authority_store_v0"; +export const FILE_AUTHORITY_LEGACY_SCHEMA = "loopx_file_authority_store_v0"; /** Test-only crash seam around the real durable publication boundary. */ export interface FileAuthorityMigrationEffects { @@ -43,7 +43,7 @@ export async function migrateFileAuthorityStore(directory: string, goal: string, return {status: "already_current", provider: "file", cursor: current.cursor, provider_revision: current.provider_revision}; } - if (value.schema_version !== LEGACY_SCHEMA || value.goal_id !== goal || value.store_identity !== identity || + if (value.schema_version !== FILE_AUTHORITY_LEGACY_SCHEMA || value.goal_id !== goal || value.store_identity !== identity || !hasExactAuthorityKeys(value, ["schema_version", "goal_id", "store_identity", "provider_revision", "cursor", "head", "committed"])) { throw new Error("Unsupported file authority format or mismatched lineage; source was not changed"); } @@ -57,7 +57,7 @@ export async function migrateFileAuthorityStore(directory: string, goal: string, const target = canonicalAuthorityBytes(compact.toDocument()); const sourceDigest = sha256(source), targetDigest = sha256(target); const backup = join(directory, "format-backups", sourceDigest); - const facts = {provider: "file", from_schema: LEGACY_SCHEMA, to_schema: FILE_AUTHORITY_JOURNAL_SCHEMA, + const facts = {provider: "file", from_schema: FILE_AUTHORITY_LEGACY_SCHEMA, to_schema: FILE_AUTHORITY_JOURNAL_SCHEMA, source_sha256: sourceDigest, target_sha256: targetDigest, logical_sha256: logicalDigest, store_identity: identity, goal_id: goal, cursor: compact.cursor, provider_revision: compact.provider_revision, bytes_before: source.length, bytes_after: target.length, backup_directory: backup}; diff --git a/loopx/control_plane/coordination/local_authority_archive.ts b/loopx/control_plane/coordination/local_authority_archive.ts index c151717565..444670ed67 100644 --- a/loopx/control_plane/coordination/local_authority_archive.ts +++ b/loopx/control_plane/coordination/local_authority_archive.ts @@ -1,5 +1,6 @@ /** Administrative archive transport. Large private state stays in local files; * the managed effect runtime returns only compact integrity/readback facts. */ +import {inspectAuthorityFormat} from "./authority_format_inspection.ts"; import {upgradeAuthorityFormats} from "./authority_format_upgrade.ts"; import {mkdir, readFile} from "node:fs/promises"; import {isAbsolute, join} from "node:path"; @@ -25,6 +26,8 @@ export async function manageLocalAuthorityArchive(value: unknown, try { const request = requireJsonObject(value, "authority archive request"); if (request.schema_version !== "loopx_authority_archive_admin_request_v0") throw new Error("archive request schema mismatch"); + if (request.action === "inspect") return {...base, status: "inspected", + inspection: await inspectAuthorityFormat(path(request.source, "source path"))}; if (request.action === "upgrade") { if (!Array.isArray(request.runtime_roots) || request.runtime_roots.some(root => typeof root !== "string")) { throw new Error("Upgrade requires explicit runtime roots"); diff --git a/loopx/control_plane/coordination/sqlite_authority_migration.ts b/loopx/control_plane/coordination/sqlite_authority_migration.ts index 2eefc75f37..301f37870b 100644 --- a/loopx/control_plane/coordination/sqlite_authority_migration.ts +++ b/loopx/control_plane/coordination/sqlite_authority_migration.ts @@ -119,6 +119,10 @@ function inspectSqliteAuthorityStore( metadata.schema_version !== SQLITE_AUTHORITY_STORE_V1_SCHEMA)) { throw new AuthorityStoreProtocolError("SQLite authority metadata or goal identity is invalid"); } + if ((version === 1 && metadata.schema_version !== SQLITE_AUTHORITY_STORE_V1_SCHEMA) || + (version === 2 && metadata.schema_version !== SQLITE_AUTHORITY_STORE_SCHEMA)) { + throw new AuthorityStoreProtocolError("SQLite schema metadata and user_version disagree"); + } if (version === 1 || version === 2) { const bounds = db.prepare(`SELECT (SELECT CAST(MIN(cursor) AS TEXT) FROM commits) AS first, diff --git a/loopx/self_update.py b/loopx/self_update.py index 806b830a8d..140a67db4d 100644 --- a/loopx/self_update.py +++ b/loopx/self_update.py @@ -1026,26 +1026,18 @@ def _execute_python_distribution_update( "reason": updated["recommended_action"], } else: - backup = ( - payload.get("plan", {}).get("backup") - if isinstance(payload.get("plan"), dict) - else {} - ) - rollback_command = ( - backup.get("rollback_command") if isinstance(backup, dict) else None - ) updated["recommended_action"] = ( - "inspect the failed update step, then use the recorded package rollback command" - if rollback_command - else "inspect the failed package-manager or host-material update step" + "inspect the failed update step and retry the verified authority upgrade; " + "do not roll back the package without checking current data-format compatibility" ) updated["next_action"] = { - "kind": "review_or_rollback", - "command": rollback_command, - "mutating": bool(rollback_command), - "requires_explicit_approval": bool(rollback_command), + "kind": "review_update_failure", + "command": "loopx --format json authority-archive upgrade --all-known", + "mutating": False, + "requires_explicit_approval": False, "reason": updated["recommended_action"], } + return updated diff --git a/loopx/windows_install.py b/loopx/windows_install.py index e78cfff69d..86a943e8d1 100644 --- a/loopx/windows_install.py +++ b/loopx/windows_install.py @@ -372,8 +372,8 @@ def install_windows( _restore_paths(snapshots) except Exception as restore_exc: # pragma: no cover - filesystem failure rollback_error = restore_exc - finally: - shutil.rmtree(release_root, ignore_errors=True) + # Keep the candidate: physical data may already have upgraded. + # Launcher rollback alone cannot restore storage compatibility. if rollback_error is not None: raise RuntimeError( "Windows installation failed and rollback was incomplete: " diff --git a/tests/control_plane/test_authority_archive.py b/tests/control_plane/test_authority_archive.py index 034428ace8..9aadb56328 100644 --- a/tests/control_plane/test_authority_archive.py +++ b/tests/control_plane/test_authority_archive.py @@ -98,6 +98,13 @@ def test_upgrade_cli_requires_migration_and_keeps_verified_backup(tmp_path, monk document["schema_version"] = "loopx_file_authority_store_v0" store.write_text(json.dumps(document)) before = store.read_bytes() + identified = subprocess.run([sys.executable, "-m", "loopx.cli", "--format", "json", + "authority-archive", "inspect", "--source", str(store)], capture_output=True, text=True, + check=True, timeout=60) + inspection = json.loads(identified.stdout)["inspection"] + assert inspection["artifact_kind"] == "authority_store" + assert inspection["upgrade_required"] is True + assert inspection["verification"] == "metadata_only" command = [sys.executable, "-m", "loopx.cli", "--registry", str(registry), "--runtime-root", str(runtime), "--format", "json", "authority-archive", "upgrade"] checked = subprocess.run([*command, "--require-current"], capture_output=True, text=True, timeout=60) diff --git a/tests/control_plane_ts/authority_format_upgrade.test.ts b/tests/control_plane_ts/authority_format_upgrade.test.ts index 201b1f55de..7848a81a8e 100644 --- a/tests/control_plane_ts/authority_format_upgrade.test.ts +++ b/tests/control_plane_ts/authority_format_upgrade.test.ts @@ -83,3 +83,69 @@ test("unsupported formats fail closed and do not bootstrap fallback stores", asy assert.equal(result.status, "failed"); assert.equal(await readFile(store.path, "utf8"), bytes); }); + +test("upgrade discovers an archived File lineage even when no active document remains", async t => { + const runtime = await root(t), goal = "archived-format"; + const store = new FileAuthorityStore(join(runtime, "authority", "file-v0"), goal); + const committed = await store.commitAuthority({...commit(null, "seed", 1, 1), next_projection: {goal_id: goal}}); + assert.equal(committed.status, "applied"); if (committed.status !== "applied") return; + const logical = await store.scanCommitted(null, 1); + assert.equal(logical.status, "page"); if (logical.status !== "page") return; + assert.equal((await store.archiveAuthorityDocument(committed.provider_revision, "retire")).status, "applied"); + const archived = store.authorityArchivePath("retire"); + const current = JSON.parse(await readFile(archived, "utf8")); + const old = JSON.stringify({...current, schema_version: "loopx_file_authority_store_v0", committed: logical.transactions}); + await writeFile(archived, old); + const result = await upgradeAuthorityFormats([runtime], true); + assert.equal(result.status, "upgraded", JSON.stringify(result)); + assert.equal((result.results as any[]).length, 1); + assert.equal((await store.archiveAuthorityDocument(committed.provider_revision, "retire")).status, "replayed"); + assert.equal((await store.loadAuthority()).status, "missing"); +}); + +test("content inspection separates store, archive, backup and selector regardless of extension", async t => { + const {inspectAuthorityFormat} = await import("../../loopx/control_plane/coordination/authority_format_inspection.ts"); + const runtime = await root(t), goal = "inspect-goal"; + const file = new FileAuthorityStore(join(runtime, "authority", "file-v0"), goal); + await file.commitAuthority({...commit(null, "seed", 1, 1), next_projection: {goal_id: goal}}); + const misleading = join(runtime, "store.sqlite"); + await writeFile(misleading, await readFile(file.path)); + const identified = await inspectAuthorityFormat(misleading); + assert.equal(identified.artifact_kind, "authority_store"); + assert.equal(identified.provider, "file"); + assert.equal(identified.verification, "metadata_only"); + const archive = join(runtime, "archive.json"); + await exportAuthorityArchive(file, goal, archive); + assert.equal((await inspectAuthorityFormat(archive)).artifact_kind, "logical_archive"); + const raw = JSON.parse(await readFile(file.path, "utf8")); + const row = raw.committed[0]; row.projection = row.state.projection; delete row.state; + raw.schema_version = "loopx_file_authority_store_v0"; + await writeFile(file.path, JSON.stringify(raw)); + const upgraded = await upgradeAuthorityFormats([runtime], true); + const backup = (upgraded.results as any[])[0].backup_directory; + assert.equal((await inspectAuthorityFormat(backup)).artifact_kind, "format_backup"); + await writeFile(join(backup, "source.json"), "corrupted"); + await assert.rejects(inspectAuthorityFormat(backup), /digest mismatch/); + const selector = join(runtime, "selector.json"); + await writeFile(selector, JSON.stringify({schema_version: "loopx_local_authority_provider_v0", + provider: "postgresql", goal_id: goal, tenant_id: "tenant-a", store_identity: "postgresql:synthetic"})); + const selected = await inspectAuthorityFormat(selector); + assert.equal(selected.artifact_kind, "provider_selector"); + assert.equal(selected.verification, "metadata_only"); + await writeFile(misleading, JSON.stringify({schema_version: "loopx_file_authority_store_v999"})); + assert.equal((await inspectAuthorityFormat(misleading)).status, "unsupported"); +}); + +test("SQLite recognition rejects conflicting version markers instead of claiming already current", async t => { + const {inspectAuthorityFormat} = await import("../../loopx/control_plane/coordination/authority_format_inspection.ts"); + const {sqliteAuthorityRuntime} = await import("../../loopx/control_plane/coordination/sqlite_runtime.ts"); + const directory = await root(t), goal = "version-mismatch"; + const source = createSqliteAuthorityStoreV1(directory, goal, [{operation_id: "seed", projection: {goal_id: goal}}]); + const before = await inspectAuthorityFormat(source.path); + assert.equal(before.provider, "sqlite"); assert.equal(before.upgrade_required, true); + const db = new (sqliteAuthorityRuntime().driver.DatabaseSync)(source.path); + try { db.exec("PRAGMA user_version = 2"); } finally { db.close(); } + assert.equal((await inspectAuthorityFormat(source.path)).status, "unsupported"); + const result = migrateSqliteAuthorityStoreV1ToV2(directory, goal, {execute: true}); + assert.equal(result.status, "failed"); assert.match(result.reason!, /disagree/); +}); From 3a7ba5103b92fbcbd1f4f1c4c3256fe3cc4d33c9 Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:32:03 +0800 Subject: [PATCH 4/5] Retain completed migration results when later store discovery fails Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- .../coordination/authority_format_upgrade.ts | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/loopx/control_plane/coordination/authority_format_upgrade.ts b/loopx/control_plane/coordination/authority_format_upgrade.ts index a70f247a21..718a959177 100644 --- a/loopx/control_plane/coordination/authority_format_upgrade.ts +++ b/loopx/control_plane/coordination/authority_format_upgrade.ts @@ -66,36 +66,41 @@ async function upgradeSqlite(directory: string, goal: string, execute: boolean): * All stores, including unselected shadows, must be readable by the new binary. */ export async function upgradeAuthorityFormats(roots: readonly string[], execute: boolean): Promise { const results: JsonObject[] = []; - for (const root of [...new Set(roots.map(requireLocalAuthorityRuntimeRoot))]) { - for (const provider of ["file", "sqlite"] as const) { - const directory = join(root, "authority", `${provider}-v0`); - const names = (await entries(directory)).filter(name => - (provider === "file" ? /^authority-store-[0-9a-f]{16}\.json$/ : /^authority-[0-9a-f]{64}\.sqlite$/).test(name)); - if (provider === "file") names.push(...(await entries(join(directory, "rollback"))) - .filter(name => /^authority-store-[0-9a-f]{24}\.json$/.test(name)).map(name => join("rollback", name))); - for (const name of names.sort()) { - const path = join(directory, name); - let goal: string | null = null; - try { - const detected = await inspectAuthorityFormat(path); - if (detected.artifact_kind !== "authority_store" || detected.provider !== provider) { - throw new Error("Detected format does not match this provider directory; no migration selected"); + try { + for (const root of [...new Set(roots.map(requireLocalAuthorityRuntimeRoot))]) { + for (const provider of ["file", "sqlite"] as const) { + const directory = join(root, "authority", `${provider}-v0`); + const names = (await entries(directory)).filter(name => + (provider === "file" ? /^authority-store-[0-9a-f]{16}\.json$/ : /^authority-[0-9a-f]{64}\.sqlite$/).test(name)); + if (provider === "file") names.push(...(await entries(join(directory, "rollback"))) + .filter(name => /^authority-store-[0-9a-f]{24}\.json$/.test(name)).map(name => join("rollback", name))); + for (const name of names.sort()) { + const path = join(directory, name); + let goal: string | null = null; + try { + const detected = await inspectAuthorityFormat(path); + if (detected.artifact_kind !== "authority_store" || detected.provider !== provider) { + throw new Error("Detected format does not match this provider directory; no migration selected"); + } + goal = detected.goal_id; + const expected = provider === "file" ? new FileAuthorityStore(directory, goal, {existingOnly: true}).path + : sqliteAuthorityPath(directory, goal); + if (!name.startsWith("rollback") && expected !== path) throw new Error("Authority filename does not match its goal"); + const result = provider === "file" ? await migrateFileAuthorityStore(directory, goal, execute, {}, name.startsWith("rollback") ? path : undefined) + : await upgradeSqlite(directory, goal, execute); + results.push({goal_id: goal, detected_format: detected.format, ...result}); + } catch (error) { + // Earlier stores may already have migrated. Never report global rollback + // or overwrite their later commits. Retry resumes per-store publication. + return {status: "failed", results, failed_provider: provider, failed_goal_id: goal, + reason: error instanceof Error ? error.message : "Format upgrade failed", retry_safe: true}; } - goal = detected.goal_id; - const expected = provider === "file" ? new FileAuthorityStore(directory, goal, {existingOnly: true}).path - : sqliteAuthorityPath(directory, goal); - if (!name.startsWith("rollback") && expected !== path) throw new Error("Authority filename does not match its goal"); - const result = provider === "file" ? await migrateFileAuthorityStore(directory, goal, execute, {}, name.startsWith("rollback") ? path : undefined) - : await upgradeSqlite(directory, goal, execute); - results.push({goal_id: goal, detected_format: detected.format, ...result}); - } catch (error) { - // Earlier stores may already have migrated. Never report global rollback - // or overwrite their later commits. Retry resumes per-store publication. - return {status: "failed", results, failed_provider: provider, failed_goal_id: goal, - reason: error instanceof Error ? error.message : "Format upgrade failed", retry_safe: true}; } } } + } catch (error) { + return {status: "failed", results, reason: error instanceof Error ? error.message : "Store discovery failed", + retry_safe: true}; } return {status: execute ? "upgraded" : "planned", results, authority_changed: false}; } From 116b58ecfc8777d8be3f9478d7627b1c72be84bd Mon Sep 17 00:00:00 2001 From: huangruiteng <14976749+huangruiteng@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:44:17 +0800 Subject: [PATCH 5/5] test(authority): qualify all-known upgrade discovery scope Signed-off-by: huangruiteng <14976749+huangruiteng@users.noreply.github.com> --- tests/control_plane/test_authority_archive.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/control_plane/test_authority_archive.py b/tests/control_plane/test_authority_archive.py index 9aadb56328..809801656f 100644 --- a/tests/control_plane/test_authority_archive.py +++ b/tests/control_plane/test_authority_archive.py @@ -120,3 +120,29 @@ def test_upgrade_cli_requires_migration_and_keeps_verified_backup(tmp_path, monk assert json.loads((backup / "manifest.json").read_text())["cursor"] == document["cursor"] assert json.loads(store.read_text())["provider_revision"] == document["provider_revision"] subprocess.run([*command, "--require-current"], capture_output=True, text=True, check=True, timeout=60) + + +def test_all_known_upgrade_roots_are_registry_owned_and_do_not_create_stores(tmp_path, monkeypatch): + from loopx.cli_commands import authority_archive + + common, project = tmp_path / "common", tmp_path / "project" + common.mkdir() + (project / ".loopx").mkdir(parents=True) + project_registry = project / ".loopx" / "registry.json" + project_registry.write_text(json.dumps({"common_runtime_root": ".loopx/runtime", "goals": [ + {"id": "markdown-only"}, {"id": "another-goal"}]})) + global_registry = common / "registry.global.json" + global_registry.write_text(json.dumps({"goals": [ + {"id": "markdown-only", "source_registry": str(project_registry)}, + {"id": "another-goal", "source_registry": str(project_registry)}, + {"id": "disconnected", "source_registry": str(tmp_path / "removed" / "registry.json")}, + ]})) + monkeypatch.delenv("LOOPX_RUNTIME_ROOT", raising=False) + monkeypatch.setattr(authority_archive, "DEFAULT_RUNTIME_ROOT", common) + before = {p: p.read_bytes() for p in (global_registry, project_registry)} + roots = authority_archive.authority_upgrade_roots(project_registry, None, all_known=True) + assert roots == sorted(map(str, [common, project / ".loopx/runtime"])) + assert authority_archive.authority_upgrade_roots(project_registry, None, all_known=False) == [str(project / ".loopx/runtime")] + assert all(p.read_bytes() == data for p, data in before.items()) + assert not (project / ".loopx/runtime").exists() + assert not (common / "authority").exists()