diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index af64aab90..af687e26f 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -1847,6 +1847,8 @@ function listen(): Server { /** Nothing in memory is worth losing to a Ctrl-C. */ let draining: Promise | undefined; +/** Startup's session reset, which replaces the held writer lease with a re-fenced one. */ +let resettingLease: Promise | undefined; function drain(): Promise { return draining ??= (async () => { @@ -1861,7 +1863,7 @@ function drain(): Promise { record(err); } }; - await attempt(() => server.stop(true)); + if (server) await attempt(() => server.stop(true)); if (researchRecoveryTimer) clearTimeout(researchRecoveryTimer); if (experimentTimer) clearInterval(experimentTimer); if (recoveringResearch) await attempt(() => recoveringResearch!); @@ -1897,6 +1899,7 @@ function drain(): Promise { } await attempt(() => shutdownHarnesses()); await attempt(() => writerLease.stop()); + if (resettingLease) await attempt(() => resettingLease!); let lease = writerLease.held; if (lease) await attempt(() => storage.leases.release(lease)); await attempt(() => storage.close()); @@ -2867,8 +2870,15 @@ try { ); if (!acquired) throw new Error("another Chopin instance owns the database"); writerLease.hold(acquired); - let reset = await storage.sessions.reset(new Date(), acquired, LEASE_TTL_MS); - writerLease.held = reset.lease; + // A stop that arrives while startup still runs must release the lease, or the next start finds it held. + process.on("SIGINT", signal); + process.on("SIGTERM", signal); + let resetting = storage.sessions.reset(new Date(), acquired, LEASE_TTL_MS).then(reset => { + writerLease.held = reset.lease; + return reset; + }); + resettingLease = resetting; + let reset = await resetting; // Before any room opens, so no child Conversation is ever loaded or served. let children = await storage.collaboration.dropChildConversations(reset.lease); if (children.cleared.length) { @@ -2926,7 +2936,5 @@ try { await storage.close().catch(() => {}); throw err; } -process.on("SIGINT", signal); -process.on("SIGTERM", signal); console.log(describe(config)); diff --git a/apps/server/src/storage/postgres/startup.test.ts b/apps/server/src/storage/postgres/startup.test.ts index 1decabd7f..0ed8efabc 100644 --- a/apps/server/src/storage/postgres/startup.test.ts +++ b/apps/server/src/storage/postgres/startup.test.ts @@ -161,6 +161,36 @@ if (database) { await storage.close(); }, 20_000); + it("releases the writer lease when stopped while it is still starting", async () => { + let storage = new PostgresStorage(database); + await storage.migrate(); + let sql = new SQL(database); + let held = async () => { + let [row] = await sql<{ owner: string }[]>` + SELECT owner FROM storage_leases + WHERE name = 'chopin:writer' AND expires_at > clock_timestamp() + `; + return row?.owner; + }; + let before = await held(); + let child = spawn(9077, "pipe"); + let owner: string | undefined; + for (let attempt = 0; attempt < 2_000 && (!owner || owner === before); attempt++) { + owner = await held(); + if (!owner || owner === before) await Bun.sleep(5); + } + expect(owner).toBeDefined(); + expect(owner).not.toBe(before); + child.kill("SIGTERM"); + await child.exited; + expect(await held()).toBeUndefined(); + let next = await storage.leases.acquire("chopin:writer", crypto.randomUUID(), 30_000); + expect(next).toBeDefined(); + await storage.leases.release(next!); + await sql.close(); + await storage.close(); + }, 20_000); + it("recovers an inline Research card without an open room, once across restarts", async () => { let storage = new PostgresStorage(database); await storage.migrate();