diff --git a/packages/daemon-contracts/src/daemon-process.ts b/packages/daemon-contracts/src/daemon-process.ts index 28869a01b5..edcc2b5be3 100644 --- a/packages/daemon-contracts/src/daemon-process.ts +++ b/packages/daemon-contracts/src/daemon-process.ts @@ -103,7 +103,22 @@ export type DaemonTerminationResult = }> | Readonly<{ status: 'retained'; - reason: 'missing-start-time' | 'identity-unverified' | 'signal-failed' | 'exit-timeout'; + reason: + | 'missing-start-time' + | 'identity-unverified' + /** + * A host owns this runtime and ends it. Callers that judge a registration answer this instead of + * calling {@link stopDaemonProcess} at all (`stopAndRetireDaemon`); this function signals + * whatever verified identity it is handed and never decides ownership itself. + */ + | 'embedded-owner' + /** + * The record states an owner this build cannot read, so nothing proves this pid is a daemon of + * its own, and signalling it is refused. The other half of the same rule as `embedded-owner`. + */ + | 'owner-unproven' + | 'signal-failed' + | 'exit-timeout'; signal?: NodeJS.Signals; }>; diff --git a/packages/daemon-contracts/src/daemon-registration-owner.ts b/packages/daemon-contracts/src/daemon-registration-owner.ts index 2e452699ad..63ae06e010 100644 --- a/packages/daemon-contracts/src/daemon-registration-owner.ts +++ b/packages/daemon-contracts/src/daemon-registration-owner.ts @@ -29,8 +29,10 @@ import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { resolveDaemonPaths, type DaemonCapability, + type DaemonOwnership, type DaemonPaths, type DaemonServerMode, + type ObservedDaemonIdentity, } from './daemon-resolution.ts'; import { findUnrecoveredRepairCommitFailure } from './session-repair-tombstone.ts'; import { @@ -40,9 +42,13 @@ import { type RegisteredDaemonOwnership, } from './daemon-registration.ts'; import { + buildDaemonShutdownCompletion, buildDaemonShutdownReport, + resolveDaemonShutdownCompletionPath, resolveDaemonShutdownReportPath, + type DaemonShutdownCompletion, type DaemonShutdownOutcome, + type DaemonShutdownStopAnswer, } from './daemon-shutdown-report.ts'; import { sweepDeadDaemonStartupPhasesFiles } from './daemon-startup-phases.ts'; @@ -59,13 +65,36 @@ export type DaemonRegistrationFields = Readonly<{ capabilities?: readonly DaemonCapability[]; /** The provider plugin inventory, so a client can read what this daemon registered. */ plugins?: readonly PluginInventoryEntry[]; + /** Who owns this process; see {@link DaemonOwnership}. Absent records are detached daemons. */ + ownership?: DaemonOwnership; }>; type DaemonRegistrationRemoval = | Readonly<{ state: 'removed' }> | Exclude; export type DaemonRegistrationOwner = Readonly<{ publish(fields: DaemonRegistrationFields): void; - finish(outcome?: DaemonShutdownOutcome): Promise; + /** + * Records that this instance kept the state directory on purpose, for the stop that addressed it. + * The release answer is NOT written here: {@link DaemonRegistrationOwner.finish} writes it from the + * removal it proves. Returns whether the record was written: a process that no longer holds the + * claim has nothing to say about this state dir. + */ + recordShutdownRetention(unfinished: readonly string[]): boolean; + /** + * Writes this instance's shutdown report, removes its registration, and answers the stop that asked + * it to leave — all under the claim this instance holds, and before it is released. The answer's + * disposition is what this call proves happened, never what the caller expected: a registration the + * unlink could not remove, or a step that threw before it, says `retained`. + * + * `unfinished` states what this close left outstanding. It matters when `completion` is absent, which + * is a close no stop asked for: the removal can still fail, and a state dir that stays claimed gets + * its retention record whatever the ledger says, because that record is what the next stop reads. + */ + finish( + outcome?: DaemonShutdownOutcome, + completion?: DaemonShutdownStopAnswer, + unfinished?: readonly string[], + ): Promise; }>; export async function tryAcquireDaemonRegistration( @@ -87,6 +116,7 @@ export async function tryAcquireDaemonRegistration( acquisition.assertHeld(); fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true }); sweepDeadDaemonStartupPhasesFiles(boundPaths.baseDir); + fs.rmSync(resolveDaemonShutdownCompletionPath(boundPaths.baseDir), { force: true }); } catch (error) { await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } @@ -116,14 +146,56 @@ export async function tryAcquireDaemonRegistration( mode: 0o600, }); }, - async finish(outcome?: DaemonShutdownOutcome) { + recordShutdownRetention(unfinished: readonly string[]) { + // The record only means something while this process provably holds the state dir: a claim + // someone else reclaimed means this runtime is no longer the one a stop is asking about. + return writeShutdownCompletion( + boundPaths.baseDir, + identity, + 'retained', + unfinished, + acquisition, + ); + }, + async finish( + outcome?: DaemonShutdownOutcome, + completion?: DaemonShutdownStopAnswer, + unfinished?: readonly string[], + ) { + // What this close reports as its work in hand. A call that is not a close — startup rollback — + // states nothing and so publishes no record: it never had a runtime that could keep serving. + const closeReport = completion ? completion.unfinished : unfinished; let removal: DaemonRegistrationRemoval; try { if (outcome) writeShutdownReport(boundPaths.baseDir, outcome, acquisition); removal = removeRegistrationUnderLock(boundPaths.infoPath, identity, acquisition); } catch (error) { + // A release that never ran kept the registration, which is the disposition a stop is owed + // before this claim goes away and this process can say nothing more. It is owed to whoever + // reads that state dir next, not only to a stop that asked. + if (closeReport) { + writeShutdownCompletion( + boundPaths.baseDir, + identity, + 'retained', + closeReport, + acquisition, + ); + } return await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath); } + // The disposition comes from the removal this call just proved, not from what the caller + // hoped: a registration an unlink could not remove is `retained` whatever the close planned. + const released = removal.state === 'removed' || removal.state === 'absent'; + if (closeReport && (completion || !released)) { + writeShutdownCompletion( + boundPaths.baseDir, + identity, + released ? 'released' : 'retained', + closeReport, + acquisition, + ); + } await acquisition.release(); return removal; }, @@ -176,6 +248,40 @@ function writeShutdownReport( } } +/** + * Writes this instance's completion answer under the claim it holds. It names the lifetime that wrote + * it, which is what a stop matches; a process that cannot prove its own start time writes nothing, + * because a record no stop can attribute to an instance would be read as somebody else's answer. + * The write follows the rule every file inside a held claim follows: one atomic publish at 0600, so a + * stop never half-reads its answer and a stranger cannot read what a daemon says about its own close. + */ +function writeShutdownCompletion( + stateDir: string, + identity: OwnerIdentity, + outcome: DaemonShutdownCompletion['outcome'], + unfinished: readonly string[], + acquisition: ProcessLockAcquisition, +): boolean { + if (identity.startTime === null) return false; + try { + acquisition.assertHeld(); + publishFileSync({ + destination: resolveDaemonShutdownCompletionPath(stateDir), + contents: `${JSON.stringify( + buildDaemonShutdownCompletion( + { pid: identity.pid, processStartTime: identity.startTime }, + outcome, + { unfinished }, + ), + )}\n`, + mode: 0o600, + }); + return true; + } catch { + return false; + } +} + /** * The daemon's stdout and stderr append to this inode, so it is emptied in place, never * replaced. On POSIX the append handle that creates the file also permits truncation, and @@ -333,7 +439,13 @@ function requirePrivateReplayState( export type DaemonRetirementInput = Readonly<{ paths: DaemonPaths; - observed: OwnerIdentity | null; + /** + * The lifetime a caller proved the daemon is, with the owner that record stated at the read. A caller + * that launched the process itself states `detached`, which is what it knows; a caller that read + * `daemon.json` passes what that record stated. Signalling on a guessed owner is the one thing this + * module must not do, so no caller may leave this out. + */ + observed: ObservedDaemonIdentity | null; termTimeoutMs?: number; killTimeoutMs?: number; lockTimeoutMs?: number; @@ -363,6 +475,13 @@ export type DaemonRetirementResult = | 'lock-busy' | 'registration-replaced' | 'metadata-unreadable' + /** A host owns the observed runtime; only that host, or its shutdown request, ends it. */ + | 'embedded-owner' + /** + * The record states an owner this build cannot read, so nothing proves the pid it names is a + * daemon of its own, and signalling it is refused. + */ + | 'owner-unproven' | 'retirement-unconfirmed'; error?: NormalizedError; repairCommitFailure?: RepairCommitFailure; @@ -395,14 +514,24 @@ export async function stopAndRetireDaemon( error: normalizeError(error), }; } + // Signalling needs a proven owner, and the observation carries the one its record stated: a record + // naming a host describes a runtime that host ends, and a record stating an owner this build cannot + // read says nothing about what is safe to end. Only a proven detached daemon is signaled. + const owner = observed?.ownership ?? 'detached'; const retirement = retireObservedDaemon( { ...input, observed }, - (identity) => - stopDaemonProcess(identity, { - mode: input.mode, - termTimeoutMs: input.termTimeoutMs ?? 3_000, - killTimeoutMs: input.killTimeoutMs ?? 1_000, - }), + observed && owner !== 'detached' + ? (identity) => + proveExitWithoutSignaling( + identity, + owner === 'embedded' ? 'embedded-owner' : 'owner-unproven', + ) + : (identity) => + stopDaemonProcess(identity, { + mode: input.mode, + termTimeoutMs: input.termTimeoutMs ?? 3_000, + killTimeoutMs: input.killTimeoutMs ?? 1_000, + }), owned, input.startupJoinTimeoutMs ?? 1_000, ); @@ -415,8 +544,8 @@ export async function stopAndRetireDaemon( function confirmOwnedStartupIdentity( owned: PrivateReplayState, - observed: OwnerIdentity | null, -): OwnerIdentity | null { + observed: ObservedDaemonIdentity | null, +): ObservedDaemonIdentity | null { if (!observed) { if (owned.startups.length === 0) return null; throw unownedStartupError(); @@ -424,7 +553,7 @@ function confirmOwnedStartupIdentity( const startup = owned.startups.find(({ launch }) => launch.pid === observed.pid); if (!startup) throw unownedStartupError(); const { launch } = startup; - const identity = observed.startTime ? observed : launch.readIdentity(); + const identity = observed.startTime ? observed : { ...observed, ...launch.readIdentity() }; if (launch.startTime && launch.startTime === identity.startTime) return identity; throw unownedStartupError(); } @@ -439,13 +568,33 @@ function unownedStartupError(): AppError { export async function recoverAbandonedDaemonRegistration( input: DaemonRetirementInput, ): Promise { - return await retireObservedDaemon(input, async (identity) => { - if (!identity.startTime?.trim()) return { status: 'retained', reason: 'missing-start-time' }; - const confirmed = { pid: identity.pid, startTime: identity.startTime }; - return (await waitForDaemonExit(confirmed, { timeoutMs: 0 })).exited - ? { status: 'exited', identity: confirmed, mode: 'already-exited' } - : { status: 'retained', reason: 'identity-unverified' }; - }); + return await retireObservedDaemon(input, (identity) => proveExitWithoutSignaling(identity)); +} + +/** + * Termination refusals that name an owner's rule rather than a lifetime this caller could not prove: + * both mean "nobody may signal this process", and both travel to the caller at the top level. + */ +function isOwnerRefusal( + reason: Extract['reason'], +): reason is 'embedded-owner' | 'owner-unproven' { + return reason === 'embedded-owner' || reason === 'owner-unproven'; +} + +/** + * Proves `observed` is gone and signals nothing, which is what a lifecycle path may do to a process + * it does not own: an abandoned registration, or one a host owns and ends itself. A lifetime whose + * exit cannot be proved stays exactly where it is. + */ +async function proveExitWithoutSignaling( + observed: OwnerIdentity, + reason: 'identity-unverified' | 'embedded-owner' | 'owner-unproven' = 'identity-unverified', +): Promise { + if (!observed.startTime?.trim()) return { status: 'retained', reason: 'missing-start-time' }; + const identity = { pid: observed.pid, startTime: observed.startTime }; + return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited + ? { status: 'exited', identity, mode: 'already-exited' } + : { status: 'retained', reason }; } async function retireObservedDaemon( @@ -468,6 +617,12 @@ async function retireObservedDaemon( const joined = await joinOwnedStartups(owned, startupJoinTimeoutMs); if (stopFailure) return { status: 'retained', reason: 'stop-failed', removedInfo: false, error: stopFailure }; + // A terminate step that refused because the record's owner ends this runtime, or because the record + // could not prove one, says so at the top: a caller must be able to tell "nobody may signal this + // process" from "this daemon is hung". + if (termination?.status === 'retained' && isOwnerRefusal(termination.reason)) { + return { status: 'retained', reason: termination.reason, termination, removedInfo: false }; + } if (termination && termination.status !== 'exited') return { status: 'retained', reason: 'exit-unconfirmed', termination, removedInfo: false }; if (!joined) { diff --git a/packages/daemon-contracts/src/daemon-registration-status.test.ts b/packages/daemon-contracts/src/daemon-registration-status.test.ts index 4458549220..684f2da41a 100644 --- a/packages/daemon-contracts/src/daemon-registration-status.test.ts +++ b/packages/daemon-contracts/src/daemon-registration-status.test.ts @@ -66,12 +66,49 @@ test('a live registered daemon reports its identity and the plugin inventory it state: 'running', stateDir, pid: owner.pid, + // A record that predates ownership is a detached daemon, which is what a reader must be able to + // rely on before it decides a stop may signal this pid. + ownership: 'detached', version: '0.21.30', transport: 'http', plugins: [ACTIVE, FAILED, { state: 'unknown' }], }); }); +test('a record published by a host reports the ownership that limits what a stop may do', () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-status-'); + const owner = readCurrentOwnerIdentity(); + publish(stateDir, { + token: 't', + port: 1, + pid: owner.pid, + processStartTime: owner.startTime, + ownership: 'embedded', + }); + + expect(readDaemonStatus(resolveDaemonPaths(stateDir))).toMatchObject({ + state: 'running', + pid: owner.pid, + ownership: 'embedded', + }); +}); + +test('a record that claims an ownership nobody knows reads as an unrecognized one', () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-status-'); + const owner = readCurrentOwnerIdentity(); + publish(stateDir, { + token: 't', + port: 1, + pid: owner.pid, + processStartTime: owner.startTime, + ownership: 'sideways', + }); + + // An ownership value outside the vocabulary says nothing about who owns the process, and `detached` + // is the one reading a caller may act on by signalling it. Status says it cannot recognize the owner. + expect(readDaemonStatus(resolveDaemonPaths(stateDir)).ownership).toBe('unknown'); +}); + test('a record whose process is gone or was replaced is not a running daemon', () => { const stateDir = mkdtempForTestSync('agent-device-daemon-status-'); const owner = readCurrentOwnerIdentity(); diff --git a/packages/daemon-contracts/src/daemon-registration.ts b/packages/daemon-contracts/src/daemon-registration.ts index fb2723e4d0..938d9bb6e7 100644 --- a/packages/daemon-contracts/src/daemon-registration.ts +++ b/packages/daemon-contracts/src/daemon-registration.ts @@ -8,9 +8,11 @@ import { type OwnerIdentity, } from '@agent-device/host-kit/process'; import { + readDaemonOwnership, readDaemonRecord, readRecordedDaemonTransport, resolveDaemonPaths, + type DaemonOwnership, type DaemonPaths, type DaemonServerMode, } from './daemon-resolution.ts'; @@ -125,6 +127,11 @@ export type DaemonStatus = Readonly<{ pid?: number; version?: string; transport?: DaemonServerMode; + /** + * Who owns the process this record names, and so who may end it. Only a running record can say: a + * stopped one names no live process to own. + */ + ownership?: DaemonOwnership; /** * The provider plugin inventory the running daemon published, as recorded; `readPluginInventory` * validates it where it is shown, so this module stays out of the inventory vocabulary's closure. @@ -171,6 +178,7 @@ export function readDaemonStatus(paths: DaemonPaths): DaemonStatus { state: 'running', stateDir: paths.baseDir, pid: identity.pid, + ownership: readDaemonOwnership(record.ownership), ...(typeof record.version === 'string' ? { version: record.version } : {}), ...(transport ? { transport } : {}), plugins: record.plugins, @@ -181,6 +189,49 @@ function readableStartTime(value: unknown): string | null { return typeof value === 'string' && value.trim().length > 0 ? value : null; } +/** + * What one read of `daemon.json` proves about the process it names, and the one observation every stop + * route judges. The lifetime and the owner come from the same read on purpose: a stop that routed on one + * record and retired on another could signal a process the record it acted on did not name. + * + * `identity` is null for a readable record that names no process, which leaves nothing to signal. + * `absent` is no record at all, so nothing claims the state directory. `unreadable` is a record that is + * present and cannot be read or parsed: it may describe a runtime that is serving, so a stop refuses it + * rather than reporting that nothing runs. + */ +export type DaemonStopIdentity = Readonly<{ + /** `absent`: no record. `unreadable`: a record whose contents cannot be used, unreadable or malformed. */ + ownership: DaemonOwnership | 'absent' | 'unreadable'; + identity: OwnerIdentity | null; +}>; + +/** + * Reads a state directory's record once for a stop: whether it names a process, and whose process that + * is. A record that states no owner describes the pre-existing detached daemon, which is what every + * record written before the ownership field named. + */ +export function readDaemonStopIdentity(infoPath: string): DaemonStopIdentity { + const read = readDaemonRecord(infoPath); + if (read.status !== 'present') { + return { ownership: read.status === 'absent' ? 'absent' : 'unreadable', identity: null }; + } + const parsed = parseRegistration(read.record); + const identity = parsed.pid === null ? null : { pid: parsed.pid, startTime: parsed.startTime }; + return { ownership: readDaemonOwnership(read.record.ownership), identity }; +} + +/** + * The lifetime a `--clean` cleanup may walk, from the read that routed the stop. Only a record proving + * the process is a daemon of its own names one, and the cleanup needs a start time to tell that lifetime + * from whatever holds its pid now. + */ +export function readStopCleanupOwner( + record: DaemonStopIdentity, +): { pid: number; processStartTime: string } | null { + if (record.ownership !== 'detached' || !record.identity?.startTime) return null; + return { pid: record.identity.pid, processStartTime: record.identity.startTime }; +} + /** * #2031: positive proof that `owner` is no longer the daemon serving its own * state dir, and so can never be asked to release what it holds — its sessions diff --git a/packages/daemon-contracts/src/daemon-resolution-lifecycle.test.ts b/packages/daemon-contracts/src/daemon-resolution-lifecycle.test.ts new file mode 100644 index 0000000000..2cff5adbc4 --- /dev/null +++ b/packages/daemon-contracts/src/daemon-resolution-lifecycle.test.ts @@ -0,0 +1,79 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import { + buildDaemonLifecycleRequest, + DAEMON_LIFECYCLE_COMMANDS, + readDaemonLifecycleCensus, + readDaemonLifecycleFence, + readDaemonLifecycleMode, + readDaemonOwnership, +} from './daemon-resolution.ts'; + +test('a census is read only when every count is a non-negative integer', () => { + const census = { + startingUp: false, + inFlightRequests: 1, + sessions: 0, + repairSessions: 2, + leases: 3, + }; + assert.deepEqual(readDaemonLifecycleCensus({ reason: 'daemon-busy', ...census }), census); + + // 2 ** 53 is an integer `Number.isInteger` accepts and arithmetic on it gets wrong, so the census + // reader has to be the one that refuses it. + for (const malformed of [-1, 1.5, '1', Number.NaN, Number.POSITIVE_INFINITY, 2 ** 53, null]) { + assert.equal(readDaemonLifecycleCensus({ ...census, leases: malformed }), undefined); + } + assert.equal(readDaemonLifecycleCensus({ sessions: 1 }), undefined); + // Only a daemon that is starting up says so; an older one omits the field. + assert.equal(readDaemonLifecycleCensus({ ...census, startingUp: true })?.startingUp, true); + const { startingUp: _omitted, ...withoutStartingUp } = census; + assert.equal(readDaemonLifecycleCensus(withoutStartingUp)?.startingUp, false); + assert.equal(readDaemonLifecycleCensus({ ...census, startingUp: 'yes' }), undefined); + assert.equal(readDaemonLifecycleCensus(undefined), undefined); +}); + +test('a fence names an integer pid and a start time or null, nothing else', () => { + assert.deepEqual(readDaemonLifecycleFence({ pid: 7, processStartTime: 'Fri' }), { + pid: 7, + processStartTime: 'Fri', + }); + assert.deepEqual(readDaemonLifecycleFence({ pid: 7, processStartTime: null }), { + pid: 7, + processStartTime: null, + }); + assert.equal(readDaemonLifecycleFence({ pid: 7 }), undefined); + assert.equal(readDaemonLifecycleFence({ pid: 7.5, processStartTime: null }), undefined); + assert.equal(readDaemonLifecycleFence('7'), undefined); +}); + +test('both lifecycle requests carry one fence shape under the command of their mode', () => { + const fence = { pid: 7, processStartTime: 'Fri Oct 17 09:00:00 2026' }; + for (const mode of ['quiesce', 'shutdown'] as const) { + const request = buildDaemonLifecycleRequest({ mode, token: 't0ken', fence, requestId: 'r-1' }); + assert.equal(request.command, DAEMON_LIFECYCLE_COMMANDS[mode]); + assert.deepEqual(request.input, fence); + assert.equal(request.meta?.requestId, 'r-1'); + assert.equal(readDaemonLifecycleMode(request.command), mode); + } + // A request that is not one of the two is dispatched like any other command, including a name the + // lookup would answer for through the object prototype. + assert.equal(readDaemonLifecycleMode('open'), undefined); + assert.equal(readDaemonLifecycleMode('toString'), undefined); + assert.equal(readDaemonLifecycleMode('__proto__'), undefined); +}); + +test('ownership names who may end the process, and an older record is detached', () => { + assert.equal(readDaemonOwnership('embedded'), 'embedded'); + assert.equal(readDaemonOwnership('detached'), 'detached'); + // An absent field is every record written before this field existed, and those are daemons that are + // their own process. + for (const absent of [undefined]) { + assert.equal(readDaemonOwnership(absent), 'detached'); + } + // Anything else is its own state: a caller that means to signal a pid reads `unknown` as a thing it + // must not do, not as permission to treat the record like a detached daemon's. + for (const unrecognized of [null, 'owned', 7, '', 'EMBEDDED', 2 ** 53]) { + assert.equal(readDaemonOwnership(unrecognized), 'unknown', String(unrecognized)); + } +}); diff --git a/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts b/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts deleted file mode 100644 index 67897417b5..0000000000 --- a/packages/daemon-contracts/src/daemon-resolution-quiesce.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'vitest'; -import { readDaemonQuiesceCensus, readDaemonQuiesceFence } from './daemon-resolution.ts'; - -test('a census is read only when every count is a non-negative integer', () => { - const census = { - startingUp: false, - inFlightRequests: 1, - sessions: 0, - repairSessions: 2, - leases: 3, - }; - assert.deepEqual(readDaemonQuiesceCensus({ reason: 'daemon-busy', ...census }), census); - - for (const malformed of [-1, 1.5, '1', Number.NaN, Number.POSITIVE_INFINITY, null]) { - assert.equal(readDaemonQuiesceCensus({ ...census, leases: malformed }), undefined); - } - assert.equal(readDaemonQuiesceCensus({ sessions: 1 }), undefined); - // Only a daemon that is starting up says so; an older one omits the field. - assert.equal(readDaemonQuiesceCensus({ ...census, startingUp: true })?.startingUp, true); - const { startingUp: _omitted, ...withoutStartingUp } = census; - assert.equal(readDaemonQuiesceCensus(withoutStartingUp)?.startingUp, false); - assert.equal(readDaemonQuiesceCensus({ ...census, startingUp: 'yes' }), undefined); - assert.equal(readDaemonQuiesceCensus(undefined), undefined); -}); - -test('a fence names an integer pid and a start time or null, nothing else', () => { - assert.deepEqual(readDaemonQuiesceFence({ pid: 7, processStartTime: 'Fri' }), { - pid: 7, - processStartTime: 'Fri', - }); - assert.deepEqual(readDaemonQuiesceFence({ pid: 7, processStartTime: null }), { - pid: 7, - processStartTime: null, - }); - assert.equal(readDaemonQuiesceFence({ pid: 7 }), undefined); - assert.equal(readDaemonQuiesceFence({ pid: 7.5, processStartTime: null }), undefined); - assert.equal(readDaemonQuiesceFence('7'), undefined); -}); diff --git a/packages/daemon-contracts/src/daemon-resolution.ts b/packages/daemon-contracts/src/daemon-resolution.ts index 1d95738b18..5105267988 100644 --- a/packages/daemon-contracts/src/daemon-resolution.ts +++ b/packages/daemon-contracts/src/daemon-resolution.ts @@ -7,6 +7,7 @@ import { AppError } from '@agent-device/kernel/errors'; import { type EnvMap } from '@agent-device/kernel/source-value'; import type { DaemonWireRequest } from '@agent-device/contracts/command'; +import type { OwnerIdentity } from '@agent-device/host-kit/process'; import type { DaemonServerMode, DaemonTransportPreference } from '@agent-device/kernel/contracts'; export type { DaemonServerMode, DaemonTransportPreference }; @@ -146,35 +147,125 @@ export function resolveDaemonTransportPreference( return 'auto'; } -// The quiesce request is how a client asks an idle daemon to leave so a compatible one can take -// its state directory. The client and the daemon share this vocabulary, so neither owns it. +// The two ways a process asks a daemon to leave, and the fence that addresses the exact instance. +// Client and daemon share this vocabulary, so neither owns it. `quiesce` asks an idle daemon to +// leave so a compatible one can take its state directory; `shutdown` asks a runtime whose owner is a +// host process to close itself, work in hand included. Both ride one request envelope, one fence, +// and one admission rule, and differ only in what the daemon may still hold when it agrees. /** What a daemon can do beyond the base RPC contract, published in its `daemon.json`. */ export const DAEMON_CAPABILITIES = ['quiesce'] as const; export type DaemonCapability = (typeof DAEMON_CAPABILITIES)[number]; -export const DAEMON_QUIESCE_COMMAND = 'daemon_quiesce'; +/** + * Who owns the process a registration names, and so who may end it. + * + * - `detached`: the daemon is its own process, which a stop may signal. This is what a record that + * predates the field names. + * - `embedded`: a host that called `agent-device/host` owns the process. Nothing in the central + * lifecycle signals that PID, not even hung recovery or a forced stop: the host closes the + * runtime through `host.close()`, and a stop from outside reaches it only as a shutdown request. + * - `unknown`: the record states an owner this build does not recognize, so no process may be + * signaled on its strength. Signalling on a guess could end a host process, and a value this + * build cannot read is exactly the case where the safe answer is to do nothing. + */ +const DAEMON_OWNERSHIPS = ['detached', 'embedded', 'unknown'] as const; +export type DaemonOwnership = (typeof DAEMON_OWNERSHIPS)[number]; + +/** + * The ownership a process can state about *itself*: `unknown` is what a reader answers for a record it + * cannot place, never what a runtime knows about its own process. So a runtime starts as detached or + * embedded, and the refusal for a record this build cannot read belongs to the code that reads the + * record — see {@link readDaemonOwnership}. + */ +export type DaemonRuntimeOwnership = Exclude; + +/** + * The lifetime a lifecycle path has proved the daemon on a state directory is, together with the owner + * its record stated at that read. Ownership travels inside the observation because a pid only says what + * is safe to end once its owner is known, and two reads of `daemon.json` can name different pairs. + */ +export type ObservedDaemonIdentity = OwnerIdentity & { + ownership: DaemonOwnership; +}; + +/** + * The ownership a `daemon.json` field records. An absent field is the pre-existing detached daemon — + * every record written before #3757 states no owner and describes a daemon that is its own process. + * A field that states something else is `unknown`, and is never folded into `detached`: the two read + * the same way to a caller that is about to signal a pid, and only one of them is safe to act on. + */ +export function readDaemonOwnership(value: unknown): DaemonOwnership { + return DAEMON_OWNERSHIPS.includes(value as DaemonOwnership) + ? (value as DaemonOwnership) + : value === undefined + ? 'detached' + : 'unknown'; +} + +/** + * The lifecycle requests, each named by the command that carries it. Both are the same envelope, + * fence and admission rule; they differ only in what the daemon may hold while it agrees. + */ +export const DAEMON_LIFECYCLE_COMMANDS = { + /** Stop if idle: refuses while the daemon holds a request, a session, or a lease. */ + quiesce: 'daemon_quiesce', + /** Stop and drain: ends what the daemon holds, and only an embedder-owned runtime agrees. */ + shutdown: 'daemon_shutdown', +} as const; +export type DaemonLifecycleMode = keyof typeof DAEMON_LIFECYCLE_COMMANDS; + +/** + * The lifecycle mode `command` names, or `undefined` for every request that is not one. Compared + * explicitly, because a lookup in an object would answer for a command like `toString` too. + */ +export function readDaemonLifecycleMode(command: string): DaemonLifecycleMode | undefined { + switch (command) { + case DAEMON_LIFECYCLE_COMMANDS.quiesce: + return 'quiesce'; + case DAEMON_LIFECYCLE_COMMANDS.shutdown: + return 'shutdown'; + default: + return undefined; + } +} /** The reason a daemon gives every request it refuses once it has begun to leave. */ export const DAEMON_SHUTTING_DOWN_REASON = 'daemon_shutting_down'; -export const DAEMON_QUIESCE_REASONS = { +/** Why a daemon refuses a lifecycle request: one entry per rule the admission seam enforces. */ +export const DAEMON_LIFECYCLE_REASONS = { + /** A quiesce found the daemon holding work, so nothing was accepted. */ busy: 'daemon-busy', + /** The request names a different daemon instance than the one answering it. */ instanceMismatch: 'daemon-instance-mismatch', + /** A quiesce was refused because a host owns this runtime and ends it through `host.close()`. */ + ownerEmbedded: 'daemon-owner-embedded', + /** A shutdown was refused because a detached daemon's owner ends it by signalling its process. */ + notEmbedded: 'daemon-not-embedded', } as const; -/** The daemon instance a quiesce request is addressed to; any other instance refuses it. */ -export type DaemonQuiesceFence = Readonly<{ +/** + * The refusal for a `daemon.json` whose owner field this build cannot place. No daemon answers it: a + * runtime always knows whether it owns itself, so `unknown` is a verdict about a file, and only a reader + * can reach it. + */ +export const DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON = 'daemon-owner-unrecognized'; + +/** The daemon instance a lifecycle request is addressed to; any other instance refuses it. */ +export type DaemonLifecycleFence = Readonly<{ pid: number; /** Null when the daemon could not prove its start time, as its `daemon.json` records. */ processStartTime: string | null; }>; /** - * What a daemon holds. It leaves a quiesce request only when all four counts are zero and it is - * not still starting up, because before that it has no drain path to run. + * What a daemon holds, and the census a busy refusal carries so a caller knows what to wait for. + * A quiesce leaves only when all four counts are zero and it is not still starting up, because before + * that it has no drain path to run. A shutdown drains whatever it finds, and refuses with this census + * only while the daemon is still starting; its acknowledgement carries the one `draining` outcome. */ -export type DaemonQuiesceCensus = Readonly<{ +export type DaemonLifecycleCensus = Readonly<{ /** True from process start until the daemon can drain. */ startingUp: boolean; inFlightRequests: number; @@ -183,23 +274,28 @@ export type DaemonQuiesceCensus = Readonly<{ leases: number; }>; -export function buildDaemonQuiesceRequest(params: { +/** + * The request that asks the fenced instance to leave. Both modes carry the fence in `input`, so a + * daemon refuses a request addressed to a predecessor that reused its pid. + */ +export function buildDaemonLifecycleRequest(params: { + mode: DaemonLifecycleMode; token: string; - fence: DaemonQuiesceFence; + fence: DaemonLifecycleFence; requestId: string; }): DaemonWireRequest { - const { token, fence, requestId } = params; + const { mode, token, fence, requestId } = params; return { token, session: 'default', - command: DAEMON_QUIESCE_COMMAND, + command: DAEMON_LIFECYCLE_COMMANDS[mode], positionals: [], input: { pid: fence.pid, processStartTime: fence.processStartTime }, meta: { requestId }, }; } -export function readDaemonQuiesceFence(input: unknown): DaemonQuiesceFence | undefined { +export function readDaemonLifecycleFence(input: unknown): DaemonLifecycleFence | undefined { if (typeof input !== 'object' || input === null) return undefined; const { pid, processStartTime } = input as Record; if (!Number.isInteger(pid)) return undefined; @@ -212,7 +308,7 @@ export function readDaemonCapabilities(value: unknown): readonly DaemonCapabilit return DAEMON_CAPABILITIES.filter((capability) => value.includes(capability)); } -export function readDaemonQuiesceCensus(details: unknown): DaemonQuiesceCensus | undefined { +export function readDaemonLifecycleCensus(details: unknown): DaemonLifecycleCensus | undefined { if (typeof details !== 'object' || details === null) return undefined; const { startingUp, inFlightRequests, sessions, repairSessions, leases } = details as Record< string, diff --git a/packages/daemon-contracts/src/daemon-shutdown-report-completion.test.ts b/packages/daemon-contracts/src/daemon-shutdown-report-completion.test.ts new file mode 100644 index 0000000000..00bf776dc1 --- /dev/null +++ b/packages/daemon-contracts/src/daemon-shutdown-report-completion.test.ts @@ -0,0 +1,181 @@ +import fs from 'node:fs'; +import { describe, expect, test } from 'vitest'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import { + DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS, + buildDaemonShutdownCompletion, + daemonShutdownCompletionAnswers, + daemonShutdownCompletionStillRetained, + readDaemonShutdownCompletion, + resolveDaemonShutdownCompletionPath, + type DaemonShutdownCompletion, +} from './daemon-shutdown-report.ts'; + +/** The instance a record in these rows describes: some other runtime, never this test process. */ +const WRITER = { pid: 4242, processStartTime: 'Sun Oct 18 09:00:00 2026' }; + +/** + * Plants a record the way a closed runtime's would look on disk. This is a fixture write, not the + * production one: the publisher runs inside the registration claim, so `daemon-registration-owner`'s + * rows in `src/__tests__/daemon-registration-embedded-owner.test.ts` cover the real write and its file + * mode. These rows judge the reader and the fence against a shape, and a fixture write keeps them + * independent of who produced the file. + */ +function plantCompletion( + stateDir: string, + outcome: Parameters[1], + unfinished: readonly string[], +) { + fs.writeFileSync( + resolveDaemonShutdownCompletionPath(stateDir), + `${JSON.stringify(buildDaemonShutdownCompletion(WRITER, outcome, { unfinished }))}\n`, + { mode: 0o600 }, + ); +} + +function stateDirFor(tag: string) { + return mkdtempForTestSync(`agent-device-shutdown-completion-${tag}-`); +} + +describe('readDaemonShutdownCompletion', () => { + test('a state directory that never held a daemon answers nothing', () => { + expect(readDaemonShutdownCompletion(stateDirFor('absent'))).toBe(null); + }); + + test('a record written by a close reads back with its verdict', () => { + const stateDir = stateDirFor('round-trip'); + plantCompletion(stateDir, 'retained', ['1 in-flight request ignored cancellation']); + + const record = readDaemonShutdownCompletion(stateDir); + expect(record).toMatchObject({ + pid: WRITER.pid, + processStartTime: WRITER.processStartTime, + outcome: 'retained', + unfinished: ['1 in-flight request ignored cancellation'], + }); + expect(record!.writtenAtMs).toBeGreaterThan(0); + }); + + test('a record a malformed writer left behind reads as nothing', () => { + const stateDir = stateDirFor('malformed'); + fs.writeFileSync(resolveDaemonShutdownCompletionPath(stateDir), '{not json', { mode: 0o600 }); + + // A stop that cannot read its answer reports that plainly, rather than acting on a half-record. + expect(readDaemonShutdownCompletion(stateDir)).toBe(null); + }); + + test('a record of another shape reads as nothing', () => { + const stateDir = stateDirFor('shape'); + // Complete but for the outcome: an unreadable verdict is not evidence about a close either. + fs.writeFileSync( + resolveDaemonShutdownCompletionPath(stateDir), + JSON.stringify({ + pid: WRITER.pid, + processStartTime: WRITER.processStartTime, + outcome: 'maybe', + unfinished: [], + writtenAtMs: Date.now(), + }), + { mode: 0o600 }, + ); + + expect(readDaemonShutdownCompletion(stateDir)).toBe(null); + }); + + test('a record naming no instance reads as nothing', () => { + const stateDir = stateDirFor('no-instance'); + fs.writeFileSync( + resolveDaemonShutdownCompletionPath(stateDir), + JSON.stringify({ + pid: WRITER.pid, + outcome: 'released', + unfinished: [], + writtenAtMs: Date.now(), + }), + { mode: 0o600 }, + ); + + // Without a start time the record cannot be fenced to a lifetime, so no stop may read it as its own. + expect(readDaemonShutdownCompletion(stateDir)).toBe(null); + }); +}); + +describe('daemonShutdownCompletionAnswers', () => { + const record = buildDaemonShutdownCompletion(WRITER, 'released', { unfinished: [] }); + + test('a stop on the instance that wrote the record is answered by it', () => { + expect(daemonShutdownCompletionAnswers(record, WRITER)).toBe(true); + }); + + test("a stop on another instance is not answered by somebody else's shutdown", () => { + expect(daemonShutdownCompletionAnswers(record, { ...WRITER, pid: 9090 })).toBe(false); + // A recycled pid is a different runtime, so the start time has to match too. + expect( + daemonShutdownCompletionAnswers(record, { + ...WRITER, + processStartTime: 'Sun Oct 18 10:00:00 2026', + }), + ).toBe(false); + }); + + test('a record older than the window is not evidence about a stop made now', () => { + expect( + daemonShutdownCompletionAnswers( + { ...record, writtenAtMs: Date.now() - DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS - 1 }, + WRITER, + ), + ).toBe(false); + }); + + test('a record stamped ahead of now is not evidence either', () => { + // A writer whose clock runs ahead must not make a stale answer look fresh, and the skew needed to + // try is small: both writers read the same machine's clock. + for (const aheadMs of [30_000, DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS + 1000]) { + expect( + daemonShutdownCompletionAnswers( + { ...record, writtenAtMs: Date.now() + aheadMs }, + WRITER, + Date.now(), + ), + `${aheadMs} ms ahead`, + ).toBe(false); + } + }); +}); + +// A retained close keeps its registration, so its record reports a state that holds rather than an answer +// to a request that is being answered now. These rows say which records hold and which expire. +describe('daemonShutdownCompletionStillRetained', () => { + const aged = (record: DaemonShutdownCompletion) => ({ + ...record, + writtenAtMs: Date.now() - DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS - 1000, + }); + + test('a retained record older than the window still describes the state directory', () => { + // The stop that arrives a day later reads the same truth the first one did: this instance closed and + // kept the registration. Nothing clears the file but a new acquisition, which is its own event. + const retained = aged( + buildDaemonShutdownCompletion(WRITER, 'retained', { unfinished: ['1 lease'] }), + ); + expect(daemonShutdownCompletionAnswers(retained, WRITER)).toBe(false); + expect(daemonShutdownCompletionStillRetained(retained, WRITER)).toBe(true); + }); + + test('a released record older than the window describes no state that holds', () => { + // A close that released its registration left nothing behind to report on, so only the stop that + // asked reads that record, and only while its answer is current. + const released = aged(buildDaemonShutdownCompletion(WRITER, 'released', { unfinished: [] })); + expect(daemonShutdownCompletionStillRetained(released, WRITER)).toBe(false); + }); + + test("a retained record does not describe another instance's state directory", () => { + const retained = buildDaemonShutdownCompletion(WRITER, 'retained', { unfinished: [] }); + expect(daemonShutdownCompletionStillRetained(retained, { ...WRITER, pid: 9090 })).toBe(false); + expect( + daemonShutdownCompletionStillRetained(retained, { + ...WRITER, + processStartTime: 'Sun Oct 18 10:00:00 2026', + }), + ).toBe(false); + }); +}); diff --git a/packages/daemon-contracts/src/daemon-shutdown-report.ts b/packages/daemon-contracts/src/daemon-shutdown-report.ts index 5208ad11ba..ec18cf555f 100644 --- a/packages/daemon-contracts/src/daemon-shutdown-report.ts +++ b/packages/daemon-contracts/src/daemon-shutdown-report.ts @@ -1,8 +1,19 @@ import fs from 'node:fs'; import path from 'node:path'; import type { DeviceLease } from '@agent-device/contracts/device'; +import type { DaemonLifecycleMode } from './daemon-resolution.ts'; + +// The two records a daemon's shutdown leaves on disk in the state directory, and the only readers of +// each. `daemon-shutdown.json` is the teardown census a detached daemon leaves for the process that +// stopped it; `daemon-shutdown-completion.json` is the bounded answer an embedder-owned runtime leaves +// to the stop it was asked for, which has no process to wait for. Both are written while the writer +// holds the state directory's claim, with mode 0600. const SHUTDOWN_REPORT_FILE = 'daemon-shutdown.json'; +const COMPLETION_FILE = 'daemon-shutdown-completion.json'; + +/** How old a completion record may be and still describe a stop made now. */ +export const DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS = 60_000; export type ProviderReleaseRecord = { leaseId: string; @@ -147,3 +158,179 @@ function isProviderReleaseRecord(value: unknown): value is ProviderReleaseRecord (record.provider === undefined || typeof record.provider === 'string') ); } + +/** + * The bounded completion record an embedder-owned runtime leaves where its stop can read it. + * + * A detached daemon reports its own exit to the process that signalled it. An embedded runtime closes + * inside another process, so a stop has no process to wait for, and the registration it held is gone by + * the time the close has anything to say. This record answers "did the instance I stopped finish", and + * two things fence it to the instance it describes: the lifetime that wrote it, and this age window, + * because a record nobody asked for is not evidence about a stop made now. + * + * It names its writer's lifetime, not a request. Several stops can address one instance — an operator + * retrying a stop that looks hung, two scripts sharing a state directory — and each of them gets this + * same honest answer instead of waiting for a record written for somebody else's request. + */ +export type DaemonShutdownCompletion = { + /** The instance that wrote this record: the pid it ran as. */ + pid: number; + /** And the process lifetime that pid had then, so a recycled number is not mistaken for it. */ + processStartTime: string; + /** + * What became of the registration this instance held: `released` means the state directory is free, + * `retained` means the runtime keeps it on purpose. What the close itself managed to finish is + * `unfinished`, which is a separate question: a close can release a registration and still report + * work it could not finish. + */ + outcome: 'released' | 'retained'; + /** What the close left unfinished, one line each. Empty means the close was clean. */ + unfinished: readonly string[]; + writtenAtMs: number; +}; + +/** The instance a completion record must describe to answer a stop. */ +export type DaemonShutdownCompletionQuery = Readonly<{ + pid: number; + processStartTime: string; +}>; + +/** What a runtime supplies about its close; the disposition is the release's to state. */ +export type DaemonShutdownStopAnswer = Readonly<{ + unfinished: readonly string[]; +}>; + +export function resolveDaemonShutdownCompletionPath(stateDir: string): string { + return path.join(stateDir, COMPLETION_FILE); +} + +/** Tracks whether a close owes a stop its completion record. */ +export type DaemonShutdownAnswerLedger = { + /** + * Records that a lifecycle request was admitted. Only a shutdown makes the answer owed: a quiesce + * asks the daemon to consider leaving on its own, and its owner already watches the process. + */ + accept(mode: DaemonLifecycleMode): void; + /** + * Claims the answer for one release. Only the first call after an acceptance returns it: a close + * has one work-in-hand to report, and a second release would report the same work twice. + */ + take(unfinished: readonly string[]): DaemonShutdownStopAnswer | undefined; +}; + +/** + * Keeps the promise a runtime makes to the stop it accepted: a close that was never asked to leave owes + * no answer to anybody, because its owner watches the process. A close that keeps the state directory + * is different, and is not the ledger's business — it publishes its record whether or not a stop asked, + * because that directory is what the next reader will find. + */ +export function createDaemonShutdownAnswerLedger(): DaemonShutdownAnswerLedger { + let accepted = false; + let answered = false; + return { + accept(mode: DaemonLifecycleMode): void { + if (mode === 'shutdown') accepted = true; + }, + take(unfinished: readonly string[]): DaemonShutdownStopAnswer | undefined { + if (!accepted || answered) return undefined; + answered = true; + return { unfinished: [...unfinished] }; + }, + }; +} + +export function buildDaemonShutdownCompletion( + instance: DaemonShutdownCompletionQuery, + outcome: DaemonShutdownCompletion['outcome'], + answer: DaemonShutdownStopAnswer, +): DaemonShutdownCompletion { + return { + pid: instance.pid, + processStartTime: instance.processStartTime, + outcome, + unfinished: [...answer.unfinished], + writtenAtMs: Date.now(), + }; +} + +export function readDaemonShutdownCompletion(stateDir: string): DaemonShutdownCompletion | null { + let raw: string; + try { + raw = fs.readFileSync(resolveDaemonShutdownCompletionPath(stateDir), 'utf8'); + } catch { + return null; + } + let parsed: unknown; + try { + parsed = JSON.parse(raw) as unknown; + } catch { + return null; + } + return isDaemonShutdownCompletion(parsed) ? parsed : null; +} + +/** + * Whether a record describes `query`'s instance and is fresh enough to be evidence about it. A record + * written by another instance is not evidence, so a waiter keeps waiting rather than reporting + * somebody else's shutdown as this one. A record stamped ahead of now is not evidence either: a writer + * whose clock runs ahead must not get to make an old answer look fresh. + * + * This bounds a close that was asked for and released, which is what the stop is waiting for. A close + * that kept its registration is not time-bounded — see {@link daemonShutdownCompletionStillRetained}. + */ +export function daemonShutdownCompletionAnswers( + completion: DaemonShutdownCompletion, + query: DaemonShutdownCompletionQuery, + nowMs = Date.now(), +): boolean { + // The age is checked in one direction only. A record stamped ahead of now cannot be evidence about + // a close: a writer's own clock may run ahead, and letting it reach backward in time would let a + // stale answer look fresh. + const ageMs = nowMs - completion.writtenAtMs; + return ( + daemonShutdownCompletionNamesInstance(completion, query) && + ageMs >= 0 && + ageMs <= DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS + ); +} + +/** + * Whether a `retained` record still reports `query`'s instance, whatever its age. + * + * A retained close keeps its registration on purpose, so its record describes a state that holds until + * someone ends it: a new daemon clears the file when it acquires the state dir, and a close that does + * not keep the registration answers with `released`. So the lifetime fence decides this and the age adds + * nothing, unlike a `released` record, whose job is to answer the stop that asked for it. + * + * The caller must prove the instance is still the process the record names. A dead host's leftover + * record is a stale file, not a runtime that agreed to leave. + */ +export function daemonShutdownCompletionStillRetained( + completion: DaemonShutdownCompletion, + query: DaemonShutdownCompletionQuery, +): boolean { + return ( + completion.outcome === 'retained' && daemonShutdownCompletionNamesInstance(completion, query) + ); +} + +function daemonShutdownCompletionNamesInstance( + completion: DaemonShutdownCompletion, + query: DaemonShutdownCompletionQuery, +): boolean { + return completion.pid === query.pid && completion.processStartTime === query.processStartTime; +} + +function isDaemonShutdownCompletion(value: unknown): value is DaemonShutdownCompletion { + if (typeof value !== 'object' || value === null) return false; + const record = value as Partial; + return ( + typeof record.pid === 'number' && + typeof record.processStartTime === 'string' && + record.processStartTime.trim().length > 0 && + (record.outcome === 'released' || record.outcome === 'retained') && + Array.isArray(record.unfinished) && + record.unfinished.every((line) => typeof line === 'string') && + Number.isSafeInteger(record.writtenAtMs) + ); +} diff --git a/packages/daemon-contracts/src/daemon-stop.test.ts b/packages/daemon-contracts/src/daemon-stop.test.ts index e5003cb030..54480e6535 100644 --- a/packages/daemon-contracts/src/daemon-stop.test.ts +++ b/packages/daemon-contracts/src/daemon-stop.test.ts @@ -7,7 +7,7 @@ vi.mock('./daemon-registration-owner.ts', () => ({ stopAndRetireDaemon: mocks.stopAndRetireDaemon, })); -import { resolveDaemonPaths } from './daemon-resolution.ts'; +import { resolveDaemonPaths, type ObservedDaemonIdentity } from './daemon-resolution.ts'; import { stopDaemon } from './daemon-stop.ts'; afterEach(() => { @@ -18,35 +18,45 @@ function createDaemonPaths(): ReturnType { const stateDir = mkdtempForTestSync('agent-device-daemon-stop-'); const paths = resolveDaemonPaths(stateDir); fs.mkdirSync(paths.baseDir, { recursive: true }); + // The record's contents are the caller's business: a stop hands over the lifetime and owner its own + // read proved, so this fixture only has to put a record on disk for a retirement to act on. fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: 'start-time' })); return paths; } +/** The lifetime the record above names, which is what a caller hands to the retirement it routes. */ +const observed: ObservedDaemonIdentity = { + pid: 123, + startTime: 'start-time', + ownership: 'detached', +}; + function removeDaemonPaths(paths: ReturnType): void { fs.rmSync(paths.baseDir, { recursive: true, force: true }); } -test('reports not-running when daemon metadata is absent', async () => { +test('a caller that read no lifetime reports not-running without retiring', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-daemon-stop-')); try { - const result = await stopDaemon({ paths }); + const result = await stopDaemon({ paths, observed: null }); expect(result).toMatchObject({ stopped: false, mode: 'not-running' }); } finally { removeDaemonPaths(paths); } + expect(mocks.stopAndRetireDaemon).not.toHaveBeenCalled(); }); test('retained identity verification is reported as failure without known cleanup', async () => { const paths = createDaemonPaths(); mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit('identity-unverified')); - await expect(stopDaemon({ paths })).rejects.toMatchObject({ + await expect(stopDaemon({ paths, observed })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_exit_unconfirmed', terminationReason: 'identity-unverified' }, }); expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith({ paths, - observed: { pid: 123, startTime: 'start-time' }, + observed, mode: 'graceful', termTimeoutMs: 10_000, killTimeoutMs: 2_000, @@ -55,19 +65,23 @@ test('retained identity verification is reported as failure without known cleanu test('missing start-time identity is passed to the owning termination operation', async () => { const paths = createDaemonPaths(); - fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123, processStartTime: ' ' })); mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit('missing-start-time')); - await expect(stopDaemon({ paths })).rejects.toMatchObject({ + await expect( + stopDaemon({ paths, observed: { pid: 123, startTime: null, ownership: 'detached' } }), + ).rejects.toMatchObject({ details: { terminationReason: 'missing-start-time' }, }); expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith( - expect.objectContaining({ paths, observed: { pid: 123, startTime: null } }), + expect.objectContaining({ + paths, + observed: { pid: 123, startTime: null, ownership: 'detached' }, + }), ); }); test('a previously exited verified lifetime is reported as not-running', async () => { mocks.stopAndRetireDaemon.mockResolvedValue(retired('already-exited')); - expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ + expect(await stopDaemon({ paths: createDaemonPaths(), observed })).toMatchObject({ stopped: false, mode: 'not-running', }); @@ -75,14 +89,15 @@ test('a previously exited verified lifetime is reported as not-running', async ( test('an already released pid without start time remains not-running without cleanup proof', async () => { const paths = createDaemonPaths(); - fs.writeFileSync(paths.infoPath, JSON.stringify({ pid: 123 })); mocks.stopAndRetireDaemon.mockResolvedValue({ status: 'retained', reason: 'exit-unconfirmed', removedInfo: false, termination: { status: 'not-running' }, }); - expect(await stopDaemon({ paths })).toMatchObject({ + expect( + await stopDaemon({ paths, observed: { pid: 123, startTime: null, ownership: 'detached' } }), + ).toMatchObject({ stopped: false, mode: 'not-running', }); @@ -91,15 +106,17 @@ test('an already released pid without start time remains not-running without cle test('confirmed TERM exit preserves graceful report behavior and configured budgets', async () => { const paths = createDaemonPaths(); mocks.stopAndRetireDaemon.mockResolvedValue(retired('graceful')); - expect(await stopDaemon({ paths, graceTimeoutMs: 11, killTimeoutMs: 7 })).toMatchObject({ - stopped: true, - mode: 'graceful', - cleanupConfidence: 'known', - providerReleases: { pending: [] }, - }); + expect(await stopDaemon({ paths, observed, graceTimeoutMs: 11, killTimeoutMs: 7 })).toMatchObject( + { + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + providerReleases: { pending: [] }, + }, + ); expect(mocks.stopAndRetireDaemon).toHaveBeenCalledWith({ paths, - observed: { pid: 123, startTime: 'start-time' }, + observed, mode: 'graceful', termTimeoutMs: 11, killTimeoutMs: 7, @@ -108,7 +125,7 @@ test('confirmed TERM exit preserves graceful report behavior and configured budg test('confirmed KILL exit preserves unknown provider cleanup', async () => { mocks.stopAndRetireDaemon.mockResolvedValue(retired('forced')); - expect(await stopDaemon({ paths: createDaemonPaths() })).toMatchObject({ + expect(await stopDaemon({ paths: createDaemonPaths(), observed })).toMatchObject({ stopped: true, mode: 'forced', cleanupConfidence: 'unknown', @@ -121,7 +138,7 @@ test.each(['signal-failed', 'exit-timeout'])( '%s cannot become a successful stop', async (reason) => { mocks.stopAndRetireDaemon.mockResolvedValue(retainedExit(reason)); - await expect(stopDaemon({ paths: createDaemonPaths() })).rejects.toMatchObject({ + await expect(stopDaemon({ paths: createDaemonPaths(), observed })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_exit_unconfirmed', terminationReason: reason }, }); @@ -162,7 +179,7 @@ test.each(['registration-replaced', 'retirement-unconfirmed'])( logPath: '/retained/daemon.log', }, }); - await expect(stopDaemon({ paths })).rejects.toMatchObject({ + await expect(stopDaemon({ paths, observed })).rejects.toMatchObject({ code: 'COMMAND_FAILED', details: { reason: 'daemon_retirement_unconfirmed', @@ -175,3 +192,13 @@ test.each(['registration-replaced', 'retirement-unconfirmed'])( expect(fs.existsSync(paths.infoPath)).toBe(true); }, ); + +test('a record that predates ownership keeps the signal path it always had', async () => { + mocks.stopAndRetireDaemon.mockResolvedValue(retired('graceful')); + + expect(await stopDaemon({ paths: createDaemonPaths(), observed })).toMatchObject({ + stopped: true, + mode: 'graceful', + }); + expect(mocks.stopAndRetireDaemon).toHaveBeenCalled(); +}); diff --git a/packages/daemon-contracts/src/daemon-stop.ts b/packages/daemon-contracts/src/daemon-stop.ts index f4a6d67639..9a23066139 100644 --- a/packages/daemon-contracts/src/daemon-stop.ts +++ b/packages/daemon-contracts/src/daemon-stop.ts @@ -1,9 +1,8 @@ import { AppError } from '@agent-device/kernel/errors'; import type { DaemonRetirementResult } from './daemon-registration-owner.ts'; -import type { OwnerIdentity } from '@agent-device/host-kit/process'; -import type { DaemonPaths } from './daemon-resolution.ts'; -import { readRegisteredDaemonIdentity } from './daemon-registration.ts'; +import type { DaemonPaths, ObservedDaemonIdentity } from './daemon-resolution.ts'; +import type { OwnerIdentity } from '@agent-device/host-kit/process'; import type { DeviceClaimRecord, ProviderReleaseRecord } from './daemon-shutdown-report.ts'; const DAEMON_STOP_GRACE_TIMEOUT_MS = 10_000; @@ -36,24 +35,32 @@ export type DaemonStopResult = { warnings: string[]; }; +/** + * Retires the daemon whose lifetime the caller read, and shapes that retirement as a stop result. + * + * The caller hands over the lifetime and the owner its own read proved, so routing and retirement judge + * one record. Only a detached daemon reaches here: a host owns an embedded runtime and ends it with its + * own close, and a record this build cannot place proves nothing about what is safe to signal. + */ export async function stopDaemon(params: { paths: DaemonPaths; + observed: ObservedDaemonIdentity | null; graceTimeoutMs?: number; killTimeoutMs?: number; }): Promise { - const info = readRegisteredDaemonIdentity(params.paths.infoPath); - if (!info) return notRunningResult(); + const observed = params.observed; + if (!observed) return notRunningResult(); const { stopAndRetireDaemon } = await import('./daemon-registration-owner.ts'); const retirement = await stopAndRetireDaemon({ paths: params.paths, - observed: info, + observed, mode: 'graceful', termTimeoutMs: params.graceTimeoutMs ?? DAEMON_STOP_GRACE_TIMEOUT_MS, killTimeoutMs: params.killTimeoutMs ?? DAEMON_STOP_KILL_TIMEOUT_MS, }); if (retirement.status === 'retained' && retirement.termination?.status === 'not-running') return notRunningResult(); - if (retirement.status !== 'retired') throw daemonRetirementError(info, retirement); + if (retirement.status !== 'retired') throw daemonRetirementError(observed, retirement); if (retirement.termination.mode === 'already-exited') return notRunningResult(); if (retirement.termination.mode === 'graceful') { return { @@ -105,12 +112,12 @@ function daemonRetirementError( }); } -export function readDaemonStopIdentity( - infoPath: string, -): { pid: number; processStartTime: string } | null { - const info = readRegisteredDaemonIdentity(infoPath); - if (!info?.startTime) return null; - return { pid: info.pid, processStartTime: info.startTime }; +/** + * The stop result for a state directory that names no daemon, from the one owner of that shape so a + * stop that reaches a runtime over its socket reports the same verdict as one that found nothing. + */ +export function daemonNotRunningStopResult(): DaemonStopResult { + return notRunningResult(); } function notRunningResult(): DaemonStopResult { diff --git a/packages/daemon/src/server/http/daemon-runtime-lifecycle.test.ts b/packages/daemon/src/server/http/daemon-runtime-lifecycle.test.ts new file mode 100644 index 0000000000..f09c8997b9 --- /dev/null +++ b/packages/daemon/src/server/http/daemon-runtime-lifecycle.test.ts @@ -0,0 +1,698 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import { afterEach, expect, test, vi } from 'vitest'; +import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; +import { + buildDaemonLifecycleRequest, + resolveDaemonPaths, + type DaemonRuntimeOwnership, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import { + daemonShutdownCompletionAnswers, + readDaemonShutdownCompletion, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; + +// A lifecycle request is the daemon's own answer to "may I leave?", so these tests boot the real +// runtime and ask over its real socket: the census, the closing of admission, the drain and the +// completion record a stop waits for are one flow. +const probe = vi.hoisted(() => ({ + /** Holds daemon startup after its servers listen, until released. */ + startupGate: undefined as Promise | undefined, + store: undefined as import('../../core/session-store.ts').SessionStore | undefined, + registry: undefined as import('../../resources/lease-registry.ts').LeaseRegistry | undefined, + held: undefined as Promise | undefined, + entered: undefined as (() => void) | undefined, +})); + +vi.mock('@agent-device/daemon-contracts/device-claims', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + reconcileOrphanedDeviceClaims: async ( + ...args: Parameters + ) => { + await probe.startupGate; + return await actual.reconcileOrphanedDeviceClaims(...args); + }, + }; +}); +vi.mock('../../core/session-store.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + SessionStore: class extends actual.SessionStore { + constructor(...args: ConstructorParameters) { + super(...args); + probe.store = this; + } + }, + }; +}); +vi.mock('../../resources/lease-registry.ts', async (importOriginal) => { + const actual = await importOriginal(); + class RecordedLeaseRegistry extends actual.LeaseRegistry { + constructor(...args: ConstructorParameters) { + super(...args); + probe.registry = this; + } + } + return { ...actual, LeaseRegistry: RecordedLeaseRegistry }; +}); +vi.mock('../request-router.ts', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createRequestHandler: (...args: Parameters) => { + const dispatch = actual.createRequestHandler(...args); + return async (request: Parameters[0]) => { + if (probe.held) { + probe.entered?.(); + await probe.held; + } + return await dispatch(request); + }; + }, + }; +}); + +import { + applicationLifecycleResourceFixture, + daemonRuntimeDepsFixture, +} from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; +import { lifecycleDeviceRuntimeGateway } from '../../__tests__/support/test-device-runtime-gateway.ts'; +import { makeRepairArmedSession, makeSession } from '../../__tests__/support/session-factories.ts'; +import { startDaemonRuntime } from './daemon-runtime.ts'; + +afterEach(() => { + probe.startupGate = undefined; + probe.held = undefined; + probe.entered = undefined; + probe.store = undefined; + probe.registry = undefined; +}); + +type Booted = { + runtime: NonNullable>>; + stateDir: string; + exit: Promise; + quiesce(overrides?: LifecycleAddress): Promise; + shutdown(overrides?: LifecycleAddress): Promise; + sessionList(): Promise; +}; + +type LifecycleAddress = { + token?: string; + pid?: number; + processStartTime?: string | null; + requestId?: string; + /** Sends the request with no `meta` at all, the shape a caller that tracks no id produces. */ + omitMeta?: boolean; +}; + +async function bootDaemon( + options: { + serverMode?: 'socket' | 'dual'; + ownership?: DaemonRuntimeOwnership; + idleTimeoutMs?: number; + requestDrainMs?: number; + } = {}, +): Promise { + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-'); + const paths = resolveDaemonPaths(stateDir); + const runtime = await startDaemonRuntime({ + composeDeps: daemonRuntimeDepsFixture(), + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: options.serverMode ?? 'socket', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: String(options.idleTimeoutMs ?? 0), + }, + }), + ownership: options.ownership, + shutdownBudgets: + options.requestDrainMs === undefined ? undefined : { requestDrainMs: options.requestDrainMs }, + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + if (!runtime?.socketPort) throw new Error('daemon did not start'); + const exit = runtime.closed.then(() => 0); + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + pid: number; + processStartTime?: string; + }; + const socketPort = runtime.socketPort; + const send = (request: object) => sendSocketRequest(socketPort, request); + const sendLifecycle = (mode: 'quiesce' | 'shutdown', overrides: LifecycleAddress) => { + const request = buildDaemonLifecycleRequest({ + mode, + token: overrides.token ?? runtime.token, + fence: { + pid: overrides.pid ?? info.pid, + processStartTime: + overrides.processStartTime !== undefined + ? overrides.processStartTime + : (info.processStartTime ?? null), + }, + requestId: overrides.requestId ?? `${mode}-test`, + }); + return overrides.omitMeta ? send({ ...request, meta: undefined }) : send(request); + }; + return { + runtime, + stateDir, + exit, + quiesce: (overrides = {}) => sendLifecycle('quiesce', overrides), + shutdown: (overrides = {}) => sendLifecycle('shutdown', overrides), + sessionList: () => + send({ + token: runtime.token, + session: 'default', + command: 'session_list', + positionals: [], + meta: { requestId: 'session-list-test' }, + }), + }; +} + +async function sendSocketRequest(port: number, request: object): Promise { + return await new Promise((resolve, reject) => { + const socket = net.createConnection({ host: '127.0.0.1', port }); + let buffer = ''; + socket.setEncoding('utf8'); + socket.on('connect', () => socket.write(`${JSON.stringify(request)}\n`)); + socket.on('data', (chunk) => { + buffer += chunk; + if (!buffer.includes('\n')) return; + socket.end(); + resolve(JSON.parse(buffer) as DaemonResponse); + }); + socket.on('error', reject); + }); +} + +function reasonOf(response: DaemonResponse): unknown { + return response.ok ? undefined : response.error.details?.reason; +} + +test('an idle daemon accepts a quiesce, drains and releases its state directory', async () => { + const daemon = await bootDaemon(); + const paths = resolveDaemonPaths(daemon.stateDir); + const published = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + capabilities?: unknown; + }; + expect(published.capabilities).toEqual(['quiesce']); + + expect(await daemon.quiesce()).toEqual({ ok: true, data: { outcome: 'draining' } }); + + expect(await daemon.exit).toBe(0); + expect(fs.existsSync(paths.infoPath)).toBe(false); + expect(fs.existsSync(paths.lockPath)).toBe(false); +}); + +test('a daemon with a request in flight refuses to quiesce and keeps serving', async () => { + const daemon = await bootDaemon(); + let release: () => void = () => {}; + const entered = new Promise((resolve) => { + probe.entered = resolve; + }); + probe.held = new Promise((resolve) => { + release = resolve; + }); + const running = daemon.sessionList(); + await entered; + + try { + const refusal = await daemon.quiesce(); + expect(refusal.ok).toBe(false); + expect(reasonOf(refusal)).toBe('daemon-busy'); + expect(!refusal.ok && refusal.error.details).toMatchObject({ inFlightRequests: 1 }); + + release(); + expect((await running).ok).toBe(true); + // Admission reopened with the refusal: the daemon still serves and can still be quiesced once idle. + probe.held = undefined; + expect((await daemon.sessionList()).ok).toBe(true); + expect((await daemon.quiesce()).ok).toBe(true); + await daemon.exit; + } finally { + release(); + await running.catch(() => {}); + await daemon.runtime.close(); + } +}); + +test('a daemon holding a session, a repair session or a live lease refuses to quiesce', async () => { + const daemon = await bootDaemon(); + try { + const openRef = probe.store!.publish('open-session', makeSession('open-session')); + const withSession = await daemon.quiesce(); + expect(reasonOf(withSession)).toBe('daemon-busy'); + expect(!withSession.ok && withSession.error.details).toMatchObject({ + sessions: 1, + repairSessions: 0, + }); + probe.store!.retire(openRef); + + const repairRef = probe.store!.publish('repairing', makeRepairArmedSession('repairing')); + const withRepair = await daemon.quiesce(); + expect(!withRepair.ok && withRepair.error.details).toMatchObject({ + sessions: 0, + repairSessions: 1, + }); + probe.store!.retire(repairRef); + + const lease = probe.registry!.allocateLease({ + tenantId: 'tenant-a', + runId: 'run-a', + ttlMs: 60_000, + }); + const withLease = await daemon.quiesce(); + expect(!withLease.ok && withLease.error.details).toMatchObject({ leases: 1 }); + probe.registry!.releaseLease({ leaseId: lease.leaseId }); + + expect((await daemon.quiesce()).ok).toBe(true); + expect(await daemon.exit).toBe(0); + } finally { + await daemon.runtime.close(); + } +}); + +test('a quiesce with the wrong token or for another daemon instance leaves the daemon running', async () => { + const daemon = await bootDaemon(); + try { + const wrongToken = await daemon.quiesce({ token: 'not-the-token' }); + expect(!wrongToken.ok && wrongToken.error.code).toBe('UNAUTHORIZED'); + + const otherInstance = await daemon.quiesce({ pid: readCurrentOwnerIdentity().pid + 1 }); + expect(reasonOf(otherInstance)).toBe('daemon-instance-mismatch'); + // A recycled pid is a different instance: the start time is part of the fence. + const recycledPid = await daemon.quiesce({ processStartTime: 'a different start time' }); + expect(reasonOf(recycledPid)).toBe('daemon-instance-mismatch'); + + expect((await daemon.sessionList()).ok).toBe(true); + } finally { + await daemon.runtime.close(); + } +}); + +async function quiesceOverHttp( + daemon: Booted, + headers: Record, +): Promise<{ result?: DaemonResponse; error?: { data?: { code?: string } } }> { + const { httpPort, token } = daemon.runtime; + const info = JSON.parse( + fs.readFileSync(resolveDaemonPaths(daemon.stateDir).infoPath, 'utf8'), + ) as { pid: number; processStartTime?: string }; + const response = await fetch(`http://127.0.0.1:${httpPort}/rpc`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + authorization: `Bearer ${token}`, + ...headers, + }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 'http-quiesce', + method: 'agent_device.command', + params: buildDaemonLifecycleRequest({ + mode: 'quiesce', + token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId: 'http-quiesce', + }), + }), + }); + return (await response.json()) as never; +} + +test('a quiesce that arrives as a remote-origin HTTP request is refused', async () => { + const daemon = await bootDaemon({ serverMode: 'dual' }); + try { + const body = await quiesceOverHttp(daemon, { 'x-agent-device-network-access': 'public-only' }); + expect(JSON.stringify(body)).toContain('"UNAUTHORIZED"'); + expect((await daemon.sessionList()).ok).toBe(true); + } finally { + await daemon.runtime.close(); + } +}); + +test('a local HTTP quiesce drains an idle daemon like a socket one', async () => { + const daemon = await bootDaemon({ serverMode: 'dual' }); + const body = await quiesceOverHttp(daemon, {}); + expect(body.result).toEqual({ ok: true, data: { outcome: 'draining' } }); + expect(await daemon.exit).toBe(0); +}); + +test('an embedded runtime records what its shutdown answered, and a stop that never asked reads nothing', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + const paths = resolveDaemonPaths(daemon.stateDir); + + expect(await daemon.shutdown({ requestId: 'embedded-stop-1' })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await daemon.exit).toBe(0); + + // The registration is gone by the time a close has a report, so this record is the only answer a + // stop of an embedded runtime can read. + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect(completion?.outcome).toBe('released'); + expect(completion?.unfinished).toEqual([]); + // The record names the instance that wrote it, so every stop addressed to that instance — including + // a second one that arrived after the first was answered — reads this same answer. + expect(completion?.pid).toBe(process.pid); + const stopped = { + pid: completion!.pid, + processStartTime: completion!.processStartTime, + }; + expect(daemonShutdownCompletionAnswers(completion!, stopped)).toBe(true); + // A different instance is not answered by this record, so it cannot be told somebody else's shutdown. + expect(daemonShutdownCompletionAnswers(completion!, { ...stopped, pid: 999_999 })).toBe(false); + expect(fs.existsSync(paths.infoPath)).toBe(false); +}); + +test('a shutdown carrying no request id is accepted and still writes a record a stop can read', async () => { + // The answer is fenced to the instance, so nothing about the close depends on the caller having an + // id: a host that tracks none gets the same completion record as one that does. + const daemon = await bootDaemon({ ownership: 'embedded' }); + const paths = resolveDaemonPaths(daemon.stateDir); + + expect(await daemon.shutdown({ omitMeta: true })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await daemon.exit).toBe(0); + + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect(completion).toMatchObject({ pid: process.pid, outcome: 'released', unfinished: [] }); + expect( + daemonShutdownCompletionAnswers(completion!, { + pid: completion!.pid, + processStartTime: completion!.processStartTime, + }), + ).toBe(true); +}); + +test('a host that closes its own runtime with nothing in hand leaves no stop to answer', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + await daemon.runtime.close(); + // The close's own `finally` offers a failure line for the case where a step rejected. A close that + // reached its end already answered, so that offer must not overwrite the real verdict: this file's + // absence is what says one stop got exactly one answer. + expect(readDaemonShutdownCompletion(resolveDaemonPaths(daemon.stateDir).baseDir)).toBe(null); +}); + +test('a host that closes its own runtime with work in hand still records what it kept', async () => { + // Nobody asked this runtime to leave, so the ledger owes no answer. The state dir is the point: the + // close keeps its registration, and a later `daemon stop` will read that record and ask. Without this + // record it waits on a listener that is already closed and reports an unconfirmed failure at a runtime + // that stopped serving. + const daemon = await bootDaemon({ ownership: 'embedded', requestDrainMs: 50 }); + const paths = resolveDaemonPaths(daemon.stateDir); + let release: () => void = () => {}; + const entered = new Promise((resolve) => { + probe.entered = resolve; + }); + probe.held = new Promise((resolve) => { + release = resolve; + }); + daemon.sessionList().catch(() => undefined); + await entered; + + try { + await daemon.runtime.close(); + + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect(completion).toMatchObject({ pid: process.pid, outcome: 'retained' }); + expect(completion?.unfinished.join('\n')).toContain('ignored cancellation'); + expect(fs.existsSync(paths.infoPath)).toBe(true); + } finally { + release(); + } +}); + +test('a close whose step rejects still answers the stop that asked for it', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-close-failure-'); + const paths = resolveDaemonPaths(stateDir); + const runtime = await startDaemonRuntime({ + composeDeps: daemonRuntimeDepsFixture(() => ({ + deviceRuntimeGateway: { + ...lifecycleDeviceRuntimeGateway, + applicationLifecycle: applicationLifecycleResourceFixture(), + shutdown: async () => { + throw new Error('gateway teardown rejected'); + }, + }, + })), + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'socket', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }), + ownership: 'embedded', + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + port: number; + token: string; + pid: number; + processStartTime?: string; + }; + const request = buildDaemonLifecycleRequest({ + mode: 'shutdown', + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId: 'failing-stop', + }); + + expect(await sendSocketRequest(info.port, request)).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + await expect(runtime.closed).rejects.toThrow('gateway teardown rejected'); + + // A stop would otherwise sit out its whole budget on a close that had already failed, so the record + // says what happened instead of saying nothing. + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect(completion?.pid).toBe(info.pid); + // A platform owner whose teardown rejected may still be driving devices, so the close keeps the + // registration and lock until this process exits — releasing them would let a second daemon take the + // same devices. The record says `retained` because that is what the state directory shows. + expect(completion?.outcome).toBe('retained'); + expect(completion?.unfinished.join('\n')).toContain('did not finish leaving'); + expect(fs.existsSync(paths.infoPath)).toBe(true); +}); + +test('a shutdown that drains work it could not cancel records what it kept', async () => { + const daemon = await bootDaemon({ ownership: 'embedded', requestDrainMs: 50 }); + const paths = resolveDaemonPaths(daemon.stateDir); + let release: () => void = () => {}; + const entered = new Promise((resolve) => { + probe.entered = resolve; + }); + // One request that answers nothing. Its own connection is still open when the listeners go away, so + // nothing about it is knowable from the client side: that is work a drain has to give up on, which is + // the one outcome a stop must be told about. + probe.held = new Promise((resolve) => { + release = resolve; + }); + daemon.sessionList().catch(() => undefined); + await entered; + + try { + expect(await daemon.shutdown({ requestId: 'draining-stop' })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + await daemon.runtime.close(); + + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect(completion?.pid).toBe(process.pid); + expect(completion?.outcome).toBe('retained'); + expect(completion?.unfinished.join('\n')).toContain('ignored cancellation'); + // The abandoned work holds the registration: releasing it would let a second daemon drive the same + // devices while this process may still be running one. + expect(fs.existsSync(paths.infoPath)).toBe(true); + } finally { + release(); + } +}); + +test('a detached daemon refuses a shutdown request and keeps serving', async () => { + const daemon = await bootDaemon(); + expect(reasonOf(await daemon.shutdown())).toBe('daemon-not-embedded'); + expect((await daemon.sessionList()).ok).toBe(true); + expect(readDaemonShutdownCompletion(resolveDaemonPaths(daemon.stateDir).baseDir)).toBe(null); + await daemon.runtime.close(); +}); + +test('an embedded runtime outlives the idle window a detached daemon reaps itself on', async () => { + // Booted first on purpose: both get the same window, so the embedded one's window ends no later than + // the detached one's. Booting it second would let this assertion run while its own window still had + // time left, which would prove nothing about the rule. + const embedded = await bootDaemon({ ownership: 'embedded', idleTimeoutMs: 40 }); + const detached = await bootDaemon({ idleTimeoutMs: 40 }); + const detachedPaths = resolveDaemonPaths(detached.stateDir); + try { + // Idle reap releases the state directory it reaped, so that is the fact to wait for. By then the + // embedded runtime has been idle at least as long, so a window that ended one daemon's life is + // proof enough that it did not end the other's. + await vi.waitFor(() => expect(fs.existsSync(detachedPaths.infoPath)).toBe(false), { + timeout: 5_000, + }); + // A gap between requests is not the owner asking to be shut down. + expect((await embedded.sessionList()).ok).toBe(true); + } finally { + await detached.runtime.close(); + await embedded.runtime.close(); + } +}); + +test('an embedded runtime refuses a quiesce, because its host decides when it leaves', async () => { + const daemon = await bootDaemon({ ownership: 'embedded' }); + expect(reasonOf(await daemon.quiesce())).toBe('daemon-owner-embedded'); + // Nothing was closed and no admission was left closed: the host's runtime still serves. + expect((await daemon.sessionList()).ok).toBe(true); + expect(await daemon.shutdown({ requestId: 'asked-to-leave' })).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await daemon.exit).toBe(0); +}); + +test('a shutdown that arrives while the runtime is starting is refused, then accepted once it can drain', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-shutdown-startup-'); + const paths = resolveDaemonPaths(stateDir); + let openGate: () => void = () => {}; + probe.startupGate = new Promise((resolve) => { + openGate = resolve; + }); + const starting = startDaemonRuntime({ + composeDeps: daemonRuntimeDepsFixture(), + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'socket', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }), + ownership: 'embedded', + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + try { + for (let polls = 0; !fs.existsSync(paths.infoPath); polls += 1) { + if (polls >= 500) throw new Error('daemon did not publish daemon.json during startup'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + port: number; + token: string; + pid: number; + processStartTime?: string; + ownership?: string; + }; + // Published while it still had no drain path, and recorded as owned by a host. + expect(info.ownership).toBe('embedded'); + const shutdownRequest = buildDaemonLifecycleRequest({ + mode: 'shutdown', + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId: 'startup-shutdown', + }); + const refused = await sendSocketRequest(info.port, shutdownRequest); + expect(reasonOf(refused)).toBe('daemon-busy'); + + openGate(); + const runtime = await starting; + const exit = runtime.closed.then(() => 0); + expect(await sendSocketRequest(info.port, shutdownRequest)).toEqual({ + ok: true, + data: { outcome: 'draining' }, + }); + expect(await exit).toBe(0); + // The record answers the instance the stop was fenced to, which is the one the record named before + // the close removed it. + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect( + completion && + daemonShutdownCompletionAnswers(completion, { + pid: info.pid, + processStartTime: info.processStartTime ?? '', + }), + ).toBe(true); + } finally { + openGate(); + await starting.then((runtime) => runtime?.close()); + } +}); + +test('a quiesce asked while the daemon is still starting up is refused as busy, then accepted', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-startup-'); + const paths = resolveDaemonPaths(stateDir); + let openGate: () => void = () => {}; + probe.startupGate = new Promise((resolve) => { + openGate = resolve; + }); + const starting = startDaemonRuntime({ + composeDeps: daemonRuntimeDepsFixture(), + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_SERVER_MODE: 'socket', + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + }, + }), + stderr: { write: () => {} }, + stdout: { write: () => {} }, + }); + try { + // Published, listening, and still held inside startup. + for (let polls = 0; !fs.existsSync(paths.infoPath); polls += 1) { + if (polls >= 500) throw new Error('daemon did not publish daemon.json during startup'); + await new Promise((resolve) => setTimeout(resolve, 10)); + } + const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + port: number; + token: string; + pid: number; + processStartTime?: string; + }; + + const quiesceRequest = buildDaemonLifecycleRequest({ + mode: 'quiesce', + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId: 'startup-quiesce', + }); + const response = await sendSocketRequest(info.port, quiesceRequest); + // The daemon is still starting up, so it refuses as busy instead of promising a drain it cannot run yet. + expect(response.ok).toBe(false); + expect(reasonOf(response)).toBe('daemon-busy'); + expect(!response.ok && response.error.details).toMatchObject({ + startingUp: true, + inFlightRequests: 0, + }); + + openGate(); + const runtime = await starting; + const exit = runtime.closed.then(() => 0); + const again = await sendSocketRequest(info.port, quiesceRequest); + expect(again).toEqual({ ok: true, data: { outcome: 'draining' } }); + expect(await exit).toBe(0); + expect(fs.existsSync(paths.infoPath)).toBe(false); + } finally { + openGate(); + await starting.then((runtime) => runtime?.close()); + } +}); diff --git a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts index f77c0fd0e0..a24e1ac01d 100644 --- a/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts +++ b/packages/daemon/src/server/http/daemon-runtime-metadata-ownership.test.ts @@ -254,16 +254,25 @@ test('both exits tear the watch down before they touch daemon.json', () => { // A watch still armed when the removal runs can report a loss for a record its own process just // deleted. The startup-failure half of this is unreachable from a test — everything after // publication needs a real toolchain — so the invariant is read off the source, the same way the - // arming order above is. Startup rollback runs its steps in reverse, so the registration that - // stops the watch comes after the one that finishes the registration; the close path stops first. + // arming order above is. const source = fs.readFileSync(new URL('./daemon-runtime.ts', import.meta.url), 'utf8'); const stops = [...source.matchAll(/stopMetadataLossWatch\(\);/g)].map((m) => m.index); - const finishes = [...source.matchAll(/await finishDaemonRegistration\(/g)].map((m) => m.index); + // Startup rollback runs its steps in reverse, so the step that stops the watch must be REGISTERED + // after the one that removes the record for it to run first. + const rollbackRemoval = source.indexOf( + 'await finishDaemonRegistration({ registration, infoPath, logPath });', + ); + // The close path: the watch is down before the release, which is the only step that removes the + // record on that exit, can run. + const closeStart = source.indexOf('const closeSteps = async'); + const closeStop = source.indexOf('stopMetadataLossWatch();', closeStart); + const closeRelease = source.indexOf('shutdownHolds.release({', closeStop); + expect(stops).toHaveLength(2); - expect(finishes).toHaveLength(2); - expect(stops[0]).toBeGreaterThan(finishes[0]!); - expect(stops[1]).toBeGreaterThan(stops[0]!); - expect(finishes[1]).toBeGreaterThan(stops[1]!); + expect(rollbackRemoval).toBeGreaterThan(0); + expect(stops[0]).toBeGreaterThan(rollbackRemoval); + expect(closeStop).toBeGreaterThan(0); + expect(closeRelease).toBeGreaterThan(closeStop); }); test('a plugin whose cleanup failed at shutdown is recorded in daemon.log', async () => { diff --git a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts b/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts deleted file mode 100644 index 110b06d9c5..0000000000 --- a/packages/daemon/src/server/http/daemon-runtime-quiesce.test.ts +++ /dev/null @@ -1,378 +0,0 @@ -import fs from 'node:fs'; -import net from 'node:net'; -import { afterEach, expect, test, vi } from 'vitest'; -import { createTestHostEnvironment, mkdtempForTestSync } from '@agent-device/test-kit'; -import { - buildDaemonQuiesceRequest, - resolveDaemonPaths, -} from '@agent-device/daemon-contracts/daemon-resolution'; -import { readCurrentOwnerIdentity } from '@agent-device/host-kit/process'; -import type { DaemonResponse } from '@agent-device/kernel/contracts'; - -// A quiesce is the daemon's own answer to "may I leave?", so these tests boot the real runtime and -// ask over its real socket: the census, the closing of admission and the drain are one flow. -const probe = vi.hoisted(() => ({ - /** Holds daemon startup after its servers listen, until released. */ - startupGate: undefined as Promise | undefined, - store: undefined as import('../../core/session-store.ts').SessionStore | undefined, - registry: undefined as import('../../resources/lease-registry.ts').LeaseRegistry | undefined, - held: undefined as Promise | undefined, - entered: undefined as (() => void) | undefined, -})); - -vi.mock('@agent-device/daemon-contracts/device-claims', async (importOriginal) => { - const actual = - await importOriginal(); - return { - ...actual, - reconcileOrphanedDeviceClaims: async ( - ...args: Parameters - ) => { - await probe.startupGate; - return await actual.reconcileOrphanedDeviceClaims(...args); - }, - }; -}); -vi.mock('../../core/session-store.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - SessionStore: class extends actual.SessionStore { - constructor(...args: ConstructorParameters) { - super(...args); - probe.store = this; - } - }, - }; -}); -vi.mock('../../resources/lease-registry.ts', async (importOriginal) => { - const actual = await importOriginal(); - class RecordedLeaseRegistry extends actual.LeaseRegistry { - constructor(...args: ConstructorParameters) { - super(...args); - probe.registry = this; - } - } - return { ...actual, LeaseRegistry: RecordedLeaseRegistry }; -}); -vi.mock('../request-router.ts', async (importOriginal) => { - const actual = await importOriginal(); - return { - ...actual, - createRequestHandler: (...args: Parameters) => { - const dispatch = actual.createRequestHandler(...args); - return async (request: Parameters[0]) => { - if (probe.held) { - probe.entered?.(); - await probe.held; - } - return await dispatch(request); - }; - }, - }; -}); - -import { daemonRuntimeDepsFixture } from '../../__tests__/support/daemon-runtime-deps-fixture.ts'; -import { makeRepairArmedSession, makeSession } from '../../__tests__/support/session-factories.ts'; -import { startDaemonRuntime } from './daemon-runtime.ts'; - -afterEach(() => { - probe.startupGate = undefined; - probe.held = undefined; - probe.entered = undefined; - probe.store = undefined; - probe.registry = undefined; -}); - -type Booted = { - runtime: NonNullable>>; - stateDir: string; - exit: Promise; - quiesce(overrides?: { - token?: string; - pid?: number; - processStartTime?: string | null; - }): Promise; - sessionList(): Promise; -}; - -async function bootDaemon(options: { serverMode?: 'socket' | 'dual' } = {}): Promise { - const stateDir = mkdtempForTestSync('agent-device-daemon-quiesce-'); - const paths = resolveDaemonPaths(stateDir); - const runtime = await startDaemonRuntime({ - composeDeps: daemonRuntimeDepsFixture(), - host: createTestHostEnvironment({ - env: { - ...process.env, - AGENT_DEVICE_STATE_DIR: stateDir, - AGENT_DEVICE_DAEMON_SERVER_MODE: options.serverMode ?? 'socket', - AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', - }, - }), - stderr: { write: () => {} }, - stdout: { write: () => {} }, - }); - if (!runtime?.socketPort) throw new Error('daemon did not start'); - const exit = runtime.closed.then(() => 0); - const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { - pid: number; - processStartTime?: string; - }; - const socketPort = runtime.socketPort; - const send = (request: object) => sendSocketRequest(socketPort, request); - return { - runtime, - stateDir, - exit, - quiesce: (overrides = {}) => - send( - buildDaemonQuiesceRequest({ - token: overrides.token ?? runtime.token, - fence: { - pid: overrides.pid ?? info.pid, - processStartTime: - overrides.processStartTime !== undefined - ? overrides.processStartTime - : (info.processStartTime ?? null), - }, - requestId: 'quiesce-test', - }), - ), - sessionList: () => - send({ - token: runtime.token, - session: 'default', - command: 'session_list', - positionals: [], - meta: { requestId: 'session-list-test' }, - }), - }; -} - -async function sendSocketRequest(port: number, request: object): Promise { - return await new Promise((resolve, reject) => { - const socket = net.createConnection({ host: '127.0.0.1', port }); - let buffer = ''; - socket.setEncoding('utf8'); - socket.on('connect', () => socket.write(`${JSON.stringify(request)}\n`)); - socket.on('data', (chunk) => { - buffer += chunk; - if (!buffer.includes('\n')) return; - socket.end(); - resolve(JSON.parse(buffer) as DaemonResponse); - }); - socket.on('error', reject); - }); -} - -function reasonOf(response: DaemonResponse): unknown { - return response.ok ? undefined : response.error.details?.reason; -} - -test('an idle daemon accepts a quiesce, drains and releases its state directory', async () => { - const daemon = await bootDaemon(); - const paths = resolveDaemonPaths(daemon.stateDir); - const published = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { - capabilities?: unknown; - }; - expect(published.capabilities).toEqual(['quiesce']); - - expect(await daemon.quiesce()).toEqual({ ok: true, data: { outcome: 'draining' } }); - - expect(await daemon.exit).toBe(0); - expect(fs.existsSync(paths.infoPath)).toBe(false); - expect(fs.existsSync(paths.lockPath)).toBe(false); -}); - -test('a daemon with a request in flight refuses to quiesce and keeps serving', async () => { - const daemon = await bootDaemon(); - let release: () => void = () => {}; - const entered = new Promise((resolve) => { - probe.entered = resolve; - }); - probe.held = new Promise((resolve) => { - release = resolve; - }); - const running = daemon.sessionList(); - await entered; - - try { - const refusal = await daemon.quiesce(); - expect(refusal.ok).toBe(false); - expect(reasonOf(refusal)).toBe('daemon-busy'); - expect(!refusal.ok && refusal.error.details).toMatchObject({ inFlightRequests: 1 }); - - release(); - expect((await running).ok).toBe(true); - // Admission reopened with the refusal: the daemon still serves and can still be quiesced once idle. - probe.held = undefined; - expect((await daemon.sessionList()).ok).toBe(true); - expect((await daemon.quiesce()).ok).toBe(true); - await daemon.exit; - } finally { - release(); - await running.catch(() => {}); - await daemon.runtime.close(); - } -}); - -test('a daemon holding a session, a repair session or a live lease refuses to quiesce', async () => { - const daemon = await bootDaemon(); - try { - const openRef = probe.store!.publish('open-session', makeSession('open-session')); - const withSession = await daemon.quiesce(); - expect(reasonOf(withSession)).toBe('daemon-busy'); - expect(!withSession.ok && withSession.error.details).toMatchObject({ - sessions: 1, - repairSessions: 0, - }); - probe.store!.retire(openRef); - - const repairRef = probe.store!.publish('repairing', makeRepairArmedSession('repairing')); - const withRepair = await daemon.quiesce(); - expect(!withRepair.ok && withRepair.error.details).toMatchObject({ - sessions: 0, - repairSessions: 1, - }); - probe.store!.retire(repairRef); - - const lease = probe.registry!.allocateLease({ - tenantId: 'tenant-a', - runId: 'run-a', - ttlMs: 60_000, - }); - const withLease = await daemon.quiesce(); - expect(!withLease.ok && withLease.error.details).toMatchObject({ leases: 1 }); - probe.registry!.releaseLease({ leaseId: lease.leaseId }); - - expect((await daemon.quiesce()).ok).toBe(true); - expect(await daemon.exit).toBe(0); - } finally { - await daemon.runtime.close(); - } -}); - -test('a quiesce with the wrong token or for another daemon instance leaves the daemon running', async () => { - const daemon = await bootDaemon(); - try { - const wrongToken = await daemon.quiesce({ token: 'not-the-token' }); - expect(!wrongToken.ok && wrongToken.error.code).toBe('UNAUTHORIZED'); - - const otherInstance = await daemon.quiesce({ pid: readCurrentOwnerIdentity().pid + 1 }); - expect(reasonOf(otherInstance)).toBe('daemon-instance-mismatch'); - // A recycled pid is a different instance: the start time is part of the fence. - const recycledPid = await daemon.quiesce({ processStartTime: 'a different start time' }); - expect(reasonOf(recycledPid)).toBe('daemon-instance-mismatch'); - - expect((await daemon.sessionList()).ok).toBe(true); - } finally { - await daemon.runtime.close(); - } -}); - -async function quiesceOverHttp( - daemon: Booted, - headers: Record, -): Promise<{ result?: DaemonResponse; error?: { data?: { code?: string } } }> { - const { httpPort, token } = daemon.runtime; - const info = JSON.parse( - fs.readFileSync(resolveDaemonPaths(daemon.stateDir).infoPath, 'utf8'), - ) as { pid: number; processStartTime?: string }; - const response = await fetch(`http://127.0.0.1:${httpPort}/rpc`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${token}`, - ...headers, - }, - body: JSON.stringify({ - jsonrpc: '2.0', - id: 'http-quiesce', - method: 'agent_device.command', - params: buildDaemonQuiesceRequest({ - token, - fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, - requestId: 'http-quiesce', - }), - }), - }); - return (await response.json()) as never; -} - -test('a quiesce that arrives as a remote-origin HTTP request is refused', async () => { - const daemon = await bootDaemon({ serverMode: 'dual' }); - try { - const body = await quiesceOverHttp(daemon, { 'x-agent-device-network-access': 'public-only' }); - expect(JSON.stringify(body)).toContain('"UNAUTHORIZED"'); - expect((await daemon.sessionList()).ok).toBe(true); - } finally { - await daemon.runtime.close(); - } -}); - -test('a local HTTP quiesce drains an idle daemon like a socket one', async () => { - const daemon = await bootDaemon({ serverMode: 'dual' }); - const body = await quiesceOverHttp(daemon, {}); - expect(body.result).toEqual({ ok: true, data: { outcome: 'draining' } }); - expect(await daemon.exit).toBe(0); -}); - -test('a quiesce asked while the daemon is still starting up is refused as busy, then accepted', async () => { - const stateDir = mkdtempForTestSync('agent-device-daemon-quiesce-startup-'); - const paths = resolveDaemonPaths(stateDir); - let openGate: () => void = () => {}; - probe.startupGate = new Promise((resolve) => { - openGate = resolve; - }); - const starting = startDaemonRuntime({ - composeDeps: daemonRuntimeDepsFixture(), - host: createTestHostEnvironment({ - env: { - ...process.env, - AGENT_DEVICE_STATE_DIR: stateDir, - AGENT_DEVICE_DAEMON_SERVER_MODE: 'socket', - AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', - }, - }), - stderr: { write: () => {} }, - stdout: { write: () => {} }, - }); - try { - // Published, listening, and still held inside startup. - for (let polls = 0; !fs.existsSync(paths.infoPath); polls += 1) { - if (polls >= 500) throw new Error('daemon did not publish daemon.json during startup'); - await new Promise((resolve) => setTimeout(resolve, 10)); - } - const info = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { - port: number; - token: string; - pid: number; - processStartTime?: string; - }; - - const quiesceRequest = buildDaemonQuiesceRequest({ - token: info.token, - fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, - requestId: 'startup-quiesce', - }); - const response = await sendSocketRequest(info.port, quiesceRequest); - // The daemon is still starting up, so it refuses as busy instead of promising a drain it cannot run yet. - expect(response.ok).toBe(false); - expect(reasonOf(response)).toBe('daemon-busy'); - expect(!response.ok && response.error.details).toMatchObject({ - startingUp: true, - inFlightRequests: 0, - }); - - openGate(); - const runtime = await starting; - const exit = runtime.closed.then(() => 0); - const again = await sendSocketRequest(info.port, quiesceRequest); - expect(again).toEqual({ ok: true, data: { outcome: 'draining' } }); - expect(await exit).toBe(0); - expect(fs.existsSync(paths.infoPath)).toBe(false); - } finally { - openGate(); - await starting.then((runtime) => runtime?.close()); - } -}); diff --git a/packages/daemon/src/server/http/daemon-runtime.ts b/packages/daemon/src/server/http/daemon-runtime.ts index aa869d6721..ea023b4064 100644 --- a/packages/daemon/src/server/http/daemon-runtime.ts +++ b/packages/daemon/src/server/http/daemon-runtime.ts @@ -12,7 +12,12 @@ import { DAEMON_CAPABILITIES, resolveDaemonPaths, resolveDaemonServerMode, + type DaemonRuntimeOwnership, } from '@agent-device/daemon-contracts/daemon-resolution'; +import { + createDaemonShutdownAnswerLedger, + type DaemonShutdownAnswerLedger, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { trackDownloadableArtifact } from '../../resources/artifact-tracking.ts'; import { installProviderDeviceAdmission, @@ -47,6 +52,7 @@ import { type WritableOutput, createPluginClose, createStartupRollback, + describeUnfinishedShutdownWork, settleWithinBudget, type ShutdownReport, type StartupRollback, @@ -55,7 +61,7 @@ import type { DaemonInvokeFn } from '../../core/daemon-request.ts'; import type { SessionRef } from '../../core/session-state.ts'; import { createDaemonIdleReap, - createDaemonQuiesce, + createDaemonLifecycleAdmission, daemonShuttingDownResponse, } from '../../sessions/server/daemon-idle-reap.ts'; import { withDaemonDiagnosticsScope } from '@agent-device/daemon-contracts/daemon-diagnostics-scope'; @@ -87,6 +93,7 @@ import { import { tryAcquireDaemonRegistration, DAEMON_STARTUP_EXIT_CODES, + type DaemonRegistrationOwner, } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { recordDaemonStartupMarker, @@ -218,6 +225,13 @@ export type DaemonRuntimeOptions = { composeDeps: ComposeDaemonRuntimeDeps; /** Shutdown time budgets, for hosts and tests that need other than the defaults. */ shutdownBudgets?: { requestDrainMs?: number; pluginMs?: number; ownerMs?: number }; + /** + * Who owns this process. A host that started this runtime in its own process passes `embedded`: the + * record says so, no idle timer ends the runtime, and it answers a shutdown request instead of a + * quiesce. Defaults to a detached daemon. `unknown` is not offered: it is what a reader answers for a + * record it cannot place, and a runtime always knows which of these two it is. + */ + ownership?: DaemonRuntimeOwnership; }; export type DaemonRuntimeController = { @@ -244,6 +258,81 @@ export function daemonStartupExitCode(error: unknown): number { return typeof exitCode === 'number' ? exitCode : 1; } +/** + * Releases the holds a close has on the state directory, and answers the stop it accepted with the + * completion record written under the claim that `finish()` releases — so the record's disposition is + * the removal that call proves, and nothing that reports the close can move past it. + * + * `finishRegistration` is passed in rather than imported: it lives in the module the runtime loads + * function-scoped to stay out of the daemon entry's eager closure (ADR 0019), and this releases + * nothing that module does not already own. + * + * Work that ignored cancellation keeps both holds: releasing them would let a second daemon drive the + * same devices. Such a close publishes a retention record, and does it once — a close reports its work in + * hand the first time, which is the specific truth, rather than once per call. + * + * Only a runtime a host owns publishes these records. Stopping a detached daemon signals its process and + * reads the shutdown report that close leaves, so a completion record would state a fact nobody reads; + * `daemon-shutdown-completion.json` means "a host-owned runtime's close", and this is where that holds. + */ +function createDaemonShutdownRelease(params: { + registration: DaemonRegistrationOwner; + infoPath: string; + logPath: string; + answers: DaemonShutdownAnswerLedger; + finishRegistration: (params: { + registration: DaemonRegistrationOwner; + infoPath: string; + logPath: string; + outcome?: Parameters[0]; + completion?: Parameters[1]; + unfinished?: Parameters[2]; + }) => Promise; + ownership: DaemonRuntimeOwnership; + clearLockConfiguration: () => Promise; +}): { + release(params: { + holdsWorkInHand: boolean; + unfinished: readonly string[]; + outcome?: Parameters[0]; + }): Promise; +} { + // A stop of this runtime reads its state dir, so a close that keeps that dir claimed owes it a record. + const recordsCloses = params.ownership !== 'detached'; + let registrationFinished = false; + let lockConfigurationCleared = false; + let workInHandReported = false; + return { + release: async (releaseParams) => { + if (releaseParams.holdsWorkInHand) { + // A close that keeps the registration leaves the state dir claimed, which is exactly what the + // next stop will find and ask about. So it publishes the record whether or not a stop asked for + // one: the ledger answers a request, and this is a fact about the state dir. Without it, a stop + // of a runtime its host already closed finds no answer and waits on a listener that is gone. + if (workInHandReported) return; + workInHandReported = true; + if (recordsCloses) params.registration.recordShutdownRetention(releaseParams.unfinished); + return; + } + const answer = params.answers.take(releaseParams.unfinished); + if (!registrationFinished) { + registrationFinished = true; + await params.finishRegistration({ + registration: params.registration, + infoPath: params.infoPath, + logPath: params.logPath, + outcome: releaseParams.outcome, + completion: answer, + unfinished: recordsCloses ? releaseParams.unfinished : undefined, + }); + } + if (lockConfigurationCleared) return; + lockConfigurationCleared = true; + await params.clearLockConfiguration(); + }, + }; +} + export async function startDaemonRuntime( options: DaemonRuntimeOptions, ): Promise { @@ -511,6 +600,9 @@ async function startDaemonRuntimeOnHost( // well after this closure captures it. let shuttingDown = false; const inFlightRequests = new Set>(); + // Who owns this process decides what it publishes and which lifecycle request it answers. A host + // that called `agent-device/host` passes `embedded`; the daemon executable leaves it detached. + const ownership = options.ownership ?? 'detached'; const idleReap = createDaemonIdleReap({ sessionStore, getInFlightRequestCount: () => inFlightRequests.size, @@ -519,6 +611,7 @@ async function startDaemonRuntimeOnHost( void close(); }, env, + ownership, }); rollback.onRollback(() => { @@ -527,23 +620,26 @@ async function startDaemonRuntimeOnHost( leaseSessionEnd.cancel(); }); - // The quiesce request is answered before dispatch and never joins `inFlightRequests`, so the - // count it reads needs no allowance for it. The servers listen and `daemon.json` is published - // before `shutdown` exists, and a daemon answers `draining` only once it has a drain path to run. + // Both lifecycle requests are answered before dispatch and never join `inFlightRequests`, so the + // count they read needs no allowance for them. The servers listen and `daemon.json` is published + // before `close` exists, and a daemon answers `draining` only once it has a drain path to run. let startingUp = true; - const quiesce = createDaemonQuiesce({ + const shutdownAnswers = createDaemonShutdownAnswerLedger(); + const lifecycleAdmission = createDaemonLifecycleAdmission({ token, identity: { pid: daemonIdentity.pid, processStartTime: daemonIdentity.startTime }, sessionStore, leaseRegistry, inFlightRequestCount: () => inFlightRequests.size, startingUp: () => startingUp, - drain: () => { + drain: (mode) => { + shutdownAnswers.accept(mode); void close(); }, + ownership: () => ownership, }); - const handleRequest: DaemonInvokeFn = quiesce.admit(async (req) => { + const handleRequest: DaemonInvokeFn = lifecycleAdmission.admit(async (req) => { if (shuttingDown) return daemonShuttingDownResponse(); idleReap.cancel(); const dispatch = runtimeContext.run(() => dispatchRequest(req)); @@ -572,6 +668,7 @@ async function startDaemonRuntimeOnHost( policyDigest: daemonPolicy?.digest, capabilities: DAEMON_CAPABILITIES, plugins: deps.pluginInventory, + ownership, }); if (socketPort) stdout.write(`AGENT_DEVICE_DAEMON_PORT=${socketPort}\n`); if (httpPort) stdout.write(`AGENT_DEVICE_DAEMON_HTTP_PORT=${httpPort}\n`); @@ -737,13 +834,18 @@ async function startDaemonRuntimeOnHost( }).catch(() => {}); }); - let registrationFinished = false; - let lockConfigurationCleared = false; - // Work that ignored cancellation may still be running, so this process keeps the registration - // and lock it holds: releasing them would let a second daemon drive the same devices. let abandonedRequests = 0; let platformOwnersPending = false; const hasUnfinishedWork = () => abandonedRequests > 0 || platformOwnersPending; + const shutdownHolds = createDaemonShutdownRelease({ + registration, + infoPath, + logPath, + answers: shutdownAnswers, + finishRegistration: finishDaemonRegistration, + ownership, + clearLockConfiguration: () => deps.ownerLifecycle.clearDaemonLockConfiguration(), + }); const drainInFlightRequests = async (): Promise => { const total = inFlightRequests.size; if (total === 0) return { drained: 0, abandoned: 0 }; @@ -765,6 +867,10 @@ async function startDaemonRuntimeOnHost( } return { drained: total - abandonedRequests, abandoned: abandonedRequests }; }; + + // A stop of an embedded runtime waits on the completion record, not on a process exit. Its + // disposition belongs to the release below, the only step that can say what became of the + // registration, so the close hands its work-in-hand to that step and never answers on its own. const closeSteps = async (closeOptions: { cause?: unknown }): Promise => { idleReap.cancel(); sessionIdleExpiry.cancel(); @@ -847,19 +953,8 @@ async function startDaemonRuntimeOnHost( releasePngWorker().catch(() => {}), sleep(DAEMON_PNG_WORKER_TERMINATE_TIMEOUT_MS), ]); - if (!hasUnfinishedWork()) { - registrationFinished = true; - await finishDaemonRegistration({ - registration, - infoPath, - logPath, - outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, - }); - lockConfigurationCleared = true; - await deps.ownerLifecycle.clearDaemonLockConfiguration(); - } const unknownLeaseIds = new Set(providerReleaseDrain.unknown.map((lease) => lease.leaseId)); - return { + const report: ShutdownReport = { providerReleases: { completed: providerReleaseDrain.released.map((lease) => lease.leaseId), pending: providerReleaseDrain.pending @@ -873,6 +968,13 @@ async function startDaemonRuntimeOnHost( sessions, plugins, }; + const unfinished = describeUnfinishedShutdownWork(report); + await shutdownHolds.release({ + holdsWorkInHand: hasUnfinishedWork(), + unfinished, + outcome: { providerReleases: providerReleaseDrain, claims: shutdownClaimLedger.claims }, + }); + return report; }; // A step that rejects must not strand what the later steps release: the PNG worker hold, the @@ -884,12 +986,15 @@ async function startDaemonRuntimeOnHost( await Promise.allSettled([ releasePngWorker(), pluginClose.close(), - registrationFinished || hasUnfinishedWork() - ? undefined - : finishDaemonRegistration({ registration, infoPath, logPath }), - lockConfigurationCleared || hasUnfinishedWork() - ? undefined - : deps.ownerLifecycle.clearDaemonLockConfiguration(), + // A close whose step rejected still owes the stop an answer, so the stop learns the close + // failed instead of timing out on a runtime that stopped serving. The release writes that + // answer with the disposition it proves. + shutdownHolds.release({ + holdsWorkInHand: hasUnfinishedWork(), + unfinished: [ + 'Daemon shutdown: a close step failed, so the runtime did not finish leaving.', + ], + }), ]); if (!hasUnfinishedWork()) runtimeContext.dispose(); } diff --git a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts index ba5ccbc2d9..831f89543e 100644 --- a/packages/daemon/src/server/http/daemon-startup-diagnostics.ts +++ b/packages/daemon/src/server/http/daemon-startup-diagnostics.ts @@ -37,9 +37,15 @@ export async function finishDaemonRegistration(params: { infoPath: string; logPath: string; outcome?: Parameters[0]; + completion?: Parameters[1]; + unfinished?: Parameters[2]; }): Promise { try { - const removal = await params.registration.finish(params.outcome); + const removal = await params.registration.finish( + params.outcome, + params.completion, + params.unfinished, + ); if (removal.state !== 'removed' && removal.state !== 'absent') { await emitDaemonDiagnostic(params.logPath, 'daemon_info_removal_declined', { infoPath: params.infoPath, diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap-lifecycle.test.ts b/packages/daemon/src/sessions/server/daemon-idle-reap-lifecycle.test.ts new file mode 100644 index 0000000000..f81acd574c --- /dev/null +++ b/packages/daemon/src/sessions/server/daemon-idle-reap-lifecycle.test.ts @@ -0,0 +1,253 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterEach, beforeEach, test, vi } from 'vitest'; +import { + buildDaemonLifecycleRequest, + DAEMON_LIFECYCLE_COMMANDS, + type DaemonLifecycleMode, + type DaemonRuntimeOwnership, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; +import type { DaemonResponse } from '@agent-device/kernel/contracts'; +import { makeIosSession } from '../../__tests__/support/session-factories.ts'; +import { SessionStore } from '../../core/session-store.ts'; +import type { DaemonRequest } from '../../core/daemon-request.ts'; +import { createDaemonLifecycleAdmission, daemonShuttingDownResponse } from './daemon-idle-reap.ts'; + +// Both lifecycle requests are answered by one seam, so these tests ask that seam and check what it +// did to admission: the two modes share the token check, the instance fence, and the rule that an +// answer and the closing of admission are one step. + +const TOKEN = 'daemon-token'; +const FENCE = { pid: 4242, processStartTime: 'Fri Oct 17 09:00:00 2026' }; + +let stateDir: string; +let sessionStore: SessionStore; + +beforeEach(() => { + stateDir = mkdtempForTestSync('agent-device-daemon-lifecycle-admission-'); + sessionStore = new SessionStore(path.join(stateDir, 'sessions')); +}); + +afterEach(() => { + fs.rmSync(stateDir, { recursive: true, force: true }); +}); + +type Seam = { + admit: (handler: Handler) => (request: DaemonRequest) => Promise; + drained: ReturnType; + served: MockHandler; +}; + +type Handler = (request: DaemonRequest) => Promise; +type MockHandler = Handler & ReturnType; + +function makeSeam( + options: { + ownership?: DaemonRuntimeOwnership; + identity?: { pid: number; processStartTime: string | null }; + startingUp?: boolean; + } = {}, +): Seam { + const drained = vi.fn(); + const admission = createDaemonLifecycleAdmission({ + token: TOKEN, + identity: options.identity ?? FENCE, + sessionStore, + leaseRegistry: { listActiveLeases: () => [] }, + inFlightRequestCount: () => 0, + startingUp: () => options.startingUp ?? false, + drain: drained, + ownership: () => options.ownership ?? 'detached', + }); + return { + admit: admission.admit, + drained, + served: vi.fn(async () => ({ ok: true })) as MockHandler, + }; +} + +function lifecycleRequest( + mode: DaemonLifecycleMode, + overrides: { + token?: string; + fence?: { pid: number; processStartTime: string | null }; + requestId?: string; + } = {}, +): DaemonRequest { + return buildDaemonLifecycleRequest({ + mode, + token: overrides.token ?? TOKEN, + fence: overrides.fence ?? FENCE, + requestId: overrides.requestId ?? `${mode}-test`, + }); +} + +function ordinaryRequest(): DaemonRequest { + return { token: TOKEN, session: 'default', command: 'devices', positionals: [] }; +} + +/** One open session, which is work a stop-if-idle must not walk away from. */ +function openOneSession(): void { + sessionStore.publish('busy-session', makeIosSession('busy-session')); +} + +function reasonOf(response: DaemonResponse): unknown { + return response.ok ? undefined : response.error.details?.reason; +} + +async function settled(): Promise { + await new Promise((resolve) => setImmediate(resolve)); +} + +test('a lifecycle request is answered at the seam and never reaches the handler it guards', async () => { + const { admit, drained, served } = makeSeam(); + assert.deepEqual(await admit(served)(lifecycleRequest('quiesce')), { + ok: true, + data: { outcome: 'draining' }, + }); + await settled(); + // The drain is told which request it answers, and nothing else: a completion record answers by + // instance, so a request id has no use this side of the seam. + assert.deepEqual(drained.mock.calls[0]?.[0], 'quiesce'); + assert.equal(served.mock.calls.length, 0); +}); + +test('a detached daemon refuses a shutdown, and a host-owned runtime refuses a quiesce', async () => { + for (const [ownership, refused, accepted] of [ + ['detached', 'shutdown', 'quiesce'], + ['embedded', 'quiesce', 'shutdown'], + ] as const) { + const { admit, drained, served } = makeSeam({ ownership }); + assert.equal( + reasonOf(await admit(served)(lifecycleRequest(refused))), + ownership === 'detached' ? 'daemon-not-embedded' : 'daemon-owner-embedded', + ownership, + ); + await settled(); + assert.equal(drained.mock.calls.length, 0, `${ownership}: a refusal never drained`); + // A refusal is not a retirement: the daemon keeps serving, and the other mode still works. + assert.deepEqual(await admit(served)(lifecycleRequest(accepted)), { + ok: true, + data: { outcome: 'draining' }, + }); + } +}); + +test('a host-owned runtime drains work in hand, where a detached daemon refuses it', async () => { + openOneSession(); + + const held = makeSeam({ ownership: 'detached' }); + const refusal = await held.admit(held.served)(lifecycleRequest('quiesce')); + assert.equal(reasonOf(refusal), 'daemon-busy'); + if (!refusal.ok) assert.equal(refusal.error.details?.sessions, 1); + await settled(); + assert.equal(held.drained.mock.calls.length, 0); + // The refusal reopened admission, so this daemon is still serving its sessions. + assert.deepEqual(await held.admit(held.served)(ordinaryRequest()), { ok: true }); + + const draining = makeSeam({ ownership: 'embedded' }); + assert.deepEqual(await draining.admit(draining.served)(lifecycleRequest('shutdown')), { + ok: true, + data: { outcome: 'draining' }, + }); + await settled(); + assert.equal(draining.drained.mock.calls.length, 1); +}); + +test('a runtime still starting accepts neither mode and keeps serving while it starts', async () => { + for (const mode of ['quiesce', 'shutdown'] as const) { + const { admit, drained, served } = makeSeam({ + ownership: mode === 'shutdown' ? 'embedded' : 'detached', + startingUp: true, + }); + const refusal = await admit(served)(lifecycleRequest(mode)); + assert.equal(reasonOf(refusal), 'daemon-busy', mode); + if (!refusal.ok) assert.equal(refusal.error.details?.startingUp, true, mode); + await settled(); + assert.equal(drained.mock.calls.length, 0, mode); + assert.deepEqual( + await admit(served)(ordinaryRequest()), + { ok: true }, + `${mode}: a refusal keeps admission open`, + ); + } +}); + +test('both modes refuse a wrong token, a remote origin, and any other instance', async () => { + for (const mode of ['quiesce', 'shutdown'] as const) { + const ownership = mode === 'shutdown' ? 'embedded' : 'detached'; + + const seam = makeSeam({ ownership }); + const wrongToken = await seam.admit(seam.served)(lifecycleRequest(mode, { token: 'other' })); + assert.equal(!wrongToken.ok && wrongToken.error.code, 'UNAUTHORIZED', mode); + assert.equal(seam.drained.mock.calls.length, 0, mode); + + const remote = await seam.admit(seam.served)({ + ...lifecycleRequest(mode), + internal: { publicNetworkOnly: true }, + }); + assert.equal(!remote.ok && remote.error.code, 'UNAUTHORIZED', mode); + + for (const [label, fence] of [ + ['recycled pid', { pid: FENCE.pid + 1, processStartTime: FENCE.processStartTime }], + ['other lifetime', { pid: FENCE.pid, processStartTime: 'Sat Oct 18 09:00:00 2026' }], + ['no recorded birth', { pid: FENCE.pid, processStartTime: null }], + ] as const) { + const addressed = makeSeam({ ownership }); + const response = await addressed.admit(addressed.served)(lifecycleRequest(mode, { fence })); + assert.equal(reasonOf(response), 'daemon-instance-mismatch', `${mode} ${label}`); + await settled(); + assert.equal(addressed.drained.mock.calls.length, 0, `${mode} ${label}`); + } + } + // A runtime that cannot prove its own birth cannot be addressed at all, in either mode: a pid alone + // cannot tell this lifetime from the process that recycled its number, and a drain would end a + // stranger's daemon. Admission stays open, so that daemon is still serving. + for (const mode of ['quiesce', 'shutdown'] as const) { + const unprovable = makeSeam({ + ownership: mode === 'shutdown' ? 'embedded' : 'detached', + identity: { ...FENCE, processStartTime: null }, + }); + const refused = unprovable.served; + assert.equal( + reasonOf(await unprovable.admit(refused)(lifecycleRequest(mode))), + 'daemon-instance-mismatch', + mode, + ); + await settled(); + assert.equal(unprovable.drained.mock.calls.length, 0, `${mode}: a refusal never drained`); + assert.deepEqual( + await unprovable.admit(refused)(ordinaryRequest()), + { ok: true }, + `${mode}: an unaddressable identity still serves its own requests`, + ); + } +}); + +test('a second lifecycle request after one was accepted is answered as draining, not re-drained', async () => { + const { admit, drained } = makeSeam({ ownership: 'embedded' }); + assert.deepEqual(await admit(async () => ({ ok: true }))(lifecycleRequest('shutdown')), { + ok: true, + data: { outcome: 'draining' }, + }); + assert.deepEqual( + await admit(async () => ({ ok: true }))(lifecycleRequest('shutdown', { requestId: 'second' })), + { ok: true, data: { outcome: 'draining' } }, + ); + await settled(); + assert.equal(drained.mock.calls.length, 1); +}); + +test('every request that is not a lifecycle one is refused once a drain was accepted', async () => { + const { admit, served } = makeSeam({ ownership: 'embedded' }); + await admit(served)(lifecycleRequest('shutdown')); + assert.deepEqual(await admit(served)(ordinaryRequest()), daemonShuttingDownResponse()); + assert.equal(served.mock.calls.length, 0); +}); + +test('the two requests ride distinct commands', () => { + assert.equal(DAEMON_LIFECYCLE_COMMANDS.quiesce, 'daemon_quiesce'); + assert.equal(DAEMON_LIFECYCLE_COMMANDS.shutdown, 'daemon_shutdown'); +}); diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts b/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts index e663a012d8..9e51d5e792 100644 --- a/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts +++ b/packages/daemon/src/sessions/server/daemon-idle-reap.test.ts @@ -7,7 +7,6 @@ import { SessionStore } from '../../core/session-store.ts'; import type { SessionState } from '../../core/session-state.ts'; import { createDaemonIdleReap, - createDaemonQuiesce, hasActiveRecording, hasReapBlockingOpenSessions, isDaemonIdle, @@ -53,6 +52,18 @@ test('resolveDaemonIdleReapMs ignores invalid overrides', () => { assert.equal(resolveDaemonIdleReapMs({ AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '-5' }), 5 * 60_000); }); +test('resolveDaemonIdleReapMs gives a host-owned runtime no window', () => { + // The env window reaches a detached daemon only. A host-owned runtime is ended by its host, and a + // build that cannot place a record's owner never reaches this call: a runtime states its own + // ownership when it starts, so "unrecognized" is a reader's verdict about daemon.json, not a state + // the owner of a running daemon can be in. + assert.equal( + resolveDaemonIdleReapMs({ AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '1200' }, 'embedded'), + 0, + ); + assert.equal(resolveDaemonIdleReapMs({}, 'embedded'), 0); +}); + test('hasReapBlockingOpenSessions reflects the session store', () => { assert.equal(hasReapBlockingOpenSessions(sessionStore), false); sessionStore.publish('default', makeSession()); @@ -294,35 +305,24 @@ test('cancel prevents a scheduled reap from firing', async () => { assert.equal(reaped, 0); }); -test('a daemon that cannot prove its start time refuses every quiesce', async () => { - const drain = vi.fn(); - const quiesce = createDaemonQuiesce({ - token: 'daemon-token', - identity: { pid: process.pid, processStartTime: null }, +test("a host that owns the runtime's lifetime never has it reaped on a timer", async () => { + vi.useFakeTimers(); + let reaped = 0; + const idleReap = createDaemonIdleReap({ sessionStore, - leaseRegistry: { listActiveLeases: () => [] }, - inFlightRequestCount: () => 0, - startingUp: () => false, - drain, - }); - const admitted = vi.fn(async () => ({ ok: true as const })); - - const response = await quiesce.admit(admitted)({ - token: 'daemon-token', - session: 'default', - command: 'daemon_quiesce', - positionals: [], - input: { pid: process.pid, processStartTime: null }, + getInFlightRequestCount: () => 0, + hasRetainedLeases: () => false, + onIdleReap: () => { + reaped++; + }, + // The env window a detached daemon would use, overridden by the owner's rule. + env: { AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '40' }, + ownership: 'embedded', }); - assert.equal(response.ok, false); - assert.equal(!response.ok && response.error.details?.reason, 'daemon-instance-mismatch'); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(drain.mock.calls.length, 0); - // Admission stays open: a refusal never leaves the daemon closed. - assert.deepEqual(await quiesce.admit(admitted)(makeRequest('devices')), { ok: true }); -}); + idleReap.noteActivity(); + await vi.advanceTimersByTimeAsync(1_000); -function makeRequest(command: string) { - return { token: 'daemon-token', session: 'default', command, positionals: [] }; -} + assert.equal(reaped, 0); + assert.equal(idleReap.idleMs, 0); +}); diff --git a/packages/daemon/src/sessions/server/daemon-idle-reap.ts b/packages/daemon/src/sessions/server/daemon-idle-reap.ts index c65cdd2fee..fd44b5ee15 100644 --- a/packages/daemon/src/sessions/server/daemon-idle-reap.ts +++ b/packages/daemon/src/sessions/server/daemon-idle-reap.ts @@ -1,37 +1,57 @@ import { - DAEMON_QUIESCE_COMMAND, - DAEMON_QUIESCE_REASONS, + DAEMON_LIFECYCLE_REASONS, DAEMON_SHUTTING_DOWN_REASON, - readDaemonQuiesceFence, - type DaemonQuiesceCensus, - type DaemonQuiesceFence, + readDaemonLifecycleMode, + readDaemonLifecycleFence, + type DaemonLifecycleCensus, + type DaemonLifecycleFence, + type DaemonLifecycleMode, + type DaemonRuntimeOwnership, } from '@agent-device/daemon-contracts/daemon-resolution'; +import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import { timingSafeStringEqual } from '@agent-device/host-kit/transport'; import type { DaemonResponse } from '@agent-device/kernel/contracts'; import { AppError, normalizeError } from '@agent-device/kernel/errors'; import type { DaemonInvokeFn, DaemonRequest } from '../../core/daemon-request.ts'; import type { LeaseRegistry } from '../../resources/lease-registry.ts'; -import { emitDiagnostic } from '@agent-device/host-kit/diagnostics'; import type { SessionStore } from '../../core/session-store.ts'; import type { SessionState } from '../../core/session-state.ts'; import { isUncommittedRepairSession } from '../../core/session-replay-transaction.ts'; -// Bounds the daemon's own lifetime when nothing is using it. Each -// AGENT_DEVICE_STATE_DIR spawns a dedicated daemon that otherwise never exits -// on its own, so short-lived sandboxes (deleted codex/claude worktrees, one-off -// CLI invocations) leave orphaned daemon processes behind indefinitely - -// observed accumulating (10+) in production. A stale daemon left running this -// way also keeps holding its iOS runner lease, which blocks a fresh daemon for -// the same device (see runner-lease.ts stale-lease takeover). +// Whether this daemon may leave now, and when it will. Two things live here because they answer one +// question from one census: +// +// - `createDaemonIdleReap` bounds the daemon's own lifetime when nothing is using it. Each +// AGENT_DEVICE_STATE_DIR spawns a dedicated daemon that otherwise never exits on its own, so +// short-lived sandboxes (deleted codex/claude worktrees, one-off CLI invocations) leave orphaned +// daemon processes behind indefinitely - observed accumulating (10+) in production. A stale daemon +// left running this way also keeps holding its iOS runner lease, which blocks a fresh daemon for the +// same device (see runner-lease.ts stale-lease takeover). // -// The window mirrors the runner idle-stop default. Idle retained runners stop with their daemon: -// their timer and listener observation cannot survive a handoff. Active eligible runners retain -// the graceful-shutdown adoption policy. +// The window mirrors the runner idle-stop default. Idle retained runners stop with their daemon: +// their timer and listener observation cannot survive a handoff. Active eligible runners retain the +// graceful-shutdown adoption policy. +// +// - `createDaemonLifecycleAdmission` answers, in place of dispatch, the two requests that ask this +// daemon to leave: `daemon_quiesce` (stop if idle, for a daemon a client may replace) and +// `daemon_shutdown` (drain, for a runtime an embedder owns). Both decide from the same census the +// idle-reap rule reads, so a stop cannot be granted by one owner and refused by the other. +// +// TODO(#3757): the file is named for the older half. A rename to a lifecycle-named module is a +// follow-up, so this PR does not spend a gate's rename tolerance on it. + const DAEMON_IDLE_REAP_DEFAULT_MS = 5 * 60_000; // AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS overrides the window; 0 disables idle // reap and restores the pre-existing "runs until killed" behavior. -export function resolveDaemonIdleReapMs(env: NodeJS.ProcessEnv = process.env): number { +// A timer exit is something only a detached daemon does: a runtime whose host owns it is ended by +// that host, and one whose owner cannot be proven is treated the same way, because nothing here can +// show the window's owner is nobody. +export function resolveDaemonIdleReapMs( + env: NodeJS.ProcessEnv = process.env, + ownership: DaemonRuntimeOwnership = 'detached', +): number { + if (ownership !== 'detached') return 0; const raw = env.AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS?.trim(); if (raw) { const parsed = Number(raw); @@ -100,8 +120,15 @@ export function createDaemonIdleReap(params: { hasRetainedLeases: () => boolean; onIdleReap: () => void; env?: NodeJS.ProcessEnv; + /** + * Who owns the process this timer runs in. A host that started the runtime in its own process owns + * its lifetime: an idle window tuned for a detached daemon nobody is watching would take the runtime + * out from under that owner, so an embedded runtime is never reaped on a timer. Defaults to a + * detached daemon. + */ + ownership?: DaemonRuntimeOwnership; }): DaemonIdleReapController { - const idleMs = resolveDaemonIdleReapMs(params.env); + const idleMs = resolveDaemonIdleReapMs(params.env, params.ownership); let timer: NodeJS.Timeout | undefined; const isIdleNow = (): boolean => @@ -144,15 +171,25 @@ export function createDaemonIdleReap(params: { } /** - * ADR 0006: a daemon leaves for a newer one only when it holds nothing, and it decides that - * itself. A client that read "idle" and then signaled would race every other client of the state - * directory; the daemon is the one place the answer and the closing of admission can be one step. + * The one seam that answers both ways a process asks this daemon to leave, in place of dispatch. + * + * ADR 0006: a daemon leaves for a newer one only when it holds nothing, and it decides that itself. + * A client that read "idle" and then signaled would race every other client of the state directory; + * the daemon is the one place the answer and the closing of admission can be one step. + * + * `quiesce` is that stop-if-idle verdict, and it is what a detached daemon answers: a host owns an + * embedded runtime and ends it itself, so a replacement asks the host instead of retiring it. + * `shutdown` is the explicit stop of an embedded runtime, where draining is the point: work in hand + * is canceled rather than refused, because the caller owns this process and asked for it to close. + * + * Both modes share the token check, the instance fence, the answer-before-dispatch rule, and the one + * `drain` the runtime owns, so a stop can never reach teardown by a route the host does not control. */ -export type DaemonQuiesce = Readonly<{ +export type DaemonLifecycleAdmission = Readonly<{ /** - * Puts the quiesce request and the closed admission it leaves behind in front of `handler`: a - * quiesce request gets its verdict, and any other request gets the shutting-down refusal once a - * quiesce has closed admission. + * Puts the lifecycle request and the closed admission it leaves behind in front of `handler`: a + * lifecycle request gets its verdict, and any other request gets the shutting-down refusal once a + * lifecycle request has closed admission. */ admit(handler: DaemonInvokeFn): DaemonInvokeFn; }>; @@ -168,22 +205,30 @@ export function daemonShuttingDownResponse(): DaemonResponse { }; } -export function createDaemonQuiesce(params: { +export function createDaemonLifecycleAdmission(params: { token: string; - identity: DaemonQuiesceFence; + identity: DaemonLifecycleFence; sessionStore: Pick; leaseRegistry: Pick; - /** Requests being served, not counting the quiesce request itself. */ + /** Requests being served, not counting the lifecycle request itself. */ inFlightRequestCount: () => number; /** Whether the daemon has yet to build the drain path `drain` runs. */ startingUp: () => boolean; - /** Starts the drain; called at most once, after the response is on its way. */ - drain: () => void; -}): DaemonQuiesce { + /** + * Starts the drain, at most once, after the response is on its way. The mode is all the runtime + * needs: a shutdown answers the stop that asked for it, and a quiesce answers nobody. + */ + drain: (mode: DaemonLifecycleMode) => void; + /** Who owns this process, and so which of the two requests it may answer. */ + ownership: () => DaemonRuntimeOwnership; +}): DaemonLifecycleAdmission { let admitting = true; - const refuse = (error: AppError): DaemonResponse => ({ ok: false, error: normalizeError(error) }); + const refuse = (error: AppError): DaemonResponse => ({ + ok: false, + error: normalizeError(error), + }); - const quiesce = (req: DaemonRequest): DaemonResponse => { + const leave = (req: DaemonRequest, mode: DaemonLifecycleMode): DaemonResponse => { if (typeof req.token !== 'string' || !timingSafeStringEqual(req.token, params.token)) { return refuse(new AppError('UNAUTHORIZED', 'Invalid daemon token.')); } @@ -191,10 +236,10 @@ export function createDaemonQuiesce(params: { // host that owns the state directory may retire the daemon on it. if (req.internal?.publicNetworkOnly) { return refuse( - new AppError('UNAUTHORIZED', 'Daemon quiesce is not available to remote clients.'), + new AppError('UNAUTHORIZED', `Daemon ${mode} is not available to remote clients.`), ); } - const fence = readDaemonQuiesceFence(req.input); + const fence = readDaemonLifecycleFence(req.input); // A pid alone can name a recycled process, so a daemon that cannot prove its start time // cannot be addressed at all. if (!fence || fence.pid !== params.identity.pid) return refuse(instanceMismatch()); @@ -202,25 +247,32 @@ export function createDaemonQuiesce(params: { if (fence.processStartTime !== params.identity.processStartTime) { return refuse(instanceMismatch()); } + // Each mode belongs to one owner: a host ends an embedded runtime it started, and the process that + // started a detached daemon ends it by signalling its pid. Neither asks a stranger. + if (mode === 'shutdown' && params.ownership() !== 'embedded') return refuse(notEmbedded()); + if (mode === 'quiesce' && params.ownership() === 'embedded') return refuse(ownerEmbedded()); // Admission closes and the census is read in the same synchronous step, so no request can be // admitted between the two; a refusal reopens admission in that same step. if (!admitting) return { ok: true, data: { outcome: 'draining' } }; admitting = false; const census = readCensus(); - if (isHeld(census)) { + // A daemon that is still starting has no drain path yet, so it accepts nothing, and a quiesce + // that finds work in hand refuses too: replacing that daemon would end another client's session. + // A shutdown drains instead of refusing — the caller owns this process and asked for it to + // close, canceling its work included — but only once a drain path exists. The census says which + // of the two applied, so a caller knows whether to retry or to ask again later. + const held = mode === 'quiesce' ? isHeld(census) : census.startingUp; + if (held) { admitting = true; - return refuse( - new AppError('COMMAND_FAILED', 'Daemon is busy and was not asked to leave.', { - reason: DAEMON_QUIESCE_REASONS.busy, - ...census, - }), - ); + return refuse(busy(census)); } - setImmediate(params.drain); + setImmediate(() => params.drain(mode)); + // One ack shape for both modes, as quiesce's shipped one already is: the census belongs in a + // refusal that a caller must act on, and in the completion record, not in an agreement. return { ok: true, data: { outcome: 'draining' } }; }; - const readCensus = (): DaemonQuiesceCensus => { + const readCensus = (): DaemonLifecycleCensus => { const sessions = params.sessionStore.toArray(); const repairSessions = sessions.filter(isUncommittedRepairSession).length; return { @@ -234,13 +286,14 @@ export function createDaemonQuiesce(params: { return { admit: (handler) => async (req) => { - if (req.command === DAEMON_QUIESCE_COMMAND) return quiesce(req); + const mode = readDaemonLifecycleMode(req.command); + if (mode) return leave(req, mode); return admitting ? await handler(req) : daemonShuttingDownResponse(); }, }; } -function isHeld(census: DaemonQuiesceCensus): boolean { +function isHeld(census: DaemonLifecycleCensus): boolean { return ( census.startingUp || census.inFlightRequests > 0 || @@ -250,8 +303,34 @@ function isHeld(census: DaemonQuiesceCensus): boolean { ); } +function busy(census: DaemonLifecycleCensus): AppError { + // The refusal's shape is the one quiesce already shipped: the reason plus the census that explains + // it. A shutdown refuses for one of those counts (`startingUp`) and a stop reads that count rather + // than a second field naming the mode it already asked for. + return new AppError('COMMAND_FAILED', 'Daemon is busy and was not asked to leave.', { + reason: DAEMON_LIFECYCLE_REASONS.busy, + ...census, + }); +} + function instanceMismatch(): AppError { - return new AppError('COMMAND_FAILED', 'Quiesce request names a different daemon instance.', { - reason: DAEMON_QUIESCE_REASONS.instanceMismatch, + return new AppError('COMMAND_FAILED', 'A lifecycle request names a different daemon instance.', { + reason: DAEMON_LIFECYCLE_REASONS.instanceMismatch, }); } + +function notEmbedded(): AppError { + return new AppError( + 'COMMAND_FAILED', + 'This daemon is not embedder-owned, so it does not accept a shutdown request.', + { reason: DAEMON_LIFECYCLE_REASONS.notEmbedded }, + ); +} + +function ownerEmbedded(): AppError { + return new AppError( + 'COMMAND_FAILED', + 'A host process owns this daemon, so only that host can retire it.', + { reason: DAEMON_LIFECYCLE_REASONS.ownerEmbedded }, + ); +} diff --git a/src/__tests__/daemon-registration-embedded-owner.test.ts b/src/__tests__/daemon-registration-embedded-owner.test.ts new file mode 100644 index 0000000000..a9d6e369c6 --- /dev/null +++ b/src/__tests__/daemon-registration-embedded-owner.test.ts @@ -0,0 +1,243 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { afterEach, test, vi } from 'vitest'; +import { readCurrentOwnerIdentity, isProcessAlive } from '@agent-device/host-kit/process'; +import { + tryAcquireDaemonRegistration, + stopAndRetireDaemon, +} from '@agent-device/daemon-contracts/daemon-registration-owner'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { + readDaemonShutdownCompletion, + resolveDaemonShutdownCompletionPath, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { mkdtempForTestSync } from '@agent-device/test-kit'; + +// What the registration owner may do to a record a host process owns: prove the lifetime is gone +// before removing anything, and never signal the process the record names. + +/** + * Whether the signal path was reached, and with what. A signal needs the observed pid to pass the + * daemon-command-line check, and the process these rows name is the vitest worker, which fails that + * check. Stubbed to accept it, the rows ask the question they mean to ask: does the record's owner keep + * the stop away from the signal path, or only a process-table lookup that happens to refuse? + */ +const signalPath = vi.hoisted(() => ({ attempted: [] as number[] })); + +vi.mock('@agent-device/daemon-contracts/daemon-process', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + stopDaemonProcess: async ( + observed: Parameters[0], + _options: Parameters[1], + ) => { + // Records the attempt instead of signaling, so the row can tell a refused stop from one that + // signaled its own test worker. An exit that cannot be confirmed is what a real signal that did + // not kill anything answers with. + signalPath.attempted.push(observed.pid); + return { status: 'retained' as const, reason: 'exit-timeout' as const }; + }, + }; +}); + +const fields = { + socketPort: 4210, + token: 'token', + version: '0.0.0-test', + codeOrigin: 'checkout' as const, + codeSignature: 'signature', +}; +const ownIdentity = readCurrentOwnerIdentity(); +const deadIdentity = { pid: 999_999_999, startTime: 'dead-start' }; + +/** What a reader that read `ownership` from the record passes on, which is what retirement judges. */ +function observedAs( + identity: { pid: number; startTime: string | null }, + ownership: 'embedded' | 'unknown' | 'detached', +) { + return { ...identity, ownership }; +} + +afterEach(() => { + vi.restoreAllMocks(); + signalPath.attempted.length = 0; +}); + +/** Writes a record that claims `ownership` for a live process, unreadable variants included. */ +function writeRecord( + paths: ReturnType, + identity: unknown, + ownership: unknown = 'embedded', +) { + const { pid, startTime } = identity as { pid: number; startTime: string }; + fs.writeFileSync( + paths.infoPath, + JSON.stringify({ + pid, + processStartTime: startTime, + token: 'token', + port: 4210, + ...(ownership === 'corrupt' ? {} : { ownership }), + }), + ); + if (ownership === 'corrupt') fs.writeFileSync(paths.infoPath, '{not json'); +} + +function writeEmbeddedRecord(paths: ReturnType, identity: unknown) { + writeRecord(paths, identity); +} + +/** A live acquisition of a state directory, so the caller holds the claim it judges. */ +async function acquire( + paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-embedded-claim-')), +) { + const attempt = await tryAcquireDaemonRegistration(paths); + assert.equal(attempt.status, 'acquired'); + if (attempt.status !== 'acquired') throw new Error('registration refused'); + return { paths, owner: attempt.owner }; +} + +// A host owns an embedded runtime and ends it with its own close, so retiring that record from outside +// would take the state directory out from under a process that is still serving. +test('retirement of an embedded record proves the lifetime is gone and signals nothing', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-embedded-retirement-')); + writeEmbeddedRecord(paths, ownIdentity); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + const signals = vi.spyOn(process, 'kill'); + + for (const mode of ['graceful', 'force'] as const) { + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(ownIdentity, 'embedded'), + mode, + lockTimeoutMs: 0, + }); + assert.equal(result.status, 'retained', JSON.stringify(result)); + if (result.status !== 'retained') assert.fail('an embedded owner was retired'); + assert.equal(result.reason, 'embedded-owner'); + assert.equal(result.termination?.status, 'retained'); + } + + // The row that gives this file its teeth: the stub above accepts this pid for the signal path, so + // only the record's owner stands between this stop and a SIGTERM to the test worker itself. + assert.deepEqual(signalPath.attempted, [], 'an embedded owner is never reached for a signal'); + assert.deepEqual( + signals.mock.calls.filter(([, requested]) => requested !== 0), + [], + 'retirement must not reach into a host process', + ); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before); + assert.equal(isProcessAlive(ownIdentity.pid), true); +}); + +// The control that keeps the row above honest: the same live pid, the same stub, and a record proving +// the process is a daemon of its own, and the signal path is taken. Without this row the embedded +// refusal could still be a process-table lookup refusing the worker rather than the owner's rule. +test('a detached record for the same live lifetime does take the signal path', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-detached-signal-')); + writeRecord(paths, ownIdentity, 'detached'); + + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(ownIdentity, 'detached'), + mode: 'graceful', + lockTimeoutMs: 0, + }); + + assert.deepEqual( + signalPath.attempted, + [ownIdentity.pid], + 'a proven detached daemon is the one record a stop signals', + ); + assert.equal(result.status, 'retained', JSON.stringify(result)); +}); + +test('an embedded record whose lifetime is provably gone retires like any other abandoned one', async () => { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-embedded-dead-')); + writeEmbeddedRecord(paths, deadIdentity); + + // A host that died without closing left the record behind; keeping it would wedge the state directory + // for every later client with nothing left to close. + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(deadIdentity, 'embedded'), + mode: 'graceful', + lockTimeoutMs: 0, + }); + assert.equal(result.status, 'retired', JSON.stringify(result)); + assert.equal(result.removedInfo, true); + assert.equal(fs.existsSync(paths.infoPath), false); +}); + +// A record this build cannot read is not a licence to signal what it names: `detached` is the one +// ownership that authorizes a signal, and an unreadable or unrecognized record proves nothing about +// who owns the process. This is the fail-closed side of the same rule the embedded row above proves. +test('retirement refuses to signal a record whose owner it cannot recognize', async () => { + for (const ownership of ['sideways', 'corrupt'] as const) { + const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-owner-unknown-')); + writeRecord(paths, ownIdentity, ownership); + const before = fs.readFileSync(paths.infoPath, 'utf8'); + const signals = vi.spyOn(process, 'kill'); + + const result = await stopAndRetireDaemon({ + paths, + observed: observedAs(ownIdentity, 'unknown'), + mode: 'graceful', + lockTimeoutMs: 0, + }); + assert.equal(result.status, 'retained', ownership); + if (result.status !== 'retained') + assert.fail(`${ownership}: an unrecognized owner was retired`); + // The reason differs from the embedded one: nothing here proves the pid is a daemon of its own, so + // an operator fixes this by inspecting or updating the record, not by closing a host. + assert.equal(result.reason, 'owner-unproven', ownership); + assert.deepEqual( + signalPath.attempted, + [], + `${ownership}: an unrecognized owner is never reached for a signal`, + ); + assert.deepEqual( + signals.mock.calls.filter(([, requested]) => requested !== 0), + [], + `${ownership}: an unrecognized owner is never signaled`, + ); + assert.equal(fs.readFileSync(paths.infoPath, 'utf8'), before, ownership); + vi.mocked(process.kill).mockClear(); + } +}); + +test('a close answers its stop through the release, and an acquisition clears the answer', async () => { + const { paths, owner } = await acquire(); + owner.publish(fields); + + await owner.finish(undefined, { unfinished: ['1 provider release left pending'] }); + const record = readDaemonShutdownCompletion(paths.baseDir); + // The disposition is the removal this call performed, not the caller's guess about it. + assert.equal(record?.outcome, 'released'); + assert.deepEqual(record?.unfinished, ['1 provider release left pending']); + assert.equal(record?.pid, process.pid, 'the record names the instance a stop is fenced to'); + // This is the production write, so it is where the file mode is a fact rather than a fixture's + // choice: a state-directory file a stranger could read leaks what a daemon said about its own close. + assert.equal( + fs.statSync(resolveDaemonShutdownCompletionPath(paths.baseDir)).mode & 0o777, + 0o600, + 'the completion record is published at 0600', + ); + + // Releasing the claim ends this process's authority to speak about the state directory. + assert.equal( + owner.recordShutdownRetention(['late']), + false, + 'a process that no longer holds the claim writes nothing', + ); + + const next = await acquire(paths); + assert.equal( + readDaemonShutdownCompletion(paths.baseDir), + null, + "a new instance must not inherit an answer to somebody else's stop", + ); + await next.owner.finish(); +}); diff --git a/src/__tests__/daemon-registration-owner.test.ts b/src/__tests__/daemon-registration-owner.test.ts index 8f86229aa3..05f9ca9440 100644 --- a/src/__tests__/daemon-registration-owner.test.ts +++ b/src/__tests__/daemon-registration-owner.test.ts @@ -16,8 +16,12 @@ import { import { resolveDaemonPaths, type DaemonPaths, + type ObservedDaemonIdentity, } from '@agent-device/daemon-contracts/daemon-resolution'; -import { readRegisteredDaemonOwnership } from '@agent-device/daemon-contracts/daemon-registration'; +import { + readDaemonStopIdentity, + readRegisteredDaemonOwnership, +} from '@agent-device/daemon-contracts/daemon-registration'; import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { mkdtempForTestSync } from '@agent-device/test-kit'; import { @@ -44,7 +48,26 @@ const fields = { codeOrigin: 'checkout' as const, codeSignature: 'signature', }; -const ownIdentity = readCurrentOwnerIdentity(); +const ownIdentity = observedOf(readCurrentOwnerIdentity()); + +/** + * Every record this file publishes describes a detached daemon: these rows write the file directly and + * state no owner, which is exactly what a record predating the ownership field states. + */ +function observedOf( + identity: T, +): ObservedDaemonIdentity { + return { ...identity, ownership: 'detached' as const }; +} + +/** The same observation a stop routes on: the record this file published, read once. */ +function readStopObserved(infoPath: string): ObservedDaemonIdentity { + const record = readDaemonStopIdentity(infoPath); + if (!record.identity || record.ownership !== 'detached') { + throw new Error(`the fixture published a ${record.ownership} record`); + } + return observedOf(record.identity); +} const report = { providerReleases: { released: [], pending: [] }, claims: { released: [], orphaned: [], superseded: [], unattributable: [] }, @@ -314,11 +337,11 @@ test('unlink disappearance is settled, other failures remain primary even when r assert.match(fs.readFileSync(second.paths.logPath, 'utf8'), /daemon_registration_release_failed/); }); -const deadIdentity = { pid: 999_999_999, startTime: 'dead-start' }; +const deadIdentity = observedOf({ pid: 999_999_999, startTime: 'dead-start' }); for (const scenario of [ { name: 'missing-start', - observed: { pid: process.pid, startTime: null }, + observed: observedOf({ pid: process.pid, startTime: null }), published: ownIdentity, retired: false, }, @@ -362,13 +385,17 @@ test('abandoned recovery never signals a live owner and absence is inspected und }); }); +// A host owns an embedded runtime and ends it with its own close, so retiring that record from outside +// would take the state directory out from under a process that is still serving. The rows that prove it +// are in daemon-registration-embedded-owner.test.ts, next to the owner rule they exercise. + test('retirement preserves a winner holding the startup lock instead of stopping or unlinking it', async () => { const { paths, owner } = await acquire(); owner.publish({ ...fields, token: 'winning-token' }); const before = fs.readFileSync(paths.infoPath, 'utf8'); const result = await stopAndRetireDaemon({ paths, - observed: { pid: 999_999_999, startTime: 'dead-start' }, + observed: observedOf({ pid: 999_999_999, startTime: 'dead-start' }), mode: 'force', lockTimeoutMs: 0, }); @@ -384,7 +411,7 @@ test('retirement preserves a winner holding the startup lock instead of stopping test('metadata failure remains primary and normalized through a secondary release failure', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-failures-')); - const observed = { pid: 999_999_999, startTime: 'dead-start' }; + const observed = observedOf({ pid: 999_999_999, startTime: 'dead-start' }); replaceInfo(paths, observed.pid, observed.startTime); const primary = Object.assign(new Error('metadata failure'), { code: 'EIO' }); const originalUnlink = fs.unlinkSync; @@ -408,7 +435,7 @@ test('metadata failure remains primary and normalized through a secondary releas test('release failure reports partial retirement rather than success', async () => { const paths = resolveDaemonPaths(mkdtempForTestSync('agent-device-retirement-partial-')); - const observed = { pid: 999_999_999, startTime: 'dead-start' }; + const observed = observedOf({ pid: 999_999_999, startTime: 'dead-start' }); replaceInfo(paths, observed.pid, observed.startTime); const original = fs.rmdirSync; vi.spyOn(fs, 'rmdirSync').mockImplementation((target, options) => { @@ -469,7 +496,7 @@ test('private retirement closes startup admission, joins the actual child and ne assert.equal((await contender.exited).exitCode, DAEMON_STARTUP_EXIT_CODES.busy); const input = { paths, - observed: { pid: launch.pid, startTime: launch.startTime }, + observed: observedOf({ pid: launch.pid, startTime: launch.startTime }), mode: 'graceful' as const, ownedStateDir, }; @@ -515,7 +542,7 @@ test('private retirement retains the directory while an earlier actual startup c await waitForFixtureFile(paths.infoPath); const result = await stopAndRetireDaemon({ paths, - observed: { pid: second.pid, startTime: second.startTime ?? null }, + observed: observedOf({ pid: second.pid, startTime: second.startTime ?? null }), mode: 'graceful', ownedStateDir, startupJoinTimeoutMs: 0, @@ -581,7 +608,7 @@ for (const [failure, reason, expectedSignals] of privateStartupFailures) { }); const pending = stopAndRetireDaemon({ paths, - observed: { pid: launch.pid, startTime: launch.startTime ?? null }, + observed: observedOf({ pid: launch.pid, startTime: launch.startTime ?? null }), mode: 'graceful', ownedStateDir, startupJoinTimeoutMs: 500, @@ -657,7 +684,7 @@ test('private startup completion stays bounded without published birth proof and const signals = vi.spyOn(process, 'kill'); const pending = stopAndRetireDaemon({ paths, - observed: { pid: launch.pid, startTime: null }, + observed: observedOf({ pid: launch.pid, startTime: null }), mode: 'force', ownedStateDir, startupJoinTimeoutMs: 0, @@ -719,7 +746,7 @@ for (const contents of [ await waitForFixtureFile(paths.infoPath); const result = await stopAndRetireDaemon({ paths, - observed: { pid: launch.pid, startTime: launch.startTime ?? null }, + observed: observedOf({ pid: launch.pid, startTime: launch.startTime ?? null }), mode: 'graceful', ownedStateDir, }); @@ -887,7 +914,14 @@ for (const mode of ['graceful', 'forced'] as const) { const child = spawnRegisteredDaemonFixture(paths, fields, undefined); try { await waitForFixtureFile(paths.infoPath); - const result = await stopDaemon({ paths, graceTimeoutMs: 30, killTimeoutMs: 1_000 }); + const result = await stopDaemon({ + paths, + // The row routes the stop the way the CLI does: one read of the record the fixture published, + // and its lifetime and owner handed to the funnel unchanged. + observed: readStopObserved(paths.infoPath), + graceTimeoutMs: 30, + killTimeoutMs: 1_000, + }); assert.equal(result.stopped, true); assert.equal(result.mode, mode); assert.equal(result.cleanupConfidence, mode === 'forced' ? 'unknown' : 'known'); diff --git a/src/cli/commands/__tests__/daemon.test.ts b/src/cli/commands/__tests__/daemon.test.ts index 580714bf22..68bed82c59 100644 --- a/src/cli/commands/__tests__/daemon.test.ts +++ b/src/cli/commands/__tests__/daemon.test.ts @@ -12,7 +12,6 @@ const mocks = vi.hoisted(() => ({ cleanupDaemonOwner: vi.fn(async () => undefined), readDaemonShutdownReport: vi.fn(), readDaemonStatus: vi.fn(), - readDaemonStopIdentity: vi.fn(), stopDaemon: vi.fn(), writeCommandOutput: vi.fn(), })); @@ -21,7 +20,17 @@ const mocks = vi.hoisted(() => ({ // mutation lane aliases only the manifest's exact entries, so the `./testing/*` pattern // form resolved outside the Stryker sandbox and the mock silently missed (#3546 CI). vi.mock('@agent-device/daemon-contracts/daemon-stop', () => ({ - readDaemonStopIdentity: mocks.readDaemonStopIdentity, + daemonNotRunningStopResult: () => ({ + stopped: false, + mode: 'not-running', + cleanupConfidence: 'known', + claimsReleased: [], + claimsOrphaned: [], + claimsSuperseded: [], + claimsUnattributable: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + warnings: [], + }), stopDaemon: mocks.stopDaemon, })); vi.mock('@agent-device/daemon-contracts/daemon-shutdown-report', () => ({ @@ -30,6 +39,8 @@ vi.mock('@agent-device/daemon-contracts/daemon-shutdown-report', () => ({ vi.mock('../../../platform-runtime-daemon-owner-cleanup.ts', () => ({ createDaemonOwnerCleanup: () => ({ cleanup: mocks.cleanupDaemonOwner }), })); +// `readDaemonStopIdentity` stays real: these tests write the record they want judged, which is the +// only way to show routing, retirement, and a `--clean` walk judge one read of one file. vi.mock('@agent-device/daemon-contracts/daemon-registration', async (importOriginal) => ({ ...(await importOriginal()), readDaemonStatus: mocks.readDaemonStatus, @@ -37,6 +48,7 @@ vi.mock('@agent-device/daemon-contracts/daemon-registration', async (importOrigi vi.mock('../shared.ts', () => ({ writeCommandOutput: mocks.writeCommandOutput })); import { daemonCommand } from '../daemon.ts'; +import * as daemonClientStop from '../../../daemon-client/daemon-client-stop.ts'; bindTestCommandHost(); @@ -52,6 +64,17 @@ const GRACEFUL_RESULT: DaemonStopResult = { warnings: [], }; +/** A detached daemon's record, which is the shape a `--clean` cleanup is allowed to follow. */ +function writeDetachedRecord(stateDir: string): void { + fs.writeFileSync( + `${stateDir}/daemon.json`, + JSON.stringify({ pid: 123, processStartTime: 'start' }), + { + mode: 0o600, + }, + ); +} + afterEach(() => { vi.clearAllMocks(); }); @@ -81,7 +104,7 @@ test('accepts only daemon stop and status', async () => { test('merges a graceful shutdown report and cleans runner leases with the start-time identity', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); + writeDetachedRecord(stateDir); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); const claim = { deviceKey: 'local:android:none:emulator-5554', @@ -105,9 +128,15 @@ test('merges a graceful shutdown report and cleans runner leases with the start- client: {} as never, }); + // The lifetime handed to the retirement is the one read out of that record, with the owner the + // same read stated: routing and retirement judge one observation. + expect(mocks.stopDaemon).toHaveBeenCalledWith({ + paths: expect.objectContaining({ baseDir: stateDir }), + observed: { pid: 123, startTime: 'start', ownership: 'detached' }, + }); expect(mocks.cleanupDaemonOwner).toHaveBeenCalledWith({ pid: 123, - startTime: 'start-time', + startTime: 'start', }); expect(mocks.writeCommandOutput).toHaveBeenCalledWith( expect.objectContaining({ clean: true, json: false }), @@ -133,7 +162,6 @@ test('merges a graceful shutdown report and cleans runner leases with the start- test('reports graceful provider cleanup as unknown when the shutdown report is unavailable', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue(null); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); mocks.readDaemonShutdownReport.mockReturnValue(null); @@ -145,6 +173,12 @@ test('reports graceful provider cleanup as unknown when the shutdown report is u client: {} as never, }); + // No record at all is not a refusal: nothing claims the state directory, so this stop hands the + // retirement no lifetime and still reports what it was told. + expect(mocks.stopDaemon).toHaveBeenCalledWith({ + paths: expect.objectContaining({ baseDir: stateDir }), + observed: null, + }); expect(mocks.writeCommandOutput).toHaveBeenCalledWith( expect.objectContaining({ json: false }), expect.objectContaining({ @@ -162,7 +196,7 @@ test('reports graceful provider cleanup as unknown when the shutdown report is u test('warns in text output when a graceful stop leaves an orphaned claim', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); + writeDetachedRecord(stateDir); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); const claim = { deviceKey: 'local:android:none:emulator-5554', @@ -200,7 +234,7 @@ test('warns in text output when a graceful stop leaves an orphaned claim', async test('routes an unattributable claim to the default status view, never to --stale', async () => { const stateDir = mkdtempForTestSync('agent-device-daemon-command-'); - mocks.readDaemonStopIdentity.mockReturnValue({ pid: 123, processStartTime: 'start-time' }); + writeDetachedRecord(stateDir); mocks.stopDaemon.mockResolvedValue(GRACEFUL_RESULT); const claim = { deviceKey: 'local:android:none:emulator-5554', @@ -247,6 +281,7 @@ test('daemon status reads the daemon record without stopping or contacting the d state: 'running', stateDir, pid: 321, + ownership: 'embedded', version: '0.21.30', plugins: [ { @@ -285,7 +320,169 @@ test('daemon status reads the daemon record without stopping or contacting the d const render = mocks.writeCommandOutput.mock.calls[0]![2] as () => string; expect(render()).toContain('testmu [store] @agent-device/testmu@1.0.0 - failed: bad plugin'); expect(render()).toContain('browserstack [builtin] builtin - active'); + // A host-owned pid changes what a stop may do to it, so status says who owns it. + expect(render()).toContain('pid 321, host-owned'); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// `--clean` walks the owner's process tree. A host process's tree is that host's whole program, so the +// flag is refused for a host-owned record rather than reaching past the runtime's own close. +test('daemon stop --clean refuses a host-owned record before stopping anything', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-command-embedded-'); + fs.writeFileSync( + `${stateDir}/daemon.json`, + JSON.stringify({ pid: 555, processStartTime: 'start-time', ownership: 'embedded' }), + { mode: 0o600 }, + ); + + try { + await expect( + daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['stop'], + flags: { clean: true, help: false, json: false, stateDir, version: false }, + client: {} as never, + }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'daemon-clean-not-detached', ownership: 'embedded', stateDir }, + }); + expect(mocks.stopDaemon).not.toHaveBeenCalled(); + expect(mocks.cleanupDaemonOwner).not.toHaveBeenCalled(); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// The same refusal for every record that cannot prove its owner. What differs is the hint, because the +// two have different fixes and only one of them still has a stop route: a readable record naming an +// odd owner can be drained without `--clean`, while an unreadable one is refused by that stop too — so +// its hint may not name a command that would answer with a second refusal. +test('daemon stop --clean refuses an unproven owner, and its hint depends on what the record says', async () => { + const rows: Array = [ + [ + 'unrecognized owner', + JSON.stringify({ pid: 555, processStartTime: 'start', ownership: 'sideways' }), + /Stop it without --clean/, + ], + ['unreadable record', '{not json', /remove .*daemon\.json/], + ]; + for (const [label, record, hintPattern] of rows) { + const stateDir = mkdtempForTestSync('agent-device-daemon-command-unknown-'); + fs.writeFileSync(`${stateDir}/daemon.json`, record, { mode: 0o600 }); + + try { + const refusal = await daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['stop'], + flags: { clean: true, help: false, json: false, stateDir, version: false }, + client: {} as never, + }).catch((error: unknown) => error); + expect(refusal, label).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'daemon-clean-not-detached', stateDir }, + }); + const hint = String((refusal as { details: { hint: string } }).details.hint); + expect(hint, label).toMatch(hintPattern); + expect(mocks.stopDaemon, label).not.toHaveBeenCalled(); + expect(mocks.cleanupDaemonOwner, label).not.toHaveBeenCalled(); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } + } +}); + +// A record that exists and cannot be read is not an absence: it may describe a runtime that is serving. +// So the stop refuses it instead of retiring a pid or reporting nothing runs. +test('daemon stop refuses an unreadable record instead of reporting nothing runs', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-command-unreadable-'); + fs.writeFileSync(`${stateDir}/daemon.json`, '{not json', { mode: 0o600 }); + + try { + await expect( + daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['stop'], + flags: { help: false, json: false, stateDir, version: false }, + client: {} as never, + }), + ).rejects.toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'daemon-record-unreadable', stateDir }, + }); + expect(mocks.stopDaemon).not.toHaveBeenCalled(); + expect(fs.readFileSync(`${stateDir}/daemon.json`, 'utf8')).toBe('{not json'); + } finally { + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// Status is the surface an operator reads before deciding what a stop may do, so an owner this build +// cannot place is said out loud instead of reading as an ordinary detached daemon. +test('daemon status names a record whose owner it cannot recognize', async () => { + mocks.readDaemonStatus.mockReturnValue({ + state: 'running', + stateDir: '/tmp/agent-device-daemon-status-unknown', + pid: 321, + ownership: 'unknown', + }); + + await daemonCommand({ + host: createTestHostEnvironment(), + positionals: ['status'], + flags: { help: false, json: false, stateDir: '/tmp/x', version: false }, + client: {} as never, + }); + + const render = mocks.writeCommandOutput.mock.calls.at(-1)![2] as () => string; + expect(render()).toContain('pid 321, owner-not-recognized'); +}); + +test('daemon stop of a host-owned record goes through the one router with the host the command runs on', async () => { + const stateDir = mkdtempForTestSync('agent-device-daemon-command-embedded-'); + fs.writeFileSync( + `${stateDir}/daemon.json`, + JSON.stringify({ pid: 555, processStartTime: 'start-time', ownership: 'embedded' }), + { mode: 0o600 }, + ); + const route = vi.spyOn(daemonClientStop, 'stopRecordedDaemon').mockResolvedValue({ + ...GRACEFUL_RESULT, + warnings: ['1 in-flight request ignored cancellation and was abandoned.'], + }); + + // A host this test can recognize: the stop must be given the command's own host, and no other host + // reaches this code path. What the router does with a host-owned record is the router's own test. + const host = createTestHostEnvironment({ + env: { ...process.env, AGENT_DEVICE_TEST_EMBEDDED_STOP_HOST: 'this-one' }, + }); + + try { + await daemonCommand({ + host, + positionals: ['stop'], + flags: { help: false, json: false, stateDir, version: false }, + client: {} as never, + }); + + // One read of the record, and the host that owns the runtime, are all the router is given. + expect(route).toHaveBeenCalledWith( + { ownership: 'embedded', identity: { pid: 555, startTime: 'start-time' } }, + { + paths: expect.objectContaining({ baseDir: stateDir }), + host: expect.objectContaining({ + env: expect.objectContaining({ AGENT_DEVICE_TEST_EMBEDDED_STOP_HOST: 'this-one' }), + }), + }, + ); + // A host-owned runtime is never signaled, so it never reaches the retirement a detached stop uses. + expect(mocks.stopDaemon).not.toHaveBeenCalled(); + const render = mocks.writeCommandOutput.mock.calls[0]![2] as () => string; + expect(render()).toContain('Daemon stopped (graceful).'); + expect(render()).toContain('ignored cancellation'); } finally { + vi.restoreAllMocks(); fs.rmSync(stateDir, { recursive: true, force: true }); } }); diff --git a/src/cli/commands/daemon.ts b/src/cli/commands/daemon.ts index 8d3499452f..0050c7881a 100644 --- a/src/cli/commands/daemon.ts +++ b/src/cli/commands/daemon.ts @@ -3,21 +3,23 @@ import { type PluginInventoryEntry, } from '@agent-device/contracts/plugin-inventory'; import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; -import { - readDaemonStopIdentity, - stopDaemon, - type DaemonStopResult, -} from '@agent-device/daemon-contracts/daemon-stop'; +import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import { isUnusableRecord, readDaemonStatus, + readDaemonStopIdentity, + readStopCleanupOwner, type DaemonStatus, + type DaemonStopIdentity, } from '@agent-device/daemon-contracts/daemon-registration'; +import type { DaemonStopResult } from '@agent-device/daemon-contracts/daemon-stop'; import { readDaemonShutdownReport } from '@agent-device/daemon-contracts/daemon-shutdown-report'; import { AppError } from '@agent-device/kernel/errors'; +import { shellQuoteIfNeeded } from '@agent-device/kernel/device-shell'; import { writeCommandOutput } from './shared.ts'; import type { ClientCommandHandler } from './router-types.ts'; import { createDaemonOwnerCleanup } from '../../platform-runtime-daemon-owner-cleanup.ts'; +import { stopRecordedDaemon } from '../../daemon-client/daemon-client-stop.ts'; const daemonOwnerCleanup = createDaemonOwnerCleanup(); @@ -36,15 +38,25 @@ export const daemonCommand: ClientCommandHandler = async ({ positionals, flags, await writeCommandOutput(flags, shown, () => renderDaemonStatus(shown)); return true; } - const identity = readDaemonStopIdentity(paths.infoPath); - const stopped = await stopDaemon({ paths }); + // One read decides every route below, so routing, retirement, and a `--clean` walk judge the same + // record rather than three snapshots of a file another process may be rewriting. + const record = readDaemonStopIdentity(paths.infoPath); + // Retained-runner cleanup works across the owner's whole process tree, which a host process shares + // with whatever else it runs. An embedded runtime's own close is the boundary a stop may cross, and + // a record that cannot prove its owner is not a licence to walk a process tree either. No record at all + // is not a refusal: there is nothing claimed, so there is nothing to clean. + if (flags.clean === true && record.ownership !== 'absent' && record.ownership !== 'detached') { + throw daemonCleanRefusedForUnprovenOwner(paths, record.ownership); + } + const stopped = await stopRecordedDaemon(record, { paths, host }); + const cleanupOwner = readStopCleanupOwner(record); const report = stopped.mode === 'graceful' ? readDaemonShutdownReport(paths.baseDir) : null; const result = mergeShutdownReport(stopped, report); - const shouldClean = flags.clean === true && identity !== null && result.stopped; + const shouldClean = flags.clean === true && cleanupOwner !== null && result.stopped; if (shouldClean) { await daemonOwnerCleanup.cleanup({ - pid: identity.pid, - startTime: identity.processStartTime, + pid: cleanupOwner.pid, + startTime: cleanupOwner.processStartTime, }); } const data = { ...result, clean: shouldClean }; @@ -130,6 +142,51 @@ function unattributableClaimWarnings( ]; } +/** + * `--clean` walks the owner's whole process tree, which for a host process includes everything else + * that host runs. An embedded runtime's own close is the only boundary a stop may cross, so this + * refuses instead of reaching past it, and the stop without `--clean` still works. + * + * A record whose owner cannot be read is refused too: the tree this would walk may be a host's, and + * the reason cleanup is allowed for a detached daemon is that its record proves the process is a + * daemon of its own. This is not a bad flag — the flag is fine, the record is the problem — so it + * reports as a failed command, like the other ownership refusals. + */ +function daemonCleanRefusedForUnprovenOwner( + paths: DaemonPaths, + record: DaemonStopIdentity['ownership'], +): AppError { + return new AppError( + 'COMMAND_FAILED', + 'This daemon record does not prove the process is a daemon of its own, so its retained-runner cleanup cannot be run from here.', + { + reason: 'daemon-clean-not-detached', + stateDir: paths.baseDir, + ownership: record, + hint: daemonCleanRefusalHint(paths, record), + }, + ); +} + +/** + * What to do next, which depends on whether a plain `daemon stop` has a route left for this record. + * A readable record naming a host or an unfamiliar owner does: that stop asks the running process to + * leave and never signals it. A record that cannot be read has none — the same stop refuses it for the + * same reason — so its hint names the record repair instead of sending the operator to a second refusal. + */ +function daemonCleanRefusalHint( + paths: DaemonPaths, + record: DaemonStopIdentity['ownership'], +): string { + if (record === 'unreadable') { + return `Nothing was signaled. Check what owns that process, then repair the record: stop it by its own means and remove ${shellQuoteIfNeeded(`${paths.baseDir}/daemon.json`)} so a daemon can register again.`; + } + if (record === 'embedded') { + return 'Stop it without --clean, which drains it through its own close, or let the host that started it close it.'; + } + return 'Stop it without --clean, which asks the running process to leave and reports what it answers.'; +} + function renderDaemonStop( result: Pick & { clean: boolean; @@ -148,7 +205,18 @@ function renderDaemonStatus( return `The daemon record for ${status.stateDir} is ${status.state === 'invalid' ? 'not a valid record' : 'unreadable'}, so whether a daemon runs there is unknown.`; } if (status.state === 'stopped') return `No running daemon for ${status.stateDir}.`; - const details = [`pid ${status.pid}`, status.version, status.transport].filter(Boolean); + const details = [ + `pid ${status.pid}`, + // An embedded pid belongs to a host process, and an unrecognized owner says the record states an + // owner this build cannot read; either way, what a stop may do to that pid changes. + status.ownership === 'embedded' + ? 'host-owned' + : status.ownership === 'unknown' + ? 'owner-not-recognized' + : null, + status.version, + status.transport, + ].filter(Boolean); return [ `Daemon running (${details.join(', ')}) for ${status.stateDir}.`, ...(status.plugins.length === 0 diff --git a/src/commands/schema/cli-help.ts b/src/commands/schema/cli-help.ts index 12fbdb0894..e27a6c46b9 100644 --- a/src/commands/schema/cli-help.ts +++ b/src/commands/schema/cli-help.ts @@ -514,6 +514,7 @@ Runner and daemon lifecycle (applies to simulators too): Each AGENT_DEVICE_STATE_DIR runs its own daemon. It self-exits after an idle window (default 5 minutes, matching the runner idle-stop default) once it has no open sessions, no in-flight requests, and no active recording; set AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS to override, or 0 to disable idle reap. A client starting a local daemon waits while the daemon process stays alive and keeps publishing startup progress, so an overloaded host makes startup slower, not failed. It stops at a ceiling (default 120 s; set AGENT_DEVICE_DAEMON_STARTUP_TIMEOUT_MS to a positive integer number of milliseconds to override) and names the outcome in error details.reason: startup-exited (the process ended before registering), startup-stalled (alive, no startup progress for a fixed 60 s window the ceiling variable does not change), or startup-ceiling (a startup was still live when the ceiling passed — this client's daemon or another client's; the client stops its own launch and never signals another client's live daemon. Only a progressing own launch outlasts the 60 s stall window, so a ceiling set below 60 s also ends a silent own launch this way). details.phases carries the startup phase timings it observed and details.startupTimeoutMs the effective ceiling. Branch on details.reason, never on the message. The variable is read only when a client must start a daemon: a value that is not a positive integer fails with INVALID_ARGS before the daemon spawns, and a client that only attaches to or reuses a running daemon never reads it. A command that needs the shared local daemon starts it for its state dir when none runs (device status and device release --stale never contact one). When the running daemon comes from another version or build, the command replaces it only if the daemon reports no work: no in-flight request, session, repair session, or live lease (a streaming resumable upload and a host human-control hold that names no lease are not counted); otherwise it fails with details.reason daemon-busy and leaves the daemon alone. A daemon that agreed to leave is waited for up to 60 s and never signaled while it drains; one still running then fails with details.reason daemon-drain-timeout unless it answers nothing and is provably a daemon process, which is force-retired. A daemon from a release that predates this check is never replaced and fails with daemon-incompatible. End a busy or older daemon deliberately with agent-device daemon stop --state-dir . Node clients pick daemonLifecycle 'attach' (never starts or replaces a daemon; fails with daemon-not-running or daemon-incompatible) or 'managed' (the default until 0.22), and ensureDaemon() starts or replaces one explicitly. + A daemon a host process started for itself records that owner. daemon status names such a record host-owned. A managed client keeps using such a daemon while it matches, and never replaces or retires one: it fails with details.reason daemon-owner-embedded and tells you to close the host. daemon stop --state-dir stops a recorded daemon with an authenticated shutdown request over its registered local transport and reports what that close could not finish as warnings; it sends that request to a record whose owner this version cannot recognize too, because the running process answers for itself and one that reports itself detached is safe to stop. Stop fails with details.reason daemon-embedded-stop-unreachable when that request cannot be delivered, because the record names no transport to send it on: nothing is signaled and nothing is deleted. A daemon.json that exists but cannot be read fails stop with details.reason daemon-record-unreadable; the record proves nothing, so no process is signaled. --clean is refused with details.reason daemon-clean-not-detached for a record that does not prove the process is a daemon of its own, which covers a host-owned record, one whose owner this version cannot recognize, and one it cannot read, because that cleanup walks the owner's whole process tree and a host process runs more than the daemon. Stop without --clean still works for the first two: it drains the runtime through its own close. Nothing outside the host signals a runtime it does not own. On a machine shared by several agents, set AGENT_DEVICE_SESSION_IDLE_TIMEOUT_MS to also expire an individual session that has taken a host-global device claim and then received no commands for that long; the claim is released and the next command on that session answers SESSION_NOT_FOUND with details.reason SESSION_IDLE_EXPIRED naming the window and the device. Off by default, because on a shared host "idle" and "thinking" are indistinguishable: only set it where an abandoned claim blocking every other agent is the worse failure. Sessions holding a remote lease or any active capture (recording, logs, audio, performance, or trace) are never expired this way. A stale iOS runner lease — its owner process dead, or its AGENT_DEVICE_STATE_DIR deleted — is reclaimed automatically instead of failing with "is already owned by another agent-device daemon". A live owner's runner is also reclaimed when the requesting daemon holds the host-global device claim for that device: claims are exclusive, so holding one proves the runner's owner released the device and merely kept the runner warm. The error remains only for owners outside claim arbitration (a pre-claims build, or daemons pointed at different claim stores). diff --git a/src/daemon-client-embedded-stop.test.ts b/src/daemon-client-embedded-stop.test.ts new file mode 100644 index 0000000000..e514fb67cf --- /dev/null +++ b/src/daemon-client-embedded-stop.test.ts @@ -0,0 +1,515 @@ +// One stop, end to end: a runtime a host process owns, the record it publishes, and the stop an outside +// caller performs. The host here is the test process itself, which is what makes the strongest assertion +// in this file: a stop that ever reached for a signal would take the test runner down with it. + +import fs from 'node:fs'; +import net from 'node:net'; +import { once } from 'node:events'; +import { spawn } from 'node:child_process'; +import { expect, test, vi } from 'vitest'; +import { startDaemonRuntime } from '@agent-device/daemon'; +import { resolveDaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; +import { readDaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { stopDaemon } from '@agent-device/daemon-contracts/daemon-stop'; +import { + DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS, + buildDaemonShutdownCompletion, + daemonShutdownCompletionAnswers, + readDaemonShutdownCompletion, + resolveDaemonShutdownCompletionPath, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { readProcessStartTime } from '@agent-device/host-kit/process'; +import { + closeLoopbackServer, + createTestHostEnvironment, + listenOnLoopback, + mkdtempForTestSync, +} from '@agent-device/test-kit'; +import { createDaemonRuntimeDeps } from './daemon-composition.ts'; +import { stopRecordedDaemon } from './daemon-client/daemon-client-stop.ts'; +import { readDaemonInfo } from './daemon-client/daemon-client-metadata.ts'; + +vi.mock('./platform-runtime.ts', () => ({ + androidObservation: {}, + createRequestPlatformProviders: () => ({ + run: async (_context: unknown, task: () => Promise) => await task(), + }), + createPlatformRuntimeGateway: () => ({ + applicationLifecycle: { + recoverStartupResources: async () => {}, + detachForDaemonShutdown: async () => [], + finalizeDaemonShutdown: async () => [], + }, + inspectFacts: async () => { + throw new Error('unused'); + }, + bind: async () => { + throw new Error('unused'); + }, + shutdown: async () => [], + }), + createPlatformDeviceInventoryGateways: () => ({}), +})); + +vi.mock('./provider-device-runtimes.ts', async () => ({ + createDaemonProviderRuntimeComposition: async () => ({ + runtimes: [], + platformModules: [], + // The composition is the one plugin-cleanup owner: its close answers with every plugin's outcomes. + shutdown: async () => [], + inventory: await ( + await import('./__tests__/test-utils/builtin-plugin-inventory.ts') + ).builtinPluginInventory(), + }), +})); + +/** + * A record whose owner field this build cannot place — the state a newer build's vocabulary, or a hand + * edit, produces. A running runtime is never in this state: it always knows whether it owns itself, so + * `unknown` is a reader's verdict about a file. The record is rewritten under a live runtime to produce + * exactly that, with everything else about the instance left intact. + */ +async function bootRuntimeWithUnplaceableRecordOwner(ownership: 'embedded' | 'detached') { + const booted = await bootRuntime(ownership); + const record = JSON.parse(fs.readFileSync(booted.paths.infoPath, 'utf8')) as Record< + string, + unknown + >; + record.ownership = 'sideways'; + fs.writeFileSync(booted.paths.infoPath, JSON.stringify(record), { mode: 0o600 }); + expect(readDaemonStopIdentity(booted.paths.infoPath).ownership).toBe('unknown'); + return booted; +} + +async function bootRuntime(ownership: 'embedded' | 'detached' = 'embedded') { + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-'); + const runtime = await startDaemonRuntime({ + host: createTestHostEnvironment({ + env: { + ...process.env, + AGENT_DEVICE_STATE_DIR: stateDir, + AGENT_DEVICE_DAEMON_IDLE_TIMEOUT_MS: '0', + AGENT_DEVICE_DAEMON_SERVER_MODE: 'http', + }, + }), + ownership, + stderr: { write: () => {} }, + stdout: { write: () => {} }, + composeDeps: createDaemonRuntimeDeps, + }); + return { stateDir, paths: resolveDaemonPaths(stateDir), runtime }; +} + +/** The stop as the CLI performs it: read the record once, then route on what it states. */ +function stopViaSocket(paths: ReturnType) { + return stopRecordedDaemon(readDaemonStopIdentity(paths.infoPath), { + paths, + host: createTestHostEnvironment(), + }); +} + +/** A port that was bound and given back, so a request to it is refused rather than served. */ +async function closedLoopbackPort(): Promise { + const server = net.createServer(); + const port = await listenOnLoopback(server); + await closeLoopbackServer(server); + return port; +} + +/** The lifetime a record can name for a child that has only just been spawned. */ +async function startTimeOf(pid: number): Promise { + for (let attempt = 0; attempt < 100; attempt += 1) { + const startTime = readProcessStartTime(pid); + if (startTime?.trim()) return startTime; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error(`child ${pid} never became readable`); +} + +/** + * The signals this stop sent at a process. `0` is the liveness probe and is not one; a `kill(pid)` with + * no signal argument is SIGTERM, so an omitted signal counts as the signal it really is. + */ +function hostSignals(killSpy: { mock: { calls: Array } }) { + return killSpy.mock.calls.filter(([, signal]) => signal !== 0); +} + +test('a host-owned runtime publishes the owner a stop has to respect', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + try { + const record = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as { + pid: number; + ownership?: string; + }; + // The record names the host's own process, which is why the signal path is closed here: stopping + // this runtime means asking it, not ending the program that asked. + expect(record.pid).toBe(process.pid); + expect(record.ownership).toBe('embedded'); + expect(readDaemonStopIdentity(paths.infoPath).ownership).toBe('embedded'); + expect(readDaemonInfo(paths.infoPath)?.ownership).toBe('embedded'); + } finally { + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('stopping a host-owned runtime asks it to leave and reports what its close said', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + + try { + // The host here stays alive the whole time, which is the case a signal-based stop cannot answer: the + // runtime closes inside a process that outlives it, so the only proof of the close is the record that + // close writes. Reading it is what ends this wait, not a timeout. + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + warnings: [], + }); + expect(Date.now() - startedAt).toBeLessThan(10_000); + // The runtime left by its own close, and the record it wrote is what the stop waited for. + await expect(runtime.closed).resolves.toMatchObject({ requests: { abandoned: 0 } }); + const completion = readDaemonShutdownCompletion(paths.baseDir); + expect(completion?.outcome).toBe('released'); + expect(completion?.unfinished).toEqual([]); + // Every stop addressed to that instance reads this answer, and a stop addressed to another one + // reads nothing — a recycled pid must not inherit an answer it was not part of. + expect( + daemonShutdownCompletionAnswers(completion!, { + pid: completion!.pid, + processStartTime: completion!.processStartTime, + }), + ).toBe(true); + expect(daemonShutdownCompletionAnswers(completion!, { pid: 42, processStartTime: 'x' })).toBe( + false, + ); + expect(fs.existsSync(paths.infoPath)).toBe(false); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('a stop performed without the shutdown route is refused and leaves the runtime serving', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + + try { + // A caller that goes straight to the retirement funnel skips the shutdown request the embedded route + // sends, and what it has left is a signal — exactly what must never reach a host process. `observed` + // carries the owner the caller read, and the funnel refuses on it rather than proving an exit it + // cannot prove. + const observed = { + pid: process.pid, + startTime: readProcessStartTime(process.pid), + ownership: 'embedded' as const, + }; + const refusal = await stopDaemon({ paths, observed }).then( + () => null, + (error: unknown) => error, + ); + expect(refusal).toMatchObject({ + code: 'COMMAND_FAILED', + details: { retirement: { reason: 'embedded-owner' } }, + }); + + expect(hostSignals(signals)).toEqual([]); + // The refusal changed nothing: the runtime the host owns is still there to close. + expect(readDaemonInfo(paths.infoPath)?.pid).toBe(process.pid); + expect(await runtime.close()).toMatchObject({ requests: { abandoned: 0 } }); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('two stops of one runtime both get the answer its close gives', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + + try { + // Started in the same tick, so both stops read the same embedded record and neither can be told it + // is already gone. The runtime admits one shutdown and answers the other with the same + // `draining` ack, so both are waiting on the one completion record this instance writes. + const [first, second] = await Promise.all([stopViaSocket(paths), stopViaSocket(paths)]); + + for (const [label, stopped] of [ + ['first', first], + ['second', second], + ] as const) { + expect(stopped, label).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'known', + }); + expect(stopped.warnings, label).toEqual([]); + } + await expect(runtime.closed).resolves.toMatchObject({ requests: { abandoned: 0 } }); + expect(fs.existsSync(paths.infoPath)).toBe(false); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('a stop of a runtime its host closed a long time ago reports what it kept', async () => { + // The state a host leaves when it closes its own runtime with work it could not cancel: the listener is + // gone, the registration stays, and the completion record that says so is older than the window a + // requested close answers within. Sitting out that window and failing with an unconfirmed shutdown would + // describe a runtime that stopped serving as one that may still be. + const { stateDir, paths, runtime } = await bootRuntime(); + const signals = vi.spyOn(process, 'kill'); + // Point the record at a port that was given back: the ask meets a closed listener, which is what a + // stop of a runtime its host already closed meets. The instance itself stays alive, as a host's does. + const record = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as Record; + record.httpPort = await closedLoopbackPort(); + fs.writeFileSync(paths.infoPath, JSON.stringify(record), { mode: 0o600 }); + const instance = { + pid: process.pid, + processStartTime: readProcessStartTime(process.pid) ?? '', + }; + // Aged past the window a requested close's answer is trusted within, so nothing about timing can + // settle this stop: only the state the record reports can. + const agedRecord = { + ...buildDaemonShutdownCompletion(instance, 'retained', { + unfinished: ['1 in-flight request ignored cancellation'], + }), + writtenAtMs: Date.now() - DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS - 1000, + }; + fs.writeFileSync( + resolveDaemonShutdownCompletionPath(paths.baseDir), + `${JSON.stringify(agedRecord)}\n`, + { + mode: 0o600, + }, + ); + const planted = readDaemonShutdownCompletion(paths.baseDir); + expect(planted).toMatchObject({ outcome: 'retained', pid: process.pid }); + expect(daemonShutdownCompletionAnswers(planted!, instance)).toBe(false); + + try { + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject({ + stopped: true, + mode: 'graceful', + cleanupConfidence: 'unknown', + }); + expect(stopped.warnings.join('\n')).toContain('1 in-flight request ignored cancellation'); + expect(stopped.warnings.join('\n')).toContain('keeps its registration'); + // Proof, not a timeout: this stop's budget is a minute, and the record settles it at once. + expect(Date.now() - startedAt).toBeLessThan(10_000); + expect(hostSignals(signals)).toEqual([]); + // Reporting the retention changes nothing about it: the state dir stays claimed for its owner. + expect(fs.existsSync(paths.infoPath)).toBe(true); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +/** + * A record naming a child process that is about to be killed, with a port nothing listens on: the shape a + * host leaves when it dies, or when its age makes its record the only thing a later stop can read. + */ +async function plantDeadWriterRecord(paths: ReturnType) { + const writer = spawn(process.execPath, ['-e', 'setTimeout(() => {}, 60_000)'], { + stdio: 'ignore', + }); + const pid = writer.pid ?? -1; + const startTime = await startTimeOf(pid); + fs.writeFileSync( + paths.infoPath, + JSON.stringify({ + pid, + processStartTime: startTime, + token: 'local-secret', + // A port nothing listens on: the ask cannot reach a runtime that is already gone. + port: await closedLoopbackPort(), + ownership: 'embedded', + }), + ); + writer.kill('SIGKILL'); + await once(writer, 'exit'); + return { pid, processStartTime: startTime }; +} + +test('a stop whose runtime died before writing its report does not wait on a dead writer', async () => { + // A host killed between removing the registration and publishing its completion record leaves a stop + // with no writer. The lifetime is the second piece of evidence, and it has to answer: sitting out the + // whole budget to report an unconfirmed failure would describe a close that had plainly ended. + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-dead-writer-'); + const paths = resolveDaemonPaths(stateDir); + await plantDeadWriterRecord(paths); + const signals = vi.spyOn(process, 'kill'); + + try { + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject({ stopped: false, mode: 'not-running' }); + // Proof, not a timeout: the budget this stop would otherwise sit out is a minute long. + expect(Date.now() - startedAt).toBeLessThan(10_000); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('an aged retention record from a writer that died is a stale file, not a held registration', async () => { + // A retained record answers a later stop while the lifetime that wrote it is still live. Once that + // process is gone, nothing holds the state directory any more, and reporting a graceful stop would + // send the operator away from the one thing left to do: the takeover path that clears the stale record. + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-dead-retention-'); + const paths = resolveDaemonPaths(stateDir); + const instance = await plantDeadWriterRecord(paths); + fs.writeFileSync( + resolveDaemonShutdownCompletionPath(paths.baseDir), + `${JSON.stringify({ + ...buildDaemonShutdownCompletion(instance, 'retained', { + unfinished: ['1 in-flight request ignored cancellation'], + }), + writtenAtMs: Date.now() - DAEMON_SHUTDOWN_COMPLETION_MAX_AGE_MS - 1000, + })}\n`, + { mode: 0o600 }, + ); + const signals = vi.spyOn(process, 'kill'); + + try { + const startedAt = Date.now(); + const stopped = await stopViaSocket(paths); + + expect(stopped).toMatchObject({ stopped: false, mode: 'not-running' }); + expect(Date.now() - startedAt).toBeLessThan(10_000); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// A record can claim a host-owned runtime and still be unusable: a file that is not JSON, or one that +// names no transport. Neither is an absence, and `not-running` would tell the operator to move on while +// a host may be serving on that very state directory. The two refusals are separate because the fixes +// are: a file that cannot be parsed says nothing about who owns the process, so nothing is even addressed. +test('a stop of a record it cannot address refuses instead of reporting nothing runs', async () => { + for (const [label, record, reason, recordState] of [ + ['unreadable', '{not json', 'daemon-record-unreadable', undefined], + [ + 'no transport', + JSON.stringify({ + pid: process.pid, + processStartTime: readProcessStartTime(process.pid), + ownership: 'embedded', + }), + 'daemon-embedded-stop-unreachable', + 'incomplete', + ], + ] as const) { + const stateDir = mkdtempForTestSync('agent-device-embedded-stop-record-'); + const paths = resolveDaemonPaths(stateDir); + fs.mkdirSync(paths.baseDir, { recursive: true }); + fs.writeFileSync(paths.infoPath, record, { mode: 0o600 }); + const signals = vi.spyOn(process, 'kill'); + + try { + const refusal = await stopViaSocket(paths).then( + () => null, + (error: unknown) => error, + ); + expect(refusal, label).toMatchObject({ + code: 'COMMAND_FAILED', + details: { + reason, + stateDir: paths.baseDir, + ...(recordState ? { record: recordState } : {}), + }, + }); + expect(hostSignals(signals), label).toEqual([]); + // Refusing changes nothing: whatever owns that record still does. + expect(fs.readFileSync(paths.infoPath, 'utf8'), label).toBe(record); + } finally { + signals.mockRestore(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } + } +}); + +// A record this build cannot place still gets the authenticated request rather than a signal: asking is +// free, and the runtime's own answer says whose process this is. What the answer must not do is name +// `daemon stop`, because that command refuses to signal a record whose owner it cannot prove either — the +// operator needs the record repair, which is the only fix for a file that states nothing usable. +test('a detached answer against an unreadable owner names the record repair, not daemon stop', async () => { + const { stateDir, paths, runtime } = await bootRuntimeWithUnplaceableRecordOwner('detached'); + const signals = vi.spyOn(process, 'kill'); + + try { + const refusal = await stopViaSocket(paths).then( + () => null, + (error: unknown) => error, + ); + // The runtime knows its own owner, and it is not a host: it answers the shutdown request the way a + // detached daemon does. + expect(refusal).toMatchObject({ + code: 'COMMAND_FAILED', + details: { reason: 'daemon-not-embedded' }, + }); + const hint = String((refusal as { details: { hint: string } }).details.hint); + expect(hint).toMatch(/daemon\.json/); + expect(hint).not.toMatch(/daemon stop/); + expect(hostSignals(signals)).toEqual([]); + expect(await runtime.close()).toMatchObject({ requests: { abandoned: 0 } }); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +// The same rewritten record under a runtime that really is host-owned. Asking is still the right move: +// the runtime answers for itself, agrees, and closes its own process. Nothing here signals, and the stop +// does not need the record to say who owns it — the answer is the proof. +test('an unrecognized owner under a host-owned runtime leaves by its own shutdown request', async () => { + const { stateDir, paths, runtime } = await bootRuntimeWithUnplaceableRecordOwner('embedded'); + const signals = vi.spyOn(process, 'kill'); + + try { + expect(await stopViaSocket(paths)).toMatchObject({ stopped: true, mode: 'graceful' }); + await expect(runtime.closed).resolves.toBeTruthy(); + expect(hostSignals(signals)).toEqual([]); + } finally { + signals.mockRestore(); + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); + +test('a second stop of a host-owned runtime reports nothing left to stop', async () => { + const { stateDir, paths, runtime } = await bootRuntime(); + + try { + expect(await stopViaSocket(paths)).toMatchObject({ stopped: true, mode: 'graceful' }); + await expect(runtime.closed).resolves.toBeTruthy(); + + // The instance is gone, so this is not a refused stop or an unconfirmed one: there is nothing there. + expect(await stopViaSocket(paths)).toMatchObject({ + stopped: false, + mode: 'not-running', + }); + } finally { + await runtime.close(); + fs.rmSync(stateDir, { recursive: true, force: true }); + } +}); diff --git a/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts new file mode 100644 index 0000000000..f02219fb9b --- /dev/null +++ b/src/daemon-client/__tests__/daemon-client-embedded-lifecycle.test.ts @@ -0,0 +1,212 @@ +// What a client may do about a runtime a host owns, and what a client pinned to one daemon instance +// will and will not do to reach it. Most of the weight here is in what did NOT happen: nothing was +// signaled, nothing was started, nothing was retired, and no request was dispatched. + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { afterEach, test, vi } from 'vitest'; +import { + closeLoopbackServer, + createTestHostEnvironment, + supportsLoopbackBind, +} from '@agent-device/test-kit'; +import { DAEMON_LIFECYCLE_REASONS } from '@agent-device/daemon-contracts/daemon-resolution'; +import { AppError } from '@agent-device/kernel/errors'; +import { isProcessAlive } from '@agent-device/host-kit/process'; +import { finishRegisteredDaemonFixtures } from '../../__tests__/test-utils/registered-daemon-fixture.ts'; +import { + launchesFreshDaemonAt, + okResult, + registerDaemon, + startScriptedDaemon, + stateDirAndPaths, +} from './explicit-lifecycle-fixture.ts'; + +vi.mock('@agent-device/host-kit/command', async (importOriginal) => ({ + ...(await importOriginal()), + runCmdDetachedMonitored: vi.fn(), +})); +import { runCmdDetachedMonitored } from '@agent-device/host-kit/command'; +import { createDaemonTransport, type sendToDaemon } from '../daemon-client.ts'; +import { resolveClientSettings, resolveTimeoutRecovery } from '../daemon-client-lifecycle.ts'; +import { readDaemonInfo } from '../daemon-client-metadata.ts'; + +const spawn = vi.mocked(runCmdDetachedMonitored); + +afterEach(async () => { + vi.restoreAllMocks(); + spawn.mockReset(); + await finishRegisteredDaemonFixtures(); +}); + +function request(stateDir: string) { + return { + session: 'default', + command: 'devices', + positionals: [], + flags: { stateDir, daemonTransport: 'http' }, + } as Parameters[0]; +} + +/** + * The signals this code sent at a process. `0` is the liveness probe and is not one; a call with no + * signal argument is SIGTERM, so an omitted signal counts. + */ +function signalsSent(killSpy: { mock: { calls: Array } }) { + return killSpy.mock.calls.filter(([, signal]) => signal !== 0); +} + +async function refusalFrom(promise: Promise): Promise { + const refusal = await promise.then( + () => assert.fail('expected the request to be refused'), + (error: unknown) => error, + ); + assert.ok(refusal instanceof AppError, String(refusal)); + return refusal; +} + +/** + * A stale daemon of an incompatible version: the shape that makes a managed client want this state + * directory. The ownership field is then the only thing that decides whether it may take it. The + * stand-in answers nothing useful on purpose: the embedded refusal happens before the client asks + * this daemon anything. + */ +async function startStaleRegistration(ownership?: 'embedded') { + const { stateDir, paths } = stateDirAndPaths(); + const daemon = await startScriptedDaemon(okResult({})); + const registered = await registerDaemon(paths, { + httpPort: daemon.port, + version: '0.0.0-stale', + capabilities: ['quiesce'], + ...(ownership ? { ownership } : {}), + }); + return { stateDir, paths, daemon, ...registered }; +} + +test('a managed client refuses to take over a runtime a host owns, and signals nothing', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback listeners are not permitted'); + const { stateDir, paths, daemon, child, info } = await startStaleRegistration('embedded'); + const killSpy = vi.spyOn(process, 'kill'); + + try { + const transport = createDaemonTransport(createTestHostEnvironment()); + const refusal = await refusalFrom(transport(request(stateDir))); + + assert.equal(info.ownership, 'embedded'); + assert.equal(refusal.details?.reason, DAEMON_LIFECYCLE_REASONS.ownerEmbedded); + assert.match(String(refusal.details?.hint), /Close the host that started it/); + assert.deepEqual(daemon.rpcCommands, [], 'the takeover was never attempted'); + assert.equal(spawn.mock.calls.length, 0, 'a host-owned runtime is never replaced'); + assert.deepEqual(signalsSent(killSpy), [], 'a host process is never signaled'); + assert.equal(isProcessAlive(child.pid), true); + assert.deepEqual(readDaemonInfo(paths.infoPath)?.pid, child.pid); + } finally { + await closeLoopbackServer(daemon.server); + } +}); + +test('a managed client refuses to replace a record whose owner it cannot recognize', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback listeners are not permitted'); + // Same setup as the embedded takeover row, with the ownership field stating something this build does + // not know. Replacing that daemon would mean signalling a process the record cannot place. + const { stateDir, paths, daemon, child, info } = await startStaleRegistration('embedded'); + const record = JSON.parse(fs.readFileSync(paths.infoPath, 'utf8')) as Record; + record.ownership = 'sideways'; + fs.writeFileSync(paths.infoPath, JSON.stringify(record), { mode: 0o600 }); + const killSpy = vi.spyOn(process, 'kill'); + + try { + assert.equal(readDaemonInfo(paths.infoPath)?.ownership, 'unknown'); + assert.notEqual(info.ownership, 'unknown', 'the row changes one field of a valid record'); + const refusal = await refusalFrom( + createDaemonTransport(createTestHostEnvironment())(request(stateDir)), + ); + + assert.equal(refusal.details?.reason, 'daemon-owner-unrecognized'); + assert.match(String(refusal.details?.hint), /Update agent-device/); + assert.deepEqual(daemon.rpcCommands, [], 'the takeover was never attempted'); + assert.equal(spawn.mock.calls.length, 0, 'a record nobody can place is never replaced'); + assert.deepEqual(signalsSent(killSpy), [], 'a process of unknown owner is never signaled'); + assert.equal(isProcessAlive(child.pid), true); + assert.equal(readDaemonInfo(paths.infoPath)?.pid, child.pid); + } finally { + await closeLoopbackServer(daemon.server); + } +}); + +test('the same record with no host owner is still the takeover this client performs', async (t) => { + if (!(await supportsLoopbackBind())) return t.skip('loopback listeners are not permitted'); + // One field differs from the row above, and it changes everything: a detached contender of the wrong + // version is a daemon this client ends and replaces. That is what makes the embedded refusal a + // boundary rather than a general rule about taking over. + const { stateDir, paths } = stateDirAndPaths(); + const stale = await startScriptedDaemon(okResult({})); + const fresh = await startScriptedDaemon(okResult({ devices: [] })); + const { child } = await registerDaemon(paths, { + httpPort: stale.port, + version: '0.0.0-stale', + capabilities: ['quiesce'], + }); + launchesFreshDaemonAt(spawn, paths, fresh.port); + const killSpy = vi.spyOn(process, 'kill'); + + try { + assert.equal(readDaemonInfo(paths.infoPath)?.ownership, 'detached'); + const response = await createDaemonTransport(createTestHostEnvironment())(request(stateDir)); + + assert.equal(response.ok, true); + assert.equal(spawn.mock.calls.length, 1, 'the detached contender was replaced'); + // The signal the embedded row proves never happens is exactly what this path is allowed to do. + assert.deepEqual( + killSpy.mock.calls.filter(([target, signal]) => target === child.pid && signal === 'SIGTERM'), + [[child.pid, 'SIGTERM']], + ); + } finally { + await closeLoopbackServer(stale.server); + await closeLoopbackServer(fresh.server); + } +}); + +// One setting that changes nothing else: an unpinned managed client on the same record still retires. +test('a record this build cannot place authorizes neither a kill nor a takeover', () => { + const { stateDir } = stateDirAndPaths(); + const settings = resolveClientSettings( + request(stateDir), + undefined, + createTestHostEnvironment(), + 'managed', + ); + const info = { + token: 'token', + pid: 4242, + processStartTime: 'start', + ownership: 'unknown' as const, + }; + + // The same fail-closed rule as the embedded row: `detached` is the only ownership that proves a pid + // may be signaled, and a hung daemon whose record cannot prove that is left alone. + assert.equal(resolveTimeoutRecovery(settings, info), 'none'); +}); + +test('timeout recovery retires nothing for a host-owned runtime', async () => { + const { stateDir } = stateDirAndPaths(); + const settings = resolveClientSettings( + request(stateDir), + undefined, + createTestHostEnvironment(), + 'managed', + ); + const detached = { + token: 'token', + pid: 4242, + processStartTime: 'start', + ownership: 'detached' as const, + }; + + assert.equal(resolveTimeoutRecovery(settings, detached), 'retire-unresponsive'); + assert.equal( + resolveTimeoutRecovery(settings, { ...detached, ownership: 'embedded' }), + 'none', + "a hung host-owned runtime is its host's to close, not a candidate for a kill", + ); +}); diff --git a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts index 30375c6bd5..6e307b1a4d 100644 --- a/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts +++ b/src/daemon-client/__tests__/daemon-client-explicit-lifecycle.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import { afterEach, test, vi } from 'vitest'; import { closeLoopbackServer, supportsLoopbackBind } from '@agent-device/test-kit'; import { AppError } from '@agent-device/kernel/errors'; -import { DAEMON_QUIESCE_COMMAND } from '@agent-device/daemon-contracts/daemon-resolution'; +import { DAEMON_LIFECYCLE_COMMANDS } from '@agent-device/daemon-contracts/daemon-resolution'; import { DAEMON_STARTUP_EXIT_CODES } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { isProcessAlive } from '@agent-device/host-kit/process'; import { readVersion } from '@agent-device/host-kit/version'; @@ -254,7 +254,7 @@ test('ensureDaemon replaces an incompatible idle daemon through a fenced quiesce }); assert.equal(connection.started, true); - assert.deepEqual(stale.rpcCommands, [DAEMON_QUIESCE_COMMAND]); + assert.deepEqual(stale.rpcCommands, [DAEMON_LIFECYCLE_COMMANDS.quiesce]); assert.equal(stale.rpcRequests[0]?.params.token, 'local-secret'); assert.deepEqual(stale.rpcRequests[0]?.params.input, { pid: info.pid, @@ -485,7 +485,7 @@ test('a live daemon on another transport is quiesced, never killed as unreachabl onStart: (reason) => started.push(reason), }); - assert.deepEqual(live.commands, [DAEMON_QUIESCE_COMMAND]); + assert.deepEqual(live.commands, [DAEMON_LIFECYCLE_COMMANDS.quiesce]); assert.deepEqual(started, ['incompatible']); assert.equal(isProcessAlive(child.pid), false, 'it left because it agreed to'); assert.deepEqual(leaver.clientSignals, [], 'the client signaled nothing'); @@ -543,7 +543,7 @@ test('ensureDaemon judges again after the daemon it addressed turned out to be g assert.equal(typeof actualSend, 'function'); assert.deepEqual( stale.rpcCommands, - [DAEMON_QUIESCE_COMMAND], + [DAEMON_LIFECYCLE_COMMANDS.quiesce], 'the second judgement asked again', ); assert.deepEqual(leaver.clientSignals, [], 'the client signaled nothing'); @@ -648,7 +648,11 @@ test('ensureDaemon quiesces an incompatible daemon that wins a startup race inst const connection = await ensureDaemon({ stateDir, daemonTransport: 'http' }); assert.equal(connection.started, true); - assert.deepEqual(stale.rpcCommands, [DAEMON_QUIESCE_COMMAND], 'the racer was asked to leave'); + assert.deepEqual( + stale.rpcCommands, + [DAEMON_LIFECYCLE_COMMANDS.quiesce], + 'the racer was asked to leave', + ); assert.ok(winner); assert.equal(isProcessAlive(winner.pid), false); assert.equal(spawn.mock.calls.length, 2, 'the second start follows the quiesce'); diff --git a/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts b/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts index d2eebf6867..0a52e50a18 100644 --- a/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts +++ b/src/daemon-client/__tests__/explicit-lifecycle-fixture.ts @@ -125,6 +125,7 @@ type Registration = { socketPort?: number; version?: string; capabilities?: readonly string[]; + ownership?: 'embedded' | 'detached'; }; /** A live child owning the registration, advertising `fields` at the stand-in's port. */ @@ -139,6 +140,7 @@ export async function registerDaemon(paths: DaemonPaths, fields: Registration) { codeOrigin: 'checkout', codeSignature: currentDaemonCodeSignature(), ...(fields.capabilities ? { capabilities: fields.capabilities as ['quiesce'] } : {}), + ...(fields.ownership ? { ownership: fields.ownership } : {}), }, undefined, ); diff --git a/src/daemon-client/daemon-client-ensure.ts b/src/daemon-client/daemon-client-ensure.ts index 3db48b3902..8fa83ec480 100644 --- a/src/daemon-client/daemon-client-ensure.ts +++ b/src/daemon-client/daemon-client-ensure.ts @@ -18,9 +18,9 @@ import { stopDaemonForTakeover, } from './daemon-client-startup.ts'; import type { DaemonClientSettings, EnsuredDaemon } from './daemon-client-lifecycle.ts'; -import { requestDaemonQuiesce } from './daemon-client-quiesce.ts'; +import { DAEMON_LEAVE_WAIT_MS, requestDaemonLifecycle } from './daemon-client-lifecycle-request.ts'; import { probeDaemonResponsive } from './daemon-client-liveness-probe.ts'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { observedDaemonIdentityOf, type DaemonInfo } from './daemon-client-metadata.ts'; import { DAEMON_CONTENDED_REASON, daemonIdentityOf, @@ -34,10 +34,6 @@ import { // a short wait, for as long as a drain can take. const ENSURE_ATTEMPTS = 40; const JUDGE_AGAIN_DELAY_MS = 250; -// How long a daemon that accepted a quiesce request gets to finish leaving. Its own shutdown is -// bounded in phases (server close, session and lease teardown, provider release), so this is the -// longest a healthy drain can take with margin, not a polling interval. -const DRAIN_DEADLINE_MS = 60_000; const DRAIN_POLL_MS = 250; const UNRESPONSIVE_PROBES = 3; const UNRESPONSIVE_PROBE_GAP_MS = 200; @@ -157,26 +153,27 @@ async function awaitDrainedDaemon( ): Promise<'left' | 'retired'> { const { exited } = await waitForDaemonExit( { pid: info.pid, startTime }, - { timeoutMs: DRAIN_DEADLINE_MS, pollMs: DRAIN_POLL_MS }, + { timeoutMs: DAEMON_LEAVE_WAIT_MS, pollMs: DRAIN_POLL_MS }, ); if (exited) { // The daemon removes its own registration; this only clears what a crashed drain left behind. await recoverAbandonedDaemonRegistration({ paths: settings.paths, - observed: { pid: info.pid, startTime }, + observed: observedDaemonIdentityOf(info), }); return 'left'; } if (await answersLivenessProbe(info)) throw await daemonDrainTimeoutError(info, settings); const retirement = await stopDaemonForTakeover(info, settings.paths, 'force'); if (retirement.status !== 'retained') return 'retired'; - // The identity check could not prove the live process is the daemon it observed (an embedded host - // runs no daemon entry; a start time or command line may also be unreadable), so nothing was - // signaled. - if ( - retirement.termination?.status === 'retained' && - retirement.termination.reason === 'identity-unverified' - ) { + // Nothing was signaled because the process could not be proved to be a daemon of its own: either its + // start time or command line is unreadable, or the record states an owner this build cannot read. + // That is the drain's own outcome: it is still leaving, or already gone, and nobody knows which. A + // refusal that names an owner instead (a host that ended up holding this state directory mid-drain) + // is a replacement that could not be confirmed, and reports as one. + const refusedReason = + retirement.termination?.status === 'retained' ? retirement.termination.reason : undefined; + if (refusedReason === 'identity-unverified' || refusedReason === 'owner-unproven') { throw await daemonDrainTimeoutError(info, settings); } throw daemonRetirementUnconfirmedError(retirement, settings.paths); @@ -191,7 +188,7 @@ async function quiesceOrRetireUnresponsive( settings: DaemonClientSettings, ): Promise<'draining' | 'gone' | 'retired'> { try { - return await requestDaemonQuiesce(info, settings); + return (await requestDaemonLifecycle(info, settings, 'quiesce')).outcome; } catch (error) { if (!(error instanceof AppError) || error.details?.reason !== 'daemon_transport_timeout') throw error; diff --git a/src/daemon-client/daemon-client-lifecycle-request.ts b/src/daemon-client/daemon-client-lifecycle-request.ts new file mode 100644 index 0000000000..31cf04b657 --- /dev/null +++ b/src/daemon-client/daemon-client-lifecycle-request.ts @@ -0,0 +1,154 @@ +import { + DAEMON_LIFECYCLE_REASONS, + DAEMON_SHUTTING_DOWN_REASON, + buildDaemonLifecycleRequest, + readDaemonLifecycleCensus, + type DaemonLifecycleMode, + type DaemonPaths, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import type { HostEnvironment } from '@agent-device/kernel/source-value'; +import { createRequestId } from '@agent-device/host-kit/diagnostics'; +import { AppError, throwDaemonError } from '@agent-device/kernel/errors'; +import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { isDaemonTransportUnavailableError, sendRequest } from './daemon-client-transport.ts'; +import { daemonBusyError, daemonNotRetirableError } from './daemon-lifecycle-errors.ts'; + +const LIFECYCLE_REQUEST_TIMEOUT_MS = 10_000; + +/** + * How long a daemon or runtime that agreed to leave gets to finish leaving. Its own shutdown is bounded + * in phases (server close, session and lease teardown, plugin disposal, provider release), so this is + * the longest a healthy close can take with margin, not a polling interval. Both waits a client keeps + * after that agreement — `ensureDaemon` replacing a detached daemon, and a stop of an embedder-owned + * runtime — use it, because the same close runs either way: a shorter one would report an unconfirmed + * failure for a close that was still succeeding. + */ +export const DAEMON_LEAVE_WAIT_MS = 60_000; + +/** What a lifecycle request needs besides the daemon it addresses: where and how to reach it. */ +export type DaemonLifecycleTarget = Readonly<{ + paths: DaemonPaths; + host: HostEnvironment; +}>; + +// A refused or reset connection: the daemon left, is leaving, or was never there. Nothing was +// accepted, so judging again is safe. +function isConnectionFailure(error: unknown): boolean { + return error instanceof AppError && error.details?.reason === 'daemon_transport_failure'; +} + +/** + * `draining`: the daemon accepted and is leaving, which is all its own close needs to know — what a + * stop waits for afterwards is the completion record, and that answers by instance, not by request. + * `gone`: the daemon the request was addressed to is no longer there to answer, because it left, is + * leaving already, or was replaced. + */ +export type DaemonLifecycleOutcome = Readonly<{ outcome: 'draining' | 'gone' }>; + +/** + * Asks the daemon `info` names to leave, by the rule `mode` states, and reports its answer. + * + * The request is fenced to that instance, so a replacement refuses it. A quiesce a daemon that holds + * work refuses with `daemon-busy`, and that refusal is the answer: nothing here signals it. A + * shutdown a detached daemon refuses names its owner instead, because a detached daemon is stopped by + * signalling its process. A timeout is not recovered here either; the caller decides what an + * unresponsive daemon deserves. + */ +export async function requestDaemonLifecycle( + info: DaemonInfo, + target: DaemonLifecycleTarget, + mode: DaemonLifecycleMode, +): Promise { + const requestId = createRequestId(); + const request = buildDaemonLifecycleRequest({ + mode, + token: info.token, + fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, + requestId, + }); + let response; + try { + response = await sendRequest( + info, + request, + // Whichever transport the daemon serves: a client that cannot reach it on its own preference + // is exactly the one that has to ask it to leave. + 'auto', + target.paths, + LIFECYCLE_REQUEST_TIMEOUT_MS, + { host: target.host, timeoutRecovery: 'none' }, + ); + } catch (error) { + if (isDaemonTransportUnavailableError(error) || isConnectionFailure(error)) + return { outcome: 'gone' }; + // Over HTTP a daemon's refusal arrives as an error carrying its reason, not as the response that + // carries it. The three ownership refusals are judged here so an embedded runtime says the same + // thing whichever transport it serves; every other reason keeps the path it shipped with. + const refusal = readOwnerRefusal(error); + if (refusal) + return await judgeRefusedAnswer(refusal, info, target, () => Promise.reject(error)); + throw error; + } + if (response.ok) return { outcome: 'draining' }; + const details = response.error.details; + return await judgeRefusedAnswer({ reason: details?.reason, details }, info, target, () => + throwDaemonError(response.error), + ); +} + +/** A daemon's refusal, as read from either transport's shape of it. */ +type RefusedDaemonAnswer = Readonly<{ reason: unknown; details: unknown }>; + +/** + * The refusals that name who owns the process. Quiesce never answers with one, so judging them on the + * error path cannot change a shipped refusal; it only gives the shutdown request the same answer a + * socket client already got. + */ +const OWNER_REFUSALS: readonly string[] = [ + DAEMON_LIFECYCLE_REASONS.ownerEmbedded, + DAEMON_LIFECYCLE_REASONS.notEmbedded, +]; + +function readOwnerRefusal(error: unknown): RefusedDaemonAnswer | undefined { + if (!(error instanceof AppError)) return undefined; + const reason = error.details?.reason; + return typeof reason === 'string' && OWNER_REFUSALS.includes(reason) + ? { reason, details: error.details } + : undefined; +} + +/** + * What the client does about a refusal the daemon answered with, and what it throws instead of the + * daemon's bare reason: this client knows the state directory, the recorded identity, and the route an + * operator can take, and none of that is in a wire error. + * + * `unanswered` runs when no rule covers the refusal, which keeps a reason this version does not know + * on its own path rather than folding it into one this function understands. + */ +async function judgeRefusedAnswer( + refusal: RefusedDaemonAnswer, + info: DaemonInfo, + target: DaemonLifecycleTarget, + unanswered: () => Promise, +): Promise { + const stateDir = target.paths.baseDir; + if (refusal.reason === DAEMON_LIFECYCLE_REASONS.busy) { + throw daemonBusyError(info, stateDir, readDaemonLifecycleCensus(refusal.details)); + } + // A detached daemon has an owner that can signal it, and a host owns an embedded runtime: either + // way, this client is not the one that ends it. The daemon's own answer says which owner is on the + // other side of that line, so it is trusted over this process's possibly-stale read of the record. + if (refusal.reason === DAEMON_LIFECYCLE_REASONS.notEmbedded) { + throw daemonNotRetirableError({ info, stateDir, owner: 'detached' }); + } + if (refusal.reason === DAEMON_LIFECYCLE_REASONS.ownerEmbedded) { + throw daemonNotRetirableError({ info, stateDir, owner: 'embedded' }); + } + if ( + refusal.reason === DAEMON_LIFECYCLE_REASONS.instanceMismatch || + refusal.reason === DAEMON_SHUTTING_DOWN_REASON + ) { + return { outcome: 'gone' }; + } + return await unanswered(); +} diff --git a/src/daemon-client/daemon-client-lifecycle.ts b/src/daemon-client/daemon-client-lifecycle.ts index a30fd19bcc..5067deb545 100644 --- a/src/daemon-client/daemon-client-lifecycle.ts +++ b/src/daemon-client/daemon-client-lifecycle.ts @@ -24,7 +24,7 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import type { findUnrecoveredRepairCommitFailure } from '@agent-device/daemon-contracts/session-repair-tombstone'; import type { DaemonInfo } from './daemon-client-metadata.ts'; import { attachActiveSessionAddressHint } from './daemon-client-address-hints.ts'; -import { isRemoteDaemon } from './daemon-client-metadata.ts'; +import { isRemoteDaemon, observedDaemonIdentityOf } from './daemon-client-metadata.ts'; import { cachedRemoteDaemonHealth } from './daemon-client-transport.ts'; import { assertDaemonPolicyMatches, @@ -90,6 +90,24 @@ export function resolveClientSettings( }; } +/** + * What a request that ran out of its time may do to the daemon it timed out on. + * + * An attach client owns no daemon, and a pinned client holds one instance it did not start, so neither + * retires a process it cannot reach. A record that names a host as its owner is not a client's to + * retire either: only that host ends the runtime behind it, whatever this client's request timed out on. + */ +export function resolveTimeoutRecovery( + settings: DaemonClientSettings, + info: DaemonInfo, +): 'retire-unresponsive' | 'none' { + if (settings.daemonLifecycle === 'attach') return 'none'; + // Only a record that proves a detached daemon authorizes a kill: an embedded one belongs to a host, + // and one whose owner this build cannot read says nothing about what is safe to signal. + if (info.ownership !== 'detached') return 'none'; + return 'retire-unresponsive'; +} + /** * The local daemon a request would use, in the terms `ensureDaemon` takes, or `undefined` when the * request has no shared local daemon to ensure: a remote connection has none, and a one-shot replay @@ -283,7 +301,7 @@ export async function cleanupDaemonAfterRequest( const result = await stopAndRetireDaemon({ paths: settings.paths, - observed: { pid: daemon.info.pid, startTime: daemon.info.processStartTime ?? null }, + observed: observedDaemonIdentityOf(daemon.info), mode: 'graceful', ownedStateDir: settings.ownedStateDir, }); diff --git a/src/daemon-client/daemon-client-metadata.ts b/src/daemon-client/daemon-client-metadata.ts index 8713947d7a..ab76c78a7b 100644 --- a/src/daemon-client/daemon-client-metadata.ts +++ b/src/daemon-client/daemon-client-metadata.ts @@ -4,11 +4,14 @@ import { isProcessPid } from '@agent-device/host-kit/process'; import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature'; import { readDaemonCapabilities, + readDaemonOwnership, readDaemonRecord, readRecordedDaemonTransport, type DaemonCapability, + type DaemonOwnership, type DaemonPaths, type DaemonServerMode, + type ObservedDaemonIdentity, } from '@agent-device/daemon-contracts/daemon-resolution'; export type DaemonInfo = { @@ -25,6 +28,11 @@ export type DaemonInfo = { policyDigest?: string; /** Absent from daemons that predate capabilities, which therefore have none. */ capabilities?: readonly DaemonCapability[]; + /** + * Who owns the process this record names, and so who may end it. A record that predates the field + * is a detached daemon, which is why an absent value means exactly that. + */ + ownership?: DaemonOwnership; /** The provider plugin inventory as published; `readPluginInventory` validates it where it is read. */ plugins?: unknown; baseUrl?: string; @@ -37,6 +45,38 @@ export type DaemonMetadataState = { hasLock: boolean; }; +/** + * Whether `daemon.json` can address a daemon, and the record when it can. + * + * This is what {@link readDaemonInfo} collapses into `null`, kept apart for the two callers that + * cannot afford the collapse: a stop of a runtime, and a check of a pin. To them, a record that is + * present but unreadable, invalid, or missing what it takes to reach a daemon is not evidence that no + * daemon runs — it is evidence that something holds this state directory and cannot be reached, which + * is a refusal they have to state, not a clean absence. + */ +export type DaemonRecordAddress = + | Readonly<{ status: 'addressable'; info: DaemonInfo }> + /** No `daemon.json` at all: nothing claims this state directory. */ + | Readonly<{ status: 'absent' }> + /** `daemon.json` exists but is not readable JSON, so nothing can be said about it. */ + | Readonly<{ status: 'unreadable' }> + /** + * A readable record that names no way to reach a daemon: no token, or no port. Something published + * it, so this state directory is claimed by a process this client cannot address. + */ + | Readonly<{ status: 'incomplete' }>; + +export function readDaemonRecordAddress(infoPath: string): DaemonRecordAddress { + const read = readDaemonRecord(infoPath); + if (read.status === 'absent') return { status: 'absent' }; + if (read.status !== 'present') return { status: 'unreadable' }; + const parsed = read.record as Partial; + const token = readRequiredDaemonToken(parsed); + const ports = readDaemonInfoPorts(parsed); + if (!token || !ports) return { status: 'incomplete' }; + return { status: 'addressable', info: readDaemonInfoFromRecord(parsed, token, ports) }; +} + export function readDaemonInfo(infoPath: string): DaemonInfo | null { const read = readDaemonRecord(infoPath); if (read.status !== 'present') return null; @@ -45,6 +85,27 @@ export function readDaemonInfo(infoPath: string): DaemonInfo | null { if (!token) return null; const ports = readDaemonInfoPorts(parsed); if (!ports) return null; + return readDaemonInfoFromRecord(parsed, token, ports); +} + +/** + * The observation a retirement judges, from a record this client already read. Ownership travels with + * the lifetime because a retirement signals only a process its record proves is a daemon of its own, and + * a client that read the record once should not read it again to answer that. + */ +export function observedDaemonIdentityOf(info: DaemonInfo): ObservedDaemonIdentity { + return { + pid: info.pid, + startTime: info.processStartTime ?? null, + ownership: readDaemonOwnership(info.ownership), + }; +} + +function readDaemonInfoFromRecord( + parsed: Partial, + token: string, + ports: Pick, +): DaemonInfo { return { token, ...ports, @@ -56,6 +117,7 @@ export function readDaemonInfo(infoPath: string): DaemonInfo | null { processStartTime: readOptionalString(parsed.processStartTime), policyDigest: readOptionalString(parsed.policyDigest), capabilities: readDaemonCapabilities(parsed.capabilities), + ownership: readDaemonOwnership(parsed.ownership), ...(Array.isArray(parsed.plugins) ? { plugins: parsed.plugins } : {}), }; } diff --git a/src/daemon-client/daemon-client-quiesce.ts b/src/daemon-client/daemon-client-quiesce.ts deleted file mode 100644 index 6a65992624..0000000000 --- a/src/daemon-client/daemon-client-quiesce.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { - DAEMON_QUIESCE_REASONS, - DAEMON_SHUTTING_DOWN_REASON, - buildDaemonQuiesceRequest, - readDaemonQuiesceCensus, -} from '@agent-device/daemon-contracts/daemon-resolution'; -import { createRequestId } from '@agent-device/host-kit/diagnostics'; -import { AppError, throwDaemonError } from '@agent-device/kernel/errors'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; -import type { DaemonClientSettings } from './daemon-client-lifecycle.ts'; -import { isDaemonTransportUnavailableError, sendRequest } from './daemon-client-transport.ts'; -import { daemonBusyError } from './daemon-lifecycle-errors.ts'; - -const QUIESCE_TIMEOUT_MS = 10_000; - -// A refused or reset connection: the daemon left, is leaving, or was never there. Nothing was -// accepted, so judging again is safe. -function isConnectionFailure(error: unknown): boolean { - return error instanceof AppError && error.details?.reason === 'daemon_transport_failure'; -} - -/** - * `draining`: the daemon accepted and is leaving. `gone`: the daemon the request was addressed to - * is no longer there to answer, because it left, is leaving already, or was replaced. - */ -export type QuiesceOutcome = 'draining' | 'gone'; - -/** - * Asks the daemon to leave, fenced to the instance `info` names. A daemon that holds work refuses - * with `daemon-busy`, and that refusal is the answer: nothing here signals it. A timeout is not - * recovered here either; the caller decides what an unresponsive daemon deserves. - */ -export async function requestDaemonQuiesce( - info: DaemonInfo, - settings: DaemonClientSettings, -): Promise { - const request = buildDaemonQuiesceRequest({ - token: info.token, - fence: { pid: info.pid, processStartTime: info.processStartTime ?? null }, - requestId: createRequestId(), - }); - let response; - try { - response = await sendRequest( - info, - request, - // Whichever transport the daemon serves: a client that cannot reach it on its own preference - // is exactly the one that has to ask it to leave. - 'auto', - settings.paths, - QUIESCE_TIMEOUT_MS, - { host: settings.host, timeoutRecovery: 'none' }, - ); - } catch (error) { - if (isDaemonTransportUnavailableError(error) || isConnectionFailure(error)) return 'gone'; - throw error; - } - if (response.ok) return 'draining'; - const reason = response.error.details?.reason; - if (reason === DAEMON_QUIESCE_REASONS.busy) { - throw daemonBusyError( - info, - settings.paths.baseDir, - readDaemonQuiesceCensus(response.error.details), - ); - } - if ( - reason === DAEMON_QUIESCE_REASONS.instanceMismatch || - reason === DAEMON_SHUTTING_DOWN_REASON - ) { - return 'gone'; - } - return throwDaemonError(response.error); -} diff --git a/src/daemon-client/daemon-client-startup.ts b/src/daemon-client/daemon-client-startup.ts index b02220d85c..fc0b4c7adb 100644 --- a/src/daemon-client/daemon-client-startup.ts +++ b/src/daemon-client/daemon-client-startup.ts @@ -32,6 +32,7 @@ import { } from './daemon-launch-spec.ts'; import { getDaemonMetadataState, + observedDaemonIdentityOf, readDaemonInfo, resolveDaemonStartupHint, type DaemonInfo, @@ -39,6 +40,8 @@ import { import { canConnect, isDaemonTransportUnavailableError } from './daemon-client-transport.ts'; import { daemonContendedError, + daemonNotRetirableError, + daemonRecordOwnerUnrecognizedError, daemonRetirementUnconfirmedError, daemonStopCommand, } from './daemon-lifecycle-errors.ts'; @@ -149,6 +152,24 @@ export async function readReusableLocalDaemon( } if (!settings.retireIncompatibleContender) throw daemonContendedError(); + // A host owns an embedded runtime, so a managed client that wants this state directory asks the host + // for it instead of retiring the runtime that holds it. + if (existing.ownership === 'embedded') { + throw daemonNotRetirableError({ + info: existing, + stateDir: settings.paths.baseDir, + owner: 'embedded', + }); + } + // A record stating an owner this build cannot read is not a licence to replace what it names: the + // whole point of reading the owner is to know whether a pid may be signaled, and this says nothing. + if (existing.ownership === 'unknown') { + throw daemonRecordOwnerUnrecognizedError({ + info: existing, + stateDir: settings.paths.baseDir, + wanted: 'replaced', + }); + } if (remainingStartupBudget(deadline) < MINIMUM_DAEMON_TAKEOVER_BUDGET_MS) return null; emitDaemonTakeoverNotice(existing, decision.reason, settings.paths.baseDir); await retireDaemonForTakeover(existing, settings.paths); @@ -196,7 +217,7 @@ export async function stopDaemonForTakeover( ): Promise { return await stopAndRetireDaemon({ paths: paths, - observed: { pid: existing.pid, startTime: existing.processStartTime ?? null }, + observed: observedDaemonIdentityOf(existing), mode, }); } @@ -434,7 +455,11 @@ async function retireStartupAttempt( ): Promise<{ cleanup: DaemonRetirementResult; joined: boolean }> { const cleanup = await stopAndRetireDaemon({ paths: settings.paths, - observed: launch.readIdentity(), + observed: { + ...launch.readIdentity(), + // A daemon this client launched is detached: a host owns a runtime only by starting one itself. + ownership: 'detached' as const, + }, mode: 'graceful', ownedStateDir: settings.ownedStateDir, termTimeoutMs: Math.min(3_000, remainingStartupBudget(deadline)), diff --git a/src/daemon-client/daemon-client-stop.ts b/src/daemon-client/daemon-client-stop.ts new file mode 100644 index 0000000000..5953e6dd97 --- /dev/null +++ b/src/daemon-client/daemon-client-stop.ts @@ -0,0 +1,251 @@ +import { + daemonShutdownCompletionAnswers, + daemonShutdownCompletionStillRetained, + readDaemonShutdownCompletion, + type DaemonShutdownCompletion, + type DaemonShutdownCompletionQuery, +} from '@agent-device/daemon-contracts/daemon-shutdown-report'; +import { waitForDaemonExit } from '@agent-device/daemon-contracts/daemon-process'; +import { + DAEMON_LIFECYCLE_REASONS, + readDaemonLifecycleCensus, + type DaemonPaths, +} from '@agent-device/daemon-contracts/daemon-resolution'; +import type { DaemonStopIdentity } from '@agent-device/daemon-contracts/daemon-registration'; +import { + daemonNotRunningStopResult, + stopDaemon, + type DaemonStopResult, +} from '@agent-device/daemon-contracts/daemon-stop'; +import { AppError } from '@agent-device/kernel/errors'; +import { sleep } from '@agent-device/host-kit/retry'; +import { readDaemonRecordAddress, type DaemonInfo } from './daemon-client-metadata.ts'; +import { + DAEMON_LEAVE_WAIT_MS, + requestDaemonLifecycle, + type DaemonLifecycleTarget, +} from './daemon-client-lifecycle-request.ts'; + +// How long a stop keeps asking while the runtime refuses because it is still starting up and has no +// drain path to run yet. +const EMBEDDED_STOP_ASK_BUDGET_MS = 5_000; +const EMBEDDED_STOP_ASK_GAP_MS = 250; +const EMBEDDED_STOP_POLL_MS = 100; +// How often the wait asks whether the runtime's host is still alive. A lifetime read costs a process +// table lookup, and the answer changes on a much slower clock than the record does. +const EMBEDDED_STOP_LIFETIME_POLL_MS = 1_000; + +/** + * Stops the daemon a state directory registers, by the owner its record states. + * + * The caller hands over the record it read so routing, retirement, and a `--clean` walk judge one + * observation of a file another process may be rewriting. Only a detached daemon is signaled: a host owns + * an embedded runtime and ends it with its own close, so this reaches it over its transport with a + * shutdown request. A record this build cannot place is asked the same way and the runtime answers with + * the rule it enforces. A record that cannot be read gets neither: it may describe a runtime that is + * serving, so this refuses rather than reporting an absence. + */ +export async function stopRecordedDaemon( + record: DaemonStopIdentity, + target: DaemonLifecycleTarget, +): Promise { + if (record.ownership === 'unreadable') throw daemonRecordUnreadableError(target.paths); + if (record.ownership === 'embedded' || record.ownership === 'unknown') { + return await stopEmbeddedDaemon(target); + } + const observed = record.identity ? { ...record.identity, ownership: 'detached' as const } : null; + return await stopDaemon({ paths: target.paths, observed }); +} + +/** + * A record that is present and cannot be read: not an absence, and nothing to address. A stop signals + * nothing and cannot ask a runtime it cannot reach, so it names the record instead of guessing that the + * daemon already left. + */ +function daemonRecordUnreadableError(paths: DaemonPaths): AppError { + return new AppError( + 'COMMAND_FAILED', + `The daemon record on ${paths.baseDir} cannot be read, so this stop cannot tell whether a daemon runs.`, + { + reason: 'daemon-record-unreadable', + stateDir: paths.baseDir, + hint: 'Nothing was signaled. Inspect or remove that record, then retry.', + }, + ); +} + +/** + * Stops the runtime a host process owns, over the transport its record publishes. + * + * Nothing here signals a process. A host owns this runtime, so the only stop available from outside is + * the authenticated shutdown request the runtime answers itself, followed by the bounded completion + * record it leaves when its close has a report to give. + */ +async function stopEmbeddedDaemon(target: DaemonLifecycleTarget): Promise { + const deadlineMs = Date.now() + DAEMON_LEAVE_WAIT_MS; + // Asking needs a transport and a token, which a record can state an owner and still omit. Such a + // record is not an absence: something holds this state directory and this stop cannot address it, + // which is a refusal rather than nothing running. + const address = readDaemonRecordAddress(target.paths.infoPath); + if (address.status === 'absent') return daemonNotRunningStopResult(); + // One reason for one fact: a record that cannot be read says so the same way on every stop route. + if (address.status === 'unreadable') throw daemonRecordUnreadableError(target.paths); + if (address.status !== 'addressable') { + throw daemonEmbeddedRecordUnaddressableError(target.paths); + } + const info = address.info; + // An answer of `gone` is waited on the same way as an agreement: a daemon that stopped answering has + // either left, is leaving, or was replaced, and the two reads below are what tells those apart. + await askToLeave(info, target, Date.now() + EMBEDDED_STOP_ASK_BUDGET_MS); + const waited = await waitForClose(target.paths, instanceOf(info), deadlineMs); + if (waited === 'instance-gone') return daemonNotRunningStopResult(); + if (!waited) throw daemonShutdownUnconfirmedError(info, target.paths); + return stopResultFromCompletion(waited); +} + +/** The instance a stop addresses: the record's own lifetime, which is what a record is matched on. */ +function instanceOf(info: { pid: number; processStartTime?: string }) { + return { pid: info.pid, processStartTime: info.processStartTime ?? '' }; +} + +/** + * Sends the shutdown request, asking again while the runtime refuses because it is starting up. What it + * answers is not this function's business: an agreement, a refusal that the daemon is already leaving, + * and a connection that cannot be made all send the stop to the same wait for evidence. + */ +async function askToLeave( + info: Parameters[0], + target: DaemonLifecycleTarget, + askUntilMs: number, +): Promise { + for (;;) { + try { + await requestDaemonLifecycle(info, target, 'shutdown'); + return; + } catch (error) { + // A runtime that is still starting has no drain path and says so with its census. That is the + // one refusal worth asking again about: it will be able to drain in a moment. + if (!refusedWhileStartingUp(error) || Date.now() >= askUntilMs) throw error; + } + await sleep(EMBEDDED_STOP_ASK_GAP_MS); + } +} + +function refusedWhileStartingUp(error: unknown): boolean { + return ( + error instanceof AppError && + error.details?.reason === DAEMON_LIFECYCLE_REASONS.busy && + readDaemonLifecycleCensus(error.details)?.startingUp === true + ); +} + +/** + * Reads the record the closing runtime publishes, and notices when its writer can no longer publish one: + * the instance's own lifetime is provably over, so what is on disk is the whole truth and nothing is left + * to wait for. + * + * Both reads are needed, and each answers what the other cannot. The record answers by instance rather + * than by request, so a second stop of the same runtime reads the same answer instead of sitting out its + * budget for a record written for somebody else's request. The lifetime check answers the two cases the + * record cannot: a host that died before writing its report would otherwise leave a stop waiting on a + * writer that is gone, and a record of a retained registration is evidence only while some process is + * left holding it. + */ +async function waitForClose( + paths: DaemonPaths, + instance: DaemonShutdownCompletionQuery, + deadlineMs: number, +): Promise { + let nextLifetimeCheckMs = 0; + // Whether the instance is still the process these records name. It starts unknown, and the only record + // that can be read before the first probe is one that answers on its own, so nothing is trusted on an + // assumption. + let instanceIsLive = false; + while (Date.now() < deadlineMs) { + const completion = readDaemonShutdownCompletion(paths.baseDir); + // A record of a close that just happened is the writer's own word and settles this stop at once, + // whatever else is true about the instance. + if (completion && daemonShutdownCompletionAnswers(completion, instance)) return completion; + if (Date.now() >= nextLifetimeCheckMs) { + nextLifetimeCheckMs = Date.now() + EMBEDDED_STOP_LIFETIME_POLL_MS; + if (await instanceEnded(instance)) return 'instance-gone'; + instanceIsLive = true; + } + // A record of a registration this instance still holds answers too, and the probe above is what makes + // it evidence rather than a dead host's leftovers. + if ( + completion && + instanceIsLive && + daemonShutdownCompletionStillRetained(completion, instance) + ) { + return completion; + } + await sleep(Math.min(EMBEDDED_STOP_POLL_MS, Math.max(0, deadlineMs - Date.now()))); + } + return null; +} + +async function instanceEnded(instance: DaemonShutdownCompletionQuery): Promise { + if (!instance.processStartTime.trim()) return false; + const identity = { pid: instance.pid, startTime: instance.processStartTime }; + return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited; +} + +/** + * The stop's own verdict, from what the runtime said about its close. Claim and provider-release + * results come from the shutdown report the close wrote, which the caller merges as it does for any + * graceful stop; what the record adds is the work the close could not finish. + */ +function stopResultFromCompletion(completion: DaemonShutdownCompletion): DaemonStopResult { + // Only a close that both released its registration and finished its work lets this stop vouch for + // what was cleaned up; anything else says the state directory may still be held. + const released = completion.outcome === 'released'; + return { + stopped: true, + mode: 'graceful', + cleanupConfidence: released && completion.unfinished.length === 0 ? 'known' : 'unknown', + claimsReleased: [], + claimsOrphaned: [], + claimsSuperseded: [], + claimsUnattributable: [], + providerReleases: { status: 'completed', released: [], pending: [] }, + warnings: [ + ...completion.unfinished, + // A runtime that could not cancel its own work stops serving but keeps its registration and + // lock, because releasing them would let a second daemon drive the same devices. + ...(released + ? [] + : [ + 'The runtime stopped serving and keeps its registration: a host process still owns it, so nothing outside that process may signal it.', + ]), + ], + }; +} + +/** + * A readable record that names a host-owned runtime and no way to reach it: the stop has no instrument + * left. It signals nothing, so it cannot end what it cannot reach, and it says so instead of guessing + * that the record describes a daemon that already left. + */ +function daemonEmbeddedRecordUnaddressableError(paths: DaemonPaths): AppError { + return new AppError( + 'COMMAND_FAILED', + `The record on ${paths.baseDir} does not name a transport and a token, so the runtime it describes cannot be reached.`, + { + reason: 'daemon-embedded-stop-unreachable', + stateDir: paths.baseDir, + record: 'incomplete', + hint: 'Nothing was signaled. Inspect the record, or close the host that owns this runtime.', + }, + ); +} + +function daemonShutdownUnconfirmedError(info: DaemonInfo, paths: DaemonPaths): AppError { + return new AppError('COMMAND_FAILED', 'The runtime did not report that it finished leaving.', { + reason: 'daemon-shutdown-unconfirmed', + pid: info.pid, + ...(info.processStartTime ? { processStartTime: info.processStartTime } : {}), + stateDir: paths.baseDir, + hint: 'It was not signaled, and it may still be serving. Ask the host that owns it to close it, or retry the stop.', + }); +} diff --git a/src/daemon-client/daemon-client-timeout.ts b/src/daemon-client/daemon-client-timeout.ts index 4b33a04b40..550e7d3887 100644 --- a/src/daemon-client/daemon-client-timeout.ts +++ b/src/daemon-client/daemon-client-timeout.ts @@ -5,7 +5,7 @@ import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; import { resolveCommandTimeoutPolicy } from '@agent-device/command-registry/registry'; import type { DaemonPaths } from '@agent-device/daemon-contracts/daemon-resolution'; import type { PlatformSelector } from '@agent-device/kernel/device'; -import type { DaemonInfo } from './daemon-client-metadata.ts'; +import { observedDaemonIdentityOf, type DaemonInfo } from './daemon-client-metadata.ts'; // Recovery for a timed-out request is scoped to that request or to nothing. The daemon owns // request-scoped recovery (#3177): destroying the timed-out connection cancels exactly that @@ -120,7 +120,7 @@ async function forceRetireDaemon( await import('@agent-device/daemon-contracts/daemon-registration-owner'); return await stopAndRetireDaemon({ paths: statePaths, - observed: { pid: info.pid, startTime: info.processStartTime ?? null }, + observed: observedDaemonIdentityOf(info), mode: 'force', }); } diff --git a/src/daemon-client/daemon-client.ts b/src/daemon-client/daemon-client.ts index 22d4a5b7a7..2b09e75e64 100644 --- a/src/daemon-client/daemon-client.ts +++ b/src/daemon-client/daemon-client.ts @@ -64,7 +64,7 @@ async function sendToDaemonOnHost( options: DaemonTransportOptions, host: HostEnvironment, ): Promise { - const { resolveClientSettings, attachSessionAddressHints } = + const { resolveClientSettings, resolveTimeoutRecovery, attachSessionAddressHints } = await import('./daemon-client-lifecycle.ts'); const { connectToDaemon } = await import('./daemon-client-attach.ts'); const requestId = req.meta?.requestId ?? createRequestId(); @@ -149,8 +149,7 @@ async function sendToDaemonOnHost( host, onProgress: options.onProgress, signal: options.signal, - timeoutRecovery: - settings.daemonLifecycle === 'attach' ? 'none' : 'retire-unresponsive', + timeoutRecovery: resolveTimeoutRecovery(settings, info), }, ), { requestId, command: req.command }, diff --git a/src/daemon-client/daemon-lifecycle-errors.ts b/src/daemon-client/daemon-lifecycle-errors.ts index 3f1e2de7dc..003df89460 100644 --- a/src/daemon-client/daemon-lifecycle-errors.ts +++ b/src/daemon-client/daemon-lifecycle-errors.ts @@ -1,8 +1,10 @@ import type { DaemonIdentity } from '@agent-device/contracts/client'; import { - DAEMON_QUIESCE_REASONS, + DAEMON_LIFECYCLE_REASONS, + DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, + type DaemonLifecycleCensus, + type DaemonOwnership, type DaemonPaths, - type DaemonQuiesceCensus, } from '@agent-device/daemon-contracts/daemon-resolution'; import type { DaemonRetirementResult } from '@agent-device/daemon-contracts/daemon-registration-owner'; import { AppError } from '@agent-device/kernel/errors'; @@ -16,12 +18,14 @@ import { } from './daemon-client-metadata.ts'; import { resolveLocalDaemonCodeIdentity, type DaemonTakeoverCause } from './daemon-launch-spec.ts'; -// The four ways an explicit lifecycle stops short of a usable daemon. Callers branch on -// `details.reason`, so each reason is one constant here and the messages are free to change. -const DAEMON_LIFECYCLE_REASONS = { +// The ways an explicit lifecycle stops short of a usable daemon, whether this client judged it or the +// daemon answered it: `busy` is the daemon's own refusal, carried through so one constant names it on +// both sides. Callers branch on `details.reason`, so each reason is one constant here and the messages +// are free to change. +const DAEMON_UNAVAILABLE_REASONS = { notRunning: 'daemon-not-running', incompatible: 'daemon-incompatible', - busy: DAEMON_QUIESCE_REASONS.busy, + busy: DAEMON_LIFECYCLE_REASONS.busy, drainTimeout: 'daemon-drain-timeout', } as const; @@ -64,7 +68,7 @@ export function daemonContendedError(): AppError { export function daemonNotRunningError(stateDir: string, registered?: DaemonInfo): AppError { return new AppError('COMMAND_FAILED', 'No daemon is running for this state directory.', { - reason: DAEMON_LIFECYCLE_REASONS.notRunning, + reason: DAEMON_UNAVAILABLE_REASONS.notRunning, stateDir, ...(registered ? { daemon: daemonIdentityOf(registered) } : {}), hint: `Start one with a local agent-device command that uses the daemon (agent-device devices --state-dir ${shellQuoteIfNeeded(stateDir)}, for example), or with ensureDaemon({ stateDir: ${JSON.stringify(stateDir)} }) from the SDK, before attaching.${registered ? ` The registered daemon (pid ${registered.pid}) does not answer; ${daemonStopCommand(stateDir)} clears it.` : ''}`, @@ -83,7 +87,9 @@ export function daemonRetirementUnconfirmedError( } export function isDaemonNotRunningError(error: unknown): boolean { - return error instanceof AppError && error.details?.reason === DAEMON_LIFECYCLE_REASONS.notRunning; + return ( + error instanceof AppError && error.details?.reason === DAEMON_UNAVAILABLE_REASONS.notRunning + ); } export async function daemonIncompatibleError(params: { @@ -93,7 +99,7 @@ export async function daemonIncompatibleError(params: { }): Promise { const { info, mismatch, stateDir } = params; return new AppError('COMMAND_FAILED', 'The running daemon cannot serve this client.', { - reason: DAEMON_LIFECYCLE_REASONS.incompatible, + reason: DAEMON_UNAVAILABLE_REASONS.incompatible, mismatch, stateDir, daemon: daemonIdentityOf(info), @@ -111,7 +117,7 @@ export async function daemonDrainTimeoutError( 'COMMAND_FAILED', 'The daemon accepted the request to leave but is still running.', { - reason: DAEMON_LIFECYCLE_REASONS.drainTimeout, + reason: DAEMON_UNAVAILABLE_REASONS.drainTimeout, stateDir, daemon: daemonIdentityOf(info), hint: `It was not signaled. Retry once it has exited, or stop it deliberately with ${daemonStopCommand(stateDir)}.`, @@ -122,10 +128,10 @@ export async function daemonDrainTimeoutError( export function daemonBusyError( info: DaemonInfo, stateDir: string, - census: DaemonQuiesceCensus | undefined, + census: DaemonLifecycleCensus | undefined, ): AppError { return new AppError('COMMAND_FAILED', 'The running daemon is busy and was not replaced.', { - reason: DAEMON_LIFECYCLE_REASONS.busy, + reason: DAEMON_UNAVAILABLE_REASONS.busy, stateDir, daemon: daemonIdentityOf(info), ...census, @@ -134,3 +140,70 @@ export function daemonBusyError( : `Wait for its requests, sessions and leases to end and retry, or stop it deliberately with ${daemonStopCommand(stateDir)}.`, }); } + +/** + * A record states an owner this build does not recognize. Nothing may act on it: replacing the daemon it + * names would signal a process whose owner is unknown, `--clean` would walk a tree that may be a host's, + * and a stop would have to trust a lifetime no owner vouches for. So every route refuses it, and this is + * the refusal they share. + * + * `verb` says what this caller wanted, since the caller knows whether it came from `daemon stop`, + * `ensureDaemon()`, or a takeover. The hint names the record and never `daemon stop`: that command refuses + * the same record for this same reason, so pointing at it would send the operator to a second refusal. + */ +export function daemonRecordOwnerUnrecognizedError(params: { + info: DaemonInfo; + stateDir: string; + wanted: 'replaced' | 'stopped'; +}): AppError { + const { info, stateDir, wanted } = params; + return new AppError( + 'COMMAND_FAILED', + `This daemon record states an owner this version cannot read, so its daemon was not ${wanted}.`, + { + reason: DAEMON_RECORD_OWNER_UNRECOGNIZED_REASON, + stateDir, + daemon: daemonIdentityOf(info), + hint: `Update agent-device if the record comes from a newer build. Otherwise check what owns pid ${info.pid}, then repair the record: stop that process by its own means and remove ${shellQuoteIfNeeded(stateDir)}/daemon.json so a daemon can register again.`, + }, + ); +} + +/** + * A caller wanted this daemon gone, and its owner's rule says otherwise: a host process owns an + * embedded runtime and ends it, and a detached daemon is stopped by signalling its own process, which is + * what `daemon stop` does for a record that proves that owner. + * + * Both cases mean the same thing to a caller: this daemon is not yours to retire. `owner` is the + * daemon's own answer about itself, so it is trusted over this process's read of the record — except for + * the hint, which reads the record too. A detached answer against a record this build cannot place is a + * corrupt state directory, and `daemon stop` refuses that record for the same reason this client just + * did, so its hint names the record repair instead of sending the operator to a second refusal. + */ +export function daemonNotRetirableError(params: { + info: DaemonInfo; + stateDir: string; + owner: Exclude; +}): AppError { + const { info, stateDir, owner } = params; + const embedded = owner === 'embedded'; + const recordUnreadable = !embedded && info.ownership === 'unknown'; + return new AppError( + 'COMMAND_FAILED', + embedded + ? 'A host process owns this daemon, so it was not replaced.' + : 'This daemon is not embedder-owned, so a shutdown request is not how it is stopped.', + { + reason: embedded + ? DAEMON_LIFECYCLE_REASONS.ownerEmbedded + : DAEMON_LIFECYCLE_REASONS.notEmbedded, + stateDir, + daemon: daemonIdentityOf(info), + hint: embedded + ? `Close the host that started it, or ask this daemon to leave with ${daemonStopCommand(stateDir)}, which sends it a shutdown request instead of a signal.` + : recordUnreadable + ? `The daemon says it is a daemon of its own, so the owner its record states is wrong. Nothing was signaled. Check what owns pid ${info.pid}, then repair the record: stop that process by its own means and remove ${shellQuoteIfNeeded(stateDir)}/daemon.json so a daemon can register again.` + : `Stop it with ${daemonStopCommand(stateDir)}.`, + }, + ); +} diff --git a/test/integration/smoke-daemon-clean.test.ts b/test/integration/smoke-daemon-clean.test.ts index 017c532faf..cffeff2f29 100644 --- a/test/integration/smoke-daemon-clean.test.ts +++ b/test/integration/smoke-daemon-clean.test.ts @@ -96,7 +96,13 @@ test('forced retirement waits for a real daemon and reclaims its abandoned regis assert.equal(fs.existsSync(paths.lockPath), true); const result = await stopAndRetireDaemon({ paths, - observed: { pid: info.pid, startTime: info.processStartTime ?? null }, + // The record is the one this row's CLI-started daemon published of itself, which is a detached + // daemon: what the row retires is that lifetime, under that owner. + observed: { + pid: info.pid, + startTime: info.processStartTime ?? null, + ownership: 'detached', + }, mode: 'force', }); assert.equal(result.status, 'retired', JSON.stringify(result)); diff --git a/website/docs/docs/client-api.md b/website/docs/docs/client-api.md index 7fa0b9daf3..5d0e3ef44d 100644 --- a/website/docs/docs/client-api.md +++ b/website/docs/docs/client-api.md @@ -90,6 +90,8 @@ if ('cloudArtifacts' in result) { A client talks to one local daemon per state directory. `daemonLifecycle` says who owns that daemon. +A daemon that a host process starts inside itself records that owner. Two things end it: the host's own close, and an authenticated shutdown request over the daemon's registered local transport. No other process signals it. A `'managed'` client uses such a daemon while it matches, as it does with any other. When such a daemon does not match, that client does not replace it, and it never retires one after a timeout: both answer `daemon-owner-embedded`, and closing the host is the fix. `agent-device daemon stop --state-dir ` makes that shutdown request and waits for the runtime to report its own close, rather than signaling a process that runs more than the daemon. A record whose owner this version cannot read gets the same request, because the running process answers for itself. That stop sends nothing to a `daemon.json` it cannot read and fails with `daemon-record-unreadable` instead of reporting that nothing runs, and it fails with `daemon-embedded-stop-unreachable` or `daemon-shutdown-unconfirmed` when a host-owned runtime cannot be reached or never reports its close. A runtime that closed while it still held work keeps its registration and records what it kept; a later stop reads that record, whenever the same host process still holds the registration, and reports the retention instead of waiting on a listener that has gone. + | Mode | What the client does with the daemon | | --- | --- | | `'managed'` (default) | Starts a missing daemon, replaces one that does not match the client, and retires an unresponsive one after a timeout. | @@ -119,6 +121,8 @@ These failures carry a typed `error.details.reason`: | `daemon-incompatible` | The running daemon does not match the client. `details.daemon` and `details.client` hold both identities. Stop it with `agent-device daemon stop --state-dir `, then call `ensureDaemon()`. A daemon from a release that predates quiescing cannot be asked to leave, so `ensureDaemon()` reports it the same way. | | `daemon-busy` | `ensureDaemon()` found an incompatible daemon that still holds work. `details` counts `inFlightRequests`, `sessions`, `repairSessions` and `leases`. Wait for them to end and retry. | | `daemon-drain-timeout` | A daemon accepted the request to leave but was still running 60 s later and could not be force-retired, either because it still answers or because its process cannot be proven to be a daemon (an embedded host). It was not signaled. Retry once it has exited, or stop it with `agent-device daemon stop --state-dir `. | +| `daemon-owner-embedded` | A host process owns the daemon on `stateDir`, so this client neither replaced it nor retired it, and signaled nothing. A daemon that matches the client is still usable. Close that host, or stop it with `agent-device daemon stop --state-dir `. | +| `daemon-owner-unrecognized` | The record on `stateDir` states an owner this version cannot read, so this client replaced nothing and signaled nothing. Update agent-device if the record comes from a newer build. Otherwise check what owns that process: a host that started it closes it with its own close, and otherwise remove `daemon.json` once that process is gone. | `ensureDaemon()` only ever manages the local daemon on `stateDir`. A client with `daemonBaseUrl` never starts or stops a local daemon in either mode, and `daemonLifecycle` does not change `daemonTransport` or `daemonServerMode`. diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index 1ccb0846c2..5611452a7b 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -85,7 +85,8 @@ Device claims and the daemon: - `device status` lists host-local device claims without starting or contacting a daemon; `--stale` shows the stale claims it hides by default (owner exited, PID reused, state directory deleted, or daemon replaced), some of whose owners may still be running. Scope it with `--platform` plus `--udid` (Apple) or `--serial` (Android). A claim held by another live or uncertain owner blocks `open`, unless the device rebooted after the claim was taken: then `open` settles that owner's durable resources, takes the claim, and reports the release in `warnings`. - `device release --stale` settles the durable resources of an owner that provably can't release its claim (its process exited or a newer daemon replaced it), then clears the claim. Live, uncertain, PID-reused, corrupt, and deleted-state-directory claims are refused with the reason. See [Sessions](/docs/sessions#release-a-device-another-session-or-workspace-holds). - A command that needs the shared local daemon starts it for its state directory when none runs; `device status` and `device release --stale` never contact one. When the running daemon comes from another version or build, the command replaces it only if it reports no work: no request in flight, no session, no repair session and no live lease. A resumable upload still streaming and a host human-control hold that names no lease are not counted, so a replacement can end them. Otherwise the command fails with `daemon-busy` and leaves the daemon alone. A daemon from a release that predates this check is never replaced, and fails with `daemon-incompatible`. End a busy or older daemon deliberately with `daemon stop`. Node.js clients choose this per client; see [Daemon lifecycle](/docs/client-api#daemon-lifecycle). -- `daemon stop --state-dir ` stops a local daemon after verifying its identity. `--clean` also removes the Apple runner processes and leases that daemon retained. +- `daemon stop --state-dir ` stops a recorded daemon with an authenticated shutdown request over its registered local transport and reports what that close could not finish as warnings. `--clean` also removes the Apple runner processes and leases that daemon retained. +- A daemon a host process started for itself is stopped the same way, and nothing else: `daemon stop` sends it the shutdown request and waits for it to report its own close, so no process outside the host is signaled. A record whose owner this version cannot recognize gets the same request, because the running process answers for itself and one reporting itself `detached` is safe to stop. If its record names no transport and token to send that request on, the stop fails with `details.reason: daemon-embedded-stop-unreachable`; stop it from the host that started it. A `daemon.json` that exists but cannot be read fails the stop with `details.reason: daemon-record-unreadable`: the record proves nothing, so no process is signalled. `--clean` is refused with `details.reason: daemon-clean-not-detached` for a record that does not prove the process is a daemon of its own — a host-owned record, one whose owner this version cannot recognize, or one it cannot read — because that cleanup walks the owner's whole process tree and a host process runs more than the daemon. Stopping without `--clean` still drains the first two. `daemon status` names such a record `host-owned`, or `owner-not-recognized`. See [Daemon lifecycle](/docs/client-api#daemon-lifecycle). - `daemon status [--state-dir ] [--json]` reads the daemon's record without contacting it. For a running daemon it lists every provider it registered, built in or installed, and the reason any plugin failed to register. See [provider plugins](/docs/plugins). Open apps and links: