Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion packages/daemon-contracts/src/daemon-process.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,22 @@ export type DaemonTerminationResult =
}>
| Readonly<{
status: 'retained';
reason: 'missing-start-time' | 'identity-unverified' | 'signal-failed' | 'exit-timeout';
reason:
| 'missing-start-time'
| 'identity-unverified'
/**
* A host owns this runtime and ends it. Callers that judge a registration answer this instead of
* calling {@link stopDaemonProcess} at all (`stopAndRetireDaemon`); this function signals
* whatever verified identity it is handed and never decides ownership itself.
*/
| 'embedded-owner'
/**
* The record states an owner this build cannot read, so nothing proves this pid is a daemon of
* its own, and signalling it is refused. The other half of the same rule as `embedded-owner`.
*/
| 'owner-unproven'
| 'signal-failed'
| 'exit-timeout';
signal?: NodeJS.Signals;
}>;

Expand Down
193 changes: 174 additions & 19 deletions packages/daemon-contracts/src/daemon-registration-owner.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@ import type { DaemonCodeOrigin } from '@agent-device/host-kit/code-signature';
import {
resolveDaemonPaths,
type DaemonCapability,
type DaemonOwnership,
type DaemonPaths,
type DaemonServerMode,
type ObservedDaemonIdentity,
} from './daemon-resolution.ts';
import { findUnrecoveredRepairCommitFailure } from './session-repair-tombstone.ts';
import {
Expand All @@ -40,9 +42,13 @@ import {
type RegisteredDaemonOwnership,
} from './daemon-registration.ts';
import {
buildDaemonShutdownCompletion,
buildDaemonShutdownReport,
resolveDaemonShutdownCompletionPath,
resolveDaemonShutdownReportPath,
type DaemonShutdownCompletion,
type DaemonShutdownOutcome,
type DaemonShutdownStopAnswer,
} from './daemon-shutdown-report.ts';
import { sweepDeadDaemonStartupPhasesFiles } from './daemon-startup-phases.ts';

Expand All @@ -59,13 +65,36 @@ export type DaemonRegistrationFields = Readonly<{
capabilities?: readonly DaemonCapability[];
/** The provider plugin inventory, so a client can read what this daemon registered. */
plugins?: readonly PluginInventoryEntry[];
/** Who owns this process; see {@link DaemonOwnership}. Absent records are detached daemons. */
ownership?: DaemonOwnership;
}>;
type DaemonRegistrationRemoval =
| Readonly<{ state: 'removed' }>
| Exclude<RegisteredDaemonOwnership, { state: 'match' }>;
export type DaemonRegistrationOwner = Readonly<{
publish(fields: DaemonRegistrationFields): void;
finish(outcome?: DaemonShutdownOutcome): Promise<DaemonRegistrationRemoval>;
/**
* Records that this instance kept the state directory on purpose, for the stop that addressed it.
* The release answer is NOT written here: {@link DaemonRegistrationOwner.finish} writes it from the
* removal it proves. Returns whether the record was written: a process that no longer holds the
* claim has nothing to say about this state dir.
*/
recordShutdownRetention(unfinished: readonly string[]): boolean;
/**
* Writes this instance's shutdown report, removes its registration, and answers the stop that asked
* it to leave — all under the claim this instance holds, and before it is released. The answer's
* disposition is what this call proves happened, never what the caller expected: a registration the
* unlink could not remove, or a step that threw before it, says `retained`.
*
* `unfinished` states what this close left outstanding. It matters when `completion` is absent, which
* is a close no stop asked for: the removal can still fail, and a state dir that stays claimed gets
* its retention record whatever the ledger says, because that record is what the next stop reads.
*/
finish(
outcome?: DaemonShutdownOutcome,
completion?: DaemonShutdownStopAnswer,
unfinished?: readonly string[],
): Promise<DaemonRegistrationRemoval>;
}>;

export async function tryAcquireDaemonRegistration(
Expand All @@ -87,6 +116,7 @@ export async function tryAcquireDaemonRegistration(
acquisition.assertHeld();
fs.rmSync(resolveDaemonShutdownReportPath(boundPaths.baseDir), { force: true });
sweepDeadDaemonStartupPhasesFiles(boundPaths.baseDir);
fs.rmSync(resolveDaemonShutdownCompletionPath(boundPaths.baseDir), { force: true });
} catch (error) {
await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath);
}
Expand Down Expand Up @@ -116,14 +146,56 @@ export async function tryAcquireDaemonRegistration(
mode: 0o600,
});
},
async finish(outcome?: DaemonShutdownOutcome) {
recordShutdownRetention(unfinished: readonly string[]) {
// The record only means something while this process provably holds the state dir: a claim
// someone else reclaimed means this runtime is no longer the one a stop is asking about.
return writeShutdownCompletion(
boundPaths.baseDir,
identity,
'retained',
unfinished,
acquisition,
);
},
async finish(
outcome?: DaemonShutdownOutcome,
completion?: DaemonShutdownStopAnswer,
unfinished?: readonly string[],
) {
// What this close reports as its work in hand. A call that is not a close — startup rollback —
// states nothing and so publishes no record: it never had a runtime that could keep serving.
const closeReport = completion ? completion.unfinished : unfinished;
let removal: DaemonRegistrationRemoval;
try {
if (outcome) writeShutdownReport(boundPaths.baseDir, outcome, acquisition);
removal = removeRegistrationUnderLock(boundPaths.infoPath, identity, acquisition);
} catch (error) {
// A release that never ran kept the registration, which is the disposition a stop is owed
// before this claim goes away and this process can say nothing more. It is owed to whoever
// reads that state dir next, not only to a stop that asked.
if (closeReport) {
writeShutdownCompletion(
boundPaths.baseDir,
identity,
'retained',
closeReport,
acquisition,
);
}
return await releaseRegistrationAfterFailure(acquisition, error, boundPaths.logPath);
}
// The disposition comes from the removal this call just proved, not from what the caller
// hoped: a registration an unlink could not remove is `retained` whatever the close planned.
const released = removal.state === 'removed' || removal.state === 'absent';
if (closeReport && (completion || !released)) {
writeShutdownCompletion(
boundPaths.baseDir,
identity,
released ? 'released' : 'retained',
closeReport,
acquisition,
);
}
await acquisition.release();
return removal;
},
Expand Down Expand Up @@ -176,6 +248,40 @@ function writeShutdownReport(
}
}

/**
* Writes this instance's completion answer under the claim it holds. It names the lifetime that wrote
* it, which is what a stop matches; a process that cannot prove its own start time writes nothing,
* because a record no stop can attribute to an instance would be read as somebody else's answer.
* The write follows the rule every file inside a held claim follows: one atomic publish at 0600, so a
* stop never half-reads its answer and a stranger cannot read what a daemon says about its own close.
*/
function writeShutdownCompletion(
stateDir: string,
identity: OwnerIdentity,
outcome: DaemonShutdownCompletion['outcome'],
unfinished: readonly string[],
acquisition: ProcessLockAcquisition,
): boolean {
if (identity.startTime === null) return false;
try {
acquisition.assertHeld();
publishFileSync({
destination: resolveDaemonShutdownCompletionPath(stateDir),
contents: `${JSON.stringify(
buildDaemonShutdownCompletion(
{ pid: identity.pid, processStartTime: identity.startTime },
outcome,
{ unfinished },
),
)}\n`,
mode: 0o600,
});
return true;
} catch {
return false;
}
}

/**
* The daemon's stdout and stderr append to this inode, so it is emptied in place, never
* replaced. On POSIX the append handle that creates the file also permits truncation, and
Expand Down Expand Up @@ -333,7 +439,13 @@ function requirePrivateReplayState(

export type DaemonRetirementInput = Readonly<{
paths: DaemonPaths;
observed: OwnerIdentity | null;
/**
* The lifetime a caller proved the daemon is, with the owner that record stated at the read. A caller
* that launched the process itself states `detached`, which is what it knows; a caller that read
* `daemon.json` passes what that record stated. Signalling on a guessed owner is the one thing this
* module must not do, so no caller may leave this out.
*/
observed: ObservedDaemonIdentity | null;
termTimeoutMs?: number;
killTimeoutMs?: number;
lockTimeoutMs?: number;
Expand Down Expand Up @@ -363,6 +475,13 @@ export type DaemonRetirementResult =
| 'lock-busy'
| 'registration-replaced'
| 'metadata-unreadable'
/** A host owns the observed runtime; only that host, or its shutdown request, ends it. */
| 'embedded-owner'
/**
* The record states an owner this build cannot read, so nothing proves the pid it names is a
* daemon of its own, and signalling it is refused.
*/
| 'owner-unproven'
| 'retirement-unconfirmed';
error?: NormalizedError;
repairCommitFailure?: RepairCommitFailure;
Expand Down Expand Up @@ -395,14 +514,24 @@ export async function stopAndRetireDaemon(
error: normalizeError(error),
};
}
// Signalling needs a proven owner, and the observation carries the one its record stated: a record
// naming a host describes a runtime that host ends, and a record stating an owner this build cannot
// read says nothing about what is safe to end. Only a proven detached daemon is signaled.
const owner = observed?.ownership ?? 'detached';
const retirement = retireObservedDaemon(
{ ...input, observed },
(identity) =>
stopDaemonProcess(identity, {
mode: input.mode,
termTimeoutMs: input.termTimeoutMs ?? 3_000,
killTimeoutMs: input.killTimeoutMs ?? 1_000,
}),
observed && owner !== 'detached'
? (identity) =>
proveExitWithoutSignaling(
identity,
owner === 'embedded' ? 'embedded-owner' : 'owner-unproven',
)
: (identity) =>
stopDaemonProcess(identity, {
mode: input.mode,
termTimeoutMs: input.termTimeoutMs ?? 3_000,
killTimeoutMs: input.killTimeoutMs ?? 1_000,
}),
owned,
input.startupJoinTimeoutMs ?? 1_000,
);
Expand All @@ -415,16 +544,16 @@ export async function stopAndRetireDaemon(

function confirmOwnedStartupIdentity(
owned: PrivateReplayState,
observed: OwnerIdentity | null,
): OwnerIdentity | null {
observed: ObservedDaemonIdentity | null,
): ObservedDaemonIdentity | null {
if (!observed) {
if (owned.startups.length === 0) return null;
throw unownedStartupError();
}
const startup = owned.startups.find(({ launch }) => launch.pid === observed.pid);
if (!startup) throw unownedStartupError();
const { launch } = startup;
const identity = observed.startTime ? observed : launch.readIdentity();
const identity = observed.startTime ? observed : { ...observed, ...launch.readIdentity() };
if (launch.startTime && launch.startTime === identity.startTime) return identity;
throw unownedStartupError();
}
Expand All @@ -439,13 +568,33 @@ function unownedStartupError(): AppError {
export async function recoverAbandonedDaemonRegistration(
input: DaemonRetirementInput,
): Promise<DaemonRetirementResult> {
return await retireObservedDaemon(input, async (identity) => {
if (!identity.startTime?.trim()) return { status: 'retained', reason: 'missing-start-time' };
const confirmed = { pid: identity.pid, startTime: identity.startTime };
return (await waitForDaemonExit(confirmed, { timeoutMs: 0 })).exited
? { status: 'exited', identity: confirmed, mode: 'already-exited' }
: { status: 'retained', reason: 'identity-unverified' };
});
return await retireObservedDaemon(input, (identity) => proveExitWithoutSignaling(identity));
}

/**
* Termination refusals that name an owner's rule rather than a lifetime this caller could not prove:
* both mean "nobody may signal this process", and both travel to the caller at the top level.
*/
function isOwnerRefusal(
reason: Extract<DaemonTerminationResult, { status: 'retained' }>['reason'],
): reason is 'embedded-owner' | 'owner-unproven' {
return reason === 'embedded-owner' || reason === 'owner-unproven';
}

/**
* Proves `observed` is gone and signals nothing, which is what a lifecycle path may do to a process
* it does not own: an abandoned registration, or one a host owns and ends itself. A lifetime whose
* exit cannot be proved stays exactly where it is.
*/
async function proveExitWithoutSignaling(
observed: OwnerIdentity,
reason: 'identity-unverified' | 'embedded-owner' | 'owner-unproven' = 'identity-unverified',
): Promise<DaemonTerminationResult> {
if (!observed.startTime?.trim()) return { status: 'retained', reason: 'missing-start-time' };
const identity = { pid: observed.pid, startTime: observed.startTime };
return (await waitForDaemonExit(identity, { timeoutMs: 0 })).exited
? { status: 'exited', identity, mode: 'already-exited' }
: { status: 'retained', reason };
}

async function retireObservedDaemon(
Expand All @@ -468,6 +617,12 @@ async function retireObservedDaemon(
const joined = await joinOwnedStartups(owned, startupJoinTimeoutMs);
if (stopFailure)
return { status: 'retained', reason: 'stop-failed', removedInfo: false, error: stopFailure };
// A terminate step that refused because the record's owner ends this runtime, or because the record
// could not prove one, says so at the top: a caller must be able to tell "nobody may signal this
// process" from "this daemon is hung".
if (termination?.status === 'retained' && isOwnerRefusal(termination.reason)) {
return { status: 'retained', reason: termination.reason, termination, removedInfo: false };
}
if (termination && termination.status !== 'exited')
return { status: 'retained', reason: 'exit-unconfirmed', termination, removedInfo: false };
if (!joined) {
Expand Down
37 changes: 37 additions & 0 deletions packages/daemon-contracts/src/daemon-registration-status.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,49 @@ test('a live registered daemon reports its identity and the plugin inventory it
state: 'running',
stateDir,
pid: owner.pid,
// A record that predates ownership is a detached daemon, which is what a reader must be able to
// rely on before it decides a stop may signal this pid.
ownership: 'detached',
version: '0.21.30',
transport: 'http',
plugins: [ACTIVE, FAILED, { state: 'unknown' }],
});
});

test('a record published by a host reports the ownership that limits what a stop may do', () => {
const stateDir = mkdtempForTestSync('agent-device-daemon-status-');
const owner = readCurrentOwnerIdentity();
publish(stateDir, {
token: 't',
port: 1,
pid: owner.pid,
processStartTime: owner.startTime,
ownership: 'embedded',
});

expect(readDaemonStatus(resolveDaemonPaths(stateDir))).toMatchObject({
state: 'running',
pid: owner.pid,
ownership: 'embedded',
});
});

test('a record that claims an ownership nobody knows reads as an unrecognized one', () => {
const stateDir = mkdtempForTestSync('agent-device-daemon-status-');
const owner = readCurrentOwnerIdentity();
publish(stateDir, {
token: 't',
port: 1,
pid: owner.pid,
processStartTime: owner.startTime,
ownership: 'sideways',
});

// An ownership value outside the vocabulary says nothing about who owns the process, and `detached`
// is the one reading a caller may act on by signalling it. Status says it cannot recognize the owner.
expect(readDaemonStatus(resolveDaemonPaths(stateDir)).ownership).toBe('unknown');
});

test('a record whose process is gone or was replaced is not a running daemon', () => {
const stateDir = mkdtempForTestSync('agent-device-daemon-status-');
const owner = readCurrentOwnerIdentity();
Expand Down
Loading
Loading