feat(sessions): offer to rebuild the sessions a host reboot destroyed

.changeset/reboot-restore-banner.md

@@ -0,0 +1,5 @@
+---
+'aicodeman': minor
+---
+
+Offer to rebuild the sessions a host reboot destroyed. A reboot takes the tmux server down with it, so every pane dies and the board comes up empty. Codeman now works out what was running, and the board offers to restore it behind a click. The conversations come back; the terminal scrollback does not, and the banner says so.

src/reboot-restore.ts

@@ -0,0 +1,241 @@
+/**
+ * @fileoverview Decide which sessions a host reboot destroyed and may be rebuilt.
+ *
+ * A server restart and a host reboot both leave `reconcileSessions()` reporting
+ * dead sessions, and they need opposite handling. A server restart leaves the
+ * tmux panes running, so recovery ATTACHES to them. A host reboot takes the tmux
+ * server down with it, so there is nothing to attach to and the pane has to be
+ * created again. This module holds the decision half of that second case, kept
+ * free of tmux and disk access so it can be unit tested without either. Every
+ * observation it reads is gathered by the caller and passed in.
+ *
+ * "Eligible" here means a session the user did not end on purpose. The rule that
+ * an intentional kill or detach is never auto-revived is enforced at runtime by
+ * an in-memory guard in `TmuxManager`, and memory does not survive a reboot. The
+ * durable equivalent is the record `cleanupSession()` leaves behind. An unpinned
+ * kill deletes the record outright, so it is already absent here. A pinned kill
+ * goes through `demoteOrRemoveSession()` and lands as `status: 'stopped'`, which
+ * is the marker this module refuses. Pruning keeps a pinned record WITHOUT
+ * touching its status, so a pinned session a reboot killed still reads `idle` or
+ * `busy` and stays eligible.
+ *
+ * @dependencies types (SessionState), config/cli-registry
+ * @consumedby web/server (plan build at boot), web/routes/reboot-restore-routes
+ *
+ * @module reboot-restore
+ */
+
+import type { SessionState } from './types.js';
+import { getCli } from './config/cli-registry/registry.js';
+
+/** Session statuses a reboot restore may rebuild. `stopped` is the kill marker. */
+const RESTORABLE_STATUSES: ReadonlySet<string> = new Set(['idle', 'busy', 'error']);
+
+/** Observations the reboot heuristic reads. Gathered by the caller, never here. */
+export interface RebootEvidence {
+  /** Sessions that still had a live pane during reconciliation. */
+  livePaneCount: number;
+  /** Sessions reconciliation just marked dead. */
+  deadSessionCount: number;
+  /** `os.uptime()`, in seconds. */
+  uptimeSeconds: number;
+  /** Newest `lastActivityAt` across the persisted records, in ms since the epoch. */
+  newestPersistedActivityAt: number;
+  /** `Date.now()` when the evidence was gathered, in ms. */
+  now: number;
+}
+
+/**
+ * Decide whether the machine plausibly rebooted rather than the server restarting.
+ *
+ * Two signals have to agree. The socket must hold no panes at all while state
+ * still lists sessions, which rules out an ordinary server restart. The host
+ * must also have booted after the newest persisted session activity, which is
+ * the corroboration `os.uptime()` provides cheaply. A wiped tmux socket on a
+ * long-uptime host fails the second test, so a user who killed the tmux server
+ * by hand does not get every session offered back to them.
+ *
+ * This heuristic decides whether to ASK, never whether to act. A wrong yes costs
+ * the user a banner they dismiss, because the restore itself waits for a click.
+ *
+ * ⚠️ `os.uptime()` reports the HOST's uptime, which a container shares. A Codeman
+ * running in Docker therefore sees a long uptime after its own container restarts,
+ * the boot test fails, and no banner appears. The feature is effectively off for
+ * containerized installs. That is the safe direction to fail in, and fixing it
+ * needs a boot signal the container actually owns rather than a wider heuristic.
+ */
+export function looksLikeHostReboot(evidence: RebootEvidence): boolean {
+  if (evidence.deadSessionCount === 0) return false;
+  if (evidence.livePaneCount > 0) return false;
+  if (evidence.newestPersistedActivityAt <= 0) return false;
+  const bootedAt = evidence.now - evidence.uptimeSeconds * 1000;
+  return bootedAt > evidence.newestPersistedActivityAt;
+}
+
+/**
+ * Pick the conversation the rebuilt pane should resume.
+ *
+ * The chain's tail is the newest conversation the session was holding, which is
+ * what a compact or a clear leaves behind; `resumeSessionId` covers a session
+ * that was itself started as a resume, and the session id is the original
+ * conversation for everything else.
+ */
+export function resolveResumeConversationId(state: SessionState): string {
+  const chain = state.claudeSessionChain;
+  const chainTail = Array.isArray(chain) && chain.length > 0 ? chain[chain.length - 1] : undefined;
+  return chainTail || state.resumeSessionId || state.id;
+}
+
+/**
+ * Why one session was passed over. Reported for logging and shown to the user.
+ *
+ * The first six are decided before anything is built. `capacity-reached` and
+ * `rebuild-failed` can only happen once a click is spending the plan, and they
+ * are the two the banner must not confuse with a missing workspace: one means
+ * "try again after closing something", the other means the CLI would not start.
+ */
+export interface RebootRestoreRejection {
+  sessionId: string;
+  reason:
+    | 'no-persisted-record'
+    | 'intentionally-ended'
+    | 'respawn-blocked'
+    | 'remote-or-docker'
+    | 'unsupported-mode'
+    | 'no-working-dir'
+    | 'workspace-missing'
+    | 'workspace-forbidden'
+    | 'already-live'
+    | 'capacity-reached'
+    | 'rebuild-failed';
+}
+
+/** One restorable session, as the banner shows it and the rebuild replays it. */
+export interface RebootRestoreEntry {
+  sessionId: string;
+  name?: string;
+  workingDir: string;
+  owner?: string;
+  mode: string;
+  /** The conversation the rebuilt pane resumes. */
+  resumeConversationId: string;
+  /**
+   * The persisted record, kept whole so the rebuild can replay what it held.
+   * Read at boot, before pruning deletes it, and held in memory until the click.
+   */
+  state: SessionState;
+}
+
+export interface RebootRestorePlan {
+  restore: RebootRestoreEntry[];
+  skipped: RebootRestoreRejection[];
+}
+
+/**
+ * Split the sessions reconciliation just killed into the ones a reboot restore
+ * may offer and the ones it must leave alone.
+ *
+ * @param deadSessionIds Session ids `reconcileSessions()` reported as dead.
+ * @param persisted The `state.json` session records, which `cleanupStaleSessions()`
+ *   has not pruned yet at the point this runs.
+ * @param workspaceExists Whether a working directory is still on disk. A tmux
+ *   session can outlive its deleted repo, and rebuilding one there would scaffold
+ *   an empty tree. The caller owns the disk access; the click re-checks, because
+ *   a repo can be deleted between the boot and the click.
+ */
+export function planRebootRestore(
+  deadSessionIds: readonly string[],
+  persisted: Readonly<Record<string, SessionState>>,
+  workspaceExists: (workingDir: string) => boolean
+): RebootRestorePlan {
+  const restore: RebootRestoreEntry[] = [];
+  const skipped: RebootRestoreRejection[] = [];
+
+  for (const sessionId of deadSessionIds) {
+    const state = persisted[sessionId];
+    if (!state) {
+      // An unpinned kill already deleted the record, so absence IS the guard.
+      skipped.push({ sessionId, reason: 'no-persisted-record' });
+      continue;
+    }
+    if (!RESTORABLE_STATUSES.has(state.status)) {
+      // A pinned kill was demoted to `stopped`. Reviving it would undo the kill.
+      skipped.push({ sessionId, reason: 'intentionally-ended' });
+      continue;
+    }
+    if (state.respawnBlocked === true) {
+      // The crash-loop breaker tripped on this pane. Re-creating it restarts the loop.
+      skipped.push({ sessionId, reason: 'respawn-blocked' });
+      continue;
+    }
+    if (state.remote || state.docker) {
+      // Both need another host or a container to be up, which a just-booted machine
+      // cannot promise. The remote reconnect watcher owns the remote case already.
+      skipped.push({ sessionId, reason: 'remote-or-docker' });
+      continue;
+    }
+    // Capability, not a CLI id: this pass resumes by handing the CLI a conversation
+    // id through the top-level `resumeSessionId`, which only a CLI whose history the
+    // claude-jsonl reader understands can consume that way. Others carry their thread
+    // id in their own `<Mode>Config`, which this pass does not thread through.
+    if (getCli(state.mode ?? 'claude')?.capabilities.transcript !== 'claude-jsonl') {
+      skipped.push({ sessionId, reason: 'unsupported-mode' });
+      continue;
+    }
+    if (!state.workingDir) {
+      skipped.push({ sessionId, reason: 'no-working-dir' });
+      continue;
+    }
+    if (!workspaceExists(state.workingDir)) {
+      skipped.push({ sessionId, reason: 'workspace-missing' });
+      continue;
+    }
+    restore.push({
+      sessionId,
+      name: state.name,
+      workingDir: state.workingDir,
+      owner: state.owner,
+      mode: state.mode ?? 'claude',
+      resumeConversationId: resolveResumeConversationId(state),
+      state,
+    });
+  }
+
+  return { restore, skipped };
+}
+
+/**
+ * Drop the entries whose conversation is already on screen.
+ *
+ * Hours can pass between the boot that built the plan and the click that spends
+ * it, and the Resume list can reach the same conversation in the meantime. Two
+ * panes running `claude --resume` on one conversation is the failure this
+ * prevents, so a match on either the session id or the conversation id is enough
+ * to skip the entry.
+ */
+export function rejectAlreadyLive(
+  entries: readonly RebootRestoreEntry[],
+  liveSessionIds: ReadonlySet<string>,
+  liveConversationIds: ReadonlySet<string>
+): RebootRestorePlan {
+  const restore: RebootRestoreEntry[] = [];
+  const skipped: RebootRestoreRejection[] = [];
+  for (const entry of entries) {
+    if (liveSessionIds.has(entry.sessionId) || liveConversationIds.has(entry.resumeConversationId)) {
+      skipped.push({ sessionId: entry.sessionId, reason: 'already-live' });
+      continue;
+    }
+    restore.push(entry);
+  }
+  return { restore, skipped };
+}
+
+/** Newest `lastActivityAt` across persisted records, or 0 when there are none. */
+export function newestPersistedActivity(persisted: Readonly<Record<string, SessionState>>): number {
+  let newest = 0;
+  for (const state of Object.values(persisted)) {
+    const stamp = state.lastActivityAt ?? state.createdAt ?? 0;
+    if (stamp > newest) newest = stamp;
+  }
+  return newest;
+}

src/session-env-clamp.ts

@@ -0,0 +1,97 @@
+/**
+ * @fileoverview The env-var half of the multi-user privilege clamp.
+ *
+ * A session's `envOverrides` can hand back privilege that the per-CLI config
+ * clamp removed, so a non-granted owner's overrides get the privileged keys
+ * stripped before the session is built. The create and resume routes are what
+ * this bites on: they clamp what a request asked for.
+ *
+ * The reboot-restore route calls it as defence in depth, and today it can strip
+ * nothing. `Session.getEnvOverridesForPersist()` keeps only `CLAUDE_CODE_*` and
+ * `CLAUDE_CONFIG_DIR` out of a session's overrides, claude's `privilegedEnvKeys`
+ * are the five `ANTHROPIC_*` names, and that pass admits claude alone — so a
+ * persisted record cannot carry a clamped key. The call is there for the day the
+ * persisted set widens. The grant re-resolution that does bite on that path is
+ * `resolveClaudeModeForUsername`, which recomputes the permission mode.
+ *
+ * This lives outside `web/routes` on purpose. The question it answers is about
+ * session privilege rather than about HTTP, and `cron/cron-service.ts` sets the
+ * precedent by importing `canUsernameRunPrivilegedCommands` from `user-store.ts`
+ * directly and re-resolving the owner's grant when a job fires. Every caller here
+ * re-resolves the grant at the moment it builds a session, for the same reason.
+ *
+ * @dependencies user-store (canUsernameRunPrivilegedCommands), config/cli-registry
+ * @consumedby web/routes/session-routes, web/routes/reboot-restore-routes
+ *
+ * @module session-env-clamp
+ */
+
+import { canUsernameRunPrivilegedCommands } from './user-store.js';
+import { enabledClis } from './config/cli-registry/registry.js';
+
+/**
+ * Env-var keys a non-granted owner must not be able to set, because each one
+ * hands back privilege `clampExternalCliBypassForOwner()` just removed, or redirects a
+ * credential-resolution endpoint.
+ *
+ * The DeepSeek three are reachable because `DSH_*` and `DEEPSEEK_*` are
+ * allowlisted `envOverrides` prefixes (schemas.ts) — which they have to be, since
+ * that is also how a user configures the harness's non-privileged knobs.
+ *
+ * - `DSH_PERMISSION_MODE` IS the harness's permission switch. Every other CLI's
+ *   bypass is a command-line FLAG, reachable only through the per-CLI config the
+ *   clamp already owns; this one is an env var, so the config clamp alone is
+ *   half a gate.
+ * - `DSH_HOME` points the launcher at a profile tree, and a profile's plugin code
+ *   executes at BOOT, before any approval row can apply. A user who can write a
+ *   workspace can put a profile in it, so this is the wider of the two.
+ * - `DEEPSEEK_BASE_URL` aims the provider endpoint, and `_configureCliEnv()`
+ *   forwards the SERVER's own `DEEPSEEK_API_KEY` into every dsh pane before
+ *   `applyEnvOverrides()` runs — so a non-granted owner who could set the base
+ *   URL would have the operator's API key sent as a bearer credential to a host
+ *   of their choosing. (`DEEPSEEK_API_KEY` itself stays overridable: supplying
+ *   your OWN key removes privilege rather than granting it.)
+ * - `OMP_AUTH_BROKER_URL`/`OMP_AUTH_BROKER_TOKEN` are where omp resolves
+ *   credentials from — the same shape as `DEEPSEEK_BASE_URL` above, reachable
+ *   because `OMP_*` is an allowlisted prefix. Unlike DeepSeek, Codeman does not
+ *   forward any operator-held key into an omp pane today (omp's provider
+ *   credentials live in `~/.omp` config files, not env vars), so there is no
+ *   known concrete exfiltration path yet — clamped defensively anyway, since a
+ *   non-granted owner redirecting where a shared multi-tenant deployment
+ *   resolves auth from is not something to allow silently (found in
+ *   Ark0N/Codeman#353 review; omp's own knobs are otherwise mostly `PI_*`,
+ *   already allowlisted for pi and not addressed here — see resolveOmpHome()).
+ */
+export function ownerClampedEnvKeys(): string[] {
+  return enabledClis().flatMap((entry) => entry.capabilities.privilegedEnvKeys);
+}
+
+/**
+ * Env-var half of the multi-user bypass clamp.
+ *
+ * `clampExternalCliBypassForOwner()` in `web/routes/session-routes.ts` clamps the
+ * per-CLI CONFIG, and for every CLI
+ * but DeepSeek that is the whole story. Here it is not: `applyEnvOverrides()` runs
+ * AFTER `_configureCliEnv()` in tmux-manager, so an override sent on the SAME
+ * request lands last and wins, and a non-granted owner could restore
+ * `danger-full-access` on the very request the config clamp downgraded.
+ *
+ * Keys are DROPPED rather than rewritten: dropping falls through to what
+ * `_configureCliEnv()` exports, which is the clamped config and the server's own
+ * `DSH_HOME`, i.e. exactly the intended state. No-op in single-user mode and for a
+ * granted owner, like every other clamp here
+ * (`canUsernameRunPrivilegedCommands()` returns true when `!isMultiUserMode()`),
+ * and it returns the caller's own object untouched when there is nothing to strip.
+ */
+export async function clampEnvOverridesForOwner(
+  owner: string | undefined,
+  envOverrides: Record<string, string> | undefined
+): Promise<Record<string, string> | undefined> {
+  if (!envOverrides) return envOverrides;
+  const keys = ownerClampedEnvKeys();
+  if (!keys.some((key) => key in envOverrides)) return envOverrides;
+  if (await canUsernameRunPrivilegedCommands(owner)) return envOverrides;
+  const clamped = { ...envOverrides };
+  for (const key of keys) delete clamped[key];
+  return clamped;
+}

src/session.ts

@@ -1499,6 +1499,19 @@ export class Session extends EventEmitter {
     this._pinnedAt = pinned ? Date.now() : null;
   }
 
+  /**
+   * Restore a pin from a persisted record, keeping the moment it was pinned.
+   *
+   * `setPinned()` stamps `pinnedAt` with now, which is right for a user pinning a
+   * session and wrong for a restore: the session-manager orders its pinned group
+   * by that stamp, so a restored session would jump to the front of a list it had
+   * been sitting further down.
+   */
+  restorePin(pinned: boolean, pinnedAt?: number): void {
+    this._pinned = pinned;
+    this._pinnedAt = pinned ? (pinnedAt ?? Date.now()) : null;
+  }
+
   get flickerFilterEnabled(): boolean {
     return this._flickerFilterEnabled;
   }