Put shared state where every replica can see it, and bound what grows

.env.example

@@ -3,6 +3,10 @@ DATABASE_URL=postgres://openbot:openbot@localhost:5432/openbot
 # development. Production refuses to start with this key.
 #   openssl rand -base64 32
 KEY_ENCRYPTION_KEY=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+# How many days of audit trail to keep. Unset keeps everything, which is the safe default and the
+# one to leave alone until somebody has decided otherwise: the trail is append-only and nothing else
+# can remove a row, so this is the only way it ever shrinks.
+# AUDIT_RETENTION_DAYS=365
 PORT=3001
 TENANT_PACKAGE_DIR=../examples/fintech
 # What this deployment calls itself, when more than one shares an Intelligence project. A copy of a

agent-computer/src/browser-eviction.ts

@@ -0,0 +1,60 @@
+/**
+ * Which Bots' browsers to close, and when.
+ *
+ * Its own file, with no Playwright import, for the reason `authorisation.ts` and `bot-id.ts` are:
+ * `profiles.ts` imports Playwright at module scope, so anything living there needs a browser merely
+ * to be imported by a test. A decision this size should be testable without one, and `profiles.ts`
+ * owns the launching, so a test that went through it would be testing Playwright rather than the
+ * choice being made.
+ *
+ * The choice is worth testing on its own. There was no cap and no timeout: a context was started the
+ * first time each Bot was used and kept, and the only things that dropped one were an explicit stop,
+ * a browser that had already died, and shutdown. A deployment where every employee has a Bot trends
+ * toward one resident Chromium per employee in a single container, at a few hundred MB each, until
+ * it is killed for memory and relaunches its way back to the same state.
+ *
+ * Closing one loses nothing. The profile is on disk, so the Bot's logins survive and its next request
+ * starts where it left off, which is already what `stop` means here.
+ */
+
+/** Enough of a live entry for either decision. Keeps this file independent of what else is on one. */
+export type Evictable = { usedAt: number };
+
+/**
+ * Which to close because there are too many running.
+ *
+ * Least recently used first: the Bot that has been quiet longest. Applied after a launch, so the Bot
+ * that just asked is the most recently used and is therefore never the one closed.
+ */
+export function chooseEvictions(
+  running: Iterable<[string, Evictable]>,
+  max: number,
+): string[] {
+  const entries = [...running];
+  if (entries.length <= max) return [];
+
+  return entries
+    .sort(([, a], [, b]) => a.usedAt - b.usedAt)
+    .slice(0, entries.length - max)
+    .map(([botId]) => botId);
+}
+
+/**
+ * Which to close because nothing has touched them.
+ *
+ * The other half of the answer: a deployment under the cap still holds a browser for a Bot used once
+ * last Tuesday, and that memory is doing nothing for anybody. A timeout of zero or less switches this
+ * off, so a deployment can keep browsers resident if it would rather.
+ */
+export function chooseIdle(
+  running: Iterable<[string, Evictable]>,
+  idleTimeoutMs: number,
+  now: number,
+): string[] {
+  if (idleTimeoutMs <= 0) return [];
+  const cutoff = now - idleTimeoutMs;
+
+  return [...running]
+    .filter(([, entry]) => entry.usedAt <= cutoff)
+    .map(([botId]) => botId);
+}

agent-computer/src/index.ts

@@ -17,12 +17,12 @@ import {
   type Screencast,
   startScreencast,
 } from "./screencast";
+import { createShell } from "./shell";
 import {
   createWorkspace,
   WorkspaceFileError,
   WorkspacePathError,
 } from "./workspace";
-import { createShell } from "./shell";
 
 /**
  * The Bot's computer: one long-lived browser, reachable over HTTP.
@@ -122,11 +122,32 @@ type BotSession = {
 
 const sessions = new Map<string, BotSession>();
 
+/**
+ * Forget the sessions of Bots whose browsers are no longer running.
+ *
+ * The map gained an entry per Bot id this process had ever seen and lost none, so a deployment where
+ * every employee has a Bot accumulated one small object per employee for the life of the container.
+ * Small, but unbounded, which is the same shape as the browsers themselves.
+ *
+ * Only entries with no live browser and nobody watching are dropped: the state is the generation
+ * counter and the control handover, and both belong to a running browser. A Bot whose browser has
+ * been closed starts a fresh session next time, which is what starting a fresh browser means.
+ */
+function forgetIdleSessions(): void {
+  for (const [botId, session] of [...sessions.entries()]) {
+    if (session.viewer) continue;
+    if (profiles.isLive(botId)) continue;
+    sessions.delete(botId);
+  }
+}
+
 function sessionFor(botId: string): BotSession {
   const existing = sessions.get(botId);
   if (existing) return existing;
   const created: BotSession = { control: createControl(), snapshotId: 0 };
   sessions.set(botId, created);
+  // Cheap, and only ever on the path that adds one, so the map cannot grow without this running.
+  if (sessions.size > 32) forgetIdleSessions();
   return created;
 }
 

agent-computer/src/profiles.ts

@@ -32,9 +32,11 @@
  * operations this process applies to its own browser, so the same design works under Compose,
  * Kubernetes or ECS, where the orchestrator's own restart policy brings a process back.
  */
+
 import { readdir, rm } from "node:fs/promises";
 import { join } from "node:path";
 import { type BrowserContext, chromium, type Page } from "playwright";
+import { chooseEvictions, chooseIdle } from "./browser-eviction";
 import { egressFor, egressLabel } from "./egress";
 
 /** The viewport, which is what a person's click coordinates are relative to. */
@@ -141,17 +143,104 @@ async function closeAndWait(context: BrowserContext): Promise<void> {
   await new Promise((resolve) => setTimeout(resolve, CLOSE_SETTLE_MS));
 }
 
+/**
+ * How many browsers one computer holds at once.
+ *
+ * There was no cap. A context was started the first time each Bot was used and kept, and the only
+ * things that dropped one were an explicit stop, a browser that had already died, and shutdown. A
+ * deployment where every employee has a Bot therefore trends toward one resident Chromium per
+ * employee in a single container, at a few hundred MB each, until the container is killed for memory
+ * and `page()` relaunches its way back to the same state.
+ *
+ * A cap rather than only an idle timeout, because the failure is concurrent breadth rather than age:
+ * fifty people using their Bots inside the same minute are fifty live browsers and none of them are
+ * idle. The least recently used is closed, which is the one whose Bot has been quiet longest.
+ *
+ * Closing is not losing anything. The profile is on disk, so a Bot whose browser was closed starts
+ * again where it left off, which is what `stop` already means here.
+ */
+const MAX_LIVE_BROWSERS = Number(process.env.COMPUTER_MAX_BROWSERS ?? 8);
+
+/**
+ * How long a browser may sit untouched before it is closed.
+ *
+ * The other half. A deployment under the cap still holds a browser per Bot that was used once last
+ * Tuesday, and that memory is doing nothing for anybody.
+ */
+const IDLE_TIMEOUT_MS = Number(
+  process.env.COMPUTER_BROWSER_IDLE_MS ?? 30 * 60_000,
+);
+
+/** How often the idle sweep looks. Cheap: it walks a map of at most `MAX_LIVE_BROWSERS`. */
+const IDLE_SWEEP_MS = 60_000;
+
 export function createProfiles(root: string) {
-  /** One running browser per Bot. */
+  /** One running browser per Bot, up to {@link MAX_LIVE_BROWSERS}. */
   const live = new Map<
     string,
-    { context: BrowserContext; page: Page; startedAt: string }
+    {
+      context: BrowserContext;
+      page: Page;
+      startedAt: string;
+      /** When this Bot last asked for its page. Decides what the cap and the sweep close. */
+      usedAt: number;
+    }
   >();
   /** Launches in flight, so a cold computer is started once however many callers ask at once. */
   const starting = new Map<string, Promise<Page>>();
 
   const directoryFor = (botId: string): string => join(root, botId);
 
+  /**
+   * Close one Bot's browser and forget it.
+   *
+   * Gracefully, so Chromium flushes the profile: the whole point of closing one is that the Bot's
+   * logins survive and its next request starts where it left off.
+   */
+  const evict = async (botId: string, reason: string): Promise<void> => {
+    const running = live.get(botId);
+    if (!running) return;
+    live.delete(botId);
+    console.info(
+      JSON.stringify({ type: "computer-browser-closed", botId, reason }),
+    );
+    await closeAndWait(running.context).catch(() => undefined);
+  };
+
+  /**
+   * Keep the number of running browsers under the cap.
+   *
+   * Least recently used first, which is the Bot that has been quiet longest. Called after a launch
+   * rather than before, so the Bot that just asked is never the one closed.
+   */
+  const enforceCap = async (): Promise<void> => {
+    for (const botId of chooseEvictions(live.entries(), MAX_LIVE_BROWSERS)) {
+      await evict(botId, "the cap on running browsers was reached");
+    }
+  };
+
+  /**
+   * Close browsers nothing has touched for a while.
+   *
+   * The cap answers concurrent breadth; this answers a Bot used once last Tuesday whose browser is
+   * still resident and doing nothing for anybody.
+   */
+  const sweepIdle = async (): Promise<void> => {
+    for (const botId of chooseIdle(
+      live.entries(),
+      IDLE_TIMEOUT_MS,
+      Date.now(),
+    )) {
+      await evict(botId, "it had been idle");
+    }
+  };
+
+  const idleSweep = setInterval(() => {
+    void sweepIdle().catch(() => undefined);
+  }, IDLE_SWEEP_MS);
+  // Housekeeping must not hold the process open on the way out.
+  idleSweep.unref?.();
+
   const sweepLocks = async (dir: string): Promise<void> => {
     await Promise.all(
       SINGLETON_FILES.map((name) =>
@@ -181,6 +270,8 @@ export function createProfiles(root: string) {
         existing?.context.browser()?.isConnected() &&
         !existing.page.isClosed()
       ) {
+        // Touched on every use, which is what makes "least recently used" mean anything.
+        existing.usedAt = Date.now();
         return existing.page;
       }
       if (existing) {
@@ -210,7 +301,15 @@ export function createProfiles(root: string) {
         });
         // Persistent contexts open with a page already; reuse it rather than leaving an extra blank tab.
         const page = context.pages()[0] ?? (await context.newPage());
-        live.set(botId, { context, page, startedAt: new Date().toISOString() });
+        live.set(botId, {
+          context,
+          page,
+          startedAt: new Date().toISOString(),
+          usedAt: Date.now(),
+        });
+        // After the new one is in the map, so the cap counts what is really running and the Bot that
+        // just asked is the most recently used and therefore never the one closed.
+        await enforceCap();
         return page;
       })();
 
@@ -291,10 +390,26 @@ export function createProfiles(root: string) {
      * here gives Chromium the chance to flush its profile within that grace period.
      */
     async closeAll(): Promise<void> {
+      clearInterval(idleSweep);
       const contexts = [...live.values()];
       live.clear();
       await Promise.all(contexts.map((c) => closeAndWait(c.context)));
     },
+
+    /** How many browsers are running. For the idle sweep's own tests, and for a status reader. */
+    liveCount(): number {
+      return live.size;
+    },
+
+    /** Whether this Bot has a browser right now, so a caller can drop state that belongs to one. */
+    isLive(botId: string): boolean {
+      return live.has(botId);
+    },
+
+    /** Run the idle sweep now. Exposed so a test does not have to wait a minute for the interval. */
+    sweepIdleNow(): Promise<void> {
+      return sweepIdle();
+    },
   };
 }
 

agent-computer/tests/browser-eviction.test.ts

@@ -0,0 +1,107 @@
+import { describe, expect, test } from "bun:test";
+import { chooseEvictions, chooseIdle } from "../src/browser-eviction";
+
+/**
+ * How many browsers one computer holds, and for how long.
+ *
+ * There was no answer to either. A context was started the first time each Bot was used and kept,
+ * and the only things that dropped one were an explicit stop, a browser that had already died, and
+ * shutdown. A deployment where every employee has a Bot trends toward one resident Chromium per
+ * employee in a single container, at a few hundred MB each, until it is killed for memory and
+ * relaunches its way back to the same state.
+ *
+ * Closing one loses nothing: the profile is on disk, so the Bot's logins survive and its next
+ * request starts where it left off. That is already what `stop` means here.
+ *
+ * These test the decision rather than the closing. `createProfiles` launches a real Chromium, so a
+ * test that went through it would be testing Playwright; the part with a wrong answer available is
+ * which browser gets picked.
+ */
+
+/** Bots, oldest use first, as a map the way the module holds them. */
+function running(...usedAt: number[]): Map<string, { usedAt: number }> {
+  return new Map(usedAt.map((at, index) => [`bot-${index}`, { usedAt: at }]));
+}
+
+describe("keeping the number of running browsers under a cap", () => {
+  test("closes nothing while there is room", () => {
+    expect(chooseEvictions(running(1, 2, 3), 8)).toEqual([]);
+  });
+
+  test("closes nothing at exactly the cap", () => {
+    // The boundary. Off by one here closes a browser somebody is using every time the last slot
+    // fills, which reads as a computer that keeps restarting itself.
+    expect(chooseEvictions(running(1, 2, 3), 3)).toEqual([]);
+  });
+
+  test("closes the least recently used", () => {
+    // Not the oldest browser: the one whose Bot has been quiet longest. A Bot started this morning
+    // and used a second ago is the wrong one to close.
+    expect(chooseEvictions(running(500, 100, 300), 2)).toEqual(["bot-1"]);
+  });
+
+  test("closes as many as it takes to get under the cap", () => {
+    expect(chooseEvictions(running(500, 100, 300, 200), 2).sort()).toEqual([
+      "bot-1",
+      "bot-3",
+    ]);
+  });
+
+  test("the one that just asked is never the one closed", () => {
+    /*
+     * The property that makes this safe. The cap is applied after a launch, so the newest entry is
+     * the most recently used; if it could be chosen, a deployment at the cap would close the browser
+     * it had just started and the Bot would never get one.
+     */
+    const now = Date.now();
+    const withNewest = running(now - 10_000, now - 5_000, now);
+
+    expect(chooseEvictions(withNewest, 2)).not.toContain("bot-2");
+  });
+
+  test("a cap of one still leaves the newest running", () => {
+    const now = Date.now();
+    const chosen = chooseEvictions(running(now - 1000, now), 1);
+
+    expect(chosen).toEqual(["bot-0"]);
+  });
+});
+
+describe("closing browsers nothing has touched", () => {
+  const NOW = 1_000_000;
+
+  test("closes what is past the timeout", () => {
+    const stale = NOW - 60_000;
+    const fresh = NOW - 1_000;
+
+    expect(chooseIdle(running(stale, fresh), 30_000, NOW)).toEqual(["bot-0"]);
+  });
+
+  test("keeps one used exactly at the boundary out of it", () => {
+    // Exactly at the cutoff counts as idle, and a moment inside it does not. Stated so the boundary
+    // is a decision rather than whatever the comparison happened to be.
+    expect(chooseIdle(running(NOW - 30_000), 30_000, NOW)).toEqual(["bot-0"]);
+    expect(chooseIdle(running(NOW - 29_999), 30_000, NOW)).toEqual([]);
+  });
+
+  test("a timeout of zero switches it off", () => {
+    // A deployment that would rather keep browsers resident says so this way, and gets the cap and
+    // nothing else.
+    expect(chooseIdle(running(0), 0, NOW)).toEqual([]);
+    expect(chooseIdle(running(0), -1, NOW)).toEqual([]);
+  });
+
+  test("closes several at once", () => {
+    const stale = NOW - 60_000;
+
+    expect(chooseIdle(running(stale, stale, NOW), 30_000, NOW).sort()).toEqual([
+      "bot-0",
+      "bot-1",
+    ]);
+  });
+
+  test("nothing running closes nothing", () => {
+    expect(chooseIdle(running(), 30_000, NOW)).toEqual([]);
+    expect(chooseEvictions(running(), 8)).toEqual([]);
+  });
+});