feat: add optional Cua Driver computer backend

.env.example

@@ -117,6 +117,9 @@ AGENT_COMPUTER_URL=http://localhost:4100
 # without this value and refuses every request that does not present it. Use a long random value;
 # `scripts/start.sh` sets a development one for you.
 COMPUTER_TOKEN=
+# Browser implementation inside each computer. `playwright` is the default. `cua-driver` uses Cua
+# Driver's native SDK, isolated named profiles, semantic browser snapshots, and native takeover.
+# COMPUTER_BACKEND=cua-driver
 # Local only. Lets a Bot browse this machine's own services; never set this in a deployment.
 AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true
 #
@@ -163,7 +166,8 @@ AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true
 # `EGRESS_PROXY_DEFAULT` covers the rest, and absent means the browser goes out directly.
 #
 # Credentials may be in the URL and are split out before Playwright sees them. Only the HOST is ever
-# reported back on the admin page or the API, so the password does not end up on a screen.
+# reported back on the admin page or the API, so the password does not end up on a screen. Egress
+# proxy settings currently apply only to the default Playwright backend.
 #
 # This is attribution, not anonymity, and it is not a boundary by itself: it gives a security team a
 # per-Bot address for network rules alongside AGENT_COMPUTER_POLICY.

.github/workflows/ci.yml

@@ -32,6 +32,10 @@ jobs:
       - run: bun run format:check
       - run: bun run lint
       - run: bun run typecheck
+      # agent-computer is intentionally outside the root workspaces because it is built as its own
+      # image. Install and typecheck its native backend against its separate lockfile explicitly.
+      - run: bun install --frozen-lockfile && bun run typecheck
+        working-directory: agent-computer
 
   test:
     name: tests
@@ -83,3 +87,5 @@ jobs:
           bun-version: 1.3.14
       - run: bun install --frozen-lockfile
       - run: bun run build
+      - run: docker build -f agent-computer/Dockerfile -t openbot-agent-computer:test .
+      - run: docker build -t openbot:test .

Dockerfile

@@ -20,15 +20,16 @@
 
 FROM mcr.microsoft.com/playwright:v1.62.1-noble AS base
 
-# unzip is not in the Playwright image and bun's installer needs it.
+# unzip is not in the Playwright image and bun's installer needs it. Openbox gives Cua Driver's
+# native input route a foreground X11 window without adding a desktop environment.
 # Bun is pinned. The installer takes whatever is newest otherwise, so the runtime drifts from the
 # one the lockfile was resolved against and an image built next month is not the image built today.
 ARG BUN_VERSION=1.3.14
 # Into /usr/local rather than /root/.bun, because the runtime stage runs as `pwuser` and cannot read
 # root's home. Set before the install, or the installer has already chosen the wrong directory.
 ENV BUN_INSTALL=/usr/local
 ENV PATH="/usr/local/bin:${PATH}"
-RUN apt-get update && apt-get install -y --no-install-recommends unzip xz-utils \
+RUN apt-get update && apt-get install -y --no-install-recommends openbox unzip xz-utils \
   && rm -rf /var/lib/apt/lists/* \
   && curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}"
 
@@ -45,8 +46,8 @@ COPY server/package.json server/package.json
 COPY worker/package.json worker/package.json
 RUN bun install --frozen-lockfile
 
-COPY agent-computer/package.json agent-computer/package.json
-RUN cd agent-computer && bun install
+COPY agent-computer/package.json agent-computer/bun.lock agent-computer/
+RUN cd agent-computer && bun install --frozen-lockfile
 
 # A second tree with the build-time dependencies left out, for the runtime stage to take. Vite,
 # biome and the test tooling are a gigabyte that nothing in a running container imports.
@@ -172,7 +173,8 @@ ENV AGENT_COMPUTER_ALLOW_PRIVATE_HOSTS=true
 # The two directories the browser writes are its workspace and its profile, the second being what
 # keeps a Bot signed in between turns. Owned here, because a non-root process cannot create them at
 # the root of the filesystem and the failure surfaces as EACCES on the first navigation.
-RUN mkdir -p /workspace /profiles \
+RUN mkdir -p /workspace /profiles /tmp/.X11-unix \
+  && chmod 1777 /tmp/.X11-unix \
   && chown -R pwuser:pwuser /workspace /profiles /app
 
 # Where the embedded database answers, when there is one. Overridden by whatever you set, so an

README.md

@@ -139,7 +139,7 @@ as one replica for now.
 
 ## Features
 
-- **A computer per Bot**: the supervisor gives each Bot its own container, its own `/workspace` volume and its own browser profile. Set `COMPUTER_RUNTIME=runsc` to run them under gVisor where the host supports it.
+- **A computer per Bot**: the supervisor gives each Bot its own container, its own `/workspace` volume and its own browser profile. Playwright is the default browser backend; set `COMPUTER_BACKEND=cua-driver` to use Cua Driver's semantic browser and native computer-use stack. Set `COMPUTER_RUNTIME=runsc` to run them under gVisor where the host supports it.
 - **A shell, not just a browser**: a Bot can run a command in its workspace, install what it needs, and process a file it saved. Through the same gate as everything else, so a rule can refuse a shell outright or refuse particular commands, and the command is on the record either way.
 - **The gateway is the only way in**: it resolves the target from a server-held snapshot, evaluates the policy, writes the audit row, and only then calls the computer. There is no path that acts without the record existing first.
 - **CEL policy, fail closed**: rules can inspect `tool.name`, `intent`, `bot.id`, `actor.id`, `page.url`, `page.host`, `element.*`, `key`, `file.*` and `mcp.*`. Deny is evaluated before allow, a missing policy permits nothing, and a broken rule refuses rather than opens.
@@ -194,6 +194,7 @@ Settings worth knowing:
 | `OPENAI_BASE_URL`                    | Answers the OpenAI-shaped calls from somewhere else: a gateway, a proxy.  |
 | `ANTHROPIC_BASE_URL`, `GOOGLE_GENERATIVE_AI_BASE_URL` | The same, for those two APIs.            |
 | `COMPUTER_TOKEN`                     | Secret every Bot computer request must present. `start.sh` sets one.      |
+| `COMPUTER_BACKEND`                   | `playwright` (default) or `cua-driver`.                                   |
 | `SUPERVISOR_TOKEN`                   | Secret the supervisor requires. `start.sh` sets one.                      |
 | `COMPUTER_SUPERVISOR_URL`            | Gives each Bot a computer of its own instead of one shared computer.      |
 | `COMPUTER_RUNTIME`                   | Set to `runsc` to run computers under gVisor, where the host has it.      |

agent-computer/Dockerfile

@@ -1,18 +1,20 @@
-# The Bot's computer uses Playwright's image so Chromium and its system libraries stay matched.
+# The Bot's computer uses Playwright's image so Chromium, Xvfb, and their system libraries stay
+# matched. Xvfb is needed by Cua Driver's optional native computer backend.
 #
 # The image tag and Playwright dependency must be pinned to the same exact version. Bump both or
 # neither.
 FROM mcr.microsoft.com/playwright:v1.62.1-noble
 
-# unzip is not in the Playwright image and bun's installer needs it.
-RUN apt-get update && apt-get install -y --no-install-recommends unzip \
+# unzip is not in the Playwright image and bun's installer needs it. Openbox gives Cua Driver's
+# foreground input route an active X11 window without adding a desktop environment.
+RUN apt-get update && apt-get install -y --no-install-recommends openbox unzip \
   && rm -rf /var/lib/apt/lists/* \
   && curl -fsSL https://bun.sh/install | bash
 ENV PATH="/root/.bun/bin:${PATH}"
 
 WORKDIR /app
-COPY agent-computer/package.json ./
-RUN bun install
+COPY agent-computer/package.json agent-computer/bun.lock ./
+RUN bun install --frozen-lockfile
 
 COPY agent-computer/src ./src
 
@@ -21,9 +23,15 @@ RUN mkdir -p /workspace
 ENV WORKSPACE_DIR=/workspace
 
 ENV PORT=4100
+ENV DISPLAY=:99
+# The image runs Chromium as root inside its container, matching the existing Playwright launch.
+# Cua Driver otherwise refuses that browser before OpenBot can apply its own container boundary.
+ENV CUA_E2E_BROWSER_NO_SANDBOX=1
 EXPOSE 4100
 # Readiness is exposed through Docker health so the supervisor need not know the network route.
 HEALTHCHECK --interval=2s --timeout=3s --start-period=2s --retries=30 \
   CMD bun -e "const r = await fetch('http://localhost:4100/health'); process.exit(r.ok ? 0 : 1)"
 
-CMD ["bun", "src/index.ts"]
+# Cua needs a private display and window manager for native keyboard/takeover input. Playwright stays
+# on its existing headless path. Fail startup if Xvfb dies instead of waiting forever for its socket.
+CMD ["sh", "-c", "if [ \"${COMPUTER_BACKEND:-playwright}\" != cua-driver ]; then exec bun src/index.ts; fi; Xvfb :99 -screen 0 1280x1024x24 -nolisten tcp & display_pid=$!; until [ -S /tmp/.X11-unix/X99 ]; do kill -0 \"$display_pid\" 2>/dev/null || exit 1; sleep 0.05; done; openbox --sm-disable & export CUA_DRIVER_BROWSER_PROFILE_ROOT=\"${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver}\"; exec bun src/index.ts"]

agent-computer/package.json

@@ -4,11 +4,12 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "dev": "bun --watch src/index.ts",
-    "start": "bun src/index.ts",
+    "dev": "CUA_DRIVER_BROWSER_PROFILE_ROOT=${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver} bun --watch src/index.ts",
+    "start": "CUA_DRIVER_BROWSER_PROFILE_ROOT=${CUA_DRIVER_BROWSER_PROFILE_ROOT:-${PROFILES_DIR:-/profiles}/cua-driver} bun src/index.ts",
     "typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@trycua/cua-driver": "0.20.0",
     "playwright": "1.62.1",
     "spiffe": "^0.5.1",
     "yaml": "^2.9.0"

agent-computer/src/browser.ts

@@ -0,0 +1,120 @@
+import type { InputMessage } from "./screencast";
+import type { ProfileSummary } from "./profiles";
+
+export { VIEWPORT } from "./profiles";
+
+export type BrowserElement = {
+  ref: string;
+  role: string;
+  name: string;
+  value?: string;
+  type?: string;
+  disabled?: boolean;
+  checked?: boolean;
+};
+
+export type PageRead = {
+  url: string;
+  title: string;
+  text: string;
+  truncated: boolean;
+};
+
+export type BrowserSnapshot = {
+  snapshotId: number;
+  url: string;
+  title: string;
+  elements: BrowserElement[];
+  truncated: boolean;
+};
+
+export type BrowserScreenshot = {
+  base64: string;
+  width: number;
+  height: number;
+  capturedAt: string;
+  url: string;
+};
+
+export type BrowserAction = {
+  action: "click" | "type" | "key" | "scroll";
+  ref?: string;
+  characters?: number;
+  submitted?: boolean;
+  key?: string;
+  deltaY?: number;
+  url: string;
+};
+
+export type HumanAction = {
+  action: "human_click" | "human_type" | "human_key" | "human_scroll";
+  characters?: number;
+  key?: string;
+  deltaY?: number;
+  url: string;
+};
+
+export type FrameMessage = {
+  type: "frame";
+  data: string;
+  width: number;
+  height: number;
+  mimeType?: string;
+};
+
+export type BrowserStream = {
+  stop: () => Promise<void>;
+  send: (message: InputMessage) => Promise<void>;
+};
+
+export class StaleSnapshotError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "StaleSnapshotError";
+  }
+}
+
+export interface BrowserComputer {
+  navigate(url: string, signal?: AbortSignal): Promise<PageRead>;
+  read(): Promise<PageRead>;
+  screenshot(): Promise<BrowserScreenshot>;
+  snapshot(): Promise<BrowserSnapshot>;
+  click(
+    ref: string,
+    snapshotId?: number,
+    signal?: AbortSignal,
+  ): Promise<BrowserAction>;
+  type(
+    ref: string,
+    text: string,
+    submit: boolean,
+    snapshotId?: number,
+    signal?: AbortSignal,
+  ): Promise<BrowserAction>;
+  key(
+    key: string,
+    ref?: string,
+    snapshotId?: number,
+    signal?: AbortSignal,
+  ): Promise<BrowserAction>;
+  scroll(deltaY: number, signal?: AbortSignal): Promise<BrowserAction>;
+  enterSecret(
+    ref: string,
+    text: string,
+  ): Promise<{ characters: number; url: string }>;
+  humanClick(x: number, y: number): Promise<HumanAction>;
+  humanType(text: string): Promise<HumanAction>;
+  humanKey(key: string): Promise<HumanAction>;
+  humanScroll(deltaY: number): Promise<HumanAction>;
+  startStream(onFrame: (frame: FrameMessage) => void): Promise<BrowserStream>;
+}
+
+export interface BrowserManager {
+  backend: "playwright" | "cua-driver";
+  computer(botId: string): BrowserComputer;
+  known(): Promise<string[]>;
+  summary(botIds: string[]): ProfileSummary[];
+  stop(botId: string): Promise<boolean>;
+  reset(botId: string): Promise<void>;
+  closeAll(): Promise<void>;
+}