infra(evals): add remote eval Docker setup (PG16, backend on 127.0.0.1:4242), WS-readiness healthcheck, Drizzle seeding, SDK WS override/skip-binary flag, and CI workflow to run single eval.

This enables isolated, reproducible remote eval runs in CI with seeded auth and explicit WS target.

🤖 Generated with Codebuff
Co-Authored-By: Codebuff <noreply@codebuff.com>

Author: brandonkachen

.github/workflows/remote-evals.yml

@@ -0,0 +1,25 @@
+name: remote-evals
+
+on:
+  workflow_dispatch:
+
+jobs:
+  remote-evals:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@v1
+        with:
+          bun-version: 1.1.34
+
+      - name: Install dependencies
+        run: bun install --frozen-lockfile
+
+      - name: Make scripts executable
+        run: chmod +x evals/scripts/*.sh
+
+      - name: Run remote eval
+        run: evals/scripts/run-remote.sh --prompt "Say hi from CI and print the working directory" --max-steps 10

backend/src/index.ts

@@ -19,6 +19,7 @@ import {
   sendRequestReconnect,
   waitForAllClientsDisconnected,
   listen as webSocketListen,
+  isWebsocketReady,
 } from './websockets/server'
 
 const app = express()
@@ -31,7 +32,10 @@ app.get('/', (req, res) => {
 })
 
 app.get('/healthz', (req, res) => {
-  res.send('ok')
+  if (!isWebsocketReady()) {
+    return res.status(503).send('starting')
+  }
+  return res.send('ok')
 })
 
 app.post('/api/usage', usageHandler)

backend/src/websockets/auth.ts

@@ -25,6 +25,18 @@ export async function getUserIdFromAuthToken(
 export async function getUserInfoFromAuthToken(
   authToken: string,
 ): Promise<UserInfo | undefined> {
+  // Test-only bypass for remote evals
+  if (process.env.NODE_ENV === 'test') {
+    const bypass = process.env.CODEBUFF_TEST_AUTH_TOKEN
+    if (bypass && authToken === bypass) {
+      return {
+        id: 'test-user',
+        email: 'evals@test.local',
+        discord_id: null,
+      }
+    }
+  }
+
   const user = await db
     .select({
       id: schema.user.id,
@@ -37,5 +49,5 @@ export async function getUserInfoFromAuthToken(
     .limit(1)
     .then((rows) => rows[0])
 
-  return user
+  return user ?? undefined
 }

backend/src/websockets/server.ts

@@ -14,6 +14,7 @@ import type { Server as HttpServer } from 'node:http'
 import type { RawData, WebSocket } from 'ws'
 
 export const SWITCHBOARD = new Switchboard()
+export let WS_READY = false
 
 // if a connection doesn't ping for this long, we assume the other side is toast
 const CONNECTION_TIMEOUT_MS = 60 * 1000
@@ -87,6 +88,7 @@ export function listen(server: HttpServer, path: string) {
   let deadConnectionCleaner: NodeJS.Timeout | undefined
   wss.on('listening', () => {
     logger.info(`Web socket server listening on ${path}.`)
+    WS_READY = true
     deadConnectionCleaner = setInterval(function ping() {
       const now = Date.now()
       try {
@@ -175,3 +177,7 @@ export function sendRequestReconnect() {
 export function waitForAllClientsDisconnected() {
   return SWITCHBOARD.waitForAllClientsDisconnected()
 }
+
+export function isWebsocketReady() {
+  return WS_READY
+}

evals/backend.Dockerfile

@@ -0,0 +1,6 @@
+FROM oven/bun:1.1.34 as base
+WORKDIR /app
+COPY . .
+RUN bun install --frozen-lockfile
+EXPOSE 4242
+CMD ["bun", "--cwd", "backend", "dev"]

evals/docker-compose.evals.yml

@@ -0,0 +1,50 @@
+version: '3.9'
+services:
+  db:
+    image: postgres:16-alpine
+    environment:
+      POSTGRES_USER: codebuff
+      POSTGRES_PASSWORD: codebuff
+      POSTGRES_DB: codebuff
+    command: [
+      "postgres",
+      "-c", "fsync=off",
+      "-c", "synchronous_commit=off", 
+      "-c", "full_page_writes=off"
+    ]
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U codebuff -d codebuff"]
+      interval: 5s
+      timeout: 3s
+      retries: 20
+
+  backend:
+    build:
+      context: ..
+      dockerfile: ./evals/backend.Dockerfile
+    environment:
+      DATABASE_URL: postgresql://codebuff:codebuff@db:5432/codebuff
+      PORT: "4242"
+      NODE_ENV: production
+    depends_on:
+      db:
+        condition: service_healthy
+    ports:
+      - "127.0.0.1:4242:4242"
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://localhost:4242/healthz"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+
+  seeder:
+    image: oven/bun:1.1.34
+    working_dir: /app
+    volumes:
+      - ..:/app:ro
+    environment:
+      DATABASE_URL: postgresql://codebuff:codebuff@db:5432/codebuff
+    entrypoint: ["bun", "run", "evals/seeds/seed-evals.ts"]
+    depends_on:
+      db:
+        condition: service_healthy

evals/scripts/run-remote.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+COMPOSE_FILE="$SCRIPT_DIR/../docker-compose.evals.yml"
+
+export CODEBUFF_WEBSOCKET_URL="ws://127.0.0.1:4242/ws"
+export CODEBUFF_SKIP_BINARY_CHECK=1
+
+# Start services
+docker compose -f "$COMPOSE_FILE" up -d --build db backend
+"$SCRIPT_DIR/wait-for-healthz.sh" "http://127.0.0.1:4242/healthz" 90 || {
+  echo 'Healthz failed; dumping backend logs...'
+  docker compose -f "$COMPOSE_FILE" logs backend --tail=200 || true
+  exit 1
+}
+
+# Drizzle seed (prints CODEBUFF_API_KEY=...)
+KEY_LINE=$(docker compose -f "$COMPOSE_FILE" run --rm seeder | tail -n1)
+export CODEBUFF_API_KEY="${KEY_LINE#CODEBUFF_API_KEY=}"
+
+# Run the eval (allow args passthrough; require prompt if not provided)
+if [[ $# -eq 0 ]]; then
+  bun scripts/git-evals/run-single-eval.ts --prompt "Say hi and print the working directory" --max-steps 10
+else
+  bun scripts/git-evals/run-single-eval.ts "$@"
+fi
+
+# Tear down
+docker compose -f "$COMPOSE_FILE" down -v

evals/scripts/wait-for-healthz.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+URL="${1:?Missing URL}"
+TIMEOUT="${2:-60}"
+for i in $(seq 1 "$TIMEOUT"); do
+  if curl -fsS "$URL" >/dev/null 2>&1; then
+    exit 0
+  fi
+  echo "waiting for backend... ($i s)"
+  sleep 1
+done
+echo "backend healthz did not become ready in $TIMEOUT seconds" >&2
+exit 1

evals/seeds/seed-evals.ts

@@ -0,0 +1,58 @@
+import 'dotenv/config'
+import crypto from 'node:crypto'
+import { Client } from 'pg'
+import { drizzle } from 'drizzle-orm/node-postgres'
+import * as schema from '../../common/db/schema'
+
+async function main() {
+  const DATABASE_URL = process.env.DATABASE_URL
+  if (!DATABASE_URL) {
+    console.error('Missing DATABASE_URL')
+    process.exit(1)
+  }
+  const client = new Client({ connectionString: DATABASE_URL })
+  await client.connect()
+  const db = drizzle(client)
+
+  const userId = 'evals-user'
+  const email = 'evals@test.local'
+  const token = crypto.randomUUID()
+
+  // Upsert user (adjust to match schema fields)
+  try {
+    // @ts-ignore - schema types may vary; keep robust
+    await db
+      .insert(schema.user)
+      .values({
+        id: userId,
+        email,
+        // Optional common columns; ignore if not present
+        created_at: new Date(),
+        updated_at: new Date(),
+      })
+      .onConflictDoNothing()
+  } catch {}
+
+  // Upsert session/api token (sessionToken + userId)
+  try {
+    // @ts-ignore - schema types may vary; keep robust
+    await db
+      .insert(schema.session)
+      .values({
+        sessionToken: token,
+        userId,
+        // Optional: expire in 24h if column exists
+        expires: new Date(Date.now() + 24 * 60 * 60 * 1000),
+        created_at: new Date(),
+      })
+      .onConflictDoNothing()
+  } catch {}
+
+  console.log(`CODEBUFF_API_KEY=${token}`)
+  await client.end()
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})