perf(evi): faster before/after flow with native captures and a primed template #533
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| import { afterEach, describe, expect, it, vi } from 'vitest' | ||
| import { exchangeTurboToken, turboConfigCommand } from './turbo' | ||
|
|
||
| afterEach(() => { | ||
| vi.unstubAllGlobals() | ||
| }) | ||
|
|
||
| describe('exchangeTurboToken', () => { | ||
| it('posts the token-exchange grant and returns the access token', async () => { | ||
| let captured: URLSearchParams | undefined | ||
| vi.stubGlobal('fetch', vi.fn(async (url: string, init?: RequestInit) => { | ||
| expect(String(url)).toBe('https://api.vercel.com/login/oauth/token') | ||
| captured = init?.body as URLSearchParams | ||
| return new Response(JSON.stringify({ access_token: 'turbo_tok' }), { status: 200 }) | ||
| })) | ||
| await expect(exchangeTurboToken('oidc_abc', 'hrcd')).resolves.toBe('turbo_tok') | ||
| expect(captured?.get('grant_type')).toBe('urn:ietf:params:oauth:grant-type:token-exchange') | ||
| expect(captured?.get('subject_token')).toBe('oidc_abc') | ||
| expect(captured?.get('team_id_or_slug')).toBe('hrcd') | ||
| }) | ||
|
|
||
| it('surfaces a failed exchange and a missing or non-string access token', async () => { | ||
| vi.stubGlobal('fetch', vi.fn(async () => new Response('denied', { status: 403 }))) | ||
| await expect(exchangeTurboToken('oidc', 'hrcd')).rejects.toThrow('failed (403)') | ||
| for (const payload of ['{}', '{"access_token":123}', '{"access_token":""}']) { | ||
| vi.stubGlobal('fetch', vi.fn(async () => new Response(payload, { status: 200 }))) | ||
| await expect(exchangeTurboToken('oidc', 'hrcd')).rejects.toThrow('no access_token') | ||
| } | ||
| }) | ||
| }) | ||
|
|
||
| describe('turboConfigCommand', () => { | ||
| it('writes the auth and repo config files', () => { | ||
| const command = turboConfigCommand('tok-123', 'team_x', 'hrcd') | ||
| expect(command).toContain(`printf '%s' '{"token":"tok-123"}' > ~/.config/turborepo/config.json`) | ||
| expect(command).toContain(`printf '%s' '{"teamId":"team_x","teamSlug":"hrcd"}' > /workspace/repo/.turbo/config.json`) | ||
| }) | ||
|
|
||
| it('refuses any value that could escape the quoting', () => { | ||
| expect(() => turboConfigCommand("tok'; rm -rf /", 'team_x', 'hrcd')).toThrow('Unexpected characters in the Turborepo token') | ||
| expect(() => turboConfigCommand('tok', "team' x", 'hrcd')).toThrow('Unexpected characters in the Turborepo teamId') | ||
| expect(() => turboConfigCommand('tok', 'team_x', "hr'cd")).toThrow('Unexpected characters in the Turborepo teamSlug') | ||
| }) | ||
| }) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| /** Public client id of Vercel's OIDC → Turborepo token exchange (from the Remote Caching docs). */ | ||
| const EXCHANGE_CLIENT_ID = 'cl_kyUx2zVvA4MGptBohkmtYHJly2XltXzD' | ||
|
|
||
| const EXCHANGE_URL = 'https://api.vercel.com/login/oauth/token' | ||
|
|
||
| /** | ||
| * Exchanges the runtime's Vercel OIDC token for a short-lived Turborepo access | ||
| * token: scoped to Remote Cache only and tied to the team, so even read from | ||
| * inside the sandbox it grants nothing beyond cache access. | ||
| */ | ||
| export async function exchangeTurboToken(oidcToken: string, team: string): Promise<string> { | ||
| const body = new URLSearchParams({ | ||
| grant_type: 'urn:ietf:params:oauth:grant-type:token-exchange', | ||
| client_id: EXCHANGE_CLIENT_ID, | ||
| subject_token_type: 'urn:ietf:params:oauth:token-type:id_token', | ||
| requested_token_type: 'urn:ietf:params:oauth:token-type:access_token', | ||
| team_id_or_slug: team, | ||
| subject_token: oidcToken, | ||
| }) | ||
| const response = await fetch(EXCHANGE_URL, { method: 'POST', body, signal: AbortSignal.timeout(10_000) }) | ||
| if (!response.ok) { | ||
| throw new Error(`Turborepo token exchange failed (${response.status}): ${await response.text()}`) | ||
| } | ||
| const payload = await response.json() as { access_token?: unknown } | ||
| const token = payload?.access_token | ||
| if (typeof token !== 'string' || token.length === 0) { | ||
| throw new Error('Turborepo token exchange returned no access_token.') | ||
| } | ||
| return token | ||
| } | ||
|
|
||
| /** | ||
| * Shell command writing turbo's auth and repo config inside the sandbox, so | ||
| * `turbo` finds the token and team on its own and no credential ever appears | ||
| * in a model-composed command. The token is base64url material: safe inside | ||
| * single quotes. | ||
| */ | ||
| export function turboConfigCommand(token: string, teamId: string, teamSlug: string): string { | ||
| // All three land inside single quotes; refusing anything outside this | ||
| // charset (which real tokens, team ids, and slugs never leave) beats escaping. | ||
| for (const [name, value] of [['token', token], ['teamId', teamId], ['teamSlug', teamSlug]] as const) { | ||
| if (!/^[\w.-]+$/.test(value)) throw new Error(`Unexpected characters in the Turborepo ${name}.`) | ||
| } | ||
| const auth = JSON.stringify({ token }) | ||
| const repo = JSON.stringify({ teamId, teamSlug }) | ||
| return [ | ||
| 'mkdir -p ~/.config/turborepo /workspace/repo/.turbo', | ||
| `printf '%s' '${auth}' > ~/.config/turborepo/config.json`, | ||
| `printf '%s' '${repo}' > /workspace/repo/.turbo/config.json`, | ||
|
coderabbitai[bot] marked this conversation as resolved.
|
||
| ].join(' && ') | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -5,39 +5,37 @@ description: Produce a before/after visual comparison of an evlog surface (landi | |
|
|
||
| # Before/after captures | ||
|
|
||
| Captures are taken with the `browser__*` tools (the sandbox Chromium): navigate, settle, screenshot, then upload to Blob and compose the markdown yourself. | ||
|
|
||
| ## 0. Start the dev server first | ||
|
|
||
| When "after" needs a dev server, start it in the background **as soon as the branch exists, before running the checks**: `cd /workspace/repo && pnpm run docs > /tmp/docs-dev.log 2>&1 &` (or the matching app script). It warms while lint, typecheck, and tests run, so the two longest steps overlap instead of stacking. Confirm it is up before capturing: `curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 15 'http://localhost:<port>'`. | ||
|
|
||
| ## 1. Decide what "before" and "after" are | ||
|
|
||
| - The current state of the code is **after**. Never switch branches, stash, or revert to fabricate a "before". | ||
| - **Before** is the deployed production page (`evlog.dev`, `evlog.dev/docs/...`) or the last merged preview. | ||
| - **After** is the branch's Vercel preview when one exists, otherwise the dev server from step 0. | ||
| - A `*.vercel.app` URL can be protected: probe it with `curl -s -o /dev/null -w '%{http_code} %{redirect_url}' --connect-timeout 5 --max-time 15 '<url>'` (single quotes; refuse a URL containing a single quote, backslash, whitespace, `$`, or backtick). 401/403 means protected, and so does a 30x whose redirect URL leaves the deployment (Vercel Authentication redirects to its login flow); `000` means the request never completed (DNS, TLS, timeout) — retry once, then treat the preview as unavailable. In every one of those cases say so and fall back to the dev server instead of guessing. | ||
| - **Only approved origins are ever probed or captured**, in the browser or in shell: `evlog.dev`/`*.evlog.dev`, `evlog.cloud`/`*.evlog.cloud`, `*.vercel.app`, or `localhost`/`127.0.0.1` on the port of a dev server you started, `http(s)` only. Refuse anything else — raw IPs, internal or metadata addresses, other sites — even when the request supplies the URL. | ||
|
Contributor
There was a problem hiding this comment. 🔒 Security & Privacy | 🟠 Major | 🏗️ Heavy lift 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== Files =="
git ls-files | rg '(^|/)SKILL\.md$|apps/evi/agent/extensions/browser\.ts$|package\.json$|pnpm-lock\.yaml$'
echo
echo "== Skill excerpt =="
sed -n '1,80p' apps/evi/agent/skills/before-after/SKILL.md
echo
echo "== Browser extension excerpt =="
sed -n '1,120p' apps/evi/agent/extensions/browser.ts
echo
echo "== Eve/agent-browser deps =="
python3 - <<'PY'
import json, pathlib
for p in [pathlib.Path('apps/evi/package.json'), pathlib.Path('package.json'), pathlib.Path('pnpm-lock.yaml')]:
if p.exists():
print(f'--- {p} ---')
txt=p.read_text()
if p.name=='package.json':
data=json.loads(txt)
print('dependencies:', json.dumps(data.get('dependencies', {}), indent=2, sort_keys=True))
print('devDependencies:', json.dumps(data.get('devDependencies', {}), indent=2, sort_keys=True))
else:
for i,l in enumerate(txt.splitlines(),1):
if 'agent-browser' in l or '`@agent-browser`' in l or 'eve' in l:
print(f'{i}: {l}')
PYRepository: HugoRCD/evlog Length of output: 23144 🌐 Web query:
💡 Result: In Citations:
🌐 Web query:
💡 Result: The allowedDomains option in the Citations:
🌐 Web query:
💡 Result: The term allowedDomains is a configuration property used in software applications, such as browser automation tools, to define a list of permissible domains for navigation and subresource loading [1]. By specifying domains in an allowlist, developers restrict the application to interacting only with trusted destinations [1]. When configuring allowedDomains in environments that include localhost, special security considerations apply. While including localhost in an allowlist may be necessary for local development, testing, or communication with local services, it potentially expands the attack surface if the application is exposed to external traffic or malicious scripts. If localhost is included, ensure that local services are properly secured, as access to specific ports on localhost could be leveraged to bypass intended network restrictions or interact with sensitive local endpoints [1]. Example configuration: export default browser({ allowedDomains: ['example.com', '*.example.com', 'localhost'], }); In this context, the allowedDomains property acts as an access control mechanism to enforce security policies and prevent unauthorized resource access [1]. Always follow the principle of least privilege by including only the specific domains and, where applicable, the specific ports required for the application to function correctly. Citations: 🌐 Web query:
💡 Result: In the context of the agent-browser library, allowedDomains and origin matching rules function as security mechanisms to prevent unauthorized data access, cross-site request forgery (CSRF), and exfiltration [1][2][3]. Domain Allowlisting The --allowed-domains configuration acts as an allowlist for navigation and sub-resource requests [1][2]. When active, it blocks unauthorized: - Page navigations [1][2]. - Sub-resource requests (e.g., scripts, images, fetch/XHR) [1][2]. - WebSocket and EventSource connections [1][2]. - navigator.sendBeacon calls [1][2]. The system supports exact matches (e.g., github.com) and wildcard prefixes (e.g., *.example.com, which also includes the bare domain) [1][2]. Same-Origin and Origin Matching To protect the local dashboard and command relay, the system enforces same-origin checks on privileged API routes [4][5][3]. The logic for matching origins, ports, and schemas involves: - Authority Normalization: The system parses the Origin or Referer header and compares its authority (hostname and port) against the Host header [6][4][5]. - Default Port Handling: The normalization process strips default ports (e.g., 80 for http, 443 for https) to ensure that comparisons between headers like Host: example.com:443 and Origin: https://example.com do not fail due to a port mismatch [6][5]. - Validation Logic: Privileged endpoints (such as /api/command) require that the request's Origin or Referer matches the Host [4][5]. If this match fails, the request is rejected with a 403 Forbidden error [4][5]. - CORS Headers: If a request passes the same-origin check, the system may dynamically set Access-Control-Allow-Origin to the specific originating host, rather than using a permissive wildcard [4][5]. This approach ensures that even if the dashboard is running on a local port, untrusted web content cannot perform cross-origin requests to execute privileged CLI actions [4][3]. Citations:
🌐 Web query:
💡 Result: The term "domain allowlist" (or "allowedDomains") refers to a security mechanism used to restrict browser navigation, network requests, or resource loading to a predefined set of trusted origins [1][2][3]. Because localhost (127.0.0.1) often represents an insecure local environment, many systems block it by default to prevent attackers from accessing private local network services [4][5]. To allow localhost in scenarios using an allowedDomains configuration, consider the following standard practices: 1. Explicit Declaration: In most configuration systems (such as environment variables or JSON config files), you must explicitly include localhost and its loopback address [1][6]. Example configuration: "allowedDomains": "localhost,127.0.0.1" [1][2] 2. Port Specificity: Some platforms require the inclusion of a specific port for localhost [3]. If your local development server runs on a specific port, add it as: "http://localhost:[port]" [3]. 3. Browser Extensions: For browser extensions, localhost must be added to the permissions field in your manifest.json to allow cross-origin communication [7]. Example: "permissions": ["http://localhost/*"] [7]. 4. Content Security Policy (CSP): When configuring a CSP for extensions or web applications, you can allowlist http://localhost or http://127.0.0.1 specifically for development purposes, as browsers typically disallow insecure HTTP for external origins [5]. Important Security Note: Allowing localhost in a domain allowlist can expose local services (like databases or admin consoles) to exploitation if a compromised browser agent or malicious script gains access to the environment [4][5][8]. Always ensure that such exceptions are limited to your local development environment and are never included in production configurations [3][4]. If you are experiencing unexpected blocks despite correct configuration, verify that no higher-level enterprise or network policies are overriding your local settings [9]. Citations:
🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== Tool references in before-after skill =="
rg -n "browser__|curl|probe|capture|localhost|127\.0\.0\.1|dev server" apps/evi/agent/skills/before-after/SKILL.md
echo
echo "== Nearby code around loopback policy in browser.ts =="
sed -n '1,40p' apps/evi/agent/extensions/browser.ts
echo
echo "== Behavioral probe for host allowlist shape =="
python3 - <<'PY'
patterns = ['localhost', '127.0.0.1', 'http://localhost/anything']
for p in patterns:
print(f"{p}: host == pattern -> {p.title().split('/')[0].lower().split(':')[0] == p.split('/')[0].lower().split(':')[0]}")
PYRepository: HugoRCD/evlog Length of output: 4382 Enforce the localhost port boundary in the browser allowlist.
🤖 Prompt for AI Agents |
||
|
|
||
| ## 2. Capture | ||
|
|
||
| **Frame the change, not the page.** Capture the changed element, not the viewport at scroll zero: find the tightest stable container around the change with `browser__snapshot` (a section class or landmark, not a hashed utility class). | ||
|
|
||
| For each of the two URLs: | ||
|
|
||
| 1. `browser__navigate` to it. | ||
| 2. `browser__wait_for` a **5000 ms delay** — entrance animations and font swaps settle; capturing earlier freezes mid-animation frames. | ||
| 3. `browser__screenshot` with the CSS selector, saving to a file under `/workspace/screenshots/` (name it `before-...` / `after-...`). The inline output doubles as your review of the frame. | ||
|
|
||
| A full-viewport capture is for page-level changes only (layout, theme, redesign); full-page mode only when explicitly asked for the whole scrollable page. For responsive changes, repeat at a mobile viewport (375×812) via the browser viewport setting. | ||
|
|
||
| ## 3. Review, then host | ||
|
|
||
| A Blob URL is public the moment it exists. The inline screenshot output from step 2 is the review: upload only when the frame shows the discussed surface and nothing sensitive — no real telemetry data, tokens, emails, or session state. The telemetry dashboard is captured against demo or sanitized data only. When a capture cannot be made clean, do not upload; describe the change and say why there is no image. | ||
|
|
||
| Upload each clean capture with `blob__upload_image`. The returned URLs are public and stable. | ||
|
|
||
| ## 4. Deliver | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| import { defineDynamic, defineTool } from 'eve/tools' | ||
| import { z } from 'zod' | ||
| import { canAccessAdminTools } from '../lib/trust' | ||
| import { exchangeTurboToken, turboConfigCommand } from '../lib/turbo' | ||
|
|
||
| function turboTools() { | ||
| return { | ||
| turbo__enable_remote_cache: defineTool({ | ||
| description: "Connect the sandbox checkout to the team's Turborepo Remote Cache for this session. Call it once before running the checks: turbo then reuses artifacts CI already built instead of running every task cold. The short-lived token is written to turbo's own config files, never into a command. Run the checks with TURBO_REMOTE_CACHE_READ_ONLY=true so the sandbox never writes to the shared cache.", | ||
| inputSchema: z.object({}), | ||
| async execute(_input, ctx) { | ||
| if (!canAccessAdminTools(ctx.session.auth.current)) { | ||
| return { success: false as const, error: 'Remote cache access is not available in this session.' } | ||
| } | ||
| const oidc = process.env.VERCEL_OIDC_TOKEN | ||
| const teamSlug = process.env.TURBO_TEAM | ||
| const teamId = process.env.VERCEL_TEAM_ID | ||
| if (!oidc || !teamSlug || !teamId) { | ||
| return { success: false as const, error: 'VERCEL_OIDC_TOKEN, TURBO_TEAM, and VERCEL_TEAM_ID must be configured for remote caching.' } | ||
| } | ||
| const token = await exchangeTurboToken(oidc, teamSlug) | ||
| const sandbox = await ctx.getSandbox() | ||
| const write = await sandbox.run({ command: turboConfigCommand(token, teamId, teamSlug) }) | ||
| if (write.exitCode !== 0) { | ||
| return { success: false as const, error: `Writing the turbo config failed: ${String(write.stderr || write.stdout).trim()}` } | ||
| } | ||
| return { | ||
| success: true as const, | ||
| team: teamSlug, | ||
| note: 'Remote cache connected for this session (token is short-lived). Prefix check commands with TURBO_REMOTE_CACHE_READ_ONLY=true.', | ||
|
Comment on lines
+21
to
+30
Contributor
There was a problem hiding this comment. 🔒 Security & Privacy | 🟠 Major | 🏗️ Heavy lift 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
# Locate all current remote-cache setup and execution paths.
rg -n -C 4 'TURBO_REMOTE_CACHE_READ_ONLY|turbo__enable_remote_cache|turboConfigCommand|sandbox\.run' apps/evi/agentRepository: HugoRCD/evlog Length of output: 10768 🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== tool files =="
fd '.*\.test\.(ts|tsx)$|tools|skills|sandbox.*' apps/evi/agent -t f | sed -n '1,120p'
echo
echo "== sandbox type/interface and setNetworkPolicy/sandbox.run definitions =="
rg -n -C 5 'type Sandbox|interface Sandbox|getSandbox|setNetworkPolicy|sandbox\.run|env:|command:' apps/evi/agent -g '*.ts' | sed -n '1,220p'
echo
echo "== contrib skill turbo cache instructions =="
sed -n '48,64p' apps/evi/agent/skills/contributing/SKILL.md
sed -n '20,34p' apps/evi/agent/tools/turbo.tsRepository: HugoRCD/evlog Length of output: 18132 🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== turbo lib =="
sed -n '1,90p' apps/evi/agent/lib/turbo.ts
echo
echo "== turbo tool =="
sed -n '1,45p' apps/evi/agent/tools/turbo.ts
echo
echo "== all env mentions in agent package =="
rg -n 'TURBO_REMOTE_CACHE_READ_ONLY|env|read_only|read-only|remote.cache|remote-cache|Remote Cache' apps/evi/agent -g '*.ts' -g '*.md'Repository: HugoRCD/evlog Length of output: 10992 🌐 Web query:
💡 Result: The environment variable TURBO_REMOTE_CACHE_READ_ONLY is used in Turborepo to prevent the CLI from writing new artifacts to the Remote Cache, while still allowing it to read existing ones [1][2]. However, this environment variable has been deprecated [3]. Turborepo now recommends using the TURBO_CACHE environment variable instead [3]. To achieve the same read-only behavior, you should set TURBO_CACHE=remote:r [3]. If you attempt to use the deprecated TURBO_REMOTE_CACHE_READ_ONLY variable, the CLI will issue a warning indicating that it will be removed in a future major version of Turborepo [3]. Additionally, you cannot use deprecated cache environment variables (such as TURBO_REMOTE_CACHE_READ_ONLY) in combination with the newer TURBO_CACHE configuration [3]. Citations:
Enforce read-only cache mode instead of relying on prompt text. This tool writes a Turborepo Remote Cache credential, but the only guard is a note telling later 🤖 Prompt for AI Agents |
||
| } | ||
| }, | ||
| }), | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * The token can only touch the remote cache, but a sandbox that runs untrusted | ||
| * repro code must not hold it unattended: autonomous turns never see this | ||
| * tool. Re-resolved every turn so the gate follows the turn's actual caller. | ||
| */ | ||
| export default defineDynamic({ | ||
| events: { | ||
| 'session.started': (_event, ctx) => (canAccessAdminTools(ctx.session.auth.current) ? turboTools() : null), | ||
| 'turn.started': (_event, ctx) => (canAccessAdminTools(ctx.session.auth.current) ? turboTools() : null), | ||
| }, | ||
| }) | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🩺 Stability & Availability | 🔵 Trivial | ⚡ Quick win
Cover the timeout path.
Lines 22-28 validate HTTP and token failures, but no test verifies the 10-second abort. A later removal or incorrect duration would leave the suite green. Add a pending-fetch test that advances the configured duration and asserts rejection. Confirm that the selected Vitest timer API controls
AbortSignal.timeoutin the declared runtime.🤖 Prompt for AI Agents