From 3b409793d206f69c088e982f6c5012445c571f16 Mon Sep 17 00:00:00 2001 From: Tofik Hasanov Date: Fri, 19 Jun 2026 11:42:59 -0400 Subject: [PATCH 01/31] feat(browserbase): add per-site auth profiles --- .../browser-auth-profile.service.spec.ts | 87 ++ .../browser-auth-profile.service.ts | 296 +++++ .../browser-auth-profiles.controller.ts | 136 ++ .../browser-automation-crud.service.ts | 203 +++ .../browser-automation-errors.spec.ts | 33 + .../browserbase/browser-automation-errors.ts | 142 +++ .../browser-automation-execution.service.ts | 250 ++++ .../browserbase/browser-evidence-execution.ts | 262 ++++ .../browser-evidence-runner.service.ts | 180 +++ .../browser-run-coordinator.spec.ts | 37 + .../browserbase/browser-run-coordinator.ts | 62 + .../browserbase-screenshot.service.ts | 100 ++ .../browserbase-session.service.ts | 202 +++ apps/api/src/browserbase/browserbase-url.ts | 12 + .../src/browserbase/browserbase.controller.ts | 24 +- .../api/src/browserbase/browserbase.module.ts | 19 +- .../browserbase/browserbase.service.spec.ts | 26 +- .../src/browserbase/browserbase.service.ts | 1099 ++--------------- apps/api/src/browserbase/credential-vault.ts | 19 + .../src/browserbase/dto/browserbase.dto.ts | 183 ++- .../run-browser-automation.spec.ts | 54 + .../run-browser-automation.ts | 54 +- .../run-browser-automations-schedule.spec.ts | 40 +- .../run-browser-automations-schedule.ts | 64 +- .../BrowserConnectionClient.test.tsx | 65 +- .../components/BrowserConnectionClient.tsx | 201 ++- .../BrowserConnectionInstructions.tsx | 33 + .../components/BrowserConnectionLiveView.tsx | 65 + .../BrowserConnectionProfileList.tsx | 71 ++ .../browser-automations/AutomationItem.tsx | 64 +- .../BrowserAutomationConfigDialog.tsx | 36 +- .../BrowserAutomationsList.test.tsx | 16 +- .../BrowserAutomationsList.tsx | 8 +- .../BrowserEmptyStates.tsx | 33 +- .../browser-automations/BrowserLiveView.tsx | 29 +- .../browser-automations/RunItem.tsx | 29 +- .../[orgId]/tasks/[taskId]/hooks/types.ts | 35 + .../tasks/[taskId]/hooks/useBrowserContext.ts | 53 +- .../migration.sql | 86 ++ .../prisma/schema/browserbase-context.prisma | 79 ++ packages/db/prisma/schema/organization.prisma | 3 +- packages/docs/openapi.json | 1039 +++++++--------- 42 files changed, 3659 insertions(+), 1870 deletions(-) create mode 100644 apps/api/src/browserbase/browser-auth-profile.service.spec.ts create mode 100644 apps/api/src/browserbase/browser-auth-profile.service.ts create mode 100644 apps/api/src/browserbase/browser-auth-profiles.controller.ts create mode 100644 apps/api/src/browserbase/browser-automation-crud.service.ts create mode 100644 apps/api/src/browserbase/browser-automation-errors.spec.ts create mode 100644 apps/api/src/browserbase/browser-automation-errors.ts create mode 100644 apps/api/src/browserbase/browser-automation-execution.service.ts create mode 100644 apps/api/src/browserbase/browser-evidence-execution.ts create mode 100644 apps/api/src/browserbase/browser-evidence-runner.service.ts create mode 100644 apps/api/src/browserbase/browser-run-coordinator.spec.ts create mode 100644 apps/api/src/browserbase/browser-run-coordinator.ts create mode 100644 apps/api/src/browserbase/browserbase-screenshot.service.ts create mode 100644 apps/api/src/browserbase/browserbase-session.service.ts create mode 100644 apps/api/src/browserbase/browserbase-url.ts create mode 100644 apps/api/src/browserbase/credential-vault.ts create mode 100644 apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts create mode 100644 apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionInstructions.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionLiveView.tsx create mode 100644 apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionProfileList.tsx create mode 100644 packages/db/prisma/migrations/20260618120000_browser_auth_profiles/migration.sql diff --git a/apps/api/src/browserbase/browser-auth-profile.service.spec.ts b/apps/api/src/browserbase/browser-auth-profile.service.spec.ts new file mode 100644 index 0000000000..2f254c0737 --- /dev/null +++ b/apps/api/src/browserbase/browser-auth-profile.service.spec.ts @@ -0,0 +1,87 @@ +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; + +jest.mock('@db', () => ({ + db: { + browserAuthProfile: { + findMany: jest.fn(), + findUnique: jest.fn(), + findUniqueOrThrow: jest.fn(), + findFirst: jest.fn(), + create: jest.fn(), + update: jest.fn(), + }, + browserbaseContext: { + findUnique: jest.fn(), + create: jest.fn(), + update: jest.fn(), + }, + }, +})); + +import { db } from '@db'; + +describe('BrowserAuthProfileService', () => { + let sessions: BrowserbaseSessionService; + let service: BrowserAuthProfileService; + + beforeEach(() => { + jest.clearAllMocks(); + sessions = new BrowserbaseSessionService(); + jest.spyOn(sessions, 'createBrowserbaseContext').mockResolvedValue('ctx_new'); + service = new BrowserAuthProfileService(sessions); + }); + + it('normalizes hostname and login identity when creating a profile', async () => { + (db.browserAuthProfile.findUnique as jest.Mock).mockResolvedValue(null); + (db.browserbaseContext.findUnique as jest.Mock).mockResolvedValue(null); + (db.browserAuthProfile.create as jest.Mock).mockResolvedValue({ + id: 'bap_1', + organizationId: 'org_1', + hostname: 'github.com', + loginIdentity: 'svc@example.com', + }); + + await service.getOrCreateProfileFromUrl({ + organizationId: 'org_1', + url: 'https://GitHub.com/acme/repo', + loginIdentity: ' SVC@EXAMPLE.COM ', + }); + + expect(db.browserAuthProfile.create).toHaveBeenCalledWith({ + data: expect.objectContaining({ + organizationId: 'org_1', + hostname: 'github.com', + loginIdentity: 'svc@example.com', + contextId: 'ctx_new', + }), + }); + }); + + it('prioritizes a verified profile for the target hostname', async () => { + (db.browserAuthProfile.findMany as jest.Mock).mockResolvedValue([ + { id: 'bap_old', status: 'needs_reauth', hostname: 'github.com' }, + { id: 'bap_verified', status: 'verified', hostname: 'github.com' }, + ]); + + const profile = await service.resolveProfileForTarget({ + organizationId: 'org_1', + targetUrl: 'https://github.com/acme/repo', + }); + + expect(profile.id).toBe('bap_verified'); + expect(db.browserAuthProfile.findMany).toHaveBeenCalledWith({ + where: { organizationId: 'org_1', hostname: 'github.com' }, + orderBy: { updatedAt: 'desc' }, + }); + }); + + it('treats a pending legacy org context as unavailable', async () => { + (db.browserbaseContext.findUnique as jest.Mock).mockResolvedValue({ + organizationId: 'org_1', + contextId: '__PENDING__', + }); + + await expect(service.getOrgContext('org_1')).resolves.toBeNull(); + }); +}); diff --git a/apps/api/src/browserbase/browser-auth-profile.service.ts b/apps/api/src/browserbase/browser-auth-profile.service.ts new file mode 100644 index 0000000000..152aab7f76 --- /dev/null +++ b/apps/api/src/browserbase/browser-auth-profile.service.ts @@ -0,0 +1,296 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { db } from '@db'; +import { + defaultProfileDisplayName, + normalizeHostnameFromUrl, + normalizeLoginIdentity, +} from './browserbase-url'; +import { BrowserbaseSessionService } from './browserbase-session.service'; + +const PENDING_CONTEXT_ID = '__PENDING__'; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const isPrismaUniqueConstraintError = (error: unknown): boolean => { + if (typeof error !== 'object' || error === null) return false; + if (!('code' in error)) return false; + const code = (error as { code?: unknown }).code; + return code === 'P2002'; +}; + +export interface AuthProfileInput { + organizationId: string; + url: string; + displayName?: string; + loginIdentity?: string; + vaultProvider?: string; + vaultExternalItemRef?: string; + vaultConnectionId?: string; +} + +@Injectable() +export class BrowserAuthProfileService { + private readonly logger = new Logger(BrowserAuthProfileService.name); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + ) {} + + async listProfiles(organizationId: string) { + return db.browserAuthProfile.findMany({ + where: { organizationId }, + orderBy: [{ hostname: 'asc' }, { updatedAt: 'desc' }], + }); + } + + async getProfile({ + profileId, + organizationId, + }: { + profileId: string; + organizationId: string; + }) { + return db.browserAuthProfile.findFirst({ + where: { id: profileId, organizationId }, + }); + } + + async getOrCreateProfileFromUrl(input: AuthProfileInput) { + const hostname = normalizeHostnameFromUrl(input.url); + const loginIdentity = normalizeLoginIdentity(input.loginIdentity); + + const existing = await db.browserAuthProfile.findUnique({ + where: { + organizationId_hostname_loginIdentity: { + organizationId: input.organizationId, + hostname, + loginIdentity, + }, + }, + }); + + if (existing) { + return { profile: existing, isNew: false }; + } + + const contextId = await this.resolveInitialContextId(input.organizationId); + + try { + const profile = await db.browserAuthProfile.create({ + data: { + organizationId: input.organizationId, + hostname, + loginIdentity, + displayName: input.displayName?.trim() || defaultProfileDisplayName(hostname), + contextId, + lastAuthCheckUrl: input.url, + vaultProvider: input.vaultProvider, + vaultExternalItemRef: input.vaultExternalItemRef, + vaultConnectionId: input.vaultConnectionId, + }, + }); + return { profile, isNew: true }; + } catch (error) { + if (!isPrismaUniqueConstraintError(error)) { + throw error; + } + + const profile = await db.browserAuthProfile.findUniqueOrThrow({ + where: { + organizationId_hostname_loginIdentity: { + organizationId: input.organizationId, + hostname, + loginIdentity, + }, + }, + }); + return { profile, isNew: false }; + } + } + + async resolveProfileForTarget(input: { + organizationId: string; + targetUrl: string; + profileId?: string; + }) { + if (input.profileId) { + const profile = await this.getProfile({ + profileId: input.profileId, + organizationId: input.organizationId, + }); + if (!profile) { + throw new Error('Browser auth profile not found'); + } + return profile; + } + + const hostname = normalizeHostnameFromUrl(input.targetUrl); + const profiles = await db.browserAuthProfile.findMany({ + where: { organizationId: input.organizationId, hostname }, + orderBy: { updatedAt: 'desc' }, + }); + + const verified = profiles.find((profile) => profile.status === 'verified'); + if (verified) return verified; + if (profiles[0]) return profiles[0]; + + const created = await this.getOrCreateProfileFromUrl({ + organizationId: input.organizationId, + url: input.targetUrl, + }); + return created.profile; + } + + async startProfileSession(input: { + organizationId: string; + profileId: string; + }): Promise<{ sessionId: string; liveViewUrl: string }> { + const profile = await this.getProfile(input); + if (!profile) { + throw new Error('Browser auth profile not found'); + } + return this.sessions.createSessionWithContext(profile.contextId); + } + + async verifyProfileSession(input: { + organizationId: string; + profileId: string; + sessionId: string; + url: string; + }) { + const profile = await this.getProfile({ + organizationId: input.organizationId, + profileId: input.profileId, + }); + if (!profile) { + throw new Error('Browser auth profile not found'); + } + + const auth = await this.sessions.checkLoginStatus(input.sessionId, input.url); + const status = auth.isLoggedIn ? 'verified' : 'needs_reauth'; + const updated = await db.browserAuthProfile.update({ + where: { id: profile.id }, + data: { + status, + lastVerifiedAt: auth.isLoggedIn ? new Date() : profile.lastVerifiedAt, + lastAuthCheckUrl: input.url, + blockedReason: auth.isLoggedIn ? null : 'Login verification failed.', + }, + }); + + return { profile: updated, auth }; + } + + async markNeedsReauth(input: { + organizationId: string; + profileId: string; + reason?: string; + }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new Error('Browser auth profile not found'); + } + + return db.browserAuthProfile.update({ + where: { id: profile.id }, + data: { + status: 'needs_reauth', + blockedReason: input.reason ?? 'Authentication needs to be refreshed.', + }, + }); + } + + async markBlocked(input: { + organizationId: string; + profileId: string; + reason: string; + }) { + const profile = await this.getProfile(input); + if (!profile) { + throw new Error('Browser auth profile not found'); + } + + return db.browserAuthProfile.update({ + where: { id: profile.id }, + data: { + status: 'blocked', + blockedReason: input.reason, + }, + }); + } + + async getOrCreateOrgContext( + organizationId: string, + ): Promise<{ contextId: string; isNew: boolean }> { + const existing = await db.browserbaseContext.findUnique({ + where: { organizationId }, + }); + + if (existing && existing.contextId !== PENDING_CONTEXT_ID) { + return { contextId: existing.contextId, isNew: false }; + } + + try { + await db.browserbaseContext.create({ + data: { organizationId, contextId: PENDING_CONTEXT_ID }, + }); + + const contextId = await this.sessions.createBrowserbaseContext(); + + await db.browserbaseContext.update({ + where: { organizationId }, + data: { contextId }, + }); + + return { contextId, isNew: true }; + } catch (error) { + if (!isPrismaUniqueConstraintError(error)) { + throw error; + } + } + + return this.waitForOrgContext(organizationId); + } + + async getOrgContext(organizationId: string): Promise<{ contextId: string } | null> { + const context = await db.browserbaseContext.findUnique({ + where: { organizationId }, + }); + + if (!context || context.contextId === PENDING_CONTEXT_ID) return null; + return { contextId: context.contextId }; + } + + private async resolveInitialContextId(organizationId: string): Promise { + const legacy = await this.getOrgContext(organizationId); + if (legacy) return legacy.contextId; + return this.sessions.createBrowserbaseContext(); + } + + private async waitForOrgContext( + organizationId: string, + ): Promise<{ contextId: string; isNew: boolean }> { + const maxWaitMs = 10_000; + const pollMs = 200; + const startedAt = Date.now(); + + while (Date.now() - startedAt < maxWaitMs) { + const current = await db.browserbaseContext.findUnique({ + where: { organizationId }, + }); + + if (current && current.contextId !== PENDING_CONTEXT_ID) { + return { contextId: current.contextId, isNew: false }; + } + + if (!current) { + return await this.getOrCreateOrgContext(organizationId); + } + + await delay(pollMs); + } + + this.logger.warn(`Timed out waiting for Browserbase context creation for org ${organizationId}`); + throw new Error('Browser context initialization is taking too long. Please retry.'); + } +} diff --git a/apps/api/src/browserbase/browser-auth-profiles.controller.ts b/apps/api/src/browserbase/browser-auth-profiles.controller.ts new file mode 100644 index 0000000000..10d2c2b347 --- /dev/null +++ b/apps/api/src/browserbase/browser-auth-profiles.controller.ts @@ -0,0 +1,136 @@ +import { Body, Controller, Get, Param, Post, UseGuards } from '@nestjs/common'; +import { + ApiBody, + ApiOperation, + ApiParam, + ApiResponse, + ApiSecurity, + ApiTags, +} from '@nestjs/swagger'; +import { OrganizationId } from '../auth/auth-context.decorator'; +import { HybridAuthGuard } from '../auth/hybrid-auth.guard'; +import { PermissionGuard } from '../auth/permission.guard'; +import { RequirePermission } from '../auth/require-permission.decorator'; +import { BrowserbaseService } from './browserbase.service'; +import { + BrowserAuthProfileResponseDto, + MarkAuthProfileNeedsReauthDto, + ResolveAuthProfileDto, + ResolveAuthProfileResponseDto, + SessionResponseDto, + VerifyAuthProfileResponseDto, + VerifyAuthProfileSessionDto, +} from './dto/browserbase.dto'; + +@ApiTags('Browserbase') +@Controller({ path: 'browserbase', version: '1' }) +@UseGuards(HybridAuthGuard, PermissionGuard) +@ApiSecurity('apikey') +export class BrowserAuthProfilesController { + constructor(private readonly browserbaseService: BrowserbaseService) {} + + @Get('profiles') + @RequirePermission('integration', 'read') + @ApiOperation({ + summary: 'List browser auth profiles', + description: + 'List per-site browser auth profiles for the organization, including hostname, verification status, and vault metadata references.', + }) + @ApiResponse({ + status: 200, + type: [BrowserAuthProfileResponseDto], + }) + async listProfiles( + @OrganizationId() organizationId: string, + ): Promise { + return (await this.browserbaseService.listAuthProfiles( + organizationId, + )) as BrowserAuthProfileResponseDto[]; + } + + @Post('profiles/resolve') + @RequirePermission('integration', 'create') + @ApiOperation({ + summary: 'Create or get browser auth profile', + description: + 'Normalize a website URL to a hostname and create or reuse the matching browser auth profile for that org and login identity.', + }) + @ApiBody({ type: ResolveAuthProfileDto }) + @ApiResponse({ + status: 201, + type: ResolveAuthProfileResponseDto, + }) + async resolveProfile( + @OrganizationId() organizationId: string, + @Body() dto: ResolveAuthProfileDto, + ): Promise { + return (await this.browserbaseService.getOrCreateAuthProfile({ + organizationId, + ...dto, + })) as ResolveAuthProfileResponseDto; + } + + @Post('profiles/:profileId/session') + @RequirePermission('integration', 'read') + @ApiOperation({ + summary: 'Start browser auth profile session', + description: + 'Create a Browserbase Live View session using a specific auth profile context so a user can log in or complete 2FA manually.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiResponse({ status: 201, type: SessionResponseDto }) + async startProfileSession( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + ): Promise { + return await this.browserbaseService.startAuthProfileSession({ + organizationId, + profileId, + }); + } + + @Post('profiles/:profileId/verify') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Verify browser auth profile session', + description: + 'Check whether the Live View session is authenticated for the profile URL and update the profile status to verified or needs reauth.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: VerifyAuthProfileSessionDto }) + @ApiResponse({ status: 200, type: VerifyAuthProfileResponseDto }) + async verifyProfileSession( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: VerifyAuthProfileSessionDto, + ): Promise { + return (await this.browserbaseService.verifyAuthProfileSession({ + organizationId, + profileId, + sessionId: dto.sessionId, + url: dto.url, + })) as VerifyAuthProfileResponseDto; + } + + @Post('profiles/:profileId/needs-reauth') + @RequirePermission('integration', 'update') + @ApiOperation({ + summary: 'Mark browser auth profile needs reauth', + description: + 'Mark a browser auth profile as needing user reauthentication without storing any raw credentials or TOTP secrets.', + }) + @ApiParam({ name: 'profileId', description: 'Browser auth profile ID' }) + @ApiBody({ type: MarkAuthProfileNeedsReauthDto }) + @ApiResponse({ status: 200, type: BrowserAuthProfileResponseDto }) + async markNeedsReauth( + @OrganizationId() organizationId: string, + @Param('profileId') profileId: string, + @Body() dto: MarkAuthProfileNeedsReauthDto, + ): Promise { + return (await this.browserbaseService.markAuthProfileNeedsReauth({ + organizationId, + profileId, + reason: dto.reason, + })) as BrowserAuthProfileResponseDto; + } +} diff --git a/apps/api/src/browserbase/browser-automation-crud.service.ts b/apps/api/src/browserbase/browser-automation-crud.service.ts new file mode 100644 index 0000000000..d13d67afc0 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-crud.service.ts @@ -0,0 +1,203 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { db, TaskFrequency } from '@db'; +import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; + +const normalizeCriteria = (value: string | null | undefined): string | null => { + if (value == null) return null; + const trimmed = value.trim(); + return trimmed.length === 0 ? null : trimmed; +}; + +@Injectable() +export class BrowserAutomationCrudService { + constructor( + private readonly screenshots: BrowserbaseScreenshotService = new BrowserbaseScreenshotService(), + ) {} + + async createBrowserAutomation( + data: { + taskId: string; + name: string; + description?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + scheduleFrequency?: TaskFrequency; + }, + organizationId?: string, + ) { + if (organizationId) { + await this.requireTaskInOrg({ taskId: data.taskId, organizationId }); + } + + return db.browserAutomation.create({ + data: { + taskId: data.taskId, + name: data.name, + description: data.description, + targetUrl: data.targetUrl, + instruction: data.instruction, + evaluationCriteria: normalizeCriteria(data.evaluationCriteria), + isEnabled: true, + ...(data.scheduleFrequency !== undefined + ? { scheduleFrequency: data.scheduleFrequency } + : {}), + }, + }); + } + + async getBrowserAutomation(automationId: string, organizationId?: string) { + const automation = await db.browserAutomation.findUnique({ + where: { id: automationId }, + include: { + task: { select: { organizationId: true } }, + runs: { orderBy: { createdAt: 'desc' }, take: 10 }, + }, + }); + return this.hideCrossOrgAutomation({ automation, organizationId }); + } + + async getBrowserAutomationsForTask(taskId: string, organizationId?: string) { + if (organizationId) { + await this.requireTaskInOrg({ taskId, organizationId }); + } + + return db.browserAutomation.findMany({ + where: { taskId }, + include: { + runs: { orderBy: { createdAt: 'desc' }, take: 1 }, + }, + orderBy: { createdAt: 'desc' }, + }); + } + + async updateBrowserAutomation( + automationId: string, + data: { + name?: string; + description?: string; + targetUrl?: string; + instruction?: string; + evaluationCriteria?: string; + isEnabled?: boolean; + scheduleFrequency?: TaskFrequency; + }, + organizationId?: string, + ) { + if (organizationId) { + await this.requireAutomationInOrg({ automationId, organizationId }); + } + + const { evaluationCriteria, scheduleFrequency, ...rest } = data; + return db.browserAutomation.update({ + where: { id: automationId }, + data: { + ...rest, + ...(evaluationCriteria !== undefined + ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } + : {}), + ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), + }, + }); + } + + async deleteBrowserAutomation(automationId: string, organizationId?: string) { + if (organizationId) { + await this.requireAutomationInOrg({ automationId, organizationId }); + } + return db.browserAutomation.delete({ where: { id: automationId } }); + } + + async getRunWithPresignedUrl(runId: string, organizationId?: string) { + const run = await db.browserAutomationRun.findUnique({ + where: { id: runId }, + include: { automation: { include: { task: true } } }, + }); + if (!run) return null; + if (organizationId && run.automation.task.organizationId !== organizationId) { + return null; + } + if (!run.screenshotUrl) return run; + const screenshotUrl = await this.screenshots.getPresignedUrl({ + key: run.screenshotUrl, + }); + return { ...run, screenshotUrl }; + } + + async getAutomationsWithPresignedUrls(taskId: string, organizationId?: string) { + const automations = await this.getBrowserAutomationsForTask(taskId, organizationId); + return Promise.all( + automations.map(async (automation) => { + const runsWithUrls = await Promise.all( + automation.runs.map(async (run) => { + if (!run.screenshotUrl) return run; + const screenshotUrl = await this.screenshots.getPresignedUrl({ + key: run.screenshotUrl, + }); + return { ...run, screenshotUrl }; + }), + ); + return { ...automation, runs: runsWithUrls }; + }), + ); + } + + async getAutomationRuns( + automationId: string, + limit = 20, + organizationId?: string, + ) { + if (organizationId) { + await this.requireAutomationInOrg({ automationId, organizationId }); + } + return db.browserAutomationRun.findMany({ + where: { automationId }, + orderBy: { createdAt: 'desc' }, + take: limit, + }); + } + + async getAutomationRun(runId: string, organizationId?: string) { + return this.getRunWithPresignedUrl(runId, organizationId); + } + + private hideCrossOrgAutomation({ + automation, + organizationId, + }: { + automation: T | null; + organizationId?: string; + }): T | null { + if (!automation) return null; + if (organizationId && automation.task.organizationId !== organizationId) { + return null; + } + return automation; + } + + private async requireAutomationInOrg(input: { + automationId: string; + organizationId: string; + }) { + const automation = await db.browserAutomation.findUnique({ + where: { id: input.automationId }, + include: { task: { select: { organizationId: true } } }, + }); + const scoped = this.hideCrossOrgAutomation({ + automation, + organizationId: input.organizationId, + }); + if (!scoped) throw new NotFoundException('Automation not found'); + } + + private async requireTaskInOrg(input: { + taskId: string; + organizationId: string; + }) { + const task = await db.task.findFirst({ + where: { id: input.taskId, organizationId: input.organizationId }, + select: { id: true }, + }); + if (!task) throw new NotFoundException('Task not found'); + } +} diff --git a/apps/api/src/browserbase/browser-automation-errors.spec.ts b/apps/api/src/browserbase/browser-automation-errors.spec.ts new file mode 100644 index 0000000000..635acee646 --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-errors.spec.ts @@ -0,0 +1,33 @@ +import { classifyBrowserAutomationError } from './browser-automation-errors'; + +describe('classifyBrowserAutomationError', () => { + it('classifies auth expiry as needs_reauth', () => { + const result = classifyBrowserAutomationError( + new Error('Session expired. User is not logged in.'), + 'auth', + ); + + expect(result.code).toBe('needs_reauth'); + expect(result.stage).toBe('auth'); + expect(result.needsReauth).toBe(true); + }); + + it('classifies 2FA and device approval as needs_user_action', () => { + const result = classifyBrowserAutomationError( + new Error('Device approval required before continuing'), + 'auth', + ); + + expect(result.code).toBe('needs_user_action'); + expect(result.blockedReason).toContain('Manual 2FA'); + }); + + it('classifies captcha and rate limit failures with stable codes', () => { + expect( + classifyBrowserAutomationError(new Error('reCAPTCHA required')).code, + ).toBe('captcha_blocked'); + expect( + classifyBrowserAutomationError(new Error('429 Too Many Requests')).code, + ).toBe('rate_limited'); + }); +}); diff --git a/apps/api/src/browserbase/browser-automation-errors.ts b/apps/api/src/browserbase/browser-automation-errors.ts new file mode 100644 index 0000000000..5430a60c6c --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-errors.ts @@ -0,0 +1,142 @@ +export type BrowserAutomationFailureCode = + | 'needs_reauth' + | 'needs_user_action' + | 'rate_limited' + | 'captcha_blocked' + | 'timeout' + | 'browser_session_lost' + | 'evaluation_failed' + | 'unknown'; + +export type BrowserAutomationFailureStage = + | 'auth' + | 'navigation' + | 'action' + | 'screenshot' + | 'evaluation' + | 'upload' + | 'session' + | 'unknown'; + +export interface ClassifiedBrowserAutomationError { + code: BrowserAutomationFailureCode; + stage: BrowserAutomationFailureStage; + userFacing: string; + needsReauth: boolean; + blockedReason?: string; +} + +const getErrorText = (error: unknown): string => { + if (error instanceof Error) return error.message; + if (typeof error === 'string') return error; + return String(error); +}; + +export function classifyBrowserAutomationError( + error: unknown, + stage: BrowserAutomationFailureStage = 'unknown', +): ClassifiedBrowserAutomationError { + const message = getErrorText(error); + const lower = message.toLowerCase(); + + if ( + lower.includes('session expired') || + lower.includes('not logged in') || + lower.includes('login') || + lower.includes('sign in') || + lower.includes('unauthorized') || + lower.includes('forbidden') + ) { + return { + code: 'needs_reauth', + stage: 'auth', + userFacing: 'Authentication is no longer valid. Reconnect this browser profile.', + needsReauth: true, + blockedReason: 'Authentication expired or could not be verified.', + }; + } + + if ( + lower.includes('two-factor') || + lower.includes('2fa') || + lower.includes('totp') || + lower.includes('device approval') || + lower.includes('approval required') || + lower.includes('verify your identity') + ) { + return { + code: 'needs_user_action', + stage: 'auth', + userFacing: 'The site requires a user action such as 2FA or device approval.', + needsReauth: true, + blockedReason: 'Manual 2FA or device approval is required.', + }; + } + + if (lower.includes('captcha') || lower.includes('recaptcha')) { + return { + code: 'captcha_blocked', + stage: 'auth', + userFacing: 'The site presented a captcha that automation cannot complete.', + needsReauth: false, + blockedReason: 'Captcha challenge blocked automation.', + }; + } + + if ( + lower.includes('rate limit') || + lower.includes('too many requests') || + lower.includes('429') + ) { + return { + code: 'rate_limited', + stage, + userFacing: 'The site or Browserbase rate limited this automation.', + needsReauth: false, + }; + } + + if ( + lower.includes('timeout') || + lower.includes('timed out') || + lower.includes('deadline') + ) { + return { + code: 'timeout', + stage, + userFacing: 'The browser automation timed out before it could finish.', + needsReauth: false, + }; + } + + if ( + lower.includes('no page') || + lower.includes('page closed') || + lower.includes('target closed') || + lower.includes('browser has been closed') || + lower.includes('session ended') + ) { + return { + code: 'browser_session_lost', + stage: 'session', + userFacing: 'The browser session ended before the automation finished.', + needsReauth: false, + }; + } + + return { + code: 'unknown', + stage, + userFacing: message || 'Browser automation failed for an unknown reason.', + needsReauth: false, + }; +} + +export function evaluationFailedError(message: string): ClassifiedBrowserAutomationError { + return { + code: 'evaluation_failed', + stage: 'evaluation', + userFacing: message, + needsReauth: false, + }; +} diff --git a/apps/api/src/browserbase/browser-automation-execution.service.ts b/apps/api/src/browserbase/browser-automation-execution.service.ts new file mode 100644 index 0000000000..fc57f08b4e --- /dev/null +++ b/apps/api/src/browserbase/browser-automation-execution.service.ts @@ -0,0 +1,250 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { db } from '@db'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { + BrowserEvidenceRunnerService, + type BrowserEvidenceRunResult, +} from './browser-evidence-runner.service'; +import { BrowserbaseSessionService } from './browserbase-session.service'; + +@Injectable() +export class BrowserAutomationExecutionService { + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + sessions, + ), + private readonly runner: BrowserEvidenceRunnerService = new BrowserEvidenceRunnerService( + sessions, + ), + ) {} + + async startAutomationWithLiveView(automationId: string, organizationId: string) { + const automation = await this.getRunnableAutomation({ + automationId, + organizationId, + }); + const profile = await this.profiles.resolveProfileForTarget({ + organizationId, + targetUrl: automation.targetUrl, + }); + const run = await this.createRun({ automationId, profileId: profile.id }); + const { sessionId, liveViewUrl } = await this.sessions.createSessionWithContext( + profile.contextId, + ); + return { runId: run.id, sessionId, liveViewUrl, profileId: profile.id }; + } + + async executeAutomationOnSession( + automationId: string, + runId: string, + sessionId: string, + organizationId: string, + ) { + const automation = await this.getRunnableAutomation({ + automationId, + organizationId, + }); + const run = await db.browserAutomationRun.findUnique({ where: { id: runId } }); + if (!run || run.automationId !== automationId) throw new Error('Run not found'); + + const profile = await this.profiles.resolveProfileForTarget({ + organizationId, + targetUrl: automation.targetUrl, + profileId: run.profileId ?? undefined, + }); + const result = await this.runner.executeEvidenceOnSession({ + organizationId, + taskId: automation.taskId, + automationId, + runId, + sessionId, + targetUrl: automation.targetUrl, + instruction: automation.instruction, + evaluationCriteria: automation.evaluationCriteria, + profile: { + id: profile.id, + hostname: profile.hostname, + contextId: profile.contextId, + }, + }); + + await this.finishRun({ runId, startedAt: run.startedAt, result }); + await this.applyProfileResult({ organizationId, profileId: profile.id, result }); + return this.toRunResponse({ runId, result }); + } + + async runBrowserAutomation(automationId: string, organizationId: string) { + const automation = await this.getRunnableAutomation({ + automationId, + organizationId, + }); + const profile = await this.profiles.resolveProfileForTarget({ + organizationId, + targetUrl: automation.targetUrl, + }); + const run = await this.createRun({ automationId, profileId: profile.id }); + + if (profile.status !== 'verified') { + const result = this.profileBlockedResult(profile.status); + await this.finishRun({ runId: run.id, startedAt: run.startedAt, result }); + return this.toRunResponse({ runId: run.id, result }); + } + + const result = await this.runner.runEvidence({ + organizationId, + taskId: automation.taskId, + automationId, + runId: run.id, + targetUrl: automation.targetUrl, + instruction: automation.instruction, + evaluationCriteria: automation.evaluationCriteria, + profile: { + id: profile.id, + hostname: profile.hostname, + contextId: profile.contextId, + }, + }); + await this.finishRun({ runId: run.id, startedAt: run.startedAt, result }); + await this.applyProfileResult({ organizationId, profileId: profile.id, result }); + return this.toRunResponse({ runId: run.id, result }); + } + + private async createRun(input: { automationId: string; profileId?: string }) { + const attemptCount = + (await db.browserAutomationRun.count({ + where: { automationId: input.automationId }, + })) + 1; + return db.browserAutomationRun.create({ + data: { + automationId: input.automationId, + profileId: input.profileId, + status: 'running', + startedAt: new Date(), + attemptCount, + }, + }); + } + + private async finishRun(input: { + runId: string; + startedAt: Date | null; + result: BrowserEvidenceRunResult; + }) { + await db.browserAutomationRun.update({ + where: { id: input.runId }, + data: { + status: input.result.status, + completedAt: new Date(), + durationMs: input.startedAt ? Date.now() - input.startedAt.getTime() : 0, + screenshotUrl: input.result.screenshotKey, + evaluationStatus: input.result.evaluationStatus ?? null, + evaluationReason: input.result.evaluationReason ?? null, + error: input.result.error, + failureCode: input.result.failureCode, + failureStage: input.result.failureStage, + blockedReason: input.result.blockedReason, + finalUrl: input.result.finalUrl, + logs: input.result.logs, + }, + }); + } + + private async applyProfileResult(input: { + organizationId: string; + profileId: string; + result: BrowserEvidenceRunResult; + }) { + if (input.result.failureCode === 'needs_reauth') { + await this.profiles.markNeedsReauth({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: input.result.blockedReason, + }); + } + + if ( + input.result.failureCode === 'captcha_blocked' || + input.result.failureCode === 'needs_user_action' + ) { + await this.profiles.markBlocked({ + organizationId: input.organizationId, + profileId: input.profileId, + reason: + input.result.blockedReason ?? + input.result.error ?? + 'User action is required before this automation can run.', + }); + } + } + + private profileBlockedResult(status: string): BrowserEvidenceRunResult { + const needsUserAction = status === 'blocked'; + return { + success: false, + status: 'blocked', + error: needsUserAction + ? 'This browser profile is blocked. Resolve the blocked state before running automations.' + : 'This browser profile is not verified. Reconnect it before running automations.', + needsReauth: !needsUserAction, + failureCode: needsUserAction ? 'needs_user_action' : 'needs_reauth', + failureStage: 'auth', + blockedReason: needsUserAction + ? 'Browser profile is blocked.' + : 'Browser profile is not verified.', + logs: [], + }; + } + + private toRunResponse(input: { + runId: string; + result: BrowserEvidenceRunResult; + }) { + return { + runId: input.runId, + success: input.result.success, + screenshotUrl: input.result.screenshotUrl, + evaluationStatus: input.result.evaluationStatus, + evaluationReason: input.result.evaluationReason, + error: input.result.error, + needsReauth: input.result.needsReauth, + failureCode: input.result.failureCode, + failureStage: input.result.failureStage, + blockedReason: input.result.blockedReason, + }; + } + + private async getRunnableAutomation(input: { + automationId: string; + organizationId: string; + }) { + const automation = await db.browserAutomation.findUnique({ + where: { id: input.automationId }, + include: { + task: { + select: { title: true, description: true, organizationId: true }, + }, + }, + }); + const scoped = this.hideCrossOrgAutomation({ + automation, + organizationId: input.organizationId, + }); + if (!scoped) throw new NotFoundException('Automation not found'); + return scoped; + } + + private hideCrossOrgAutomation({ + automation, + organizationId, + }: { + automation: T | null; + organizationId?: string; + }): T | null { + if (!automation) return null; + if (organizationId && automation.task.organizationId !== organizationId) { + return null; + } + return automation; + } +} diff --git a/apps/api/src/browserbase/browser-evidence-execution.ts b/apps/api/src/browserbase/browser-evidence-execution.ts new file mode 100644 index 0000000000..8e1eec4bf0 --- /dev/null +++ b/apps/api/src/browserbase/browser-evidence-execution.ts @@ -0,0 +1,262 @@ +import { Logger } from '@nestjs/common'; +import { z } from 'zod'; +import { renderOverlay } from './screenshot-overlay'; +import type { BrowserbaseSessionService } from './browserbase-session.service'; +import { + type BrowserAutomationFailureCode, + type BrowserAutomationFailureStage, + type ClassifiedBrowserAutomationError, + classifyBrowserAutomationError, + evaluationFailedError, +} from './browser-automation-errors'; +import type { BrowserEvidenceSessionInput } from './browser-evidence-runner.service'; + +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const STAGEHAND_CUA_MODEL = 'anthropic/claude-sonnet-4-6'; +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export interface BrowserEvidenceLog { + timestamp: string; + stage: string; + message: string; +} + +export interface BrowserEvidenceExecutionResult { + success: boolean; + screenshot?: string; + finalUrl?: string; + evaluationStatus?: 'pass' | 'fail'; + evaluationReason?: string; + error?: string; + needsReauth?: boolean; + failureCode?: BrowserAutomationFailureCode; + failureStage?: BrowserAutomationFailureStage; + blockedReason?: string; + logs: BrowserEvidenceLog[]; +} + +export async function executeBrowserEvidence({ + input, + sessions, + logger, +}: { + input: BrowserEvidenceSessionInput; + sessions: BrowserbaseSessionService; + logger: Logger; +}): Promise { + const logs: BrowserEvidenceLog[] = []; + const log = (stage: string, message: string) => { + logs.push({ timestamp: new Date().toISOString(), stage, message }); + }; + let stagehand: Stagehand | null = null; + + try { + log('session', 'Initializing Stagehand session.'); + stagehand = await sessions.createStagehand(input.sessionId); + let page = await sessions.ensureActivePage(stagehand); + + log('navigation', `Opening ${input.targetUrl}.`); + await page.goto(input.targetUrl, { + waitUntil: 'domcontentloaded', + timeoutMs: 30000, + }); + await delay(1000); + + const authCheck = await checkAuth(stagehand); + if (!authCheck.isLoggedIn) { + const classified = classifyBrowserAutomationError( + new Error('Session expired. User is not logged in.'), + 'auth', + ); + log('auth', classified.userFacing); + return toExecutionFailure({ classified, logs }); + } + + log('action', 'Running navigation instruction.'); + const instruction = `${input.instruction}. After completing all navigation steps, stop and wait.`; + await stagehand + .agent({ + cua: true, + model: { + modelName: STAGEHAND_CUA_MODEL, + apiKey: process.env.ANTHROPIC_API_KEY, + }, + }) + .execute({ instruction, maxSteps: 20 }); + + await delay(2000); + page = await sessions.ensureActivePage(stagehand); + const finalUrl = page.url(); + + log('screenshot', 'Capturing screenshot.'); + const rawScreenshot = await page.screenshot({ + type: 'jpeg', + quality: 80, + fullPage: false, + }); + + const screenshot = await renderScreenshot({ + logger, + logs, + rawScreenshot, + instruction: input.instruction, + finalUrl, + }); + const evaluation = await evaluateIfNeeded({ + stagehand, + criteria: input.evaluationCriteria, + logs, + }); + + if (!evaluation.success) { + return { + success: false, + screenshot, + finalUrl, + evaluationStatus: evaluation.evaluationStatus, + evaluationReason: evaluation.evaluationReason, + error: evaluation.error, + failureCode: evaluation.failureCode, + failureStage: evaluation.failureStage, + logs, + }; + } + + return { + success: true, + screenshot, + finalUrl, + evaluationStatus: evaluation.evaluationStatus, + evaluationReason: evaluation.evaluationReason, + logs, + }; + } catch (err) { + const classified = classifyBrowserAutomationError(err, 'action'); + log(classified.stage, classified.userFacing); + logger.error('Failed to execute browser evidence run', err); + return toExecutionFailure({ classified, logs }); + } finally { + if (stagehand) { + await sessions.safeCloseStagehand(stagehand); + } + } +} + +async function renderScreenshot({ + logger, + logs, + rawScreenshot, + instruction, + finalUrl, +}: { + logger: Logger; + logs: BrowserEvidenceLog[]; + rawScreenshot: Buffer; + instruction: string; + finalUrl: string; +}): Promise { + let finalBuffer: Buffer = rawScreenshot; + try { + finalBuffer = await renderOverlay({ + buffer: rawScreenshot, + instruction, + sourceUrl: finalUrl, + capturedAt: new Date(), + }); + } catch (overlayErr) { + logger.warn('Screenshot overlay render failed; uploading raw image', { + error: overlayErr instanceof Error ? overlayErr.message : String(overlayErr), + }); + logs.push({ + timestamp: new Date().toISOString(), + stage: 'screenshot', + message: 'Overlay failed; captured raw screenshot.', + }); + } + return finalBuffer.toString('base64'); +} + +async function checkAuth(stagehand: Stagehand): Promise<{ isLoggedIn: boolean }> { + const loginSchema = z.object({ isLoggedIn: z.boolean() }); + return stagehand.extract( + 'Check if the user is logged in to this website. Look for a user avatar, profile menu, account dropdown, or login/sign-in buttons. Return true if logged in, false if you see login buttons or a login form.', + loginSchema, + ); +} + +async function evaluateIfNeeded({ + stagehand, + criteria, + logs, +}: { + stagehand: Stagehand; + criteria?: string | null; + logs: BrowserEvidenceLog[]; +}): Promise<{ + success: boolean; + evaluationStatus?: 'pass' | 'fail'; + evaluationReason?: string; + error?: string; + failureCode?: BrowserAutomationFailureCode; + failureStage?: BrowserAutomationFailureStage; +}> { + const normalizedCriteria = criteria?.trim(); + if (!normalizedCriteria) return { success: true }; + + logs.push({ + timestamp: new Date().toISOString(), + stage: 'evaluation', + message: 'Evaluating final page against criteria.', + }); + + try { + const evalSchema = z.object({ + pass: z.boolean(), + reason: z.string(), + }); + const evaluation = await stagehand.extract( + `You are an auditor reviewing the current page after an automation has finished navigating. Decide whether the page clearly satisfies this criteria. Only return pass=true if the evidence is unambiguously present and visible. If it is ambiguous, missing, or contradicted, return pass=false. Always provide a short reason (max 220 characters).\n\nCriteria: ${normalizedCriteria}`, + evalSchema, + ); + + return { + success: true, + evaluationStatus: evaluation.pass ? 'pass' : 'fail', + evaluationReason: evaluation.reason, + }; + } catch (err) { + const classified = evaluationFailedError( + 'The automation captured evidence, but evaluation failed. Review the screenshot manually.', + ); + logs.push({ + timestamp: new Date().toISOString(), + stage: classified.stage, + message: err instanceof Error ? err.message : String(err), + }); + return { + success: false, + error: classified.userFacing, + failureCode: classified.code, + failureStage: classified.stage, + }; + } +} + +function toExecutionFailure({ + classified, + logs, +}: { + classified: ClassifiedBrowserAutomationError; + logs: BrowserEvidenceLog[]; +}): BrowserEvidenceExecutionResult { + return { + success: false, + error: classified.userFacing, + needsReauth: classified.needsReauth, + failureCode: classified.code, + failureStage: classified.stage, + blockedReason: classified.blockedReason, + logs, + }; +} diff --git a/apps/api/src/browserbase/browser-evidence-runner.service.ts b/apps/api/src/browserbase/browser-evidence-runner.service.ts new file mode 100644 index 0000000000..54e188e2d7 --- /dev/null +++ b/apps/api/src/browserbase/browser-evidence-runner.service.ts @@ -0,0 +1,180 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Prisma } from '@db'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; +import { + type BrowserAutomationFailureCode, + type BrowserAutomationFailureStage, +} from './browser-automation-errors'; +import { + executeBrowserEvidence, + type BrowserEvidenceLog, + type BrowserEvidenceExecutionResult, +} from './browser-evidence-execution'; +import { browserRunCoordinator } from './browser-run-coordinator'; +import { + type BrowserCredentialVaultAdapter, + NoopBrowserCredentialVaultAdapter, +} from './credential-vault'; + +export interface BrowserEvidenceRunnerInput { + organizationId: string; + taskId?: string; + automationId: string; + runId: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string | null; + profile: { + id: string; + hostname: string; + contextId: string; + }; +} + +export interface BrowserEvidenceSessionInput extends BrowserEvidenceRunnerInput { + sessionId: string; +} + +export interface BrowserEvidenceRunResult { + success: boolean; + status: 'completed' | 'failed' | 'blocked'; + screenshotKey?: string; + screenshotUrl?: string; + finalUrl?: string; + evaluationStatus?: 'pass' | 'fail'; + evaluationReason?: string; + error?: string; + needsReauth?: boolean; + failureCode?: BrowserAutomationFailureCode; + failureStage?: BrowserAutomationFailureStage; + blockedReason?: string; + logs: Prisma.InputJsonValue; +} + +const toJsonLogs = (logs: BrowserEvidenceLog[]): Prisma.InputJsonArray => + logs.map( + (log): Prisma.InputJsonObject => ({ + timestamp: log.timestamp, + stage: log.stage, + message: log.message, + }), + ); + +@Injectable() +export class BrowserEvidenceRunnerService { + private readonly logger = new Logger(BrowserEvidenceRunnerService.name); + private readonly credentialVault: BrowserCredentialVaultAdapter = + new NoopBrowserCredentialVaultAdapter(); + + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly screenshots: BrowserbaseScreenshotService = new BrowserbaseScreenshotService(), + ) {} + + async runEvidence( + input: BrowserEvidenceRunnerInput, + ): Promise { + return browserRunCoordinator.withProfileLock({ + profileId: input.profile.id, + hostname: input.profile.hostname, + run: async () => { + const { sessionId } = await this.sessions.createSessionWithContext( + input.profile.contextId, + ); + + try { + return await this.executeEvidenceOnSession({ ...input, sessionId }); + } finally { + await this.closeSession(sessionId); + } + }, + }); + } + + async executeEvidenceOnSession( + input: BrowserEvidenceSessionInput, + ): Promise { + await this.credentialVault.resolveCredentialReference({ + profileId: input.profile.id, + }); + + const execution = await executeBrowserEvidence({ + input, + sessions: this.sessions, + logger: this.logger, + }); + const uploaded = await this.uploadCapturedScreenshot({ input, execution }); + + if (!execution.success) { + return { + success: false, + status: this.blockedStatusForCode(execution.failureCode), + screenshotKey: uploaded?.screenshotKey, + screenshotUrl: uploaded?.screenshotUrl, + finalUrl: execution.finalUrl, + evaluationStatus: execution.evaluationStatus, + evaluationReason: execution.evaluationReason, + error: execution.error, + needsReauth: execution.needsReauth, + failureCode: execution.failureCode, + failureStage: execution.failureStage, + blockedReason: execution.blockedReason, + logs: toJsonLogs(execution.logs), + }; + } + + return { + success: true, + status: 'completed', + screenshotKey: uploaded?.screenshotKey, + screenshotUrl: uploaded?.screenshotUrl, + finalUrl: execution.finalUrl, + evaluationStatus: execution.evaluationStatus, + evaluationReason: execution.evaluationReason, + logs: toJsonLogs(execution.logs), + }; + } + + private async uploadCapturedScreenshot({ + input, + execution, + }: { + input: BrowserEvidenceRunnerInput; + execution: BrowserEvidenceExecutionResult; + }): Promise<{ screenshotKey: string; screenshotUrl: string } | null> { + if (!execution.screenshot) return null; + + const screenshotKey = await this.screenshots.uploadScreenshot({ + organizationId: input.organizationId, + automationId: input.automationId, + runId: input.runId, + base64Screenshot: execution.screenshot, + }); + const screenshotUrl = await this.screenshots.getPresignedUrl({ + key: screenshotKey, + }); + + return { screenshotKey, screenshotUrl }; + } + + private async closeSession(sessionId: string): Promise { + try { + await this.sessions.closeSession(sessionId); + } catch (err) { + this.logger.warn('Failed to close Browserbase session (ignored)', { + sessionId, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + private blockedStatusForCode( + code: BrowserAutomationFailureCode | undefined, + ): 'failed' | 'blocked' { + if (code === 'captcha_blocked' || code === 'needs_user_action') { + return 'blocked'; + } + return 'failed'; + } +} diff --git a/apps/api/src/browserbase/browser-run-coordinator.spec.ts b/apps/api/src/browserbase/browser-run-coordinator.spec.ts new file mode 100644 index 0000000000..271da0862f --- /dev/null +++ b/apps/api/src/browserbase/browser-run-coordinator.spec.ts @@ -0,0 +1,37 @@ +import { BrowserRunCoordinator } from './browser-run-coordinator'; + +describe('BrowserRunCoordinator', () => { + it('serializes runs for the same profile', async () => { + process.env.BROWSER_AUTOMATION_DOMAIN_THROTTLE_MS = '1'; + const coordinator = new BrowserRunCoordinator(); + const events: string[] = []; + let releaseFirst: () => void = () => {}; + + const first = coordinator.withProfileLock({ + profileId: 'bap_1', + hostname: 'example.com', + run: async () => { + events.push('first:start'); + await new Promise((resolve) => { + releaseFirst = resolve; + }); + events.push('first:end'); + }, + }); + + const second = coordinator.withProfileLock({ + profileId: 'bap_1', + hostname: 'example.com', + run: async () => { + events.push('second:start'); + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(events).toEqual(['first:start']); + + releaseFirst(); + await Promise.all([first, second]); + expect(events).toEqual(['first:start', 'first:end', 'second:start']); + }); +}); diff --git a/apps/api/src/browserbase/browser-run-coordinator.ts b/apps/api/src/browserbase/browser-run-coordinator.ts new file mode 100644 index 0000000000..dc14b32c45 --- /dev/null +++ b/apps/api/src/browserbase/browser-run-coordinator.ts @@ -0,0 +1,62 @@ +const DEFAULT_DOMAIN_THROTTLE_MS = 5_000; + +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + if (!value) return fallback; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export class BrowserRunCoordinator { + private readonly profileLocks = new Map>(); + private readonly lastDomainRunAt = new Map(); + + async withProfileLock({ + profileId, + hostname, + run, + }: { + profileId: string; + hostname: string; + run: () => Promise; + }): Promise { + const previous = this.profileLocks.get(profileId) ?? Promise.resolve(); + let release: () => void = () => undefined; + const current = new Promise((resolve) => { + release = resolve; + }); + const chained = previous.then(() => current); + this.profileLocks.set(profileId, chained); + + await previous; + + try { + await this.waitForDomainTurn(hostname); + return await run(); + } finally { + release(); + if (this.profileLocks.get(profileId) === chained) { + this.profileLocks.delete(profileId); + } + } + } + + private async waitForDomainTurn(hostname: string): Promise { + const throttleMs = parsePositiveInt( + process.env.BROWSER_AUTOMATION_DOMAIN_THROTTLE_MS, + DEFAULT_DOMAIN_THROTTLE_MS, + ); + const now = Date.now(); + const lastRunAt = this.lastDomainRunAt.get(hostname) ?? 0; + const waitMs = Math.max(0, lastRunAt + throttleMs - now); + + if (waitMs > 0) { + await delay(waitMs); + } + + this.lastDomainRunAt.set(hostname, Date.now()); + } +} + +export const browserRunCoordinator = new BrowserRunCoordinator(); diff --git a/apps/api/src/browserbase/browserbase-screenshot.service.ts b/apps/api/src/browserbase/browserbase-screenshot.service.ts new file mode 100644 index 0000000000..9f430ad4c7 --- /dev/null +++ b/apps/api/src/browserbase/browserbase-screenshot.service.ts @@ -0,0 +1,100 @@ +import { Injectable, NotFoundException } from '@nestjs/common'; +import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { BUCKET_NAME, getSignedUrl, s3Client } from '@/app/s3'; +import { db } from '@db'; + +@Injectable() +export class BrowserbaseScreenshotService { + private get s3Client(): S3Client { + if (!s3Client) { + throw new Error( + 'S3 client not configured — set APP_AWS_ACCESS_KEY_ID, APP_AWS_SECRET_ACCESS_KEY, APP_AWS_REGION, APP_AWS_BUCKET_NAME in apps/api/.env', + ); + } + return s3Client; + } + + private get bucketName(): string { + if (!BUCKET_NAME) { + throw new Error( + 'APP_AWS_BUCKET_NAME is not set — configure S3 credentials in apps/api/.env', + ); + } + return BUCKET_NAME; + } + + async uploadScreenshot({ + organizationId, + automationId, + runId, + base64Screenshot, + }: { + organizationId: string; + automationId: string; + runId: string; + base64Screenshot: string; + }): Promise { + const buffer = Buffer.from(base64Screenshot, 'base64'); + const key = `browser-automations/${organizationId}/${automationId}/${runId}.jpg`; + + await this.s3Client.send( + new PutObjectCommand({ + Bucket: this.bucketName, + Key: key, + Body: buffer, + ContentType: 'image/jpeg', + }), + ); + + return key; + } + + async getPresignedUrl({ + key, + expiresIn, + responseContentDisposition, + }: { + key: string; + expiresIn?: number; + responseContentDisposition?: string; + }): Promise { + const command = new GetObjectCommand({ + Bucket: this.bucketName, + Key: key, + ResponseContentDisposition: responseContentDisposition, + }); + return getSignedUrl(this.s3Client, command, { + expiresIn: expiresIn ?? 3600, + }); + } + + async getScreenshotRedirectUrl(input: { + runId: string; + organizationId: string; + download?: boolean; + }): Promise { + const { runId, organizationId, download } = input; + + const run = await db.browserAutomationRun.findUnique({ + where: { id: runId }, + include: { automation: { include: { task: true } } }, + }); + + if (!run || !run.screenshotUrl) { + throw new NotFoundException('Screenshot not found'); + } + + if (run.automation.task.organizationId !== organizationId) { + throw new NotFoundException('Screenshot not found'); + } + + const responseContentDisposition = download + ? `attachment; filename="screenshot-${runId}.jpg"` + : undefined; + + return this.getPresignedUrl({ + key: run.screenshotUrl, + responseContentDisposition, + }); + } +} diff --git a/apps/api/src/browserbase/browserbase-session.service.ts b/apps/api/src/browserbase/browserbase-session.service.ts new file mode 100644 index 0000000000..f1baf92b60 --- /dev/null +++ b/apps/api/src/browserbase/browserbase-session.service.ts @@ -0,0 +1,202 @@ +import { Injectable, Logger } from '@nestjs/common'; +import Browserbase from '@browserbasehq/sdk'; +import { z } from 'zod'; +import { isNoPageError } from './run-error-formatter'; + +type Stagehand = import('@browserbasehq/stagehand').Stagehand; + +const BROWSER_WIDTH = 1440; +const BROWSER_HEIGHT = 900; +const STAGEHAND_MODEL = 'anthropic/claude-sonnet-4-6'; + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +@Injectable() +export class BrowserbaseSessionService { + private readonly logger = new Logger(BrowserbaseSessionService.name); + + getBrowserbase() { + return new Browserbase({ + apiKey: process.env.BROWSERBASE_API_KEY, + }); + } + + getProjectId() { + return process.env.BROWSERBASE_PROJECT_ID || ''; + } + + async createBrowserbaseContext(): Promise { + const context = await this.getBrowserbase().contexts.create({ + projectId: this.getProjectId(), + }); + return context.id; + } + + async createSessionWithContext( + contextId: string, + ): Promise<{ sessionId: string; liveViewUrl: string }> { + const bb = this.getBrowserbase(); + + const session = await bb.sessions.create({ + projectId: this.getProjectId(), + browserSettings: { + context: { + id: contextId, + persist: true, + }, + fingerprint: { + screen: { + maxHeight: BROWSER_HEIGHT, + maxWidth: BROWSER_WIDTH, + minHeight: BROWSER_HEIGHT, + minWidth: BROWSER_WIDTH, + }, + }, + viewport: { width: BROWSER_WIDTH, height: BROWSER_HEIGHT }, + }, + keepAlive: true, + }); + + const debug = await bb.sessions.debug(session.id); + + return { + sessionId: session.id, + liveViewUrl: debug.debuggerFullscreenUrl, + }; + } + + async closeSession(sessionId: string): Promise { + await this.getBrowserbase().sessions.update(sessionId, { + projectId: this.getProjectId(), + status: 'REQUEST_RELEASE', + }); + } + + async createStagehand(sessionId: string): Promise { + const { Stagehand } = await import('@browserbasehq/stagehand'); + const stagehand = new Stagehand({ + env: 'BROWSERBASE', + apiKey: process.env.BROWSERBASE_API_KEY, + projectId: this.getProjectId(), + browserbaseSessionID: sessionId, + model: { + modelName: STAGEHAND_MODEL, + apiKey: process.env.ANTHROPIC_API_KEY, + }, + verbose: 1, + }); + + await stagehand.init(); + return stagehand; + } + + async safeCloseStagehand(stagehand: Stagehand): Promise { + try { + await stagehand.close(); + } catch (err) { + this.logger.warn('Failed to close stagehand (ignored)', { + error: err instanceof Error ? err.message : String(err), + }); + } + } + + async ensureActivePage(stagehand: Stagehand) { + const maxWaitMs = 5_000; + const pollMs = 250; + const startedAt = Date.now(); + + while (Date.now() - startedAt < maxWaitMs) { + const page = stagehand.context.pages().find((candidate) => { + const maybePage = candidate as { isClosed?: () => boolean }; + return maybePage.isClosed ? !maybePage.isClosed() : true; + }); + if (page) return page; + await delay(pollMs); + } + + return await stagehand.context.newPage(); + } + + async navigateToUrl( + sessionId: string, + url: string, + ): Promise<{ success: boolean; error?: string }> { + let stagehand: Stagehand | null = null; + + try { + stagehand = await this.createStagehand(sessionId); + const page = await this.ensureActivePage(stagehand); + + await page.sendCDP('WebAuthn.enable'); + await page.sendCDP('WebAuthn.addVirtualAuthenticator', { + options: { + protocol: 'ctap2', + transport: 'internal', + hasResidentKey: true, + hasUserVerification: true, + isUserVerified: true, + automaticPresenceSimulation: true, + }, + }); + + await page.goto(url, { + waitUntil: 'domcontentloaded', + timeoutMs: 30000, + }); + + return { success: true }; + } catch (err) { + this.logger.error('Failed to navigate to URL', err); + if (stagehand) { + await this.safeCloseStagehand(stagehand); + } + return { + success: false, + error: err instanceof Error ? err.message : 'Unknown error', + }; + } + } + + async checkLoginStatus( + sessionId: string, + url: string, + ): Promise<{ isLoggedIn: boolean; username?: string }> { + const stagehand = await this.createStagehand(sessionId); + + try { + const page = await this.ensureActivePage(stagehand); + + await page.goto(url, { + waitUntil: 'domcontentloaded', + timeoutMs: 30000, + }); + await delay(1500); + + const loginSchema = z.object({ + isLoggedIn: z + .boolean() + .describe('Whether the user is currently logged in to this site'), + username: z.string().optional().describe('The username if logged in'), + }); + + const result = await stagehand.extract( + 'Check if the user is logged in to this website. Look for a user avatar, profile menu, or account dropdown in the header/navigation. If logged in, extract the username if visible.', + loginSchema, + ); + + return { + isLoggedIn: result.isLoggedIn, + username: result.username, + }; + } catch (err) { + if (isNoPageError(err)) { + throw new Error( + 'Browser session ended before we could verify login status. Please retry.', + ); + } + throw err; + } finally { + await this.safeCloseStagehand(stagehand); + } + } +} diff --git a/apps/api/src/browserbase/browserbase-url.ts b/apps/api/src/browserbase/browserbase-url.ts new file mode 100644 index 0000000000..9bf2def94f --- /dev/null +++ b/apps/api/src/browserbase/browserbase-url.ts @@ -0,0 +1,12 @@ +export function normalizeHostnameFromUrl(url: string): string { + const parsed = new URL(url); + return parsed.hostname.toLowerCase().replace(/\.$/, ''); +} + +export function defaultProfileDisplayName(hostname: string): string { + return `${hostname} browser profile`; +} + +export function normalizeLoginIdentity(loginIdentity?: string | null): string { + return loginIdentity?.trim().toLowerCase() ?? ''; +} diff --git a/apps/api/src/browserbase/browserbase.controller.ts b/apps/api/src/browserbase/browserbase.controller.ts index a0c7047fee..a02d1fb69c 100644 --- a/apps/api/src/browserbase/browserbase.controller.ts +++ b/apps/api/src/browserbase/browserbase.controller.ts @@ -13,6 +13,7 @@ import { import type { Response } from 'express'; import { ApiOperation, + ApiBody, ApiParam, ApiResponse, ApiSecurity, @@ -32,6 +33,7 @@ import { ContextResponseDto, CreateBrowserAutomationDto, CreateSessionDto, + ExecuteAutomationSessionDto, NavigateToUrlDto, RunAutomationResponseDto, SessionResponseDto, @@ -171,10 +173,12 @@ export class BrowserbaseController { type: BrowserAutomationResponseDto, }) async createAutomation( + @OrganizationId() organizationId: string, @Body() dto: CreateBrowserAutomationDto, ): Promise { return (await this.browserbaseService.createBrowserAutomation( dto, + organizationId, )) as BrowserAutomationResponseDto; } @@ -191,9 +195,11 @@ export class BrowserbaseController { }) async getAutomationsForTask( @Param('taskId') taskId: string, + @OrganizationId() organizationId: string, ): Promise { return (await this.browserbaseService.getAutomationsWithPresignedUrls( taskId, + organizationId, )) as BrowserAutomationResponseDto[]; } @@ -210,9 +216,11 @@ export class BrowserbaseController { }) async getAutomation( @Param('automationId') automationId: string, + @OrganizationId() organizationId: string, ): Promise { return (await this.browserbaseService.getBrowserAutomation( automationId, + organizationId, )) as BrowserAutomationResponseDto | null; } @@ -229,11 +237,13 @@ export class BrowserbaseController { }) async updateAutomation( @Param('automationId') automationId: string, + @OrganizationId() organizationId: string, @Body() dto: UpdateBrowserAutomationDto, ): Promise { return (await this.browserbaseService.updateBrowserAutomation( automationId, dto, + organizationId, )) as BrowserAutomationResponseDto; } @@ -249,8 +259,12 @@ export class BrowserbaseController { }) async deleteAutomation( @Param('automationId') automationId: string, + @OrganizationId() organizationId: string, ): Promise<{ success: boolean }> { - await this.browserbaseService.deleteBrowserAutomation(automationId); + await this.browserbaseService.deleteBrowserAutomation( + automationId, + organizationId, + ); return { success: true }; } @@ -291,13 +305,14 @@ export class BrowserbaseController { description: 'Runs the automation on a pre-created session', }) @ApiParam({ name: 'automationId', description: 'Automation ID' }) + @ApiBody({ type: ExecuteAutomationSessionDto }) @ApiResponse({ status: 200, description: 'Execution result', }) async executeAutomationOnSession( @Param('automationId') automationId: string, - @Body() body: { runId: string; sessionId: string }, + @Body() body: ExecuteAutomationSessionDto, @OrganizationId() organizationId: string, ): Promise<{ success: boolean; @@ -350,9 +365,12 @@ export class BrowserbaseController { }) async getAutomationRuns( @Param('automationId') automationId: string, + @OrganizationId() organizationId: string, ): Promise { return (await this.browserbaseService.getAutomationRuns( automationId, + 20, + organizationId, )) as BrowserAutomationRunResponseDto[]; } @@ -369,9 +387,11 @@ export class BrowserbaseController { }) async getRunById( @Param('runId') runId: string, + @OrganizationId() organizationId: string, ): Promise { return (await this.browserbaseService.getRunWithPresignedUrl( runId, + organizationId, )) as BrowserAutomationRunResponseDto | null; } diff --git a/apps/api/src/browserbase/browserbase.module.ts b/apps/api/src/browserbase/browserbase.module.ts index 9dfec69375..304a90007e 100644 --- a/apps/api/src/browserbase/browserbase.module.ts +++ b/apps/api/src/browserbase/browserbase.module.ts @@ -1,12 +1,27 @@ import { Module } from '@nestjs/common'; +import { BrowserAutomationCrudService } from './browser-automation-crud.service'; +import { BrowserAutomationExecutionService } from './browser-automation-execution.service'; +import { BrowserAuthProfilesController } from './browser-auth-profiles.controller'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; import { BrowserbaseController } from './browserbase.controller'; +import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; +import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserbaseService } from './browserbase.service'; import { AuthModule } from '../auth/auth.module'; @Module({ imports: [AuthModule], - controllers: [BrowserbaseController], - providers: [BrowserbaseService], + controllers: [BrowserbaseController, BrowserAuthProfilesController], + providers: [ + BrowserbaseService, + BrowserbaseSessionService, + BrowserAutomationCrudService, + BrowserAutomationExecutionService, + BrowserAuthProfileService, + BrowserbaseScreenshotService, + BrowserEvidenceRunnerService, + ], exports: [BrowserbaseService], }) export class BrowserbaseModule {} diff --git a/apps/api/src/browserbase/browserbase.service.spec.ts b/apps/api/src/browserbase/browserbase.service.spec.ts index 7f8cdba32f..daf41651b7 100644 --- a/apps/api/src/browserbase/browserbase.service.spec.ts +++ b/apps/api/src/browserbase/browserbase.service.spec.ts @@ -1,6 +1,12 @@ // apps/api/src/browserbase/browserbase.service.spec.ts import { Test } from '@nestjs/testing'; import { NotFoundException } from '@nestjs/common'; +import { BrowserAutomationCrudService } from './browser-automation-crud.service'; +import { BrowserAutomationExecutionService } from './browser-automation-execution.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; +import { BrowserbaseSessionService } from './browserbase-session.service'; import { BrowserbaseService } from './browserbase.service'; jest.mock('@db', () => ({ @@ -37,7 +43,15 @@ describe('BrowserbaseService.getScreenshotRedirectUrl', () => { beforeEach(async () => { jest.clearAllMocks(); const moduleRef = await Test.createTestingModule({ - providers: [BrowserbaseService], + providers: [ + BrowserbaseService, + BrowserbaseSessionService, + BrowserAutomationCrudService, + BrowserAutomationExecutionService, + BrowserAuthProfileService, + BrowserbaseScreenshotService, + BrowserEvidenceRunnerService, + ], }).compile(); service = moduleRef.get(BrowserbaseService); }); @@ -144,7 +158,15 @@ describe('BrowserbaseService schedule frequency passthrough', () => { beforeEach(async () => { jest.clearAllMocks(); const moduleRef = await Test.createTestingModule({ - providers: [BrowserbaseService], + providers: [ + BrowserbaseService, + BrowserbaseSessionService, + BrowserAutomationCrudService, + BrowserAutomationExecutionService, + BrowserAuthProfileService, + BrowserbaseScreenshotService, + BrowserEvidenceRunnerService, + ], }).compile(); service = moduleRef.get(BrowserbaseService); }); diff --git a/apps/api/src/browserbase/browserbase.service.ts b/apps/api/src/browserbase/browserbase.service.ts index da81adadea..696133553f 100644 --- a/apps/api/src/browserbase/browserbase.service.ts +++ b/apps/api/src/browserbase/browserbase.service.ts @@ -1,403 +1,117 @@ -import { Injectable, Logger, NotFoundException } from '@nestjs/common'; -import Browserbase from '@browserbasehq/sdk'; -// Lazy-imported in createStagehand() to avoid Node v25 crash -// (SlowBuffer.prototype was removed — @browserbasehq/stagehand bundles buffer-equal-constant-time which uses it) -type Stagehand = import('@browserbasehq/stagehand').Stagehand; -import { db, TaskFrequency } from '@db'; -import { z } from 'zod'; -import { - GetObjectCommand, - PutObjectCommand, - S3Client, -} from '@aws-sdk/client-s3'; -import { BUCKET_NAME, getSignedUrl, s3Client } from '@/app/s3'; -import { renderOverlay } from './screenshot-overlay'; -import { isNoPageError, toRunErrorMessage } from './run-error-formatter'; - -const BROWSER_WIDTH = 1440; -const BROWSER_HEIGHT = 900; - -/** Stagehand v3 requires 'provider/model' format. */ -const STAGEHAND_MODEL = 'anthropic/claude-sonnet-4-6'; -const STAGEHAND_CUA_MODEL = 'anthropic/claude-sonnet-4-6'; - -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -const PENDING_CONTEXT_ID = '__PENDING__'; - -/** Empty strings become null; any actual text is trimmed and kept. */ -const normalizeCriteria = (value: string | null | undefined): string | null => { - if (value == null) return null; - const trimmed = value.trim(); - return trimmed.length === 0 ? null : trimmed; -}; - -const isPrismaUniqueConstraintError = (error: unknown): boolean => { - if (typeof error !== 'object' || error === null) return false; - if (!('code' in error)) return false; - const code = (error as { code?: unknown }).code; - return code === 'P2002'; -}; +import { Injectable } from '@nestjs/common'; +import { TaskFrequency } from '@db'; +import { BrowserAutomationCrudService } from './browser-automation-crud.service'; +import { BrowserAutomationExecutionService } from './browser-automation-execution.service'; +import { BrowserAuthProfileService } from './browser-auth-profile.service'; +import { BrowserEvidenceRunnerService } from './browser-evidence-runner.service'; +import { BrowserbaseScreenshotService } from './browserbase-screenshot.service'; +import { BrowserbaseSessionService } from './browserbase-session.service'; +import { normalizeHostnameFromUrl } from './browserbase-url'; @Injectable() export class BrowserbaseService { - private readonly logger = new Logger(BrowserbaseService.name); - - private get s3Client(): S3Client { - if (!s3Client) { - throw new Error( - 'S3 client not configured — set APP_AWS_ACCESS_KEY_ID, APP_AWS_SECRET_ACCESS_KEY, APP_AWS_REGION, APP_AWS_BUCKET_NAME in apps/api/.env', - ); - } - return s3Client; + constructor( + private readonly sessions: BrowserbaseSessionService = new BrowserbaseSessionService(), + private readonly profiles: BrowserAuthProfileService = new BrowserAuthProfileService( + sessions, + ), + private readonly screenshots: BrowserbaseScreenshotService = new BrowserbaseScreenshotService(), + private readonly runner: BrowserEvidenceRunnerService = new BrowserEvidenceRunnerService( + sessions, + screenshots, + ), + private readonly automationCrud: BrowserAutomationCrudService = new BrowserAutomationCrudService( + screenshots, + ), + private readonly automationExecution: BrowserAutomationExecutionService = + new BrowserAutomationExecutionService(sessions, profiles, runner), + ) {} + + async listAuthProfiles(organizationId: string) { + return this.profiles.listProfiles(organizationId); + } + + async getOrCreateAuthProfile(input: { + organizationId: string; + url: string; + displayName?: string; + loginIdentity?: string; + vaultProvider?: string; + vaultExternalItemRef?: string; + vaultConnectionId?: string; + }) { + return this.profiles.getOrCreateProfileFromUrl(input); } - private get bucketName(): string { - if (!BUCKET_NAME) { - throw new Error( - 'APP_AWS_BUCKET_NAME is not set — configure S3 credentials in apps/api/.env', - ); - } - return BUCKET_NAME; + async startAuthProfileSession(input: { + organizationId: string; + profileId: string; + }) { + return this.profiles.startProfileSession(input); } - private getBrowserbase() { - return new Browserbase({ - apiKey: process.env.BROWSERBASE_API_KEY, - }); + async verifyAuthProfileSession(input: { + organizationId: string; + profileId: string; + sessionId: string; + url: string; + }) { + return this.profiles.verifyProfileSession(input); } - private getProjectId() { - return process.env.BROWSERBASE_PROJECT_ID || ''; + async markAuthProfileNeedsReauth(input: { + organizationId: string; + profileId: string; + reason?: string; + }) { + return this.profiles.markNeedsReauth(input); } - /** - * Stagehand sometimes has no active page (or the page gets closed mid-run), - * which causes errors like: "No Page found for awaitActivePage: no page available". - * Ensure there's at least one non-closed page available, and create one if needed. - */ - private async ensureActivePage(stagehand: Stagehand) { - const MAX_WAIT_MS = 5000; - const POLL_MS = 250; - const startedAt = Date.now(); - - while (Date.now() - startedAt < MAX_WAIT_MS) { - // Stagehand's Page type doesn't always expose Playwright's `isClosed()` in typings. - // We still want to filter out closed pages at runtime when possible. - const pages = stagehand.context.pages().filter((p) => { - const maybeIsClosed = (p as { isClosed?: () => boolean }).isClosed; - return typeof maybeIsClosed === 'function' ? !maybeIsClosed() : true; - }); - if (pages[0]) return pages[0]; - await delay(POLL_MS); - } - - // Last resort: create a page (may still fail if the CDP session already died) - return await stagehand.context.newPage(); + async getOrCreateOrgContext(organizationId: string) { + return this.profiles.getOrCreateOrgContext(organizationId); } - // ===== Organization Context Management ===== - - async getOrCreateOrgContext( - organizationId: string, - ): Promise<{ contextId: string; isNew: boolean }> { - // Fast path: already created - const existing = await db.browserbaseContext.findUnique({ - where: { organizationId }, - }); - - if (existing && existing.contextId !== PENDING_CONTEXT_ID) { - return { contextId: existing.contextId, isNew: false }; - } - - try { - await db.browserbaseContext.create({ - data: { - organizationId, - contextId: PENDING_CONTEXT_ID, - }, - }); - - const bb = this.getBrowserbase(); - const context = await bb.contexts.create({ - projectId: this.getProjectId(), - }); - - await db.browserbaseContext.update({ - where: { organizationId }, - data: { contextId: context.id }, - }); - - return { contextId: context.id, isNew: true }; - } catch (error) { - if (!isPrismaUniqueConstraintError(error)) { - throw error; - } - } - - const MAX_WAIT_MS = 10_000; - const POLL_MS = 200; - const startedAt = Date.now(); - - while (Date.now() - startedAt < MAX_WAIT_MS) { - const current = await db.browserbaseContext.findUnique({ - where: { organizationId }, - }); - - if (current && current.contextId !== PENDING_CONTEXT_ID) { - return { contextId: current.contextId, isNew: false }; - } - - if (!current) { - return await this.getOrCreateOrgContext(organizationId); - } - - await delay(POLL_MS); - } - - this.logger.warn( - `Timed out waiting for Browserbase context creation for org ${organizationId}`, - ); - throw new Error( - 'Browser context initialization is taking too long. Please retry.', - ); + async getOrgContext(organizationId: string) { + return this.profiles.getOrgContext(organizationId); } - async getOrgContext( - organizationId: string, - ): Promise<{ contextId: string } | null> { - const context = await db.browserbaseContext.findUnique({ - where: { organizationId }, - }); - - if (!context) return null; - return { contextId: context.contextId }; - } - - // ===== Session Management ===== - - async createSessionWithContext( - contextId: string, - ): Promise<{ sessionId: string; liveViewUrl: string }> { - const bb = this.getBrowserbase(); - - const session = await bb.sessions.create({ - projectId: this.getProjectId(), - browserSettings: { - context: { - id: contextId, - persist: true, - }, - fingerprint: { - screen: { - maxHeight: BROWSER_HEIGHT, - maxWidth: BROWSER_WIDTH, - minHeight: BROWSER_HEIGHT, - minWidth: BROWSER_WIDTH, - }, - }, - viewport: { width: BROWSER_WIDTH, height: BROWSER_HEIGHT }, - }, - keepAlive: true, - }); - - const debug = await bb.sessions.debug(session.id); - - return { - sessionId: session.id, - liveViewUrl: debug.debuggerFullscreenUrl, - }; + async createSessionWithContext(contextId: string) { + return this.sessions.createSessionWithContext(contextId); } async closeSession(sessionId: string): Promise { - const bb = this.getBrowserbase(); - await bb.sessions.update(sessionId, { - projectId: this.getProjectId(), - status: 'REQUEST_RELEASE', - }); - } - - // ===== Stagehand helpers ===== - - private async createStagehand(sessionId: string): Promise { - const { Stagehand } = await import('@browserbasehq/stagehand'); - const stagehand = new Stagehand({ - env: 'BROWSERBASE', - apiKey: process.env.BROWSERBASE_API_KEY, - projectId: this.getProjectId(), - browserbaseSessionID: sessionId, - model: { - modelName: STAGEHAND_MODEL, - apiKey: process.env.ANTHROPIC_API_KEY, - }, - verbose: 1, - }); - - await stagehand.init(); - return stagehand; - } - - private async safeCloseStagehand(stagehand: Stagehand) { - try { - await stagehand.close(); - } catch (err) { - // IMPORTANT: never let cleanup errors override a successful run - this.logger.warn('Failed to close stagehand (ignored)', { - error: err instanceof Error ? err.message : String(err), - }); - } + return this.sessions.closeSession(sessionId); } - // ===== Browser Actions ===== - - async navigateToUrl( - sessionId: string, - url: string, - ): Promise<{ success: boolean; error?: string }> { - let stagehand: Stagehand | null = null; - - try { - stagehand = await this.createStagehand(sessionId); - const page = stagehand.context.pages()[0]; - if (!page) { - throw new Error('No page found in browser session'); - } - - // Set up virtual authenticator to bypass passkeys via CDP - await page.sendCDP('WebAuthn.enable'); - await page.sendCDP('WebAuthn.addVirtualAuthenticator', { - options: { - protocol: 'ctap2', - transport: 'internal', - hasResidentKey: true, - hasUserVerification: true, - isUserVerified: true, - automaticPresenceSimulation: true, - }, - }); - - await page.goto(url, { - waitUntil: 'domcontentloaded', - timeoutMs: 30000, - }); - - return { success: true }; - } catch (err) { - this.logger.error('Failed to navigate to URL', err); - if (stagehand) { - try { - await stagehand.close(); - } catch (closeErr) { - this.logger.warn('Failed to close stagehand after navigation error', { - closeErr: - closeErr instanceof Error - ? closeErr.message - : 'Unknown close error', - }); - } - } - return { - success: false, - error: err instanceof Error ? err.message : 'Unknown error', - }; - } - // Don't close - user needs to interact via Live View + async navigateToUrl(sessionId: string, url: string) { + return this.sessions.navigateToUrl(sessionId, url); } - async checkLoginStatus( - sessionId: string, - url: string, - ): Promise<{ isLoggedIn: boolean; username?: string }> { - const stagehand = await this.createStagehand(sessionId); - - try { - const page = await this.ensureActivePage(stagehand); - - await page.goto(url, { - waitUntil: 'domcontentloaded', - timeoutMs: 30000, - }); - await delay(1500); - - // Use extract to check login status - const loginSchema = z.object({ - isLoggedIn: z - .boolean() - .describe('Whether the user is currently logged in to this site'), - username: z.string().optional().describe('The username if logged in'), - }); - - const result = (await stagehand.extract( - 'Check if the user is logged in to this website. Look for a user avatar, profile menu, or account dropdown in the header/navigation. If logged in, extract the username if visible.', - loginSchema as any, - )) as { isLoggedIn: boolean; username?: string }; - - return { - isLoggedIn: result.isLoggedIn, - username: result.username, - }; - } catch (err) { - if (isNoPageError(err)) { - throw new Error( - 'Browser session ended before we could verify login status. Please retry.', - ); - } - throw err; - } finally { - await this.safeCloseStagehand(stagehand); - } + async checkLoginStatus(sessionId: string, url: string) { + return this.sessions.checkLoginStatus(sessionId, url); } - // ===== Browser Automation CRUD ===== - - async createBrowserAutomation(data: { - taskId: string; - name: string; - description?: string; - targetUrl: string; - instruction: string; - evaluationCriteria?: string; - scheduleFrequency?: TaskFrequency; - }) { - return db.browserAutomation.create({ - data: { - taskId: data.taskId, - name: data.name, - description: data.description, - targetUrl: data.targetUrl, - instruction: data.instruction, - evaluationCriteria: normalizeCriteria(data.evaluationCriteria), - isEnabled: true, // Enable by default so scheduled runs work - ...(data.scheduleFrequency !== undefined - ? { scheduleFrequency: data.scheduleFrequency } - : {}), - }, - }); + async createBrowserAutomation( + data: { + taskId: string; + name: string; + description?: string; + targetUrl: string; + instruction: string; + evaluationCriteria?: string; + scheduleFrequency?: TaskFrequency; + }, + organizationId?: string, + ) { + return this.automationCrud.createBrowserAutomation(data, organizationId); } - async getBrowserAutomation(automationId: string) { - return db.browserAutomation.findUnique({ - where: { id: automationId }, - include: { - runs: { - orderBy: { createdAt: 'desc' }, - take: 10, - }, - }, - }); + async getBrowserAutomation(automationId: string, organizationId?: string) { + return this.automationCrud.getBrowserAutomation(automationId, organizationId); } - async getBrowserAutomationsForTask(taskId: string) { - return db.browserAutomation.findMany({ - where: { taskId }, - include: { - runs: { - orderBy: { createdAt: 'desc' }, - take: 1, - }, - }, - orderBy: { createdAt: 'desc' }, - }); + async getBrowserAutomationsForTask(taskId: string, organizationId?: string) { + return this.automationCrud.getBrowserAutomationsForTask(taskId, organizationId); } async updateBrowserAutomation( @@ -411,634 +125,83 @@ export class BrowserbaseService { isEnabled?: boolean; scheduleFrequency?: TaskFrequency; }, + organizationId?: string, ) { - const { evaluationCriteria, scheduleFrequency, ...rest } = data; - return db.browserAutomation.update({ - where: { id: automationId }, - data: { - ...rest, - ...(evaluationCriteria !== undefined - ? { evaluationCriteria: normalizeCriteria(evaluationCriteria) } - : {}), - ...(scheduleFrequency !== undefined ? { scheduleFrequency } : {}), - }, - }); + return this.automationCrud.updateBrowserAutomation( + automationId, + data, + organizationId, + ); } - async deleteBrowserAutomation(automationId: string) { - return db.browserAutomation.delete({ - where: { id: automationId }, - }); + async deleteBrowserAutomation(automationId: string, organizationId?: string) { + return this.automationCrud.deleteBrowserAutomation(automationId, organizationId); } - // ===== Browser Automation Execution ===== - - /** - * Start an automation run with a live session that the user can watch - */ - async startAutomationWithLiveView( - automationId: string, - organizationId: string, - ): Promise<{ - runId: string; - sessionId: string; - liveViewUrl: string; - error?: string; - needsReauth?: boolean; - }> { - const automation = await db.browserAutomation.findUnique({ - where: { id: automationId }, - }); - - if (!automation) { - throw new Error('Automation not found'); - } - - const context = await this.getOrgContext(organizationId); - if (!context) { - return { - runId: '', - sessionId: '', - liveViewUrl: '', - needsReauth: true, - error: 'No browser context found. Please connect your browser first.', - }; - } - - // Create a run record - const run = await db.browserAutomationRun.create({ - data: { - automationId, - status: 'running', - startedAt: new Date(), - }, - }); - - // Create session with live view - const { sessionId, liveViewUrl } = await this.createSessionWithContext( - context.contextId, + async startAutomationWithLiveView(automationId: string, organizationId: string) { + return this.automationExecution.startAutomationWithLiveView( + automationId, + organizationId, ); - - return { - runId: run.id, - sessionId, - liveViewUrl, - }; } - /** - * Execute an automation on an existing session (for live view runs) - */ async executeAutomationOnSession( automationId: string, runId: string, sessionId: string, organizationId: string, - ): Promise<{ - success: boolean; - screenshotUrl?: string; - evaluationStatus?: 'pass' | 'fail'; - evaluationReason?: string; - error?: string; - needsReauth?: boolean; - }> { - const automation = await db.browserAutomation.findUnique({ - where: { id: automationId }, - include: { - task: { - select: { title: true, description: true }, - }, - }, - }); - - if (!automation) { - throw new Error('Automation not found'); - } - - const run = await db.browserAutomationRun.findUnique({ - where: { id: runId }, - }); - - if (!run) { - throw new Error('Run not found'); - } - - try { - const result = await this.executeAutomation( - sessionId, - automation.targetUrl, - automation.instruction, - { - title: automation.task.title, - description: automation.task.description, - evaluationCriteria: automation.evaluationCriteria, - }, - ); - - if (!result.success) { - // Store evaluation data even on failure (requirement not met) - await db.browserAutomationRun.update({ - where: { id: runId }, - data: { - status: 'failed', - completedAt: new Date(), - durationMs: run.startedAt - ? Date.now() - run.startedAt.getTime() - : 0, - error: result.error, - evaluationStatus: result.evaluationStatus, - evaluationReason: result.evaluationReason, - }, - }); - - return { - success: false, - error: result.error, - evaluationStatus: result.evaluationStatus, - evaluationReason: result.evaluationReason, - needsReauth: result.needsReauth, - }; - } - - // Upload screenshot to S3 - let screenshotKey: string | undefined; - let presignedUrl: string | undefined; - if (result.screenshot) { - screenshotKey = await this.uploadScreenshot( - organizationId, - automationId, - runId, - result.screenshot, - ); - presignedUrl = await this.getPresignedUrl(screenshotKey); - } - - // Update run as completed. Only persist an evaluation verdict when - // the caller's automation had criteria configured. - await db.browserAutomationRun.update({ - where: { id: runId }, - data: { - status: 'completed', - completedAt: new Date(), - durationMs: run.startedAt ? Date.now() - run.startedAt.getTime() : 0, - screenshotUrl: screenshotKey, - evaluationStatus: result.evaluationStatus ?? null, - evaluationReason: result.evaluationReason ?? null, - }, - }); - - return { - success: true, - screenshotUrl: presignedUrl, - evaluationStatus: result.evaluationStatus, - evaluationReason: result.evaluationReason, - }; - } catch (err) { - this.logger.error('Failed to execute automation on session', err); - const { userFacing, needsReauth } = toRunErrorMessage(err); - - await db.browserAutomationRun.update({ - where: { id: runId }, - data: { - status: 'failed', - completedAt: new Date(), - durationMs: run.startedAt ? Date.now() - run.startedAt.getTime() : 0, - error: userFacing, - }, - }); - - return { - success: false, - error: userFacing, - needsReauth: needsReauth ? true : undefined, - }; - } - } - - async runBrowserAutomation( - automationId: string, - organizationId: string, - ): Promise<{ - runId: string; - success: boolean; - screenshotUrl?: string; - evaluationStatus?: 'pass' | 'fail'; - evaluationReason?: string; - error?: string; - needsReauth?: boolean; - }> { - // Get the automation with task context - const automation = await db.browserAutomation.findUnique({ - where: { id: automationId }, - include: { - task: { - select: { title: true, description: true }, - }, - }, - }); - - if (!automation) { - throw new Error('Automation not found'); - } - - // Get org context - const context = await this.getOrgContext(organizationId); - if (!context) { - return { - runId: '', - success: false, - needsReauth: true, - error: 'No browser context found. Please connect your browser first.', - }; - } - - // Create a run record - const run = await db.browserAutomationRun.create({ - data: { - automationId, - status: 'running', - startedAt: new Date(), - }, - }); - - try { - // Create a session - const { sessionId } = await this.createSessionWithContext( - context.contextId, - ); - - try { - const result = await this.executeAutomation( - sessionId, - automation.targetUrl, - automation.instruction, - { - title: automation.task.title, - description: automation.task.description, - evaluationCriteria: automation.evaluationCriteria, - }, - ); - - if (!result.success) { - // Update run as failed - include evaluation data if requirement not met - await db.browserAutomationRun.update({ - where: { id: run.id }, - data: { - status: 'failed', - completedAt: new Date(), - durationMs: Date.now() - run.startedAt!.getTime(), - error: result.error, - evaluationStatus: result.evaluationStatus, - evaluationReason: result.evaluationReason, - }, - }); - - return { - runId: run.id, - success: false, - error: result.error, - evaluationStatus: result.evaluationStatus, - evaluationReason: result.evaluationReason, - needsReauth: result.needsReauth, - }; - } - - // Upload screenshot to S3 (only taken if evaluation passed) - let screenshotKey: string | undefined; - let presignedUrl: string | undefined; - if (result.screenshot) { - screenshotKey = await this.uploadScreenshot( - organizationId, - automationId, - run.id, - result.screenshot, - ); - presignedUrl = await this.getPresignedUrl(screenshotKey); - } - - // Update run as completed. Only persist an evaluation verdict when - // the automation had criteria configured. - await db.browserAutomationRun.update({ - where: { id: run.id }, - data: { - status: 'completed', - completedAt: new Date(), - durationMs: Date.now() - run.startedAt!.getTime(), - screenshotUrl: screenshotKey, - evaluationStatus: result.evaluationStatus ?? null, - evaluationReason: result.evaluationReason ?? null, - }, - }); - - return { - runId: run.id, - success: true, - screenshotUrl: presignedUrl, - evaluationStatus: result.evaluationStatus, - evaluationReason: result.evaluationReason, - }; - } finally { - // Always attempt to close the session, but never let cleanup override success - try { - await this.closeSession(sessionId); - } catch (err) { - this.logger.warn('Failed to close Browserbase session (ignored)', { - sessionId, - error: err instanceof Error ? err.message : String(err), - }); - } - } - } catch (err) { - this.logger.error('Failed to run browser automation', err); - const { userFacing, needsReauth } = toRunErrorMessage(err); - - // Update run as failed - await db.browserAutomationRun.update({ - where: { id: run.id }, - data: { - status: 'failed', - completedAt: new Date(), - durationMs: run.startedAt ? Date.now() - run.startedAt.getTime() : 0, - error: userFacing, - }, - }); - - return { - runId: run.id, - success: false, - error: userFacing, - needsReauth: needsReauth ? true : undefined, - }; - } - } - - private async executeAutomation( - sessionId: string, - targetUrl: string, - instruction: string, - taskContext?: { - title: string; - description?: string | null; - evaluationCriteria?: string | null; - }, - ): Promise<{ - success: boolean; - screenshot?: string; - evaluationStatus?: 'pass' | 'fail'; - evaluationReason?: string; - error?: string; - needsReauth?: boolean; - }> { - const stagehand = await this.createStagehand(sessionId); - - try { - let page = await this.ensureActivePage(stagehand); - - // Navigate to target URL - await page.goto(targetUrl, { - waitUntil: 'domcontentloaded', - timeoutMs: 30000, - }); - await delay(1000); - - // Check if we need to authenticate (look for login page indicators) - const loginSchema = z.object({ - isLoggedIn: z.boolean(), - }); - const authCheck = (await stagehand.extract( - 'Check if the user is logged in to this website. Look for a user avatar, profile menu, account dropdown, or login/sign-in buttons. Return true if logged in, false if you see login buttons or a login form.', - loginSchema as any, - )) as { isLoggedIn: boolean }; - - if (!authCheck.isLoggedIn) { - return { - success: false, - needsReauth: true, - error: 'Session expired. Please re-authenticate in browser settings.', - }; - } - - // Execute the navigation instruction using Stagehand agent - const fullInstruction = `${instruction}. After completing all navigation steps, stop and wait.`; - - await stagehand - .agent({ - cua: true, - model: { - modelName: STAGEHAND_CUA_MODEL, - apiKey: process.env.ANTHROPIC_API_KEY, - }, - }) - .execute({ - instruction: fullInstruction, - maxSteps: 20, - }); - - // Wait for final page to settle - await delay(2000); - - // Always take a screenshot at the end (no pass/fail criteria gate) - page = await this.ensureActivePage(stagehand); - const sourceUrl = page.url(); - const rawScreenshot = await page.screenshot({ - type: 'jpeg', - quality: 80, - fullPage: false, - }); - - let finalBuffer: Buffer = rawScreenshot; - try { - finalBuffer = await renderOverlay({ - buffer: rawScreenshot, - instruction, - sourceUrl, - capturedAt: new Date(), - }); - } catch (overlayErr) { - this.logger.warn( - 'Screenshot overlay render failed; uploading raw image', - { - error: - overlayErr instanceof Error - ? overlayErr.message - : String(overlayErr), - }, - ); - } - - // Optional evaluation: if the automation was configured with - // natural-language criteria, ask Stagehand to inspect the page and - // produce a pass/fail verdict with a short reason. - let evaluationStatus: 'pass' | 'fail' | undefined; - let evaluationReason: string | undefined; - const criteria = taskContext?.evaluationCriteria?.trim(); - if (criteria) { - try { - const evalSchema = z.object({ - pass: z.boolean(), - reason: z.string(), - }); - const evaluation = (await stagehand.extract( - `You are an auditor reviewing the current page after an automation has finished navigating. Decide whether the page clearly satisfies this criteria. Only return pass=true if the evidence is unambiguously present and visible. If it is ambiguous, missing, or contradicted, return pass=false. Always provide a short reason (max 220 characters).\n\nCriteria: ${criteria}`, - evalSchema as any, - )) as { pass: boolean; reason: string }; - - evaluationStatus = evaluation.pass ? 'pass' : 'fail'; - evaluationReason = evaluation.reason; - } catch (evalErr) { - this.logger.warn( - 'Evaluation step failed; returning screenshot without verdict', - { - error: - evalErr instanceof Error ? evalErr.message : String(evalErr), - }, - ); - } - } - - return { - success: true, - screenshot: finalBuffer.toString('base64'), - evaluationStatus, - evaluationReason, - }; - } catch (err) { - this.logger.error('Failed to execute automation', err); - const { userFacing, needsReauth } = toRunErrorMessage(err); - return { - success: false, - needsReauth: needsReauth ? true : undefined, - error: userFacing, - }; - } finally { - await this.safeCloseStagehand(stagehand); - } - } - - private async uploadScreenshot( - organizationId: string, - automationId: string, - runId: string, - base64Screenshot: string, - ): Promise { - const buffer = Buffer.from(base64Screenshot, 'base64'); - const key = `browser-automations/${organizationId}/${automationId}/${runId}.jpg`; - - await this.s3Client.send( - new PutObjectCommand({ - Bucket: this.bucketName, - Key: key, - Body: buffer, - ContentType: 'image/jpeg', - }), + ) { + return this.automationExecution.executeAutomationOnSession( + automationId, + runId, + sessionId, + organizationId, ); - - // Return just the key - we'll generate presigned URLs when viewing - return key; } - async getPresignedUrl( - key: string, - options: { expiresIn?: number; responseContentDisposition?: string } = {}, - ): Promise { - const command = new GetObjectCommand({ - Bucket: this.bucketName, - Key: key, - ResponseContentDisposition: options.responseContentDisposition, - }); - return getSignedUrl(this.s3Client, command, { - expiresIn: options.expiresIn ?? 3600, - }); + async runBrowserAutomation(automationId: string, organizationId: string) { + return this.automationExecution.runBrowserAutomation( + automationId, + organizationId, + ); } - /** - * Resolve a run's S3 screenshot key to a freshly signed presigned URL, - * scoped to the caller's organization. Used by the controller's - * GET runs/:runId/screenshot redirect endpoint so that the "Open full size" - * UI link never serves an expired URL. - * - * When `download` is true, the presigned URL is signed with an - * attachment Content-Disposition so the browser downloads the image - * instead of rendering it inline. - */ async getScreenshotRedirectUrl(input: { runId: string; organizationId: string; download?: boolean; }): Promise { - const { runId, organizationId, download } = input; - - const run = await db.browserAutomationRun.findUnique({ - where: { id: runId }, - include: { automation: { include: { task: true } } }, - }); - - if (!run || !run.screenshotUrl) { - throw new NotFoundException('Screenshot not found'); - } - - if (run.automation.task.organizationId !== organizationId) { - throw new NotFoundException('Screenshot not found'); - } - - const responseContentDisposition = download - ? `attachment; filename="screenshot-${runId}.jpg"` - : undefined; - - return this.getPresignedUrl(run.screenshotUrl, { - responseContentDisposition, - }); + return this.screenshots.getScreenshotRedirectUrl(input); } - async getRunWithPresignedUrl(runId: string) { - const run = await db.browserAutomationRun.findUnique({ - where: { id: runId }, - }); - - if (!run) return null; - - if (run.screenshotUrl) { - const presignedUrl = await this.getPresignedUrl(run.screenshotUrl); - return { ...run, screenshotUrl: presignedUrl }; - } - - return run; + async getRunWithPresignedUrl(runId: string, organizationId?: string) { + return this.automationCrud.getRunWithPresignedUrl(runId, organizationId); } - async getAutomationsWithPresignedUrls(taskId: string) { - const automations = await this.getBrowserAutomationsForTask(taskId); - - return Promise.all( - automations.map(async (automation) => { - const runsWithUrls = await Promise.all( - automation.runs.map(async (run) => { - if (run.screenshotUrl) { - const presignedUrl = await this.getPresignedUrl( - run.screenshotUrl, - ); - return { ...run, screenshotUrl: presignedUrl }; - } - return run; - }), - ); - return { ...automation, runs: runsWithUrls }; - }), + async getAutomationsWithPresignedUrls(taskId: string, organizationId?: string) { + return this.automationCrud.getAutomationsWithPresignedUrls( + taskId, + organizationId, ); } - // ===== Run History ===== + async getAutomationRuns( + automationId: string, + limit = 20, + organizationId?: string, + ) { + return this.automationCrud.getAutomationRuns( + automationId, + limit, + organizationId, + ); + } - async getAutomationRuns(automationId: string, limit = 20) { - return db.browserAutomationRun.findMany({ - where: { automationId }, - orderBy: { createdAt: 'desc' }, - take: limit, - }); + async getAutomationRun(runId: string, organizationId?: string) { + return this.automationCrud.getAutomationRun(runId, organizationId); } - async getAutomationRun(runId: string) { - return db.browserAutomationRun.findUnique({ - where: { id: runId }, - }); + getHostnameFromUrl(url: string): string { + return normalizeHostnameFromUrl(url); } } diff --git a/apps/api/src/browserbase/credential-vault.ts b/apps/api/src/browserbase/credential-vault.ts new file mode 100644 index 0000000000..cf04fcf4ea --- /dev/null +++ b/apps/api/src/browserbase/credential-vault.ts @@ -0,0 +1,19 @@ +export interface RuntimeCredentialMaterial { + username?: string; + password?: string; + totpCode?: string; +} + +export interface BrowserCredentialVaultAdapter { + resolveCredentialReference(params: { + profileId: string; + }): Promise; +} + +export class NoopBrowserCredentialVaultAdapter + implements BrowserCredentialVaultAdapter +{ + async resolveCredentialReference(): Promise { + return null; + } +} diff --git a/apps/api/src/browserbase/dto/browserbase.dto.ts b/apps/api/src/browserbase/dto/browserbase.dto.ts index 2a7334a782..1fb7b29b18 100644 --- a/apps/api/src/browserbase/dto/browserbase.dto.ts +++ b/apps/api/src/browserbase/dto/browserbase.dto.ts @@ -54,6 +54,136 @@ export class CloseSessionDto { sessionId: string; } +export class AuthStatusResponseDto { + @ApiProperty() + isLoggedIn: boolean; + + @ApiPropertyOptional() + username?: string; +} + +// ===== Auth Profile DTOs ===== + +export class ResolveAuthProfileDto { + @ApiProperty({ description: 'Website URL to normalize into an auth profile hostname' }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + url: string; + + @ApiPropertyOptional({ description: 'Human-readable profile name' }) + @IsString() + @IsOptional() + displayName?: string; + + @ApiPropertyOptional({ + description: 'Login identity label, such as a service account email', + }) + @IsString() + @IsOptional() + loginIdentity?: string; + + @ApiPropertyOptional({ description: 'External vault provider name' }) + @IsString() + @IsOptional() + vaultProvider?: string; + + @ApiPropertyOptional({ description: 'External vault item reference' }) + @IsString() + @IsOptional() + vaultExternalItemRef?: string; + + @ApiPropertyOptional({ description: 'External vault connection ID' }) + @IsString() + @IsOptional() + vaultConnectionId?: string; +} + +export class VerifyAuthProfileSessionDto { + @ApiProperty({ description: 'Browserbase session ID' }) + @IsString() + @IsNotEmpty() + sessionId: string; + + @ApiProperty({ description: 'URL to verify authentication on' }) + @IsUrl({}, { message: 'url must be a valid URL' }) + @IsSafeUrl({ message: 'The provided URL is not allowed.' }) + @IsString() + @IsNotEmpty() + url: string; +} + +export class MarkAuthProfileNeedsReauthDto { + @ApiPropertyOptional({ description: 'Reason the profile needs re-authentication' }) + @IsString() + @IsOptional() + reason?: string; +} + +export class BrowserAuthProfileResponseDto { + @ApiProperty() + id: string; + + @ApiProperty() + organizationId: string; + + @ApiProperty() + hostname: string; + + @ApiProperty() + loginIdentity: string; + + @ApiProperty() + displayName: string; + + @ApiProperty() + contextId: string; + + @ApiProperty({ enum: ['unverified', 'verified', 'needs_reauth', 'blocked'] }) + status: string; + + @ApiPropertyOptional() + lastVerifiedAt?: Date; + + @ApiPropertyOptional() + lastAuthCheckUrl?: string; + + @ApiPropertyOptional() + blockedReason?: string; + + @ApiPropertyOptional() + vaultProvider?: string; + + @ApiPropertyOptional() + vaultExternalItemRef?: string; + + @ApiPropertyOptional() + vaultConnectionId?: string; + + @ApiProperty() + createdAt: Date; + + @ApiProperty() + updatedAt: Date; +} + +export class ResolveAuthProfileResponseDto { + @ApiProperty({ type: BrowserAuthProfileResponseDto }) + profile: BrowserAuthProfileResponseDto; + + @ApiProperty() + isNew: boolean; +} + +export class VerifyAuthProfileResponseDto { + @ApiProperty({ type: BrowserAuthProfileResponseDto }) + profile: BrowserAuthProfileResponseDto; + + @ApiProperty({ type: () => AuthStatusResponseDto }) + auth: AuthStatusResponseDto; +} + // ===== Browser Automation DTOs ===== export class CreateBrowserAutomationDto { @@ -146,6 +276,18 @@ export class UpdateBrowserAutomationDto { scheduleFrequency?: TaskFrequency; } +export class ExecuteAutomationSessionDto { + @ApiProperty({ description: 'Browser automation run ID' }) + @IsString() + @IsNotEmpty() + runId: string; + + @ApiProperty({ description: 'Browserbase session ID' }) + @IsString() + @IsNotEmpty() + sessionId: string; +} + // ===== Response DTOs ===== export class ContextResponseDto { @@ -164,14 +306,6 @@ export class SessionResponseDto { liveViewUrl: string; } -export class AuthStatusResponseDto { - @ApiProperty() - isLoggedIn: boolean; - - @ApiPropertyOptional() - username?: string; -} - export class BrowserAutomationResponseDto { @ApiProperty() id: string; @@ -208,6 +342,9 @@ export class BrowserAutomationRunResponseDto { @ApiProperty() automationId: string; + @ApiPropertyOptional() + profileId?: string; + @ApiProperty() status: string; @@ -226,6 +363,21 @@ export class BrowserAutomationRunResponseDto { @ApiPropertyOptional() error?: string; + @ApiPropertyOptional() + failureCode?: string; + + @ApiPropertyOptional() + failureStage?: string; + + @ApiPropertyOptional() + blockedReason?: string; + + @ApiPropertyOptional() + finalUrl?: string; + + @ApiPropertyOptional() + attemptCount?: number; + @ApiProperty() createdAt: Date; } @@ -245,4 +397,19 @@ export class RunAutomationResponseDto { @ApiPropertyOptional() needsReauth?: boolean; + + @ApiPropertyOptional() + evaluationStatus?: string; + + @ApiPropertyOptional() + evaluationReason?: string; + + @ApiPropertyOptional() + failureCode?: string; + + @ApiPropertyOptional() + failureStage?: string; + + @ApiPropertyOptional() + blockedReason?: string; } diff --git a/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts b/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts new file mode 100644 index 0000000000..d445fa8497 --- /dev/null +++ b/apps/api/src/trigger/browser-automation/run-browser-automation.spec.ts @@ -0,0 +1,54 @@ +jest.mock('@db', () => ({ + db: { + browserAutomation: { findUnique: jest.fn(), update: jest.fn() }, + browserAutomationRun: { create: jest.fn() }, + task: { findUnique: jest.fn(), update: jest.fn() }, + organization: { findUnique: jest.fn() }, + member: { findMany: jest.fn() }, + }, +})); + +jest.mock('@trigger.dev/sdk', () => ({ + logger: { info: jest.fn(), warn: jest.fn(), error: jest.fn() }, + tags: { add: jest.fn() }, + task: (config: unknown) => config, +})); + +jest.mock('../../email/trigger-email', () => ({ + triggerEmail: jest.fn(), +})); + +jest.mock('@trycompai/email', () => ({ + isUserUnsubscribed: jest.fn().mockResolvedValue(false), +})); + +import { shouldMarkTaskDoneAfterBrowserRun } from './run-browser-automation'; + +describe('shouldMarkTaskDoneAfterBrowserRun', () => { + it('allows screenshot-only automations to complete tasks', () => { + expect( + shouldMarkTaskDoneAfterBrowserRun({ + screenshotUrl: 'https://example.com/screenshot.jpg', + evaluationCriteria: null, + }), + ).toBe(true); + }); + + it('requires a passing evaluation when criteria are configured', () => { + expect( + shouldMarkTaskDoneAfterBrowserRun({ + screenshotUrl: 'https://example.com/screenshot.jpg', + evaluationCriteria: 'Branch protection is enabled', + evaluationStatus: 'pass', + }), + ).toBe(true); + + expect( + shouldMarkTaskDoneAfterBrowserRun({ + screenshotUrl: 'https://example.com/screenshot.jpg', + evaluationCriteria: 'Branch protection is enabled', + evaluationStatus: 'fail', + }), + ).toBe(false); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automation.ts b/apps/api/src/trigger/browser-automation/run-browser-automation.ts index 26b54945a2..05dbf1d024 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automation.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automation.ts @@ -7,6 +7,25 @@ import { isUserUnsubscribed } from '@trycompai/email'; const browserbaseService = new BrowserbaseService(); +const browserAutomationConcurrencyLimit = (): number => { + const parsed = Number.parseInt( + process.env.BROWSER_AUTOMATION_GLOBAL_CONCURRENCY ?? '20', + 10, + ); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 20; +}; + +export function shouldMarkTaskDoneAfterBrowserRun(input: { + screenshotUrl?: string; + evaluationCriteria?: string | null; + evaluationStatus?: 'pass' | 'fail'; +}): boolean { + if (!input.screenshotUrl) return false; + const criteria = input.evaluationCriteria?.trim(); + if (!criteria) return true; + return input.evaluationStatus === 'pass'; +} + /** * Send email notifications for task status change */ @@ -171,7 +190,7 @@ export const runBrowserAutomation = task({ id: 'run-browser-automation', maxDuration: 1000 * 60 * 10, // 10 minutes per automation queue: { - concurrencyLimit: 80, + concurrencyLimit: browserAutomationConcurrencyLimit(), }, retry: { maxAttempts: 2, @@ -217,29 +236,6 @@ export const runBrowserAutomation = task({ }); const taskTitle = taskDetails?.title ?? 'Unknown Task'; - // Check if org has browser context - const context = await browserbaseService.getOrgContext(organizationId); - if (!context) { - logger.error(`No browser context for org ${organizationId}`); - - // Create a failed run record - await db.browserAutomationRun.create({ - data: { - automationId, - status: 'failed', - startedAt: new Date(), - completedAt: new Date(), - error: 'No browser context. Please connect your browser in settings.', - }, - }); - - return { - success: false, - error: 'No browser context', - needsReauth: true, - }; - } - // Run the automation const result = await browserbaseService.runBrowserAutomation( automationId, @@ -252,8 +248,13 @@ export const runBrowserAutomation = task({ screenshotUrl: result.screenshotUrl ? 'captured' : 'none', }); - // Update task status to done if screenshot was captured - if (result.screenshotUrl) { + if ( + shouldMarkTaskDoneAfterBrowserRun({ + screenshotUrl: result.screenshotUrl, + evaluationCriteria: automation.evaluationCriteria, + evaluationStatus: result.evaluationStatus, + }) + ) { const currentTask = await db.task.findUnique({ where: { id: taskId }, select: { status: true, frequency: true }, @@ -349,6 +350,7 @@ export const runBrowserAutomation = task({ screenshotUrl: result.screenshotUrl, error: result.error, needsReauth: result.needsReauth, + failureCode: result.failureCode, }; }, }); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts index 2ae7fb5a7d..0b1761361d 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.spec.ts @@ -1,5 +1,8 @@ import { TaskFrequency } from '@trycompai/db'; -import { filterDueAutomations } from './run-browser-automations-schedule'; +import { + filterDueAutomations, + limitAutomationBatch, +} from './run-browser-automations-schedule'; // Mock @db at the module boundary so importing the orchestrator does not try // to connect to Postgres. We never call the scheduled `run` function itself @@ -116,3 +119,38 @@ describe('filterDueAutomations (browser automation orchestrator)', () => { expect(due).toEqual([]); }); }); + +describe('limitAutomationBatch', () => { + it('limits due automations by organization and hostname', () => { + const automations = [ + { + id: 'ba_1', + targetUrl: 'https://github.com/a', + task: { organizationId: 'org_1' }, + }, + { + id: 'ba_2', + targetUrl: 'https://github.com/b', + task: { organizationId: 'org_1' }, + }, + { + id: 'ba_3', + targetUrl: 'https://gitlab.com/a', + task: { organizationId: 'org_1' }, + }, + { + id: 'ba_4', + targetUrl: 'https://github.com/c', + task: { organizationId: 'org_2' }, + }, + ]; + + const limited = limitAutomationBatch({ + automations, + maxPerOrg: 2, + maxPerHostname: 2, + }); + + expect(limited.map((automation) => automation.id)).toEqual(['ba_1', 'ba_2']); + }); +}); diff --git a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts index dedd4b5660..dcdaf262aa 100644 --- a/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts +++ b/apps/api/src/trigger/browser-automation/run-browser-automations-schedule.ts @@ -2,6 +2,7 @@ import { db, TaskFrequency } from '@db'; import { logger, schedules } from '@trigger.dev/sdk'; import { runBrowserAutomation } from './run-browser-automation'; import { isDueToday } from '../shared/is-due-today'; +import { normalizeHostnameFromUrl } from '../../browserbase/browserbase-url'; /** * Pure helper extracted for unit testing. Filters a list of candidate @@ -25,6 +26,48 @@ export function filterDueAutomations< ); } +const parsePositiveInt = (value: string | undefined, fallback: number): number => { + if (!value) return fallback; + const parsed = Number.parseInt(value, 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +}; + +export function limitAutomationBatch< + T extends { + targetUrl: string; + task: { organizationId: string }; + }, +>({ + automations, + maxPerOrg, + maxPerHostname, +}: { + automations: T[]; + maxPerOrg: number; + maxPerHostname: number; +}): T[] { + const orgCounts = new Map(); + const hostnameCounts = new Map(); + const selected: T[] = []; + + for (const automation of automations) { + const organizationId = automation.task.organizationId; + const hostname = normalizeHostnameFromUrl(automation.targetUrl); + const orgCount = orgCounts.get(organizationId) ?? 0; + const hostnameCount = hostnameCounts.get(hostname) ?? 0; + + if (orgCount >= maxPerOrg || hostnameCount >= maxPerHostname) { + continue; + } + + orgCounts.set(organizationId, orgCount + 1); + hostnameCounts.set(hostname, hostnameCount + 1); + selected.push(automation); + } + + return selected; +} + /** * Daily scheduled task (orchestrator) that finds all enabled browser automations * and triggers individual runs for each. @@ -48,6 +91,7 @@ export const browserAutomationsSchedule = schedules.task({ id: true, name: true, taskId: true, + targetUrl: true, scheduleFrequency: true, lastRunAt: true, task: { @@ -89,8 +133,26 @@ export const browserAutomationsSchedule = schedules.task({ return { success: true, automationsTriggered: 0 }; } + const limitedAutomations = limitAutomationBatch({ + automations, + maxPerOrg: parsePositiveInt( + process.env.BROWSER_AUTOMATION_ORG_CONCURRENCY, + 5, + ), + maxPerHostname: parsePositiveInt( + process.env.BROWSER_AUTOMATION_HOST_CONCURRENCY, + 3, + ), + }); + + if (limitedAutomations.length < automations.length) { + logger.info( + `Deferred ${automations.length - limitedAutomations.length} automation(s) due to org/domain concurrency limits`, + ); + } + // Build payloads for batch triggering - const triggerPayloads = automations.map((automation) => ({ + const triggerPayloads = limitedAutomations.map((automation) => ({ payload: { automationId: automation.id, automationName: automation.name, diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx index 08992aced7..be1e1bdca1 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.test.tsx @@ -1,4 +1,11 @@ import { render, screen } from '@testing-library/react'; +import type { + ButtonHTMLAttributes, + HTMLAttributes, + InputHTMLAttributes, + LabelHTMLAttributes, + ReactNode, +} from 'react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { setMockPermissions, @@ -16,44 +23,46 @@ vi.mock('@/hooks/use-permissions', () => ({ vi.mock('@/lib/api-client', () => ({ apiClient: { - get: vi.fn().mockResolvedValue({ data: { hasContext: false } }), + get: vi.fn(() => new Promise(() => undefined)), post: vi.fn().mockResolvedValue({ data: {} }), }, })); -vi.mock('@trycompai/ui/badge', () => ({ - Badge: ({ children }: any) => {children}, -})); - -vi.mock('@trycompai/ui/button', () => ({ - Button: ({ children, disabled, onClick, ...props }: any) => ( - ), + Card: ({ children }: HTMLAttributes) =>
{children}
, + CardContent: ({ children }: HTMLAttributes) =>
{children}
, + CardDescription: ({ children }: HTMLAttributes) =>

{children}

, + CardHeader: ({ children }: HTMLAttributes) =>
{children}
, + CardTitle: ({ children }: HTMLAttributes) =>

{children}

, + Input: (props: InputHTMLAttributes) => , + Label: ({ children, ...props }: LabelHTMLAttributes) => ( + + ), + Spinner: () => , })); -vi.mock('@trycompai/ui/card', () => ({ - Card: ({ children }: any) =>
{children}
, - CardContent: ({ children }: any) =>
{children}
, - CardDescription: ({ children }: any) =>

{children}

, - CardHeader: ({ children }: any) =>
{children}
, - CardTitle: ({ children }: any) =>

{children}

, -})); - -vi.mock('@trycompai/ui/input', () => ({ - Input: (props: any) => , -})); - -vi.mock('@trycompai/ui/label', () => ({ - Label: ({ children, ...props }: any) => , -})); - -vi.mock('lucide-react', () => ({ +vi.mock('@trycompai/design-system/icons', () => ({ Globe: () => , - Loader2: () => , - MonitorSmartphone: () => , - RefreshCw: () => , + Renew: () => , + Screen: () => , })); import { BrowserConnectionClient } from './BrowserConnectionClient'; diff --git a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx index 6dbcdf7638..ce1565d1c5 100644 --- a/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx +++ b/apps/app/src/app/(app)/[orgId]/settings/browser-connection/components/BrowserConnectionClient.tsx @@ -2,16 +2,29 @@ import { apiClient } from '@/lib/api-client'; import { usePermissions } from '@/hooks/use-permissions'; -import { Badge } from '@trycompai/ui/badge'; -import { Button } from '@trycompai/ui/button'; -import { Card, CardContent, CardDescription, CardHeader, CardTitle } from '@trycompai/ui/card'; -import { Input } from '@trycompai/ui/input'; -import { Label } from '@trycompai/ui/label'; -import { Globe, Loader2, MonitorSmartphone, RefreshCw } from 'lucide-react'; +import { + Badge, + Button, + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, + Input, + Label, + Spinner, +} from '@trycompai/design-system'; +import { Globe, Screen } from '@trycompai/design-system/icons'; import { useCallback, useEffect, useState } from 'react'; +import { + BrowserConnectionProfileList, + type BrowserConnectionProfile, +} from './BrowserConnectionProfileList'; +import { BrowserConnectionInstructions } from './BrowserConnectionInstructions'; +import { BrowserConnectionLiveView } from './BrowserConnectionLiveView'; -interface ContextResponse { - contextId: string; +interface ResolveProfileResponse { + profile: BrowserConnectionProfile & { contextId: string }; isNew: boolean; } @@ -25,6 +38,11 @@ interface AuthStatusResponse { username?: string; } +interface VerifyProfileResponse { + profile: BrowserConnectionProfile; + auth: AuthStatusResponse; +} + type Status = 'idle' | 'loading' | 'session-active' | 'checking'; interface BrowserConnectionClientProps { @@ -36,22 +54,23 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli const canManageBrowser = hasPermission('integration', 'create'); const [status, setStatus] = useState('idle'); const [hasContext, setHasContext] = useState(false); - const [contextId, setContextId] = useState(null); + const [profileId, setProfileId] = useState(null); + const [profiles, setProfiles] = useState([]); const [sessionId, setSessionId] = useState(null); const [liveViewUrl, setLiveViewUrl] = useState(null); const [urlToCheck, setUrlToCheck] = useState('https://github.com'); const [authStatus, setAuthStatus] = useState(null); const [error, setError] = useState(null); - // Check if org has a browser context const checkContextStatus = useCallback(async () => { try { - const res = await apiClient.get<{ hasContext: boolean; contextId?: string }>( - '/v1/browserbase/org-context', - ); + const res = await apiClient.get('/v1/browserbase/profiles'); if (res.data) { - setHasContext(res.data.hasContext); - setContextId(res.data.contextId || null); + setProfiles(res.data); + const verifiedProfile = res.data.find((profile) => profile.status === 'verified'); + const firstProfile = verifiedProfile ?? res.data[0]; + setHasContext(Boolean(verifiedProfile)); + setProfileId(firstProfile?.id ?? null); } } catch { // Ignore @@ -68,21 +87,22 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli setError(null); setStatus('loading'); - // Get or create org context - const contextRes = await apiClient.post( - '/v1/browserbase/org-context', - {}, + const profileRes = await apiClient.post( + '/v1/browserbase/profiles/resolve', + { url: urlToCheck }, ); - if (contextRes.error || !contextRes.data) { - throw new Error(contextRes.error || 'Failed to create context'); + if (profileRes.error || !profileRes.data) { + throw new Error(profileRes.error || 'Failed to create auth profile'); } - setContextId(contextRes.data.contextId); - setHasContext(true); + setProfileId(profileRes.data.profile.id); + setProfiles((currentProfiles) => { + const rest = currentProfiles.filter((profile) => profile.id !== profileRes.data?.profile.id); + return profileRes.data ? [profileRes.data.profile, ...rest] : currentProfiles; + }); - // Create session const sessionRes = await apiClient.post( - '/v1/browserbase/session', - { contextId: contextRes.data.contextId }, + `/v1/browserbase/profiles/${profileRes.data.profile.id}/session`, + {}, ); if (sessionRes.error || !sessionRes.data) { throw new Error(sessionRes.error || 'Failed to create session'); @@ -119,26 +139,32 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli }; const handleCheckAuth = async () => { - if (!sessionId) return; + if (!sessionId || !profileId) return; try { setError(null); setStatus('checking'); - const res = await apiClient.post( - '/v1/browserbase/check-auth', + const res = await apiClient.post( + `/v1/browserbase/profiles/${profileId}/verify`, { sessionId, url: urlToCheck }, ); if (res.error || !res.data) { throw new Error(res.error || 'Failed to check auth'); } - setAuthStatus(res.data); + setAuthStatus(res.data.auth); + setProfiles((currentProfiles) => { + const rest = currentProfiles.filter((profile) => profile.id !== res.data?.profile.id); + return res.data ? [res.data.profile, ...rest] : currentProfiles; + }); + setHasContext(res.data.profile.status === 'verified'); // Close the session after checking await apiClient.post('/v1/browserbase/session/close', { sessionId }); setSessionId(null); setLiveViewUrl(null); + setProfileId(null); setStatus('idle'); } catch (err) { setError(err instanceof Error ? err.message : 'Failed to check auth'); @@ -156,6 +182,7 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli } setSessionId(null); setLiveViewUrl(null); + setProfileId(null); setStatus('idle'); }; @@ -167,14 +194,14 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli
- +
- Browser Session + Browser Session {hasContext - ? 'Your organization has a browser context configured' - : 'No browser context configured yet'} + ? 'At least one browser auth profile is verified' + : 'No verified browser auth profile yet'}
@@ -191,16 +218,20 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli
- setUrlToCheck(e.target.value)} - className="flex-1" - /> +
+ setUrlToCheck(e.target.value)} + /> +
{canManageBrowser && ( - )} @@ -232,89 +263,25 @@ export function BrowserConnectionClient({ organizationId }: BrowserConnectionCli {status === 'loading' && (
- + Starting browser session...
)} - {/* Live View */} {(status === 'session-active' || status === 'checking') && liveViewUrl && ( - - -
-
- Browser Session - - Log in to websites below. Your session will be saved for automations. - -
-
- - -
-
- - -
-