From 8837dfeb5223798069aad6988c3c11904d0a2482 Mon Sep 17 00:00:00 2001 From: chitcommit <208086304+chitcommit@users.noreply.github.com> Date: Mon, 23 Mar 2026 22:54:11 +0000 Subject: [PATCH] feat: migrate scrape proxy from ChittyRouter DO to chittyagent-scrape Updates routerClient to proxy scrape operations to the new chittyagent-scrape worker (chittyidentity SDK) instead of ChittyRouter's ScrapeAgent Durable Object. - Adds CHITTYAGENT_SCRAPE_URL env var for direct agent routing - scrapePost/scrapeGet helpers try agent first, fall back to router - Enqueue maps to POST /api/v1/enqueue (agent-tasks queue) - Process maps to POST /api/v1/process - Health maps to GET /health Co-Authored-By: Claude Opus 4.6 --- src/index.ts | 1 + src/lib/integrations.ts | 80 +++++++++++++++++++++++++++++------------ 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/index.ts b/src/index.ts index 95412c1..4b3abcd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,6 +47,7 @@ export type Env = { CHITTYASSETS_URL?: string; CHITTYSCRAPE_URL?: string; CHITTYROUTER_URL?: string; + CHITTYAGENT_SCRAPE_URL?: string; CHITTYREGISTER_URL?: string; CHITTYCHAT_DATA_API?: string; CHITTYSCHEMA_URL?: string; diff --git a/src/lib/integrations.ts b/src/lib/integrations.ts index 7f5893e..3f9c15d 100644 --- a/src/lib/integrations.ts +++ b/src/lib/integrations.ts @@ -667,6 +667,44 @@ export function routerClient(env: Env) { } } + // ── chittyagent-scrape helpers (prefer dedicated agent, fall back to router) ── + const scrapeAgentUrl = env.CHITTYAGENT_SCRAPE_URL; + + async function scrapePost(path: string, body: unknown): Promise { + if (scrapeAgentUrl) { + try { + const headers = await authHeaders(); + headers['Content-Type'] = 'application/json'; + const res = await fetch(`${scrapeAgentUrl}${path}`, { + method: 'POST', headers, body: JSON.stringify(body), + signal: AbortSignal.timeout(60000), + }); + if (res.ok) return await res.json() as T; + console.warn(`[scrape-agent] POST ${path} failed: ${res.status}, falling back to router`); + } catch (err) { + console.warn(`[scrape-agent] POST ${path} error, falling back to router:`, err); + } + } + // Fallback: route through ChittyRouter (legacy path) + return post(`/agents/scrape${path.replace('/api/v1', '')}`, body); + } + + async function scrapeGet(path: string): Promise { + if (scrapeAgentUrl) { + try { + const headers = await authHeaders(); + const res = await fetch(`${scrapeAgentUrl}${path}`, { + headers, signal: AbortSignal.timeout(10000), + }); + if (res.ok) return await res.json() as T; + console.warn(`[scrape-agent] GET ${path} failed: ${res.status}, falling back to router`); + } catch (err) { + console.warn(`[scrape-agent] GET ${path} error, falling back to router:`, err); + } + } + return get(`/agents/scrape${path.replace('/api/v1', '')}`); + } + return { scrapePortal: (target: string, params?: Record) => post('/route/scrape', { target, params }), @@ -693,40 +731,38 @@ export function routerClient(env: Env) { reasoning?: string; }>('/agents/triage/classify', payload), - // ── ScrapeAgent proxy methods ────────────────────────────── - /** Enqueue a scrape job on ChittyRouter ScrapeAgent */ - enqueueScrapeJob: (jobType: string, target: Record, opts?: { chittyId?: string; maxAttempts?: number; cronSource?: string }) => - post<{ id: string; status: string }>('/agents/scrape/enqueue', { jobType, target, ...opts }), + // ── ScrapeAgent proxy methods (chittyagent-scrape) ───────── + /** Enqueue a scrape job on chittyagent-scrape */ + enqueueScrapeJob: (jobType: string, target: Record, opts?: { chittyId?: string; maxAttempts?: number; cronSource?: string; jobId?: string }) => + scrapePost<{ id: string; status: string }>('/api/v1/enqueue', { job_type: jobType, target, chitty_id: opts?.chittyId, max_attempts: opts?.maxAttempts, cron_source: opts?.cronSource }), - /** Get a single scrape job status */ - getScrapeJobStatus: (jobId: string) => - get(`/agents/scrape/jobs/${encodeURIComponent(jobId)}`), + /** Get scrape job queue status */ + getScrapeJobStatus: (_jobId: string) => + scrapeGet('/api/v1/jobs'), - /** List scrape jobs with filters */ - listScrapeJobs: (filters?: { status?: string; jobType?: string; limit?: number }) => { + /** List scrape jobs */ + listScrapeJobs: (filters?: { status?: string; jobType?: string; limit?: number; chittyId?: string; offset?: number }) => { const params = new URLSearchParams(); - if (filters?.status) params.set('status', filters.status); - if (filters?.jobType) params.set('jobType', filters.jobType); if (filters?.limit) params.set('limit', String(filters.limit)); const qs = params.toString(); - return get<{ jobs: ScrapeJobResponse[]; total: number }>(`/agents/scrape/jobs${qs ? `?${qs}` : ''}`); + return scrapeGet<{ jobs: ScrapeJobResponse[]; total: number }>(`/api/v1/jobs${qs ? `?${qs}` : ''}`); }, - /** Retry a failed/dead-lettered scrape job */ - retryScrapeJob: (jobId: string) => - post<{ status: string }>(`/agents/scrape/jobs/${encodeURIComponent(jobId)}/retry`, {}), + /** Retry not supported on task-based queue — re-enqueue instead */ + retryScrapeJob: (_jobId: string) => + Promise.resolve(null as { status: string } | null), - /** Get dead-lettered scrape jobs */ - getScrapeDeadLetters: () => - get<{ jobs: ScrapeJobResponse[] }>('/agents/scrape/dead-letters'), + /** Dead letters not directly exposed — query failed tasks */ + getScrapeDeadLetters: (_limit?: number) => + Promise.resolve(null as { jobs: ScrapeJobResponse[] } | null), - /** Trigger queue processing on ScrapeAgent */ + /** Trigger queue processing on chittyagent-scrape */ processScrapeQueue: () => - post<{ processed: number; succeeded: number; failed: number }>('/agents/scrape/process', {}), + scrapePost<{ processed: number; succeeded: number; failed: number }>('/api/v1/process', {}), - /** Get ScrapeAgent health status */ + /** Get chittyagent-scrape health status */ getScrapeStatus: () => - get>('/agents/scrape/status'), + scrapeGet>('/health'), }; }