diff --git a/README.md b/README.md index d63b4772f9..975b3c3280 100644 --- a/README.md +++ b/README.md @@ -1012,3 +1012,26 @@ firecrawl setup workflows ## Documentation For more details, visit the [Firecrawl Documentation](https://docs.firecrawl.dev). + +### Alexandria provider terms (beta) + +When a provider returns `THIRD_PARTY_DATA_TERMS_REQUIRED`, review its linked terms. +Read the current provider agreement and metadata with: + +```bash +npx firecrawl-cli@alexandria alexandria terms show benzinga --pretty +``` + +After reviewing it, explicitly accept the exact version and digest for the organization +associated with your Firecrawl API key: + +```bash +npx firecrawl-cli@alexandria alexandria terms accept benzinga \ + --version '' --digest '' --confirm +``` + +This posts to `/exchange/provider-terms/accept`. No automatic acceptance or retry +occurs. A `409 terms_changed` requires reviewing the new agreement before retrying. +The terms catalog may remain access-gated even when the acceptance endpoint is +available. A failed catalog lookup does not imply acceptance is unavailable. +After confirmed success, rerun the original provider command; its normal credits apply. diff --git a/package.json b/package.json index 958fc273ce..2db166ad4f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.23.4-alexandria-beta.10", + "version": "1.23.4-alexandria-beta.13", "publishConfig": { "tag": "alexandria" }, diff --git a/src/commands/alexandria.ts b/src/commands/alexandria.ts index c2b32cab60..df82edb57f 100644 --- a/src/commands/alexandria.ts +++ b/src/commands/alexandria.ts @@ -119,6 +119,11 @@ export async function handleAlexandria( !envelope.success || envelope.data?.alexandria?.some((item: any) => item.error); if (failed) process.exitCode = 1; + if (envelope.code === 'THIRD_PARTY_DATA_TERMS_REQUIRED') { + console.error( + 'Review the provider terms with firecrawl alexandria terms show . After review, accept with firecrawl alexandria terms accept --version --digest --confirm.' + ); + } writeOutput( JSON.stringify(envelope, null, options.pretty ? 2 : undefined), options.output, diff --git a/src/commands/list.test.ts b/src/commands/list.test.ts new file mode 100644 index 0000000000..6289845c54 --- /dev/null +++ b/src/commands/list.test.ts @@ -0,0 +1,119 @@ +import { beforeEach, expect, it, vi } from 'vitest'; +import { Command } from 'commander'; +import { createAlexandriaCommand } from './list'; +import { requestAlexandria } from './alexandria'; +vi.mock('./alexandria', async (original) => ({ + ...(await original()), + requestAlexandria: vi.fn(), +})); +vi.mock('../utils/output', () => ({ writeOutput: vi.fn() })); +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(requestAlexandria).mockResolvedValue({ + success: true, + data: { + alexandria: [{ data: { level: 'providers', items: [], total: 1 } }], + }, + } as any); +}); +it.each( + 'ai-models apps companies software finance health jobs news people places podcasts government real-estate restaurants shopping social sports skills travel'.split( + ' ' + ) +)('routes %s directly to category discovery', async (category) => { + await new Command() + .addCommand(createAlexandriaCommand()) + .parseAsync(['alexandria', category, '--json'], { from: 'user' }); + expect(requestAlexandria).toHaveBeenCalledWith( + [ + { + provider: 'firecrawl', + capability: 'find-tools', + options: { + categories: [category], + level: 'providers', + limit: 20, + }, + }, + ], + expect.anything() + ); +}); +it('preserves explicit provider browsing', async () => { + await new Command() + .addCommand(createAlexandriaCommand()) + .parseAsync(['alexandria', 'list', 'benzinga', '--json'], { from: 'user' }); + expect(requestAlexandria).toHaveBeenCalledWith( + [ + { + provider: 'firecrawl', + capability: 'find-tools', + options: { providers: ['benzinga'], level: 'tools', limit: 20 }, + }, + ], + expect.anything() + ); +}); + +it('lists compact provider tools before expanding a selected contract', async () => { + const run = (path: string[]) => + new Command() + .addCommand(createAlexandriaCommand()) + .parseAsync(['alexandria', ...path, '--json'], { from: 'user' }); + await run(['people', 'fullenrich']); + expect(requestAlexandria).toHaveBeenLastCalledWith( + [ + { + provider: 'firecrawl', + capability: 'find-tools', + options: { + categories: ['people'], + providers: ['fullenrich'], + level: 'tools', + limit: 20, + }, + }, + ], + expect.anything() + ); + await run(['people', 'fullenrich', 'people/search']); + expect(requestAlexandria).toHaveBeenLastCalledWith( + [ + { + provider: 'firecrawl', + capability: 'find-tools', + options: { + categories: ['people'], + providers: ['fullenrich'], + capabilities: ['people/search'], + level: 'tools', + expand: ['options', 'response', 'examples'], + limit: 20, + }, + }, + ], + expect.anything() + ); +}); +it('expands category contracts only when requested', async () => { + await new Command() + .addCommand(createAlexandriaCommand()) + .parseAsync(['alexandria', 'people', '--contracts', '--json'], { + from: 'user', + }); + expect(requestAlexandria).toHaveBeenLastCalledWith( + [ + { + provider: 'firecrawl', + capability: 'find-tools', + options: { + categories: ['people'], + level: 'tools', + expand: ['options', 'response', 'examples'], + limit: 20, + }, + }, + ], + expect.anything() + ); +}); diff --git a/src/commands/list.ts b/src/commands/list.ts index 78f4a8f404..9dbdfe6565 100644 --- a/src/commands/list.ts +++ b/src/commands/list.ts @@ -1,3 +1,4 @@ +import { createTermsCommand } from './terms'; import { Command, InvalidArgumentError } from 'commander'; import { randomUUID } from 'node:crypto'; import { @@ -13,6 +14,7 @@ import { getApiKey, getConfig } from '../utils/config'; type Selectors = Record; type ListOptions = AlexandriaOptions & { category?: boolean; + contracts?: boolean; limit?: number; request?: string; providers?: boolean; @@ -161,7 +163,7 @@ function renderCategories(items: Category[]): string { ' Browse a category below, or jump directly to a provider.', '', 'Calling it', - ' Browse: firecrawl alexandria list --category', + ' Browse: firecrawl alexandria ', ' Tools: firecrawl alexandria list ', ' Inspect: firecrawl alexandria list ', " Execute: firecrawl scrape --alexandria / --options ''", @@ -359,6 +361,14 @@ export async function handleList( if (options.request) return fetchPage(parseFindToolsRequest(options.request).options); if (!path.length) return fetchPage({ level: 'providers', limit }); + if (options.contracts && options.category && path.length === 1) { + return fetchPage({ + categories: [categoryId(path[0])], + level: 'tools', + expand: ['options', 'response', 'examples'], + limit, + }); + } let scope: Selectors = { providers: [path[0]] }; let remaining = path.slice(1); let result = options.category @@ -428,9 +438,10 @@ export async function handleList( } } -export function createListCommand(): Command { - return new Command('list') - .alias('list-tools') +export function createListCommand(name = 'list'): Command { + const command = new Command(name); + if (name === 'list') command.alias('list-tools'); + return command .description( 'Start with the Alexandria category index, then browse providers and tool contracts; discovery only' ) @@ -442,6 +453,7 @@ export function createListCommand(): Command { '--category', 'Treat the first ID as a category when a provider has the same ID' ) + .option('--contracts', 'Include full tool contracts for a category') .option('--providers', 'List all providers instead of the category index') .option( '--limit ', @@ -468,3 +480,20 @@ export function createListCommand(): Command { ) .action(handleList); } + +export function createAlexandriaCommand(): Command { + const browse = createListCommand('browse').action( + (path: string[], options: ListOptions) => + handleList(path, { + ...options, + category: path.length > 0, + }) + ); + return new Command('alexandria') + .description( + 'Browse categories with alexandria , or inspect providers with alexandria list' + ) + .addCommand(createListCommand()) + .addCommand(createTermsCommand()) + .addCommand(browse, { isDefault: true, hidden: true }); +} diff --git a/src/commands/terms.test.ts b/src/commands/terms.test.ts new file mode 100644 index 0000000000..d567e86d1a --- /dev/null +++ b/src/commands/terms.test.ts @@ -0,0 +1,74 @@ +import { afterEach, expect, it, vi } from 'vitest'; +import { requestTerms } from './terms'; +vi.mock('../utils/config', () => ({ + getApiKey: () => 'test-key', + getConfig: () => ({}), +})); +afterEach(() => vi.unstubAllGlobals()); +const options = { version: 'v1', digest: 'a'.repeat(64), confirm: true }; +it('requires explicit confirmation without making any request', async () => { + const fetcher = vi.fn(); + vi.stubGlobal('fetch', fetcher); + await expect( + requestTerms('benzinga', { ...options, confirm: false }, true) + ).rejects.toThrow('--confirm'); + expect(fetcher).not.toHaveBeenCalled(); +}); +it('submits only the reviewed provider version and digest to the new API', async () => { + const fetcher = vi + .fn() + .mockResolvedValue( + new Response(JSON.stringify({ success: true, provider: 'benzinga' })) + ); + vi.stubGlobal('fetch', fetcher); + expect(await requestTerms('benzinga', options, true)).toMatchObject({ + success: true, + }); + const [url, init] = fetcher.mock.calls[0]!; + expect(url).toBe('https://api.firecrawl.dev/exchange/provider-terms/accept'); + expect(JSON.parse(init.body)).toEqual({ + provider: 'benzinga', + version: 'v1', + digest: 'a'.repeat(64), + confirmed: true, + }); + expect(init.redirect).toBe('error'); + expect(fetcher).toHaveBeenCalledTimes(1); +}); +it('preserves changed-terms errors without retrying', async () => { + const fetcher = vi + .fn() + .mockResolvedValue( + new Response( + JSON.stringify({ error: 'Terms changed', code: 'terms_changed' }), + { status: 409 } + ) + ); + vi.stubGlobal('fetch', fetcher); + expect(await requestTerms('benzinga', options, true)).toMatchObject({ + success: false, + status: 409, + code: 'terms_changed', + }); + expect(fetcher).toHaveBeenCalledTimes(1); +}); +it('shows only the requested provider and rejects HTML responses', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce( + new Response( + JSON.stringify({ + providers: [{ provider: 'benzinga', terms: { version: 'v1' } }], + }) + ) + ) + .mockResolvedValueOnce(new Response('')) + ); + expect(await requestTerms('benzinga', {})).toMatchObject({ + provider: 'benzinga', + terms: { version: 'v1' }, + }); + await expect(requestTerms('benzinga', {})).rejects.toThrow('non-JSON'); +}); diff --git a/src/commands/terms.ts b/src/commands/terms.ts new file mode 100644 index 0000000000..1f0140261d --- /dev/null +++ b/src/commands/terms.ts @@ -0,0 +1,136 @@ +import { Command } from 'commander'; +import { getApiKey, getConfig } from '../utils/config'; +import { writeOutput } from '../utils/output'; + +type TermsOptions = { + apiKey?: string; + apiUrl?: string; + version?: string; + digest?: string; + confirm?: boolean; + json?: boolean; + pretty?: boolean; +}; + +export async function requestTerms( + provider: string, + options: TermsOptions, + accept = false +): Promise> { + if (!provider.trim() || provider.length > 200) + throw new Error('Provide a provider ID of 1-200 characters.'); + if ( + accept && + (!options.confirm || + !options.version?.trim() || + options.version.length > 200 || + !/^[a-f0-9]{64}$/.test(options.digest ?? '')) + ) { + throw new Error( + 'Review the terms, then supply --version, --digest (64 lowercase hex characters), and --confirm.' + ); + } + const key = getApiKey(options.apiKey); + if (!key) + throw new Error('A Firecrawl API key is required. Run firecrawl login.'); + const base = ( + options.apiUrl || + getConfig().apiUrl || + 'https://api.firecrawl.dev' + ).replace(/\/$/, ''); + const response = await fetch( + `${base}/exchange/provider-terms${accept ? '/accept' : ''}`, + { + method: accept ? 'POST' : 'GET', + headers: { + Authorization: `Bearer ${key}`, + 'Content-Type': 'application/json', + }, + ...(accept + ? { + body: JSON.stringify({ + provider, + version: options.version, + digest: options.digest, + confirmed: true, + }), + } + : {}), + redirect: 'error', + signal: AbortSignal.timeout(getConfig().timeoutMs ?? 30000), + } + ); + const body = await response.json().catch(() => null); + if (!body || typeof body !== 'object') + throw new Error( + `Terms endpoint returned non-JSON (HTTP ${response.status}).` + ); + if (!response.ok || body.success === false) + return { ...body, success: false, status: response.status }; + if (accept) { + if (body.success !== true) + throw new Error( + 'Acceptance response did not confirm success. Check status before retrying.' + ); + return body; + } + if (!Array.isArray(body.providers)) + throw new Error('Terms endpoint returned an invalid catalog.'); + const item = body.providers.find((entry: any) => entry.provider === provider); + if (!item) + throw new Error('Provider not found in the accessible terms catalog.'); + return { success: true, ...item }; +} + +async function handle( + provider: string, + options: TermsOptions, + accept: boolean +) { + try { + const result = await requestTerms(provider, options, accept); + if (result.success === false) process.exitCode = 1; + writeOutput(JSON.stringify(result, null, options.pretty ? 2 : undefined)); + } catch (error) { + process.exitCode = 1; + writeOutput( + JSON.stringify({ + success: false, + error: error instanceof Error ? error.message : 'Terms request failed', + }) + ); + } +} + +export function createTermsCommand(): Command { + const terms = new Command('terms').description( + 'Read provider terms or explicitly accept an exact version for your API key organization' + ); + for (const name of ['show', 'accept'] as const) { + const command = new Command(name) + .argument('', 'Exact provider ID') + .option('-k, --api-key ', 'Firecrawl API key') + .option('--api-url ', 'Firecrawl API URL') + .option('--json', 'Output JSON (default)') + .option('--pretty', 'Format JSON'); + if (name === 'accept') + command + .requiredOption( + '--version ', + 'Exact version of the terms you reviewed' + ) + .requiredOption( + '--digest ', + 'Exact SHA-256 digest of the terms you reviewed' + ) + .option( + '--confirm', + 'Confirm acceptance for the organization associated with your API key' + ); + command.action((provider, options) => + handle(provider, options, name === 'accept') + ); + terms.addCommand(command); + } + return terms; +} diff --git a/src/index.ts b/src/index.ts index 2f1308fe29..1be04672ca 100644 --- a/src/index.ts +++ b/src/index.ts @@ -80,7 +80,7 @@ import type { ScrapeFormat } from './types/scrape'; import type { RelatedPapersOptions } from './types/research'; import type { AgentWebhookConfig } from 'firecrawl'; import { createCreateCommand } from './commands/create'; -import { createListCommand } from './commands/list'; +import { createListCommand, createAlexandriaCommand } from './commands/list'; // Initialize global configuration from environment variables initializeConfig(); @@ -2201,11 +2201,7 @@ program.addCommand(createMonitorCommand()); program.addCommand(createSearchCommand()); program.addCommand(createFindToolsCommand()); program.addCommand(createListCommand()); -program.addCommand( - new Command('alexandria') - .description('Alexandria catalogue commands') - .addCommand(createListCommand()) -); +program.addCommand(createAlexandriaCommand()); program.addCommand(createDeveloperCommand()); program.addCommand(createResearchCommand()); program.addCommand(createFeedbackCommand());