diff --git a/README.md b/README.md index d9766319f7..07e5fb9a64 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ npx -y firecrawl-cli@latest init -y --browser Use the Alexandria beta with your existing Firecrawl login or API key: ```bash -npx firecrawl-cli@alexandria list # visible providers +npx firecrawl-cli@alexandria alexandria list # introduction and live categories +npx firecrawl-cli@alexandria list --providers # flat provider list npx firecrawl-cli@alexandria list finance # providers in a category npx firecrawl-cli@alexandria list benzinga # provider's tools npx firecrawl-cli@alexandria list-tools benzinga # same browsing interface @@ -32,11 +33,11 @@ npx firecrawl-cli@alexandria list benzinga calendar --group npx firecrawl-cli@alexandria list benzinga --json ``` -Available in `1.23.4-alexandria-beta.8` onward. `list` and `list-tools` are interchangeable, including under `firecrawl alexandria`. Provider IDs work directly; you do not need their category. Lists stay compact until you select a capability, which reveals its inputs, response, examples, and price. Browsable results include next commands; selecting a capability displays the final contract. Generated commands use `firecrawl`; when using `npx`, replace that prefix with `npx firecrawl-cli@alexandria`. +The category overview is available in `1.23.4-alexandria-beta.9` onward. `list` and `list-tools` are interchangeable, including under `firecrawl alexandria`. The root explains how to find and call tools and lists live category descriptions. Choose a category to see its providers, or jump directly to a provider. Selecting a capability reveals its inputs, response, examples, and price. Browsable results expose next commands in JSON; the text guide explains how to select each category. Selecting a capability displays the final contract. Generated commands use `firecrawl`; when using `npx`, replace that prefix with `npx firecrawl-cli@alexandria`. -Discovery is free and never executes the listed tools. Results come from the live Find Tools catalog through Scrape and respect your access. `--limit` controls page size (default 20, maximum 100); follow the returned `More` command for the next page. `--json` includes the API envelope, next commands, and receipts for every discovery request. +Discovery is free and never executes the listed tools. The root reads `GET /exchange/discover` on the configured Firecrawl API using your existing credentials; provider and tool lookups use the Find Tools meta tool through Scrape. Category membership and descriptions stay on the server. The root shows all returned categories; `--limit` controls provider/tool page size (default 20, maximum 100). Follow `More` to continue a page. Root `--json` exposes categories at `data.items`; provider/tool JSON keeps the Scrape envelope at `data.alexandria[0].data`. Both include request IDs and navigation where available. -Provider IDs take precedence over category IDs; use `--category` to select a category explicitly. Exact capability IDs take precedence over group IDs; use `--group` to select a group explicitly. Use `firecrawl search --sources alexandria` to find tools by task, or `firecrawl find-tools` for URL lookup and raw catalog selectors. +Provider IDs take precedence over category IDs; use `--category` to select a category explicitly. Display names such as `retail`, `developer`, and `public-records` also resolve to their catalog category IDs. Exact capability IDs take precedence over group IDs; use `--group` to select a group explicitly. Use `firecrawl search --sources alexandria` to find tools by task, or `firecrawl find-tools` for URL lookup and raw catalog selectors. ### Setup Skills, Workflows, and MCP diff --git a/beta-skills/firecrawl-alexandria/SKILL.md b/beta-skills/firecrawl-alexandria/SKILL.md index c82122aa77..854241071d 100644 --- a/beta-skills/firecrawl-alexandria/SKILL.md +++ b/beta-skills/firecrawl-alexandria/SKILL.md @@ -25,10 +25,10 @@ A complete contract returned by search needs no additional discovery call. Check ## Browse the catalog progressively -With beta `1.23.4-alexandria-beta.8` or newer, use `list` when the user wants to browse providers, categories, or a known provider's tools: +With beta `1.23.4-alexandria-beta.9` or newer, use `list` when the user wants to browse categories, providers, or a known provider's tools: ```sh -npx firecrawl-cli@alexandria list +npx firecrawl-cli@alexandria alexandria list npx firecrawl-cli@alexandria list finance npx firecrawl-cli@alexandria list benzinga npx firecrawl-cli@alexandria list benzinga --groups @@ -36,9 +36,9 @@ npx firecrawl-cli@alexandria list benzinga calendar --group npx firecrawl-cli@alexandria list benzinga --json ``` -The root lists visible providers. A category filters providers; a provider lists compact tools directly. Selecting a capability expands only that contract, including price, inputs, response, and examples. Categories are optional: use returned provider and capability IDs directly. `list-tools` is an alias for `list`; both also work under `alexandria`. `--category` and `--group` resolve ambiguous IDs explicitly. +The root shows an introduction, discovery/execution commands, and live categories with descriptions. A category lists its providers; a provider lists compact tools directly. Selecting a capability expands only that contract, including price, inputs, response, and examples. Categories are optional: use returned provider and capability IDs directly. Use `list --providers` only when a flat provider inventory is needed. `list-tools` is an alias for `list`; both also work under `alexandria`. `--category` and `--group` resolve ambiguous IDs explicitly. Category display names such as `retail`, `developer`, and `public-records` are accepted alongside the returned canonical IDs. -`list` only calls the free Find Tools meta tool through Scrape. `--json` preserves the API envelope and adds `discoveryRequests` receipts and `nextCommand` navigation. Page size defaults to 20 (`--limit 1–100`); follow the returned `More`/`nextCommand` only when needed. Generated commands start with `firecrawl`; replace that prefix with `npx firecrawl-cli@alexandria` to stay on this beta. Raw `--request` next requests preserve selectors and pagination and must not be mixed with a path or filters. +The root reads the free public `GET /exchange/discover` route on the configured Firecrawl API with existing credentials. Category rows are at `data.items` in JSON. Provider and tool lookups use the free Find Tools meta tool through Scrape, with rows at `data.alexandria[0].data.items`. Both include `discoveryRequests` and `nextCommand` navigation; no listed tool is executed. The root shows all categories. Provider/tool page size defaults to 20 (`--limit 1–100`); follow `More`/`nextCommand` only when needed. Generated commands start with `firecrawl`; replace that prefix with `npx firecrawl-cli@alexandria` to stay on this beta. Raw `--request` next requests preserve selectors and pagination and must not be mixed with a path or filters. ## Explicit requests for tools: search then the Find Tools meta tool diff --git a/package.json b/package.json index 530fb8c403..3b0d453160 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.23.4-alexandria-beta.8", + "version": "1.23.4-alexandria-beta.9", "publishConfig": { "tag": "alexandria" }, diff --git a/src/__tests__/alexandria-beta.test.ts b/src/__tests__/alexandria-beta.test.ts index ad25374c9a..d0aa03e1f6 100644 --- a/src/__tests__/alexandria-beta.test.ts +++ b/src/__tests__/alexandria-beta.test.ts @@ -91,16 +91,84 @@ function catalogue(level: string, items: any[], next?: unknown) { }; } +it('starts with a live category guide and preserves category-discovery access errors', async () => { + response = { + cohorts: [ + { + cohort: 'finance', + about: 'Live market data description.', + providers: 2, + }, + { + cohort: 'new-category', + about: 'A category added by the server.', + providers: 0, + }, + ], + }; + const guide = await cli(['alexandria', 'list']); + expect(guide.code).toBe(0); + expect(guide.stdout).toContain('Firecrawl Alexandria'); + expect(guide.stdout).toContain( + 'Finance (finance): Live market data description.' + ); + expect(guide.stdout).toContain('New Category (new-category)'); + expect(guide.stdout).not.toContain('Providers ('); + expect(requests[0]).toMatchObject({ + url: '/exchange/discover', + headers: { authorization: 'Bearer fc-test' }, + }); + expect(requests[0].body).toBeUndefined(); + const json = await cli(['list-tools', '--json']); + expect(json.code).toBe(0); + expect(JSON.parse(json.stdout).data).toMatchObject({ + level: 'categories', + total: 2, + items: [ + { + id: 'finance', + nextCommand: 'firecrawl alexandria list finance --category', + }, + { id: 'new-category' }, + ], + }); + status = 403; + response = { + success: false, + error: 'Access required', + code: 'ACCESS_REQUIRED', + requiresAction: { type: 'request_access' }, + }; + const denied = await cli(['list', '--json']); + expect(denied.code).toBe(1); + expect(JSON.parse(denied.stdout)).toMatchObject({ + ...response, + discoveryRequests: [{ requestId: expect.any(String) }], + }); + expect( + requests.every((request) => request.url === '/exchange/discover') + ).toBe(true); +}); + it('browses live provider IDs directly or through a category without expanding contracts', async () => { responseFor = (body) => { const options = body.alexandria[0].options; if (options.providers?.[0] === 'finance') return catalogue(options.level, []); + if ( + ['retail', 'Retail', 'Public records'].includes(options.categories?.[0]) + ) + return catalogue(options.level, []); return catalogue(options.level, [ { id: 'benzinga', provider: 'benzinga', name: 'Benzinga' }, ]); }; - for (const args of [[], ['finance'], ['benzinga'], ['finance', 'benzinga']]) { + for (const args of [ + ['--providers'], + ['finance'], + ['benzinga'], + ['finance', 'benzinga'], + ]) { expect((await cli(['list', ...args])).code).toBe(0); } expect((await cli(['alexandria', 'list', 'benzinga', '--groups'])).code).toBe( @@ -108,6 +176,8 @@ it('browses live provider IDs directly or through a category without expanding c ); expect((await cli(['list-tools', 'benzinga'])).code).toBe(0); expect((await cli(['alexandria', 'list-tools', 'benzinga'])).code).toBe(0); + expect((await cli(['list', 'Retail', '--category'])).code).toBe(0); + expect((await cli(['list', 'Public records', '--category'])).code).toBe(0); expect(requests.map((request) => request.body.alexandria[0].options)).toEqual( [ { level: 'providers', limit: 20 }, @@ -125,6 +195,10 @@ it('browses live provider IDs directly or through a category without expanding c { providers: ['benzinga'], level: 'groups', limit: 20 }, { providers: ['benzinga'], level: 'tools', limit: 20 }, { providers: ['benzinga'], level: 'tools', limit: 20 }, + { categories: ['Retail'], level: 'providers', limit: 20 }, + { categories: ['shopping'], level: 'providers', limit: 20 }, + { categories: ['Public records'], level: 'providers', limit: 20 }, + { categories: ['government'], level: 'providers', limit: 20 }, ] ); expect( @@ -135,6 +209,16 @@ it('browses live provider IDs directly or through a category without expanding c body.alexandria[0].capability === 'find-tools' ) ).toBe(true); + responseFor = () => + catalogue('providers', [ + { id: 'future-retailer', provider: 'future-retailer' }, + ]); + const canonical = await cli(['list', 'retail', '--category']); + expect(canonical.code).toBe(0); + expect(canonical.stdout).toContain('future-retailer'); + expect(requests.at(-1)?.body.alexandria[0].options.categories).toEqual([ + 'retail', + ]); }); it('expands only a selected tool and falls back to a compact group listing', async () => { @@ -254,7 +338,7 @@ it('refuses execution through list and propagates discovery access errors', asyn creditsCost: 0, }, }; - const result = await cli(['list', '--json']); + const result = await cli(['list', '--providers', '--json']); expect(result.code).toBe(1); expect(JSON.parse(result.stdout).data.alexandria[0].error).toEqual( response.data.alexandria[0].error diff --git a/src/commands/list.ts b/src/commands/list.ts index e84892bbef..d46df267bd 100644 --- a/src/commands/list.ts +++ b/src/commands/list.ts @@ -1,11 +1,14 @@ import { Command, InvalidArgumentError } from 'commander'; +import { randomUUID } from 'node:crypto'; import { apiFailure, parseFindToolsRequest, requestAlexandria, + requireAlexandriaKey, type AlexandriaOptions, } from './alexandria'; import { writeOutput } from '../utils/output'; +import { getApiKey, getConfig } from '../utils/config'; type Selectors = Record; type ListOptions = AlexandriaOptions & { @@ -14,6 +17,30 @@ type ListOptions = AlexandriaOptions & { group?: boolean; limit?: number; request?: string; + providers?: boolean; +}; +const CATEGORY_NAMES: Record = { + software: 'Developer', + government: 'Public records', + shopping: 'Retail', + restaurants: 'Restaurant', + companies: 'Company', + skills: 'Tools', +}; +function categoryId(id: string): string { + const normalized = id.trim().toLowerCase().replace(/\s+/g, '-'); + return ( + Object.entries(CATEGORY_NAMES).find( + ([, name]) => name.toLowerCase().replaceAll(' ', '-') === normalized + )?.[0] ?? normalized + ); +} + +type Category = { + id: string; + name: string; + description: string; + nextCommand: string; }; type Item = { id: string; @@ -53,6 +80,109 @@ function quote(value: string): string { : `'${value.replaceAll("'", "'\\''")}'`; } +async function requestCategories( + options: ListOptions +): Promise> { + requireAlexandriaKey(options.apiKey); + const requestId = randomUUID(); + console.error(`Request ID: ${requestId}`); + const base = ( + options.apiUrl || + getConfig().apiUrl || + 'https://api.firecrawl.dev' + ).replace(/\/$/, ''); + try { + const response = await fetch(`${base}/exchange/discover`, { + headers: { + Authorization: `Bearer ${getApiKey(options.apiKey)}`, + 'X-Request-ID': requestId, + }, + signal: AbortSignal.timeout(getConfig().timeoutMs ?? 30000), + redirect: 'error', + }); + const body: any = await response.json().catch(() => ({})); + if (!response.ok || body?.success === false) { + return { + ...apiFailure({ + response: { + data: { + ...body, + error: + body?.error || + `Category discovery failed (HTTP ${response.status}).`, + }, + }, + }), + requestId, + }; + } + if ( + !Array.isArray(body?.cohorts) || + body.cohorts.some( + (row: any) => + !row || + typeof row.cohort !== 'string' || + !row.cohort.trim() || + typeof row.about !== 'string' || + !Number.isInteger(row.providers) || + row.providers < 0 + ) + ) + throw new Error('Discovery returned an invalid category index.'); + const items: Category[] = body.cohorts + .map((row: any) => ({ + id: row.cohort, + name: Object.hasOwn(CATEGORY_NAMES, row.cohort) + ? CATEGORY_NAMES[row.cohort] + : row.cohort + .split('-') + .map( + (word: string) => word.charAt(0).toUpperCase() + word.slice(1) + ) + .join(' '), + description: row.about, + nextCommand: `firecrawl alexandria list ${quote(row.cohort)} --category${options.limit === undefined ? '' : ` --limit ${options.limit}`}`, + })) + .sort((a: Category, b: Category) => a.name.localeCompare(b.name)); + return { + success: true, + requestId, + data: { level: 'categories', items, total: items.length }, + }; + } catch (error) { + return { ...apiFailure(error), requestId }; + } +} + +function renderCategories(items: Category[]): string { + return [ + 'Firecrawl Alexandria', + 'A trusted data layer for agents to access high-provenance data.', + '', + 'Finding data', + ' firecrawl search "" --sources alexandria', + ' Browse a category below, or jump directly to a provider.', + '', + 'Calling it', + ' Browse: firecrawl alexandria list --category', + ' Tools: firecrawl alexandria list ', + ' Inspect: firecrawl alexandria list ', + " Execute: firecrawl scrape --alexandria / --options ''", + ' Catalog browsing is free. Check the contract and price before executing.', + ' Use your existing Firecrawl login or FIRECRAWL_API_KEY.', + '', + `Categories (${items.length})`, + ...items.map((item) => ` ${item.name} (${item.id}): ${item.description}`), + ...(!items.length ? [' No categories are currently visible.'] : []), + '', + 'Developer and Research indexes have native commands:', + ' firecrawl developer --help', + ' firecrawl research --help', + '', + 'All providers: firecrawl alexandria list --providers', + ].join('\n'); +} + function nextCommand(request: unknown): string { const call = parseFindToolsRequest(JSON.stringify(request)); return `firecrawl list --request ${quote(JSON.stringify(call))}`; @@ -163,6 +293,7 @@ export async function handleList( options.category || options.groups || options.group || + options.providers || options.limit !== undefined) ) throw new Error( @@ -176,6 +307,30 @@ export async function handleList( ); if ((options.category || options.groups) && !path.length) throw new Error('Provide a category or provider ID.'); + if ( + options.providers && + (path.length || options.category || options.groups || options.group) + ) + throw new Error( + '--providers lists all providers; omit the path and other selectors.' + ); + if (!path.length && !options.request && !options.providers) { + const result = await requestCategories(options); + receipts.push({ requestId: result.requestId }); + if (!result.success) throw new DiscoveryFailure(result); + writeOutput( + asJson + ? JSON.stringify( + { ...result, discoveryRequests: receipts }, + null, + options.pretty ? 2 : undefined + ) + : renderCategories(result.data.items), + options.output, + !!options.output + ); + return; + } const limit = options.limit ?? 20; async function fetchPage(selectors: Selectors) { @@ -231,6 +386,10 @@ export async function handleList( if (!result?.page.total) { scope = { categories: [path[0]] }; result = await fetchPage({ ...scope, level: 'providers', limit }); + if (!result.page.total && categoryId(path[0]) !== path[0]) { + scope = { categories: [categoryId(path[0])] }; + result = await fetchPage({ ...scope, level: 'providers', limit }); + } if (!result.page.total || !remaining.length) return result; scope.providers = [remaining[0]]; remaining = remaining.slice(1); @@ -299,7 +458,7 @@ export function createListCommand(): Command { return new Command('list') .alias('list-tools') .description( - 'Browse providers, categories and tool contracts through Find Tools; discovery only' + 'Start with the Alexandria category index, then browse providers and tool contracts; discovery only' ) .argument( '[path...]', @@ -310,13 +469,14 @@ export function createListCommand(): Command { 'Treat the first ID as a category when a provider has the same ID' ) .option('--groups', 'List a provider’s tool groups instead of its tools') + .option('--providers', 'List all providers instead of the category index') .option( '--group', 'Treat the final path as a group instead of a capability' ) .option( '--limit ', - 'Results per page (1-100; default: 20)', + 'Provider/tool results per page (1-100; default: 20); the root shows all categories', (raw) => { const value = Number(raw); if (!Number.isInteger(value) || value < 1 || value > 100) @@ -335,7 +495,7 @@ export function createListCommand(): Command { .option('--pretty', 'Format JSON') .addHelpText( 'after', - '\nExamples:\n firecrawl list\n firecrawl list finance\n firecrawl list benzinga\n firecrawl list benzinga --groups\n firecrawl list benzinga calendar --group\n firecrawl list benzinga --json\n\nProvider IDs take precedence over categories; exact capabilities take precedence over groups.\nNo listed tool is executed. Search by task with firecrawl search --sources alexandria.\n' + '\nExamples:\n firecrawl alexandria list\n firecrawl list --providers\n firecrawl list finance\n firecrawl list benzinga\n firecrawl list benzinga --groups\n firecrawl list benzinga calendar --group\n firecrawl list benzinga --json\n\nProvider IDs take precedence over categories; exact capabilities take precedence over groups.\nNo listed tool is executed. Search by task with firecrawl search --sources alexandria.\n' ) .action(handleList); }