diff --git a/README.md b/README.md index 4c2c637570..654219b914 100644 --- a/README.md +++ b/README.md @@ -370,20 +370,21 @@ endpoint feedback calls silently. #### Feedback Options -| Option | Description | -| -------------------------------- | -------------------------------------------- | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| --------------------------------------------- | ------------------------------------------------------------------ | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--valuable-result-positions ` | Search only: 1-indexed `data.web` positions of every useful result | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index 5494c1e3cb..ae17fce5f5 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -193,9 +193,7 @@ The `check` response then carries a per-field diff (paths like `plans[0].price`) }, "snapshot": { "json": { - "plans": [ - /* current full extraction */ - ] + "plans": [/* current full extraction */] } } } @@ -255,11 +253,11 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li These patterns are useful when working with file-based output (`-o` flag) for complex tasks: ```bash -# Extract URLs from search -jq -r '.data.web[].url' .firecrawl/search.json +# Extract URLs from search with their 1-indexed positions (needed for feedback) +jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json -# Get titles and URLs -jq -r '.data.web[] | "\(.title): \(.url)"' .firecrawl/search.json +# Get positions, titles, and URLs +jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.title): \(.value.url)"' .firecrawl/search.json ``` ## After search: send feedback (refunds 1 credit) @@ -271,13 +269,13 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Authoritative"}]' \ + --valuable-result-positions "1,3" \ --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ --query-suggestions "Boost react.dev for react-hooks queries" \ --silent & ``` -The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. +The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. Mark useful results with `--valuable-result-positions` (1-indexed into `data.web`, list **every** useful one — unlisted results count as not useful); reserve `--valuable-sources` for useful URLs that were not in `data.web`. **Opt out:** `export FIRECRAWL_NO_SEARCH_FEEDBACK=1` makes the CLI skip every feedback call silently. Respect that flag — do not try to work around it. See [firecrawl-search](../firecrawl-search/SKILL.md) for the full pattern. diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 87b426cf92..d8437f5c90 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -49,7 +49,7 @@ firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json - **`--scrape` fetches full content** — don't re-scrape URLs from search results. This saves credits and avoids redundant fetches. - Always write results to `.firecrawl/` with `-o` to avoid context window bloat. -- Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` +- Use `jq` to extract URLs or titles **with their 1-indexed positions** (you'll need the positions for feedback): `jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` ## Send feedback after every search (refunds 1 credit) @@ -62,9 +62,10 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Time window:** must be sent within ~2 minutes of the search. Late feedback is rejected. - **`--missing-content` is the most important field.** It's a list of _specific pieces_ of content you expected but did not find. One topic per entry — do not pack multiple topics into one string. These aggregate across teams and tell us what to index next. +- **`--valuable-result-positions` marks which results were useful.** 1-indexed positions into `data.web` (web results only). **Be exhaustive** — list every result that was actually useful; unlisted results are treated as not useful, so a partial list corrupts the signal. Reserve `--valuable-sources` for useful URLs that were NOT in `data.web` (e.g. a page you found by following a result's link) — never report the same result in both. - **Substantive content required** (zero-effort feedback is rejected with HTTP 400): - - `good` → must include at least one `--valuable-sources` entry. - - `partial` → must include `--valuable-sources` or `--missing-content`. + - `good` → must include `--valuable-result-positions` or at least one `--valuable-sources` entry. + - `partial` → must include `--valuable-result-positions`, `--valuable-sources`, or `--missing-content`. - `bad` → must include `--missing-content` or `--query-suggestions`. - **Daily refund cap (per team, per UTC day, default 100 credits).** Once your team has been refunded 100 credits today, further submissions still record feedback but no longer refund credits. The response includes `creditsRefundedToday` / `dailyRefundCap` / `dailyCapReached`. **When `dailyCapReached: true`, stop calling `search-feedback` for the rest of the UTC day** — it won't refund anything and you're wasting bandwidth. - **Idempotent:** re-submitting for the same search id returns success but no extra refund. @@ -79,10 +80,10 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) Then send feedback. Pick the rating that matches what actually happened: ```bash -# Results were useful, with notes on what was still missing +# Results were useful — positions 1 and 3 of data.web answered the question firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ + --valuable-result-positions "1,3" \ --missing-content '[ {"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}, {"topic":"useTransition","description":"No coverage of useTransition for routing"} diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 14318a8c9e..e14e643672 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -20,6 +20,7 @@ export interface EndpointFeedbackOptions { tags?: string[]; note?: string; valuableSources?: ValuableSourceInput[]; + valuableResultPositions?: number[]; missingContent?: MissingContentInput[]; querySuggestions?: string; url?: string; @@ -111,8 +112,9 @@ export function parseFeedbackListArg( return normalizeList(trimmed.split(',')); } -export function parsePageNumbersArg( - raw: string | undefined +function parsePositiveIntArrayArg( + raw: string | undefined, + flag: string ): number[] | undefined { if (!raw) return undefined; const trimmed = raw.trim(); @@ -123,12 +125,12 @@ export function parsePageNumbersArg( try { const parsed = JSON.parse(trimmed); if (!Array.isArray(parsed)) { - throw new Error('--page-numbers must be a JSON array.'); + throw new Error(`${flag} must be a JSON array.`); } values = parsed; } catch { throw new Error( - '--page-numbers must be a comma-separated list or valid JSON array.' + `${flag} must be a comma-separated list or valid JSON array.` ); } } else { @@ -144,6 +146,18 @@ export function parsePageNumbersArg( return numbers.length > 0 ? numbers : undefined; } +export function parsePageNumbersArg( + raw: string | undefined +): number[] | undefined { + return parsePositiveIntArrayArg(raw, '--page-numbers'); +} + +export function parseValuableResultPositionsArg( + raw: string | undefined +): number[] | undefined { + return parsePositiveIntArrayArg(raw, '--valuable-result-positions'); +} + export function parseMetadataArg( raw: string | undefined, filePath: string | undefined @@ -207,6 +221,7 @@ export function parseEndpointFeedbackCliOptions(options: { metadata?: string; metadataFile?: string; valuableSources?: string; + valuableResultPositions?: string; missingContent?: string | string[]; rating?: string; }) { @@ -217,6 +232,9 @@ export function parseEndpointFeedbackCliOptions(options: { pageNumbers: parsePageNumbersArg(options.pageNumbers), metadata: parseMetadataArg(options.metadata, options.metadataFile), valuableSources: parseValuableSourcesArg(options.valuableSources), + valuableResultPositions: parseValuableResultPositionsArg( + options.valuableResultPositions + ), missingContent: parseMissingContentArg(options.missingContent), }; } @@ -261,6 +279,7 @@ export async function executeEndpointFeedback( ['tags', normalizeList(options.tags)], ['note', options.note], ['valuableSources', options.valuableSources], + ['valuableResultPositions', options.valuableResultPositions], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], ['url', options.url], diff --git a/src/commands/search-feedback.ts b/src/commands/search-feedback.ts index 35898e78b4..e5c9d46d2a 100644 --- a/src/commands/search-feedback.ts +++ b/src/commands/search-feedback.ts @@ -17,6 +17,7 @@ export interface SearchFeedbackOptions { searchId: string; rating: SearchFeedbackRating; valuableSources?: ValuableSourceInput[]; + valuableResultPositions?: number[]; missingContent?: MissingContentInput[]; querySuggestions?: string; apiKey?: string; @@ -118,6 +119,12 @@ export async function executeSearchFeedback( ...(s.reason ? { reason: s.reason } : {}), })); } + if ( + options.valuableResultPositions && + options.valuableResultPositions.length > 0 + ) { + body.valuableResultPositions = options.valuableResultPositions; + } if (options.missingContent && options.missingContent.length > 0) { body.missingContent = options.missingContent .filter((m) => !!m.topic) diff --git a/src/index.ts b/src/index.ts index 5c1ae11f3f..a5069533d8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,6 +37,7 @@ import { handleEndpointFeedbackCommand, parseEndpointFeedbackCliOptions, parseEndpointFeedbackEndpoint, + parseValuableResultPositionsArg, } from './commands/feedback'; import { handleAgentCommand } from './commands/agent'; import { @@ -1294,9 +1295,16 @@ function createSearchFeedbackCommand(): Command { ) .argument('', 'The id returned by `firecrawl search ... --json`') .requiredOption('--rating ', 'Overall rating: good | bad | partial') + .option( + '--valuable-result-positions ', + '1-indexed positions in data.web of every result that was useful ' + + '(e.g. "1,3" or [1,3]). Unlisted results are treated as not useful.' + ) .option( '--valuable-sources ', - 'Comma-separated URLs OR JSON array of {url, reason} entries' + 'Comma-separated URLs OR JSON array of {url, reason} entries. ' + + 'For useful URLs NOT in data.web; use --valuable-result-positions ' + + 'for returned web results.' ) .option( '--missing-content ', @@ -1337,6 +1345,19 @@ function createSearchFeedbackCommand(): Command { process.exit(1); } + let valuableResultPositions; + try { + valuableResultPositions = parseValuableResultPositionsArg( + options.valuableResultPositions + ); + } catch (error: any) { + console.error( + 'Error:', + error?.message || 'Invalid --valuable-result-positions' + ); + process.exit(1); + } + let missingContent; try { missingContent = parseMissingContentArg(options.missingContent); @@ -1349,6 +1370,7 @@ function createSearchFeedbackCommand(): Command { searchId, rating: rating as SearchFeedbackRating, valuableSources, + valuableResultPositions, missingContent, querySuggestions: options.querySuggestions, apiKey: options.apiKey, @@ -1385,6 +1407,11 @@ function createFeedbackCommand(): Command { '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries' ) + .option( + '--valuable-result-positions ', + 'Search only: 1-indexed positions in data.web of every useful result ' + + '(e.g. "1,3" or [1,3])' + ) .option( '--missing-content ', 'Specific pieces of content missing from results. ' + @@ -1440,6 +1467,7 @@ function createFeedbackCommand(): Command { tags: parsed.tags, note: options.note, valuableSources: parsed.valuableSources, + valuableResultPositions: parsed.valuableResultPositions, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, url: options.url,