From 2e9cadc8833c5220ad9ace69a38f81bf02dae6c9 Mon Sep 17 00:00:00 2001 From: David Tippett Date: Thu, 23 Jul 2026 20:26:08 -0400 Subject: [PATCH] Add --valuable-result-positions to search feedback Wire the API's new valuableResultPositions field through both feedback commands so agents can attribute usefulness to specific data.web results by 1-indexed position. Skills now surface positions in jq extraction snippets and require exhaustive position marking (unlisted results count as not useful), with valuableSources reserved for URLs outside data.web. Requires an API with position-based search feedback support (firecrawl/firecrawl#4109); the field is only sent when the flag is provided. Co-Authored-By: Claude Fable 5 --- README.md | 29 +++++++++++++++-------------- skills/firecrawl-cli/SKILL.md | 16 +++++++--------- skills/firecrawl-search/SKILL.md | 11 ++++++----- src/commands/feedback.ts | 27 +++++++++++++++++++++++---- src/commands/search-feedback.ts | 7 +++++++ src/index.ts | 30 +++++++++++++++++++++++++++++- 6 files changed, 87 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 4c2c637570..654219b914 100644 --- a/README.md +++ b/README.md @@ -370,20 +370,21 @@ endpoint feedback calls silently. #### Feedback Options -| Option | Description | -| -------------------------------- | -------------------------------------------- | -| `--rating ` | Required: `good`, `partial`, or `bad` | -| `--issues ` | Comma-separated issue codes or JSON array | -| `--tags ` | Comma-separated tags or JSON array | -| `--note ` | Short human-readable feedback | -| `--valuable-sources ` | JSON array of `{url, reason}` entries | -| `--missing-content ` | JSON array of `{topic, description}` entries | -| `--query-suggestions ` | Search/query improvement notes | -| `--url ` | Relevant URL for scrape or parse feedback | -| `--page-numbers ` | Comma-separated page numbers or JSON array | -| `--metadata ` | Small JSON object with extra context | -| `--metadata-file ` | Path to small metadata JSON object | -| `--silent` | Suppress output for background agent calls | +| Option | Description | +| --------------------------------------------- | ------------------------------------------------------------------ | +| `--rating ` | Required: `good`, `partial`, or `bad` | +| `--issues ` | Comma-separated issue codes or JSON array | +| `--tags ` | Comma-separated tags or JSON array | +| `--note ` | Short human-readable feedback | +| `--valuable-sources ` | JSON array of `{url, reason}` entries | +| `--valuable-result-positions ` | Search only: 1-indexed `data.web` positions of every useful result | +| `--missing-content ` | JSON array of `{topic, description}` entries | +| `--query-suggestions ` | Search/query improvement notes | +| `--url ` | Relevant URL for scrape or parse feedback | +| `--page-numbers ` | Comma-separated page numbers or JSON array | +| `--metadata ` | Small JSON object with extra context | +| `--metadata-file ` | Path to small metadata JSON object | +| `--silent` | Suppress output for background agent calls | --- diff --git a/skills/firecrawl-cli/SKILL.md b/skills/firecrawl-cli/SKILL.md index 5494c1e3cb..ae17fce5f5 100644 --- a/skills/firecrawl-cli/SKILL.md +++ b/skills/firecrawl-cli/SKILL.md @@ -193,9 +193,7 @@ The `check` response then carries a per-field diff (paths like `plans[0].price`) }, "snapshot": { "json": { - "plans": [ - /* current full extraction */ - ] + "plans": [/* current full extraction */] } } } @@ -255,11 +253,11 @@ Single format outputs raw content. Multiple formats (e.g., `--format markdown,li These patterns are useful when working with file-based output (`-o` flag) for complex tasks: ```bash -# Extract URLs from search -jq -r '.data.web[].url' .firecrawl/search.json +# Extract URLs from search with their 1-indexed positions (needed for feedback) +jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json -# Get titles and URLs -jq -r '.data.web[] | "\(.title): \(.url)"' .firecrawl/search.json +# Get positions, titles, and URLs +jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.title): \(.value.url)"' .firecrawl/search.json ``` ## After search: send feedback (refunds 1 credit) @@ -271,13 +269,13 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Authoritative"}]' \ + --valuable-result-positions "1,3" \ --missing-content '[{"topic":"useDeferredValue example"},{"topic":"Server Components hooks"}]' \ --query-suggestions "Boost react.dev for react-hooks queries" \ --silent & ``` -The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. +The most useful field is `--missing-content`: an _array_ of specific pieces of content you expected to find but didn't. Use one entry per missing topic. Bad/partial feedback with detailed `--missing-content` is just as valuable as good feedback. Mark useful results with `--valuable-result-positions` (1-indexed into `data.web`, list **every** useful one — unlisted results count as not useful); reserve `--valuable-sources` for useful URLs that were not in `data.web`. **Opt out:** `export FIRECRAWL_NO_SEARCH_FEEDBACK=1` makes the CLI skip every feedback call silently. Respect that flag — do not try to work around it. See [firecrawl-search](../firecrawl-search/SKILL.md) for the full pattern. diff --git a/skills/firecrawl-search/SKILL.md b/skills/firecrawl-search/SKILL.md index 87b426cf92..d8437f5c90 100644 --- a/skills/firecrawl-search/SKILL.md +++ b/skills/firecrawl-search/SKILL.md @@ -49,7 +49,7 @@ firecrawl search "your query" --sources news --tbs qdr:d -o .firecrawl/news.json - **`--scrape` fetches full content** — don't re-scrape URLs from search results. This saves credits and avoids redundant fetches. - Always write results to `.firecrawl/` with `-o` to avoid context window bloat. -- Use `jq` to extract URLs or titles: `jq -r '.data.web[].url' .firecrawl/search.json` +- Use `jq` to extract URLs or titles **with their 1-indexed positions** (you'll need the positions for feedback): `jq -r '.data.web | to_entries[] | "\(.key + 1)\t\(.value.url)"' .firecrawl/search.json` - Naming convention: `.firecrawl/search-{query}.json` or `.firecrawl/search-{query}-scraped.json` ## Send feedback after every search (refunds 1 credit) @@ -62,9 +62,10 @@ Search costs 2 credits. After you've actually used the results (or decided they - **Time window:** must be sent within ~2 minutes of the search. Late feedback is rejected. - **`--missing-content` is the most important field.** It's a list of _specific pieces_ of content you expected but did not find. One topic per entry — do not pack multiple topics into one string. These aggregate across teams and tell us what to index next. +- **`--valuable-result-positions` marks which results were useful.** 1-indexed positions into `data.web` (web results only). **Be exhaustive** — list every result that was actually useful; unlisted results are treated as not useful, so a partial list corrupts the signal. Reserve `--valuable-sources` for useful URLs that were NOT in `data.web` (e.g. a page you found by following a result's link) — never report the same result in both. - **Substantive content required** (zero-effort feedback is rejected with HTTP 400): - - `good` → must include at least one `--valuable-sources` entry. - - `partial` → must include `--valuable-sources` or `--missing-content`. + - `good` → must include `--valuable-result-positions` or at least one `--valuable-sources` entry. + - `partial` → must include `--valuable-result-positions`, `--valuable-sources`, or `--missing-content`. - `bad` → must include `--missing-content` or `--query-suggestions`. - **Daily refund cap (per team, per UTC day, default 100 credits).** Once your team has been refunded 100 credits today, further submissions still record feedback but no longer refund credits. The response includes `creditsRefundedToday` / `dailyRefundCap` / `dailyCapReached`. **When `dailyCapReached: true`, stop calling `search-feedback` for the rest of the UTC day** — it won't refund anything and you're wasting bandwidth. - **Idempotent:** re-submitting for the same search id returns success but no extra refund. @@ -79,10 +80,10 @@ SEARCH_ID=$(jq -r '.id' .firecrawl/search-react-hooks.json) Then send feedback. Pick the rating that matches what actually happened: ```bash -# Results were useful, with notes on what was still missing +# Results were useful — positions 1 and 3 of data.web answered the question firecrawl search-feedback "$SEARCH_ID" \ --rating good \ - --valuable-sources '[{"url":"https://react.dev/reference/react/hooks","reason":"Most authoritative"}]' \ + --valuable-result-positions "1,3" \ --missing-content '[ {"topic":"useDeferredValue","description":"No example of useDeferredValue with Suspense"}, {"topic":"useTransition","description":"No coverage of useTransition for routing"} diff --git a/src/commands/feedback.ts b/src/commands/feedback.ts index 14318a8c9e..e14e643672 100644 --- a/src/commands/feedback.ts +++ b/src/commands/feedback.ts @@ -20,6 +20,7 @@ export interface EndpointFeedbackOptions { tags?: string[]; note?: string; valuableSources?: ValuableSourceInput[]; + valuableResultPositions?: number[]; missingContent?: MissingContentInput[]; querySuggestions?: string; url?: string; @@ -111,8 +112,9 @@ export function parseFeedbackListArg( return normalizeList(trimmed.split(',')); } -export function parsePageNumbersArg( - raw: string | undefined +function parsePositiveIntArrayArg( + raw: string | undefined, + flag: string ): number[] | undefined { if (!raw) return undefined; const trimmed = raw.trim(); @@ -123,12 +125,12 @@ export function parsePageNumbersArg( try { const parsed = JSON.parse(trimmed); if (!Array.isArray(parsed)) { - throw new Error('--page-numbers must be a JSON array.'); + throw new Error(`${flag} must be a JSON array.`); } values = parsed; } catch { throw new Error( - '--page-numbers must be a comma-separated list or valid JSON array.' + `${flag} must be a comma-separated list or valid JSON array.` ); } } else { @@ -144,6 +146,18 @@ export function parsePageNumbersArg( return numbers.length > 0 ? numbers : undefined; } +export function parsePageNumbersArg( + raw: string | undefined +): number[] | undefined { + return parsePositiveIntArrayArg(raw, '--page-numbers'); +} + +export function parseValuableResultPositionsArg( + raw: string | undefined +): number[] | undefined { + return parsePositiveIntArrayArg(raw, '--valuable-result-positions'); +} + export function parseMetadataArg( raw: string | undefined, filePath: string | undefined @@ -207,6 +221,7 @@ export function parseEndpointFeedbackCliOptions(options: { metadata?: string; metadataFile?: string; valuableSources?: string; + valuableResultPositions?: string; missingContent?: string | string[]; rating?: string; }) { @@ -217,6 +232,9 @@ export function parseEndpointFeedbackCliOptions(options: { pageNumbers: parsePageNumbersArg(options.pageNumbers), metadata: parseMetadataArg(options.metadata, options.metadataFile), valuableSources: parseValuableSourcesArg(options.valuableSources), + valuableResultPositions: parseValuableResultPositionsArg( + options.valuableResultPositions + ), missingContent: parseMissingContentArg(options.missingContent), }; } @@ -261,6 +279,7 @@ export async function executeEndpointFeedback( ['tags', normalizeList(options.tags)], ['note', options.note], ['valuableSources', options.valuableSources], + ['valuableResultPositions', options.valuableResultPositions], ['missingContent', options.missingContent], ['querySuggestions', options.querySuggestions], ['url', options.url], diff --git a/src/commands/search-feedback.ts b/src/commands/search-feedback.ts index 35898e78b4..e5c9d46d2a 100644 --- a/src/commands/search-feedback.ts +++ b/src/commands/search-feedback.ts @@ -17,6 +17,7 @@ export interface SearchFeedbackOptions { searchId: string; rating: SearchFeedbackRating; valuableSources?: ValuableSourceInput[]; + valuableResultPositions?: number[]; missingContent?: MissingContentInput[]; querySuggestions?: string; apiKey?: string; @@ -118,6 +119,12 @@ export async function executeSearchFeedback( ...(s.reason ? { reason: s.reason } : {}), })); } + if ( + options.valuableResultPositions && + options.valuableResultPositions.length > 0 + ) { + body.valuableResultPositions = options.valuableResultPositions; + } if (options.missingContent && options.missingContent.length > 0) { body.missingContent = options.missingContent .filter((m) => !!m.topic) diff --git a/src/index.ts b/src/index.ts index 5c1ae11f3f..a5069533d8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -37,6 +37,7 @@ import { handleEndpointFeedbackCommand, parseEndpointFeedbackCliOptions, parseEndpointFeedbackEndpoint, + parseValuableResultPositionsArg, } from './commands/feedback'; import { handleAgentCommand } from './commands/agent'; import { @@ -1294,9 +1295,16 @@ function createSearchFeedbackCommand(): Command { ) .argument('', 'The id returned by `firecrawl search ... --json`') .requiredOption('--rating ', 'Overall rating: good | bad | partial') + .option( + '--valuable-result-positions ', + '1-indexed positions in data.web of every result that was useful ' + + '(e.g. "1,3" or [1,3]). Unlisted results are treated as not useful.' + ) .option( '--valuable-sources ', - 'Comma-separated URLs OR JSON array of {url, reason} entries' + 'Comma-separated URLs OR JSON array of {url, reason} entries. ' + + 'For useful URLs NOT in data.web; use --valuable-result-positions ' + + 'for returned web results.' ) .option( '--missing-content ', @@ -1337,6 +1345,19 @@ function createSearchFeedbackCommand(): Command { process.exit(1); } + let valuableResultPositions; + try { + valuableResultPositions = parseValuableResultPositionsArg( + options.valuableResultPositions + ); + } catch (error: any) { + console.error( + 'Error:', + error?.message || 'Invalid --valuable-result-positions' + ); + process.exit(1); + } + let missingContent; try { missingContent = parseMissingContentArg(options.missingContent); @@ -1349,6 +1370,7 @@ function createSearchFeedbackCommand(): Command { searchId, rating: rating as SearchFeedbackRating, valuableSources, + valuableResultPositions, missingContent, querySuggestions: options.querySuggestions, apiKey: options.apiKey, @@ -1385,6 +1407,11 @@ function createFeedbackCommand(): Command { '--valuable-sources ', 'Comma-separated URLs OR JSON array of {url, reason} entries' ) + .option( + '--valuable-result-positions ', + 'Search only: 1-indexed positions in data.web of every useful result ' + + '(e.g. "1,3" or [1,3])' + ) .option( '--missing-content ', 'Specific pieces of content missing from results. ' + @@ -1440,6 +1467,7 @@ function createFeedbackCommand(): Command { tags: parsed.tags, note: options.note, valuableSources: parsed.valuableSources, + valuableResultPositions: parsed.valuableResultPositions, missingContent: parsed.missingContent, querySuggestions: options.querySuggestions, url: options.url,