Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ firecrawl scrape https://firecrawl.dev https://firecrawl.dev/blog https://docs.f
| `--exclude-tags <tags>` | Exclude specific HTML tags |
| `--max-age <milliseconds>` | Maximum age of cached content in milliseconds |
| `--lockdown` | Enable lockdown mode for the scrape |
| `--redact-pii` | Redact personally identifiable information from output |
| `--schema <json>` | JSON schema for structured extraction |
| `--schema-file <path>` | Path to JSON schema file for structured extraction |
| `--actions <json>` | JSON actions array to run during scrape |
Expand Down
2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "firecrawl-cli",
"version": "1.18.5",
"version": "1.18.6",
"description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.",
"main": "dist/index.js",
"bin": {
Expand Down
1 change: 1 addition & 0 deletions skills/firecrawl-scrape/SKILL.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ firecrawl scrape "https://example.com/pricing" --query "What is the enterprise p
| `--wait-for <ms>` | Wait for JS rendering before scraping |
| `--include-tags <tags>` | Only include these HTML tags |
| `--exclude-tags <tags>` | Exclude these HTML tags |
| `--redact-pii` | Redact personally identifiable information from output |
| `-o, --output <path>` | Output file path |

## Tips
Expand Down
16 changes: 16 additions & 0 deletions src/__tests__/commands/scrape.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,22 @@ describe('executeScrape', () => {
});
});

it('should include redactPII when provided', async () => {
const mockResponse = { markdown: '# Test' };
mockClient.scrape.mockResolvedValue(mockResponse);

await executeScrape({
url: 'https://example.com',
redactPII: true,
});

expect(mockClient.scrape).toHaveBeenCalledWith('https://example.com', {
formats: ['markdown'],
integration: 'cli',
redactPII: true,
});
});

it('should not include location parameter when not provided', async () => {
const mockResponse = { markdown: '# Test' };
mockClient.scrape.mockResolvedValue(mockResponse);
Expand Down
13 changes: 13 additions & 0 deletions src/__tests__/utils/options.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,17 @@ describe('Option Parsing Utilities', () => {
expect(result.timing).toBe(true);
});

it('should parse redactPii option from the CLI flag', () => {
const options = {
url: 'https://example.com',
redactPii: true,
};

const result = parseScrapeOptions(options);

expect(result.redactPII).toBe(true);
});

it('should handle undefined format', () => {
const options = {
url: 'https://example.com',
Expand All @@ -328,6 +339,7 @@ describe('Option Parsing Utilities', () => {
output: '.firecrawl/output.json',
pretty: true,
timing: true,
redactPii: true,
};

const result = parseScrapeOptions(options);
Expand All @@ -345,6 +357,7 @@ describe('Option Parsing Utilities', () => {
output: '.firecrawl/output.json',
pretty: true,
timing: true,
redactPII: true,
});
});

Expand Down
4 changes: 4 additions & 0 deletions src/commands/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ export async function executeScrape(
scrapeParams.lockdown = true;
}

if (options.redactPII) {
scrapeParams.redactPII = true;
}

// Execute scrape with timing - only wrap the scrape call in try-catch
const requestStartTime = Date.now();

Expand Down
5 changes: 5 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,11 @@ function createScrapeCommand(): Command {
'Load existing profile data without saving changes (default: saves changes)'
)
.option('--lockdown', 'Enable lockdown mode for the scrape', false)
.option(
'--redact-pii',
'Redact personally identifiable information from returned content',
false
)
.option('--schema <json>', 'JSON schema for structured extraction')
.option('--schema-file <path>', 'Path to JSON schema file')
.option('--actions <json>', 'JSON actions array to run during scrape')
Expand Down
2 changes: 2 additions & 0 deletions src/types/scrape.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ export interface ScrapeOptions {
};
/** Enable lockdown mode for the scrape */
lockdown?: boolean;
/** Redact personally identifiable information from returned content */
redactPII?: boolean;
}

export interface ScrapeResult {
Expand Down
1 change: 1 addition & 0 deletions src/utils/options.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,5 +122,6 @@ export function parseScrapeOptions(options: any): ScrapeOptions {
query: options.query,
profile,
lockdown: options.lockdown,
redactPII: options.redactPii ?? options.redactPII,
};
}
Loading