From 783d96b838d64e7ec1710173a4bc55677f7b6347 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Wed, 29 Jul 2026 13:23:02 +0200 Subject: [PATCH 1/6] feat(search): add contentTypes filter Search now accepts contentTypes so callers can pick which content types get fetched. Adds application/json and text/markdown to fetchContentTypeSchema to match the API enum. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 11 +++++++++++ src/schemas.ts | 8 +++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 37b1dad..514f19d 100644 --- a/README.md +++ b/README.md @@ -155,9 +155,20 @@ const res = await sgai.search({ timeRange: "past_week", // optional locationGeoCode: "us", // optional fetchConfig: { /* ... */ }, // optional + contentTypes: [ // optional, defaults to text-like types + "text/html", + "application/json", + "text/markdown", + "text/plain", + ], }); ``` +By default `search` only scrapes text-like results (`text/html`, `application/json`, +`text/markdown`, `text/plain`); PDFs, office documents and images are skipped. Pass +`contentTypes` to change that, for example `contentTypes: ["text/html", "application/pdf"]` +to include PDFs. + ### crawl Crawl a website and its linked pages. diff --git a/src/schemas.ts b/src/schemas.ts index 6a273c9..6ee5635 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -10,6 +10,10 @@ export const htmlModeSchema = z.enum(["normal", "reader", "prune"]); export const fetchContentTypeSchema = z.enum([ "text/html", "application/json", + "text/markdown", + "text/plain", + "text/csv", + "application/x-latex", "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.presentationml.presentation", @@ -25,9 +29,6 @@ export const fetchContentTypeSchema = z.enum([ "application/epub+zip", "application/rtf", "application/vnd.oasis.opendocument.text", - "text/csv", - "text/plain", - "application/x-latex", ]); export const userPromptSchema = z.string().min(1).max(10_000); @@ -249,6 +250,7 @@ export const searchRequestSchema = z prompt: userPromptSchema.optional(), schema: z.record(z.string(), z.unknown()).optional(), locationGeoCode: z.string().max(10).optional(), + contentTypes: z.array(fetchContentTypeSchema).optional(), timeRange: z .enum(["past_hour", "past_24_hours", "past_week", "past_month", "past_year"]) .optional(), From f82f7da8d1961c7ed3bfb086b765750cd95c245b Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 14:36:45 +0200 Subject: [PATCH 2/6] fix(search): align PDF options with API --- README.md | 15 +++++---------- src/schemas.ts | 7 ++++++- tests/scrapegraphai.test.ts | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 514f19d..dc8b2ef 100644 --- a/README.md +++ b/README.md @@ -155,19 +155,14 @@ const res = await sgai.search({ timeRange: "past_week", // optional locationGeoCode: "us", // optional fetchConfig: { /* ... */ }, // optional - contentTypes: [ // optional, defaults to text-like types - "text/html", - "application/json", - "text/markdown", - "text/plain", - ], + allowedTypes: ["text/html", "application/pdf"], // optional MIME allowlist + processors: [{ type: "pdf", maxPages: 10 }], // optional PDF page cap }); ``` -By default `search` only scrapes text-like results (`text/html`, `application/json`, -`text/markdown`, `text/plain`); PDFs, office documents and images are skipped. Pass -`contentTypes` to change that, for example `contentTypes: ["text/html", "application/pdf"]` -to include PDFs. +By default `search` accepts every supported content type, including PDFs, and processes up to 25 +pages per PDF. Use `allowedTypes` to restrict accepted MIME types. Configure PDF processing +separately with `processors`; `maxPages` accepts `1`–`500`, or `-1` for no page limit. ### crawl diff --git a/src/schemas.ts b/src/schemas.ts index 6ee5635..5e0f253 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -30,6 +30,10 @@ export const fetchContentTypeSchema = z.enum([ "application/rtf", "application/vnd.oasis.opendocument.text", ]); +export const pdfProcessorSchema = z.object({ + type: z.literal("pdf"), + maxPages: z.union([z.literal(-1), z.number().int().min(1).max(500)]).default(25), +}); export const userPromptSchema = z.string().min(1).max(10_000); const PUBLIC_DOMAIN_RE = @@ -250,7 +254,8 @@ export const searchRequestSchema = z prompt: userPromptSchema.optional(), schema: z.record(z.string(), z.unknown()).optional(), locationGeoCode: z.string().max(10).optional(), - contentTypes: z.array(fetchContentTypeSchema).optional(), + allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), + processors: z.array(pdfProcessorSchema).min(1).optional(), timeRange: z .enum(["past_hour", "past_24_hours", "past_week", "past_month", "past_year"]) .optional(), diff --git a/tests/scrapegraphai.test.ts b/tests/scrapegraphai.test.ts index edfe47a..02eb83c 100644 --- a/tests/scrapegraphai.test.ts +++ b/tests/scrapegraphai.test.ts @@ -710,6 +710,25 @@ describe("search", () => { expect(res.status).toBe("success"); expectRequest(0, "POST", "/search", searchParams); }); + + test("with PDF options", async () => { + const body = { + results: [], + metadata: { search: {}, pages: { requested: 1, scraped: 0 } }, + }; + fetchSpy = spyOn(globalThis, "fetch").mockResolvedValueOnce(json(body)); + const searchParams = { + query: "papers", + numResults: 1, + allowedTypes: ["application/pdf"] as const, + processors: [{ type: "pdf" as const, maxPages: 10 }], + }; + + const res = await sdk.search(API_KEY, searchParams); + + expect(res.status).toBe("success"); + expectRequest(0, "POST", "/search", searchParams); + }); }); describe("getCredits", () => { From c336efbfd977f763fd6b939e7b9979d8bc325a28 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 14:59:45 +0200 Subject: [PATCH 3/6] fix(sdk): support PDF options across endpoints --- src/schemas.ts | 7 ++++++- tests/scrapegraphai.test.ts | 5 +++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/schemas.ts b/src/schemas.ts index 5e0f253..d0c7d25 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -219,6 +219,8 @@ export const scrapeFormatEntrySchema = z.discriminatedUnion("type", [ export const scrapeRequestSchema = z.object({ url: urlSchema, contentType: fetchContentTypeSchema.optional(), + allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), + processors: z.array(pdfProcessorSchema).min(1).optional(), fetchConfig: fetchConfigSchema.optional(), formats: z .array(scrapeFormatEntrySchema) @@ -238,6 +240,8 @@ export const extractRequestBaseSchema = z prompt: userPromptSchema, schema: z.record(z.string(), z.unknown()).optional(), contentType: fetchContentTypeSchema.optional(), + allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), + processors: z.array(pdfProcessorSchema).min(1).optional(), fetchConfig: fetchConfigSchema.optional(), }) .refine((d) => d.url || d.html || d.markdown, { @@ -524,7 +528,8 @@ export const crawlRequestSchema = z.object({ .describe( 'Glob-style URL patterns to exclude. Use "*/" for first-level paths and "**//**" for nested paths.', ), - contentTypes: z.array(fetchContentTypeSchema).optional(), + allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), + processors: z.array(pdfProcessorSchema).min(1).optional(), fetchConfig: fetchConfigSchema.optional(), }); diff --git a/tests/scrapegraphai.test.ts b/tests/scrapegraphai.test.ts index 02eb83c..31c6634 100644 --- a/tests/scrapegraphai.test.ts +++ b/tests/scrapegraphai.test.ts @@ -881,7 +881,7 @@ describe("crawl", () => { expectRequest(0, "POST", "/crawl", patternParams); }); - test("start with fetchConfig and contentTypes", async () => { + test("start with fetchConfig, allowedTypes, and processors", async () => { const body = { id: "crawl-abc", status: "running", @@ -893,7 +893,8 @@ describe("crawl", () => { const configParams = { url: "https://example.com", - contentTypes: ["text/html" as const, "application/pdf" as const], + allowedTypes: ["text/html" as const, "application/pdf" as const], + processors: [{ type: "pdf" as const, maxPages: 10 }], fetchConfig: { mode: "js" as const, stealth: true, From e76686821a80b634d9ecc717b86c48e2f43d7add Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 15:09:29 +0200 Subject: [PATCH 4/6] fix(sdk): match content validation with API --- src/schemas.ts | 32 +++++++++++++++++------ tests/schemas.test.ts | 61 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 tests/schemas.test.ts diff --git a/src/schemas.ts b/src/schemas.ts index d0c7d25..333f0ff 100644 --- a/src/schemas.ts +++ b/src/schemas.ts @@ -34,6 +34,22 @@ export const pdfProcessorSchema = z.object({ type: z.literal("pdf"), maxPages: z.union([z.literal(-1), z.number().int().min(1).max(500)]).default(25), }); +export const allowedTypesSchema = z + .array(fetchContentTypeSchema) + .min(1) + .refine((types) => new Set(types).size === types.length, { + message: "duplicate allowed types not allowed", + }); +export const processorsSchema = z + .array(pdfProcessorSchema) + .min(1) + .refine( + (processors) => + new Set(processors.map((processor) => processor.type)).size === processors.length, + { + message: "duplicate processor types not allowed", + }, + ); export const userPromptSchema = z.string().min(1).max(10_000); const PUBLIC_DOMAIN_RE = @@ -219,8 +235,8 @@ export const scrapeFormatEntrySchema = z.discriminatedUnion("type", [ export const scrapeRequestSchema = z.object({ url: urlSchema, contentType: fetchContentTypeSchema.optional(), - allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), - processors: z.array(pdfProcessorSchema).min(1).optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), fetchConfig: fetchConfigSchema.optional(), formats: z .array(scrapeFormatEntrySchema) @@ -240,8 +256,8 @@ export const extractRequestBaseSchema = z prompt: userPromptSchema, schema: z.record(z.string(), z.unknown()).optional(), contentType: fetchContentTypeSchema.optional(), - allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), - processors: z.array(pdfProcessorSchema).min(1).optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), fetchConfig: fetchConfigSchema.optional(), }) .refine((d) => d.url || d.html || d.markdown, { @@ -258,8 +274,8 @@ export const searchRequestSchema = z prompt: userPromptSchema.optional(), schema: z.record(z.string(), z.unknown()).optional(), locationGeoCode: z.string().max(10).optional(), - allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), - processors: z.array(pdfProcessorSchema).min(1).optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), timeRange: z .enum(["past_hour", "past_24_hours", "past_week", "past_month", "past_year"]) .optional(), @@ -528,8 +544,8 @@ export const crawlRequestSchema = z.object({ .describe( 'Glob-style URL patterns to exclude. Use "*/" for first-level paths and "**//**" for nested paths.', ), - allowedTypes: z.array(fetchContentTypeSchema).min(1).optional(), - processors: z.array(pdfProcessorSchema).min(1).optional(), + allowedTypes: allowedTypesSchema.optional(), + processors: processorsSchema.optional(), fetchConfig: fetchConfigSchema.optional(), }); diff --git a/tests/schemas.test.ts b/tests/schemas.test.ts new file mode 100644 index 0000000..7525827 --- /dev/null +++ b/tests/schemas.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { + crawlRequestSchema, + extractRequestBaseSchema, + scrapeRequestSchema, + searchRequestSchema, +} from "../src/schemas.js"; + +const requests = [ + ["scrape", scrapeRequestSchema, { url: "https://example.com" }], + ["extract", extractRequestBaseSchema, { url: "https://example.com", prompt: "title" }], + ["search", searchRequestSchema, { query: "example" }], + ["crawl", crawlRequestSchema, { url: "https://example.com" }], +] as const; + +describe("content type and PDF processor validation", () => { + for (const [name, schema, base] of requests) { + test(`${name} accepts the documented PDF configuration`, () => { + expect( + schema.safeParse({ + ...base, + allowedTypes: ["application/pdf"], + processors: [{ type: "pdf", maxPages: 10 }], + }).success, + ).toBe(true); + }); + + test(`${name} rejects empty and duplicate arrays`, () => { + expect(schema.safeParse({ ...base, allowedTypes: [] }).success).toBe(false); + expect( + schema.safeParse({ + ...base, + allowedTypes: ["application/pdf", "application/pdf"], + }).success, + ).toBe(false); + expect(schema.safeParse({ ...base, processors: [] }).success).toBe(false); + expect( + schema.safeParse({ + ...base, + processors: [ + { type: "pdf", maxPages: 1 }, + { type: "pdf", maxPages: 10 }, + ], + }).success, + ).toBe(false); + }); + + test(`${name} enforces the documented PDF page limits`, () => { + for (const maxPages of [1, 500, -1]) { + expect(schema.safeParse({ ...base, processors: [{ type: "pdf", maxPages }] }).success).toBe( + true, + ); + } + for (const maxPages of [0, -2, 501, 1.5]) { + expect(schema.safeParse({ ...base, processors: [{ type: "pdf", maxPages }] }).success).toBe( + false, + ); + } + }); + } +}); From 5a1fd3c1d3be291a51f933dabe69e3ca13465c41 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 15:37:09 +0200 Subject: [PATCH 5/6] fix(ci): quote local package rewrite steps --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 039af49..d3b3132 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,8 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - name: Use local scrapegraph-js package - run: sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json + run: | + sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json - run: bun install - run: bun run test @@ -25,7 +26,8 @@ jobs: - uses: actions/checkout@v4 - uses: oven-sh/setup-bun@v2 - name: Use local scrapegraph-js package - run: sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json + run: | + sed -i 's/"scrapegraph-js": "\^2.2.0"/"scrapegraph-js": "file:..\/.."/' packages/ai-sdk/package.json - run: bun install - run: bun run build - run: bun run check From af182bf6e77ff0a8f68edd7961d68f70012315f5 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 16:40:10 +0200 Subject: [PATCH 6/6] fix(types): allow default PDF page cap --- README.md | 6 +++--- src/types.ts | 8 ++++---- tests/schemas.test.ts | 13 +++++++++++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index dc8b2ef..dc99cfe 100644 --- a/README.md +++ b/README.md @@ -156,13 +156,13 @@ const res = await sgai.search({ locationGeoCode: "us", // optional fetchConfig: { /* ... */ }, // optional allowedTypes: ["text/html", "application/pdf"], // optional MIME allowlist - processors: [{ type: "pdf", maxPages: 10 }], // optional PDF page cap }); ``` By default `search` accepts every supported content type, including PDFs, and processes up to 25 -pages per PDF. Use `allowedTypes` to restrict accepted MIME types. Configure PDF processing -separately with `processors`; `maxPages` accepts `1`–`500`, or `-1` for no page limit. +pages per PDF. You do not need to send `processors` or `maxPages` for this default. Use +`allowedTypes` to restrict accepted MIME types. Only configure `processors` to override the cap; +`{ type: "pdf" }` also defaults to 25, while `maxPages` accepts `1`–`500`, or `-1` for no page limit. ### crawl diff --git a/src/types.ts b/src/types.ts index d2fd8ea..0fec5b1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -134,7 +134,7 @@ export type HealthResponse = z.infer; // ─── scrape ────────────────────────────────────────────────────────────────── -export type ScrapeRequest = z.infer; +export type ScrapeRequest = z.input; export type FetchConfig = z.infer; export type FetchMode = z.infer; export type FetchContentType = z.infer; @@ -202,7 +202,7 @@ export type ScrapeEvent = // ─── extract ───────────────────────────────────────────────────────────────── -export type ExtractRequestBase = z.infer; +export type ExtractRequestBase = z.input; export type LlmConfig = z.infer; export type ExtractResponse = z.infer; @@ -217,7 +217,7 @@ export type ExtractEvent = // ─── search ────────────────────────────────────────────────────────────────── -export type SearchRequest = z.infer; +export type SearchRequest = z.input; export type SearchResult = z.infer; export type SearchMetadata = z.infer; @@ -313,7 +313,7 @@ export type MonitorEvent = // ─── crawl ─────────────────────────────────────────────────────────────────── -export type CrawlRequest = z.infer; +export type CrawlRequest = z.input; export type CrawlStatus = z.infer; export type CrawlPageStatus = z.infer; diff --git a/tests/schemas.test.ts b/tests/schemas.test.ts index 7525827..a9fd249 100644 --- a/tests/schemas.test.ts +++ b/tests/schemas.test.ts @@ -5,6 +5,12 @@ import { scrapeRequestSchema, searchRequestSchema, } from "../src/schemas.js"; +import type { SearchRequest } from "../src/types.js"; + +const requestWithoutExplicitPageCap: SearchRequest = { + query: "example", + processors: [{ type: "pdf" }], +}; const requests = [ ["scrape", scrapeRequestSchema, { url: "https://example.com" }], @@ -14,6 +20,13 @@ const requests = [ ] as const; describe("content type and PDF processor validation", () => { + test("request types and schemas allow the default 25-page PDF cap", () => { + expect(searchRequestSchema.parse(requestWithoutExplicitPageCap).processors).toEqual([ + { type: "pdf", maxPages: 25 }, + ]); + expect(searchRequestSchema.parse({ query: "example" }).processors).toBeUndefined(); + }); + for (const [name, schema, base] of requests) { test(`${name} accepts the documented PDF configuration`, () => { expect(