From a23f01634935cb0cb66d20e32d79f285ed3e058f Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Wed, 29 Jul 2026 13:22:35 +0200 Subject: [PATCH 1/5] feat(search): add contentTypes filter Search now accepts contentTypes so callers can pick which content types get fetched. Adds application/json and text/markdown to FetchContentType to match the API enum. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 11 +++++++++++ src/scrapegraph_py/async_client.py | 2 ++ src/scrapegraph_py/client.py | 2 ++ src/scrapegraph_py/schemas.py | 8 +++++--- 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2a10a6c..6bef72c 100644 --- a/README.md +++ b/README.md @@ -152,9 +152,20 @@ res = sgai.search( schema={...}, # optional time_range="past_week", # optional location_geo_code="us", # optional + content_types=[ # optional, defaults to text-like types + "text/html", + "application/json", + "text/markdown", + "text/plain", + ], ) ``` +By default `search` only scrapes text-like results (`text/html`, `application/json`, +`text/markdown`, `text/plain`); PDFs, office documents and images are skipped. Pass +`content_types` to change that, for example `content_types=["text/html", "application/pdf"]` +to include PDFs. + ### crawl Crawl a website and its linked pages. diff --git a/src/scrapegraph_py/async_client.py b/src/scrapegraph_py/async_client.py index f3afa2f..830ab77 100644 --- a/src/scrapegraph_py/async_client.py +++ b/src/scrapegraph_py/async_client.py @@ -395,6 +395,7 @@ async def search( schema: dict[str, object] | None = None, fetch_config: FetchConfig | None = None, location_geo_code: str | None = None, + content_types: list[FetchContentType] | None = None, time_range: TimeRange | None = None, ) -> ApiResult[SearchResponse]: req = SearchRequest( @@ -407,6 +408,7 @@ async def search( schema=schema, fetch_config=fetch_config, location_geo_code=location_geo_code, + content_types=content_types, time_range=time_range, ) ) diff --git a/src/scrapegraph_py/client.py b/src/scrapegraph_py/client.py index b6bf53b..984ba94 100644 --- a/src/scrapegraph_py/client.py +++ b/src/scrapegraph_py/client.py @@ -395,6 +395,7 @@ def search( schema: dict[str, object] | None = None, fetch_config: FetchConfig | None = None, location_geo_code: str | None = None, + content_types: list[FetchContentType] | None = None, time_range: TimeRange | None = None, ) -> ApiResult[SearchResponse]: req = SearchRequest( @@ -407,6 +408,7 @@ def search( schema=schema, fetch_config=fetch_config, location_geo_code=location_geo_code, + content_types=content_types, time_range=time_range, ) ) diff --git a/src/scrapegraph_py/schemas.py b/src/scrapegraph_py/schemas.py index 20301e6..9d40be7 100644 --- a/src/scrapegraph_py/schemas.py +++ b/src/scrapegraph_py/schemas.py @@ -27,6 +27,10 @@ class ResponseModel(BaseModel): FetchContentType = Literal[ "text/html", "application/json", + "text/markdown", + "text/plain", + "text/csv", + "application/x-latex", "application/pdf", "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "application/vnd.openxmlformats-officedocument.presentationml.presentation", @@ -42,9 +46,6 @@ class ResponseModel(BaseModel): "application/epub+zip", "application/rtf", "application/vnd.oasis.opendocument.text", - "text/csv", - "text/plain", - "application/x-latex", ] ScrapeContentFormat = Literal["markdown", "html", "links", "images", "summary", "json", "branding"] @@ -187,6 +188,7 @@ class SearchRequest(CamelModel): prompt: Annotated[str, Field(min_length=1, max_length=10000)] | None = None schema_: dict[str, object] | None = Field(default=None, alias="schema") location_geo_code: Annotated[str, Field(max_length=10)] | None = None + content_types: list[FetchContentType] | None = None time_range: TimeRange | None = None @model_validator(mode="after") From 8415e100b25bb7275f1454c85b35c3fbad6e550e Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 14:36:35 +0200 Subject: [PATCH 2/5] fix(search): align PDF options with API --- README.md | 17 ++++++----------- src/scrapegraph_py/__init__.py | 2 ++ src/scrapegraph_py/async_client.py | 7 +++++-- src/scrapegraph_py/client.py | 7 +++++-- src/scrapegraph_py/schemas.py | 9 ++++++++- tests/test_client.py | 16 ++++++++++++++++ 6 files changed, 42 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 6bef72c..3210cc5 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ res = sgai.extract( Search the web and optionally extract structured data. ```python -from scrapegraph_py import ScrapeGraphAI +from scrapegraph_py import PdfProcessor, ScrapeGraphAI sgai = ScrapeGraphAI() @@ -152,19 +152,14 @@ res = sgai.search( schema={...}, # optional time_range="past_week", # optional location_geo_code="us", # optional - content_types=[ # optional, defaults to text-like types - "text/html", - "application/json", - "text/markdown", - "text/plain", - ], + allowed_types=["text/html", "application/pdf"], # optional MIME allowlist + processors=[PdfProcessor(max_pages=10)], # optional PDF page cap ) ``` -By default `search` only scrapes text-like results (`text/html`, `application/json`, -`text/markdown`, `text/plain`); PDFs, office documents and images are skipped. Pass -`content_types` to change that, for example `content_types=["text/html", "application/pdf"]` -to include PDFs. +By default `search` accepts every supported content type, including PDFs, and processes up to 25 +pages per PDF. Use `allowed_types` to restrict accepted MIME types. Configure PDF processing +separately with `processors`; `max_pages` accepts `1`–`500`, or `-1` for no page limit. ### crawl diff --git a/src/scrapegraph_py/__init__.py b/src/scrapegraph_py/__init__.py index 0e88ba0..063b19b 100644 --- a/src/scrapegraph_py/__init__.py +++ b/src/scrapegraph_py/__init__.py @@ -39,6 +39,7 @@ MonitorTickEntry, MonitorTickStatus, MonitorUpdateRequest, + PdfProcessor, ScrapeBrandingFormatEntry, ScrapeCaptureFormat, ScrapeContentFormat, @@ -104,6 +105,7 @@ "FetchConfig", "FetchContentType", "FetchMode", + "PdfProcessor", "FormatConfig", "ScrapeFormat", "ScrapeContentFormat", diff --git a/src/scrapegraph_py/async_client.py b/src/scrapegraph_py/async_client.py index 830ab77..c0b5658 100644 --- a/src/scrapegraph_py/async_client.py +++ b/src/scrapegraph_py/async_client.py @@ -34,6 +34,7 @@ MonitorCreateRequest, MonitorResponse, MonitorUpdateRequest, + PdfProcessor, ScrapeRequest, ScrapeResponse, SearchRequest, @@ -395,7 +396,8 @@ async def search( schema: dict[str, object] | None = None, fetch_config: FetchConfig | None = None, location_geo_code: str | None = None, - content_types: list[FetchContentType] | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, time_range: TimeRange | None = None, ) -> ApiResult[SearchResponse]: req = SearchRequest( @@ -408,7 +410,8 @@ async def search( schema=schema, fetch_config=fetch_config, location_geo_code=location_geo_code, - content_types=content_types, + allowed_types=allowed_types, + processors=processors, time_range=time_range, ) ) diff --git a/src/scrapegraph_py/client.py b/src/scrapegraph_py/client.py index 984ba94..6b155a8 100644 --- a/src/scrapegraph_py/client.py +++ b/src/scrapegraph_py/client.py @@ -34,6 +34,7 @@ MonitorCreateRequest, MonitorResponse, MonitorUpdateRequest, + PdfProcessor, ScrapeRequest, ScrapeResponse, SearchRequest, @@ -395,7 +396,8 @@ def search( schema: dict[str, object] | None = None, fetch_config: FetchConfig | None = None, location_geo_code: str | None = None, - content_types: list[FetchContentType] | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, time_range: TimeRange | None = None, ) -> ApiResult[SearchResponse]: req = SearchRequest( @@ -408,7 +410,8 @@ def search( schema=schema, fetch_config=fetch_config, location_geo_code=location_geo_code, - content_types=content_types, + allowed_types=allowed_types, + processors=processors, time_range=time_range, ) ) diff --git a/src/scrapegraph_py/schemas.py b/src/scrapegraph_py/schemas.py index 9d40be7..5cd1c6e 100644 --- a/src/scrapegraph_py/schemas.py +++ b/src/scrapegraph_py/schemas.py @@ -48,6 +48,12 @@ class ResponseModel(BaseModel): "application/vnd.oasis.opendocument.text", ] + +class PdfProcessor(CamelModel): + type: Literal["pdf"] = "pdf" + max_pages: Literal[-1] | Annotated[int, Field(ge=1, le=500)] = 25 + + ScrapeContentFormat = Literal["markdown", "html", "links", "images", "summary", "json", "branding"] ScrapeCaptureFormat = Literal["screenshot"] ScrapeFormat = Literal[ @@ -188,7 +194,8 @@ class SearchRequest(CamelModel): prompt: Annotated[str, Field(min_length=1, max_length=10000)] | None = None schema_: dict[str, object] | None = Field(default=None, alias="schema") location_geo_code: Annotated[str, Field(max_length=10)] | None = None - content_types: list[FetchContentType] | None = None + allowed_types: list[FetchContentType] | None = None + processors: list[PdfProcessor] | None = None time_range: TimeRange | None = None @model_validator(mode="after") diff --git a/tests/test_client.py b/tests/test_client.py index b889c13..deabdcf 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -10,6 +10,7 @@ JsonFormatConfig, LinksFormatConfig, MarkdownFormatConfig, + PdfProcessor, ScrapeGraphAI, ScreenshotFormatConfig, ) @@ -260,6 +261,21 @@ def test_with_extraction(self): _, kwargs = mock.call_args assert kwargs["json"]["prompt"] == "Summarize results" + def test_with_pdf_options(self): + body = {"results": [], "metadata": {"search": {}, "pages": {}}} + with patch.object(httpx.Client, "request", return_value=mock_response(body)) as mock: + sgai = ScrapeGraphAI(api_key=API_KEY) + res = sgai.search( + "papers", + allowed_types=["application/pdf"], + processors=[PdfProcessor(max_pages=10)], + ) + + assert res.status == "success" + _, kwargs = mock.call_args + assert kwargs["json"]["allowedTypes"] == ["application/pdf"] + assert kwargs["json"]["processors"] == [{"type": "pdf", "maxPages": 10}] + class TestCrawl: def test_start(self): From 9caba6fab55b00c1dad792ca15504c9b7b56d8d8 Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 14:59:37 +0200 Subject: [PATCH 3/5] fix(sdk): support PDF options across endpoints --- src/scrapegraph_py/async_client.py | 14 ++++++++++++-- src/scrapegraph_py/client.py | 14 ++++++++++++-- src/scrapegraph_py/schemas.py | 7 ++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/scrapegraph_py/async_client.py b/src/scrapegraph_py/async_client.py index c0b5658..71dd637 100644 --- a/src/scrapegraph_py/async_client.py +++ b/src/scrapegraph_py/async_client.py @@ -97,7 +97,8 @@ async def start( allow_external: bool | None = None, include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, - content_types: list[FetchContentType] | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, fetch_config: FetchConfig | None = None, ) -> ApiResult[CrawlResponse]: req = CrawlRequest( @@ -110,7 +111,8 @@ async def start( allow_external=allow_external, include_patterns=include_patterns, exclude_patterns=exclude_patterns, - content_types=content_types, + allowed_types=allowed_types, + processors=processors, fetch_config=fetch_config, ) ) @@ -348,6 +350,8 @@ async def scrape( formats: list[FormatConfig] | None = None, fetch_config: FetchConfig | None = None, content_type: FetchContentType | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, ) -> ApiResult[ScrapeResponse]: req = ScrapeRequest( **_compact( @@ -355,6 +359,8 @@ async def scrape( formats=formats, fetch_config=fetch_config, content_type=content_type, + allowed_types=allowed_types, + processors=processors, ) ) return await self._post("/scrape", req, ScrapeResponse) @@ -370,6 +376,8 @@ async def extract( mode: HtmlMode | None = None, fetch_config: FetchConfig | None = None, content_type: FetchContentType | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, ) -> ApiResult[ExtractResponse]: req = ExtractRequest( **_compact( @@ -381,6 +389,8 @@ async def extract( mode=mode, fetch_config=fetch_config, content_type=content_type, + allowed_types=allowed_types, + processors=processors, ) ) return await self._post("/extract", req, ExtractResponse) diff --git a/src/scrapegraph_py/client.py b/src/scrapegraph_py/client.py index 6b155a8..b583e7f 100644 --- a/src/scrapegraph_py/client.py +++ b/src/scrapegraph_py/client.py @@ -97,7 +97,8 @@ def start( allow_external: bool | None = None, include_patterns: list[str] | None = None, exclude_patterns: list[str] | None = None, - content_types: list[FetchContentType] | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, fetch_config: FetchConfig | None = None, ) -> ApiResult[CrawlResponse]: req = CrawlRequest( @@ -110,7 +111,8 @@ def start( allow_external=allow_external, include_patterns=include_patterns, exclude_patterns=exclude_patterns, - content_types=content_types, + allowed_types=allowed_types, + processors=processors, fetch_config=fetch_config, ) ) @@ -348,6 +350,8 @@ def scrape( formats: list[FormatConfig] | None = None, fetch_config: FetchConfig | None = None, content_type: FetchContentType | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, ) -> ApiResult[ScrapeResponse]: req = ScrapeRequest( **_compact( @@ -355,6 +359,8 @@ def scrape( formats=formats, fetch_config=fetch_config, content_type=content_type, + allowed_types=allowed_types, + processors=processors, ) ) return self._post("/scrape", req, ScrapeResponse) @@ -370,6 +376,8 @@ def extract( mode: HtmlMode | None = None, fetch_config: FetchConfig | None = None, content_type: FetchContentType | None = None, + allowed_types: list[FetchContentType] | None = None, + processors: list[PdfProcessor] | None = None, ) -> ApiResult[ExtractResponse]: req = ExtractRequest( **_compact( @@ -381,6 +389,8 @@ def extract( mode=mode, fetch_config=fetch_config, content_type=content_type, + allowed_types=allowed_types, + processors=processors, ) ) return self._post("/extract", req, ExtractResponse) diff --git a/src/scrapegraph_py/schemas.py b/src/scrapegraph_py/schemas.py index 5cd1c6e..d04917d 100644 --- a/src/scrapegraph_py/schemas.py +++ b/src/scrapegraph_py/schemas.py @@ -154,6 +154,8 @@ class ScrapeBrandingFormatEntry(CamelModel): class ScrapeRequest(CamelModel): url: HttpUrl content_type: FetchContentType | None = None + allowed_types: list[FetchContentType] | None = None + processors: list[PdfProcessor] | None = None fetch_config: FetchConfig | None = None formats: list[ScrapeFormatEntry] = Field(default_factory=lambda: [ScrapeMarkdownFormatEntry()]) @@ -173,6 +175,8 @@ class ExtractRequestBase(CamelModel): prompt: Annotated[str, Field(min_length=1, max_length=10000)] schema_: dict[str, object] | None = Field(default=None, alias="schema") content_type: FetchContentType | None = None + allowed_types: list[FetchContentType] | None = None + processors: list[PdfProcessor] | None = None fetch_config: FetchConfig | None = None @model_validator(mode="after") @@ -246,7 +250,8 @@ class CrawlRequest(CamelModel): allow_external: bool = False include_patterns: list[str] | None = None exclude_patterns: list[str] | None = None - content_types: list[FetchContentType] | None = None + allowed_types: list[FetchContentType] | None = None + processors: list[PdfProcessor] | None = None fetch_config: FetchConfig | None = None @model_validator(mode="after") From 5215bc5ab6deb6bda266ceae0ab1e9ae0697891d Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 15:09:29 +0200 Subject: [PATCH 4/5] fix(sdk): match content validation with API --- src/scrapegraph_py/schemas.py | 37 ++++++++++++++++----- tests/test_schemas.py | 62 +++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 tests/test_schemas.py diff --git a/src/scrapegraph_py/schemas.py b/src/scrapegraph_py/schemas.py index d04917d..971dbfc 100644 --- a/src/scrapegraph_py/schemas.py +++ b/src/scrapegraph_py/schemas.py @@ -2,7 +2,7 @@ from typing import Annotated, Generic, Literal, TypeVar -from pydantic import BaseModel, ConfigDict, Field, HttpUrl, model_validator +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, HttpUrl, model_validator from pydantic.alias_generators import to_camel T = TypeVar("T") @@ -54,6 +54,25 @@ class PdfProcessor(CamelModel): max_pages: Literal[-1] | Annotated[int, Field(ge=1, le=500)] = 25 +def _unique_allowed_types(values: list[FetchContentType]) -> list[FetchContentType]: + if len(values) != len(set(values)): + raise ValueError("duplicate allowed types not allowed") + return values + + +def _unique_processors(values: list[PdfProcessor]) -> list[PdfProcessor]: + types = [processor.type for processor in values] + if len(types) != len(set(types)): + raise ValueError("duplicate processor types not allowed") + return values + + +AllowedTypes = Annotated[ + list[FetchContentType], Field(min_length=1), AfterValidator(_unique_allowed_types) +] +Processors = Annotated[list[PdfProcessor], Field(min_length=1), AfterValidator(_unique_processors)] + + ScrapeContentFormat = Literal["markdown", "html", "links", "images", "summary", "json", "branding"] ScrapeCaptureFormat = Literal["screenshot"] ScrapeFormat = Literal[ @@ -154,8 +173,8 @@ class ScrapeBrandingFormatEntry(CamelModel): class ScrapeRequest(CamelModel): url: HttpUrl content_type: FetchContentType | None = None - allowed_types: list[FetchContentType] | None = None - processors: list[PdfProcessor] | None = None + allowed_types: AllowedTypes | None = None + processors: Processors | None = None fetch_config: FetchConfig | None = None formats: list[ScrapeFormatEntry] = Field(default_factory=lambda: [ScrapeMarkdownFormatEntry()]) @@ -175,8 +194,8 @@ class ExtractRequestBase(CamelModel): prompt: Annotated[str, Field(min_length=1, max_length=10000)] schema_: dict[str, object] | None = Field(default=None, alias="schema") content_type: FetchContentType | None = None - allowed_types: list[FetchContentType] | None = None - processors: list[PdfProcessor] | None = None + allowed_types: AllowedTypes | None = None + processors: Processors | None = None fetch_config: FetchConfig | None = None @model_validator(mode="after") @@ -198,8 +217,8 @@ class SearchRequest(CamelModel): prompt: Annotated[str, Field(min_length=1, max_length=10000)] | None = None schema_: dict[str, object] | None = Field(default=None, alias="schema") location_geo_code: Annotated[str, Field(max_length=10)] | None = None - allowed_types: list[FetchContentType] | None = None - processors: list[PdfProcessor] | None = None + allowed_types: AllowedTypes | None = None + processors: Processors | None = None time_range: TimeRange | None = None @model_validator(mode="after") @@ -250,8 +269,8 @@ class CrawlRequest(CamelModel): allow_external: bool = False include_patterns: list[str] | None = None exclude_patterns: list[str] | None = None - allowed_types: list[FetchContentType] | None = None - processors: list[PdfProcessor] | None = None + allowed_types: AllowedTypes | None = None + processors: Processors | None = None fetch_config: FetchConfig | None = None @model_validator(mode="after") diff --git a/tests/test_schemas.py b/tests/test_schemas.py new file mode 100644 index 0000000..338bbfa --- /dev/null +++ b/tests/test_schemas.py @@ -0,0 +1,62 @@ +import pytest +from pydantic import ValidationError + +from scrapegraph_py import ( + CrawlRequest, + ExtractRequest, + ScrapeRequest, + SearchRequest, +) + +REQUESTS = [ + (ScrapeRequest, {"url": "https://example.com"}), + (ExtractRequest, {"url": "https://example.com", "prompt": "title"}), + (SearchRequest, {"query": "example"}), + (CrawlRequest, {"url": "https://example.com"}), +] + + +@pytest.mark.parametrize(("request_model", "base"), REQUESTS) +def test_documented_pdf_configuration(request_model, base): + request = request_model( + **base, + allowed_types=["application/pdf"], + processors=[{"type": "pdf", "max_pages": 10}], + ) + + assert request.model_dump(by_alias=True, mode="json", exclude_none=True)["processors"] == [ + {"type": "pdf", "maxPages": 10} + ] + + +@pytest.mark.parametrize(("request_model", "base"), REQUESTS) +@pytest.mark.parametrize( + "invalid", + [ + {"allowed_types": []}, + {"allowed_types": ["application/pdf", "application/pdf"]}, + {"processors": []}, + { + "processors": [ + {"type": "pdf", "max_pages": 1}, + {"type": "pdf", "max_pages": 10}, + ] + }, + ], +) +def test_rejects_empty_and_duplicate_arrays(request_model, base, invalid): + with pytest.raises(ValidationError): + request_model(**base, **invalid) + + +@pytest.mark.parametrize(("request_model", "base"), REQUESTS) +@pytest.mark.parametrize("max_pages", [1, 500, -1]) +def test_accepts_documented_pdf_page_limits(request_model, base, max_pages): + request_model(**base, processors=[{"type": "pdf", "max_pages": max_pages}]) + + +@pytest.mark.parametrize(("request_model", "base"), REQUESTS) +@pytest.mark.parametrize("max_pages", [0, -2, 501, 1.5]) +def test_rejects_invalid_pdf_page_limits(request_model, base, max_pages): + with pytest.raises(ValidationError): + request_model(**base, processors=[{"type": "pdf", "max_pages": max_pages}]) From 034da97ad350c45388a81731f96b6227fa9c534b Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Thu, 30 Jul 2026 16:40:10 +0200 Subject: [PATCH 5/5] docs(sdk): clarify default PDF page cap --- README.md | 8 ++++---- tests/test_schemas.py | 11 +++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3210cc5..cae0399 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,7 @@ res = sgai.extract( Search the web and optionally extract structured data. ```python -from scrapegraph_py import PdfProcessor, ScrapeGraphAI +from scrapegraph_py import ScrapeGraphAI sgai = ScrapeGraphAI() @@ -153,13 +153,13 @@ res = sgai.search( time_range="past_week", # optional location_geo_code="us", # optional allowed_types=["text/html", "application/pdf"], # optional MIME allowlist - processors=[PdfProcessor(max_pages=10)], # optional PDF page cap ) ``` By default `search` accepts every supported content type, including PDFs, and processes up to 25 -pages per PDF. Use `allowed_types` to restrict accepted MIME types. Configure PDF processing -separately with `processors`; `max_pages` accepts `1`–`500`, or `-1` for no page limit. +pages per PDF. You do not need to send `processors` or `max_pages` for this default. Use +`allowed_types` to restrict accepted MIME types. Only configure `processors` to override the cap; +`PdfProcessor()` also defaults to 25, while `max_pages` accepts `1`–`500`, or `-1` for no page limit. ### crawl diff --git a/tests/test_schemas.py b/tests/test_schemas.py index 338bbfa..0bfcf90 100644 --- a/tests/test_schemas.py +++ b/tests/test_schemas.py @@ -29,6 +29,17 @@ def test_documented_pdf_configuration(request_model, base): ] +@pytest.mark.parametrize(("request_model", "base"), REQUESTS) +def test_default_pdf_cap_can_be_omitted(request_model, base): + request = request_model(**base) + assert "processors" not in request.model_dump(by_alias=True, mode="json", exclude_none=True) + + request = request_model(**base, processors=[{"type": "pdf"}]) + assert request.model_dump(by_alias=True, mode="json", exclude_none=True)["processors"] == [ + {"type": "pdf", "maxPages": 25} + ] + + @pytest.mark.parametrize(("request_model", "base"), REQUESTS) @pytest.mark.parametrize( "invalid",