fix(docker): support PDF scraping by default

deploy/docker/api.py

@@ -865,15 +865,20 @@ async def handle_stream_crawl_request(
         # mirroring handle_crawl_request. The streaming path previously skipped
         # this, leaving /crawl/stream (and /crawl with stream=true) unguarded.
         urls = _normalize_and_validate_seeds(urls)
-        browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED)
+        browser_config = BrowserConfig.load(
+            browser_config, provenance=Provenance.UNTRUSTED
+        )
         # browser_config.verbose = True # Set to False or remove for production stress testing
         browser_config.verbose = False
         from egress_broker import enforce_egress
+
         enforce_egress(browser_config)
-        crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)
+        crawler_config = CrawlerRunConfig.load(
+            crawler_config, provenance=Provenance.UNTRUSTED
+        )
         from governor import clamp_deep_crawl
+
         clamp_deep_crawl(crawler_config)
-        crawler_config.scraping_strategy = LXMLWebScrapingStrategy()
         crawler_config.stream = True
 
         # Deep crawl streaming supports exactly one start URL
@@ -941,7 +946,7 @@ async def handle_stream_crawl_request(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=str(e)
         )
-        
+
 async def handle_crawl_job(
     redis,
     background_tasks: BackgroundTasks,
@@ -1023,4 +1028,4 @@ async def _runner():
     except HTTPException:
         await redis.delete(f"task:{task_id}")
         raise
-    return {"task_id": task_id}
+    return {"task_id": task_id}

deploy/docker/requirements.txt

@@ -14,3 +14,4 @@ PyJWT==2.10.1
 mcp>=1.18.0
 websockets>=15.0.1
 httpx[http2]>=0.27.2
+pypdf

tests/test_issue_2127_docker_pdf.py

@@ -0,0 +1,31 @@
+import ast
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+
+
+def test_default_docker_dependencies_include_pypdf():
+    requirements = (
+        (ROOT / "deploy" / "docker" / "requirements.txt").read_text().splitlines()
+    )
+
+    assert "pypdf" in requirements
+
+
+def test_stream_handler_preserves_requested_scraping_strategy():
+    tree = ast.parse((ROOT / "deploy" / "docker" / "api.py").read_text())
+    handler = next(
+        node
+        for node in tree.body
+        if isinstance(node, ast.AsyncFunctionDef)
+        and node.name == "handle_stream_crawl_request"
+    )
+    assigned_attributes = {
+        target.attr
+        for node in ast.walk(handler)
+        if isinstance(node, ast.Assign)
+        for target in node.targets
+        if isinstance(target, ast.Attribute)
+    }
+
+    assert "scraping_strategy" not in assigned_attributes