fix(docker-api): apply base_config to fields the client did not send

deploy/docker/api.py

@@ -644,6 +644,19 @@ def _normalize_and_validate_seeds(urls: List[str]) -> List[str]:
     return urls
 
 
+def _apply_base_config(cfg, base_config: dict, raw: Optional[dict]) -> None:
+    """Apply the server-side base_config to fields absent from the raw payload.
+
+    Only the raw keys tell "omitted" apart from "sent, equal to the default" —
+    a default of False or 0 is indistinguishable from unset once loaded. The
+    payload is either a flat dict or {"type": ..., "params": {...}}.
+    """
+    provided = set(raw.get("params", raw)) if raw else set()
+    for key, value in base_config.items():
+        if key not in provided and hasattr(cfg, key):
+            setattr(cfg, key, value)
+
+
 async def handle_crawl_request(
     urls: List[str],
     browser_config: dict,
@@ -671,6 +684,7 @@ async def handle_crawl_request(
 
     try:
         urls = _normalize_and_validate_seeds(urls)
+        raw_crawler_config = crawler_config
         browser_config = BrowserConfig.load(browser_config, provenance=Provenance.UNTRUSTED)
         crawler_config = CrawlerRunConfig.load(crawler_config, provenance=Provenance.UNTRUSTED)
         from egress_broker import enforce_egress
@@ -700,20 +714,12 @@ async def handle_crawl_request(
         if crawler_configs and len(urls) > 1:
             # Per-URL config list: deserialize each and apply base_config
             config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]
-            for cfg in config_list:
-                for key, value in base_config.items():
-                    if hasattr(cfg, key):
-                        current_value = getattr(cfg, key)
-                        if current_value is None or current_value == "":
-                            setattr(cfg, key, value)
+            for cfg, raw in zip(config_list, crawler_configs):
+                _apply_base_config(cfg, base_config, raw)
             effective_config = config_list
         else:
             # Single config (original behavior)
-            for key, value in base_config.items():
-                if hasattr(crawler_config, key):
-                    current_value = getattr(crawler_config, key)
-                    if current_value is None or current_value == "":
-                        setattr(crawler_config, key, value)
+            _apply_base_config(crawler_config, base_config, raw_crawler_config)
             effective_config = crawler_config
 
         results = []

tests/test_base_config_defaults.py

@@ -0,0 +1,37 @@
+"""Regression tests: crawler.base_config must apply to fields the client omitted.
+
+The old guard treated only None/"" as "not provided", so any base_config key
+whose CrawlerRunConfig default is False or 0 (simulate_user, magic,
+override_navigator, check_robots_txt, ...) was silently dropped.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'deploy', 'docker'))
+
+from crawl4ai import CrawlerRunConfig
+from crawl4ai.async_configs import Provenance
+from api import _apply_base_config
+
+
+def test_boolean_base_config_applies_when_client_omits():
+    cfg = CrawlerRunConfig()
+    _apply_base_config(cfg, {"simulate_user": True}, None)
+    assert cfg.simulate_user is True
+
+
+def test_client_value_wins():
+    """#1505: an explicitly sent value must not be clobbered by base_config.
+
+    Uses check_robots_txt because simulate_user is forbidden on untrusted
+    bodies — base_config is the only way that one is ever set.
+    """
+    raw = {"type": "CrawlerRunConfig", "params": {"check_robots_txt": False}}
+    cfg = CrawlerRunConfig.load(raw, provenance=Provenance.UNTRUSTED)
+    _apply_base_config(cfg, {"check_robots_txt": True}, raw)
+    assert cfg.check_robots_txt is False
+
+    flat = CrawlerRunConfig()
+    _apply_base_config(flat, {"check_robots_txt": True}, {"check_robots_txt": False})
+    assert flat.check_robots_txt is False

tests/test_issue_1837_config_list.py

@@ -105,7 +105,7 @@ def test_api_applies_base_config_to_each(self):
         """Base config should be applied to each config in the list."""
         with open("deploy/docker/api.py") as f:
             source = f.read()
-        assert "for cfg in config_list:" in source
+        assert "for cfg, raw in zip(config_list, crawler_configs):" in source
 
 
 # -- Server endpoint passes crawler_configs --