fix(config): make body visibility timeout configurable

crawl4ai/async_configs.py

@@ -244,6 +244,7 @@ class UntrustedConfigError(ValueError):
         "fetch_ssl_certificate",
         # timing / waiting
         "wait_until", "page_timeout", "wait_for", "wait_for_timeout",
+        "body_visibility_timeout",
         "wait_for_images", "delay_before_return_html", "mean_delay", "max_range",
         # scrolling / rendering
         "ignore_body_visibility", "scan_full_page", "scroll_delay",
@@ -1463,6 +1464,8 @@ class CrawlerRunConfig():
                         Default: False.
         ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding.
                                        Default: True.
+        body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible.
+                                       Default: 30000.
         scan_full_page (bool): If True, scroll through the entire page to load all content.
                                Default: False.
         scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True.
@@ -1640,6 +1643,7 @@ def __init__(
         c4a_script: Union[str, List[str]] = None,
         js_only: bool = False,
         ignore_body_visibility: bool = True,
+        body_visibility_timeout: int = 30000,
         scan_full_page: bool = False,
         scroll_delay: float = 0.2,
         max_scroll_steps: Optional[int] = None,
@@ -1770,6 +1774,7 @@ def __init__(
         self.c4a_script = c4a_script
         self.js_only = js_only
         self.ignore_body_visibility = ignore_body_visibility
+        self.body_visibility_timeout = body_visibility_timeout
         self.scan_full_page = scan_full_page
         self.scroll_delay = scroll_delay
         self.max_scroll_steps = max_scroll_steps
@@ -2137,6 +2142,7 @@ def to_dict(self):
             "js_code_before_wait": self.js_code_before_wait,
             "js_only": self.js_only,
             "ignore_body_visibility": self.ignore_body_visibility,
+            "body_visibility_timeout": self.body_visibility_timeout,
             "scan_full_page": self.scan_full_page,
             "scroll_delay": self.scroll_delay,
             "max_scroll_steps": self.max_scroll_steps,

crawl4ai/async_crawler_strategy.py

@@ -823,7 +823,7 @@ async def handle_request_failed_capture(request):
                                         style.opacity !== '0';
                         return isVisible;
                     }""",
-                    timeout=30000,
+                    timeout=config.body_visibility_timeout,
                 )
 
                 if not is_visible and not config.ignore_body_visibility:

tests/test_config_defaults.py

@@ -226,13 +226,17 @@ def test_dump_load_survives_reset(self):
         assert loaded.headless is False
 
     def test_crawler_run_config_dump_load(self):
-        CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True)
+        assert CrawlerRunConfig().body_visibility_timeout == 30000
+        CrawlerRunConfig.set_defaults(
+            verbose=False, scan_full_page=True, body_visibility_timeout=2000
+        )
         cfg = CrawlerRunConfig()
         data = cfg.dump()
         CrawlerRunConfig.reset_defaults()
         loaded = CrawlerRunConfig.load(data)
         assert loaded.verbose is False
         assert loaded.scan_full_page is True
+        assert loaded.body_visibility_timeout == 2000
 
     def test_to_dict_includes_user_default_values(self):
         BrowserConfig.set_defaults(headless=False)