diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 27320fd4b..084060f9f 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -244,6 +244,7 @@ class UntrustedConfigError(ValueError): "fetch_ssl_certificate", # timing / waiting "wait_until", "page_timeout", "wait_for", "wait_for_timeout", + "body_visibility_timeout", "wait_for_images", "delay_before_return_html", "mean_delay", "max_range", # scrolling / rendering "ignore_body_visibility", "scan_full_page", "scroll_delay", @@ -1463,6 +1464,8 @@ class CrawlerRunConfig(): Default: False. ignore_body_visibility (bool): If True, ignore whether the body is visible before proceeding. Default: True. + body_visibility_timeout (int): Maximum time in ms to wait for the body to become visible. + Default: 30000. scan_full_page (bool): If True, scroll through the entire page to load all content. Default: False. scroll_delay (float): Delay in seconds between scroll steps if scan_full_page is True. @@ -1640,6 +1643,7 @@ def __init__( c4a_script: Union[str, List[str]] = None, js_only: bool = False, ignore_body_visibility: bool = True, + body_visibility_timeout: int = 30000, scan_full_page: bool = False, scroll_delay: float = 0.2, max_scroll_steps: Optional[int] = None, @@ -1770,6 +1774,7 @@ def __init__( self.c4a_script = c4a_script self.js_only = js_only self.ignore_body_visibility = ignore_body_visibility + self.body_visibility_timeout = body_visibility_timeout self.scan_full_page = scan_full_page self.scroll_delay = scroll_delay self.max_scroll_steps = max_scroll_steps @@ -2137,6 +2142,7 @@ def to_dict(self): "js_code_before_wait": self.js_code_before_wait, "js_only": self.js_only, "ignore_body_visibility": self.ignore_body_visibility, + "body_visibility_timeout": self.body_visibility_timeout, "scan_full_page": self.scan_full_page, "scroll_delay": self.scroll_delay, "max_scroll_steps": self.max_scroll_steps, diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 265c376e9..9202124ab 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -823,7 +823,7 @@ async def handle_request_failed_capture(request): style.opacity !== '0'; return isVisible; }""", - timeout=30000, + timeout=config.body_visibility_timeout, ) if not is_visible and not config.ignore_body_visibility: diff --git a/tests/test_config_defaults.py b/tests/test_config_defaults.py index 700886aa9..93a9484ca 100644 --- a/tests/test_config_defaults.py +++ b/tests/test_config_defaults.py @@ -226,13 +226,17 @@ def test_dump_load_survives_reset(self): assert loaded.headless is False def test_crawler_run_config_dump_load(self): - CrawlerRunConfig.set_defaults(verbose=False, scan_full_page=True) + assert CrawlerRunConfig().body_visibility_timeout == 30000 + CrawlerRunConfig.set_defaults( + verbose=False, scan_full_page=True, body_visibility_timeout=2000 + ) cfg = CrawlerRunConfig() data = cfg.dump() CrawlerRunConfig.reset_defaults() loaded = CrawlerRunConfig.load(data) assert loaded.verbose is False assert loaded.scan_full_page is True + assert loaded.body_visibility_timeout == 2000 def test_to_dict_includes_user_default_values(self): BrowserConfig.set_defaults(headless=False)