From 6e62e56c9c02a188e9dfc4d17178c388efaea9b2 Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 13:59:44 +0200 Subject: [PATCH] fix(crawl): wait at most 30 s between crawl_busy stop retries stop waited whatever Retry-After the server sent, so a value of 3600 made one stop wait an hour. It now waits at most 30 s, and 1 s when the header is missing or unusable (negative, NaN). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01HJnfsEB5DBXR4e7KxBqvJV --- src/zenrows/crawl/client.py | 16 ++++++++++++++-- tests/test_crawl_client.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 2b61a45..e4e439a 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -18,6 +18,7 @@ import json import logging +import math import random import time from collections.abc import Callable, Iterator @@ -50,6 +51,9 @@ # 503 `crawl_busy` on stop: nothing was saved and the crawl is still # running, so the stop is safe to repeat after `Retry-After`. _CRAWL_BUSY = "crawl_busy" +# The longest a crawl_busy `Retry-After` may make a stop wait, so a large +# server value cannot hang it. +_MAX_STOP_RETRY_AFTER = 30.0 _POLL_INTERVAL = 2.0 _POLL_BACKOFF = 1.5 @@ -270,7 +274,7 @@ def stop(self, crawl_id: str) -> CrawlStop: A 503 `crawl_busy` means the stop was not saved and the crawl is still running; `stop` retries it up to `retries` times, waiting - `Retry-After` seconds each time, then raises it. + `Retry-After` seconds (at most 30) each time, then raises it. """ attempt = 0 while True: @@ -281,7 +285,7 @@ def stop(self, crawl_id: str) -> CrawlStop: except CrawlAPIError as err: if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: raise - time.sleep(err.retry_after if err.retry_after is not None else 1.0) + time.sleep(_stop_retry_wait(err.retry_after)) attempt += 1 # ===== contents ===== @@ -342,6 +346,14 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) +def _stop_retry_wait(retry_after: float | None) -> float: + """Seconds `stop` waits after a 503 crawl_busy: the server's + `Retry-After`, 1s when it is missing or unusable, at most 30s.""" + if retry_after is None or not math.isfinite(retry_after) or retry_after < 0: + return 1.0 + return min(retry_after, _MAX_STOP_RETRY_AFTER) + + def _crawl_status(response: httpx.Response) -> CrawlStatus | None: status = response.headers.get("X-Crawl-Status") return CrawlStatus(status) if status else None diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index d48256e..b56671c 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -572,3 +572,19 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): assert crawl.status is CrawlStatus.FAILED assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND assert no_sleep == [] + + +@pytest.mark.parametrize( + ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)] +) +@respx.mock +def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1) + respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": retry_after}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + client.stop("c_1") + assert no_sleep == [slept]