From 77be02c0024c973344a51288b45bbc42fa464a47 Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 9 Oct 2026 21:35:13 +0200 Subject: [PATCH 1/3] fix(crawl): retry stop on 503 crawl_busy, honouring Retry-After A stop that collides with the crawl saving its own progress answers 503 crawl_busy with Retry-After. Nothing was saved and the crawl keeps running, and the stop is safe to repeat, but stop raised at once. stop now retries that answer up to `retries` times, waiting Retry-After seconds between tries, then raises it. Other errors, and other 503s, are raised as before. --- src/zenrows/crawl/client.py | 24 +++++++++++++++++-- tests/test_crawl_client.py | 47 +++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index d43b893..2b61a45 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -47,6 +47,10 @@ # 429 `too_many_crawls` is a capacity limit, not a transient failure. _CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} +# 503 `crawl_busy` on stop: nothing was saved and the crawl is still +# running, so the stop is safe to repeat after `Retry-After`. +_CRAWL_BUSY = "crawl_busy" + _POLL_INTERVAL = 2.0 _POLL_BACKOFF = 1.5 _MAX_POLL_INTERVAL = 15.0 @@ -117,9 +121,11 @@ def __init__( and network errors) on idempotent requests — GETs, and a `create` that carries an `idempotency_key` (never on 429 for `create`). Retries use jittered exponential backoff and honor - `Retry-After`; set `retries=0` to disable.""" + `Retry-After`; set `retries=0` to disable. It also bounds the + retries of a `stop` answered 503 `crawl_busy`.""" if not api_key: raise ValueError("ZenRowsCrawlClient: api_key is required.") + self._retries = max(0, retries) self._t = _Transport( base_url=base_url or _DEFAULT_BASE_URL, api_key=api_key, @@ -261,8 +267,22 @@ def stop(self, crawl_id: str) -> CrawlStop: Idempotent: a crawl that already ended answers as it ended. The answer carries no counts; read the final coverage and results with `get` once pages already in flight have finished. + + A 503 `crawl_busy` means the stop was not saved and the crawl is + still running; `stop` retries it up to `retries` times, waiting + `Retry-After` seconds each time, then raises it. """ - return CrawlStop.model_validate(self._t.request_json("POST", f"/crawls/{crawl_id}/stop")) + attempt = 0 + while True: + try: + return CrawlStop.model_validate( + self._t.request_json("POST", f"/crawls/{crawl_id}/stop") + ) + except CrawlAPIError as err: + if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: + raise + time.sleep(err.retry_after if err.retry_after is not None else 1.0) + attempt += 1 # ===== contents ===== diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index 0384a66..d48256e 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -248,6 +248,53 @@ def test_stop_posts_without_body(client: ZenRowsCrawlClient): assert route.calls.last.request.content == b"" +@respx.mock +def test_stop_retries_crawl_busy_after_retry_after(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": "1"}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert route.call_count == 2 + assert no_sleep == [1.0] + + +@respx.mock +def test_stop_surfaces_crawl_busy_after_retries(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "crawl_busy", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError) as exc_info: + client.stop("c_1") + + assert exc_info.value.status_code == 503 + assert exc_info.value.code == "crawl_busy" + assert route.call_count == 3 + assert no_sleep == [1.0, 1.0] + + +@respx.mock +def test_stop_does_not_retry_other_503(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "service_unavailable", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError): + client.stop("c_1") + + assert route.call_count == 1 + assert no_sleep == [] + + # ----- results scanner ----- From c5f167271dc63a6d8a1f072cec7e3c1428a0583d Mon Sep 17 00:00:00 2001 From: Gonzalo Date: Fri, 9 Oct 2026 21:36:32 +0200 Subject: [PATCH 2/3] docs(crawl): list 503 crawl_busy in the crawl errors table --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7b92847..d05ad1a 100644 --- a/README.md +++ b/README.md @@ -604,6 +604,7 @@ except CrawlAPIError as exc: | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | | 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. | | 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. | +| 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | `code` is None when the response has no problem body with a code (for example a 502 from a proxy). A crawl that ended `failed` is not an error From 2edc5706948140b4644b641809842a69e8a3946c Mon Sep 17 00:00:00 2001 From: Stefano Guerrini Date: Mon, 12 Oct 2026 15:49:09 +0200 Subject: [PATCH 3/3] fix(crawl): wait at most 30 s between crawl_busy stop retries --- src/zenrows/crawl/client.py | 16 ++++++++++++++-- tests/test_crawl_client.py | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 2b61a45..e4e439a 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -18,6 +18,7 @@ import json import logging +import math import random import time from collections.abc import Callable, Iterator @@ -50,6 +51,9 @@ # 503 `crawl_busy` on stop: nothing was saved and the crawl is still # running, so the stop is safe to repeat after `Retry-After`. _CRAWL_BUSY = "crawl_busy" +# The longest a crawl_busy `Retry-After` may make a stop wait, so a large +# server value cannot hang it. +_MAX_STOP_RETRY_AFTER = 30.0 _POLL_INTERVAL = 2.0 _POLL_BACKOFF = 1.5 @@ -270,7 +274,7 @@ def stop(self, crawl_id: str) -> CrawlStop: A 503 `crawl_busy` means the stop was not saved and the crawl is still running; `stop` retries it up to `retries` times, waiting - `Retry-After` seconds each time, then raises it. + `Retry-After` seconds (at most 30) each time, then raises it. """ attempt = 0 while True: @@ -281,7 +285,7 @@ def stop(self, crawl_id: str) -> CrawlStop: except CrawlAPIError as err: if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: raise - time.sleep(err.retry_after if err.retry_after is not None else 1.0) + time.sleep(_stop_retry_wait(err.retry_after)) attempt += 1 # ===== contents ===== @@ -342,6 +346,14 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) +def _stop_retry_wait(retry_after: float | None) -> float: + """Seconds `stop` waits after a 503 crawl_busy: the server's + `Retry-After`, 1s when it is missing or unusable, at most 30s.""" + if retry_after is None or not math.isfinite(retry_after) or retry_after < 0: + return 1.0 + return min(retry_after, _MAX_STOP_RETRY_AFTER) + + def _crawl_status(response: httpx.Response) -> CrawlStatus | None: status = response.headers.get("X-Crawl-Status") return CrawlStatus(status) if status else None diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index d48256e..b56671c 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -572,3 +572,19 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): assert crawl.status is CrawlStatus.FAILED assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND assert no_sleep == [] + + +@pytest.mark.parametrize( + ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)] +) +@respx.mock +def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1) + respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": retry_after}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + client.stop("c_1") + assert no_sleep == [slept]