diff --git a/README.md b/README.md index 465e94a..97a67ac 100644 --- a/README.md +++ b/README.md @@ -605,6 +605,7 @@ except CrawlAPIError as exc: | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. | | 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. | | 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. | +| 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. | `code` is None when the response has no problem body with a code (for example a 502 from a proxy). A crawl that ended `failed` is not an error diff --git a/src/zenrows/crawl/client.py b/src/zenrows/crawl/client.py index 1a8a6ac..59bff3d 100644 --- a/src/zenrows/crawl/client.py +++ b/src/zenrows/crawl/client.py @@ -18,6 +18,7 @@ import json import logging +import math import random import time from collections.abc import Callable, Iterator @@ -47,6 +48,13 @@ # 429 `too_many_crawls` is a capacity limit, not a transient failure. _CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429} +# 503 `crawl_busy` on stop: nothing was saved and the crawl is still +# running, so the stop is safe to repeat after `Retry-After`. +_CRAWL_BUSY = "crawl_busy" +# The longest a crawl_busy `Retry-After` may make a stop wait, so a large +# server value cannot hang it. +_MAX_STOP_RETRY_AFTER = 30.0 + _POLL_INTERVAL = 2.0 _POLL_BACKOFF = 1.5 _MAX_POLL_INTERVAL = 15.0 @@ -117,9 +125,11 @@ def __init__( and network errors) on idempotent requests — GETs, and a `create` that carries an `idempotency_key` (never on 429 for `create`). Retries use jittered exponential backoff and honor - `Retry-After`; set `retries=0` to disable.""" + `Retry-After`; set `retries=0` to disable. It also bounds the + retries of a `stop` answered 503 `crawl_busy`.""" if not api_key: raise ValueError("ZenRowsCrawlClient: api_key is required.") + self._retries = max(0, retries) self._t = _Transport( base_url=base_url or _DEFAULT_BASE_URL, api_key=api_key, @@ -261,8 +271,22 @@ def stop(self, crawl_id: str) -> CrawlStop: Idempotent: a crawl that already ended answers as it ended. The answer carries no counts; read the final coverage and results with `get` once pages already in flight have finished. + + A 503 `crawl_busy` means the stop was not saved and the crawl is + still running; `stop` retries it up to `retries` times, waiting + `Retry-After` seconds (at most 30) each time, then raises it. """ - return CrawlStop.model_validate(self._t.request_json("POST", f"/crawls/{crawl_id}/stop")) + attempt = 0 + while True: + try: + return CrawlStop.model_validate( + self._t.request_json("POST", f"/crawls/{crawl_id}/stop") + ) + except CrawlAPIError as err: + if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries: + raise + time.sleep(_stop_retry_wait(err.retry_after)) + attempt += 1 # ===== contents ===== @@ -322,6 +346,14 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl: interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL) +def _stop_retry_wait(retry_after: float | None) -> float: + """Seconds `stop` waits after a 503 crawl_busy: the server's + `Retry-After`, 1s when it is missing or unusable, at most 30s.""" + if retry_after is None or not math.isfinite(retry_after) or retry_after < 0: + return 1.0 + return min(retry_after, _MAX_STOP_RETRY_AFTER) + + def _crawl_status(response: httpx.Response) -> CrawlStatus | None: status = response.headers.get("X-Crawl-Status") return CrawlStatus(status) if status else None diff --git a/tests/test_crawl_client.py b/tests/test_crawl_client.py index eba3b15..c93fd20 100644 --- a/tests/test_crawl_client.py +++ b/tests/test_crawl_client.py @@ -248,6 +248,53 @@ def test_stop_posts_without_body(client: ZenRowsCrawlClient): assert route.calls.last.request.content == b"" +@respx.mock +def test_stop_retries_crawl_busy_after_retry_after(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": "1"}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + + stopped = client.stop("c_1") + + assert stopped.status is CrawlStatus.STOPPED + assert route.call_count == 2 + assert no_sleep == [1.0] + + +@respx.mock +def test_stop_surfaces_crawl_busy_after_retries(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "crawl_busy", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError) as exc_info: + client.stop("c_1") + + assert exc_info.value.status_code == 503 + assert exc_info.value.code == "crawl_busy" + assert route.call_count == 3 + assert no_sleep == [1.0, 1.0] + + +@respx.mock +def test_stop_does_not_retry_other_503(no_sleep): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2) + route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + return_value=problem(503, "service_unavailable", headers={"Retry-After": "1"}) + ) + + with pytest.raises(CrawlAPIError): + client.stop("c_1") + + assert route.call_count == 1 + assert no_sleep == [] + + # ----- results scanner ----- @@ -527,6 +574,22 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep): assert no_sleep == [] +@pytest.mark.parametrize( + ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)] +) +@respx.mock +def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float): + client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1) + respx.post(f"{BASE_URL}/crawls/c_1/stop").mock( + side_effect=[ + problem(503, "crawl_busy", headers={"Retry-After": retry_after}), + Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}), + ] + ) + client.stop("c_1") + assert no_sleep == [slept] + + @respx.mock def test_retry_after_above_30_seconds_waits_30_seconds(no_sleep): client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2)