fix(crawl): retry stop on 503 crawl_busy, honoring a capped Retry-After

README.md

@@ -605,6 +605,7 @@ except CrawlAPIError as exc:
 | 422 | `invalid_parameter`, `invalid_start_url`, `domain_not_allowed` | Fix the request. Do not retry it as is. |
 | 422 | `idempotency_key_reused` | Use a new key, or no key. Do not retry it as is. |
 | 429 | `too_many_crawls` | The account has reached its limit of active jobs (3 by default), shared with its Batch jobs. Retry after `exc.retry_after` seconds. `create` does not retry it. |
+| 503 | `crawl_busy` | `stop` could not be saved yet; the crawl is still running. `stop` retries it up to `retries` times, waiting `exc.retry_after` seconds, then raises it. Repeating the stop is safe. |
 
 `code` is None when the response has no problem body with a code (for
 example a 502 from a proxy). A crawl that ended `failed` is not an error

src/zenrows/crawl/client.py

@@ -18,6 +18,7 @@
 
 import json
 import logging
+import math
 import random
 import time
 from collections.abc import Callable, Iterator
@@ -47,6 +48,13 @@
 # 429 `too_many_crawls` is a capacity limit, not a transient failure.
 _CREATE_RETRY_STATUSES = _RETRYABLE_STATUSES - {429}
 
+# 503 `crawl_busy` on stop: nothing was saved and the crawl is still
+# running, so the stop is safe to repeat after `Retry-After`.
+_CRAWL_BUSY = "crawl_busy"
+# The longest a crawl_busy `Retry-After` may make a stop wait, so a large
+# server value cannot hang it.
+_MAX_STOP_RETRY_AFTER = 30.0
+
 _POLL_INTERVAL = 2.0
 _POLL_BACKOFF = 1.5
 _MAX_POLL_INTERVAL = 15.0
@@ -117,9 +125,11 @@ def __init__(
         and network errors) on idempotent requests — GETs, and a
         `create` that carries an `idempotency_key` (never on 429 for
         `create`). Retries use jittered exponential backoff and honor
-        `Retry-After`; set `retries=0` to disable."""
+        `Retry-After`; set `retries=0` to disable. It also bounds the
+        retries of a `stop` answered 503 `crawl_busy`."""
         if not api_key:
             raise ValueError("ZenRowsCrawlClient: api_key is required.")
+        self._retries = max(0, retries)
         self._t = _Transport(
             base_url=base_url or _DEFAULT_BASE_URL,
             api_key=api_key,
@@ -261,8 +271,22 @@ def stop(self, crawl_id: str) -> CrawlStop:
         Idempotent: a crawl that already ended answers as it ended.
         The answer carries no counts; read the final coverage and
         results with `get` once pages already in flight have finished.
+
+        A 503 `crawl_busy` means the stop was not saved and the crawl is
+        still running; `stop` retries it up to `retries` times, waiting
+        `Retry-After` seconds (at most 30) each time, then raises it.
         """
-        return CrawlStop.model_validate(self._t.request_json("POST", f"/crawls/{crawl_id}/stop"))
+        attempt = 0
+        while True:
+            try:
+                return CrawlStop.model_validate(
+                    self._t.request_json("POST", f"/crawls/{crawl_id}/stop")
+                )
+            except CrawlAPIError as err:
+                if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries:
+                    raise
+                time.sleep(_stop_retry_wait(err.retry_after))
+                attempt += 1
 
     # ===== contents =====
 
@@ -322,6 +346,14 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl:
             interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL)
 
 
+def _stop_retry_wait(retry_after: float | None) -> float:
+    """Seconds `stop` waits after a 503 crawl_busy: the server's
+    `Retry-After`, 1s when it is missing or unusable, at most 30s."""
+    if retry_after is None or not math.isfinite(retry_after) or retry_after < 0:
+        return 1.0
+    return min(retry_after, _MAX_STOP_RETRY_AFTER)
+
+
 def _crawl_status(response: httpx.Response) -> CrawlStatus | None:
     status = response.headers.get("X-Crawl-Status")
     return CrawlStatus(status) if status else None

tests/test_crawl_client.py

@@ -248,6 +248,53 @@ def test_stop_posts_without_body(client: ZenRowsCrawlClient):
     assert route.calls.last.request.content == b""
 
 
+@respx.mock
+def test_stop_retries_crawl_busy_after_retry_after(no_sleep):
+    client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2)
+    route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(
+        side_effect=[
+            problem(503, "crawl_busy", headers={"Retry-After": "1"}),
+            Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}),
+        ]
+    )
+
+    stopped = client.stop("c_1")
+
+    assert stopped.status is CrawlStatus.STOPPED
+    assert route.call_count == 2
+    assert no_sleep == [1.0]
+
+
+@respx.mock
+def test_stop_surfaces_crawl_busy_after_retries(no_sleep):
+    client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2)
+    route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(
+        return_value=problem(503, "crawl_busy", headers={"Retry-After": "1"})
+    )
+
+    with pytest.raises(CrawlAPIError) as exc_info:
+        client.stop("c_1")
+
+    assert exc_info.value.status_code == 503
+    assert exc_info.value.code == "crawl_busy"
+    assert route.call_count == 3
+    assert no_sleep == [1.0, 1.0]
+
+
+@respx.mock
+def test_stop_does_not_retry_other_503(no_sleep):
+    client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2)
+    route = respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(
+        return_value=problem(503, "service_unavailable", headers={"Retry-After": "1"})
+    )
+
+    with pytest.raises(CrawlAPIError):
+        client.stop("c_1")
+
+    assert route.call_count == 1
+    assert no_sleep == []
+
+
 # ----- results scanner -----
 
 
@@ -527,6 +574,22 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep):
     assert no_sleep == []
 
 
+@pytest.mark.parametrize(
+    ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)]
+)
+@respx.mock
+def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float):
+    client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1)
+    respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(
+        side_effect=[
+            problem(503, "crawl_busy", headers={"Retry-After": retry_after}),
+            Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}),
+        ]
+    )
+    client.stop("c_1")
+    assert no_sleep == [slept]
+
+
 @respx.mock
 def test_retry_after_above_30_seconds_waits_30_seconds(no_sleep):
     client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=2)