fix(crawl): wait at most 30 s between crawl_busy stop retries

src/zenrows/crawl/client.py

@@ -18,6 +18,7 @@
 
 import json
 import logging
+import math
 import random
 import time
 from collections.abc import Callable, Iterator
@@ -50,6 +51,9 @@
 # 503 `crawl_busy` on stop: nothing was saved and the crawl is still
 # running, so the stop is safe to repeat after `Retry-After`.
 _CRAWL_BUSY = "crawl_busy"
+# The longest a crawl_busy `Retry-After` may make a stop wait, so a large
+# server value cannot hang it.
+_MAX_STOP_RETRY_AFTER = 30.0
 
 _POLL_INTERVAL = 2.0
 _POLL_BACKOFF = 1.5
@@ -270,7 +274,7 @@ def stop(self, crawl_id: str) -> CrawlStop:
 
         A 503 `crawl_busy` means the stop was not saved and the crawl is
         still running; `stop` retries it up to `retries` times, waiting
-        `Retry-After` seconds each time, then raises it.
+        `Retry-After` seconds (at most 30) each time, then raises it.
         """
         attempt = 0
         while True:
@@ -281,7 +285,7 @@ def stop(self, crawl_id: str) -> CrawlStop:
             except CrawlAPIError as err:
                 if err.status_code != 503 or err.code != _CRAWL_BUSY or attempt >= self._retries:
                     raise
-                time.sleep(err.retry_after if err.retry_after is not None else 1.0)
+                time.sleep(_stop_retry_wait(err.retry_after))
                 attempt += 1
 
     # ===== contents =====
@@ -342,6 +346,14 @@ def wait(self, crawl_id: str, *, timeout: float = 600.0) -> Crawl:
             interval = min(interval * _POLL_BACKOFF, _MAX_POLL_INTERVAL)
 
 
+def _stop_retry_wait(retry_after: float | None) -> float:
+    """Seconds `stop` waits after a 503 crawl_busy: the server's
+    `Retry-After`, 1s when it is missing or unusable, at most 30s."""
+    if retry_after is None or not math.isfinite(retry_after) or retry_after < 0:
+        return 1.0
+    return min(retry_after, _MAX_STOP_RETRY_AFTER)
+
+
 def _crawl_status(response: httpx.Response) -> CrawlStatus | None:
     status = response.headers.get("X-Crawl-Status")
     return CrawlStatus(status) if status else None

tests/test_crawl_client.py

@@ -572,3 +572,19 @@ def test_wait_returns_failed_crawl(client: ZenRowsCrawlClient, no_sleep):
     assert crawl.status is CrawlStatus.FAILED
     assert crawl.error is not None and crawl.error.code is CrawlErrorCode.NO_ITEMS_FOUND
     assert no_sleep == []
+
+
+@pytest.mark.parametrize(
+    ("retry_after", "slept"), [("3600", 30.0), ("-1", 1.0), ("nan", 1.0), ("2", 2.0)]
+)
+@respx.mock
+def test_stop_waits_at_most_30_seconds_for_crawl_busy(no_sleep, retry_after: str, slept: float):
+    client = ZenRowsCrawlClient(api_key=API_KEY, base_url=BASE_URL, retries=1)
+    respx.post(f"{BASE_URL}/crawls/c_1/stop").mock(
+        side_effect=[
+            problem(503, "crawl_busy", headers={"Retry-After": retry_after}),
+            Response(200, json={"crawl_id": "c_1", "status": "stopped", "stop_reason": "user"}),
+        ]
+    )
+    client.stop("c_1")
+    assert no_sleep == [slept]